bundle.lisp 18.3 KB
Newer Older
1 2 3
;;;; -------------------------------------------------------------------------
;;;; ASDF-Bundle

4 5
(asdf/package:define-package :asdf/bundle
  (:recycle :asdf/bundle :asdf)
6
  (:use :common-lisp :asdf/driver :asdf/upgrade
7 8 9
   :asdf/component :asdf/system :asdf/find-system :asdf/find-component :asdf/operation
   :asdf/action :asdf/lisp-action :asdf/plan :asdf/operate)
  (:export
10
   #:bundle-op #:bundle-op-build-args #:bundle-type #:bundle-system #:bundle-pathname-type
11
   #:fasl-op #:load-fasl-op #:lib-op #:dll-op #:binary-op
12
   #:monolithic-op #:monolithic-bundle-op #:direct-dependency-files
13
   #:monolithic-binary-op #:monolithic-fasl-op #:monolithic-lib-op #:monolithic-dll-op
14
   #:program-op
15
   #:compiled-file #:precompiled-system #:prebuilt-system
16 17
   #:operation-monolithic-p
   #:user-system-p #:user-system #:trivial-system-p
18
   #+ecl #:make-build
19
   #:register-pre-built-system
20
   #:build-args #:name-suffix #:prologue-code #:epilogue-code #:static-library
21 22
   #:component-bundle-pathname #:bundle-pathname
   #:component-entry-point #:entry-point))
23 24 25 26 27
(in-package :asdf/bundle)

(defclass bundle-op (operation)
  ((build-args :initarg :args :initform nil :accessor bundle-op-build-args)
   (name-suffix :initarg :name-suffix :initform nil)
28
   (bundle-type :initform :no-output-file :reader bundle-type)
29 30 31 32 33
   #+ecl (lisp-files :initform nil :accessor bundle-op-lisp-files)
   #+mkcl (do-fasb :initarg :do-fasb :initform t :reader bundle-op-do-fasb-p)
   #+mkcl (do-static-library :initarg :do-static-library :initform t :reader bundle-op-do-static-library-p)))

(defclass fasl-op (bundle-op)
34 35
  ;; create a single fasl for the entire library
  ((bundle-type :initform :fasl)))
36

37 38 39
(defclass load-fasl-op (basic-load-op)
  ;; load a single fasl for the entire library
  ())
40 41

(defclass lib-op (bundle-op)
42 43 44
  ;; On ECL: compile the system and produce linkable .a library for it.
  ;; On others: just compile the system.
  ((bundle-type :initform #+(or ecl mkcl) :lib #-(or ecl mkcl) :no-output-file)))
45 46

(defclass dll-op (bundle-op)
47 48
  ;; Link together all the dynamic library used by this system into a single one.
  ((bundle-type :initform :dll)))
49 50

(defclass binary-op (bundle-op)
51 52
  ;; On ECL: produce lib and fasl for the system.
  ;; On "normal" Lisps: produce just the fasl.
53 54 55 56 57 58 59 60 61
  ())

(defclass monolithic-op (operation) ()) ;; operation on a system and its dependencies

(defclass monolithic-bundle-op (monolithic-op bundle-op)
  ((prologue-code :accessor monolithic-op-prologue-code)
   (epilogue-code :accessor monolithic-op-epilogue-code)))

(defclass monolithic-binary-op (binary-op monolithic-bundle-op)
62 63
  ;; On ECL: produce lib and fasl for combined system and dependencies.
  ;; On "normal" Lisps: produce an image file from system and dependencies.
64 65
  ())

66 67 68
(defclass monolithic-fasl-op (monolithic-bundle-op fasl-op)
  ;; Create a single fasl for the system and its dependencies.
  ())
69 70

(defclass monolithic-lib-op (monolithic-bundle-op lib-op)
71 72
  ;; ECL: Create a single linkable library for the system and its dependencies.
  ((bundle-type :initform :lib)))
73 74

(defclass monolithic-dll-op (monolithic-bundle-op dll-op)
75
  ((bundle-type :initform :dll)))
76 77

(defclass program-op (monolithic-bundle-op)
78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
  ;; All: create an executable file from the system and its dependencies
  ((bundle-type :initform :program)))

(defgeneric* component-bundle-pathname (component))
(defgeneric* component-entry-point (component))

(defmethod component-bundle-pathname ((c component))
  (declare (ignorable c))
  nil)
(defmethod component-entry-point ((c component))
  (declare (ignorable c))
  nil)

(defclass bundle-system (system)
  ((bundle-pathname
    :initform nil :initarg :bundle-pathname :accessor component-bundle-pathname)
   (entry-point
95
    :initform nil :initarg :entry-point :accessor component-entry-point)))
96 97 98 99 100 101 102

(defun* bundle-pathname-type (bundle-type)
  (etypecase bundle-type
    ((eql :no-output-file) nil) ;; should we error out instead?    
    ((or null string) bundle-type)
    ((eql :fasl) #-(or ecl mkcl) (compile-file-type) #+(or ecl mkcl) "fasb")
    #+ecl
103
    ((member :binary :dll :lib :static-library :program :object :program)
104 105 106 107 108 109 110 111 112 113 114 115 116
     (compile-file-type :type bundle-type))
    ((eql :binary) "image")
    ((eql :dll) (cond ((os-unix-p) "so") ((os-windows-p) "dll")))
    ((member :lib :static-library) (cond ((os-unix-p) "a") ((os-windows-p) "lib")))
    ((eql :program) (cond ((os-unix-p) nil) ((os-windows-p) "exe")))))

(defun* bundle-output-files (o c)
  (let ((bundle-type (bundle-type o)))
    (unless (eq bundle-type :no-output-file) ;; NIL already means something regarding type.
      (let ((name (or (component-bundle-pathname c)
                      (format nil "~A~@[~A~]" (component-name c) (slot-value o 'name-suffix))))
            (type (bundle-pathname-type bundle-type)))
        (values (list (subpathname (component-pathname c) name :type type))
117
                (eq (type-of o) (component-build-operation c)))))))
118 119 120 121 122 123 124 125 126 127 128 129 130 131

(defmethod output-files ((o bundle-op) (c system))
  (bundle-output-files o c))

#-(or ecl mkcl)
(progn
  (defmethod perform ((o program-op) (c system))
    (let ((output-file (output-file o c)))
      (setf *image-entry-point* (ensure-function (component-entry-point c)))
      (dump-image output-file :executable t)))

  (defmethod perform ((o monolithic-binary-op) (c system))
    (let ((output-file (output-file o c)))
      (dump-image output-file))))
132 133

(defclass compiled-file (file-component)
134
  ((type :initform #-(or ecl mkcl) (compile-file-type) #+(or ecl mkcl) "fasb")))
135 136

(defclass precompiled-system (system)
137
  ((bundle-pathname :initarg :fasl)))
138 139

(defclass prebuilt-system (system)
140 141
  ((bundle-pathname :initarg :static-library :initarg :lib
                    :accessor prebuilt-system-static-library)))
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160

;;;
;;; BUNDLE-OP
;;;
;;; This operation takes all components from one or more systems and
;;; creates a single output file, which may be
;;; a FASL, a statically linked library, a shared library, etc.
;;; The different targets are defined by specialization.
;;;

(defun* operation-monolithic-p (op)
  (typep op 'monolithic-op))

(defmethod initialize-instance :after ((instance bundle-op) &rest initargs
                                       &key (name-suffix nil name-suffix-p)
                                       &allow-other-keys)
  (declare (ignorable initargs name-suffix))
  (unless name-suffix-p
    (setf (slot-value instance 'name-suffix)
161 162
          (unless (typep instance 'program-op)
            (if (operation-monolithic-p instance) ".all-systems" #-ecl ".system"))))
163 164 165 166 167 168
  (when (typep instance 'monolithic-bundle-op)
    (destructuring-bind (&rest original-initargs
                         &key lisp-files prologue-code epilogue-code
                         &allow-other-keys)
        (operation-original-initargs instance)
      (setf (operation-original-initargs instance)
169
            (remove-plist-keys '(:lisp-files :epilogue-code :prologue-code) original-initargs)
170 171
            (monolithic-op-prologue-code instance) prologue-code
            (monolithic-op-epilogue-code instance) epilogue-code)
172
      #-ecl (assert (null (or lisp-files epilogue-code prologue-code)))
173 174
      #+ecl (setf (bundle-op-lisp-files instance) lisp-files)))
  (setf (bundle-op-build-args instance)
175
        (remove-plist-keys '(:type :monolithic :name-suffix)
176 177 178 179 180 181 182 183
                     (operation-original-initargs instance))))

(defmethod bundle-op-build-args :around ((o lib-op))
  (declare (ignorable o))
  (let ((args (call-next-method)))
    (remf args :ld-flags)
    args))

184 185
(defun* bundlable-file-p (pathname)
  (let ((type (pathname-type pathname)))
186
    (declare (ignorable type))
187 188
    (or #+ecl (or (equal type (compile-file-type :type :object))
                  (equal type (compile-file-type :type :static-library)))
189 190 191
        #+mkcl (equal type (compile-file-type :fasl-p nil))
        #+(or allegro clisp clozure cmu lispworks sbcl scl xcl) (equal type (compile-file-type)))))

192
(defgeneric* (trivial-system-p) (component))
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209

(defun* user-system-p (s)
  (and (typep s 'system)
       (not (builtin-system-p s))
       (not (trivial-system-p s))))

(deftype user-system () '(and system (satisfies user-system-p)))

;;;
;;; First we handle monolithic bundles.
;;; These are standalone systems which contain everything,
;;; including other ASDF systems required by the current one.
;;; A PROGRAM is always monolithic.
;;;
;;; MONOLITHIC SHARED LIBRARIES, PROGRAMS, FASL
;;;

210 211
(defmethod component-depends-on ((o monolithic-lib-op) (c system))
  (declare (ignorable o))
212
  `((lib-op ,@(required-components c :other-systems t :component-type 'system
213
                                     :goal-operation 'load-op
214
                                     :keep-operation 'compile-op))))
215

216 217
(defmethod component-depends-on ((o monolithic-fasl-op) (c system))
  (declare (ignorable o))
218
  `((fasl-op ,@(required-components c :other-systems t :component-type 'system
219
                                      :goal-operation 'load-fasl-op
220
                                      :keep-operation 'fasl-op))))
221 222 223

(defmethod component-depends-on ((o program-op) (c system))
  (declare (ignorable o))
224 225
  #+(or ecl mkcl) (component-depends-on (make-operation 'monolithic-lib-op) c)
  #-(or ecl mkcl) `((load-op ,c)))
226 227 228 229 230 231 232 233 234

(defmethod component-depends-on ((o binary-op) (c system))
  (declare (ignorable o))
  `((fasl-op ,c)
    (lib-op ,c)))

(defmethod component-depends-on ((o monolithic-binary-op) (c system))
  `((,(find-operation o 'monolithic-fasl-op) ,c)
    (,(find-operation o 'monolithic-lib-op) ,c)))
235 236 237

(defmethod component-depends-on ((o lib-op) (c system))
  (declare (ignorable o))
238
  `((compile-op ,@(required-components c :other-systems nil :component-type '(not system)
239
                                         :goal-operation 'load-op
240
                                         :keep-operation 'compile-op))))
241 242 243 244 245 246 247 248 249

(defmethod component-depends-on ((o fasl-op) (c system))
  (declare (ignorable o))
  #+ecl `((lib-op ,c))
  #-ecl
  (component-depends-on (find-operation o 'lib-op) c))

(defmethod component-depends-on ((o dll-op) c)
  (component-depends-on (find-operation o 'lib-op) c))
250 251 252 253 254

(defmethod component-depends-on ((o bundle-op) c)
  (declare (ignorable o c))
  nil)

255 256
(defmethod component-depends-on :around ((o bundle-op) (c component))
  (declare (ignorable o c))
257
  (if-let (op (and (eq (type-of o) 'bundle-op) (component-build-operation c)))
258 259 260
      `((,op ,c))
      (call-next-method)))

261
(defun* direct-dependency-files (o c &key (test 'identity) (key 'output-files) &allow-other-keys)
262
  (while-collecting (collect)
263 264 265 266
    (map-direct-dependencies
     o c #'(lambda (sub-o sub-c)
             (loop :for f :in (funcall key sub-o sub-c)
                   :when (funcall test f) :do (collect f))))))
267 268

(defmethod input-files ((o bundle-op) (c system))
269
  (direct-dependency-files o c :test 'bundlable-file-p :key 'output-files))
270

271
(defun* select-bundle-operation (type &optional monolithic)
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
  (ecase type
    ((:binary)
     (if monolithic 'monolithic-binary-op 'binary-op))
    ((:dll :shared-library)
     (if monolithic 'monolithic-dll-op 'dll-op))
    ((:lib :static-library)
     (if monolithic 'monolithic-lib-op 'lib-op))
    ((:fasl)
     (if monolithic 'monolithic-fasl-op 'fasl-op))
    ((:program)
     'program-op)))

(defun* make-build (system &rest args &key (monolithic nil) (type :fasl)
                   (move-here nil move-here-p)
                   &allow-other-keys)
287
  (let* ((operation-name (select-bundle-operation type monolithic))
288 289 290 291 292 293
         (move-here-path (if (and move-here
                                  (typep move-here '(or pathname string)))
                             (pathname move-here)
                             (merge-pathnames "./asdf-output/")))
         (operation (apply #'operate operation-name
                           system
294
                           (remove-plist-keys '(:monolithic :type :move-here) args)))
295 296 297 298
         (system (find-system system))
         (files (and system (output-files operation system))))
    (if (or move-here (and (null move-here-p)
                           (member operation-name '(:program :binary))))
299
        (loop :with dest-path = (resolve-symlinks* (ensure-directories-exist move-here-path))
300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
              :for f :in files
              :for new-f = (make-pathname :name (pathname-name f)
                                :type (pathname-type f)
                                :defaults dest-path)
              :do (rename-file-overwriting-target f new-f)
              :collect new-f)
        files)))

;;;
;;; LOAD-FASL-OP
;;;
;;; This is like ASDF's LOAD-OP, but using monolithic fasl files.
;;;

(defmethod component-depends-on ((o load-fasl-op) (c system))
  (declare (ignorable o))
316 317
  `((,o ,@(loop :for dep :in (component-sibling-dependencies c)
                :collect (resolve-dependency-spec c dep)))
318 319 320 321 322 323 324 325 326 327 328 329
    (,(if (user-system-p c) 'fasl-op 'load-op) ,c)
    ,@(call-next-method)))

(defmethod input-files ((o load-fasl-op) (c system))
  (when (user-system-p c)
    (output-files (find-operation o 'fasl-op) c)))

(defmethod perform ((o load-fasl-op) c)
  (declare (ignorable o c))
  nil)

(defmethod perform ((o load-fasl-op) (c system))
330
  (perform-lisp-load-fasl o c))
331 332

(defmethod mark-operation-done :after ((o load-fasl-op) (c system))
333
  (mark-operation-done (find-operation o 'load-op) c))
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351

;;;
;;; PRECOMPILED FILES
;;;
;;; This component can be used to distribute ASDF systems in precompiled form.
;;; Only useful when the dependencies have also been precompiled.
;;;

(defmethod trivial-system-p ((s system))
  (every #'(lambda (c) (typep c 'compiled-file)) (component-children s)))

(defmethod output-files (o (c compiled-file))
  (declare (ignorable o c))
  nil)
(defmethod input-files (o (c compiled-file))
  (declare (ignorable o))
  (component-pathname c))
(defmethod perform ((o load-op) (c compiled-file))
352
  (perform-lisp-load-fasl o c))
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
(defmethod perform ((o load-source-op) (c compiled-file))
  (perform (find-operation o 'load-op) c))
(defmethod perform ((o load-fasl-op) (c compiled-file))
  (perform (find-operation o 'load-op) c))
(defmethod perform (o (c compiled-file))
  (declare (ignorable o c))
  nil)

;;;
;;; Pre-built systems
;;;
(defmethod trivial-system-p ((s prebuilt-system))
  (declare (ignorable s))
  t)

(defmethod perform ((o lib-op) (c prebuilt-system))
  (declare (ignorable o c))
  nil)

372
(defmethod component-depends-on ((o lib-op) (c prebuilt-system))
373 374 375
  (declare (ignorable o c))
  nil)

376
(defmethod component-depends-on ((o monolithic-lib-op) (c prebuilt-system))
377 378 379
  (declare (ignorable o))
  nil)

380

381 382 383 384 385 386
;;;
;;; PREBUILT SYSTEM CREATOR
;;;


(defmethod output-files ((o binary-op) (s system))
387 388
  (list (make-pathname :name (component-name s) :type "asd"
                       :defaults (component-pathname s))))
389 390

(defmethod perform ((o binary-op) (s system))
391 392 393 394 395
  (let* ((dependencies (component-depends-on o s))
         (fasl (first (apply #'output-files (first dependencies))))
         (library (first (apply #'output-files (second dependencies))))
         (asd (first (output-files o s)))
         (name (pathname-name asd))
396
         (name-keyword (intern (string name) (find-package :keyword))))
397
    (with-open-file (s asd :direction :output :if-exists :supersede
398 399 400 401 402 403 404 405 406 407 408 409
                       :if-does-not-exist :create)
      (format s ";;; Prebuilt ASDF definition for system ~A" name)
      (format s ";;; Built for ~A ~A on a ~A/~A ~A"
              (lisp-implementation-type)
              (lisp-implementation-version)
              (software-type)
              (machine-type)
              (software-version))
      (let ((*package* (find-package :keyword)))
        (pprint `(defsystem ,name-keyword
                     :class prebuilt-system
                     :components ((:compiled-file ,(pathname-name fasl)))
410
                     :lib ,(and library (file-namestring library)))
411 412 413
                s)))))

#-(or ecl mkcl)
414
(defmethod perform ((o fasl-op) (c system))
415
  (let* ((input-files (input-files o c))
416 417
         (fasl-files (remove (compile-file-type) input-files :key #'pathname-type :test-not #'string=))
         (non-fasl-files (remove (compile-file-type) input-files :key #'pathname-type :test #'string=))
418 419
         (output-files (output-files o c))
         (output-file (first output-files)))
420
    (unless input-files (format t "WTF no input-files for ~S on ~S !???" o c))
421 422 423 424 425 426 427 428 429 430 431 432 433 434
    (when input-files
      (assert output-files)
      (when non-fasl-files
        (error "On ~A, asdf-bundle can only bundle FASL files, but these were also produced: ~S"
               (implementation-type) non-fasl-files))
      (when (and (typep o 'monolithic-bundle-op)
                 (or (monolithic-op-prologue-code o) (monolithic-op-epilogue-code o)))
        (error "prologue-code and epilogue-code are not supported on ~A"
               (implementation-type)))
      (with-staging-pathname (output-file)
        (combine-fasls fasl-files output-file)))))

(defmethod input-files ((o load-op) (s precompiled-system))
  (declare (ignorable o))
435
  (bundle-output-files (find-operation o 'fasl-op) s))
436 437 438 439 440 441 442 443 444 445 446 447

(defmethod component-depends-on ((o load-fasl-op) (s precompiled-system))
  (declare (ignorable o))
  `((load-op ,s) ,@(call-next-method)))

#| ;; Example use:
(asdf:defsystem :precompiled-asdf-utils :class asdf::precompiled-system :fasl (asdf:apply-output-translations (asdf:system-relative-pathname :asdf-utils "asdf-utils.system.fasl")))
(asdf:load-system :precompiled-asdf-utils)
|#

#+ecl
(defmethod perform ((o bundle-op) (c system))
448 449 450
  (let* ((object-files (input-files o c))
         (output (output-files o c))
         (bundle (first output))
451 452 453 454 455 456 457 458 459 460 461 462
         (kind (bundle-type o)))
    (create-image
     bundle (append object-files (bundle-op-lisp-files o))
     :kind kind
     :entry-point (component-entry-point c)
     :prologue-code
     (when (typep o 'monolithic-bundle-op)
       (monolithic-op-prologue-code o))
     :epilogue-code
     (when (typep o 'monolithic-bundle-op)
       (monolithic-op-epilogue-code o))
     :build-args (bundle-op-build-args o))))
463 464 465

#+mkcl
(progn
466 467 468
  (defmethod perform ((o lib-op) (s system))
    (apply #'compiler::build-static-library (first output)
           :lisp-object-files (input-files o s) (bundle-op-build-args o)))
469

470 471 472
  (defmethod perform ((o fasl-op) (s system))
    (apply #'compiler::build-bundle (second output)
           :lisp-object-files (input-files o s) (bundle-op-build-args o)))
473

474 475 476
  (defun* bundle-system (system &rest args &key force (verbose t) version &allow-other-keys)
    (declare (ignore force verbose version))
    (apply #'operate 'binary-op system args)))
477 478 479 480

#+(or ecl mkcl)
(defun* register-pre-built-system (name)
  (register-system (make-instance 'system :name (coerce-name name) :source-file nil)))