Rework ABCL-BUILD as a contrib

ABCL-BUILD now more usefully installs necessary build machinery.

Refactor the ABCL-BUILD Lisp based build to use use Ant, which allows
us to canonicalize the build instructions in a much more configurable
manner, restoring the cross-implementation build of ABCL for CL users
who really do want to know about Java tooling.

Since we want to use ASDF/UIOP anyways, this code is best expressed as
an abcl contrib where such behavior can profitably take place.

TODO: does not pass all tests

TODO: API compatibility so that build-from-lisp.bash works

TODO: If an invocable Ant cannot be found, then offer to install one via restart

TODO: work backwards to the scenario that we are booting in pure
interepreted mode without access to compilers.
parent 42989ed4
......@@ -130,7 +130,7 @@ preferred (and most tested way) is to being to use the Ant build tool:
implementations for this process: SBCL, CMUCL, OpenMCL, Allegro
CL, LispWorks or CLISP.
In all cases you need a Java 5 or later JDK (JDK 1.[5678] have been
In all cases you will need a Java 6 or later JDK (JDK 1.[678] have been
tested). Just the JRE isn't enough, as you need the Java compiler
('javac') to compile the Java source of the ABCL implementation.
......@@ -178,21 +178,9 @@ debug process.
Building from Lisp
------------------
Building from another Common Lisp implementation is the most venerable
and least tested way of building ABCL. It produces a "non-standard"
version of the distribution that doesn't share build instructions with
the previous two methods, but it still may be of interest to those who
absolutely don't want to know anything about Java. The other Common
Lisp implementation does not actually perform any compliation, but is
merely used to orchestrate the invocation of tools in the JDK.
First, copy the file 'customizations.lisp.in' to 'customization.lisp',
in the directory containing this README file, editing to suit your
situation, paying attention to the comments in the file. The critical
step is to have Lisp special variable '*JDK*' point to the root of the
Java Development Kit. Underneath the directory referenced by the
value of '*JDK*' there should be an executable Java compiler in
'bin/javac' ('bin/javac.exe' under Windows).
You need to have a Java compiler installed locally. If the Ant build
tool does not exist, it will be downloaded and used to invoke the Ant
build.
Then, one may either use the 'build-from-lisp.bash' shell script or load
the necessary files into your Lisp image by hand.
......@@ -229,7 +217,7 @@ into limitations on command line length (but is a lot slower).
### Building from another Lisp manually
There is also an ASDF definition in 'abcl.asd' for the BUILD-ABCL
There is also an ASDF definition in 'abcl.asd' for ABCL/BUILD
which can be used to load the necessary Lisp definitions, after which
CL-USER> (build-abcl:build-abcl :clean t :full t)
......
......@@ -138,42 +138,6 @@ be in a directory named '../ansi-test/'."
(:module wrapper :pathname "test/lisp/cl-bench/"
:depends-on (package) :components
((:file "wrapper")))))
;;; aka the "Lisp-hosted build system" which doesn't share build
;;; instructions with the canonical build system in <file:build.xml>
;;; Works for: abcl, sbcl, clisp, cmu, lispworks, allegro, openmcl
(defsystem abcl/build
:version "2.0.0"
:description "Build ABCL from a Lisp. Downloads necessary build-time tools to local cache."
:in-order-to ((test-op (test-op abcl/build/t)))
:components ((:module package
:pathname "src/org/abcl/lisp/build/"
:components ((:file "package")))
(:module build
:pathname "src/org/abcl/lisp/build/"
:depends-on (package)
:serial t
:components (;;; TODO optionally parse a local configuration for customization
(:file "platform")
(:file "customizations-default")
(:file "install")
(:file "ant")
(:file "build-abcl")))))
(defsystem abcl/build/t
:description "Test ABCL build system."
:defsystem-depends-on (prove-asdf)
:depends-on (abcl/build
prove)
:perform (asdf:test-op (op c)
(uiop:symbol-call :prove-asdf 'run-test-system c))
:components ((:module package
:pathname "t/"
:components ((:file "package")))
(:module build
:depends-on (package)
:pathname "t/"
:components ((:test-file "abcl-build")))))
(defsystem abcl/documentation
:description "Tools to generate LaTeX source from docstrings."
:depends-on (swank)
......
......@@ -5,11 +5,10 @@
usage()
{
echo "WARNING: This build method can produce slightly different results than"
echo " the canonical Ant build method."
echo
echo "USAGE:"
echo "$0 <implementation> [[ --clean=T | --full=T | --batch=NIL ]]"
echo "$0 <implementation>"
# echo "$0 <implementation> [[ --clean=T | --full=T | --batch=NIL ]]"
}
if [ -z "$1" ]; then
......@@ -39,10 +38,12 @@ if [ $? != 0 ] ; then
fi
eval set -- "$TEMP"
# TODO all ignored
CLEAN="t"
FULL="t"
BATCH="t"
# TODO all ignored
while true ; do
case "$1" in
--clean)
......@@ -65,21 +66,8 @@ while true ; do
esac
done
FORM="(build-abcl:build-abcl :clean $CLEAN :full $FULL :batch $BATCH)"
FILE="build-abcl.lisp"
if [[ ! -r customizations.lisp ]]; then
cat <<-EOF
Failed to find configuration in 'customizations.lisp'. Please copy
'customizations.lisp.in' to 'customizations.lisp', editing it to
reflect local configuration.
EOF
exit 1
fi
FORM="(abcl/build:abcl/build)"
FILE="src/org/abcl/lisp/build/build-abcl.lisp"
abcl()
{
......
......@@ -79,10 +79,6 @@ For help on the automatic tests available, use the Ant target 'help.test'.
<exclude name="org/armedbear/lisp/j.lisp"/>
<include if="abcl.jsr-223.p" name="org/armedbear/lisp/scripting/lisp/*.lisp"/>
</patternset>
<patternset id="abcl.source.lisp.build">
<include name="org/abcl/lisp/build/*.lisp"/>
</patternset>
<!-- Lisp files required at runtime -->
<patternset id="abcl.source.lisp.dist">
......@@ -210,8 +206,6 @@ For help on the automatic tests available, use the Ant target 'help.test'.
<src path="${src.dir}"/>
<patternset refid="abcl.source.java"/>
</javac>
<echo message="${build}"
file="${build.classes.dir}/org/armedbear/lisp/build"/>
</target>
<!-- Additional artifacts to stage relative to Ant ${basedir} -->
......@@ -233,9 +227,6 @@ For help on the automatic tests available, use the Ant target 'help.test'.
<fileset dir="${basedir}/">
<patternset refid="abcl.stage"/>
</fileset>
<fileset dir="${src.dir}">
<patternset refid="abcl.source.lisp.build"/>
</fileset>
</copy>
</target>
......
......@@ -65,11 +65,16 @@ abcl-introspect
Advanced introspection of Java and Lisp runtime classes
representation.
abcl-build
The ABCL build system plus associated utilities for manipulating
path executables
# Colophon
Mark <evenson.not.org@gmail.com>
Created: 2011-09-11
Revised: 2016-12-28
Revised: 2017-05-22
<> abcl:documents <abcl.org/release/1.5.0/#abcl-contrib.jar> .
......
ABCL-BUILD
==========
Installing the necessary toolchain to build ABCL.
Plus some utility functions to download needed artifacts from the
network.
;;; aka the "Lisp-hosted build system" which doesn't share build
;;; instructions with the canonical build system in <file:build.xml>
;;; Works for: abcl, sbcl, clisp, cmu, lispworks, allegro, openmcl
(defsystem abcl-build
:version "2.0.0"
:description "Build ABCL from a Lisp. Downloads necessary build-time tools to local cache."
:in-order-to ((test-op (test-op abcl-build/t)))
:components ((:module package
:pathname "build/"
:components ((:file "package")))
(:module util
:pathname "build/"
:depends-on (package)
:components ((:file "util")))
(:module build
:pathname "build/"
:depends-on (util)
:serial t
:components (;;; TODO optionally parse a local configuration for customization
(:file "customizations-default")
(:file "install")
(:file "maven")
(:file "ant")
(:file "abcl-build") ;; TODO: support API
(:file "deprecated")))))
(defsystem abcl-build/t
:description "Test ABCL build system."
:defsystem-depends-on (prove-asdf)
:depends-on (abcl-build
prove)
:perform (asdf:test-op (op c)
(uiop:symbol-call :prove-asdf 'run-test-system c))
:components ((:module package
:pathname "build/t/"
:components ((:file "package")))
(:module build
:depends-on (package)
:pathname "build/t/"
:components ((:test-file "util")
(:test-file "install")
(:test-file "ant")
(:test-file "maven")
(:test-file "abcl-build")))))
(in-package :abcl/build)
(defun make-dist (version-string)
(warn "Unimplemented"))
(defun build-abcl (&key force
(batch t)
compile-system
jar
clean
full)
(warn "Unimplemented"))
(in-package :abcl/build)
;; TODO function to deal with looking up a locally preferred mirrors
(defun ant-zip-uri ()
#p"http://archive.apache.org/dist/ant/binaries/apache-ant-1.9.4-bin.zip"
#+nil ;; https on OPEN fails; probably attempting to upgrade
#p"https://archive.apache.org/dist/ant/binaries/apache-ant-1.9.4-bin.zip"
#+nil ;; need apache-ant-1.9 for JVM version 49.0
#p"http://www-eu.apache.org/dist/ant/binaries/apache-ant-1.10.1-bin.zip")
(defun xdg/ant-executable ()
(probe-file
(localize-executable-name
(merge-pathnames #p"bin/ant"
(xdg/abcl-install-root (ant-zip-uri))))))
(defun ant/install ()
(unless (xdg/ant-executable)
(xdg/install (ant-zip-uri) :type :unzip))
(values
(xdg/ant-executable)
(directory (merge-pathnames "**/*"
(xdg/abcl-install-root (ant-zip-uri))))))
(defparameter *ant-home* nil)
(define-condition no-installed-ant (error)
((searched))
(:report (lambda (condition stream)
(format stream "Unable to introspect Apache Ant installation."))))
;; TODO after this routines executes *ANT-EXECUTABLE-DIRECTORY* and XDG/ANT-EXECUTABLE will work
(defun ensure-ant (&key (ant-home nil ant-home-p))
(cond
((and (null ant-home) ant-home-p)
(warn "Unimplemented explicit auto-configuration run."))
((and ant-home ant-home-p)
(warn "Unimplemented explicit configuration with specified directory directory."))
(t
(if *ant-home*
*ant-home*
(restart-case
(let ((ant-home (some-directory-containing "ant")))
(unless ant-home
(signal 'no-installed-ant))
(setf *ant-home ant-home))
(install-ant ()
(ant/install)))))))
(defmacro with-ensured-ant ((ant) &body body)
`(progn
(unless ,ant
(setf ,ant (ensure-ant)))
,@body))
(defun ant/call (ant-file target-or-targets)
(let (ant)
(with-ensured-ant (ant)
(uiop:run-program
`(,ant "-buildfile"
,(stringify ant-file)
,@(listify target-or-targets))
:output :string))))
(in-package :abcl/build)
(defun abcl/build ()
(abcl-build:ant/call (asdf:system-relative-pathname :abcl "build.xml")
"abcl"))
(defun abcl/dist ()
(abcl-build:ant/call (asdf:system-relative-pathname :abcl "build.xml")
"abcl.release"))
(defun abcl/test ()
(abcl-build:ant/call (asdf:system-relative-pathname :abcl "build.xml")
"abcl.test"))
......@@ -17,10 +17,22 @@
;;; Directories should be specified with a trailing slash (or, on Windows, a
;;; trailing backslash).
(in-package "BUILD-ABCL")
(in-package :abcl/build)
;; Standard compiler options.
(setq *javac-options* "-g")
(setq *jikes-options* "+D -g")
(defparameter *javac-options*
"-g")
(defparameter *jikes-options*
"+D -g")
(defparameter *jdk*
(cond
((uiop:os-macosx-p)
"/usr/")
(t
(introspect-path-for "javac"))))
;;; build-abcl.lisp
;;;; Historic cross platform build infrastructure
;;;; N.b. currently unused in favor of canonicalizing build.xml
#+abcl
(require 'format)
(in-package :abcl/build)
(in-package #:build-abcl)
(defun comp (string char)
(defun chop-end-from-char (string char)
"Chops off the character at the end of `string' if it matches char"
(let ((len (length string)))
(if (eql char (char string (1- len)))
......@@ -16,11 +14,10 @@
(let ((string (namestring pathname)))
(when (position #\space string)
(setf string (concatenate 'string "\""
(comp string #\\)
(chop-end-from-char string #\\)
"\"")))
string))
(defun child-pathname (pathname parent)
"Returns `pathname' relative to `parent', assuming that it
is infact a child of it while being rooted at the same root as `parent'."
......@@ -41,127 +38,11 @@ is infact a child of it while being rooted at the same root as `parent'."
(> (file-write-date orig)
(file-write-date artifact))))
(defparameter *file-separator-char*
(if (eq *platform* :windows) #\\ #\/))
(if (uiop:os-windows-p) #\\ #\/))
(defparameter *path-separator-char*
(if (eq *platform* :windows) #\; #\:))
#+sbcl
(defun run-shell-command (command &key directory (output *standard-output*))
(when directory
(setf command (concatenate 'string
"\\cd \""
(namestring (pathname directory))
"\" && "
command)))
(sb-ext:process-exit-code
(sb-ext:run-program
"/bin/sh"
(list "-c" command)
:input nil :output output)))
#+cmu
(defun run-shell-command (command &key directory (output *standard-output*))
(when directory
(setf command (concatenate 'string
"\\cd \""
(namestring (pathname directory))
"\" && "
command)))
(ext::process-exit-code
(ext:run-program
"/bin/sh"
(list "-c" command)
:input nil :output output)))
#+clisp
(defun run-shell-command (command &key directory (output *standard-output*))
(declare (ignore output)) ;; FIXME
(let (status old-directory)
(when directory
(setf old-directory (ext:cd))
(ext:cd directory))
(unwind-protect
(setf status (ext:shell command))
(when old-directory
(ext:cd old-directory)))
(cond ((numberp status)
status)
((or (eq status t) (null status)) ;; clisp 2.47 returns NIL on success
0)
(t
-1))))
#+lispworks
(defun run-shell-command (command &key directory (output *standard-output*))
(when directory
(unless (eq *platform* :windows)
(setf command (concatenate 'string
"\\cd \""
(namestring (pathname directory))
"\" && "
command))))
(system:call-system-showing-output command
:shell-type "/bin/sh"
:output-stream output))
#+allegro
(defun run-shell-command (command &key directory (output *standard-output*))
(excl:run-shell-command command
:directory directory
:input nil
:output #+ide nil #-ide output))
#+openmcl
(defun run-shell-command (command &key directory (output *standard-output*))
(when directory
(setf command (concatenate 'string
"\\cd \""
(namestring (pathname directory))
"\" && "
command)))
(multiple-value-bind (status exitcode)
(ccl:external-process-status
(ccl:run-program
"/bin/sh"
(list "-c" command)
:wait t :input nil :output output))
(declare (ignore status))
exitcode))
#+ecl
(defun run-shell-command (command &key directory (output *standard-output*))
(when directory
(if (member :windows *features*)
(error "Unimplemented.")
(setf command (concatenate 'string
"\\cd \""
(namestring (pathname directory))
"\" && "
command))))
(ext:system command))
;; (multiple-value-bind (stream exit details)
;; (ext:run-program
;; "/bin/sh" (list "-c" command)
;; :input nil :output :stream :error :output)
;; (declare (ignore details))
;; (loop for line = (read-line stream nil)
;; while line do (format output "~A~%" line))
;; exit))
#+(or sbcl cmu lispworks openmcl ecl)
(defun probe-directory (pathspec)
(let* ((truename (probe-file pathspec)) ; TRUENAME is a pathname.
(namestring (and truename (namestring truename)))) ; NAMESTRING is a string.
(and namestring
(> (length namestring) 0)
(eql (char namestring (1- (length namestring))) *file-separator-char*)
truename)))
(if (uiop:os-windows-p) #\; #\:))
(defparameter *tree-root*
(make-pathname :device (pathname-device *load-truename*)
......@@ -173,7 +54,6 @@ is infact a child of it while being rooted at the same root as `parent'."
(defparameter *dist-root*
(merge-pathnames "dist/" *tree-root*))
(defparameter *customizations-file*
(merge-pathnames "customizations.lisp" *tree-root*))
......@@ -192,30 +72,20 @@ is infact a child of it while being rooted at the same root as `parent'."
(defvar *java-compiler-command-line-prefix*)
(defun initialize-build ()
(setf *jdk* nil
*java-compiler* nil
*javac-options* nil
*jikes-options* nil
*jar* nil)
(load *customizations-file*)
(setf *java* (probe-file (merge-pathnames (if (eq *platform* :windows)
"bin\\java.exe"
"bin/java")
*jdk*)))
;;; FIXME: highly breakable; user shouldn't be reading
(load (asdf:system-relative-pathname :build-abcl
"src/org/abcl/lisp/build/customizations-default.lisp"))
(setf *java*
(introspect-path-for "java"))
(unless *java*
(error "Can't find Java executable."))
(unless *java-compiler*
(setf *java-compiler* (merge-pathnames (if (eq *platform* :windows)
"bin/javac.exe"
"bin/javac")
*jdk*)))
(setf *java-compiler* (introspect-path-for "java")))
(unless *jar*
(setf *jar* (merge-pathnames (if (eq *platform* :windows)
"bin/jar.exe"
"bin/jar")
*jdk*)))
(setf *jar* (introspect-path-for "jar")))
(let ((classpath-components (list *source-root*
(if (eq *platform* :darwin)
(if (uiop:os-macosx-p)
#p"/System/Library/Frameworks/JavaVM.framework/Classes/classes.jar"
(merge-pathnames "jre/lib/rt.jar" *jdk*)))))
(setf *classpath*
......@@ -270,8 +140,14 @@ is infact a child of it while being rooted at the same root as `parent'."
(namestring source-file)))
(defun java-compile-file (source-file)
(let ((cmdline (build-javac-command-line source-file)))
(equal 0 (run-shell-command cmdline :directory *abcl-dir*))))
(let ((command-line (build-javac-command-line source-file)))
;; TODO: detect failure of invocation
(values
(uiop:run-program command-line
:directory *abcl-dir*
:output :string))
cmdline))
(defun do-compile-classes (force batch)
(let* ((source-files
......@@ -322,9 +198,9 @@ is infact a child of it while being rooted at the same root as `parent'."
(when (position #\space jar-namestring)
(setf jar-namestring (concatenate 'string "\"" jar-namestring "\"")))
(let ((substitutions-alist (acons "@JAR@" jar-namestring nil))
(source-file (if (eq *platform* :windows) "make-jar.bat.in" "make-jar.in"))
(target-file (if (eq *platform* :windows) "make-jar.bat" "make-jar"))
(command (if (eq *platform* :windows) "make-jar.bat" "sh make-jar")))
(source-file (if (uiop:os-windows-p) "make-jar.bat.in" "make-jar.in"))
(target-file (if (uiop:os-windows-p) "make-jar.bat" "make-jar"))
(command (if (uiop:os-windows-p) "make-jar.bat" "sh make-jar")))
(copy-with-substitutions source-file target-file substitutions-alist)
(ensure-directories-exist *dist-root*)
(let ((status (run-shell-command command :directory *tree-root*)))
......@@ -340,14 +216,14 @@ is infact a child of it while being rooted at the same root as `parent'."
status
(abcl-home (substitute-in-string
(namestring *abcl-dir*)
(when (eq *platform* :windows)
(when (uiop:os-windows-p)
'(("\\" . "/")
("/" . "\\\\")))))
(output-path (substitute-in-string
(namestring
(merge-pathnames "build/classes/org/armedbear/lisp/"
*tree-root*))
(when (eq *platform* :windows)
(when (uiop:os-windows-p)
'(("\\" . "/")))))
(cmdline (format nil
"~A -cp build/classes -Dabcl.home=\"~A\" ~
......@@ -367,7 +243,7 @@ org.armedbear.lisp.Main --noinit --nosystem ~
(defun make-launch-script ()
;; Use the -Xss4M and -Xmx256M flags so that the default launch script can be
;; used to build sbcl.
(cond ((eq *platform* :windows)
(cond ((uiop:os-windows-p)
(with-open-file (s
(merge-pathnames "abcl.bat" *tree-root*)
:direction :output
......@@ -436,7 +312,7 @@ org.armedbear.lisp.Main --noinit --nosystem ~
(delete-files (mapcan #'(lambda (name)
(directory (merge-pathnames name default)))
(cdr f)))))))
#+(or)
(defun build-abcl (&key force
(batch t)
compile-system
......@@ -446,16 +322,11 @@ org.armedbear.lisp.Main --noinit --nosystem ~
(let ((start (get-internal-real-time)))
#+lispworks
(when (eq *platform* :windows)
(when (uiop:os-windows-p)
(setf batch nil))
(initialize-build)
(format t "~&Platform: ~A~%"
(case *platform*
(:windows "Windows")
(:linux "Linux")
(:darwin "Mac OS X")
(t (software-type))))
(format t "~&Platform: ~A~%" (software-type))
(finish-output)
;; clean
(when clean
......@@ -521,8 +392,8 @@ org.armedbear.lisp.Main --noinit --nosystem ~
(merge-pathnames file target-dir))))
(defun make-dist-dir (version-string)
(unless (eq *platform* :linux)
(error "MAKE-DIST is only supported on Linux."))
(unless (uiop:os-unix-p)
(error "MAKE-DIST is only supported on Unices."))
(let ((target-root (pathname (concatenate 'string "/var/tmp/" version-string "/"))))
(when (probe-directory target-root)
(error "Target directory ~S already exists." target-root))
......@@ -562,6 +433,7 @@ org.armedbear.lisp.Main --noinit --nosystem ~
(copy-files files source-dir target-dir))
target-root))
#+(or)
(defun make-dist (version-string)
(let* ((dist-dir (make-dist-dir version-string))
(parent-dir (merge-pathnames (make-pathname :directory '(:relative :back))
......
#-abcl (error "Sorry, but this only currently works with the Bear.")
(in-package :abcl/build)
(defun xdg/abcl-install-root (uri)
"Return the private xdg rooted installation location for URI."
(merge-pathnames
(make-pathname :directory `(:relative "abcl" "install" ,(pathname-name uri)))
(uiop/configuration:xdg-data-home)))
(defun xdg/abcl-download-root (&key (for-uri nil for-uri-p))
(declare (ignore for-uri-p))
(let ((root (merge-pathnames
(make-pathname :directory '(:relative "abcl" "dist"))
(uiop/configuration:xdg-data-home)))) ;; TODO move to proper XDG cache hierarchy
(unless for-uri
(return-from xdg/abcl-download-root root))
(let* ((uri (if (pathnamep for-uri)
for-uri
(pathname for-uri)))
(name (pathname-name uri)))
(merge-pathnames
(make-pathname :directory `(:relative ,name))
root))))
(defgeneric xdg/install ((uri pathname) &key type)
(:method ((uri pathname) &key (type :unzip))
(declare (ignore type))
(download-and-unzip uri)))
(defun download-and-unzip (uri)
(let ((archive
(download uri))
(root
(xdg/abcl-download-root :for-uri uri)))
(ensure-directories-exist root)
(sys:unzip archive root)
(values
root
(directory (merge-pathnames "**/*" root)))))
(defun download (uri &key (destination
(merge-pathnames
(make-pathname :defaults uri :host nil :device nil :directory nil)
(xdg/abcl-download-root))))
"Download the contents of URI to DESTINATION.
Returns the local pathname of the download artifact."
(ensure-directories-exist destination)