Commit 13dc2b29 authored by Dave Cooper's avatar Dave Cooper
Browse files

commonized containerization scripts in ./docker/

parent e62fc8bb
Loading
Loading
Loading
Loading
Loading
+29 −0
Original line number Diff line number Diff line
@@ -78,4 +78,33 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
- View inputs only: `(the (message-list :category :inputs))`
- For uncertain slots: `(defaulting (the slot-name) default-value)`

## Interactive Evaluation vs File Compilation

When developing Gendl objects, there's an important distinction between interactive evaluation and proper file compilation:

**Interactive Evaluation** (via REPL, HTTP API, or MCP):
- Direct evaluation of `define-object` forms
- May produce misleading syntax errors due to parsing context
- Example: "duplicate slot names" errors that don't reflect actual code issues
- Useful for quick testing but not reliable for syntax validation

**File Compilation** (recommended approach):
```lisp
;; Create a proper .gendl file
(with-temp-file "/path/to/object.gendl" 
  (insert "(define-object my-object (base-object) ...)"))

;; Compile and load properly
(load (compile-file "/path/to/object.gendl"))
```

**Key Benefits of File Compilation:**
- Accurate syntax error reporting
- Proper macro expansion context
- Generates compiled `.lx64fsl` files for faster loading
- Clean compilation indicates syntactically correct code
- No false positive errors from evaluation context

**Best Practice:** When troubleshooting syntax issues or developing complex objects, always test with proper file compilation rather than relying solely on interactive evaluation.

also see in gendl codebase `documentation/historical/doc/usage.txt`
+20 −18
Original line number Diff line number Diff line
@@ -6,39 +6,41 @@ ARG CONT_USER=gendl-user

RUN apt-get -y update && apt-get -y upgrade && apt-get -y autoremove && \
    apt-get -y install openssl libssl-dev iputils-ping telnet ghostscript \
    wamerican rsync curl zlib1g-dev libc6-dev && \
    wamerican rsync curl zlib1g-dev libc6-dev procps && \
    rm -rf /var/cache/apt/* && apt-get clean && \
    groupadd -g 1000 $CONT_USER && \
    useradd -rm -s /bin/bash -u 1000 -g $CONT_USER $CONT_USER && \
    ln -s /home /Users && \
    mkdir -p /opt/gendl && chown -R $CONT_USER:$CONT_USER /opt && \
    mkdir -p /opt/ && chown -R $CONT_USER:$CONT_USER /opt && \
    mkdir -p /src/ && chown -R $CONT_USER:$CONT_USER /src && \
    curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-24.0.7.tgz | \
    tar xz --strip-components=1 -C /usr/local/bin docker/docker \ &&
    tar xz --strip-components=1 -C /usr/local/bin docker/docker && \
    chmod +x /usr/local/bin/docker	  

COPY --chown=$CONT_USER:$CONT_USER gendl/. /opt/gendl/
COPY --chown=$CONT_USER:$CONT_USER gendl-src/docker/build-build/run-gendl-services \
  gendl-src/docker/build-build/start-services.lisp \
  gendl-src/docker/build-build/run-pipeline-tests.sh \
  ./gendl-src/CLAUDE.md /opt/gendl/

COPY --chown=$CONT_USER:$CONT_USER gendl-build/. /opt/gendl/
COPY --chown=$CONT_USER:$CONT_USER gendl/docker/build-build/run-gendl-services \
  gendl/docker/build-build/start-services.lisp \
  gendl/docker/build-build/run-pipeline-tests.sh \
  ./gendl/CLAUDE.md /opt/gendl/
COPY --chown=$CONT_USER:$CONT_USER gendl-src/docker/pipeline-tests.lisp /opt/gendl/gdl/

COPY --chown=$CONT_USER:$CONT_USER gendl/docker/pipeline-tests.lisp /opt/gendl/gdl/
COPY --chown=$CONT_USER:$CONT_USER gendl-src/docker/build-build/etc/rc.local /etc/

COPY --chown=$CONT_USER:$CONT_USER gendl/docker/build-build/etc/rc.local /etc/
COPY --chown=$CONT_USER:$CONT_USER ./gendl-src/regression/ /opt/gendl-regression/

COPY --chown=$CONT_USER:$CONT_USER ./gendl/regression/ /opt/gendl-regression/
COPY --chown=$CONT_USER:$CONT_USER ./gendl-src/gwl/lisply-backend/CLAUDE.md /opt/gendl/CLAUDE-Lisply.md

COPY --chown=$CONT_USER:$CONT_USER ./gendl/gwl/lisply-backend/CLAUDE.md /opt/gendl/CLAUDE-Lisply.md
COPY --chown=$CONT_USER:$CONT_USER ./gendl-src/ /src/gendl/


RUN chown $CONT_USER /opt
RUN chmod +x /opt/gendl/run-pipeline-tests.sh 

USER $CONT_USER
WORKDIR /home/$CONT_USER
ENV HOME=/home/$CONT_USER

RUN chmod +x /opt/gendl/run-pipeline-tests.sh && /opt/gendl/run-pipeline-tests.sh

USER root

RUN chown $CONT_USER /opt
RUN /opt/gendl/run-pipeline-tests.sh

CMD ["/etc/rc.local"]
+86 −71
Original line number Diff line number Diff line
#!/bin/bash
pushd .
set -e
cd $(dirname $0)/../

#
#
# Don't try running this script at home, kids! This depends on the
# existence of build output from our legacy build system which is not
# self-contained in this repo. Hopefully a self-contained docker build
# will be coming soon.
#
#

#
# FLAG -- this build appears to make the build context way too huge!!
#
echo "Containerizing from `pwd`"

if [ -n "${CI_COMMIT_REF_NAME}" ];
then
    branch=${CI_COMMIT_REF_NAME//\//--}
else
    branch="local"
fi

set -e
# export CI_COMMIT_REF_NAME="release/1598"
echo "CI_COMMIT_REF_NAME is: ${CI_COMMIT_REF_NAME}" 
if [ -n "${CI_PIPELINE_ID}" ];
then
    pipeline=${CI_PIPELINE_ID}
else
    pipeline=local
fi

if [ -z "${CI_COMMIT_REF_NAME}" ];
if [ -e /home/builder/genworks/manager/staging/ ];
then
    branch="unspecified"
    staging=/home/builder/genworks/manager/staging/
else
    branch=${CI_COMMIT_REF_NAME//\//--}
    staging=~/build/
fi

build_area=~/tmp/docker-gendl-build/

echo "branch is: ${branch}" 
rm -rf ${build_area}
mkdir -p ${build_area}

cp ./docker/Dockerfile ${build_area} || { echo "Failed to copy Dockerfile"; exit 1; }

cd `dirname $0`/../../
# FLAG -- copy entire gdl source when we start doing the full build in-container
# rsync -a --include='docker/***' --include='regression-gg/***' --exclude='*' ./ ${build_area}/gdl-src/
rsync -a --exclude='.git' ./ ${build_area}/gendl-src/

echo "Present Working Directory: " `pwd`
cd ${build_area}

tag="${branch}"

CI_PIPELINE_ID=${CI_PIPELINE_ID:="9711"}

for impl in  "ccl" "sbcl"
#for impl in  "ccl"
do

    dist_name=gendl-build${CI_PIPELINE_ID}L-linux-${impl}
    dist_name_release=gendl1598-linux-${impl}
    dist_name_release_beta=gendl1598-beta-linux-${impl}

    dist_zip_server=/home/builder/genworks/manager/staging/gendl/${CI_PIPELINE_ID}L/distribution/${dist_name}.zip
    dist_zip_server_release=/home/builder/genworks/manager/staging/gendl/${CI_PIPELINE_ID}L/distribution/${dist_name_release}.zip
    dist_zip_server_release_beta=/home/builder/genworks/manager/staging/gendl/${CI_PIPELINE_ID}L/distribution/${dist_name_release_beta}.zip

    dist_zip_local=~/genworks/builds/${dist_name}.zip

    dist_zip_release_beta_local=~/genworks/builds/${dist_name_release_beta}.zip

    dist_zip_fallback=/tmp/gendl.zip


    if [ -f "$dist_zip_server_release_beta" ]; then
        dist_zip=$dist_zip_server_release_beta
        dist_name=$dist_name_release_beta
    elif [ -f "$dist_zip_server_release" ]; then
        dist_zip=$dist_zip_server_release
        dist_name=$dist_name_release
    elif [ -f "$dist_zip_server" ]; then
        dist_zip=$dist_zip_server
    elif  [ -f "$dist_zip_local" ]; then
        dist_zip=$dist_zip_local
    elif  [ -f "$dist_zip_release_beta_local" ]; then
        dist_zip=$dist_zip_release_beta_local
        dist_name=$dist_name_release_beta
    elif  [ -f "$dist_zip_fallback" ]; then
        dist_zip=$dist_zip_fallback
	dist_name=gendl-bare-local-${impl}
    else
        echo "Neither $dist_zip_server_release nor $dist_zip_server nor $dist_zip_local was found. Exiting."
        exit 1
    if [ "$impl" = "ccl" ]; then
	impl_suffix="-ccl"
    elif [ "$impl" = "sbcl" ]; then
	impl_suffic="-sbcl"
    else echo "Invalid impl: $impl" && exit 1
    fi


    cp $dist_zip .
    if [ "$pipeline" = "local" ]; then
	distro_dir="${HOME}/build/"
	dist_zip="gendl1598-beta-linux${impl_suffix}.zip"
    else
	distro_dir=${staging}/${pipeline}${suffix}/distribution/
	dist_zip=$(basename -- $(find ${distro_dir} -name "*.zip" | grep $maybe_minus_v ${impl_suffi}))
    fi

    unzip -q ${dist_name}.zip
    echo pipeline: ${pipeline}, branch: ${branch}
    echo tag: ${tag}, smp_file_suffix: ${smp_file_suffix}
    echo impl: ${impl}, impl_suffix: ${impl_suffix}
    
    rm ${dist_name}.zip
    echo Copying ${distro_dir}${dist_zip} into place...
    cp ${distro_dir}${dist_zip} ./

    rm -rf gendl-build/
    unzip -q ${dist_zip}
    dist_dir=$(echo ${dist_zip} | sed 's/.zip//')

    mv ${dist_name} gendl-build/
    echo dist_dir is ${dist_dir}
    echo "Moving ${dist_dir} to \"gendl\" in the docker build context area..."
    mv ${dist_dir} gendl
    newtag=${tag}${impl_suffix}

    ls -l ./Dockerfile || { echo "Dockerfile not found"; exit 1; }
    cat  ./Dockerfile || { echo "Failed to read Dockerfile"; exit 1; }
	
    export DOCKER_BUILDKIT=1
    docker build --progress=plain -t genworks/gendl:${branch}-${impl} -f ./gendl/docker/Dockerfile .
    echo "About to do: docker build -t genworks/gendl:${newtag} -f ./Dockerfile --build-arg TAG=${newtag}"
    echo "From: $(pwd)"
    echo ""
	
    rm -rf gendl-build/
    # export DOCKER_BUILDKIT=0
    
    if ! [ ${branch} = "unspecified" ];
    then
        docker push genworks/gendl:${branch}-${impl}
	# echo "Not Pushing for temp build"
    fi
    docker build --progress=plain -t genworks/gendl:"${newtag}" -f ./Dockerfile --build-arg TAG="${newtag}" .
    
    rm -rf gendl
    
    # echo tagging and pushing gendl:${newtag} to the Genworks container registry...
    docker image tag genworks/gendl:${newtag} genworks/gendl:${newtag}
    DOCKER_CONFIG=~/.docker-dockerhub-genworks docker push genworks/gendl:${newtag}
done

#
# Remove things piecewise for safety
#

echo "Cleaning up..."

rm *.zip
rm -r gendl-src/
rm Dockerfile


cd ..

echo "Removing (presumably now empty) directory $build_area"
rmdir $build_area

popd

echo ""
echo "Done with docker build job."
echo ""
+8 −1
Original line number Diff line number Diff line
@@ -167,7 +167,14 @@ if [ -d "/tmp/.X11-unix" ]; then
    echo "X11 forwarding enabled"
fi

DOCKER_GENDL_CID=$(docker run -id --rm -h gendl --name gendl-${branch}-${impl} \
if [ "$INTERACTIVE" = "true" ]; then
    DOCKER_SWITCHES="-it"
else
    DOCKER_SWITCHES="-id"
fi


DOCKER_GENDL_CID=$(docker run $DOCKER_SWITCHES --rm -h gendl --name gendl-${branch}-${impl} \
                          -p ${SWANK_HOST_PORT}:${SWANK_PORT} -p ${HTTP_HOST_PORT}:${HTTP_PORT} \
                          -p ${HTTPS_HOST_PORT}:${HTTPS_PORT} \
                          -e HTTP_PORT -e HTTPS_PORT -e HTTP_HOST_PORT -e HTTPS_HOST_PORT \