Alpine 3.23, Test vector, Clean up output, reduce messages

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   -
     repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       -
         id: check-ast

build.sh

@@ -24,8 +24,21 @@ if ! >"$log" 2>&1 docker build \
     --build-arg "BASE_IMAGE_DIGEST=${BASE_IMAGE_DIGEST}" \
     --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
     --build-arg "BUILD_ROOT=/d" \
-    -f Dockerfile.alpine \
-    -t "$IMAGE_TAG" \
+    -f buildroot/Dockerfile.alpine \
+    -t "${IMAGE_TAG}-buildroot" \
+    . ; then
+    >&2 echo 'Unable to build '"${IMAGE_TAG}-buildroot"'!'
+    >&2 cat "$log"
+    exit 8;
+fi
+if ! >"$log" 2>&1 docker build \
+    --build-arg "ALPINE_VERSION=${ALPINE_VERSION}" \
+    --build-arg "BASE_IMAGE_DIGEST=${BASE_IMAGE_DIGEST}" \
+    --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
+    --build-arg "BUILD_ROOT=/d" \
+    --build-arg "SOURCE_IMAGE=${IMAGE_TAG}-buildroot" \
+    -f buildroot/Dockerfile.alpine \
+    -t "${IMAGE_TAG}" \
     . ; then
     >&2 echo 'Unable to build '"$IMAGE_TAG"'!'
     >&2 cat "$log"

Dockerfile.alpine → buildroot/Dockerfile.alpine

@@ -1,12 +1,16 @@
-#syntax=docker/dockerfile:1
+# syntax=docker/dockerfile:1.20-labs
+
 
 ARG ALPINE_VERSION=3.20
 ARG PYTHON_VERSION=3.12
 ARG SOURCE_IMAGE=docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
 ARG BASE_IMAGE_DIGEST
 
-FROM ${SOURCE_IMAGE}@${BASE_IMAGE_DIGEST} AS buildroot
+FROM ${SOURCE_IMAGE}@${BASE_IMAGE_DIGEST} AS source
+FROM source AS buildroot
 ARG PYTHON_VERSION=3.12
+ARG TARGETARCH
+ARG TARGETVARIANT
 
 ARG BUILD_ROOT='/dest'
 ARG CACHE_ROOT='/cache'
@@ -17,16 +21,17 @@
     _apk_add="/usr/bin/env apk add --root $BUILD_ROOT --no-cache" \
     _apk_del="/usr/bin/env apk del --root $BUILD_ROOT --purge" \
     _sh="chroot $BUILD_ROOT sh" \
-    _ln="chroot $BUILD_ROOT ln" \
+    _ln="chroot $BUILD_ROOT /bin/ln" \
     _chroot="chroot $BUILD_ROOT"
 
+COPY --from=source /dev /$BUILD_ROOT/dev
 ADD --chmod=0755 chroot-apk.sh /usr/local/bin/chroot-apk
 ADD --chmod=0755 chroot-pip.sh /usr/local/bin/chroot-pip
 ADD --chmod=0755 chroot-ln.sh /usr/local/bin/chroot-ln
 ADD --chmod=0755 remove-py-if-pyc-exists.sh /usr/local/bin/remove-py-if-pyc-exists
-RUN set -eu ; \
-    python -m pip install -U pip setuptools ; \
-    # Add to buildroot:
+ADD --chmod=0755 chroot-exec.sh /usr/local/bin/chroot-exec
+RUN \
+    set -eu ; \
     $_sys_apk_add \
         dash \
         # TLS certs
@@ -36,18 +41,16 @@
         # be imported from. This makes the stdlib immutable.
         zip \
     ; \
-    # remove all ``__pycache__`` directories
-    find /usr/local/lib/python$PYTHON_VERSION -type d -name '__pycache__' -print0 | xargs -0 rm -rf ; \
-    # compile all py to an adjacent pyc and remove the original, leaving only the bytecode
-    python -m compileall -b /usr/local/lib/python$PYTHON_VERSION ; \
-    find -type f -name '*.py' -exec sh -c "remove-py-if-pyc-exists {}" \; ;\
     # make the new root:
     mkdir -p \
         $CACHE_ROOT/ \
         $BUILD_ROOT/etc \
         $BUILD_ROOT/bin \
         $BUILD_ROOT/usr/local/lib/python$PYTHON_VERSION/site-packages \
         $BUILD_ROOT/usr/local/bin \
+        $BUILD_ROOT/proc \
+        $BUILD_ROOT/sys \
+        $BUILD_ROOT/dev \
     ; \
     # copy the apk related confs
     cp -R /etc/apk $BUILD_ROOT/etc/apk ; \
@@ -57,68 +60,73 @@
         alpine-release \
         musl \
         libffi \
-        dash \
-        dash-binsh \
-        coreutils-env \
         ; \
+    cp -p /bin/busybox $BUILD_ROOT/bin/busybox ; \
+    chroot $BUILD_ROOT /bin/busybox busybox ln -sf /bin/busybox /bin/ln
+
+RUN --security=insecure \
+    set -eu ; \
+    mount -t proc none $BUILD_ROOT/proc ; \
+    mount -t sysfs none $BUILD_ROOT/sys ; \
+    mount --rbind /dev /$BUILD_ROOT/dev ; \
     $_apk_add \
+        busybox \
+        dash \
+        dash-binsh \
+    ; \
+    T=$(mktemp -d) ; \
+    if [ -f $BUILD_ROOT/lib/apk/db/scripts.tar.gz ]; then \
+        tar -C "$T" -xzpf $BUILD_ROOT/lib/apk/db/scripts.tar.gz ; \
+        rm -f $BUILD_ROOT/lib/apk/db/scripts.tar.gz ; \
+        find "$T" -name 'busybox-*' -delete ; \
+        tar -C "$T" -cpvzf $BUILD_ROOT/lib/apk/db/scripts.tar.gz  . ; \
+        rm -rf "$T" ; \
+    fi ;  \
+    tar -C "$BUILD_ROOT" -cpf - etc/apk bin/ln bin/busybox var/cache/apk usr/share/apk | tar -C "$CACHE_ROOT" -xpf - ; \
+    rm -rf $BUILD_ROOT/bin/ln $BUILD_ROOT/bin/busybox $BUILD_ROOT/etc/apk $BUILD_ROOT/var/cache/apk $BUILD_ROOT/usr/share/apk && \
+    chroot-apk add \
         ca-certificates \
         # needed for update-ca-certificates to work:
-        run-parts \
-        # install the runtime dependencies for python
+        run-parts
+
+RUN \
+    --mount=type=cache,id=pip-cache-${TARGETARCH}${TARGETVARIANT},sharing=shared,target=/root/.cache/pip \
+    set -eu ; \
+    export SOURCE_DATE_EPOCH=0 ; \
+    chroot-apk add \
+        coreutils-env \
         $(apk info -R .python-rundeps | grep -vE ':$') \
         ; \
-    cp -p /bin/busybox $BUILD_ROOT/bin/busybox ; \
-    ls -lt $BUILD_ROOT/bin/busybox ; \
-    chroot $BUILD_ROOT /bin/busybox ln -sf /bin/busybox /bin/ln ; \
-    # copy dash into the container so we can use it as the default bin/sh
-    # tar -C / -cpf - $(\
-    #     apk info -L \
-    #         dash \
-    #         dash-binsh \
-    #         ca-certificates \
-    #     | grep -vE ':$' \
-    #     ) | tar -C $BUILD_ROOT -xpf - ; \
-    # $_ln -sf /usr/bin/dash /bin/sh ; \
+    python -m pip install -U pip setuptools ; \
+    # remove all ``__pycache__`` directories
+    find /usr/local/lib/python$PYTHON_VERSION -type d -name '__pycache__' -print0 | xargs -0 rm -rf ; \
+    # compile all py to an adjacent pyc and remove the original, leaving only the bytecode
+    python -m compileall -q -b /usr/local/lib/python$PYTHON_VERSION ; \
+    find -type f -name '*.py' -exec sh -c "remove-py-if-pyc-exists -q {}" \; ;\
     (\
         cd /usr/local/lib && \
         tar -C /usr/local/lib -cpf - python$PYTHON_VERSION/lib-dynload libpython*  | tar -C $BUILD_ROOT/usr/local/lib -xpf - ; \
         tar -C /usr/local/bin -cpf - python*  | tar -C $BUILD_ROOT/usr/local/bin -xpf -; \
-        (cd python$PYTHON_VERSION && zip -9 -X $BUILD_ROOT/usr/local/lib/python$(echo $PYTHON_VERSION | tr -d '.').zip $(\
+        (cd python$PYTHON_VERSION && zip -q -9 -X $BUILD_ROOT/usr/local/lib/python$(echo $PYTHON_VERSION | tr -d '.').zip $(\
             find . | grep -vE "(__pycache__|^\./(test|site-packages|lib-dynload|idlelib|lib2to3|tkinter|turtle|ensurepip))" \
         )); \
         cp -p python$PYTHON_VERSION/os.pyc $BUILD_ROOT/usr/local/lib/python$PYTHON_VERSION/os.pyc ; \
         touch $BUILD_ROOT/usr/local/lib/python$PYTHON_VERSION/ensurepip.py ; \
         rm $BUILD_ROOT/usr/local/lib/python$PYTHON_VERSION/lib-dynload/_tkinter* ; \
     ) && \
-    $_ln -sf /usr/local/bin/python$PYTHON_VERSION /usr/local/bin/python3 && \
-    $_ln -sf /usr/local/bin/python$PYTHON_VERSION /usr/local/bin/python && \
-    tar -C "$BUILD_ROOT" -cpf - etc/apk bin/ln bin/busybox var/cache/apk usr/share/apk | tar -C "$CACHE_ROOT" -xpf - ; \
-    rm -rf $BUILD_ROOT/bin/ln $BUILD_ROOT/bin/busybox $BUILD_ROOT/etc/apk $BUILD_ROOT/var/cache/apk $BUILD_ROOT/usr/share/apk && \
+    chroot-ln -sf /usr/local/bin/python$PYTHON_VERSION /usr/local/bin/python3 && \
+    chroot-ln -sf /usr/local/bin/python$PYTHON_VERSION /usr/local/bin/python && \
     # regenerate the ca-certs!
-    chroot $BUILD_ROOT update-ca-certificates && \
-    chroot-pip install --force-reinstall setuptools
+    chroot-exec update-ca-certificates && \
+    chroot-pip --optimize install --force-reinstall setuptools
 
-FROM scratch AS distroless-python
-ARG ALPINE_VERSION=3.20
-ARG PYTHON_VERSION=3.12
-ARG SOURCE_IMAGE=docker.io/python:${PYTHON_VERSION}-alpine${ALPINE_VERSION}
-ARG BASE_IMAGE_DIGEST
-ARG BUILD_ROOT='/dest'
-ENV BUILD_ROOT=$BUILD_ROOT \
-    PYTHON_VERSION=$PYTHON_VERSION \
-    ALPINE_VERSION=$ALPINE_VERSION
-COPY --from=buildroot $BUILD_ROOT /
-LABEL \
-    org.opencontainers.image.authors="distroless-python image developers <[email protected]>" \
-    org.opencontainers.image.source="https://github.com/autumnjolitz/distroless-python" \
-    org.opencontainers.image.title="Distroless Python ${PYTHON_VERSION} on alpine${ALPINE_VERSION}" \
-    org.opencontainers.image.description="Distroless, optimized Python images distilled from the DockerHub official Python images. These images only have a python interpreter and the dash shell." \
-    org.opencontainers.image.base.digest="${BASE_IMAGE_DIGEST}" \
-    org.opencontainers.image.base.name="$SOURCE_IMAGE" \
-    distroless.python-version="${PYTHON_VERSION}" \
-    distroless.alpine-version="${ALPINE_VERSION}" \
-    distroless.base-image="alpine${ALPINE_VERSION}"
+COPY --chmod=755 <<EOF /$BUILD_ROOT/usr/local/lib/python$PYTHON_VERSION/ensurepip.py
+import sys
 
-SHELL ["/usr/bin/dash", "-c"]
-ENTRYPOINT [ "/usr/local/bin/python" ]
+if __name__ == "__main__":
+    print("""
+ensurepip is removed to save space
+please install the package in the buildroot image via chroot-pip
+and then COPY it over (using --from=) into your image root.
+""", sys.stderr)
+EOF

chroot-apk.sh

@@ -5,31 +5,56 @@ if [ "x$CACHE_ROOT" = 'x' ] || [ "x$BUILD_ROOT" = 'x' ]; then
     exit 1
 fi
 
+DEBUG="${CHROOT_APK_DEBUG:-0}"
+
 set -e
 set -o pipefail
 
 setup () {
-    >&2 echo "Grafting $CACHE_ROOT into $BUILD_ROOT..."
-    tar -C "$CACHE_ROOT" -cpf - . | tar -C "$BUILD_ROOT" -xpf -
+    local extra=''
+    if [ "$DEBUG" = '1' ]; then
+        extra='-v'
+        >&2 echo "Grafting $CACHE_ROOT into $BUILD_ROOT..."
+    fi
+    tar -C "$CACHE_ROOT" -cpf - . | eval tar -C "$BUILD_ROOT" -xpf $extra -
     return $?
 }
 
 fini () {
-    >&2 echo "Removing APK data from $BUILD_ROOT, storing in $CACHE_ROOT"
-    tar -C "$BUILD_ROOT" -cpf - etc/apk bin/ln bin/busybox var/cache/apk usr/share/apk | tar -C "$CACHE_ROOT" -xpf -
+    local rc=$?
+    local extra=''
+    if [ "$DEBUG" = '1' ]; then
+        >&2 echo "Removing APK data from $BUILD_ROOT, storing in $CACHE_ROOT"
+        extra='-v'
+    fi
+    local T="$(mktemp -d)"
+    if [ -f $BUILD_ROOT/lib/apk/db/scripts.tar.gz ]; then
+        tar -C "$T" -xzpf $BUILD_ROOT/lib/apk/db/scripts.tar.gz
+        rm -f $BUILD_ROOT/lib/apk/db/scripts.tar.gz
+        sed -i'' 's|^#!busybox sh|#!/usr/bin/dash|g' $(find "$T" -type f -print)
+        sed -i'' 's|^#!/bin/sh|#!/usr/bin/dash|g' $(find "$T" -type f -print)
+        sed -i'' 's|^#!/bin/busybox sh|#!/usr/bin/dash|g' $(find "$T" -type f -print)
+        cat $(find "$T" -type f -print)
+        tar -C "$T" -cpvzf $BUILD_ROOT/lib/apk/db/scripts.tar.gz  .
+        rm -rf "$T"
+    fi
+
+    mkdir -p $BUILD_ROOT/var/cache/apk
+    tar -C "$BUILD_ROOT" -cpf - etc/apk bin/ln bin/busybox var/cache/apk usr/share/apk | eval tar -C "$CACHE_ROOT" -xpf $extra -
     $_chroot /bin/ln -sf /usr/bin/dash /bin/sh.bak
     rm -rf $BUILD_ROOT/bin/ln $BUILD_ROOT/bin/busybox $BUILD_ROOT/etc/apk $BUILD_ROOT/var/cache/apk $BUILD_ROOT/usr/share/apk
     if $_chroot /usr/bin/dash -c '[ ! -x /bin/sh ]'; then
         >&2 echo '/bin/sh in chroot failed the vibe check, replacing with a symlink to /usr/bin/dash!'
         mv $BUILD_ROOT/bin/sh.bak $BUILD_ROOT/bin/sh
     else
-        >&2 echo '/bin/sh passed the vibe check'
+        if [ "$DEBUG" = '1' ]; then
+            >&2 echo '/bin/sh passed the vibe check'
+        fi
         rm $BUILD_ROOT/bin/sh.bak
     fi
-    return $?
+    exit $rc
 }
 
 trap fini EXIT
 setup
-set -x
-apk --root "$BUILD_ROOT" $@
+apk --root "$BUILD_ROOT" $@

chroot-exec.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env sh
+
+if [ "x$CACHE_ROOT" = 'x' ] || [ "x$BUILD_ROOT" = 'x' ]; then
+    >&2 echo "CACHE_ROOT (${CACHE_ROOT:-not-set}) and/or BUILD_ROOT (${BUILD_ROOT:-not-set}) is not set!"
+    exit 1
+fi
+
+DEBUG="${CHROOT_EXEC_DEBUG:-0}"
+
+set -e
+set -o pipefail
+
+setup () {
+    local extra=
+    if [ "$DEBUG" = '1' ]; then
+        >&2 echo "Grafting $CACHE_ROOT into $BUILD_ROOT..."
+        extra='-v'
+    fi
+    tar -C "$CACHE_ROOT" -cpf - . | eval tar -C "$BUILD_ROOT" $extra -xpf -
+    return $?
+}
+
+fini () {
+    local rc=$?
+    local extra=
+    if [ "$DEBUG" = '1' ]; then
+        >&2 echo "Removing APK data from $BUILD_ROOT, storing in $CACHE_ROOT"
+        extra=-v
+    fi
+    mkdir -p $BUILD_ROOT/var/cache/apk
+    tar -C "$BUILD_ROOT" -cpf - etc/apk bin/ln bin/busybox var/cache/apk usr/share/apk | eval tar -C "$CACHE_ROOT" $extra -xpf -
+    $_chroot /bin/ln -sf /usr/bin/dash /bin/sh.bak
+    rm -rf $BUILD_ROOT/bin/ln $BUILD_ROOT/bin/busybox $BUILD_ROOT/etc/apk $BUILD_ROOT/var/cache/apk $BUILD_ROOT/usr/share/apk
+    if $_chroot /usr/bin/dash -c '[ ! -x /bin/sh ]'; then
+        >&2 echo '/bin/sh in chroot failed the vibe check, replacing with a symlink to /usr/bin/dash!'
+        mv $BUILD_ROOT/bin/sh.bak $BUILD_ROOT/bin/sh
+    else
+        if [ "$DEBUG" = '1' ]; then
+            >&2 echo '/bin/sh passed the vibe check'
+        fi
+        rm $BUILD_ROOT/bin/sh.bak
+    fi
+    exit $rc
+}
+
+trap fini EXIT
+
+while [ "x$1" = x/usr/bin/env -o "x$1" = xenv ]; do
+    shift
+done
+
+if [ "x$@" = x -o x"$@" = 'x-h' -o x"$@" = 'x--help' -o x"$@" = 'x-?' ]; then
+    >&2 echo 'chroot-exec [NAME=VALUE]... [COMMAND [ARG]...]
+
+let NAME=VALUE denote environment variables
+
+All file accesses in this command are relative to the '"$BUILD_ROOT"'
+'
+    exit 1
+fi
+
+setup
+
+chroot $BUILD_ROOT /usr/bin/env $@