Noble cut over

ci/bats/tasks/deploy-director.sh

@@ -50,4 +50,9 @@ bosh-cli create-env \
   --vars-store director-creds.yml \
   director.yml
 
-cat bosh-release/version
+version_file="bosh-release/version"
+if [ -f "${version_file}" ]; then
+  cat "${version_file}"
+else
+  echo "Version file '${version_file}' was not present"
+fi

ci/configure.sh

@@ -4,5 +4,4 @@ set -euo pipefail
 REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
 fly -t "${CONCOURSE_TARGET:-bosh}" set-pipeline -p bosh-director \
-    -c "${REPO_ROOT}/ci/pipeline.yml" \
-    --var=branch_name=main
+    -c "${REPO_ROOT}/ci/pipeline.yml"

ci/dockerfiles/docker-cpi/Dockerfile

@@ -29,11 +29,10 @@ RUN docker_gpg="/etc/apt/trusted.gpg.d/docker.gpg" \
 RUN sed -i 's/\(ulimit -Hn [0-9]*\)/#\1/' /etc/init.d/docker
 
 COPY bosh-deployment /usr/local/bosh-deployment/
-RUN curl -o /usr/local/bosh.tgz "$(bosh int /usr/local/bosh-deployment/bosh.yml --path /releases/name=bosh/url)" \
-    && curl -o /usr/local/bpm.tgz "$(bosh int /usr/local/bosh-deployment/bosh.yml --path /releases/name=bpm/url)"
+RUN mkdir -p /usr/local/releases \
+    && curl -o /usr/local/releases/bosh.tgz "$(bosh int /usr/local/bosh-deployment/bosh.yml --path /releases/name=bosh/url)" \
+    && curl -o /usr/local/releases/bpm.tgz "$(bosh int /usr/local/bosh-deployment/bosh.yml --path /releases/name=bpm/url)"
 
 COPY local-releases.yml /usr/local/local-releases.yml
-COPY noble-updates.yml /usr/local/noble-updates.yml
 COPY start-bosh.sh /usr/local/bin/start-bosh
-
 RUN chmod +x /usr/local/bin/start-bosh

ci/dockerfiles/docker-cpi/latest-bosh-release.yml

@@ -20,21 +20,21 @@
   path: /releases/name=bpm/sha1
 
 - type: replace
-  path: /releases/name=bosh-docker-cpi/version
+  path: /releases/name=os-conf/version
   value: latest
 
 - type: remove
-  path: /releases/name=bosh-docker-cpi/url
+  path: /releases/name=os-conf/url
 
 - type: remove
-  path: /releases/name=bosh-docker-cpi/sha1
+  path: /releases/name=os-conf/sha1
 
 - type: replace
-  path: /releases/name=os-conf/version
+  path: /releases/name=bosh-docker-cpi/version
   value: latest
 
 - type: remove
-  path: /releases/name=os-conf/url
+  path: /releases/name=bosh-docker-cpi/url
 
 - type: remove
-  path: /releases/name=os-conf/sha1
+  path: /releases/name=bosh-docker-cpi/sha1

ci/dockerfiles/docker-cpi/local-releases.yml

@@ -1,8 +1,8 @@
 ---
 - type: replace
   path: /releases/name=bosh/url
-  value: file:///usr/local/bosh.tgz
+  value: file:///usr/local/releases/bosh.tgz
 
 - type: replace
   path: /releases/name=bpm/url
-  value: file:///usr/local/bpm.tgz
+  value: file:///usr/local/releases/bpm.tgz

ci/dockerfiles/docker-cpi/start-bosh.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 
 set -e
+if [[ -n "${DEBUG:-}" ]]; then
+  set -x
+  export BOSH_LOG_LEVEL=debug
+fi
 
 function generate_certs() {
   local certs_dir
@@ -39,9 +43,9 @@ EOF
    bosh int ./certs.yml --path=/client_docker_tls/private_key > ./key.pem
     # generate certs in json format
     #
-   ruby -e 'puts File.read("./ca.pem").split("\n").join("\\n")' > $certs_dir/ca_json_safe.pem
-   ruby -e 'puts File.read("./cert.pem").split("\n").join("\\n")' > $certs_dir/client_certificate_json_safe.pem
-   ruby -e 'puts File.read("./key.pem").split("\n").join("\\n")' > $certs_dir/client_private_key_json_safe.pem
+   ruby -e 'puts File.read("./ca.pem").split("\n").join("\\n")' > "${certs_dir}/ca_json_safe.pem"
+   ruby -e 'puts File.read("./cert.pem").split("\n").join("\\n")' > "${certs_dir}/client_certificate_json_safe.pem"
+   ruby -e 'puts File.read("./key.pem").split("\n").join("\\n")' > "${certs_dir}/client_private_key_json_safe.pem"
   popd > /dev/null
 }
 
@@ -52,13 +56,14 @@ function sanitize_cgroups() {
 
   mount -o remount,rw /sys/fs/cgroup
 
+  # shellcheck disable=SC2034
   sed -e 1d /proc/cgroups | while read sys hierarchy num enabled; do
     if [ "$enabled" != "1" ]; then
       # subsystem disabled; skip
       continue
     fi
 
-    grouping="$(cat /proc/self/cgroup | cut -d: -f2 | grep "\\<$sys\\>")"
+    grouping="$(cut -d: -f2 < /proc/self/cgroup | grep "\\<$sys\\>")"
     if [ -z "$grouping" ]; then
       # subsystem not mounted anywhere; mount it on its own
       grouping="$sys"
@@ -90,10 +95,12 @@ function stop_docker() {
 }
 
 function start_docker() {
+  local certs_dir
+  certs_dir="${1}"
   # docker will fail starting with the new iptables. it throws:
   # iptables v1.8.7 (nf_tables): Could not fetch rule set generation id: ....
   update-alternatives --set iptables /usr/sbin/iptables-legacy
-  generate_certs $1
+  generate_certs "${certs_dir}"
   mkdir -p /var/log
   mkdir -p /var/run
 
@@ -110,7 +117,8 @@ function start_docker() {
     mount -o remount,rw /proc/sys
   fi
 
-  local mtu=$(cat /sys/class/net/$(ip route get 169.254.169.254|awk '{ print $5 }')/mtu)
+  local mtu
+  mtu=$(cat "/sys/class/net/$(ip route get 169.254.169.254|awk '{ print $5 }')/mtu")
 
   [[ ! -d /etc/docker ]] && mkdir /etc/docker
   cat <<EOF > /etc/docker/daemon.json
@@ -130,12 +138,9 @@ EOF
 
   service docker start
 
-  export DOCKER_TLS_VERIFY=1
-  export DOCKER_CERT_PATH=$1
-
   rc=1
   for i in $(seq 1 100); do
-    echo waiting for docker to come up...
+    echo "waiting for docker to come up... (${i})"
     sleep 1
     set +e
     docker info
@@ -150,69 +155,100 @@ EOF
     exit 1
   fi
 
-  echo $certs_dir
+  echo "${certs_dir}"
 }
 
 function main() {
-  export OUTER_CONTAINER_IP=$(ruby -rsocket -e 'puts Socket.ip_address_list
-                          .reject { |addr| !addr.ip? || addr.ipv4_loopback? || addr.ipv6? }
-                          .map { |addr| addr.ip_address }')
-
-  export DOCKER_HOST="tcp://${OUTER_CONTAINER_IP}:4243"
+  OUTER_CONTAINER_IP=$(
+    ip addr \
+    | grep 'inet ' \
+    | grep -v -E ' (127\.|172\.|10\.245)' \
+    | cut -d/ -f 1 \
+    | cut -d' ' -f6
+  )
+  export OUTER_CONTAINER_IP
+
+  if [[ "${OUTER_CONTAINER_IP}" == *$'\n'* ]] ; then
+    echo "OUTER_CONTAINER_IP had more than one ip: '${OUTER_CONTAINER_IP}'" >&2
+    exit 1
+  fi
 
   local certs_dir
   certs_dir=$(mktemp -d)
-  start_docker "${certs_dir}"
 
   local local_bosh_dir
   local_bosh_dir="/tmp/local-bosh/director"
+  mkdir -p ${local_bosh_dir}
+
+  export DOCKER_HOST="tcp://${OUTER_CONTAINER_IP}:4243"
+  export DOCKER_TLS_VERIFY=1
+  export DOCKER_CERT_PATH="${certs_dir}"
+  cat <<EOF > "${local_bosh_dir}/docker-env"
+export DOCKER_HOST="tcp://${OUTER_CONTAINER_IP}:4243"
+export DOCKER_TLS_VERIFY=1
+export DOCKER_CERT_PATH="${certs_dir}"
+
+EOF
+  echo "Source '${local_bosh_dir}/docker-env' to run docker" >&2
 
-  docker network create -d bridge --subnet=10.245.0.0/16 director_network
+  start_docker "${certs_dir}"
+
+  local docker_network_name="director_network"
+  if docker network ls | grep -q "${docker_network_name}"; then
+    echo "A docker network named '${docker_network_name}' already exists, skipping creation" >&2
+  else
+    docker network create -d bridge --subnet=10.245.0.0/16 "${docker_network_name}"
+  fi
 
-  pushd ${BOSH_DEPLOYMENT_PATH:-/usr/local/bosh-deployment} > /dev/null
+  pushd "${BOSH_DEPLOYMENT_PATH:-/usr/local/bosh-deployment}" > /dev/null
       export BOSH_DIRECTOR_IP="10.245.0.3"
       export BOSH_ENVIRONMENT="docker-director"
 
-      mkdir -p ${local_bosh_dir}
+      cat <<EOF > "${local_bosh_dir}/docker_tls.json"
+{
+  "ca": "$(cat "${certs_dir}/ca_json_safe.pem")",
+  "certificate": "$(cat "${certs_dir}/client_certificate_json_safe.pem")",
+  "private_key": "$(cat "${certs_dir}/client_private_key_json_safe.pem")"
+}
 
-      additional_ops_files=""
-      if [ "$(lsb_release -cs)" != "jammy" ]; then
-        additional_ops_files="-o /usr/local/noble-updates.yml"
-      fi
+EOF
 
-      command bosh int bosh.yml \
+      bosh int bosh.yml \
         -o docker/cpi.yml \
         -o jumpbox-user.yml \
         -o /usr/local/local-releases.yml \
-        ${additional_ops_files} \
         -v director_name=docker \
         -v internal_cidr=10.245.0.0/16 \
         -v internal_gw=10.245.0.1 \
         -v internal_ip="${BOSH_DIRECTOR_IP}" \
         -v docker_host="${DOCKER_HOST}" \
-        -v network=director_network \
-        -v docker_tls="{\"ca\": \"$(cat ${certs_dir}/ca_json_safe.pem)\",\"certificate\": \"$(cat ${certs_dir}/client_certificate_json_safe.pem)\",\"private_key\": \"$(cat ${certs_dir}/client_private_key_json_safe.pem)\"}" \
-        ${@} > "${local_bosh_dir}/bosh-director.yml"
+        -v network="${docker_network_name}" \
+        -v docker_tls="$(cat "${local_bosh_dir}/docker_tls.json")" \
+        "${@}" > "${local_bosh_dir}/bosh-director.yml"
+
+      bosh create-env "${local_bosh_dir}/bosh-director.yml" \
+        --vars-store="${local_bosh_dir}/creds.yml" \
+        --state="${local_bosh_dir}/state.json"
 
-      command bosh create-env "${local_bosh_dir}/bosh-director.yml" \
-              --vars-store="${local_bosh_dir}/creds.yml" \
-              --state="${local_bosh_dir}/state.json"
+      bosh int "${local_bosh_dir}/creds.yml" --path /director_ssl/ca \
+        > "${local_bosh_dir}/ca.crt"
+      bosh_client_secret="$(bosh int "${local_bosh_dir}/creds.yml" --path /admin_password)"
 
-      bosh int "${local_bosh_dir}/creds.yml" --path /director_ssl/ca > "${local_bosh_dir}/ca.crt"
       bosh -e "${BOSH_DIRECTOR_IP}" --ca-cert "${local_bosh_dir}/ca.crt" alias-env "${BOSH_ENVIRONMENT}"
 
       cat <<EOF > "${local_bosh_dir}/env"
       export BOSH_ENVIRONMENT="${BOSH_ENVIRONMENT}"
       export BOSH_CLIENT=admin
-      export BOSH_CLIENT_SECRET=`bosh int "${local_bosh_dir}/creds.yml" --path /admin_password`
+      export BOSH_CLIENT_SECRET=${bosh_client_secret}
       export BOSH_CA_CERT="${local_bosh_dir}/ca.crt"
 
 EOF
+      echo "Source '${local_bosh_dir}/env' to run bosh" >&2
       source "${local_bosh_dir}/env"
 
-      bosh -n update-cloud-config docker/cloud-config.yml -v network=director_network
+      bosh -n update-cloud-config docker/cloud-config.yml -v network="${docker_network_name}"
 
   popd > /dev/null
 }
 
-main $@
+main "${@}"

ci/dockerfiles/docker-cpi/start-inner-bosh-parallel.sh

@@ -4,19 +4,13 @@ set -euo pipefail
 set -x
 
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-src_dir="${script_dir}/../../../"
 node_number=${1}
 
-pushd ${BOSH_DEPLOYMENT_PATH} > /dev/null
-  inner_bosh_dir="/tmp/inner-bosh/director/$node_number"
-  mkdir -p ${inner_bosh_dir}
+pushd "${BOSH_DEPLOYMENT_PATH}" > /dev/null
+  inner_bosh_dir="/tmp/inner-bosh/director/${node_number}"
+  mkdir -p "${inner_bosh_dir}"
 
-  export BOSH_DIRECTOR_IP="10.245.0.$((10+$node_number))"
-
-  additional_ops_files=""
-  if [ "$(lsb_release -cs)" != "jammy" ]; then
-    additional_ops_files="-o /usr/local/noble-updates.yml"
-  fi
+  export BOSH_DIRECTOR_IP="10.245.0.$((10 + node_number))"
 
   bosh int bosh.yml \
     -o "$script_dir/inner-bosh-ops.yml" \
@@ -31,16 +25,16 @@ pushd ${BOSH_DEPLOYMENT_PATH} > /dev/null
     -o "${BOSH_DEPLOYMENT_PATH}/misc/source-releases/bosh.yml" \
     -o "$script_dir/latest-bosh-release.yml" \
     -o "$script_dir/deployment-name.yml" \
-    ${additional_ops_files} \
-    -v deployment_name="bosh-$node_number" \
-    ${@:2} > "${inner_bosh_dir}/bosh-director.yml"
+    -v deployment_name="bosh-${node_number}" \
+    "${@:2}" > "${inner_bosh_dir}/bosh-director.yml"
 
-  bosh -n deploy -d "bosh-$node_number" "${inner_bosh_dir}/bosh-director.yml" --vars-store="${inner_bosh_dir}/creds.yml"
+  bosh -n deploy -d "bosh-${node_number}" "${inner_bosh_dir}/bosh-director.yml" --vars-store="${inner_bosh_dir}/creds.yml"
 
   # set up inner director
   export BOSH_ENVIRONMENT="docker-inner-director-${node_number}"
   export BOSH_CONFIG="${inner_bosh_dir}/config"
-  export BOSH_CLIENT_SECRET=$(bosh int "${inner_bosh_dir}/creds.yml" --path /admin_password)
+  BOSH_CLIENT_SECRET=$(bosh int "${inner_bosh_dir}/creds.yml" --path /admin_password)
+  export BOSH_CLIENT_SECRET
 
   bosh int "${inner_bosh_dir}/creds.yml" --path /director_ssl/ca > "${inner_bosh_dir}/ca.crt"
   bosh -e "${BOSH_DIRECTOR_IP}" --ca-cert "${inner_bosh_dir}/ca.crt" alias-env "${BOSH_ENVIRONMENT}"
@@ -64,7 +58,7 @@ EOF
 
   "${inner_bosh_dir}/bosh" -n update-cloud-config \
     "$script_dir/inner-bosh-cloud-config.yml" \
-    -v node_number="$((${node_number} * 4))" \
+    -v node_number="$((node_number * 4))" \
     -v network=director_network
 
 popd > /dev/null

ci/dockerfiles/docker-cpi/start-inner-bosh.sh

@@ -8,7 +8,7 @@ bosh_release_path=""
 
 src_dir="${script_dir}/../../../"
 
-default_stemcell_path="${src_dir}/../stemcell/*.tgz"
+default_stemcell_path="$(ls "${src_dir}/../stemcell/*.tgz")"
 stemcell="${CANDIDATE_STEMCELL_TARBALL_PATH:-$default_stemcell_path}"
 
 pushd "${bosh_path}" > /dev/null
@@ -21,7 +21,7 @@ popd > /dev/null
 
 export bosh_release_path
 
-pushd ${BOSH_DEPLOYMENT_PATH} > /dev/null
+pushd "${BOSH_DEPLOYMENT_PATH}" > /dev/null
   inner_bosh_dir="/tmp/inner-bosh/director"
 
   export BOSH_DIRECTOR_IP="10.245.0.34"
@@ -44,9 +44,9 @@ pushd ${BOSH_DEPLOYMENT_PATH} > /dev/null
     -o "${BOSH_DEPLOYMENT_PATH}/misc/source-releases/bosh.yml" \
     -o "${BOSH_DEPLOYMENT_PATH}/local-bosh-release-tarball.yml" \
     -v local_bosh_release="${bosh_release_path}" \
-    ${@} > "${inner_bosh_dir}/bosh-director.yml"
+    "${@}" > "${inner_bosh_dir}/bosh-director.yml"
 
-  bosh upload-stemcell ${stemcell}
+  bosh upload-stemcell "${stemcell}"
 
   deployment_name="--deployment=bosh"
   if [[ "${1}" = "--deployment="* ]]; then
@@ -64,12 +64,14 @@ pushd ${BOSH_DEPLOYMENT_PATH} > /dev/null
   bosh int "${inner_bosh_dir}/creds.yml" --path /jumpbox_ssh/private_key > "${inner_bosh_dir}/jumpbox_private_key.pem"
   chmod 600 "${inner_bosh_dir}/jumpbox_private_key.pem"
 
+  bosh_client_secret="$(bosh int "${inner_bosh_dir}/creds.yml" --path /admin_password)"
+
   cat <<EOF > "${inner_bosh_dir}/bosh"
 #!/bin/bash
 
 export BOSH_ENVIRONMENT="${BOSH_ENVIRONMENT}"
 export BOSH_CLIENT=admin
-export BOSH_CLIENT_SECRET=`bosh int "${inner_bosh_dir}/creds.yml" --path /admin_password`
+export BOSH_CLIENT_SECRET=${bosh_client_secret}
 export BOSH_CA_CERT="${inner_bosh_dir}/ca.crt"
 
 $(which bosh) "\$@"