[WIP] Add functionality for updating an existing compatibility layer

.github/workflows/ansible-lint.yml

@@ -21,3 +21,4 @@ jobs:
           # demote var-naming[no-role-prefix] to warnings, as we only have a single role,
           # and prefixing all variables in that role with the role name is really ugly
           args: "--warn-list var-naming[no-role-prefix]"
+          requirements_file: "ansible/galaxy-requirements.yml"

ansible/galaxy-requirements.yml

@@ -0,0 +1,8 @@
+#
+# Install roles and collections from the default Ansible Galaxy server.
+#
+---
+collections:
+  - name: community.general
+    version: '>=10.7.3'
+...

ansible/playbooks/roles/compatibility_layer/defaults/main.yml

@@ -1,33 +1,33 @@
 # Defaults file for the compatibility layer role.
 ---
 eessi_version: "2025.06"
 
 custom_overlays:
   - name: eessi
     source: git
     url: https://github.com/EESSI/gentoo-overlay.git
     eclass-overrides: true
 
 cvmfs_repository: software.eessi.io
 
 gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}
 
 # How to build the prefix.
 gentoo_git_repo: https://github.com/gentoo/gentoo.git
 # Select a specific commit in the gentoo_git_repo that should be used for the bootstrap,
 # e.g. by checking: https://github.com/gentoo/gentoo/commits/master
 # June 7 2025 (083e38cef302128d595e9f9cfd029ad8f67ec2b7) commit that includes Lmod 8.7.60 and glibc 2.41
 gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7
 prefix_required_space: 15 GB
 prefix_user_defined_trusted_dirs:
-  - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override"
-  - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia"
-  - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd"
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override"
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia"
+  - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd"
 prefix_mask_packages: |
   # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer,
   # see for example https://github.com/EESSI/software-layer/issues/151
   >=sys-devel/gcc-14
 prefix_unmask_packages: |
   # if necessary, unmask for instance an older GCC to make it installable
 prefix_bootstrap_use_flags: |
   # only build libnss, don't build the daemon (use the one from the host)

ansible/playbooks/roles/compatibility_layer/handlers/main.yml

@@ -4,3 +4,8 @@
 - name: Generate locales
   ansible.builtin.command: locale-gen
   changed_when: true
+
+- name: Sync overlays
+  community.general.portage:
+    sync: 'yes'
+    verbose: true

ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml

@@ -17,6 +17,7 @@
     dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/{{ item.name }}.conf"
     mode: "0644"
   loop: "{{ custom_overlays }}"
+  notify: Sync overlays
 
 - name: Make configuration file with overlays that can override eclasses
   ansible.builtin.copy:
@@ -29,10 +30,8 @@
         selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ')
       }}
 
-- name: Sync the repositories
-  community.general.portage:
-    sync: 'yes'
-    verbose: true
+- name: Flush handlers to make sure that overlays are synced
+  ansible.builtin.meta: flush_handlers
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:

ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml

@@ -1,9 +1,15 @@
 # Store some information and scripts that were used for this installation.
 ---
 
-- name: Make a subdirectory for storing build information
+- name: Determine timestamped reprod directory for storing information about this build
+  ansible.builtin.set_fact:
+    reprod_dir: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ '%Y%m%d_%H%M%SUTC' | strftime(ansible_facts.date_time.epoch | int, utc=True) }}"
+  tags:
+    - reprod
+
+- name: Make a timestamped subdirectory for storing build information
   ansible.builtin.file:
-    path: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}"
+    path: "{{ reprod_dir }}"
     state: directory
     mode: '0755'
   tags:
@@ -12,10 +18,11 @@
 - name: Copy the used bootstrap script
   ansible.builtin.copy:
     src: "{{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }}"
-    dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/bootstrap-prefix.sh"
+    dest: "{{ reprod_dir }}/bootstrap-prefix.sh"
     mode: '0644'
   tags:
     - reprod
+  when: not startprefix.stat.exists
 
 - name: Get list of installed packages
   ansible.builtin.command: "qlist -IRv"
@@ -27,15 +34,15 @@
 - name: Dump list of installed packages to a file
   ansible.builtin.copy:
     content: "{{ qlist.stdout }}"
-    dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_packages_file }}"
+    dest: "{{ reprod_dir }}/{{ prefix_packages_file }}"
     mode: '0644'
   tags:
     - reprod
 
 - name: Store other metadata of build in a json file
   ansible.builtin.copy:
     content: "{{ metadata | to_nice_json }}"
-    dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_metadata_json }}"
+    dest: "{{ reprod_dir }}/{{ prefix_metadata_json }}"
     mode: '0644'
   vars:
     metadata:

bot/build.sh

@@ -85,27 +85,48 @@ job_repo=$(cfg_get_value "repository" "repo_name")
 eessi_repo=${job_repo:-software.eessi.io}
 tar_topdir=/cvmfs/${eessi_repo}/versions
 
-if [ "${eessi_arch}" != "${host_arch}" ]; then
-  echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!"
+# Check the architecture, which has to match the architecture of the build host
+if [[ "${eessi_arch}" != "${host_arch}" ]]; then
+  echo "bot/build.sh: requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!"
   exit 1
 fi
 
-# option -k is used for retaining ${eessi_tmp}
-# store output in local file such that the temporary directory ${STORAGE}/eessi.XXXXXXXXXX
-# can be determined
+# Read and check the build type (new or update, case-insensitive)
+if [[ ! -f bot/build_type ]]; then
+    echo 'bot/build.sh: cannot determine build type, please add a file "bot/build_type" containing either "new" or "update".'
+    exit 1
+fi
+build_type=$(cat bot/build_type | tr '[:upper:]' '[:lower:]')
+if [[ "${build_type}" != "new" && "${build_type}" != "update" ]]; then
+    echo 'bot/build.sh: invalid build type! It has to be "new" or "update".'
+    exit 1
+fi
+echo "bot/build.sh: requested build type '${build_type}'"
+
+# Set up the script arguments and run the installation script
 script_out="install_stdout.log"
-./install_compatibility_layer.sh -a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k --verbose 2>&1 | tee -a ${script_out}
-
-# TODO handle errors (no outfile, no tmp directory found)
-eessi_tmp=$(cat ${script_out} | grep 'To resume work add' | cut -f 2 -d \' | cut -f 2 -d ' ')
-eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir})
-# create tarball -> should go into a separate script when this is supported by the bot
-target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz
-if [ -d ${eessi_tmp}${tar_topdir}/${eessi_version} ]; then
-  echo ">> Creating tarball ${target_tgz} from ${eessi_tmp}${tar_topdir}..."
-  tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch}
-  echo ${target_tgz} created!
-else
-  echo "Directory ${eessi_tmp}${tar_topdir}/${eessi_version} was not created, not creating tarball."
-  exit 1
+script_args=(-a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k)
+# for (very!) verbose output, uncomment:
+#script_args+=(--verbose)
+if [[ "${build_type}" == "update" ]]; then
+    script_args+=(-u)
+fi
+./install_compatibility_layer.sh ${script_args[@]} 2>&1 | tee -a ${script_out}
+
+# Create a tarball of the new or updated build
+# TODO handle errors (no outfile, no tmp directory found, etc)
+eessi_tmp=$(grep '^To resume work add' ${script_out} | cut -f 2 -d \' | cut -f 2 -d ' ' | tail -n 1)
+echo "bot/build.sh: creating compatibility layer tarball..."
+if [[ "${build_type}" == "update" ]]; then
+    # Resume the build container session and tar the entire /cvmfs/$repo tree
+    container_tmp=$(grep -oP '(?<=^Using ).*(?= as tmp directory)' ${script_out} | tail -n 1)
+    container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1)
+    eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions)
+    target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz
+    ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch}
+elif [[ "${build_type}" == "new" ]]; then
+    # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo
+    eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir})
+    target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz
+    tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch}
 fi

bot/build_type

@@ -0,0 +1 @@
+update

install_compatibility_layer.sh

@@ -5,17 +5,18 @@
 #
 
 ARCH=
-CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11
+CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-12
 REPOSITORY="software.eessi.io"
 RESUME=
 RETAIN_TMP=0
 STORAGE=
+UPDATE=0
 VERSION=
 VERBOSE=
 
 # Debian 11 does not support RISC-V, so we use a Debian 13 container instead.
 if [[ $(uname -m) = "riscv64" ]]; then
-  CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian-13
+  CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-13
 fi
 
 display_help() {
@@ -46,6 +47,11 @@ display_help() {
   echo "    -t | --resume TMPDIR"
   echo "        tmp directory to resume from [default: None]"
   echo ""
+  echo "    -u | --update"
+  echo "        update an existing compatibility layer by"
+  echo "        doing a fuse mount of the given repository"
+  echo "        [default: not set]"
+  echo ""
   echo "    -v | --version VERSION"
   echo "        override the EESSI stack version set in Ansible's"
   echo "        defaults/main.yml file [default: None]"
@@ -87,6 +93,10 @@ while [[ $# -gt 0 ]]; do
       RESUME="$2"
       shift 2
       ;;
+    -u|--update)
+      UPDATE=1
+      shift
+      ;;
     -v|--version)
       VERSION="$2"
       shift 2
@@ -116,9 +126,6 @@ if [ ! -f "${SCRIPT_DIR}/ansible/playbooks/install.yml" ]; then
     exit 1
 fi
 
-# source utils.sh (for get_container_runtime and check_exit_code)
-source ${SCRIPT_DIR}/scripts/utils.sh
-
 # Check if the target architecture is set to the architecture of the current host,
 # as that's the only thing that's currently supported by this script
 HOST_ARCH=$(uname -m)
@@ -129,7 +136,6 @@ fi
 if [[ -z ${ARCH} ]]; then
   ARCH=${HOST_ARCH}
 fi
-echo "A compatibility layer for architecture ${ARCH} will be built."
 
 # Make a temporary directory on the host for storing the installation and some temporary files
 if [[ ! -z ${RESUME} ]] && [[ -d ${RESUME} ]]; then
@@ -143,40 +149,12 @@ else
 fi
 echo "Using $EESSI_TMPDIR as temporary storage..."
 
-# Create temporary directories
-mkdir -p ${EESSI_TMPDIR}/cvmfs
-mkdir -p ${EESSI_TMPDIR}/home
-mkdir -p ${EESSI_TMPDIR}/tmp
-
-RUNTIME=$(get_container_runtime)
-exit_code=$?
-[[ ${VERBOSE} == '-vvv' ]] && echo "RUNTIME='${RUNTIME}'"
-check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available"
-
-# Set up paths and mount points for Apptainer
-if [[ -z ${APPTAINER_CACHEDIR} ]]; then
-  export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
-  [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_CACHEDIR='${APPTAINER_CACHEDIR}'"
-fi
-export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
-export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp"
-[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_BIND='${APPTAINER_BIND}'"
-export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
-[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_HOME='${APPTAINER_HOME}'"
-
-# also define SINGULARITY_* env vars
-if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
-  export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
-  [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
-fi
-export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
-export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp"
-[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'"
-export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
-[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'"
+# Clone the EESSI/software-layer-scripts repository
+git clone https://github.com/EESSI/software-layer-scripts ${EESSI_TMPDIR}/software-layer-scripts
+#cp ../eessi_container.sh ${EESSI_TMPDIR}/software-layer-scripts/eessi_container.sh
 
 # Construct the Ansible playbook command
-ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=$(uname -m)"
+ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=${ARCH}"
 if [[ ! -z ${VERSION} ]]; then
     ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e eessi_version=${VERSION}"
 fi
@@ -187,12 +165,29 @@ if [[ ! -z ${VERBOSE} ]]; then
     ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} ${VERBOSE}"
 fi
 ANSIBLE_COMMAND="ansible-playbook ${ANSIBLE_OPTIONS} /compatibility-layer/ansible/playbooks/install.yml"
+
+# Set the options for the EESSI container script
+CONTAINER_OPTIONS="-c ${CONTAINER} -g ${EESSI_TMPDIR}"
+if [[ $UPDATE -eq 0 ]]; then
+    # For a new compatibility layer, we bind mount an empty host directory as /cvmfs.
+    # This is a lot faster than (unnecessarily) using an overlay on top of a fuse-mounted /cvmfs.
+    mkdir "${EESSI_TMPDIR}/cvmfs"
+    CONTAINER_OPTIONS="${CONTAINER_OPTIONS} -r none -b ${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
+else
+    # To update an existing compatibility layer, we do have to use an overlay.
+    CONTAINER_OPTIONS="${CONTAINER_OPTIONS} --access rw -r ${REPOSITORY} -b ${SCRIPT_DIR}:/compatibility-layer"
+fi
+
 # Finally, run Ansible inside the container to do the actual installation
 echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..."
-${RUNTIME} shell ${CONTAINER} <<EOF
+${EESSI_TMPDIR}/software-layer-scripts/eessi_container.sh ${CONTAINER_OPTIONS} <<EOF
 # The Gentoo Prefix bootstrap script will complain if $LD_LIBRARY_PATH is set
 unset LD_LIBRARY_PATH
 unset PKG_CONFIG_PATH
+unset RFM_CONFIG_FILES
+export LC_ALL=C.UTF-8
+
+ansible-galaxy install -r /compatibility-layer/ansible/galaxy-requirements.yml
 ${ANSIBLE_COMMAND} | tee /tmp/ansible.log
 EOF
 

test/compat_layer.py

@@ -219,15 +219,20 @@ def __init__(self):
         self.descr = 'Verify that the env file for sys-libs/glibc was created and is picked up by emerge.'
         self.command = 'equery has --package glibc EXTRA_EMAKE'
 
-        trusted_dir = os.path.join(
-            self.eessi_repo_dir,
-            'host_injections',
-            self.eessi_version,
-            'compat',
-            self.eessi_os,
-            self.eessi_arch,
-            'lib'
-        )
+        # in 2023.06 we had a single trusted directory in host_injections,
+        # in 2025.06 we introduced three subdirectories (override, nvidia, amd) in the lib dir of the compat layer itself.
+        if self.eessi_version == '2023.06':
+            trusted_dirs = os.path.join(
+                self.eessi_repo_dir,
+                'host_injections',
+                self.eessi_version,
+                'compat',
+                self.eessi_os,
+                self.eessi_arch,
+                'lib'
+            )
+        else:
+            trusted_dirs = [os.path.join(self.compat_dir, 'lib', subdir) for subdir in ['override', 'nvidia', 'amd']]
 
         self.sanity_patterns = sn.assert_found(
             f'user-defined-trusted-dirs={trusted_dir}',
@@ -242,26 +247,20 @@ def __init__(self):
         self.descr = 'Verify that glibc was compiled with the custom user-defined trusted dirs.'
         self.command = 'ld.so --help'
 
-        libdir = os.path.join(
-            self.eessi_repo_dir,
-            'host_injections',
-            self.eessi_version,
-            'compat',
-            self.eessi_os,
-            self.eessi_arch,
-            'lib'
-        )
-
-        # in 2023.06 we had a single trusted directory,
-        # in 2025.06 we introduced three subdirectories (override, nvidia, amd).
+        # in 2023.06 we had a single trusted directory in host_injections,
+        # in 2025.06 we introduced three subdirectories (override, nvidia, amd) in the lib dir of the compat layer itself.
         if self.eessi_version == '2023.06':
-            trusted_dirs = [libdir]
+            trusted_dirs = os.path.join(
+                self.eessi_repo_dir,
+                'host_injections',
+                self.eessi_version,
+                'compat',
+                self.eessi_os,
+                self.eessi_arch,
+                'lib'
+            )
         else:
-            trusted_dirs = [
-                os.path.join(libdir, 'override'),
-                os.path.join(libdir, 'nvidia'),
-                os.path.join(libdir, 'amd'),
-            ]
+            trusted_dirs = [os.path.join(self.compat_dir, 'lib', subdir) for subdir in ['override', 'nvidia', 'amd']]
 
         # ld.so --help prints the trusted directories as:
         #   /path/to/dir (system search path)