Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
02898a7
feat: extract build infrastructure into standalone scripts
wdconinc Apr 6, 2026
e88b5d4
chore: chmod u+x build-eic.sh build-base.sh
plexoos Apr 6, 2026
8427bfe
Add ghcr.io/eic/buildcache as default cache-from
wdconinc Apr 6, 2026
f3365c5
More pipefail
wdconinc Apr 6, 2026
c884c7a
fix: apk add bash
wdconinc Apr 8, 2026
9de546f
ci(github): rewrite build jobs to call build-base.sh and build-eic.sh
wdconinc Apr 8, 2026
cf32384
refactor: unify push-by-digest output for all CI modes
wdconinc Apr 8, 2026
ad03e20
build: default JOBS to $(nproc) instead of hardcoded 4
wdconinc Apr 8, 2026
114c440
build(eic): drop unused spack version file sources
wdconinc Apr 8, 2026
f8e894f
build scripts: harmonize github/gitlab by normalizing CI_REGISTRY_PREFIX
wdconinc Apr 9, 2026
cee8c2b
ci: add pre-commit shellcheck hook for root-level shell scripts
wdconinc Apr 9, 2026
b01db8d
fix: docs: --jobs default to $(nproc)
wdconinc Apr 9, 2026
23894df
docs: --jobs default nproc
wdconinc Apr 9, 2026
36208ab
fix: preserve arch/vN suffix
wdconinc Apr 9, 2026
2b3f8a2
docs: default base is now debian:stable-slim
wdconinc Apr 9, 2026
3f0fdb0
docs: clarify argument and env var mismatch allowed
wdconinc Apr 9, 2026
350fc7c
fix: strip \r from GITHUB_REF_POINT_SLUG/GITHUB_BASE_REF_SLUG
wdconinc Apr 9, 2026
206bd1f
refactor: replace github-slug-action with inline slugify helper
wdconinc Apr 9, 2026
228a6d9
fix: strip CRLF line endings that embedded \r into ARCH variable
wdconinc Apr 10, 2026
1720fc6
fix: remove non-existent tag extraction from image@digest files
wdconinc Apr 10, 2026
d89e030
fix: fix set up QEMU job step
wdconinc Apr 10, 2026
d01a97f
fix: use getconf _NPROCESSORS_ONLN instead of nproc
wdconinc Apr 10, 2026
87392ed
fix: add mirrors.yaml to .gitignore to prevent secrets upload
wdconinc Apr 11, 2026
42676e1
feat: support -h/--help
wdconinc Apr 13, 2026
245ac5b
fix: update default tag handling to avoid accidental misuse
wdconinc Apr 13, 2026
fbac841
feat: move gitlab ci pipeline to separate manifest merging job
wdconinc Apr 14, 2026
68a792f
docs: fix stale BASE_IMAGE default (amd64/debian:stable-slim → debian…
wdconinc Apr 14, 2026
6986380
fix: use repo default branch for cache key fallback on push events
wdconinc Apr 14, 2026
9afe7f2
fix: escape sed replacements and use temp file for mirrors.yaml
wdconinc Apr 14, 2026
6bc2cf1
fix: mention both base images in local-fallback message
wdconinc Apr 14, 2026
e397195
fix: mark registry credential secrets as required=false in Dockerfile
wdconinc Apr 14, 2026
546a47e
fix: apk add coreutils
wdconinc Apr 15, 2026
f11aef2
fix(docs): build-eic --jobs: default: nproc
wdconinc Apr 15, 2026
0710afd
fix: build-*.sh: cd "${SCRIPT_DIR}"
wdconinc Apr 15, 2026
87c455b
fix(docs): consistentify
wdconinc Apr 15, 2026
545b988
build-eic: support comma-separated --build-type (default,nightly) (#252)
wdconinc Apr 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Enforce LF line endings for all text files tracked in this repository.
# This prevents Windows editors from introducing CRLF endings in shell scripts,
# which causes bash to embed '\r' into variable values (e.g. ARCH=$(cmd)\r).
* text=auto eol=lf

# Explicitly mark binary files so they are never munged
*.png binary
*.jpg binary
*.ico binary
275 changes: 72 additions & 203 deletions .github/workflows/build-push.yml

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# build-*.sh products
mirrors.yaml

# install.sh products
eic-shell
local/
Expand Down
310 changes: 110 additions & 200 deletions .gitlab-ci.yml

Large diffs are not rendered by default.

200 changes: 200 additions & 0 deletions build-base.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
#!/bin/bash
# Build the EIC base container image (debian_stable_base, cuda_devel, or cuda_runtime).
#
# This script is used in GitLab CI, GitHub Actions, and for local builds.
# CI mode is detected via CI_REGISTRY (GitLab) or GITHUB_ACTIONS=true (GitHub Actions).
#
# Run `bash build-base.sh --help` for usage, options, and CI-specific details.

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
Comment thread
wdconinc marked this conversation as resolved.
cd "${SCRIPT_DIR}"

print_help() {
cat <<EOF
Build the EIC base container image (debian_stable_base, cuda_devel, or cuda_runtime).

Usage (local):
bash build-base.sh [options]

Usage (CI, called from .gitlab-ci.yml or build-push.yml with matrix variables in env):
bash build-base.sh

Options:
--image IMAGE Image to build: debian_stable_base, cuda_devel, cuda_runtime
(default: \$BUILD_IMAGE or debian_stable_base)
--base-image IMAGE Upstream base image (default: derived from --image)
--platform PLATFORM Build platform, e.g. linux/amd64, linux/arm64
(default: \$PLATFORM or linux/amd64)
--jobs N Number of parallel Spack build jobs (default: \$JOBS or \$(nproc)
or \$(getconf _NPROCESSORS_ONLN))
--tag TAG Local tag for the image (default: local; ignored in CI)
-h, --help Show this help and exit

GitHub Actions mode (GITHUB_ACTIONS=true):
Set GH_REGISTRY, GH_REGISTRY_USER, JOBS. The script derives cache-key slugs
from GITHUB_REF_NAME (current branch), GITHUB_BASE_REF (PR target branch, empty
on push events), and DEFAULT_BRANCH (repo default branch, used as fallback when
GITHUB_BASE_REF is empty). Writes the image digest to METADATA_FILE (default:
/tmp/build-metadata.json).
EOF
}

## Defaults (may be overridden by env vars set from CI matrix or command-line flags)
BUILD_IMAGE="${BUILD_IMAGE:-debian_stable_base}"
BASE_IMAGE="${BASE_IMAGE:-}"
PLATFORM="${PLATFORM:-linux/amd64}"
JOBS="${JOBS:-$(getconf _NPROCESSORS_ONLN)}"
Comment thread
wdconinc marked this conversation as resolved.
LOCAL_TAG="${LOCAL_TAG:-local}"
METADATA_FILE="${METADATA_FILE:-/tmp/build-metadata.json}"
Comment thread
wdconinc marked this conversation as resolved.
## CUDA defaults (used when building cuda_devel or cuda_runtime)
CUDA_VERSION="${CUDA_VERSION:-12.5.1}"
CUDA_OS="${CUDA_OS:-ubuntu24.04}"

while [[ $# -gt 0 ]]; do
case "$1" in
-h|--help) print_help; exit 0 ;;
--image) BUILD_IMAGE="$2"; shift 2 ;;
--base-image) BASE_IMAGE="$2"; shift 2 ;;
--platform) PLATFORM="$2"; shift 2 ;;
--jobs) JOBS="$2"; shift 2 ;;
--tag) LOCAL_TAG="$2"; shift 2 ;;
*) echo "Unknown argument: $1" >&2; echo "Try 'bash build-base.sh --help' for usage." >&2; exit 1 ;;
esac
done

## Source version files
source "${SCRIPT_DIR}/spack.sh"
source "${SCRIPT_DIR}/spack-packages.sh"
source "${SCRIPT_DIR}/key4hep-spack.sh"
source "${SCRIPT_DIR}/eic-spack.sh"

## Convert an arbitrary git ref/branch name to a valid OCI tag component.
## Mirrors GitLab's CI_COMMIT_REF_SLUG: lowercase, non-alnum runs → '-',
## strip leading/trailing '-', truncate to 63 chars.
slugify() {
echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]\+/-/g; s/^-//; s/-$//' | cut -c1-63
}

## Detect CI mode and normalise environment variables
if [ -n "${CI_REGISTRY}" ]; then
## GitLab CI — all CI_* variables are already set by the runner
CI_MODE="gitlab"
elif [ "${GITHUB_ACTIONS}" = "true" ]; then
## GitHub Actions — map GitHub variables to the names used below.
## GITHUB_REF_NAME (current branch) and GITHUB_BASE_REF (PR target branch,
## empty on push events) are standard runner variables. DEFAULT_BRANCH should
## be supplied by the workflow (github.event.repository.default_branch) so
## that cache keys are correct even when GITHUB_BASE_REF is empty.
CI_MODE="github"
CI_REGISTRY="${GH_REGISTRY}"
CI_PROJECT_PATH="${GH_REGISTRY_USER}"
CI_COMMIT_REF_SLUG="$(slugify "${GITHUB_REF_NAME:-master}")"
CI_DEFAULT_BRANCH_SLUG="$(slugify "${GITHUB_BASE_REF:-${DEFAULT_BRANCH:-master}}")"
INTERNAL_TAG="${INTERNAL_TAG:-pipeline-${GITHUB_RUN_ID}}"
else
CI_MODE="local"
fi

## Derive BASE_IMAGE from BUILD_IMAGE if not provided
if [ -z "${BASE_IMAGE}" ]; then
case "${BUILD_IMAGE}" in
debian_stable_base) BASE_IMAGE="debian:trixie-slim" ;;
cuda_devel) BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-devel-${CUDA_OS}" ;;
cuda_runtime) BASE_IMAGE="nvidia/cuda:${CUDA_VERSION}-runtime-${CUDA_OS}" ;;
*) echo "Unknown BUILD_IMAGE '${BUILD_IMAGE}'; please specify --base-image" >&2; exit 1 ;;
esac
Comment on lines +86 to +107
fi

## Resolve SHAs (network calls — skipped if version is already a SHA)
echo "Resolving git SHAs..."
SPACK_SHA=$(sh "${SCRIPT_DIR}/.ci/resolve_git_ref" "${SPACK_ORGREPO}" "${SPACK_VERSION}")
SPACKPACKAGES_SHA=$(sh "${SCRIPT_DIR}/.ci/resolve_git_ref" "${SPACKPACKAGES_ORGREPO}" "${SPACKPACKAGES_VERSION}")
KEY4HEPSPACK_SHA=$(sh "${SCRIPT_DIR}/.ci/resolve_git_ref" "${KEY4HEPSPACK_ORGREPO}" "${KEY4HEPSPACK_VERSION}")
EICSPACK_SHA=$(sh "${SCRIPT_DIR}/.ci/resolve_git_ref" "${EICSPACK_ORGREPO}" "${EICSPACK_VERSION}")

## Normalize arch string for cache tag names while preserving platform variants
## Examples: linux/amd64 -> amd64, linux/amd64/v3 -> amd64_v3, linux/arm/v7 -> arm_v7
ARCH=$(echo "${PLATFORM}" | sed 's|^linux/||; s|/|_|g')

## Build the docker buildx command as an array for safe quoting
build_cmd=(docker buildx build)
# shellcheck disable=SC2206 # word splitting is intentional: BUILD_OPTIONS is a space-separated list
build_cmd+=(${BUILD_OPTIONS})

## Derive shared registry prefix (used for image push, caching, and DOCKER_REGISTRY build-arg)
CI_REGISTRY_PREFIX="${CI_REGISTRY}/${CI_PROJECT_PATH}"
IMAGE_REPO="${CI_REGISTRY_PREFIX}/${BUILD_IMAGE}"

## Output mode: push-by-digest in all CI modes; load locally
if [ "${CI_MODE}" != "local" ]; then
## Push by digest; CI wrapper creates final tags via imagetools create
build_cmd+=(--output "type=image,name=${IMAGE_REPO},push-by-digest=true,name-canonical=true,push=true")
build_cmd+=(--metadata-file "${METADATA_FILE}")
else
build_cmd+=(--load)
fi

## Cache sources: CI registry (if in CI) plus public ghcr.io/eic (GitLab and local modes)
BUILDCACHE_REPOS=()
[ "${CI_MODE}" != "local" ] && BUILDCACHE_REPOS+=("${CI_REGISTRY_PREFIX}")
[ "${CI_MODE}" != "github" ] && BUILDCACHE_REPOS+=("ghcr.io/eic")
for REPO in "${BUILDCACHE_REPOS[@]}"; do
build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG:-master}-${ARCH}")
build_cmd+=(--cache-from "type=registry,ref=${REPO}/buildcache:${BUILD_IMAGE}-${CI_DEFAULT_BRANCH_SLUG:-master}-${ARCH}")
done

## Cache destination (CI only)
if [ "${CI_MODE}" != "local" ]; then
build_cmd+=(--cache-to "type=registry,ref=${CI_REGISTRY_PREFIX}/buildcache:${BUILD_IMAGE}-${CI_COMMIT_REF_SLUG:-master}-${ARCH},mode=max")
fi

## Image tag (local only; CI creates tags via imagetools create after build)
if [ "${CI_MODE}" = "local" ]; then
build_cmd+=(--tag "${BUILD_IMAGE}:${LOCAL_TAG}")
fi

## Dockerfile and platform
build_cmd+=(--file containers/debian/Dockerfile)
build_cmd+=(--platform "${PLATFORM}")

## Build arguments
build_cmd+=(--build-arg "BASE_IMAGE=${BASE_IMAGE}")
build_cmd+=(--build-arg "BUILD_IMAGE=${BUILD_IMAGE}")
case "${BUILD_IMAGE}" in
cuda*)
build_cmd+=(--build-arg "NVIDIA_VISIBLE_DEVICES=all")
build_cmd+=(--build-arg "NVIDIA_DRIVER_CAPABILITIES=all")
;;
esac
build_cmd+=(--build-arg "SPACK_ORGREPO=${SPACK_ORGREPO}")
build_cmd+=(--build-arg "SPACK_VERSION=${SPACK_VERSION}")
build_cmd+=(--build-arg "SPACK_SHA=${SPACK_SHA}")
build_cmd+=(--build-arg "SPACK_CHERRYPICKS=${SPACK_CHERRYPICKS}")
build_cmd+=(--build-arg "SPACK_CHERRYPICKS_FILES=${SPACK_CHERRYPICKS_FILES}")
build_cmd+=(--build-arg "SPACKPACKAGES_ORGREPO=${SPACKPACKAGES_ORGREPO}")
build_cmd+=(--build-arg "SPACKPACKAGES_VERSION=${SPACKPACKAGES_VERSION}")
build_cmd+=(--build-arg "SPACKPACKAGES_SHA=${SPACKPACKAGES_SHA}")
build_cmd+=(--build-arg "SPACKPACKAGES_CHERRYPICKS=${SPACKPACKAGES_CHERRYPICKS}")
build_cmd+=(--build-arg "SPACKPACKAGES_CHERRYPICKS_FILES=${SPACKPACKAGES_CHERRYPICKS_FILES}")
build_cmd+=(--build-arg "KEY4HEPSPACK_ORGREPO=${KEY4HEPSPACK_ORGREPO}")
build_cmd+=(--build-arg "KEY4HEPSPACK_VERSION=${KEY4HEPSPACK_VERSION}")
build_cmd+=(--build-arg "KEY4HEPSPACK_SHA=${KEY4HEPSPACK_SHA}")
build_cmd+=(--build-arg "EICSPACK_ORGREPO=${EICSPACK_ORGREPO}")
build_cmd+=(--build-arg "EICSPACK_VERSION=${EICSPACK_VERSION}")
build_cmd+=(--build-arg "EICSPACK_SHA=${EICSPACK_SHA}")
build_cmd+=(--build-arg "jobs=${JOBS}")
build_cmd+=(--build-arg "BUILDWEEK=${BUILDWEEK:-0}")

## Suppress provenance attestation (matches CI behaviour)
build_cmd+=(--provenance false)

## Build context
build_cmd+=(containers/debian)

## Execute
set -o xtrace -o pipefail
"${build_cmd[@]}" 2>&1 | tee build.log


Loading
Loading