Support mirrored local installation best

INSTALL_LOCAL_MIRROR.md

@@ -0,0 +1,52 @@
+# Providing a self-hosted installation mirror for `cmake-re`
+`TIPI_INSTALL_LEGACY_PACKAGES=ON` won't be supported by self-hosted installation, but this LEGACY mode is only used in official tipi 
+dockers for backward compatibility reasons.
+
+In some context it can be necessary to shield the installation from the internet and use a mirror instead in order to provide a fully 
+self-hosted installation from validated sources.
+
+The `install/container/*.sh` containers initialization scripts provide customization points to perform the installation.
+
+## `install/container/centos.sh` customization point 
+
+- You have to configure a local YUM mirror in the Docker image before running the script, as we install packages like openssh-server
+- [ ] **TODO: Implement** `env:TIPI_CLIENT_INSTALL_SCRIPT_SOURCE` `cmake-re` client install script URL https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh 
+
+## `install/container/ubuntu.sh` customization point 
+- You have to configure a local apt mirror in the Docker image before running the script, as we install packages like openssh-server
+- [ ] **TODO: Implement** `env:TIPI_CLIENT_INSTALL_SCRIPT_SOURCE` `cmake-re` client install script URL https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh 
+- [ ] **TODO: Implement** `env:UTIL_LINUX_SOURCES_MIRROR` For Ubuntu release 16.04 or older and URL to replace https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.39/util-linux-2.39.tar.gz
+
+## cmake-re Distro Customization Mode
+You will need to provide :
+- `TIPI_INSTALL_SOURCE` (e.g. `TIPI_INSTALL_SOURCE=file:///tipi-linux-x86_64.zip` )
+
+### Local Mirroring of `TIPI_DISTRO_JSON`, `TIPI_DISTRO_JSON_SHA1` 
+
+`cmake-re` only requires the default tooling to be present, so the customized distro.json should only include updated URL to : 
+- openssh
+- environments
+- ssh-over-http-proxy
+- cmake
+- make
+- ninja
+- cmake-tipi-provider
+- reclient
+
+## Build the Docker
+
+In order for this build to work, we will locally host the downloaded distro tools at `http://127.0.0.1:8080/tools/` using a lightweigth `httpd`. This URL will be accessible as `http://host.docker.internal:8080/tools` during the docker build phase, so that is what is being passed into the `download_distro.sh` script:
+
+```
+export TIPI_DISTRO_PLATFORM=linux
+install/tools/distro-downloader/download_distro.sh install/container/environments/linux-offline.pkr.js/ http://host.docker.internal:8080/tools/
+
+# start the localhost http server and remember <container id>
+docker run -d --rm -v $PWD/install/container/environments/linux-offline.pkr.js/tools/:/www/tools -p 127.0.0.1:8080:80 busybox busybox httpd -f -p 80 -h /www
+
+# build the container
+docker build install/container/environments/linux-offline.pkr.js/ -f install/container/environments/linux-offline.pkr.js/linux-offline.Dockerfile -t linux-offline:latest
+
+# stop the server once you're done
+docker stop <container id>
+```

install/container/centos.sh

@@ -122,7 +122,8 @@ git config --system --add safe.directory "*"
 
 export TIPI_DISTRO_MODE=${TIPI_DISTRO_MODE:-default}
 export TIPI_ENV_WHITELIST=${TIPI_ENV_WHITELIST:-TIPI_INSTALL_SOURCE,TIPI_DISTRO_MODE,TIPI_DISTRO_JSON,TIPI_DISTRO_JSON_SHA1}
-su tipi -c "cd ~ && curl -fsSL https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh -o install_for_macos_linux.sh && /bin/bash install_for_macos_linux.sh"
+TIPI_CLIENT_INSTALL_SCRIPT_SOURCE=${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE:-https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh}
+su tipi -c 'cd ~ && curl -fsSL ${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE} -o install_for_macos_linux.sh && /bin/bash install_for_macos_linux.sh'
 
 rm -rf ./main \
   && rm -rf /usr/local/share/.tipi/downloads/* \

install/container/ubuntu.sh

@@ -74,8 +74,8 @@ if [ ${DISTRIB_RELEASE_MAJOR} -le 16 ]; then
     && rm -rf /var/lib/apt/lists/*
 
 
-
-  wget https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.39/util-linux-2.39.tar.gz  \
+  TIPI_UTIL_LINUX_SOURCES_MIRROR=${TIPI_UTIL_LINUX_SOURCES_MIRROR:-https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.39/util-linux-2.39.tar.gz}
+  wget ${TIPI_UTIL_LINUX_SOURCES_MIRROR} \
     && tar xvzf util-linux-2.39.tar.gz  \
     && cd util-linux-2.39/  \
     && ./configure \
@@ -171,7 +171,8 @@ if [ "$TIPI_INSTALL_LEGACY_PACKAGES" = "ON" ]; then
 fi
 
 # INCLUDE+ common/Dockerfile.install-tipi
-su tipi -c 'cd ~ && curl -fsSL https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh -o install_for_macos_linux.sh && /bin/bash install_for_macos_linux.sh'
+TIPI_CLIENT_INSTALL_SCRIPT_SOURCE=${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE:-https://raw.githubusercontent.com/tipi-build/cli/v0.0.85/install/install_for_macos_linux.sh}
+su tipi -c 'cd ~ && curl -fsSL ${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE} -o install_for_macos_linux.sh && /bin/bash install_for_macos_linux.sh'
 
 rm -rf ./main \
   && rm -rf /usr/local/share/.tipi/downloads/* \

install/tools/distro-downloader/download_distro.sh

@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+DISTRO_URL="${TIPI_DISTRO_JSON:-https://raw.githubusercontent.com/tipi-build/distro/refs/heads/main/distro.json}"
+
+RAW_DIR=${1:-.}
+CLEANED_DIR=$(echo "$RAW_DIR" | tr -s '/')
+CLEANED_DIR="${CLEANED_DIR%/}"
+DISTRO_DOWNLOAD_OUTPUT_DIR=${CLEANED_DIR:-/}
+
+TOOLS_DIR="${DISTRO_DOWNLOAD_OUTPUT_DIR}/tools"
+DISTRO_JSON="${DISTRO_DOWNLOAD_OUTPUT_DIR}/distro.json"
+DISTRO_LOCAL="${DISTRO_DOWNLOAD_OUTPUT_DIR}/distro.local.json"
+
+BASE_URL=${2:-"file://${TOOLS_DIR}"}
+BASE_URL="${BASE_URL%/}" # drop any trailing slashes
+
+TIPI_INSTALL_SOURCE="${TIPI_INSTALL_SOURCE:-https://github.com/tipi-build/cli/releases/download/v0.0.85/tipi-v0.0.85-linux-x86_64.zip}"
+TIPI_CLIENT_INSTALL_SCRIPT_SOURCE="${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE:-https://raw.githubusercontent.com/tipi-build/cli/refs/heads/master/install/install_for_macos_linux.sh}"
+TIPI_CONTAINER_INSTALL_SCRIPT="${TIPI_CONTAINER_INSTALL_SCRIPT:-https://raw.githubusercontent.com/tipi-build/cli/refs/heads/master/install/container/ubuntu.sh}"
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+red()   { printf '\033[0;31m%s\033[0m\n' "$*"; }
+green() { printf '\033[0;32m%s\033[0m\n' "$*"; }
+cyan()  { printf '\033[0;36m%s\033[0m\n' "$*"; }
+warn()  { printf '\033[0;33mWARN: %s\033[0m\n' "$*" >&2; }
+
+require() {
+  command -v "$1" &>/dev/null || { red "Required tool not found: $1"; exit 1; }
+}
+
+# ── pre-flight ────────────────────────────────────────────────────────────────
+
+require curl
+require jq
+
+mkdir -p "$TOOLS_DIR"
+
+# ── 1. Download distro.json ───────────────────────────────────────────────────
+
+cyan "==> Downloading distro.json from cmake-re..."
+curl -fsSL "$DISTRO_URL" -o "$DISTRO_JSON"
+green "    Saved to $DISTRO_JSON"
+
+# ── 2. Resolve mode filter (optional) ────────────────────────────────────────
+
+MODE="${TIPI_DISTRO_MODE:-default}"
+
+if [[ -n "$MODE" ]]; then
+  # Validate the mode exists in the JSON
+  mode_exists=$(jq -r --arg m "$MODE" 'if .modes[$m] then "yes" else "no" end' "$DISTRO_JSON")
+  if [[ "$mode_exists" != "yes" ]]; then
+    red "Error: mode '$MODE' not found in distro.json"
+    echo "Available modes: $(jq -r '.modes | keys | join(", ")' "$DISTRO_JSON")"
+    exit 1
+  fi
+
+  # Collect the tool names for this mode into a bash array
+  mapfile -t MODE_TOOLS < <(jq -r --arg m "$MODE" '.modes[$m][]' "$DISTRO_JSON")
+  cyan "==> Mode '$MODE' selected — filtering to: ${MODE_TOOLS[*]}"
+
+  # Build a jq-compatible JSON array of tool names for filtering
+  MODE_TOOLS_JSON=$(jq -n --argjson tools "$(printf '%s\n' "${MODE_TOOLS[@]}" | jq -R . | jq -s .)" '$tools')
+else
+  cyan "==> No TIPI_DISTRO_MODE set — downloading all tools"
+  MODE_TOOLS_JSON="null"
+fi
+
+# ── 2b. Resolve platform filter (optional) ───────────────────────────────────
+
+PLATFORM="${TIPI_DISTRO_PLATFORM:-}"
+
+if [[ -n "$PLATFORM" ]]; then
+  cyan "==> Platform '$PLATFORM' selected — only '$PLATFORM' and 'all' platform entries will be downloaded"
+else
+  cyan "==> No TIPI_DISTRO_PLATFORM set — downloading all platforms (including 'all')"
+fi
+
+# ── 3. Collect URLs (filtered by mode and/or platform) ───────────────────────
+
+mapfile -t URLS < <(jq -r --argjson filter "$MODE_TOOLS_JSON" --arg platform "$PLATFORM" '
+  .distro[]
+  | select(
+      if $filter == null then true
+      else .name as $n | ($filter | index($n)) != null
+      end
+    )
+  | .platforms
+  | to_entries[]
+  | select(
+      # "all" platform entries are unconditionally included
+      .key == "all"
+      or
+      if $platform == "" then true
+      else .key == $platform
+      end
+    )
+  | .value
+  | to_entries[]
+  | .value.url
+  | select(. != null)
+' "$DISTRO_JSON")
+
+total="${#URLS[@]}"
+cyan "==> Found $total tool archive(s) to download"
+
+# ── 4. Download tool archives in parallel ────────────────────────────────────
+
+# Max simultaneous downloads (default 8, override with TIPI_DOWNLOAD_JOBS)
+MAX_JOBS="${TIPI_DOWNLOAD_JOBS:-8}"
+
+# Temp file to collect failures from subshells
+FAIL_LOG="$(mktemp)"
+trap 'rm -f "$FAIL_LOG"' EXIT
+
+# Semaphore: wait until fewer than MAX_JOBS background jobs are running
+_throttle() {
+  while (( $(jobs -rp | wc -l) >= MAX_JOBS )); do
+    wait -n 2>/dev/null || true
+  done
+}
+
+declare -A PIDS   # pid → filename, for result reporting
+
+# Also Download cmake-re and install scripts
+URLS+=("${TIPI_INSTALL_SOURCE}")
+URLS+=("${TIPI_CONTAINER_INSTALL_SCRIPT}")
+URLS+=("${TIPI_CLIENT_INSTALL_SCRIPT_SOURCE}")
+
+for url in "${URLS[@]}"; do
+  filename="$(basename "$url")"
+  dest="$TOOLS_DIR/$filename"
+
+  if [[ -f "$dest" ]]; then
+    echo "  [skip]     $filename  (already exists)"
+    continue
+  fi
+
+  _throttle
+  echo "  [start]    $filename"
+  (
+    if curl -fsSL --retry 3 --retry-delay 2 "$url" -o "$dest"; then
+      echo "  [done]     $filename"
+    else
+      warn "Failed to download: $url"
+      rm -f "$dest"
+      echo "$url" >> "$FAIL_LOG"
+    fi
+  ) &
+  PIDS[$!]="$filename"
+done
+
+# Wait for all remaining background jobs to finish
+wait
+
+failed=$(wc -l < "$FAIL_LOG" | tr -d ' ')
+if (( failed > 0 )); then
+  warn "$failed download(s) failed:"
+  while IFS= read -r url; do
+    warn "  $url"
+  done < "$FAIL_LOG"
+fi
+
+# ── 5. Rewrite distro.json with formatted URLs (strip to mode if set) ────────
+
+cyan "==> Writing $DISTRO_LOCAL with rewritten URLs..."
+
+jq --arg base_url "$BASE_URL" --argjson filter "$MODE_TOOLS_JSON" --arg platform "$PLATFORM" '
+  # Keep only the relevant mode entry (or all modes if no filter)
+  .modes |= (
+    if $filter == null then .
+    else
+      to_entries
+      | map(select(
+          .value | (map(. as $t | ($filter | index($t)) != null) | any)
+        ))
+      | from_entries
+    end
+  )
+  |
+  # Strip distro entries not in the mode filter
+  .distro |= map(
+    select(
+      if $filter == null then true
+      else .name as $n | ($filter | index($n)) != null
+      end
+    )
+  )
+  |
+  # Strip platforms not matching the platform filter.
+  # "all" platform entries are unconditionally kept — they are platform-agnostic.
+  .distro[] |= (
+    .platforms |= with_entries(
+      select(
+        .key == "all"
+        or
+        if $platform == "" then true
+        else .key == $platform
+        end
+      )
+    )
+  )
+  |
+  # Rewrite all url fields using the normalized base_url
+  .distro[] |= (
+    .platforms |= with_entries(
+      .value |= with_entries(
+        if (.value | type) == "object" and .value.url then
+          .value.url |= ($base_url + "/" + (. | split("/") | last))
+        else
+          .
+        end
+      )
+    )
+  )
+' "$DISTRO_JSON" > "$DISTRO_LOCAL"
+
+rm "$DISTRO_JSON"
+
+green "==> Done!"
+echo ""
+echo "  Original distro :  $DISTRO_JSON, removed."
+echo "  Local distro    :  $DISTRO_LOCAL"
+echo "  cmake-re release:  $(basename "$TIPI_INSTALL_SOURCE")"
+echo "  Tool archives   :  $TOOLS_DIR/"
+echo "  Base URL        :  $BASE_URL"
+if [[ -n "$MODE" ]]; then
+  echo "  Mode filter     :  $MODE  (${#MODE_TOOLS[@]} tools)"
+fi
+if [[ -n "$PLATFORM" ]]; then
+  echo "  Platform filter :  $PLATFORM  (+ 'all')"
+fi
+echo ""
+echo "Use '$DISTRO_LOCAL' in your cmake-re configuration to point at the local archives."