Skip to content

CICD.yml: Publish individual bins for stable tag, prepare computing bin size for all platforms #10403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
oech3 wants to merge 1 commit into from
Closed

CICD.yml: Publish individual bins for stable tag, prepare computing bin size for all platforms #10403

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 95 additions & 127 deletions .github/workflows/CICD.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,130 +478,6 @@ jobs:
flags: nightly,${{ matrix.job.os }}
fail_ci_if_error: false

compute_size:
name: Binary sizes
needs: [ min_version, deps ]
runs-on: ${{ matrix.job.os }}
permissions:
contents: write
env:
SCCACHE_GHA_ENABLED: "true"
RUSTC_WRAPPER: "sccache"
CARGO_INCREMENTAL: 0
strategy:
fail-fast: false
matrix:
job:
- { os: ubuntu-latest , features: feat_os_unix }
steps:
- uses: actions/checkout@v6
with:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Run sccache-cache
uses: mozilla-actions/sccache-action@v0.0.9
- name: "`make install PROFILE=release-fast`"
shell: bash
run: |
export CARGO_TARGET_DIR=cargo-target RUSTFLAGS="${RUSTFLAGS} -C strip=symbols" PROFILE=release-fast MANPAGES=n COMPLETIONS=n LOCALES=n
mkdir -p "${CARGO_TARGET_DIR}" && sudo mount -t tmpfs -o noatime,size=16G tmpfs "${CARGO_TARGET_DIR}"
make install DESTDIR=target/size-release/
make install COMPLETIONS=n MULTICALL=y LN="ln -vf" DESTDIR=target/size-multi-release/
ZSTD_CLEVEL=19 tar --zstd -caf individual-x86_64-unknown-linux-gnu.tar.zst -C target/size-release/usr/local bin
- name: Publish
uses: softprops/action-gh-release@v2
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
with:
tag_name: latest-commit
draft: false
prerelease: true
files: |
individual-x86_64-unknown-linux-gnu.tar.zst
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Test for hardlinks
shell: bash
run: |
[ $(stat -c %i target/size-multi-release/usr/local/bin/cp) = $(stat -c %i target/size-multi-release/usr/local/bin/coreutils) ]
- name: Compute uutil release sizes
shell: bash
run: |
## Compute uutil release sizes
DATE=$(date --rfc-email)
find target/size-release/usr/local/bin -type f -printf '%f\0' | sort -z |
while IFS= read -r -d '' name; do
size=$(du -s target/size-release/usr/local/bin/$name | awk '{print $1}')
echo "\"$name\""
echo "$size"
done | \
jq -n \
--arg date "$DATE" \
--arg sha "$GITHUB_SHA" \
'reduce inputs as $name ({}; . + { ($name): input }) | { ($date): {sha: $sha, sizes: map_values(.)} }' > individual-size-result.json
SIZE=$(cat individual-size-result.json | jq '[.[] | .sizes | .[]] | reduce .[] as $num (0; . + $num)')
SIZE_MULTI=$(du -s target/size-multi-release/usr/local/bin/coreutils | awk '{print $1}')
jq -n \
--arg date "$DATE" \
--arg sha "$GITHUB_SHA" \
--arg size "$SIZE" \
--arg multisize "$SIZE_MULTI" \
'{($date): { sha: $sha, size: $size, multisize: $multisize, }}' > size-result.json
- name: Download the previous individual size result
uses: dawidd6/action-download-artifact@v14
with:
workflow: CICD.yml
name: individual-size-result
repo: uutils/coreutils
path: dl
- name: Download the previous size result
uses: dawidd6/action-download-artifact@v14
with:
workflow: CICD.yml
name: size-result
repo: uutils/coreutils
path: dl
- name: Check uutil release sizes
shell: bash
run: |
check() {
# Warn if the size increases by more than 5%
threshold='1.05'

if [[ "$2" -eq 0 || "$3" -eq 0 ]]; then
echo "::warning file=$4::Invalid size for $1. Sizes cannot be 0."
return
fi

ratio=$(jq -n "$2 / $3")
echo "$1: size=$2, previous_size=$3, ratio=$ratio, threshold=$threshold"
if [[ "$(jq -n "$ratio > $threshold")" == 'true' ]]; then
echo "::warning file=$4::Size of $1 increases by more than 5%"
fi
}
## Check individual size result
while read -r name previous_size; do
size=$(cat individual-size-result.json | jq -r ".[] | .sizes | .\"$name\"")
check "\`$name\` binary" "$size" "$previous_size" 'individual-size-result.json'
done < <(cat dl/individual-size-result.json | jq -r '.[] | .sizes | to_entries[] | "\(.key) \(.value)"')
## Check size result
size=$(cat size-result.json | jq -r '.[] | .size')
previous_size=$(cat dl/size-result.json | jq -r '.[] | .size')
check 'multiple binaries' "$size" "$previous_size" 'size-result.json'
multisize=$(cat size-result.json | jq -r '.[] | .multisize')
previous_multisize=$(cat dl/size-result.json | jq -r '.[] | .multisize')
check 'multicall binary' "$multisize" "$previous_multisize" 'size-result.json'
- name: Upload the individual size result
uses: actions/upload-artifact@v6
with:
name: individual-size-result
path: individual-size-result.json
- name: Upload the size result
uses: actions/upload-artifact@v6
with:
name: size-result
path: size-result.json

build:
permissions:
contents: write # to create GitHub release (softprops/action-gh-release)
Expand All @@ -628,7 +504,7 @@ jobs:
- { os: ubuntu-latest , target: i686-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross }
- { os: ubuntu-latest , target: i686-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross }
- { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,test_risky_names", use-cross: use-cross, skip-publish: true }
- { os: ubuntu-latest , target: x86_64-unknown-linux-gnu , features: "feat_os_unix,uudoc" , use-cross: no, workspace-tests: true }
- { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, features: "feat_os_unix,uudoc", use-cross: no, workspace-tests: true, individual: "--workspace --bins --config=profile.release.codegen-units=1 --exclude uu_runcon --exclude uu_chcon" }
- { os: ubuntu-latest , target: x86_64-unknown-linux-musl , features: feat_os_unix_musl , use-cross: use-cross }
- { os: ubuntu-latest , target: x86_64-unknown-redox , features: feat_os_unix_redox , use-cross: redoxer , skip-tests: true }
- { os: ubuntu-latest , target: wasm32-unknown-unknown , default-features: false, features: uucore/format, skip-tests: true, skip-package: true, skip-publish: true }
Expand Down Expand Up @@ -859,8 +735,8 @@ jobs:
shell: bash
if: matrix.job.skip-publish != true
run: |
## Build
${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release --config=profile.release.strip=true \
# libstdbuf is not stripped by cargo's config
RUSTFLAGS="-C strip=symbols" ${{ steps.vars.outputs.CARGO_CMD }} ${{ steps.vars.outputs.CARGO_CMD_OPTIONS }} build --release ${{ matrix.job.individual }} \
--target=${{ matrix.job.target }} ${{ matrix.job.cargo-options }} ${{ steps.vars.outputs.CARGO_FEATURES_OPTION }} ${{ steps.vars.outputs.CARGO_DEFAULT_FEATURES_OPTION }}
- name: Package
if: matrix.job.skip-publish != true
Expand All @@ -881,6 +757,13 @@ jobs:
*) tar czf '${{ steps.vars.outputs.PKG_NAME }}' '${{ steps.vars.outputs.PKG_BASENAME }}'/* ;;
esac
popd >/dev/null
- name: Package individual binaries
if: matrix.job.individual != ''
run: |
mkdir -p individual
(cd target/${{ matrix.job.target }}/release && mv $(./coreutils --list) -t ../../../individual ||:)
individual/true --version
ZSTD_CLEVEL=19 tar --zstd -caf individual-${{ matrix.job.target }}.tar.zst individual
- name: Package manpages and completions
if: matrix.job.target == 'x86_64-unknown-linux-gnu' && matrix.job.features == 'feat_os_unix,uudoc'
run: |
Expand All @@ -903,6 +786,7 @@ jobs:
files: |
${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }}
docs.tar.zst
individual-${{ matrix.job.target }}.tar.zst
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Publish latest commit
Expand All @@ -917,8 +801,92 @@ jobs:
files: |
${{ steps.vars.outputs.STAGING }}/${{ steps.vars.outputs.PKG_NAME }}
docs.tar.zst
individual-${{ matrix.job.target }}.tar.zst
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Compute uutil release sizes
if: matrix.job.individual != '' # todo: support all platforms
shell: bash
run: |
## Compute uutil release sizes
DATE=$(date --rfc-email)
find individual -type f -printf '%f\0' | sort -z |
while IFS= read -r -d '' name; do
size=$(du -s individual/$name | awk '{print $1}')
echo "\"$name\""
echo "$size"
done | \
jq -n \
--arg date "$DATE" \
--arg sha "$GITHUB_SHA" \
'reduce inputs as $name ({}; . + { ($name): input }) | { ($date): {sha: $sha, sizes: map_values(.)} }' > individual-size-result.json
SIZE=$(cat individual-size-result.json | jq '[.[] | .sizes | .[]] | reduce .[] as $num (0; . + $num)')
SIZE_MULTI=$(du -s target/${{ matrix.job.target }}/release/coreutils | awk '{print $1}')
jq -n \
--arg date "$DATE" \
--arg sha "$GITHUB_SHA" \
--arg size "$SIZE" \
--arg multisize "$SIZE_MULTI" \
'{($date): { sha: $sha, size: $size, multisize: $multisize, }}' > size-result.json
- name: Download the previous individual size result
if: matrix.job.individual != ''
uses: dawidd6/action-download-artifact@v12
with:
workflow: CICD.yml
name: individual-size-result
repo: uutils/coreutils
path: dl
- name: Download the previous size result
if: matrix.job.individual != ''
uses: dawidd6/action-download-artifact@v12
with:
workflow: CICD.yml
name: size-result
repo: uutils/coreutils
path: dl
- name: Check uutil release sizes
if: matrix.job.individual != ''
shell: bash
run: |
check() {
# Warn if the size increases by more than 5%
threshold='1.05'

if [[ "$2" -eq 0 || "$3" -eq 0 ]]; then
echo "::warning file=$4::Invalid size for $1. Sizes cannot be 0."
return
fi

ratio=$(jq -n "$2 / $3")
echo "$1: size=$2, previous_size=$3, ratio=$ratio, threshold=$threshold"
if [[ "$(jq -n "$ratio > $threshold")" == 'true' ]]; then
echo "::warning file=$4::Size of $1 increases by more than 5%"
fi
}
## Check individual size result
while read -r name previous_size; do
size=$(cat individual-size-result.json | jq -r ".[] | .sizes | .\"$name\"")
check "\`$name\` binary" "$size" "$previous_size" 'individual-size-result.json'
done < <(cat dl/individual-size-result.json | jq -r '.[] | .sizes | to_entries[] | "\(.key) \(.value)"')
## Check size result
size=$(cat size-result.json | jq -r '.[] | .size')
previous_size=$(cat dl/size-result.json | jq -r '.[] | .size')
check 'multiple binaries' "$size" "$previous_size" 'size-result.json'
multisize=$(cat size-result.json | jq -r '.[] | .multisize')
previous_multisize=$(cat dl/size-result.json | jq -r '.[] | .multisize')
check 'multicall binary' "$multisize" "$previous_multisize" 'size-result.json'
- name: Upload the individual size result
if: matrix.job.individual != ''
uses: actions/upload-artifact@v6
with:
name: individual-size-result
path: individual-size-result.json
- name: Upload the size result
if: matrix.job.individual != ''
uses: actions/upload-artifact@v6
with:
name: size-result
path: size-result.json

test_busybox:
name: Tests/BusyBox test suite
Expand Down
Loading