Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
206 changes: 143 additions & 63 deletions contrib/merge-prs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,58 @@
export LC_ALL=C
set -eo pipefail

# Setup base branch and Bitcoin/Elements remote names.
BASE_ORIG=merged-master
BASE="${BASE_ORIG}"
BITCOIN_UPSTREAM_REMOTE=bitcoin
BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
# ELEMENTS_UPSTREAM_REMOTE=upstream
# ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"
export BITCOIN_UPSTREAM="${BITCOIN_UPSTREAM_REMOTE}/master"
ELEMENTS_UPSTREAM_REMOTE=upstream
export ELEMENTS_UPSTREAM="${ELEMENTS_UPSTREAM_REMOTE}/master"

# START USER CONFIG:
# Set your target upstream here
TARGET_UPSTREAM=$BITCOIN_UPSTREAM
TARGET_NAME="Bitcoin"
PR_PREFIX="bitcoin/bitcoin"
# TARGET_UPSTREAM=$ELEMENTS_UPSTREAM
# TARGET_NAME="Elements"
# PR_PREFIX="ElementsProject/elements"

# Set your git worktree location here. This is where the merges will be done, and where you should checkout the merged-master branch.
WORKTREE="/home/byron/code/elements-worktree"

# Set your parallellism during build/test. You probably want as many cores as possible.
# Parallel functional tests can somewhat exceed your core count, depends on the build machine CPU/RAM.
PARALLEL_BUILD=23 # passed to make -j
PARALLEL_TEST=46 # passed to test_runner.py --jobs
PARALLEL_FUZZ=12 # passed to test_runner.py -j when fuzzing

# Setup a ccache dir if necessary.
#export CCACHE_DIR="/tmp/ccache"
#export CCACHE_MAXSIZE="20G"

# Set and export a WEBHOOK environment variable to a Discord webhook URL outside of this script to get notifications of progress and failures.

# We don't currently fuzz during merging. Check fuzzing and CI after a merge run.
# Replace this with the location where we should put the fuzz test corpus
BITCOIN_QA_ASSETS="${HOME}/.tmp/bitcoin/qa-assets"
FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"
mkdir -p "$(dirname "${BITCOIN_QA_ASSETS}")"

# BEWARE: On some systems /tmp/ gets periodically cleaned, which may cause
# random files from this directory to disappear based on timestamp, and
# make git very confused
WORKTREE="${HOME}/.tmp/elements-merge-worktree"
mkdir -p "${HOME}/.tmp"

# These should be tuned to your machine; below values are for an 8-core
# 16-thread macbook pro
PARALLEL_BUILD=4 # passed to make -j
PARALLEL_TEST=12 # passed to test_runner.py --jobs
PARALLEL_FUZZ=8 # passed to test_runner.py -j when fuzzing
# BITCOIN_QA_ASSETS="${HOME}/code/bitcoin/qa-assets"
# FUZZ_CORPUS="${BITCOIN_QA_ASSETS}/fuzz_seed_corpus/"

# END USER CONFIG

# Script
SKIP_MERGE=0
DO_BUILD=1
KEEP_GOING=1
DO_TEST=1
DO_FUZZ=0
NUM=15
COUNT=0

if [[ "$1" == "setup" ]]; then
echo "Setting up..."
echo
git config remote.upstream.url >/dev/null || remote add upstream "https://github.com/ElementsProject/elements.git"
git config remote.upstream.url >/dev/null || git remote add upstream "https://github.com/ElementsProject/elements.git"
git config remote.bitcoin.url >/dev/null || git remote add bitcoin "https://github.com/bitcoin/bitcoin.git"
if git worktree list --porcelain | grep --silent prunable; then
echo "You have stale git worktrees, please either fix them or run 'git worktree prune'."
Expand All @@ -45,14 +65,7 @@ if [[ "$1" == "setup" ]]; then
echo "Fetching all remotes..."
echo
git fetch --all
echo
#echo "Cloning fuzz test corpus..."
#echo
#if [[ ! -d "${BITCOIN_QA_ASSETS}" ]]; then
# cd "$(dirname ${BITCOIN_QA_ASSETS})" && git clone https://github.com/bitcoin-core/qa-assets.git
#fi
#echo
echo "Done! Remember to also check out merged-master, and push it back up when finished."
echo "Done! Remember to also checkout merged-master at ${WORKTREE}"
exit 0
elif [[ "$1" == "continue" ]]; then
SKIP_MERGE=1
Expand All @@ -62,17 +75,35 @@ elif [[ "$1" == "list-only" ]]; then
DO_BUILD=0
elif [[ "$1" == "step" ]]; then
KEEP_GOING=0
elif [[ "$1" == "merge-only" ]]; then
SKIP_MERGE=0
KEEP_GOING=0
DO_BUILD=0
DO_TEST=0
elif [[ "$1" == "step-continue" ]]; then
SKIP_MERGE=1
KEEP_GOING=0
elif [[ "$1" == "step-test" ]]; then
SKIP_MERGE=1
KEEP_GOING=0
DO_BUILD=0
elif [[ "$1" == "step-fuzz" ]]; then
SKIP_MERGE=1
KEEP_GOING=0
DO_BUILD=0
DO_TEST=0
elif [[ "$1" == "analyze" ]]; then
DO_BUILD=0
DO_TEST=0
else
echo "Usage: $0 <setup|list-only|go|continue|step|step-continue>"
echo "Usage: $0 <setup|list-only|go|continue|step|step-continue|analyze>"
echo " setup will configure your repository for the first run of this script"
echo " list-only will simply list all the PRs yet to be done"
echo " go will try to merge every PR, building/testing each"
echo " continue assumes the first git-merge has already happened, and starts with building"
echo " step will try to merge/build/test a single PR"
echo " step-continue assumes the first git-merge has already happened, and will try to build/test a single PR"
echo " analyze will analyze the next $NUM PRs and count conflicts NB: DO NOT CTRL+C THIS PROCESS"
echo
echo "Prior to use, please create a git worktree for the elements repo at:"
echo " $WORKTREE"
Expand Down Expand Up @@ -101,54 +132,95 @@ if [[ "$SKIP_MERGE" == "1" ]]; then
fi

## Get full list of merges
# for elements
# COMMITS=$(git -C "$WORKTREE" log "$ELEMENTS_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Elements %s')
# for bitcoin
COMMITS=$(git -C "$WORKTREE" log "$BITCOIN_UPSTREAM" --not $BASE --merges --first-parent --pretty='format:%ct %cI %h Bitcoin %s')
COMMITS=$(git -C "$WORKTREE" log "$TARGET_UPSTREAM" --not $BASE --merges --first-parent --pretty="format:%ct %cI %h $TARGET_NAME %s")

cd "$WORKTREE"
cd "$WORKTREE" || exit 1

VERBOSE=1

echo start > merge.log

quietly () {
if [[ "$VERBOSE" == "1" ]]; then
"$@"
date | tee --append merge.log
time "$@" 2>&1 | tee --append merge.log
else
chronic "$@"
fi
}

notify () {
local MESSAGE="$1"
local JSON="{\"content\": \"$MESSAGE\"}"
if [ -n "$WEBHOOK" ]; then
curl -d "$JSON" -H "Content-Type: application/json" "$WEBHOOK"
else
echo "$MESSAGE"
fi
if [[ "$2" == "1" ]]; then
exit 1
fi
}

## Sort by unix timestamp and iterate over them
#echo "$ELT_COMMITS" "$BTC_COMMITS" | sort -n -k1 | while read line
echo "$COMMITS" | tac | while read -r line
do
echo
echo "=-=-=-=-=-=-=-=-=-=-="
echo

echo -e "$line"
## Extract data and output what we're doing
DATE=$(echo "$line" | cut -d ' ' -f 2)
HASH=$(echo "$line" | cut -d ' ' -f 3)
CHAIN=$(echo "$line" | cut -d ' ' -f 4)
PR_ID=$(echo "$line" | cut -d ' ' -f 6 | tr -d :)
PR_ID_ALT=$(echo "$line" | cut -d ' ' -f 8 | tr -d :)
PR_ID=$(echo "$line" | grep -o -P "#\d+")

if [[ "$PR_ID" == "pull" ]]; then
PR_ID="${PR_ID_ALT}"
fi
echo -e "$CHAIN PR \e[37m$PR_ID \e[33m$HASH\e[0m on \e[32m$DATE\e[0m "
GIT_HEAD=$(git rev-parse HEAD)

## Do it
if [[ "$1" == "list-only" ]]; then
echo -e "$line"
continue
fi
if [[ "$1" == "analyze" ]]; then
COUNT=$((COUNT + 1))
# todo: count conflicts in critical files
# CRITICAL_FILES=("src/wallet/spend.h", "src/wallet/spend.cpp")
MERGE_FILE="/tmp/$HASH.merge"
DIFF_FILE="/tmp/$HASH.diff"
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" > "$MERGE_FILE" || true
git -C "$WORKTREE" diff > "$DIFF_FILE"
git -C "$WORKTREE" reset --hard "$GIT_HEAD" > /dev/null
# FILES=$(grep "CONFLICT" "$MERGE_FILE")
NUM_FILES=$(grep -c "CONFLICT" "$MERGE_FILE")
NUM_CONFLICTS=$(grep -c "<<<<<<<" "$DIFF_FILE")
echo "$COUNT. Merge up to $PR_ID ($HASH) has $NUM_CONFLICTS conflicts in $NUM_FILES files."
if [[ "$COUNT" == "$NUM" ]]; then
exit 0
else
continue
fi
fi
notify "starting merge of $PR_ID"

# check for stoppers and halt if found
# a stopper is normally the PR after the version has been changed
# ie. the branch point we want to stop at for this version
STOPPERS=(
"#30716" # bitcoin v28
"#32041" # bitcoin v29
)
for STOPPER in "${STOPPERS[@]}"
do
if [[ "$PR_ID" == *"$STOPPER"* ]]; then
echo "Found $STOPPER in $PR_ID! Exiting."
notify "hit stopper, exiting"
exit 1
else
echo "Didn't find $STOPPER in $PR_ID. Continuing."
fi
done

if [[ "$SKIP_MERGE" == "1" ]]; then
echo -e "Continuing build of \e[37m$PR_ID\e[0m at $(date)"
else
echo -e "Start merge/build of \e[37m$PR_ID\e[0m at $(date)"
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_ID)"
git -C "$WORKTREE" merge "$HASH" --no-ff -m "Merge $HASH into merged_master ($CHAIN PR $PR_PREFIX$PR_ID)" || notify "fail merge" 1
fi

if [[ "$DO_BUILD" == "1" ]]; then
Expand All @@ -163,35 +235,43 @@ do
# The following is an expansion of `make check` that skips the libsecp
# tests and also the benchmarks (though it does build them!)
echo "Building"
quietly make -j"$PARALLEL_BUILD" -k
# quietly make -j1 check
echo "Linting"
quietly ./ci/lint/06_script.sh
quietly make -j"$PARALLEL_BUILD" -k || notify "fail build" 1
# todo: fix linting step
# echo "Linting"
# quietly ./ci/lint/06_script.sh || notify "fail lint"
fi

if [[ "$DO_TEST" == "1" ]]; then
echo "Testing"
quietly ./src/qt/test/test_elements-qt
quietly ./src/test/test_bitcoin
quietly ./src/bench/bench_bitcoin
quietly ./test/util/bitcoin-util-test.py
quietly ./test/util/rpcauth-test.py
quietly make -C src/univalue/ check
quietly ./src/qt/test/test_elements-qt || notify "fail test qt" 1
quietly ./src/test/test_bitcoin || notify "fail test bitcoin" 1
quietly ./src/bench/bench_bitcoin || notify "fail test bench" 1
quietly ./test/util/test_runner.py || notify "fail test util" 1
quietly ./test/util/rpcauth-test.py || notify "fail test rpc" 1
echo "Functional testing"
quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST"
quietly ./test/functional/test_runner.py --jobs="$PARALLEL_TEST" || notify "fail test runner" 1
fi

if [[ "$DO_FUZZ" == "1" ]]; then
echo "Cleaning for fuzz"
quietly make distclean || true
quietly git -C "$WORKTREE" clean -xf
echo "Building for fuzz"
quietly ./autogen.sh
# TODO turn on `,integer` after this rebase
quietly ./configure --with-incompatible-bdb --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC=clang CXX=clang++
quietly ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined CC="ccache clang" CXX="ccache clang++"
quietly make -j"$PARALLEL_BUILD" -k
echo "Fuzzing"
quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}"
quietly ./test/fuzz/test_runner.py -j"$PARALLEL_FUZZ" "${FUZZ_CORPUS}" || notify "fail fuzz" 1
fi

if [[ "$KEEP_GOING" == "0" ]]; then
exit 1
notify "$PR_ID done, exiting"
exit 0
else
echo "$PR_ID done, continuing"
fi

# bummer1.sh
SKIP_MERGE=0
echo "end" >> merge.log
done
10 changes: 5 additions & 5 deletions doc/github-merge-script.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ github-merge.py <PR_NUMBER>
For example for PR 1518 I see this output:

```
ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)

Dropping you on a shell so you can try building/testing the merged source.
Run 'git diff HEAD~' to show the changes being merged.
Expand All @@ -47,10 +47,10 @@ exit
In our example this results in the following output:

```
[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propogation
[pull/1518/local-merge 5e1a950e52] Merge ElementsProject/elements#1518: Avoid Simplicity header dependency propagation
Date: Thu Jan 22 14:49:26 2026 +0200
ElementsProject/elements#1518 Avoid Simplicity header dependency propogation into master
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propogation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
ElementsProject/elements#1518 Avoid Simplicity header dependency propagation into master
* 6c7788adf373cd8f0dc5c4fe2b7674625631721e Avoid Simplicity header dependency propagation (Russell O'Connor) (upstream/simplicity, pull/1518/head)
ACKs:
* ACK 6c7788a; built and tested locally (delta1)
* ACK 6c7788adf373cd8f0dc5c4fe2b7674625631721e; successfully ran local tests (apoelstra)
Expand Down