Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions circleci/images/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@
DOCKER_REPO ?= ghcr.io/citusdata
SHA_SUFFIX := $(shell git rev-parse --short HEAD)

# Upstream postgres git ref used to source-build PG19 (which is unreleased
# and therefore not available via apt.postgresql.org). Override at build
# time, e.g. `make PG19_SOURCE_REF=master push-extbuilder-19devel`.
PG19_SOURCE_REF ?= 43649b6a53e9b3964e921cc962f5c977bce688fd

# auto generated variables
ifdef RELEASE
TAG_SUFFIX := -v${SHA_SUFFIX}
Expand All @@ -16,8 +21,11 @@ PG_UPGRADE_TESTER_VERSION=$(shell echo ${PG_VERSIONS}|tr ' ' '-'|sed 's/~//g')

STYLE_CHECKER_TOOLS_VERSION=0.8.33

# Upgrade tests for the PG major versions from PG_VERSIONS file
CITUS_UPGRADE_PG_VERSIONS=$(shell head PG_VERSIONS|cut -c 6-|tr '\n' ' ')
# Upgrade tests for the PG major versions from PG_VERSIONS file.
# PG19 is excluded because no Citus release supports it yet; filter it
# out by name so reordering PG_VERSIONS does not silently change which
# major is dropped.
CITUS_UPGRADE_PG_VERSIONS=$(shell grep -v '^PG19=' PG_VERSIONS|cut -c 6-|tr '\n' ' ')
# 12.1.10 is the latest release of Citus 12 when the test is expanded to cover Citus 12
CITUS_UPGRADE_VERSIONS_16=v12.1.12
# Latest minor version of Citus 13
Expand All @@ -43,6 +51,7 @@ build-extbuilder-$1:
-f extbuilder/Dockerfile \
--build-arg=PG_VERSION=$1 \
--build-arg=PG_MAJOR=$2 \
--build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \
--tag=${DOCKER_REPO}/extbuilder:$3${TAG_SUFFIX}

build-all:: build-extbuilder-$1
Expand All @@ -62,6 +71,7 @@ build-exttester-$1:
--build-arg=PG_VERSION=$1 \
--build-arg=PG_MAJOR=$2 \
--build-arg=PG_VERSION_CLEAN=$3 \
--build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \
--build-arg=CITUS_VERSION="$(call get_last_citus_version,$2)" \
--tag=${DOCKER_REPO}/exttester:$3${TAG_SUFFIX}

Expand All @@ -82,6 +92,7 @@ build-failtester-$1:
-f failtester/Dockerfile \
--build-arg=PG_VERSION=$1 \
--build-arg=PG_MAJOR=$2 \
--build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \
--tag=${DOCKER_REPO}/failtester:$3${TAG_SUFFIX}

build-all:: build-failtester-$1
Expand Down Expand Up @@ -110,6 +121,7 @@ build-citusupgradetester-$1:
-f citusupgradetester/Dockerfile \
--build-arg=PG_VERSION=$1 \
--build-arg=PG_MAJOR=$2 \
--build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \
--build-arg=CITUS_VERSIONS="$(call get_citus_versions,$2)" \
--tag=${DOCKER_REPO}/citusupgradetester:$3${TAG_SUFFIX}

Expand All @@ -131,6 +143,7 @@ build-pgupgradetester:
pgupgradetester/ \
-f pgupgradetester/Dockerfile \
--build-arg=PG_VERSIONS="${PG_VERSIONS}" \
--build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \
--tag=${DOCKER_REPO}/pgupgradetester:${PG_UPGRADE_TESTER_VERSION}${TAG_SUFFIX}

build-all:: build-pgupgradetester
Expand Down
7 changes: 4 additions & 3 deletions circleci/images/PG_VERSIONS
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
PG16=16.13
PG17=17.9
PG18=18.3
PG16=16.14
PG17=17.10
PG18=18.4
PG19=19devel
88 changes: 71 additions & 17 deletions circleci/images/citusupgradetester/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -45,31 +45,85 @@ RUN locale-gen

ARG PG_VERSION
ARG PG_MAJOR
ARG PG19_SOURCE_REF
ENV PG_VERSION=$PG_VERSION
ENV PG_MAJOR=$PG_MAJOR
ENV PG19_SOURCE_REF=$PG19_SOURCE_REF


RUN <<'EOF'
# install postgres ecosystem for pg version: $PG_VERSION
set -eux

# install key and repositories
curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list
echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list

apt-get update

# infer the pgdgversion of postgres based on the $PG_VERSION
pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 )

apt-get install -y --no-install-recommends --allow-downgrades \
libpq-dev=${pgdg_version} \
libpq5=${pgdg_version} \
postgresql-${PG_MAJOR}=${pgdg_version} \
postgresql-client-${PG_MAJOR}=${pgdg_version} \
postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \
postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \
if [ "${PG_MAJOR}" = "19" ]; then
# PG19 is unreleased and not on apt.postgresql.org. Build from a pinned
# upstream source ref so this image can still be produced. Note that no
# Citus release currently supports PG19, so the Makefile excludes PG19
# from CITUS_UPGRADE_PG_VERSIONS and this branch should not be hit by
# CI -- it exists so manual builds don't fail noisily.
apt-get update
apt-get install -y --no-install-recommends \
bison \
flex \
libxml2-dev \
pkg-config \
zlib1g-dev

src_dir=/tmp/pg19-src
rm -rf "${src_dir}"
mkdir -p "${src_dir}"
curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \
| tar -xzf - -C "${src_dir}" --strip-components=1
cd "${src_dir}"
./configure \
--prefix=/usr/lib/postgresql/${PG_MAJOR} \
--datarootdir=/usr/share/postgresql/${PG_MAJOR} \
--includedir=/usr/include/postgresql/${PG_MAJOR} \
--sysconfdir=/etc/postgresql-common \
--with-openssl \
--with-icu \
--with-libxml \
--with-libxslt \
--with-lz4 \
--with-zstd \
--enable-rpath \
--with-system-tzdata=/usr/share/zoneinfo
make -sj"$(nproc)" world-bin
make install-world-bin
cd /
rm -rf "${src_dir}"

# apt-installed postgres packages normally create this user/group; do
# it manually for the source-build path so the chown in the next RUN
# step still works.
if ! getent group postgres >/dev/null; then
groupadd --system postgres
fi
if ! id postgres >/dev/null 2>&1; then
useradd --system --gid postgres --home-dir /var/lib/postgresql --shell /bin/bash postgres
fi
mkdir -p /var/lib/postgresql
chown -R postgres:postgres /var/lib/postgresql
else
# install key and repositories
curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list
echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list

apt-get update

# infer the pgdgversion of postgres based on the $PG_VERSION
pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 )

apt-get install -y --no-install-recommends --allow-downgrades \
libpq-dev=${pgdg_version} \
libpq5=${pgdg_version} \
postgresql-${PG_MAJOR}=${pgdg_version} \
postgresql-client-${PG_MAJOR}=${pgdg_version} \
postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \
postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \

fi

# clear apt cache
rm -rf /var/lib/apt/lists/*
Expand Down
70 changes: 56 additions & 14 deletions circleci/images/extbuilder/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -39,34 +39,76 @@ EOF

ARG PG_VERSION
ARG PG_MAJOR
ARG PG19_SOURCE_REF
ENV PG_VERSION=$PG_VERSION
ENV PG_MAJOR=$PG_MAJOR
ENV PG19_SOURCE_REF=$PG19_SOURCE_REF

RUN <<'EOF'
# install postgres ecosystem for pg version: $PG_VERSION
set -eux

# install key and repositories
curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list
echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list
if [ "${PG_MAJOR}" = "19" ]; then
# PG19 is unreleased and not on apt.postgresql.org. Build from a pinned
# upstream source ref so this image can still be produced.
apt-get update
apt-get install -y --no-install-recommends \
bison \
flex \
libxml2-dev \
pkg-config \
zlib1g-dev

apt-get update
src_dir=/tmp/pg19-src
rm -rf "${src_dir}"
mkdir -p "${src_dir}"
curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \
| tar -xzf - -C "${src_dir}" --strip-components=1
cd "${src_dir}"
./configure \
--prefix=/usr/lib/postgresql/${PG_MAJOR} \
--datarootdir=/usr/share/postgresql/${PG_MAJOR} \
--includedir=/usr/include/postgresql/${PG_MAJOR} \
--sysconfdir=/etc/postgresql-common \
--with-openssl \
--with-icu \
--with-libxml \
--with-libxslt \
--with-lz4 \
--with-zstd \
--enable-rpath \
--with-system-tzdata=/usr/share/zoneinfo
make -sj"$(nproc)" world-bin
make install-world-bin
cd /
rm -rf "${src_dir}"
else
# install key and repositories
curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add -
echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list
echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list

apt-get update

# infer the pgdgversion of postgres based on the $PG_VERSION
pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 )
# infer the pgdgversion of postgres based on the $PG_VERSION
pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 )

apt-get install -y --no-install-recommends --allow-downgrades \
libpq-dev=${pgdg_version} \
libpq5=${pgdg_version} \
postgresql-${PG_MAJOR}=${pgdg_version} \
postgresql-client-${PG_MAJOR}=${pgdg_version} \
postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \
postgresql-server-dev-${PG_MAJOR}=${pgdg_version}
apt-get install -y --no-install-recommends --allow-downgrades \
libpq-dev=${pgdg_version} \
libpq5=${pgdg_version} \
postgresql-${PG_MAJOR}=${pgdg_version} \
postgresql-client-${PG_MAJOR}=${pgdg_version} \
postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \
postgresql-server-dev-${PG_MAJOR}=${pgdg_version}
fi

# clear apt cache
rm -rf /var/lib/apt/lists/*
EOF

# make pg_config (installed by postgresql-server-dev-${PG_MAJOR} from apt or
# by the PG19 source-build path) discoverable to extension builds
ENV PATH=/usr/lib/postgresql/$PG_MAJOR/bin/:$PATH

USER circleci
WORKDIR /home/circleci
Loading