Optimize Dockerfile and allow UV in Makefile

Author: Tomás Link

.dockerignore

@@ -1 +1,7 @@
-tests/test_init_project.py
+tests/test_init_project.py
+
+**/__pycache__/
+**/*.pyc
+**/*.pyo
+**/*.pyd
+.pytest_cache/

.github/workflows/main.yaml

@@ -2,18 +2,18 @@ name: main
 
 on:
   push:
-    branches: [main, develop]
+    branches: [main]
   pull_request:
 
 jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
 
       - name: Set up Python
-        uses: actions/setup-python@v4.1.0
+        uses: actions/setup-python@v6
         with:
           python-version: 3.12
 
@@ -24,28 +24,31 @@ jobs:
         run: flake8 --count
 
   tests:
-    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    name: tests - Python ${{ matrix.python-version }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         os: ["ubuntu-latest"]
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.13"]
     env:
       VENV: .venv
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip'
           # cache option make the step fail if you don´t have requirements.txt or pyproject.toml on root.
           # https://github.com/actions/setup-python/issues/807.
 
+      - name: Install UV
+        run: make uv
+
       - name: Create virtual environment
-        run: python -m venv $VENV
+        run: uv venv $VENV
 
       - name: Install package with test dependencies
         run: |
@@ -60,16 +63,17 @@ jobs:
 
       - name: Upload coverage reports to Codecov
         if: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') && matrix.python-version == '3.12'
-        uses: codecov/codecov-action@v4.0.1
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
   tests-in-docker:
+    name: tests - Python 3.12 on Docker
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4.1.0
+        uses: actions/setup-python@v6
 
       - name: Build docker image
         run: make docker-build

.gitignore

@@ -170,3 +170,6 @@ terraform.tfstate.backup
 # project stuff
 scripts/config.sh
 test/
+
+# UV
+.python-version

CONTRIBUTING.md

@@ -65,13 +65,18 @@ make docker-build
 make docker-gcp
 ```
 
+Install UV for faster installs (otherwise modify Makefile to use regular pip): 
+```shell
+make uv
+```
+
 4. Create virtual environment and activate it:
 ```shell
 make venv
 ./.venv/bin/activate
 ```
 
-5. Install dependencies and the python package:
+5. Install all dependencies for development and the python package in editable mode:
 ```shell
 make install
 ```

Dockerfile

@@ -1,69 +1,62 @@
 # ---------------------------------------------------------------------------------------
-# BASE IMAGE
+# BUILDER
 # ---------------------------------------------------------------------------------------
-FROM python:3.12.10-slim-bookworm AS base
+FROM python:3.12-slim-bookworm AS builder
 
-# Setup a volume for configuration and authtentication.
 VOLUME ["/root/.config"]
 
-# Update system and install build tools. Remove unneeded stuff afterwards.
-# Upgrade PIP.
-# Create working directory.
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends gcc g++ build-essential && \
-    rm -rf /var/lib/apt/lists/* && \
-    pip install --upgrade pip && \
-    mkdir -p /opt/project
+# Use uv for high-speed installs
+COPY --from=ghcr.io/astral-sh/uv:0.10.9 /uv /usr/local/bin/uv
 
-# Set working directory.
-WORKDIR /opt/project
+ENV UV_COMPILE_BYTECODE=1
 
-# ---------------------------------------------------------------------------------------
-# DEPENDENCIES IMAGE (installed project dependencies)
-# ---------------------------------------------------------------------------------------
-# We do this first so when we modify code while development, this layer is reused
-# from cache and only the layer installing the package executes again.
-FROM base AS deps
-COPY requirements.txt .
-RUN pip install -r requirements.txt
+WORKDIR /install
 
-# ---------------------------------------------------------------------------------------
-# Apache Beam integration IMAGE
-# ---------------------------------------------------------------------------------------
-FROM deps AS beam
-# Copy files from official SDK image, including script/dependencies.
-# IMPORTANT: This version must match the one in requirements.txt
-COPY --from=apache/beam_python3.12_sdk:2.64.0 /opt/apache/beam /opt/apache/beam
+COPY pyproject.toml requirements.txt README.md MANIFEST.in ./
+COPY src ./src
 
-# Set the entrypoint to Apache Beam SDK launcher.
-ENTRYPOINT ["/opt/apache/beam/boot"]
+RUN uv pip install --system --upgrade pip && \
+    uv pip install --system build && \
+    uv pip install --system --prefix=/install -r requirements.txt && \
+    uv pip install --system --prefix=/install --no-deps .
 
 # ---------------------------------------------------------------------------------------
 # PRODUCTION IMAGE
 # ---------------------------------------------------------------------------------------
-# If you need Apache Beam integration, replace "deps" base image with "beam".
-FROM deps AS prod
+FROM python:3.12-slim-bookworm AS prod
+
+ENV PYTHONUNBUFFERED=1
 
-COPY . /opt/project
-RUN pip install . && \
-    rm -rf /root/.cache/pip && \
-    rm -rf /opt/project/*
+# Copy the pre-compiled packages from builder
+COPY --from=builder /install /usr/local
+
+# APACHE BEAM INTEGRATION (Uncomment if needed)
+# COPY --from=apache/beam_python3.12_sdk:2.71.0 /opt/apache/beam /opt/apache/beam
+# ENTRYPOINT ["/opt/apache/beam/boot"]
+
+WORKDIR /opt/project
 
 # ---------------------------------------------------------------------------------------
-# DEVELOPMENT IMAGE (editable install and development tools)
+# DEVELOPMENT IMAGE
 # ---------------------------------------------------------------------------------------
-# If you need Apache Beam integration, replace "deps" base image with "beam".
-FROM deps AS dev
+FROM builder AS dev
 
-COPY . /opt/project
-RUN make install
+WORKDIR /opt/project
+
+COPY . .
+RUN uv pip install --system -e .[lint,dev,build] && \
+    uv pip install --system -r requirements-test.txt
 
 # ---------------------------------------------------------------------------------------
-# TEST IMAGE (This one allows to check that package is properly installed in prod image)
+# TEST IMAGE
 # ---------------------------------------------------------------------------------------
 FROM prod AS test
 
-COPY ./tests /opt/project/tests
-COPY ./requirements-test.txt /opt/project/
+COPY ./requirements-test.txt .
+RUN pip install -r requirements-test.txt
+
+COPY ./tests ./tests
 
-RUN pip install -r requirements-test.txt
+# Suppress all warnings during tests
+# To see/address warnings, run tests in your development environment.
+ENV PYTHONWARNINGS=ignore

Makefile

@@ -2,17 +2,25 @@
 
 VENV_NAME:=.venv
 REQS_PROD:=requirements.txt
+SETUP_FILE:=pyproject.toml
+SOURCES = src
+
 DOCKER_DEV_SERVICE:=dev
-DOCKER_CI_TEST_SERVICE:=test
-DOCKER_ISOLATED_SERVICE:=isolated
+DOCKER_DEV_NO_GCP_SERVICE:=dev_no_gcp
+DOCKER_PROD_SERVICE:=prod
+DOCKER_TEST_SERVICE:=test
 
 GCP_PROJECT:=world-fishing-827
 GCP_DOCKER_VOLUME:=gcp
 
-sources = python_app_template
+PYTHON_VERSION:=3.12
+UV_VERSION := 0.10.9
+
+VENV:=uv venv
+PIP:=uv pip
+PIP_COMPILE:=uv pip compile
+
 
-PYTHON:=python
-PIP:=${PYTHON} -m pip
 
 # ---------------------
 # DOCKER
@@ -34,29 +42,34 @@ docker-gcp: docker-volume
 
 .PHONY: docker-ci-test ## Runs tests using prod image, exporting coverage.xml report.
 docker-ci-test:
-	docker compose run --rm ${DOCKER_CI_TEST_SERVICE}
+	docker compose run --rm ${DOCKER_TEST_SERVICE}
 
 .PHONY: docker-shell ## Enters to docker container shell.
 docker-shell: docker-volume
 	docker compose run --rm -it ${DOCKER_DEV_SERVICE}
 
-.PHONY: reqs  ## Compiles requirements.txt with pip-tools.
+.PHONY: docker-reqs  ## Compiles requirements.txt with pip-tools.
 reqs:
-	docker compose run --rm ${DOCKER_ISOLATED_SERVICE} -c \
-		'pip-compile -o ${REQS_PROD} -v'
+	docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \
+		'${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -v'
 
-.PHONY: reqs-upgrade  ## Upgrades requirements.txt with pip-tools.
+.PHONY: docker-reqs-upgrade  ## Upgrades requirements.txt with pip-tools.
 reqs-upgrade:
-	docker compose run --rm ${DOCKER_ISOLATED_SERVICE} -c \
-		'pip-compile -o ${REQS_PROD} -U -v'
+	docker compose run --rm ${DOCKER_DEV_NO_GCP_SERVICE} -c \
+		'${PIP_COMPILE} -o ${REQS_PROD} ${SETUP_FILE} -U -v'
 
 # ---------------------
 # VIRTUAL ENVIRONMENT
 # ---------------------
 
+.PHONY: uv  ## Installs UV
+uv: 
+	curl -LsSf https://astral.sh/uv/install.sh | UV_VERSION=$(UV_VERSION) sh
+	uv python pin ${PYTHON_VERSION}
+
 .PHONY: venv  ## Creates virtual environment.
 venv:
-	${PYTHON} -m venv ${VENV_NAME}
+	${VENV} ${VENV_NAME}
 
 .PHONY: upgrade-pip  ## Upgrades pip.
 upgrade-pip:
@@ -68,48 +81,49 @@ install-test: upgrade-pip
 
 .PHONY: install  ## Install the package in editable mode & all dependencies for local development.
 install: upgrade-pip
-	${PIP} install -e .[lint,dev,build,test]
+	${PIP} install -e .[lint,dev,build]
+	make install-test
 
 .PHONY: test  ## Run all unit tests exporting coverage.xml report.
 test:
-	${PYTHON} -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(sources)
+	python -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(SOURCES)
 
 # ---------------------
 # QUALITY CHECKS
 # ---------------------
 
 .PHONY: hooks  ## Install and pre-commit hooks.
 hooks:
-	${PYTHON} -m pre_commit install --install-hooks
-	${PYTHON} -m pre_commit install --hook-type commit-msg
+	python -m pre_commit install --install-hooks
+	python -m pre_commit install --hook-type commit-msg
 
 .PHONY: format  ## Auto-format python source files according with PEP8.
 format:
-	${PYTHON} -m black $(sources)
-	${PYTHON} -m ruff check --fix $(sources)
-	${PYTHON} -m ruff format $(sources)
+	python -m black $(SOURCES)
+	python -m ruff check --fix $(SOURCES)
+	python -m ruff format $(SOURCES)
 
 .PHONY: lint  ## Lint python source files.
 lint:
-	${PYTHON} -m ruff check $(sources)
-	${PYTHON} -m ruff format --check $(sources)
-	${PYTHON} -m black $(sources) --check --diff
+	python -m ruff check $(SOURCES)
+	python -m ruff format --check $(SOURCES)
+	python -m black $(SOURCES) --check --diff
 
 .PHONY: codespell  ## Use Codespell to do spell checking.
 codespell:
-	${PYTHON} -m codespell
+	python -m codespell
 
 .PHONY: typecheck  ## Perform type-checking.
 typecheck:
-	${PYTHON} -m mypy
+	python -m mypy
 
 .PHONY: audit  ## Use pip-audit to scan for known vulnerabilities.
 audit:
-	${PYTHON} -m pip_audit .
+	python -m pip_audit .
 
 .PHONY: pre-commit  ## Run all pre-commit hooks.
 pre-commit:
-	${PYTHON} -m pre_commit run --all-files
+	python -m pre_commit run --all-files
 
 .PHONY: all  ## Run the standard set of checks performed in CI.
 all: lint codespell typecheck audit test
@@ -121,7 +135,7 @@ all: lint codespell typecheck audit test
 
 .PHONY: build  ## Build a source distribution and a wheel distribution.
 build: all clean
-	${PYTHON} -m build
+	python -m build
 
 .PHONY: publish  ## Publish the distribution to PyPI.
 publish: build

README.md

@@ -8,7 +8,7 @@
     <img src="https://codecov.io/gh/GlobalFishingWatch/python-app-template/graph/badge.svg?token=uZTb6EphP8"/>
   </a>
   <a>
-    <img alt="Python versions" src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue">
+    <img alt="Python versions" src="https://img.shields.io/badge/python-3.12%20%7C%203.13-blue">
   </a>
   <a>
     <img alt="Last release" src="https://img.shields.io/github/v/release/GlobalFishingWatch/python-app-template">