Add a worker process to archive PDFs

.devcontainer/.gitignore

@@ -0,0 +1,2 @@
+# Ignore host-specific extensions to the Docker Compose dev stack.
+docker-compose.extend.yml

.devcontainer/devcontainer.json

@@ -1,16 +1,24 @@
 {
   "name": "Archivist",
-  "dockerComposeFile": "docker-compose.yml",
+  "dockerComposeFile": [
+    "docker-compose.yml",
+    "docker-compose.extend.yml"
+  ],
+  "remoteUser": "vscode",
   "service": "archivist",
   "workspaceFolder": "/workspace",
   "customizations": {
     "vscode": {
       "settings": {
+        "editor.formatOnSave": true,
+        "editor.rulers": [
+          98
+        ],
         "terminal.integrated.defaultProfile.linux": "zsh"
       },
       "extensions": [
         "eamodio.gitlens",
-        "jakebecker.elixir-ls",
+        "JakeBecker.elixir-ls",
         "ms-azuretools.vscode-docker"
       ]
     }

.devcontainer/docker-compose.yml

@@ -2,7 +2,8 @@ services:
   archivist:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: ../Dockerfile
+      target: devcontainer
     volumes:
       - ..:/workspace:cached
       - shell_histories:/root/.cache/erlang-history
@@ -11,20 +12,11 @@ services:
     network_mode: service:ollama
 
   ollama:
-    image: ollama/ollama:0.5.5
-    environment:
-      POSTGRES_PASSWORD: postgres
+    image: ollama/ollama:0.5.7
     ports:
       - "11434:11434"
     volumes:
       - ollama_data:/root/.ollama
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
 
 volumes:
   shell_histories:

.github/workflows/elixir.yaml

@@ -0,0 +1,160 @@
+name: Elixir CI
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+  workflow_dispatch:
+
+env:
+  MIX_ENV: test
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Restore file modification timestamps
+      uses: chetan/git-restore-mtime-action@v2
+
+    - name: Build the Docker image
+      run: |
+        docker login -u ${{ github.actor }} -p ${{ github.token }} ghcr.io
+
+        image="ghcr.io/${{ github.repository }}"
+        sha_tag="${image}:${{ github.sha }}"
+        branch="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+        branch_tag="${image}:${branch}"
+
+        builder_tag="${branch_tag}_builder"
+        main_builder_tag="${image}:main_builder"
+
+        main_tag="${image}:main"
+
+        docker buildx build --push --tag $builder_tag \
+          --target builder \
+          --cache-to type=inline \
+          --cache-from $main_builder_tag \
+          --cache-from $builder_tag \
+          .
+
+        docker buildx build --push --tag $branch_tag --tag $sha_tag \
+          --target runner \
+          --cache-to type=inline \
+          --cache-from $builder_tag \
+          --cache-from $main_tag \
+          --cache-from $branch_tag \
+          .
+
+  build_devcontainer:
+    name: Build devcontainer
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.build_docker_image.outputs.image }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Restore file modification timestamps
+      uses: chetan/git-restore-mtime-action@v2
+
+    - id: build_docker_image
+      name: Build the Docker image
+      run: |
+        docker login -u ${{ github.actor }} -p ${{ github.token }} ghcr.io
+
+        image="ghcr.io/${{ github.repository }}"
+        sha_tag="${image}:${{ github.sha }}"
+        branch="${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+        branch_tag="${image}:${branch}"
+
+        devcontainer_tag="${branch_tag}_devcontainer"
+        main_devcontainer_tag="${image}:main_devcontainer"
+
+        docker buildx build --push --tag $devcontainer_tag \
+          --target devcontainer \
+          --build-arg USER_UID=1001 \
+          --cache-to type=inline \
+          --cache-from $main_devcontainer_tag \
+          --cache-from $devcontainer_tag \
+          .
+
+        echo "image=${devcontainer_tag}" >> "$GITHUB_OUTPUT"
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    needs: build_devcontainer
+    container:
+      image: ${{ needs.build_devcontainer.outputs.image }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
+      options: --user 1001
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Restore file modification timestamps
+        uses: chetan/git-restore-mtime-action@v2
+
+      - name: Cache deps
+        id: cache-deps
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-elixir-deps
+        with:
+          path: deps
+          key: ${{ runner.os }}-mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-${{ env.cache-name }}-
+
+      - name: Cache compiled build
+        id: cache-build
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-compiled-build
+        with:
+          path: _build
+          key: ${{ runner.os }}-mix-${{ env.cache-name }}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-${{ env.cache-name }}-
+            ${{ runner.os }}-mix-
+
+      - name: Clean to rule out incremental build as a source of flakiness
+        if: github.run_attempt != '1'
+        run: |
+          mix deps.clean --all
+          mix clean
+        shell: sh
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      - name: Compiles without warnings
+        run: mix compile --warnings-as-errors
+
+      - name: Check Formatting
+        run: mix format --check-formatted
+
+      - name: Check for retired dependencies
+        run: mix hex.audit
+
+      - name: Check for unused dependencies
+        run: mix deps.unlock --check-unused
+
+      - name: Run tests
+        run: mix test

Dockerfile

@@ -0,0 +1,117 @@
+ARG ELIXIR_VERSION=1.18.2
+ARG OTP_VERSION=27.2.1
+ARG DEBIAN_VERSION=bookworm-20250113-slim
+
+ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
+ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
+
+FROM ${BUILDER_IMAGE} AS devcontainer
+
+RUN apt-get update && apt-get install -y \
+  curl \
+  git \
+  gnupg2 \
+  ocrmypdf \
+  poppler-utils \
+  sudo \
+  zsh
+
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Add vscode user
+RUN groupadd --gid $USER_GID $USERNAME \
+  && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME \
+  && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+  && chmod 0440 /etc/sudoers.d/$USERNAME
+
+USER $USERNAME
+
+# Let the BEAM change its clock when the system time changes.
+ENV ERL_FLAGS="+C multi_time_warp"
+
+# Enable history in IEX.
+ENV ERL_AFLAGS="-kernel shell_history enabled"
+
+# Install Oh My Zsh
+RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
+
+# Co-locate the zsh history with IEx shell history for convenience
+RUN mkdir -p /home/${USERNAME}/.cache/erlang-history \
+  && SNIPPET="export HISTFILE=/home/${USERNAME}/.cache/erlang-history/.zsh_history" \
+  && echo "$SNIPPET" >> "/home/${USERNAME}/.zshrc"
+
+RUN mix local.hex --force && mix local.rebar --force
+
+WORKDIR /workspace
+
+
+FROM ${BUILDER_IMAGE} AS builder
+
+# install build dependencies
+RUN apt-get update -y && apt-get install -y build-essential git \
+  && apt-get clean && rm -f /var/lib/apt/lists/*_*
+
+# prepare build dir
+WORKDIR /app
+
+# install hex + rebar
+RUN mix local.hex --force && \
+  mix local.rebar --force
+
+# set build ENV
+ENV MIX_ENV="prod"
+
+# install mix dependencies
+COPY mix.exs mix.lock ./
+RUN mix deps.get --only $MIX_ENV
+RUN mkdir config
+
+# copy compile-time config files before we compile dependencies to ensure any relevant config
+# change will trigger the dependencies to be re-compiled.
+COPY config/config.exs config/${MIX_ENV}.exs config/
+RUN mix deps.compile
+
+COPY lib ./lib
+
+# Compile the release
+RUN mix compile
+
+# Changes to config/runtime.exs don't require recompiling the code
+COPY config/runtime.exs config/
+
+RUN mix release
+
+
+FROM ${RUNNER_IMAGE} AS runner
+
+RUN apt-get update -y && apt-get install -y \
+  ca-certificates \
+  libncurses5 \
+  libstdc++6 \
+  locales \
+  ocrmypdf \
+  openssl \
+  poppler-utils \
+  && apt-get clean && rm -f /var/lib/apt/lists/*_*
+
+# Set the locale
+RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
+
+ENV LANG="en_US.UTF-8"
+ENV LANGUAGE="en_US:en"
+ENV LC_ALL="en_US.UTF-8"
+
+WORKDIR "/app"
+RUN chown nobody /app
+
+# set runner ENV
+ENV MIX_ENV="prod"
+
+# Only copy the final release from the build stage
+COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/archivist ./
+
+USER nobody
+
+CMD ["/app/bin/archivist", "start"]

config/config.exs

@@ -0,0 +1,12 @@
+import Config
+
+ollama_timeout_seconds =
+  String.to_integer(System.get_env("ARCHIVIST_OLLAMA_TIMEOUT_SECONDS", "60"))
+
+config :archivist, :ollama,
+  base_url: System.get_env("ARCHIVIST_OLLAMA_BASE_URL", "http://localhost:11434/api"),
+  receive_timeout: to_timeout(second: ollama_timeout_seconds)
+
+# Import environment specific config.  This must remain at the bottom of this file so it overrides
+# the configuration defined above.
+import_config "#{config_env()}.exs"

config/dev.exs

@@ -0,0 +1,9 @@
+import Config
+
+config :archivist,
+  worker: [
+    archive: "/archive",
+    check_interval: to_timeout(minute: 1),
+    inbox: "/inbox",
+    llm_timeout: to_timeout(minute: 5)
+  ]

config/prod.exs

@@ -0,0 +1 @@
+import Config