[Docker] Fix networking and startup issues with Docker Compose scripts. Update scripts to align with best practices

.github/workflows/docker.yml

@@ -57,4 +57,117 @@ jobs:
       # Enable redeploy of sandbox & demo if the branch for this image matches the deployment branch of
       # these sites as specified in reusable-docker-build.xml
       REDEPLOY_SANDBOX_URL:  ${{ secrets.REDEPLOY_SANDBOX_URL }}
-      REDEPLOY_DEMO_URL:  ${{ secrets.REDEPLOY_DEMO_URL }}
+      REDEPLOY_DEMO_URL:  ${{ secrets.REDEPLOY_DEMO_URL }}
+
+  #################################################################################
+  # Test Deployment via Docker to ensure newly built images are working properly
+  #################################################################################
+  docker-deploy:
+    # Ensure this job never runs on forked repos. It's only executed for 'dspace/dspace-angular'
+    if: github.repository == 'dspace/dspace-angular'
+    runs-on: ubuntu-latest
+        # Must run after all major images are built
+    needs: [dspace-angular, dspace-angular-dist]
+    env:
+      # Override default dspace.server.url & REST 'host' because backend starts at http://127.0.0.1:8080
+      dspace__P__server__P__url: http://127.0.0.1:8080/server
+      DSPACE_REST_HOST: 127.0.0.1
+      # Override default dspace.ui.url to also use 127.0.0.1.
+      dspace__P__ui__P__url: http://127.0.0.1:4000
+      # Docker Registry to use for Docker compose scripts below.
+      # If this is a PR, then we need to use docker.io (as the registry must be public),
+      # Otherwise we default to ghcr.io to avoid aggressive rate limits at DockerHub.
+      DOCKER_REGISTRY: ${{ github.event_name == 'pull_request' && 'docker.io' || 'ghcr.io' }}
+    steps:
+      # Checkout our codebase (to get access to Docker Compose scripts)
+      - name: Checkout codebase
+        uses: actions/checkout@v4
+      # Download Docker image artifacts (which were just built by reusable-docker-build.yml)
+      - name: Download Docker image artifacts
+        uses: actions/download-artifact@v4
+        with:
+          # Download all amd64 Docker images (TAR files) into the /tmp/docker directory
+          pattern: docker-image-*-linux-amd64
+          path: /tmp/docker
+          merge-multiple: true
+      # Load each of the images into Docker by calling "docker image load" for each.
+      # This ensures we are using the images just built & not any prior versions on DockerHub
+      - name: Load all downloaded Docker images
+        run: |
+          find /tmp/docker -type f -name "*.tar" -exec docker image load --input "{}" \;
+          docker image ls -a
+      # Start backend using our compose script in the codebase.
+      - name: Start backend in Docker
+        run: |
+          docker compose -f docker/docker-compose-rest.yml up -d
+          sleep 10
+          docker container ls
+      # Create a test admin account. Load test data from a simple set of AIPs as defined in cli.ingest.yml
+      - name: Load test data into Backend
+        run: |
+          docker compose -f docker/cli.yml run --rm dspace-cli create-administrator -e test@test.edu -f admin -l user -p admin -c en
+          docker compose -f docker/cli.yml -f docker/cli.ingest.yml run --rm dspace-cli
+      # Verify backend started successfully.
+      # 1. Make sure root endpoint is responding (check for dspace.name defined in docker-compose.yml)
+      # 2. Also check /collections endpoint to ensure the test data loaded properly (check for a collection name in AIPs)
+      - name: Verify backend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace Started with Docker Compose\""
+          result=$(wget -O- -q http://127.0.0.1:8080/server/api/core/collections)
+          echo "$result"
+          echo "$result" |  grep -oE "\"Dog in Yard\""
+      # Start production frontend using our compose script in the codebase.
+      - name: Start production frontend in Docker
+        # Specify the GHCR copy of the production frontend, so that we use the newly built image
+        env:
+          DOCKER_REGISTRY: ghcr.io
+        run: |
+          docker compose -f docker/docker-compose-dist.yml up -d
+          sleep 10
+          docker container ls
+      # Verify production frontend started successfully.
+      # 1. Make sure /home path has "DSpace software" (this is in the footer of the page)
+      # 2. Also check /community-list page lists one of the test Communities in the loaded test data
+      - name: Verify production frontend is responding properly
+        run: |
+          result=$(wget -O- -q http://127.0.0.1:4000/home)
+          echo "$result"
+          echo "$result" |  grep -oE "\"DSpace software\""
+      - name: Error logs of production frontend (if error in startup)
+        if: ${{ failure() }}
+        run: |
+          docker compose -f docker/docker-compose-dist.yml logs
+      # Now shutdown the production frontend image and startup the development frontend image
+      - name: Shutdown production frontend
+        run: |
+          docker compose -f docker/docker-compose-dist.yml down
+          sleep 10
+          docker container ls
+      - name: Startup development frontend
+        # Specify the GHCR copy of the development frontend, so that we use the newly built image
+        env:
+          DOCKER_REGISTRY: ghcr.io
+        run: |
+          docker compose -f docker/docker-compose.yml up -d
+          sleep 10
+          docker container ls
+      # Verify development frontend started successfully.
+      # 1. First, keep requesting the frontend every 10 seconds to wait until its up. Timeout after 10 minutes.
+      # 2. Once it's responding, check to see if the word "DSpace" appears.
+      #    We cannot check for anything more specific because development mode doesn't have SSR.
+      - name: Verify development frontend is responding properly
+        run: |
+          timeout 10m wget --retry-connrefused -t 0 --waitretry=10 http://127.0.0.1:4000
+          result=$(wget -O- -q http://127.0.0.1:4000)
+          echo "$result"
+          echo "$result" |  grep -oE "DSpace"
+      - name: Error logs of development frontend (if error in startup)
+        if: ${{ failure() }}
+        run: |
+          docker compose -f docker/docker-compose.yml logs
+      # Shutdown our containers
+      - name: Shutdown running Docker containers
+        run: |
+          docker compose -f docker/docker-compose.yml -f docker/docker-compose-rest.yml down

Dockerfile

@@ -1,26 +1,34 @@
 # This image will be published as dspace/dspace-angular
 # See https://github.com/DSpace/dspace-angular/tree/main/docker for usage details
 
-FROM docker.io/node:20-alpine
+FROM docker.io/node:22-alpine
 
 # Ensure Python and other build tools are available
 # These are needed to install some node modules, especially on linux/arm64
 RUN apk add --update python3 make g++ && rm -rf /var/cache/apk/*
 
 WORKDIR /app
-ADD . /app/
-EXPOSE 4000
 
-RUN npm install
+# Copy over package files first, so this layer will only be rebuilt if those files change.
+COPY package.json package-lock.json ./
+# NOTE: "ci" = clean install from package files
+RUN npm ci
+
+# Add the rest of the source code
+COPY . /app/
 
 # When running in dev mode, 4GB of memory is required to build & launch the app.
 # This default setting can be overridden as needed in your shell, via an env file or in docker-compose.
 # See Docker environment var precedence: https://docs.docker.com/compose/environment-variables/envvars-precedence/
 ENV NODE_OPTIONS="--max_old_space_size=4096"
 
 # On startup, run in DEVELOPMENT mode (this defaults to live reloading enabled, etc).
-# Listen / accept connections from all IP addresses.
-# NOTE: At this time it is only possible to run Docker container in Production mode
-# if you have a public URL. See https://github.com/DSpace/dspace-angular/issues/1485
 ENV NODE_ENV=development
-CMD npm run serve -- --host 0.0.0.0
+
+EXPOSE 4000
+
+# On startup, run this command to start application in dev mode
+ENTRYPOINT [ "npm", "run", "serve" ]
+# By default set host to 0.0.0.0 to listen/accept connections from all IP addresses.
+# Poll for changes every 5 seconds (if any detected, app will rebuild/restart)
+CMD ["--", "--host 0.0.0.0", "--poll 5000"]

Dockerfile.dist

@@ -4,28 +4,46 @@
 # Test build:
 # docker build -f Dockerfile.dist -t dspace/dspace-angular:latest-dist .
 
-FROM docker.io/node:20-alpine AS build
+# Step 1 - Build code for production
+FROM docker.io/node:22-alpine AS build
 
 # Ensure Python and other build tools are available
 # These are needed to install some node modules, especially on linux/arm64
 RUN apk add --update python3 make g++ && rm -rf /var/cache/apk/*
 
 WORKDIR /app
+# Copy over package files first, so this layer will only be rebuilt if those files change.
 COPY package.json package-lock.json ./
-RUN npm install
+# NOTE: "ci" = clean install from package files
+RUN npm ci
 
-ADD . /app/
+# Around 4GB of memory is required to build the app for production.
+# This default setting can be overridden as needed in your shell, via an env file or in docker-compose.
+# See Docker environment var precedence: https://docs.docker.com/compose/environment-variables/envvars-precedence/
+ENV NODE_OPTIONS="--max_old_space_size=4096"
+
+COPY . /app/
 RUN npm run build:prod
 
-FROM node:20-alpine
+# Step 2 - Start up UI via PM2
+FROM docker.io/node:22-alpine
+
+# Install PM2
 RUN npm install --global pm2
 
+# Copy pre-built code from build image
 COPY --chown=node:node --from=build /app/dist /app/dist
+# Copy configs and PM2 startup script from local machine
 COPY --chown=node:node config /app/config
 COPY --chown=node:node docker/dspace-ui.json /app/dspace-ui.json
 
+# Start up UI in PM2 in production mode
 WORKDIR /app
 USER node
 ENV NODE_ENV=production
 EXPOSE 4000
-CMD pm2-runtime start dspace-ui.json --json
+
+# On startup, run start the DSpace UI in PM2
+ENTRYPOINT [ "pm2-runtime", "start", "dspace-ui.json" ]
+# By default, pass param that specifies to use JSON format logs.
+CMD ["--json"]

docker/README.md

@@ -20,7 +20,8 @@ the Docker compose scripts in this 'docker' folder.
 
 ### Dockerfile
 
-This Dockerfile is used to build a *development* DSpace Angular UI image, published as 'dspace/dspace-angular'
+This Dockerfile is used to build a *development* mode DSpace Angular UI image, published as 'dspace/dspace-angular'. Because it uses development mode, this image supports "live reloading" of the user interface
+when local source code is modified.
 
 ```
 docker build -t dspace/dspace-angular:latest .
@@ -35,7 +36,7 @@ docker push dspace/dspace-angular:latest
 
 ### Dockerfile.dist
 
-The `Dockerfile.dist` is used to generate a *production* build and runtime environment.
+The `Dockerfile.dist` is used to build a *production* mode DSpace Angular UI image, published as 'dspace/dspace-angular' with a `*-dist` tag. Because it uses production mode, this image supports Server Side Rendering (SSR).
 
 ```bash
 # build the latest image

docker/cli.yml

@@ -16,7 +16,7 @@ networks:
   # Default to using network named 'dspacenet' from docker-compose-rest.yml.
   # Its full name will be prepended with the project name (e.g. "-p d7" means it will be named "d7_dspacenet")
   # If COMPOSITE_PROJECT_NAME is missing, default value will be "docker" (name of folder this file is in)
-  default:
+  dspacenet:
     name: ${COMPOSE_PROJECT_NAME:-docker}_dspacenet
     external: true
 services:
@@ -28,19 +28,15 @@ services:
       # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
       # __P__ => "." (e.g. dspace__P__dir => dspace.dir)
       # __D__ => "-" (e.g. google__D__metadata => google-metadata)
-      # dspace.dir
-      dspace__P__dir: /dspace
       # db.url: Ensure we are using the 'dspacedb' image for our database
       db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
       # solr.server: Ensure we are using the 'dspacesolr' image for Solr
       solr__P__server: http://dspacesolr:8983/solr
+    networks:
+    - dspacenet
     volumes:
     # Keep DSpace assetstore directory between reboots
     - assetstore:/dspace/assetstore
-    entrypoint: /dspace/bin/dspace
-    command: help
-    tty: true
-    stdin_open: true
 
 volumes:
   assetstore:

docker/db.entities.yml

@@ -34,4 +34,4 @@ services:
       - |
         while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
         /dspace/bin/dspace database migrate ignored
-        java -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace
+        java -jar /dspace/webapps/server-boot.jar

docker/docker-compose-ci.yml

@@ -75,8 +75,6 @@ services:
     ports:
     - published: 5432
       target: 5432
-    stdin_open: true
-    tty: true
     volumes:
      # Keep Postgres data directory between reboots
     - pgdata:/pgdata
@@ -89,8 +87,6 @@ services:
     ports:
     - published: 8983
       target: 8983
-    stdin_open: true
-    tty: true
     working_dir: /var/solr/data
     volumes:
     # Keep Solr data directory between reboots
@@ -116,6 +112,8 @@ services:
       cp -r /opt/solr/server/solr/configsets/qaevent/* qaevent
       precreate-core suggestion /opt/solr/server/solr/configsets/suggestion
       cp -r /opt/solr/server/solr/configsets/suggestion/* suggestion
+      precreate-core audit /opt/solr/server/solr/configsets/audit
+      cp -r /opt/solr/server/solr/configsets/audit/* audit
       chown -R solr:solr /var/solr
       runuser -u solr -- solr-foreground
 volumes:

docker/docker-compose-dist.yml

@@ -9,7 +9,11 @@
 # Docker Compose for running the DSpace Angular UI dist build
 # for previewing with the DSpace Demo site backend
 networks:
+  # Default to using network named 'dspacenet' from docker-compose.yml.
+  # Its full name will be prepended with the project name (e.g. "-p d7" means it will be named "d7_dspacenet")
   dspacenet:
+    name: ${COMPOSE_PROJECT_NAME}_dspacenet
+    external: true
 services:
   dspace-angular:
     container_name: dspace-angular
@@ -18,22 +22,18 @@ services:
       DSPACE_UI_HOST: dspace-angular
       DSPACE_UI_PORT: '4000'
       DSPACE_UI_NAMESPACE: /
-      # NOTE: When running the UI in production mode (which the -dist image does),
-      # these DSPACE_REST_* variables MUST point at a public, HTTPS URL.
-      # This is because Server Side Rendering (SSR) currently requires a public URL,
-      # see this bug: https://github.com/DSpace/dspace-angular/issues/1485
-      DSPACE_REST_SSL: 'true'
-      DSPACE_REST_HOST: sandbox.dspace.org
-      DSPACE_REST_PORT: 443
+      DSPACE_REST_SSL: 'false'
+      DSPACE_REST_HOST: localhost
+      DSPACE_REST_PORT: 8080
       DSPACE_REST_NAMESPACE: /server
+      # Ensure SSR can use the 'dspace' Docker image directly (see docker-compose-rest.yml)
+      DSPACE_REST_SSRBASEURL: http://dspace:8080/server
     image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-angular:${DSPACE_VER:-latest}-dist"
     build:
       context: ..
       dockerfile: Dockerfile.dist
     networks:
-      dspacenet:
+    - dspacenet
     ports:
     - published: 4000
       target: 4000
-    stdin_open: true
-    tty: true