Merge PR to keep BBC Images Main in step with GNM Main - 2026-05-26

.github/dependabot.yml

@@ -52,15 +52,3 @@ updates:
 #      interval: 'weekly'
 #    commit-message:
 #      prefix: "chore(deps): "
-#  - package-ecosystem: 'npm'
-#    directory: '/s3watcher/lambda'
-#    schedule:
-#      interval: 'weekly'
-#    commit-message:
-#      prefix: "chore(deps): "
-#  - package-ecosystem: 'npm'
-#    directory: '/s3watcher/scripts'
-#    schedule:
-#      interval: 'weekly'
-#    commit-message:
-#      prefix: "chore(deps): "

.github/workflows/check-labels.yaml

@@ -0,0 +1,9 @@
+name: CORE4 label enforcement
+permissions:
+  contents: read
+on:
+  pull_request:
+    types: [opened, labeled, unlabeled, synchronize, reopened, edited]
+jobs:
+  require-label:
+    uses: guardian/.github/.github/workflows/require-label.yaml@4cb5024736632ffcc564b7f4b772c38b8e5ce739 #  v2.0.0

.github/workflows/ci.yml

@@ -36,11 +36,13 @@ jobs:
           --health-timeout 5s
           --health-retries 10
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v6.0.2
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v6
         with:
           node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: "cdk/package-lock.json"
       - name: Kahuna (client-side)
         working-directory: ./kahuna
         run: |
@@ -66,13 +68,16 @@ jobs:
           npm install-clean
           npm test
           npm run compile
-      - name: S3Watcher
-        working-directory: ./s3watcher/lambda
+      - name: Image Embedder CDK
+        run: ./scripts/embedder-deploy/cdk.sh
+      - name: Image Embedder tests
+        working-directory: ./image-embedder-lambda
         run: |
           npm install-clean
           npm test
-          npm run build
-      - uses: guardian/actions-riff-raff@v4.2.3
+      - name: Image Embedder bundling
+        run: ./scripts/embedder-deploy/build.sh
+      - uses: guardian/actions-riff-raff@v4.2.4
         if: "! github.event.pull_request.head.repo.fork"
         with:
           githubToken: ${{ secrets.GITHUB_TOKEN }}
@@ -108,4 +113,9 @@ jobs:
             image-counter-lambda:
               - image-counter-lambda/dist/image-counter-lambda.zip
 
-
+            cdk.out:
+              - cdk/cdk.out
+            image-embedder-lambda:
+              - image-embedder.zip
+            image-embedder-backfill-lambda:
+              - image-embedder.zip

.github/workflows/push-pr.yml

@@ -18,7 +18,7 @@ jobs:
       contents: write
       actions: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v6.0.2
       - name: push-pr
         env:
           PR: ${{ inputs.prNumber }}

.github/workflows/sbt-dependency-graph.yaml

@@ -10,10 +10,10 @@ jobs:
     steps:
       - name: Checkout branch
         id: checkout
-        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1
       - name: Install Java
         id: java
-        uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.2.0
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
         with:
           distribution: corretto
           java-version: 17
@@ -22,7 +22,7 @@ jobs:
         uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14
       - name: Submit dependencies
         id: submit
-        uses: scalacenter/sbt-dependency-submission@64084844d2b0a9b6c3765f33acde2fbe3f5ae7d3 # v3.1.0
+        uses: scalacenter/sbt-dependency-submission@f43202114d7522a4b233e052f82c2eea8d658134 # v3.2.1
       - name: Log snapshot for user validation
         id: validate
         run: cat ${{ steps.submit.outputs.snapshot-json-path }} | jq

.gitignore

@@ -28,3 +28,9 @@ dev/config/permissions.json
 
 coverage/
 .bsp
+
+image-embedder.zip
+
+# Integration test data (cached S3 downloads and outputs)
+image-embedder-lambda/__tests__/embedder/integration/test-data/
+image-embedder-lambda/__tests__/embedder/test-data

auth/app/auth/AuthController.scala

@@ -11,7 +11,7 @@ import play.api.libs.json.Json
 import play.api.mvc.{BaseController, ControllerComponents, Result}
 
 import java.net.URI
-import java.util.Date
+import java.time.Instant
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
@@ -34,10 +34,13 @@ class AuthController(auth: Authentication, providers: AuthenticationProviders, v
 
   def cookieMonster = auth { request =>
     providers.userProvider match {
-      case panda: PandaAuthenticationProvider =>{
-        val cookieBatter = panda.readAuthenticatedUser(request).map(user => panda.generateCookie(user.copy(expires = new Date().getTime)))
+      case panda: PandaAuthenticationProvider =>
+        val cookieBatter = panda.readAuthenticatedUser(request)
+          // Note the cookie monster does not expire the cookie itself, but instead expires the panda token stored
+          // by the cookie. The cookie will remain in the browser storage, but if decoded will declare that it expired
+          // at the epoch in 1970.
+          .map(user => panda.generateCookie(user.copy(expires = Instant.ofEpochMilli(0L))))
         cookieBatter.fold(respond("Me want cookie."))(cookie => respond("Cookies are a sometimes food.").withCookies(cookie))
-      }
       case _ => respond("Me want cookie.")
     }
   }

build.sbt

@@ -20,8 +20,22 @@ ThisBuild / packageOptions += FixedTimestamp(Package.keepTimestamps)
 ThisBuild / libraryDependencySchemes +=
   "org.scala-lang.modules" %% "scala-java8-compat" % VersionScheme.Always
 
+lazy val jacksonVersion = "2.21.1"
+lazy val jacksonAnnotationsVersion = "2.21"
+lazy val jacksonOverrides = Seq(
+  "com.fasterxml.jackson.core" % "jackson-core",
+  "com.fasterxml.jackson.core" % "jackson-databind",
+  "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor",
+  "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8",
+  "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310",
+  "com.fasterxml.jackson.module" % "jackson-module-parameter-names",
+  "com.fasterxml.jackson.module" %% "jackson-module-scala"
+).map(_ % jacksonVersion) ++ Seq(
+  "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonAnnotationsVersion
+)
+
 val commonSettings = Seq(
-  scalaVersion := "2.13.16",
+  scalaVersion := "2.13.18",
   description := "grid",
   organization := "com.gu",
   version := "0.1",
@@ -40,7 +54,7 @@ val commonSettings = Seq(
     "org.mockito" % "mockito-core" % "2.18.0" % Test,
     "org.scalamock" %% "scalamock" % "5.1.0" % Test,
   ),
-  dependencyOverrides += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.19.0",
+  dependencyOverrides ++= jacksonOverrides,
 
   Compile / doc / sources := Seq.empty,
   Compile / packageDoc / publishArtifact := false
@@ -63,8 +77,9 @@ Global / concurrentRestrictions := Seq(
 )
 
 val awsSdkVersion = "1.12.470"
-val awsSdkV2Version = "2.32.33"
-val elastic4sVersion = "8.18.2"
+val awsSdkV2Version = "2.42.25"
+val elastic4sVersion = "8.19.1"
+val awsKclVersion = "3.4.3"
 val okHttpVersion = "3.12.1"
 
 val bbcBuildProcess: Boolean = System.getenv().asScala.get("BUILD_ORG").contains("bbc")
@@ -77,17 +92,16 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj
 lazy val commonLib = project("common-lib").settings(
   libraryDependencies ++= Seq(
     "com.gu" %% "editorial-permissions-client" % "4.0.0",
-    "com.gu" %% "pan-domain-auth-play_3-0" % "9.0.0",
+    "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0",
     "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion,
-    "com.amazonaws" % "aws-java-sdk-cloudwatch" % awsSdkVersion,
-    "com.amazonaws" % "aws-java-sdk-cloudfront" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion,
     "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion,
+    "software.amazon.awssdk" % "s3" % awsSdkV2Version,
     "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion,
     "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion,
@@ -107,11 +121,13 @@ lazy val commonLib = project("common-lib").settings(
     "org.scanamo" %% "scanamo" % "2.0.0",
     // declare explicit dependency on desired version of aws sdk v2 dynamo
     "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
+    "software.amazon.awssdk" % "dynamodb-enhanced" % awsSdkV2Version,
+    "software.amazon.awssdk" % "cloudwatch" % awsSdkV2Version,
     // declare explicit dependency on desired version of aws sdk v2 bedrock runtime
     "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version,
     "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version,
     ws,
-    "org.testcontainers" % "elasticsearch" % "1.19.2" % Test
+    "org.testcontainers" % "elasticsearch" % "1.21.4" % Test
   ),
   dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test
 )
@@ -132,7 +148,7 @@ lazy val cropper = playProject("cropper", 9006)
 
 lazy val imageLoader = playProject("image-loader", 9003).settings {
   libraryDependencies ++= Seq(
-    "org.apache.tika" % "tika-core" % "1.28.5",
+    "org.apache.tika" % "tika-core" % "3.2.3",
     "com.drewnoakes" % "metadata-extractor" % "2.19.0"
   )
 }
@@ -150,6 +166,7 @@ lazy val mediaApi = playProject("media-api", 9001)
       "org.apache.commons" % "commons-email" % "1.5",
       "org.parboiled" %% "parboiled" % "2.1.7",
       "org.http4s" %% "http4s-core" % "0.23.17",
+      "com.github.blemale" %% "scaffeine" % "5.3.0"
     )
   )
 
@@ -161,19 +178,15 @@ lazy val thrall = playProject("thrall", 9002)
     pipelineStages := Seq(digest, gzip),
     libraryDependencies ++= Seq(
       "org.codehaus.groovy" % "groovy-json" % "3.0.7",
-      // TODO upgrading kcl to v3? check if you can remove avro override below
-      "software.amazon.kinesis" % "amazon-kinesis-client" % "2.6.1",
+      "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion,
       // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl
       "software.amazon.awssdk" % "kinesis" % awsSdkV2Version,
       "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
-      "com.gu" %% "kcl-pekko-stream" % "0.1.0",
+      "com.gu" %% "kcl-pekko-stream" % "0.1.2",
       "org.testcontainers" % "elasticsearch" % "1.19.2" % Test,
       "com.google.protobuf" % "protobuf-java" % "3.19.6"
     ),
-    // amazon-kinesis-client 2.6.0 brings in a critically vulnerable version of apache avro,
-    // but we cannot upgrade amazon-kinesis-client further without performing the v2->v3 upgrade https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html
     dependencyOverrides ++= Seq(
-      "org.apache.avro" % "avro" % "1.11.4",
       "org.apache.pekko" %% "pekko-stream" % "1.0.3"
     )
   )
@@ -183,17 +196,12 @@ lazy val usage = playProject("usage", 9009).settings(
     "com.gu" %% "content-api-client-default" % "32.0.0",
     "com.gu" %% "content-api-client-aws" % "0.7.6",
     "io.reactivex" %% "rxscala" % "0.27.0",
-    // amazon-kinesis-client brings in a critical vulnerability warning through apache avro, resolved in versions 1.11.4 and 1.12.0.
-    // updating amazon-kinesis-client? check if the override below can be removed
-    "software.amazon.kinesis" % "amazon-kinesis-client" % "3.0.2",
+    "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion,
     // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl
     "software.amazon.awssdk" % "kinesis" % awsSdkV2Version,
     "software.amazon.awssdk" % "dynamodb" % awsSdkV2Version,
     "com.google.protobuf" % "protobuf-java" % "3.19.6"
   ),
-  dependencyOverrides ++= Seq(
-    "org.apache.avro" % "avro" % "1.11.4",
-  )
 )
 
 lazy val scripts = project("scripts")

cdk/.gitignore

@@ -0,0 +1,11 @@
+*.js
+!jest.config.js
+!jest.setup.js
+!.eslintrc.js
+*.d.ts
+node_modules
+dist
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out

cdk/README.md

@@ -0,0 +1,5 @@
+# Infrastructure
+
+This directory defines the components to be deployed to AWS.
+
+See [`package.json`](./package.json) for a list of available scripts.

cdk/bin/cdk.ts

@@ -0,0 +1,7 @@
+import "source-map-support/register";
+import { GuRoot } from "@guardian/cdk/lib/constructs/root";
+import { ImageEmbedder } from "../lib/image-embedder-lambda";
+
+const app = new GuRoot();
+new ImageEmbedder(app, "ImageEmbedderLambda-euwest-1-PROD", { stack: "media-service", stage: "PROD", env: { region: "eu-west-1" } });
+new ImageEmbedder(app, "ImageEmbedderLambda-euwest-1-TEST", { stack: "media-service", stage: "TEST", env: { region: "eu-west-1" } });

cdk/cdk.json

@@ -0,0 +1,7 @@
+{
+  "app": "npx ts-node bin/cdk.ts",
+  "context": {
+    "aws-cdk:enableDiffNoFail": "true",
+    "@aws-cdk/core:stackRelativeExports": "true"
+  }
+}

cdk/eslint.config.mjs

@@ -0,0 +1,6 @@
+import guardian from '@guardian/eslint-config';
+
+export default [
+	...guardian.configs.recommended,
+	...guardian.configs.jest
+];

cdk/jest.setup.js

@@ -0,0 +1,2 @@
+/* eslint-disable no-undef -- jest globals are missing due to guardian.recommended.jest eslint rules not covering jest.setup.js */
+jest.mock("@guardian/cdk/lib/constants/tracking-tag");