cherry-pick allowing numpy 2.3.x

.github/trigger_files/IO_Iceberg_Integration_Tests.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 3
+    "modification": 4
 }

.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 5
+  "modification": 1
 }

.github/trigger_files/IO_Iceberg_Managed_Integration_Tests_Dataflow.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 5
+  "modification": 1
 }

.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Direct.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run",
-  "modification": 6
+  "modification": 1
 }

.github/trigger_files/beam_PostCommit_SQL.json

@@ -1,4 +1,4 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run ",
-  "modification": 4
+  "modification": 2
 }

.github/workflows/build_wheels_manual.yml

@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Build Python 3.11 and 3.13 wheels (Manual)
+
+on:
+  workflow_dispatch:
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.sha || github.head_ref || github.ref }}-${{ github.event.sender.login}}'
+  cancel-in-progress: true
+
+jobs:
+  build_source:
+    runs-on: ubuntu-latest
+    name: Build python source distribution
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Build source
+        working-directory: ./sdks/python
+        run: pip install -U build && python -m build --sdist
+      - name: Add checksums
+        working-directory: ./sdks/python/dist
+        run: |
+          file=$(ls | grep .tar.gz | head -n 1)
+          sha512sum $file > ${file}.sha512
+      - name: Unzip source
+        working-directory: ./sdks/python
+        run: tar -xzvf dist/$(ls dist | grep .tar.gz | head -n 1)
+      - name: Rename source directory
+        working-directory: ./sdks/python
+        # https://github.com/pypa/setuptools/issues/4300 changed naming. Match both old and new names.
+        run: mv $(ls | grep "apache-beam-\|apache_beam-") apache-beam-source
+      - name: Upload source as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: source
+          path: sdks/python/apache-beam-source
+      - name: Upload compressed sources as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: source_zip
+          path: sdks/python/dist
+
+  build_wheels:
+    name: Build python ${{matrix.py_version}} wheels on ${{matrix.os_python.arch}} for ${{ matrix.os_python.os }}
+    needs:
+      - build_source
+    env:
+      CIBW_ARCHS_LINUX: ${{matrix.os_python.arch}}
+    runs-on: ${{ matrix.os_python.runner }}
+    timeout-minutes: 480
+    strategy:
+      fail-fast: false
+      matrix:
+        os_python: [
+          {"os": "ubuntu-latest", "runner": "ubuntu-latest", "arch": "x86_64" },
+          {"os": "macos-latest", "runner": "macos-latest", "arch": "x86_64" },
+          {"os": "ubuntu-latest", "runner": "ubuntu-latest", "arch": "aarch64" }
+        ]
+        # Only Python 3.11 and 3.13
+        py_version: ["cp311-", "cp313-"]
+    steps:
+    - name: Download python source distribution from artifacts
+      uses: actions/download-artifact@v4.1.8
+      with:
+        name: source
+        path: apache-beam-source
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+    - uses: docker/setup-qemu-action@v1
+      if: ${{matrix.os_python.arch == 'aarch64'}}
+      name: Set up QEMU
+    - name: Install cibuildwheel
+      run: pip install cibuildwheel==3.1.3 setuptools
+    - name: Fix distlib conflict on macOS
+      if: startsWith(matrix.os_python.os, 'macos')
+      working-directory: apache-beam-source
+      run: |
+        sed -i.bak 's/distlib==0.3.7/distlib>=0.3.7,<1.0.0/g' pyproject.toml
+    - name: Build wheel
+      working-directory: apache-beam-source
+      env:
+        CIBW_BUILD: ${{ matrix.py_version }}*
+        # TODO: https://github.com/apache/beam/issues/23048
+        CIBW_SKIP: "*-musllinux_*"
+        CIBW_BEFORE_BUILD: pip install cython==0.29.36 numpy --config-settings=setup-args="-Dallow-noblas=true" && pip install --upgrade setuptools
+      run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
+      shell: bash
+    - name: install sha512sum on MacOS
+      if: startsWith(matrix.os_python.os, 'macos')
+      run: brew install coreutils
+    - name: Add checksums
+      working-directory: apache-beam-source/wheelhouse/
+      run: |
+        for file in *.whl; do
+          sha512sum $file > ${file}.sha512
+        done
+      shell: bash
+    - name: Upload wheels as artifacts
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheelhouse-${{ matrix.py_version }}${{ matrix.os_python.os }}-${{ matrix.os_python.arch }}
+        path: apache-beam-source/wheelhouse/

CHANGES.md

@@ -70,6 +70,7 @@
 ## I/Os
 
 * Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
+* Upgraded Iceberg dependency to 1.9.2 ([#35981](https://github.com/apache/beam/pull/35981))
 
 ## New Features / Improvements
 
@@ -98,6 +99,7 @@
   improves support for BigQuery and other SQL dialects. Note: Minor behavior changes are observed such as output
   significant digits related to casting.
 * (Python) Prism runner now enabled by default for most Python pipelines using the direct runner ([#34612](https://github.com/apache/beam/pull/34612)). This may break some tests, see https://github.com/apache/beam/pull/34612 for details on how to handle issues.
+* Dropped Java 8 support for [IO expansion-service](https://central.sonatype.com/artifact/org.apache.beam/beam-sdks-java-io-expansion-service). Cross-language pipelines using this expansion service will need a Java11+ runtime ([#35981](https://github.com/apache/beam/pull/35981).
 
 ## Deprecations
 

build.gradle.kts

@@ -253,6 +253,7 @@ tasks.register("javaPreCommit") {
   dependsOn(":examples:java:sql:preCommit")
   dependsOn(":examples:java:twitter:build")
   dependsOn(":examples:java:twitter:preCommit")
+  dependsOn(":examples:java:iceberg:build")
   dependsOn(":examples:multi-language:build")
   dependsOn(":model:fn-execution:build")
   dependsOn(":model:job-management:build")
@@ -380,6 +381,7 @@ tasks.register("sqlPreCommit") {
   dependsOn(":sdks:java:extensions:sql:datacatalog:build")
   dependsOn(":sdks:java:extensions:sql:expansion-service:build")
   dependsOn(":sdks:java:extensions:sql:hcatalog:build")
+  dependsOn(":sdks:java:extensions:sql:iceberg:build")
   dependsOn(":sdks:java:extensions:sql:jdbc:build")
   dependsOn(":sdks:java:extensions:sql:jdbc:preCommit")
   dependsOn(":sdks:java:extensions:sql:perf-tests:build")
@@ -426,6 +428,7 @@ tasks.register("sqlPostCommit") {
   dependsOn(":sdks:java:extensions:sql:postCommit")
   dependsOn(":sdks:java:extensions:sql:jdbc:postCommit")
   dependsOn(":sdks:java:extensions:sql:datacatalog:postCommit")
+  dependsOn(":sdks:java:extensions:sql:iceberg:integrationTest")
   dependsOn(":sdks:java:extensions:sql:hadoopVersionsTest")
 }
 

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -617,7 +617,7 @@ class BeamModulePlugin implements Plugin<Project> {
     // [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
     def grpc_version = "1.71.0"
     def guava_version = "33.1.0-jre"
-    def hadoop_version = "3.4.1"
+    def hadoop_version = "3.4.2"
     def hamcrest_version = "2.1"
     def influxdb_version = "2.19"
     def httpclient_version = "4.5.13"

examples/java/build.gradle

@@ -71,9 +71,6 @@ dependencies {
   implementation project(":sdks:java:extensions:python")
   implementation project(":sdks:java:io:google-cloud-platform")
   implementation project(":sdks:java:io:kafka")
-  runtimeOnly project(":sdks:java:io:iceberg")
-  runtimeOnly project(":sdks:java:io:iceberg:bqms")
-  implementation project(":sdks:java:managed")
   implementation project(":sdks:java:extensions:ml")
   implementation library.java.avro
   implementation library.java.bigdataoss_util

examples/java/iceberg/build.gradle

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+plugins {
+  id 'java'
+  id 'org.apache.beam.module'
+  id 'com.gradleup.shadow'
+}
+
+applyJavaNature(
+  exportJavadoc: false,
+  automaticModuleName: 'org.apache.beam.examples.iceberg',
+  // iceberg requires Java11+
+  requireJavaVersion: JavaVersion.VERSION_11
+)
+
+description = "Apache Beam :: Examples :: Java :: Iceberg"
+ext.summary = """Apache Beam Java SDK examples using IcebergIO."""
+
+/** Define the list of runners which execute a precommit test.
+ * Some runners are run from separate projects, see the preCommit task below
+ * for details.
+ */
+def preCommitRunners = ["directRunner", "flinkRunner", "sparkRunner"]
+// The following runners have configuration created but not added to preCommit
+def nonPreCommitRunners = ["dataflowRunner", "prismRunner"]
+for (String runner : preCommitRunners) {
+  configurations.create(runner + "PreCommit")
+}
+for (String runner: nonPreCommitRunners) {
+  configurations.create(runner + "PreCommit")
+}
+configurations.sparkRunnerPreCommit {
+  // Ban certain dependencies to prevent a StackOverflow within Spark
+  // because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
+  exclude group: "org.slf4j", module: "jul-to-slf4j"
+  exclude group: "org.slf4j", module: "slf4j-jdk14"
+}
+
+dependencies {
+  implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
+  runtimeOnly project(":sdks:java:io:iceberg")
+  runtimeOnly project(":sdks:java:io:iceberg:bqms")
+  implementation project(path: ":sdks:java:core", configuration: "shadow")
+  implementation project(":sdks:java:extensions:google-cloud-platform-core")
+  implementation project(":sdks:java:io:google-cloud-platform")
+  implementation project(":sdks:java:managed")
+  implementation library.java.google_auth_library_oauth2_http
+  implementation library.java.joda_time
+  runtimeOnly project(path: ":runners:direct-java", configuration: "shadow")
+  implementation library.java.vendored_guava_32_1_2_jre
+  runtimeOnly library.java.hadoop_client
+  runtimeOnly library.java.bigdataoss_gcs_connector
+
+  // Add dependencies for the PreCommit configurations
+  // For each runner a project level dependency on the examples project.
+  for (String runner : preCommitRunners) {
+    delegate.add(runner + "PreCommit", project(path: ":examples:java", configuration: "testRuntimeMigration"))
+  }
+  directRunnerPreCommit project(path: ":runners:direct-java", configuration: "shadow")
+  flinkRunnerPreCommit project(":runners:flink:${project.ext.latestFlinkVersion}")
+  sparkRunnerPreCommit project(":runners:spark:3")
+  sparkRunnerPreCommit project(":sdks:java:io:hadoop-file-system")
+  dataflowRunnerPreCommit project(":runners:google-cloud-dataflow-java")
+  dataflowRunnerPreCommit project(":runners:google-cloud-dataflow-java:worker") // v2 worker
+  dataflowRunnerPreCommit project(":sdks:java:harness") // v2 worker
+  prismRunnerPreCommit project(":runners:prism:java")
+
+  // Add dependency if requested on command line for runner
+  if (project.hasProperty("runnerDependency")) {
+    runtimeOnly project(path: project.getProperty("runnerDependency"))
+  }
+}

examples/java/src/main/java/org/apache/beam/examples/cookbook/IcebergBatchWriteExample.java → examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/IcebergBatchWriteExample.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.cookbook;
+package org.apache.beam.examples.iceberg;
 
 import java.io.IOException;
 import java.util.Map;

examples/java/src/main/java/org/apache/beam/examples/cookbook/IcebergRestCatalogCDCExample.java → examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/IcebergRestCatalogCDCExample.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.cookbook;
+package org.apache.beam.examples.iceberg;
 
 import static org.apache.beam.sdk.managed.Managed.ICEBERG_CDC;
 

examples/java/src/main/java/org/apache/beam/examples/cookbook/IcebergRestCatalogStreamingWriteExample.java → examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/IcebergRestCatalogStreamingWriteExample.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.cookbook;
+package org.apache.beam.examples.iceberg;
 
 import com.google.auth.oauth2.GoogleCredentials;
 import java.io.IOException;

examples/java/src/main/java/org/apache/beam/examples/cookbook/IcebergTaxiExamples.java → examples/java/iceberg/src/main/java/org/apache/beam/examples/iceberg/IcebergTaxiExamples.java

@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.beam.examples.cookbook;
+package org.apache.beam.examples.iceberg;
 
 import java.util.Arrays;
 import java.util.Map;

gradle.properties

@@ -30,8 +30,8 @@ signing.gnupg.useLegacyGpg=true
 # buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy.
 # To build a custom Beam version make sure you change it in both places, see
 # https://github.com/apache/beam/issues/21302.
-version=2.68.0-SNAPSHOT
-sdk_version=2.68.0.dev
+version=2.68.0-post2
+sdk_version=2.68.0.post2
 
 javaVersion=1.8
 

runners/google-cloud-dataflow-java/build.gradle

@@ -52,8 +52,8 @@ evaluationDependsOn(":sdks:java:container:java11")
 
 ext.dataflowLegacyEnvironmentMajorVersion = '8'
 ext.dataflowFnapiEnvironmentMajorVersion = '8'
-ext.dataflowLegacyContainerVersion = 'beam-master-20250811'
-ext.dataflowFnapiContainerVersion = 'beam-master-20250811'
+ext.dataflowLegacyContainerVersion = '2.68.0'
+ext.dataflowFnapiContainerVersion = '2.68.0'
 ext.dataflowContainerBaseRepository = 'gcr.io/cloud-dataflow/v1beta3'
 
 processResources {

runners/google-cloud-dataflow-java/worker/build.gradle

@@ -131,7 +131,7 @@ applyJavaNature(
             dependencies {
                 // We have to include jetty-server/jetty-servlet and all of its transitive dependencies
                 // which includes several org.eclipse.jetty artifacts + servlet-api
-                include(dependency("org.eclipse.jetty:.*:9.4.54.v20240208"))
+                include(dependency("org.eclipse.jetty:.*:9.4.57.v20241219"))
                 include(dependency("javax.servlet:javax.servlet-api:3.1.0"))
             }
             relocate("org.eclipse.jetty", getWorkerRelocatedPath("org.eclipse.jetty"))
@@ -200,8 +200,8 @@ dependencies {
     compileOnly "org.conscrypt:conscrypt-openjdk-uber:2.5.1"
 
     implementation "javax.servlet:javax.servlet-api:3.1.0"
-    implementation "org.eclipse.jetty:jetty-server:9.4.54.v20240208"
-    implementation "org.eclipse.jetty:jetty-servlet:9.4.54.v20240208"
+    implementation "org.eclipse.jetty:jetty-server:9.4.57.v20241219"
+    implementation "org.eclipse.jetty:jetty-servlet:9.4.57.v20241219"
     implementation library.java.avro
     implementation library.java.jackson_annotations
     implementation library.java.jackson_core

sdks/go/pkg/beam/core/core.go

@@ -27,7 +27,7 @@ const (
 	// SdkName is the human readable name of the SDK for UserAgents.
 	SdkName = "Apache Beam SDK for Go"
 	// SdkVersion is the current version of the SDK.
-	SdkVersion = "2.68.0.dev"
+	SdkVersion = "2.68.0"
 
 	// DefaultDockerImage represents the associated image for this release.
 	DefaultDockerImage = "apache/beam_go_sdk:" + SdkVersion