Skip to content

Ship hadoop-common and hadoop-mapreduce-client-core to Flink lib #266

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
velo merged 4 commits into from
Dec 23, 2025
Merged

Ship hadoop-common and hadoop-mapreduce-client-core to Flink lib #266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
044b40f
Ship hadoop-common and hadoop-mapreduce-client-core to Flink lib
velo Dec 23, 2025
eb882eb
Move hadoop dependency management to parent pom
velo Dec 23, 2025
c5dfab3
Fix Avro version to 1.11.4 for flink-avro test-jar compatibility
velo Dec 23, 2025
faef1d3
Exclude avro from hadoop dependencies instead of version override
velo Dec 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions flink-sql-runner/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,18 @@
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<scope>provided</scope>
</dependency>

<!-- Add logging framework, to produce console output when running in the IDE. -->
<!-- These dependencies are excluded from the application JAR by default. -->
<dependency>
Expand Down Expand Up @@ -525,6 +537,30 @@
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</execution>
<execution>
<id>copy-hadoop-common-for-iceberg</id>
<goals>
<goal>copy-dependencies</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<includeGroupIds>org.apache.hadoop</includeGroupIds>
<includeArtifactIds>hadoop-common</includeArtifactIds>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</execution>
<execution>
<id>copy-hadoop-mapreduce-client-core-for-iceberg</id>
<goals>
<goal>copy-dependencies</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<includeGroupIds>org.apache.hadoop</includeGroupIds>
<includeArtifactIds>hadoop-mapreduce-client-core</includeArtifactIds>
<outputDirectory>${project.build.directory}</outputDirectory>
</configuration>
</execution>
<execution>
<id>copy-stdlib-utils</id>
<goals>
Expand Down
2 changes: 2 additions & 0 deletions flink-sql-runner/src/main/docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ RUN mkdir -p /opt/flink/plugins/flink-sql-runner

COPY aws-msk-iam-auth-*.jar /opt/flink/lib
COPY flink-s3-fs-hadoop-*.jar /opt/flink/lib
COPY hadoop-common-*.jar /opt/flink/lib
COPY hadoop-hdfs-client-*.jar /opt/flink/lib
COPY hadoop-mapreduce-client-core-*.jar /opt/flink/lib
COPY iceberg-flink-runtime-*.jar /opt/flink/lib
COPY iceberg-aws-bundle-*.jar /opt/flink/lib
COPY stdlib-utils-*.jar /opt/flink/lib
Expand Down
56 changes: 56 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,62 @@
<artifactId>testcontainers-postgresql</artifactId>
<version>${testcontainers.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down
11 changes: 0 additions & 11 deletions stdlib/stdlib-iceberg/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>ch.qos.reload4j</groupId>
<artifactId>reload4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
Expand Down