Upgrade testcontainers to 2.x

pinot-integration-tests/pom.xml

@@ -250,7 +250,7 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>kafka</artifactId>
+      <artifactId>testcontainers-kafka</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/MultiStageEngineExplainIntegrationTest.java

@@ -21,12 +21,12 @@
 import java.io.File;
 import java.util.List;
 import java.util.Map;
+import org.apache.commons.io.FileUtils;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.util.TestUtils;
-import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;

pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/QueryThreadContextIntegrationTest.java

@@ -23,13 +23,13 @@
 import com.jayway.jsonpath.JsonPath;
 import java.io.File;
 import java.util.List;
+import org.apache.commons.io.FileUtils;
 import org.apache.pinot.core.query.utils.QueryIdUtils;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.util.TestUtils;
-import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/SpoolIntegrationTest.java

@@ -26,14 +26,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.Schema;
 import org.apache.pinot.spi.env.PinotConfiguration;
 import org.apache.pinot.spi.utils.CommonConstants;
 import org.apache.pinot.util.TestUtils;
 import org.intellij.lang.annotations.Language;
-import org.testcontainers.shaded.org.apache.commons.io.FileUtils;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/kafka/schemaregistry/SchemaRegistryStarter.java

@@ -69,7 +69,8 @@ public void start() {
 
       _network = Network.newNetwork();
 
-      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME).withNetwork(_network).withNetworkAliases("kafka")
+      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME)
+          .withNetwork(_network).withNetworkAliases("kafka")
           .withCreateContainerCmdModifier(it -> it.withHostName("kafka")).waitingFor(Wait.forListeningPort());
       _kafkaContainer.start();
 

pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/udf/UdfTest.java

@@ -24,6 +24,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.util.concurrent.MoreExecutors;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
@@ -54,7 +55,6 @@
 import org.assertj.core.api.Assertions;
 import org.assertj.core.util.diff.DiffUtils;
 import org.assertj.core.util.diff.Patch;
-import org.testcontainers.shaded.com.google.common.util.concurrent.MoreExecutors;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

pinot-plugins/pinot-file-system/pinot-s3/pom.xml

@@ -49,9 +49,8 @@
     </dependency>
 
     <dependency>
-      <groupId>com.adobe.testing</groupId>
-      <artifactId>s3mock-testcontainers</artifactId>
-      <version>${s3mock.version}</version>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>testcontainers</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

pinot-plugins/pinot-file-system/pinot-s3/src/test/java/org/apache/pinot/plugin/filesystem/S3PinotFSTest.java

@@ -18,7 +18,6 @@
  */
 package org.apache.pinot.plugin.filesystem;
 
-import com.adobe.testing.s3mock.testcontainers.S3MockContainer;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -33,6 +32,8 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.pinot.spi.filesystem.FileMetadata;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -56,7 +57,8 @@
 
 @Test
 public class S3PinotFSTest {
-  private static final String S3MOCK_VERSION = System.getProperty("s3mock.version", "2.12.2");
+  private static final String S3MOCK_VERSION = System.getProperty("s3mock.version", "3.12.0");
+  private static final int S3MOCK_HTTP_PORT = 9090;
   private static final File TEMP_FILE = new File(FileUtils.getTempDirectory(), "S3PinotFSTest");
 
   private static final String S3_SCHEME = "s3";
@@ -66,7 +68,7 @@ public class S3PinotFSTest {
   private static final String FILE_FORMAT = "%s://%s/%s";
   private static final String DIR_FORMAT = "%s://%s";
 
-  private S3MockContainer _s3MockContainer;
+  private GenericContainer<?> _s3MockContainer;
   private S3PinotFS _s3PinotFS;
   private S3Client _s3Client;
 
@@ -77,9 +79,12 @@ public static Object[][] schemes() {
 
   @BeforeClass
   public void setUp() {
-    _s3MockContainer = new S3MockContainer(S3MOCK_VERSION);
+    _s3MockContainer = new GenericContainer<>("adobe/s3mock:" + S3MOCK_VERSION)
+        .withExposedPorts(S3MOCK_HTTP_PORT)
+        .waitingFor(Wait.forHttp("/favicon.ico").forPort(S3MOCK_HTTP_PORT).forStatusCode(200));
     _s3MockContainer.start();
-    String endpoint = _s3MockContainer.getHttpEndpoint();
+    String endpoint = "http://" + _s3MockContainer.getHost() + ":"
+        + _s3MockContainer.getMappedPort(S3MOCK_HTTP_PORT);
     _s3Client = createS3ClientV2(endpoint);
     _s3PinotFS = new S3PinotFS();
     _s3PinotFS.init(_s3Client);

pinot-plugins/pinot-input-format/pinot-confluent-json/pom.xml

@@ -74,7 +74,7 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>kafka</artifactId>
+      <artifactId>testcontainers-kafka</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

pinot-plugins/pinot-input-format/pinot-confluent-json/src/test/java/org/apache/pinot/plugin/inputformat/json/confluent/kafka/schemaregistry/SchemaRegistryStarter.java

@@ -69,7 +69,8 @@ public void start() {
 
       _network = Network.newNetwork();
 
-      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME).withNetwork(_network).withNetworkAliases("kafka")
+      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME)
+          .withNetwork(_network).withNetworkAliases("kafka")
           .withCreateContainerCmdModifier(it -> it.withHostName("kafka")).waitingFor(Wait.forListeningPort());
       _kafkaContainer.start();
 

pinot-plugins/pinot-input-format/pinot-confluent-protobuf/pom.xml

@@ -61,7 +61,7 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>kafka</artifactId>
+      <artifactId>testcontainers-kafka</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

pinot-plugins/pinot-input-format/pinot-confluent-protobuf/src/test/java/org/apache/pinot/plugin/inputformat/protobuf/kafka/schemaregistry/SchemaRegistryStarter.java

@@ -69,7 +69,8 @@ public void start() {
 
       _network = Network.newNetwork();
 
-      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME).withNetwork(_network).withNetworkAliases("kafka")
+      _kafkaContainer = new KafkaContainer(KAFKA_DOCKER_IMAGE_NAME)
+          .withNetwork(_network).withNetworkAliases("kafka")
           .withCreateContainerCmdModifier(it -> it.withHostName("kafka")).waitingFor(Wait.forListeningPort());
       _kafkaContainer.start();
 

pinot-plugins/pinot-input-format/pinot-protobuf/pom.xml

@@ -61,7 +61,7 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>kafka</artifactId>
+      <artifactId>testcontainers-kafka</artifactId>
       <scope>test</scope>
     </dependency>
   </dependencies>

pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/pom.xml

@@ -63,7 +63,7 @@
     <!-- Testcontainers for embedded Kafka testing with KRaft mode -->
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>kafka</artifactId>
+      <artifactId>testcontainers-kafka</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

pinot-plugins/pinot-stream-ingestion/pinot-pulsar/pom.xml

@@ -49,7 +49,7 @@
     </dependency>
     <dependency>
       <groupId>org.testcontainers</groupId>
-      <artifactId>pulsar</artifactId>
+      <artifactId>testcontainers-pulsar</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

pinot-plugins/pinot-stream-ingestion/pinot-pulsar/src/test/java/org/apache/pinot/plugin/stream/pulsar/PulsarConsumerTest.java

@@ -43,7 +43,7 @@
 import org.apache.pulsar.client.api.Schema;
 import org.apache.pulsar.client.api.TopicMetadata;
 import org.apache.pulsar.client.impl.BatchMessageIdImpl;
-import org.testcontainers.containers.PulsarContainer;
+import org.testcontainers.pulsar.PulsarContainer;
 import org.testcontainers.utility.DockerImageName;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;

pom.xml

@@ -188,7 +188,6 @@
     <plexus-classworlds.version>2.9.0</plexus-classworlds.version>
     <scala-xml.version>2.4.0</scala-xml.version>
     <fmpp.version>0.9.16</fmpp.version>
-    <s3mock.version>2.17.0</s3mock.version>
     <localstack-utils.version>0.2.23</localstack-utils.version>
     <protobuf-dynamic.version>1.0.1</protobuf-dynamic.version>
     <enforcer-api.version>3.6.2</enforcer-api.version>
@@ -308,7 +307,7 @@
     <testng.version>7.12.0</testng.version>
     <mockito-core.version>5.17.0</mockito-core.version>
     <equalsverifier.version>3.19.4</equalsverifier.version>
-    <testcontainers.version>1.21.4</testcontainers.version>
+    <testcontainers.version>2.0.3</testcontainers.version>
     <h2.version>2.4.240</h2.version>
     <jnr-posix.version>3.1.21</jnr-posix.version>
     <scalatest.version>3.2.19</scalatest.version>