remove legacy delta producer [WIP

.github/workflows/release-docker-image.yaml

@@ -54,7 +54,7 @@ jobs:
 
   e2e-test:
     name: E2E Test
-    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@v3
+    uses: IABTechLab/uid2-shared-actions/.github/workflows/shared-run-e2e-tests.yaml@ian-UID2-6945-sqs-e2e
     needs: Image
     with:
       operator_type: public

conf/default-config.json

@@ -9,9 +9,7 @@
   "optout_observe_only": false,
   "optout_s3_path_compat": false,
   "optout_partner_endpoint_mock": false,
-  "optout_add_entry_timeout_ms": 1000,
   "optout_data_dir": "/opt/uid2/optout/",
-  "optout_replica_uris": null,
   "optout_producer_replica_id": -1,
   "optout_producer_replica_id_offset": 0,
   "optout_producer_max_replicas": 3,
@@ -24,8 +22,6 @@
   "optout_delete_expired": true,
   "optout_s3_bucket": null,
   "optout_s3_folder": null,
-  "optout_legacy_producer_s3_folder": null,
-  "optout_sqs_s3_folder": null,
   "optout_sqs_queue_url": null,
   "cloud_download_threads": 8,
   "cloud_upload_threads": 2,
@@ -36,7 +32,6 @@
   "partners_config_path": "partners/config.json",
   "operator_type": "public",
   "uid_instance_id_prefix": "local-optout",
-  "optout_enqueue_sqs_enabled": false,
   "optout_sqs_max_queue_size": 0,
   "optout_sqs_max_messages_per_poll": 10,
   "optout_sqs_visibility_timeout": 300,
@@ -46,4 +41,4 @@
   "traffic_filter_config_path": null,
   "traffic_calc_config_path": null,
   "manual_override_s3_path": null
-}
+}

conf/integ-config.json

@@ -5,12 +5,11 @@
   "att_token_enc_salt": "<salt-for-attestation-token>",
   "optout_s3_folder": "optout/",
   "optout_internal_api_token": "test-optout-internal-key",
-  "optout_replica_uris": "http://localhost:8081/optout/write,http://localhost:8081/optout/write,http://localhost:8081/optout/write",
   "partners_config_path": "/com.uid2.core/test/partners/config.json",
   "partners_metadata_path": "http://localhost:8088/partners/refresh",
   "operators_metadata_path": "http://localhost:8088/operators/refresh",
   "core_attest_url": "http://localhost:8088/attest",
   "core_api_token": "trusted-partner-key",
   "enforceJwt": false,
   "uid_instance_id_prefix": "local-optout"
-}
+}

conf/local-config.json

@@ -7,7 +7,6 @@
   "optout_partner_endpoint_mock": true,
   "optout_internal_api_token": "test-optout-internal-key",
   "optout_producer_max_replicas": 1,
-  "optout_replica_uris": "http://127.0.0.1:8081/optout/write,http://127.0.0.1:8081/optout/write,http://127.0.0.1:8081/optout/write",
   "partners_config_path": "/com.uid2.core/test/partners/config.json",
   "partners_metadata_path": "/com.uid2.core/test/partners/metadata.json",
   "clients_metadata_path": "/com.uid2.core/test/clients/metadata.json",
@@ -18,4 +17,4 @@
   "optout_url": "http://localhost:8081",
   "enforceJwt": false,
   "uid_instance_id_prefix": "local-optout"
-}
+}

conf/local-e2e-docker-config.json

@@ -1,20 +1,25 @@
 {
   "service_instances": 1,
+  "service_verbose": true,
   "storage_mock": false,
   "aws_s3_endpoint": "http://localstack:5001",
+  "aws_sqs_endpoint": "http://localstack:5001",
   "optout_s3_bucket": "test-optout-bucket",
+  "optout_s3_bucket_dropped_requests": "test-optout-bucket",
+  "optout_sqs_queue_url": "http://localstack:5001/000000000000/optout-queue",
   "pre_signed_url_expiry": 1800,
-  "aws_region": "us-east-2",
+  "aws_region": "us-east-1",
   "aws_access_key_id": "no access key needed for test",
   "aws_secret_access_key": "no secret key needed for test",
   "att_token_enc_key": "<key-for-attestation-token>",
   "att_token_enc_salt": "<salt-for-attestation-token>",
   "optout_producer_max_replicas": 1,
+  "optout_producer_buffer_size": 65536,
   "optout_s3_folder": "optout-v2/",
   "optout_internal_api_token": "test-optout-internal-key",
-  "optout_replica_uris": "http://localhost:8081/optout/write,http://localhost:8081/optout/write,http://localhost:8081/optout/write",
   "optout_producer_replica_id": 0,
   "optout_delta_rotate_interval": 60,
+  "optout_sqs_delta_window_seconds": 0,
   "partners_metadata_path": "http://core:8088/partners/refresh",
   "operators_metadata_path": "http://core:8088/operators/refresh",
   "core_attest_url": "http://core:8088/attest",

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-optout</artifactId>
-    <version>4.7.5</version>
+    <version>4.7.58-alpha-207-SNAPSHOT</version>
     <name>uid2-optout</name>
     <url>https://github.com/IABTechLab/uid2-optout</url>
 

src/main/java/com/uid2/optout/Const.java

@@ -11,16 +11,12 @@ public static class Config extends com.uid2.shared.Const.Config {
         public static final String OptOutPartnerEndpointMockProp = "optout_partner_endpoint_mock";
         public static final String OptOutObserveOnlyProp = "optout_observe_only";
         public static final String OptOutS3PathCompatProp = "optout_s3_path_compat";
-        public static final String OptOutAddEntryTimeoutMsProp = "optout_add_entry_timeout_ms";
         public static final String OptOutProducerBufferSizeProp = "optout_producer_buffer_size";
         public static final String OptOutSenderReplicaIdProp = "optout_sender_replica_id";
         public static final String OptOutDeleteExpiredProp = "optout_delete_expired";
         public static final String PartnersConfigPathProp = "partners_config_path";
         public static final String PartnersMetadataPathProp = "partners_metadata_path";
         public static final String OptOutSqsQueueUrlProp = "optout_sqs_queue_url";
-        public static final String OptOutSqsEnabledProp = "optout_enqueue_sqs_enabled";
-        public static final String OptOutSqsS3FolderProp = "optout_sqs_s3_folder"; // sqs delta producer writes to this folder
-        public static final String OptOutLegacyProducerS3FolderProp = "optout_legacy_producer_s3_folder"; // legacy producer writes to this folder
         public static final String OptOutSqsMaxMessagesPerPollProp = "optout_sqs_max_messages_per_poll";
         public static final String OptOutSqsVisibilityTimeoutProp = "optout_sqs_visibility_timeout";
         public static final String OptOutDeltaJobTimeoutSecondsProp = "optout_delta_job_timeout_seconds";
@@ -35,14 +31,13 @@ public static class Config extends com.uid2.shared.Const.Config {
         public static final String OptOutTrafficCalcAllowlistRangesProp = "traffic_calc_allowlist_ranges";
         public static final String OptOutSqsDeltaWindowSecondsProp = "optout_sqs_delta_window_seconds";
         public static final String OptOutSqsMaxQueueSizeProp = "optout_sqs_max_queue_size";
+        public static final String AwsSqsEndpointProp = "aws_sqs_endpoint";
     }
 
     public static class Event {
-        public static final String DeltaProduce = "delta.produce";
         public static final String DeltaProduced = "delta.produced";
         public static final String DeltaSentRemote = "delta.sent_remote";
         public static final String PartitionProduce = "partition.produce";
         public static final String PartitionProduced = "partition.produced";
-        public static final String EntryAdd = "entry.add";
     }
 }