Merge branch '41-feature-queue-processing-fargate-service' into 'develop'

Resolve "Implement QueueProcessingFargateService"

Closes #41

See merge request verbose-equals-true/django-postgres-vue-gitlab-ecs!81

Author: briancaffey

awscdk/awscdk/awscdk.egg-info/requires.txt

@@ -15,3 +15,4 @@ aws-cdk.aws_elasticloadbalancingv2==1.38.0
 aws-cdk.aws_ecs==1.38.0
 aws-cdk.aws_ecs_patterns==1.38.0
 aws-cdk.aws_autoscaling==1.38.0
+aws-cdk.aws_sqs==1.38.0

awscdk/awscdk/cdk_app_root.py

@@ -142,9 +142,14 @@ def __init__(
             self.backend.backend_task.task_role,
             self.backend_tasks.collectstatic_task.task_role,
             self.backend_tasks.create_superuser_task.task_role,
-            self.celery_worker_service.celery_default_worker_task.task_role,
+            # self.celery_worker_service.celery_default_worker_task.task_role,
+            self.celery_worker_service.celery_default_worker_service.task_definition.task_role,
         ]
 
+        self.celery_worker_service.celery_default_worker_service.service.connections.security_groups.append(
+            self.vpc.vpc_default_security_group
+        )
+
         for task_role in task_roles:
             self.assets.assets_bucket.grant_read_write(task_role)
 

awscdk/awscdk/celery_workers.py

@@ -4,6 +4,7 @@
     aws_ecs as ecs,
     aws_ecs_patterns as ecs_patterns,
     aws_elasticloadbalancingv2 as elbv2,
+    aws_sqs as sqs,
 )
 
 
@@ -13,16 +14,50 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
             scope, id, **kwargs,
         )
 
-        self.celery_default_worker_task = ecs.FargateTaskDefinition(
-            self, "DefaultCeleryWorkerTask"
+        # self.celery_default_worker_task = ecs.FargateTaskDefinition(
+        #     self, "DefaultCeleryWorkerTask"
+        # )
+
+        # self.celery_default_worker_task.add_container(
+        #     "DefaultCeleryWorkerContaienr",
+        #     image=scope.image,
+        #     logging=ecs.LogDrivers.aws_logs(
+        #         stream_prefix="CeleryDefaultWorker"
+        #     ),
+        #     environment=scope.variables.regular_variables,
+        #     secrets=scope.variables.secret_variables,
+        #     command=[
+        #         'celery',
+        #         'worker',
+        #         '-A',
+        #         'backend.celery_app:app',
+        #         '-l',
+        #         'info',
+        #     ],
+        # )
+
+        # self.celery_default_worker_service = ecs.FargateService(
+        #     self,
+        #     "DefaultCeleryWorkerService",
+        #     task_definition=self.celery_default_worker_task,
+        #     assign_public_ip=True,
+        #     cluster=scope.ecs.cluster,
+        #     security_group=ec2.SecurityGroup.from_security_group_id(
+        #         self,
+        #         "CeleryDefaultWorkerSG",
+        #         security_group_id=scope.vpc.vpc_default_security_group,
+        #     ),
+        # )
+
+        self.default_celery_worker_queue = sqs.Queue(
+            self, "DefaultCeleryWorkerQueue", queue_name="default"
         )
 
-        self.celery_default_worker_task.add_container(
-            "DefaultCeleryWorkerContaienr",
+        self.celery_default_worker_service = ecs_patterns.QueueProcessingFargateService(
+            self,
+            "CeleryDefaultWorkerService",
             image=scope.image,
-            logging=ecs.LogDrivers.aws_logs(
-                stream_prefix="CeleryDefaultWorker"
-            ),
+            queue=self.default_celery_worker_queue,
             environment=scope.variables.regular_variables,
             secrets=scope.variables.secret_variables,
             command=[
@@ -33,17 +68,10 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
                 '-l',
                 'info',
             ],
-        )
-
-        self.celery_default_worker_service = ecs.FargateService(
-            self,
-            "DefaultCeleryWorkerService",
-            task_definition=self.celery_default_worker_task,
-            assign_public_ip=True,
-            cluster=scope.ecs.cluster,
-            security_group=ec2.SecurityGroup.from_security_group_id(
-                self,
-                "CeleryDefaultWorkerSG",
-                security_group_id=scope.vpc.vpc_default_security_group,
+            family="celery-default-worker",
+            log_driver=ecs.LogDrivers.aws_logs(
+                stream_prefix="CeleryDefaultWorker"
             ),
+            vpc=scope.vpc,
+            desired_task_count=1,
         )

awscdk/setup.py

@@ -32,6 +32,7 @@
         "aws-cdk.aws_ecs==1.38.0",
         "aws-cdk.aws_ecs_patterns==1.38.0",
         "aws-cdk.aws_autoscaling==1.38.0",
+        "aws-cdk.aws_sqs==1.38.0",
     ],
     python_requires=">=3.6",
     classifiers=[