Merge branch '42-refactor-single-cdk-construct-to-nested-stacks' into 'develop'

Resolve "Split single core.Construct into nested Stacks"

Closes #42

See merge request verbose-equals-true/django-postgres-vue-gitlab-ecs!82

Author: briancaffey

.gitignore

@@ -17,4 +17,5 @@ backend/celerybeat-schedule
 .pytest_cache
 .~lock.*
 notes/
-.vscode
+.vscode
+cdk.out

README.md

@@ -4,7 +4,6 @@ Documentation for this project can be found here:
 
 [https://verbose-equals-true.gitlab.io/django-postgres-vue-gitlab-ecs/](https://verbose-equals-true.gitlab.io/django-postgres-vue-gitlab-ecs/)
 
-
 ## Architecture
 
 ![png](/architecture.png)
@@ -17,6 +16,13 @@ First, copy `.env.template` to a new file in the project's root directory called
 
 Currently I am working on replacing CloudFormation with CDK for infrastructure and deployment.
 
+To work with CDK, do the following:
+
+- Make sure you are using at least version 10 of node: `nvm use 13`
+- Activate the virtual environment with `source awscdk/.env/bin/activate`
+- `pip install -e awscdk` to install CDK dependencies
+- run `cdk synth --app awscdk/app.py --output awscdk/cdk.out` and view the resulting JSON for the nested CloudFormation stacks in `awscdk/cdk.out`
+
 ### Social Authentication Keys
 
 To use social sign on in development, you will need to create an application with the given provider.
@@ -50,7 +56,6 @@ docker-compose -f compose/docs.yml up --build
 
 This will make the docs available at `http://localhost:8082/docs/`. Hot-reloading through websockets is supported, so changes will show up as they are saved in your code editor.
 
-
 ### Access Django Shell in Jupyter Notebook
 
 With all containers running, run the following commands:

awscdk/app.py

@@ -6,13 +6,18 @@
 from awscdk.cdk_app_root import ApplicationStack
 
 # naming conventions, also used for ACM certs, DNS Records, resource naming
-# Note: dynamically generated resource names created in CDK are used
-# in GitLab CI, such as cluster name, task definitions, etc.
+# Dynamically generated resource names created in CDK are used in GitLab CI
+# such as cluster name, task definitions, etc.
 environment_name = f"{os.environ.get('ENVIRONMENT', 'dev')}"
 base_domain_name = os.environ.get("DOMAIN_NAME", "mysite.com")
+# if the the production environent subdomain should nott be included in the URL
+# redefine `full_domain_name` to `base_domain_name` for that environment
 full_domain_name = f"{environment_name}.{base_domain_name}"  # dev.mysite.com
+# if environment_name == "prod":
+#     full_domain_name = base_domain_name
 base_app_name = os.environ.get("APP_NAME", "mysite-com")
 full_app_name = f"{environment_name}-{base_app_name}"  # dev-mysite-com
+aws_region = os.environ.get("AWS_DEFAULT_REGION", "us-east-1")
 
 
 app = core.App()
@@ -24,7 +29,7 @@
     full_domain_name=full_domain_name,
     base_app_name=base_app_name,
     full_app_name=full_app_name,
-    env={"region": "us-east-1"},
+    env={"region": aws_region},
 )
 
 # in order to be able to tag ECS resources, you need to go to

awscdk/awscdk/alb.py

@@ -1,27 +1,17 @@
 from aws_cdk import (
-    aws_iam as iam,
     aws_ec2 as ec2,
-    aws_route53 as route53,
-    aws_certificatemanager as acm,
     aws_elasticloadbalancingv2 as elbv2,
+    aws_cloudformation as cloudformation,
     core,
 )
 
 
-class ApplicationLoadBalancerResources(core.Construct):
-    def __init__(
-        self,
-        scope: core.Construct,
-        id: str,
-        hosted_zone: route53.IHostedZone,
-        certificate: acm.ICertificate,
-        vpc: ec2.IVpc,
-        **kwargs
-    ) -> None:
+class AlbStack(cloudformation.NestedStack):
+    def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
         super().__init__(scope, id, **kwargs)
 
         self.alb = elbv2.ApplicationLoadBalancer(
-            self, "ALB", internet_facing=True, vpc=vpc
+            self, "ALB", internet_facing=True, vpc=scope.vpc
         )
 
         self.alb.connections.allow_from_any_ipv4(
@@ -37,7 +27,10 @@ def __init__(
         )
 
         self.https_listener = self.alb.add_listener(
-            "HTTPSListener", port=443, certificates=[certificate], open=True
+            "HTTPSListener",
+            port=443,
+            certificates=[scope.certificate],
+            open=True,
         )
 
         # self.listener.add_redirect_response(
@@ -49,7 +42,8 @@ def __init__(
             "DefaultTargetGroup",
             port=80,
             protocol=elbv2.ApplicationProtocol.HTTP,
-            vpc=vpc,
+            vpc=scope.vpc,
+            target_type=elbv2.TargetType.IP,
         )
 
         self.listener.add_target_groups(

awscdk/awscdk/awscdk.egg-info/requires.txt

@@ -1,18 +1,20 @@
-aws-cdk.core==1.38.0
-aws-cdk.aws_certificatemanager==1.38.0
-aws-cdk.aws_secretsmanager==1.38.0
-aws-cdk.aws_route53==1.38.0
-aws-cdk.aws_s3==1.38.0
-aws_cdk.aws_s3_deployment==1.38.0
-aws-cdk.aws_cloudfront==1.38.0
-aws-cdk.aws_route53_targets==1.38.0
-aws-cdk.aws_ecr==1.38.0
-aws-cdk.aws_ec2==1.38.0
-aws-cdk.aws_rds==1.38.0
-aws-cdk.aws_ssm==1.38.0
-aws-cdk.aws_elasticache==1.38.0
-aws-cdk.aws_elasticloadbalancingv2==1.38.0
-aws-cdk.aws_ecs==1.38.0
-aws-cdk.aws_ecs_patterns==1.38.0
-aws-cdk.aws_autoscaling==1.38.0
-aws-cdk.aws_sqs==1.38.0
+aws-cdk.core==1.41.0
+aws-cdk.aws_cloudformation==1.41.0
+aws-cdk.aws_certificatemanager==1.41.0
+aws-cdk.aws_logs==1.41.0
+aws-cdk.aws_secretsmanager==1.41.0
+aws-cdk.aws_route53==1.41.0
+aws-cdk.aws_s3==1.41.0
+aws_cdk.aws_s3_deployment==1.41.0
+aws-cdk.aws_cloudfront==1.41.0
+aws-cdk.aws_route53_targets==1.41.0
+aws-cdk.aws_ecr==1.41.0
+aws-cdk.aws_ec2==1.41.0
+aws-cdk.aws_rds==1.41.0
+aws-cdk.aws_ssm==1.41.0
+aws-cdk.aws_elasticache==1.41.0
+aws-cdk.aws_elasticloadbalancingv2==1.41.0
+aws-cdk.aws_ecs==1.41.0
+aws-cdk.aws_ecs_patterns==1.41.0
+aws-cdk.aws_autoscaling==1.41.0
+aws-cdk.aws_sqs==1.41.0

awscdk/awscdk/backend.py

@@ -1,41 +1,40 @@
-import os
-
 from aws_cdk import (
     core,
     aws_ec2 as ec2,
     aws_ecs as ecs,
-    aws_ecs_patterns as ecs_patterns,
+    aws_logs as logs,
+    aws_cloudformation as cloudformation,
     aws_elasticloadbalancingv2 as elbv2,
 )
 
 
-class Backend(core.Construct):
-    def __init__(
-        self,
-        scope: core.Construct,
-        id: str,
-        image: ecs.AssetImage,
-        https_listener: elbv2.IApplicationListener,
-        cluster: ecs.ICluster,
-        environment_variables: core.Construct,
-        security_group: str,
-        **kwargs,
-    ) -> None:
+class BackendServiceStack(cloudformation.NestedStack):
+    def __init__(self, scope: core.Construct, id: str, **kwargs,) -> None:
         super().__init__(
             scope, id, **kwargs,
         )
 
         self.backend_task = ecs.FargateTaskDefinition(self, "BackendTask")
 
         self.backend_task.add_container(
-            "DjangoBackend",
-            image=image,
-            logging=ecs.LogDrivers.aws_logs(stream_prefix="Backend"),
-            environment=environment_variables.regular_variables,
-            secrets=environment_variables.secret_variables,
+            "BackendContainer",
+            image=scope.image,
+            logging=ecs.LogDrivers.aws_logs(
+                stream_prefix="BackendContainer",
+                log_retention=logs.RetentionDays.ONE_WEEK,
+            ),
+            environment=scope.variables.regular_variables,
+            secrets=scope.variables.secret_variables,
             command=["/start_prod.sh"],
         )
 
+        scope.backend_assets_bucket.grant_read_write(
+            self.backend_task.task_role
+        )
+
+        for secret in [scope.variables.django_secret_key, scope.rds.db_secret]:
+            secret.grant_read(self.backend_task.task_role)
+
         port_mapping = ecs.PortMapping(
             container_port=8000, protocol=ecs.Protocol.TCP
         )
@@ -46,13 +45,15 @@ def __init__(
             "BackendService",
             task_definition=self.backend_task,
             assign_public_ip=True,
-            cluster=cluster,
+            cluster=scope.ecs.cluster,
             security_group=ec2.SecurityGroup.from_security_group_id(
-                self, "BackendSecurityGroup", security_group_id=security_group
+                self,
+                "BackendServiceSecurityGroup",
+                security_group_id=scope.vpc.vpc_default_security_group,
             ),
         )
 
-        https_listener.add_targets(
+        scope.https_listener.add_targets(
             "BackendTarget",
             port=80,
             targets=[self.backend_service],

awscdk/awscdk/assets.py awscdk/awscdk/backend_assets.pyawscdk/awscdk/assets.py renamed to awscdk/awscdk/backend_assets.py

@@ -1,14 +1,17 @@
-from aws_cdk import aws_iam as iam, aws_s3 as s3, core
+from aws_cdk import (
+    aws_iam as iam,
+    aws_s3 as s3,
+    core,
+    aws_cloudformation as cloudformation,
+)
 
 
-class Assets(core.Construct):
-    def __init__(
-        self, scope: core.Construct, id: str, full_app_name: str, **kwargs
-    ) -> None:
+class BackendAssetsStack(cloudformation.NestedStack):
+    def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
         super().__init__(scope, id, **kwargs)
 
         self.assets_bucket = s3.Bucket(
-            self, "AssetsBucket", bucket_name=f"{full_app_name}-assets"
+            self, "AssetsBucket", bucket_name=f"{scope.full_app_name}-assets"
         )
 
         self.policy_statement = iam.PolicyStatement(

awscdk/awscdk/backend_tasks.py

@@ -1,65 +1,76 @@
 import os
 
-from aws_cdk import (
-    core,
-    aws_ecs as ecs,
-)
+from aws_cdk import core, aws_ecs as ecs, aws_cloudformation as cloudformation
 
+# These tasks are executed from manual GitLab CI jobs. The cluster is
+# specified with:
+# `aws ecs run-task --cluster ${ENVIRONMENT}-${APP_NAME}-cluster [...]`
+# TODO: consider making this more DRY
 
-class BackendTasks(core.Construct):
-    def __init__(
-        self,
-        scope: core.Construct,
-        id: str,
-        image: ecs.AssetImage,
-        cluster: ecs.ICluster,
-        environment_variables: core.Construct,
-        full_app_name: str,
-        **kwargs,
-    ) -> None:
+
+class BackendTasksStack(cloudformation.NestedStack):
+    def __init__(self, scope: core.Construct, id: str, **kwargs,) -> None:
         super().__init__(
             scope, id, **kwargs,
         )
 
+        # migrate
         self.migrate_task = ecs.FargateTaskDefinition(
-            self, "MigrateTask", family=f"{full_app_name}-migrate"
+            self, "MigrateTask", family=f"{scope.full_app_name}-migrate"
         )
 
+        for secret in [scope.variables.django_secret_key, scope.rds.db_secret]:
+            secret.grant_read(self.migrate_task.task_role)
+
         self.migrate_task.add_container(
             "MigrateCommand",
-            image=image,
-            environment=environment_variables.regular_variables,
-            secrets=environment_variables.secret_variables,
+            image=scope.image,
+            environment=scope.variables.regular_variables,
+            secrets=scope.variables.secret_variables,
             command=["python3", "manage.py", "migrate", "--no-input"],
             logging=ecs.LogDrivers.aws_logs(stream_prefix="MigrateCommand"),
         )
 
+        # collectstatic
         self.collectstatic_task = ecs.FargateTaskDefinition(
-            self, "CollecstaticTask", family=f"{full_app_name}-collectstatic"
+            self,
+            "CollecstaticTask",
+            family=f"{scope.full_app_name}-collectstatic",
         )
 
+        scope.backend_assets_bucket.grant_read_write(
+            self.collectstatic_task.task_role
+        )
+
+        for secret in [scope.variables.django_secret_key, scope.rds.db_secret]:
+            secret.grant_read(self.collectstatic_task.task_role)
+
         self.collectstatic_task.add_container(
             "CollecstaticCommand",
-            image=image,
-            environment=environment_variables.regular_variables,
-            secrets=environment_variables.secret_variables,
+            image=scope.image,
+            environment=scope.variables.regular_variables,
+            secrets=scope.variables.secret_variables,
             command=["python3", "manage.py", "collectstatic", "--no-input"],
             logging=ecs.LogDrivers.aws_logs(
                 stream_prefix="CollectstaticCommand"
             ),
         )
 
+        # createsuperuser
         self.create_superuser_task = ecs.FargateTaskDefinition(
             self,
             "CreateSuperuserTask",
-            family=f"{full_app_name}-create-superuser",
+            family=f"{scope.full_app_name}-create-superuser",
         )
 
+        for secret in [scope.variables.django_secret_key, scope.rds.db_secret]:
+            secret.grant_read(self.create_superuser_task.task_role)
+
         self.create_superuser_task.add_container(
             "CreateSuperuserCommand",
-            image=image,
-            environment=environment_variables.regular_variables,
-            secrets=environment_variables.secret_variables.update(
+            image=scope.image,
+            environment=scope.variables.regular_variables,
+            secrets=scope.variables.secret_variables.update(
                 {
                     "SUPERUSER_PASSWORD": os.environ.get(
                         "SUPERUSER_PASSWORD", "password"