Merge branch 'develop' into 35-add-flower-for-celery-monitoring

Author: briancaffey

.env.template

@@ -18,8 +18,8 @@ GOOGLE_OAUTH2_SECRET=abcde-fghij
 SOCIAL_AUTH_FACEBOOK_KEY = ''
 SOCIAL_AUTH_FACEBOOK_SECRET = ''
 
-CELERY_BROKER_URL=redis://redis:6379
-CELERY_RESULT_BACKEND=redis://redis:6379
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/1
 
 DJANGO_EMAIL_HOST=mailhog
 DJANGO_EMAIL_PORT=1025

awscdk/awscdk/backend.py

@@ -60,6 +60,6 @@ def __init__(self, scope: core.Construct, id: str, **kwargs,) -> None:
             priority=1,
             path_patterns=["*"],
             health_check=elbv2.HealthCheck(
-                healthy_http_codes="200-299", path="/api/hello-world",
+                healthy_http_codes="200-299", path="/api/health-check/",
             ),
         )

awscdk/awscdk/cdk_app_root.py

@@ -55,6 +55,11 @@ def __init__(
         self.static_site_stack = StaticSiteStack(self, "StaticSiteStack")
         self.static_site_bucket = self.static_site_stack.static_site_bucket
 
+        self.backend_assets = BackendAssetsStack(self, "BackendAssetsStack")
+        self.backend_assets_bucket = self.backend_assets.assets_bucket
+
+        self.cloudfront = CloudFrontStack(self, "CloudFrontStack")
+
         if os.path.isdir("./quasar/dist/pwa"):
             s3_deployment.BucketDeployment(
                 self,
@@ -64,11 +69,6 @@ def __init__(
                 distribution=self.cloudfront.distribution,
             )
 
-        self.backend_assets = BackendAssetsStack(self, "BackendAssetsStack")
-        self.backend_assets_bucket = self.backend_assets.assets_bucket
-
-        self.cloudfront = CloudFrontStack(self, "CloudFrontStack")
-
         self.ecs = EcsStack(self, "EcsStack")
 
         self.rds = RdsStack(self, "RdsStack")

awscdk/awscdk/celery_default.py

@@ -4,7 +4,6 @@
     aws_ecs as ecs,
     aws_logs as logs,
     aws_cloudformation as cloudformation,
-    aws_sqs as sqs,
 )
 
 

awscdk/awscdk/cloudfront.py

@@ -65,10 +65,11 @@ def __init__(self, scope: core.Construct, id: str, **kwargs,) -> None:
                     behaviors=[
                         cloudfront.Behavior(
                             allowed_methods=ALL_METHODS,
+                            forwarded_values={"query_string": True},
                             path_pattern=path_pattern,
-                            max_ttl=core.Duration.seconds(0),
                             min_ttl=core.Duration.seconds(0),
-                            forwarded_values={"query_string": True},
+                            default_ttl=core.Duration.seconds(0),
+                            max_ttl=core.Duration.seconds(0),
                         )
                         for path_pattern in ["/static/*", "/media/*"]
                     ],

backend/apps/core/management/commands/__init__.py

@@ -6,21 +6,32 @@
 https://avilpage.com/2017/05/how-to-auto-reload-celery-workers-in-development.html
 """
 
+import os
 import shlex
 import subprocess
 
 from django.core.management.base import BaseCommand
 from django.utils import autoreload
 
 
-def restart_celery():
-    cmd = "pkill -9 celery"
+def restart_celery(queue=None, concurrency=None):
+    cmd = 'pkill -9 celery'
     subprocess.call(shlex.split(cmd))
-    cmd = "celery worker --app=backend.celery_app:app --loglevel=info --concurrency=2 --max-memory-per-child=150000"  # noqa
+    cmd = f"celery worker --app=backend.celery_app:app --loglevel=info -Q {queue} -n worker-{queue}@%h --concurrency={os.environ.get('CONCURRENT_WORKERS', 2)} --max-memory-per-child=150000"  # noqa
     subprocess.call(shlex.split(cmd))
 
 
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-q', '--queue', nargs=1, default='celery', type=str
+        )
+        parser.add_argument('-c', '--concurrency', type=str)
+
     def handle(self, *args, **options):
-        print("Starting celery worker with autoreload...")
-        autoreload.run_with_reloader(restart_celery)
+        queue = options['queue'][0]
+        concurrency = options['concurrency'] or 1
+        print('Starting celery worker with autoreload...')
+        autoreload.run_with_reloader(
+            restart_celery, queue=queue, concurrency=concurrency
+        )

backend/apps/core/management/commands/watch_celery.py

@@ -15,7 +15,6 @@ class BaseTask(celery.Task):
 @task(bind=True, base=BaseTask)
 def debug_task(self):
     time.sleep(10)
-    print("Task is done")
 
 
 @periodic_task(
@@ -30,9 +29,15 @@ def debug_periodic_task():
 @task(bind=True, base=BaseTask)
 def send_test_email_task(self):
     send_mail(
-        "Subject here",
-        "Here is the message.",
+        "Email subject",
+        "Email message.",
         "from@example.com",
         ["to@example.com"],
         fail_silently=False,
     )
+
+
+@task(bind=True, base=BaseTask)
+def sleep_task(self, seconds):
+    time.sleep(int(seconds))
+    return f"Slept {seconds} seconds"

backend/apps/core/tasks.py

@@ -3,9 +3,8 @@
 from . import views
 
 urlpatterns = [
-    path("", views.home, name="home"),
-    path("hello-world", views.hello_world, name="hello-world"),
-    path("debug-task/", views.debug_task_view, name="debug-task",),
+    path("health-check/", views.health_check, name="health-check"),
+    path("celery/sleep-task/", views.sleep_task_view, name="sleep-task"),
     path(
         "debug/send-test-email/",
         views.send_test_email,

backend/apps/core/urls.py

@@ -2,14 +2,11 @@
 
 from django.conf import settings
 from django.http import JsonResponse
-from django.views.decorators.cache import never_cache
 from django.views.generic import TemplateView
 from rest_framework import viewsets
+from rest_framework.decorators import api_view
 
-from apps.core.tasks import debug_task, send_test_email_task
-
-# Serve Vue Application via template for GitLab CI
-index_view = never_cache(TemplateView.as_view(template_name="index.html"))
+from apps.core.tasks import debug_task, send_test_email_task, sleep_task
 
 r = settings.REDIS
 
@@ -21,46 +18,34 @@ def get(self, request):
         value = r.get("cached_value")
 
         if value:
-            print(value)
             count = value
         return JsonResponse({"count": count})
 
     def post(self, request):
         new_count = int(request.data["count"])
-        print("setting value to cache:")
-        print(new_count)
         r.set("cached_value", new_count)
         new_count = r.get("cached_value")
-        print("value from cache is...")
-        print(new_count)
         return JsonResponse({"count": new_count})
 
     def delete(self, request):
         r.delete("cached_value")
         return JsonResponse({"count": r.get("cached_value")})
 
 
-def hello_world(request):
-    response = JsonResponse(
-        {
-            "message": "Hello, World!",
-            "git_sha": os.environ.get("GIT_SHA", "<git SHA>"),
-            "debug": settings.DEBUG,
-            "format": "JSON",
-            "ssm_param": os.environ.get("MY_PARAM", "param_value"),
-        }
-    )
+def health_check(request):
+    response = JsonResponse({"message": "OK"})
     return response
 
 
-def home(request):
-    response = JsonResponse({"message": "Root"})
-    return response
-
-
-def debug_task_view(request):
-    debug_task.delay()
-    return JsonResponse({"message": "Task sent to queue."})
+@api_view(["POST"])
+def sleep_task_view(request):
+    sleep_seconds = request.data.get("seconds")
+    sleep_task.apply_async(
+        [sleep_seconds], queue=settings.CELERY_QUEUE_DEFAULT
+    )
+    return JsonResponse(
+        {"message": f"Sleep task submitted ({sleep_seconds} seconds)"}
+    )
 
 
 def send_test_email(request):