Skip to content
This repository was archived by the owner on Mar 27, 2023. It is now read-only.

Commit fbc6049

Browse files
briancaffeybriancaffey
committed
added autoscaling to default celery queue
1 parent 14bebac commit fbc6049

File tree

7 files changed

+164
-22
lines changed

7 files changed

+164
-22
lines changed

‎awscdk/awscdk/awscdk.egg-info/requires.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
aws-cdk.core==1.41.0
22
aws-cdk.aws_cloudformation==1.41.0
3+
aws-cdk.aws_autoscaling==1.41.0
4+
aws-cdk.aws_applicationautoscaling==1.41.0
35
aws-cdk.aws_certificatemanager==1.41.0
6+
aws-cdk.aws_cloudwatch==1.41.0
47
aws-cdk.aws_logs==1.41.0
58
aws-cdk.aws_secretsmanager==1.41.0
69
aws-cdk.aws_route53==1.41.0

‎awscdk/awscdk/cdk_app_root.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ def __init__(
7171
)
7272

7373
self.ecs = EcsStack(self, "EcsStack")
74+
self.cluster = self.ecs.cluster
7475

7576
self.rds = RdsStack(self, "RdsStack")
7677

‎awscdk/awscdk/celery_default.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@
22
core,
33
aws_ec2 as ec2,
44
aws_ecs as ecs,
5+
aws_ecs_patterns as ecs_patterns,
56
aws_logs as logs,
67
aws_cloudformation as cloudformation,
8+
aws_cloudwatch as cloudwatch,
9+
aws_applicationautoscaling as app_autoscaling,
710
)
811

912

@@ -46,6 +49,7 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
4649
task_definition=self.celery_default_worker_task,
4750
assign_public_ip=True,
4851
cluster=scope.ecs.cluster,
52+
desired_count=0,
4953
security_group=ec2.SecurityGroup.from_security_group_id(
5054
self,
5155
"CeleryDefaultWorkerSG",
@@ -61,3 +65,51 @@ def __init__(self, scope: core.Construct, id: str, **kwargs) -> None:
6165
secret.grant_read(
6266
self.celery_default_worker_service.task_definition.task_role
6367
)
68+
69+
self.default_celery_queue_cloudwatch_metric = cloudwatch.Metric(
70+
namespace=scope.full_app_name, metric_name="default"
71+
)
72+
73+
self.celery_default_queue_asg = self.celery_default_worker_service.auto_scale_task_count(
74+
min_capacity=0, max_capacity=2
75+
)
76+
77+
self.celery_default_queue_asg.scale_on_metric(
78+
"CeleryDefaultQueueAutoscaling",
79+
metric=self.default_celery_queue_cloudwatch_metric,
80+
scaling_steps=[
81+
app_autoscaling.ScalingInterval(change=1, lower=0),
82+
app_autoscaling.ScalingInterval(change=-1, lower=1),
83+
],
84+
adjustment_type=app_autoscaling.AdjustmentType.CHANGE_IN_CAPACITY,
85+
)
86+
self.celery_default_cloudwatch_monitoring_task = ecs.FargateTaskDefinition(
87+
self, "CeleryDefaultCloudWatchMonitoringTask"
88+
)
89+
90+
self.celery_default_cloudwatch_monitoring_task.add_container(
91+
"CeleryDefaultCloudWatchMonitoringContainer",
92+
image=scope.image,
93+
logging=ecs.LogDrivers.aws_logs(
94+
stream_prefix="CeleryDefaultCloudWatchMonitoringContainerLogs",
95+
log_retention=logs.RetentionDays.ONE_DAY,
96+
),
97+
environment=scope.variables.regular_variables,
98+
secrets=scope.variables.secret_variables,
99+
command=["python3", "manage.py", "put_celery_cloudwatch_metrics"],
100+
)
101+
self.celery_default_cloudwatch_monitoring_scheduled_task = ecs_patterns.ScheduledFargateTask(
102+
self,
103+
"CeleryDefaultFargateScheduledTask",
104+
scheduled_fargate_task_definition_options=ecs_patterns.ScheduledFargateTaskDefinitionOptions(
105+
task_definition=self.celery_default_cloudwatch_monitoring_task,
106+
),
107+
schedule=app_autoscaling.Schedule.cron(minute="0/5"),
108+
cluster=scope.cluster,
109+
subnet_selection=ec2.SubnetSelection(
110+
subnet_type=ec2.SubnetType.PUBLIC
111+
),
112+
)
113+
self.default_celery_queue_cloudwatch_metric.grant_put_metric_data(
114+
self.celery_default_cloudwatch_monitoring_scheduled_task.task_definition.task_role
115+
)

‎awscdk/awscdk/env_vars.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def __init__(
3131
"DJANGO_SETTINGS_MODULE": "backend.settings.production",
3232
"DEBUG": "",
3333
"FULL_DOMAIN_NAME": full_domain_name,
34+
"FULL_APP_NAME": scope.full_app_name,
3435
"AWS_STORAGE_BUCKET_NAME": bucket_name,
3536
"POSTGRES_SERVICE_HOST": postgres_host,
3637
"POSTGRES_PASSWORD": db_secret.secret_value_from_json(

‎awscdk/setup.py‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,26 +15,29 @@
1515
package_dir={"": "awscdk"},
1616
packages=setuptools.find_packages(where="awscdk"),
1717
install_requires=[
18-
"aws-cdk.core==1.41.0",
19-
"aws-cdk.aws_cloudformation==1.41.0",
20-
"aws-cdk.aws_certificatemanager==1.41.0",
21-
"aws-cdk.aws_logs==1.41.0",
22-
"aws-cdk.aws_secretsmanager==1.41.0",
23-
"aws-cdk.aws_route53==1.41.0",
24-
"aws-cdk.aws_s3==1.41.0",
25-
"aws_cdk.aws_s3_deployment==1.41.0",
26-
"aws-cdk.aws_cloudfront==1.41.0",
27-
"aws-cdk.aws_route53_targets==1.41.0",
28-
"aws-cdk.aws_ecr==1.41.0",
29-
"aws-cdk.aws_ec2==1.41.0",
30-
"aws-cdk.aws_rds==1.41.0",
31-
"aws-cdk.aws_ssm==1.41.0",
32-
"aws-cdk.aws_elasticache==1.41.0",
33-
"aws-cdk.aws_elasticloadbalancingv2==1.41.0",
34-
"aws-cdk.aws_ecs==1.41.0",
35-
"aws-cdk.aws_ecs_patterns==1.41.0",
36-
"aws-cdk.aws_autoscaling==1.41.0",
37-
"aws-cdk.aws_sqs==1.41.0",
18+
"aws-cdk.core==1.42.0",
19+
"aws-cdk.aws_cloudformation==1.42.0",
20+
"aws-cdk.aws_autoscaling==1.42.0",
21+
"aws-cdk.aws_applicationautoscaling==1.42.0",
22+
"aws-cdk.aws_certificatemanager==1.42.0",
23+
"aws-cdk.aws_cloudwatch==1.42.0",
24+
"aws-cdk.aws_logs==1.42.0",
25+
"aws-cdk.aws_secretsmanager==1.42.0",
26+
"aws-cdk.aws_route53==1.42.0",
27+
"aws-cdk.aws_s3==1.42.0",
28+
"aws_cdk.aws_s3_deployment==1.42.0",
29+
"aws-cdk.aws_cloudfront==1.42.0",
30+
"aws-cdk.aws_route53_targets==1.42.0",
31+
"aws-cdk.aws_ecr==1.42.0",
32+
"aws-cdk.aws_ec2==1.42.0",
33+
"aws-cdk.aws_rds==1.42.0",
34+
"aws-cdk.aws_ssm==1.42.0",
35+
"aws-cdk.aws_elasticache==1.42.0",
36+
"aws-cdk.aws_elasticloadbalancingv2==1.42.0",
37+
"aws-cdk.aws_ecs==1.42.0",
38+
"aws-cdk.aws_ecs_patterns==1.42.0",
39+
"aws-cdk.aws_autoscaling==1.42.0",
40+
"aws-cdk.aws_sqs==1.42.0",
3841
],
3942
python_requires=">=3.6",
4043
classifiers=[

‎backend/apps/core/management/commands/put_celery_cloudwatch_metrics.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import os
2+
3+
import redis
4+
5+
from django.conf import settings
6+
from django.contrib.auth import get_user_model
7+
from django.core.management.base import BaseCommand
8+
9+
from apps.core import celery_app
10+
import boto3
11+
12+
13+
class Command(BaseCommand):
14+
help = "Creates a default superuser for local development"
15+
16+
def active_and_reserved_tasks_by_queue_name(self, queue_name):
17+
"""
18+
i.active() returns a dictionary where keys are worker names
19+
and values are lists of active tasks for the worker
20+
21+
"""
22+
i = celery_app.control.inspect()
23+
24+
active = i.active()
25+
active_count = 0
26+
if active:
27+
for _, active_tasks in active.items():
28+
active_count += len(
29+
[
30+
task
31+
for task in active_tasks
32+
if task['delivery_info']['routing_key'] == queue_name
33+
]
34+
)
35+
36+
reserved = i.reserved()
37+
reserved_count = 0
38+
if reserved:
39+
for _, reserved_tasks in reserved.items():
40+
reserved_count += len(
41+
[
42+
task
43+
for task in reserved_tasks
44+
if task['delivery_info']['routing_key'] == queue_name
45+
]
46+
)
47+
48+
r = redis.Redis(
49+
host=settings.REDIS_SERVICE_HOST,
50+
port=6379,
51+
db=1,
52+
charset="utf-8",
53+
decode_responses=True,
54+
)
55+
56+
queue_length = r.llen("default")
57+
print(f"Active count: {active_count}")
58+
print(f"Reserved count: {reserved_count}")
59+
print(f"Queue length: {queue_length}")
60+
61+
return active_count + reserved_count + queue_length
62+
63+
def publish_queue_metrics(self, queue_names):
64+
metric_data = {
65+
queue_name: self.active_and_reserved_tasks_by_queue_name(
66+
queue_name
67+
)
68+
for queue_name in queue_names
69+
}
70+
if not settings.DEBUG:
71+
client = boto3.client('cloudwatch')
72+
client.put_metric_data(
73+
Namespace=os.environ.get("FULL_APP_NAME", "FULL_APP_NAME"),
74+
MetricData=[
75+
{"MetricName": metric_name, "Value": value}
76+
for metric_name, value in metric_data.items()
77+
],
78+
)
79+
80+
def handle(self, *args, **options):
81+
82+
self.publish_queue_metrics(["default"])

‎gitlab-ci/aws/cdk.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ cdk deploy:
3030
services:
3131
- docker:19.03.5-dind
3232
stage: deploy
33-
only:
34-
- master
33+
# only:
34+
# - master
3535
variables:
3636
ENVIRONMENT: dev
3737
DOCKER_TLS_CERTDIR: ""

0 commit comments

Comments
 (0)