test: unify exponential backoff use

tests/acm-certificate/index.test.ts

@@ -3,15 +3,14 @@ import * as automation from '../automation';
 import { InlineProgramArgs } from '@pulumi/pulumi/automation';
 import { ACMClient } from '@aws-sdk/client-acm';
 import { Route53Client } from '@aws-sdk/client-route-53';
-import { backOff } from 'exponential-backoff';
 import {
   DescribeCertificateCommand,
   CertificateType,
 } from '@aws-sdk/client-acm';
 import { ListResourceRecordSetsCommand } from '@aws-sdk/client-route-53';
 import { AcmCertificateTestContext } from './test-context';
 import { describe, it, before, after } from 'node:test';
-import { requireEnv } from '../util';
+import { backOff, requireEnv } from '../util';
 import * as infraConfig from './infrastructure/config';
 
 const programArgs: InlineProgramArgs = {
@@ -30,13 +29,6 @@ const ctx: AcmCertificateTestContext = {
     certificateDomain: infraConfig.certificateDomain,
     sanCertificateDomain: infraConfig.sanCertificateDomain,
     certificateSANs: infraConfig.certificateSANs,
-    exponentialBackOffConfig: {
-      delayFirstAttempt: true,
-      numOfAttempts: 5,
-      startingDelay: 2000,
-      timeMultiple: 1.5,
-      jitter: 'full',
-    },
   },
   clients: {
     acm: new ACMClient({ region }),
@@ -76,7 +68,7 @@ describe('ACM Certificate component deployment', () => {
         CertificateType.AMAZON_ISSUED,
         'Should be Amazon issued certificate',
       );
-    }, ctx.config.exponentialBackOffConfig);
+    });
   });
 
   it('should have validation record with correct resource record value', async () => {
@@ -157,6 +149,6 @@ describe('ACM Certificate component deployment', () => {
 
       const cert = certResult.Certificate;
       assert.ok(cert, 'Certificate should exist');
-    }, ctx.config.exponentialBackOffConfig);
+    });
   });
 });

tests/acm-certificate/test-context.ts

@@ -6,13 +6,6 @@ interface AcmCertificateTestConfig {
   certificateDomain: string;
   sanCertificateDomain: string;
   certificateSANs: string[];
-  exponentialBackOffConfig: {
-    delayFirstAttempt: boolean;
-    numOfAttempts: number;
-    startingDelay: number;
-    timeMultiple: number;
-    jitter: 'full' | 'none';
-  };
 }
 
 interface ConfigContext {
@@ -32,6 +25,4 @@ interface AwsContext {
 }
 
 export interface AcmCertificateTestContext
-  extends ConfigContext,
-    PulumiProgramContext,
-    AwsContext {}
+  extends ConfigContext, PulumiProgramContext, AwsContext {}

tests/database/util.ts

@@ -5,7 +5,7 @@ import {
   DescribeDBInstancesCommand,
   DescribeDBSnapshotsCommand,
 } from '@aws-sdk/client-rds';
-import { backOff } from '../util';
+import { NonRetryableError, backOff } from '../util';
 import { createSpinner } from 'nanospinner';
 import { DatabaseTestContext } from './test-context';
 import * as studion from '@studion/infra-code-blocks';
@@ -64,59 +64,59 @@ export async function cleanupReplicas(ctx: DatabaseTestContext) {
 async function deleteReplica(ctx: DatabaseTestContext, db: studion.Database) {
   const replicaDBInstanceId = db.replica!.instance
     .identifier as unknown as string;
-  const deleteCommand = new DeleteDBInstanceCommand({
-    DBInstanceIdentifier: replicaDBInstanceId,
-    SkipFinalSnapshot: true,
-  });
-  await ctx.clients.rds.send(deleteCommand);
+
+  try {
+    const deleteCommand = new DeleteDBInstanceCommand({
+      DBInstanceIdentifier: replicaDBInstanceId,
+      SkipFinalSnapshot: true,
+    });
+
+    await ctx.clients.rds.send(deleteCommand);
+  } catch (err) {
+    if (err instanceof DBInstanceNotFoundFault) {
+      return;
+    }
+
+    throw err;
+  }
 
   // Wait for replica to be deleted
-  await backOff(
-    async () => {
-      try {
-        const describeCommand = new DescribeDBInstancesCommand({
-          DBInstanceIdentifier: replicaDBInstanceId,
-        });
-        const { DBInstances } = await ctx.clients.rds.send(describeCommand);
-
-        if (!DBInstances || !DBInstances.length) {
-          return;
-        }
-
-        const [DBInstance] = DBInstances;
-        if (DBInstance.DBInstanceStatus === 'deleting') {
-          throw new Error('DB instance still deleting');
-        }
-      } catch (err: unknown) {
-        if (err instanceof DBInstanceNotFoundFault) {
-          return;
-        }
-
-        throw new Error('Something went wrong');
+  await backOff(async () => {
+    try {
+      const describeCommand = new DescribeDBInstancesCommand({
+        DBInstanceIdentifier: replicaDBInstanceId,
+      });
+      const { DBInstances } = await ctx.clients.rds.send(describeCommand);
+
+      if (DBInstances![0].DBInstanceStatus === 'deleting') {
+        throw new Error('DB instance still deleting');
+      }
+    } catch (err) {
+      if (err instanceof DBInstanceNotFoundFault) {
+        return;
       }
-    },
-    { numOfAttempts: 10 },
-  );
+
+      throw err;
+    }
+  });
 
   // Wait for primary instance to exit modifying state
-  await backOff(
-    async () => {
+  await backOff(async () => {
+    try {
       const primaryDBInstanceId = db.instance
         .dbInstanceIdentifier as unknown as string;
       const describeCommand = new DescribeDBInstancesCommand({
         DBInstanceIdentifier: primaryDBInstanceId,
       });
       const { DBInstances } = await ctx.clients.rds.send(describeCommand);
 
-      if (!DBInstances || !DBInstances.length) {
-        throw new Error('DB instance not found');
-      }
-
-      const [DBInstance] = DBInstances;
-      if (DBInstance.DBInstanceStatus === 'modifying') {
+      if (DBInstances![0].DBInstanceStatus === 'modifying') {
         throw new Error('DB instance still modifying');
       }
-    },
-    { numOfAttempts: 10 },
-  );
+    } catch (err) {
+      if (err instanceof DBInstanceNotFoundFault) {
+        throw new NonRetryableError('Db instance not found', { cause: err });
+      }
+    }
+  });
 }

tests/ecs-service/index.test.ts

@@ -12,15 +12,14 @@ import { ElasticLoadBalancingV2Client } from '@aws-sdk/client-elastic-load-balan
 import { ServiceDiscoveryClient } from '@aws-sdk/client-servicediscovery';
 import { ApplicationAutoScalingClient } from '@aws-sdk/client-application-auto-scaling';
 import { EFSClient } from '@aws-sdk/client-efs';
-import { backOff } from 'exponential-backoff';
 import * as automation from '../automation';
 import { EcsTestContext } from './test-context';
 import { testConfigurableEcsService } from './configuration.test';
 import { testEcsServiceWithLb } from './load-balancer.test';
 import { testEcsServiceWithStorage } from './persistent-storage.test';
 import { testEcsServiceWithServiceDiscovery } from './service-discovery.test';
 import { testEcsServiceWithAutoscaling } from './autoscaling.test';
-import { requireEnv } from '../util';
+import { backOff, requireEnv } from '../util';
 
 const programArgs: InlineProgramArgs = {
   stackName: 'dev',
@@ -33,13 +32,6 @@ const ctx: EcsTestContext = {
   outputs: {},
   config: {
     minEcsName: 'ecs-test-min',
-    exponentialBackOffConfig: {
-      delayFirstAttempt: true,
-      numOfAttempts: 5,
-      startingDelay: 1000,
-      timeMultiple: 2,
-      jitter: 'full',
-    },
   },
   clients: {
     ecs: new ECSClient({ region }),
@@ -109,7 +101,7 @@ describe('EcsService component deployment', () => {
         service.runningCount,
         `Service should have ${service.desiredCount} running tasks`,
       );
-    }, ctx.config.exponentialBackOffConfig);
+    });
   });
 
   it('should have running tasks with the correct task definition', async () => {

tests/ecs-service/load-balancer.test.ts

@@ -1,11 +1,11 @@
 import { it } from 'node:test';
 import * as assert from 'node:assert';
-import { backOff } from 'exponential-backoff';
 import { EcsTestContext } from './test-context';
 import {
   DescribeTargetGroupsCommand,
   DescribeTargetHealthCommand,
 } from '@aws-sdk/client-elastic-load-balancing-v2';
+import { backOff } from '../util';
 
 export function testEcsServiceWithLb(ctx: EcsTestContext) {
   it('should properly configure load balancer when provided', async () => {
@@ -49,29 +49,23 @@ export function testEcsServiceWithLb(ctx: EcsTestContext) {
       TargetGroupArn: targetGroupArn,
     });
 
-    return backOff(
-      async () => {
-        const { TargetHealthDescriptions } =
-          await ctx.clients.elb.send(describeHealth);
-        assert.ok(
-          TargetHealthDescriptions && TargetHealthDescriptions.length > 0,
-          'Target group should have registered targets',
-        );
+    return backOff(async () => {
+      const { TargetHealthDescriptions } =
+        await ctx.clients.elb.send(describeHealth);
+      assert.ok(
+        TargetHealthDescriptions && TargetHealthDescriptions.length > 0,
+        'Target group should have registered targets',
+      );
 
-        // At least one target should be healthy
-        const healthyTargets = TargetHealthDescriptions.filter(
-          (target: any) => target.TargetHealth?.State === 'healthy',
-        );
-        assert.ok(
-          healthyTargets.length > 0,
-          'At least one target should be healthy',
-        );
-      },
-      {
-        ...ctx.config.exponentialBackOffConfig,
-        numOfAttempts: 10,
-      },
-    );
+      // At least one target should be healthy
+      const healthyTargets = TargetHealthDescriptions.filter(
+        (target: any) => target.TargetHealth?.State === 'healthy',
+      );
+      assert.ok(
+        healthyTargets.length > 0,
+        'At least one target should be healthy',
+      );
+    });
   });
 
   it('should be able to access the service via load balancer URL', async () => {
@@ -90,6 +84,6 @@ export function testEcsServiceWithLb(ctx: EcsTestContext) {
         text.includes('Simple PHP App'),
         'Response should contain expected content',
       );
-    }, ctx.config.exponentialBackOffConfig);
+    });
   });
 }

tests/ecs-service/persistent-storage.test.ts

@@ -13,7 +13,7 @@ import {
   GetLogEventsCommand,
 } from '@aws-sdk/client-cloudwatch-logs';
 import { DescribeTasksCommand, ListTasksCommand } from '@aws-sdk/client-ecs';
-import { backOff } from 'exponential-backoff';
+import { backOff } from '../util';
 import { EcsTestContext } from './test-context';
 
 export function testEcsServiceWithStorage(ctx: EcsTestContext) {
@@ -275,7 +275,7 @@ export function testEcsServiceWithStorage(ctx: EcsTestContext) {
       assert.ok(taskArns && taskArns.length > 0, 'Task should be running');
 
       return taskArns;
-    }, ctx.config.exponentialBackOffConfig);
+    });
 
     const describeTasksCommand = new DescribeTasksCommand({
       cluster: clusterName,
@@ -295,37 +295,29 @@ export function testEcsServiceWithStorage(ctx: EcsTestContext) {
       logStreamNamePrefix: `ecs/test-container/${taskId}`,
     });
 
-    return backOff(
-      async () => {
-        const { logStreams = [] } = await logsClient.send(
-          describeStreamsCommand,
-        );
-        assert.ok(logStreams.length > 0, 'Log stream should exist');
-
-        const getLogsCommand = new GetLogEventsCommand({
-          logGroupName: ecsServiceWithStorage.logGroup.name,
-          logStreamName: logStreams[0].logStreamName,
-          startFromHead: true,
-        });
-
-        const { events } = await logsClient.send(getLogsCommand);
-        assert.ok(events && events.length > 0, 'Log events should exist');
-
-        const logContent = events.map(e => e.message).join('\n');
-        assert.ok(
-          logContent.includes('Successfully wrote to and read from EFS volume'),
-          'Logs should indicate successful EFS operation',
-        );
-        assert.ok(
-          !logContent.includes('Failed to write to or read from EFS volume'),
-          'Logs should not contain failure messages',
-        );
-      },
-      {
-        ...ctx.config.exponentialBackOffConfig,
-        numOfAttempts: 8,
-      },
-    );
+    return backOff(async () => {
+      const { logStreams = [] } = await logsClient.send(describeStreamsCommand);
+      assert.ok(logStreams.length > 0, 'Log stream should exist');
+
+      const getLogsCommand = new GetLogEventsCommand({
+        logGroupName: ecsServiceWithStorage.logGroup.name,
+        logStreamName: logStreams[0].logStreamName,
+        startFromHead: true,
+      });
+
+      const { events } = await logsClient.send(getLogsCommand);
+      assert.ok(events && events.length > 0, 'Log events should exist');
+
+      const logContent = events.map(e => e.message).join('\n');
+      assert.ok(
+        logContent.includes('Successfully wrote to and read from EFS volume'),
+        'Logs should indicate successful EFS operation',
+      );
+      assert.ok(
+        !logContent.includes('Failed to write to or read from EFS volume'),
+        'Logs should not contain failure messages',
+      );
+    });
   });
 
   it('should create ECS service when empty volumes array argument is passed', async () => {

tests/ecs-service/service-discovery.test.ts

@@ -1,11 +1,11 @@
 import { it } from 'node:test';
 import * as assert from 'node:assert';
-import { backOff } from 'exponential-backoff';
 import { EcsTestContext } from './test-context';
 import {
   GetNamespaceCommand,
   ListInstancesCommand,
 } from '@aws-sdk/client-servicediscovery';
+import { backOff } from '../util';
 
 export function testEcsServiceWithServiceDiscovery(ctx: EcsTestContext) {
   it('should create a private DNS namespace for service discovery', async () => {
@@ -44,6 +44,6 @@ export function testEcsServiceWithServiceDiscovery(ctx: EcsTestContext) {
         Instances && Instances.length > 0,
         'Service should have registered instances',
       );
-    }, ctx.config.exponentialBackOffConfig);
+    });
   });
 }