Merge latest changes from topic-release to main

SRE/Egress/Setup-templates/egressresources.yml

@@ -1,19 +1,28 @@
-# (c) 2022 Amazon Web Services, Inc. or its affiliates. All Rights Reserved.
-# This AWS Content is provided subject to the terms of the AWS Customer
-# Agreement available at https://aws.amazon.com/agreement or other written
-# agreement between Customer and Amazon Web Services, Inc
-
-###############################################################################
-
 AWSTemplateFormatVersion: "2010-09-09"
-Description: Provision all the required resources for TRE Egress application setup in RG.
+Description: >
+  Combined CloudFormation template for TRE Egress application setup in RG.
+  Creates egressstore and egressnotification buckets, Lambda function, and all required resources.
+  Lambda permissions are restricted to only the created buckets and KMS key.
+
+Parameters:
+  LambdaFunctionName:
+    Type: String
+    Description: Name of the Lambda function
+    AllowedPattern: ^[a-zA-Z0-9\-_.]{3,63}
+    Default: rg-EgressLambda
+  S3BucketName:
+    Type: String
+    Description: Name of the S3 bucket containing the Lambda zip file
+  S3ObjectKey:
+    Type: String
+    Description: Key of the Lambda zip file in the S3 bucket
+    Default: egress-copy.zip
 
 Resources:
   EgressStoreEncryptionKey:
     Type: AWS::KMS::Key
     Properties:
-      Description: >-
-        Master key used to encrypt objects stored in the egress-store bucket
+      Description: Master key used to encrypt objects stored in the egress-store bucket
       EnableKeyRotation: true
       KeyPolicy:
         Version: "2012-10-17"
@@ -27,39 +36,13 @@ Resources:
             Action:
               - "kms:*"
             Resource: "*"
-          # - Sid: Allow API access to create object and update policy for new workspaces
-          #   Effect: Allow
-          #   Principal:
-          #     AWS:
-          #       - !Sub "arn:aws:iam::${AWS::AccountId}:role/RGPortalUser"
-          #   Action:
-          #     - kms:GenerateDataKey
-          #     - kms:DescribeKey
-          #     - kms:GetKeyPolicy
-          #     - kms:PutKeyPolicy
-          #   Resource: "*"
-          # - Sid: Allow workflows to update key policy for new workspaces
-          #   Effect: Allow
-          #   Principal:
-          #     AWS:
-          #       - !Sub "arn:aws:iam::${AWS::AccountId}:role/RGPortalUser"
-          #   Action:
-          #     - kms:GenerateDataKey
-          #     - kms:Decrypt
-          #     - kms:DescribeKey
-          #     - kms:Encrypt
-          #     - kms:ReEncrypt*
-          #     - kms:GetKeyPolicy
-          #     - kms:PutKeyPolicy
-          #   Resource: "*"
 
   EgressStoreEncryptionKeyAlias:
     Type: AWS::KMS::Alias
     Properties:
       AliasName: !Sub "alias/${AWS::AccountId}-rg/s3/egressstorever"
       TargetKeyId: !Ref EgressStoreEncryptionKey
 
-  # S3 bucket used to store egress data from workspace
   EgressStoreBucket:
     Type: AWS::S3::Bucket
     Properties:
@@ -72,9 +55,6 @@ Resources:
       OwnershipControls:
         Rules:
           - ObjectOwnership: BucketOwnerPreferred
-      # LoggingConfiguration:
-      #   DestinationBucketName: "915161961510-treprod-ldn-pj1-logging"
-      #   LogFilePrefix: egressStore/
       VersioningConfiguration:
         Status: Enabled
       CorsConfiguration:
@@ -85,12 +65,12 @@ Resources:
               - POST
             ExposedHeaders:
               - ETag
-      PublicAccessBlockConfiguration: # Block all public access configuration for the S3 bucket
+      PublicAccessBlockConfiguration:
         BlockPublicAcls: true
         BlockPublicPolicy: true
         IgnorePublicAcls: true
         RestrictPublicBuckets: true
-  # S3 bucket policy used to store egress data from workspace
+
   EgressStoreBucketPolicy:
     Type: AWS::S3::BucketPolicy
     Properties:
@@ -117,7 +97,7 @@ Resources:
             Condition:
               StringNotEquals:
                 s3:signatureversion: "AWS4-HMAC-SHA256"
-  # S3 snapshot(egress copy) bucket used to copy the egress data from workspace bucket
+
   EgressNotificationBucket:
     Type: AWS::S3::Bucket
     Properties:
@@ -130,9 +110,8 @@ Resources:
       OwnershipControls:
         Rules:
           - ObjectOwnership: BucketOwnerPreferred
-      # LoggingConfiguration:
-      #   DestinationBucketName: "915161961510-treprod-ldn-pj1-logging"
-      #   LogFilePrefix: egressNotificationBucket/
+      VersioningConfiguration:
+        Status: Enabled
       CorsConfiguration:
         CorsRules:
           - AllowedOrigins:
@@ -141,14 +120,12 @@ Resources:
               - POST
             ExposedHeaders:
               - ETag
-      PublicAccessBlockConfiguration: # Block all public access configuration for the S3 bucket
+      PublicAccessBlockConfiguration:
         BlockPublicAcls: true
         BlockPublicPolicy: true
         IgnorePublicAcls: true
         RestrictPublicBuckets: true
-      VersioningConfiguration:
-        Status: Enabled
-  # S3 snapshot(copy egress) bucket policy used to copy the egress data from workspace bucket
+
   EgressNotificationBucketPolicy:
     Type: AWS::S3::BucketPolicy
     Properties:
@@ -175,14 +152,14 @@ Resources:
             Condition:
               StringNotEquals:
                 s3:signatureversion: "AWS4-HMAC-SHA256"
-  #SNS topic is created as part of this stack but Subscriptions would happen from egress stack
+
   EgressNotificationTopic:
     Type: AWS::SNS::Topic
     Properties:
       DisplayName: "rg-EgressTopicTest"
       KmsMasterKeyId: !Ref EgressStoreEncryptionKey
       TopicName: "EgressTopicTest"
-  #Egress Store dynamoDB
+
   EgressStoreDb:
     Type: AWS::DynamoDB::Table
     Properties:
@@ -203,3 +180,124 @@ Resources:
               KeyType: "HASH"
           Projection:
             ProjectionType: "ALL"
+
+  LambdaExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: "Allow"
+            Principal:
+              Service:
+                - "lambda.amazonaws.com"
+            Action:
+              - "sts:AssumeRole"
+      Policies:
+        - PolicyName: EgressLambdaBasicExecution
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Sid: RGLambdaBasics
+                Effect: "Allow"
+                Action:
+                  - "logs:CreateLogGroup"
+                  - "logs:CreateLogStream"
+                  - "logs:PutLogEvents"
+                Resource: "*"
+              - Sid: AllowEgressLambdaToCopyFromStoreToNotification
+                Effect: "Allow"
+                Action:
+                  - "s3:GetObject"
+                  - "s3:PutObject"
+                  - "s3:ListBucket"
+                  - "s3:DeleteObject"
+                  - "s3:GetBucketLocation"
+                  - "s3:ListBucketMultipartUploads"
+                  - "s3:AbortMultipartUpload"
+                  - "s3:ListMultipartUploadParts"
+                  - "s3:PutObjectAcl"
+                  - "s3:GetObjectAcl"
+                  - "s3:DeleteObjectVersion"
+                  - "s3:GetObjectVersion"
+                  - "s3:PutObjectVersionAcl"
+                  - "s3:GetObjectVersionAcl"
+                Resource:
+                  - !GetAtt EgressStoreBucket.Arn
+                  - !Join ["/", [!GetAtt EgressStoreBucket.Arn, "*"]]
+                  - !GetAtt EgressNotificationBucket.Arn
+                  - !Join ["/", [!GetAtt EgressNotificationBucket.Arn, "*"]]
+              - Sid: AllowKMSForEgressBuckets
+                Effect: "Allow"
+                Action:
+                  - "kms:Decrypt"
+                  - "kms:Encrypt"
+                  - "kms:GenerateDataKey"
+                  - "kms:DescribeKey"
+                Resource: !GetAtt EgressStoreEncryptionKey.Arn
+
+  LambdaFunction:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: !Ref LambdaFunctionName
+      Handler: index.handler
+      Role: !GetAtt LambdaExecutionRole.Arn
+      Runtime: nodejs18.x
+      Code:
+        S3Bucket: !Ref S3BucketName
+        S3Key: !Ref S3ObjectKey
+      Timeout: 900
+
+  LambdaFunctionUrl:
+    Type: AWS::Lambda::Url
+    Properties:
+      TargetFunctionArn: !Ref LambdaFunction
+      AuthType: AWS_IAM
+
+  InvokeLambdaPermissionPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      ManagedPolicyName:
+        Fn::Sub: "InvokeLambda-${LambdaFunctionName}"
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Allow
+            Action: "lambda:InvokeFunctionUrl"
+            Resource: !GetAtt LambdaFunction.Arn
+            Condition:
+              StringEquals:
+                lambda:FunctionUrlAuthType: "AWS_IAM"
+
+Outputs:
+  EgressStoreBucketArn:
+    Description: ARN of the Egress Store S3 bucket
+    Value: !GetAtt EgressStoreBucket.Arn
+
+  EgressStoreKmsArn:
+    Description: ARN of the Egress Store KMS Key
+    Value: !GetAtt EgressStoreEncryptionKey.Arn
+
+  EgressStoreBucketName:
+    Description: Name of the Egress Store S3 bucket
+    Value: !Ref EgressStoreBucket
+
+  EgressNotificationBucketName:
+    Description: Name of the Egress Notification S3 bucket
+    Value: !Ref EgressNotificationBucket
+
+  EgressStoreLambdaUrl:
+    Description: Lambda Function Url
+    Value: !GetAtt LambdaFunctionUrl.FunctionUrl
+
+  EgressStoreLambdaArn:
+    Description: ARN of the Lambda function
+    Value: !GetAtt LambdaFunction.Arn
+
+  EgressStoreDbTableName:
+    Description: Name of the Egress Store DynamoDB table
+    Value: !Ref EgressStoreDb
+
+  EgressNotificationTopicArn:
+    Description: ARN of the Egress Notification SNS Topic
+    Value: !Ref EgressNotificationTopic

SRE/lambdaresources/index.mjs

@@ -0,0 +1,94 @@
+import AWS from 'aws-sdk';
+import archiver from 'archiver';
+import stream from 'stream';
+
+const s3 = new AWS.S3();
+
+export const handler = async (event) => {
+
+    try {
+        const reqBody = JSON.parse(event.body);
+        const sourceBucket = reqBody.sourceBucket;
+        const sourcePrefix = reqBody.sourcePrefix;
+        const destinationBucket = reqBody.destinationBucket;
+        const destinationPrefix = reqBody.destinationPrefix;
+        console.log(`Zipping and uploading files from ${sourcePrefix} to ${destinationPrefix}`);
+
+        // List objects in the source S3 folder
+        const listParams = {
+            Bucket: sourceBucket,
+            Prefix: sourcePrefix
+        };
+        const contents = [];
+        let continuationToken;
+
+        do {
+            const page = await s3.listObjectsV2({
+                ...listParams,
+                ContinuationToken: continuationToken
+            }).promise();
+            contents.push(...(page.Contents ?? []));
+            continuationToken = page.IsTruncated ? page.NextContinuationToken : undefined;
+        } while (continuationToken);	    
+
+        if (listedObjects.Contents.length === 0) {
+            console.log('No files found in the source folder');
+            return { statusCode: 404, body: 'No files found in the source folder' };
+        }
+
+        // PassThrough stream and archive setup
+        const passThroughStream = new stream.PassThrough();
+        const archive = archiver('zip', { zlib: { level: 9 } });
+
+        archive.on('error', (error) => {
+            throw new Error(`Archiving error: ${error.message}`);
+        });
+
+        archive.pipe(passThroughStream);
+
+        // Start S3 upload process asynchronously
+        const uploadParams = {
+            Bucket: destinationBucket,
+            Key: destinationPrefix,
+            Body: passThroughStream,
+            ContentType: 'application/zip'
+        };
+        const uploadPromise = s3.upload(uploadParams).promise();
+
+        // Add files to archive
+        for (const obj of contents) {
+            const fileKey = obj.Key;
+            const fileStream = s3.getObject({ Bucket: sourceBucket, Key: fileKey }).createReadStream();
+            const filePath = fileKey.replace(sourcePrefix, '');
+
+            if (!filePath) continue;  // Skip empty paths
+
+            archive.append(fileStream, { name: filePath });
+            console.log(`Appended file to archive: ${filePath}`);
+        }
+
+        // Finalize the archive to signal completion
+        await archive.finalize();
+
+        // Wait for S3 upload to complete
+        await uploadPromise;
+
+        console.log('Zip file created and uploaded successfully');
+
+        return {
+            statusCode: 201,
+            headers: { "Content-Type": "application/json" },
+            body: 'Zip file created and uploaded successfully',
+            isBase64Encoded: false
+        };
+
+    } catch (error) {
+        console.error('Error creating or uploading zip file:', error.message);
+        return {
+            statusCode: 500,
+            headers: { "Content-Type": "application/json" },
+            body: `Error creating or uploading zip file: ${error.message}`,
+            isBase64Encoded: false
+        };
+    }
+};