Feature/s3 rework

.github/workflows/publish_docker.yaml

@@ -68,4 +68,5 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
+          # platforms: linux/amd64,linux/arm64

packages/backend/src/common/SigV4Util.ts

@@ -125,8 +125,15 @@ export class S3SigV4Auth {
     // Or: GET /key/path instead of GET /bucket/key/path
     const [pathPart, query] = originalUrl.split('?');
 
-    // Strip /s3 prefix first
+    // Strip /s3 prefix first and try that path too
     const s3Path = this.extractS3Path(pathPart);
+    const s3PathWithQuery = query ? `${s3Path}?${query}` : s3Path;
+
+    // Add the path without /s3 prefix as a variation (clients sign without the base route)
+    if (s3PathWithQuery !== originalUrl) {
+      pathsToTry.push(s3PathWithQuery);
+    }
+
     const s3PathSegments = s3Path.split('/').filter(s => s);
 
     if (s3PathSegments.length >= 1) {
@@ -528,8 +535,8 @@ export class S3SigV4Auth {
     // Signed headers (semicolon-separated list)
     const signedHeadersString = signedHeaders.join(';');
 
-    // Payload hash
-    const payloadHash = this.hashPayload(body);
+    // Payload hash - check for x-amz-content-sha256 header first
+    const payloadHash = this.hashPayload(body, headers);
 
     // Combine into canonical request
     return [
@@ -633,7 +640,21 @@ export class S3SigV4Auth {
    * @param body - Request body (can be undefined for GET/HEAD requests)
    * @returns Hex-encoded SHA256 hash
    */
-  private static hashPayload(body: string | Buffer | undefined): string {
+  private static hashPayload(body: string | Buffer | undefined, headers?: IncomingHttpHeaders): string {
+    // Check if client provided pre-computed content hash or UNSIGNED-PAYLOAD
+    if (headers) {
+      const contentSha256 = headers['x-amz-content-sha256'] || headers['X-Amz-Content-Sha256'];
+      if (contentSha256 && typeof contentSha256 === 'string') {
+        // Special case: UNSIGNED-PAYLOAD means the client wants to skip payload verification
+        // In this case, we use the literal string 'UNSIGNED-PAYLOAD' in the canonical request
+        if (contentSha256 === 'UNSIGNED-PAYLOAD') {
+          return 'UNSIGNED-PAYLOAD';
+        }
+        // Otherwise, use the provided hash (common for large uploads to avoid re-computing)
+        return contentSha256;
+      }
+    }
+
     if (!body || (typeof body === 'string' && body.length === 0)) {
       // Empty body
       return createHash('sha256').update('').digest('hex');

packages/backend/src/services/connections/s3/handlers/AbortMultipartUpload.ts

@@ -0,0 +1,115 @@
+import express from 'express';
+import { prisma, redis } from '../../../../fork';
+import { S3SigV4Auth } from '../../../../common/SigV4Util';
+import fs from 'fs/promises';
+
+interface UploadSession {
+    bucket: {
+        id: string;
+        name: string;
+        ownerId: string;
+        access: string;
+        storageQuota: number;
+        createdAt: Date;
+        updatedAt: Date;
+    };
+    filename: string;
+    parent: {
+        id: string;
+        bucketId: string;
+    } | null;
+    tempFileBase: string;
+    tempFileParts: {
+        partNum: number;
+        path: string;
+        etag: string;
+    }[];
+    mimeType: string;
+}
+
+export default function S3Handlers_AbortMultipartUpload(router: express.Router) {
+    router.delete('/:bucket{/*objectPath}', async (req: express.Request, res) => {
+        try {
+            const { uploadId } = req.query;
+
+            // Only handle abort multipart upload requests
+            if (!uploadId) {
+                return;
+            }
+
+            const { bucket } = req.params;
+
+            // Get bucket
+            const bucketObj = await prisma.bucket.findFirst({
+                where: {
+                    name: bucket
+                }
+            });
+
+            if (!bucketObj) {
+                return res.status(404).send('Bucket not found');
+            }
+
+            if (bucketObj.access === 'private' || bucketObj.access === 'public-read') {
+                const accessKeyId = S3SigV4Auth.extractAccessKeyId(req.headers);
+
+                if (!accessKeyId) {
+                    return res.status(401).send('Unauthorized');
+                }
+
+                const credentialsInDb = await prisma.s3Credential.findUnique({
+                    where: {
+                        accessKey: accessKeyId,
+                        bucketAccess: { some: { bucketId: bucketObj.id } }
+                    },
+                    include: {
+                        user: true
+                    }
+                });
+                if (!credentialsInDb) {
+                    return res.status(401).send('Unauthorized');
+                }
+
+                const result = S3SigV4Auth.verifyWithPathDetection(
+                    req.method,
+                    req.originalUrl,
+                    req.path,
+                    req.headers,
+                    undefined,
+                    accessKeyId,
+                    credentialsInDb.secretKey
+                );
+
+                if (!result.isValid) {
+                    return res.status(403).send('Invalid signature');
+                }
+            }
+
+            // Get upload session
+            const uploadSession = await redis.json.GET(`s3:multipartupload:${uploadId as string}`) as UploadSession | null;
+
+            if (!uploadSession) {
+                return res.status(404).send('Upload session not found');
+            }
+
+            // Delete all part files
+            for (const part of uploadSession.tempFileParts) {
+                try {
+                    await fs.unlink(part.path);
+                } catch (err) {
+                    console.error(`Failed to delete part file ${part.path}:`, err);
+                }
+            }
+
+            // Delete the upload session from Redis
+            await redis.del(`s3:multipartupload:${uploadId as string}`);
+
+            console.log(`[S3] Aborted multipart upload: ${uploadId}`);
+
+            return res.status(204).send();
+        } catch (err: any) {
+            console.error('Error in AbortMultipartUpload handler:', err);
+            return res.status(500).send(`Internal server error: ${err?.message || err}`);
+        }
+    });
+}

packages/backend/src/services/connections/s3/handlers/DeleteObjects.ts

@@ -0,0 +1,180 @@
+import express from 'express';
+import { prisma } from '../../../../fork';
+import { S3SigV4Auth } from '../../../../common/SigV4Util';
+import { getObjectPath, resolvePath } from '../../../../common/object-nesting';
+import fs from 'fs/promises';
+import { XMLParser } from 'fast-xml-parser';
+import { escapeXml } from '../utils/xmlEscape';
+
+interface DeleteRequest {
+    Delete: {
+        Object: Array<{ Key: string }> | { Key: string };
+        Quiet?: boolean | string;
+    };
+}
+
+export default function S3Handlers_DeleteObjects(router: express.Router) {
+    router.post('/:bucket', async (req: express.Request, res) => {
+        try {
+            // Check if this is a delete request
+            if (!req.query.delete) {
+                return;
+            }
+
+            const { bucket } = req.params;
+
+            // Get bucket
+            const bucketObj = await prisma.bucket.findFirst({
+                where: {
+                    name: bucket
+                },
+                include: {
+                    BucketConfig: true
+                }
+            });
+
+            if (!bucketObj) {
+                return res.status(404).json({
+                    message: 'Bucket not found'
+                });
+            }
+
+            if (bucketObj.access === 'private' || bucketObj.access === 'public-read') {
+                const accessKeyId = S3SigV4Auth.extractAccessKeyId(req.headers);
+
+                if (!accessKeyId) {
+                    return res.status(401).json({ message: 'Unauthorized' });
+                }
+
+                const credentialsInDb = await prisma.s3Credential.findUnique({
+                    where: {
+                        accessKey: accessKeyId,
+                        bucketAccess: { some: { bucketId: bucketObj.id } }
+                    },
+                    include: {
+                        user: true
+                    }
+                });
+                if (!credentialsInDb) {
+                    return res.status(401).json({ message: 'Unauthorized' });
+                }
+
+                const result = S3SigV4Auth.verifyWithPathDetection(
+                    req.method,
+                    req.originalUrl,
+                    req.path,
+                    req.headers,
+                    req.body,
+                    accessKeyId,
+                    credentialsInDb.secretKey
+                );
+
+                if (!result.isValid) {
+                    return res.status(403).json({ message: 'Invalid signature' });
+                }
+            }
+
+            // Parse XML body
+            const xmlBody = typeof req.body === 'string' ? req.body : req.body.toString();
+
+            const parser = new XMLParser({
+                ignoreAttributes: false,
+                parseTagValue: true
+            });
+
+            try {
+                const result: DeleteRequest = parser.parse(xmlBody);
+
+                const objects = Array.isArray(result.Delete.Object) 
+                    ? result.Delete.Object 
+                    : [result.Delete.Object];
+
+                const quiet = result.Delete.Quiet === true || result.Delete.Quiet === 'true';
+
+                const deleted: Array<{ Key: string }> = [];
+                const errors: Array<{ Key: string; Code: string; Message: string }> = [];
+
+                for (const obj of objects) {
+                    try {
+                        const key = obj.Key;
+                        const pathSegments = key.split('/').filter((p: string) => p);
+
+                        const object = await resolvePath(bucketObj.name, pathSegments);
+
+                        if (!object) {
+                            if (!quiet) {
+                                errors.push({
+                                    Key: key,
+                                    Code: 'NoSuchKey',
+                                    Message: 'The specified key does not exist.'
+                                });
+                            }
+                            continue;
+                        }
+
+                        await prisma.object.delete({
+                            where: {
+                                id: object.id
+                            }
+                        });
+
+                        // Clean up empty parent folders if configured
+                        if (bucketObj.BucketConfig.find(c => c.key === 's3_clear_empty_parents')?.value === 'true' && object.parentId) {
+                            const { _count } = await prisma.object.aggregate({
+                                where: {
+                                    parentId: object.parentId,
+                                    bucketId: bucketObj.id
+                                },
+                                _count: {
+                                    id: true
+                                }
+                            });
+
+                            if (_count.id === 0) {
+                                await prisma.object.delete({
+                                    where: {
+                                        id: object.parentId
+                                    }
+                                }).catch(() => {});
+                            }
+                        }
+
+                        await fs.unlink(getObjectPath(bucketObj.name, object.id)).catch(() => {});
+
+                        deleted.push({ Key: key });
+                    } catch (error: any) {
+                        if (!quiet) {
+                            errors.push({
+                                Key: obj.Key,
+                                Code: 'InternalError',
+                                Message: error?.message || 'Internal error'
+                            });
+                        }
+                    }
+                }
+
+                // Build XML response
+                const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">${deleted.map(d => `
+    <Deleted>
+        <Key>${escapeXml(d.Key)}</Key>
+    </Deleted>`).join('')}${errors.map(e => `
+    <Error>
+        <Key>${escapeXml(e.Key)}</Key>
+        <Code>${escapeXml(e.Code)}</Code>
+        <Message>${escapeXml(e.Message)}</Message>
+    </Error>`).join('')}
+</DeleteResult>`;
+
+                res.setHeader('Content-Type', 'application/xml');
+                return res.status(200).send(xml);
+            } catch (parseError: any) {
+                return res.status(400).json({ message: `Invalid XML: ${parseError?.message || parseError}` });
+            }
+
+        } catch (err: any) {
+            console.error('Error in DeleteObjects handler:', err);
+            return res.status(500).json({ message: `Internal server error: ${err?.message || err}` });
+        }
+    });
+}