Skip to content

feat: Implement chunked file uploading #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WaveringAna wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: Implement chunked file uploading #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions app/lib/cleanup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import * as fsPromises from 'fs/promises';
import path from 'path';

const TMP_UPLOAD_DIR = path.join('uploads', 'tmp');
const DEFAULT_MAX_AGE_HOURS = 24; // Default to 24 hours

/**
* Cleans up orphaned chunk directories from the temporary upload directory.
* Orphaned chunks are directories that haven't been modified for a certain period.
*
* @param maxAgeInHours - The maximum age in hours for a directory to be considered orphaned.
* Defaults to DEFAULT_MAX_AGE_HOURS.
*/
export async function cleanupOrphanedChunks(maxAgeInHours: number = DEFAULT_MAX_AGE_HOURS): Promise<void> {
console.log(`[CLEANUP_SERVICE] Starting orphaned chunk cleanup. Max age: ${maxAgeInHours} hours.`);
const now = Date.now();
const maxAgeInMillis = maxAgeInHours * 60 * 60 * 1000;

try {
// Ensure the temporary directory exists
try {
await fsPromises.access(TMP_UPLOAD_DIR);
} catch (e: any) {
if (e.code === 'ENOENT') {
console.log(`[CLEANUP_SERVICE] Temporary upload directory ${TMP_UPLOAD_DIR} does not exist. Nothing to clean.`);
return;
}
throw e; // Other access errors
}

const entries = await fsPromises.readdir(TMP_UPLOAD_DIR, { withFileTypes: true });
let cleanedCount = 0;
let errorCount = 0;

for (const entry of entries) {
if (entry.isDirectory()) {
const dirPath = path.join(TMP_UPLOAD_DIR, entry.name);
try {
const stats = await fsPromises.stat(dirPath);
const dirAgeInMillis = now - stats.mtime.getTime();

if (dirAgeInMillis > maxAgeInMillis) {
console.log(`[CLEANUP_SERVICE] Deleting orphaned directory: ${dirPath} (age: ${Math.round(dirAgeInMillis / (60 * 60 * 1000))} hours)`);
await fsPromises.rm(dirPath, { recursive: true, force: true });
cleanedCount++;
}
} catch (statError: any) {
// If we can't stat the directory, it might have been deleted by another process, or permissions issue.
console.error(`[CLEANUP_SERVICE_ERROR] Error stating directory ${dirPath}: ${statError.message}. Skipping.`);
errorCount++;
}
}
}

if (cleanedCount > 0) {
console.log(`[CLEANUP_SERVICE] Successfully deleted ${cleanedCount} orphaned chunk director(ies).`);
} else {
console.log('[CLEANUP_SERVICE] No orphaned chunk directories found to delete.');
}
if (errorCount > 0) {
console.warn(`[CLEANUP_SERVICE] Encountered ${errorCount} errors during stat/delete operations.`);
}
console.log('[CLEANUP_SERVICE] Orphaned chunk cleanup finished.');

} catch (error: any) {
console.error(`[CLEANUP_SERVICE_FATAL] Error during cleanup process: ${error.message}`, error);
}
}

// Example of how it might be called (e.g., from a scheduler, not part of this subtask to implement the scheduler)
// (async () => {
// await cleanupOrphanedChunks(48); // Clean up anything older than 48 hours
// })();
206 changes: 159 additions & 47 deletions app/public/js/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ const getMediaType = filename => {

class FileUploader {
constructor() {
this.CHUNK_SIZE = 5 * 1024 * 1024; // 5MB
this.LARGE_FILE_THRESHOLD = 20 * 1024 * 1024; // 20MB
this.MAX_CHUNK_RETRIES = 3;

this.dropArea = document.getElementById('dropArea');
this.gallery = document.getElementById('gallery');
this.setupEventListeners();
Expand Down Expand Up @@ -132,63 +136,171 @@ class FileUploader {
}

async uploadFileWithProgress(file) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
const formData = new FormData();

formData.append('fileupload', file);
formData.append('expire', document.getElementById('expire').value);

// Show the progress UI
const container = document.getElementById('uploadProgressContainer');
const percentElem = document.getElementById('uploadPercent');
const barElem = document.getElementById('uploadBar');
container.style.display = 'block';

// Upload progress event
xhr.upload.addEventListener('progress', e => {
if (e.lengthComputable) {
const percent = (e.loaded / e.total) * 100;
percentElem.textContent = percent.toFixed(1) + '%';
barElem.style.width = percent + '%';
}
});

xhr.upload.addEventListener('load', () => {
console.log('Upload completed for', file.name);
});
const expire = document.getElementById('expire').value;
const container = document.getElementById('uploadProgressContainer');
const percentElem = document.getElementById('uploadPercent');
const barElem = document.getElementById('uploadBar');

// Show progress UI
container.style.display = 'block';
barElem.style.width = '0%';
percentElem.textContent = '0%';

if (file.size > this.LARGE_FILE_THRESHOLD) {
console.log(`File ${file.name} is large, using chunked upload.`);
try {
await this.uploadFileInChunks(file, expire, percentElem, barElem);
// Success message or UI update for chunked upload completion is handled in finalize
} catch (error) {
console.error(`Chunked upload failed for ${file.name}:`, error);
alert(`Upload failed for ${file.name}: ${error.message}`);
// Hide progress bar on error
container.style.display = 'none';
barElem.style.width = '0%';
percentElem.textContent = '0%';
throw error; // Re-throw to be caught by handleFiles if necessary
}
} else {
console.log(`File ${file.name} is small, using direct upload.`);
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
const formData = new FormData();

formData.append('fileupload', file);
formData.append('expire', expire);

xhr.upload.addEventListener('progress', e => {
if (e.lengthComputable) {
const percent = (e.loaded / e.total) * 100;
percentElem.textContent = percent.toFixed(1) + '%';
barElem.style.width = percent + '%';
}
});

xhr.onreadystatechange = () => {
if (xhr.readyState === 4) {
if (xhr.status === 200) {
// We got a success from the server, re-render the file list
console.log('Server returned success for', file.name);
xhr.upload.addEventListener('load', () => {
console.log('Direct upload completed for', file.name);
});

xhr.onreadystatechange = () => {
if (xhr.readyState === 4) {
// Hide & reset progress bar
container.style.display = 'none';
barElem.style.width = '0%';
percentElem.textContent = '0%';

// Insert updated partial into #embedder-list
document.getElementById('embedder-list').innerHTML = xhr.responseText;
htmx.process(document.getElementById('embedder-list'))
// Clear any "preview" in the gallery
this.gallery.innerHTML = '';

resolve();
} else {
// Some error from the server
const msg = `Upload failed: ${xhr.status} - ${xhr.responseText}`;
console.error(msg);
alert(msg);
reject(new Error(msg));
if (xhr.status === 200) {
console.log('Server returned success for direct upload:', file.name);
document.getElementById('embedder-list').innerHTML = xhr.responseText;
htmx.process(document.getElementById('embedder-list'));
this.gallery.innerHTML = '';
resolve();
} else {
const msg = `Upload failed: ${xhr.status} - ${xhr.responseText}`;
console.error(msg);
alert(msg);
reject(new Error(msg));
}
}
};
xhr.open('POST', '/');
xhr.send(formData);
});
}
}

async uploadFileInChunks(file, expire, percentElem, barElem) {
const totalChunks = Math.ceil(file.size / this.CHUNK_SIZE);
console.log(`Uploading ${file.name} in ${totalChunks} chunks of size ${this.CHUNK_SIZE} bytes.`);

for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
const start = chunkIndex * this.CHUNK_SIZE;
const end = Math.min(start + this.CHUNK_SIZE, file.size);
const chunk = file.slice(start, end);

const formData = new FormData();
formData.append('chunk', chunk, file.name); // file.name for the blob
formData.append('originalFilename', file.name);
formData.append('chunkIndex', String(chunkIndex));
formData.append('totalChunks', String(totalChunks));
// 'expire' is not sent with chunks, but with /complete

let retries = 0;
let success = false;
while (retries < this.MAX_CHUNK_RETRIES && !success) {
try {
console.log(`Uploading chunk ${chunkIndex + 1}/${totalChunks} for ${file.name} (Attempt ${retries + 1})`);
const response = await fetch('/upload/chunk', {
method: 'POST',
body: formData,
});

if (!response.ok) {
const errorData = await response.text();
throw new Error(`Chunk upload failed: ${response.status} - ${errorData}`);
}

const result = await response.json();
console.log(`Chunk ${result.chunkIndex} uploaded successfully for ${result.originalFilename}`);
success = true;
} catch (error) {
retries++;
console.error(`Error uploading chunk ${chunkIndex} for ${file.name}, attempt ${retries}:`, error);
if (retries >= this.MAX_CHUNK_RETRIES) {
throw new Error(`Failed to upload chunk ${chunkIndex} for ${file.name} after ${this.MAX_CHUNK_RETRIES} attempts. Error: ${error.message}`);
}
// Optional: Add a small delay before retrying
await new Promise(resolve => setTimeout(resolve, 1000 * retries));
}
};
}
// Update progress after each chunk
const percent = ((chunkIndex + 1) / totalChunks) * 100;
percentElem.textContent = percent.toFixed(1) + '%';
barElem.style.width = percent + '%';
}

xhr.open('POST', '/');
xhr.send(formData);
});
// All chunks uploaded, finalize
console.log(`All chunks uploaded for ${file.name}. Finalizing...`);
await this.finalizeChunkedUpload(file.name, totalChunks, expire);
}

async finalizeChunkedUpload(originalFilename, totalChunks, expire) {
const formData = new FormData();
formData.append('originalFilename', originalFilename);
formData.append('totalChunks', String(totalChunks));
formData.append('expire', expire); // Send expire with the completion request

try {
const response = await fetch('/upload/complete', {
method: 'POST',
body: formData,
});

if (!response.ok) {
const errorData = await response.text();
throw new Error(`Finalization failed: ${response.status} - ${errorData}`);
}

// Success, update UI
const responseText = await response.text();
document.getElementById('embedder-list').innerHTML = responseText;
htmx.process(document.getElementById('embedder-list'));
this.gallery.innerHTML = ''; // Clear any "preview"

// Hide progress bar elements after successful completion
const container = document.getElementById('uploadProgressContainer');
const barElem = document.getElementById('uploadBar');
const percentElem = document.getElementById('uploadPercent');
if (container) container.style.display = 'none';
if (barElem) barElem.style.width = '0%';
if (percentElem) percentElem.textContent = '0%';

console.log(`File ${originalFilename} successfully uploaded and processed.`);

} catch (error) {
console.error(`Error finalizing upload for ${originalFilename}:`, error);
alert(`Failed to finalize upload for ${originalFilename}: ${error.message}`);
throw error; // Re-throw for higher level handling if needed
}
}
}

Expand Down
Loading