File Handling in Full Stack Applications

Advanced File Handling Patterns

Chunked File Uploads

When dealing with very large files, it's often better to split them into smaller chunks for upload. This approach has several advantages:

• Improved reliability for large file uploads
• Ability to resume interrupted uploads
• Better progress tracking
• Reduced memory usage on both client and server

sequenceDiagram
    participant Client
    participant Server
    participant Storage
    
    Client->>Client: Split file into chunks
    Client->>Server: Upload chunk 1
    Server->>Storage: Store chunk 1
    Server->>Client: Acknowledge chunk 1
    Client->>Server: Upload chunk 2
    Server->>Storage: Store chunk 2
    Server->>Client: Acknowledge chunk 2
    Client->>Server: Upload chunk N
    Server->>Storage: Store chunk N
    Server->>Client: Acknowledge chunk N
    Client->>Server: Request file completion
    Server->>Storage: Merge chunks
    Storage->>Server: Return final file
    Server->>Client: Upload complete
                

Client-side Implementation

// Client-side chunked upload implementation
class ChunkedUploader {
  constructor(file, options = {}) {
    this.file = file;
    this.options = {
      chunkSize: 1024 * 1024 * 5, // 5MB chunks
      retries: 3,
      retryDelay: 1000,
      concurrentUploads: 3,
      onProgress: null,
      onComplete: null,
      onError: null,
      ...options
    };
    
    this.uploadId = null;
    this.chunks = this.prepareChunks();
    this.uploadedChunks = [];
    this.activeUploads = 0;
    this.totalChunks = this.chunks.length;
    this.uploadedBytes = 0;
    this.status = 'ready'; // ready, uploading, paused, completed, error
  }
  
  prepareChunks() {
    const chunks = [];
    let start = 0;
    let end = this.options.chunkSize;
    
    while (start < this.file.size) {
      chunks.push({
        index: chunks.length,
        start,
        end: Math.min(end, this.file.size),
        status: 'pending', // pending, uploading, completed, error
        retries: 0
      });
      
      start = end;
      end = start + this.options.chunkSize;
    }
    
    return chunks;
  }
  
  async start() {
    if (this.status === 'uploading') {
      return;
    }
    
    this.status = 'uploading';
    
    try {
      // Initialize upload session
      await this.initializeUpload();
      
      // Start uploading chunks concurrently
      this.processNextChunks();
    } catch (error) {
      this.handleError(error);
    }
  }
  
  async initializeUpload() {
    try {
      const response = await fetch('/api/uploads/chunked/initialize', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({
          fileName: this.file.name,
          fileSize: this.file.size,
          fileType: this.file.type,
          totalChunks: this.totalChunks
        })
      });
      
      if (!response.ok) {
        throw new Error(`Failed to initialize upload: ${response.statusText}`);
      }
      
      const data = await response.json();
      this.uploadId = data.uploadId;
    } catch (error) {
      console.error('Failed to initialize upload:', error);
      throw error;
    }
  }
  
  processNextChunks() {
    // Find pending chunks to upload
    const pendingChunks = this.chunks.filter(chunk => chunk.status === 'pending');
    
    if (pendingChunks.length === 0 && this.activeUploads === 0) {
      // All chunks uploaded, complete the upload
      this.completeUpload();
      return;
    }
    
    // Calculate how many new uploads to start
    const availableSlots = Math.max(0, this.options.concurrentUploads - this.activeUploads);
    const chunksToUpload = pendingChunks.slice(0, availableSlots);
    
    // Start new uploads
    chunksToUpload.forEach(chunk => {
      this.uploadChunk(chunk);
    });
  }
  
  async uploadChunk(chunk) {
    // Mark chunk as uploading
    chunk.status = 'uploading';
    this.activeUploads++;
    
    try {
      // Get chunk data as Blob
      const chunkData = this.file.slice(chunk.start, chunk.end);
      
      // Create FormData with chunk data
      const formData = new FormData();
      formData.append('uploadId', this.uploadId);
      formData.append('chunkIndex', chunk.index);
      formData.append('totalChunks', this.totalChunks);
      formData.append('chunkData', chunkData);
      
      // Upload chunk
      const response = await fetch('/api/uploads/chunked/chunk', {
        method: 'POST',
        body: formData
      });
      
      if (!response.ok) {
        throw new Error(`Failed to upload chunk ${chunk.index}: ${response.statusText}`);
      }
      
      // Mark chunk as completed
      chunk.status = 'completed';
      this.uploadedChunks.push(chunk.index);
      this.uploadedBytes += (chunk.end - chunk.start);
      
      // Report progress
      if (this.options.onProgress) {
        const percentage = (this.uploadedBytes / this.file.size) * 100;
        this.options.onProgress(percentage, this.uploadedBytes, this.file.size);
      }
    } catch (error) {
      console.error(`Chunk ${chunk.index} upload error:`, error);
      
      // Handle retry logic
      if (chunk.retries < this.options.retries) {
        chunk.retries++;
        chunk.status = 'pending';
        
        // Delay retry
        setTimeout(() => {
          // Only process next chunks if we're still uploading
          if (this.status === 'uploading') {
            this.processNextChunks();
          }
        }, this.options.retryDelay);
      } else {
        chunk.status = 'error';
        this.handleError(error);
      }
    } finally {
      this.activeUploads--;
      
      // Only process next chunks if we're still uploading
      if (this.status === 'uploading') {
        this.processNextChunks();
      }
    }
  }
  
  async completeUpload() {
    try {
      const response = await fetch('/api/uploads/chunked/complete', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({
          uploadId: this.uploadId
        })
      });
      
      if (!response.ok) {
        throw new Error(`Failed to complete upload: ${response.statusText}`);
      }
      
      const data = await response.json();
      
      this.status = 'completed';
      
      if (this.options.onComplete) {
        this.options.onComplete(data);
      }
    } catch (error) {
      console.error('Failed to complete upload:', error);
      this.handleError(error);
    }
  }
  
  pause() {
    if (this.status === 'uploading') {
      this.status = 'paused';
    }
  }
  
  resume() {
    if (this.status === 'paused') {
      this.status = 'uploading';
      this.processNextChunks();
    }
  }
  
  cancel() {
    this.status = 'cancelled';
    
    // Notify server to clean up
    fetch(`/api/uploads/chunked/cancel/${this.uploadId}`, {
      method: 'POST'
    }).catch(error => {
      console.error('Failed to cancel upload on server:', error);
    });
  }
  
  handleError(error) {
    this.status = 'error';
    
    if (this.options.onError) {
      this.options.onError(error);
    }
  }
}

// Example usage
document.getElementById('fileInput').addEventListener('change', (e) => {
  const file = e.target.files[0];
  if (!file) return;
  
  const progressBar = document.getElementById('progressBar');
  const uploadBtn = document.getElementById('uploadBtn');
  const pauseBtn = document.getElementById('pauseBtn');
  const resumeBtn = document.getElementById('resumeBtn');
  const cancelBtn = document.getElementById('cancelBtn');
  
  const uploader = new ChunkedUploader(file, {
    onProgress: (percentage, uploaded, total) => {
      progressBar.style.width = `${percentage}%`;
      progressBar.textContent = `${Math.round(percentage)}%`;
      
      const uploadedMB = (uploaded / (1024 * 1024)).toFixed(2);
      const totalMB = (total / (1024 * 1024)).toFixed(2);
      document.getElementById('uploadStatus').textContent = 
        `Uploaded ${uploadedMB} MB of ${totalMB} MB`;
    },
    onComplete: (data) => {
      progressBar.style.width = '100%';
      progressBar.textContent = 'Completed';
      document.getElementById('uploadStatus').textContent = 'Upload completed';
      
      // Display file URL
      const fileLink = document.createElement('a');
      fileLink.href = data.url;
      fileLink.textContent = 'Access your file';
      fileLink.target = '_blank';
      document.getElementById('fileLink').appendChild(fileLink);
    },
    onError: (error) => {
      document.getElementById('uploadStatus').textContent = `Error: ${error.message}`;
      progressBar.classList.add('error');
    }
  });
  
  uploadBtn.addEventListener('click', () => uploader.start());
  pauseBtn.addEventListener('click', () => uploader.pause());
  resumeBtn.addEventListener('click', () => uploader.resume());
  cancelBtn.addEventListener('click', () => uploader.cancel());
});
            

Server-side Implementation

// Server-side implementation of chunked uploads
const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const { S3Client, PutObjectCommand, CompleteMultipartUploadCommand, 
        CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3');
const crypto = require('crypto');

const app = express();

// Configure multer for handling chunk uploads
const upload = multer({
  storage: multer.diskStorage({
    destination: function(req, file, cb) {
      const uploadId = req.body.uploadId;
      const tempDir = path.join(__dirname, 'temp', uploadId);
      
      // Create directory if it doesn't exist
      if (!fs.existsSync(tempDir)) {
        fs.mkdirSync(tempDir, { recursive: true });
      }
      
      cb(null, tempDir);
    },
    filename: function(req, file, cb) {
      // Use chunk index as filename
      const chunkIndex = req.body.chunkIndex;
      cb(null, `chunk-${chunkIndex}`);
    }
  })
});

// Configure S3 client
const s3Client = new S3Client({
  region: process.env.AWS_REGION || 'us-east-1',
  credentials: {
    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
  }
});

const bucketName = process.env.AWS_S3_BUCKET;

// In-memory storage for upload sessions
const uploadSessions = new Map();

// Initialize a new chunked upload
app.post('/api/uploads/chunked/initialize', async (req, res) => {
  try {
    const { fileName, fileSize, fileType, totalChunks } = req.body;
    
    if (!fileName || !fileSize || !totalChunks) {
      return res.status(400).json({ error: 'Missing required parameters' });
    }
    
    // Generate upload ID
    const uploadId = crypto.randomBytes(16).toString('hex');
    
    // Generate a final target filename
    const fileExtension = path.extname(fileName);
    const baseName = path.basename(fileName, fileExtension).replace(/\s+/g, '-').toLowerCase();
    const finalFileName = `${Date.now()}-${crypto.randomBytes(8).toString('hex')}-${baseName}${fileExtension}`;
    
    // Store upload session info
    uploadSessions.set(uploadId, {
      fileName,
      fileSize: parseInt(fileSize),
      fileType,
      totalChunks: parseInt(totalChunks),
      finalFileName,
      receivedChunks: 0,
      chunkStatus: Array(parseInt(totalChunks)).fill(false),
      createdAt: new Date(),
      s3UploadId: null,
      s3Parts: []
    });
    
    // Create temp directory for chunks
    const tempDir = path.join(__dirname, 'temp', uploadId);
    if (!fs.existsSync(tempDir)) {
      fs.mkdirSync(tempDir, { recursive: true });
    }
    
    // For S3 multipart upload, initialize the multipart upload
    if (process.env.STORAGE_TYPE === 's3') {
      const command = new CreateMultipartUploadCommand({
        Bucket: bucketName,
        Key: finalFileName,
        ContentType: fileType
      });
      
      const { UploadId } = await s3Client.send(command);
      
      // Store S3 multipart upload ID
      const session = uploadSessions.get(uploadId);
      session.s3UploadId = UploadId;
      uploadSessions.set(uploadId, session);
    }
    
    res.status(200).json({
      uploadId,
      maxChunkSize: 1024 * 1024 * 5 // 5MB, for client reference
    });
    
    // Set up cleanup for abandoned uploads (e.g., after 24 hours)
    setTimeout(() => cleanupUpload(uploadId), 24 * 60 * 60 * 1000);
  } catch (error) {
    console.error('Upload initialization error:', error);
    res.status(500).json({ error: 'Failed to initialize upload' });
  }
});

// Handle chunk upload
app.post('/api/uploads/chunked/chunk', upload.single('chunkData'), async (req, res) => {
  try {
    const { uploadId, chunkIndex, totalChunks } = req.body;
    
    // Validate parameters
    if (!uploadId || chunkIndex === undefined || !totalChunks) {
      return res.status(400).json({ error: 'Missing required parameters' });
    }
    
    // Get upload session
    const session = uploadSessions.get(uploadId);
    if (!session) {
      return res.status(404).json({ error: 'Upload session not found' });
    }
    
    const index = parseInt(chunkIndex);
    
    // For S3 multipart upload
    if (process.env.STORAGE_TYPE === 's3' && session.s3UploadId) {
      // Read the chunk file
      const chunkFile = req.file.path;
      const fileContent = fs.readFileSync(chunkFile);
      
      // Upload part to S3
      const command = new UploadPartCommand({
        Bucket: bucketName,
        Key: session.finalFileName,
        UploadId: session.s3UploadId,
        PartNumber: index + 1, // S3 part numbers start at 1
        Body: fileContent
      });
      
      const { ETag } = await s3Client.send(command);
      
      // Store part info for completing the multipart upload later
      session.s3Parts[index] = {
        ETag,
        PartNumber: index + 1
      };
      
      // Clean up temp file
      fs.unlinkSync(chunkFile);
    }
    
    // Update session status
    session.chunkStatus[index] = true;
    session.receivedChunks++;
    uploadSessions.set(uploadId, session);
    
    res.status(200).json({
      success: true,
      uploadId,
      chunkIndex: index,
      receivedChunks: session.receivedChunks,
      totalChunks: session.totalChunks
    });
  } catch (error) {
    console.error('Chunk upload error:', error);
    res.status(500).json({ error: 'Failed to upload chunk' });
  }
});

// Complete the upload and merge chunks
app.post('/api/uploads/chunked/complete', async (req, res) => {
  try {
    const { uploadId } = req.body;
    
    if (!uploadId) {
      return res.status(400).json({ error: 'Upload ID is required' });
    }
    
    // Get upload session
    const session = uploadSessions.get(uploadId);
    if (!session) {
      return res.status(404).json({ error: 'Upload session not found' });
    }
    
    // Verify all chunks are received
    if (session.receivedChunks !== session.totalChunks) {
      return res.status(400).json({
        error: 'Not all chunks have been uploaded',
        receivedChunks: session.receivedChunks,
        totalChunks: session.totalChunks
      });
    }
    
    let fileUrl;
    
    // Complete the multipart upload for S3
    if (process.env.STORAGE_TYPE === 's3' && session.s3UploadId) {
      // Sort parts by part number
      const sortedParts = session.s3Parts
        .filter(part => part !== null)
        .sort((a, b) => a.PartNumber - b.PartNumber);
      
      // Complete the multipart upload
      const command = new CompleteMultipartUploadCommand({
        Bucket: bucketName,
        Key: session.finalFileName,
        UploadId: session.s3UploadId,
        MultipartUpload: {
          Parts: sortedParts
        }
      });
      
      await s3Client.send(command);
      
      // Generate URL for the file
      fileUrl = `https://${bucketName}.s3.amazonaws.com/${session.finalFileName}`;
    } else {
      // For local storage, merge chunks into a single file
      const tempDir = path.join(__dirname, 'temp', uploadId);
      const uploadDir = path.join(__dirname, 'uploads');
      
      // Ensure uploads directory exists
      if (!fs.existsSync(uploadDir)) {
        fs.mkdirSync(uploadDir);
      }
      
      const outputPath = path.join(uploadDir, session.finalFileName);
      
      // Create write stream for the final file
      const writeStream = fs.createWriteStream(outputPath);
      
      // Merge chunks
      for (let i = 0; i < session.totalChunks; i++) {
        const chunkPath = path.join(tempDir, `chunk-${i}`);
        
        // Read chunk and append to the final file
        const chunkData = fs.readFileSync(chunkPath);
        writeStream.write(chunkData);
        
        // Clean up chunk file
        fs.unlinkSync(chunkPath);
      }
      
      // Close write stream
      writeStream.end();
      
      // Generate URL for the file
      fileUrl = `/files/${session.finalFileName}`;
    }
    
    // Clean up temp directory
    const tempDir = path.join(__dirname, 'temp', uploadId);
    if (fs.existsSync(tempDir)) {
      fs.rmSync(tempDir, { recursive: true, force: true });
    }
    
    // Remove upload session from memory
    uploadSessions.delete(uploadId);
    
    res.status(200).json({
      success: true,
      file: {
        name: session.fileName,
        size: session.fileSize,
        type: session.fileType,
        url: fileUrl
      }
    });
  } catch (error) {
    console.error('Complete upload error:', error);
    res.status(500).json({ error: 'Failed to complete upload' });
  }
});

// Cancel upload and clean up
app.post('/api/uploads/chunked/cancel/:uploadId', async (req, res) => {
  try {
    const { uploadId } = req.params;
    
    if (!uploadId) {
      return res.status(400).json({ error: 'Upload ID is required' });
    }
    
    // Get upload session
    const session = uploadSessions.get(uploadId);
    if (!session) {
      return res.status(404).json({ error: 'Upload session not found' });
    }
    
    // For S3, abort multipart upload
    if (process.env.STORAGE_TYPE === 's3' && session.s3UploadId) {
      const command = new AbortMultipartUploadCommand({
        Bucket: bucketName,
        Key: session.finalFileName,
        UploadId: session.s3UploadId
      });
      
      await s3Client.send(command);
    }
    
    // Clean up temp directory
    const tempDir = path.join(__dirname, 'temp', uploadId);
    if (fs.existsSync(tempDir)) {
      fs.rmSync(tempDir, { recursive: true, force: true });
    }
    
    // Remove upload session from memory
    uploadSessions.delete(uploadId);
    
    res.status(200).json({ success: true });
  } catch (error) {
    console.error('Cancel upload error:', error);
    res.status(500).json({ error: 'Failed to cancel upload' });
  }
});

// Helper function to clean up abandoned uploads
function cleanupUpload(uploadId) {
  try {
    const session = uploadSessions.get(uploadId);
    if (!session) return;
    
    // For S3, abort multipart upload
    if (process.env.STORAGE_TYPE === 's3' && session.s3UploadId) {
      const command = new AbortMultipartUploadCommand({
        Bucket: bucketName,
        Key: session.finalFileName,
        UploadId: session.s3UploadId
      });
      
      s3Client.send(command).catch(error => {
        console.error('Failed to abort S3 multipart upload:', error);
      });
    }
    
    // Clean up temp directory
    const tempDir = path.join(__dirname, 'temp', uploadId);
    if (fs.existsSync(tempDir)) {
      fs.rmSync(tempDir, { recursive: true, force: true });
    }
    
    // Remove upload session from memory
    uploadSessions.delete(uploadId);
  } catch (error) {
    console.error('Cleanup upload error:', error);
  }
}

// Serve static files
app.use('/files', express.static('uploads'));

// Start server
app.listen(3000, () => {
  console.log('Server running on port 3000');
});
            

File Type Validation with Magic Numbers

MIME types sent by browsers can be spoofed. For more secure file validation, you can check file "magic numbers" - signatures in the file's binary data that identify its true type.

// Install required packages
// npm install file-type multer

const express = require('express');
const multer = require('multer');
const { fileTypeFromBuffer } = require('file-type');
const fs = require('fs');
const path = require('path');

const app = express();

// Configure multer for memory storage
const upload = multer({
  storage: multer.memoryStorage(),
  limits: {
    fileSize: 10 * 1024 * 1024 // 10MB
  }
});

// Validate file type using magic numbers
async function validateFileType(buffer, allowedTypes) {
  try {
    // Detect file type from buffer
    const fileType = await fileTypeFromBuffer(buffer);
    
    // If file type couldn't be detected
    if (!fileType) {
      return { valid: false, detectedType: 'unknown' };
    }
    
    // Check if detected type is in allowed types
    const isValid = allowedTypes.some(type => {
      // Match MIME type (e.g., 'image/jpeg')
      if (type === fileType.mime) {
        return true;
      }
      
      // Match category (e.g., 'image/*')
      if (type.endsWith('/*')) {
        const category = type.split('/')[0];
        return fileType.mime.startsWith(`${category}/`);
      }
      
      return false;
    });
    
    return {
      valid: isValid,
      detectedType: fileType.mime,
      extension: fileType.ext
    };
  } catch (error) {
    console.error('File type validation error:', error);
    return { valid: false, detectedType: 'error' };
  }
}

// Upload route with magic number validation
app.post('/api/upload/secure', upload.single('file'), async (req, res) => {
  try {
    if (!req.file) {
      return res.status(400).json({ error: 'No file uploaded' });
    }
    
    // Define allowed types based on upload context
    const allowedTypes = ['image/jpeg', 'image/png', 'image/gif', 'application/pdf'];
    
    // Validate file using magic numbers
    const validation = await validateFileType(req.file.buffer, allowedTypes);
    
    if (!validation.valid) {
      return res.status(400).json({
        error: 'Invalid file type',
        detectedType: validation.detectedType,
        allowedTypes
      });
    }
    
    // At this point, the file type is validated
    // Continue with upload processing...
    
    // For example, save the file with the correct extension
    const fileName = `${Date.now()}-${Math.random().toString(36).substring(2, 15)}.${validation.extension}`;
    const filePath = path.join(__dirname, 'uploads', fileName);
    
    // Ensure uploads directory exists
    if (!fs.existsSync(path.join(__dirname, 'uploads'))) {
      fs.mkdirSync(path.join(__dirname, 'uploads'), { recursive: true });
    }
    
    // Write the file
    fs.writeFileSync(filePath, req.file.buffer);
    
    res.status(200).json({
      success: true,
      file: {
        name: req.file.originalname,
        size: req.file.buffer.length,
        type: validation.detectedType,
        url: `/files/${fileName}`
      }
    });
  } catch (error) {
    console.error('Secure upload error:', error);
    res.status(500).json({ error: 'File upload failed' });
  }
});

// Serve uploaded files
app.use('/files', express.static('uploads'));

// Start server
app.listen(3000, () => {
  console.log('Server running on port 3000');
});