Download all your AWS S3 bucket folders in single script

#aws #programming #javascript #webdev

Today when i need to downlaod all s3 bucket folders. but in bucket the button is getting disabled. so i have created a script that download all the folders
like images, videos, fonts etc.

I hope it can help you in some projects.

const { S3Client, ListObjectsV2Command, GetObjectCommand } = require('@aws-sdk/client-s3');
const fs = require('fs');
const path = require('path');
const { pipeline } = require('stream/promises');
require('dotenv').config();

const region = process.env.AWS_REGION || 'us-east-1';
const accessKeyId = process.env.AWS_ACCESS_KEY_ID || '';
const secretAccessKey = process.env.AWS_SECRET_ACCESS_KEY || '';
const bucketName = process.env.AWS_S3_BUCKET || '';

if (!bucketName) {
  console.error('Error: AWS_S3_BUCKET is not defined in your environment variables.');
  process.exit(1);
}

const s3Client = new S3Client({
  region,
  credentials: {
    accessKeyId,
    secretAccessKey,
  },
});

async function downloadAll() {
  const publicDir = path.join(__dirname, 'public');
  if (!fs.existsSync(publicDir)) {
    fs.mkdirSync(publicDir, { recursive: true });
  }

  let isTruncated = true;
  let continuationToken = undefined;

  console.log(`Starting download from S3 Bucket: ${bucketName}...`);

  while (isTruncated) {
    const listParams = {
      Bucket: bucketName,
      ContinuationToken: continuationToken,
    };

    const listResponse = await s3Client.send(new ListObjectsV2Command(listParams));

    if (listResponse.Contents && listResponse.Contents.length > 0) {
      for (const object of listResponse.Contents) {
        const key = object.Key;
        // Skip directory placeholder objects (objects with size 0 that end in a slash)
        if (key.endsWith('/')) {
          continue;
        }

        const localPath = path.join(publicDir, key);
        console.log(`Downloading ${key} -> ${localPath}...`);

        try {
          const getParams = {
            Bucket: bucketName,
            Key: key,
          };
          const getResponse = await s3Client.send(new GetObjectCommand(getParams));

          // Ensure parent directory exists for the file
          const dir = path.dirname(localPath);
          if (!fs.existsSync(dir)) {
            fs.mkdirSync(dir, { recursive: true });
          }

          await pipeline(getResponse.Body, fs.createWriteStream(localPath));
          console.log(`Successfully downloaded ${key}`);
        } catch (error) {
          console.error(`Failed to download ${key}:`, error);
        }
      }
    }

    isTruncated = listResponse.IsTruncated;
    continuationToken = listResponse.NextContinuationToken;
  }

  console.log('Download complete.');
}

downloadAll().catch((err) => {
  console.error('Fatal error during download:', err);
  process.exit(1);
});