index.js

import fs from "fs";
import path from "path";
import dotenv from "dotenv";
import { program } from "commander";
import ora from "ora";
import chalk from "chalk";
import { list } from "@vercel/blob";

dotenv.config();

const VERCEL_PROJECT_ID = process.env.VERCEL_PROJECT_ID;
const VERCEL_ACCESS_TOKEN = process.env.VERCEL_ACCESS_TOKEN;
const VERCEL_STORE_ID = process.env.VERCEL_STORE_ID;
const MAX_RETRIES = 3;
const CONCURRENT_DOWNLOADS = 5;

program
  .option("-d, --dir <path>", "Download directory", "./downloads")
  .option("-f, --force", "Force download existing files", false)
  .option(
    "-c, --concurrency <number>",
    "Number of concurrent downloads",
    CONCURRENT_DOWNLOADS
  )
  .option("-s, --store <id>", "Vercel Blob Store ID")
  .parse(process.argv);

const options = program.opts();
const DOWNLOAD_DIR = path.resolve(process.cwd(), options.dir);

async function listFiles() {
  const spinner = ora("Fetching file list...").start();
  try {
    let cursor;
    const allBlobs = [];

    do {
      const response = await list({
        cursor,
        limit: 1000,
      });

      allBlobs.push(...response.blobs);
      cursor = response.cursor;
    } while (cursor);

    spinner.succeed(`Found ${allBlobs.length} files`);
    return allBlobs;
  } catch (error) {
    spinner.fail("Failed to fetch file list");
    throw error;
  }
}

async function downloadFile(blob) {
  const filePath = path.join(DOWNLOAD_DIR, blob.pathname);

  if (!options.force && fs.existsSync(filePath)) {
    return { skipped: true, path: blob.pathname };
  }

  // Get the download URL from the blob object
  const downloadUrl = blob.downloadUrl || blob.url;
  const response = await fetch(downloadUrl);

  if (!response.ok) {
    throw new Error(
      `Failed to download: ${response.statusText} (${response.status})`
    );
  }

  // Ensure the directory exists
  fs.mkdirSync(path.dirname(filePath), { recursive: true });

  // Download and write the file
  const buffer = await response.arrayBuffer();
  fs.writeFileSync(filePath, Buffer.from(buffer));

  return { skipped: false, path: blob.pathname };
}

async function downloadAllFiles() {
  try {
    if (!VERCEL_PROJECT_ID || !VERCEL_ACCESS_TOKEN) {
      throw new Error(
        "Missing required environment variables: VERCEL_PROJECT_ID or VERCEL_ACCESS_TOKEN"
      );
    }

    const blobs = await listFiles();
    if (blobs.length === 0) {
      console.log(chalk.yellow("No files found to download."));
      return;
    }

    console.log(chalk.blue("\nStarting downloads..."));
    const results = { successful: 0, skipped: 0, failed: 0 };
    const errors = [];

    // Process downloads in chunks
    for (let i = 0; i < blobs.length; i += options.concurrency) {
      const chunk = blobs.slice(i, i + options.concurrency);
      const downloads = chunk.map(async (blob) => {
        try {
          const result = await downloadFile(blob);
          if (result.skipped) {
            console.log(
              chalk.yellow(`Skipped: ${result.path} (already exists)`)
            );
            results.skipped++;
          } else {
            console.log(chalk.green(`Downloaded: ${result.path}`));
            results.successful++;
          }
        } catch (error) {
          console.log(chalk.red(`Failed: ${blob.pathname}`));
          errors.push({ path: blob.pathname, error: error.message });
          results.failed++;
        }
      });

      await Promise.all(downloads);
    }

    // Print summary
    console.log("\nDownload Summary:");
    console.log(
      chalk.green(`✓ Successfully downloaded: ${results.successful}`)
    );
    console.log(chalk.yellow(`⚠ Skipped: ${results.skipped}`));
    console.log(chalk.red(`✗ Failed: ${results.failed}`));

    if (errors.length > 0) {
      console.log("\nErrors:");
      errors.forEach(({ path, error }) => {
        console.log(chalk.red(`${path}: ${error}`));
      });
    }
  } catch (error) {
    console.error(chalk.red(`\nError: ${error.message}`));
    process.exit(1);
  }
}

downloadAllFiles();