Convert Any Website to CSV or JSON: A Practical Guide

#javascript #beginners #tutorial #webdev

Clients always want data in CSV or JSON. Here's how to scrape a website and output clean, structured files.

Step 1: Scrape the Data

const cheerio = require('cheerio');

async function scrapeTable(url) {
  const res = await fetch(url, {
    headers: { 'User-Agent': 'DataBot/1.0' }
  });
  const $ = cheerio.load(await res.text());

  const headers = $('table th').map((i, el) => $(el).text().trim()).get();
  const rows = [];
  $('table tr').each((i, row) => {
    if (i === 0) return; // skip header
    const cells = $(row).find('td').map((j, cell) => $(cell).text().trim()).get();
    if (cells.length) rows.push(cells);
  });

  return { headers, rows };
}

Step 2: Export as JSON

const fs = require('fs');

function toJSON(headers, rows) {
  const data = rows.map(row => {
    const obj = {};
    headers.forEach((h, i) => obj[h] = row[i] || '');
    return obj;
  });
  fs.writeFileSync('output.json', JSON.stringify(data, null, 2));
  console.log(`Saved ${data.length} records to output.json`);
}

Step 3: Export as CSV

function toCSV(headers, rows) {
  const escape = (s) => `"${(s || '').replace(/"/g, '""')}"`;
  const lines = [headers.map(escape).join(',')];
  rows.forEach(row => lines.push(row.map(escape).join(',')));
  fs.writeFileSync('output.csv', lines.join('\n'));
  console.log(`Saved ${rows.length} records to output.csv`);
}

Step 4: Export as Excel

const XLSX = require('xlsx');

function toExcel(headers, rows) {
  const data = [headers, ...rows];
  const ws = XLSX.utils.aoa_to_sheet(data);
  const wb = XLSX.utils.book_new();
  XLSX.utils.book_append_sheet(wb, ws, 'Data');
  XLSX.writeFile(wb, 'output.xlsx');
}

The Complete Script

const { headers, rows } = await scrapeTable(url);
toJSON(headers, rows);
toCSV(headers, rows);
toExcel(headers, rows);
// Done! Three formats in one script.