Fetching a large JSON, streaming it to a worker for filtering, and rendering chunks incrementally.

#snippets #ai
I am inauguring a new serie called "DevGuides" with the help of AI
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <title>Streaming API Workflow Demo with Progress</title>
  <style>
    body { font-family: sans-serif; padding: 20px; max-width: 900px; margin: auto; }
    h1 { color: #333; }
    h2 { color: #4CAF50; margin-top: 30px; }
    .step { margin: 20px 0; padding: 15px; border-left: 4px solid #4CAF50; background: #f9f9f9; }
    .diagram { font-family: monospace; white-space: pre; background: #eee; padding: 10px; margin-top: 10px; }
    #progress-container { width: 100%; background: #eee; border: 1px solid #ccc; border-radius: 5px; height: 20px; margin: 10px 0; }
    #download-progress, #filter-progress { width: 0%; height: 100%; border-radius: 5px; transition: width 0.2s; }
    #download-progress { background: #2196F3; }
    #filter-progress { background: #4CAF50; margin-top: 5px; }
    #product-list { max-height: 400px; overflow-y: auto; border: 1px solid #ccc; padding: 10px; background: #fafafa; }
    .product { margin: 2px 0; }
  </style>
</head>
<body>
  <h1>Streaming API Workflow Demo with Progress</h1>

  <div class="step">
    <h2>Step 1: Fetch Large JSON Data</h2>
    <p>The main thread fetches JSON from a public API as a stream, updating the download progress as chunks arrive.</p>
  </div>

  <div class="step">
    <h2>Step 2: Send Data to Worker</h2>
    <p>Once chunks are received, they are combined into an array and sent to a Web Worker for filtering to avoid blocking the UI.</p>
  </div>

  <div class="step">
    <h2>Step 3: Worker Filters Data</h2>
    <p>The worker filters items according to criteria and sends filtered chunks back to the main thread.</p>
  </div>

  <div class="step">
    <h2>Step 4: Incremental Rendering</h2>
    <p>The main thread renders filtered chunks immediately while updating the filter progress bar.</p>
  </div>

  <div class="step">
    <h2>Workflow Diagram</h2>
    <div class="diagram">
Main Thread                 Worker
------------                ------------
1. fetch()  --------------&gt;
                              2. receive products array
                              3. filter in chunks
                              4. postMessage(filtered chunk)
5. receive filtered chunk &lt;-------------
6. render chunk in UI
(repeat for each chunk)
    </div>
  </div>

  <h2>Live Demo</h2>
  <p>Blue bar: download progress | Green bar: filtering progress</p>
  <div id="progress-container">
    <div id="download-progress"></div>
  </div>
  <div id="progress-container">
    <div id="filter-progress"></div>
  </div>
  <div id="product-list"></div>

  <script>
    // --- Worker code ---
    const workerBlob = new Blob([`
      self.onmessage = function(e) {
        const { type, products, minPrice, minRating, chunkSize } = e.data;
        if (type === 'PROCESS_PRODUCTS') {
          try {
            let filteredChunk = [];
            for (let i = 0; i < products.length; i++) {
              const p = products[i];
              if (p.price > minPrice && p.rating > minRating) {
                filteredChunk.push(p);
              }
              if (filteredChunk.length === chunkSize) {
                self.postMessage({ type: 'CHUNK', data: filteredChunk, progress: i + 1, total: products.length });
                filteredChunk = [];
              }
            }
            if (filteredChunk.length) {
              self.postMessage({ type: 'CHUNK', data: filteredChunk, progress: products.length, total: products.length });
            }
          } catch (err) {
            self.postMessage({ type: 'ERROR', message: err.message });
          }
        }
      };
    `], { type: 'application/javascript' });

    const worker = new Worker(URL.createObjectURL(workerBlob));

    const container = document.getElementById('product-list');
    const downloadBar = document.getElementById('download-progress');
    const filterBar = document.getElementById('filter-progress');

    worker.onmessage = e => {
      const msg = e.data;
      if (msg.type === 'CHUNK') {
        appendToUI(msg.data);
        const percent = Math.round((msg.progress / msg.total) * 100);
        filterBar.style.width = percent + '%';
      }
      if (msg.type === 'ERROR') console.error('Worker error:', msg.message);
    };

    function appendToUI(products) {
      for (const p of products) {
        const div = document.createElement('div');
        div.className = 'product';
        div.textContent = p.title + ' - $' + p.price + ' - rating ' + p.rating;
        container.appendChild(div);
      }
    }

    async function fetchAndProcess(url, minPrice = 500, minRating = 4.5, chunkSize = 20) {
      const res = await fetch(url);
      const reader = res.body.getReader();
      const decoder = new TextDecoder();
      let buffer = '';
      let totalBytes = res.headers.get('Content-Length') ? parseInt(res.headers.get('Content-Length')) : 0;
      let loadedBytes = 0;
      let products = [];

      // Read stream incrementally
      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        loadedBytes += value.length;
        downloadBar.style.width = totalBytes ? Math.round((loadedBytes / totalBytes) * 100) + '%' : '50%';

        buffer += decoder.decode(value, { stream: true });
      }

      // Parse JSON once fully loaded
      try {
        const data = JSON.parse(buffer);
        products = data.products || [];
      } catch (err) {
        console.error('JSON parse error:', err);
        return;
      }

      downloadBar.style.width = '100%';

      // Send to worker for filtering
      worker.postMessage({ type: 'PROCESS_PRODUCTS', products, minPrice, minRating, chunkSize }); 
    }

    // --- Start demo ---
    fetchAndProcess('https://dummyjson.com/products?limit=1000');   
  </script>
</body>
</html>
Demo
DEV Community

Fetching a large JSON, streaming it to a worker for filtering, and rendering chunks incrementally.

Top comments (0)
