DEV Community

Cover image for Fetch any Website HTML without a cors error | Free API
Sh Raj
Sh Raj

Posted on • Updated on

Fetch any Website HTML without a cors error | Free API

#javascript

Fetch any Website HTML without a cors error | Free API

If you want to fetch some website's HTML content or want to scrap that using JavaScript on browser you may face this error of cors error kuch bhi likho abd
if yiu want to fetch some website html content or want to serap that using 

let i = await fetch('https://example.com')
Enter fullscreen mode Exit fullscreen mode

Image description

But if you want to avoid the cors error and scrap or fetch or get the HTML of the website. Just like view-source:--websiteURL-- . You can use the API Provided by WhollyAPI - https://sh20raj.gitbook.io/whollyapi/

i.e. :- https://whollyapi.vercel.app/api/html?url=

For Example :- https://whollyapi.vercel.app/api/html?url=https://example.com

Now, Let's Fetch

let i = await fetch('https://whollyapi.vercel.app/api/html?url=https://example.com')

// Directly Fetching in Text Format
 i = await (await fetch('https://whollyapi.vercel.app/api/html?url=https://example.com')).text()

/* Responce :- <!doctype html>
<html>
<head>
    <title>Example Domain</title>

    <meta charset="utf-8" />
    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <style type="text/css">
    body {
        background-color: #f0f0f2;
        margin: 0;
        padding: 0;
        font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;

    }
    div {
        width: 600px;
        margin: 5em auto;
        padding: 2em;
        background-color: #fdfdff;
        border-radius: 0.5em;
        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
    }
    a:link, a:visited {
        color: #38488f;
        text-decoration: none;
    }
    @media (max-width: 700px) {
        div {
            margin: 0 auto;
            width: auto;
        }
    }
    </style>    
</head>

<body>
<div>
    <h1>Example Domain</h1>
    <p>This domain is for use in illustrative examples in documents. You may use this
    domain in literature without prior coordination or asking for permission.</p>
    <p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
 */
Enter fullscreen mode Exit fullscreen mode

Get More API's Here :- https://sh20raj.gitbook.io/whollyapi/

Source Code :-

import { NextResponse } from "next/server";


let i = async (url) => {
  let data = await fetch(url || 'https://example.com');
  data = await data.text();
  return data;
}

function getParameterByName(name, url = window.location.href) {
  name = name.replace(/\[\\\[\\\]\]/g, '\\\\$&');
  var regex = new RegExp('\[?&\]' + name + '(=(\[^&#\]\*)|&|#|$)'),
    results = regex.exec(url);
  if (!results) return null;
  if (!results[2]) return '';
  return decodeURIComponent(results[2].replace(/\\+/g, ' '));
}

export async function GET(req, res) {

  const { url: rawUrl } = req;
  const url = getParameterByName("url", rawUrl);
  console.log(rawUrl);

  const result = await i(url);

  return new NextResponse(result, {
    status: 200,
    headers: {
      'Access-Control-Allow-Origin': '*',
      'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
      'Access-Control-Allow-Headers': 'Content-Type, Authorization',
    }
  });
}
Enter fullscreen mode Exit fullscreen mode

No-Cors Mode

fetch('https://example.com/', {
  mode: 'no-cors'
})
  .then(response => console.log(response))
  .catch(error => console.error(error));
Enter fullscreen mode Exit fullscreen mode

Comment for some Upgradation in the code....

Top comments (4)

Collapse
 
sh20raj profile image
Sh Raj •

Which is better

Collapse
 
sh20raj profile image
Sh Raj •

CORS Error Fixed Here

import { NextResponse } from "next/server";


let i = async (url) => {
  let data = await fetch(url || 'https://example.com');
  data = await data.text();
  return data;
}

function getParameterByName(name, url = window.location.href) {
  name = name.replace(/\[\\\[\\\]\]/g, '\\\\$&');
  var regex = new RegExp('\[?&\]' + name + '(=(\[^&#\]\*)|&|#|$)'),
    results = regex.exec(url);
  if (!results) return null;
  if (!results[2]) return '';
  return decodeURIComponent(results[2].replace(/\\+/g, ' '));
}

export async function GET(req, res) {

  const { url: rawUrl } = req;
  const url = getParameterByName("url", rawUrl);
  console.log(rawUrl);

  const result = await i(url);

  return new NextResponse(result, {
    status: 200,
    headers: {
      'Access-Control-Allow-Origin': '*',
      'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS',
      'Access-Control-Allow-Headers': 'Content-Type, Authorization',
    }
  });
}
Enter fullscreen mode Exit fullscreen mode
Collapse
 
sh20raj profile image
Sh Raj • 
import { NextResponse } from "next/server";
import Cors from 'cors';

const cors = Cors({
  methods: ['GET'],
  origin: '*'
});

let i = async (url) => {
  try {
    const response = await fetch(url || 'https://example.com');
    if (!response.ok) {
      throw new Error(`HTTP error! status: ${response.status}`);
    }
    const data = await response.text();
    return data;
  } catch (error) {
    console.error('Error:', error);
    throw error;
  }
}

function getParameterByName(name, url = window.location.href) {
  const urlObj = new URL(url);
  return urlObj.searchParams.get(name);
}

export async function GET(req, res) {
  try {
    await cors(req, res);

    const { url: rawUrl } = req;
    const url = getParameterByName("url", rawUrl);
    console.log(rawUrl);

    const result = await i(url);

    return new NextResponse(result, { status: 200 });
  } catch (error) {
    console.error('Error:', error);
    return new NextResponse('An error occurred while processing the request.', { status: 500 });
  }
}
Enter fullscreen mode Exit fullscreen mode
Collapse
 
sh20raj profile image
Sh Raj • 
import { NextResponse } from "next/server";
import Cors from 'cors';

const cors = Cors({
  methods: ['GET'],
});

let i = async (url) => {
  let data = await fetch(url || 'https://example.com');
  data = await data.text();
  return data;
}

function getParameterByName(name, url = window.location.href) {
  name = name.replace(/\[\\\[\\\]\]/g, '\\\\$&');
  var regex = new RegExp('\[?&\]' + name + '(=(\[^&#\]\*)|&|#|$)'),
    results = regex.exec(url);
  if (!results) return null;
  if (!results[2]) return '';
  return decodeURIComponent(results[2].replace(/\\+/g, ' '));
}

export async function GET(req, res) {
  await cors(req, res);

  const { url: rawUrl } = req;
  const urlSearchParams = new URLSearchParams(rawUrl.search);
  const url = getParameterByName("url", rawUrl);
  console.log(rawUrl);

  const result = await i(url);

  return new NextResponse(`${result}`);
}
Enter fullscreen mode Exit fullscreen mode

Read next

ashsajal profile image

JavaScript Event Loop Explained in 1 minute.

Ashfiquzzaman Sajal -

arshadayvid profile image

How to have AI onboard your users (Next.js, OpenAI, CopilotKit)

David Asaolu -

amankumarsingh01 profile image

React native and the JavaScript bridge.

A -

dillonheadley profile image

HTMZ inspired form subission

Dillon Headley -