DEV Community: Mikhail Zub

7500 hotels price research: scraping using JavaScript module hotels-scraper-js and data analysis

Mikhail Zub — Sat, 06 May 2023 17:47:58 +0000

About

This demo project is a practical showcase of using hotels-scraper-js NPM package plus how extracted data could be used in exploratory data analysis.

hotels-scraper-js is an open source tool designed to parse popular website hotels from a single module. It is used to scrape hotels list and hotel info from Airbnb, Booking, and Hotels.com websites.

You can download the dataset at Kaggle: 7500 hotels from Airbnb, Booking and Hotels.com.

I'll cover:

Getting 500 hotel data (for each website) from each city from the list (I take 5 European capitals for this example).
Getting hotels with the lowest, median, and highest prices.
Saving parsed data in .json files.
Saving charts images with analyzed data.

Full code (if you don't need an explanation)

import fs from "fs/promises";
import { ChartJSNodeCanvas } from "chartjs-node-canvas";
import { airbnb, booking, hotelsCom, saveToJSON } from "hotels-scraper-js";

const locations = ["London", "Berlin", "Paris", "Madrid", "Rome"];

const getHotels = async (location) => {
  const airbnbHotels = await airbnb.getHotels(undefined, "USD", 500, location);
  const bookingHotels = await booking.getHotels(undefined, "USD", 500, location);
  const hotelsComHotels = await hotelsCom.getHotels(undefined, undefined, undefined, "United States", undefined, 500, location);

  return { airbnbHotels, bookingHotels, hotelsComHotels };
};

const getResults = async () => {
  const results = {};
  for (const location of locations) {
    // There are two options (only one can be uncommented):
    // 1. If you need to get and save parsed data into .json files:

    const hotels = await getHotels(location);
    saveToJSON(hotels, `data/${location}`);

    // 2. If you saved files earlier and now you need to read data from these files:

    //   const hotels = await fs.readFile(`data/${location}.json`, "utf-8");
    //   results[`${location}`] = JSON.parse(hotels);

    results[`${location}`] = hotels;
  }
  return results;
};

const saveChart = async (chartData, labels, chartName, title) => {
  const width = 500;
  const height = 500;
  const backgroundColour = "white";
  const chartJSNodeCanvas = new ChartJSNodeCanvas({ width, height, backgroundColour });

  const configuration = {
    type: "bar",
    data: {
      labels,
      datasets: [
        {
          data: chartData,
          backgroundColor: [
            "rgba(255, 99, 132, 0.2)",
            "rgba(255, 159, 64, 0.2)",
            "rgba(75, 192, 192, 0.2)",
            "rgba(54, 162, 235, 0.2)",
            "rgba(153, 102, 255, 0.2)",
          ],
          borderColor: ["rgb(255, 99, 132)", "rgb(255, 159, 64)", "rgb(75, 192, 192)", "rgb(54, 162, 235)", "rgb(153, 102, 255)"],
          borderWidth: 1,
        },
      ],
    },
    options: {
      scales: {
        y: {
          title: {
            display: true,
            text: "USD",
          },
        },
      },
      plugins: {
        title: {
          display: true,
          text: title,
        },
        legend: {
          display: false,
        },
      },
    },
  };

  const base64Image = await chartJSNodeCanvas.renderToDataURL(configuration);

  const base64Data = base64Image.replace(/^data:image\/png;base64,/, "");

  fs.writeFile(`${chartName}.png`, base64Data, "base64")
    .then(() => console.log("Chart saved!"))
    .catch(console.log);
};

getResults().then(async (results) => {
  const airbnbLowest = [];
  const bookingLowest = [];
  const hotelsComLowest = [];
  const airbnbHighest = [];
  const bookingHighest = [];
  const hotelsComHighest = [];
  const airbnbMedian = [];
  const bookingMedian = [];
  const hotelsComMedian = [];
  for (const location of locations) {
    const airbnbPrices = results[location].airbnbHotels.map((el) => el.price.value).sort((a, b) => a - b);
    const bookingPrices = results[location].bookingHotels.map((el) => el.price.value).sort((a, b) => a - b);
    const hotelsComPrices = results[location].hotelsComHotels.map((el) => el.price.value).sort((a, b) => a - b);
    airbnbLowest.push(airbnbPrices[0]);
    bookingLowest.push(bookingPrices[0]);
    hotelsComLowest.push(hotelsComPrices[0]);
    airbnbHighest.push(airbnbPrices[airbnbPrices.length - 1]);
    bookingHighest.push(bookingPrices[bookingPrices.length - 1]);
    hotelsComHighest.push(hotelsComPrices[hotelsComPrices.length - 1]);
    airbnbMedian.push(
      airbnbPrices.length % 2 === 0
        ? (airbnbPrices[airbnbPrices.length / 2 - 1] + airbnbPrices[airbnbPrices.length / 2]) / 2
        : Math.ceil(airbnbPrices[airbnbPrices.length / 2])
    );
    bookingMedian.push(
      bookingPrices.length % 2 === 0
        ? (bookingPrices[bookingPrices.length / 2 - 1] + bookingPrices[bookingPrices.length / 2]) / 2
        : Math.ceil(bookingPrices[bookingPrices.length / 2])
    );
    hotelsComMedian.push(
      hotelsComPrices.length % 2 === 0
        ? (hotelsComPrices[hotelsComPrices.length / 2 - 1] + hotelsComPrices[hotelsComPrices.length / 2]) / 2
        : Math.ceil(hotelsComPrices[hotelsComPrices.length / 2])
    );
  }
  await saveChart(airbnbLowest, locations, `charts/airbnbLowest`, "Airbnb Lowest Prices");
  await saveChart(bookingLowest, locations, `charts/bookingLowest`, "Booking Lowest Prices");
  await saveChart(hotelsComLowest, locations, `charts/hotelsComLowest`, "Hotels.com Lowest Prices");
  await saveChart(airbnbHighest, locations, `charts/airbnbHighest`, "Airbnb Highest Prices");
  await saveChart(bookingHighest, locations, `charts/bookingHighest`, "Booking Highest Prices");
  await saveChart(hotelsComHighest, locations, `charts/hotelsComHighest`, "Hotels.com Highest Prices");
  await saveChart(airbnbMedian, locations, `charts/airbnbMedian`, "Airbnb Median Prices");
  await saveChart(bookingMedian, locations, `charts/bookingMedian`, "Booking Median Prices");
  await saveChart(hotelsComMedian, locations, `charts/hotelsComMedian`, "Hotels.com Median Prices");
});

Code explanation

Preparation

First, we need to create a Node.js* project and add npm packages hotels-scraper-js to parse hotel lists, chart.js to build chart from received data and chartjs-node-canvas to render chart with Chart.js using canvas.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i hotels-scraper-js chart.js chartjs-node-canvas

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

Process

📌Note: Only ES modules import statement is available.

First, we need to import the required modules and define the locations variable with the cities in which need to search hotels:

import fs from "fs/promises";
import { ChartJSNodeCanvas } from "chartjs-node-canvas";
import { airbnb, booking, hotelsCom, saveToJSON } from "hotels-scraper-js";

const locations = ["London", "Berlin", "Paris", "Madrid", "Rome"];

Then we write getHotels function that takes location and returns hotel lists from each website:

const getHotels = async (location) => {
  const airbnbHotels = await airbnb.getHotels(undefined, "USD", 500, location);
  const bookingHotels = await booking.getHotels(undefined, "USD", 500, location);
  const hotelsComHotels = await hotelsCom.getHotels(undefined, undefined, undefined, "United States", undefined, 500, location);

  return { airbnbHotels, bookingHotels, hotelsComHotels };
};

In this function, we define currency ("USD") and set the number of results (500) for each parser.

Next, we write getResults function that handles what we want: get results from the parser, save and return them or read saved earlier data from files and return it:

const getResults = async () => {
  const results = {};
  for (const location of locations) {
    // There are two options (only one can be uncommented):
    // 1. If you need to get and save parsed data into .json files:

    const hotels = await getHotels(location);
    saveToJSON(hotels, `data/${location}`);
    results[`${location}`] = hotels;

    // 2. If you saved files earlier and now you need to read data from these files:

    //   const hotels = await fs.readFile(`data/${location}.json`, "utf-8");
    //   results[`${location}`] = JSON.parse(hotels);
  }
  return results;
};

Next, we write saveChart function that allows build and save chart images:

const saveChart = async (chartData, labels, chartName, title) => {
  const width = 500;
  const height = 500;
  const backgroundColour = "white";
  const chartJSNodeCanvas = new ChartJSNodeCanvas({ width, height, backgroundColour });

  const configuration = {
    type: "bar",
    data: {
      labels,
      datasets: [
        {
          data: chartData,
          backgroundColor: [
            "rgba(255, 99, 132, 0.2)",
            "rgba(255, 159, 64, 0.2)",
            "rgba(75, 192, 192, 0.2)",
            "rgba(54, 162, 235, 0.2)",
            "rgba(153, 102, 255, 0.2)",
          ],
          borderColor: ["rgb(255, 99, 132)", "rgb(255, 159, 64)", "rgb(75, 192, 192)", "rgb(54, 162, 235)", "rgb(153, 102, 255)"],
          borderWidth: 1,
        },
      ],
    },
    options: {
      scales: {
        y: {
          title: {
            display: true,
            text: "USD",
          },
        },
      },
      plugins: {
        title: {
          display: true,
          text: title,
        },
        legend: {
          display: false,
        },
      },
    },
  };

  const base64Image = await chartJSNodeCanvas.renderToDataURL(configuration);

  const base64Data = base64Image.replace(/^data:image\/png;base64,/, "");

  fs.writeFile(`${chartName}.png`, base64Data, "base64")
    .then(() => console.log("Chart saved!"))
    .catch(console.log);
};

And finally, we call getResults function, wait for results, make some data operations and save charts:

getResults().then(async (results) => {
    ...
});

In this function first, we declare analyzed results arrays:

const airbnbLowest = [];
const bookingLowest = [];
const hotelsComLowest = [];
const airbnbHighest = [];
const bookingHighest = [];
const hotelsComHighest = [];
const airbnbMedian = [];
const bookingMedian = [];
const hotelsComMedian = [];

Then we need to fill each array with the necessary data. We need to sort data in ascending order, for lowest price we get the first elements, for highest - last. And for medians we get one (for odd length arrays) or two (for even length arrays) values from the middle of the arrays:

for (const location of locations) {
  const airbnbPrices = results[location].airbnbHotels.map((el) => el.price.value).sort((a, b) => a - b);
  const bookingPrices = results[location].bookingHotels.map((el) => el.price.value).sort((a, b) => a - b);
  const hotelsComPrices = results[location].hotelsComHotels.map((el) => el.price.value).sort((a, b) => a - b);
  airbnbLowest.push(airbnbPrices[0]);
  bookingLowest.push(bookingPrices[0]);
  hotelsComLowest.push(hotelsComPrices[0]);
  airbnbHighest.push(airbnbPrices[airbnbPrices.length - 1]);
  bookingHighest.push(bookingPrices[bookingPrices.length - 1]);
  hotelsComHighest.push(hotelsComPrices[hotelsComPrices.length - 1]);
  airbnbMedian.push(
    airbnbPrices.length % 2 === 0
      ? (airbnbPrices[airbnbPrices.length / 2 - 1] + airbnbPrices[airbnbPrices.length / 2]) / 2
      : Math.ceil(airbnbPrices[airbnbPrices.length / 2])
  );
  bookingMedian.push(
    bookingPrices.length % 2 === 0
      ? (bookingPrices[bookingPrices.length / 2 - 1] + bookingPrices[bookingPrices.length / 2]) / 2
      : Math.ceil(bookingPrices[bookingPrices.length / 2])
  );
  hotelsComMedian.push(
    hotelsComPrices.length % 2 === 0
      ? (hotelsComPrices[hotelsComPrices.length / 2 - 1] + hotelsComPrices[hotelsComPrices.length / 2]) / 2
      : Math.ceil(hotelsComPrices[hotelsComPrices.length / 2])
  );
}

After all data operations, we save charts images:

await saveChart(airbnbLowest, locations, `charts/airbnbLowest`, "Airbnb Lowest Prices");
await saveChart(bookingLowest, locations, `charts/bookingLowest`, "Booking Lowest Prices");
await saveChart(hotelsComLowest, locations, `charts/hotelsComLowest`, "Hotels.com Lowest Prices");
await saveChart(airbnbHighest, locations, `charts/airbnbHighest`, "Airbnb Highest Prices");
await saveChart(bookingHighest, locations, `charts/bookingHighest`, "Booking Highest Prices");
await saveChart(hotelsComHighest, locations, `charts/hotelsComHighest`, "Hotels.com Highest Prices");
await saveChart(airbnbMedian, locations, `charts/airbnbMedian`, "Airbnb Median Prices");
await saveChart(bookingMedian, locations, `charts/bookingMedian`, "Booking Median Prices");
await saveChart(hotelsComMedian, locations, `charts/hotelsComMedian`, "Hotels.com Median Prices");

Results

Lowest price:

Airbnb

<img src="https://user-images.githubusercontent.com/64033139/233854764-6dcbe6a7-c3f8-45b0-a3d0-4b43f266835a.png" alt="hotels-scraper-js">

Booking

<img src="https://user-images.githubusercontent.com/64033139/233854768-a0ff5d80-8396-424b-983b-fcf955b2eaf9.png" alt="hotels-scraper-js">

Hotels.com

<img src="https://user-images.githubusercontent.com/64033139/233854771-5307e51e-6b16-4bfb-85c5-da45429939b2.png" alt="hotels-scraper-js">

Highest price:

Airbnb

<img src="https://user-images.githubusercontent.com/64033139/233854759-3c8b30e4-b518-4e9e-8987-1481b3d8db08.png" alt="hotels-scraper-js">

Booking

<img src="https://user-images.githubusercontent.com/64033139/233854767-37d40733-80b9-4fe8-86f0-ce8d40d02772.png" alt="hotels-scraper-js">

Hotels.com

<img src="https://user-images.githubusercontent.com/64033139/233854770-3fa3e562-509d-44c3-a720-eee2abdc7cc7.png" alt="hotels-scraper-js">

Median price:

Airbnb

<img src="https://user-images.githubusercontent.com/64033139/233854765-aa1a8696-7682-4e80-bf53-9ff6951655df.png" alt="hotels-scraper-js">

Booking

<img src="https://user-images.githubusercontent.com/64033139/233854769-d0b29687-946e-4a6d-9a8c-1a45eefe3e8c.png" alt="hotels-scraper-js">

Hotels.com

<img src="https://user-images.githubusercontent.com/64033139/233854772-309a0fad-ac77-46f3-a542-0e83ef1d5ea3.png" alt="hotels-scraper-js">

Links

If you want other functionality added to this demo project or if you want to see some other projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

JavaScript Hotels Parser

Mikhail Zub — Sat, 15 Apr 2023 08:10:39 +0000

About

hotels-scraper-js is a Node.js package designed to parse popular website hotels from a single module.

The package is available on npm and can be installed and used in your Node.js project. It is used to scrape hotels list and hotel info from Airbnb, Booking, and Hotels.com websites. Soon we planning to add Google Hotels or other websites.

The long-term plan of this project is to combine API and non-API solutions as SerpApi started developing support for Google Hotels so later users can choose the desired backend to work with (free with limitations or paid without limitations). SerpApi's backend could be also useful for people who don't know how to use it, for example, SerpApi pagination which could be enabled with a simple function argument pagination=True.

Development is sponsored by SerpApi.

Limitations and problems

The main problem is that sites that parse often change their structure or selectors, so from time to time one of the scraping fields can't be gotten (sometimes the parser can crash).
Hotels.com sometimes does not load the place input field (which happens in the usual browser too) and the place autocomplete doesn't work, so the parser can crash.
If your IP address is blocked, or there is no access from your country, and if one of the sites from this parser doesn't work in your browser you can't get results from this parser.

What you can get with `hotels-scraper-js`?

Airbnb:

List of recommended hotels by categories

List of hotels by selected location and dates

Selected hotel info

Booking:

List of hotels by selected location and dates

Selected hotel info

Hotels.com:

List of hotels by selected location and dates

Selected hotel info

Installation & using

You can get the installation guide and all available options from the documentation

Basic Examples

In these basic examples, we'll get links to each hotel from the 3 parsers. And then those links will be used to get hotel info.

Airbnb hotels result from the selected category and with selected currency.

First, we need to get all available categories and currencies:

console.log(airbnb.getFilters());

Output:

{
   "currencies":[
      {
         "name":"Australian dollar",
         "value":"AUD"
      },
      {
         "name":"Brazilian real",
         "value":"BRL"
      },
    ... and other currencies
   ],
   "categories":[
      {
         "name":"A-frames",
         "value":"8148"
      },
      {
         "name":"Design",
         "value":"8528"
      },
     ... and other categories
   ]
}

Next, using got before category and currency get hotels list with 50 results:

📌Note: you can use both name or value.

airbnb.getHotels("Design", "BRL", 50).then(console.log);

Output:

[
   {
      "thumbnail":"https://a0.muscache.com/im/pictures/74102172/9815f41f_original.jpg?im_w=720",
      "title":"Bolzano, Italy",
      "subtitles":[
         "Designed by Peter Pichler",
         "Apr 15 – 20"
      ],
      "price":{
         "currency":"R$",
         "value":1417,
         "period":"night"
      },
      "rating":4.8,
      "link":"https://www.airbnb.com/rooms/4594196"
   },
   {
      "thumbnail":"https://a0.muscache.com/im/pictures/397ed026-16a9-4418-af19-1d2b31fe0aa8.jpg?im_w=720",
      "title":"Korčula, Croatia",
      "subtitles":[
         "Luxury Lifestyle Awards: Best Luxury Villa Croatia",
         "Apr 1 – 6"
      ],
      "price":{
         "currency":"R$",
         "value":11290,
         "period":"night"
      },
      "rating":"No rating",
      "link":"https://www.airbnb.com/luxury/listing/34594127"
   },
   ... and other hotels
]

And then using one of the hotel links get hotel info:

airbnb.getHotelInfo("https://www.airbnb.com/rooms/4594196").then((data) => {
  console.dir(data, { depth: null });
});

Output:

{
   "name":"Mirror House Sud",
   "shortDescription":"Entire home hosted by Sabina Angela",
   "shortOverview":[
      "4 guests",
      "1 bedroom",
      "1 bed",
      "1 bath"
   ],
   "highlights":[
      {
         "title":"Designed by",
         "subtitle":"Peter Pichler"
      },
      {
         "title":"Featured in",
         "subtitle":"Designboom, December 2014ArchDaily, December 2014"
      },
      {
         "title":"Sabina Angela is a Superhost",
         "subtitle":"Superhosts are experienced, highly rated hosts who are committed to providing great stays for guests."
      }
   ],
   "price":{
      "currency":"₴",
      "amount":8828,
      "period":"night"
   },
   "description":"Mirror Houses are two small houses immersed in a beautiful scenery of apple orchards just outside, in the wonderful surroundings of the South Tyrolean Dolomites.The Mirror Houses offer a unique opportunity to spend a wonderful holiday surrounded by contemporary architecture of the highest standards in close contact with one of the most evocative landscapes that nature can offer.The spaceThe Mirror House Sud (45sqm) consists of 1 bedroom with double bed, 1 bathroom with spacious shower and 1 living room / kitchen (single space) with a double sofa bed. The location is ideal for 2 people, but can accommodate up to a maximum of 4 people.If you are 4 adults we recommend booking both Mirror House North and South.In addition there is a spacious terrace that + a private garden and
in the summer months a shared garden with a common pool and a self-service pool bar.Under the houses there is also a common storage room for sports equipment or other.",
   "sleepOptions":[

   ],
   "host":{
      "name":"Sabina Angela",
      "joined":"Joined in March 2013",
      "overview":[
         "127 Reviews",
         "Identity verified",
         "Superhost"
      ],
      "additionalInfo":[
         "Response rate: 87%",
         "Response time: within an hour"
      ]
   },
   "link":"https://www.airbnb.com/rooms/4594196",
   "placeOffers":[
      "Hair dryer",
      "Shampoo",
      "Hot water",
   ... and other place offers
   ],
   "houseRules":[
      "4 guests maximum",
      "No pets",
      "Check-in: 2:00 PM - 9:00 PM",
      "Checkout before 11:00 AM",
      "No commercial photography",
      "No parties or events",
      "No smoking"
   ],
   "safetyAndProperty":[
      "Carbon monoxide alarm not reported",
      "Smoke alarm not reported",
      "Some spaces are shared"
   ],
   "photos":[
      "https://a0.muscache.com/im/pictures/74102421/46d207f2_original.jpg?aki_policy=large",
      "https://a0.muscache.com/im/pictures/74102172/9815f41f_original.jpg?aki_policy=large",
      "https://a0.muscache.com/im/pictures/309bee53-311d-4f07-a2e7-14daadbbfb77.jpg?aki_policy=large",
   ... and other photos
   ],
   "reviewsInfo":{
      "totalReviews":53,
      "rating":{
         "total":4.81,
         "cleanliness":4.9,
         "accuracy":4.9,
         "communication":4.8,
         "location":4.7,
         "checkIn":4.8,
         "value":4.6
      },
      "reviews":[
         {
            "name":"Nadav",
            "avatar":"https://a0.muscache.com/im/pictures/user/cbecae28-d8cc-46b9-9294-c1b0fcb512e9.jpg?im_w=240",
            "userPage":"https://www.airbnb.com/users/show/10949335",
            "date":"March 2023",
            "review":"Had an amazing time at the Mirror House. Not only was the place very beautifully designed, but the location also made it very special. A must-stay for a romantic vacation in Bolzano.
In addition to this, Sabine was probably the best host I've had while using AirBnb. She was very helpful in helping us get reservations to local restaurants and actually made us a cake (this was an engagement trip, so she offered to make us one instead of actually getting one at a bakery). Overall, we had an amazing vacation. Sabine and the Mirror House definitely made it very special for us!"
         },
         {
            "name":"Khaled",
            "avatar":"https://a0.muscache.com/im/pictures/user/f7804757-f867-46d5-8557-33d9be786657.jpg?im_w=240",
            "userPage":"https://www.airbnb.com/users/show/179920339",
            "date":"March 2023",
            "review":"Beautiful place and private surroundingIt was a good experienceHighly recommended"
         },
      ... and other reviews
      ]
   }
}

Booking hotels result with selected filters, currency, location, dates, amount of guests, and set for business purposes.

First, we need to get all available categories and currencies:

📌Note: you need to use console.dir method to see the depth of inner keys of more than two in the result object.

console.dir(booking.getFilters(), { depth: null });

Output:

{
   "currencies":[
      {
         "name":"Property's Currency",
         "value":"hotel_currency"
      },
      {
         "name":"Argentine Peso",
         "value":"ARS"
      },
    ... and other currencies
   ],
   "filters":{
      "budget":[
         {
            "name":"US$0 – US$50",
            "value":"pri=1"
         },
         {
            "name":"US$50 – US$100",
            "value":"pri=2"
         },
        ... and other budget options
      ],
      "propertyRating":[
         {
            "name":"1 star",
            "value":"class=1"
         },
         {
            "name":"2 stars",
            "value":"class=2"
         },
         {
            "name":"3 stars",
            "value":"class=3"
         },
        ... and other rating options
      ],
      ... and other filters
   }
}

Next, using got before an array of filters (["pri=2", "3 stars"]), currency, set location, dates, amount of guests, and set for business purposes:

📌Note: you can use both name or value for filters and currencies.

booking
  .getHotels(["pri=2", "3 stars"], "Argentine Peso", undefined, "Miami", "22/05/2023", "27/05/2023", 3, 1, undefined, "business")
  .then(console.log);

["pri=2", "3 stars"] - we choose a budget filter with a price limit of "US$50 – US$100" and a property rating filter with a "3 stars" limit;
"Argentine Peso" - selected currency;
undefined - didn't set the resultsLimit argument. That means we leave the default results limit (35 for booking);
"Miami" - selected location;
"22/05/2023" - check in date;
"27/05/2023" - check out date;
3 - adults amount;
1 - children amount;
undefined - didn't set the rooms argument. That means we leave the default needed rooms amount (1 for booking);
"business" - we choose "traveling for work" filter;

Output:

[
   {
      "thumbnail":"https://cf.bstatic.com/xdata/images/hotel/square200/277583459.webp?k=d90d11e2d824655526e06df6a5ddc759b715e9a27cb6ed1d8351a1a3a4e67b03&o=&s=1",
      "title":"Holiday Inn Express Doral Miami, an IHG Hotel",
      "stars":3,
      "preferredBadge":true,
      "promotedBadge":false,
      "location":"Doral",
      "subwayAccess":false,
      "distanceFromCenter":17.7,
      "highlights":[

      ],
      "rating":{
         "score":9,
         "scoreDescription":"Wonderful",
         "reviews":1396
      },
      "link":"https://www.booking.com/hotel/us/holiday-inn-express-doral-miami.html?lang=en-us"
   },
   {
      "thumbnail":"https://cf.bstatic.com/xdata/images/hotel/square200/379622649.webp?k=ed10fc85de5b55f776d6d5f2337bd823566039afd770a5f8a395d24e5308399f&o=&s=1",
      "title":"Best Western Plus Windsor Inn",
      "stars":3,
      "preferredBadge":true,
      "promotedBadge":false,
      "location":"North Miami",
      "subwayAccess":false,
      "distanceFromCenter":13.3,
      "highlights":[

      ],
      "rating":{
         "score":7.7,
         "scoreDescription":"Good",
         "reviews":644
      },
      "link":"https://www.booking.com/hotel/us/best-western-windsor-inn.html?lang=en-us"
   },
   ... and other hotels
]

And then using one of the hotel links get hotel info:

booking.getHotelInfo("https://www.booking.com/hotel/us/holiday-inn-express-doral-miami.html?lang=en-us").then((data) => {
  console.dir(data, { depth: null });
});

Output:

{
   "title":"Holiday Inn Express Doral Miami, an IHG Hotel",
   "type":"Hotel",
   "stars":3,
   "preferredBadge":true,
   "subwayAccess":false,
   "sustainability":"",
   "address":"1691 NW 107th Ave, Doral, FL 33172, United States of America",
   "highlights":[
      "Free WiFi",
      "Free parking",
      "Air conditioning",
      "Private Bathroom",
      "24-hour front desk",
      "Key card access",
      "Daily housekeeping",
      "Non-smoking rooms",
      "Safe",
      "Baggage storage"
   ],
   "description":"Located a 13-minute walk from Miami International Mall, Holiday Inn Express Doral Miami, an IHG Hotel offers 3-star accommodations in Doral and has a shared lounge. Featuring a fitness center, the 3-star hotel has air-conditioned rooms with free WiFi, each with a private bathroom. Guests can have a drink at the snack bar.\n""+""All rooms at the hotel come with a seating area, a flat-screen TV with satellite channels and a safety deposit box. At Holiday Inn Express Doral Miami, an IHG Hotel all rooms include bed linen and towels.\n""+""An American breakfast is available every morning at the accommodation.\n""+""Guests at Holiday Inn Express Doral Miami, an IHG Hotel will be able to enjoy activities in and around Doral, like cycling.\n""+""Free private parking and a business center are available, as well as a 24-hour front desk.\n""+""Dolphin Mall is a 17-minute walk from the hotel, while University of Miami is 17.7 km from the property. The nearest airport is Miami International Airport, 6 km from Holiday Inn Express Doral Miami, an IHG Hotel.",
   "descriptionHighlight":"Couples in particular like the location – they rated it 9.2 for a two-person trip.",
   "descriptionSummary":"Holiday Inn Express Doral Miami, an IHG Hotel has been welcoming Booking.com guests since Oct 19, 2020\n""+""\n""+""Hotel chain/brand:\n""+""Holiday Inn Express",
   "facilities":[
      "Free parking",
      "Free WiFi",
      "Fitness center",
      "Facilities for disabled guests",
      "Non-smoking rooms",
      "Tea/Coffee Maker in All Rooms",
      "Free parking",
      "Free WiFi",
      "Fitness center",
      "Facilities for disabled guests",
      "Non-smoking rooms",
      "Tea/Coffee Maker in All Rooms"
   ],
   "areaInfo":[
      {
         "What's nearby":[
            {
               "place":"Veteran's Park",
               "distance":"1.8 km"
            },
            {
               "place":"The Women's Park",
               "distance":"2.3 km"
            },
         ... and other nearby places
         ]
      },
      ... and other area info
   ],
   "link":"https://www.booking.com/hotel/us/holiday-inn-express-doral-miami.html?lang=en-us",
   "photos":[
      "https://cf.bstatic.com/xdata/images/hotel/max1024x768/277583459.jpg?k=da2b83dc681cb5c13878eb05d0bb838298092038e7d61b6df72a86631dcce9aa&o=&hp=1",
      "https://cf.bstatic.com/xdata/images/hotel/max1024x768/294863203.jpg?k=8ad830f1aa24e541616b02550d32d9da8018a5889730b509db4ad801981b6588&o=&hp=1",
      "https://cf.bstatic.com/xdata/images/hotel/max1024x768/277583540.jpg?k=ef279b7064a7b7c538f9e8fb437ef593fa4451f7186894af4ce9f4fd0602ecb0&o=&hp=1",
   ... and other photos
   ],
   "reviewsInfo":{
      "score":9,
      "scoreDescription":"Rated wonderful",
      "totalReviews":1419,
      "categoriesRating":[
         {
            "Staff":9.2
         },
         {
            "Facilities":9.2
         },
         {
            "Cleanliness":9.3
         },
         {
            "Comfort":9.3
         },
         {
            "Value for money":8.8
         },
         {
            "Location":9.2
         },
         {
            "Free WiFi":8.8
         }
      ],
      "reviews":[
         {
            "name":"Barbara",
            "avatar":"https://cf.bstatic.com/static/img/review/avatars/ava-b/8103dfb0481c4cedc201d849f5666a270512f538.png",
            "country":"Bermuda",
            "date":"April 3, 2023",
            "reting":"10",
            "review":[
               {
                  "liked":"Breakfast was very good.\nThe food was hot\nCoffee was good"
               },
               {
                  "didNotLike":"Nothing"
               }
            ],
            "hotelResponse":"Hello, thank you for your feedback! We are glad you enjoyed your stay."
         },
      ... and other reviews
      ]
   }
}

Hotels.com selected hotel info results.

First, we need to get hotels links:

hotelsCom.getHotels().then(console.log);

Output:

[
   {
      "title":"Hotel 10 Opera",
      "isAd":true,
      "location":"Paris",
      "snippet":{
         "title":"3* hotel located near the Opera",
         "text":"In the heart of the 9th district: customized offers according to the length of stay, flexibility & reinforced sanitary measures."
      },
      "paymentOptions":[
         "Fully refundable",
         "Reserve now, pay later"
      ],
      "highlightedAmenities":[

      ],
      "price":{
         "currency":"$",
         "value":274,
         "withTaxesAndCharges":305
      },
      "rating":{
         "score":8.8,
         "reviews":36
      },
      "link":"https://www.hotels.com/ho282954/hotel-10-opera-paris-france/"
   },
   ... and other hotels
]

And then we use the hotel link and get the hotel info result:

hotelsCom.getHotelInfo("https://www.hotels.com/ho282954/hotel-10-opera-paris-france/").then((result) => console.dir(result, { depth: null }));

Output:

{
   "title":"Hotel 10 Opera",
   "stars":3,
   "shortDescription":"City-center hotel within walking distance of Palais Garnier",
   "address":"10, Rue du Helder, Paris, Paris, 75009",
   "description":"Hotel 10 Opera offers a great location, putting you within a 15-minute walk of Palais Garnier and Louvre Museum. Also, Champs-Élysées and La Machine du Moulin Rouge are just a 5-minute drive away. Fellow travelers say great things about the local sightseeing and quiet guestrooms. The property is just a short walk to public transportation: Chaussee d'Antin - La Fayette Station is 3
minutes and Opéra Station is 3 minutes.",
   "languages":"English, French, Portuguese, Spanish",
   "roomOptions":[
      "Room",
      "Twin Room",
      "Double Room Single Use",
      "Family Room"
   ],
   "areaInfo":[
      {
         "What's nearby":[
            {
               "place":"In Paris City Center"
            },
            {
               "place":"Galeries Lafayette",
               "distance":"3 min walk"
            },
            ... and other nearby places
         ]
      },
      ... and other aria info
   ],
   "CleaningAndSafety":[
      {
         "Enhanced cleanliness measures":[
            "Disinfectant is used to clean the property",
            "High-touch surfaces are cleaned and disinfected",
            "Sheets and towels are washed at 60°C/140°F or hotter"
         ]
      },
    ... and other cleaning and safety options
   ],
   "atAGlance":[
      {
         "Hotel size":[
            "34 rooms",
            "Arranged over 6 floors"
         ]
      },
    ... and other at a glance options
   ],
   "propertyAmenities":[
      {
         "Food and drink":[
            "Buffet breakfast (surcharge) each morning 7 AM–10:30 AM",
            "Coffee/tea in a common area",
            "Room service (limited hours)"
         ]
      },
     ... and other property amenities
   ],
   "roomAmenities":[
      {
         "Be entertained":[
            "Television",
            "Satellite TV channels"
         ]
      },
     ... and other room amenities
   ],
   "specialFeatures":[],
   "feesAndPolicies":[
      {
         "Mandatory fees":[
            "A tax is imposed by the city: EUR 1.88 per person, per night. This tax does not apply to children under 18 years of age."
         ]
      },
    ... and other fees and policies
   ],
   "link":"https://www.hotels.com/ho282954/hotel-10-opera-paris-france/",
   "photos":[
      "https://images.trvl-media.com/lodging/2000000/1550000/1542800/1542774/b08e7322.jpg?impolicy=resizecrop&rw=1200&ra=fit",
      "https://images.trvl-media.com/lodging/2000000/1550000/1542800/1542774/8975206c.jpg?impolicy=resizecrop&rw=1200&ra=fit",
     ... and other photo
   ],
   "reviewsInfo":{
      "score":8.8,
      "scoreDescription":"Fabulous",
      "totalReviews":36,
      "categoriesRating":[
         {
            "Cleanliness":9.4
         },
        ... and more category rating
      ],
      "reviews":[
         {
            "date":"Mar 21, 2023",
            "reting":8,
            "reviewTitle":"Will stay here again!!",
            "review":"The hotel is located around around the corner of the Opera House and a 5 minute walk to Galerie Lafayette. It is 15 minutes walk to the Louvre. The location is perfect.  The hotel is
clean, safe and the staff is friendly. Free water, coffee and tea in reception. Two things to be aware of: 1. small, simple room and tiny bathroom (you get what you pay for) and 2. Across the street from hotel is a popular restaurant with people on street. The restaurant closes at midnight so noise is not a problem if you are in the front.  I plan to stay there again given the price, location, and service.",
            "hotelResponse":"Dear Karen,Thank you for your feedback, we are delighted to see that you enjoyed your stay! Please feel free to tell us what we could improve to deserve a 10!Looking forward to welcoming you again"
         },
        ... and other reviews
      ]
   }
}

Advanced Example

Find the cheapest hotels from all sites, get top 10 hotel info, and save the results.

Full code:

import { airbnb, booking, hotelsCom, saveToJSON } from "./index.js";

const getHotels = async (location, checkIn, checkOut) => {
  const hotels = [
    ...(await airbnb.getHotels(undefined, "USD", 50, location, checkIn, checkOut)),
    ...(await booking.getHotels(undefined, "USD", 50, location, checkIn, checkOut)),
    ...(await hotelsCom.getHotels(undefined, undefined, undefined, "United States", undefined, 50, location, checkIn, checkOut)),
  ];

  return hotels
    .sort((a, b) => {
      let aValue = a.price?.value;
      let bValue = b.price?.value;
      if (a.price?.taxesAndCharges || (a.price?.period && a.price?.period !== "night")) {
        aValue = a.price?.value / 10;
      }
      if (b.price?.taxesAndCharges || (b.price?.period && b.price?.period !== "night")) {
        bValue = b.price?.value / 10;
      }
      return aValue - bValue;
    })
    .slice(0, 10);
};

const getInfo = async (hotels) => {
  const results = [];
  for (const hotel of hotels) {
    const { link, price } = hotel;
    if (link.includes("airbnb.com")) {
      results.push({ ...(await airbnb.getHotelInfo(link)), price });
    } else if (link.includes("booking.com")) {
      results.push({ ...(await booking.getHotelInfo(link)), price });
    } else results.push({ ...(await hotelsCom.getHotelInfo(link)), price });
  }
  return results;
};

getHotels("Miami", "07/18/2023", "07/28/2023").then(getInfo).then(saveToJSON);

Explanation:

First, we get hotels from each website located in "Miami" and free from 07/18/2023 to 07/28/2023:

getHotels("Miami", "07/18/2023", "07/28/2023");

In getHotels function we get 50 hotels from each website, then sort them by price ascendent, and leave the 10 most cheap:

const getHotels = async (location, checkIn, checkOut) => {
  const hotels = [
    ...(await airbnb.getHotels(undefined, "USD", 50, location, checkIn, checkOut)),
    ...(await booking.getHotels(undefined, "USD", 50, location, checkIn, checkOut)),
    ...(await hotelsCom.getHotels(undefined, undefined, undefined, "United States", undefined, 50, location, checkIn, checkOut)),
  ];

  return hotels
    .sort((a, b) => {
      let aValue = a.price?.value;
      let bValue = b.price?.value;
      if (a.price?.taxesAndCharges || (a.price?.period && a.price?.period !== "night")) {
        aValue = a.price?.value / 10;
      }
      if (b.price?.taxesAndCharges || (b.price?.period && b.price?.period !== "night")) {
        bValue = b.price?.value / 10;
      }
      return aValue - bValue;
    })
    .filter((el, i) => i < 10);
};

After hotels are received and filtered, we call then function and give received hotels into getInfo function:

.then(getInfo)

In getInfo we iterate over all hotels, receive each hotel info, push them in results array and return it:

const getInfo = async (hotels) => {
  const results = [];
  for (const hotel of hotels) {
    const { link, price } = hotel;
    if (link.includes("airbnb.com")) {
      results.push({ ...(await airbnb.getHotelInfo(link)), price });
    } else if (link.includes("booking.com")) {
      results.push({ ...(await booking.getHotelInfo(link)), price });
    } else results.push({ ...(await hotelsCom.getHotelInfo(link)), price });
  }
  return results;
};

And the last one, we call then function and give received hotels into saveToJSON function:

.then(saveToJSON)

The result is the saved parsed_results.json file in your app directory with top-10 cheapest hotels with full description.

Links

If you want other functionality added to this demo project or if you want to see some other projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

JavaScript SEO Keywords Research Tool: Google Autocomplete, People Also Ask, and People Also Search For

Mikhail Zub — Tue, 07 Mar 2023 19:36:59 +0000

About

js-seo-keywords-generator is a Node.js package designed to generate keywords for SEO purposes. The package is available on npm and can be installed and used in your Node.js project or as CLI. It is used to generate keyword ideas using Google's Autocomplete, People also search and People also ask results and save them to TXT, CSV, JSON file. Scrape results were obtained using SerpApi.

Currently supports:

Google search
other engines soon.

Here's a simple user guide to help you get started with using js-seo-keywords-generator.

Installation

📌Note: If you want to use this tool for personal or commercial purposes, you must register on SerpApi and get your own API key (no credit card needed, free plan available). The default API key used is for informational purposes only.

Make sure you have Node.js installed on your computer.

If you want to use SEO Keywords generator as CLI and save results in a file, you need to install it globally:

npm i -g js-seo-keywords-generator

If you want to get results from SEO Keywords generator in your Node.js app, you need to add it to your project dependency:

npm i js-seo-keywords-generator

Before use, you need to define your API key from SerpApi. There are several ways to do this:

Add an environment variable API_KEY with your key. Read more how to add it in different systems;
Add your key directly in code: SEO.api_key = "your_api_key" (for in-code usage only).

CLI Usage

You can see available usage options by typing in the terminal:

seo help get

Available arguments

  -e, --engine       Parameter defines the engine(s) of use. It can be set to "paa" (People Also Ask),
                                   "rs" (Related Searches), "ac" (Autocomplete), or in any combination, e.g.
                                   "paa,rs,ac" (default). (default: ["paa","rs","ac"])
  -d, --domain      Parameter defines the Google domain to use. It defaults to google.com. Head to the
                                   https://serpapi.com/google-domains for a full list of supported Google domains.
                                   (default: "google.com")
  -gl, --country     Parameter defines the country to use for the Google search. Head to the
                                   https://serpapi.com/google-countries for a full list of supported Google countries.
                                   (default: "us")
  -hl, --language   Parameter defines the language to use for the Google search. Head to the
                                   https://serpapi.com/google-languages for a full list of supported Google languages.
                                   (default: "en")
  -dl, --depth        Parameter defines the depth of disclosure of questions for People Also Ask (default:
                                   "1")
  -ft, --type           Parameter defines the type of file to save. (choices: "json", "txt", "csv", default:
                                   "json")
  -h, --help                       display help for command

📌Note: You can use any of the available options in any combination. You can see the full description of each of them below.

Engines use example

To get results from all supported engines you need to use the app with no engine argument:

seo get <search  query>

If you want to get results from one or two selected engines you need to define the --engine option (or short: -e) divided by a comma, e.g. paa, ac (People also ask and Autocomplete):

seo get <search  query> -e <engine(s)>

paa - People Also Ask engine;
rs - Related Searches (People Also Search) engine;
ac - Autocomplete engine.

Use different Google domain, country and language

You can change the default google domain google.com to any of supported domains list by defining the --domain (-d) option:

seo get <search  query> -d <google domain>

Also, you can change the default country us (United States) of the search using the --country (-gl) option and set it to any of supported country codes list:

seo get <search  query> -gl <country code>

If you want to change the default language en (English) of the search, you need to use the --language (-hl) option and set it to any of supported language codes list:

seo get <search  query> -hl <language code>

Available file types to save search results

The default file type to save search results is json. You can change it to txt or csv using the --type (-ft) option:

seo get <search  query> -ft <file type>

What is the depth parameter? (only for People Also Ask results)

When you open People Also Ask result, it loads more results:

Default:

One of the results is open:

So if you open all initial results that was mean you see next-depth results.

The depth limit parameter can be set from 1 to 3. This limit is set because, for each depth value, the number of results returned grows exponentially, for example, if the number of results is 5 at each level, for a depth of 3, the total number of results will be equal 125.

Output example

Terminal output:

Saved file:

{
  "people_also_ask": [
    "Was ist ein Node in der IT?",
    "Für was brauche ich node js?",
    "Ist Node ein Framework?",
    "Wie installiere ich Node?",
    "Warum eine Bitcoin Node?",
    "Wer steuert den Bitcoin?",
    "Was ist ein Ethereum Node?",
    "Wie funktioniert eine Node?",
    "Was kostet eine Bitcoin Node?",
    "Welcher Coin wird Bitcoin ersetzen?",
    "Was ist der billigste Bitcoin?",
    "Was kostet 1 ganzer Bitcoin?",
    "Ist node js kostenlos?",
    "Was kann man mit Node-RED machen?",
    "Wer nutzt node js?",
    "Was macht ein NPM?",
    "Wie funktioniert eine Node?",
    "Was kostet eine Bitcoin Node?",
    "Was bringt ein Bitcoin Node?",
    "Wer hat den Bitcoin entwickelt?",
    "Was bringt es eine Node zu betreiben?",
    "Was macht ein Bitcoin Node?",
    "Was ist ein Ethereum Node?",
    "Wie funktioniert eine Node?",
    "Wer nutzt node js?",
    "Ist node js kostenlos?",
    "Wie funktioniert eine Node?",
    "Welche Datenbank für node js?",
    "Ist Node-RED kostenlos?",
    "Wie funktioniert eine Node?",
    "Was kann man alles mit Node-RED machen?",
    "Welches Betriebssystem für Node-RED?",
    "Ist node js kostenlos?",
    "Wie funktioniert eine Node?",
    "Was kann man mit Node-RED machen?",
    "Wer nutzt node js?"
  ],
  "related_searches": [
    "node übersetzung",
    "node crypto",
    "node lts",
    "node version check",
    "node update",
    "node.js tutorial",
    "node server",
    "node version manager"
  ]
}

In-code Usage

Import SEO to your file:

import SEO from "seo-keywords";

Available methods:

getAll(query[, domain[, country[, language[, depthlimit]]]])
getCustom(engines, query[, domain[, country[, language[, depthlimit]]]])
getPeopleAlsoAsk(query[, domain[, country[, language[, depthlimit]]]])
getRelatedSearch(query[, domain[, country[, language]]])
getAutocomplete(query[, domain[, country[, language]]])

query - search query.
engines - parameter defines the engine(s) of use:
- paa (People Also Ask)
- rs (Related Searches)
- ac (Autocomplete)
- could be combined: paa, ac
domain - parameter defines the Google domain to use. Head to the Google domains page for a full list of supported Google domains. Default is google.com.
country - parameter defines the country to use for the Google search. Head to the Google countries page for a full list of supported Google countries. Default is us.
language - parameter defines the language to use for the Google search. Head to the Google languages page for a full list of supported Google languages. Default is en.
depthlimit - parameter defines the depth of disclosure of questions for People Also Ask. Default is 1 (Only for People Also Ask engine).

Engines use example

To get results from all supported engines you need to use getAll method:

(async () => {
  console.log(await SEO.getAll(query[, domain[, country[, language[, depthlimit]]]]));
})();

query argument is required;
domain, country, language, and depthlimit are optional arguments.

If you want to get results from one or two selected engines you need to use getCustom method with the engines argument divided by a comma, e.g.paa, ac (Peaople also ask and Autocomplete):

(async () => {
  console.log(await SEO.getCustom(engines, query[, domain[, country[, language[, depthlimit]]]]));
})();

engines argument is required;
- paa - People Also Ask engine;
- rs - Related Searches (People Also Search) engine;
- ac - Autocomplete engine.
query argument is required;
domain, country, language, and depthlimit are optional arguments.

If you want to get results from only one engine you need to use getPeopleAlsoAsk, getRelatedSearch, or getAutocomplete methods:

(async () => {
  console.log(await SEO.ggetPeopleAlsoAsk(query[, domain[, country[, language[, depthlimit]]]]));
  console.log(await SEO.getRelatedSearch(query[, domain[, country[, language]]]));
  console.log(await SEO.getAutocomplete(query[, domain[, country[, language]]]));
})();

query argument is required;
domain, country, language, and depthlimit are optional arguments.

Use different Google domain, country and language

You can change the default google domain google.com to any of supported domains list by adding a domain argument for any method.

Also, you can change the default country us (United States) of the search using the country argument and set it to any of supported country codes list.

If you want to change the default language en (English) of the search, you need to use the language argument and set it to any of supported language codes list.

What is the depth parameter? (only for People Also Ask results)

When you open People Also Ask result, it loads more results:

Default:

One of the results is open:

So if you open all initial results that was mean you see next-depth results.

Output example

{
  "people_also_ask": ["What exactly is a node?", "What is node used for?", "Why it is called node?", "Why is node better than Python?"],
  "related_searches": [
    "node js",
    "what is node in networking",
    "node plant",
    "node version",
    "node js documentation",
    "nodes meaning in human body",
    "how does node work?",
    "node js download"
  ],
  "autocomplete": ["node js", "nodes of ranvier", "node unblocker", "node js download", "node red", "node definition", "node-fetch", "node version"]
}

Links

If you want other functionality added to this demo project or if you want to see some other projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Google Hotels with Nodejs

Mikhail Zub — Fri, 03 Mar 2023 22:03:12 +0000

Currently, we don't have an API that supports extracting data from Google Hotels page, however, it's currently in the development stage which you can track at our public-roadmap.

This blog post is to show you way how you can do it yourself with provided DIY solution below while we're working on releasing our proper API.

What will be scraped

If you don't need an explanation, have a look at the full code example in the online IDE

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const searchQuery = "Honolulu";

const URL = `https://www.google.com/travel/hotels/${encodeURI(searchQuery)}`;

async function getHotelsInfo(page) {
  let lastHeight = await page.evaluate(`document.querySelector(".zQTmif").scrollHeight`);
  while (true) {
    await page.waitForTimeout(500);
    await page.keyboard.press("End");
    await page.waitForTimeout(500);
    await page.keyboard.press("PageUp");
    await page.waitForTimeout(5000);
    let newHeight = await page.evaluate(`document.querySelector(".zQTmif").scrollHeight`);
    if (newHeight === lastHeight) {
      break;
    }
    lastHeight = newHeight;
  }
  return await page.evaluate(() =>
    Array.from(document.querySelectorAll(".TNNk1.nzwZbc")).map((el) => {
      const adFrom = el.querySelector(".hVE5 .ogfYpf")?.textContent.trim();
      return {
        link: `https://www.google.com/${el.querySelector(".PVOOXe")?.getAttribute("href")}`,
        images: Array.from(el.querySelectorAll(".pb2I5 img"))
          .map((el) => el.getAttribute("src"))
          .filter((el) => el),
        title: el.querySelector(".BgYkof")?.textContent.trim(),
        price: el.querySelector(".xquSSe .kixHKb > span:first-child")?.textContent.trim(),
        rating: parseFloat(el.querySelector(".KFi5wf")?.textContent.trim()) || "No rating",
        reviews:
          parseInt(
            el
              .querySelector(".jdzyld")
              ?.textContent.trim()
              .replace(/[\(|\)|\s]/gm, "")
          ) || "No reviews",
        stars: parseInt(el.querySelector(".UqrZme")?.textContent.trim()),
        options: Array.from(el.querySelectorAll(".XX3dkb > .LtjZ2d ")).map((el) => el.textContent.trim()),
        adFrom,
      };
    })
  );
}

async function getHotelsResults() {
  const browser = await puppeteer.launch({
    headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);

  await page.waitForSelector(".TNNk1.nzwZbc");

  const hotels = await getHotelsInfo(page);

  await browser.close();

  return hotels;
}

getHotelsResults().then(console.log);

Preparation

First, we need to create a Node.js* project and add npm packages puppeteer, puppeteer-extra and puppeteer-extra-plugin-stealth to control Chromium (or Chrome, or Firefox, but now we work only with Chromium which is used by default) over the DevTools Protocol in headless or non-headless mode.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

📌Note: also, you can use puppeteer without any extensions, but I strongly recommended use it with puppeteer-extra with puppeteer-extra-plugin-stealth to prevent website detection that you are using headless Chromium or that you are using web driver. You can check it on Chrome headless tests website. The screenshot below shows you a difference.

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Process

We need to extract data from HTML elements. The process of getting the right CSS selectors is fairly easy via SelectorGadget Chrome extension which able us to grab CSS selectors by clicking on the desired element in the browser. However, it is not always working perfectly, especially when the website is heavily used by JavaScript.

We have a dedicated Web Scraping with CSS Selectors blog post at SerpApi if you want to know a little bit more about them.

The Gif below illustrates the approach of selecting different parts of the results using SelectorGadget.

Code explanation

Declare puppeteer to control Chromium browser from puppeteer-extra library and StealthPlugin to prevent website detection that you are using web driver from puppeteer-extra-plugin-stealth library:

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

Next, we "say" to puppeteer use StealthPlugin, write the search query and the search URL:

puppeteer.use(StealthPlugin());

const searchQuery = "Honolulu";

const URL = `https://www.google.com/travel/hotels/${encodeURI(searchQuery)}`;

Next, we write a function to get hotels info from the page:

async function getHotelsInfo(page) {
  ...
}

In this function we'll use the next methods and properties to get the necessary information:

First, we need to scroll the page for loads all results. To do this we get the results container scrollHeight and set it to lastHeight variable, then scroll the page in the while loop by press "End" button and check if new scrollHeight isn't change we stop the loop:

let lastHeight = await page.evaluate(`document.querySelector(".zQTmif").scrollHeight`);
while (true) {
  await page.waitForTimeout(500);
  await page.keyboard.press("End");
  await page.waitForTimeout(500);
  await page.keyboard.press("PageUp");
  await page.waitForTimeout(5000);
  let newHeight = await page.evaluate(`document.querySelector(".zQTmif").scrollHeight`);
  if (newHeight === lastHeight) {
    break;
  }
  lastHeight = newHeight;
}

Then, we get and return all hotels info from the page (using evaluate() method):

return await page.evaluate(() =>
  Array.from(document.querySelectorAll(".TNNk1.nzwZbc")).map((el) => {
    const adFrom = el.querySelector(".hVE5 .ogfYpf")?.textContent.trim();
    return {
      link: `https://www.google.com/${el.querySelector(".PVOOXe")?.getAttribute("href")}`,
      images: Array.from(el.querySelectorAll(".pb2I5 img"))
        .map((el) => el.getAttribute("src"))
        .filter((el) => el),
      title: el.querySelector(".BgYkof")?.textContent.trim(),
      price: el.querySelector(".xquSSe .kixHKb > span:first-child")?.textContent.trim(),
      rating: parseFloat(el.querySelector(".KFi5wf")?.textContent.trim()) || "No rating",
      reviews:
        parseInt(
          el
            .querySelector(".jdzyld")
            ?.textContent.trim()
            .replace(/[\(|\)|\s]/gm, "") // this RegEx matches "(", or ")", or any white space
        ) || "No reviews",
      stars: parseInt(el.querySelector(".UqrZme")?.textContent.trim()),
      options: Array.from(el.querySelectorAll(".XX3dkb > .LtjZ2d ")).map((el) => el.textContent.trim()),
      adFrom,
    };
  })
);

Next, we write a function to control the browser, and get information from each category:

async function getHotelsResults() {
  ...
}

In this function first we need to define browser using puppeteer.launch({options}) method with current options, such as headless: true and args: ["--no-sandbox", "--disable-setuid-sandbox"].

These options mean that we use headless mode and array with arguments which we use to allow the launch of the browser process in the online IDE. And then we open a new page:

const browser = await puppeteer.launch({
  headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();

Next, we change the default (30 sec) time for waiting for selectors to 60000 ms (1 min) for slow internet connection with .setDefaultNavigationTimeout() method, go to URL with .goto() method:

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);

Then we wait until ".TNNk1.nzwZbc" selector is load (waitForSelector() method) and add hotels information from the page to hotels constant:

await page.waitForSelector(".TNNk1.nzwZbc");

const hotels = await getHotelsInfo(page);

And finally, we close the browser, and return the received data:

await browser.close();

return hotels;

Now we can launch our parser:

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

Output

[
   {
      "link":"https://www.google.com//travel/hotels/Honolulu/entity/CgoIsva7guWXuqkpEAEae0FBQm5CM2xEcERidFBBWHRBV2x0TWt0UnhmQW4xVjJMWjJDVExrWWd0eG9JVWRfZXlnWWxteHBpaEh1cXptb3pDTWpqUlFkSmJpdmlHRlRLT3NJbER6QW1vMUV5c3VOOW5VR3FFb0xZTlgxLWQ2UllDVnVmNE04QWVRSQ?utm_campaign=sharing&utm_medium=link&utm_source=htls&ved=2ahUKEwjw8ur3wJr9AhVyBRwAHcDoCHoQyvcEegQIAxAr&ts=CAESABogCgIaABIaEhQKBwjnDxACGBMSBwjnDxACGBQYATICEAAqCQoFOgNVU0QaAA&rp=OAE",
      "images":[
         "https://lh5.googleusercontent.com/p/AF1QipNLY_J7UdmiBu8U1ivimm1OFZGASPJxxsdRPXlv=w592-h404-n-k-no-v1",
         "https://lh5.googleusercontent.com/p/AF1QipOkC6UOW2plYus-TxCYqhMtBMi-0HkqyUYmrMzu=w592-h404-n-k-no-v1"
      ],
      "title":"Hilton Waikiki Beach",
      "rating":4.1,
      "reviews":823,
      "stars":4,
      "options":[
         "Breakfast ($)",
         "Free Wi-Fi"
      ],
      "adFrom":"From  Hilton Waikiki Beach"
   },
   {
      "link":"https://www.google.com//travel/hotels/Honolulu/entity/CgoI866GmIeXmos7EAEafkFBQm5CM2xuTDdXWDI2cDRoWllQXzVVazI3Vkd3OWVDbnczM215U05wRmhOOGhiMHpOaGQ4TTRoQ2ZfZVEzbU5yWDZ2U2t2THRDMy1sMlc1YWhLel9GUE9TWEhpd1ZRclgxYjhZY0VFM2ZlOEwzNTlONlJiVDNnd0k2SHpsdw?utm_campaign=sharing&utm_medium=link&utm_source=htls&ved=2ahUKEwjw8ur3wJr9AhVyBRwAHcDoCHoQyvcEegQIAxBE&ts=CAESABogCgIaABIaEhQKBwjnDxACGBMSBwjnDxACGBQYATICEAAqCQoFOgNVU0QaAA&rp=OAE",
      "images":[
         "https://lh5.googleusercontent.com/p/AF1QipMVQjGzCM8CwfDJrdrmY_CCdzwa-LQeSuF2bs5O=w592-h404-n-k-no-v1",
         "https://lh5.googleusercontent.com/p/AF1QipNSGmtQ44aKHkdXo9yU1jqZgjcu9azk7gZ4An1J=w592-h404-n-k-no-v1"
      ],
      "title":"Coconut Waikiki Hotel",
      "rating":4.2,
      "reviews":802,
      "stars":3,
      "options":[
         "Free Wi-Fi",
         "Parking ($)"
      ],
      "adFrom":"From  Booking.com"
   },
  ... and other hotels
]

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Google Flights with Nodejs

Mikhail Zub — Tue, 28 Feb 2023 16:50:16 +0000

Currently, we don't have an API that supports extracting data from Google flights page. This blog post is to show you way how you can do it yourself with provided DIY solution.

What will be scraped

📌Note: The solution I'm showing you only gets flight results for the "One Way", "1 person" and "Economy" options applied.

If you don't need an explanation, have a look at the full code example in the online IDE

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const from = "Seattle";
const to = "Las Vegas";
const leaveDate = "5-15-2023"; // mm-dd-yyyy format

const URL = `https://www.google.com/travel/flights?hl=en-US&curr=USD`;

async function getFlightsFromPage(page) {
  return await page.evaluate(() =>
    Array.from(document.querySelectorAll(".pIav2d")).map((el) => {
      const thumbnailString = el.querySelector(".EbY4Pc")?.getAttribute("style");
      const startIndex = thumbnailString?.indexOf("url(");
      const endIndex = thumbnailString?.indexOf(";");
      const thumbnail = thumbnailString?.slice(startIndex + 4, endIndex - 1).replaceAll("\\", "") || "No thumbnail";
      const layover = el.querySelector(".BbR8Ec .sSHqwe")?.getAttribute("aria-label");
      return {
        thumbnail,
        companyName: el.querySelector(".Ir0Voe .sSHqwe")?.textContent.trim(),
        description: el.querySelector(".mv1WYe")?.getAttribute("aria-label"),
        duration: el.querySelector(".gvkrdb")?.textContent.trim(),
        airportLeave: el.querySelectorAll(".Ak5kof .sSHqwe .eoY5cb")[0]?.textContent.trim(),
        airportArive: el.querySelectorAll(".Ak5kof .sSHqwe .eoY5cb")[1]?.textContent.trim(),
        layover: layover || "Nonstop",
        emisions: el.querySelector(".V1iAHe > div")?.getAttribute("aria-label").replace(". Learn more about this emissions estimate", " "),
        price: el.querySelector(".U3gSDe .YMlIz > span")?.textContent.trim(),
        priceDescription: el.querySelector(".U3gSDe .JMnxgf > span > span > span")?.getAttribute("aria-label"),
      };
    })
  );
}

async function getFlightsResults() {
  const browser = await puppeteer.launch({
    headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();
  page.setViewport({
    width: 1280,
    height: 720,
  });

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);

  await page.waitForSelector(".e5F5td");
  const inputs = await page.$$(".e5F5td");
  // type "from"
  await inputs[0].click();
  await page.waitForTimeout(1000);
  await page.keyboard.type(from);
  await page.keyboard.press("Enter");
  // type "to"
  await inputs[1].click();
  await page.waitForTimeout(1000);
  await page.keyboard.type(to);
  await page.waitForTimeout(1000);
  await page.keyboard.press("Enter");
  await page.waitForTimeout(1000);
  // type "Leave date"
  await page.click(".rIZzse .d5wCYc");
  await page.waitForTimeout(1000);
  await page.keyboard.type(leaveDate);
  await page.waitForTimeout(1000);
  await page.keyboard.press("Enter");
  await page.waitForTimeout(1000);
  // choose "One way"
  await page.click(".UGrfjc .VfPpkd-RLmnJb");
  await page.waitForTimeout(1000);
  await page.click(".VfPpkd-qPzbhe-JNdkSc > li:last-child");
  await page.waitForTimeout(1000);
  // press "Done"
  await page.click(".A8nfpe .akjk5c  .VfPpkd-vQzf8d");
  await page.waitForTimeout(1000);
  await page.keyboard.press("Enter");
  // press "Search"
  await page.waitForTimeout(1000);
  await page.keyboard.press("Enter");

  await page.waitForSelector(".pIav2d");

  const moreButton = await page.$(".XsapA");
  if (moreButton) {
    await moreButton.click();
    await page.waitForTimeout(2000);
  }

  const flights = await getFlightsFromPage(page);

  await browser.close();

  return flights;
}

getFlightsResults().then(console.log);

Preparation

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Process

We have a dedicated Web Scraping with CSS Selectors blog post at SerpApi if you want to know a little bit more about them.

The Gif below illustrates the approach of selecting different parts of the results using SelectorGadget.

Code explanation

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

Next, we "say" to puppeteer use StealthPlugin, write the city of departure (from) and destination (to) and the date of departure (leaveDate) and the search URL:

puppeteer.use(StealthPlugin());

const from = "Seattle";
const to = "Las Vegas";
const leaveDate = "5-15-2023"; // mm-dd-yyyy format

const URL = `https://www.google.com/travel/flights?hl=en-US&curr=USD`;

Next, we write a function to get places from the page:

async function getFlightsFromPage(page) {
  ...
}

In this function we'll use the next methods and properties to get the necessary information:

First, we need to get thumbnail URL from the page. To do this we get the style attribute from ".EbY4Pc" selector that contains URL. Then we find startIndex that matches "url(" and endIndex that matches ";" and cut thumbnail URL:

const thumbnailString = el.querySelector(".EbY4Pc")?.getAttribute("style");
const startIndex = thumbnailString?.indexOf("url(");
const endIndex = thumbnailString?.indexOf(";");
const thumbnail = thumbnailString?.slice(startIndex + 4, endIndex - 1).replaceAll("\\", "") || "No thumbnail";

Next, we need to get layover information if it exists:

const layover = el.querySelector(".BbR8Ec .sSHqwe")?.getAttribute("aria-label");

Then, we get and return all flights info from the page:

return {
  thumbnail,
  companyName: el.querySelector(".Ir0Voe .sSHqwe")?.textContent.trim(),
  description: el.querySelector(".mv1WYe")?.getAttribute("aria-label"),
  duration: el.querySelector(".gvkrdb")?.textContent.trim(),
  airportLeave: el.querySelectorAll(".Ak5kof .sSHqwe .eoY5cb")[0]?.textContent.trim(),
  airportArive: el.querySelectorAll(".Ak5kof .sSHqwe .eoY5cb")[1]?.textContent.trim(),
  layover: layover || "Nonstop",
  emisions: el.querySelector(".V1iAHe > div")?.getAttribute("aria-label").replace(". Learn more about this emissions estimate", " "),
  price: el.querySelector(".U3gSDe .YMlIz > span")?.textContent.trim(),
  priceDescription: el.querySelector(".U3gSDe .JMnxgf > span > span > span")?.getAttribute("aria-label"),
};

Next, we write a function to control the browser, and get information from each category:

async function getFlightsResults() {
  ...
}

These options mean that we use headless mode and array with arguments which we use to allow the launch of the browser process in the online IDE. And then we open a new page and set viewport size equal to 1280x720 pixels:

const browser = await puppeteer.launch({
  headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();
page.setViewport({
  width: 1280,
  height: 720,
});

Next, we change the default (30 sec) time for waiting for selectors to 60000 ms (1 min) for slow internet connection with .setDefaultNavigationTimeout() method, go to URL with .goto() method:

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);

Then we wait for the ".e5F5td" selector to load (method waitForSelector()), get the input fields from and to click on each of them and enter from and to cities (method keyboard.type()), press "Enter" button (method keyboard.press()) after typing, and then enter leaveDate and the "one way ticket" option:

await page.waitForSelector(".e5F5td");
const inputs = await page.$$(".e5F5td");
// type "from"
await inputs[0].click();
await page.waitForTimeout(1000);
await page.keyboard.type(from);
await page.keyboard.press("Enter");
// type "to"
await inputs[1].click();
await page.waitForTimeout(1000);
await page.keyboard.type(to);
await page.waitForTimeout(1000);
await page.keyboard.press("Enter");
await page.waitForTimeout(1000);
// type "Leave date"
await page.click(".rIZzse .d5wCYc");
await page.waitForTimeout(1000);
await page.keyboard.type(leaveDate);
await page.waitForTimeout(1000);
await page.keyboard.press("Enter");
await page.waitForTimeout(1000);
// choose "One way"
await page.click(".UGrfjc .VfPpkd-RLmnJb");
await page.waitForTimeout(1000);
await page.click(".VfPpkd-qPzbhe-JNdkSc > li:last-child");
await page.waitForTimeout(1000);
// press "Done"
await page.click(".A8nfpe .akjk5c  .VfPpkd-vQzf8d");
await page.waitForTimeout(1000);
await page.keyboard.press("Enter");

After filling in all fields we press the "Enter" button and wait for loads of flight results. Click on the "Show more" button and save flights results to the flights constant:

// press "Search"
await page.waitForTimeout(1000);
await page.keyboard.press("Enter");

await page.waitForSelector(".pIav2d");

const moreButton = await page.$(".XsapA");
if (moreButton) {
  await moreButton.click();
  await page.waitForTimeout(2000);
}

const flights = await getFlightsFromPage(page);

And finally, we close the browser, and return the received data:

await browser.close();

return flights;

Now we can launch our parser:

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

Output

[
   {
      "thumbnail":"https://www.gstatic.com/flights/airline_logos/70px/AA.png",
      "companyName":"American",
      "description":"Leaves Seattle-Tacoma International Airport at 11:55 PM on Monday, May 15 and arrives at Harry Reid International Airport at 5:54 PM on Tuesday, May 16.",
      "duration":"17 hr 59 min",
      "airportLeave":"Seattle-Tacoma International Airport",
      "airportArive":"Harry Reid International Airport",
      "layover":"Layover (1 of 1) is a 11 hr 4 min layover at Dallas/Fort Worth International Airport in Dallas.",
      "emisions":"Carbon emissions estimate: 315 kilograms. +184% emissions ",
      "price":"$318"
   },
   {
      "thumbnail":"https://www.gstatic.com/flights/airline_logos/70px/AS.png",
      "companyName":"Alaska",
      "description":"Leaves Seattle-Tacoma International Airport at 7:10 PM on Monday, May 15 and arrives at Harry Reid International Airport at 4:36 PM on Tuesday, May 16.",
      "duration":"21 hr 26 min",
      "airportLeave":"Seattle-Tacoma International Airport",
      "airportArive":"Harry Reid International Airport",
      "layover":"Layover (1 of 1) is a 17 hr 42 min overnight layover at San Francisco International Airport in San Francisco.",
      "emisions":"Carbon emissions estimate: 176 kilograms. +59% emissions ",
      "price":"$323"
   }
   ... and other flights results
]

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Google Things To Do with Nodejs

Mikhail Zub — Thu, 16 Feb 2023 16:43:46 +0000

What will be scraped

If you don't need an explanation, have a look at the full code example in the online IDE

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

puppeteer.use(StealthPlugin());

const searchQuery = "Honolulu";

const URL = `https://www.google.com/travel/things-to-do`;

async function getPlaces(page) {
  const height = await page.evaluate(() => document.querySelector(".zQTmif").scrollHeight);
  const scrollIterationCount = 10;
  for (let i = 0; i < scrollIterationCount; i++) {
    await page.mouse.wheel({ deltaY: height / scrollIterationCount });
    await page.waitForTimeout(2000);
  }
  return await page.evaluate(() =>
    Array.from(document.querySelectorAll(".f4hh3d")).map((el) => ({
      thumbnail: el.querySelector(".kXlUEb img")?.getAttribute("src") || "No thumbnail",
      title: el.querySelector(".GwjAi .skFvHc")?.textContent.trim(),
      description: el.querySelector(".GwjAi .nFoFM")?.textContent.trim() || "No description",
      rating: parseFloat(el.querySelector(".GwjAi .KFi5wf")?.textContent.trim()) || "No rating",
      reviews:
        parseInt(
          el
            .querySelector(".GwjAi .jdzyld")
            ?.textContent.trim()
            .replace(/[\(|\)|\s]/gm, "")
        ) || "No reviews",
    }))
  );
}

async function getThingsToDoResults() {
  const browser = await puppeteer.launch({
    headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);

  await page.waitForSelector("[type='text']");
  await page.click("[type='text']");
  await page.waitForTimeout(1000);
  await page.keyboard.type(searchQuery);
  await page.waitForTimeout(1000);
  await page.keyboard.press("Enter");
  await page.waitForSelector(".GtiGue button");
  await page.click(".GtiGue button");
  await page.waitForSelector(".f4hh3d");
  await page.waitForTimeout(2000);

  const options = Array.from(await page.$$(".iydyUc"));

  const places = {
    all: await getPlaces(page),
  };

  for (const option of options) {
    await option.click();
    await page.waitForSelector(".f4hh3d");
    await page.waitForTimeout(2000);
    const optionName = await option.$eval(".m1GHmf", (node) => node.textContent.trim());
    places[`${optionName}`] = await getPlaces(page);
  }

  await browser.close();

  return places;
}

getThingsToDoResults().then(console.log);

Preparation

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Process

We have a dedicated Web Scraping with CSS Selectors blog post at SerpApi if you want to know a little bit more about them.

The Gif below illustrates the approach of selecting different parts of the results using SelectorGadget.

Code explanation

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

Next, we "say" to puppeteer use StealthPlugin, write the search query and the search URL:

puppeteer.use(StealthPlugin());

const searchQuery = "Honolulu";

const URL = `https://www.google.com/travel/things-to-do`;

Next, we write a function to get places from the page:

async function getPlaces(page) {
  ...
}

In this function we'll use the next methods and properties to get the necessary information:

First, we need to scroll the page for loads all thumbnails. To do this we get the page scrollHeight, define scrollIterationCount (you need to greater this value if not all thumbnails are loaded), and then scroll the page in the for loop:

const height = await page.evaluate(() => document.querySelector(".zQTmif").scrollHeight);
const scrollIterationCount = 10;
for (let i = 0; i < scrollIterationCount; i++) {
  await page.mouse.wheel({ deltaY: height / scrollIterationCount });
  await page.waitForTimeout(2000);
}

Then, we get and return all places info from the page (using evaluate() method):

return await page.evaluate(() =>
  Array.from(document.querySelectorAll(".f4hh3d")).map((el) => ({
    thumbnail: el.querySelector(".kXlUEb img")?.getAttribute("src") || "No thumbnail",
    title: el.querySelector(".GwjAi .skFvHc")?.textContent.trim(),
    description: el.querySelector(".GwjAi .nFoFM")?.textContent.trim() || "No description",
    rating: parseFloat(el.querySelector(".GwjAi .KFi5wf")?.textContent.trim()) || "No rating",
    reviews:
      parseInt(
        el
          .querySelector(".GwjAi .jdzyld")
          ?.textContent.trim()
          .replace(/[\(|\)|\s]/gm, "") // this RegEx matches "(", or ")", or any white space
      ) || "No reviews",
  }))
);

Next, we write a function to control the browser, and get information from each category:

async function getThingsToDoResults() {
  ...
}

These options mean that we use headless mode and array with arguments which we use to allow the launch of the browser process in the online IDE. And then we open a new page:

const browser = await puppeteer.launch({
  headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();

Next, we change the default (30 sec) time for waiting for selectors to 60000 ms (1 min) for slow internet connection with .setDefaultNavigationTimeout() method, go to URL with .goto() method:

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);

Then we wait until "[type='text']" selector is load (waitForSelector() method), click on this input field and type searchQuery (keyboard.type() method), press "Enter" button (keyboard.press() method) and then click on "See all top sights" button:

await page.waitForSelector("[type='text']");
await page.click("[type='text']");
await page.waitForTimeout(1000);
await page.keyboard.type(searchQuery);
await page.waitForTimeout(1000);
await page.keyboard.press("Enter");
await page.waitForSelector(".GtiGue button");
await page.click(".GtiGue button");
await page.waitForSelector(".f4hh3d");
await page.waitForTimeout(2000);

Then we define the places object and add places information from the page to all key:

const places = {
  all: await getPlaces(page),
};

Next, we need to get all categories from the page and get all the places information from each category by clicking on each and setting to places object keys with the category name:

const categories = Array.from(await page.$$(".iydyUc"));

for (const category of categories) {
  await category.click();
  await page.waitForSelector(".f4hh3d");
  await page.waitForTimeout(2000);
  const categoryName = await category.$eval(".m1GHmf", (node) => node.textContent.trim());
  places[`${categoryName}`] = await getPlaces(page);
}

And finally, we close the browser, and return the received data:

await browser.close();

return places;

Now we can launch our parser:

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

Output

{
  "all": [
    {
      "thumbnail": "https://encrypted-tbn1.gstatic.com/licensed-image?q=tbn:ANd9GcSNARkYcqi7DBwaNx9w-qMSlFVL_nYNTuu0bX8zgIswYAjlyIx9oIpilLInYWdr7xWXGdy2zSTyhYnO_GjbBYhOJQ",
      "title": "Tonggs Beach",
      "description": "Surfing and beach",
      "rating": 4.4,
      "reviews": 68
    },
    {
      "thumbnail": "https://encrypted-tbn1.gstatic.com/images?q=tbn:ANd9GcRMnzB_-HjKVPLtoD-QSTeLiLbxb87JCCaKmiI_179MO1zj1uRo30CQ41icaJrOEihrQQYwXFpvojMpEg",
      "title": "Kaluahole Beach",
      "description": "Beach",
      "rating": 2.5,
      "reviews": 2
    },
    ...and other places
  ],
  "History": [
    {
      "thumbnail": "https://encrypted-tbn2.gstatic.com/licensed-image?q=tbn:ANd9GcRlsOO0zJJJhXHxJdoms3a0VSDHdTSOlARXlcyBI7THZ64LnuaSAuBdlvYYxliXdo8fO666Fu3QSisgG-cWt9pt-Q",
      "title": "Pearl Harbor Aviation Museum",
      "description": "Exhibits on WWII aviation in the Pacific",
      "rating": 4.6,
      "reviews": 4
    },
    {
      "thumbnail": "https://encrypted-tbn1.gstatic.com/licensed-image?q=tbn:ANd9GcSgKRnVx6y-cH0Jq-h64UDAc50iwHHMOaARxnQN8xH2n_CBGIMSgQM0QGTs_qZWY65VS0sOtmgLEN9rI87k03MQiA",
      "title": "Bishop Museum",
      "description": "Polynesian culture & natural history",
      "rating": 4.6,
      "reviews": 3
    },
    ...and other places
  ],
  ...and other categories
}

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Product Comparison App (JS Demo Project)

Mikhail Zub — Sat, 04 Feb 2023 14:43:18 +0000

Intro

The main idea for creating this eBay and Walmart dropshipping product research demo app was to show a practical example of one of the many options for using SerpApi.

This application could be improved to be an actual dropshipping tool (similar to ZikAnalytics for example) to research products to sell, and ability to compare the prices of certain products on eBay and Walmart to calculate profit and other possible criteria.

The backend is written in Node.js, the frontend in React. To create this app I was use the next npm packages:

How to run locally?

📌Note: If you don't want to run the app locally, you can skip this part.

You can download the application files to your computer and run locally:

Run backend

To run the backend you need to create .env file in the backend directory and write your API_KEY in it:

API_KEY="paste here your api key from https://serpapi.com/manage-api-key"

Then run the terminal and enter (you need to be installed Node.js):

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

npm i

And then:

npm start

Now your server is working. To check it go to http://localhost:4004 in your browser. If you see "Cannot GET /" everything is OK.

Run frontend

Let's start our frontend. First, you need to change the backend URL. To do this open src/requests.js and change baseURL to http://localhost:4004. Now your app will send requests to your local backend.

Next, run the terminal in the frontend directory and enter:

npm i

And then:

npm start

After the application starts you can go to http://localhost:3000 in your browser and you may see first screen:

How to use?

First, you need to enter a search query with a product name, choose where you want to sell a product and name match accuracy (default is more or equal to 50%) and click the "search" button.

📌Note: because when searching, the Levenshtein distance is used to compare the names of products, the search query must be similar to the full name of the product, otherwise the search will not return matches.

Then you need to choose the product you are interested in and click the "compare results" button:

And choose a product with a suitable profit and save it in "xlsx" format if you want:

How it works?

Because the frontend job is just to show you the information, the main part of the work makes on the backend. So I will try to explain to you how it works.

I'll skip the part where we create the server with express and focus on the part where we parse and process the results. If you don't know how to do it, here's a Hello world express.js example from official docs.

First, when the server receives a request on route "/store", it handles it with getResultsToChoose function:

//index.js
app.post("/store", getResultsToChoose);

In this function we get scraped results depending on targetStore, filter it and send it back to the frontend in the response:

//storeResultsHandler.js
export const getResultsToChoose = async (req, res) => {
  const { targetStore, query, accuracy } = req.body;
  let results;
  if (targetStore === "ebay") {
    const ebayResults = await getStoreResults(targetStore, query);
    results = ebayFiltering(query, accuracy, ebayResults);
  } else {
    const walmartResults = await getStoreResults(targetStore, query);
    results = walmartFiltering(query, accuracy, walmartResults);
  }
  res.status(200).json({ filteredResults: results });
};

getStoreResults is used to get the organic results from the store using Walmart Search Engine Results API and Ebay Search Engine Results API from SerpApi:

//serpApiHandler.js
import dotenv from "dotenv";
import { config, getJson } from "serpapi";

dotenv.config();
config.api_key = process.env.API_KEY; // your SerpApi private key from .env file

// make params to make a request to SerpApi depending on target store
const getParams = (store, query) => {
  switch (store) {
    case "ebay":
      return {
        _nkw: query,
      };
    case "walmart":
      return {
        query,
      };
    default:
      break;
  }
};

// get JSON with results and return organic_results array
const getStoreResults = async (store, query) => {
  const params = getParams(store, query);
  const results = await getJson(store, params);
  return results.organic_results;
};

export default getStoreResults;

After we get organic results (for example from eBay) we filter it, leaving only products which have a buy now price, are in new condition, and match our search query with Levenshtain distance method. And finally sort them in ascending order of price:

//storeResultsHandler.js
const ebayFiltering = (query, accuracy, results) => {
  return results
    ?.filter((result) => {
      const { title, condition, price } = result;
      const extractedPrice = price?.extracted;
      if (100 - levenshtein(query, title) >= accuracy && condition === "Brand New" && extractedPrice) {
        return true;
      } else return false;
    })
    ?.sort((a, b) => {
      if (a.price.extracted > b.price.extracted) return 1;
      else return -1;
    });
};

After receiving the results, the user selects those that suit him and again sends a request to the server on route "/comparison":

//index.js
app.post("/comparison", buildComparisonTable);

This time, our server, using SerpApi, searches and filters products on another site (if eBay was selected as the target site, then the search is performed on Walmart) for each product selected by the user:

//storeResultsHandler.js
export const buildComparisonTable = async (req, res) => {
  const { targetStore, selectedResults, accuracy } = req.body;
  let comparisonResults = [];
  for (const result of selectedResults) {
    const { title } = result;
    if (targetStore === "ebay") {
      const walmartResults = await getStoreResults("walmart", title);
      comparisonResults.push({ product: result, comparison: walmartFiltering(title, accuracy, walmartResults) });
    } else {
      const ebayResults = await getStoreResults("ebay", title);
      comparisonResults.push({ product: result, comparison: ebayFiltering(title, accuracy, ebayResults) });
    }
  }
  res.status(200).json({ comparisonResults });
};

The final result is returned to the user for the selection of products with the greatest benefit and the ability to save the selected products in the "xlsx" file:

Limitations & things to improve

Limitations:

You need to specify the name of the product as accurately as possible;
You need to choose the appropriate search results yourself;
Only two marketplaces are currently supported;
Ability to save results only in xslx format.

Things to improve:

Connect more marketplaces;
Create a PDF template to make saved results look prettier and more readable;
Use AI for result matching instead of levenshtein. Possibly match via images also.

Links

If you want other functionality added to this demo project or if you want to see some other projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Walmart Search with Nodejs

Mikhail Zub — Thu, 26 Jan 2023 18:19:33 +0000

What will be scraped

Full code

If you don't need an explanation, have a look at the full code example in the online IDE

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com
const resultsLimit = 40; // hardcoded limit for demonstration purpose

const engine = "walmart"; // search engine
const params = {
  query: "macnook pro", // Parameter defines the search query
  page: 1, // Value is used to get the items on a specific page
  device: "desktop", // Parameter defines the device to use to get the results
  store_id: "2280", //Store ID to filter the products by the specific store only
  //other parameters: https://serpapi.com/walmart-search-api#api-parameters
};

const getResults = async () => {
  const results = {
    fixedQuery: null,
    organicResults: [],
  };
  while (results.organicResults.length < resultsLimit) {
    const json = await getJson(engine, params);
    if (!results.fixedQuery) results.fixedQuery = json.search_information?.spelling_fix;
    if (json.organic_results) {
      results.organicResults.push(...json.organic_results);
      params.page += 1;
    } else break;
  }
  return results;
};

getResults().then((result) => console.dir(result, { depth: null }));

Why use Walmart Search Engine Results API from SerpApi?

Using API generally solves all or most problems that might get encountered while creating own parser or crawler. From webscraping perspective, our API can help to solve the most painful problems:

Bypass blocks from supported search engines by solving CAPTCHA or IP blocks.
No need to create a parser from scratch and maintain it.
Pay for proxies, and CAPTCHA solvers.
Don't need to use browser automation if there's a need to extract data in large amounts faster.

Head to the Playground for a live and interactive demo.

Preparation

First, we need to create a Node.js* project and add npm packages serpapi and dotenv.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i serpapi dotenv

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

SerpApi package is used to scrape and parse search engine results using SerpApi. Get search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay, and more.
dotenv package is a zero-dependency module that loads environment variables from a .env file into process.env.

Next, we need to add a top-level "type" field with a value of "module" in our package.json file to allow using ES6 modules in Node.JS:

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Code explanation

First, we need to import dotenv from dotenv library, and config and getJson from serpapi library:

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

Then, we apply some config. Call dotenv config() method, set your SerpApi Private API key to global config object, and how many results we want to receive (resultsLimit constant).

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com
const resultsLimit = 40; // hardcoded limit for demonstration purpose

dotenv.config() will read your .env file, parse the contents, assign it to process.env, and return an object with a parsed key containing the loaded content or an error key if it failed.
config.api_keyallows you declare a global api_key value by modifying the config object.

Next, we write search engine and write the necessary search parameters for making a request (get the full JSON list of supported Walmart Stores):

📌Note: I specifically made a mistake in the search query to demonstrate how Walmart Spell Check API works.

const engine = "walmart"; // search engine
const params = {
  query: "macnook pro", // Parameter defines the search query
  page: 1, // Value is used to get the items on a specific page
  device: "desktop", // Parameter defines the device to use to get the results
  store_id: "2280", //Store ID to filter the products by the specific store only
};

📌Note: Also see SerpApi Python demo project of extracting data from 500 Walmart stores and analyzing extracted data if you want to know more about scraping Walmart.

You can see all available parameters in the API documentation.

Next, we declare the function getResult that gets data from the page and return it:

const getResults = async () => {
  ...
};

In this function we need to declare an object two keys: fixedQuery is equal to null, and empty organicResults array, then and using while loop get json with results, add spelling_fix to the fixedQuery on the first iteration, and add organic_results to organicResults array (push() method) from each page and set next page index (to params.page value).

If there are no more results on the page or if the number of received results is more than reviewsLimit we stop the loop (using break) and return an array with results:

const results = {
  fixedQuery: null,
  organicResults: [],
};
while (results.organicResults.length < resultsLimit) {
  const json = await getJson(engine, params);
  if (!results.fixedQuery) results.fixedQuery = json.search_information?.spelling_fix;
  if (json.organic_results) {
    results.organicResults.push(...json.organic_results);
    params.page += 1;
  } else break;
}
return results;

And finally, we run the getResults function and print all the received information in the console with the console.dir method, which allows you to use an object with the necessary parameters to change default output options:

getResults().then((result) => console.dir(result, { depth: null }));

Output

{
   "fixedQuery":"macbook pro",
   "organicResults":[
      {
         "us_item_id":"121393924",
         "product_id":"18F5MJ3R95JG",
         "title":"Apple MacBook Air, 13.3-inch, Intel Core i5, 4GB RAM, Mac OS, 128GB SSD, Bundle: Black Case, Wireless Mouse, Bluetooth Headset - Silver",
         "thumbnail":"https://i5.walmartimages.com/asr/60e5ea72-ac15-4bdc-b112-572f76776e83.77df8900f9478a7a581dad9a6698ecd5.jpeg?odnHeight=180&odnWidth=180&odnBg=FFFFFF",
         "rating":3.5,
         "reviews":100,
         "seller_id":"F86EF73A620D4265AEE28E9FD77A4ED1",
         "seller_name":"Certified 2 Day Express",
         "fulfillment_badges":[
            "2-day shipping"
         ],
         "two_day_shipping":false,
         "out_of_stock":false,
         "sponsored":true,
         "muliple_options_available":false,
         "primary_offer":{
            "offer_id":"53A0316C100D4D1EBD2AD8753FC4FE25",
            "offer_price":349,
            "min_price":0
         },
         "price_per_unit":{
            "unit":"each",
            "amount":""
         },
         "product_page_url":"https://www.walmart.com/ip/Apple-MacBook-Air-13-3-inch-Intel-Core-i5-4GB-RAM-Mac-OS-128GB-SSD-Bundle-Black-Case-Wireless-Mouse-Bluetooth-Headset-Silver/121393924",
         "serpapi_product_page_url":"https://serpapi.com/search.json?device=desktop&engine=walmart_product&product_id=121393924"
      },
      ... and other results
   ]
}

Links

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping The Home Depot Product Info with Nodejs

Mikhail Zub — Thu, 12 Jan 2023 19:17:42 +0000

What will be scraped

Full code

If you don't need an explanation, have a look at the full code example in the online IDE

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com

const engine = "home_depot_product"; // search engine
const params = {
  product_id: "318783386", // The Home Depot identifier of a product
};

const getResults = async () => {
  const json = await getJson(engine, params);
  return json.product_results;
};

getResults().then((result) => console.dir(result, { depth: null }));

Why use The Home Depot Product API from SerpApi?

Using API generally solves all or most problems that might get encountered while creating own parser or crawler. From webscraping perspective, our API can help to solve the most painful problems:

Bypass blocks from supported search engines by solving CAPTCHA or IP blocks.
No need to create a parser from scratch and maintain it.
Pay for proxies, and CAPTCHA solvers.
Don't need to use browser automation if there's a need to extract data in large amounts faster.

Head to the Playground for a live and interactive demo.

Preparation

First, we need to create a Node.js* project and add npm packages serpapi and dotenv.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i serpapi dotenv

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

SerpApi package is used to scrape and parse search engine results using SerpApi. Get search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay, and more.
dotenv package is a zero-dependency module that loads environment variables from a .env file into process.env.

Next, we need to add a top-level "type" field with a value of "module" in our package.json file to allow using ES6 modules in Node.JS:

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Code explanation

First, we need to import dotenv from dotenv library, and config and getJson from serpapi library:

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

Then, we apply some config. Call dotenv config() method and set your SerpApi Private API key to global config object.

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com

dotenv.config() will read your .env file, parse the contents, assign it to process.env, and return an object with a parsed key containing the loaded content or an error key if it failed.
config.api_keyallows you declare a global api_key value by modifying the config object.

Next, we write search engine and write the necessary search parameters for making a request:

const engine = "home_depot_product"; // search engine
const params = {
  product_id: "318783386", // The Home Depot identifier of a product
};

You can use the next search params:

product_id The Home Depot identifier of a product.
delivery_zip ZIP Postal code. To filter the shipping products by a selected area.
store_id Store Id to filter the products by the specific store only.
no_cache parameter will force SerpApi to fetch the App Store Search results even if a cached version is already present. A cache is served only if the query and all parameters are exactly the same. Cache expires after 1h. Cached searches are free, and are not counted towards your searches per month. It can be set to false (default) to allow results from the cache, or true to disallow results from the cache. no_cache and async parameters should not be used together.
async parameter defines the way you want to submit your search to SerpApi. It can be set to false (default) to open an HTTP connection and keep it open until you got your search results, or true to just submit your search to SerpApi and retrieve them later. In this case, you'll need to use our Searches Archive API to retrieve your results. async and no_cache parameters should not be used together. async should not be used on accounts with Ludicrous Speed enabled.

Next, we declare the function getResult that gets data from the page and return it:

const getResults = async () => {
  ...
};

In this function we get json with results, and return product_results from the received json.

const json = await getJson(engine, params);
return json.product_results;

getResults().then((result) => console.dir(result, { depth: null }));

Output

{
   "product_id":"318783386",
   "title":"2500-Watt Recoil Start Ultra-Light Portable Gas and Propane Powered Dual Fuel Inverter Generator with CO Shield",
   "description":"The Champion Power Equipment 201122 2500-Watt Portable Inverter Generator with CO Shield is ideal for camping or tailgating. Weighing in at an ultralight 39 pounds, this model is one of the lightest 2500-watt inverters in the industry. Included are a covered 120V 20A household duplex outlet (5-20R) plus two handy 2.1A USB ports you can use to power your phone, laptop, or similar device. Just add oil (included 10W-30), and operate your Dual-Fuel generator right out of the box on gasoline or propane, and easily switch fuels with the fuel selector dial. When the 1.1-gallon tank of gasoline is full, the 79cc Champion engine produces 2500 starting watts and 1850 running watts and will run for up to 11.5 hours at 25% load. When using a 20-pound propane tank, it produces 2500 starting watts and 1665 running watts and will run for up to 34 hours at 25% load. CO Shield technology monitors the accumulation of carbon monoxide (CO), a poisonous gas produced by engine exhaust when the generator is running. If CO Shield detects unsafe elevated levels of CO gas, it automatically shuts off the engine. CO Shield is not a substitute for an indoor carbon monoxide alarm or for safe operation. DO NOT allow engine exhaust fumes to enter a confined area through windows, doors, vents or other openings. Generators must ALWAYS be used outdoors, far away from occupied buildings with engine exhaust pointed away from people and buildings. Meets the requirements of ANSI/PGMA G300-2018.",
   "link":"https://www.homedepot.com/p/Champion-Power-Equipment-2500-Watt-Recoil-Start-Ultra-Light-Portable-Gas-and-Propane-Powered-Dual-Fuel-Inverter-Generator-with-CO-Shield-201122/318783386",
   "upc":"817198025742",
   "model_number":"201122",
   "rating":"4.7821",
   "reviews":"1606",
   "price":899,
   "highlights":[
      "Ultra-quiet operation, Dual fuel flexibility (gas or propane)",
      "Engine oil and oil funnel included",
      "2500 starting watts and 1850 running watts"
   ],
   "brand":{
      "name":"Champion Power Equipment",
      "link":"https://www.homedepot.com/b/Sports-Outdoors-Tailgating-Gear-Tailgating-Portable-Gas-Power/Champion-Power-Equipment/N-5yc1vZcbwtZ9xs"
   },
   "images":[
      [
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_65.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_100.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_145.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_300.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_400.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_600.jpg",
         "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_1000.jpg"
      ],
      ...and other images
   ],
   "bullets":[
      "Need help with service or repair? Champion Power Equipment is available to help 24 hours a day, 7 days a week. Call us at 1-877-338-0999.",
      "Operate your 2500-Watt portable generator right out of the box on either gasoline or propane, plus at only 39 lbs., this inverter is 1 of the lightest 2500-Watt inverters in the industry",
      "With an ultra-quiet 53 dBA from 23 ft., enjoy 2500-Watt starting watt, 1850-Watt running watt and up to 11.5-hours run time on gasoline and 1665-Watt running watt and up to 34-hours on propane",
      "Optional, sold-separately parallel kit enables this inverter to connect with another 2500-Watt Champion inverter to double your output power",
      "Includes a 120-Volt 20 Amp household duplex outlet (5-20R) with clean electricity (less than 3% THD) and 2 convenient USB ports",
      "Includes 3-year limited warranty with free lifetime technical support from dedicated experts",
      "<br /><br /><center><img src=\"https://inlinecontent.thdstatic.com/28I/CHAMPION POWER EQUIPMENT/Champinline.jpg\"></center><br />"
   ],
   "info_and_guides":[
      {
         "title":"SDS",
         "link":"https://images.thdstatic.com/catalog/pdfImages/15/15de2a59-eb0b-443f-85dd-f6b773d618df.pdf"
      },
      {
         "title":"Replacement Part List",
         "link":"https://images.thdstatic.com/catalog/pdfImages/f9/f999c0c7-2df4-432d-86e1-fff36b2d3de9.pdf"
      },
      {
         "title":"Service and Repairs",
         "link":"https://images.thdstatic.com/catalog/pdfImages/70/70c2dc77-a1d9-4d45-9610-a723523171ab.pdf"
      },
      {
         "title":"Product Brochure",
         "link":"https://images.thdstatic.com/catalog/pdfImages/c7/c767f0c7-bd5e-4b78-ad4d-0bf2272ca158.pdf"
      },
      {
         "title":"Product Label in Spanish",
         "link":"https://images.thdstatic.com/catalog/pdfImages/be/be815184-f129-40b7-829d-34d3629168fe.pdf"
      },
      {
         "title":"Full Product Manual",
         "link":"https://images.thdstatic.com/catalog/pdfImages/a5/a531c5cb-34f4-4d3f-8992-86ab7b774b1c.pdf"
      }
   ],
   "specifications":[
      {
         "key":"Details",
         "value":[
            {
               "name":"Application",
               "value":"Campsite,Recreation,Tailgating"
            },
            {
               "name":"Built-in inverter",
               "value":"Yes"
            },
           ... and other details
         ]
      },
      {
         "key":"Warranty / Certifications",
         "value":[
            {
               "name":"Certifications and Listings",
               "value":"CARB Compliant,CARB Compliant,EPA Approved,EPA Approved,FCC Listed"
            },
            {
               "name":"Manufacturer Warranty",
               "value":"3 Year Limited Warranty"
            }
         ]
      },
      {
         "key":"Dimensions",
         "value":[
            {
               "name":"Product Height (in.)",
               "value":"17.7 in"
            },
            {
               "name":"Product Length (in.)",
               "value":"17.3 in"
            },
            {
               "name":"Product Width (in.)",
               "value":"11.5 in"
            }
         ]
      }
   ],
   "fulfillment":{
      "countity":1385,
      "store":"Bangor",
      "options":[
         {
            "type":"Ship to Home",
            "title":"Get it by",
            "arrival_time":[
               "Dec 26",
               "Dec 26"
            ],
            "bottom":"Free delivery"
         },
         {
            "type":"Schedule delivery",
            "title":"Not available for this item"
         },
         {
            "type":"Ship to store",
            "title":"Pickup",
            "arrival_time":[
               "Dec 22",
               "Dec 28"
            ],
            "bottom":"FREE"
         }
      ]
   }
}

How to extract products results and then extract product data

You can get products (with product_id) using our Web scraping The Home Depot Search with Nodejs blog post and then get product data with extracted product_id. Or you can use the simple code example, which shows you how to get products and get all these products info:

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com

const getResults = async () => {
  const productsInfo = [];
  const { products } = await getJson("home_depot", { q: "refrigerator" });
  for (const { product_id } of products) {
    const { product_results } = await getJson("home_depot_product", { product_id });
    productsInfo.push(product_results);
  }
  return productsInfo;
};

getResults().then((result) => console.dir(result, { depth: null }));

First, we get and destructure product from the received JSON with getJson function(we use "home_depot" search engine and params object with search query (q) "refrigerator").

Then, we use for...of loop, destructure product_id from each received product and get and destructure product_results from received JSON with getJson function(for now we use "home_depot_product" search engine and params object with product_id). And then we add the received results to the productsInfo array (using push() method).

Links

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping The Home Depot Search with Nodejs

Mikhail Zub — Tue, 10 Jan 2023 21:27:19 +0000

What will be scraped

Full code

If you don't need an explanation, have a look at the full code example in the online IDE

import dotenv from "dotenv";
import { config, getJson } from "serpapi";
import readline from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";

dotenv.config();
const rl = readline.createInterface({ input, output });
config.api_key = process.env.API_KEY; //your API key from serpapi.com

const engine = "home_depot"; // search engine
const resultsLimit = 40; // hardcoded limit for demonstration purpose
const params = {
  q: "inverter generatot", // Parameter defines the search query
  page: 1, // Value is used to get the items on a specific page
};

const getSearchRefinement = async () => {
  const preResults = await getJson(engine, params);
  let fixedQuery;
  if (preResults?.search_information?.spelling_fix) {
    fixedQuery = preResults.search_information.spelling_fix;
  }
  return { query: fixedQuery, filters: preResults.filters };
};

const applyNewSearchParams = async ({ query, filters }) => {
  if (query) {
    const answer = await rl.question(`Do you want to change the search query from "${params.q}" to "${query}"? y/n: `);
    if (answer.toLowerCase() === "y") {
      params.q = query;
      console.log(`Now the search query is: "${query}"`);
    } else {
      console.log(`The search query didn't change`);
    }
  }
  if (filters) {
    const appliedFilters = [];
    let tokens = "";
    for (const filter of filters) {
      const answer = await rl.question(`Do you want to apply some filter from "${filter.key}" category? y/n: `);
      if (answer.toLowerCase() === "y") {
        for (const filterValue of filter.value) {
          const answer = await rl.question(`Do you want to apply "${filterValue.name}" filter from "${filter.key}" category? y/n: `);
          if (answer.toLowerCase() === "y") {
            tokens += `${filterValue.value},`;
            appliedFilters.push(`${filter.key}: ${filterValue.name}`);
          }
        }
      }
    }
    rl.close();
    if (tokens) {
      params.hd_filter_tokens = tokens.slice(0, -1);
    }
  }
};

const getResults = async () => {
  const results = [];
  while (true) {
    const json = await getJson(engine, params);
    if (json.products) {
      results.push(...json.products);
      params.page += 1;
    } else break;
    if (results.length >= resultsLimit) break;
  }
  return results;
};

getSearchRefinement()
  .then(applyNewSearchParams)
  .then(getResults)
  .then((result) => console.dir(result, { depth: null }));

Why use The Home Depot API from SerpApi?

Using API generally solves all or most problems that might get encountered while creating own parser or crawler. From webscraping perspective, our API can help to solve the most painful problems:

Bypass blocks from supported search engines by solving CAPTCHA or IP blocks.
No need to create a parser from scratch and maintain it.
Pay for proxies, and CAPTCHA solvers.
Don't need to use browser automation if there's a need to extract data in large amounts faster.

Head to the Playground for a live and interactive demo.

Preparation

First, we need to create a Node.js* project and add npm packages serpapi and dotenv.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i serpapi dotenv

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

SerpApi package is used to scrape and parse search engine results using SerpApi. Get search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay, and more.
dotenv package is a zero-dependency module that loads environment variables from a .env file into process.env.

Next, we need to add a top-level "type" field with a value of "module" in our package.json file to allow using ES6 modules in Node.JS:

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Code explanation

First, we need to import dotenv from dotenv library, config and getJson from serpapi library, readline from readline Node.js built-in library, stdin and stdout (declare them as input and output) from process Node.js built-in library:

import dotenv from "dotenv";
import { config, getJson } from "serpapi";
import readline from "node:readline/promises";
import { stdin as input, stdout as output } from "node:process";

Then, we apply some config. Call dotenv config() method, call readline createInterface method with parameters input and output and set it to rl constant, set your SerpApi Private API key to global config object.

dotenv.config();
const rl = readline.createInterface({ input, output });
config.api_key = process.env.API_KEY; //your API key from serpapi.com

dotenv.config() will read your .env file, parse the contents, assign it to process.env, and return an object with a parsed key containing the loaded content or an error key if it failed.
readline.createInterface() creates a new readlinePromises.Interface instance. Once the readlinePromises.Interface instance is created, the most common case is to listen for the 'line' event.
config.api_keyallows you declare a global api_key value by modifying the config object.

Next, we write the necessary search parameters. We define search engine, how many results we want to receive (resultsLimit constant) and params object with q and page parameters for making a request:

📌Note: I specifically made a mistake in the search query to demonstrate how The Home Depot Spell Check API works.

const engine = "home_depot"; // search engine
const resultsLimit = 40; // hardcoded limit for demonstration purpose
const params = {
  q: "inverter generatot", // Parameter defines the search query
  page: 1, // Value is used to get the items on a specific page
};

You can use the next search params:

q parameter defines the search query. You can use anything that you would use in a regular The Home Depot search.
hd_sort parameter defines results sorted by diferent options. It can be set to: top_sellers (Top Sellers), price_low_to_high (Price Low to High), price_high_to_low (Price High to Low), top_rated (Top Rated), best_match (Best Match).
hd_filter_tokens used to pass filter tokens divided by comma. Filter tokens can be obtained from API response.
delivery_zip ZIP Postal code. To filter the shipping products by a selected area.
store_id store Id to filter the products by the specific store only.
lowerbound defines lower bound for price in USD.
upperbound defines upper bound for price in USD.
nao defines offset for products result. A single page contains 24 products. First page offset is 0, second -> 24, third -> 48 and so on.
page value is used to get the items on a specific page. (e.g., 1 (default) is the first page of results, 2 is the 2nd page of results, 3 is the 3rd page of results, etc.).
ps determines the number of items per page. There are scenarios where Home depot overrides the ps value. By default Home depot returns 24 results.
no_cache parameter will force SerpApi to fetch the App Store Search results even if a cached version is already present. A cache is served only if the query and all parameters are exactly the same. Cache expires after 1h. Cached searches are free, and are not counted towards your searches per month. It can be set to false (default) to allow results from the cache, or true to disallow results from the cache. no_cache and async parameters should not be used together.
async parameter defines the way you want to submit your search to SerpApi. It can be set to false (default) to open an HTTP connection and keep it open until you got your search results, or true to just submit your search to SerpApi and retrieve them later. In this case, you'll need to use our Searches Archive API to retrieve your results. async and no_cache parameters should not be used together. async should not be used on accounts with Ludicrous Speed enabled.

Next, we declare the function getSearchRefinement that gets preliminary results. In this function we return available filters for this search and check if preResults has spelling_fix, set it to fixedQuery, and return it:

const getSearchRefinement = async () => {
  const preResults = await getJson(engine, params);
  let fixedQuery;
  if (preResults?.search_information?.spelling_fix) {
    fixedQuery = preResults.search_information.spelling_fix;
  }
  return { query: fixedQuery, filters: preResults.filters };
};

Next, we declare the function applyNewSearchParams which allows setting fixed query and filters. We need to destructure function arguments object to query and filters:

const applyNewSearchParams = async ({ query, filters }) => {
  ...
};

In this function, we need to check if query is present, and after that, we print a question about the change search query in the console (question() method) and wait for the answer from the user. If the user's answer is 'y' we set a new search query to the params object:

if (query) {
  const answer = await rl.question(`Do you want to change the search query from "${params.q}" to "${query}"? y/n: `);
  if (answer.toLowerCase() === "y") {
    params.q = query;
    console.log(`Now the search query is: "${query}"`);
  } else {
    console.log(`The search query didn't change`);
  }
}

Next, we need to ask the same questions about applying received filters, by running each filter result using for...of loop:

if (filters) {
  const appliedFilters = [];
  let tokens = "";
  for (const filter of filters) {
    const answer = await rl.question(`Do you want to apply some filter from "${filter.key}" category? y/n: `);
    if (answer.toLowerCase() === "y") {
      for (const filterValue of filter.value) {
        const answer = await rl.question(`Do you want to apply "${filterValue.name}" filter from "${filter.key}" category? y/n: `);
        if (answer.toLowerCase() === "y") {
          tokens += `${filterValue.value},`;
          appliedFilters.push(`${filter.key}: ${filterValue.name}`);
        }
      }
    }
  }
  ...
}

Next, we close readline stream (rl.close() method) and if the user chooses some filter, remove the comma at the end of tokens string and set it to the hd_filter_tokens value in the params object:

rl.close();
if (tokens) {
  params.hd_filter_tokens = tokens.slice(0, -1);
}

Next, we declare the function getResults that gets results from all pages (using pagination) and return it:

const getResults = async () => {
  ...
};

In this function we need to declare an empty results array, then using while loop to get json with results, add products from each page and set next page index (to params.page value).

If there are no more results on the page or if the number of received results is more than resultsLimit we stop the loop (using break) and return an array with results:

const results = [];
while (true) {
  const json = await getJson(engine, params);
  if (json.products) {
    results.push(...json.products);
    params.page += 1;
  } else break;
  if (results.length >= resultsLimit) break;
}
return results;

And finally, run the getSearchRefinement function and using Promise chaining run applyNewSearchParams and getResults functions. Then we print all the received information in the console with the console.dir method, which allows you to use an object with the necessary parameters to change default output options:

getSearchRefinement()
  .then(applyNewSearchParams)
  .then(getResults)
  .then((result) => console.dir(result, { depth: null }));

Output

[
  {
    "position": 1,
    "product_id": "318783386",
    "title": "2500-Watt Recoil Start Ultra-Light Portable Gas and Propane Powered Dual Fuel Inverter Generator with CO Shield",
    "thumbnails": [
      [
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_65.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_100.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_145.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_300.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_400.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_600.jpg",
        "https://images.thdstatic.com/productImages/95951f39-703a-4efc-b12b-ad3a3276e73c/svn/champion-power-equipment-inverter-generators-201122-64_1000.jpg"
      ],
      [
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_65.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_100.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_145.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_300.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_400.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_600.jpg",
        "https://images.thdstatic.com/productImages/27667698-e4fc-488f-b713-7f51e83d5b5b/svn/champion-power-equipment-inverter-generators-201122-e4_1000.jpg"
      ]
    ],
    "link": "https://www.homedepot.com/p/Champion-Power-Equipment-2500-Watt-Recoil-Start-Ultra-Light-Portable-Gas-and-Propane-Powered-Dual-Fuel-Inverter-Generator-with-CO-Shield-201122/318783386",
    "serpapi_link": "https://serpapi.com/search.json?delivery_zip=04401&engine=home_depot_product&product_id=318783386&store_id=2414",
    "model_number": "201122",
    "brand": "Champion Power Equipment",
    "collection": "https://www.homedepot.com/collection/Outdoors/Champion-Inverter-Generators-Accessories-Collection/Family-311405669?omsid=318783386",
    "rating": 4.7821,
    "reviews": 1606,
    "price": 899,
    "delivery": {
      "free": true,
      "free_delivery_threshold": false
    },
    "pickup": {
      "free_ship_to_store": true
    }
  }
]

Links

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Apple App Store Product Info And Reviews with Nodejs

Mikhail Zub — Thu, 05 Jan 2023 17:16:38 +0000

What will be scraped

Full code

If you don't need an explanation, have a look at the full code example in the online IDE

import dotenv from "dotenv";
dotenv.config();
import { getJson } from "serpapi";

const getSearchParams = (searchType) => {
  const isProduct = searchType === "product";
  const reviewsLimit = 10; // hardcoded limit for demonstration purpose
  const engine = isProduct ? "apple_product" : "apple_reviews"; // search engine
  const params = {
    api_key: process.env.API_KEY, //your API key from serpapi.com
    product_id: "1507782672", // Parameter defines the ID of a product you want to get the reviews for
    country: "us", // Parameter defines the country to use for the search
    type: isProduct ? "app" : undefined, // Parameter defines the type of Apple Product to get the product page of
    page: isProduct ? undefined : 1, // Parameter is used to get the items on a specific page
    sort: isProduct ? undefined : "mostrecent", // Parameter is used for sorting reviews
  };
  return { engine, params, reviewsLimit };
};

const getProductInfo = async () => {
  const { engine, params } = getSearchParams("product");
  const json = await getJson(engine, params);
  delete json.search_metadata;
  delete json.search_parameters;
  delete json.search_information;
  return json;
};

const getReviews = async () => {
  const reviews = [];
  const { engine, params, reviewsLimit } = getSearchParams();
  while (true) {
    const json = await getJson(engine, params);
    if (json.reviews) {
      reviews.push(...json.reviews);
      params.page += 1;
    } else break;
    if (reviews.length >= reviewsLimit) break;
  }
  return reviews;
};

const getResults = async () => {
  return { productInfo: await getProductInfo(), reviews: await getReviews() };
};

getResults().then((result) => console.dir(result, { depth: null }));

Why use Apple Product Page Scraper and Apple App Store Reviews Scraper APIs from SerpApi?

Using API generally solves all or most problems that might get encountered while creating own parser or crawler. From webscraping perspective, our API can help to solve the most painful problems:

Bypass blocks from supported search engines by solving CAPTCHA or IP blocks.
No need to create a parser from scratch and maintain it.
Pay for proxies, and CAPTCHA solvers.
Don't need to use browser automation if there's a need to extract data in large amounts faster.

Head to the Apple Product Page playground and Apple App Store Reviews playground for a live and interactive demo.

Preparation

First, we need to create a Node.js* project and add npm packages serpapi and dotenv.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i serpapi dotenv

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

SerpApi package is used to scrape and parse search engine results using SerpApi. Get search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay, and more.
dotenv package is a zero-dependency module that loads environment variables from a .env file into process.env.

Next, we need to add a top-level "type" field with a value of "module" in our package.json file to allow using ES6 modules in Node.JS:

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Code explanation

First, we need to import dotenv from dotenv library and call config() method, then import getJson from serpapi library:

import dotenv from "dotenv";
dotenv.config();
import { getJson } from "serpapi";

config() will read your .env file, parse the contents, assign it to process.env, and return an Object with a parsed key containing the loaded content or an error key if it failed.
getJson() allows you to get a JSON response based on search parameters.

Next, we write getSearchParams function, to make the necessary search parameters for two different APIs. In this function, we define and set isProduct constant depending on the searchType argument.

Next, we define and return different search parameters for Product Page API and Reviews API: search engine; how many reviews we want to receive (reviewsLimit constant); search parameters for making a request:

const getSearchParams = (searchType) => {
  const isProduct = searchType === "product";
  const reviewsLimit = 10; // hardcoded limit for demonstration purpose
  const engine = isProduct ? "apple_product" : "apple_reviews"; // search engine
  const params = {
    api_key: process.env.API_KEY, //your API key from serpapi.com
    product_id: "1507782672", // Parameter defines the ID of a product you want to get the reviews for
    country: "us", // Parameter defines the country to use for the search
    type: isProduct ? "app" : undefined, // Parameter defines the type of Apple Product to get the product page of
    page: isProduct ? undefined : 1, // Parameter is used to get the items on a specific page
    sort: isProduct ? undefined : "mostrecent", // Parameter is used for sorting reviews
  };
  return { engine, params, reviewsLimit };
};

When we run this function, we receive different search parameters for:

Product Page API:
Reviews API:

You can use the next search params:

Common params:

api_key parameter defines the SerpApi private key to use.
product_id parameter defines the ID of a product you want to get the reviews for. You can get the ID of a product from our Web scraping Apple App Store Search with Nodejs blog post. You can also get it from the URL of the app. For example product_id of "https://apps.apple.com/us/app/the-great-coffee-app/id534220544", is the long numerical value that comes after "id", 534220544.
country parameter defines the country to use for the search. It's a two-letter country code. (e.g., us (default) for the United States, uk for United Kingdom, or fr for France). Head to the Apple Regions for a full list of supported Apple Regions.
no_cache parameter will force SerpApi to fetch the App Store Search results even if a cached version is already present. A cache is served only if the query and all parameters are exactly the same. Cache expires after 1h. Cached searches are free, and are not counted towards your searches per month. It can be set to false (default) to allow results from the cache, or true to disallow results from the cache. no_cache and async parameters should not be used together.
async parameter defines the way you want to submit your search to SerpApi. It can be set to false (default) to open an HTTP connection and keep it open until you got your search results, or true to just submit your search to SerpApi and retrieve them later. In this case, you'll need to use our Searches Archive API to retrieve your results. async and no_cache parameters should not be used together. async should not be used on accounts with Ludicrous Speed enabled.

Product Page params:

type parameter defines the type of Apple Product to get the product page of. It defaults to app.

Reviews params:

page parameter is used to get the items on a specific page. (e.g., 1 (default) is the first page of results, 2 is the 2nd page of results, 3 is the 3rd page of results, etc.).
sort parameter is used for sorting reviews. It can be set to: mostrecent (Most recent (default)) or mosthelpful (Most helpful).

Next, we declare the function getProductInfo that gets all product information from the page and returns it. In this function we receive and destructure engine and params from getSearchParams function with "product" argument. Next, we get json with results, delete unnecessary keys, and return it:

const getProductInfo = async () => {
  const { engine, params } = getSearchParams("product");
  const json = await getJson(engine, params);
  delete json.search_metadata;
  delete json.search_parameters;
  delete json.search_information;
  return json;
};

Next, we declare the function getReviews that gets reviews results from all pages (using pagination) and return it:

const getReviews = async () => {
  ...
};

In this function we need to declare an empty reviews array, receive and destructure engine, params and reviewsLimit from getSearchParams function without arguments, then and using while loop get json with results, add reviews from each page and set next page index (to params.page value).

If there are no more results on the page or if the number of received results is more than reviewsLimit we stop the loop (using break) and return an array with results:

const reviews = [];
const { engine, params, reviewsLimit } = getSearchParams();
while (true) {
  const json = await getJson(engine, params);
  if (json.reviews) {
    reviews.push(...json.reviews);
    params.page += 1;
  } else break;
  if (reviews.length >= reviewsLimit) break;
}
return reviews;

And finally, we declare and run the getResults function, in which we make an object with results from getProductInfo and getReviews functions. Then we print all the received information in the console with the console.dir method, which allows you to use an object with the necessary parameters to change default output options:

const getResults = async () => {
  return { productInfo: await getProductInfo(), reviews: await getReviews() };
};

getResults().then((result) => console.dir(result, { depth: null }));

Output

{
   "productInfo":{
      "title":"Pixea",
      "snippet":"The invisible image viewer",
      "id":"1507782672",
      "age_rating":"4+",
      "developer":{
         "name":"ImageTasks Inc",
         "link":"https://apps.apple.com/us/developer/imagetasks-inc/id450316587"
      },
      "rating":4.6,
      "rating_count":"594 Ratings",
      "price":"Free",
      "logo":"https://is3-ssl.mzstatic.com/image/thumb/Purple118/v4/f6/93/b6/f693b68f-9b14-3689-7521-c19a83fb0d88/AppIcon-1x_U007emarketing-85-220-6.png/320x0w.webp",
      "mac_screenshots":[
         "https://is3-ssl.mzstatic.com/image/thumb/PurpleSource124/v4/b1/8c/fb/b18cfb80-cb5c-d67d-2edc-ee1f6666e012/35b8d5a7-b493-4a80-bdbd-3e9d564601dd_Pixea-1.jpg/643x0w.webp",
         "https://is1-ssl.mzstatic.com/image/thumb/PurpleSource124/v4/96/08/83/9608834d-3d2b-5c0b-570c-f022407ff5cc/1836573e-1b6a-421c-b654-6ae2f915d755_Pixea-2.jpg/643x0w.webp",
         "https://is1-ssl.mzstatic.com/image/thumb/PurpleSource124/v4/58/fd/db/58fddb5d-9480-2536-8679-92d6b067d285/98e22b63-1575-4ee6-b08d-343b9e0474ea_Pixea-3.jpg/643x0w.webp",
         "https://is2-ssl.mzstatic.com/image/thumb/PurpleSource124/v4/c3/f3/f3/c3f3f3b5-deb0-4b58-4afc-79073373b7b9/28f51f38-bc59-4a61-a5a1-bff553838267_Pixea-4.jpg/643x0w.webp"
      ],
      "description":"Pixea is an image viewer for macOS with a nice minimal modern user interface. Pixea works great with JPEG, HEIC, PSD, RAW, WEBP, PNG, GIF, and many other formats. Provides basic image processing, including flip and rotate, shows a color histogram, EXIF, and other information. Supports keyboard shortcuts and trackpad gestures. Shows images inside archives, without extracting them.Supported formats:JPEG, HEIC, GIF, PNG, TIFF, Photoshop (PSD), BMP, Fax images, macOS and Windows icons, Radiance images, Google's WebP. RAW formats: Leica DNG and RAW, Sony ARW, Olympus ORF, Minolta MRW, Nikon NEF, Fuji RAF, Canon CR2 and CRW, Hasselblad 3FR. Sketch files (preview only). ZIP-archives.Export formats:JPEG, JPEG-2000, PNG, TIFF, BMP.Found a bug? Have a suggestion? Please, send it to support@imagetasks.comFollow us on Twitter @imagetasks!",
      "version_history":[
         {
            "release_version":"1.4",
            "release_notes":"- New icon- macOS Big Sur support- Universal Binary- Bug fixes and improvements",
            "release_date":"2020-11-09"
         },
        ... and other versions
      ],
      "ratings_and_reviews":{
         "rating_percentage":{
            "5_star":"76%",
            "4_star":"14%",
            "3_star":"4%",
            "2_star":"2%",
            "1_star":"3%"
         },
         "review_examples":[
            {
               "rating":"5 out of 5",
               "username":"MyrtleBlink182",
               "review_date":"01/18/2022",
               "review_title":"Full-Screen Perfection",
               "review_text":"This photo-viewer is by far the best in the biz. I thoroughly enjoy viewing photos with it. I tried a couple of others out, but this one is exactly what I was looking for. There is no dead space or any extra design baggage when viewing photos. Pixea knocks it out of the park keeping the design minimalistic while ensuring the functionality is through the roof"
            },
            ... and other reviews examples
         ]
      },
      "privacy":{
         "description":"The developer, ImageTasks Inc, indicated that the app’s privacy practices may include handling of data as described below. For more information, see the developer’s privacy policy.",
         "privacy_policy_link":"https://www.imagetasks.com/Pixea-policy.txt",
         "cards":[
            {
               "title":"Data Not Collected",
               "description":"The developer does not collect any data from this app."
            }
         ],
         "sidenote":"Privacy practices may vary, for example, based on the features you use or your age. Learn More",
         "learn_more_link":"https://apps.apple.com/story/id1538632801"
      },
      "information":{
         "seller":"ImageTasks Inc",
         "price":"Free",
         "size":"5.8 MB",
         "categories":[
            "Photo & Video"
         ],
         "compatibility":[
            {
               "device":"Mac",
               "requirement":"Requires macOS 10.12 or later."
            }
         ],
         "supported_languages":[
            "English"
         ],
         "age_rating":{
            "rating":"4+"
         },
         "copyright":"Copyright © 2020 Andrey Tsarkov. All rights reserved.",
         "developer_website":"https://www.imagetasks.com",
         "app_support_link":"https://www.imagetasks.com/pixea",
         "privacy_policy_link":"https://www.imagetasks.com/Pixea-policy.txt"
      },
      "more_by_this_developer":{
         "apps":[
            {
               "logo":"https://is3-ssl.mzstatic.com/image/thumb/Purple118/v4/f6/93/b6/f693b68f-9b14-3689-7521-c19a83fb0d88/AppIcon-1x_U007emarketing-85-220-6.png/320x0w.webp",
               "link":"https://apps.apple.com/us/app/istatistica/id1126874522",
               "serpapi_link":"https://serpapi.com/search.json?country=us&engine=apple_product&product_id=1507782672&type=app",
               "name":"iStatistica",
               "category":"Utilities"
            },
            ... and other apps
         ],
         "result_type":"Full",
         "see_all_link":"https://apps.apple.com/us/app/id1507782672#see-all/developer-other-apps"
      }
   },
   "reviews":[
      {
         "position":1,
         "id":"9332275235",
         "title":"Doesn't respect aspect ratios",
         "text":"Seemingly no way to maintain the aspect ratio of an image. It always wants to fill the photo to the window size, no matter what sizing options you pick. How useless is that?",
         "rating":3,
         "review_date":"2022-11-26 13:29:43 UTC",
         "author":{
            "name":"soren121",
            "link":"https://itunes.apple.com/us/reviews/id33706024"
         }
      },
      ... and other reviews
   ]
}

Links

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞

Web scraping Google Lens Results with Nodejs

Mikhail Zub — Wed, 04 Jan 2023 17:09:38 +0000

What will be scraped

Why use Google Lens API from SerpApi?

Using API generally solves all or most problems that might get encountered while creating own parser or crawler. From webscraping perspective, our API can help to solve the most painful problems:

Bypass blocks from supported search engines by solving CAPTCHA or IP blocks.
No need to create a parser from scratch and maintain it.
Pay for proxies, and CAPTCHA solvers.
Don't need to use browser automation if there's a need to extract data in large amounts faster.

Head to the Playground for a live and interactive demo.

Preparation

First, we need to create a Node.js* project and add npm packages serpapi and dotenv.

To do this, in the directory with our project, open the command line and enter:

$ npm init -y

And then:

$ npm i serpapi dotenv

*If you don't have Node.js installed, you can download it from nodejs.org and follow the installation documentation.

SerpApi package is used to scrape and parse search engine results using SerpApi. Get search results from Google, Bing, Baidu, Yandex, Yahoo, Home Depot, eBay, and more.
dotenv package is a zero-dependency module that loads environment variables from a .env file into process.env.

Next, we need to add a top-level "type" field with a value of "module" in our package.json file to allow using ES6 modules in Node.JS:

For now, we complete the setup Node.JS environment for our project and move to the step-by-step code explanation.

Code explanation

First, we need to import dotenv from dotenv library, and config and getJson from serpapi library:

import dotenv from "dotenv";
import { config, getJson } from "serpapi";

Then, we apply some config. Call dotenv config() method and set your SerpApi Private API key to global config object.

dotenv.config();
config.api_key = process.env.API_KEY; //your API key from serpapi.com

dotenv.config() will read your .env file, parse the contents, assign it to process.env, and return an object with a parsed key containing the loaded content or an error key if it failed.
config.api_keyallows you declare a global api_key value by modifying the config object.

Next, we write search engine and write the necessary search parameters for making a request:

const engine = "google_lens"; // search engine
const params = {
  url: "https://user-images.githubusercontent.com/64033139/209465038-010d1e56-16db-41d6-9769-6707d6f57111.png", //Parameter defines the URL of an image to perform the Google Lens search
  hl: "en", // Parameter defines the language to use for the Google search
};

You can use the next search params:

url Parameter defines the URL of an image to perform the Google Lens search.
hl Parameter defines the language to use for the Google Lens search. It's a two-letter language code. (e.g., en for English, es for Spanish, or fr for French). Head to the Google languages page for a full list of supported Google languages.
no_cache parameter will force SerpApi to fetch the App Store Search results even if a cached version is already present. A cache is served only if the query and all parameters are exactly the same. Cache expires after 1h. Cached searches are free, and are not counted towards your searches per month. It can be set to false (default) to allow results from the cache, or true to disallow results from the cache. no_cache and async parameters should not be used together.
async parameter defines the way you want to submit your search to SerpApi. It can be set to false (default) to open an HTTP connection and keep it open until you got your search results, or true to just submit your search to SerpApi and retrieve them later. In this case, you'll need to use our Searches Archive API to retrieve your results. async and no_cache parameters should not be used together. async should not be used on accounts with Ludicrous Speed enabled.

Next, we declare the function getResult that gets data from the page and return it:

const getResults = async () => {
  ...
};

In this function, we need to get json with results,delete unnecessary keys from JSON, and return it.

const results = await getJson(engine, params);
delete results.search_metadata;
delete results.search_parameters;
return results;

getResults().then((result) => console.dir(result, { depth: null }));

Output

{
   "reverse_image_search":{
      "link":"https://www.google.com/search?tbs=sbi:AMhZZitWp1e8SyEO5dl4qkTCxU70zIpDW4BZqoAlVXqVRYoNA58nc3GQXbry3Wfl6SnzaHZ-xCdVfXXvQmYBmRz2VytmMNasi8-oL3MaHsJM80etUlB7YESZvKTW3gD6f49AdV9TV5A-Sm-Nl9p6tDjuCTPPO6XuZA"
   },
   "text_results":[
      {
         "text":"18",
         "link":"https://www.google.com/search?q=18&hl=en",
         "serpapi_link":"https://serpapi.com/search.json?device=desktop&engine=google&google_domain=google.com&hl=en&q=18"
      },
      {
         "text":"M",
         "link":"https://www.google.com/search?q=M&hl=en",
         "serpapi_link":"https://serpapi.com/search.json?device=desktop&engine=google&google_domain=google.com&hl=en&q=M"
      }
   ],
   "knowledge_graph":[
      {
         "title":"BMW i8",
         "subtitle":"Sports car",
         "link":"https://www.google.com/search?q=BMW+i8&kgmid=/m/080607h&hl=en&gl=US",
         "more_images":{
            "link":"https://www.google.com/search?q=BMW+i8&kgmid=/m/080607h&ved=0EOTpBwgAKAAwAA&source=.lens.button&tbm=isch&hl=en&gl=US",
            "serpapi_link":"https://serpapi.com/search.json?device=desktop&engine=google&gl=US&google_domain=google.com&hl=en&q=BMW+i8&tbm=isch"
         },
         "thumbnail":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH",
         "images":[
            {
               "title":"Image #1 for BMW i8",
               "source":"https://www.youtube.com/watch?v=fOXuBey9xo8",
               "link":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH",
               "size":{
                  "width":300,
                  "height":168
               }
            },
            {
               "title":"Image #2 for BMW i8",
               "source":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSLMCx-TtUdq5kmOhdczYL-PWIgaWtUntnHVAJWbbo7vin_obsM",
               "link":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS8sXyFSX0NjlLBQcQbJAC85f-Aw9WIEX2D-Y3by-C7pd23R3C2",
               "size":{
                  "width":276,
                  "height":182
               }
            }
         ]
      },
        ... and other knowledge graph results
   ],
   "visual_matches":[
      {
         "position":1,
         "title":"The 2023 BMW i8 M (All we know so far) - YouTube",
         "link":"https://www.youtube.com/watch?v=fOXuBey9xo8",
         "source":"youtube.com",
         "source_icon":"https://encrypted-tbn0.gstatic.com/favicon-tbn?q=tbn:ANd9GcQHQslzHLLyLc_qne5jxn7JocidlmUPyegZ8ojX3WVlorFk8BxW9a3vJWjDzN99UHVTqSaNBj_-6XykhxuVQfF3Ye7xScSWSuc2QHXi0a12CkVwBmo",
         "thumbnail":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH"
      },
        ... and other visual matches results
   ]
}

DIY solution

This section is to show the comparison between our solution and the DIY solution.

If you don't need an explanation, have a look at the full code example in the online IDE

import puppeteer from "puppeteer-extra";
import StealthPlugin from "puppeteer-extra-plugin-stealth";

puppeteer.use(StealthPlugin());

const searchParams = {
  imageUrl: "https://user-images.githubusercontent.com/64033139/209465038-010d1e56-16db-41d6-9769-6707d6f57111.png", //Parameter defines the URL of an image to perform the Google Lens search
  hl: "en", //Parameter defines the language to use for the Google search
};

const URL = `https://lens.google.com/uploadbyurl?url=${searchParams.imageUrl}&hl=${searchParams.hl}&gl=${searchParams.gl}`;

async function getResultsFromPage(page) {
  const reverseImageSearch = await page.evaluate(() => ({ link: document.querySelector(".z3qvzf .WpHeLc").getAttribute("href") }));
  const knowledgeGraphItems = await page.$$(".sHc5hf > div");
  const knowledgeGraph = [];
  for (const item of knowledgeGraphItems) {
    await item.click();
    await page.waitForTimeout(2000);
    const thumbnail = await item.$eval(".FH8DCc", (node) => node.getAttribute("src"));
    knowledgeGraph.push(
      await page.evaluate(
        (thumbnail) => ({
          title: document.querySelector(".DeMn2d").textContent,
          subtitle: document.querySelector(".XNTym").textContent,
          link: document.querySelector(".wNPKTe a").getAttribute("href"),
          moreImagesLink: document.querySelector(".XkkoHf + div .Tc2PU > a").getAttribute("href"),
          thumbnail,
          images: Array.from(document.querySelectorAll(".XkkoHf + div .Tc2PU .Y02Gld a")).map((el) => ({
            title: el.getAttribute("aria-label"),
            source: el.getAttribute("href"),
            link: el.querySelector("img").getAttribute("src"),
          })),
        }),
        thumbnail
      )
    );
  }
  const height = await page.evaluate(() => document.querySelector(".jXKZBd").scrollHeight);
  const scrollIterationCount = 5;
  for (let i = 0; i < scrollIterationCount; i++) {
    await page.mouse.wheel({ deltaY: height / scrollIterationCount });
    await page.waitForTimeout(2000);
  }
  const visualMatches = await page.evaluate(() => {
    return Array.from(document.querySelectorAll(".Vd9M6 a")).map((el) => ({
      title: el.getAttribute("aria-label"),
      link: el.getAttribute("href"),
      source: el.querySelector(".PlAMyb span").textContent,
      sourceIcon: el.querySelector(".PlAMyb img").getAttribute("src"),
      thumbnail: el.querySelector(".Me0cf img").getAttribute("src"),
    }));
  });
  await page.click("#text");
  await page.waitForTimeout(2000);
  const textResultsItems = await page.$$(".WfGljd[aria-label]");
  const textResults = [];
  for (const item of textResultsItems) {
    await page.click("#text");
    await page.waitForTimeout(2000);
    await item.click();
    await page.waitForTimeout(2000);
    textResults.push(
      await page.evaluate(() => ({
        text: document.querySelector(".C3t4Ke").textContent,
        link: document.querySelector("a[aria-label=Search]").getAttribute("href"),
      }))
    );
  }
  return { reverseImageSearch, textResults, knowledgeGraph, visualMatches };
}

async function getLensResults() {
  const browser = await puppeteer.launch({
    headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
    args: ["--no-sandbox", "--disable-setuid-sandbox"],
  });

  const page = await browser.newPage();

  await page.setDefaultNavigationTimeout(60000);
  await page.goto(URL);

  const results = await getResultsFromPage(page);

  await browser.close();

  return results;
}

getLensResults().then((result) => console.dir(result, { depth: null }));

Preparation

First, we need to add npm packages puppeteer, puppeteer-extra and puppeteer-extra-plugin-stealth to control Chromium (or Chrome, or Firefox, but now we work only with Chromium which is used by default) over the DevTools Protocol in headless or non-headless mode.

To do this, in the directory with our project, open the command line and enter:

$ npm i puppeteer puppeteer-extra puppeteer-extra-plugin-stealth

Process

We have a dedicated Web Scraping with CSS Selectors blog post at SerpApi if you want to know a little bit more about them.

The Gif below illustrates the approach of selecting different parts of the results using SelectorGadget.

Code explanation

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");

Next, we "say" to puppeteer use StealthPlugin, write the search parameters and the search URL:

puppeteer.use(StealthPlugin());

const searchParams = {
  imageUrl: "https://user-images.githubusercontent.com/64033139/209465038-010d1e56-16db-41d6-9769-6707d6f57111.png", //Parameter defines the URL of an image to perform the Google Lens search
  hl: "en", //Parameter defines the language to use for the Google search
};

const URL = `https://lens.google.com/uploadbyurl?url=${searchParams.imageUrl}&hl=${searchParams.hl}&gl=${searchParams.gl}`;

Next, we write a function to get results from the page:

async function getResultsFromPage(page) {
  ...
}

In this function we'll use the next methods to get the necessary information:

Then, we get link to the Google Reverse Image from the page context (using evaluate() method) and save it in the reverseImageSearch constant:

const reverseImageSearch = await page.evaluate(() => ({ link: document.querySelector(".z3qvzf .WpHeLc").getAttribute("href") }));

Next, we get knowledge graph info. To do this, we need to get all knowledgeGraphItems from the page (with .$$() method).

Then, using for...of loop we click (click() method) on each item and get and add (push() method) information from this knowledge graph to the knowledgeGraph array:

const knowledgeGraphItems = await page.$$(".sHc5hf > div");
const knowledgeGraph = [];
for (const item of knowledgeGraphItems) {
  await item.click();
  await page.waitForTimeout(2000);
  const thumbnail = await item.$eval(".FH8DCc", (node) => node.getAttribute("src"));
  knowledgeGraph.push(
    await page.evaluate(
      (thumbnail) => ({
        title: document.querySelector(".DeMn2d").textContent,
        subtitle: document.querySelector(".XNTym").textContent,
        link: document.querySelector(".wNPKTe a").getAttribute("href"),
        moreImagesLink: document.querySelector(".XkkoHf + div .Tc2PU > a").getAttribute("href"),
        thumbnail,
        images: Array.from(document.querySelectorAll(".XkkoHf + div .Tc2PU .Y02Gld a")).map((el) => ({
          title: el.getAttribute("aria-label"),
          source: el.getAttribute("href"),
          link: el.querySelector("img").getAttribute("src"),
        })),
      }),
      thumbnail
    )
  );
}

Next, we need to scroll the page to load all thumbnails (if we don't, we get only a 1x1 pixels size placeholder). To do this we get page height, set scrollIterationCount to define how parts we want to divide the page, and then scroll it (mouse.wheel() method):

const height = await page.evaluate(() => document.querySelector(".jXKZBd").scrollHeight);
const scrollIterationCount = 5;
for (let i = 0; i < scrollIterationCount; i++) {
  await page.mouse.wheel({ deltaY: height / scrollIterationCount });
  await page.waitForTimeout(2000);
}

Next, we get info for visual match results and save it in the visualMatches constant.

const visualMatches = await page.evaluate(() => {
  return Array.from(document.querySelectorAll(".Vd9M6 a")).map((el) => ({
    title: el.getAttribute("aria-label"),
    link: el.getAttribute("href"),
    source: el.querySelector(".PlAMyb span").textContent,
    sourceIcon: el.querySelector(".PlAMyb img").getAttribute("src"),
    thumbnail: el.querySelector(".Me0cf img").getAttribute("src"),
  }));
});

To get the text results we click on the "Text" button, get all text items and set info from each item to the textResults array:

await page.click("#text");
await page.waitForTimeout(2000);
const textResultsItems = await page.$$(".WfGljd[aria-label]");
const textResults = [];
for (const item of textResultsItems) {
  await page.click("#text");
  await page.waitForTimeout(2000);
  await item.click();
  await page.waitForTimeout(2000);
  textResults.push(
    await page.evaluate(() => ({
      text: document.querySelector(".C3t4Ke").textContent,
      link: document.querySelector("a[aria-label=Search]").getAttribute("href"),
    }))
  );
}

Then, we return the object with all the info that we get in this function:

return { reverseImageSearch, textResults, knowledgeGraph, visualMatches };

Next, write a function to control the browser, and get information:

async function getLensResults() {
  ...
}

These options mean that we use headless mode and array with arguments which we use to allow the launch of the browser process in the online IDE. And then we open a new page:

const browser = await puppeteer.launch({
  headless: true, // if you want to see what the browser is doing, you need to change this option to "false"
  args: ["--no-sandbox", "--disable-setuid-sandbox"],
});

const page = await browser.newPage();

Next, we change the default (30 sec) time for waiting for selectors to 60000 ms (1 min) for slow internet connection with .setDefaultNavigationTimeout() method, go to URL with .goto() method:

await page.setDefaultNavigationTimeout(60000);
await page.goto(URL);

Then we define the results array and save results from page to it:

const results = await getResultsFromPage(page);

And finally, we close the browser, and return the received data:

await browser.close();

return results;

Now we can launch our parser:

$ node YOUR_FILE_NAME # YOUR_FILE_NAME is the name of your .js file

Output

{
   "reverseImageSearch":{
      "link":"https://www.google.com/search?tbs=sbi:AMhZZis56hg-YEqgOHS8pa8l_1QxyQa2cxtCmEZoD71xt6dE6WPxXnbPXQV6EuTR9Yhnyb56yuK1pVQveu_1_1lXI49V16nuPJPYy5LFOIu9_16JkSKvotCdgwacqAKAxnmZ8mjkw0qYAZqvGzmkOCPnXM6oko0pADoQ4A"
   },
   "textResults":[
      {
         "text":"\"18\"",
         "link":"https://www.google.com/search?q=18%20"
      },
      {
         "text":"\"M\"",
         "link":"https://www.google.com/search?q=M"
      }
   ],
   "knowledgeGraph":[
      {
         "title":"BMW i8",
         "subtitle":"Sports car",
         "link":"https://www.google.com/search?q=BMW+i8&kgmid=/m/080607h&hl=en&gl=US",
         "moreImagesLink":"https://www.google.com/search?q=BMW+i8&kgmid=/m/080607h&ved=0EOTpBwgAKAAwAA&source=.lens.button&tbm=isch&hl=en&gl=US",
         "thumbnail":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH",
         "images":[
            {
               "title":"Image #1 for BMW i8",
               "source":"https://www.youtube.com/watch?v=fOXuBey9xo8",
               "link":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH"
            },
            {
               "title":"Image #2 for BMW i8",
               "source":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSLMCx-TtUdq5kmOhdczYL-PWIgaWtUntnHVAJWbbo7vin_obsM",
               "link":"https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcS8sXyFSX0NjlLBQcQbJAC85f-Aw9WIEX2D-Y3by-C7pd23R3C2"
            }
         ]
      },
        ... and other knowledge graph results
   ],
   "visualMatches":[
      {
         "title":"The 2023 BMW i8 M (All we know so far) - YouTube",
         "link":"https://www.youtube.com/watch?v=fOXuBey9xo8",
         "source":"youtube.com",
         "sourceIcon":"https://encrypted-tbn0.gstatic.com/favicon-tbn?q=tbn:ANd9GcQHQslzHLLyLc_qne5jxn7JocidlmUPyegZ8ojX3WVlorFk8BxW9a3vJWjDzN99UHVTqSaNBj_-6XykhxuVQfF3Ye7xScSWSuc2QHXi0a12CkVwBmo",
         "thumbnail":"https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcS7p4dD_8PlhsKlZ_5N2szF0jRpzjc-KiboC0GrNmPQhFsd8rAH"
      },
        ... and other visual matches results
   ]
}

Links

If you want other functionality added to this blog post or if you want to see some projects made with SerpApi, write me a message.

Join us on Twitter | YouTube

Add a Feature Request💫 or a Bug🐞