Cypress Web Scraping in 20 minutes

/// <reference types="cypress" />
let myName = 'HolyToledo';
let myPath = 'C:\\Users\\someSecretPath\\assets\\devArticles'

context("dev.to", () => {
  it("get all my articles", () => {
    // get all of my dev.to articles
    cy.request(`https://dev.to/api/articles? 
       username=${myName}&per_page=2000`).then(
      (result) => {
        let titles = result.body.map((item) => {
          // put all articles into an array of objects with title and url
          let result = { title: item.title, url: item.url };
          return result;
        });
        let sorted = titles.sort();
        // make it suitable for a json file.
        let object = { Articles: sorted };
        let path = `${myPath}\\articles.json`;
        // write it to my assets folder for the web display.
        cy.writeFile(path, object);
      }
    );
  });

Cypress has a built in HTTPClient, using the Cy.request command! The response object has a body property, in this case an array. We then change the format slightly to create links in our website! When we're done and have saved the content into the Angular assets folder, where we can build a GUI around it easily.

We like the material table, so let's put it to use.

// using out own Grid directive...
<div Grid columns=".1fr .7fr">
  // This is the search control
  <label>Search</label>
  <input #search (keyup)='onSearchKeyUp($event, search)' type="text">
</div> 
// The searchable, sort-able, fiter-able table. 
// See the script for dataSource below, this is our json file.
<table mat-table  [dataSource]="dataSource">
  <ng-container matColumnDef="articles">
     <th mat-header-cell *matHeaderCellDef>Dev.to Articles</th>
     <td mat-cell *matCellDef="let article">
      // Each article has a url and a title, this is the binding
      <a Grid [(href)]='article.url'> <label>{{article.title}}</label></a>
     </td>
  </ng-container>
  <tr mat-header-row *matHeaderRowDef="displayedColumns"></tr>
  <tr mat-row *matRowDef="let row; columns: displayedColumns"></tr>
</table>
<mat-paginator 
      [length]="length"
      [pageSize]="size"
      [pageSizeOptions]="[10, 20, 50, 100]">
</mat-paginator>

import { Component, OnInit, AfterViewInit, ViewChild } from "@angular/core";
// Notice we are using import to read the json file!
import { Articles } from "src/assets/devArticles/articles.json";
import { MatTableDataSource } from "@angular/material/table";
import { MatPaginator } from "@angular/material/paginator";
import { MatSort, MatSortable } from '@angular/material/sort';

@Component({
  selector: "app-dev-articles",
  templateUrl: "./dev-articles.component.html",
  styleUrls: ["./dev-articles.component.css"],
})
export class DevArticlesComponent implements OnInit, AfterViewInit {
  @ViewChild(MatPaginator) paginator;
  dataSource: MatTableDataSource<any>;
  displayedColumns;
  length;
  size;
  constructor() {}

  ngOnInit(): void {
    this.dataSource = new MatTableDataSource();
    // Sets the data for the datasource to the JSON content.
    this.dataSource.data = Articles;
    this.displayedColumns = ["articles"];
  }
  ngAfterViewInit() {
    this.setPaginatorConfig();
    this.setDataSourceConfig();
  }

  onSearchKeyUp(event, search){
    var currentFilter = search.value;
    // use the datasource.filter so we don't have to change the items source
    this.dataSource.filter = currentFilter;
  }

  private setDataSourceConfig() {
    // Want it sorted?
    this.dataSource.data = Articles.sort(this.sorter);  
    this.dataSource.paginator = this.paginator;
    let direction:MatSort;
    direction.direction="asc"
    let data=[];
    this.dataSource.sortData(data,direction)


  }

  private setPaginatorConfig() {
    setTimeout(() => {
      this.length = this.dataSource.data.length;
      this.size = 10;
    }, 5);
  }

  sorter = (a, b) => {
    if (a.title > b.title) return 1;
    if (a.title < b.title) return -1;
    return 0;
  };
}

Result

Jwp2020