DEV Community: Linda Sebastian

Benchmarking CSV File Processing: Golang vs NestJS vs PHP vs Python

Linda Sebastian — Mon, 12 Aug 2024 12:57:52 +0000

Introduction

Processing large CSV files efficiently is a common requirement in many applications, from data analysis to ETL (Extract, Transform, Load) processes. In this article, I want to benchmark the performance of four popular programming languages—Golang, NodeJS with NestJS, PHP, and Python—in handling large CSV files on a MacBook Pro M1. I aim to determine which language provides the best performance for this task.

Test Environment

Hardware: MacBook Pro M1, 256GB SSD, 8GB RAM

Software:

macOS Sonoma 14.5
PHP 8.3.6
Golang 1.22.4
Node.js 22.0.0 with NestJS
Python 3.12.3

Test Data

I used a synthetic CSV file named sales_data.csv with approximately 1 million rows, each containing transaction details such as transaction_id, product_id, quantity, price, and timestamp.

Task Description

For each language, the script performs the following tasks:

Reads the CSV file.
Calculates the total sales amount.
Identifies the product with the highest sales.

Implementation

Here are the scripts used for each language:

Golang Script:

sales.go



package main

import (
    "encoding/csv"
    "fmt"
    "os"
    "strconv"
    "time"
)

func main() {
    start := time.Now()

    file, err := os.Open("../generate-csv/sales_data.csv")
    if err != nil {
        fmt.Println("Error:", err)
        return
    }
    defer file.Close()

    reader := csv.NewReader(file)
    _, _ = reader.Read() // Skip header

    totalSales := 0.0
    productSales := make(map[string]float64)

    for {
        line, err := reader.Read()
        if err != nil {
            break
        }
        productID := line[1]
        quantity, _ := strconv.Atoi(line[2])
        price, _ := strconv.ParseFloat(line[3], 64)
        total := float64(quantity) * price

        totalSales += total
        productSales[productID] += total
    }

    var topProduct string
    var topSales float64
    for product, sales := range productSales {
        if sales > topSales {
            topProduct = product
            topSales = sales
        }
    }

    elapsed := time.Since(start)
    fmt.Printf("Golang Execution time: %s\n", elapsed)
    fmt.Printf("Total Sales: $%.2f\n", totalSales)
    fmt.Printf("Top Product: %s with sales $%.2f\n", topProduct, topSales)
}

NestJS Script:

csv.service.ts



import { Injectable } from '@nestjs/common';
import * as fs from 'fs';
import * as fastcsv from 'fast-csv';

// path file CSV
const GLOBAL_CSV_PATH = '../generate-csv/sales_data.csv';

@Injectable()
@Injectable()
export class CsvService {
  async parseCsv(): Promise<{
    nestExecutionTime: number;
    totalSales: number;
    topProductSales: number;
  }> {
    return new Promise((resolve, reject) => {
      const startTime = process.hrtime();

      let totalSales = 0;
      const productSales: { [key: string]: number } = {};

      fs.createReadStream(GLOBAL_CSV_PATH)
        .pipe(fastcsv.parse({ headers: true, delimiter: ',' }))
        .on('data', (row) => {
          const productID = row.product_id;
          const quantity = parseInt(row.quantity, 10);
          const price = parseFloat(row.price);
          const total = quantity * price;
          totalSales += total;
          if (!productSales[productID]) {
            productSales[productID] = 0;
          }
          productSales[productID] += total;
        })
        .on('end', () => {
          const topProduct = Object.keys(productSales).reduce((a, b) =>
            productSales[a] > productSales[b] ? a : b,
          );
          const topProductSales = productSales[topProduct] || 0;
          const endTime = process.hrtime(startTime);
          const nestExecutionTime = endTime[0] + endTime[1] / 1e9;

          console.log(`NestJS Execution time: ${nestExecutionTime} seconds`);
          console.log(`Total Sales: $${totalSales}`);
          console.log(
            `Top Product: ${topProduct} with sales $${topProductSales}`,
          );

          resolve({
            nestExecutionTime,
            totalSales,
            topProductSales,
          });
        })
        .on('error', (error) => reject(error));
    });
  }
}

csv.controller.ts



import { Controller, Get } from '@nestjs/common';
import { CsvService } from './csv.service';

@Controller('csv')
export class CsvController {
  constructor(private readonly csvService: CsvService) {}

  @Get('parse')
  async parseCsv(): Promise<{
    nestExecutionTime: number;
    totalSales: number;
    topProductSales: number;
  }> {
    return this.csvService.parseCsv();
  }
}

PHP Script

sales.php



<?php
$start_time = microtime(true);

$file = fopen("../generate-csv/sales_data.csv", "r");
$total_sales = 0;
$product_sales = [];

fgetcsv($file); // Skip header
while (($line = fgetcsv($file)) !== false) {
    $product_id = $line[1];
    $quantity = (int)$line[2];
    $price = (float)$line[3];
    $total = $quantity * $price;

    $total_sales += $total;
    if (!isset($product_sales[$product_id])) {
        $product_sales[$product_id] = 0;
    }
    $product_sales[$product_id] += $total;
}
fclose($file);

arsort($product_sales);
$top_product = array_key_first($product_sales);

$end_time = microtime(true);
$execution_time = ($end_time - $start_time);

echo "PHP Execution time: ".$execution_time." seconds\n";
echo "Total Sales: $".$total_sales."\n";
echo "Top Product: ".$top_product." with sales $".$product_sales[$top_product]."\n";

Python Script



import csv
import time

# Input file name config
input_file = '../generate-csv/sales_data.csv'


def parse_csv(file_path):
    start_time = time.time()

    total_sales = 0
    product_sales = {}

    with open(file_path, mode='r') as file:
        reader = csv.DictReader(file)

        for row in reader:
            product_id = row['product_id']
            quantity = int(row['quantity'])
            price = float(row['price'])
            total = quantity * price
            total_sales += total

            if product_id not in product_sales:
                product_sales[product_id] = 0
            product_sales[product_id] += total

    top_product = max(product_sales, key=product_sales.get)
    execution_time = time.time() - start_time

    return {
        'total_sales': total_sales,
        'top_product': top_product,
        'top_product_sales': product_sales[top_product],
        'execution_time': execution_time,
    }


if __name__ == "__main__":
    result = parse_csv(input_file)
    print(f"Python Execution time: {result['execution_time']:.2f} seconds")
    print(f"Total Sales: ${result['total_sales']:.2f}")
    print(f"Top Product: {result['top_product']} with sales ${
          result['top_product_sales']:.2f}")

Results

Here are the results of our benchmark tests:

Golang

Execution time: 466.69975ms
Total Sales: $274654985.36
Top Product: Product 1126 with sales $305922.81

NestJS

Execution time: 6.730134208 seconds
Total Sales: $274654985.36000216
Top Product: 1126 with sales $305922.8099999997

PHP

Execution time: 1.5142710208893 seconds
Total Sales: $274654985.36
Top Product: 1126 with sales $305922.81

Python

Execution time: 2.56 seconds
Total Sales: $274654985.36
Top Product: 1126 with sales $305922.81

Analysis

My benchmark reveals several interesting insights:

Execution Time: Golang performed the best in terms of execution time, followed closely by PHP8, while NestJS took the longest time to complete the task.
Memory Usage: Build NestJS demonstrated efficient memory usage, while Python showed higher memory consumption.
Ease of Implementation: Golang provided the most straightforward implementation, while NestJS required more lines of code and complexity.

Conclusion

Based on my findings, Golang offers the best performance speed and memory efficiency, making it an excellent choice for handling large datasets.

Complete Code

You can get the full code on My Github repository
csv-parsing-battle.

Update Refactor Go

I refactor the part of Go from two loops to one loop for simplicity.



package main

import (
    "encoding/csv"
    "fmt"
    "os"
    "strconv"
    "time"
)

func main() {
    start := time.Now()

    file, err := os.Open("../generate-csv/sales_data.csv")
    if err != nil {
        fmt.Println("Error:", err)
        return
    }
    defer file.Close()

    reader := csv.NewReader(file)
    _, _ = reader.Read() // Skip header

    totalSales := 0.0
    productSales := make(map[string]float64)

    var topProduct string
    var topSales float64

    for {
        line, err := reader.Read()
        if err != nil {
            break
        }

        productID := line[1]
        quantity, _ := strconv.Atoi(line[2])
        price, _ := strconv.ParseFloat(line[3], 64)
        total := float64(quantity) * price

        totalSales += total
        productSales[productID] += total

        // Update topProduct and topSales in the same loop
        if productSales[productID] > topSales {
            topProduct = productID
            topSales = productSales[productID]
        }
    }

    elapsed := time.Since(start)
    fmt.Printf("Golang Execution time: %s\n", elapsed)
    fmt.Printf("Total Sales: $%.2f\n", totalSales)
    fmt.Printf("Top Product: %s with sales $%.2f\n", topProduct, topSales)
}

The result barely any difference.

Execution time: 457.721917ms
Total Sales: $274654985.36
Top Product: Product 1126 with sales $305922.81

Building Apps With Bubble.io

Linda Sebastian — Wed, 10 Nov 2021 10:50:19 +0000

If you want to build an app but don't have any skills for coding or maybe you just know little, I think bubble.io is one of the answers. Bubble.io is one of the low-code development platforms, this amazing tool allows you to create a website application with just a few steps.

You can go to the website [https://bubble.io/welcome]

Bubble Also provides you step by step how to use this platform, for websites and mobile web.

Bubble Pricing is cheaper compared to other platforms.

Even though bubble.io is a great platform but there is some concerns about scalability, speed performance, and security. I have my list of pros and cons about this platform.

Pros

Very easy just drag and drop
Building apps faster compare than traditional hand-coding
There are s many free plugins that you can use
You don't have to set up a server or hosting

Cons

Speed performance is not so good
I'm not sure if bubble.io can handle such complex logic
There's no versioning! If you did something wrong in production there's no going back.
Scalability. The capacity limit for production planning is just 50 GB.

I think that's my impression of bubble.io. Cheers!

Mockup API With JSON Server

Linda Sebastian — Wed, 13 Oct 2021 08:05:37 +0000

Overview

In a software development project, sometimes the annoying common problem is frontend developer needs to wait until the backend developer finishes working on an API. That's why we need to make a mockup API, therefore both backend and frontend developers can work parallel.

JSON server is an easy way to make a fake REST API, even a beginner or non-programmer can make it. The only prerequisite is you need to understand JSON structure. Here's a tutorial to make a fake REST API with JSON Server.

Setup

Install node js: [https://nodejs.org/en/download/]
Open a terminal (bash/shell/linux)
Create a directory call mock-api (or whatever you want)
initial new project. Type npm init then enter until show like below, and type yes.

Working with JSON Server
install json server npm install -g json-server if you want install globally or npm i json-server if you want install locally. In this tutorial I will install globally

npm install -g json-server

create file db.json
write this structure json

{
    "users": [
        {
            "id": 1,
            "first_name": "Sonny",
            "last_name": "Allward",
            "email": "sallward0@umn.edu",
            "gender": "Genderfluid",
            "ip_address": "24.3.199.140"
        },
        {
            "id": 2,
            "first_name": "Manfred",
            "last_name": "Erickson",
            "email": "merickson1d@mapquest.com",
            "gender": "Genderfluid",
            "ip_address": "103.30.222.192"
        }
    ],
    "products": [
        {
            "id": 1,
            "product_name": "Beets",
            "sku": "54949-004",
            "price": 27,
            "category": "Beauty",
            "quantity": 69
        },
        {
            "id": 2,
            "product_name": "Wine - Mondavi Coastal Private",
            "sku": "46122-146",
            "price": 63,
            "category": "Home",
            "quantity": 12
        }
    ]
}

running JSON Server

npm run json:server --watch db.json

Call Request

Get all data

http://localhost:3000/users

Get one data, method GET

http://localhost:3000/users/1

Search data, method GET

http://localhost:3000/users?first_name=Manfred

Pagination, method GET

http://localhost:3000/users?_limit=10&_page=5

Sorting, method GET

http://localhost:3000/users?_sort=id&_order=DESC

Create data, method POST

http://localhost:3000/users
body:
{
    "id": 51,
    "first_name": "Itje",
    "last_name": "Juice",
    "email": "itje@yale.edu",
    "gender": "Helicopter",
    "ip_address": "44.73.130.666"
}

Delete data, method DELETE

http://localhost:3000/users/1

Github URL: [https://github.com/rocklinda/mock-api]
NPM JSON Server: [https://www.npmjs.com/package/json-server]