IP Visualizer live demo & redis bulk loading with node.

Soo.. demo is live, woohoo.. 🙂
Head over to ipv.karate.monster and try it out. And yeah, don't run it on chrome mobile browser, the touch stuff is dodgy and not working like it should. Seems to work on firefox mobile though, but well, it was never really developed for mobile so your mileage my vary with that, just use a computer 😉.

How to use? press the crosshair button, click and drag on the map to make a circle.
Visualize the stuff, change layers and settings as you like. In some layers can click on the drawn items to get data for it.

Let's talk about redis bulk loading.
So for my project I need to parse a csv-file and add the data to redis.

So at first i was just parsing a csv-file, shoveled promises with commands into an array and then sending it regularly with the node redis client. This approach is slooooow and i do not recommend it.

I needed a better way, so started googling around and reading.
Stumbled upon a github repository showing how to pipe commands from node to redis-cli.

Implemented the pipeline in my code and also the protocol needed for bulkloading

Below is the code, maybe it will be useful for someone trying to do something similar.

const spawn = require('child_process').spawn;
const fs = require('fs');
const CsvReadableStream = require('csv-reader');
const redisPipe = spawn('redis-cli', ['--pipe']);

redisPipe.stdout.setEncoding('utf8');
redisPipe.stdout.pipe(process.stdout);
redisPipe.stderr.pipe(process.stderr);

const file = 'my.csv';
const BUFFER_SIZE = 524288; // 512KB
let buffer = '';

async function run() {
    let inputStream = fs.createReadStream(file, 'utf8');
    console.log('Let the piping commence!');

    inputStream
        .pipe(new CsvReadableStream({ asObject: true })) //reads in every row of file as object with keys being taken from csv header
        .on('data', async function (row) {
            //check that the row acutally have coordinates, if not i dont want it
            if (row.longitude && row.latitude) {
                //encode the string with the redis command and add it to the buffer
                buffer += encodeRedis(`geoadd ips ${row.longitude} ${row.latitude} "${row.network}"`);

                //when buffer is filled then write it and then empty buffer.
                if (buffer.length > BUFFER_SIZE) {
                    redisPipe.stdin.write(buffer);
                    buffer = '';
                }
            }
        })
        .on('end', async function () {
            redisPipe.stdin.write(buffer); //write the remaining buffer if any left
            redisPipe.stdin.end(); //end the pipe
            console.log('Update complete');
            process.exit();
        });
}

function encodeRedis(dataString) {
    const dataArr = dataString.split(' '); //split data into array
    let msg = '*' + dataArr.length + '\r\n'; //create start of message with amount of args

    for (let i = 0; i < dataArr.length; i++) {
        msg += '$' + dataArr[i].length + '\r\n' + dataArr[i] + '\r\n'; //encode the data
    }
    return msg; //return the encoded message
}

run();