DEV Community: J

Tradme 100K JSON ETL, MongoDB Atlas, Processing and Visualizing

Wed, 27 Dec 2023 13:27:35 +0000

Trademe First

Mon, 04 Sep 2023 00:59:51 +0000

import time
import asyncio
import aiohttp
from datetime import datetime
import csv
import re


async def get_responses(number):
    stringnumber = str(number)
    print("Start at " + stringnumber)
    params = {
        'page': f'{number}',
        'rows':'3000',
        }
    async with aiohttp.ClientSession() as session:
        async with session.get(url = url,headers = headers, params = params) as resp:
            very_raw = await resp.json()
            for current_dic_number in range(len(very_raw["List"])):
                local_dic = very_raw["List"][current_dic_number]
                local_key_list = []
                final_list = []
                for keys in local_dic:
                    if keys not in low_list:
                        local_key_list.append(keys)
                print(local_key_list)
                for key in long:
                    if key not in local_key_list:
                        final_list.append("/")
                    elif key == "StartDate" or key == "EndDate":
                        new_key = str(datetime.fromtimestamp(int(re.findall('\d+', local_dic[key])[0]) / 1000))
                        final_list.append(new_key)
                    elif key == "OpenHomes":
                        if len(local_dic[key]) == 1:
                            local_dic[key][0]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["Start"])[0]) / 1000))
                            local_dic[key][0]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["End"])[0]) / 1000))
                            final_list.append(local_dic[key])
                        elif len(local_dic[key]) == 2:
                            local_dic[key][0]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["Start"])[0]) / 1000))
                            local_dic[key][0]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["End"])[0]) / 1000))
                            local_dic[key][1]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][1]["Start"])[0]) / 1000))
                            local_dic[key][1]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][1]["End"])[0]) / 1000))
                            final_list.append(local_dic[key])
                        else:
                            final_list.append(" ")
                    else:
                        final_list.append(local_dic[key])

                # with open("120920223.csv", "a", newline="") as file:
                #     writer = csv.writer(file)
                #     writer.writerow(final_list)
            print("Finished " + stringnumber)



async def main():
    tasks = []
    for number in range(1, /):
        tasks.append(get_responses(number))
    await asyncio.gather(*tasks)


if __name__ == '__main__':

    long = ['ListingId', 'StartDate', 'Title', 'EndDate', 'Region', 'Suburb', 'OpenHomes', 'GeographicLocation',
            'PriceDisplay', 'HasEmbeddedVideo', 'Has3DTour', 'Address', 'District', 'Area', 'LandArea', 'Bathrooms',
            'Bedrooms', 'Parking', 'PropertyType', 'RateableValue', 'AdjacentSuburbNames', 'Agency', 'TotalParking']

    low_list = ['StartPrice', 'Category', 'AsAt', 'CategoryPath', 'PictureHref', 'RegionId', 'ListingLength',
                'IsFeatured', 'HasGallery', 'IsBold', 'IsHighlighted', 'IsClassified', 'NoteDate', 'SuburbId',
                'ReserveState', 'PhotoUrls', 'AdditionalData', 'ListingGroup', 'ListingExtras', 'MemberId',
                'AgencyReference', 'DistrictId', 'AdjacentSuburbIds' ]
    headers = {
        'authority': 'api.trademe.co.nz',
        'method': 'GET',
        'accept': 'pplication/json, text/plain, */*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en,en-US;q=0.9',
        'cache-control': 'max-age=0',
        'newrelic': 'eyJ2IjpbMCwx=XSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjQzODYzOCIsImFwIjoiMzgwMDc2Nzg0IiwiaWQiOiIxOGZjMWRkNzM1YTE4MmViIiwidHIiOiI4NGRiNzIyODE2Yjk1NzhjZDEzMTIxYjc3MGQ3MzQwMCIsInRpIjoxNjcwMjg3Mjg0NzU2fX0=',
        'origin': 'https://www.trademe.co.nz',
        'referer': 'https://www.trademe.co.nz/',
        'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'some-site',
        'upgrade-insecure-requests': '1',
        # 'x-trademe-uniqueclientid': '51f6a3d6-24e6-c673-45fe-8646ad15be04'

    }
    url = /

    with open("Processing and Visualization/120920222.csv", "a", newline="") as file:
        writer = csv.writer(file)
        writer.writerow(long)
    start = time.time()

    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(main())
    finally:
        loop.close()
    stop = time.time()
    print(stop - start)

Reverse Engineering JS

Sun, 03 Sep 2023 23:07:30 +0000

<head>
        <style>
            div {
                font-family:'Courier';
            }

            #c {
                width:720px;
            }

            #canary {
                position: absolute;
                z-index: 9;
                background-color: #f1f1f1;
                border: 1px solid #d3d3d3;
                text-align: center;
            }
        </style>
        <title>Can you crack me?</title>
        <!-- Global site tag (gtag.js) - Google Analytics -->
        <script type="text/javascript" async="" src="https://www.google-analytics.com/analytics.js"></script><script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-26894442-4"></script>
        <script>
          window.dataLayer = window.dataLayer || [];
          function gtag(){dataLayer.push(arguments);}
          gtag('js', new Date());

          gtag('config', 'UA-26894442-4');
        </script>
        <script type="text/javascript">

            let runes = [];
            let ee = eval("\145\x76" + atob("yw".toUpperCase() + "w=")), b = atob;

            let spikes = ["ymF2aWdhdG9b", "0XNlckFnZW5d", "=W5kZXhPZg=a", "=EFNQkVSVA=T", "ZklQTEVU", "=UxERVJTT04Q", "=kxBQ0tTRUEQ", "yGlZ", "=2xlYXIY", "=2x1ZQ=Y", "52FuYXJY", "=WxwaGEY", "=mV0YQ=Y", "=2FtbWEZ"]

            let tY = (function(){
                function n(s){
                    bb = s[0], nn = s[s.length-1];
                    return nn + s.substring(1, s.length-1) + bb;
                }
                return n;
            })()

            let clue = (function(){
                if (window[b(tY(spikes[0x00]))][b(tY(spikes[0x01]))].indexOf(b(tY(spikes[0x06]))) == 9){
                    return b(tY(spikes[0x04]));
                } else {
                    return b(tY(spikes[0x03])); 
                }; 
            })();

            var el = new Image();
            Object.defineProperty(el, 'id', {
                get: function () {
                    clue = b(tY(spikes[0x05]));
                }
            });         
            requestAnimationFrame(function check() {
                console.dir(el);
                console.clear();
                requestAnimationFrame(check);
            });           

            setInterval(() => {
                ee('dLb'.replace('L', 'e') +'ug' + '14r'.replace('14', 'ge'));
                if (runes.length > 2 && (function (targ) {
                                            tt = 0;
                                            for (i = 0; i < targ.length; i++) {
                                                tt += targ.charCodeAt(i);
                                            }
                                            return tt == 469;
                                        })(clue)){
                    ee(b("YWxlcn"+"QoIkNvbmdy"+"YXR1bGF0aW9uc"+"yBPaCAxMzM3I"+"E9uZSEgWW91IGh"+"hdmUgZm91bmQgbXkg"+"dHJlYXN1cmUhIik="));
                    runes=[];
                }
            }, 2000);

            function stage(){
                document.location.hash='';
                d(document.getElementById("c"+"an"+"ar"+"y"));
            }

            function IChing(x,y){
                document.getElementById("gps").innerHTML = btoa(y.toString() + "," + x.toString()).replace("=", "\n\<br>\\n");
                if (y == clue.charCodeAt(0) && x == clue.charCodeAt(1) && !runes.length){
                    runes.push(b(tY(spikes[11])));
                } else if (y == clue.charCodeAt(2) && x == clue.charCodeAt(3) && runes.length == 1){
                    runes.push(b(tY(spikes[12])));
                } else if (y == clue.charCodeAt(4) && x == clue.charCodeAt(5) && runes.length == 2){
                    runes.push(b(tY(spikes[13])));
                }
            }

           function d(e) { 
               var n = 0, t = 0, o = 0, u = 0; function d(e) { (e = e || window.event).preventDefault(), o = e.clientX, u = e.clientY, document.onmouseup = c, document.onmousemove = l } function l(d) { (d = d || window.event).preventDefault(), n = o - d.clientX, t = u - d.clientY, o = d.clientX, u = d.clientY, e.style.top = e.offsetTop - t + "px", e.style.left = e.offsetLeft - n + "px", IChing(u, o) } function c() { document.onmouseup = null, document.onmousemove = null } document.getElementById(e.id + "header") ? document.getElementById(e.id + "header").onmousedown = d : e.onmousedown = d 
               }
        </script>
    </head>
    <body onload="stage()">
        <div id="c">
            <hr>
            The notorious pirate Captain Platinumbeard ravaged the high seas for decades. 
            <br><br>
            After every plunder, he stashed his booty in a hidden alcove on a tropical isle. 
            <br><br>
            Confined to his death bed after falling ill due to scurvy, he spent his last living hours coding a cryptic map in the hope a 1337 adventurer would one day come and find the treasure. 
            <br><br>
            Can you solve the mystery of Captain Platinumbeard and find the hidden riches?
            <hr>
            Remember: 
            <br><br>
            <div id="canary">X</div>&nbsp;&nbsp;&nbsp;&nbsp;marks the spot.
            <br><br>
            <hr>
            <div id="gps"></div>
        </div>
</body>

let runes = [];
let execute = eval("ev" + atob("yw".toUpperCase() + "w=")), decodeBase64 = atob;


decodeBase64 = atob

let transformString = function () {
  function manipulate(s) {
    bb = s[0], nn = s[s.length - 1];
    return nn + s.substring(1, s.length - 1) + bb;
  }
  return manipulate;
}();



let determineValue = function () {
  if (window[decodeBase64(transformString("ymF2aWdhdG9b")))][decodeBase64(transformString("0XNlckFnZW5d"))].indexOf(decodeBase64(transformString("=kxBQ0tTRUEQ"))) == 9) {
    return decodeBase64(transformString("ZklQTEVU"));
  } else {
    return decodeBase64(transformString("=EFNQkVSVA=T"));
  }
}();



var imageElement = new Image;
Object.defineProperty(imageElement, "id", {get: function () {
  determineValue = decodeBase64(transformString("=UxERVJTT04Q"));
}});
requestAnimationFrame(function check() {
  console.dir(imageElement);
  console.clear();
  requestAnimationFrame(check);
});




// it reset the runes every 2 seconds, the runes.length first need to be >2 and meet function (targ)
setInterval(() => {
  execute("dLb".replace("L", "e") + "ug" + "14r".replace("14", "ge"));
  if (runes.length > 2 && function (targ) {
    tt = 0;
    for (i = 0; i < targ.length; i++) {
        // exxucte every length time
        // and sum all the deteminevalue one by one
        // and return tt==469
      tt += targ.charCodeAt(i);
    }
    return tt == 469;
  }(determineValue)) {
    execute(decodeBase64("YWxlcnQoIkNvbmdyYXR1bGF0aW9ucyBPaCAxMzM3IE9uZSEgWW91IGhhdmUgZm91bmQgbXkgdHJlYXN1cmUhIik="));
    runes = [];
  }
}, 2000);



function moveElement() {
  document.location.hash = "";
  move(document.getElementById("canary"));
}
function interactWithElement(x, y) {
  document.getElementById("gps").innerHTML = decodeBase64(btoa(y.toString() + "," + x.toString())).replace("=", "\n<br>\\n");
  if (y == determineValue.charCodeAt(0) && x == determineValue.charCodeAt(1) && !runes.length) {
    runes.push(decodeBase64(transformString("=WxwaGEY")));
  } else if (y == determineValue.charCodeAt(2) && x == determineValue.charCodeAt(3) && runes.length == 1) {
    runes.push(decodeBase64(transformString("=mV0YQ=Y")));
  } else if (y == determineValue.charCodeAt(4) && x == determineValue.charCodeAt(5) && runes.length == 2) {
    runes.push(decodeBase64(transformString("=2FtbWEZ")));
  }
}




function handleMouseActions(e) {
  var offsetX = 0, offsetY = 0, lastX = 0, lastY = 0;
  function startDrag(e) {
    e = e || window.event;
    e.preventDefault();
    lastX = e.clientX;
    lastY = e.clientY;
    document.onmouseup = stopDrag;
    document.onmousemove = drag;
  }
  function drag(e) {
    e = e || window.event;
    e.preventDefault();
    offsetX = lastX - e.clientX;
    offsetY = lastY - e.clientY;
    lastX = e.clientX;
    lastY = e.clientY;
    e.style.top = e.offsetTop - offsetY + "px";
    e.style.left = e.offsetLeft - offsetX + "px";
    interactWithElement(lastY, lastX);
  }
  function stopDrag() {
    document.onmouseup = null;
    document.onmousemove = null;
  }
  if (document.getElementById(e.id + "header")) {
    document.getElementById(e.id + "header").onmousedown = startDrag;
  } else {
    e.onmousedown = startDrag;
  }
}

Mavel Snap Auto

Fri, 25 Aug 2023 17:22:40 +0000

Made in 3 days.
3000 matches, 24/7, 0 error, 21 win-rate , first published gen.

I Don't Know How To Describle.

Sun, 20 Aug 2023 18:37:54 +0000

from datetime import datetime
from logger_config import Logger_config
import asyncio
import aiohttp

class Trademe_Scraper:
    def __init__(self):
        self.logger = Logger_config().get_logger()
        self.url = ??? Who I AM
        self.headers = {
        'authority': 'api.trademe.co.nz',
        'method': 'GET',
        'accept': 'pplication/json, text/plain, */*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en,en-US;q=0.9',
        'cache-control': 'max-age=0',
        'newrelic': 'eyJ2IjpbMCwx=XSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjQzODYzOCIsImFwIjoiMzgwMDc2Nzg0IiwiaWQiOiIxOGZjMWRkNzM1YTE4MmViIiwidHIiOiI4NGRiNzIyODE2Yjk1NzhjZDEzMTIxYjc3MGQ3MzQwMCIsInRpIjoxNjcwMjg3Mjg0NzU2fX0=',
        'origin': 'https://www.trademe.co.nz',
        'referer': 'https://www.trademe.co.nz/',
        'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'some-site',
        'upgrade-insecure-requests': '1',
    }
        self.total_batches = []
        self.box = 0
        self.timestamp = datetime.utcnow()


    async def scrape(self, ID):
        params = {
            'page': f'{ID}',
            'rows': '3000',
        }
        async with aiohttp.ClientSession() as session:
            async with session.get(url=self.url, headers=self.headers, params=params) as r:
                if r:
                    json_data = await r.json()
                    batch = json_data['List']

                    def convert_milliseconds(inputs):
                        # Convert to Seconds
                        seconds = int(inputs) / 1000.0
                        # Return IsoDate
                        return datetime.utcfromtimestamp(seconds).isoformat()

                    for x in range(len(batch)):
                        batch[x]["G_Date"] = self.timestamp  # Converted to ISO
                        try:
                            batch[x]["StartDate"] = convert_milliseconds(batch[x]["StartDate"][6:-2])
                        except:
                            pass
                        try:
                            batch[x]["EndDate"] = convert_milliseconds(batch[x]["EndDate"][6:-2])
                        except:
                            pass
                        try:
                            batch[x]["AsAt"] = convert_milliseconds(batch[x]["AsAt"][6:-2])
                        except:
                            pass
                        try:
                            batch[x]["OpenHomes"][0]["Start"] = convert_milliseconds(
                                batch[x]["OpenHomes"][0]["Start"][6:-2])
                            batch[x]["OpenHomes"][0]["End"] = convert_milliseconds(
                                batch[x]["OpenHomes"][0]["End"][6:-2])
                        except:
                            pass
                        try:
                            batch[x]["OpenHomes"][1]["Start"] = convert_milliseconds(
                                batch[x]["OpenHomes"][1]["Start"][6:-2])
                            batch[x]["OpenHomes"][1]["End"] = convert_milliseconds(
                                batch[x]["OpenHomes"][1]["End"][6:-2])
                        except:
                            pass
                    self.total_batches.extend(batch)
                    self.box += 1
                    self.logger.info(f"box: {self.box}, len: {len(self.total_batches)}")
                else:
                    self.logger.critical(f"Scraping ERROR {ID}")

    async def Gather(self):
        tasks = []
        for ID in range(1,/):
            tasks.append(self.scrape(ID))
        await asyncio.gather(*tasks)

    def run(self):
        asyncio.run(self.Gather())
        if len(self.total_batches) > 31990:  #Data Integrity Simple Checker!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
            self.logger.info(f"Scraping Success, return total_batches. box: {self.box}, {len(self.total_batches)}")
            return self.total_batches
        else:
            self.logger.critical(
                f"Scraping unsuccess, Not meet batch requirements. box: {self.box}, {len(self.total_batches)}")
            return False

A.B.O.M.N.S (A Based on My Needs System) - Dangerous Illegal Self-Designed Proton Mail Bulk Automation in 5 Days.

Sun, 20 Aug 2023 17:19:29 +0000

DEMO: Multiprocessing, 2 Proton CAPTCHAs, 1 Hcaptcha

Like more?
Lemin CAPTCHA Two Hours Domination Showcase

a

Sun, 20 Aug 2023 05:16:51 +0000

"C:\Projects\Web projects\venv\Scripts\python.exe" "C:\Projects\Web projects\Trademe_MongoDB\For Show\one batch.py" 
INFO: 2023-08-20 09:07:47,751: Connected to MongoDB server  
INFO: 2023-08-20 09:07:53,439: box: 1, len: 500  
INFO: 2023-08-20 09:07:53,442: Scraping Success, return total_batches. box: 1, 500  
{'AdditionalData': {'BulletPoints': [], 'Tags': []},
 'Address': '603A Maunganui Road',
 'AdjacentSuburbIds': [431, 432, 504, 481],
 'AdjacentSuburbNames': ['Mount Maunganui', 'Omanu', 'Bayfair', 'Te Maunga'],
 'Agency': {'Agents': [{'FullName': 'Paige Smith',
                        'MembershipType': 2,
                        'OfficePhoneNumber': '(027) 3551476',
                        'Photo': 'https://trademe.tmcdn.co.nz/property/agent_profile/160x160/agent_individual_profile6253437v4.jpg',
                        'UrlSlug': 'Paige-Smith'},
                       {'FullName': 'Dee Gillard',
                        'MembershipType': 2,
                        'OfficePhoneNumber': '(021) 1737555',
                        'Photo': 'https://trademe.tmcdn.co.nz/property/agent_profile/160x160/agent_individual_profile6356755v4.jpg',
                        'UrlSlug': 'Dee-Gillard'}],
            'Branding': {'BackgroundColor': '#ffffff',
                         'LargeBannerURL': 'https://trademe.tmcdn.co.nz/tm/property/agent_logos/office_branding/7945776-office_branding-2.jpg',
                         'OfficeLocation': '',
                         'StrokeColor': '#ffffff',
                         'TextColor': '#000000'},
            'Id': 15318,
            'IsLicensedPropertyAgency': True,
            'IsRealEstateAgency': True,
            'Logo': 'https://trademe.tmcdn.co.nz/tm/property/agent_logos/7945776-2.jpg',
            'Logo2': 'https://trademe.tmcdn.co.nz/tm/property/agent_logos/7945776-2.logo2.jpg',
            'Name': 'Collab Realty Ltd, (Licensed: REAA 2008)',
            'PhoneNumber': '+64-7-3551476',
            'Website': 'http://www.collabrealty.co.nz'},
 'AgencyReference': 'RX3775811',
 'Area': 138,
 'AsAt': '/Date(1692508068584)/',
 'Bathrooms': 2,
 'Bedrooms': 3,
 'Category': '0350-5748-3399-',
 'CategoryPath': '/Trade-Me-Property/Residential/For-Sale',
 'District': 'Tauranga',
 'DistrictId': 23,
 'EndDate': '/Date(1697162758840)/',
 'GeographicLocation': {'Accuracy': 1,
                        'Easting': 1883469,
                        'Latitude': -37.6695398,
                        'Longitude': 176.2138743,
                        'Northing': 5825988},
 'HasGallery': True,
 'IsBold': True,
 'IsClassified': True,
 'IsFeatured': True,
 'IsHighlighted': True,
 'IsSuperFeatured': True,
 'LandArea': 304,
 'ListingExtras': ['premium_listing'],
 'ListingGroup': 'PROPERTY',
 'ListingId': 4280331591,
 'ListingLength': None,
 'Lounges': 1,
 'MemberId': 7945776,
 'NoteDate': '/Date(0)/',
 'OpenHomes': [],
 'Parking': '',
 'PhotoUrls': ['https://trademe.tmcdn.co.nz/photoserver/thumb/2063397889.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397894.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397903.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397907.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397912.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397916.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397924.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397929.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397939.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397961.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397965.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397976.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397999.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398008.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398013.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398017.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398022.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398024.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398028.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398032.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398037.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398054.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398061.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398066.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398073.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398075.jpg',
               'https://trademe.tmcdn.co.nz/photoserver/thumb/2063398078.jpg'],
 'PictureHref': 'https://trademe.tmcdn.co.nz/photoserver/thumb/2063397883.jpg',
 'PremiumPackageCode': 'PlatinumPackage',
 'PriceDisplay': 'Price by negotiation',
 'PropertyType': 'House',
 'RateableValue': 910000,
 'Region': 'Bay Of Plenty',
 'RegionId': 2,
 'ReserveState': 3,
 'StartDate': '/Date(1692248038840)/',
 'StartPrice': 0,
 'Suburb': 'Mount Maunganui',
 'SuburbId': 431,
 'Title': 'Elevate your Lifestyle'}
Exception ignored in: <function _ProactorBasePipeTransport.__del__ at 0x0000028995C98E50>
Traceback (most recent call last):
  File "C:\Users\acer\AppData\Local\Programs\Python\Python39\lib\asyncio\proactor_events.py", line 116, in __del__
    self.close()
  File "C:\Users\acer\AppData\Local\Programs\Python\Python39\lib\asyncio\proactor_events.py", line 108, in close
    self._loop.call_soon(self._call_connection_lost, None)
  File "C:\Users\acer\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 751, in call_soon
    self._check_closed()
  File "C:\Users\acer\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 515, in _check_closed
    raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed

Process finished with exit code 0

Asynchronous all the way, exploring MongoDB and first build data pipeline.

Sat, 05 Aug 2023 08:51:27 +0000

(This post will get me hired.)
Today, finally, all my useless data will get dumped (dump harder) into a big hole.

Imagine my first, official, data scraping project aimed for New Zealand Biggest Amazon platform - Trademe.com. I need to trim all the data, sensitive data and select only "valid" data to store. "Manually" compare all the keys from dictionaries.
A one of 500 JSON data call:

Imagine strak gold from a newer.
How can i store this fucking shit?
Here is goes:

import time
import asyncio
import aiohttp
from datetime import datetime
import csv
import re

async def get_responses(number):
    stringnumber = str(number)
    print("Start at " + stringnumber)
    params = {
        'page': f'{number}',
        'rows':'3000',
        }
    async with aiohttp.ClientSession() as session:
        async with session.get(url = url,headers = headers, params = params) as resp:
            very_raw = await resp.json()
            for current_dic_number in range(len(very_raw["List"])):
                local_dic = very_raw["List"][current_dic_number]
                local_key_list = []
                final_list = []
                for keys in local_dic:
                    if keys not in low_list:
                        local_key_list.append(keys)
                        print(local_key_list)
                for key in long:
                    if key not in local_key_list:
                        final_list.append("/")
                    elif key == "StartDate" or key == "EndDate":
                        new_key = str(datetime.fromtimestamp(int(re.findall('\d+', local_dic[key])[0]) / 1000))
                        final_list.append(new_key)
                    elif key == "OpenHomes":
                        if len(local_dic[key]) == 1:
                            local_dic[key][0]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["Start"])[0]) / 1000))
                            local_dic[key][0]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["End"])[0]) / 1000))
                            final_list.append(local_dic[key])
                        elif len(local_dic[key]) == 2:
                            local_dic[key][0]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["Start"])[0]) / 1000))
                            local_dic[key][0]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][0]["End"])[0]) / 1000))
                            local_dic[key][1]["Start"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][1]["Start"])[0]) / 1000))
                            local_dic[key][1]["End"] = str(datetime.fromtimestamp(
                                int(re.findall('\d+', local_dic["OpenHomes"][1]["End"])[0]) / 1000))
                            final_list.append(local_dic[key])
                        else:
                            final_list.append(" ")
                    else:
                        final_list.append(local_dic[key])

                # with open("120920223.csv", "a", newline="") as file:
                #     writer = csv.writer(file)
                #     writer.writerow(final_list)
            print("Finished " + stringnumber)



async def main():
    tasks = []
    for number in range(/):
        tasks.append(get_responses(number))
    await asyncio.gather(*tasks)


if __name__ == '__main__':

    long = ['ListingId', 'StartDate', 'Title', 'EndDate', 'Region', 'Suburb', 'OpenHomes', 'GeographicLocation',
            'PriceDisplay', 'HasEmbeddedVideo', 'Has3DTour', 'Address', 'District', 'Area', 'LandArea', 'Bathrooms',
            'Bedrooms', 'Parking', 'PropertyType', 'RateableValue', 'AdjacentSuburbNames', 'Agency', 'TotalParking']
    low_list = ['StartPrice', 'Category', 'AsAt', 'CategoryPath', 'PictureHref', 'RegionId', 'ListingLength',
                'IsFeatured', 'HasGallery', 'IsBold', 'IsHighlighted', 'IsClassified', 'NoteDate', 'SuburbId',
                'ReserveState', 'PhotoUrls', 'AdditionalData', 'ListingGroup', 'ListingExtras', 'MemberId',
                'AgencyReference', 'DistrictId', 'AdjacentSuburbIds' ]
    headers = {
        'authority': 'api.trademe.co.nz',
        'method': 'GET',
        'accept': 'pplication/json, text/plain, */*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en,en-US;q=0.9',
        'cache-control': 'max-age=0',
        'newrelic': 'eyJ2IjpbMCwx=XSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjQzODYzOCIsImFwIjoiMzgwMDc2Nzg0IiwiaWQiOiIxOGZjMWRkNzM1YTE4MmViIiwidHIiOiI4NGRiNzIyODE2Yjk1NzhjZDEzMTIxYjc3MGQ3MzQwMCIsInRpIjoxNjcwMjg3Mjg0NzU2fX0=',
        'origin': 'https://www.trademe.co.nz',
        'referer': 'https://www.trademe.co.nz/',
        'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'some-site',
        'upgrade-insecure-requests': '1',
        # 'x-trademe-uniqueclientid': '51f6a3d6-24e6-c673-45fe-8646ad15be04'

    }
    url = "Protecting a vulnerable API"

    with open("Processing and Visualization/120920222.csv", "a", newline="") as file:
        writer = csv.writer(file)
        writer.writerow(long)
    start = time.time()

    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(main())
    finally:
        loop.close()
    stop = time.time()
    print(stop - start)

Those lines literally took me 2 days. Till today i still don't (or don't want to) know how i wrote these:

  local_dic = very_raw["List"][current_dic_number]
  local_key_list = []
  final_list = []
  for keys in local_dic:
     if keys not in low_list:
           local_key_list.append(keys)
           print(local_key_list)
           for key in long:
               if key not in local_key_list:

A nice shit.

In 20 seconds:

But i want all the useless data, like agency phone numbers so i can call them at midnight when i scared to sleep.
Before dump it i wrote a little testing code to know something better.

import asyncio
import time
import psutil
# Start 50,000 requests to test is there any latency or memory lose when program reading from local or shared data. The answer is they are same.

var = 100 (shared)
async def async_scraper(ID):
    # var = 100 (local)
    var_final = var + 1000
    print(var_final)
    pid = psutil.Process()
    memory_info = pid.memory_info()
    memory_used_mb = memory_info.rss / (1024 ** 2)
    print("Memory used:", memory_used_mb, "MB")

async def Gather():
    tasks = []
    for ID in range(1, 50000):
        tasks.append(async_scraper(ID))
    await asyncio.gather(*tasks)

start = time.time()
loop = asyncio.get_event_loop()
try:
    loop.run_until_complete(Gather())
finally:
    loop.close()
stop = time.time()
print(stop-start)

- As now the behave of import and reading function are the same to look up the location inside the memory. But i tested for starting 50,000 requests to if there is any changes in run time and memory usage. - It's the same

import asyncio
import time
from TestingModules import Gather

start = time.time()
loop = asyncio.get_event_loop()
try:
    loop.run_until_complete(Gather())
finally:
    loop.close()
stop = time.time()
print(stop-start)

import time
import asyncio
import aiohttp
from pymongo import MongoClient, errors
from pymongo.server_api import ServerApi
from Credentials import Credentials
from datetime import datetime
from motor.motor_asyncio import AsyncIOMotorClient

def convert_milliseconds(ms):
    seconds = int(ms) / 1000.0
    return datetime.fromtimestamp(seconds)

async def async_scraper(ID):
    params = {
        'page': f'{ID}',
        'rows': '3000',
    }
    async with aiohttp.ClientSession() as session:
        async with session.get(url=url, headers=headers, params=params) as r:
            json = await r.json()
            batch = json['List']
            print(len(batch))
            for x in range(len(batch)):
                batch[x]["G_Date"] = timestamp
                try:
                    batch[x]["StartDate"] = convert_milliseconds(batch[x]["StartDate"][6:-2])
                except:
                    pass
                try:
                    batch[x]["EndDate"] = convert_milliseconds(batch[x]["EndDate"][6:-2])
                except:
                    pass
                try:
                    batch[x]["AsAt"] = convert_milliseconds(batch[x]["AsAt"][6:-2])
                except:
                    pass
                try:
                    batch[x]["OpenHomes"][0]["Start"] = convert_milliseconds(batch[x]["OpenHomes"][0]["Start"][6:-2])
                    batch[x]["OpenHomes"][0]["End"] = convert_milliseconds(batch[x]["OpenHomes"][0]["End"][6:-2])
                except:
                    pass
                try:
                    batch[x]["OpenHomes"][1]["Start"] = convert_milliseconds(batch[x]["OpenHomes"][1]["Start"][6:-2])
                    batch[x]["OpenHomes"][1]["End"] = convert_milliseconds(batch[x]["OpenHomes"][1]["End"][6:-2])
                except:
                    pass
            result = await coll.insert_many(batch, ordered=False )
            if result.acknowledged:
                print(f"batch: {ID} Completed")
                # print(batch[0])
                return True
            else:
                print(f">>>> batch: {ID} ERROR")
async def Gather():
    tasks = []
    for ID in range(/):
        tasks.append(async_scraper(ID))
    await asyncio.gather(*tasks)

def loop():
    loop = asyncio.get_event_loop()
    try:
        loop.run_until_complete(Gather())
    finally:
        loop.close()

headers = {
        'authority': 'api.trademe.co.nz',
        'method': 'GET',
        'accept': 'pplication/json, text/plain, */*',
        'accept-encoding': 'gzip, deflate, br',
        'accept-language': 'en,en-US;q=0.9',
        'cache-control': 'max-age=0',
        'newrelic': 'eyJ2IjpbMCwx=XSwiZCI6eyJ0eSI6IkJyb3dzZXIiLCJhYyI6IjQzODYzOCIsImFwIjoiMzgwMDc2Nzg0IiwiaWQiOiIxOGZjMWRkNzM1YTE4MmViIiwidHIiOiI4NGRiNzIyODE2Yjk1NzhjZDEzMTIxYjc3MGQ3MzQwMCIsInRpIjoxNjcwMjg3Mjg0NzU2fX0=',
        'origin': 'https://www.trademe.co.nz',
        'referer': 'https://www.trademe.co.nz/',
        'sec-ch-ua': '"Not?A_Brand";v="8", "Chromium";v="108", "Google Chrome";v="108"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-dest': 'empty',
        'sec-fetch-mode': 'cors',
        'sec-fetch-site': 'some-site',
        'upgrade-insecure-requests': '1',
    }
url = "Protecting a vulnerable API"

uri = Credentials.uri
client = AsyncIOMotorClient(uri, server_api=ServerApi('1'))
try:
    if client.server_info():
            print("Connected to MongoDB server")
            start = time.time()
            db = client["Tradme"]
            coll = db["Tradme_Property_Test"]
            current_datetime = datetime.now()
            timestamp = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
            loop()
            stop = time.time()
            print(stop-start)
            coll.estimated_document_count()
            client.close()


except errors.ConnectionFailure as e:
    print("Failed to connect to MongoDB server:", e)
except Exception as e:
    print("An error occurred:", e)

Solving performance issues:
20s to 197s? I know there is something wrong here.

Added await as there is some bottleneck.