DEV Community: Eko Priyanto

scrape data masjid SIMAS Kemenag

Eko Priyanto — Sat, 09 May 2026 10:12:22 +0000

get all URL:


<?php

/**
 * =========================================================
 * SIMAS KEMENAG SCRAPER
 * SQLITE + MULTICURL + AUTO RESUME
 * =========================================================
 *
 * FITUR:
 * - multicurl batch aman
 * - sqlite database
 * - auto resume dari sqlite
 * - anti duplicate
 * - anti banned ringan
 * - sanitize utf8
 * - low memory
 * - stable long scraping
 * - restart otomatis tinggal jalankan ulang
 *
 * OUTPUT:
 * simas.db
 *
 * =========================================================
 */

set_time_limit(0);
ini_set('memory_limit', '-1');

date_default_timezone_set('Asia/Jakarta');

// =========================================================
// CONFIG
// =========================================================

$startPage = 1;
$endPage   = 16000;

$batchSize = 5;

$baseUrl =
    "https://simas.kemenag.go.id/page/profilmasjid/0/0/0/0/0?page=";

$dbFile = __DIR__ . '/simas.db';

$failedLog = __DIR__ . '/failed.txt';

// =========================================================
// SQLITE
// =========================================================

$db = new PDO("sqlite:" . $dbFile);

$db->setAttribute(
    PDO::ATTR_ERRMODE,
    PDO::ERRMODE_EXCEPTION
);

// =========================================================
// SQLITE OPTIMIZATION
// =========================================================

$db->exec("
    PRAGMA journal_mode = WAL;
");

$db->exec("
    PRAGMA synchronous = NORMAL;
");

$db->exec("
    PRAGMA cache_size = 100000;
");

$db->exec("
    PRAGMA temp_store = MEMORY;
");

$db->exec("
    PRAGMA busy_timeout = 30000;
");

// =========================================================
// TABLE
// =========================================================

$db->exec("
CREATE TABLE IF NOT EXISTS masjid (
    id INTEGER PRIMARY KEY AUTOINCREMENT,

    page INTEGER,

    nama TEXT,

    alamat TEXT,

    link_detail TEXT UNIQUE,

    link_peta TEXT,

    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
");

// =========================================================
// INDEX
// =========================================================

$db->exec("
CREATE INDEX IF NOT EXISTS idx_page
ON masjid(page)
");

$db->exec("
CREATE INDEX IF NOT EXISTS idx_nama
ON masjid(nama)
");

// =========================================================
// PREPARED STATEMENT
// =========================================================

$insertStmt = $db->prepare("
INSERT OR IGNORE INTO masjid
(
    page,
    nama,
    alamat,
    link_detail,
    link_peta
)
VALUES
(
    :page,
    :nama,
    :alamat,
    :link_detail,
    :link_peta
)
");

// =========================================================
// AUTO RESUME
// =========================================================

$lastPage = $db->query("
    SELECT MAX(page) as last_page
    FROM masjid
")->fetch(PDO::FETCH_ASSOC);

if (!empty($lastPage['last_page'])) {

    $startPage = ((int)$lastPage['last_page']) + 1;

    echo "RESUME FROM PAGE : {$startPage}\n";
}

// =========================================================
// USER AGENT
// =========================================================

$userAgents = [

    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/136.0 Safari/537.36',

    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/135.0 Safari/537.36',

    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/134.0 Safari/537.36',

];

// =========================================================
// CLEAN TEXT
// =========================================================

function cleanText($text)
{
    if ($text === null) {
        return '';
    }

    $text = html_entity_decode(
        $text,
        ENT_QUOTES | ENT_HTML5,
        'UTF-8'
    );

    $text = preg_replace(
        '/[\x00-\x1F\x7F]/u',
        ' ',
        $text
    );

    $text = preg_replace(
        '/\s+/u',
        ' ',
        $text
    );

    return trim($text);
}

// =========================================================
// CURL HANDLE
// =========================================================

function createCurlHandle($url, $userAgent)
{
    $ch = curl_init();

    curl_setopt_array($ch, [

        CURLOPT_URL => $url,

        CURLOPT_RETURNTRANSFER => true,

        CURLOPT_FOLLOWLOCATION => true,

        CURLOPT_SSL_VERIFYPEER => false,

        CURLOPT_CONNECTTIMEOUT => 20,

        CURLOPT_TIMEOUT => 60,

        CURLOPT_ENCODING => 'gzip',

        CURLOPT_USERAGENT => $userAgent,

        CURLOPT_HTTPHEADER => [

            'Accept: text/html,application/xhtml+xml',

            'Cache-Control: no-cache',

            'Pragma: no-cache',

        ]

    ]);

    return $ch;
}

// =========================================================
// PARSER
// =========================================================

function parsePage($html, $page)
{
    $rows = [];

    libxml_use_internal_errors(true);

    $dom = new DOMDocument();

    if (!$dom->loadHTML($html)) {
        return [];
    }

    $xpath = new DOMXPath($dom);

    $items = $xpath->query(
        "//div[contains(@class,'search-result-item')]"
    );

    foreach ($items as $item) {

        $nama = '';
        $alamat = '';
        $linkDetail = '';
        $linkPeta = '';

        // =================================================
        // NAMA
        // =================================================

        $namaNode = $xpath->query(
            ".//h4",
            $item
        );

        if ($namaNode->length > 0) {

            $nama = trim(
                $namaNode->item(0)->textContent
            );
        }

        // =================================================
        // ALAMAT
        // =================================================

        $alamatNode = $xpath->query(
            ".//div[contains(@class,'search-result-content')]/p[1]",
            $item
        );

        if ($alamatNode->length > 0) {

            $alamat = trim(
                $alamatNode->item(0)->textContent
            );
        }

        // =================================================
        // LINK DETAIL
        // =================================================

        $detailNode = $xpath->query(
            ".//a[contains(text(),'Lihat Detil')]",
            $item
        );

        if ($detailNode->length > 0) {

            $linkDetail = trim(
                $detailNode
                    ->item(0)
                    ->getAttribute('href')
            );
        }

        // =================================================
        // LINK PETA
        // =================================================

        $mapNode = $xpath->query(
            ".//a[contains(text(),'Lihat di Peta')]",
            $item
        );

        if ($mapNode->length > 0) {

            $linkPeta = trim(
                $mapNode
                    ->item(0)
                    ->getAttribute('href')
            );
        }

        // =================================================
        // VALIDASI
        // =================================================

        if (
            empty($nama) &&
            empty($alamat) &&
            empty($linkDetail)
        ) {
            continue;
        }

        $rows[] = [

            'page' => $page,

            'nama' => cleanText($nama),

            'alamat' => cleanText($alamat),

            'link_detail' => cleanText($linkDetail),

            'link_peta' => cleanText($linkPeta)

        ];
    }

    unset($dom);
    unset($xpath);
    unset($items);

    gc_collect_cycles();

    return $rows;
}

// =========================================================
// MAIN LOOP
// =========================================================

$totalData = 0;
$totalSuccess = 0;
$totalFailed = 0;

$startTime = time();

for ($i = $startPage; $i <= $endPage; $i += $batchSize) {

    echo "\n==================================================\n";
    echo "BATCH : {$i}\n";
    echo "TIME  : " . date('Y-m-d H:i:s') . "\n";

    $multiHandle = curl_multi_init();

    $handles = [];

    // =====================================================
    // CREATE MULTICURL
    // =====================================================

    for ($x = 0; $x < $batchSize; $x++) {

        $page = $i + $x;

        if ($page > $endPage) {
            break;
        }

        $url = $baseUrl . $page;

        echo "QUEUE PAGE {$page}\n";

        $ch = createCurlHandle(
            $url,
            $userAgents[array_rand($userAgents)]
        );

        $handles[$page] = [
            'handle' => $ch,
            'url' => $url
        ];

        curl_multi_add_handle(
            $multiHandle,
            $ch
        );

        usleep(rand(100000, 300000));
    }

    // =====================================================
    // EXECUTE MULTICURL
    // =====================================================

    $running = null;

    do {

        curl_multi_exec(
            $multiHandle,
            $running
        );

        curl_multi_select(
            $multiHandle,
            1
        );

    } while ($running > 0);

    // =====================================================
    // HANDLE RESULT
    // =====================================================

    foreach ($handles as $page => $info) {

        $ch = $info['handle'];

        $html = curl_multi_getcontent($ch);

        $httpCode = curl_getinfo(
            $ch,
            CURLINFO_HTTP_CODE
        );

        $error = curl_error($ch);

        curl_multi_remove_handle(
            $multiHandle,
            $ch
        );

        curl_close($ch);

        // =================================================
        // ERROR CURL
        // =================================================

        if (!empty($error)) {

            echo "ERROR PAGE {$page} : {$error}\n";

            file_put_contents(
                $failedLog,
                $page . PHP_EOL,
                FILE_APPEND
            );

            $totalFailed++;

            continue;
        }

        // =================================================
        // HTTP ERROR
        // =================================================

        if ($httpCode != 200) {

            echo "HTTP {$httpCode} PAGE {$page}\n";

            file_put_contents(
                $failedLog,
                $page . PHP_EOL,
                FILE_APPEND
            );

            $totalFailed++;

            continue;
        }

        // =================================================
        // PARSE HTML
        // =================================================

        $rows = parsePage($html, $page);

        if (count($rows) == 0) {

            echo "EMPTY PAGE {$page}\n";

            file_put_contents(
                $failedLog,
                $page . PHP_EOL,
                FILE_APPEND
            );

            $totalFailed++;

            continue;
        }

        // =================================================
        // INSERT SQLITE
        // =================================================

        $db->beginTransaction();

        foreach ($rows as $row) {

            $insertStmt->execute([

                ':page' => $row['page'],

                ':nama' => $row['nama'],

                ':alamat' => $row['alamat'],

                ':link_detail' => $row['link_detail'],

                ':link_peta' => $row['link_peta']

            ]);

            $totalData++;
        }

        $db->commit();

        echo "SUCCESS PAGE {$page} => "
            . count($rows)
            . " DATA\n";

        $totalSuccess++;

        unset($rows);
        unset($html);
    }

    curl_multi_close($multiHandle);

    // =====================================================
    // COOL DOWN
    // =====================================================

    if ($i % 100 == 0) {

        echo "\nCOOL DOWN 15 DETIK...\n";

        sleep(15);

    } else {

        usleep(rand(700000, 2000000));
    }

    gc_collect_cycles();

    // =====================================================
    // STATS
    // =====================================================

    $elapsed = time() - $startTime;

    echo "\nTOTAL DATA   : {$totalData}\n";
    echo "SUCCESS PAGE : {$totalSuccess}\n";
    echo "FAILED PAGE  : {$totalFailed}\n";
    echo "ELAPSED      : {$elapsed} detik\n";
}

// =========================================================
// FINISH
// =========================================================

$db = null;

$elapsed = time() - $startTime;

echo "\n==================================================\n";
echo "SCRAPING SELESAI\n";
echo "TOTAL DATA   : {$totalData}\n";
echo "SUCCESS PAGE : {$totalSuccess}\n";
echo "FAILED PAGE  : {$totalFailed}\n";
echo "TOTAL TIME   : {$elapsed} detik\n";
echo "DATABASE     : {$dbFile}\n";
echo "FAILED LOG   : {$failedLog}\n";
echo "==================================================\n";

Get detail, update ke table uri


<?php

/**
 * =========================================================
 * SIMAS DETAIL SCRAPER
 * SQLITE + AUTO RESUME + MULTICURL
 * =========================================================
 *
 * REQUIREMENT:
 * composer require symfony/dom-crawler symfony/css-selector
 *
 * FITUR:
 * - ambil semua link dari sqlite
 * - scrape detail otomatis
 * - auto resume
 * - multicurl batch
 * - anti duplicate
 * - aman karakter aneh
 * - prepared statement
 * - rollback transaction
 * - restart tinggal jalankan ulang
 *
 * =========================================================
 */

set_time_limit(0);
ini_set('memory_limit', '-1');

require 'vendor/autoload.php';

use Symfony\Component\DomCrawler\Crawler;

// =========================================================
// SQLITE
// =========================================================

$db = new PDO('sqlite:' . __DIR__ . '/simas.db');

$db->setAttribute(
    PDO::ATTR_ERRMODE,
    PDO::ERRMODE_EXCEPTION
);

// =========================================================
// SQLITE OPTIMIZATION
// =========================================================

$db->exec("PRAGMA journal_mode=WAL");
$db->exec("PRAGMA synchronous=NORMAL");
$db->exec("PRAGMA cache_size=100000");
$db->exec("PRAGMA temp_store=MEMORY");
$db->exec("PRAGMA busy_timeout=30000");

// =========================================================
// TAMBAH KOLOM JIKA BELUM ADA
// =========================================================

$columns = [

    'detail_scraped INTEGER DEFAULT 0',

    'jenis TEXT',
    'no_id_masjid_mushalla TEXT',
    'didirikan_tahun TEXT',

    'alamat_baris_1 TEXT',
    'alamat_baris_2 TEXT',
    'alamat_baris_3 TEXT',

    'alamat_email TEXT',
    'alamat_web TEXT',

    'jumlah_pengurus TEXT',
    'jumlah_imam TEXT',
    'jumlah_khatib TEXT',
    'jumlah_muazin TEXT',
    'jumlah_remaja_masjid TEXT',

    'luas_tanah TEXT',
    'status_tanah TEXT',
    'luas_bangunan TEXT',
    'daya_tampung_jamaah TEXT',

    'sejarah_masjid TEXT',
    'foto_masjid TEXT'
];

$tableInfo = $db->query("
    PRAGMA table_info(masjid)
")->fetchAll(PDO::FETCH_ASSOC);

$existingColumns = [];

foreach ($tableInfo as $col) {
    $existingColumns[] = $col['name'];
}

foreach ($columns as $col) {

    $colName = explode(' ', trim($col))[0];

    if (!in_array($colName, $existingColumns)) {

        $db->exec("
            ALTER TABLE masjid
            ADD COLUMN {$col}
        ");

        echo "ADD COLUMN : {$colName}\n";
    }
}

// =========================================================
// CONFIG
// =========================================================

$batchSize = 5;

$userAgents = [

    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/137 Safari/537.36',

    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/136 Safari/537.36',

    'Mozilla/5.0 (Macintosh; Intel Mac OS X) AppleWebKit/537.36 Chrome/135 Safari/537.36',

];

// =========================================================
// CLEAN TEXT
// =========================================================

function cleanText(?string $text): string
{
    $text = html_entity_decode(
        $text ?? '',
        ENT_QUOTES | ENT_HTML5,
        'UTF-8'
    );

    $text = strip_tags($text);

    $text = preg_replace(
        '/[\x00-\x1F\x7F]/u',
        ' ',
        $text
    );

    $text = preg_replace('/\s+/u', ' ', $text);

    return trim($text);
}

// =========================================================
// GET TEXT
// =========================================================

function getNodeText(
    Crawler $crawler,
    string $selector,
    int $index = 0
): string {

    try {

        return cleanText(
            $crawler
                ->filter($selector)
                ->eq($index)
                ->text()
        );

    } catch (\Throwable $e) {

        return '';
    }
}

// =========================================================
// GET PROFIL VALUE
// =========================================================

function getProfilValue(
    Crawler $crawler,
    string $labelCari
): string {

    $hasil = '';

    try {

        $crawler
            ->filter('.section-content-info-wrapper .row')
            ->each(function ($row)
            use ($labelCari, &$hasil) {

                $cols = $row->filter('.label');

                if ($cols->count() >= 2) {

                    $label = cleanText(
                        $cols->eq(0)->text()
                    );

                    $value = cleanText(
                        $cols->eq(1)->text()
                    );

                    if (
                        stripos($label, $labelCari)
                        !== false
                    ) {

                        $hasil = $value;
                    }
                }
            });

    } catch (\Throwable $e) {
    }

    return $hasil;
}

// =========================================================
// PARSE DETAIL
// =========================================================

function parseMasjidDetail($html)
{
    $crawler = new Crawler($html);

    // =====================================================
    // BASIC
    // =====================================================

    $jenis = getNodeText(
        $crawler,
        '.masjid-sub-title'
    );

    $nama = getNodeText(
        $crawler,
        '.masjid-title'
    );

    $noId = getNodeText(
        $crawler,
        '.masjid-card a'
    );

    // =====================================================
    // TAHUN
    // =====================================================

    $didirikan = '';

    try {

        $crawler
            ->filter('.masjid-alamat-calendar p')
            ->each(function ($node)
            use (&$didirikan) {

                $text = cleanText(
                    $node->text()
                );

                if (
                    preg_match(
                        '/(\d{4})/',
                        $text,
                        $match
                    )
                ) {

                    $didirikan = $match[1];
                }
            });

    } catch (\Throwable $e) {
    }

    // =====================================================
    // ALAMAT
    // =====================================================

    $alamat = '';
    $alamat1 = '';
    $alamat2 = '';
    $alamat3 = '';

    try {

        $alamatNode = $crawler
            ->filter('.masjid-alamat-location p')
            ->first();

        if ($alamatNode->count()) {

            $htmlAlamat = $alamatNode->html();

            $lines = preg_split(
                '/<br\s*\/?>/i',
                $htmlAlamat
            );

            $lines = array_map(function ($v) {

                return cleanText($v);

            }, $lines);

            $lines = array_values(
                array_filter($lines)
            );

            $alamat1 = $lines[0] ?? '';
            $alamat2 = $lines[1] ?? '';
            $alamat3 = $lines[2] ?? '';

            $alamat = implode(', ', $lines);
        }

    } catch (\Throwable $e) {
    }

    // =====================================================
    // EMAIL & WEBSITE
    // =====================================================

    $email = '';
    $website = '';

    try {

        $crawler
            ->filter('.masjid-alamat-phone')
            ->each(function ($node)
            use (&$email, &$website) {

                $icon = '';

                try {

                    $icon = $node
                        ->filter('i')
                        ->attr('class');

                } catch (\Throwable $e) {
                }

                if (
                    strpos($icon, 'ti-email')
                    !== false
                ) {

                    $email = cleanText(
                        $node
                            ->filter('p')
                            ->text()
                    );
                }

                if (
                    strpos($icon, 'ti-desktop')
                    !== false
                ) {

                    $website = cleanText(
                        $node
                            ->filter('p')
                            ->text()
                    );
                }
            });

    } catch (\Throwable $e) {
    }

    // =====================================================
    // SUMMARY
    // =====================================================

    $summary = [];

    try {

        $crawler
            ->filter('.summary-item')
            ->each(function ($node)
            use (&$summary) {

                $judul = cleanText(
                    $node->filter('h4')->text()
                );

                $nilai = cleanText(
                    $node->filter('span')->text()
                );

                $summary[$judul] = $nilai;
            });

    } catch (\Throwable $e) {
    }

    // =====================================================
    // PROFIL
    // =====================================================

    $luasTanah = getProfilValue(
        $crawler,
        'Luas Tanah'
    );

    $statusTanah = getProfilValue(
        $crawler,
        'Status Tanah'
    );

    $luasBangunan = getProfilValue(
        $crawler,
        'Luas Bangunan'
    );

    $dayaTampung = getProfilValue(
        $crawler,
        'Daya Tampung Jamaah'
    );

    // =====================================================
    // SEJARAH
    // =====================================================

    $sejarah = '';

    try {

        $crawler
            ->filter('#content-sejarah')
            ->each(function ($node)
            use (&$sejarah) {

                $sejarah .= ' ' .
                    cleanText(
                        $node->text()
                    );
            });

        $sejarah = trim($sejarah);

    } catch (\Throwable $e) {
    }

    // =====================================================
    // FOTO
    // =====================================================

    $fotos = [];

    try {

        $crawler
            ->filter('.profil-masjid-photos a')
            ->each(function ($node)
            use (&$fotos) {

                $href = trim(
                    $node->attr('href')
                );

                if (!empty($href)) {
                    $fotos[] = $href;
                }
            });

    } catch (\Throwable $e) {
    }

    $fotos = array_unique($fotos);

    return [

        'nama' => $nama,

        'jenis' => $jenis,

        'no_id_masjid_mushalla' => $noId,

        'didirikan_tahun' => $didirikan,

        'alamat' => $alamat,

        'alamat_baris_1' => $alamat1,
        'alamat_baris_2' => $alamat2,
        'alamat_baris_3' => $alamat3,

        'alamat_email' => $email,
        'alamat_web' => $website,

        'jumlah_pengurus' =>
            $summary['Jumlah Pengurus'] ?? '',

        'jumlah_imam' =>
            $summary['Jumlah Imam'] ?? '',

        'jumlah_khatib' =>
            $summary['Jumlah Khatib'] ?? '',

        'jumlah_muazin' =>
            $summary['Jumlah Muazin'] ?? '',

        'jumlah_remaja_masjid' =>
            $summary['Jumlah Remaja Masjid'] ?? '',

        'luas_tanah' => $luasTanah,
        'status_tanah' => $statusTanah,
        'luas_bangunan' => $luasBangunan,
        'daya_tampung_jamaah' => $dayaTampung,

        'sejarah_masjid' => $sejarah,

        'foto_masjid' => implode(
            ' | ',
            $fotos
        )
    ];
}

// =========================================================
// UPDATE QUERY
// =========================================================

$updateStmt = $db->prepare("
UPDATE masjid SET

    nama = :nama,

    jenis = :jenis,

    no_id_masjid_mushalla = :no_id_masjid_mushalla,

    didirikan_tahun = :didirikan_tahun,

    alamat = :alamat,

    alamat_baris_1 = :alamat_baris_1,
    alamat_baris_2 = :alamat_baris_2,
    alamat_baris_3 = :alamat_baris_3,

    alamat_email = :alamat_email,
    alamat_web = :alamat_web,

    jumlah_pengurus = :jumlah_pengurus,
    jumlah_imam = :jumlah_imam,
    jumlah_khatib = :jumlah_khatib,
    jumlah_muazin = :jumlah_muazin,
    jumlah_remaja_masjid = :jumlah_remaja_masjid,

    luas_tanah = :luas_tanah,
    status_tanah = :status_tanah,
    luas_bangunan = :luas_bangunan,
    daya_tampung_jamaah = :daya_tampung_jamaah,

    sejarah_masjid = :sejarah_masjid,

    foto_masjid = :foto_masjid,

    detail_scraped = 1

WHERE id = :id
");

// =========================================================
// TOTAL
// =========================================================

$total = $db->query("
    SELECT COUNT(*)
    FROM masjid
    WHERE detail_scraped = 0
")->fetchColumn();

echo "TOTAL BELUM SCRAPE : {$total}\n";

// =========================================================
// MAIN LOOP
// =========================================================

while (true) {

    $stmt = $db->prepare("
        SELECT id, link_detail
        FROM masjid
        WHERE detail_scraped = 0
        LIMIT :limit
    ");

    $stmt->bindValue(
        ':limit',
        $batchSize,
        PDO::PARAM_INT
    );

    $stmt->execute();

    $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);

    if (!$rows) {
        break;
    }

    echo "\n===================================\n";
    echo "NEW BATCH\n";

    $mh = curl_multi_init();

    $handles = [];

    // =====================================================
    // QUEUE CURL
    // =====================================================

    foreach ($rows as $row) {

        echo "QUEUE : {$row['link_detail']}\n";

        $ch = curl_init();

        curl_setopt_array($ch, [

            CURLOPT_URL => $row['link_detail'],

            CURLOPT_RETURNTRANSFER => true,

            CURLOPT_FOLLOWLOCATION => true,

            CURLOPT_SSL_VERIFYPEER => false,

            CURLOPT_TIMEOUT => 30,

            CURLOPT_ENCODING => 'gzip',

            CURLOPT_USERAGENT =>
                $userAgents[array_rand($userAgents)]

        ]);

        curl_multi_add_handle($mh, $ch);

        $handles[] = [
            'id' => $row['id'],
            'url' => $row['link_detail'],
            'handle' => $ch
        ];

        usleep(rand(100000, 300000));
    }

    // =====================================================
    // EXECUTE
    // =====================================================

    $running = null;

    do {

        curl_multi_exec($mh, $running);

        curl_multi_select($mh);

    } while ($running > 0);

    // =====================================================
    // RESULT
    // =====================================================

    foreach ($handles as $item) {

        $id = $item['id'];

        $url = $item['url'];

        $ch = $item['handle'];

        $html = curl_multi_getcontent($ch);

        $http = curl_getinfo(
            $ch,
            CURLINFO_HTTP_CODE
        );

        $error = curl_error($ch);

        curl_multi_remove_handle($mh, $ch);

        curl_close($ch);

        // =================================================
        // ERROR CURL
        // =================================================

        if (!empty($error)) {

            echo "CURL ERROR : {$url}\n";
            echo $error . "\n";

            continue;
        }

        // =================================================
        // HTTP ERROR
        // =================================================

        if ($http != 200) {

            echo "HTTP {$http} : {$url}\n";

            continue;
        }

        // =================================================
        // PARSE + SAVE
        // =================================================

        try {

            $detail = parseMasjidDetail($html);

            $detail['id'] = $id;

            if ($db->inTransaction()) {
                $db->rollBack();
            }

            $db->beginTransaction();

            $updateStmt->execute($detail);

            $db->commit();

            echo "SUCCESS : {$url}\n";

        } catch (\Throwable $e) {

            if ($db->inTransaction()) {
                $db->rollBack();
            }

            echo "FAILED : {$url}\n";
            echo $e->getMessage() . "\n";
        }

        unset($html);
        unset($detail);

        gc_collect_cycles();
    }

    curl_multi_close($mh);

    echo "COOLDOWN...\n";

    usleep(rand(1000000, 3000000));
}

// =========================================================
// FINISH
// =========================================================

echo "\n===================================\n";
echo "SELESAI\n";
echo "===================================\n";

cara menjalankannya adalah dengan menjalankan di CMD atau terminal

nohup php scrape.php

Database sekolah Indonesia

Eko Priyanto — Sat, 09 May 2026 10:07:47 +0000

Database sekolah Indonesia dengan cara scrape dari api tidak resmi https://api-sekolah-indonesia.vercel.app/sekolah/s?sekolah=&perPage=100

berikut script multicurl untuk generate csv file dari api tersebut.

<?php

set_time_limit(0);

// CONFIG
$baseUrl   = "https://api-sekolah-indonesia.vercel.app/sekolah/s?sekolah=&perPage=100&page=";
$maxBatch  = 10; // jumlah request paralel
$output    = __DIR__ . "/sekolah.csv";

// buka file CSV
$fp = fopen($output, 'w');

// header CSV
fputcsv($fp, [
    'npsn', 'sekolah', 'bentuk', 'status',
    'alamat', 'kecamatan', 'kabupaten', 'provinsi',
    'lintang', 'bujur'
]);

$page = 1;
$running = true;

while ($running) {

    $mh = curl_multi_init();
    $handles = [];

    // batching request
    for ($i = 0; $i < $maxBatch; $i++) {
        $url = $baseUrl . ($page + $i);

        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT => 30,
        ]);

        curl_multi_add_handle($mh, $ch);
        $handles[$page + $i] = $ch;
    }

    // eksekusi multicurl
    do {
        curl_multi_exec($mh, $runningCurl);
        curl_multi_select($mh);
    } while ($runningCurl > 0);

    // proses hasil
    $running = false;

    foreach ($handles as $p => $ch) {
        $response = curl_multi_getcontent($ch);
        $data = json_decode($response, true);

        if (!empty($data['dataSekolah'])) {
            $running = true;

            foreach ($data['dataSekolah'] as $row) {

                fputcsv($fp, [
                    trim($row['npsn']),
                    trim($row['sekolah']),
                    trim($row['bentuk']),
                    trim($row['status']),
                    trim($row['alamat_jalan']),
                    trim($row['kecamatan']),
                    trim($row['kabupaten_kota']),
                    trim($row['propinsi']),
                    trim($row['lintang']),
                    trim($row['bujur']),
                ]);
            }

            echo "Page $p selesai\n";
        }

        curl_multi_remove_handle($mh, $ch);
        curl_close($ch);
    }

    curl_multi_close($mh);

    $page += $maxBatch;

    echo "Batch selesai, lanjut ke page $page\n";
}

fclose($fp);

echo "Selesai! File: $output\n";

Database sekolah Indonesia

Eko Priyanto — Sun, 12 Apr 2026 02:22:51 +0000

Database sekolah Indonesia dengan cara scrape dari api tidak resmi https://api-sekolah-indonesia.vercel.app/sekolah/s?sekolah=&perPage=100

berikut script multicurl untuk generate csv file dari api tersebut.

<?php

set_time_limit(0);

// CONFIG
$baseUrl   = "https://api-sekolah-indonesia.vercel.app/sekolah/s?sekolah=&perPage=100&page=";
$maxBatch  = 10; // jumlah request paralel
$output    = __DIR__ . "/sekolah.csv";

// buka file CSV
$fp = fopen($output, 'w');

// header CSV
fputcsv($fp, [
    'npsn', 'sekolah', 'bentuk', 'status',
    'alamat', 'kecamatan', 'kabupaten', 'provinsi',
    'lintang', 'bujur'
]);

$page = 1;
$running = true;

while ($running) {

    $mh = curl_multi_init();
    $handles = [];

    // batching request
    for ($i = 0; $i < $maxBatch; $i++) {
        $url = $baseUrl . ($page + $i);

        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT => 30,
        ]);

        curl_multi_add_handle($mh, $ch);
        $handles[$page + $i] = $ch;
    }

    // eksekusi multicurl
    do {
        curl_multi_exec($mh, $runningCurl);
        curl_multi_select($mh);
    } while ($runningCurl > 0);

    // proses hasil
    $running = false;

    foreach ($handles as $p => $ch) {
        $response = curl_multi_getcontent($ch);
        $data = json_decode($response, true);

        if (!empty($data['dataSekolah'])) {
            $running = true;

            foreach ($data['dataSekolah'] as $row) {

                fputcsv($fp, [
                    trim($row['npsn']),
                    trim($row['sekolah']),
                    trim($row['bentuk']),
                    trim($row['status']),
                    trim($row['alamat_jalan']),
                    trim($row['kecamatan']),
                    trim($row['kabupaten_kota']),
                    trim($row['propinsi']),
                    trim($row['lintang']),
                    trim($row['bujur']),
                ]);
            }

            echo "Page $p selesai\n";
        }

        curl_multi_remove_handle($mh, $ch);
        curl_close($ch);
    }

    curl_multi_close($mh);

    $page += $maxBatch;

    echo "Batch selesai, lanjut ke page $page\n";
}

fclose($fp);

echo "Selesai! File: $output\n";

Mencoba global forest watch API

Eko Priyanto — Tue, 27 Jan 2026 02:04:54 +0000

Saat pertama coba selalu gagal API key tidak sesuai

ini mungkin works

<?php
$apiKey = 'ini-api-key-tanpa-domain-restriction'; // API key tanpa domain restriction

// Define geometry (polygon untuk area Trenggalek)
$geometry = [
    "type" => "Polygon",
    "coordinates" => [[
        [103.19732666015625, 0.5537709801264608],
        [103.24882507324219, 0.5647567848663363],
        [103.21277618408203, 0.5932511181408705],
        [103.19732666015625, 0.5537709801264608]
    ]]
];

// Step 1: Create geostore dari geometry
$geostoreUrl = 'https://data-api.globalforestwatch.org/geostore';
$geostorePayload = json_encode(["geojson" => $geometry]);

$ch = curl_init($geostoreUrl);

curl_setopt_array($ch, [
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_POST => true,
    CURLOPT_FOLLOWLOCATION => true,
    CURLOPT_HTTPHEADER => [
        'x-api-key: ' . $apiKey,
        'Content-Type: application/json',
        'Content-Length: ' . strlen($geostorePayload)
    ],
    CURLOPT_POSTFIELDS => $geostorePayload,
]);

$geostoreResponse = curl_exec($ch);
$geostoreHttpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);

$geostoreResult = json_decode($geostoreResponse, true);

if (!isset($geostoreResult['data']['id'])) {
    die("Gagal membuat geostore. HTTP: $geostoreHttpCode\n");
}

$geostoreId = $geostoreResult['data']['id'];
echo "Geostore ID: $geostoreId\n";

// Step 2: Query dataset dengan geostore dan geometry
$queryUrl = "https://data-api.globalforestwatch.org/dataset/umd_tree_cover_loss/v1.12/query/json?geostore=$geostoreId";

$queryPayload = json_encode([
    "sql" => "SELECT SUM(area__ha) as total_area FROM results WHERE umd_tree_cover_loss__year = 2019",
    "geometry" => $geometry
]);

$ch = curl_init($queryUrl);

curl_setopt_array($ch, [
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_POST => true,
    CURLOPT_FOLLOWLOCATION => true,
    CURLOPT_HTTPHEADER => [
        'x-api-key: ' . $apiKey,
        'Content-Type: application/json',
        'Content-Length: ' . strlen($queryPayload)
    ],
    CURLOPT_POSTFIELDS => $queryPayload,
]);

$queryResponse = curl_exec($ch);
$queryHttpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);

echo "Query HTTP: $queryHttpCode\n";

$result = json_decode($queryResponse, true);

echo "<pre>";
print_r($result);
echo "</pre>";

ini dokumentasi yang bagus:
https://developer.openepi.io/how-tos/getting-started-using-global-forest-watch-data-api

Grab JDIHN berdasarkan jenis dokumen

Eko Priyanto — Tue, 25 Nov 2025 05:13:55 +0000

Grab JDIHN berdasarkan jenis dokumen.

<?php

/**
 * Skrip interaktif untuk mengambil data dari API jdihn.go.id
 * dan mengonversinya menjadi file CSV.
 * Pengguna dapat memilih jenis dokumen dari daftar yang tersedia.
 */

// Non-aktifkan waktu eksekusi maksimum PHP, karena prosesnya bisa memakan waktu.
// Hapus baris ini jika Anda berada di lingkungan hosting yang membatasi fungsi ini.
set_time_limit(0);

// --- Langkah 0: Tampilkan daftar jenis dokumen dan minta input dari pengguna ---

// Data JSON yang berisi daftar jenis dokumen
 $jenisDokumenJson = '[
    {"id":4,"name":"MONOGRAFI HUKUM"},
    {"id":5,"name":"ARTIKEL HUKUM"},
    {"id":7,"name":"UNDANG-UNDANG DASAR"},
    {"id":8,"name":"KETETAPAN MPR"},
    {"id":9,"name":"UNDANG-UNDANG"},
    {"id":10,"name":"PERATURAN PEMERINTAH PENGGANTI UNDANG-UNDANG"},
    {"id":11,"name":"PERATURAN PEMERINTAH"},
    {"id":12,"name":"PERATURAN PRESIDEN"},
    {"id":13,"name":"KEPUTUSAN PRESIDEN"},
    {"id":14,"name":"INSTRUKSI PRESIDEN"},
    {"id":15,"name":"PENETAPAN PRESIDEN"},
    {"id":16,"name":"UNDANG-UNDANG DARURAT"},
    {"id":17,"name":"PENGATURAN PENGUASA PERANG TERTINGGI"},
    {"id":27,"name":"PERATURAN KEMENTERIAN"},
    {"id":28,"name":"PERATURAN BERSAMA"},
    {"id":29,"name":"KEPUTUSAN KEMENTERIAN"},
    {"id":30,"name":"KEPUTUSAN BERSAMA"},
    {"id":31,"name":"INSTRUKSI KEMENTERIAN"},
    {"id":33,"name":"SURAT EDARAN KEMENTERIAN"},
    {"id":34,"name":"PERATURAN ESELON 1"},
    {"id":35,"name":"KEPUTUSAN ESELON 1"},
    {"id":36,"name":"PERATURAN LEMBAGA NEGARA"},
    {"id":42,"name":"PERATURAN BPK"},
    {"id":44,"name":"KEPUTUSAN LEMBAGA NEGARA"},
    {"id":45,"name":"PERATURAN LEMBAGA PEMERINTAH NON KEMENTERIAN"},
    {"id":47,"name":"PERATURAN KEPALA"},
    {"id":48,"name":"KEPUTUSAN LEMBAGA PEMERINTAH NON KEMENTERIAN"},
    {"id":49,"name":"PERATURAN LEMBAGA NON STRUKTURAL"},
    {"id":56,"name":"KEPUTUSAN LEMBAGA NON STRUKTURAL"},
    {"id":57,"name":"PERATURAN DAERAH PROVINSI"},
    {"id":58,"name":"PERATURAN DAERAH KABUPATEN"},
    {"id":59,"name":"PERATURAN DAERAH KOTA"},
    {"id":60,"name":"PERATURAN DPRD PROVINSI"},
    {"id":61,"name":"PERATURAN DPRD KABUPATEN"},
    {"id":62,"name":"PERATURAN DPRD KOTA"},
    {"id":63,"name":"PERATURAN GUBERNUR"},
    {"id":64,"name":"PERATURAN BUPATI"},
    {"id":65,"name":"PERATURAN WALIKOTA"},
    {"id":67,"name":"PERATURAN DESA"},
    {"id":68,"name":"KEPUTUSAN GUBERNUR"},
    {"id":69,"name":"KEPUTUSAN BUPATI"},
    {"id":70,"name":"KEPUTUSAN WALIKOTA"},
    {"id":71,"name":"KEPUTUSAN DPRD"},
    {"id":72,"name":"INSTRUKSI GUBERNUR"},
    {"id":73,"name":"INSTRUKSI BUPATI"},
    {"id":74,"name":"INSTRUKSI WALIKOTA"},
    {"id":75,"name":"SURAT EDARAN PEMERINTAH DAERAH"},
    {"id":76,"name":"STAATSBLAD"},
    {"id":77,"name":"BUKU HUKUM"},
    {"id":78,"name":"NASKAH AKADEMIK KEMENKUMHAM"},
    {"id":79,"name":"NASKAH AKADEMIK"},
    {"id":80,"name":"PENELITIAN HUKUM DAN HAM"},
    {"id":81,"name":"PENGKAJIAN HUKUM"},
    {"id":83,"name":"PENULISAN KARYA ILMIAH"},
    {"id":84,"name":"KOMPENDIUM HUKUM"},
    {"id":85,"name":"ANALISIS DAN EVALUASI"},
    {"id":86,"name":"MAJALAH HUKUM NASIONAL"},
    {"id":87,"name":"MAJALAH HUKUM"},
    {"id":89,"name":"KLIPING MAJALAH KORAN"},
    {"id":90,"name":"JURNAL HUKUM"},
    {"id":91,"name":"YURISPRUDENSI"},
    {"id":92,"name":"PUTUSAN MAHKAMAH AGUNG"},
    {"id":93,"name":"PUTUSAN MAHKAMAH KONSTITUSI"},
    {"id":94,"name":"RANCANGAN"},
    {"id":95,"name":"RANCANGAN PERATURAN DAERAH"},
    {"id":96,"name":"RANCANGAN PERATURAN DAERAH PROVINSI"},
    {"id":97,"name":"RANCANGAN PERATURAN DAERAH KABUPATEN"},
    {"id":98,"name":"RANCANGAN PERATURAN DAERAH KOTA"},
    {"id":100,"name":"RANCANGAN PERATURAN BUPATI"},
    {"id":103,"name":"MEMORANDUM OF UNDERSTANDING"},
    {"id":104,"name":"PERJANJIAN KERJA SAMA"},
    {"id":105,"name":"ARTIKEL"},
    {"id":162,"name":"SURAT EDARAN"},
    {"id":163,"name":"INSTRUKSI LEMBAGA PEMERINTAH NON KEMENTERIAN"},
    {"id":217,"name":"LOKAKARYA"},
    {"id":492,"name":"OSAMU SEIREI"},
    {"id":493,"name":"OSAMU KANREI"},
    {"id":624,"name":"PENELITIAN HUKUM"},
    {"id":683,"name":"RANCANGAN PERATURAN BADAN"},
    {"id":685,"name":"SURAT EDARAN LEMBAGA NEGARA"},
    {"id":686,"name":"SURAT EDARAN LEMBAGA PEMERINTAH NON KEMENTERIAN"},
    {"id":687,"name":"SURAT EDARAN LEMBAGA NON STRUKTURAL"},
    {"id":688,"name":"RISALAH SIDANG"},
    {"id":696,"name":"KEPUTUSAN PIMPINAN DPRD"},
    {"id":720,"name":"RISALAH DPRD"},
    {"id":794,"name":"KEPUTUSAN REKTOR"},
    {"id":804,"name":"SURAT EDARAN KEPALA DESA"},
    {"id":815,"name":"SURAT KEPUTUSAN DPRD KABUPATEN"},
    {"id":816,"name":"KEPUTUSAN BERSAMA DPRD"},
    {"id":817,"name":"SURAT KEPUTUSAN"},
    {"id":823,"name":"KEPUTUSAN ESELON II"},
    {"id":826,"name":"PERATURAN KEPALA DESA"},
    {"id":831,"name":"HIMPUNAN"},
    {"id":833,"name":"KESEPAKATAN BERSAMA BUPATI"},
    {"id":841,"name":"PERATURAN DEWAN"},
    {"id":844,"name":"HIMPUNAN PERATURAN"},
    {"id":892,"name":"STATUTEN"},
    {"id":911,"name":"PERATURAN KOMISI"},
    {"id":973,"name":"KEPUTUSAN BADAN MUSYAWARAH DPRD"},
    {"id":1013,"name":"BERITA ACARA"},
    {"id":1027,"name":"KEPUTUSAN KEPALA DESA"},
    {"id":1031,"name":"RANCANGAN DPRD"},
    {"id":1033,"name":"KEPUTUSAN SEKRETARIS DPRD"},
    {"id":1035,"name":"KARYA TULIS"},
    {"id":1103,"name":"PERATURAN PERGURUAN TINGGI"},
    {"id":1104,"name":"KEPUTUSAN PERGURUAN TINGGI"},
    {"id":1105,"name":"SURAT EDARAN PERGURUAN TINGGI"},
    {"id":1108,"name":"INSTRUKSI PERGURUAN TINGGI"},
    {"id":1110,"name":"CONVENTION"},
    {"id":1111,"name":"TREATY"},
    {"id":1112,"name":"DECLARATION"},
    {"id":1113,"name":"PROTOCOL"},
    {"id":1114,"name":"COVENANT"},
    {"id":1115,"name":"AGREEMENT"},
    {"id":1147,"name":"PERJANJIAN KERJA SAMA PERGURUAN TINGGI"},
    {"id":1155,"name":"PERJANJIAN KERJASAMA"},
    {"id":1157,"name":"INSTRUKSI MENTERI"},
    {"id":1159,"name":"KEPUTUSAN MENTERI"},
    {"id":1198,"name":"PERATURAN DPR"},
    {"id":1363,"name":"ZYOOREI"},
    {"id":1364,"name":"SYUUREI"},
    {"id":1365,"name":"MAKLOEMAT GUNSEIKAN"},
    {"id":1366,"name":"OSAMU GUNREI"},
    {"id":1689,"name":"PERATURAN ESELON 2 DAERAH"},
    {"id":1690,"name":"KEPUTUSAN ESELON 1 DAERAH"},
    {"id":1691,"name":"KEPUTUSAN ESELON 2 DAERAH"}
]';

// Decode JSON menjadi array asosiatif
 $jenisDokumenList = json_decode($jenisDokumenJson, true);

// Buat array untuk pencarian cepat (id => name)
 $lookup = array_column($jenisDokumenList, 'name', 'id');

echo "========================================\n";
echo "    Daftar Jenis Dokumen JDIHN\n";
echo "========================================\n";
foreach ($jenisDokumenList as $dokumen) {
    // Menggunakan sprintf untuk formatting output yang rapi
    echo sprintf("ID: %-5s | Nama: %s\n", $dokumen['id'], $dokumen['name']);
}
echo "========================================\n";

// Loop untuk meminta input yang valid dari pengguna
do {
    $input = readline("Masukkan kode jenis dokumen yang ingin diambil (ID): ");
    if (!ctype_digit($input)) {
        echo "Error: Input harus berupa angka. Silakan coba lagi.\n";
    } elseif (!isset($lookup[$input])) {
        echo "Error: Kode '$input' tidak ditemukan dalam daftar. Silakan coba lagi.\n";
    } else {
        // Input valid, keluar dari loop
        break;
    }
} while (true);

 $jenisId = (int)$input;
 $jenisName = $lookup[$jenisId];

echo "\nAnda memilih: ID {$jenisId} ({$jenisName}).\n";
echo "Memulai proses pengambilan data...\n\n";


// --- Langkah 1: Dapatkan total halaman dari halaman pertama ---
 $baseUrl = 'https://jdihn.go.id/api/search?jenis=' . $jenisId . '&page=';
 $totalPages = 0;
try {
    $firstPageContent = @file_get_contents($baseUrl . '1');
    if ($firstPageContent === false) {
        throw new Exception("Gagal mengambil data dari halaman pertama. Periksa koneksi internet atau ketersediaan API.");
    }

    $firstPageData = json_decode($firstPageContent, true);
    if (json_last_error() !== JSON_ERROR_NONE) {
        throw new Exception("Gagal mem-parsing JSON dari halaman pertama: " . json_last_error_msg());
    }

    if (!isset($firstPageData['meta']['last_page'])) {
        throw new Exception("Struktur JSON tidak sesuai, 'meta.last_page' tidak ditemukan.");
    }

    $totalPages = $firstPageData['meta']['last_page'];
    echo "Ditemukan total {$totalPages} halaman.\n\n";

} catch (Exception $e) {
    die("Error: " . $e->getMessage() . "\n");
}

// --- Langkah 2: Buat nama file CSV secara dinamis ---
 $startPage = 1;
 $source = 'JDIHN';
// Bersihkan nama dokumen untuk digunakan sebagai nama file
 $fileNameSafe = str_replace([' ', '/'], '_', $jenisName);
 $outputFile = $jenisId . '_' . $fileNameSafe . '_page_' . $startPage . '_' . $totalPages . '_' . $source . '.csv';

echo "Membuka file CSV untuk ditulis: {$outputFile}\n";
 $handle = fopen($outputFile, 'w');
if ($handle === false) {
    die("Gagal membuka file {$outputFile} untuk ditulis. Periksa izin penulisan folder.\n");
}

// Tentukan header kolom CSV
 $csvHeader = [
    'judul', 'nomor', 'id_anggota', 'id_dokumen', 'hit_see', 'hit_download',
    'jenis_peraturan_id', 'jenis_peraturan_second_id', 'jenis_peraturan_parent_id', 'jenis_peraturan_name',
    'jenis_peraturan_status', 'jenis_peraturan_created_by', 'jenis_peraturan_updated_by',
    'jenis_peraturan_created_time', 'jenis_peraturan_updated_time', 'jenis_peraturan_setPublish',
    'tahun_terbit', 'instansi', 'download'
];

// Tulis header ke file CSV
fputcsv($handle, $csvHeader);


// --- Langkah 3: Looping melalui semua halaman dan tulis data ---
for ($page = $startPage; $page <= $totalPages; $page++) {
    echo "Memproses halaman {$page} dari {$totalPages}... ";

    $pageContent = @file_get_contents($baseUrl . $page);

    if ($pageContent === false) {
        echo "[GAGAL] Tidak dapat mengambil halaman {$page}. Melewati.\n";
        continue;
    }

    $pageData = json_decode($pageContent, true);
    if (json_last_error() !== JSON_ERROR_NONE || !isset($pageData['data'])) {
        echo "[GAGAL] Gagal mem-parsing JSON pada halaman {$page}. Melewati.\n";
        continue;
    }

    $itemsOnPage = count($pageData['data']);
    foreach ($pageData['data'] as $item) {
        $csvRow = [
            $item['judul'] ?? '', $item['nomor'] ?? '', $item['id_anggota'] ?? '', $item['id_dokumen'] ?? '',
            $item['hit_see'] ?? '', $item['hit_download'] ?? '',
            $item['jenis_peraturan']['id'] ?? '', $item['jenis_peraturan']['second_id'] ?? '',
            $item['jenis_peraturan']['parent_id'] ?? '', $item['jenis_peraturan']['name'] ?? '',
            $item['jenis_peraturan']['status'] ?? '', $item['jenis_peraturan']['_created_by'] ?? '',
            $item['jenis_peraturan']['_updated_by'] ?? '', $item['jenis_peraturan']['_created_time'] ?? '',
            $item['jenis_peraturan']['_updated_time'] ?? '', $item['jenis_peraturan']['setPublish'] ?? '',
            $item['tahun_terbit'] ?? '', $item['instansi'] ?? '', $item['download'] ?? ''
        ];
        fputcsv($handle, $csvRow);
    }

    echo "[SUKSES] ({$itemsOnPage} item ditambahkan)\n";
    sleep(1);
}

// --- Langkah 4: Tutup file dan selesai ---
fclose($handle);

echo "\nProses selesai! Semua data telah disimpan dalam file '{$outputFile}'.\n";

?>

Hasilnya:

Enak pakai CSV atau supabase?

Grab detail JDIHN

Eko Priyanto — Tue, 25 Nov 2025 05:09:39 +0000

Grab detail JDIHN berdasarkan id dan menyimpannya menjadi csv

<?php
// Pastikan file simple_html_dom.php ada di satu folder
require 'simple_html_dom.php';

// --------------------------------------------------------------------------
// INPUT DARI USER
// --------------------------------------------------------------------------
echo "Masukkan ID awal: ";
 $awal = trim(readline());
echo "Masukkan ID akhir: ";
 $akhir = trim(readline());

// Validasi input
if (!is_numeric($awal) || !is_numeric($akhir) || $awal > $akhir) {
    die("Input tidak valid. Pastikan awal dan akhir adalah angka dan awal <= akhir.\n");
}

// --------------------------------------------------------------------------
// SETUP NAMA FILE DINAMIS
// --------------------------------------------------------------------------
 $filename = $awal."_".$akhir."_detail_JDIHN.csv";

echo "--------------------------------------------------\n";
echo "Proses akan dimulai dari ID $awal hingga $akhir.\n";
echo "Hasil akan disimpan dalam file: $filename\n";
echo "--------------------------------------------------\n";


// --------------------------------------------------------------------------
// FUNGSI-FUNGSI HELPER
// --------------------------------------------------------------------------

// FITUR: Mengambil ID dari URL (tetap dipertahankan untuk modularitas)
function getIdFromUrl($url) {
    $path = parse_url($url, PHP_URL_PATH);
    $segments = explode('/', $path);
    $key = array_search('detail', $segments);
    if ($key !== false && isset($segments[$key + 1])) {
        return $segments[$key + 1];
    }
    return 'unknown';
}

// LANGKAH 1: Ambil HTML via cURL
function getHTML($url) {
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false); 
    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'); 
    curl_setopt($ch, CURLOPT_TIMEOUT, 30); // Timeout 30 detik
    $output = curl_exec($ch);
    $http_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    curl_close($ch);

    if ($http_code != 200) {
        return null; // Kembalikan null jika gagal (misal 404)
    }

    return $output;
}

// --------------------------------------------------------------------------
// SETUP CSV
// --------------------------------------------------------------------------
 $file_exists = file_exists($filename);

// Buka file untuk append ('a'). Ini akan menambah data di akhir file.
 $fp = fopen($filename, 'a');

if (!$fp) {
    die("Gagal membuka/membuat file CSV. Cek permission folder.\n");
}

// Jika file belum ada, tulis header dan BOM untuk kompatibilitas Excel
if (!$file_exists) {
    // Tambahkan BOM agar UTF-8 terbaca benar di Excel
    fputs($fp, $bom = (chr(0xEF) . chr(0xBB) . chr(0xBF)));

    // Definisikan header secara manual agar lebih robust
    $headers = [
        'id', 'judul', 'jenis_peraturan', 'nomor_peraturan', 'tahun_peraturan', 
        'tentang', 'tempat_penetapan', 'tanggal_penetapan', 'tanggal_pengundangan', 
        'status', 'bahasa', 'lokasi', 'sumber', 'bidang_hukum', 'sub_bidang_hukum', 'link_download'
    ];
    fputcsv($fp, $headers);
    echo "File $filename dibuat dan header ditulis.\n";
}

// --------------------------------------------------------------------------
// PROSES LOOPING
// ----------
echo "Memulai proses...\n";

for ($i = $awal; $i <= $akhir; $i++) {
    echo "Memproses ID: $i\r"; // Gunakan \r untuk menimpa baris, agar output lebih rapi

    // 1. Konfigurasi URL (dinamis berdasarkan loop)
    $url = "https://jdihn.go.id/pencarian/detail/" . $i;

    // 2. Ambil HTML
    $html_content = getHTML($url);

    if ($html_content === null) {
        // echo "Halaman tidak ditemukan (HTTP Error) atau timeout. Melewati ID $i.\n";
        continue; // Lanjut ke iterasi berikutnya
    }

    // 3. Parsing HTML
    $html = str_get_html($html_content);
    if (!$html) {
        // echo "Gagal mem-parsing HTML. Melewati ID $i.\n";
        continue;
    }

    // Inisialisasi array data untuk ID saat ini
    $data = [
        'id' => $i // Gunakan $i langsung sebagai ID
    ];

    // A. Ambil Judul
    $title_element = $html->find('h4', 0); 
    $data['judul'] = $title_element ? trim($title_element->plaintext) : '';

    // Jika judul kosong, kemungkinan besar ini adalah halaman 404 atau tidak valid
    if (empty($data['judul'])) {
        // echo "Judul tidak ditemukan, kemungkinan halaman tidak valid. Melewati ID $i.\n";
        $html->clear();
        continue;
    }

    // B. Ambil Metadata Tabel
    foreach($html->find('table tr') as $row) {
        $cols = $row->find('td');
        if (count($cols) >= 2) {
            $label = trim(str_replace(':', '', $cols[0]->plaintext));
            $value = trim(end($cols)->plaintext);

            if (!empty($label)) {
                $key = strtolower(preg_replace('/[^a-zA-Z0-9]/', '_', $label));
                $key = preg_replace('/_+/', '_', $key);
                $data[$key] = $value;
            }
        }
    }

    // C. Ambil Link Download
    $download_link = "";
    foreach($html->find('a') as $element) {
        if (strpos(strtolower($element->plaintext), 'unduh') !== false || strpos(strtolower($element->plaintext), 'download') !== false) {
            $download_link = $element->href;
            break; 
        }
    }
    if ($download_link && strpos($download_link, 'http') === false) {
        $download_link = "https://jdihn.go.id" . $download_link;
    }
    $data['link_download'] = $download_link;

    // Bersihkan memori DOM
    $html->clear();

    // Simpan data ke CSV (hanya baris data, bukan header)
    fputcsv($fp, $data);
}

// --------------------------------------------------------------------------
// SELESAI
// --------------------------------------------------------------------------
// Tutup file setelah loop selesai
fclose($fp);

echo "\n--------------------------------------------------\n";
echo "Proses selesai! Semua data telah disimpan ke file: $filename\n";
echo "--------------------------------------------------\n";

hasil berupa csv:

id,judul,jenis_peraturan,nomor_peraturan,tahun_peraturan,tentang,tempat_penetapan,tanggal_penetapan,tanggal_pengundangan,status,bahasa,lokasi,sumber,bidang_hukum,sub_bidang_hukum,link_download
327,"TIM ASISTENSI E-BUDGETING RENCANA KERJA SATUAN KERJA PERANGKAT DAERAH TINGKAT KOTA BANDUNG TAHUN ANGGARAN 2017","JDIH Pemerintah Kota Bandung","KEPUTUSAN WALIKOTA",050/Kep.697-Bappeda/2016,2016,,KEPWAL,,-,2023-09-04,-,-,-,,,20190806061944531547339.pdf,20190806061944531547339.pdf,,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3382</button>,https://jdihn.go.id/unduhan
329,"PERUBAHAN ATAS PERATURAN WALIKOTA BANDUNG NOMOR 548 TAHUN 2016 TENTANG TATA CARA PENERAPAN SANKSI ADMINISTRATIF DALAM PENYELENGGARAAN BANGUNAN GEDUNG","JDIH Pemerintah Kota Bandung","PERATURAN WALIKOTA",1032,2016,,PERWAL,,-,2023-09-04,-,-,-,,,"Perwal 1032 Tahun  2016 SANKSI OTENTIFIKASI.pdf","Perwal 1032 Tahun  2016 SANKSI OTENTIFIKASI.pdf",,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3383</button>,https://jdihn.go.id/unduhan
335,KEPEMUDAAN,"JDIH Pemerintah Kota Bandung","PERATURAN DAERAH KOTA",1,2016,,PERDA,,-,2023-09-04,-,-,-,,,"Perda No. 1 tahun 2016 ttg. Kepemudaan.pdf","Perda No. 1 tahun 2016 ttg. Kepemudaan.pdf",,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3384</button>,https://jdihn.go.id/unduhan
348,"PERUBAHAN KEDUA ATAS PERATURAN WALIKOTA BANDUNG NOMOR 495 TAHUN 2015 TENTANG STANDAR OPERASIONAL PROSEDUR PELAYANAN PERIZINAN TERPADU","JDIH Pemerintah Kota Bandung","PERATURAN WALIKOTA",265,2016,,PERWAL,,-,2023-09-04,-,-,-,,,20201201003442704186689.pdf,20201201003442704186689.pdf,,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3260</button>,https://jdihn.go.id/unduhan
363,"Tim Ahli Bangunan Gedung Kota Bandung Masa Bakti 2015-2016","JDIH Pemerintah Kota Bandung","KEPUTUSAN WALIKOTA",640/Kep.977-DisTarCip/2015,2015,,KEPWAL,,-,2023-09-04,-,-,-,,,"SLN SK 640_Kep.977-DisTarCip_2015 perub TABG 2015-2016.pdf","SLN SK 640_Kep.977-DisTarCip_2015 perub TABG 2015-2016.pdf",,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3248</button>,https://jdihn.go.id/unduhan
364,"PERUBAHAN ATAS PERATURAN DAERAH KOTA BANDUNG NOMOR 24 TAHUN 2012 TENTANG PENYELENGGARAAN DAN PENANGANAN KESEJAHTERAAN SOSIAL","JDIH Pemerintah Kota Bandung","PERATURAN DAERAH KOTA",5,2015,,PERDA,,-,2023-09-04,-,-,-,,,"Perda No. 5 Tahun 2015  kesos.pdf","Perda No. 5 Tahun 2015  kesos.pdf",,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3270</button>,https://jdihn.go.id/unduhan
368,"Pembentukan Forum Komunikasi Anak Kota Bandung Periode Tahun 2015 2017","JDIH Pemerintah Kota Bandung","KEPUTUSAN WALIKOTA","464/Kep.981 - BPPKB/2015",2015,,KEPWAL,,-,2023-09-04,-,-,-,,,201907220300001983419264.pdf,201907220300001983419264.pdf,,,https://jdih.bandung.go.id/home/produk-hukum/peraturan-perundang-undangan-daerah/3249</button>,https://jdihn.go.id/unduhan
448,"Peraturan Daerah Kotamadya Daerah Tingkat II Salatiga Nomor 18 Tahun 1977 tentang Mengubah untuk Ketiga kalinya Peraturan Daerah tentang Peraturan Daerahgangan dan Penjualan Susu","JDIH Pemerintah Kota Salatiga","PERATURAN DAERAH KOTA",18,1977,-,PERDA,,-,1978-02-27,-,Berlaku,-,"LD Kotamadya Daerah Tingkat II Salatiga  1977 (18)",Indonesia,1977pd3332018.pdf,,,,https://jdih.salatiga.go.id/dokumen/view?id=1363</button>,https://jdihn.go.id/unduhan
463,"Peraturan Daerah Kotamadya Daerah Tingkat II Salatiga Nomor 2 Tahun 1974 tentang Pasar yang Dikuasai oleh Pemerintah Daerah Kotamadya Salatiga","JDIH Pemerintah Kota Salatiga","PERATURAN DAERAH KOTA",2,1974,-,PERDA,,-,1978-11-28,-,Berlaku,-,"LD Kotamadya Daerah Tingkat II Salatiga  1974 (2)",Indonesia,1974pd3332002.pdf,,,,https://jdih.salatiga.go.id/dokumen/view?id=1351</button>,https://jdihn.go.id/unduhan
464,"Peraturan Daerah Kotamadya Daerah Tingkat II Salatiga Nomor 5 Tahun 1980 tentang Penetapan Sisa Perhitungan Anggaran Pendapatan dan Belanja Daerah Kotamadya Daerah Tingkat II Salatiga Tahun Anggaran 1976/1977","JDIH Pemerintah Kota Salatiga","PERATURAN DAERAH KOTA",5,1980,-,PERDA,,-,1980-06-04,-,Berlaku,-,"LD Kotamadya Daerah Tingkat II Salatiga  1980 (5)",Indonesia,1980pd3332005.pdf,,,,https://jdih.salatiga.go.id/dokumen/view?id=1373</button>,https://jdihn.go.id/unduhan
468,"Peraturan Daerah Kotamadya Daerah Tingkat II Salatiga Nomor 1 Tahun 1975 tentang Mengubah Untuk Ketiga Kali Peraturan Daerah tentang Pajak kendaraan","JDIH Pemerintah Kota Salatiga","PERATURAN DAERAH KOTA",1,1975,-,PERDA,,-,1976-11-01,-,Berlaku,-,"LD Kotamadya Daerah Tingkat II Salatiga  1975 (1)",Indonesia,1975pd3332001.pdf,,,,https://jdih.salatiga.go.id/dokumen/view?id=1352</button>,https://jdihn.go.id/unduhan

Peta curah hujan dari BMKG

Eko Priyanto — Wed, 04 Jun 2025 10:18:38 +0000

Ternyata BMKG juga menyediakan peta curah hujan, walaupun tampilannya kurang menarik sih...

Ini sumber datanya: https://gis.bmkg.go.id/portal/dataapi

Berikut setelah diimplementasikan dalam bentuk peta

<!DOCTYPE html>
<html lang="id">
<head>
  <meta charset="UTF-8">
  <title>Peta Curah Hujan dan Hari Hujan - BMKG</title>
  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  <!-- Leaflet CSS -->
  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.4/dist/leaflet.css" />

  <style>
    body, html {
      margin: 0;
      padding: 0;
      height: 100%;
    }
    #map {
      width: 100%;
      height: 100vh;
    }
    .legend {
      position: absolute;
      bottom: 10px;
      left: 10px;
      background: white;
      padding: 10px;
      border-radius: 5px;
      box-shadow: 0 0 5px rgba(0,0,0,0.3);
      font-size: 12px;
      line-height: 1.5;
      z-index: 1000;
    }
    .legend img {
      width: 100px;
      display: block;
    }
  </style>
</head>
<body>

  <div id="map"></div>
  <div class="legend" id="legend">
    <strong>Legenda:</strong>
    <div id="curahLegend" style="margin-top:5px;"></div>
  </div>

  <!-- Leaflet JS -->
  <script src="https://unpkg.com/leaflet@1.9.4/dist/leaflet.js"></script>

  <script>
    // Inisialisasi peta
    var map = L.map('map').setView([-2, 118], 5); // Koordinat tengah Indonesia

    // Tambahkan layer dasar dari OpenStreetMap
    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
      attribution: '© OpenStreetMap contributors'
    }).addTo(map);

    // Layer Curah Hujan (layer 2)
    var curahHujan = L.tileLayer.wms('https://gis.bmkg.go.id/arcgis/services/Peta_Curah_Hujan_dan_Hari_Hujan_/MapServer/WMSServer', {
      layers: '2',
      format: 'image/png',
      transparent: true,
      version: '1.3.0',
      attribution: 'BMKG'
    }).addTo(map);

    // Layer Hari Hujan (layer 1)
    var hariHujan = L.tileLayer.wms('https://gis.bmkg.go.id/arcgis/services/Peta_Curah_Hujan_dan_Hari_Hujan_/MapServer/WMSServer', {
      layers: '1',
      format: 'image/png',
      transparent: true,
      version: '1.3.0',
      attribution: 'BMKG'
    });

    // Kontrol layer (checkbox untuk mengaktifkan/mematikan layer)
    L.control.layers(null, {
      "Curah Hujan": curahHujan,
      "Hari Hujan": hariHujan
    }).addTo(map);

    // Tampilkan legenda default (Curah Hujan)
    document.getElementById("curahLegend").innerHTML = '<img src="https://gis.bmkg.go.id/arcgis/services/Peta_Curah_Hujan_dan_Hari_Hujan_/MapServer/WMSServer?request=GetLegendGraphic&version=1.3.0&format=image/png&layer=2" alt="Legenda Curah Hujan">';

    // Tambahkan handler untuk update legenda saat layer aktif berubah
    map.on('overlayadd', function (e) {
      if (e.name === 'Curah Hujan') {
        document.getElementById("curahLegend").innerHTML = '<img src="https://gis.bmkg.go.id/arcgis/services/Peta_Curah_Hujan_dan_Hari_Hujan_/MapServer/WMSServer?request=GetLegendGraphic&version=1.3.0&format=image/png&layer=2" alt="Legenda Curah Hujan">';
      } else if (e.name === 'Hari Hujan') {
        document.getElementById("curahLegend").innerHTML = '<img src="https://gis.bmkg.go.id/arcgis/services/Peta_Curah_Hujan_dan_Hari_Hujan_/MapServer/WMSServer?request=GetLegendGraphic&version=1.3.0&format=image/png&layer=1" alt="Legenda Hari Hujan">';
      }
    });
  </script>

</body>
</html>

rain map

Eko Priyanto — Sat, 31 May 2025 21:34:36 +0000

Membuat rain map dengan leaflet dan rainviewer

<!DOCTYPE html>
<html>
<head>
  <title>Peta Hujan - RainViewer + Leaflet</title>
  <meta charset="utf-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <link rel="stylesheet" href="https://unpkg.com/leaflet@1.9.3/dist/leaflet.css" />
  <style>
    #map { height: 100vh; }
  </style>
</head>
<body>
  <div id="map"></div>

  <script src="https://unpkg.com/leaflet@1.9.3/dist/leaflet.js"></script>
  <script>
    // Inisialisasi peta di koordinat Indonesia
    var map = L.map('map').setView([-7.65, 111.6], 7); // Trenggalek

    // Tambahkan tile dasar (OpenStreetMap)
    L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
      attribution: '© OpenStreetMap'
    }).addTo(map);

    // Ambil data radar terbaru dari RainViewer
    fetch('https://api.rainviewer.com/public/weather-maps.json')
      .then(response => response.json())
      .then(data => {
        // Ambil path radar terbaru
        const radar = data.radar;
        const latestFrame = radar.past[radar.past.length - 1];
        const tileUrl = `https://tilecache.rainviewer.com${latestFrame.path}/256/{z}/{x}/{y}/2/1_1.png`;

        // Tambahkan layer radar ke peta
        L.tileLayer(tileUrl, {
          attribution: 'Radar data © RainViewer',
          opacity: 0.6,
          transparent: true
        }).addTo(map);
      })
      .catch(error => {
        console.error('Gagal memuat data radar:', error);
      });
  </script>
</body>
</html>

Code analisis termudah

Eko Priyanto — Sat, 17 May 2025 03:11:43 +0000

Ingin code yang aman tanpa capek-capek vuln scan? tapi gratis.

Awalnya nemu web ini https://snyk.io/ yang menampilkan vuln terbaru. Namun ternyata mereka juga punya apps bagus yang gratis. code analisis yang enak, tinggal hubungkan github kita, pilih projectnya maka hasil analisis kode kita ketemu. Langsung dengan kode perbaikannya.

https://app.snyk.io/

PHP security checklist

Eko Priyanto — Mon, 14 Apr 2025 03:35:10 +0000

https://freedium.cfd/https://levelup.gitconnected.com/secure-your-php-code-8cd5921047a0

Security is crucial when developing with PHP, as poorly written code can expose vulnerabilities such as SQL injection, XSS (Cross-Site Scripting), and CSRF (Cross-Site Request Forgery). Below are best practices and secure code examples.

Validate and Sanitize User Input
User input should never be trusted. Always validate and sanitize input before using it.

✅ Example: Sanitizing User Input

Copy

<?php
// Using filter_input() to sanitize user input
$name = filter_input(INPUT_POST, 'name', FILTER_SANITIZE_STRING);
$email = filter_input(INPUT_POST, 'email', FILTER_VALIDATE_EMAIL);

if (!$email) {
    echo "Invalid email!";
} else {
    echo "Hello, " . htmlspecialchars($name);
}
?>

FILTER_SANITIZE_STRING: Removes harmful characters from strings.
FILTER_VALIDATE_EMAIL: Ensures the email is valid.
htmlspecialchars(): Prevents XSS attacks by escaping special characters.
Use Prepared Statements to Prevent SQL Injection
SQL Injection happens when user input is directly used in an SQL query. Always use prepared statements.

❌ Insecure SQL Query (Vulnerable to SQL Injection)

Copy

<?php
$conn = new mysqli("localhost", "user", "password", "database");
$user_input = $_GET['id'];
$sql = "SELECT * FROM users WHERE id = '$user_input'"; // 🚨 Unsafe: User can manipulate the query!
$result = $conn->query($sql);
?>

A hacker can enter: 1' OR '1'='1 This makes the query return all users instead of just one.

✅ Secure SQL Query (Using Prepared Statements)

Copy

<?php
$conn = new mysqli("localhost", "user", "password", "database");
$stmt = $conn->prepare("SELECT * FROM users WHERE id = ?");
$stmt->bind_param("i", $_GET['id']); // "i" stands for integer
$stmt->execute();
$result = $stmt->get_result();
?>

Prevents SQL injection by binding user input.
Used "i" to indicate that it id is an integer.
Secure Password Handling
Never store passwords in plain text. Always hash passwords before saving them.

✅ Hash a Password Before Storing

Copy

<?php
$password = "my_secure_password";
$hashed_password = password_hash($password, PASSWORD_BCRYPT);
echo $hashed_password; // Store this in the database
?>

PASSWORD_BCRYPT generates a strong, salted hash.
✅ Verify Password During Login

Copy

<?php
$entered_password = "my_secure_password";
$stored_hash = '$2y$10$Qj4X...'; // Retrieved from the database

if (password_verify($entered_password, $stored_hash)) {
    echo "Login successful!";
} else {
    echo "Invalid password!";
}
?>

password_verify() compares user input with the stored hash.
Protect Against Cross-Site Scripting (XSS)
XSS occurs when malicious scripts are injected into web pages. To prevent XSS:

✅ Escape Output Using htmlspecialchars()

Copy

<?php
$comment = $_POST['comment'];
echo htmlspecialchars($comment, ENT_QUOTES, 'UTF-8'); // Prevents JavaScript execution
?>

If a user inputs:alert('Hacked!'); Without htmlspecialchars(), it would execute as JavaScript!

Prevent Cross-Site Request Forgery (CSRF)
CSRF happens when malicious websites trick users into performing unwanted actions. Use CSRF tokens.

✅ Generate and Validate a CSRF Token

<?php
session_start();
if (empty($_SESSION['csrf_token'])) {
    $_SESSION['csrf_token'] = bin2hex(random_bytes(32));
}
?>

<?php
if ($_SERVER['REQUEST_METHOD'] == 'POST') {
    if (!hash_equals($_SESSION['csrf_token'], $_POST['csrf_token'])) {
        die("CSRF validation failed!");
    }
    echo "Form submitted successfully!";
}
?>

The hidden input field sends the token.
hash_equals() ensures the token matches.
Disable Error Display in Production
Errors can expose sensitive information. Always log errors instead of displaying them.

✅ Turn Off the Error Display in php.ini

Copy
display_errors = Off
log_errors = On
error_log = /var/log/php_errors.log
✅ Manually Log Errors

<?php
error_log("Something went wrong!", 3, "/var/log/php_errors.log");
?>

Secure File Uploads
If your site allows file uploads, validate and store file types securely.

✅ Check File Type Before Uploading

Copy

<?php
$allowed_types = ['image/jpeg', 'image/png'];
if (!in_array($_FILES['file']['type'], $allowed_types)) {
    die("Invalid file type!");
}
?>

Avoid storing uploads in a web-accessible directory to prevent direct access.

Restrict PHP Execution in Uploads
Prevent attackers from executing PHP files in the upload folder by disabling PHP execution with a .htaccess file.

✅ Block PHP Execution in Uploads

Copy

Deny from all

Use Secure Headers
Set HTTP headers to enhance security.

✅ Enable Secure Headers

Copy

<?php
header("X-Frame-Options: DENY"); // Prevents clickjacking
header("X-XSS-Protection: 1; mode=block"); // Prevents XSS
header("X-Content-Type-Options: nosniff"); // Prevents MIME-type sniffing
?>

Final Thoughts
By following these best security practices, you can protect your PHP applications from common threats. 🚀

Scrape data masjid dengan multi threads

Eko Priyanto — Thu, 10 Apr 2025 00:45:24 +0000

Mencoba untuk melakukan scrape masjid di simas kemenag menggunakan PHP dan multi threads dengan multi curl

Scrape akan melakukan scrape masjid id dan lokasi masjid atau koordinatnya di gmaps.

Scriptnya:

<?php
$startPage   = 1;
$endPage     = 100;
$maxThreads  = 10;
$namaFile    = "hasil_masjid_page{$startPage}-{$endPage}.txt";

// Buka file dan tulis header
$fp = fopen($namaFile, 'w');
fwrite($fp, "\"s_id\",\"link_detil\",\"link_peta\"\n");

// Ambil banyak halaman secara paralel
function fetchPagesMultiCurl($urls) {
    $multiHandle = curl_multi_init();
    $curlHandles = [];
    $results = [];

    foreach ($urls as $key => $url) {
        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_USERAGENT => 'Mozilla/5.0',
            CURLOPT_TIMEOUT => 10,
        ]);
        curl_multi_add_handle($multiHandle, $ch);
        $curlHandles[$key] = $ch;
    }

    $running = null;
    do {
        curl_multi_exec($multiHandle, $running);
        curl_multi_select($multiHandle, 1.0); // Optimasi I/O blocking
    } while ($running > 0);

    foreach ($curlHandles as $key => $ch) {
        $results[$key] = curl_multi_getcontent($ch);
        curl_multi_remove_handle($multiHandle, $ch);
        curl_close($ch);
    }

    curl_multi_close($multiHandle);
    return $results;
}

// Batch processing
for ($batchStart = $startPage; $batchStart <= $endPage; $batchStart += $maxThreads) {
    $batchEnd = min($batchStart + $maxThreads - 1, $endPage);
    $urls = [];

    for ($i = $batchStart; $i <= $batchEnd; $i++) {
        $urls[$i] = "https://simas.kemenag.go.id/page/profilmasjid/0/0/0/0/0?page=$i";
    }

    printf("🔄 Memproses halaman %d - %d\n", $batchStart, $batchEnd);
    $responses = fetchPagesMultiCurl($urls);

    foreach ($responses as $page => $html) {
        if (!$html) {
            echo "⚠️ Gagal mengambil halaman $page\n";
            continue;
        }

        libxml_use_internal_errors(true); // suppress warning
        $dom = new DOMDocument();
        $dom->loadHTML($html, LIBXML_NOERROR | LIBXML_NOWARNING);
        libxml_clear_errors();

        $xpath = new DOMXPath($dom);
        $linksDetil = $xpath->query("//a[contains(text(), 'Lihat Detil')]");
        $linksPeta  = $xpath->query("//a[contains(text(), 'Lihat di Peta')]");

        $limit = min($linksDetil->length, $linksPeta->length);

        for ($i = 0; $i < $limit; $i++) {
            $detilHref = $linksDetil->item($i)->getAttribute('href');
            $petaHref  = $linksPeta->item($i)->getAttribute('href');

            $row = [
                "{$page}-" . ($i + 1),
                "https://simas.kemenag.go.id" . $detilHref,
                $petaHref
            ];

            fwrite($fp, '"' . implode('","', $row) . '"' . "\n");
        }

        // Bebaskan memori DOM
        unset($dom, $xpath, $linksDetil, $linksPeta);
    }
}

fclose($fp);
echo "🎉 Selesai! File berhasil dibuat: $namaFile\n";

Setelah scrape per batch yang bisa diatur maka akan menyimpan ke file txt dengan format csv

cara menjalankannya adalah dengan menjalankan di CMD atau terminal

nohup php scrape.php

Semoga bermanfaat

scrape peraturan.go.id

Eko Priyanto — Thu, 13 Mar 2025 22:28:47 +0000

Untuk contoh di sini saya pakai diDOM

<?php
require 'vendor/autoload.php'; // Pastikan DiDOM terinstall melalui Composer

use DiDom\Document;

$max_pages = 5; // Jumlah halaman yang ingin di-scrape

for ($page = 1; $page <= $max_pages; $page++) {
    $url = "https://peraturan.go.id/cariglobal?PeraturanSearch%5Bidglobal%5D=&page=$page";
    $options = [
        'http' => [
            'method' => 'GET',
            'header' => "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
        ]
    ];
    $context = stream_context_create($options);
    $html = file_get_contents($url, false, $context);

    // Cek apakah HTML berhasil diambil
    if (!$html) {
        die("Gagal mengambil data dari URL pada halaman $page.\n");
    }

    $document = new Document();
    $document->loadHtml($html);

    // Mencari semua elemen dalam <div class="strip grid">
    $items = $document->find('div.strip.grid');

    foreach ($items as $item) {
        // Mendapatkan jenis peraturan
        $jenis = $item->first('a.float-right') ? trim($item->first('a.float-right')->text()) : 'Tidak tersedia';

        // Mendapatkan judul peraturan
        $judul = $item->first('p') ? trim($item->first('p')->text()) : 'Tidak tersedia';

        // Mendapatkan link dokumen PDF
        $pdf = $item->first('a[target="_blank"]') ? trim($item->first('a[target="_blank"]')->getAttribute('href')) : 'Tidak tersedia';

        // Mendapatkan link detail peraturan dan uraian
        $detail_link_element = $item->first('p a[data-pjax="0"]');
        $link = $detail_link_element ? trim($detail_link_element->getAttribute('href')) : 'Tidak tersedia';
        $uraian = $detail_link_element ? trim($detail_link_element->text()) : 'Tidak tersedia';

        // Mendapatkan nama kementerian atau lembaga yang mengeluarkan peraturan
        $lembaga_element = $item->first('div.score span.loc_open');
        $lembaga = $lembaga_element ? trim($lembaga_element->text()) : 'Tidak tersedia';

        echo "Halaman: $page\n";
        echo "Jenis: $jenis\n";
        echo "Judul: $judul\n";
        echo "PDF: $pdf\n";
        echo "Link: $link\n";
        echo "Uraian: $uraian\n";
        echo "Lembaga: $lembaga\n";
        echo "-----------------------------\n";
    }
}

Contoh kode ini akan scrape 5 halaman pertama di pencarian peraturan.go.id

cara menjalankannya adalah dengan menjalankan di CMD atau terminal

nohup php scrape.php