Implementing IndexNow Client in PHP for Bing and Yandex

#php #seo #webdev #tutorial

When your content changes frequently, waiting for search engines to crawl is too slow. IndexNow lets you proactively notify Bing, Yandex, and other engines about new and updated URLs. Here's a complete PHP implementation used at TrendVidStream.

What is IndexNow?

IndexNow is an open protocol that lets website owners instantly notify search engines about content changes. Instead of waiting for crawlers, you push URLs directly.

Supported engines: Bing, Yandex, Seznam.cz, Naver, and others.

Implementation

Key Generation

<?php

class IndexNowKey
{
    public static function generate(): string
    {
        return bin2hex(random_bytes(16));
    }

    public static function deploy(string $key, string $publicDir): void
    {
        file_put_contents(
            $publicDir . '/' . $key . '.txt',
            $key
        );
    }
}

// Generate and deploy key
$key = IndexNowKey::generate();
IndexNowKey::deploy($key, '/var/www/html');
echo "Key file created: {$key}.txt\n";

Client Class

<?php

class IndexNowClient
{
    private string $host;
    private string $key;
    private string $keyLocation;
    private string $engine;

    public function __construct(
        string $host,
        string $key,
        ?string $keyLocation = null,
        string $engine = 'api.indexnow.org'
    ) {
        $this->host = $host;
        $this->key = $key;
        $this->keyLocation = $keyLocation ?? "https://{$host}/{$key}.txt";
        $this->engine = $engine;
    }

    /**
     * Submit a single URL
     */
    public function submit(string $url): bool
    {
        $apiUrl = "https://{$this->engine}/indexnow"
            . '?url=' . urlencode($url)
            . '&key=' . urlencode($this->key);

        $ch = curl_init($apiUrl);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT => 10,
            CURLOPT_HTTPHEADER => ['Content-Type: application/json'],
        ]);

        $response = curl_exec($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);

        return $httpCode >= 200 && $httpCode < 300;
    }

    /**
     * Submit multiple URLs in batch (up to 10,000)
     */
    public function submitBatch(array $urls): array
    {
        if (empty($urls)) {
            return ['submitted' => 0, 'success' => true];
        }

        // IndexNow batch limit is 10,000 per request
        $chunks = array_chunk($urls, 10000);
        $totalSubmitted = 0;
        $errors = [];

        foreach ($chunks as $chunk) {
            $payload = json_encode([
                'host' => $this->host,
                'key' => $this->key,
                'keyLocation' => $this->keyLocation,
                'urlList' => array_values($chunk),
            ]);

            $ch = curl_init("https://{$this->engine}/indexnow");
            curl_setopt_array($ch, [
                CURLOPT_POST => true,
                CURLOPT_POSTFIELDS => $payload,
                CURLOPT_RETURNTRANSFER => true,
                CURLOPT_TIMEOUT => 30,
                CURLOPT_HTTPHEADER => [
                    'Content-Type: application/json; charset=utf-8',
                    'Content-Length: ' . strlen($payload),
                ],
            ]);

            $response = curl_exec($ch);
            $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
            curl_close($ch);

            if ($httpCode >= 200 && $httpCode < 300) {
                $totalSubmitted += count($chunk);
            } else {
                $errors[] = [
                    'http_code' => $httpCode,
                    'response' => $response,
                    'url_count' => count($chunk),
                ];
            }
        }

        return [
            'submitted' => $totalSubmitted,
            'errors' => $errors,
            'success' => empty($errors),
        ];
    }
}

Integration with Cron

<?php

// After fetching new trending videos
$fetcher = new VideoFetcher($db);
$newVideos = $fetcher->run();

// Build URLs for new/updated pages
$urls = [];
foreach ($newVideos as $video) {
    $urls[] = "https://trendvidstream.com/watch/{$video['slug']}";
}

// Also include updated category pages
foreach ($updatedCategories as $cat) {
    $urls[] = "https://trendvidstream.com/category/{$cat['slug']}";
}

// Submit to IndexNow
if (!empty($urls)) {
    $indexNow = new IndexNowClient(
        host: 'trendvidstream.com',
        key: getenv('INDEXNOW_KEY')
    );

    $result = $indexNow->submitBatch($urls);
    echo "IndexNow: {$result['submitted']} URLs submitted\n";

    if (!$result['success']) {
        error_log('IndexNow errors: ' . json_encode($result['errors']));
    }
}

Rate Limiting

<?php

class RateLimitedIndexNow extends IndexNowClient
{
    private string $stateFile;
    private int $maxPerHour;

    public function __construct(string $host, string $key, int $maxPerHour = 10000)
    {
        parent::__construct($host, $key);
        $this->stateFile = sys_get_temp_dir() . '/indexnow_rate.json';
        $this->maxPerHour = $maxPerHour;
    }

    public function submitBatch(array $urls): array
    {
        $state = $this->loadState();
        $hourAgo = time() - 3600;

        // Remove entries older than 1 hour
        $state['submissions'] = array_filter(
            $state['submissions'] ?? [],
            fn($ts) => $ts > $hourAgo
        );

        $remaining = $this->maxPerHour - count($state['submissions']);
        if ($remaining <= 0) {
            return ['submitted' => 0, 'throttled' => true, 'success' => false];
        }

        $batch = array_slice($urls, 0, $remaining);
        $result = parent::submitBatch($batch);

        // Record submissions
        foreach ($batch as $_) {
            $state['submissions'][] = time();
        }
        $this->saveState($state);

        return $result;
    }
}