DEV Community

ahmet gedik
ahmet gedik

Posted on

Rate Limiting and API Key Management for YouTube Data API

#api #ratelimiting #youtube #backend

TopVideoHub fetches trending video data from 9 Asia-Pacific regions — US, GB, JP, KR, TW, SG, VN, TH, HK. Each YouTube Data API v3 videos.list call costs 1 unit. Each search.list call costs 100 units. With a default quota of 10,000 units per key per day, we need careful management to keep all 9 regions refreshed.

Here is the full key rotation and rate limiting system.

Quota Anatomy

Operation Units Usage
videos.list (trending, 50 results) 1 Every cron cycle per region
search.list (keyword) 100 On-demand, cached 6h
videoCategories.list 1 Daily, cached 24h

With 9 regions and a 4-hour cron cycle, trending fetches cost 9 × 6 = 54 units/day. Negligible. The budget killer is search — 100 units per unique query.

Key Storage in SQLite 

CREATE TABLE api_keys (
    id       INTEGER PRIMARY KEY AUTOINCREMENT,
    key_val  TEXT NOT NULL UNIQUE,
    provider TEXT NOT NULL DEFAULT 'youtube',
    quota_used   INTEGER NOT NULL DEFAULT 0,
    quota_limit  INTEGER NOT NULL DEFAULT 10000,
    quota_reset  TEXT,           -- ISO 8601 UTC timestamp of next reset
    is_active    INTEGER NOT NULL DEFAULT 1,
    error_count  INTEGER NOT NULL DEFAULT 0,
    last_error   TEXT,
    created_at   TEXT DEFAULT (datetime('now'))
);

CREATE INDEX idx_api_keys_active ON api_keys(provider, is_active, quota_used);
Enter fullscreen mode Exit fullscreen mode

QuotaManager 

<?php

class QuotaManager
{
    private PDO $db;
    private ?array $activeKey = null;

    public function __construct(PDO $db)
    {
        $this->db = $db;
    }

    /**
     * Get the key with most remaining quota.
     * Returns null if all keys are exhausted.
     */
    public function getActiveKey(): ?array
    {
        if ($this->activeKey !== null) {
            return $this->activeKey;
        }

        $stmt = $this->db->prepare(<<<SQL
            SELECT *
            FROM api_keys
            WHERE provider = 'youtube'
              AND is_active = 1
              AND (quota_used < quota_limit)
              AND (
                  quota_reset IS NULL
                  OR quota_reset < datetime('now')
                  OR quota_used < quota_limit
              )
            ORDER BY (quota_limit - quota_used) DESC
            LIMIT 1
        SQL);
        $stmt->execute();

        $this->activeKey = $stmt->fetch(PDO::FETCH_ASSOC) ?: null;
        return $this->activeKey;
    }

    public function consume(string $keyVal, int $units): void
    {
        $this->db->prepare(<<<SQL
            UPDATE api_keys
            SET quota_used = quota_used + ?,
                quota_reset = COALESCE(
                    quota_reset,
                    datetime('now', '+1 day', 'start of day')
                )
            WHERE key_val = ?
        SQL)->execute([$units, $keyVal]);

        // Bust the in-memory cache so next call re-fetches
        $this->activeKey = null;
    }

    public function markError(string $keyVal, string $error): void
    {
        $this->db->prepare(<<<SQL
            UPDATE api_keys
            SET error_count = error_count + 1,
                last_error  = ?,
                is_active   = CASE WHEN error_count >= 5 THEN 0 ELSE is_active END
            WHERE key_val = ?
        SQL)->execute([$error, $keyVal]);

        $this->activeKey = null;
    }

    public function canFetch(int $unitsNeeded = 1): bool
    {
        $key = $this->getActiveKey();
        if ($key === null) {
            return false;
        }
        return ($key['quota_limit'] - $key['quota_used']) >= $unitsNeeded;
    }

    public function resetDailyQuotas(): void
    {
        // Called at midnight UTC by cron
        $this->db->exec(<<<SQL
            UPDATE api_keys
            SET quota_used = 0,
                error_count = 0,
                is_active = 1,
                quota_reset = NULL
            WHERE quota_reset < datetime('now')
        SQL);
    }
}
Enter fullscreen mode Exit fullscreen mode

YouTube API Client with Rotation 

<?php

class YouTubeApi
{
    private QuotaManager $quota;
    private int $retryDelay = 2; // seconds

    public function __construct(QuotaManager $quota)
    {
        $this->quota = $quota;
    }

    public function fetchTrending(string $region, int $maxResults = 50): array
    {
        $key = $this->quota->getActiveKey();
        if ($key === null) {
            throw new QuotaExhaustedException("All API keys exhausted for region {$region}");
        }

        $url = 'https://www.googleapis.com/youtube/v3/videos?' . http_build_query([
            'part'           => 'snippet,statistics,contentDetails',
            'chart'          => 'mostPopular',
            'regionCode'     => $region,
            'maxResults'     => $maxResults,
            'videoCategoryId' => '',
            'key'            => $key['key_val'],
        ]);

        $response = $this->get($url);

        if ($response['http_code'] === 403) {
            // Quota exceeded — disable this key, try next
            $this->quota->markError($key['key_val'], 'quota_exceeded');
            $this->quota->consume($key['key_val'], $key['quota_limit']); // Force exhaustion
            return $this->fetchTrending($region, $maxResults); // Recurse with next key
        }

        if ($response['http_code'] !== 200) {
            $this->quota->markError($key['key_val'], "HTTP {$response['http_code']}");
            throw new ApiException("YouTube API error: {$response['http_code']}");
        }

        // Charge the quota: 1 unit for videos.list
        $this->quota->consume($key['key_val'], 1);

        $body = json_decode($response['body'], true);
        return $body['items'] ?? [];
    }

    private function get(string $url): array
    {
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT        => 20,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_HTTPHEADER     => ['Accept: application/json'],
        ]);
        $body     = curl_exec($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        $error    = curl_error($ch);
        curl_close($ch);

        if ($error) {
            throw new NetworkException("cURL error: {$error}");
        }

        return ['http_code' => $httpCode, 'body' => $body];
    }
}
Enter fullscreen mode Exit fullscreen mode

Per-Region Budget Enforcement

Search queries are expensive (100 units each). Budget them per region:

<?php

class RegionBudget
{
    // Units per day per region for search
    private const BUDGETS = [
        'JP' => 2000, // High-traffic
        'KR' => 2000,
        'US' => 1500,
        'GB' => 1000,
        'TW' => 800,
        'SG' => 600,
        'HK' => 600,
        'VN' => 400,
        'TH' => 400,
    ];

    public function canSearch(string $region): bool
    {
        $used = $this->getUsedToday($region);
        $budget = self::BUDGETS[$region] ?? 200;
        return $used + 100 <= $budget;
    }

    private function getUsedToday(string $region): int
    {
        // Read from SQLite daily_budget table
        $stmt = $this->db->prepare(<<<SQL
            SELECT COALESCE(SUM(units_used), 0)
            FROM quota_log
            WHERE region = ? AND DATE(used_at) = DATE('now')
        SQL);
        $stmt->execute([$region]);
        return (int) $stmt->fetchColumn();
    }
}
Enter fullscreen mode Exit fullscreen mode

Circuit Breaker 

<?php

class CircuitBreaker
{
    private const THRESHOLD   = 5;   // failures before opening
    private const TIMEOUT     = 300; // seconds before half-open

    public function isOpen(string $service): bool
    {
        $state = $this->getState($service);
        if ($state['status'] === 'open') {
            if (time() - $state['opened_at'] > self::TIMEOUT) {
                $this->halfOpen($service);
                return false; // Allow one test request
            }
            return true; // Still open
        }
        return false;
    }

    public function recordSuccess(string $service): void
    {
        $this->setState($service, ['status' => 'closed', 'failures' => 0]);
    }

    public function recordFailure(string $service): void
    {
        $state = $this->getState($service);
        $failures = ($state['failures'] ?? 0) + 1;

        if ($failures >= self::THRESHOLD) {
            $this->setState($service, [
                'status'    => 'open',
                'failures'  => $failures,
                'opened_at' => time(),
            ]);
            error_log("Circuit breaker OPEN for {$service}");
        } else {
            $this->setState($service, ['status' => 'closed', 'failures' => $failures]);
        }
    }
}
Enter fullscreen mode Exit fullscreen mode

Auto-Import from Config File 

<?php

// api_keys.conf (deployed alongside the app)
// youtube_key=AIzaSyABC123...
// youtube_key=AIzaSyDEF456...
// youtube_key=AIzaSyGHI789...

public function autoImportApiKeys(): void
{
    $conf = parse_ini_file(__DIR__ . '/../api_keys.conf', false, INI_SCANNER_RAW);
    $keys = (array)($conf['youtube_key'] ?? []);

    $stmt = $this->pdo->prepare(<<<SQL
        INSERT OR IGNORE INTO api_keys (key_val, provider)
        VALUES (?, 'youtube')
    SQL);

    foreach ($keys as $key) {
        $stmt->execute([trim($key)]);
    }
}
Enter fullscreen mode Exit fullscreen mode

With 3 YouTube API keys and a combined 30,000 daily units, TopVideoHub can refresh all 9 Asia-Pacific regions on schedule every day with budget to spare for user-initiated searches.

This is part of the "Building TopVideoHub" series, documenting the architecture behind a video discovery platform covering 9 Asia-Pacific regions.

Top comments (0)