DEV Community

drake
drake

Posted on

prometheus 监控scrapy-redis队列

import redis
from prometheus_client import (
    Gauge,
    start_http_server,
)
import time


redis_client = redis.Redis(
    host="redis",
    port=6379,
    decode_responses=True,
)

QUEUE_SIZE = Gauge(
    "scrapy_redis_queue_size",
    "Pending Scrapy Redis requests",
    ["spider"],
)

DUPEFILTER_SIZE = Gauge(
    "scrapy_redis_dupefilter_size",
    "Scrapy Redis deduplication set size",
    ["spider"],
)


SPIDERS = ["spider_a", "spider_b", "spider_c"]


def collect_metrics():
    for spider in SPIDERS:

        queue_key = f"{spider}:requests"
        dupefilter_key = f"{spider}:dupefilter"

        # 示例假设 requests 使用 Redis List
        queue_size = redis_client.llen(queue_key)

        # 示例假设 dupefilter 使用 Redis Set
        dupefilter_size = redis_client.scard(
            dupefilter_key
        )

        QUEUE_SIZE.labels(
            spider=spider
        ).set(queue_size)

        DUPEFILTER_SIZE.labels(
            spider=spider
        ).set(dupefilter_size)


if __name__ == "__main__":

    start_http_server(9200)

    while True:
        try:
            collect_metrics()
        except Exception:
            pass

        time.sleep(15)
Enter fullscreen mode Exit fullscreen mode

Top comments (0)