import redis
from prometheus_client import (
Gauge,
start_http_server,
)
import time
redis_client = redis.Redis(
host="redis",
port=6379,
decode_responses=True,
)
QUEUE_SIZE = Gauge(
"scrapy_redis_queue_size",
"Pending Scrapy Redis requests",
["spider"],
)
DUPEFILTER_SIZE = Gauge(
"scrapy_redis_dupefilter_size",
"Scrapy Redis deduplication set size",
["spider"],
)
SPIDERS = ["spider_a", "spider_b", "spider_c"]
def collect_metrics():
for spider in SPIDERS:
queue_key = f"{spider}:requests"
dupefilter_key = f"{spider}:dupefilter"
# 示例假设 requests 使用 Redis List
queue_size = redis_client.llen(queue_key)
# 示例假设 dupefilter 使用 Redis Set
dupefilter_size = redis_client.scard(
dupefilter_key
)
QUEUE_SIZE.labels(
spider=spider
).set(queue_size)
DUPEFILTER_SIZE.labels(
spider=spider
).set(dupefilter_size)
if __name__ == "__main__":
start_http_server(9200)
while True:
try:
collect_metrics()
except Exception:
pass
time.sleep(15)
Top comments (0)