From bccba777b1e8e8e5443abe65be4267cb43ca64aa Mon Sep 17 00:00:00 2001 From: tunjayoff Date: Tue, 13 Oct 2026 16:11:54 +0300 Subject: [PATCH 1/4] feat(research): add a soak recorder for the live push sources scripts/soak_live.py runs the live service as it is (LiveService with the direct or page source, the arbiter, the poll source) for hours and records push frames, connection events, leader switches, REST reads with the frame-to-event-read latency, the per-minute fallback reads the owner set for the c project, and the memory of this process and of each browser. Kills of the connection or the pages test the switch back. Nothing it writes holds the credential, the CONNECT body, server addresses, the INFO body or an IP address. --- scripts/soak_live.py | 2008 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2008 insertions(+) create mode 100644 scripts/soak_live.py diff --git a/scripts/soak_live.py b/scripts/soak_live.py new file mode 100644 index 00000000..7aaa3274 --- /dev/null +++ b/scripts/soak_live.py @@ -0,0 +1,2008 @@ +""" +S2: canlı push kaynağının saatler süren ölçüm kaydı (soak). c projesi (bu kodu taşıyacak oran motoru) için uygulamanın +canlı yolunu olduğu gibi ölçer: push kaynağı (varsayılan `direct`: sofascore_scraper/services/live/direct_source.py; +`--source page` ile sayfa dinleme: push_source.py), hakem (arbiter.py), yoklama yedeği (poll_source.py) ve maç sayfası +okumaları için köprü (normal istemci, ortak istek bütçesi). Yeniden yazım yok: servisin kendi parçaları +(LiveService, DirectConnection/DirectSource ya da PageSource/BrowserPageOpener, SportArbiter, PollSource) çalışır; +script yalnızca onların kenarlarına kayıt kancaları takar. + + python scripts/soak_live.py --sports football,basketball,tennis --hours 6 [--source direct|page] [--run-id ID] + [--max-requests N] [--rate 1.0] [--list-share 0.5] [--priority-ids 1,2] + [--kill-every DAKİKA] [--kill-at DAKİKA] [--compare-credential-sports] [--out KLASÖR] + +**`direct` açık seçimdir** (docs/deploy/watch.md, "The `direct` source"): sitenin kendi istemci kimlik bilgisini +sitenin istemcisi dışında kullanır; kimlik bilgisi ya da sunucu değişirse haber vermeden bozulabilir; IP adresinin +engellenmesine yol açabilir; kullanım şartları açısından gri alandır. Script yalnızca bilerek çalıştırılır. +Uygulamanın gönderdiği `direct` istemcisi bu koşuda ilk kez canlıda ölçülür: 2026-10-01'deki deney aiohttp ile +yapılmıştı, uygulamanınki standart kütüphaneyle yazılmış WebSocket istemcisidir (summary.json `notes`). + +Nasıl kurulur: + * Servis (`RecordingLiveService`, LiveService'in alt sınıfı) seçilen push kaynağıyla ve her sporun **bütün** canlı + maçlarını kapsayan bir kapsamla (AllScope) çalışır: hakem, push bağlantısı ve yoklama kaynağının turları (canlı + liste; düşen, bitişe yakın, takılı, gecikmiş maçların sayfaları) bugünkü gibidir. Servisin bitiş onayı kapalıdır + (`confirm=False`): bitişin `/event/{id}` okumasını kaydedici yapar (aşağıda), iki kez okunmasın. + * Push önde: hakemin güvenlik yoklaması (liste 120 sn'de bir ve her (yeniden) bağlanmada bir tur) ve ilgili her + değişiklikte (durum sınıfı, skor, yeni canlı maç, bitiş, listeden düşme) bir `/event/{id}` okuması, c'nin yapacağı + gibi. Okuma değişikliği henüz göstermiyorsa (CDN) servisin bitiş onayı gibi CONFIRM_RETRY_SECONDS sonra, en çok + CONFIRM_ATTEMPTS kez yinelenir. Gecikme = tetikleyen kare ya da liste değişikliği → değişikliği gösteren ilk okuma. + * Yoklama önde (push kopuk, sessiz ya da öldürüldü; sahibin c için kuralı): canlı liste 30 sn'de bir (hakemin + `poll_interval`'ı) ve yoklama kaynağının kendi okumaları, ARTI her canlı maçın sayfası dakikada en az bir kez + (FallbackReads): önce öncelikli id'ler (`--priority-ids`, varsayılan boş: "c fikstürüne bağlı" sinyali burada + yok), sonra bitişe yakınlar, sonra en uzun süredir okunmayanlar. Dakikanın payı `60 × --rate × (1 − + --list-share)` okumadır; sığmayan okuma atlanır ve sayılır (fallback_skipped), bir sonraki dakikaya taşınmaz. + Dakika başına fallback.jsonl: spor başına canlı maç, yapılan okuma, atlanan, kapsama (dakika içinde okunan payı). + * Kancalar: DirectConnection'ın alt sınıfı (oturum başı/sonu, yeniden deneme), bağlantı işlevinin sarmalayıcısı + (el sıkışma, hat üstü işlemler; öldürme), sayfa açıcının sarmalayıcısı (`page`; öldürme = sayfayı kapatma), her + sporun PushFeed'ini saran musluk (MSG kareleri; `page`'de PING/PONG/INFO/-ERR de), kimlik bilgisi okuyucusunun + sarmalayıcısı, servisin `_system`'i (hakemin Switch'leri) ve istek işlevi. + * Bütçe koşu başınadır (explore_all_sports.py gibi): `--max-requests` REST okumalarını (liste ve maç sayfası) sayar; + varsayılan `--hours × 3600 × --rate` (ortalama ≤ 1 istek/sn). Bitince REST okuması yapılmaz (servisin isteği None + alır, kaydedicinin okuması atlanır); push kaydı sürer. Her REST okuması `--rate` hızında sıra alır, sonra + araştırma script'lerinin ortak kilidinden (scripts/_research_common.py: süreçler arası ≤ 1 istek/sn). + * `--kill-every DAKİKA` / `--kill-at DAKİKA`: `direct`'te bağlantının soketi zorla kapatılır (shutdown), `page`'de + sporların sayfaları kapatılır. Kaynak bunu olağan bir kopma gibi görür; hakem yoklamaya geçer, bağlantı ya da sayfa + yeniden açılır, hakem push'a döner. Hepsi kayıtlıdır (connection.jsonl `kill`, leader.jsonl). + * Bellek (resources.jsonl, 30 sn'de bir; kimlik bilgisi okunurken 5 sn'de bir): bu Python süreci, ana köprünün + tarayıcısı (REST okumaları; ilk okumada açılır, koşu boyunca açık kalır) ve canlı profilin tarayıcısı (`page` + sayfaları ya da `direct`'in kimlik bilgisi okuyucusu) ayrı ayrı, toplamı ve zirvesi. Tarayıcı süreçleri profil + dizinine göre ayrılır; dizin yazılmaz. + +Çıktı: `research/soak//` (ya da `--out`): frames.jsonl, connection.jsonl, leader.jsonl, reads.jsonl, +fallback.jsonl, resources.jsonl, summary.json (10 dakikada bir ve sonda). Servisin deposu (state.db) çıktı klasörüne +değil `--data-dir`'e (varsayılan ~/.cache/sofascore_research/soak_store/) yazılır; ortak bütçenin dosyası +`SOFASCORE_CLIENT__THROTTLE_DIR` verilmezse ~/.cache/sofascore_research/throttle. Tarayıcı profili +~/.cache/sofascore_research/chrome_soak (canlı sayfalar ve kimlik bilgisi okuyucusu `-live`). + +GİZLİLİK (kesin kural): push kimlik bilgisi, CONNECT gövdesi, sunucu adresleri/URL'leri, INFO gövdesi ve client_ip +hiçbir yere yazılmaz. Hat üstünde yalnızca işlemin adı ve konusu tutulur; INFO'dan yalnızca geldiği an; -ERR'den +yalnızca bilinen NATS hata metni (push_source.NatsReader). `page` kaynağı giden kareleri hiç dinlemez. Kimlik +bilgilerinin karşılaştırılması bellekte, rastgele tuzlu HMAC ile yapılır; ne kimlik bilgisi ne özeti yazılır, +yalnızca "same"/"different". Yazılan her satır sofascore_scraper/redact.py'den (çalışma anında öğrenilen kimlik bilgisi +dahil) ve araştırma maskesinden (explore_all_sports.redact: IPv4/IPv6, istemci konumu) geçer. tests/test_soak_live.py +ve tests/test_research_privacy.py kayıtları tarar. +""" +from __future__ import annotations + +import argparse +import bisect +import collections +import datetime as dt +import hashlib +import hmac +import json +import math +import os +import re +import secrets +import signal +import socket +import statistics +import sys +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Mapping, Optional, Sequence, Set, Tuple + +# 3.1'in okuduğu adlar (FX-32). Köprü profilini ilk içe aktarılışında okur: bu satırlar sofascore_scraper'dan önce +# çalışmalı. Araştırma uygulamanın profilini paylaşmaz. +os.environ.setdefault("SOFASCORE_LOG__LEVEL", "WARNING") +os.environ.setdefault( + "SOFASCORE_CLIENT__BROWSER_PROFILE", os.path.expanduser("~/.cache/sofascore_research/chrome_soak") +) + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import _research_common as rc # noqa: E402 +from explore_all_sports import redact as research_redact # noqa: E402 + +from sofascore_scraper import redact as app_redact # noqa: E402 +from sofascore_scraper.services.live import direct_source as ds # noqa: E402 +from sofascore_scraper.services.live import supervisor as sv # noqa: E402 +from sofascore_scraper.services.live.arbiter import LEADER_POLL, SILENCE_SECONDS # noqa: E402 +from sofascore_scraper.services.live.push_source import NatsOp, NatsReader, PushFeed, frame_body # noqa: E402 +from sofascore_scraper.services.live.reducer import TERMINAL_CLASSES # noqa: E402 +from sofascore_scraper.status import StatusClass, classify_code, classify_status # noqa: E402 + +ROOT = rc.ROOT +OUT = os.path.join(ROOT, "research", "soak") +STORE_ROOT = os.path.expanduser("~/.cache/sofascore_research/soak_store") +THROTTLE_DIR = os.path.expanduser("~/.cache/sofascore_research/throttle") +DEFAULT_SPORTS = ("football", "basketball", "tennis") +SOURCES = ("direct", "page") +DEFAULT_HOURS = 6.0 +DEFAULT_RATE = 1.0 # istek/sn (bütçenin hızı) +DEFAULT_LIST_SHARE = 0.5 # hızın keşfe (liste ve yoklama kaynağının okumaları) ayrılan payı +FALLBACK_WINDOW_SECONDS = 60.0 # yoklama öndeyken her canlı maçın sayfası bu sürede en az bir kez +RESOURCE_SECONDS = 30.0 +READING_RESOURCE_SECONDS = 5.0 # kimlik bilgisi okunurken (ikinci Chromium açık) örnekleme aralığı +SUMMARY_SECONDS = 600.0 +MAX_PENDING_READS = 1000 +TOP_GAPS = 5 + +FILES = ("frames", "connection", "leader", "reads", "fallback", "resources") +LIVE = StatusClass.LIVE.value +# Karede değeri yazılan alanlar (geri kalanların yalnızca yolu yazılır) +STATUS_VALUE_KEYS = ("status.code", "status.type") +SCORE_PREFIXES = ("homeScore.", "awayScore.") +CHANGE_TS = "changes.changeTimestamp" +# Tetikleyici türleri (bir okuma birden çok türü taşıyabilir) +KIND_FINISH, KIND_STATUS, KIND_NEW, KIND_DROPPED, KIND_SCORE = "finish", "status", "new_live", "dropped", "score" +CLIENT_NOTE = ("First live run of the shipped direct client: the standard-library WebSocket client in " + "sofascore_scraper/services/live/direct_source.py. The probe measured on 2026-10-01 " + "(scripts/push_direct_probe.py) used aiohttp.") +SCOPE_NOTE = ("The live service ran with the chosen push source and a scope of every live match of each sport: the " + "arbiter, the push connection and the poll source's rounds are the service's own. Its confirmation " + "read was off; the recorder reads /event/{id} once per relevant change instead, through the service's " + "fetch, and while polling leads it reads every live match's page at least once a minute (fallback).") +PAGE_NOTE = ("Source 'page': the live browser listens to the push connection of SofaScore's own sport pages " + "(push_source.py); a kill closes the pages.") +LIST_PATH = re.compile(r"^/sport/([a-z0-9-]+)/events/live$") +EVENT_PATH = re.compile(r"^/event/(\d+)$") + + +# --- maskeleme ve kayıt ---------------------------------------------------------------------------------------- + + +def scrub(obj: Any) -> Any: + """Yazılacak her şey: uygulamanın maskesi (bilinen ve çalışma anında öğrenilen gizli değerler) + araştırmanınki.""" + return research_redact(app_redact.redact_obj(obj)) + + +def utc(ts: Optional[float]) -> Optional[str]: + if ts is None: + return None + return dt.datetime.fromtimestamp(ts, dt.timezone.utc).isoformat(timespec="seconds") + + +def describe(values: Sequence[float]) -> Dict[str, Any]: + """n, medyan, p90, en küçük, en büyük (3 basamak).""" + vals = sorted(float(v) for v in values) + if not vals: + return {"n": 0} + p90 = vals[min(len(vals) - 1, int(round(0.9 * (len(vals) - 1))))] + return {"n": len(vals), "median": round(statistics.median(vals), 3), "p90": round(p90, 3), + "max": round(vals[-1], 3), "min": round(vals[0], 3)} + + +class Recorder: + """Koşunun JSONL dosyaları. Her satır `ts` (epoch) ve `mono` (monotonik saat) taşır ve maskeden geçer.""" + + def __init__(self, out_dir: str, *, wall: Callable[[], float] = time.time, + mono: Callable[[], float] = time.monotonic) -> None: + os.makedirs(out_dir, exist_ok=True) + self.out_dir = out_dir + self._wall = wall + self._mono = mono + self._lock = threading.Lock() + self._files = {name: open(os.path.join(out_dir, f"{name}.jsonl"), "a", encoding="utf-8") for name in FILES} + self.lines: Dict[str, int] = {name: 0 for name in FILES} + + def write(self, name: str, row: Mapping[str, Any], *, ts: Optional[float] = None, + mono: Optional[float] = None) -> None: + ts = self._wall() if ts is None else ts + mono = self._mono() if mono is None else mono + line = json.dumps(scrub({"ts": round(ts, 3), "mono": round(mono, 3), **row}), ensure_ascii=False) + with self._lock: + f = self._files.get(name) + if f is None or f.closed: + return + f.write(line + "\n") + f.flush() + self.lines[name] += 1 + + def write_summary(self, summary: Mapping[str, Any]) -> None: + rc.write_json(os.path.join(self.out_dir, "summary.json"), scrub(summary)) + + def close(self) -> None: + with self._lock: + for f in self._files.values(): + f.close() + + +# --- değişiklik izleyici: hangi değişiklik bir maç sayfası okuması ister --------------------------------------- + + +@dataclass +class EventState: + code: Any = None + stype: Optional[str] = None + cls: Optional[str] = None + home: Any = None + away: Any = None + ts: Optional[int] = None # changes.changeTimestamp + + def expect(self) -> Dict[str, Any]: + return {"class": self.cls, "home": self.home, "away": self.away} + + +def _class_of(code: Any, stype: Optional[str]) -> Optional[str]: + if stype: + status: Dict[str, Any] = {"type": stype} + if code is not None: + status["code"] = code + return classify_status({"status": status}).value + if code is not None: + return classify_code(code).value + return None + + +def _ts(value: Any) -> Optional[int]: + return value if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def _score(event: Mapping[str, Any], side: str) -> Any: + node = event.get(side) + return node.get("current") if isinstance(node, dict) else None + + +class ChangeTracker: + """ + Spor ve maç başına son bilinen durum sınıfı, skor ve değişiklik anı. Push karesi ya da canlı liste ilgili bir + değişiklik gösterirse tetikleyici türlerini verir. Sporun ilk listesi tohumdur (tetikleyici yok). Liste, son + bilinen değişiklik anından eskiyse (CDN) o maç için yok sayılır: eski gözlem yeniyi ezmez (LastKnown gibi). + Okunan maç sayfası da durumu günceller (tetikleyicisiz): okunmuş bir değişiklik yeniden okuma istemez. + """ + + def __init__(self) -> None: + self._events: Dict[Tuple[str, int], EventState] = {} + self._listed: Dict[str, Set[int]] = {} + + def state(self, sport: str, eid: int) -> Optional[EventState]: + return self._events.get((sport, eid)) + + @staticmethod + def _kinds(new: bool, prev_cls: Optional[str], cls: Optional[str], score_changed: bool, + had_score: bool) -> Set[str]: + kinds: Set[str] = set() + if cls is not None and cls != prev_cls: + if new or prev_cls is None: + if cls in TERMINAL_CLASSES: + kinds.add(KIND_FINISH) + elif cls == LIVE: + kinds.add(KIND_NEW) + else: + kinds.add(KIND_FINISH if cls in TERMINAL_CLASSES else KIND_STATUS) + if score_changed: + if (new or not had_score) and cls in (None, LIVE): + if not kinds: + kinds.add(KIND_NEW) + elif not new and had_score: + kinds.add(KIND_SCORE) + return kinds + + def frame(self, sport: str, body: Mapping[str, Any]) -> Tuple[Set[str], Dict[str, Any], Optional[str]]: + """Bir push karesi → (tetikleyiciler, beklenen durum, karenin gösterdiği sınıf).""" + eid = int(body["id"]) + st = self._events.get((sport, eid)) + new = st is None + st = st or EventState() + prev_cls, had_score = st.cls, st.home is not None or st.away is not None + code = body.get("status.code", st.code) + stype = st.stype + if "status.type" in body: + stype = body.get("status.type") + elif "status.code" in body and stype: + # merge_frame gibi: yalnız kod geldiyse ve eski tür onunla çelişiyorsa eski tür düşer + by_code = classify_code(code) + if by_code is not StatusClass.UNKNOWN and by_code.value != _class_of(code, stype): + stype = None + status_seen = "status.code" in body or "status.type" in body + cls = _class_of(code, stype) if status_seen else st.cls + home = body.get("homeScore.current", st.home) + away = body.get("awayScore.current", st.away) + score_changed = ("homeScore.current" in body or "awayScore.current" in body) and (home, away) != (st.home, st.away) + kinds = self._kinds(new, prev_cls, cls if status_seen else None, score_changed, had_score) + st.code, st.stype, st.cls, st.home, st.away = code, stype, cls, home, away + st.ts = _ts(body.get(CHANGE_TS)) or st.ts + self._events[(sport, eid)] = st + return kinds, st.expect(), cls if status_seen else None + + def _apply_event(self, sport: str, ev: Mapping[str, Any], *, triggers: bool) -> Optional[Tuple[int, Set[str]]]: + try: + eid = int(str(ev.get("id"))) + except ValueError: + return None + st = self._events.get((sport, eid)) + new = st is None + st = st or EventState() + changes = ev.get("changes") + ts = _ts(changes.get("changeTimestamp")) if isinstance(changes, dict) else None + if not new and st.ts is not None and ts is not None and ts < st.ts: + return eid, set() # CDN'in eski yanıtı + status = ev.get("status") if isinstance(ev.get("status"), dict) else {} + cls = classify_status(dict(ev)).value if status else None + home, away = _score(ev, "homeScore"), _score(ev, "awayScore") + had_score = st.home is not None or st.away is not None + score_changed = (home, away) != (st.home, st.away) and (home is not None or away is not None) + kinds = self._kinds(new, st.cls, cls, score_changed, had_score) if triggers else set() + st.code, st.stype, st.cls = status.get("code"), status.get("type"), cls + st.home, st.away = home, away + st.ts = ts or st.ts + self._events[(sport, eid)] = st + return eid, kinds + + def listing(self, sport: str, events: Iterable[Mapping[str, Any]]) -> List[Tuple[int, Set[str], Dict[str, Any]]]: + """Bir canlı liste → [(maç, tetikleyiciler, beklenen durum)]; sporun ilk listesi tohumdur.""" + first = sport not in self._listed + out: List[Tuple[int, Set[str], Dict[str, Any]]] = [] + ids: Set[int] = set() + for ev in events: + applied = self._apply_event(sport, ev, triggers=not first) + if applied is None: + continue + eid, kinds = applied + ids.add(eid) + if kinds: + out.append((eid, kinds, self._events[(sport, eid)].expect())) + if not first: + for eid in sorted(self._listed[sport] - ids): + st = self._events.get((sport, eid)) + if st is not None and st.cls == LIVE: + out.append((eid, {KIND_DROPPED}, st.expect())) + self._listed[sport] = ids + return out + + def observe_event(self, sport: str, event: Mapping[str, Any]) -> None: + """Okunan maç sayfası: durum güncellenir, tetikleyici üretilmez.""" + self._apply_event(sport, event, triggers=False) + + +def reflects(kinds: Set[str], expect: Mapping[str, Any], event: Mapping[str, Any]) -> bool: + """Maç sayfası tetikleyen değişikliği gösteriyor mu (CDN eski yanıt verebilir).""" + cls = classify_status(dict(event)).value + if kinds == {KIND_DROPPED}: + return cls != LIVE + wanted = expect.get("class") + if KIND_FINISH in kinds and cls not in TERMINAL_CLASSES: + return False + if kinds & {KIND_STATUS, KIND_NEW} and wanted is not None and cls != wanted and not ( + KIND_FINISH in kinds and cls in TERMINAL_CLASSES): + return False + if KIND_SCORE in kinds and (_score(event, "homeScore"), _score(event, "awayScore")) != ( + expect.get("home"), expect.get("away")): + return False + return True + + +# --- REST okumaları: bütçe, hız, durum kodu --------------------------------------------------------------------- + + +@dataclass +class Outcome: + status: Any + seconds: float + data: Optional[Dict[str, Any]] + waited: float = 0.0 + blocked: Optional[BaseException] = None + refused: bool = False # bütçe bitti: ağa çıkılmadı + + +def status_of(exc: Optional[BaseException]) -> Optional[Any]: + """Servisin istek işlevinin hatasından HTTP durumu (biliniyorsa).""" + if exc is None: + return None + from sofascore_scraper.exceptions import CircuitOpenError, RateLimitError, ResourceNotFoundError + + if isinstance(exc, ResourceNotFoundError): + return 404 + if isinstance(exc, CircuitOpenError): + return "circuit_open" + code = getattr(exc, "status_code", None) + if isinstance(code, int): + return code + if isinstance(exc, RateLimitError): + return 429 + return None + + +class Pacer: + """Süreç içi hız: ardışık REST okumaları arasında en az 1/rate sn (thread'ler arası sıra).""" + + def __init__(self, rate: float, *, mono: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep) -> None: + self.interval = 1.0 / rate if rate > 0 else 0.0 + self._mono = mono + self._sleep = sleep + self._lock = threading.Lock() + self._next = 0.0 + + def wait(self) -> None: + with self._lock: + now = self._mono() + at = max(now, self._next) + self._next = at + self.interval + if at > now: + self._sleep(at - now) + + +def default_gate(rate: float) -> Callable[[], None]: + pacer = Pacer(rate) + + def gate() -> None: + pacer.wait() + rc._wait_rate_slot() # araştırma süreçlerinin ortak kilidi (≤ 1 istek/sn) + + return gate + + +class ReadClient: + """ + Kaydedicinin ve servisin REST okumaları: koşu bütçesi, sıra (`gate`), süre ve durum kodu. `base` servisin gerçek + istek işlevidir (supervisor._default_fetch: 404 → None, engellenme → Blocked). + """ + + def __init__(self, base: Callable[[str], Optional[Dict[str, Any]]], *, budget: int, + gate: Callable[[], None], mono: Callable[[], float] = time.monotonic) -> None: + self._base = base + self.budget = max(0, int(budget)) + self._gate = gate + self._mono = mono + self._lock = threading.Lock() + self.used = 0 + self.refused = 0 + + @property + def exhausted(self) -> bool: + return self.used >= self.budget + + def request(self, path: str) -> Outcome: + with self._lock: + if self.used >= self.budget: + self.refused += 1 + return Outcome("budget", 0.0, None, refused=True) + self.used += 1 + t0 = self._mono() + self._gate() + t1 = self._mono() + try: + data = self._base(path) + except sv.Blocked as e: + return Outcome(status_of(e.__cause__) or "blocked", self._mono() - t1, None, t1 - t0, blocked=e) + except Exception as e: # tek isteğin hatası: kayda geçer, kaydedici sürer + return Outcome(status_of(e) or type(e).__name__, self._mono() - t1, None, t1 - t0) + return Outcome(200 if data is not None else 404, self._mono() - t1, data, t1 - t0) + + +@dataclass +class Pending: + sport: str + eid: int + kinds: Set[str] + sources: Set[str] + first_ts: float + first_mono: float + expect: Dict[str, Any] + due: float + attempts: int = 0 + + +class EventReads: + """ + Kaydedicinin değişiklikle tetiklenen maç sayfası okumaları (kendi thread'i). Aynı maçın bekleyen okuması + birleştirilir: en erken tetikleme anı kalır, beklenen durum en sonuncusudur. Değişikliği göstermeyen okuma `retry` + sn sonra, en çok `attempts` kez yinelenir; engellenmede artan aralıklarla beklenir. Bekleyen okuma sayısı sınırlıdır. + """ + + def __init__(self, soak: "Soak", *, retry: float = sv.CONFIRM_RETRY_SECONDS, attempts: int = sv.CONFIRM_ATTEMPTS, + max_pending: int = MAX_PENDING_READS, blocked_first: float = sv.BLOCKED_FIRST_SECONDS, + blocked_max: float = sv.BLOCKED_MAX_SECONDS) -> None: + self._soak = soak + self.retry = retry + self.attempts = max(1, attempts) + self._max = max_pending + self._blocked = (blocked_first, blocked_max) + self._blocked_rounds = 0 + self._cond = threading.Condition() + self._pending: "collections.OrderedDict[int, Pending]" = collections.OrderedDict() + self.overflow = 0 + + def want(self, p: Pending) -> None: + with self._cond: + old = self._pending.get(p.eid) + if old is not None: + old.kinds |= p.kinds + old.sources |= p.sources + old.expect = p.expect + old.due = min(old.due, p.due) # yinelemeyi bekleyen okuma: yeni değişiklikle hemen + elif len(self._pending) >= self._max: + self.overflow += 1 + return + else: + self._pending[p.eid] = p + self._cond.notify() + + def pending(self) -> int: + with self._cond: + return len(self._pending) + + def _next(self, stop: threading.Event) -> Optional[Pending]: + with self._cond: + while not stop.is_set(): + now = self._soak.mono() + if self._pending: + item = min(self._pending.values(), key=lambda p: (p.due, p.first_mono)) + if item.due <= now: + return self._pending.pop(item.eid) + self._cond.wait(min(0.5, item.due - now)) + else: + self._cond.wait(0.5) + return None + + def run(self, stop: threading.Event) -> None: + from sofascore_scraper.client import request_context + + with request_context(breaker=None): + while not stop.is_set(): + item = self._next(stop) + if item is not None: + self._read(item) + + def wake(self) -> None: + with self._cond: + self._cond.notify_all() + + def _read(self, p: Pending) -> None: + soak = self._soak + out = soak.client.request(f"/event/{p.eid}") + done_ts, done_mono = soak.wall(), soak.mono() + if out.refused: + soak.on_event_skipped(p) + return + p.attempts += 1 + event = (out.data or {}).get("event") if out.data is not None else None + ok = bool(event) and reflects(p.kinds, p.expect, event) + final = ok or (out.blocked is None and p.attempts >= self.attempts) + retry_in: Optional[float] = None + if out.blocked is not None: + self._blocked_rounds += 1 + first, cap = self._blocked + retry_in = min(cap, first * 2 ** (self._blocked_rounds - 1)) + final = False + else: + self._blocked_rounds = 0 + if not final: + retry_in = self.retry + latency = done_mono - p.first_mono if ok else None + soak.on_event_read(p, out, event, ok=ok, final=final, latency=latency, retry_in=retry_in, + ts=done_ts, mono=done_mono) + if retry_in is not None: + p.due = done_mono + retry_in + self.want(p) + + +class FallbackReads: + """ + Yoklama öndeyken (sahibin c için kuralı) her canlı maçın sayfası dakikada en az bir kez: dakikanın başında o + sporların son canlı listesindeki maçlar sıralanır (öncelikli id'ler, bitişe yakınlar, en uzun süredir okunmayanlar) + ve dakikanın payı (`cap`) kadar okunur. Dakika içinde başka bir yoldan (değişiklik okuması, yoklama kaynağı) + okunmuş maç yeniden okunmaz, kapsanmış sayılır. Sığmayan okuma atlanır ve sayılır; bir sonraki dakikaya taşınmaz. + """ + + def __init__(self, soak: "Soak", *, rate: float, share: float, window: float = FALLBACK_WINDOW_SECONDS, + priority: Iterable[int] = ()) -> None: + self._soak = soak + self.window = float(window) + self.cap = max(0, int(math.floor(self.window * rate * (1.0 - share) + 1e-9))) + self.priority: FrozenSet[int] = frozenset(int(p) for p in priority) + + def run(self, stop: threading.Event) -> None: + from sofascore_scraper.client import request_context + + with request_context(breaker=None): + while not stop.is_set(): + sports = self._soak.poll_led_sports() + if not sports: + stop.wait(min(1.0, self.window / 4)) + continue + self._minute(sports, stop) + + def _rank(self, sport: str, eid: int) -> Tuple[int, int, float]: + last = self._soak.last_read.get((sport, eid)) + return (0 if eid in self.priority else 1, 0 if self._soak.near_end(sport, eid) else 1, + -1.0 if last is None else last) + + def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: + soak = self._soak + start_ts, start = soak.wall(), soak.mono() + end = start + self.window + live = {sport: soak.latest_list(sport) for sport in sports} + ranked = sorted(((s, e) for s in sports for e in live[s]), key=lambda c: self._rank(*c)) + made: Dict[str, int] = collections.defaultdict(int) + attempted = 0 + cut = None + for sport, eid in ranked: + if stop.is_set(): + cut = "stopped" + break + if sport not in soak.poll_led_sports(): + continue # push yeniden önde: bu sporun dakikalık okumaları durur + if soak.read_since(sport, eid, start): + continue # bu dakika başka bir yoldan okundu + if attempted >= self.cap: + cut = "share" + break + if soak.mono() >= end: + cut = "minute" + break + out = soak.fallback_read(sport, eid, window_start_ts=start_ts) + attempted += 1 + made[sport] += 1 + if out.refused or out.blocked is not None: + cut = "budget" if out.refused else "blocked" + break + # Dakikanın sonuna kadar beklenir; push bütün bu sporlarda yeniden önde olursa dakika erken biter + while not stop.is_set() and soak.mono() < end and set(sports) & set(soak.poll_led_sports()): + stop.wait(min(0.5, max(0.0, end - soak.mono()))) + led = set(soak.poll_led_sports()) + early = soak.mono() < end - 1e-3 + for sport in sports: + ids = live[sport] + covered = sum(1 for eid in ids if soak.read_since(sport, eid, start)) + # yarım dakika (durduruldu ya da push döndü) atlama oranına sayılmaz + soak.on_fallback_minute(sport, start_ts, live=len(ids), made=made[sport], covered=covered, + priority=sum(1 for eid in ids if eid in self.priority), cap=self.cap, cut=cut, + partial=early or sport not in led) + + +# --- push: musluk, soket, okuyucu, bağlantı, sayfa açıcı ------------------------------------------------------- + + +class TapSession: + """`page` kaynağında sayfanın bir WebSocket'i (musluğun gördüğü kadarıyla).""" + + def __init__(self, session: int, handshake_mono: float) -> None: + self.session = session + self.handshake_mono = handshake_mono # sayfada WebSocket'in açıldığı an + self.info_seen = False + self.killed: Optional[str] = None + + +class FeedTap: + """ + Bir sporun PushFeed'i ile kaynak arasındaki musluk: her şey olduğu gibi akışa geçer (servis aynı işaretleri görür); + MSG kareleri alındıkları an kaydedilir. `direct`'te bağlantının hat üstü olaylarını RecordingSocket kaydeder; + `page`'de (`page=True`) INFO, PING, PONG ve -ERR de buradan kaydedilir. Kaynağın akışta kullandığı dört işlemi + taşır (connection, received, closed, gone). + """ + + def __init__(self, feed: PushFeed, soak: "Soak", *, page: bool = False) -> None: + self._feed = feed + self._soak = soak + self._page = page + self.sport = feed.sport + self._readers: Dict[int, NatsReader] = {} + self._sessions: Dict[int, TapSession] = {} + + def connection(self) -> int: + conn = self._feed.connection() + self._readers[conn] = NatsReader() + if self._page: + self._sessions[conn] = self._soak.on_page_socket(self.sport) + else: + self._soak.on_sport_open(self.sport) + return conn + + def received(self, conn: int, payload: Any) -> None: + ts, mono = self._soak.wall(), self._soak.mono() + self._feed.received(conn, payload) + reader = self._readers.get(conn) + if reader is None: + return + for op in reader.feed(payload): + if op.kind == "MSG": + body = frame_body(op) + if body is not None: + self._soak.on_frame(self.sport, op.subject, body, ts, mono) + elif self._page: + session = self._sessions.get(conn) + if session is None: + continue + if op.kind == "INFO" and not session.info_seen: + self._soak.on_wire(session, op) + self._soak.on_sport_open(self.sport, session=session.session) + elif session.info_seen or op.kind != "INFO": + self._soak.on_wire(session, op) + + def closed(self, conn: int) -> None: + self._feed.closed(conn) + self._readers.pop(conn, None) + session = self._sessions.pop(conn, None) + if self._page: + if session is not None and session.info_seen: + self._soak.on_page_socket_closed(self.sport, session) + else: + self._soak.on_sport_closed(self.sport, "closed") + + def gone(self, reason: str) -> None: + self._feed.gone(reason) + self._readers.clear() + self._sessions.clear() + self._soak.on_sport_closed(self.sport, reason, gone=True) + + +class RecordingSocket: + """ + Bağlantı işlevinin döndürdüğü WebSocket'in sarmalayıcısı: hat üstü işlemlerin adları ve zamanları kaydedilir + (CONNECT gövdesi, INFO gövdesi, adres asla). `kill` soketi kapatır: istemci olağan bir kopma görür. + """ + + def __init__(self, ws: Any, soak: "Soak", session: int, handshake_mono: float) -> None: + self._ws = ws + self._soak = soak + self.session = session + self.handshake_mono = handshake_mono + self._reader = NatsReader() + self.info_seen = False + self.killed: Optional[str] = None + + def send_text(self, text: str) -> None: + self._ws.send_text(text) + for line in text.split("\r\n"): + if line: + self._soak.on_sent(self, line) + + def recv(self, timeout: float) -> Optional[bytes]: + data = self._ws.recv(timeout) + if data is not None: + for op in self._reader.feed(data): + self._soak.on_wire(self, op) + return data + + def close(self) -> None: + self._ws.close() + + def kill(self, cause: str) -> None: + self.killed = cause + sock = getattr(self._ws, "_sock", None) + try: + if sock is not None: + sock.shutdown(socket.SHUT_RDWR) + else: + self._ws.close() + except OSError: + pass + + +_SALT = secrets.token_bytes(32) # yalnızca bellek: kimlik bilgilerinin karşılaştırılması + + +def _digest(value: Any) -> bytes: + """Bellekte karşılaştırma için tuzlu HMAC; ne girdisi ne kendisi hiçbir yere yazılır.""" + text = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str) + return hmac.new(_SALT, text.encode("utf-8"), hashlib.sha256).digest() + + +def _same(a: Optional[bytes], b: Optional[bytes]) -> Optional[str]: + if a is None or b is None: + return None + return "same" if hmac.compare_digest(a, b) else "different" + + +class RecordingReader: + """ + Kimlik bilgisi okuyucusunun sarmalayıcısı: hangi sporun sayfasından, ne kadar sürede, başarılı mı, bir öncekiyle + aynı mı (yalnızca "same"/"different"). Okuma sürerken kaynak örneklemesi 5 sn'ye iner (ikinci Chromium). + `compare_sports` verilirse ilk başarılı okumadan sonra diğer sporların sayfalarından da okunur ve karşılaştırılır. + """ + + def __init__(self, inner: Any, soak: "Soak", *, compare_factory: Optional[Callable[[], Any]] = None, + compare_sports: Sequence[str] = ()) -> None: + self._inner = inner + self._soak = soak + self._compare_factory = compare_factory + self._compare_sports = tuple(compare_sports) + self._lock = threading.Lock() # okumalar sırayla: aynı canlı profil + self._previous: Optional[Tuple[bytes, bytes]] = None + self._first: Optional[Tuple[str, bytes, bytes]] = None + self.reads = 0 + self.compare_thread: Optional[threading.Thread] = None + + def _read_with(self, reader: Any, sport: str, purpose: str) -> Any: + self.reads += 1 + token = self._soak.on_credential_start(sport, self.reads, purpose) + try: + credential = reader.read(sport) + except ds.CredentialUnavailable as e: + self._soak.on_credential_end(token, ok=False, error=str(e)[:200]) + raise + except Exception as e: + self._soak.on_credential_end(token, ok=False, error=type(e).__name__) + raise + digests = (_digest(dict(credential.options)), _digest(credential.url)) + extra: Dict[str, Any] = {} + if purpose == "connect": + prev = self._previous + # Anahtar adları sofascore_scraper/redact.py'nin gizli saydığı adlara (…CREDENTIAL…) benzemez: değer maskelenmesin + extra = {"connect_vs_previous": _same(prev[0] if prev else None, digests[0]), + "address_vs_previous": _same(prev[1] if prev else None, digests[1])} + self._previous = digests + if self._first is None: + self._first = (sport, digests[0], digests[1]) + self._soak.on_credential_end(token, ok=True, **extra) + return credential if purpose == "connect" else digests + + def read(self, sport: str) -> Any: + with self._lock: + credential = self._read_with(self._inner, sport, "connect") + others = [s for s in self._compare_sports if s != sport] + if others and self._compare_factory is not None and self.compare_thread is None: + self.compare_thread = threading.Thread(target=self._compare, args=(others,), name="soak-compare", + daemon=True) + self.compare_thread.start() + return credential + + def _compare(self, others: Sequence[str]) -> None: + assert self._first is not None and self._compare_factory is not None + base_sport, base_options, base_address = self._first + for other in others: + if self._soak.stopping: + return + reader = self._compare_factory() + try: + with self._lock: + digests = self._read_with(reader, other, "compare") + except Exception: + self._soak.on_credential_compare(base_sport, other, None, None) + continue + finally: + try: + reader.close() + except Exception: + pass + self._soak.on_credential_compare(base_sport, other, _same(base_options, digests[0]), + _same(base_address, digests[1])) + + def close(self) -> None: + self._inner.close() + + +class RecordingConnection(ds.DirectConnection): + """Uygulamanın DirectConnection'ı; oturum başı/sonu, yeniden deneme ve akışlar kaydedilir, bağlantı öldürülebilir.""" + + def __init__(self, reader: Any, soak: "Soak", **options: Any) -> None: + self._soak = soak + inner = options.pop("connect", None) or ds.default_connect + self.current: Optional[RecordingSocket] = None + self._session_no = 0 + + def connect(credential: Any) -> RecordingSocket: + t0 = soak.mono() + try: + ws = inner(credential) + except ds.HandshakeRejected as e: + soak.on_handshake(self._session_no, soak.mono() - t0, status=e.status) + raise + except (OSError, ds.WebSocketError) as e: + soak.on_handshake(self._session_no, soak.mono() - t0, error=type(e).__name__) + raise + soak.on_handshake(self._session_no, soak.mono() - t0, status=101) + sock = RecordingSocket(ws, soak, self._session_no, soak.mono()) + self.current = sock + return sock + + super().__init__(reader, connect=connect, **options) + + def subscribe(self, sport: str, feed: PushFeed) -> None: + super().subscribe(sport, FeedTap(feed, self._soak)) # type: ignore[arg-type] + + def _session(self, credential: Any) -> Tuple[str, str, float]: + self._session_no += 1 + n = self._session_no + self._soak.on_session_start(n) + outcome, reason, lasted = super()._session(credential) + sock, self.current = self.current, None + self._soak.on_session_end(n, outcome, reason, lasted, killed=sock.killed if sock is not None else None) + return outcome, reason, lasted + + def _failed(self, reason: str, *, reading: bool = False) -> None: + self._soak.on_retry(reason, failures=self.failures + 1, reading=reading, + unhealthy=self.unhealthy or self.failures + 1 >= self._unhealthy_after) + super()._failed(reason, reading=reading) + + def kill(self, cause: str) -> bool: + sock = self.current + if sock is None: + return False + sock.kill(cause) + return True + + +class RecordingOpener: + """`page` kaynağının sayfa açıcısı (BrowserPageOpener) sarmalanır: akışa musluk takılır, sayfalar kapatılabilir.""" + + def __init__(self, inner: Any, soak: "Soak") -> None: + self._inner = inner + self._soak = soak + self._handles: Dict[str, Any] = {} + self._lock = threading.Lock() + + def open(self, sport: str, feed: PushFeed) -> Any: + self._soak.recorder.write("connection", {"event": "page_open", "sport": sport}) + handle = self._inner.open(sport, FeedTap(feed, self._soak, page=True)) + with self._lock: + self._handles[sport] = handle + return handle + + def kill(self) -> List[str]: + """Açık sayfaların hepsini kapatır (sayfa kapanışı akışa `gone` olarak düşer, PageSource yeniden açar).""" + with self._lock: + handles, self._handles = dict(self._handles), {} + for handle in handles.values(): + try: + handle.close() + except Exception: + pass + return sorted(handles) + + def close(self) -> None: + self._inner.close() + + +class AllScope(sv.SportScope): + """Sporun bütün canlı maçları kapsamda (servisin yoklama kaynağı her canlı maçı bugünkü kurallarla izler).""" + + def listed(self, event: Mapping[str, Any]) -> bool: + return True + + +class RecordingLiveService(sv.LiveService): + """LiveService; her sistem olayı (hakemin Switch'i, engellenme) kaydediciye de verilir.""" + + def __init__(self, *args: Any, soak: "Soak", **kwargs: Any) -> None: + self._soak_hook = soak + super().__init__(*args, **kwargs) + + def _system(self, type_: str, data: Mapping[str, Any]) -> None: + super()._system(type_, data) + self._soak_hook.on_system(type_, dict(data)) + + +# --- kaynaklar -------------------------------------------------------------------------------------------------- + + +def _rss_kb(pid: int) -> Optional[int]: + try: + with open(f"/proc/{pid}/status", encoding="utf-8") as f: + for line in f: + if line.startswith("VmRSS:"): + return int(line.split()[1]) + except (OSError, ValueError, IndexError): + return None + return 0 + + +def _process_table() -> Dict[int, int]: + """pid → ppid (procfs).""" + table: Dict[int, int] = {} + try: + names = os.listdir("/proc") + except OSError: + return table + for name in names: + if not name.isdigit(): + continue + try: + with open(f"/proc/{name}/stat", encoding="utf-8") as f: + stat = f.read() + table[int(name)] = int(stat.rsplit(")", 1)[1].split()[1]) + except (OSError, ValueError, IndexError): + continue + return table + + +def _profile_of(pid: int) -> Optional[str]: + """Tarayıcının ana sürecinin `--user-data-dir`'i (yalnızca sınıflandırmada kullanılır, yazılmaz).""" + try: + with open(f"/proc/{pid}/cmdline", "rb") as f: + args = f.read().split(b"\0") + except OSError: + return None + for arg in args: + if arg.startswith(b"--user-data-dir="): + return arg.split(b"=", 1)[1].decode("utf-8", "replace").rstrip("/") + return None + + +def classify_browser(profile: Optional[str]) -> str: + """Profil dizini → "live" (canlı profil `-live`: sayfalar ya da kimlik bilgisi okuyucusu) ya da "bridge".""" + if profile is None: + return "other" + return "live" if profile.endswith("-live") else "bridge" + + +def sample_resources() -> Dict[str, Any]: + """ + Bu sürecin RSS'i, alt süreçlerinin tarayıcıya göre RSS'i (ana köprü, canlı profil, diğer: Playwright sürücüsü), + toplam ve açık soket sayısı. Bir alt süreç, kendisinin ya da en yakın atasının `--user-data-dir`'ine göre ayrılır. + """ + pid = os.getpid() + self_kb = _rss_kb(pid) + if self_kb is None: # /proc yok (Linux dışı) + import resource + + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss # Linux'ta kB, macOS'ta bayt; zirve + mb = peak / (1024 * 1024) if sys.platform == "darwin" else peak / 1024 + return {"rss_self_mb": round(mb, 1), "rss_bridge_mb": None, "rss_live_mb": None, + "rss_other_mb": None, "rss_total_mb": None, "processes": None, "sockets": None, "procfs": False} + table = _process_table() + children: Dict[int, List[int]] = collections.defaultdict(list) + for child, parent in table.items(): + children[parent].append(child) + kb = {"bridge": 0, "live": 0, "other": 0} + procs = {"bridge": 0, "live": 0, "other": 0} + todo: List[Tuple[int, Optional[str]]] = [(c, None) for c in children.get(pid, [])] + while todo: + p, inherited = todo.pop() + kind = inherited + if kind is None: + profile = _profile_of(p) + kind = classify_browser(profile) if profile is not None else None + bucket = kind or "other" + kb[bucket] += _rss_kb(p) or 0 + procs[bucket] += 1 + todo.extend((c, kind) for c in children.get(p, [])) + try: + sockets = sum(1 for fd in os.listdir("/proc/self/fd") + if os.readlink(f"/proc/self/fd/{fd}").startswith("socket:")) + except OSError: + sockets = None + total = self_kb + sum(kb.values()) + return {"rss_self_mb": round(self_kb / 1024, 1), "rss_bridge_mb": round(kb["bridge"] / 1024, 1), + "rss_live_mb": round(kb["live"] / 1024, 1), "rss_other_mb": round(kb["other"] / 1024, 1), + "rss_total_mb": round(total / 1024, 1), "processes": procs, "sockets": sockets} + + +# --- zaman çizelgeleri ------------------------------------------------------------------------------------------ + + +def step_positive(points: Sequence[Tuple[float, int]], start: float, end: float) -> List[Tuple[float, float]]: + """(an, canlı maç sayısı) noktalarından sayının > 0 olduğu aralıklar (bir nokta bir sonrakine kadar geçerli).""" + out: List[Tuple[float, float]] = [] + pts = sorted(p for p in points if p[0] <= end) + for i, (t, n) in enumerate(pts): + s = max(t, start) + e = min(pts[i + 1][0] if i + 1 < len(pts) else end, end) + if n > 0 and e > s: + if out and abs(out[-1][1] - s) < 1e-9: + out[-1] = (out[-1][0], e) + else: + out.append((s, e)) + return out + + +def overlap(intervals: Sequence[Tuple[float, float]], a: float, b: float) -> float: + return sum(max(0.0, min(e, b) - max(s, a)) for s, e in intervals) + + +def quiet_gaps(frames: Sequence[float], positive: Sequence[Tuple[float, float]], start: float, + end: float) -> List[Tuple[float, float]]: + """MSG karesi gelmeyen ve canlı listenin > 0 maç gösterdiği kesintisiz aralıklar (uzundan kısaya).""" + bounds = [start] + sorted(t for t in frames if start <= t <= end) + [end] + out: List[Tuple[float, float]] = [] + for a, b in zip(bounds, bounds[1:], strict=False): + for s, e in positive: + lo, hi = max(a, s), min(b, e) + if hi > lo: + out.append((lo, hi)) + out.sort(key=lambda iv: iv[1] - iv[0], reverse=True) + return out + + +def leader_intervals(timeline: Sequence[Tuple[float, str]], start: float, end: float, + leader: str) -> List[Tuple[float, float]]: + out: List[Tuple[float, float]] = [] + pts = sorted(timeline) + for i, (t, who) in enumerate(pts): + s, e = max(t, start), min(pts[i + 1][0] if i + 1 < len(pts) else end, end) + if who == leader and e > s: + out.append((s, e)) + return out + + +def complement(intervals: Sequence[Tuple[float, float]], start: float, end: float) -> List[Tuple[float, float]]: + out: List[Tuple[float, float]] = [] + cursor = start + for s, e in sorted(intervals): + if s > cursor: + out.append((cursor, min(s, end))) + cursor = max(cursor, e) + if cursor < end: + out.append((cursor, end)) + return [(s, e) for s, e in out if e > s] + + +def count_between(times: Sequence[float], a: float, b: float) -> int: + return bisect.bisect_right(times, b) - bisect.bisect_left(times, a) + + +def longest_without(times: Sequence[float], a: float, b: float) -> float: + bounds = [a] + [t for t in times if a <= t <= b] + [b] + return max((y - x for x, y in zip(bounds, bounds[1:], strict=False)), default=b - a) + + +class KillSchedule: + """`--kill-every` (dakika, tekrar) ve `--kill-at` (dakika, bir kez); `due(geçen sn)` → neden ya da None.""" + + def __init__(self, every_minutes: Optional[float] = None, at_minutes: Optional[float] = None) -> None: + self.every = every_minutes * 60.0 if every_minutes else None + self.at = at_minutes * 60.0 if at_minutes is not None else None + self._next_every = self.every + self._at_done = False + + def due(self, elapsed: float) -> Optional[str]: + if self.at is not None and not self._at_done and elapsed >= self.at: + self._at_done = True + return "kill-at" + if self._next_every is not None and self.every is not None and elapsed >= self._next_every: + while self._next_every <= elapsed: + self._next_every += self.every + return "kill-every" + return None + + +def default_budget(hours: float, rate: float = DEFAULT_RATE) -> int: + """Koşunun REST bütçesi: süre × hız (ortalama ≤ 1 istek/sn).""" + return max(1, int(hours * 3600 * min(1.0, rate))) + + +# --- koşu ------------------------------------------------------------------------------------------------------- + + +@dataclass +class SportStats: + frames: List[float] = field(default_factory=list) # MSG karelerinin alındığı an (epoch) + status_frames: int = 0 + score_frames: int = 0 + finish_frames: int = 0 + events: Set[int] = field(default_factory=set) + frame_lag: List[float] = field(default_factory=list) # alındı − changeTimestamp, her kare + finish_lag: List[float] = field(default_factory=list) # aynısı, bitiş kareleri + triggers: Dict[str, int] = field(default_factory=lambda: collections.defaultdict(int)) + read_latency: Dict[str, List[float]] = field(default_factory=lambda: collections.defaultdict(list)) + not_reflected: int = 0 + reads_ok: int = 0 + skipped_budget: int = 0 + live_counts: List[Tuple[float, int]] = field(default_factory=list) + leader: List[Tuple[float, str]] = field(default_factory=list) + first_push_at: Optional[float] = None + open_intervals: List[List[Any]] = field(default_factory=list) # [başlangıç, bitiş, kapanış nedeni] + open_since: Optional[float] = None + fallback: List[Dict[str, Any]] = field(default_factory=list) # dakika satırları + + +class Soak: + """ + Bir koşu: kaydedici, servis (kendi thread'inde), değişiklik okumaları, yedek dakika okumaları ve kaynak örneklemesi. + `start` ve `stop` testlerden de çağrılır; `run` gerçek koşunun döngüsüdür (öldürme takvimi, 10 dakikalık özet). + + Testler gerçek okuyucu, bağlantı, sayfa açıcı ve istek işlevi yerine sahtelerini verir (`reader`, + `connection_options`, `page_opener`, `base_fetch`, `gate`, `compare_factory`); varsayılanlar uygulamanın gerçek + parçalarıdır. + """ + + def __init__(self, out_dir: str, sports: Sequence[str], *, run_id: str, data_dir: str, + budget: int, source: str = "direct", hours: Optional[float] = None, + kill_every: Optional[float] = None, kill_at: Optional[float] = None, poll_interval: float = 30.0, + rate: float = DEFAULT_RATE, list_share: float = DEFAULT_LIST_SHARE, + priority_ids: Iterable[int] = (), fallback_window: float = FALLBACK_WINDOW_SECONDS, + base_fetch: Optional[Callable[[str], Optional[Dict[str, Any]]]] = None, + gate: Optional[Callable[[], None]] = None, reader: Any = None, page_opener: Any = None, + compare_sports: bool = False, compare_factory: Optional[Callable[[], Any]] = None, + connection_options: Optional[Dict[str, Any]] = None, + resource_seconds: float = RESOURCE_SECONDS, reading_resource_seconds: float = READING_RESOURCE_SECONDS, + event_retry_seconds: float = sv.CONFIRM_RETRY_SECONDS, store: Any = None, + args_note: Optional[Dict[str, Any]] = None) -> None: + from sofascore_scraper.store import open_store + + if source not in SOURCES: + raise ValueError(f"source must be one of {SOURCES}") + self.wall = time.time + self.mono = time.monotonic + self.run_id = run_id + self.source = source + self.push_name = source # hakemin push önderken yazdığı ad + self.sports = tuple(dict.fromkeys(sports)) + self.hours = hours + self.recorder = Recorder(out_dir, wall=self.wall, mono=self.mono) + self.kills = KillSchedule(kill_every, kill_at) + self.args_note = dict(args_note or {}) + self._lock = threading.RLock() + self._stop = threading.Event() + self.stopping = False + self.client = ReadClient(base_fetch or sv._default_fetch, budget=budget, + gate=gate if gate is not None else default_gate(rate), mono=self.mono) + self.tracker = ChangeTracker() + self.reads = EventReads(self, retry=event_retry_seconds) + self.fallback = FallbackReads(self, rate=rate, share=list_share, window=fallback_window, + priority=priority_ids) + self.stats: Dict[str, SportStats] = {s: SportStats() for s in self.sports} + self.last_read: Dict[Tuple[str, int], float] = {} # son başarılı maç sayfası okuması (mono), her yoldan + self._latest_list: Dict[str, List[int]] = {} + self._resource_seconds = resource_seconds + self._reading_resource_seconds = reading_resource_seconds + self._wake_sampler = threading.Event() + self._budget_noted = False + # bağlantı + self.session_no = 0 + self.sessions: List[Dict[str, Any]] = [] + self._session: Dict[str, Any] = {} + self._conn_open = False + self._kill_cause: Dict[str, str] = {} + self.ping_sent: List[float] = [] + self.pongs: List[float] = [] + self.server_pings: List[float] = [] + self.rtt: List[float] = [] + self.handshake_to_info: List[float] = [] + self.handshake_seconds: List[float] = [] + self.errors: Dict[str, int] = collections.defaultdict(int) + self.drops: Dict[str, int] = collections.defaultdict(int) + self.kill_log: List[Dict[str, Any]] = [] + self.retries = 0 + self.credential_reads: List[Dict[str, Any]] = [] + self.compares: List[Dict[str, Any]] = [] + self._reading: Optional[Dict[str, Any]] = None + self.peak_rss_mb: Dict[str, float] = {"self": 0.0, "bridge": 0.0, "live": 0.0, "total": 0.0} + self.last_resources: Optional[Dict[str, Any]] = None + self.requests: Dict[str, int] = collections.defaultdict(int) + self.request_latency: Dict[str, List[float]] = collections.defaultdict(list) + self.switches: List[Dict[str, Any]] = [] + self.blocked: List[Dict[str, Any]] = [] + self.service_error: Optional[str] = None + # bileşenler + self.reader: Optional[RecordingReader] = None + self.connection: Optional[RecordingConnection] = None + self.opener: Optional[RecordingOpener] = None + if source == "direct": + inner_reader = reader if reader is not None else ds.BrowserCredentialReader() + factory = compare_factory or (ds.BrowserCredentialReader if reader is None else None) + self.reader = RecordingReader(inner_reader, self, compare_factory=factory, + compare_sports=self.sports if compare_sports else ()) + self.connection = RecordingConnection(self.reader, self, **dict(connection_options or {})) + else: + from sofascore_scraper.services.live.push_source import BrowserPageOpener + + self.opener = RecordingOpener(page_opener if page_opener is not None else BrowserPageOpener(), self) + self.store = store if store is not None else open_store(data_dir) + scope = sv.LiveScope(sports=tuple(AllScope(s) for s in self.sports)) + self.service = RecordingLiveService( + self.store, scope, fetch=self._service_fetch, poll_interval=poll_interval, confirm=False, + requested_source=source, direct_connection=self.connection, page_opener=self.opener, soak=self) + self.started_ts: Optional[float] = None + self.started_mono: Optional[float] = None + self.ended_ts: Optional[float] = None + self._threads: List[threading.Thread] = [] + self._last_summary = 0.0 + + # --- yaşam döngüsü ------------------------------------------------------------------------------------ + + def start(self) -> None: + self.started_ts, self.started_mono = self.wall(), self.mono() + self._last_summary = self.started_mono + for sport in self.sports: + self.stats[sport].leader.append((self.started_ts, LEADER_POLL)) + self.recorder.write("leader", {"event": "start", "sport": sport, "leader": LEADER_POLL}) + self.recorder.write("connection", {"event": "run_start", "source": self.source, "sports": list(self.sports), + "run_id": self.run_id}) + self._spawn("soak-service", self._run_service) + self._spawn("soak-event-reads", lambda: self.reads.run(self._stop)) + self._spawn("soak-fallback", lambda: self.fallback.run(self._stop)) + self._spawn("soak-resources", self._sample_loop) + + def _spawn(self, name: str, target: Callable[[], None]) -> None: + t = threading.Thread(target=target, name=name, daemon=True) + t.start() + self._threads.append(t) + + def _run_service(self) -> None: + try: + self.service.run(self._stop) + except Exception as e: # servis düştü: kayda geçer, koşu biter + self.service_error = type(e).__name__ + self.recorder.write("connection", {"event": "service_error", "error": type(e).__name__}) + self._stop.set() + + @property + def running(self) -> bool: + return not self._stop.is_set() + + def stop(self, timeout: float = 30.0) -> None: + self.stopping = True + self._stop.set() + self.reads.wake() + self._wake_sampler.set() + for t in self._threads: + t.join(timeout) + thread = self.reader.compare_thread if self.reader is not None else None + if thread is not None: + thread.join(timeout) + self.ended_ts = self.wall() + for sport in self.sports: + self._close_sport(sport, self.ended_ts, "run end") + self.recorder.write("connection", {"event": "run_end"}) + self.write_summary(final=True) + self.recorder.close() + try: + self.store.close() + except Exception: + pass + + def run(self, hours: float, *, tick: float = 1.0) -> None: + """Gerçek koşu: süre dolana ya da durdurulana kadar; öldürme takvimi ve 10 dakikalık özet.""" + self.start() + deadline = self.mono() + hours * 3600.0 + try: + while self.running and self.mono() < deadline: + self._stop.wait(tick) + cause = self.kills.due(self.mono() - (self.started_mono or self.mono())) + if cause: + self.kill(cause) + if self.mono() - self._last_summary >= SUMMARY_SECONDS: + self.write_summary(final=False) + finally: + self.stop() + + def request_stop(self) -> None: + self._stop.set() + + def kill(self, cause: str) -> bool: + """`direct`: bağlantının soketini kapatır; `page`: sporların sayfalarını kapatır.""" + sports: List[str] = [] + if self.connection is not None: + done = self.connection.kill(cause) + else: + assert self.opener is not None + with self._lock: + for sport in self.sports: + self._kill_cause[sport] = cause + sports = self.opener.kill() + done = bool(sports) + with self._lock: + self.kill_log.append({"ts": self.wall(), "cause": cause, "done": done}) + row: Dict[str, Any] = {"event": "kill", "cause": cause, "done": done, "session": self.session_no} + if self.opener is not None: + row["pages"] = sports + self.recorder.write("connection", row) + return done + + # --- push ------------------------------------------------------------------------------------------------ + + def on_frame(self, sport: str, subject: str, body: Mapping[str, Any], ts: float, mono: float) -> None: + with self._lock: + kinds, expect, cls = self.tracker.frame(sport, body) + st = self.stats.setdefault(sport, SportStats()) + st.frames.append(ts) + st.events.add(int(body["id"])) + change = _ts(body.get(CHANGE_TS)) + lag = ts - change if change is not None else None + if lag is not None: + st.frame_lag.append(lag) + if "status.code" in body or "status.type" in body: + st.status_frames += 1 + if any(str(k).startswith(SCORE_PREFIXES) for k in body): + st.score_frames += 1 + if cls in TERMINAL_CLASSES: + st.finish_frames += 1 + if lag is not None and KIND_FINISH in kinds: + st.finish_lag.append(lag) + for kind in kinds: + st.triggers[kind] += 1 + values = {k: v for k, v in body.items() if k in STATUS_VALUE_KEYS or str(k).startswith(SCORE_PREFIXES)} + row = {"sport": sport, "event_id": int(body["id"]), + "subject": subject.split(".", 1)[0] if subject.startswith(("sport.", "event.")) else "other", + "paths": sorted(str(k) for k in body if k != "id"), "values": values, + "change_ts": change, "lag_s": round(lag, 3) if lag is not None else None, + "triggers": sorted(kinds)} + self.recorder.write("frames", row, ts=ts, mono=mono) + if kinds: + self.reads.want(Pending(sport, int(body["id"]), set(kinds), {"frame"}, ts, mono, expect, mono)) + + def on_sport_open(self, sport: str, *, session: Optional[int] = None) -> None: + ts = self.wall() + session = self.session_no if session is None else session + with self._lock: + st = self.stats.setdefault(sport, SportStats()) + if st.open_since is None: + st.open_since = ts + first = self.connection is not None and not self._conn_open + if first: + self._conn_open = True + hs = self._session.get("handshake_mono") + if first: + since = round(self.mono() - hs, 3) if hs is not None else None + self.recorder.write("connection", {"event": "open", "session": session, "handshake_to_open_s": since}) + self.recorder.write("connection", {"event": "sport_open", "sport": sport, "session": session}) + + def _close_sport(self, sport: str, ts: float, reason: str) -> None: + with self._lock: + st = self.stats.setdefault(sport, SportStats()) + if st.open_since is not None: + st.open_intervals.append([st.open_since, ts, reason]) + st.open_since = None + + def on_sport_closed(self, sport: str, reason: str, *, gone: bool = False) -> None: + with self._lock: + killed = self._kill_cause.pop(sport, None) if self.opener is not None else None + why = reason + (f" ({killed})" if killed else "") + self._close_sport(sport, self.wall(), why) + self.recorder.write("connection", {"event": "page_gone" if gone and self.opener is not None else "sport_closed", + "sport": sport, "reason": why[:200], "session": self.session_no}) + + # `page`: sayfanın her WebSocket'i bir oturumdur + def on_page_socket(self, sport: str) -> TapSession: + with self._lock: + self.session_no += 1 + n = self.session_no + return TapSession(n, self.mono()) + + def on_page_socket_closed(self, sport: str, session: TapSession) -> None: + ts = self.wall() + with self._lock: + killed = self._kill_cause.pop(sport, None) + self.sessions.append({"n": session.session, "sport": sport, "outcome": "closed", "reason": "closed", + "lasted": self.mono() - session.handshake_mono, "killed": killed, "end_ts": ts}) + self.drops["closed" + (f" ({killed})" if killed else "")] += 1 + self._close_sport(sport, ts, "websocket closed" + (f" ({killed})" if killed else "")) + self.recorder.write("connection", {"event": "session_end", "sport": sport, "session": session.session, + "outcome": "closed", "killed": killed}) + + # --- bağlantı (`direct`) ------------------------------------------------------------------------------------ + + def on_session_start(self, n: int) -> None: + with self._lock: + self.session_no = n + self._session = {"n": n, "start_mono": self.mono(), "msgs": 0, "subs": 0} + self.recorder.write("connection", {"event": "session_start", "session": n}) + + def on_handshake(self, n: int, seconds: float, *, status: Optional[int] = None, + error: Optional[str] = None) -> None: + with self._lock: + if status == 101: + self.handshake_seconds.append(seconds) + self._session["handshake_mono"] = self.mono() + self.recorder.write("connection", {"event": "handshake", "session": n, "status": status, "error": error, + "seconds": round(seconds, 3)}) + + def on_sent(self, sock: RecordingSocket, line: str) -> None: + verb, _, rest = line.partition(" ") + verb = verb.upper() + row: Dict[str, Any] = {"session": sock.session} + if verb == "CONNECT": + row["event"] = "connect_sent" # gövde (kimlik bilgisi) asla + elif verb == "SUB": + subject = rest.split(" ")[0] + with self._lock: + self._session["subs"] = self._session.get("subs", 0) + 1 + resub = any(s.get("subs") for s in self.sessions) + row.update(event="subscribe", subject=subject if subject.startswith(ds.SUBJECT_PREFIX) else "other", + resubscribe=resub) + elif verb == "UNSUB": + row["event"] = "unsubscribe" + elif verb == "PING": + with self._lock: + self.ping_sent.append(self.wall()) + self._session["last_ping_mono"] = self.mono() + row["event"] = "ping_sent" + elif verb == "PONG": + row["event"] = "pong_sent" + else: + row["event"] = "sent_other" + self.recorder.write("connection", row) + + def on_wire(self, sock: Any, op: NatsOp) -> None: + """Sunucudan gelen işlem (MSG dışında): yalnızca türü, zamanı ve -ERR'in bilinen metni.""" + ts, mono = self.wall(), self.mono() + if op.kind == "MSG": + with self._lock: + self._session["msgs"] = self._session.get("msgs", 0) + 1 + return + row: Dict[str, Any] = {"session": sock.session} + if op.kind == "INFO": + if sock.info_seen: + return + sock.info_seen = True + since = mono - sock.handshake_mono + with self._lock: + self.handshake_to_info.append(since) + row.update(event="info", handshake_to_info_s=round(since, 3)) # gövde asla + elif op.kind == "PING": + with self._lock: + gap = ts - self.server_pings[-1] if self.server_pings else None + self.server_pings.append(ts) + row.update(event="server_ping", gap_s=round(gap, 3) if gap is not None else None) + elif op.kind == "PONG": + with self._lock: + gap = ts - self.pongs[-1] if self.pongs else None + self.pongs.append(ts) + sent = self._session.get("last_ping_mono") if self.connection is not None else None + rtt = mono - sent if sent is not None else None + if rtt is not None: + self.rtt.append(rtt) + row.update(event="pong", rtt_s=round(rtt, 3) if rtt is not None else None, + gap_s=round(gap, 3) if gap is not None else None) + elif op.kind == "-ERR": + with self._lock: + self.errors[op.text] += 1 + row.update(event="error", text=op.text) # bilinen NATS metni ya da "other" + else: + row.update(event="received_other", op="+OK" if op.kind == "+OK" else "OTHER") + self.recorder.write("connection", row, ts=ts, mono=mono) + + def on_session_end(self, n: int, outcome: str, reason: str, lasted: float, *, killed: Optional[str]) -> None: + ts = self.wall() + why = f"{outcome}: {reason}" + (f" ({killed})" if killed else "") + with self._lock: + self._conn_open = False + session = dict(self._session) + session.update(outcome=outcome, reason=reason, lasted=lasted, killed=killed, end_ts=ts) + self.sessions.append(session) + self.drops[why] += 1 + # musluk sporları oturum bitmeden kapattı ("closed"); pencerenin nedeni oturumunki olsun + for sport in self.sports: + st = self.stats[sport] + if st.open_since is not None: + st.open_intervals.append([st.open_since, ts, why]) + st.open_since = None + elif st.open_intervals and st.open_intervals[-1][2] == "closed" and ts - st.open_intervals[-1][1] < 5: + st.open_intervals[-1][2] = why + self.recorder.write("connection", {"event": "session_end", "session": n, "outcome": outcome, + "reason": reason[:200], "lasted_s": round(lasted, 3), "killed": killed, + "messages": session.get("msgs", 0)}) + + def on_retry(self, reason: str, *, failures: int, reading: bool, unhealthy: bool) -> None: + with self._lock: + self.retries += 1 + self.recorder.write("connection", {"event": "retry_wait", "reason": reason[:200], "failures": failures, + "reading": reading, "unhealthy": unhealthy}) + + def on_credential_start(self, sport: str, n: int, purpose: str) -> Dict[str, Any]: + token = {"sport": sport, "n": n, "purpose": purpose, "start_mono": self.mono(), "peak_mb": 0.0, + "samples": 0} + with self._lock: + self._reading = token + self.recorder.write("connection", {"event": "credential_read_start", "sport": sport, "read": n, + "purpose": purpose}) + self._wake_sampler.set() + return token + + def on_credential_end(self, token: Dict[str, Any], *, ok: bool, error: Optional[str] = None, + **extra: Any) -> None: + self._sample(token) # okuma bitmeden son bir ölçüm (tarayıcı kapanmadan önceki zirve kaçmasın) + with self._lock: + self._reading = None + entry = {"sport": token["sport"], "read": token["n"], "purpose": token["purpose"], "ok": ok, + "error": error, "seconds": round(self.mono() - token["start_mono"], 3), + "peak_rss_total_mb": token["peak_mb"], "samples": token["samples"], **extra} + self.credential_reads.append(entry) + self.recorder.write("connection", {"event": "credential_read", **entry}) + + def on_credential_compare(self, base_sport: str, other: str, options: Optional[str], + address: Optional[str]) -> None: + entry = {"read_from": base_sport, "compared_with": other, "connect_options": options or "unavailable", + "address": address or "unavailable"} + with self._lock: + self.compares.append(entry) + self.recorder.write("connection", {"event": "credential_compare", **entry}) + + def on_system(self, type_: str, data: Dict[str, Any]) -> None: + ts = self.wall() + if type_ == sv.SYSTEM_SOURCE_CHANGED: + sport, to = str(data.get("sport")), str(data.get("to")) + fallback = "start" if to == LEADER_POLL else "stop" if data.get("from") == LEADER_POLL else None + with self._lock: + st = self.stats.setdefault(sport, SportStats()) + st.leader.append((ts, to)) + if to == self.push_name and st.first_push_at is None: + st.first_push_at = ts + self.switches.append({"ts": ts, **data}) + self.recorder.write("leader", {"event": "switch", "sport": sport, "from": data.get("from"), "to": to, + "reason": data.get("reason"), "fallback": fallback}) + elif type_ in (sv.SYSTEM_BLOCKED, sv.SYSTEM_RECOVERED): + with self._lock: + self.blocked.append({"ts": ts, "type": type_}) + self.recorder.write("leader", {"event": type_.split(".", 1)[1], + **{k: data.get(k) for k in ("retry_in_s", "blocked_for_s")}}) + + def poll_led_sports(self) -> List[str]: + with self._lock: + return [s for s in self.sports if self.stats[s].leader and self.stats[s].leader[-1][1] == LEADER_POLL] + + # --- REST --------------------------------------------------------------------------------------------------- + + def _note_budget(self) -> None: + with self._lock: + if self._budget_noted: + return + self._budget_noted = True + self.recorder.write("reads", {"kind": "budget_exhausted", "used": self.client.used, + "budget": self.client.budget}) + + def _count(self, by: str, kind: str, out: Outcome) -> None: + with self._lock: + self.requests[f"{by}:{kind}:{out.status}"] += 1 + self.request_latency[kind].append(out.seconds) + + def _event_seen(self, sport: Optional[str], eid: int, event: Optional[Mapping[str, Any]]) -> None: + if not event: + return + with self._lock: + sports = [sport] if sport else [s for s in self.sports if self.tracker.state(s, eid) is not None] + for s in sports: + self.last_read[(s, eid)] = self.mono() + self.tracker.observe_event(s, event) + + def _service_fetch(self, path: str) -> Optional[Dict[str, Any]]: + """Servisin istek işlevi: bütçe, sıra, kayıt; canlı liste ve maç sayfaları kaydediciye de verilir.""" + trigger_ts = self.wall() + out = self.client.request(path) + m, e = LIST_PATH.match(path), EVENT_PATH.match(path) + kind = "list" if m else "event" if e else "other" + if out.refused: + self._note_budget() + return None + row: Dict[str, Any] = {"kind": kind, "by": "service", "status": out.status, "latency_s": round(out.seconds, 3), + "waited_s": round(out.waited, 3)} + if m: + row["sport"] = m.group(1) + if e: + row["event_id"] = int(e.group(1)) + events: Optional[List[Any]] = None + if m and out.data is not None: + events = [ev for ev in (out.data.get("events") or []) if isinstance(ev, dict)] + row["live_count"] = len(events) + self._count("service", kind, out) + self.recorder.write("reads", row) + if m and events is not None: + self._on_list(m.group(1), events, trigger_ts) + if e and out.data is not None: + self._event_seen(None, int(e.group(1)), out.data.get("event")) + if out.blocked is not None: + raise out.blocked + return out.data + + def _on_list(self, sport: str, events: List[Dict[str, Any]], ts: float) -> None: + mono = self.mono() + with self._lock: + changes = self.tracker.listing(sport, events) + st = self.stats.setdefault(sport, SportStats()) + st.live_counts.append((self.wall(), len(events))) + ids: List[int] = [] + for ev in events: + try: + ids.append(int(str(ev.get("id")))) + except ValueError: + continue + self._latest_list[sport] = ids + for _, kinds, _ in changes: + for kind in kinds: + st.triggers[kind] += 1 + for eid, kinds, expect in changes: + self.reads.want(Pending(sport, eid, set(kinds), {"list"}, ts, mono, expect, mono)) + + def latest_list(self, sport: str) -> List[int]: + with self._lock: + return list(self._latest_list.get(sport, [])) + + def read_since(self, sport: str, eid: int, since: float) -> bool: + with self._lock: + last = self.last_read.get((sport, eid)) + return last is not None and last >= since + + def near_end(self, sport: str, eid: int) -> bool: + """Servisin yoklama kaynağının işareti (indirgeyicinin `near_end`'i); okunamazsa False.""" + try: + slot = self.service._slots.get(sport) + return bool(slot is not None and (slot.tracker.state.get(str(eid)) or {}).get("near_end")) + except Exception: + return False + + def on_event_read(self, p: Pending, out: Outcome, event: Optional[Mapping[str, Any]], *, ok: bool, final: bool, + latency: Optional[float], retry_in: Optional[float], ts: float, mono: float) -> None: + cls = classify_status(dict(event)).value if event else None + self._count("recorder", "event", out) + self._event_seen(p.sport, p.eid, event) + with self._lock: + st = self.stats.setdefault(p.sport, SportStats()) + if ok and latency is not None: + st.reads_ok += 1 + for kind in p.kinds: + st.read_latency[kind].append(latency) + elif final: + st.not_reflected += 1 + self.recorder.write("reads", { + "kind": "event", "by": "recorder", "sport": p.sport, "event_id": p.eid, "status": out.status, + "latency_s": round(out.seconds, 3), "waited_s": round(out.waited, 3), "attempt": p.attempts, + "trigger": {"kinds": sorted(p.kinds), "sources": sorted(p.sources), "ts": round(p.first_ts, 3)}, + "read_class": cls, "reflects": ok, "trigger_to_read_s": round(latency, 3) if latency is not None else None, + "retry_in_s": retry_in, "final": final, + }, ts=ts, mono=mono) + + def on_event_skipped(self, p: Pending) -> None: + self._note_budget() + with self._lock: + self.stats.setdefault(p.sport, SportStats()).skipped_budget += 1 + + def fallback_read(self, sport: str, eid: int, *, window_start_ts: float) -> Outcome: + out = self.client.request(f"/event/{eid}") + if out.refused: + self._note_budget() + return out + event = (out.data or {}).get("event") if out.data is not None else None + self._count("fallback", "event", out) + self._event_seen(sport, eid, event) + self.recorder.write("reads", { + "kind": "event", "by": "fallback", "sport": sport, "event_id": eid, "status": out.status, + "latency_s": round(out.seconds, 3), "waited_s": round(out.waited, 3), + "read_class": classify_status(dict(event)).value if event else None, + "minute": round(window_start_ts, 3)}) + return out + + def on_fallback_minute(self, sport: str, start_ts: float, *, live: int, made: int, covered: int, priority: int, + cap: int, cut: Optional[str], partial: bool) -> None: + skipped = max(0, live - covered) + row = {"sport": sport, "minute": round(start_ts, 3), "live": live, "reads_made": made, "covered": covered, + "skipped": skipped, "coverage": round(covered / live, 4) if live else None, "priority_live": priority, + "cap": cap, "cut": cut, "partial": partial} + with self._lock: + self.stats.setdefault(sport, SportStats()).fallback.append(row) + self.recorder.write("fallback", row) + + # --- kaynaklar --------------------------------------------------------------------------------------------- + + def _sample(self, token: Optional[Dict[str, Any]] = None) -> None: + sample = sample_resources() + with self._lock: + reading = token or self._reading + total = sample.get("rss_total_mb") or sample.get("rss_self_mb") or 0.0 + if reading is not None: + reading["peak_mb"] = round(max(reading["peak_mb"], total), 1) + reading["samples"] += 1 + for key, field_name in (("self", "rss_self_mb"), ("bridge", "rss_bridge_mb"), ("live", "rss_live_mb"), + ("total", "rss_total_mb")): + self.peak_rss_mb[key] = max(self.peak_rss_mb[key], sample.get(field_name) or 0.0) + self.last_resources = sample + open_now = any(st.open_since is not None for st in self.stats.values()) + self.recorder.write("resources", {**sample, "credential_read": reading is not None, "push_open": open_now}) + + def _sample_loop(self) -> None: + while not self._stop.is_set(): + self._sample() + with self._lock: + interval = self._reading_resource_seconds if self._reading is not None else self._resource_seconds + self._wake_sampler.wait(interval) + self._wake_sampler.clear() + + # --- özet ---------------------------------------------------------------------------------------------------- + + @staticmethod + def _windows(intervals: Sequence[Sequence[Any]], start: float, end: float) -> List[Dict[str, Any]]: + """Açık aralıkların tümleyeni: kesinti pencereleri ve nedenleri (pencereden önce kapanan aralığın nedeni).""" + ends = {round(float(iv[1]), 6): str(iv[2]) for iv in intervals} + out = [] + for s, e in complement([(float(iv[0]), float(iv[1])) for iv in intervals], start, end): + cause = "start" if abs(s - start) < 1e-6 else ends.get(round(s, 6), "unknown") + out.append({"start": utc(s), "end": utc(e), "seconds": round(e - s, 3), "cause": cause, + "until_run_end": e >= end - 1e-6}) + return out + + @staticmethod + def _fallback_summary(rows: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: + full = [r for r in rows if not r.get("partial")] + live = sum(r["live"] for r in full) + skipped = sum(r["skipped"] for r in full) + scored = [r for r in full if r["live"]] + worst = min(scored, key=lambda r: (r["coverage"], -r["skipped"]), default=None) + return {"minutes": len(full), "live_total": live, "covered_total": sum(r["covered"] for r in full), + "skipped_total": skipped, "reads_made": sum(r["reads_made"] for r in full), + "skip_rate": round(skipped / live, 4) if live else None, + "worst_minute": None if worst is None else { + "at": utc(worst["minute"]), "sport": worst["sport"], "live": worst["live"], + "covered": worst["covered"], "skipped": worst["skipped"], "coverage": worst["coverage"], + "cut": worst["cut"]}} + + def summary(self, *, final: bool) -> Dict[str, Any]: + with self._lock: + start = self.started_ts or self.wall() + end = self.ended_ts if (final and self.ended_ts) else self.wall() + duration = max(1e-9, end - start) + pongs, server_pings, client_pings = sorted(self.pongs), sorted(self.server_pings), sorted(self.ping_sent) + signs = sorted(pongs + server_pings) + all_open: List[List[Any]] = [] + sports: Dict[str, Any] = {} + fallback_rows: List[Mapping[str, Any]] = [] + for sport, st in sorted(self.stats.items()): + opens = [list(iv) for iv in st.open_intervals] + if st.open_since is not None: + opens.append([st.open_since, end, "open"]) + all_open.extend(opens) + open_iv = [(float(a), float(b)) for a, b, _ in opens] + push = leader_intervals(st.leader, start, end, self.push_name) + positive = step_positive(st.live_counts, start, end) + gaps = [] + for s, e in quiet_gaps(st.frames, positive, start, end)[:TOP_GAPS]: + length = max(1e-9, e - s) + before = [n for t, n in st.live_counts if t <= s][-1:] + during = [n for t, n in st.live_counts if s - 1e-6 <= t <= e] + gaps.append({ + "seconds": round(e - s, 3), "start": utc(s), "end": utc(e), + "live_count_max": max(before + during or [0]), + "leader_push_share": round(overlap(push, s, e) / length, 3), + "connection_open_share": round(overlap(open_iv, s, e) / length, 3), + "pongs": count_between(pongs, s, e), "server_pings": count_between(server_pings, s, e), + "client_pings": count_between(client_pings, s, e), + "longest_without_ping_or_pong_s": round(longest_without(signs, s, e), 3), + }) + fallback_rows.extend(st.fallback) + sports[sport] = { + "frames": len(st.frames), "status_frames": st.status_frames, "score_frames": st.score_frames, + "finish_frames": st.finish_frames, "distinct_events": len(st.events), + "triggers": dict(sorted(st.triggers.items())), + "event_reads_reflecting": st.reads_ok, "event_reads_not_reflected": st.not_reflected, + "event_reads_skipped_budget": st.skipped_budget, + "frame_lag_s": describe(st.frame_lag), + "finish_push_lag_s": describe(st.finish_lag), + "trigger_to_event_read_s": {k: describe(v) for k, v in sorted(st.read_latency.items())}, + "live_list_reads": len(st.live_counts), + "live_count_max": max((n for _, n in st.live_counts), default=0), + "fallback_time_share": round((duration - overlap(push, start, end)) / duration, 4), + "time_to_first_push_s": round(st.first_push_at - start, 3) if st.first_push_at else None, + "downtime_windows": self._windows(opens, start, end), + "longest_msg_gaps_while_live": gaps, + "fallback": self._fallback_summary(st.fallback), + } + conn_windows = self._windows(all_open, start, end) + reads = self.credential_reads + notes = [SCOPE_NOTE, + "Push lag = receive time - changes.changeTimestamp (SofaScore's change time); the local clock's " + "offset is part of it.", + "longest_msg_gaps_while_live: intervals with no MSG frame of that sport while its last live list " + "showed more than 0 matches. leader_push_share 1.0 means the arbiter judged push healthy for the " + f"whole gap (a PING or PONG within {SILENCE_SECONDS:.0f} s keeps it healthy without frames).", + "fallback: while polling leads, every live match's /event/{id} at least once a minute within " + f"{self.fallback.cap} reads a minute (rate x (1 - list share)); a match not read within its minute " + "is skipped and counted, never carried over.", + "RSS: bridge = the main bridge's browser (REST reads), live = the live profile's browser (page " + "source pages or the direct credential reader), other = other children (Playwright driver)."] + notes.insert(0, CLIENT_NOTE if self.source == "direct" else PAGE_NOTE) + return { + "run_id": self.run_id, "source": self.source, "final": final, "written_at": utc(self.wall()), + "started": utc(start), "ended": utc(end) if final else None, "duration_s": round(end - start, 3), + "sports": list(self.sports), "args": self.args_note, "notes": notes, + "sports_detail": sports, + "fallback": self._fallback_summary(fallback_rows), + "connection": { + "sessions": len(self.sessions), + "opens": self.connection.opens if self.connection is not None else None, + "credential_reads": reads, "credential_compares": self.compares, + "credential_read_peak_rss_total_mb": max((c["peak_rss_total_mb"] for c in reads), default=None), + "rejections": self.connection.rejections if self.connection is not None else None, + "retries": self.retries, + "unhealthy": self.connection.unhealthy if self.connection is not None else None, + "session_ends": dict(sorted(self.drops.items())), + "session_seconds": describe([s["lasted"] for s in self.sessions if s.get("outcome") != "stopped"]), + "handshake_s": describe(self.handshake_seconds), + "handshake_to_info_s": describe(self.handshake_to_info), + "pong_rtt_s": describe(self.rtt), + "pong_gap_s": describe([b - a for a, b in zip(pongs, pongs[1:], strict=False)]), + "server_ping_gap_s": describe([b - a for a, b in zip(server_pings, server_pings[1:], + strict=False)]), + "errors": dict(sorted(self.errors.items())), + "kills": [{"at": utc(k["ts"]), "cause": k["cause"], "done": k["done"]} for k in self.kill_log], + "downtime_windows": conn_windows, + "downtime_s": round(sum(w["seconds"] for w in conn_windows), 3), + }, + "leader_switches": [{"at": utc(s["ts"]), "sport": s.get("sport"), "from": s.get("from"), + "to": s.get("to"), "reason": s.get("reason")} for s in self.switches], + "blocked": [{"at": utc(b["ts"]), "type": b["type"]} for b in self.blocked], + "requests": {"used": self.client.used, "budget": self.client.budget, "refused": self.client.refused, + "by_who_kind_status": dict(sorted(self.requests.items())), + "latency_s": {k: describe(v) for k, v in sorted(self.request_latency.items())}, + "pending_event_reads": self.reads.pending(), "dropped_overflow": self.reads.overflow}, + "resources": {"peak_rss_mb": dict(self.peak_rss_mb), "last": self.last_resources}, + "service": {"error": self.service_error, **{k: v for k, v in self.service.report.to_dict().items() + if k in ("rounds", "requests", "push_frames", + "source_switches", "leaders")}}, + } + + def write_summary(self, *, final: bool) -> None: + self._last_summary = self.mono() + self.recorder.write_summary(self.summary(final=final)) + + +# --- komut satırı ---------------------------------------------------------------------------------------------- + + +def _sports(value: str) -> List[str]: + sports = [s.strip().lower() for s in value.split(",") if s.strip()] + if not sports or not all(re.fullmatch(r"[a-z0-9-]+", s) for s in sports): + raise argparse.ArgumentTypeError("a comma-separated list of sport slugs") + return list(dict.fromkeys(sports)) + + +def _ids(value: str) -> List[int]: + try: + return [int(v) for v in value.split(",") if v.strip()] + except ValueError: + raise argparse.ArgumentTypeError("a comma-separated list of event ids") + + +def _share(value: str) -> float: + share = float(value) + if not 0.0 <= share < 1.0: + raise argparse.ArgumentTypeError("a share from 0 up to (not including) 1") + return share + + +def _parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="soak_live.py", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--sports", type=_sports, default=list(DEFAULT_SPORTS), + help="comma-separated sports (default football,basketball,tennis)") + p.add_argument("--source", choices=SOURCES, default="direct", help="push source to measure (default direct)") + p.add_argument("--hours", type=float, default=DEFAULT_HOURS, help=f"run length (default {DEFAULT_HOURS:g})") + p.add_argument("--run-id", default=None, help="id of the run (default: UTC timestamp); a used id is refused") + p.add_argument("--max-requests", type=int, default=None, + help="budget of REST reads (lists and event pages) for this run (default --hours x 3600 x --rate)") + p.add_argument("--rate", type=float, default=DEFAULT_RATE, help="REST reads per second (default 1)") + p.add_argument("--list-share", type=_share, default=DEFAULT_LIST_SHARE, + help="share of the rate kept for discovery and list reads while polling leads (default 0.5)") + p.add_argument("--priority-ids", type=_ids, default=[], metavar="IDS", + help="event ids read first by the per-minute fallback (default none)") + p.add_argument("--kill-every", type=float, default=None, metavar="MINUTES", + help="force-close the push connection (direct) or the pages (page) every MINUTES") + p.add_argument("--kill-at", type=float, default=None, metavar="MINUTES", + help="force-close it once, MINUTES after the start") + p.add_argument("--compare-credential-sports", action="store_true", + help="direct: also read the credential from the other sports' pages; records same/different only") + p.add_argument("--poll-interval", type=float, default=30.0, help="list interval while polling leads (default 30)") + p.add_argument("--out", default=OUT, help="output root (default research/soak); the run writes //") + p.add_argument("--data-dir", default=None, + help=f"the live service's store (default {STORE_ROOT}/); never inside --out") + return p + + +def main(argv: Optional[List[str]] = None) -> int: + args = _parser().parse_args(argv) + if args.hours <= 0 or args.rate <= 0: + print("error: --hours and --rate must be positive", file=sys.stderr) + return 2 + run_id = args.run_id or dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + if not re.fullmatch(r"[A-Za-z0-9._-]+", run_id): + print("error: --run-id may contain letters, digits, '.', '_' and '-' only", file=sys.stderr) + return 2 + out_dir = os.path.join(args.out, run_id) + if os.path.isdir(out_dir) and os.listdir(out_dir): + print(f"error: run {run_id!r} already has output in {out_dir}; choose another --run-id", file=sys.stderr) + return 2 + data_dir = args.data_dir or os.path.join(STORE_ROOT, run_id) + if os.path.abspath(data_dir).startswith(os.path.abspath(out_dir) + os.sep): + print("error: --data-dir must not be inside the run's output folder", file=sys.stderr) + return 2 + if args.compare_credential_sports and args.source != "direct": + print("error: --compare-credential-sports needs --source direct", file=sys.stderr) + return 2 + budget = args.max_requests if args.max_requests is not None else default_budget(args.hours, args.rate) + os.environ.setdefault("SOFASCORE_CLIENT__THROTTLE_DIR", THROTTLE_DIR) + os.environ["SOFASCORE_STORAGE__DATA_DIR"] = data_dir + os.makedirs(data_dir, exist_ok=True) + note = {"source": args.source, "hours": args.hours, "max_requests": budget, "rate": args.rate, + "list_share": args.list_share, "priority_ids": len(args.priority_ids), + "kill_every_min": args.kill_every, "kill_at_min": args.kill_at, + "compare_credential_sports": args.compare_credential_sports, "poll_interval_s": args.poll_interval} + soak = Soak(out_dir, args.sports, run_id=run_id, data_dir=data_dir, budget=budget, source=args.source, + hours=args.hours, kill_every=args.kill_every, kill_at=args.kill_at, poll_interval=args.poll_interval, + rate=args.rate, list_share=args.list_share, priority_ids=args.priority_ids, + compare_sports=args.compare_credential_sports, args_note=note) + + def on_signal(signum: int, frame: Any) -> None: + soak.request_stop() + + signal.signal(signal.SIGINT, on_signal) + signal.signal(signal.SIGTERM, on_signal) + print(f"soak {run_id}: source {args.source}, sports {', '.join(args.sports)}, {args.hours:g} h, budget {budget} " + f"REST reads at {args.rate:g}/s; output {out_dir}", flush=True) + soak.run(args.hours) + print(f"soak {run_id}: done; summary {os.path.join(out_dir, 'summary.json')}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 2d2cf98d3fe36ea54850f5a2162ff431c31246b5 Mon Sep 17 00:00:00 2001 From: tunjayoff Date: Tue, 13 Oct 2026 16:11:54 +0300 Subject: [PATCH 2/4] test(research): cover the soak recorder offline A fake NATS server, a fake page and a fake API drive the recorder: frames and event-read latency, kill and switch back for both sources, the per-minute fallback share and ranking, the budget, and a privacy scan of a recorded run. The script's research profile joins the settings check. --- tests/test_research_settings.py | 1 + tests/test_soak_live.py | 635 ++++++++++++++++++++++++++++++++ 2 files changed, 636 insertions(+) create mode 100644 tests/test_soak_live.py diff --git a/tests/test_research_settings.py b/tests/test_research_settings.py index 702b65c9..24c028c0 100644 --- a/tests/test_research_settings.py +++ b/tests/test_research_settings.py @@ -54,6 +54,7 @@ "push_channel_run": "chrome_push_run", "push_light_probe": "chrome_push_light", "push_direct_probe": "chrome_push_direct", + "soak_live": "chrome_soak", } diff --git a/tests/test_soak_live.py b/tests/test_soak_live.py new file mode 100644 index 00000000..4d36ebc3 --- /dev/null +++ b/tests/test_soak_live.py @@ -0,0 +1,635 @@ +""" +S2: canlı push kaynağının soak kaydedicisi (scripts/soak_live.py). + +Ağ yok, SofaScore yok, tarayıcı yok. `direct` için push sunucusu tests/test_live_direct_source.py'deki süreç içi +sahte NATS sunucusudur (yalnızca 127.0.0.1; kayıtlı INFO ve kareler); `page` için sayfa tests/test_live_push_source.py'deki +sahte Playwright sayfasıdır. Maç sayfaları ve canlı liste sahte bir istek işlevidir (tests/test_live_service.py +FakeApi). Kimlik bilgisi her testte açıkça sahte parçalardan kurulur. Kaydedici uygulamanın gerçek parçalarını +çalıştırır: LiveService, DirectConnection ya da PageSource, hakem, yoklama kaynağı. + + * kareler kaydedilir: yollar, yalnızca durum ve skor değerleri, changeTimestamp; tetiklenen maç sayfası okuması + ve "kare → /event/{id}" gecikmesi (CDN'in eski yanıtı yinelenir) + * öldürme: hakem yoklamaya geçer, bağlantı (ya da sayfa) yeniden açılır, hakem push'a döner; kesinti penceresi, + yedek süresi, el sıkışma → INFO süresi + * yoklama öndeyken dakikalık okumalar: öncelik, en uzun süredir okunmayan, dakikanın payı, atlananlar + * bütçe bitince REST okuması yapılmaz + * gizlilik: kayıtta IPv4/IPv6 yok, user/pass/auth_token değeri yok, kimlik bilgisi, sunucu adresi, INFO gövdesi + yok; kimlik bilgisi karşılaştırması yalnızca same/different; tests/test_research_privacy.py denetimi temiz +""" +from __future__ import annotations + +import json +import os +import sys +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, Iterator, List + +import pytest + +import conftest +import test_research_privacy as privacy +from sofascore_scraper.services.live import push_source as ps +from sofascore_scraper.services.live import supervisor as sv +from sofascore_scraper.services.live.push_source import PushFeed, attach +from test_live_direct_source import FakeNatsServer, FakeReader, fake_credential, fake_value, server_frame +from test_live_push_source import FakePage, finish_frame, wait_for, wire +from test_live_service import FB_DONE, FB_LIVE, FakeApi, fx + +pytest.importorskip("fcntl") # araştırma script'lerinin ortak kilidi (Linux/macOS araçları) + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPTS = os.path.join(ROOT, "scripts") + + +def _import_soak() -> Any: + """Script içe aktarılırken ortam değişkeni koyar (profil, log seviyesi): test sürecinin ortamı değişmesin.""" + saved = dict(os.environ) + sys.path.insert(0, SCRIPTS) + try: + import soak_live + finally: + sys.path.remove(SCRIPTS) + os.environ.clear() + os.environ.update(saved) + return soak_live + + +soak_live = _import_soak() + +# Sayfanın INFO karesi istemcinin adresini taşır (belgeleme aralığından sahte adres): hiçbir yere yazılmamalı +PAGE_INFO = "INFO " + json.dumps({"version": "2.12.15", "auth_required": True, "client_ip": "198.51.100.9", + "connect_urls": ["203.0.113.4:9222"]}) + "\r\n" + + +# --- yardımcılar ------------------------------------------------------------------------------------------ + + +def rows(run: Path, name: str) -> List[Dict[str, Any]]: + path = run / f"{name}.jsonl" + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def events(run: Path, name: str, event: str) -> List[Dict[str, Any]]: + return [r for r in rows(run, name) if r.get("event") == event] + + +def reads_by(run: Path, who: str) -> List[Dict[str, Any]]: + return [r for r in rows(run, "reads") if r.get("by") == who] + + +def summary(run: Path) -> Dict[str, Any]: + return json.loads((run / "summary.json").read_text(encoding="utf-8")) + + +class SlowReader(FakeReader): + """Kimlik bilgisi okuması biraz sürer: okuma sırasında kaynak örneklemesi sıklaşmalı.""" + + def read(self, sport: str) -> Any: + time.sleep(0.3) + return super().read(sport) + + +class KillableOpener: + """Sayfa açıcının sahtesi: gerçek dinleyiciler (attach) sahte sayfaya bağlanır; tutamacın kapanışı sayfayı kapatır.""" + + def __init__(self) -> None: + self.pages: Dict[str, FakePage] = {} + self.opened: List[str] = [] + + def open(self, sport: str, feed: PushFeed) -> Any: + self.opened.append(sport) + page = FakePage() + attach(page, feed) + self.pages[sport] = page + state = {"closed": False} + + def close() -> None: + if not state["closed"]: + state["closed"] = True + page.emit("close") + + return SimpleNamespace(ready=True, failed=None, close=close) + + def close(self) -> None: + pass + + +@pytest.fixture +def data_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + path = tmp_path / "data" + monkeypatch.setenv("SOFASCORE_STORAGE__DATA_DIR", str(path)) + monkeypatch.setenv("SOFASCORE_REFRESH__WINDOW_HOURS", "48") + return path + + +@pytest.fixture +def fast_service(monkeypatch: pytest.MonkeyPatch) -> None: + """Servis karelere 50 ms'de bir bakar, istekleri arasında 1 sn beklemez, kapanan sayfayı 0,3 sn sonra açar.""" + monkeypatch.setattr(sv, "PUSH_DRAIN_SECONDS", 0.05) + monkeypatch.setattr(sv, "MIN_REQUEST_SPACING_SECONDS", 0.01) + monkeypatch.setattr(ps, "OPEN_RETRY_FIRST_SECONDS", 0.3) + + +@pytest.fixture +def servers() -> Iterator[Any]: + made: List[FakeNatsServer] = [] + + def make(**options: Any) -> FakeNatsServer: + made.append(FakeNatsServer(**options)) + return made[-1] + + yield make + for s in made: + s.close() + + +@pytest.fixture +def soaks() -> Iterator[Any]: + made: List[Any] = [] + + def make(tmp_path: Path, data_dir: Path, srv: Any, api: FakeApi, *, sports=("football",), + budget: int = 100, reader: Any = None, **options: Any) -> Any: + run = tmp_path / "research" / "soak" / "t-run" + conn = dict(blocked=lambda: None, recv_poll=0.02, reconnect_first=0.5, reconnect_max=1.0, + read_retry_first=0.05, read_retry_max=0.1, stable_after=0.0) + conn.update(options.pop("connection_options", {})) + if options.get("source") != "page": + options["reader"] = reader or FakeReader(fake_credential(srv.url, "soak")) + options["connection_options"] = conn + options.setdefault("fallback_window", 60.0) + soak = soak_live.Soak( + str(run), list(sports), run_id="t-run", data_dir=str(data_dir), budget=budget, base_fetch=api, + gate=lambda: None, resource_seconds=0.2, reading_resource_seconds=0.05, event_retry_seconds=0.2, + **options) + made.append(soak) + return soak, run + + yield make + for soak in made: + if soak.running: + soak.stop() + + +def live_and_done(eid: int = 500) -> Any: + live = fx(FB_LIVE, eid, tournament_id=17) + done = fx(FB_DONE, eid, tournament_id=17) + done["startTimestamp"] = live["startTimestamp"] + return live, done + + +def first_half(eid: int) -> Dict[str, Any]: + """İlk yarıda bir maç: bitişe yakın değil, servisin yoklama kaynağı sayfasını okumaz.""" + ev = fx(FB_LIVE, eid, tournament_id=17) + ev["status"] = {"code": 6, "type": "inprogress", "description": "1st half"} + return ev + + +def switches(soak: Any, sport: str = "football") -> List[str]: + return [f"{s['from']}>{s['to']}" for s in soak.switches if s.get("sport") == sport] + + +# --- kareler ve maç sayfası okuması --------------------------------------------------------------------------- + + +def test_frames_are_recorded_and_a_finish_triggers_one_event_read_with_its_latency( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, done = live_and_done() + api = FakeApi({"football": [live]}, {500: live}) # maç sayfası (CDN) önce eski hali verir + soak, run = soaks(tmp_path, data_dir, srv, api, reader=SlowReader(fake_credential(srv.url, "soak"))) + soak.start() + wait_for(lambda: switches(soak) == ["poll>direct"] and api.calls.count("/sport/football/events/live") >= 1) + frame = finish_frame(500, live["startTimestamp"]) + srv.msg("sport.football", frame) + wait_for(lambda: len(reads_by(run, "recorder")) >= 1) + done["homeScore"]["current"], done["awayScore"]["current"] = frame["homeScore.current"], frame["awayScore.current"] + api.events[500] = done # sayfa bitişi ve son skoru artık gösteriyor: yineleme onu okur + wait_for(lambda: soak.stats["football"].reads_ok == 1) + soak.stop() + + [row] = rows(run, "frames") + assert row["sport"] == "football" and row["event_id"] == 500 and row["subject"] == "sport" + assert row["paths"] == sorted(k for k in frame if k != "id") + assert "winnerCode" in row["paths"] and "status.description" in row["paths"] + # değer yalnızca durum kodu/türü ve skor alanları için + assert set(row["values"]) == {k for k in frame if k in ("status.code", "status.type") + or k.startswith(("homeScore.", "awayScore."))} + assert row["values"]["status.type"] == "finished" and row["values"]["homeScore.current"] == frame["homeScore.current"] + assert row["change_ts"] == frame["changes.changeTimestamp"] and row["triggers"] == ["finish", "score"] + assert row["lag_s"] == pytest.approx(row["ts"] - frame["changes.changeTimestamp"], abs=0.01) + + reads = reads_by(run, "recorder") + assert [(r["attempt"], r["reflects"], r["final"]) for r in reads] == [(1, False, False), (2, True, True)] + assert reads[0]["retry_in_s"] == pytest.approx(0.2) and reads[0]["read_class"] == "live" + assert reads[1]["trigger"]["kinds"] == ["finish", "score"] and reads[1]["trigger"]["sources"] == ["frame"] + assert reads[1]["trigger_to_read_s"] >= 0.2 and reads[1]["status"] == 200 + assert reads[1]["trigger"]["ts"] == pytest.approx(row["ts"], abs=0.01) + lists = [r for r in rows(run, "reads") if r["kind"] == "list"] + assert lists and all(r["by"] == "service" and r["sport"] == "football" and r["live_count"] == 1 for r in lists) + + s = summary(run) + fb = s["sports_detail"]["football"] + assert s["final"] is True and s["source"] == "direct" and fb["frames"] == 1 and fb["finish_frames"] == 1 + assert fb["finish_push_lag_s"]["n"] == 1 and fb["trigger_to_event_read_s"]["finish"]["n"] == 1 + assert fb["event_reads_reflecting"] == 1 and fb["triggers"] == {"finish": 1, "score": 1} + assert s["requests"]["used"] == len(api.calls) and s["requests"]["by_who_kind_status"]["recorder:event:200"] == 2 + assert any("aiohttp" in n and "standard-library" in n for n in s["notes"]) + # kimlik bilgisi okunurken kaynaklar sık örneklenir; okumanın zirvesi özette + [read] = s["connection"]["credential_reads"] + assert read["sport"] == "football" and read["ok"] and read["samples"] >= 2 and read["peak_rss_total_mb"] > 0 + assert read["connect_vs_previous"] is None # ilk okuma: karşılaştırılacak önceki yok + assert s["connection"]["credential_read_peak_rss_total_mb"] == read["peak_rss_total_mb"] + resources = rows(run, "resources") + assert any(r["credential_read"] for r in resources) + assert {"rss_self_mb", "rss_bridge_mb", "rss_live_mb", "rss_other_mb", "rss_total_mb"} <= set(resources[0]) + assert set(s["resources"]["peak_rss_mb"]) == {"self", "bridge", "live", "total"} + + +def test_the_longest_msg_gap_while_the_list_shows_live_matches_is_reported( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, _ = live_and_done() + api = FakeApi({"football": [live]}, {500: live}) + soak, run = soaks(tmp_path, data_dir, srv, api) + soak.start() + wait_for(lambda: switches(soak) == ["poll>direct"]) + time.sleep(0.3) + soak.stop() + [gap, *_] = summary(run)["sports_detail"]["football"]["longest_msg_gaps_while_live"] + assert gap["seconds"] > 0.3 and gap["live_count_max"] == 1 + assert 0 < gap["leader_push_share"] < 1 and gap["pongs"] >= 1 # açılıştan önce yoklama önde + assert {"connection_open_share", "server_pings", "client_pings", "longest_without_ping_or_pong_s"} <= set(gap) + + +# --- öldürme: yoklamaya geçiş ve geri dönüş ---------------------------------------------------------------------- + + +def test_a_kill_switches_to_poll_reconnects_and_switches_back( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, _ = live_and_done() + api = FakeApi({"football": [live]}, {500: live}) + soak, run = soaks(tmp_path, data_dir, srv, api) + soak.start() + wait_for(lambda: switches(soak) == ["poll>direct"]) + lists_before = api.calls.count("/sport/football/events/live") + assert soak.kill("kill-every") + wait_for(lambda: switches(soak) == ["poll>direct", "direct>poll", "poll>direct"]) + assert srv.connections == 2 and srv.subs == {"sport.football": 1} + soak.stop() + + leader = [r for r in rows(run, "leader") if r["event"] == "switch"] + assert [(r["from"], r["to"], r["reason"], r["fallback"]) for r in leader] == [ + ("poll", "direct", "push_connected", "stop"), ("direct", "poll", "push_disconnected", "start"), + ("poll", "direct", "push_connected", "stop")] + assert api.calls.count("/sport/football/events/live") > lists_before # yoklama turu yedekte + [kill] = events(run, "connection", "kill") + assert kill["cause"] == "kill-every" and kill["done"] is True and kill["session"] == 1 + ends = events(run, "connection", "session_end") + assert ends[0]["killed"] == "kill-every" and ends[0]["outcome"] == "dropped" + subs = events(run, "connection", "subscribe") + assert [(s["session"], s["subject"], s["resubscribe"]) for s in subs] == [ + (1, "sport.football", False), (2, "sport.football", True)] + infos = events(run, "connection", "info") + assert [i["session"] for i in infos] == [1, 2] and all(i["handshake_to_info_s"] >= 0 for i in infos) + assert [h["status"] for h in events(run, "connection", "handshake")] == [101, 101] + assert len(events(run, "connection", "credential_read")) == 1 # kopmada kimlik bilgisi yeniden okunmaz + + s = summary(run) + windows = s["connection"]["downtime_windows"] + assert windows[0]["cause"] == "start" + killed = [w for w in windows if w["cause"] == "dropped: closed by the server (kill-every)"] + assert len(killed) == 1 and not killed[0]["until_run_end"] + assert s["sports_detail"]["football"]["downtime_windows"][1]["cause"] == killed[0]["cause"] + assert 0 < s["sports_detail"]["football"]["fallback_time_share"] < 1 + assert s["connection"]["handshake_to_info_s"]["n"] == 2 and s["connection"]["sessions"] >= 2 + assert [k["cause"] for k in s["connection"]["kills"]] == ["kill-every"] + assert [w["to"] for w in s["leader_switches"]] == ["direct", "poll", "direct"] + + +def test_the_page_source_is_recorded_and_a_kill_closes_its_page( + tmp_path: Path, data_dir: Path, fast_service: None, soaks: Any) -> None: + live, _ = live_and_done() + api = FakeApi({"football": [live]}, {500: live}) + opener = KillableOpener() + soak, run = soaks(tmp_path, data_dir, None, api, source="page", page_opener=opener) + soak.start() + wait_for(lambda: "football" in opener.pages and api.calls.count("/sport/football/events/live") >= 1) + page = opener.pages["football"] + page.connect(credential=fake_value("page"), info=PAGE_INFO) # sitenin kodu CONNECT gönderir: dinlenmez + wait_for(lambda: switches(soak) == ["poll>page"]) + page.ws.receive(("PING\r\n" + wire("sport.football", finish_frame(500, live["startTimestamp"])) + + "-ERR 'Slow Consumer'\r\n").encode()) + wait_for(lambda: len(rows(run, "frames")) == 1 and soak.errors) + assert soak.kill("kill-at") + wait_for(lambda: switches(soak) == ["poll>page", "page>poll"] and len(opener.opened) == 2) + opener.pages["football"].connect(info=PAGE_INFO) + wait_for(lambda: switches(soak) == ["poll>page", "page>poll", "poll>page"]) + soak.stop() + + [frame] = rows(run, "frames") + assert frame["event_id"] == 500 and frame["triggers"] + assert [(r["from"], r["to"], r["reason"]) for r in rows(run, "leader") if r["event"] == "switch"] == [ + ("poll", "page", "push_connected"), ("page", "poll", "push_unavailable"), ("poll", "page", "push_connected")] + [kill] = events(run, "connection", "kill") + assert kill["pages"] == ["football"] and kill["done"] is True + gone = events(run, "connection", "page_gone") + assert gone[0]["reason"] == "page closed (kill-at)" + assert [i["session"] for i in events(run, "connection", "info")] == [1, 2] + assert [e["text"] for e in events(run, "connection", "error")] == ["Slow Consumer"] + assert len(events(run, "connection", "server_ping")) == 1 + assert events(run, "connection", "page_open")[0]["sport"] == "football" + s = summary(run) + assert s["source"] == "page" and any("page" in n for n in s["notes"][:1]) + assert any(w["cause"] == "page closed (kill-at)" for w in s["sports_detail"]["football"]["downtime_windows"]) + assert s["connection"]["credential_reads"] == [] + assert_private(run, needles=(fake_value("page"), "198.51.100.9", "203.0.113.4", "client_ip", "connect_urls")) + + +def test_run_follows_the_kill_schedule_and_writes_the_summary( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, _ = live_and_done() + api = FakeApi({"football": [live]}, {500: live}) + soak, run = soaks(tmp_path, data_dir, srv, api, kill_at=0.01) # 0,6 sn sonra bir kez + soak.run(hours=2.0 / 3600, tick=0.05) + assert not soak.running + [kill] = events(run, "connection", "kill") + assert kill["cause"] == "kill-at" and kill["done"] is True + s = summary(run) + assert s["final"] and 1.9 < s["duration_s"] < 10 and switches(soak)[:2] == ["poll>direct", "direct>poll"] + + +def test_the_kill_schedule() -> None: + k = soak_live.KillSchedule(every_minutes=10, at_minutes=3) + assert [k.due(t) for t in (0, 179, 180, 181, 599, 600, 601, 1250, 1300)] == [ + None, None, "kill-at", None, None, "kill-every", None, "kill-every", None] + assert soak_live.KillSchedule().due(10 ** 6) is None + + +# --- yoklama öndeyken dakikalık okumalar ---------------------------------------------------------------------- + + +def test_while_polling_leads_every_live_match_is_read_each_minute_within_its_share( + tmp_path: Path, data_dir: Path, fast_service: None, soaks: Any) -> None: + matches = {eid: first_half(eid) for eid in (601, 602, 603)} + api = FakeApi({"football": list(matches.values())}, dict(matches)) + # push hiç bağlanmaz (kimlik bilgisi yok): yoklama hep önde. Dakika 1 sn, hız 4/sn, listeye pay 0,5 → dakikada 2 + soak, run = soaks(tmp_path, data_dir, None, api, reader=FakeReader(), fallback_window=1.0, rate=4.0, + list_share=0.5, priority_ids=[603], poll_interval=1.0) + soak.start() + wait_for(lambda: len([r for r in rows(run, "fallback") if r["live"]]) >= 3, seconds=10) + soak.stop() + + minutes = [r for r in rows(run, "fallback") if r["live"] and not r["partial"]] + assert all((r["live"], r["reads_made"], r["covered"], r["skipped"], r["cap"]) == (3, 2, 2, 1, 2) + for r in minutes) + assert all(r["coverage"] == pytest.approx(2 / 3, abs=1e-3) and r["cut"] == "share" for r in minutes) + fallback_reads = reads_by(run, "fallback") + by_minute: Dict[float, List[int]] = {} + for r in fallback_reads: + by_minute.setdefault(r["minute"], []).append(r["event_id"]) + full = [ids for m, ids in sorted(by_minute.items()) if any(x["minute"] == m for x in minutes)] + assert all(ids[0] == 603 for ids in full) # öncelikli id önce + # sonra en uzun süredir okunmayan: 601 ile 602 sırayla + assert [ids[1] for ids in full[:3]] in ([601, 602, 601], [602, 601, 602]) + assert reads_by(run, "recorder") == [] # liste değişmedi: tetiklenen okuma yok + + s = summary(run) + fb = s["sports_detail"]["football"]["fallback"] + assert fb["skip_rate"] == pytest.approx(1 / 3, abs=1e-3) and fb["skipped_total"] == len(minutes) + assert fb["worst_minute"]["coverage"] == pytest.approx(2 / 3, abs=1e-3) and fb["worst_minute"]["skipped"] == 1 + assert s["fallback"]["skip_rate"] == fb["skip_rate"] + assert s["sports_detail"]["football"]["fallback_time_share"] == 1.0 + + +def test_while_push_leads_there_are_no_minute_reads( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + matches = {eid: first_half(eid) for eid in (601, 602)} + api = FakeApi({"football": list(matches.values())}, dict(matches)) + soak, run = soaks(tmp_path, data_dir, srv, api, fallback_window=0.5) + soak.start() + wait_for(lambda: switches(soak) == ["poll>direct"]) + reads_then = len(reads_by(run, "fallback")) + time.sleep(1.2) + soak.stop() + later = [r for r in rows(run, "fallback") if r["minute"] > soak.switches[0]["ts"] + 0.5] + assert later == [] and len(reads_by(run, "fallback")) == reads_then + + +def test_the_fallback_share_and_ranking() -> None: + fb = soak_live.FallbackReads.__new__(soak_live.FallbackReads) + soak_live.FallbackReads.__init__(fb, SimpleNamespace(), rate=1.0, share=0.5, window=60.0, priority=[7]) + assert fb.cap == 30 and fb.priority == frozenset({7}) + soak = SimpleNamespace(last_read={("football", 1): 10.0, ("football", 2): 5.0}, + near_end=lambda sport, eid: eid == 3) + fb._soak = soak + ranked = sorted([("football", e) for e in (1, 2, 3, 4, 7)], key=lambda c: fb._rank(*c)) + assert [e for _, e in ranked] == [7, 3, 4, 2, 1] # öncelik, bitişe yakın, hiç okunmamış, en eski okuma + + +# --- bütçe ------------------------------------------------------------------------------------------------------ + + +def test_the_budget_stops_rest_reads(tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, + soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, done = live_and_done() + api = FakeApi({"football": [live]}, {500: done}) + soak, run = soaks(tmp_path, data_dir, srv, api, budget=1) + soak.start() + wait_for(lambda: switches(soak) == ["poll>direct"]) + srv.msg("sport.football", finish_frame(500, live["startTimestamp"])) + wait_for(lambda: soak.stats["football"].skipped_budget == 1) + soak.stop() + assert api.calls == ["/sport/football/events/live"] # bütçe tek okumaydı: sonrası ağa çıkmadı + [note] = [r for r in rows(run, "reads") if r["kind"] == "budget_exhausted"] # bir kez yazılır + assert (note["used"], note["budget"]) == (1, 1) + s = summary(run) + assert s["requests"]["used"] == 1 and s["requests"]["refused"] >= 2 + assert s["sports_detail"]["football"]["event_reads_skipped_budget"] == 1 + assert len(rows(run, "frames")) == 1 # push kaydı sürer + + +def test_the_default_budget_stays_below_one_request_per_second() -> None: + for hours in (0.5, 1, 6, 24): + for rate in (0.5, 1.0, 3.0): + assert soak_live.default_budget(hours, rate) <= hours * 3600 + + +def test_the_pacer_spaces_reads() -> None: + now = [0.0] + slept: List[float] = [] + + def sleep(s: float) -> None: + slept.append(s) + + pacer = soak_live.Pacer(2.0, mono=lambda: now[0], sleep=sleep) + for _ in range(3): + pacer.wait() + assert slept == [0.5, 1.0] + + +# --- gizlilik --------------------------------------------------------------------------------------------------- + + +SECRET_KEYS = {"user", "pass", "auth_token", "jwt", "nkey", "sig", "token"} + + +def secret_keys(o: Any) -> List[str]: + if isinstance(o, dict): + found = [k for k in o if k in SECRET_KEYS] + return found + [k for v in o.values() for k in secret_keys(v)] + if isinstance(o, list): + return [k for v in o for k in secret_keys(v)] + return [] + + +def assert_private(run: Path, needles: Any = ()) -> None: + texts = {p.name: p.read_text(encoding="utf-8") for p in sorted(run.iterdir()) if p.is_file()} + assert set(texts) == {"frames.jsonl", "connection.jsonl", "leader.jsonl", "reads.jsonl", "fallback.jsonl", + "resources.jsonl", "summary.json"} + for name, text in texts.items(): + # IPv4/IPv6 yok: yerel ve belgeleme aralıkları da (izin verilen aralık olmadan) + assert privacy.ip_literals(text, allowed=()) == [], name + for needle in (*needles, "ws://", "wss://", "CONNECT {", "auth_required", "2.12.15", "***"): + assert needle not in text, (name, needle) + docs = [json.loads(text)] if name.endswith(".json") else [json.loads(x) for x in text.splitlines() if x] + for doc in docs: + assert secret_keys(doc) == [], (name, doc) + # araştırma ağacının gizlilik denetimi bu koşunun klasöründe de temiz + assert privacy.scan(str(run.parent.parent)) == {"ipv4": [], "ipv6": [], "geo": []} + + +def test_a_recorded_run_holds_no_credential_address_or_info_body( + tmp_path: Path, data_dir: Path, fast_service: None, servers: Any, soaks: Any) -> None: + srv = servers(accept=fake_value("soak")) + live, done = live_and_done() + api = FakeApi({"football": [live]}, {500: done}) + compared: List[FakeReader] = [] + + def compare_reader() -> FakeReader: + compared.append(FakeReader(fake_credential(srv.url, "other"))) + return compared[-1] + + soak, run = soaks(tmp_path, data_dir, srv, api, sports=("football", "tennis"), compare_sports=True, + compare_factory=compare_reader) + soak.start() + wait_for(lambda: switches(soak, "football") == ["poll>direct"] and len(soak.compares) == 1) + frame = finish_frame(500, live["startTimestamp"]) + done["homeScore"]["current"], done["awayScore"]["current"] = frame["homeScore.current"], frame["awayScore.current"] + srv.msg("sport.football", frame) + srv.send(server_frame(0x2, b"-ERR 'Slow Consumer'\r\n")) + srv.send(server_frame(0x2, b"-ERR 'not a known text from 198.51.100.7 and 2001:db8::7'\r\n")) + wait_for(lambda: soak.stats["football"].reads_ok == 1 and sum(soak.errors.values()) == 2) + assert soak.kill("kill-at") + wait_for(lambda: switches(soak, "football")[-1:] == ["poll>direct"] and srv.connections == 2) + soak.stop() + + # kimlik bilgisi tenis sayfasından da okundu; yalnızca same/different yazılır + [cmp] = events(run, "connection", "credential_compare") + assert (cmp["read_from"], cmp["compared_with"], cmp["connect_options"], cmp["address"]) == ( + "football", "tennis", "different", "same") + assert compared and compared[0].reads == ["tennis"] + assert sorted(e["text"] for e in events(run, "connection", "error")) == ["Slow Consumer", "other"] + connect = events(run, "connection", "connect_sent") + assert len(connect) == 2 and all(set(c) == {"ts", "mono", "event", "session"} for c in connect) + port = srv.url.rsplit(":", 1)[1].strip("/") + assert_private(run, needles=(fake_value("soak"), fake_value("soakp"), fake_value("other"), fake_value("otherp"), + srv.url, "127.0.0.1", f":{port}", "198.51.100.7")) + + +# --- değişiklik izleyici ----------------------------------------------------------------------------------------- + + +def _ev(eid: int, code: int, stype: str, home: int, away: int, ts: int) -> Dict[str, Any]: + return {"id": eid, "status": {"code": code, "type": stype}, "homeScore": {"current": home}, + "awayScore": {"current": away}, "changes": {"changeTimestamp": ts}} + + +def test_the_change_tracker_finds_the_changes_that_need_an_event_read() -> None: + t = soak_live.ChangeTracker() + assert t.listing("football", [_ev(1, 6, "inprogress", 0, 0, 100), _ev(2, 7, "inprogress", 1, 1, 100)]) == [] + kinds = lambda changes: {eid: sorted(k) for eid, k, _ in changes} # noqa: E731 + # skor değişimi, yeni canlı maç, listeden düşme + changes = t.listing("football", [_ev(1, 6, "inprogress", 1, 0, 110), _ev(3, 6, "inprogress", 0, 0, 110)]) + assert kinds(changes) == {1: ["score"], 3: ["new_live"], 2: ["dropped"]} + # CDN'in eski listesi yok sayılır + assert t.listing("football", [_ev(1, 6, "inprogress", 0, 0, 105), _ev(3, 6, "inprogress", 0, 0, 110)]) == [] + # kare: bitiş (skorla birlikte tek kare), devre arası (durum), yalnızca kodlu kare + k, expect, cls = t.frame("football", {"id": 1, "status.code": 100, "status.type": "finished", + "homeScore.current": 2, "changes.changeTimestamp": 120}) + assert k == {"finish", "score"} and cls == "completed" and expect == {"class": "completed", "home": 2, "away": 0} + assert t.frame("football", {"id": 3, "status.code": 31, "status.type": "inprogress"})[0] == set() + assert t.frame("football", {"id": 3, "cardsCode": "01"})[0] == set() + # bilinmeyen maçın başlama karesi: yeni canlı maç + assert t.frame("football", {"id": 9, "status.code": 6, "status.type": "inprogress"})[0] == {"new_live"} + assert t.frame("football", {"id": 9, "status.code": 70})[0] == {"status"} # tür düşer, kod iptal (void) + # okunan maç sayfası durumu tetikleyicisiz günceller: aynı skor karesi yeniden okuma istemez + t.observe_event("football", _ev(3, 6, "inprogress", 1, 0, 130)) + assert t.frame("football", {"id": 3, "homeScore.current": 1})[0] == set() + + +def test_reflects_checks_what_the_trigger_expects() -> None: + finished = _ev(1, 100, "finished", 2, 0, 1) + live = _ev(1, 7, "inprogress", 2, 0, 1) + assert soak_live.reflects({"finish"}, {"class": "completed"}, finished) + assert not soak_live.reflects({"finish"}, {"class": "completed"}, live) + assert soak_live.reflects({"score"}, {"class": "live", "home": 2, "away": 0}, live) + assert not soak_live.reflects({"score"}, {"class": "live", "home": 3, "away": 0}, live) + assert soak_live.reflects({"dropped"}, {}, finished) and not soak_live.reflects({"dropped"}, {}, live) + + +def test_quiet_gaps_count_only_time_with_live_matches() -> None: + positive = soak_live.step_positive([(0.0, 0), (10.0, 3), (50.0, 0), (60.0, 2)], 0.0, 100.0) + assert positive == [(10.0, 50.0), (60.0, 100.0)] + gaps = soak_live.quiet_gaps([20.0, 30.0, 80.0], positive, 0.0, 100.0) + assert gaps[:3] == [(30.0, 50.0), (60.0, 80.0), (80.0, 100.0)] + assert soak_live.complement([(5.0, 10.0), (20.0, 30.0)], 0.0, 40.0) == [(0.0, 5.0), (10.0, 20.0), (30.0, 40.0)] + assert soak_live.leader_intervals([(0.0, "poll"), (5.0, "direct"), (9.0, "poll")], 0.0, 10.0, "direct") == [ + (5.0, 9.0)] + + +def test_browser_processes_are_told_apart_by_their_profile() -> None: + assert soak_live.classify_browser("/home/x/.cache/sofascore_research/chrome_soak-live") == "live" + assert soak_live.classify_browser("/home/x/.cache/sofascore_research/chrome_soak") == "bridge" + assert soak_live.classify_browser(None) == "other" + sample = soak_live.sample_resources() + assert sample["rss_self_mb"] > 0 + if sample.get("procfs", True): + assert sample["rss_total_mb"] >= sample["rss_self_mb"] and set(sample["processes"]) == { + "bridge", "live", "other"} + + +# --- komut satırı ------------------------------------------------------------------------------------------------ + + +def test_a_used_run_id_is_refused_before_anything_starts(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + out = tmp_path / "soak" + (out / "r1").mkdir(parents=True) + (out / "r1" / "frames.jsonl").write_text("", encoding="utf-8") + assert soak_live.main(["--out", str(out), "--run-id", "r1", "--hours", "1"]) == 2 + assert "already has output" in capsys.readouterr().err + assert soak_live.main(["--out", str(out), "--run-id", "r 2"]) == 2 + assert soak_live.main(["--out", str(out), "--run-id", "r3", "--data-dir", str(out / "r3" / "data")]) == 2 + assert soak_live.main(["--out", str(out), "--run-id", "r4", "--source", "page", + "--compare-credential-sports"]) == 2 + assert not (out / "r3").exists() and not (out / "r4").exists() + + +def test_the_options_parse() -> None: + args = soak_live._parser().parse_args(["--sports", "football, tennis", "--kill-every", "40", "--kill-at", "5", + "--compare-credential-sports", "--priority-ids", "1,2", "--rate", "0.8", + "--list-share", "0.25"]) + assert args.sports == ["football", "tennis"] and args.kill_every == 40 and args.kill_at == 5 + assert args.compare_credential_sports and args.max_requests is None and args.hours == soak_live.DEFAULT_HOURS + assert args.priority_ids == [1, 2] and args.rate == 0.8 and args.list_share == 0.25 and args.source == "direct" + for bad in (["--sports", "foot ball"], ["--list-share", "1"], ["--source", "auto"], ["--priority-ids", "x"]): + with pytest.raises(SystemExit): + soak_live._parser().parse_args(bad) + + +assert conftest # sınır denetimi ve kancalar conftest'te kurulur From 6765be39aa285ef34f505c8611c878296052fcb4 Mon Sep 17 00:00:00 2001 From: tunjayoff Date: Tue, 13 Oct 2026 16:27:31 +0300 Subject: [PATCH 3/4] feat(research): read joined matches every minute and the rest as a backlog The owner's revised fallback rule for the c project: while the list leads, matches in a joined set are read every 60 s, other live matches as the rate allows, longest unread first; reads that do not fit are skipped and counted per sport. The joined set comes from --joined-ids and from --joined-random-soccer (20 random live football matches by default, replaced when they finish). The summary reports read-gap percentiles for joined and other matches, the list rate achieved and the skip rates. --- scripts/soak_live.py | 306 ++++++++++++++++++++++++++++++---------- tests/test_soak_live.py | 124 +++++++++++----- 2 files changed, 327 insertions(+), 103 deletions(-) diff --git a/scripts/soak_live.py b/scripts/soak_live.py index 7aaa3274..3ddf556d 100644 --- a/scripts/soak_live.py +++ b/scripts/soak_live.py @@ -7,7 +7,8 @@ script yalnızca onların kenarlarına kayıt kancaları takar. python scripts/soak_live.py --sports football,basketball,tennis --hours 6 [--source direct|page] [--run-id ID] - [--max-requests N] [--rate 1.0] [--list-share 0.5] [--priority-ids 1,2] + [--max-requests N] [--rate 1.0] [--list-share 0.5] [--joined-ids DOSYA] + [--joined-random-soccer 20] [--kill-every DAKİKA] [--kill-at DAKİKA] [--compare-credential-sports] [--out KLASÖR] **`direct` açık seçimdir** (docs/deploy/watch.md, "The `direct` source"): sitenin kendi istemci kimlik bilgisini @@ -25,12 +26,15 @@ değişiklikte (durum sınıfı, skor, yeni canlı maç, bitiş, listeden düşme) bir `/event/{id}` okuması, c'nin yapacağı gibi. Okuma değişikliği henüz göstermiyorsa (CDN) servisin bitiş onayı gibi CONFIRM_RETRY_SECONDS sonra, en çok CONFIRM_ATTEMPTS kez yinelenir. Gecikme = tetikleyen kare ya da liste değişikliği → değişikliği gösteren ilk okuma. - * Yoklama önde (push kopuk, sessiz ya da öldürüldü; sahibin c için kuralı): canlı liste 30 sn'de bir (hakemin - `poll_interval`'ı) ve yoklama kaynağının kendi okumaları, ARTI her canlı maçın sayfası dakikada en az bir kez - (FallbackReads): önce öncelikli id'ler (`--priority-ids`, varsayılan boş: "c fikstürüne bağlı" sinyali burada - yok), sonra bitişe yakınlar, sonra en uzun süredir okunmayanlar. Dakikanın payı `60 × --rate × (1 − - --list-share)` okumadır; sığmayan okuma atlanır ve sayılır (fallback_skipped), bir sonraki dakikaya taşınmaz. - Dakika başına fallback.jsonl: spor başına canlı maç, yapılan okuma, atlanan, kapsama (dakika içinde okunan payı). + * Liste önde (push kopuk, sessiz ya da öldürüldü; sahibin c için kuralı): canlı liste 30 sn'de bir (hakemin + `poll_interval`'ı) ve yoklama kaynağının kendi okumaları (bitişe yakın, düşen, takılı, gecikmiş) bugünkü gibi; + ARTI (FallbackReads) "joined" kümedeki canlı maçlar 60 sn'de bir (acil, neden "joined") ve diğer canlı maçlar hız + elverdikçe, en uzun süredir okunmayan önce (birikmiş iş, sıra "listed"). "c fikstürüne bağlı" sinyali burada yok: + küme `--joined-ids DOSYA` ile `--joined-random-soccer N`'in (varsayılan 20) rastgele seçtiği canlı futbol + maçlarıdır; biten seçimin yerine yenisi seçilir. Dakikanın payı `60 × --rate × (1 − --list-share)` okumadır; kalan + pay keşfe ayrılır. Sığmayan okuma atlanır ve spor başına sayılır; hiçbir şey sınırsız beklemez. fallback.jsonl: + dakika ve spor başına canlı maç, joined canlı maç, okumalar (joined/listed/diğer), atlananlar; joined küme + değişiklikleri. Özet: joined ve diğer maçların okuma aralıkları (medyan/p90/maks), keşif hızı, atlama oranları. * Kancalar: DirectConnection'ın alt sınıfı (oturum başı/sonu, yeniden deneme), bağlantı işlevinin sarmalayıcısı (el sıkışma, hat üstü işlemler; öldürme), sayfa açıcının sarmalayıcısı (`page`; öldürme = sayfayı kapatma), her sporun PushFeed'ini saran musluk (MSG kareleri; `page`'de PING/PONG/INFO/-ERR de), kimlik bilgisi okuyucusunun @@ -72,6 +76,7 @@ import json import math import os +import random import re import secrets import signal @@ -111,7 +116,8 @@ DEFAULT_HOURS = 6.0 DEFAULT_RATE = 1.0 # istek/sn (bütçenin hızı) DEFAULT_LIST_SHARE = 0.5 # hızın keşfe (liste ve yoklama kaynağının okumaları) ayrılan payı -FALLBACK_WINDOW_SECONDS = 60.0 # yoklama öndeyken her canlı maçın sayfası bu sürede en az bir kez +FALLBACK_WINDOW_SECONDS = 60.0 # liste öndeyken dakikalık okumaların penceresi (joined maçlar 60 sn'de bir) +DEFAULT_JOINED_RANDOM = 20 # --joined-random-soccer: rastgele seçilen canlı futbol maçı sayısı RESOURCE_SECONDS = 30.0 READING_RESOURCE_SECONDS = 5.0 # kimlik bilgisi okunurken (ikinci Chromium açık) örnekleme aralığı SUMMARY_SECONDS = 600.0 @@ -569,20 +575,66 @@ def _read(self, p: Pending) -> None: self.want(p) +class JoinedSet: + """ + "c fikstürüne bağlı" maçların kümesi (sahibin kuralı: bu maçlar liste öndeyken 60 sn'de bir okunur). Burada o + sinyal yok: küme `--joined-ids` dosyasındaki sabit id'ler ile `--joined-random-soccer N`'in seçtiği N rastgele + canlı futbol maçıdır. Seçilen maç canlı listeden düşünce (bitti) yerine yenisi seçilir; maç canlı oldukça küme + N'de kalır. + """ + + SPORT = "football" + + def __init__(self, fixed: Iterable[int] = (), random_n: int = 0, *, rng: Optional[random.Random] = None) -> None: + self.fixed: FrozenSet[int] = frozenset(int(x) for x in fixed) + self.random_n = max(0, int(random_n)) + self.picks: Set[int] = set() + self.ever: Set[int] = set(self.fixed) # bir an kümede olan her maç (okuma aralıkları buna göre ayrılır) + self._rng = rng or random.Random() + self._lock = threading.Lock() + + def ids(self) -> FrozenSet[int]: + with self._lock: + return self.fixed | frozenset(self.picks) + + def refresh(self, sport: str, live_ids: Iterable[int]) -> List[Tuple[str, int]]: + """Sporun canlı listesi geldi: biten seçim düşer, eksik kadar rastgele canlı futbol maçı seçilir.""" + if sport != self.SPORT or not self.random_n: + return [] + live = set(live_ids) + changes: List[Tuple[str, int]] = [] + with self._lock: + for eid in sorted(self.picks - live): + self.picks.discard(eid) + changes.append(("leave", eid)) + need = self.random_n - len(self.picks) + pool = sorted(live - self.picks - self.fixed) + if need > 0 and pool: + for eid in sorted(self._rng.sample(pool, min(need, len(pool)))): + self.picks.add(eid) + self.ever.add(eid) + changes.append(("join", eid)) + return changes + + class FallbackReads: """ - Yoklama öndeyken (sahibin c için kuralı) her canlı maçın sayfası dakikada en az bir kez: dakikanın başında o - sporların son canlı listesindeki maçlar sıralanır (öncelikli id'ler, bitişe yakınlar, en uzun süredir okunmayanlar) - ve dakikanın payı (`cap`) kadar okunur. Dakika içinde başka bir yoldan (değişiklik okuması, yoklama kaynağı) - okunmuş maç yeniden okunmaz, kapsanmış sayılır. Sığmayan okuma atlanır ve sayılır; bir sonraki dakikaya taşınmaz. + Liste öndeyken (push kopuk, sessiz ya da öldürüldü) sahibin c için kuralı, servisin yoklama kaynağının üstüne: + + * "joined" kümedeki canlı maçlar acildir: her biri 60 sn'de bir okunur (neden "joined"); + * diğer canlı maçlar birikmiş iştir: hız elverdikçe, en uzun süredir okunmayan önce (sıra "listed"); + * bitişe yakın, düşen, takılı ve gecikmiş maçların okumaları yoklama kaynağında bugünkü gibi kalır. + + Hız: dakikada `60 × rate × (1 − share)` okuma; kalan pay keşfe (canlı liste ve yoklama kaynağının okumaları) + ayrılır. Dakikanın başında o sporların son canlı listesindeki maçlar sıralanır; dakika içinde başka bir yoldan + okunmuş maç yeniden okunmaz. Sığmayan okuma atlanır ve spor başına sayılır; hiçbir şey sınırsız beklemez (sıra her + dakika listeden yeniden kurulur). """ - def __init__(self, soak: "Soak", *, rate: float, share: float, window: float = FALLBACK_WINDOW_SECONDS, - priority: Iterable[int] = ()) -> None: + def __init__(self, soak: "Soak", *, rate: float, share: float, window: float = FALLBACK_WINDOW_SECONDS) -> None: self._soak = soak self.window = float(window) self.cap = max(0, int(math.floor(self.window * rate * (1.0 - share) + 1e-9))) - self.priority: FrozenSet[int] = frozenset(int(p) for p in priority) def run(self, stop: threading.Event) -> None: from sofascore_scraper.client import request_context @@ -595,21 +647,29 @@ def run(self, stop: threading.Event) -> None: continue self._minute(sports, stop) - def _rank(self, sport: str, eid: int) -> Tuple[int, int, float]: - last = self._soak.last_read.get((sport, eid)) - return (0 if eid in self.priority else 1, 0 if self._soak.near_end(sport, eid) else 1, - -1.0 if last is None else last) + def queue(self, sports: Sequence[str], live: Mapping[str, Sequence[int]], + joined: FrozenSet[int]) -> List[Tuple[str, str, int]]: + """(neden, spor, maç): önce joined, sonra listed; ikisinde de en uzun süredir okunmayan (hiç okunmamış) önce.""" + last = self._soak.last_read + + def key(c: Tuple[str, int]) -> float: + value = last.get(c) + return -1.0 if value is None else value + + joined_c = sorted(((s, e) for s in sports for e in live[s] if e in joined), key=key) + listed_c = sorted(((s, e) for s in sports for e in live[s] if e not in joined), key=key) + return [("joined", s, e) for s, e in joined_c] + [("listed", s, e) for s, e in listed_c] def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: soak = self._soak start_ts, start = soak.wall(), soak.mono() end = start + self.window + joined = soak.joined.ids() live = {sport: soak.latest_list(sport) for sport in sports} - ranked = sorted(((s, e) for s in sports for e in live[s]), key=lambda c: self._rank(*c)) - made: Dict[str, int] = collections.defaultdict(int) + made: Dict[str, Dict[str, int]] = {s: {"joined": 0, "listed": 0} for s in sports} attempted = 0 cut = None - for sport, eid in ranked: + for reason, sport, eid in self.queue(sports, live, joined): if stop.is_set(): cut = "stopped" break @@ -623,9 +683,9 @@ def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: if soak.mono() >= end: cut = "minute" break - out = soak.fallback_read(sport, eid, window_start_ts=start_ts) + out = soak.fallback_read(sport, eid, reason=reason, window_start_ts=start_ts) attempted += 1 - made[sport] += 1 + made[sport][reason] += 1 if out.refused or out.blocked is not None: cut = "budget" if out.refused else "blocked" break @@ -636,11 +696,14 @@ def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: early = soak.mono() < end - 1e-3 for sport in sports: ids = live[sport] - covered = sum(1 for eid in ids if soak.read_since(sport, eid, start)) - # yarım dakika (durduruldu ya da push döndü) atlama oranına sayılmaz - soak.on_fallback_minute(sport, start_ts, live=len(ids), made=made[sport], covered=covered, - priority=sum(1 for eid in ids if eid in self.priority), cap=self.cap, cut=cut, - partial=early or sport not in led) + j = [e for e in ids if e in joined] + o = [e for e in ids if e not in joined] + soak.on_fallback_minute( + sport, start_ts, live=len(ids), joined_live=len(j), made=made[sport], + other=soak.reads_between(sport, start, soak.mono(), exclude="fallback"), + skipped_joined=sum(1 for e in j if not soak.read_since(sport, e, start)), + skipped_listed=sum(1 for e in o if not soak.read_since(sport, e, start)), + cap=self.cap, cut=cut, partial=early or sport not in led) # yarım dakika oranlara sayılmaz # --- push: musluk, soket, okuyucu, bağlantı, sayfa açıcı ------------------------------------------------------- @@ -1167,6 +1230,8 @@ class SportStats: open_intervals: List[List[Any]] = field(default_factory=list) # [başlangıç, bitiş, kapanış nedeni] open_since: Optional[float] = None fallback: List[Dict[str, Any]] = field(default_factory=list) # dakika satırları + list_reads: List[float] = field(default_factory=list) # servisin canlı liste okumaları (epoch) + read_events: List[Tuple[float, str]] = field(default_factory=list) # başarılı maç sayfası okuması (mono, kim) class Soak: @@ -1183,7 +1248,8 @@ def __init__(self, out_dir: str, sports: Sequence[str], *, run_id: str, data_dir budget: int, source: str = "direct", hours: Optional[float] = None, kill_every: Optional[float] = None, kill_at: Optional[float] = None, poll_interval: float = 30.0, rate: float = DEFAULT_RATE, list_share: float = DEFAULT_LIST_SHARE, - priority_ids: Iterable[int] = (), fallback_window: float = FALLBACK_WINDOW_SECONDS, + joined_ids: Iterable[int] = (), joined_random: int = DEFAULT_JOINED_RANDOM, + joined_rng: Optional[random.Random] = None, fallback_window: float = FALLBACK_WINDOW_SECONDS, base_fetch: Optional[Callable[[str], Optional[Dict[str, Any]]]] = None, gate: Optional[Callable[[], None]] = None, reader: Any = None, page_opener: Any = None, compare_sports: bool = False, compare_factory: Optional[Callable[[], Any]] = None, @@ -1212,10 +1278,13 @@ def __init__(self, out_dir: str, sports: Sequence[str], *, run_id: str, data_dir gate=gate if gate is not None else default_gate(rate), mono=self.mono) self.tracker = ChangeTracker() self.reads = EventReads(self, retry=event_retry_seconds) - self.fallback = FallbackReads(self, rate=rate, share=list_share, window=fallback_window, - priority=priority_ids) + self.rate = rate + self.list_share = list_share + self.joined = JoinedSet(joined_ids, joined_random, rng=joined_rng) + self.fallback = FallbackReads(self, rate=rate, share=list_share, window=fallback_window) self.stats: Dict[str, SportStats] = {s: SportStats() for s in self.sports} self.last_read: Dict[Tuple[str, int], float] = {} # son başarılı maç sayfası okuması (mono), her yoldan + self.read_log: Dict[Tuple[str, int], List[float]] = collections.defaultdict(list) # okuma anları (epoch) self._latest_list: Dict[str, List[int]] = {} self._resource_seconds = resource_seconds self._reading_resource_seconds = reading_resource_seconds @@ -1620,15 +1689,24 @@ def _count(self, by: str, kind: str, out: Outcome) -> None: self.requests[f"{by}:{kind}:{out.status}"] += 1 self.request_latency[kind].append(out.seconds) - def _event_seen(self, sport: Optional[str], eid: int, event: Optional[Mapping[str, Any]]) -> None: + def _event_seen(self, sport: Optional[str], eid: int, event: Optional[Mapping[str, Any]], who: str) -> None: if not event: return with self._lock: sports = [sport] if sport else [s for s in self.sports if self.tracker.state(s, eid) is not None] + now, ts = self.mono(), self.wall() for s in sports: - self.last_read[(s, eid)] = self.mono() + self.last_read[(s, eid)] = now + self.read_log[(s, eid)].append(ts) + self.stats.setdefault(s, SportStats()).read_events.append((now, who)) self.tracker.observe_event(s, event) + def reads_between(self, sport: str, a: float, b: float, *, exclude: str = "") -> int: + """Sporun [a, b] (mono) aralığındaki başarılı maç sayfası okumaları (`exclude` yolundakiler hariç).""" + with self._lock: + return sum(1 for t, who in self.stats.setdefault(sport, SportStats()).read_events + if a <= t <= b and who != exclude) + def _service_fetch(self, path: str) -> Optional[Dict[str, Any]]: """Servisin istek işlevi: bütçe, sıra, kayıt; canlı liste ve maç sayfaları kaydediciye de verilir.""" trigger_ts = self.wall() @@ -1653,7 +1731,7 @@ def _service_fetch(self, path: str) -> Optional[Dict[str, Any]]: if m and events is not None: self._on_list(m.group(1), events, trigger_ts) if e and out.data is not None: - self._event_seen(None, int(e.group(1)), out.data.get("event")) + self._event_seen(None, int(e.group(1)), out.data.get("event"), "service") if out.blocked is not None: raise out.blocked return out.data @@ -1664,6 +1742,7 @@ def _on_list(self, sport: str, events: List[Dict[str, Any]], ts: float) -> None: changes = self.tracker.listing(sport, events) st = self.stats.setdefault(sport, SportStats()) st.live_counts.append((self.wall(), len(events))) + st.list_reads.append(self.wall()) ids: List[int] = [] for ev in events: try: @@ -1674,6 +1753,9 @@ def _on_list(self, sport: str, events: List[Dict[str, Any]], ts: float) -> None: for _, kinds, _ in changes: for kind in kinds: st.triggers[kind] += 1 + for change, eid in self.joined.refresh(sport, ids): + self.recorder.write("fallback", {"event": "joined_" + change, "sport": sport, "event_id": eid, + "joined_now": len(self.joined.ids())}) for eid, kinds, expect in changes: self.reads.want(Pending(sport, eid, set(kinds), {"list"}, ts, mono, expect, mono)) @@ -1686,19 +1768,11 @@ def read_since(self, sport: str, eid: int, since: float) -> bool: last = self.last_read.get((sport, eid)) return last is not None and last >= since - def near_end(self, sport: str, eid: int) -> bool: - """Servisin yoklama kaynağının işareti (indirgeyicinin `near_end`'i); okunamazsa False.""" - try: - slot = self.service._slots.get(sport) - return bool(slot is not None and (slot.tracker.state.get(str(eid)) or {}).get("near_end")) - except Exception: - return False - def on_event_read(self, p: Pending, out: Outcome, event: Optional[Mapping[str, Any]], *, ok: bool, final: bool, latency: Optional[float], retry_in: Optional[float], ts: float, mono: float) -> None: cls = classify_status(dict(event)).value if event else None self._count("recorder", "event", out) - self._event_seen(p.sport, p.eid, event) + self._event_seen(p.sport, p.eid, event, "recorder") with self._lock: st = self.stats.setdefault(p.sport, SportStats()) if ok and latency is not None: @@ -1720,27 +1794,30 @@ def on_event_skipped(self, p: Pending) -> None: with self._lock: self.stats.setdefault(p.sport, SportStats()).skipped_budget += 1 - def fallback_read(self, sport: str, eid: int, *, window_start_ts: float) -> Outcome: + def fallback_read(self, sport: str, eid: int, *, reason: str, window_start_ts: float) -> Outcome: out = self.client.request(f"/event/{eid}") if out.refused: self._note_budget() return out event = (out.data or {}).get("event") if out.data is not None else None self._count("fallback", "event", out) - self._event_seen(sport, eid, event) + self._event_seen(sport, eid, event, "fallback") self.recorder.write("reads", { - "kind": "event", "by": "fallback", "sport": sport, "event_id": eid, "status": out.status, + "kind": "event", "by": "fallback", "reason": reason, "sport": sport, "event_id": eid, + "status": out.status, "latency_s": round(out.seconds, 3), "waited_s": round(out.waited, 3), "read_class": classify_status(dict(event)).value if event else None, "minute": round(window_start_ts, 3)}) return out - def on_fallback_minute(self, sport: str, start_ts: float, *, live: int, made: int, covered: int, priority: int, - cap: int, cut: Optional[str], partial: bool) -> None: - skipped = max(0, live - covered) - row = {"sport": sport, "minute": round(start_ts, 3), "live": live, "reads_made": made, "covered": covered, - "skipped": skipped, "coverage": round(covered / live, 4) if live else None, "priority_live": priority, - "cap": cap, "cut": cut, "partial": partial} + def on_fallback_minute(self, sport: str, start_ts: float, *, live: int, joined_live: int, + made: Mapping[str, int], other: int, skipped_joined: int, skipped_listed: int, cap: int, + cut: Optional[str], partial: bool) -> None: + row = {"event": "minute", "sport": sport, "minute": round(start_ts, 3), "live": live, + "joined_live": joined_live, + "reads": {"joined": made.get("joined", 0), "listed": made.get("listed", 0), "other": other}, + "skipped": {"joined": skipped_joined, "listed": skipped_listed}, "cap": cap, "cut": cut, + "partial": partial} with self._lock: self.stats.setdefault(sport, SportStats()).fallback.append(row) self.recorder.write("fallback", row) @@ -1785,19 +1862,55 @@ def _windows(intervals: Sequence[Sequence[Any]], start: float, end: float) -> Li @staticmethod def _fallback_summary(rows: Sequence[Mapping[str, Any]]) -> Dict[str, Any]: - full = [r for r in rows if not r.get("partial")] + """Dakika satırlarından (yarım dakikalar hariç): okumalar, atlananlar, atlama oranları, en kötü dakika.""" + full = [r for r in rows if r.get("event") == "minute" and not r.get("partial")] live = sum(r["live"] for r in full) - skipped = sum(r["skipped"] for r in full) + joined_live = sum(r["joined_live"] for r in full) + skipped = {k: sum(r["skipped"][k] for r in full) for k in ("joined", "listed")} + reads = {k: sum(r["reads"][k] for r in full) for k in ("joined", "listed", "other")} + + def rate(n: int, d: int) -> Optional[float]: + return round(n / d, 4) if d else None + scored = [r for r in full if r["live"]] - worst = min(scored, key=lambda r: (r["coverage"], -r["skipped"]), default=None) - return {"minutes": len(full), "live_total": live, "covered_total": sum(r["covered"] for r in full), - "skipped_total": skipped, "reads_made": sum(r["reads_made"] for r in full), - "skip_rate": round(skipped / live, 4) if live else None, + worst = min(scored, key=lambda r: (-r["skipped"]["joined"], + -(r["skipped"]["joined"] + r["skipped"]["listed"]) / r["live"]), + default=None) + return {"minutes": len(full), "live_total": live, "joined_live_total": joined_live, "reads": reads, + "skipped": skipped, + "skip_rate": {"joined": rate(skipped["joined"], joined_live), + "listed": rate(skipped["listed"], live - joined_live), + "all": rate(skipped["joined"] + skipped["listed"], live)}, "worst_minute": None if worst is None else { "at": utc(worst["minute"]), "sport": worst["sport"], "live": worst["live"], - "covered": worst["covered"], "skipped": worst["skipped"], "coverage": worst["coverage"], + "joined_live": worst["joined_live"], "skipped": worst["skipped"], "reads": worst["reads"], "cut": worst["cut"]}} + def _read_gaps(self, sport: str, list_led: Sequence[Tuple[float, float]]) -> Dict[str, Any]: + """ + Liste öndeyken maç sayfası okumaları arasındaki süreler: aynı liste-önde aralığındaki ardışık iki okuma (her + yoldan). Joined maçlar maç başına ve toplu; diğerleri toplu. + """ + joined: List[float] = [] + others: List[float] = [] + per_joined: Dict[str, Any] = {} + other_matches = 0 + for (s, eid), times in sorted(self.read_log.items()): + if s != sport: + continue + ts = sorted(times) + gaps = [b - a for a, b in zip(ts, ts[1:], strict=False) + if any(lo <= a and b <= hi for lo, hi in list_led)] + if eid in self.joined.ever: + joined.extend(gaps) + if gaps: + per_joined[str(eid)] = describe(gaps) + elif gaps: + others.extend(gaps) + other_matches += 1 + return {"joined": describe(joined), "joined_per_match": per_joined, "others": describe(others), + "others_matches": other_matches} + def summary(self, *, final: bool) -> Dict[str, Any]: with self._lock: start = self.started_ts or self.wall() @@ -1808,6 +1921,9 @@ def summary(self, *, final: bool) -> Dict[str, Any]: all_open: List[List[Any]] = [] sports: Dict[str, Any] = {} fallback_rows: List[Mapping[str, Any]] = [] + pooled: Dict[str, List[float]] = {"joined": [], "others": []} + list_led_total = 0.0 + list_reads_led = 0 for sport, st in sorted(self.stats.items()): opens = [list(iv) for iv in st.open_intervals] if st.open_since is not None: @@ -1831,6 +1947,19 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "longest_without_ping_or_pong_s": round(longest_without(signs, s, e), 3), }) fallback_rows.extend(st.fallback) + list_led = leader_intervals(st.leader, start, end, LEADER_POLL) + led_s = overlap(list_led, start, end) + led_lists = sum(1 for t in st.list_reads if any(lo <= t <= hi for lo, hi in list_led)) + list_led_total += led_s + list_reads_led += led_lists + read_gaps = self._read_gaps(sport, list_led) + for (s_, eid), times in self.read_log.items(): + if s_ != sport: + continue + ts_ = sorted(times) + bucket = "joined" if eid in self.joined.ever else "others" + pooled[bucket].extend(b - a for a, b in zip(ts_, ts_[1:], strict=False) + if any(lo <= a and b <= hi for lo, hi in list_led)) sports[sport] = { "frames": len(st.frames), "status_frames": st.status_frames, "score_frames": st.score_frames, "finish_frames": st.finish_frames, "distinct_events": len(st.events), @@ -1847,6 +1976,10 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "downtime_windows": self._windows(opens, start, end), "longest_msg_gaps_while_live": gaps, "fallback": self._fallback_summary(st.fallback), + "read_gaps_while_list_leads_s": read_gaps, + "discovery_while_list_leads": { + "minutes": round(led_s / 60, 2), "list_reads": led_lists, + "list_reads_per_minute": round(led_lists / (led_s / 60), 3) if led_s else None}, } conn_windows = self._windows(all_open, start, end) reads = self.credential_reads @@ -1856,9 +1989,12 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "longest_msg_gaps_while_live: intervals with no MSG frame of that sport while its last live list " "showed more than 0 matches. leader_push_share 1.0 means the arbiter judged push healthy for the " f"whole gap (a PING or PONG within {SILENCE_SECONDS:.0f} s keeps it healthy without frames).", - "fallback: while polling leads, every live match's /event/{id} at least once a minute within " - f"{self.fallback.cap} reads a minute (rate x (1 - list share)); a match not read within its minute " - "is skipped and counted, never carried over.", + "fallback: while the list leads, joined matches are read every 60 s, other live matches as " + f"the rate allows (longest unread first), within {self.fallback.cap} reads a minute (rate x (1 - " + "list share)); a read that does not fit its minute is skipped and counted, never carried over. " + "Joined = --joined-ids plus --joined-random-soccer random live football matches (refreshed when " + "they finish). Read gaps count two consecutive reads of a match (any path) inside one interval " + "in which the list leads.", "RSS: bridge = the main bridge's browser (REST reads), live = the live profile's browser (page " "source pages or the direct credential reader), other = other children (Playwright driver)."] notes.insert(0, CLIENT_NOTE if self.source == "direct" else PAGE_NOTE) @@ -1868,6 +2004,14 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "sports": list(self.sports), "args": self.args_note, "notes": notes, "sports_detail": sports, "fallback": self._fallback_summary(fallback_rows), + "read_gaps_while_list_leads_s": {"joined": describe(pooled["joined"]), + "others": describe(pooled["others"])}, + "discovery_while_list_leads": { + "sport_minutes": round(list_led_total / 60, 2), "list_reads": list_reads_led, + "list_reads_per_sport_minute": round(list_reads_led / (list_led_total / 60), 3) + if list_led_total else None}, + "joined": {"fixed": len(self.joined.fixed), "random_n": self.joined.random_n, + "ever": len(self.joined.ever), "now": len(self.joined.ids())}, "connection": { "sessions": len(self.sessions), "opens": self.connection.opens if self.connection is not None else None, @@ -1893,6 +2037,8 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "to": s.get("to"), "reason": s.get("reason")} for s in self.switches], "blocked": [{"at": utc(b["ts"]), "type": b["type"]} for b in self.blocked], "requests": {"used": self.client.used, "budget": self.client.budget, "refused": self.client.refused, + "rate": self.rate, "list_share": self.list_share, + "achieved_per_second": round(self.client.used / duration, 4), "by_who_kind_status": dict(sorted(self.requests.items())), "latency_s": {k: describe(v) for k, v in sorted(self.request_latency.items())}, "pending_event_reads": self.reads.pending(), "dropped_overflow": self.reads.overflow}, @@ -1917,11 +2063,24 @@ def _sports(value: str) -> List[str]: return list(dict.fromkeys(sports)) -def _ids(value: str) -> List[int]: +def parse_ids(text: str) -> List[int]: + """Bir id dosyasının metni: boşluk, virgül ya da satırla ayrılmış sayılar; `#`'tan sonrası yorum.""" + ids: List[int] = [] + for line in text.splitlines(): + for token in re.split(r"[\s,;]+", line.split("#", 1)[0]): + if token: + if not token.isdigit(): + raise ValueError(f"not an event id: {token[:20]!r}") + ids.append(int(token)) + return list(dict.fromkeys(ids)) + + +def _joined_file(path: str) -> List[int]: try: - return [int(v) for v in value.split(",") if v.strip()] - except ValueError: - raise argparse.ArgumentTypeError("a comma-separated list of event ids") + with open(path, encoding="utf-8") as f: + return parse_ids(f.read()) + except (OSError, ValueError) as e: + raise argparse.ArgumentTypeError(f"--joined-ids: {e}") def _share(value: str) -> float: @@ -1944,8 +2103,11 @@ def _parser() -> argparse.ArgumentParser: p.add_argument("--rate", type=float, default=DEFAULT_RATE, help="REST reads per second (default 1)") p.add_argument("--list-share", type=_share, default=DEFAULT_LIST_SHARE, help="share of the rate kept for discovery and list reads while polling leads (default 0.5)") - p.add_argument("--priority-ids", type=_ids, default=[], metavar="IDS", - help="event ids read first by the per-minute fallback (default none)") + p.add_argument("--joined-ids", type=_joined_file, default=[], metavar="FILE", + help="file of event ids treated as joined (read every 60 s while the list leads)") + p.add_argument("--joined-random-soccer", type=int, default=DEFAULT_JOINED_RANDOM, metavar="N", + help=f"also join N random live football matches, refreshed when they finish " + f"(default {DEFAULT_JOINED_RANDOM}; 0 turns it off)") p.add_argument("--kill-every", type=float, default=None, metavar="MINUTES", help="force-close the push connection (direct) or the pages (page) every MINUTES") p.add_argument("--kill-at", type=float, default=None, metavar="MINUTES", @@ -1984,12 +2146,14 @@ def main(argv: Optional[List[str]] = None) -> int: os.environ["SOFASCORE_STORAGE__DATA_DIR"] = data_dir os.makedirs(data_dir, exist_ok=True) note = {"source": args.source, "hours": args.hours, "max_requests": budget, "rate": args.rate, - "list_share": args.list_share, "priority_ids": len(args.priority_ids), + "list_share": args.list_share, "joined_ids": len(args.joined_ids), + "joined_random_soccer": args.joined_random_soccer, "kill_every_min": args.kill_every, "kill_at_min": args.kill_at, "compare_credential_sports": args.compare_credential_sports, "poll_interval_s": args.poll_interval} soak = Soak(out_dir, args.sports, run_id=run_id, data_dir=data_dir, budget=budget, source=args.source, hours=args.hours, kill_every=args.kill_every, kill_at=args.kill_at, poll_interval=args.poll_interval, - rate=args.rate, list_share=args.list_share, priority_ids=args.priority_ids, + rate=args.rate, list_share=args.list_share, joined_ids=args.joined_ids, + joined_random=args.joined_random_soccer, compare_sports=args.compare_credential_sports, args_note=note) def on_signal(signum: int, frame: Any) -> None: diff --git a/tests/test_soak_live.py b/tests/test_soak_live.py index 4d36ebc3..33c4315b 100644 --- a/tests/test_soak_live.py +++ b/tests/test_soak_live.py @@ -159,6 +159,7 @@ def make(tmp_path: Path, data_dir: Path, srv: Any, api: FakeApi, *, sports=("foo options["reader"] = reader or FakeReader(fake_credential(srv.url, "soak")) options["connection_options"] = conn options.setdefault("fallback_window", 60.0) + options.setdefault("joined_random", 0) soak = soak_live.Soak( str(run), list(sports), run_id="t-run", data_dir=str(data_dir), budget=budget, base_fetch=api, gate=lambda: None, resource_seconds=0.2, reading_resource_seconds=0.05, event_retry_seconds=0.2, @@ -369,40 +370,87 @@ def test_the_kill_schedule() -> None: assert soak_live.KillSchedule().due(10 ** 6) is None -# --- yoklama öndeyken dakikalık okumalar ---------------------------------------------------------------------- +# --- liste öndeyken: joined maçlar 60 sn'de bir, diğerleri hız elverdikçe ----------------------------------- -def test_while_polling_leads_every_live_match_is_read_each_minute_within_its_share( +def minute_rows(run: Path) -> List[Dict[str, Any]]: + return [r for r in rows(run, "fallback") if r.get("event") == "minute"] + + +def test_while_the_list_leads_joined_matches_are_read_every_minute_and_others_as_the_rate_allows( tmp_path: Path, data_dir: Path, fast_service: None, soaks: Any) -> None: matches = {eid: first_half(eid) for eid in (601, 602, 603)} api = FakeApi({"football": list(matches.values())}, dict(matches)) - # push hiç bağlanmaz (kimlik bilgisi yok): yoklama hep önde. Dakika 1 sn, hız 4/sn, listeye pay 0,5 → dakikada 2 + # push hiç bağlanmaz (kimlik bilgisi yok): liste hep önde. Dakika 1 sn, hız 4/sn, listeye pay 0,5 → dakikada 2 soak, run = soaks(tmp_path, data_dir, None, api, reader=FakeReader(), fallback_window=1.0, rate=4.0, - list_share=0.5, priority_ids=[603], poll_interval=1.0) + list_share=0.5, joined_ids=[603], poll_interval=1.0) soak.start() - wait_for(lambda: len([r for r in rows(run, "fallback") if r["live"]]) >= 3, seconds=10) + wait_for(lambda: len([r for r in minute_rows(run) if r["live"] and not r["partial"]]) >= 4, seconds=15) soak.stop() - minutes = [r for r in rows(run, "fallback") if r["live"] and not r["partial"]] - assert all((r["live"], r["reads_made"], r["covered"], r["skipped"], r["cap"]) == (3, 2, 2, 1, 2) - for r in minutes) - assert all(r["coverage"] == pytest.approx(2 / 3, abs=1e-3) and r["cut"] == "share" for r in minutes) - fallback_reads = reads_by(run, "fallback") - by_minute: Dict[float, List[int]] = {} - for r in fallback_reads: - by_minute.setdefault(r["minute"], []).append(r["event_id"]) - full = [ids for m, ids in sorted(by_minute.items()) if any(x["minute"] == m for x in minutes)] - assert all(ids[0] == 603 for ids in full) # öncelikli id önce - # sonra en uzun süredir okunmayan: 601 ile 602 sırayla - assert [ids[1] for ids in full[:3]] in ([601, 602, 601], [602, 601, 602]) + minutes = [r for r in minute_rows(run) if r["live"] and not r["partial"]] + assert all((r["live"], r["joined_live"], r["cap"], r["cut"]) == (3, 1, 2, "share") for r in minutes) + assert all(r["reads"] == {"joined": 1, "listed": 1, "other": 0} for r in minutes) + assert all(r["skipped"] == {"joined": 0, "listed": 1} for r in minutes) + by_minute: Dict[float, List[Any]] = {} + for r in reads_by(run, "fallback"): + by_minute.setdefault(r["minute"], []).append((r["reason"], r["event_id"])) + full = [by_minute[m["minute"]] for m in minutes] + assert all(reads[0] == ("joined", 603) for reads in full) # acil: joined önce + # sonra birikmiş iş, en uzun süredir okunmayan önce: 601 ile 602 sırayla + listed = [reads[1][1] for reads in full[:4]] + assert listed in ([601, 602, 601, 602], [602, 601, 602, 601]) and all(r[1][0] == "listed" for r in full) assert reads_by(run, "recorder") == [] # liste değişmedi: tetiklenen okuma yok s = summary(run) fb = s["sports_detail"]["football"]["fallback"] - assert fb["skip_rate"] == pytest.approx(1 / 3, abs=1e-3) and fb["skipped_total"] == len(minutes) - assert fb["worst_minute"]["coverage"] == pytest.approx(2 / 3, abs=1e-3) and fb["worst_minute"]["skipped"] == 1 + assert fb["skip_rate"]["joined"] == 0.0 and fb["skip_rate"]["listed"] == pytest.approx(0.5, abs=1e-3) + assert fb["skipped"] == {"joined": 0, "listed": len(minutes)} and fb["worst_minute"]["skipped"]["listed"] == 1 assert s["fallback"]["skip_rate"] == fb["skip_rate"] + gaps = s["sports_detail"]["football"]["read_gaps_while_list_leads_s"] + assert gaps["joined"]["n"] >= 3 and gaps["joined"]["median"] == pytest.approx(1.0, abs=0.3) + assert gaps["others"]["median"] == pytest.approx(2.0, abs=0.4) and gaps["others_matches"] == 2 + assert list(gaps["joined_per_match"]) == ["603"] + assert s["read_gaps_while_list_leads_s"]["joined"]["n"] == gaps["joined"]["n"] + discovery = s["sports_detail"]["football"]["discovery_while_list_leads"] + assert discovery["list_reads"] >= 3 and discovery["list_reads_per_minute"] > 0 assert s["sports_detail"]["football"]["fallback_time_share"] == 1.0 + assert s["requests"]["achieved_per_second"] > 0 and s["joined"]["fixed"] == 1 + + +def test_random_live_football_matches_are_joined_and_replaced_when_they_finish( + tmp_path: Path, data_dir: Path, fast_service: None, soaks: Any) -> None: + matches = {eid: first_half(eid) for eid in (601, 602, 603)} + api = FakeApi({"football": list(matches.values())}, dict(matches)) + soak, run = soaks(tmp_path, data_dir, None, api, reader=FakeReader(), fallback_window=0.5, rate=20.0, + joined_random=2, joined_rng=__import__("random").Random(7), poll_interval=1.0) + soak.start() + wait_for(lambda: len(soak.joined.ids()) == 2) + first = set(soak.joined.ids()) + gone = min(first) + api.live["football"] = [m for eid, m in matches.items() if eid != gone] # biri bitti: listeden düştü + wait_for(lambda: gone not in soak.joined.ids() and len(soak.joined.ids()) == 2, seconds=5) + wait_for(lambda: any(r["joined_live"] == 2 and r["live"] == 2 for r in minute_rows(run)), seconds=5) + soak.stop() + assert soak.joined.ids() == frozenset(set(matches) - {gone}) + changes = [(r["event"], r["event_id"]) for r in rows(run, "fallback") if r["event"].startswith("joined_")] + assert changes[:2] == [("joined_join", e) for e in sorted(first)] + assert ("joined_leave", gone) in changes and len([c for c in changes if c[0] == "joined_join"]) == 3 + + +def test_the_joined_set() -> None: + import random + + js = soak_live.JoinedSet([99], 2, rng=random.Random(1)) + assert js.refresh("tennis", [1, 2, 3]) == [] and js.ids() == {99} + joined = js.refresh("football", [1, 2, 3, 99]) + assert [c for c, _ in joined] == ["join", "join"] and 99 not in {e for _, e in joined} + picks = set(js.ids()) - {99} + keep = min(picks) + changes = js.refresh("football", [keep, 5, 6]) + assert ("leave", max(picks)) in changes and len(js.ids() - {99}) == 2 and keep in js.ids() + assert js.ever >= picks | {99} + assert soak_live.JoinedSet([], 0).refresh("football", [1]) == [] def test_while_push_leads_there_are_no_minute_reads( @@ -416,19 +464,20 @@ def test_while_push_leads_there_are_no_minute_reads( reads_then = len(reads_by(run, "fallback")) time.sleep(1.2) soak.stop() - later = [r for r in rows(run, "fallback") if r["minute"] > soak.switches[0]["ts"] + 0.5] + later = [r for r in minute_rows(run) if r["minute"] > soak.switches[0]["ts"] + 0.5] assert later == [] and len(reads_by(run, "fallback")) == reads_then -def test_the_fallback_share_and_ranking() -> None: +def test_the_fallback_share_and_queue_order() -> None: fb = soak_live.FallbackReads.__new__(soak_live.FallbackReads) - soak_live.FallbackReads.__init__(fb, SimpleNamespace(), rate=1.0, share=0.5, window=60.0, priority=[7]) - assert fb.cap == 30 and fb.priority == frozenset({7}) - soak = SimpleNamespace(last_read={("football", 1): 10.0, ("football", 2): 5.0}, - near_end=lambda sport, eid: eid == 3) - fb._soak = soak - ranked = sorted([("football", e) for e in (1, 2, 3, 4, 7)], key=lambda c: fb._rank(*c)) - assert [e for _, e in ranked] == [7, 3, 4, 2, 1] # öncelik, bitişe yakın, hiç okunmamış, en eski okuma + soak_live.FallbackReads.__init__(fb, SimpleNamespace(), rate=1.0, share=0.5, window=60.0) + assert fb.cap == 30 + fb._soak = SimpleNamespace(last_read={("football", 1): 10.0, ("football", 2): 5.0, ("football", 7): 3.0, + ("football", 8): 1.0}) + order = fb.queue(["football"], {"football": [1, 2, 3, 7, 8]}, frozenset({7, 8, 3})) + # joined önce (hiç okunmamış, sonra en eski), sonra listed (en uzun süredir okunmayan önce) + assert order == [("joined", "football", 3), ("joined", "football", 8), ("joined", "football", 7), + ("listed", "football", 2), ("listed", "football", 1)] # --- bütçe ------------------------------------------------------------------------------------------------------ @@ -622,14 +671,25 @@ def test_a_used_run_id_is_refused_before_anything_starts(tmp_path: Path, capsys: def test_the_options_parse() -> None: args = soak_live._parser().parse_args(["--sports", "football, tennis", "--kill-every", "40", "--kill-at", "5", - "--compare-credential-sports", "--priority-ids", "1,2", "--rate", "0.8", - "--list-share", "0.25"]) + "--compare-credential-sports", "--rate", "0.8", "--list-share", "0.25", + "--joined-random-soccer", "5"]) assert args.sports == ["football", "tennis"] and args.kill_every == 40 and args.kill_at == 5 assert args.compare_credential_sports and args.max_requests is None and args.hours == soak_live.DEFAULT_HOURS - assert args.priority_ids == [1, 2] and args.rate == 0.8 and args.list_share == 0.25 and args.source == "direct" - for bad in (["--sports", "foot ball"], ["--list-share", "1"], ["--source", "auto"], ["--priority-ids", "x"]): + assert args.joined_ids == [] and args.joined_random_soccer == 5 and args.rate == 0.8 and args.list_share == 0.25 + assert args.source == "direct" and soak_live._parser().parse_args([]).joined_random_soccer == 20 + for bad in (["--sports", "foot ball"], ["--list-share", "1"], ["--source", "auto"], + ["--joined-ids", "/nonexistent/ids.txt"]): with pytest.raises(SystemExit): soak_live._parser().parse_args(bad) assert conftest # sınır denetimi ve kancalar conftest'te kurulur + + +def test_the_joined_ids_file(tmp_path: Path) -> None: + path = tmp_path / "ids.txt" + path.write_text("# c fixtures\n17124861, 16947598\n17124861 15453137 # again\n", encoding="utf-8") + assert soak_live._parser().parse_args(["--joined-ids", str(path)]).joined_ids == [17124861, 16947598, 15453137] + path.write_text("12 abc\n", encoding="utf-8") + with pytest.raises(SystemExit): + soak_live._parser().parse_args(["--joined-ids", str(path)]) From 01b2518ca92900ff8d398264c3472591aeed340a Mon Sep 17 00:00:00 2001 From: tunjayoff Date: Tue, 13 Oct 2026 16:42:42 +0300 Subject: [PATCH 4/4] feat(research): rate phases, per-minute request stats and c's joined-ids shape --rate-schedule "60:1,120:5" runs one rate after another in a single run. requests.jsonl records each minute's requests by kind and path, the peak in any one-second window, 403s (curl challenge, browser challenge, browser), 429s and Turnstile solves counted from the client's own log lines, and latency p50/p95; the summary gives the same per rate phase. --joined-ids now reads c's message shape {"ids": [...]} and ignores a malformed file or one with more than 50 000 ids, as c does. --- scripts/soak_live.py | 351 ++++++++++++++++++++++++++++++++++++---- tests/test_soak_live.py | 103 ++++++++++-- 2 files changed, 411 insertions(+), 43 deletions(-) diff --git a/scripts/soak_live.py b/scripts/soak_live.py index 3ddf556d..b1762b2a 100644 --- a/scripts/soak_live.py +++ b/scripts/soak_live.py @@ -8,7 +8,7 @@ python scripts/soak_live.py --sports football,basketball,tennis --hours 6 [--source direct|page] [--run-id ID] [--max-requests N] [--rate 1.0] [--list-share 0.5] [--joined-ids DOSYA] - [--joined-random-soccer 20] + [--joined-random-soccer 20] [--rate-schedule "60:1,120:5"] [--kill-every DAKİKA] [--kill-at DAKİKA] [--compare-credential-sports] [--out KLASÖR] **`direct` açık seçimdir** (docs/deploy/watch.md, "The `direct` source"): sitenin kendi istemci kimlik bilgisini @@ -35,6 +35,12 @@ pay keşfe ayrılır. Sığmayan okuma atlanır ve spor başına sayılır; hiçbir şey sınırsız beklemez. fallback.jsonl: dakika ve spor başına canlı maç, joined canlı maç, okumalar (joined/listed/diğer), atlananlar; joined küme değişiklikleri. Özet: joined ve diğer maçların okuma aralıkları (medyan/p90/maks), keşif hızı, atlama oranları. + * Hız evreleri (`--rate-schedule "60:1,120:5"`: 60 dk 1 istek/sn, sonra 5 istek/sn; son evre sürer): requests.jsonl + dakika başına istekleri (türe ve yola göre, toplam), herhangi 1 sn'deki zirveyi, 429 ve 403 sayılarını (curl + challenge, tarayıcı challenge, tarayıcı), Turnstile çözümlerini ve yanıt süresinin p50/p95'ini yazar; özet aynısını + evre başına verir. 403/429/Turnstile sayıları istemcinin kendi log satırlarından sayılır (ChallengeCounter); + satırların metni yazılmaz. 1 istek/sn'den hızlı evrede araştırma kilidi alınmaz. + * `--joined-ids DOSYA`: c'nin mesaj biçimi `{"ids": [...]}`; bozuk dosya ya da 50.000'den çok id yok sayılır. * Kancalar: DirectConnection'ın alt sınıfı (oturum başı/sonu, yeniden deneme), bağlantı işlevinin sarmalayıcısı (el sıkışma, hat üstü işlemler; öldürme), sayfa açıcının sarmalayıcısı (`page`; öldürme = sayfayı kapatma), her sporun PushFeed'ini saran musluk (MSG kareleri; `page`'de PING/PONG/INFO/-ERR de), kimlik bilgisi okuyucusunun @@ -74,6 +80,7 @@ import hashlib import hmac import json +import logging import math import os import random @@ -124,7 +131,8 @@ MAX_PENDING_READS = 1000 TOP_GAPS = 5 -FILES = ("frames", "connection", "leader", "reads", "fallback", "resources") +FILES = ("frames", "connection", "leader", "reads", "requests", "fallback", "resources") +MAX_JOINED_IDS = 50_000 # c gibi: daha büyük joined dosyası yok sayılır LIVE = StatusClass.LIVE.value # Karede değeri yazılan alanlar (geri kalanların yalnızca yolu yazılır) STATUS_VALUE_KEYS = ("status.code", "status.type") @@ -169,6 +177,25 @@ def describe(values: Sequence[float]) -> Dict[str, Any]: "max": round(vals[-1], 3), "min": round(vals[0], 3)} +def percentile(values: Sequence[float], q: float) -> Optional[float]: + """En yakın sıra yüzdeliği (q 0..1); boşsa None.""" + vals = sorted(float(v) for v in values) + if not vals: + return None + return round(vals[min(len(vals) - 1, max(0, int(math.ceil(q * len(vals))) - 1))], 3) + + +def peak_in_window(times: Sequence[float], window: float = 1.0) -> int: + """Herhangi bir `window` sn'lik aralıktaki en çok olay (sıralı anlardan, iki işaretçiyle).""" + ts = sorted(times) + best, lo = 0, 0 + for hi, t in enumerate(ts): + while t - ts[lo] >= window: + lo += 1 + best = max(best, hi - lo + 1) + return best + + class Recorder: """Koşunun JSONL dosyaları. Her satır `ts` (epoch) ve `mono` (monotonik saat) taşır ve maskeden geçer.""" @@ -380,6 +407,7 @@ class Outcome: waited: float = 0.0 blocked: Optional[BaseException] = None refused: bool = False # bütçe bitti: ağa çıkılmadı + sent: float = 0.0 # isteğin sıradan çıkıp gönderildiği an (mono) def status_of(exc: Optional[BaseException]) -> Optional[Any]: @@ -403,9 +431,9 @@ def status_of(exc: Optional[BaseException]) -> Optional[Any]: class Pacer: """Süreç içi hız: ardışık REST okumaları arasında en az 1/rate sn (thread'ler arası sıra).""" - def __init__(self, rate: float, *, mono: Callable[[], float] = time.monotonic, + def __init__(self, rate: Any, *, mono: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep) -> None: - self.interval = 1.0 / rate if rate > 0 else 0.0 + self._rate: Callable[[], float] = rate if callable(rate) else (lambda: float(rate)) self._mono = mono self._sleep = sleep self._lock = threading.Lock() @@ -415,21 +443,79 @@ def wait(self) -> None: with self._lock: now = self._mono() at = max(now, self._next) - self._next = at + self.interval + rate = self._rate() + self._next = at + (1.0 / rate if rate > 0 else 0.0) if at > now: self._sleep(at - now) -def default_gate(rate: float) -> Callable[[], None]: +def default_gate(rate: Callable[[], float]) -> Callable[[], None]: + """ + O anki hızda sıra (`--rate` ya da `--rate-schedule`'un evresi). Hız ≤ 1 istek/sn iken araştırma script'lerinin + ortak kilidinden de sıra alınır; daha hızlı bir evrede o kilit (≥ 1 sn aralık) evreyi boğacağı için alınmaz. + """ pacer = Pacer(rate) def gate() -> None: pacer.wait() - rc._wait_rate_slot() # araştırma süreçlerinin ortak kilidi (≤ 1 istek/sn) + if rate() <= 1.0: + rc._wait_rate_slot() # araştırma süreçlerinin ortak kilidi (≤ 1 istek/sn) return gate +class RateSchedule: + """ + `--rate-schedule "60:1,120:5"`: evreler (dakika, istek/sn) sırayla; son evrenin hızı koşu bitene kadar sürer. + Evre yoksa tek evre: `--rate`. + """ + + def __init__(self, phases: Sequence[Tuple[float, float]] = (), default: float = DEFAULT_RATE) -> None: + self.phases: List[Tuple[float, float]] = [(float(m), float(r)) for m, r in phases] or [(math.inf, default)] + + @classmethod + def parse(cls, text: str) -> "RateSchedule": + phases: List[Tuple[float, float]] = [] + for part in text.split(","): + minutes, sep, rate = part.strip().partition(":") + if not sep: + raise ValueError(f"not MINUTES:RATE: {part.strip()[:20]!r}") + m, r = float(minutes), float(rate) + if not (m > 0 and r > 0 and math.isfinite(m) and math.isfinite(r)): + raise ValueError("minutes and rate must be positive") + phases.append((m, r)) + if not phases: + raise ValueError("empty schedule") + return cls(phases) + + def bounds(self) -> List[Tuple[float, float, float]]: + """(başlangıç sn, bitiş sn, hız); son evrenin bitişi sonsuz.""" + out: List[Tuple[float, float, float]] = [] + t = 0.0 + for i, (minutes, rate) in enumerate(self.phases): + end = math.inf if i == len(self.phases) - 1 else t + minutes * 60.0 + out.append((t, end, rate)) + t = end + return out + + def phase_at(self, elapsed: float) -> int: + for i, (a, b, _) in enumerate(self.bounds()): + if a <= elapsed < b: + return i + return len(self.phases) - 1 + + def rate_at(self, elapsed: float) -> float: + return self.bounds()[self.phase_at(elapsed)][2] + + @property + def max_rate(self) -> float: + return max(r for _, r in self.phases) + + def requests_in(self, seconds: float) -> float: + """Bu sürede evrelerin hızıyla yapılabilecek istek sayısı (bütçe).""" + return sum(max(0.0, min(b, seconds) - a) * r for a, b, r in self.bounds()) + + class ReadClient: """ Kaydedicinin ve servisin REST okumaları: koşu bütçesi, sıra (`gate`), süre ve durum kodu. `base` servisin gerçek @@ -462,10 +548,102 @@ def request(self, path: str) -> Outcome: try: data = self._base(path) except sv.Blocked as e: - return Outcome(status_of(e.__cause__) or "blocked", self._mono() - t1, None, t1 - t0, blocked=e) + return Outcome(status_of(e.__cause__) or "blocked", self._mono() - t1, None, t1 - t0, blocked=e, sent=t1) except Exception as e: # tek isteğin hatası: kayda geçer, kaydedici sürer - return Outcome(status_of(e) or type(e).__name__, self._mono() - t1, None, t1 - t0) - return Outcome(200 if data is not None else 404, self._mono() - t1, data, t1 - t0) + return Outcome(status_of(e) or type(e).__name__, self._mono() - t1, None, t1 - t0, sent=t1) + return Outcome(200 if data is not None else 404, self._mono() - t1, data, t1 - t0, sent=t1) + + +class ChallengeCounter(logging.Handler): + """ + İstemcinin kendi log satırlarından (sofascore_scraper/client/transport.py "Utils", bridge.py "ChallengeSolver") + curl ve tarayıcı yanıtlarının 403/429'larını ve Turnstile çözümlerini sayar; satırın metni hiçbir yere yazılmaz. + Ürün kodu değişmez: bu sayılar başka bir yoldan dışarı verilmiyor. + """ + + LOGGERS = ("Utils", "ChallengeSolver") + PATTERNS = ( + (re.compile(r"^Rate limited or server busy \((\d{3})\)"), "curl_{0}"), + (re.compile(r"^Rate limited or server busy, no attempts left"), "curl_429_or_503_last"), + (re.compile(r"^403 Forbidden \(attempt"), "curl_403"), + (re.compile(r"^Turnstile challenge detected; fetching through the browser bridge"), "curl_403_challenge"), + (re.compile(r"^The API answered 403 with a challenge"), "browser_403_challenge"), + (re.compile(r"^Browser fetch failed \(status (\d+)\)"), "browser_{0}"), + (re.compile(r"^Solving the Cloudflare Turnstile challenge"), "turnstile_attempts"), + (re.compile(r"^Turnstile challenge solved"), "turnstile_solved"), + (re.compile(r"^The Turnstile challenge could not be solved"), "turnstile_failed"), + ) + + def __init__(self, mono: Callable[[], float] = time.monotonic) -> None: + super().__init__(level=logging.INFO) + self._mono = mono + self.events: List[Tuple[float, str]] = [] + self._lock_events = threading.Lock() + self._saved: List[Tuple[logging.Logger, int, bool]] = [] + self._forward = _Forward() + + def emit(self, record: logging.LogRecord) -> None: + try: + text = record.getMessage() + except Exception: + return + for pattern, key in self.PATTERNS: + m = pattern.match(text) + if m: + with self._lock_events: + self.events.append((self._mono(), key.format(*m.groups()))) + return + + def between(self, a: float, b: float) -> Dict[str, int]: + with self._lock_events: + counts: Dict[str, int] = collections.defaultdict(int) + for t, key in self.events: + if a <= t < b: + counts[key] += 1 + return dict(sorted(counts.items())) + + def install(self) -> None: + """INFO satırları da gelsin diye iki logger INFO'ya iner; yalnız WARNING ve üstü kök logger'a iletilir.""" + for name in self.LOGGERS: + lg = logging.getLogger(name) + self._saved.append((lg, lg.level, lg.propagate)) + lg.setLevel(logging.INFO) + lg.propagate = False + lg.addHandler(self) + lg.addHandler(self._forward) + + def uninstall(self) -> None: + for lg, level, propagate in self._saved: + lg.removeHandler(self) + lg.removeHandler(self._forward) + lg.setLevel(level) + lg.propagate = propagate + self._saved.clear() + + +class _Forward(logging.Handler): + """Yalnız WARNING ve üstünü kök logger'ın handler'larına iletir (konsol INFO ile dolmasın).""" + + def __init__(self) -> None: + super().__init__(level=logging.WARNING) + + def emit(self, record: logging.LogRecord) -> None: + if record.levelno >= logging.getLogger().getEffectiveLevel(): + logging.getLogger().handle(record) + + +def challenge_summary(counts: Mapping[str, int]) -> Dict[str, Any]: + """Sayaçlardan 429 ve 403 toplamları ile Turnstile çözümleri.""" + c = dict(counts) + return { + "429": c.get("curl_429", 0) + c.get("browser_429", 0), + "503": c.get("curl_503", 0) + c.get("browser_503", 0), + "rate_limited_last_attempt": c.get("curl_429_or_503_last", 0), + "403": {"curl": c.get("curl_403", 0), "curl_challenge": c.get("curl_403_challenge", 0), + "browser_challenge": c.get("browser_403_challenge", 0), "browser": c.get("browser_403", 0)}, + "turnstile": {"attempts": c.get("turnstile_attempts", 0), "solved": c.get("turnstile_solved", 0), + "failed": c.get("turnstile_failed", 0)}, + } @dataclass @@ -631,10 +809,16 @@ class FallbackReads: dakika listeden yeniden kurulur). """ - def __init__(self, soak: "Soak", *, rate: float, share: float, window: float = FALLBACK_WINDOW_SECONDS) -> None: + def __init__(self, soak: "Soak", *, rate: Any, share: float, window: float = FALLBACK_WINDOW_SECONDS) -> None: self._soak = soak self.window = float(window) - self.cap = max(0, int(math.floor(self.window * rate * (1.0 - share) + 1e-9))) + self.share = float(share) + self._rate: Callable[[], float] = rate if callable(rate) else (lambda: float(rate)) + + @property + def cap(self) -> int: + """Bu dakikanın payı: o anki hızla (evre değişebilir).""" + return max(0, int(math.floor(self.window * self._rate() * (1.0 - self.share) + 1e-9))) def run(self, stop: threading.Event) -> None: from sofascore_scraper.client import request_context @@ -667,6 +851,7 @@ def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: joined = soak.joined.ids() live = {sport: soak.latest_list(sport) for sport in sports} made: Dict[str, Dict[str, int]] = {s: {"joined": 0, "listed": 0} for s in sports} + cap = self.cap attempted = 0 cut = None for reason, sport, eid in self.queue(sports, live, joined): @@ -677,7 +862,7 @@ def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: continue # push yeniden önde: bu sporun dakikalık okumaları durur if soak.read_since(sport, eid, start): continue # bu dakika başka bir yoldan okundu - if attempted >= self.cap: + if attempted >= cap: cut = "share" break if soak.mono() >= end: @@ -703,7 +888,7 @@ def _minute(self, sports: Sequence[str], stop: threading.Event) -> None: other=soak.reads_between(sport, start, soak.mono(), exclude="fallback"), skipped_joined=sum(1 for e in j if not soak.read_since(sport, e, start)), skipped_listed=sum(1 for e in o if not soak.read_since(sport, e, start)), - cap=self.cap, cut=cut, partial=early or sport not in led) # yarım dakika oranlara sayılmaz + cap=cap, cut=cut, partial=early or sport not in led) # yarım dakika oranlara sayılmaz # --- push: musluk, soket, okuyucu, bağlantı, sayfa açıcı ------------------------------------------------------- @@ -1202,8 +1387,13 @@ def due(self, elapsed: float) -> Optional[str]: return None -def default_budget(hours: float, rate: float = DEFAULT_RATE) -> int: - """Koşunun REST bütçesi: süre × hız (ortalama ≤ 1 istek/sn).""" +def default_budget(hours: float, rate: float = DEFAULT_RATE, schedule: Optional[RateSchedule] = None) -> int: + """ + Koşunun REST bütçesi: süre × hız (ortalama ≤ 1 istek/sn). `--rate-schedule` verildiyse evrelerin hızlarıyla + (hızlı bir evre bilerek seçilir; bütçe onu boğmasın). + """ + if schedule is not None: + return max(1, int(schedule.requests_in(hours * 3600))) return max(1, int(hours * 3600 * min(1.0, rate))) @@ -1248,6 +1438,7 @@ def __init__(self, out_dir: str, sports: Sequence[str], *, run_id: str, data_dir budget: int, source: str = "direct", hours: Optional[float] = None, kill_every: Optional[float] = None, kill_at: Optional[float] = None, poll_interval: float = 30.0, rate: float = DEFAULT_RATE, list_share: float = DEFAULT_LIST_SHARE, + rate_schedule: Optional[RateSchedule] = None, minute_seconds: float = 60.0, joined_ids: Iterable[int] = (), joined_random: int = DEFAULT_JOINED_RANDOM, joined_rng: Optional[random.Random] = None, fallback_window: float = FALLBACK_WINDOW_SECONDS, base_fetch: Optional[Callable[[str], Optional[Dict[str, Any]]]] = None, @@ -1274,14 +1465,18 @@ def __init__(self, out_dir: str, sports: Sequence[str], *, run_id: str, data_dir self._lock = threading.RLock() self._stop = threading.Event() self.stopping = False + self.schedule = rate_schedule or RateSchedule((), rate) self.client = ReadClient(base_fetch or sv._default_fetch, budget=budget, - gate=gate if gate is not None else default_gate(rate), mono=self.mono) + gate=gate if gate is not None else default_gate(self.current_rate), mono=self.mono) + self.challenges = ChallengeCounter(self.mono) + self.req_log: List[Tuple[float, str, str, Any, float]] = [] # (gönderildi mono, kim, tür, durum, süre) + self._minute_seconds = float(minute_seconds) self.tracker = ChangeTracker() self.reads = EventReads(self, retry=event_retry_seconds) self.rate = rate self.list_share = list_share self.joined = JoinedSet(joined_ids, joined_random, rng=joined_rng) - self.fallback = FallbackReads(self, rate=rate, share=list_share, window=fallback_window) + self.fallback = FallbackReads(self, rate=self.current_rate, share=list_share, window=fallback_window) self.stats: Dict[str, SportStats] = {s: SportStats() for s in self.sports} self.last_read: Dict[Tuple[str, int], float] = {} # son başarılı maç sayfası okuması (mono), her yoldan self.read_log: Dict[Tuple[str, int], List[float]] = collections.defaultdict(list) # okuma anları (epoch) @@ -1355,6 +1550,8 @@ def start(self) -> None: self._spawn("soak-event-reads", lambda: self.reads.run(self._stop)) self._spawn("soak-fallback", lambda: self.fallback.run(self._stop)) self._spawn("soak-resources", self._sample_loop) + self.challenges.install() + self._spawn("soak-minutes", self._minute_loop) def _spawn(self, name: str, target: Callable[[], None]) -> None: t = threading.Thread(target=target, name=name, daemon=True) @@ -1384,6 +1581,7 @@ def stop(self, timeout: float = 30.0) -> None: if thread is not None: thread.join(timeout) self.ended_ts = self.wall() + self.challenges.uninstall() for sport in self.sports: self._close_sport(sport, self.ended_ts, "run end") self.recorder.write("connection", {"event": "run_end"}) @@ -1412,6 +1610,13 @@ def run(self, hours: float, *, tick: float = 1.0) -> None: def request_stop(self) -> None: self._stop.set() + def elapsed(self) -> float: + return 0.0 if self.started_mono is None else self.mono() - self.started_mono + + def current_rate(self) -> float: + """O anki hız (`--rate-schedule`'un evresi ya da `--rate`).""" + return self.schedule.rate_at(self.elapsed()) + def kill(self, cause: str) -> bool: """`direct`: bağlantının soketini kapatır; `page`: sporların sayfalarını kapatır.""" sports: List[str] = [] @@ -1688,6 +1893,44 @@ def _count(self, by: str, kind: str, out: Outcome) -> None: with self._lock: self.requests[f"{by}:{kind}:{out.status}"] += 1 self.request_latency[kind].append(out.seconds) + self.req_log.append((out.sent, by, kind, out.status, out.seconds)) + + def request_stats(self, a: float, b: float) -> Dict[str, Any]: + """[a, b) (mono) aralığında gönderilen REST okumaları: türe ve yola göre, 1 sn'lik zirve, durumlar, süre.""" + with self._lock: + rows = [r for r in self.req_log if a <= r[0] < b] + by_kind: Dict[str, int] = collections.defaultdict(int) + by_who: Dict[str, int] = collections.defaultdict(int) + status: Dict[str, int] = collections.defaultdict(int) + for _, who, kind, st, _ in rows: + by_kind[kind] += 1 + by_who[who] += 1 + status[str(st)] += 1 + latency = [r[4] for r in rows] + return {"total": len(rows), "by_kind": dict(sorted(by_kind.items())), "by_who": dict(sorted(by_who.items())), + "status": dict(sorted(status.items())), "peak_1s": peak_in_window([r[0] for r in rows]), + "latency_s": {"p50": percentile(latency, 0.5), "p95": percentile(latency, 0.95)}, + "client": challenge_summary(self.challenges.between(a, b))} + + def _minute_loop(self) -> None: + """Her dakika (koşu başından hizalı) istek satırı: requests.jsonl.""" + assert self.started_mono is not None + index = 0 + while not self._stop.is_set(): + a = self.started_mono + index * self._minute_seconds + b = a + self._minute_seconds + self._stop.wait(max(0.0, b - self.mono())) + if self._stop.is_set() and self.mono() < b: + b = self.mono() # son, yarım dakika + partial = True + else: + partial = False + elapsed = a - self.started_mono + row = {"event": "minute", "minute": index, "start": round((self.started_ts or 0) + elapsed, 3), + "phase": self.schedule.phase_at(elapsed), "rate": self.schedule.rate_at(elapsed), + "partial": partial, **self.request_stats(a, b)} + self.recorder.write("requests", row) + index += 1 def _event_seen(self, sport: Optional[str], eid: int, event: Optional[Mapping[str, Any]], who: str) -> None: if not event: @@ -1990,7 +2233,7 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "showed more than 0 matches. leader_push_share 1.0 means the arbiter judged push healthy for the " f"whole gap (a PING or PONG within {SILENCE_SECONDS:.0f} s keeps it healthy without frames).", "fallback: while the list leads, joined matches are read every 60 s, other live matches as " - f"the rate allows (longest unread first), within {self.fallback.cap} reads a minute (rate x (1 - " + f"the rate allows (longest unread first), within {self.fallback.cap} reads a minute now (rate x (1 - " "list share)); a read that does not fit its minute is skipped and counted, never carried over. " "Joined = --joined-ids plus --joined-random-soccer random live football matches (refreshed when " "they finish). Read gaps count two consecutive reads of a match (any path) inside one interval " @@ -2036,6 +2279,7 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "leader_switches": [{"at": utc(s["ts"]), "sport": s.get("sport"), "from": s.get("from"), "to": s.get("to"), "reason": s.get("reason")} for s in self.switches], "blocked": [{"at": utc(b["ts"]), "type": b["type"]} for b in self.blocked], + "rate_phases": self._phase_summary(end), "requests": {"used": self.client.used, "budget": self.client.budget, "refused": self.client.refused, "rate": self.rate, "list_share": self.list_share, "achieved_per_second": round(self.client.used / duration, 4), @@ -2048,6 +2292,21 @@ def summary(self, *, final: bool) -> Dict[str, Any]: "source_switches", "leaders")}}, } + def _phase_summary(self, end_ts: float) -> List[Dict[str, Any]]: + """`--rate-schedule`'un her evresi için (koşunun gördüğü kısmı) istek istatistikleri.""" + if self.started_mono is None: + return [] + run_s = max(0.0, end_ts - (self.started_ts or end_ts)) + out = [] + for i, (a, b, rate) in enumerate(self.schedule.bounds()): + if a >= run_s: + break + hi = min(b, run_s) + stats = self.request_stats(self.started_mono + a, self.started_mono + hi) + out.append({"phase": i, "rate": rate, "start_min": round(a / 60, 4), "end_min": round(hi / 60, 4), + "achieved_per_second": round(stats["total"] / (hi - a), 4) if hi > a else None, **stats}) + return out + def write_summary(self, *, final: bool) -> None: self._last_summary = self.mono() self.recorder.write_summary(self.summary(final=final)) @@ -2063,24 +2322,42 @@ def _sports(value: str) -> List[str]: return list(dict.fromkeys(sports)) -def parse_ids(text: str) -> List[int]: - """Bir id dosyasının metni: boşluk, virgül ya da satırla ayrılmış sayılar; `#`'tan sonrası yorum.""" - ids: List[int] = [] - for line in text.splitlines(): - for token in re.split(r"[\s,;]+", line.split("#", 1)[0]): - if token: - if not token.isdigit(): - raise ValueError(f"not an event id: {token[:20]!r}") - ids.append(int(token)) +def parse_joined(text: str) -> Optional[List[int]]: + """ + c'nin mesaj biçimi: `{"ids": [olay id'leri]}`. Bozuk dosya (JSON değil, nesne değil, `ids` liste değil, tam sayı + olmayan id) ya da MAX_JOINED_IDS'ten çok id: None (c gibi yok sayılır). + """ + try: + doc = json.loads(text) + except ValueError: + return None + ids = doc.get("ids") if isinstance(doc, dict) else None + if not isinstance(ids, list) or len(ids) > MAX_JOINED_IDS: + return None + if not all(isinstance(x, int) and not isinstance(x, bool) and x > 0 for x in ids): + return None return list(dict.fromkeys(ids)) def _joined_file(path: str) -> List[int]: try: with open(path, encoding="utf-8") as f: - return parse_ids(f.read()) - except (OSError, ValueError) as e: - raise argparse.ArgumentTypeError(f"--joined-ids: {e}") + ids = parse_joined(f.read()) + except (OSError, UnicodeDecodeError) as e: + print(f"warning: --joined-ids: the file could not be read ({type(e).__name__}); ignoring it", file=sys.stderr) + return [] + if ids is None: + print(f"warning: --joined-ids: not a {{\"ids\": [...]}} object with at most {MAX_JOINED_IDS} ids; " + "ignoring it", file=sys.stderr) + return [] + return ids + + +def _schedule(value: str) -> RateSchedule: + try: + return RateSchedule.parse(value) + except ValueError as e: + raise argparse.ArgumentTypeError(f"--rate-schedule: {e}") def _share(value: str) -> float: @@ -2101,6 +2378,9 @@ def _parser() -> argparse.ArgumentParser: p.add_argument("--max-requests", type=int, default=None, help="budget of REST reads (lists and event pages) for this run (default --hours x 3600 x --rate)") p.add_argument("--rate", type=float, default=DEFAULT_RATE, help="REST reads per second (default 1)") + p.add_argument("--rate-schedule", type=_schedule, default=None, metavar="MIN:RATE,...", + help="rate phases, e.g. \"60:1,120:5\" (60 min at 1/s, then 5/s; the last phase lasts); " + "overrides --rate") p.add_argument("--list-share", type=_share, default=DEFAULT_LIST_SHARE, help="share of the rate kept for discovery and list reads while polling leads (default 0.5)") p.add_argument("--joined-ids", type=_joined_file, default=[], metavar="FILE", @@ -2141,18 +2421,23 @@ def main(argv: Optional[List[str]] = None) -> int: if args.compare_credential_sports and args.source != "direct": print("error: --compare-credential-sports needs --source direct", file=sys.stderr) return 2 - budget = args.max_requests if args.max_requests is not None else default_budget(args.hours, args.rate) + schedule = args.rate_schedule + budget = args.max_requests if args.max_requests is not None else default_budget(args.hours, args.rate, schedule) + if schedule is not None and schedule.max_rate > 5.0 and "SOFASCORE_CLIENT__RATE" not in os.environ: + # uygulamanın ortak bütçesi (client.rate, varsayılan 5/sn) daha hızlı bir evreyi boğmasın + os.environ["SOFASCORE_CLIENT__RATE"] = f"{schedule.max_rate:g}" os.environ.setdefault("SOFASCORE_CLIENT__THROTTLE_DIR", THROTTLE_DIR) os.environ["SOFASCORE_STORAGE__DATA_DIR"] = data_dir os.makedirs(data_dir, exist_ok=True) note = {"source": args.source, "hours": args.hours, "max_requests": budget, "rate": args.rate, "list_share": args.list_share, "joined_ids": len(args.joined_ids), + "rate_schedule": None if schedule is None else [list(p) for p in schedule.phases], "joined_random_soccer": args.joined_random_soccer, "kill_every_min": args.kill_every, "kill_at_min": args.kill_at, "compare_credential_sports": args.compare_credential_sports, "poll_interval_s": args.poll_interval} soak = Soak(out_dir, args.sports, run_id=run_id, data_dir=data_dir, budget=budget, source=args.source, hours=args.hours, kill_every=args.kill_every, kill_at=args.kill_at, poll_interval=args.poll_interval, - rate=args.rate, list_share=args.list_share, joined_ids=args.joined_ids, + rate=args.rate, list_share=args.list_share, rate_schedule=schedule, joined_ids=args.joined_ids, joined_random=args.joined_random_soccer, compare_sports=args.compare_credential_sports, args_note=note) diff --git a/tests/test_soak_live.py b/tests/test_soak_live.py index 33c4315b..3654d1b8 100644 --- a/tests/test_soak_live.py +++ b/tests/test_soak_live.py @@ -19,6 +19,7 @@ from __future__ import annotations import json +import logging import os import sys import time @@ -539,8 +540,8 @@ def secret_keys(o: Any) -> List[str]: def assert_private(run: Path, needles: Any = ()) -> None: texts = {p.name: p.read_text(encoding="utf-8") for p in sorted(run.iterdir()) if p.is_file()} - assert set(texts) == {"frames.jsonl", "connection.jsonl", "leader.jsonl", "reads.jsonl", "fallback.jsonl", - "resources.jsonl", "summary.json"} + assert set(texts) == {"frames.jsonl", "connection.jsonl", "leader.jsonl", "reads.jsonl", "requests.jsonl", + "fallback.jsonl", "resources.jsonl", "summary.json"} for name, text in texts.items(): # IPv4/IPv6 yok: yerel ve belgeleme aralıkları da (izin verilen aralık olmadan) assert privacy.ip_literals(text, allowed=()) == [], name @@ -677,8 +678,10 @@ def test_the_options_parse() -> None: assert args.compare_credential_sports and args.max_requests is None and args.hours == soak_live.DEFAULT_HOURS assert args.joined_ids == [] and args.joined_random_soccer == 5 and args.rate == 0.8 and args.list_share == 0.25 assert args.source == "direct" and soak_live._parser().parse_args([]).joined_random_soccer == 20 + schedule = soak_live._parser().parse_args(["--rate-schedule", "60:1, 120:5"]).rate_schedule + assert schedule.phases == [(60.0, 1.0), (120.0, 5.0)] for bad in (["--sports", "foot ball"], ["--list-share", "1"], ["--source", "auto"], - ["--joined-ids", "/nonexistent/ids.txt"]): + ["--rate-schedule", "60"], ["--rate-schedule", "60:0"], ["--rate-schedule", "x:1"]): with pytest.raises(SystemExit): soak_live._parser().parse_args(bad) @@ -686,10 +689,90 @@ def test_the_options_parse() -> None: assert conftest # sınır denetimi ve kancalar conftest'te kurulur -def test_the_joined_ids_file(tmp_path: Path) -> None: - path = tmp_path / "ids.txt" - path.write_text("# c fixtures\n17124861, 16947598\n17124861 15453137 # again\n", encoding="utf-8") - assert soak_live._parser().parse_args(["--joined-ids", str(path)]).joined_ids == [17124861, 16947598, 15453137] - path.write_text("12 abc\n", encoding="utf-8") - with pytest.raises(SystemExit): - soak_live._parser().parse_args(["--joined-ids", str(path)]) +def test_the_joined_ids_file_uses_the_c_message_shape(tmp_path: Path, capsys: pytest.CaptureFixture) -> None: + path = tmp_path / "joined.json" + path.write_text(json.dumps({"ids": [17124861, 16947598, 17124861]}), encoding="utf-8") + assert soak_live._parser().parse_args(["--joined-ids", str(path)]).joined_ids == [17124861, 16947598] + # bozuk dosya ya da 50.000'den çok id: c gibi yok sayılır (hata değil, uyarı) + for bad in ("17124861 16947598", json.dumps([1, 2]), json.dumps({"ids": "1"}), json.dumps({"ids": [1, "2"]}), + json.dumps({"ids": [True]}), json.dumps({"ids": list(range(1, 50_002))})): + path.write_text(bad, encoding="utf-8") + assert soak_live._parser().parse_args(["--joined-ids", str(path)]).joined_ids == [] + assert soak_live._parser().parse_args(["--joined-ids", str(tmp_path / "missing.json")]).joined_ids == [] + assert soak_live.parse_joined(json.dumps({"ids": list(range(1, 50_001))})) is not None + assert "ignoring it" in capsys.readouterr().err + + +# --- hız evreleri ve dakikalık istek satırları --------------------------------------------------------------------- + + +def test_the_rate_schedule() -> None: + sched = soak_live.RateSchedule.parse("60:1,120:5") + assert sched.bounds() == [(0.0, 3600.0, 1.0), (3600.0, float("inf"), 5.0)] + assert [sched.rate_at(t) for t in (0, 3599, 3600, 10 ** 6)] == [1.0, 1.0, 5.0, 5.0] + assert sched.phase_at(3600) == 1 and sched.max_rate == 5.0 + assert sched.requests_in(3 * 3600) == 3600 * 1 + 7200 * 5 # son evre koşu bitene kadar sürer + assert soak_live.default_budget(3, schedule=sched) == 39600 + assert soak_live.RateSchedule((), 0.5).rate_at(10 ** 6) == 0.5 + + +def test_peak_and_percentiles() -> None: + assert soak_live.peak_in_window([0.0, 0.2, 0.9, 1.0, 1.5, 3.0]) == 3 + assert soak_live.peak_in_window([]) == 0 + assert soak_live.percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.5) == 5 + assert soak_live.percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.95) == 10 and soak_live.percentile([], 0.5) is None + + +def test_the_challenge_counter_reads_the_client_s_own_log_lines() -> None: + counter = soak_live.ChallengeCounter(mono=lambda: 5.0) + counter.install() + try: + utils, solver = logging.getLogger("Utils"), logging.getLogger("ChallengeSolver") + utils.warning("Rate limited or server busy (429); waiting 5 s") + utils.warning("403 Forbidden (attempt 1/3): https://api.example.invalid/x") + utils.info("Turnstile challenge detected; fetching through the browser bridge") + solver.info("The API answered 403 with a challenge; solving Turnstile") + solver.info("Solving the Cloudflare Turnstile challenge (captcha.html)") + solver.info("Turnstile challenge solved.") + solver.warning("Browser fetch failed (status 403): nope") + utils.info("an unrelated line") + finally: + counter.uninstall() + summary_ = soak_live.challenge_summary(counter.between(0.0, 10.0)) + assert summary_["429"] == 1 and summary_["403"] == {"curl": 1, "curl_challenge": 1, "browser_challenge": 1, + "browser": 1} + assert summary_["turnstile"] == {"attempts": 1, "solved": 1, "failed": 0} + assert logging.getLogger("Utils").propagate and logging.getLogger("Utils").level == logging.NOTSET + + +def test_requests_are_recorded_per_minute_and_per_rate_phase( + tmp_path: Path, data_dir: Path, fast_service: None, soaks: Any) -> None: + matches = {eid: first_half(eid) for eid in (601, 602)} + api = FakeApi({"football": list(matches.values())}, dict(matches)) + # "dakika" 0,5 sn; ilk evre 0,9 sn 2/sn, sonra 50/sn + sched = soak_live.RateSchedule([(0.015, 2.0), (1.0, 50.0)]) + soak, run = soaks(tmp_path, data_dir, None, api, reader=FakeReader(), rate_schedule=sched, minute_seconds=0.5, + fallback_window=0.5, list_share=0.5, poll_interval=1.0) + soak.start() + logging.getLogger("Utils").info("Turnstile challenge detected; fetching through the browser bridge") + logging.getLogger("ChallengeSolver").info("Turnstile challenge solved.") + wait_for(lambda: len([r for r in rows(run, "requests") if r["phase"] == 1]) >= 2, seconds=10) + soak.stop() + + minutes = rows(run, "requests") + assert [m["minute"] for m in minutes] == list(range(len(minutes))) + assert minutes[0]["phase"] == 0 and minutes[0]["rate"] == 2.0 and minutes[-1]["rate"] == 50.0 + assert minutes[0]["client"]["403"]["curl_challenge"] == 1 and minutes[0]["client"]["turnstile"]["solved"] == 1 + for m in minutes: + assert m["total"] == sum(m["by_kind"].values()) == sum(m["by_who"].values()) + assert m["peak_1s"] <= m["total"] and set(m["latency_s"]) == {"p50", "p95"} + assert sum(m["total"] for m in minutes) == soak.client.used + assert any(m["by_who"].get("fallback") for m in minutes) + s = summary(run) + phases = s["rate_phases"] + assert [p["rate"] for p in phases] == [2.0, 50.0] and phases[0]["end_min"] == pytest.approx(0.015) + assert sum(p["total"] for p in phases) == soak.client.used and phases[0]["client"]["turnstile"]["solved"] == 1 + assert phases[1]["achieved_per_second"] > 0 and "peak_1s" in phases[1] + + +assert conftest # sınır denetimi ve kancalar conftest'te kurulur