""" DeepSearch configuration. Everything is tunable via environment variables (see .env.example). The engine runs 100% keyless out of the box; adding API keys silently upgrades the provider pool to premium sources. """ from __future__ import annotations import os from dataclasses import dataclass, field from typing import Dict, List try: # optional .env support from dotenv import load_dotenv # type: ignore load_dotenv() except Exception: # pragma: no cover pass def _env(key: str, default: str = "") -> str: return os.getenv(key, default).strip() def _env_int(key: str, default: int) -> int: try: return int(os.getenv(key, str(default))) except (TypeError, ValueError): return default def _env_float(key: str, default: float) -> float: try: return float(os.getenv(key, str(default))) except (TypeError, ValueError): return default def _env_bool(key: str, default: bool) -> bool: v = os.getenv(key) if v is None: return default return v.strip().lower() in {"1", "true", "yes", "on"} # -------------------------------------------------------------------------- # Depth presets: the single knob that trades latency for thoroughness. # -------------------------------------------------------------------------- @dataclass(frozen=True) class DepthProfile: name: str sub_queries: int # how many parallel query variants to plan results_per_provider: int # raw SERP rows pulled per provider per query pages_to_read: int # full pages fetched + extracted max_chunks: int # evidence chunks fed to the synthesizer llm_rerank: bool # use the LLM as a cross-encoder-ish reranker planner: bool # use the LLM to plan sub-queries budget_seconds: float # soft wall-clock budget for retrieval DEPTH_PROFILES: Dict[str, DepthProfile] = { # Lightning: single-shot, no LLM planning. Typically ~1-3s. "instant": DepthProfile("instant", 1, 8, 0, 8, False, False, 6.0), # Balanced default: planned fan-out + page reading. ~4-9s. "fast": DepthProfile("fast", 3, 10, 5, 14, False, True, 12.0), # Deep: wide fan-out, LLM reranking, heavy reading. ~10-25s. "deep": DepthProfile("deep", 6, 12, 12, 26, True, True, 30.0), # Extreme: maximum recall, multi-hop follow-up research. ~25-70s. "extreme": DepthProfile("extreme", 10, 14, 20, 40, True, True, 75.0), # Ultra: TeCoxBeta maximum. Rewind's 2M-context models + parallel # subagents make this tier possible - it reads far more of the web than # any single-context engine can hold. "ultra": DepthProfile("ultra", 16, 16, 36, 90, True, True, 150.0), } DEFAULT_DEPTH = _env("DEEPSEARCH_DEPTH", "fast") @dataclass class Settings: # ---- server ----------------------------------------------------------- host: str = _env("HOST", "0.0.0.0") port: int = _env_int("PORT", 8000) # ---- networking ------------------------------------------------------- http_timeout: float = _env_float("HTTP_TIMEOUT", 12.0) provider_timeout: float = _env_float("PROVIDER_TIMEOUT", 8.0) scrape_timeout: float = _env_float("SCRAPE_TIMEOUT", 9.0) max_connections: int = _env_int("MAX_CONNECTIONS", 160) max_keepalive: int = _env_int("MAX_KEEPALIVE", 60) search_concurrency: int = _env_int("SEARCH_CONCURRENCY", 32) scrape_concurrency: int = _env_int("SCRAPE_CONCURRENCY", 20) max_page_bytes: int = _env_int("MAX_PAGE_BYTES", 2_500_000) user_agent: str = _env( "USER_AGENT", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", ) # ---- cache ------------------------------------------------------------ cache_enabled: bool = _env_bool("CACHE_ENABLED", True) serp_cache_ttl: int = _env_int("SERP_CACHE_TTL", 600) # 10 min page_cache_ttl: int = _env_int("PAGE_CACHE_TTL", 3600) # 1 hour answer_cache_ttl: int = _env_int("ANSWER_CACHE_TTL", 300) # 5 min cache_max_items: int = _env_int("CACHE_MAX_ITEMS", 4096) # ---- models (the synthesis layer) -------------------------------------------------- model_fast: str = _env("MODEL_FAST", "google/gemini-2.5-flash-lite") model_balanced: str = _env("MODEL_BALANCED", "google/gemini-2.5-flash") model_power: str = _env("MODEL_POWER", "google/gemini-2.5-pro") llm_timeout: float = _env_float("LLM_TIMEOUT", 90.0) llm_retries: int = _env_int("LLM_RETRIES", 2) # Rewind accepts enormous prompts (verified 300k chars in ~5s). This is # the per-context budget for the LEAD agent; subagents get their own # isolated windows, so total evidence processed far exceeds this. max_prompt_chars: int = _env_int("MAX_PROMPT_CHARS", 120000) map_reduce_workers: int = _env_int("MAP_REDUCE_WORKERS", 8) # ---- ranking weights -------------------------------------------------- w_rrf: float = _env_float("W_RRF", 1.00) w_bm25: float = _env_float("W_BM25", 0.85) w_authority: float = _env_float("W_AUTHORITY", 0.55) w_freshness: float = _env_float("W_FRESHNESS", 0.35) w_agreement: float = _env_float("W_AGREEMENT", 0.60) w_llm: float = _env_float("W_LLM", 1.30) rrf_k: int = _env_int("RRF_K", 60) # ---- api keys (all optional) ----------------------------------------- tavily_key: str = _env("TAVILY_API_KEY") exa_key: str = _env("EXA_API_KEY") brave_key: str = _env("BRAVE_API_KEY") serper_key: str = _env("SERPER_API_KEY") firecrawl_key: str = _env("FIRECRAWL_API_KEY") # ---- misc ------------------------------------------------------------- api_key: str = _env("DEEPSEARCH_API_KEY") # optional auth for your API # 0 = unlimited. No per-user caps by default. rate_limit_per_min: int = _env_int("RATE_LIMIT_PER_MIN", 0) searx_instances: List[str] = field( default_factory=lambda: [ s.strip() for s in _env( "SEARX_INSTANCES", "https://opnxng.com,https://searxng.site,https://searx.be,https://search.inetol.net,https://priv.au,https://baresearch.org", ).split(",") if s.strip() ] ) def has_premium(self) -> bool: return any([self.tavily_key, self.exa_key, self.brave_key, self.serper_key]) settings = Settings() # -------------------------------------------------------------------------- # Domain authority priors. Not exhaustive - a heuristic prior that gets # blended with live signals, never the sole decider. # -------------------------------------------------------------------------- AUTHORITY: Dict[str, float] = { # reference "wikipedia.org": 0.90, "britannica.com": 0.86, "nature.com": 0.97, "science.org": 0.96, "sciencedirect.com": 0.92, "springer.com": 0.90, "arxiv.org": 0.90, "pubmed.ncbi.nlm.nih.gov": 0.95, "ncbi.nlm.nih.gov": 0.93, "acm.org": 0.90, "ieee.org": 0.92, "jstor.org": 0.88, "plos.org": 0.88, "doi.org": 0.85, "openalex.org": 0.80, "semanticscholar.org": 0.85, # docs / dev "docs.python.org": 0.95, "developer.mozilla.org": 0.94, "github.com": 0.88, "stackoverflow.com": 0.86, "kubernetes.io": 0.90, "pytorch.org": 0.91, "tensorflow.org": 0.90, "huggingface.co": 0.88, "docs.rust-lang.org": 0.92, "go.dev": 0.90, "postgresql.org": 0.92, "nginx.org": 0.88, "fastapi.tiangolo.com": 0.90, "readthedocs.io": 0.82, "npmjs.com": 0.80, "pypi.org": 0.84, "gitlab.com": 0.80, "cloud.google.com": 0.88, "learn.microsoft.com": 0.89, "docs.aws.amazon.com": 0.89, # news / press "reuters.com": 0.93, "apnews.com": 0.93, "bbc.com": 0.90, "bbc.co.uk": 0.90, "ft.com": 0.89, "economist.com": 0.89, "wsj.com": 0.89, "bloomberg.com": 0.89, "nytimes.com": 0.88, "theguardian.com": 0.86, "npr.org": 0.86, "aljazeera.com": 0.82, "cnbc.com": 0.82, "theverge.com": 0.78, "arstechnica.com": 0.84, "techcrunch.com": 0.76, "wired.com": 0.79, "thehindu.com": 0.84, "indianexpress.com": 0.82, "livemint.com": 0.80, "economictimes.indiatimes.com": 0.79, "business-standard.com": 0.80, # official "who.int": 0.95, "cdc.gov": 0.94, "nih.gov": 0.94, "nasa.gov": 0.95, "europa.eu": 0.90, "un.org": 0.90, "imf.org": 0.90, "worldbank.org": 0.90, "oecd.org": 0.89, "gov.uk": 0.90, "rbi.org.in": 0.92, "sebi.gov.in": 0.90, "irs.gov": 0.92, "sec.gov": 0.93, "nist.gov": 0.93, "ietf.org": 0.92, # community "news.ycombinator.com": 0.72, "reddit.com": 0.62, "medium.com": 0.52, "quora.com": 0.42, "substack.com": 0.55, "dev.to": 0.58, } # Domains that are almost never useful as primary evidence. DOMAIN_PENALTIES: Dict[str, float] = { "pinterest.com": -0.55, "facebook.com": -0.35, "instagram.com": -0.35, "tiktok.com": -0.30, "x.com": -0.10, "twitter.com": -0.10, "answers.yahoo.com": -0.40, "ask.com": -0.35, "coursehero.com": -0.45, "scribd.com": -0.40, "slideshare.net": -0.25, "issuu.com": -0.35, "w3schools.com": -0.10, "geeksforgeeks.org": -0.05, } # Never worth fetching/reading. BLOCKED_PATTERNS = ( "/login", "/signin", "/signup", "/register", "/cart", "/checkout", "javascript:", "mailto:", "/privacy-policy", "/terms-of-service", "accounts.google.com", "doubleclick.net", "googleadservices", ) BINARY_EXTENSIONS = ( ".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".ico", ".mp4", ".mp3", ".avi", ".mov", ".zip", ".tar", ".gz", ".rar", ".7z", ".exe", ".dmg", ".iso", ".woff", ".woff2", ".ttf", ".css", ".js", ".xml", ".rss", )