agent.py

#!/usr/bin/env python3
"""
Scout OSS

Mission-driven web scout that:
- plans a run
- explores with tool calls until budget is exhausted
- writes a structured markdown brief

It keeps the core architecture simple: plan, explore, compose, save.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import subprocess
import time
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib import error, request

import anthropic
import httpx
from dotenv import load_dotenv


BASE_DIR = Path(__file__).parent
load_dotenv(BASE_DIR / ".env", override=False)

ADAPTERS_DIR = BASE_DIR / "adapters"
OUTPUT_DIR = Path(os.getenv("SCOUT_OUTPUT_DIR", str(BASE_DIR / "inbox"))).expanduser()
MEMORY_FILE = Path(os.getenv("SCOUT_MEMORY_FILE", str(BASE_DIR / "MEMORY.md"))).expanduser()
STATE_FILE = BASE_DIR / "STATE.json"
CONTEXT_DIR = Path(os.getenv("SCOUT_CONTEXT_DIR", str(BASE_DIR / "examples" / "sample-context"))).expanduser()
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
CONTEXT_DIR.mkdir(parents=True, exist_ok=True)

MODEL = os.getenv("SCOUT_MODEL", os.getenv("MODEL", "gpt-5.4"))
ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "")
AWS_BEARER_TOKEN = os.getenv("AWS_BEARER_TOKEN_BEDROCK", "")
AWS_REGION = os.getenv("AWS_REGION", "us-east-1")
LOOP_INTERVAL_MINUTES = int(os.getenv("SCOUT_LOOP_INTERVAL_MINUTES", os.getenv("LOOP_INTERVAL_MINUTES", "30")))
CODEX_HOME = Path(os.getenv("CODEX_HOME", str(Path.home() / ".codex"))).expanduser()
CODEX_AUTH_FILE = CODEX_HOME / "auth.json"
CHATGPT_URL = "https://chatgpt.com/backend-api/codex/responses"

DEFAULT_SCAN_PLAN = {
    "mission_type": "general_scan",
    "goal": "Run a broad autonomous scan and follow the strongest leads.",
    "search_style": "broad_then_deep",
    "priority_order": ["recent signal", "official sources", "non-obvious findings", "actionable opportunities"],
    "source_include": ["hn_front", "exa_search", "tavily_search", "reddit_search"],
    "source_avoid": [],
    "followup_rules": [
        "After finding a promising lead, read the original source page.",
        "Prefer official pages over summaries when verifying facts.",
        "Use note whenever something is worth reporting.",
    ],
    "minimum_tool_calls": 4,
    "compose_requirements": [
        "Be concrete and include URLs.",
        "Separate verified facts from interpretation.",
    ],
}

SCOUT_TOOLS = [
    {
        "name": "hn_front",
        "description": "Get Hacker News front page.",
        "input_schema": {"type": "object", "properties": {"n": {"type": "integer"}}, "required": []},
    },
    {
        "name": "hn_search",
        "description": "Search Hacker News posts by keyword.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "n": {"type": "integer"},
                "points": {"type": "integer"},
                "date": {"type": "boolean"},
            },
            "required": ["query"],
        },
    },
    {
        "name": "hn_item",
        "description": "Get a specific HN post and comments by ID.",
        "input_schema": {"type": "object", "properties": {"item_id": {"type": "string"}}, "required": ["item_id"]},
    },
    {
        "name": "github_trending",
        "description": "Get trending GitHub repositories.",
        "input_schema": {
            "type": "object",
            "properties": {
                "period": {"type": "string", "enum": ["day", "week", "month"]},
                "language": {"type": "string"},
                "n": {"type": "integer"},
            },
            "required": [],
        },
    },
    {
        "name": "exa_search",
        "description": "Semantic web search via Exa.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "n": {"type": "integer"},
                "category": {"type": "string"},
                "domains": {"type": "array", "items": {"type": "string"}},
                "search_type": {"type": "string", "enum": ["auto", "neural", "keyword"]},
            },
            "required": ["query"],
        },
    },
    {
        "name": "exa_research",
        "description": "Deep autonomous research via Exa.",
        "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
    },
    {
        "name": "exa_similar",
        "description": "Find pages similar to a URL.",
        "input_schema": {
            "type": "object",
            "properties": {"url": {"type": "string"}, "n": {"type": "integer"}},
            "required": ["url"],
        },
    },
    {
        "name": "exa_answer",
        "description": "Get a direct answer with web sources via Exa.",
        "input_schema": {"type": "object", "properties": {"question": {"type": "string"}}, "required": ["question"]},
    },
    {
        "name": "arxiv_search",
        "description": "Search arXiv for papers.",
        "input_schema": {
            "type": "object",
            "properties": {"query": {"type": "string"}, "n": {"type": "integer"}, "cat": {"type": "string"}},
            "required": ["query"],
        },
    },
    {
        "name": "lobsters",
        "description": "Browse Lobste.rs.",
        "input_schema": {
            "type": "object",
            "properties": {
                "command": {"type": "string", "enum": ["hot", "new", "tag"]},
                "tag": {"type": "string"},
                "n": {"type": "integer"},
            },
            "required": ["command"],
        },
    },
    {
        "name": "x_search",
        "description": "Search X/Twitter for recent discussions.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "sort": {"type": "string", "enum": ["likes", "recent", "impressions", "retweets"]},
                "since": {"type": "string"},
                "limit": {"type": "integer"},
            },
            "required": ["query"],
        },
    },
    {
        "name": "reddit_search",
        "description": "Search Reddit discussions.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "n": {"type": "integer"},
                "sort": {"type": "string", "enum": ["relevance", "new", "top", "comments"]},
                "subreddit": {"type": "string"},
            },
            "required": ["query"],
        },
    },
    {
        "name": "reddit_subreddit",
        "description": "Browse a subreddit directly.",
        "input_schema": {
            "type": "object",
            "properties": {
                "subreddit": {"type": "string"},
                "n": {"type": "integer"},
                "sort": {"type": "string", "enum": ["hot", "new", "top"]},
            },
            "required": ["subreddit"],
        },
    },
    {
        "name": "reddit_thread",
        "description": "Read a Reddit thread and top comments.",
        "input_schema": {
            "type": "object",
            "properties": {"url": {"type": "string"}, "comments": {"type": "integer"}},
            "required": ["url"],
        },
    },
    {
        "name": "youtube_search",
        "description": "Search YouTube videos.",
        "input_schema": {"type": "object", "properties": {"query": {"type": "string"}, "n": {"type": "integer"}}, "required": ["query"]},
    },
    {
        "name": "youtube_video",
        "description": "Get metadata for a YouTube video.",
        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
    },
    {
        "name": "youtube_transcript",
        "description": "Extract a YouTube transcript.",
        "input_schema": {
            "type": "object",
            "properties": {"url": {"type": "string"}, "lang": {"type": "string"}},
            "required": ["url"],
        },
    },
    {
        "name": "linkedin_read",
        "description": "Read a public LinkedIn page via adapter.",
        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
    },
    {
        "name": "tavily_search",
        "description": "Fast web search via Tavily.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "n": {"type": "integer"},
                "depth": {"type": "string", "enum": ["basic", "advanced"]},
                "time": {"type": "string", "enum": ["day", "week", "month", "year"]},
                "domains": {"type": "array", "items": {"type": "string"}},
            },
            "required": ["query"],
        },
    },
    {
        "name": "tavily_research",
        "description": "AI-synthesized multi-source research via Tavily.",
        "input_schema": {
            "type": "object",
            "properties": {"topic": {"type": "string"}, "model": {"type": "string", "enum": ["mini", "pro"]}},
            "required": ["topic"],
        },
    },
    {
        "name": "gemini_search",
        "description": "Google-grounded search via Gemini.",
        "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
    },
    {
        "name": "web_fetch",
        "description": "Fetch and read full text of a URL.",
        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
    },
    {
        "name": "jina_read",
        "description": "Fetch clean markdown from a URL via adapter.",
        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
    },
    {
        "name": "note",
        "description": "Save a finding for the final brief.",
        "input_schema": {
            "type": "object",
            "properties": {
                "category": {
                    "type": "string",
                    "enum": ["direct_hit", "worth_attention", "surprising", "opportunity", "research_queue", "person"],
                },
                "title": {"type": "string"},
                "content": {"type": "string"},
            },
            "required": ["category", "title", "content"],
        },
    },
]

KEEP_ROUNDS = 4
KNOWN_HEADER = "## Scout Known Items"
KNOWN_END = "<!-- scout-known-end -->"


def _escape(value: str) -> str:
    return value.replace('"', '\\"')


def _adapter_script(*parts: str) -> Path:
    return ADAPTERS_DIR.joinpath(*parts)


def _adapter_cmd(name: str, args: dict) -> str | None:
    if name == "hn_front":
        script = _adapter_script("hn", "scripts", "hn.py")
        return f'python3 "{script}" front -n {args.get("n", 25)}'
    if name == "hn_search":
        script = _adapter_script("hn", "scripts", "hn.py")
        pts = f'--points {args["points"]}' if args.get("points") else ""
        dt = "--date" if args.get("date") else ""
        return f'python3 "{script}" search "{_escape(args["query"])}" -n {args.get("n", 10)} {pts} {dt}'
    if name == "hn_item":
        script = _adapter_script("hn", "scripts", "hn.py")
        return f'python3 "{script}" item {args["item_id"]}'
    if name == "github_trending":
        script = _adapter_script("github", "scripts", "githunt.py")
        lang = f'-l {args["language"]}' if args.get("language") else ""
        return f'python3 "{script}" -p {args.get("period", "day")} -n {args.get("n", 15)} {lang}'
    if name == "exa_search":
        script = _adapter_script("exa", "scripts", "exa.py")
        cat = f'--category "{args["category"]}"' if args.get("category") else ""
        st = f'--type {args["search_type"]}' if args.get("search_type") else ""
        domains = ("--include-domains " + " ".join(args["domains"])) if args.get("domains") else ""
        return f'python3 "{script}" search "{_escape(args["query"])}" -n {args.get("n", 8)} {cat} {st} {domains}'
    if name == "exa_research":
        script = _adapter_script("exa", "scripts", "exa.py")
        return f'python3 "{script}" research "{_escape(args["query"])}"'
    if name == "exa_similar":
        script = _adapter_script("exa", "scripts", "exa.py")
        return f'python3 "{script}" similar "{_escape(args["url"])}" -n {args.get("n", 6)}'
    if name == "exa_answer":
        script = _adapter_script("exa", "scripts", "exa.py")
        return f'python3 "{script}" answer "{_escape(args["question"])}"'
    if name == "arxiv_search":
        script = _adapter_script("arxiv", "scripts", "arxiv_search.py")
        cat = f'--cat {args["cat"]}' if args.get("cat") else ""
        return f'python3 "{script}" "{_escape(args["query"])}" -n {args.get("n", 5)} {cat}'
    if name == "x_search":
        script = _adapter_script("x", "scripts", "x_search.py")
        return (
            f'python3 "{script}" "{_escape(args["query"])}" '
            f'--sort {args.get("sort", "likes")} --since {args.get("since", "7d")} --limit {args.get("limit", 15)}'
        )
    if name == "reddit_search":
        script = _adapter_script("reddit", "scripts", "reddit.py")
        subreddit = f' --subreddit {args["subreddit"]}' if args.get("subreddit") else ""
        return f'python3 "{script}" search "{_escape(args["query"])}" -n {args.get("n", 5)} --sort {args.get("sort", "relevance")}{subreddit}'
    if name == "reddit_subreddit":
        script = _adapter_script("reddit", "scripts", "reddit.py")
        return f'python3 "{script}" subreddit "{args["subreddit"]}" -n {args.get("n", 5)} --sort {args.get("sort", "hot")}'
    if name == "reddit_thread":
        script = _adapter_script("reddit", "scripts", "reddit.py")
        return f'python3 "{script}" thread "{args["url"]}" --comments {args.get("comments", 6)}'
    if name == "youtube_search":
        script = _adapter_script("youtube", "scripts", "youtube.py")
        return f'python3 "{script}" search "{_escape(args["query"])}" -n {args.get("n", 5)}'
    if name == "youtube_video":
        script = _adapter_script("youtube", "scripts", "youtube.py")
        return f'python3 "{script}" video "{args["url"]}"'
    if name == "youtube_transcript":
        script = _adapter_script("youtube", "scripts", "youtube.py")
        return f'python3 "{script}" transcript "{args["url"]}" --lang "{args.get("lang", "en.*")}"'
    if name == "linkedin_read":
        script = _adapter_script("linkedin", "scripts", "linkedin.py")
        return f'python3 "{script}" "{args["url"]}"'
    if name == "lobsters":
        script = _adapter_script("lobsters", "scripts", "lobsters.py")
        cmd = args.get("command", "hot")
        n = args.get("n", 15)
        if cmd == "tag" and args.get("tag"):
            return f'python3 "{script}" tag "{args["tag"]}" -n {n}'
        return f'python3 "{script}" {cmd} -n {n}'
    if name == "tavily_search":
        script = _adapter_script("tavily", "scripts", "tavily_search.py")
        domains = ("--domains " + " ".join(args["domains"])) if args.get("domains") else ""
        time_flag = f'--time {args["time"]}' if args.get("time") else ""
        return f'python3 "{script}" "{_escape(args["query"])}" -n {args.get("n", 8)} --depth {args.get("depth", "basic")} {time_flag} {domains}'
    if name == "tavily_research":
        script = _adapter_script("tavily", "scripts", "tavily_research.py")
        return f'python3 "{script}" "{_escape(args["topic"])}" --model {args.get("model", "mini")}'
    if name == "gemini_search":
        script = _adapter_script("gemini", "scripts", "gemini_search.py")
        return f'python3 "{script}" "{_escape(args["query"])}"'
    if name == "jina_read":
        script = _adapter_script("jina", "scripts", "jina_read.py")
        return f'python3 "{script}" "{args["url"]}"'
    return None


def _run_tool(name: str, args: dict, notes: list[dict[str, str]]) -> str:
    max_chars = 8000
    try:
        if name == "web_fetch":
            return _web_fetch(args["url"])
        if name == "note":
            notes.append({"category": args["category"], "title": args["title"], "content": args["content"]})
            return f"Noted [{args['category']}]: {args['title']}"

        cmd = _adapter_cmd(name, args)
        if cmd is None:
            return f"Unknown tool: {name}"

        script_match = re.search(r'"([^"]+\.py)"', cmd)
        if script_match and not Path(script_match.group(1)).exists():
            return f"Adapter unavailable for {name}: missing {script_match.group(1)}"

        timeout = 120 if name in ("tavily_research", "gemini_search", "youtube_video", "youtube_transcript") else 60
        result = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout, cwd=str(BASE_DIR))
        output = (result.stdout + result.stderr).strip() or "(no output)"
        return output[:max_chars]
    except subprocess.TimeoutExpired:
        return f"(timeout - {name})"
    except Exception as exc:
        return f"(error in {name}: {exc})"


def _bedrock_model_id(model: str) -> str:
    if model.startswith("us.") or model.startswith("anthropic.") or ":" in model:
        return model
    mapping = {
        "claude-haiku-4-5-20251001": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
        "claude-sonnet-4-5-20250929": "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
        "claude-sonnet-4-6": "us.anthropic.claude-sonnet-4-6",
        "claude-opus-4-5": "us.anthropic.claude-opus-4-5",
        "sonnet": "us.anthropic.claude-sonnet-4-6",
        "haiku": "us.anthropic.claude-haiku-4-5-20251001-v1:0",
        "opus": "us.anthropic.claude-opus-4-5",
    }
    return mapping.get(model, f"us.anthropic.{model}-v1:0")


def _anthropic_tools_to_bedrock(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
    return [{"toolSpec": {"name": t["name"], "description": t.get("description", ""), "inputSchema": {"json": t["input_schema"]}}} for t in tools]


def _anthropic_messages_to_bedrock(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
    out = []
    for msg in messages:
        role = msg["role"]
        content = msg["content"]
        if isinstance(content, str):
            out.append({"role": role, "content": [{"text": content}]})
            continue
        blocks = []
        for block in content:
            if block.get("type") == "text":
                blocks.append({"text": block["text"]})
            elif block.get("type") == "tool_use":
                blocks.append({"toolUse": {"toolUseId": block["id"], "name": block["name"], "input": block["input"]}})
            elif block.get("type") == "tool_result":
                blocks.append({"toolResult": {"toolUseId": block["tool_use_id"], "content": [{"text": str(block["content"])}]}})
        if blocks:
            out.append({"role": role, "content": blocks})
    return out


def _load_codex_auth() -> tuple[str, str | None] | None:
    if not CODEX_AUTH_FILE.exists():
        return None
    try:
        data = json.loads(CODEX_AUTH_FILE.read_text(encoding="utf-8"))
    except Exception:
        return None
    tokens = data.get("tokens") or {}
    access = tokens.get("access_token")
    if not isinstance(access, str) or not access:
        return None
    account = tokens.get("account_id")
    if not isinstance(account, str) or not account:
        account = None
    return access, account


def _anthropic_tools_to_openai(tools: list[dict[str, Any]]) -> list[dict[str, Any]]:
    out = []
    for tool in tools:
        schema = dict(tool["input_schema"])
        schema.setdefault("additionalProperties", False)
        out.append({
            "type": "function",
            "name": tool["name"],
            "description": tool.get("description", ""),
            "parameters": schema,
        })
    return out


def _messages_to_openai_input(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
    out = []
    for msg in messages:
        role = msg["role"]
        content = msg["content"]
        if isinstance(content, str):
            out.append({"role": role, "content": [{"type": "input_text" if role == "user" else "output_text", "text": content}]})
            continue
        text_parts = []
        for block in content:
            if block.get("type") == "text":
                text_parts.append(block["text"])
            elif block.get("type") == "tool_use":
                out.append({"type": "function_call", "call_id": block["id"], "name": block["name"], "arguments": json.dumps(block["input"])})
            elif block.get("type") == "tool_result":
                out.append({"type": "function_call_output", "call_id": block["tool_use_id"], "output": str(block["content"])})
        if text_parts:
            out.append({"role": role, "content": [{"type": "input_text" if role == "user" else "output_text", "text": "\n".join(text_parts)}]})
    return out


def _openai_call(messages: list[dict[str, Any]], system: str, tools: list[dict[str, Any]], model: str, max_output_tokens: int = 4096) -> dict[str, Any]:
    auth = _load_codex_auth()
    if not auth:
        raise RuntimeError("Codex auth not found. Run `codex login --device-auth` first.")
    access, account = auth
    body: dict[str, Any] = {
        "model": model,
        "input": _messages_to_openai_input(messages),
        "instructions": system,
        "stream": True,
        "store": False,
        "reasoning": {"effort": "low"},
    }
    if tools:
        body["tools"] = _anthropic_tools_to_openai(tools)
        body["tool_choice"] = "auto"
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {access}",
        "User-Agent": "scout-oss/0.1",
        "originator": "scout-oss",
        "Accept": "text/event-stream",
    }
    if account:
        headers["ChatGPT-Account-Id"] = account
    req = request.Request(CHATGPT_URL, data=json.dumps(body).encode("utf-8"), headers=headers, method="POST")
    try:
        with request.urlopen(req, timeout=180) as resp:
            raw = resp.read().decode("utf-8", errors="replace")
    except error.HTTPError as exc:
        detail = exc.read().decode("utf-8", errors="replace")[:2000]
        raise RuntimeError(f"OpenAI/Codex request failed ({exc.code}): {detail}") from exc

    events = []
    for chunk in raw.split("\n\n"):
        chunk = chunk.strip()
        if not chunk:
            continue
        for line in chunk.splitlines():
            if not line.startswith("data: "):
                continue
            payload = line[6:]
            if payload == "[DONE]":
                continue
            events.append(json.loads(payload))
    completed = next((event for event in reversed(events) if event.get("type") == "response.completed"), None)
    if not completed:
        raise RuntimeError("OpenAI/Codex response missing response.completed event")
    return completed["response"]


def _parse_openai_response(raw: dict[str, Any]) -> tuple[list[dict[str, Any]], str, bool, int, int]:
    content_blocks = []
    text_parts = []
    has_tool_calls = False
    for item in raw.get("output", []):
        if item.get("type") == "function_call":
            has_tool_calls = True
            try:
                args = json.loads(item.get("arguments") or "{}")
            except Exception:
                args = {}
            content_blocks.append({"type": "tool_use", "id": item.get("call_id") or item.get("id"), "name": item["name"], "input": args})
            continue
        if item.get("type") != "message":
            continue
        for part in item.get("content", []):
            if part.get("type") != "output_text":
                continue
            text = part.get("text", "")
            if text:
                text_parts.append(text)
                content_blocks.append({"type": "text", "text": text})
    usage = raw.get("usage", {})
    return content_blocks, " ".join(text_parts), has_tool_calls, usage.get("input_tokens", 0), usage.get("output_tokens", 0)


def _strip_code_fences(text: str) -> str:
    text = text.strip()
    if text.startswith("```"):
        lines = text.splitlines()
        if len(lines) >= 3:
            text = "\n".join(lines[1:-1])
    return text.strip()


def _extract_json_object(text: str) -> dict[str, Any] | None:
    cleaned = _strip_code_fences(text)
    try:
        parsed = json.loads(cleaned)
        return parsed if isinstance(parsed, dict) else None
    except Exception:
        pass
    start = cleaned.find("{")
    end = cleaned.rfind("}")
    if start == -1 or end == -1 or end <= start:
        return None
    try:
        parsed = json.loads(cleaned[start : end + 1])
        return parsed if isinstance(parsed, dict) else None
    except Exception:
        return None


def _normalize_scan_plan(plan: dict[str, Any] | None) -> dict[str, Any]:
    merged = dict(DEFAULT_SCAN_PLAN)
    if isinstance(plan, dict):
        merged.update({k: v for k, v in plan.items() if v not in (None, "")})
    for key in ("priority_order", "source_include", "source_avoid", "followup_rules", "compose_requirements"):
        value = merged.get(key)
        if not isinstance(value, list):
            merged[key] = list(DEFAULT_SCAN_PLAN[key])
        else:
            merged[key] = [str(item) for item in value if str(item).strip()]
    try:
        merged["minimum_tool_calls"] = max(2, int(merged.get("minimum_tool_calls", 4)))
    except Exception:
        merged["minimum_tool_calls"] = DEFAULT_SCAN_PLAN["minimum_tool_calls"]
    for key in ("mission_type", "goal", "search_style"):
        merged[key] = str(merged.get(key, DEFAULT_SCAN_PLAN[key]))
    return merged


def _planner_model_call(prompt: str, system: str, max_output_tokens: int = 1200) -> str:
    use_chatgpt = MODEL.startswith("gpt-") and _load_codex_auth() is not None
    use_bedrock = bool(AWS_BEARER_TOKEN)
    if use_chatgpt:
        raw = _openai_call([{"role": "user", "content": prompt}], system, [], MODEL, max_output_tokens=max_output_tokens)
        _, text, _, _, _ = _parse_openai_response(raw)
        return text
    if use_bedrock:
        raw = _bedrock_call([{"role": "user", "content": prompt}], system, [], MODEL)
        _, text, _ = _parse_bedrock_response(raw)
        return text
    if not ANTHROPIC_API_KEY:
        raise RuntimeError("No provider available. Set Codex auth, Bedrock, or ANTHROPIC_API_KEY.")
    client: Any = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
    resp: Any = client.messages.create(model=MODEL, max_tokens=max_output_tokens, system=system, messages=[{"role": "user", "content": prompt}])
    if not resp.content:
        return ""
    first = resp.content[0]
    return getattr(first, "text", "")


def _plan_scan(mission: str, token_budget: int, context_packet: str, memory: str, now: str) -> dict[str, Any]:
    tool_names = ", ".join(tool["name"] for tool in SCOUT_TOOLS)
    prompt = f"""Design the best autonomous scouting plan for this run.

Current time: {now}
Token budget: {token_budget}
Mission: {mission or 'General intelligence scan'}

Available tools:
{tool_names}

Relevant context:
{context_packet[:3500]}

Recent memory:
{memory[:1200]}

Return JSON only with this shape:
{{
  "mission_type": "short label",
  "goal": "one sentence",
  "search_style": "broad_then_deep or another concise strategy",
  "priority_order": ["priority 1", "priority 2"],
  "source_include": ["tool_name", "tool_name"],
  "source_avoid": ["tool_name"],
  "followup_rules": ["rule", "rule"],
  "minimum_tool_calls": 4,
  "compose_requirements": ["requirement", "requirement"]
}}

Rules:
- Be general and adaptive, not brittle.
- Let the scout choose tools autonomously, but set a strong initial plan.
- Prefer source diversity first, then depth.
- Include source_avoid only when a source is clearly low-value for this mission.
- Keep the plan compact and actionable.
"""
    system = "You are a planning engine for an autonomous research scout. Return only valid JSON."
    try:
        text = _planner_model_call(prompt, system, max_output_tokens=1200)
        return _normalize_scan_plan(_extract_json_object(text))
    except Exception as exc:
        print(f"  [planner] fallback to default plan ({exc})")
        return _normalize_scan_plan(None)


def _render_scan_plan(plan: dict[str, Any]) -> str:
    lines = [
        f"mission_type: {plan['mission_type']}",
        f"goal: {plan['goal']}",
        f"search_style: {plan['search_style']}",
        f"minimum_tool_calls: {plan['minimum_tool_calls']}",
    ]
    if plan.get("priority_order"):
        lines.append("priority_order:")
        lines.extend(f"- {item}" for item in plan["priority_order"])
    if plan.get("source_include"):
        lines.append("source_include:")
        lines.extend(f"- {item}" for item in plan["source_include"])
    if plan.get("source_avoid"):
        lines.append("source_avoid:")
        lines.extend(f"- {item}" for item in plan["source_avoid"])
    if plan.get("followup_rules"):
        lines.append("followup_rules:")
        lines.extend(f"- {item}" for item in plan["followup_rules"])
    if plan.get("compose_requirements"):
        lines.append("compose_requirements:")
        lines.extend(f"- {item}" for item in plan["compose_requirements"])
    return "\n".join(lines)


def _bedrock_call(messages: list[dict[str, Any]], system: str, tools: list[dict[str, Any]], model: str) -> dict[str, Any]:
    model_id = _bedrock_model_id(model)
    url = f"https://bedrock-runtime.{AWS_REGION}.amazonaws.com/model/{model_id}/converse"
    body: dict[str, Any] = {
        "messages": _anthropic_messages_to_bedrock(messages),
        "system": [{"text": system}],
        "inferenceConfig": {"maxTokens": 4096, "temperature": 0.7},
    }
    if tools:
        body["toolConfig"] = {"tools": _anthropic_tools_to_bedrock(tools), "toolChoice": {"auto": {}}}
    resp = httpx.post(
        url,
        json=body,
        headers={"Authorization": f"Bearer {AWS_BEARER_TOKEN}", "Content-Type": "application/json"},
        timeout=120,
    )
    resp.raise_for_status()
    return resp.json()


def _parse_bedrock_response(raw: dict[str, Any]) -> tuple[list[dict[str, Any]], str, bool]:
    output = raw.get("output", {}).get("message", {})
    stop_reason = raw.get("stopReason", "end_turn")
    content_blocks = []
    text_parts = []
    has_tool_calls = stop_reason == "tool_use"
    for block in output.get("content", []):
        if "text" in block:
            text_parts.append(block["text"])
            content_blocks.append({"type": "text", "text": block["text"]})
        elif "toolUse" in block:
            tool_use = block["toolUse"]
            content_blocks.append({"type": "tool_use", "id": tool_use["toolUseId"], "name": tool_use["name"], "input": tool_use["input"]})
    return content_blocks, " ".join(text_parts), has_tool_calls


def _web_fetch(url: str) -> str:
    try:
        result = subprocess.run(["curl", "-s", "-L", "--max-time", "30", "-H", "User-Agent: Mozilla/5.0", url], capture_output=True, text=True, timeout=35)
        content = result.stdout
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL)
        content = re.sub(r"<[^>]+>", " ", content)
        content = re.sub(r"\s+", " ", content).strip()
        return content[:15000] if content else "(empty response)"
    except Exception as exc:
        return f"Fetch error: {exc}"


def _load_state() -> dict[str, Any]:
    if STATE_FILE.exists():
        try:
            return json.loads(STATE_FILE.read_text(encoding="utf-8"))
        except Exception:
            pass
    return {"scan_count": 0, "last_scan": None}


def _save_state(state: dict[str, Any]) -> None:
    STATE_FILE.write_text(json.dumps(state, indent=2), encoding="utf-8")


def _prune_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
    if len(messages) <= 1 + KEEP_ROUNDS * 2:
        return messages
    init = messages[0]
    pairs = []
    index = 1
    while index + 1 < len(messages):
        pairs.append((messages[index], messages[index + 1]))
        index += 2
    trailing = messages[index] if index < len(messages) else None
    old_pairs, keep_pairs = pairs[:-KEEP_ROUNDS], pairs[-KEEP_ROUNDS:]

    lines = ["[EARLIER TOOL CALLS - summarised]"]
    for assistant_msg, user_msg in old_pairs:
        for block in assistant_msg.get("content") or []:
            if isinstance(block, dict) and block.get("type") == "tool_use":
                lines.append(f"  called {block['name']}({json.dumps(block.get('input', {}))[:50]})")
        for block in user_msg.get("content") or []:
            if isinstance(block, dict) and block.get("type") == "tool_result":
                text = str(block.get("content", ""))
                if text.startswith("Noted ["):
                    lines.append(f"  -> {text[:80]}")
    lines.append("[END SUMMARY]")

    pruned = [init, {"role": "user", "content": "\n".join(lines)}]
    for assistant_msg, user_msg in keep_pairs:
        pruned.extend([assistant_msg, user_msg])
    if trailing:
        pruned.append(trailing)
    return pruned


def _est_tokens(messages: list[dict[str, Any]]) -> int:
    chars = sum(len(json.dumps(message)) for message in messages)
    return chars // 4


def _read_text(path: Path, max_chars: int = 0) -> str:
    if not path.exists() or not path.is_file():
        return ""
    text = path.read_text(encoding="utf-8")
    return text[:max_chars] if max_chars else text


def _build_context_packet() -> str:
    parts = []
    files = sorted([path for path in CONTEXT_DIR.glob("**/*") if path.is_file() and path.suffix.lower() in {".md", ".txt", ".json"}])
    for path in files[:12]:
        text = _read_text(path, 3000).strip()
        if text:
            parts.append(f"## {path.name}\nSource: {path}\n{text}")
    if parts:
        return "\n\n".join(parts)
    return (
        "No explicit context files were provided. Assume a generic user who wants a useful, "
        "well-sourced brief with concrete links, dates, and next actions."
    )


def _load_known_titles() -> set[str]:
    if not MEMORY_FILE.exists():
        return set()
    text = MEMORY_FILE.read_text(encoding="utf-8")
    if KNOWN_HEADER not in text:
        return set()
    start = text.index(KNOWN_HEADER) + len(KNOWN_HEADER)
    end = text.index(KNOWN_END) if KNOWN_END in text else len(text)
    return {line[2:].strip().lower() for line in text[start:end].splitlines() if line.startswith("- ")}


def _save_known_titles(notes: list[dict[str, str]]) -> None:
    if not notes:
        return
    new = "\n".join(f"- {note['title']}" for note in notes if note.get("title"))
    if not new:
        return
    memory = MEMORY_FILE.read_text(encoding="utf-8") if MEMORY_FILE.exists() else ""
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
    if KNOWN_HEADER in memory and KNOWN_END in memory:
        memory = memory.replace(KNOWN_END, f"\n{new}\n{KNOWN_END}")
    else:
        memory += f"\n\n{KNOWN_HEADER}\n_Updated: {timestamp}_\n{new}\n{KNOWN_END}\n"
    MEMORY_FILE.write_text(memory, encoding="utf-8")


def _compose_sections(notes: list[dict[str, str]]) -> str:
    by_category: dict[str, list[dict[str, str]]] = {}
    for note in notes:
        by_category.setdefault(note["category"], []).append(note)
    text = ""
    for category, items in by_category.items():
        text += f"\n### {category}\n"
        text += "".join(f"- **{item['title']}**: {item['content']}\n" for item in items)
    return text


def _compose_brief(
    notes: list[dict[str, str]],
    tool_log: list[dict[str, Any]],
    total_calls: int,
    context_packet: str,
    now: str,
    plan_block: str,
    prev_titles: list[str],
) -> tuple[str, int, int]:
    notes_text = _compose_sections(notes)
    tool_summary = "\n".join(f"- {entry['tool']}({json.dumps(entry['args'])[:55]})" for entry in tool_log)
    new_block = "\n".join(f"- [{note['category']}] **{note['title']}**" for note in notes if note["title"].lower() not in prev_titles) or "(all items new or first scan)"
    prompt = f"""You are an intelligence scout. Exploration complete - {total_calls} tool calls made.

## Findings
{notes_text or '(no explicit notes - synthesize from tool log)'}

## Tool Calls Made
{tool_summary}

## What's New vs Last Scan
{new_block}

## User Context
{context_packet[:2200]}

## Current Time
{now}

## Planner requirements for this run
{plan_block}

Produce the final brief. Be specific: titles, URLs, deadlines, numbers. No filler. Follow the compose_requirements from the planner.

# Intel Brief - {now}

## New Since Last Scan
Only items not in the previous brief. Format: "- **Title** - why it matters"

## Direct Hits
Anything directly relevant to the user's watchlist or active mission. If none: "(none this scan)"

## Worth Your Attention
Up to 5 items to act on or read this week. Include URLs.

## Surprising
1 to 3 unexpected findings or connections.

## Opportunities
Programs, grants, fellowships, events, or calls with real URLs and deadlines. Skip if none.

## People to Follow
Anyone found with category="person". Include who they are, what they do, and link.

## Research Queue
2 to 3 papers, repos, or sources to inspect next. Skip if none.

---
_Tool calls: {total_calls} | Notes: {len(notes)} | {now}_
"""
    system = "You are an intelligence scout. Produce the briefing exactly as requested. Be specific and concrete."
    use_chatgpt = MODEL.startswith("gpt-") and _load_codex_auth() is not None
    if use_chatgpt:
        raw = _openai_call([{"role": "user", "content": prompt}], system, [], MODEL, max_output_tokens=4096)
        _, brief, _, c_in, c_out = _parse_openai_response(raw)
        return brief, c_in, c_out
    use_bedrock = bool(AWS_BEARER_TOKEN)
    if use_bedrock:
        raw = _bedrock_call([{"role": "user", "content": prompt}], system, [], MODEL)
        _, brief, _ = _parse_bedrock_response(raw)
        return brief, raw.get("usage", {}).get("inputTokens", 0), raw.get("usage", {}).get("outputTokens", 0)
    if not ANTHROPIC_API_KEY:
        raise RuntimeError("No provider available. Set Codex auth, Bedrock, or ANTHROPIC_API_KEY.")
    client: Any = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
    resp: Any = client.messages.create(model=MODEL, max_tokens=2048, system=system, messages=[{"role": "user", "content": prompt}])
    brief = getattr(resp.content[0], "text", "") if resp.content else ""
    return brief, resp.usage.input_tokens if resp.usage else 0, resp.usage.output_tokens if resp.usage else 0


def run_scout(token_budget: int = 15000, mission: str = "") -> str:
    run_start = time.time()
    use_chatgpt = MODEL.startswith("gpt-") and _load_codex_auth() is not None
    use_bedrock = bool(AWS_BEARER_TOKEN)
    now = datetime.now().strftime("%Y-%m-%d %H:%M %A")
    print(f"\n[{datetime.now().strftime('%H:%M:%S')}] Scout | budget={token_budget:,}")

    context_packet = _build_context_packet()
    memory = MEMORY_FILE.read_text(encoding="utf-8") if MEMORY_FILE.exists() else "(empty)"
    known = _load_known_titles()
    known_block = ("\n".join(f"  - {title}" for title in sorted(known)[:30]) + f"\n  ({len(known)} total)") if known else "  (none yet - first scan)"

    explore_budget = int(token_budget * 0.80)
    print(f"  Known items: {len(known)} | Explore budget: {explore_budget:,}")

    scan_plan = _plan_scan(mission=mission, token_budget=token_budget, context_packet=context_packet, memory=memory, now=now)
    plan_block = _render_scan_plan(scan_plan)
    print(f"  [plan] {scan_plan['mission_type']} | min_calls={scan_plan['minimum_tool_calls']} | style={scan_plan['search_style']}")

    system = f"""You are an autonomous intelligence scout.

## User context
{context_packet}

## Memory from previous scans
{memory[:2000]}

## Items already reported (skip unless something significant changed)
{known_block}

## Current time
{now}

## Run plan chosen by the planner
{plan_block}

## Your tools
You have access to HN, GitHub trending, Exa, arXiv, X, Reddit, YouTube, LinkedIn, Lobsters, Tavily, Gemini, web_fetch, jina_read, and note.

## Exploration strategy
Work in two phases.

Phase 1: cast wide. Cover at least 4 different source types before going deep.
Phase 2: follow the strongest leads. Read full pages, verify facts, and note what matters.

## What makes a great scan
- surface non-obvious findings
- prioritize actionable and time-sensitive items
- quality over quantity
- use note for every significant finding

## note categories
- direct_hit: directly relevant to the user's mission, watchlist, or active work
- opportunity: program, grant, fellowship, event, or role with a real deadline or URL
- worth_attention: something the user should act on or read this week
- surprising: non-obvious connection or unexpected signal
- research_queue: paper, repo, or source worth reading next
- person: builder, researcher, or operator worth tracking
"""

    init_msg = (
        f"Today is {now}. Your mission:\n\n{mission}\n\nPlanner summary:\n{plan_block}\n\n"
        "Use the plan as your starting strategy. Sample broadly first, then go deep on the strongest leads. Use note for every significant finding."
    ) if mission else (
        f"Today is {now}. Run a full intelligence scan.\n\n"
        "Cover opportunities, builders, research worth reading, and anything surprising from the last week.\n\n"
        f"Planner summary:\n{plan_block}\n\nStart broad, then follow the best threads. Use note for every significant finding."
    )

    messages: list[dict[str, Any]] = [{"role": "user", "content": init_msg}]
    notes: list[dict[str, str]] = []
    tool_log: list[dict[str, Any]] = []
    tool_stats: dict[str, dict[str, int]] = {}
    tokens_in = 0
    tokens_out = 0
    total_calls = 0
    iteration = 0
    prune_counter = 0

    while True:
        iteration += 1
        prune_counter += 1
        if prune_counter >= KEEP_ROUNDS:
            before = _est_tokens(messages)
            messages = _prune_messages(messages)
            after = _est_tokens(messages)
            if before - after > 200:
                print(f"  [prune] ~{before:,}->{after:,} est tokens")
            prune_counter = 0

        cumulative = tokens_in + tokens_out
        remaining = explore_budget - cumulative
        est_next = _est_tokens(messages) + 500
        min_calls = scan_plan.get("minimum_tool_calls", 4)
        if (remaining < 1000 or est_next > remaining * 2) and total_calls >= min_calls:
            print(f"  [budget] {cumulative:,} used, ~{remaining:,} remaining -> stopping explore")
            break

        try:
            if use_chatgpt:
                raw = _openai_call(messages, system, SCOUT_TOOLS, MODEL, max_output_tokens=2048)
                content_blocks, _, has_tools, step_in, step_out = _parse_openai_response(raw)
            elif use_bedrock:
                raw = _bedrock_call(messages, system, SCOUT_TOOLS, MODEL)
                content_blocks, _, has_tools = _parse_bedrock_response(raw)
                usage = raw.get("usage", {})
                step_in = usage.get("inputTokens", 0)
                step_out = usage.get("outputTokens", 0)
            else:
                if not ANTHROPIC_API_KEY:
                    raise RuntimeError("No provider available. Set Codex auth, Bedrock, or ANTHROPIC_API_KEY.")
                client: Any = anthropic.Anthropic(api_key=ANTHROPIC_API_KEY)
                resp: Any = client.messages.create(model=MODEL, system=system, messages=messages, tools=SCOUT_TOOLS, max_tokens=2048)
                content_blocks = []
                for block in resp.content:
                    if getattr(block, "type", None) == "text" and getattr(block, "text", None):
                        content_blocks.append({"type": "text", "text": block.text})
                    elif getattr(block, "type", None) == "tool_use":
                        content_blocks.append({"type": "tool_use", "id": block.id, "name": block.name, "input": block.input})
                has_tools = resp.stop_reason == "tool_use"
                step_in = resp.usage.input_tokens if resp.usage else 0
                step_out = resp.usage.output_tokens if resp.usage else 0
        except Exception as exc:
            print(f"  [LLM error iter {iteration}] {exc}")
            break

        tokens_in += step_in
        tokens_out += step_out
        messages.append({"role": "assistant", "content": content_blocks})

        if not has_tools:
            print(f"  [iter {iteration}] Agent stopped voluntarily. {tokens_in + tokens_out:,} tokens used.")
            break

        tool_results = []
        for block in content_blocks:
            if block.get("type") != "tool_use":
                continue
            tool_name = block["name"]
            tool_args = block["input"]
            total_calls += 1
            start = time.time()
            result = _run_tool(tool_name, tool_args, notes)
            elapsed_ms = int((time.time() - start) * 1000)
            stat = tool_stats.setdefault(tool_name, {"calls": 0, "chars": 0, "ms": 0})
            stat["calls"] += 1
            stat["chars"] += len(result)
            stat["ms"] += elapsed_ms
            print(f"  [{tool_name}] {json.dumps(tool_args)[:70]} -> {result[:90].replace(chr(10), ' ')}")
            tool_log.append({"tool": tool_name, "args": tool_args, "ms": elapsed_ms})
            remaining = explore_budget - (tokens_in + tokens_out)
            hint = f"\n\n[BUDGET: ~{remaining:,} tokens left - wrap up and note findings]" if remaining < 5000 else ""
            tool_results.append({"type": "tool_result", "tool_use_id": block["id"], "content": result + hint})
        messages.append({"role": "user", "content": tool_results})

    explore_duration = round(time.time() - run_start, 1)
    print(f"  Explore done: {total_calls} calls | {tokens_in:,}in/{tokens_out:,}out | {explore_duration}s")
    print("  Composing brief...")

    previous_titles: list[str] = []
    briefs = sorted(OUTPUT_DIR.glob("*.md"))
    if briefs:
        previous_content = briefs[-1].read_text(encoding="utf-8")
        previous_titles = [title.lower() for title in re.findall(r'\*\*(.+?)\*\*', previous_content)]

    compose_start = time.time()
    try:
        brief, compose_in, compose_out = _compose_brief(notes, tool_log, total_calls, context_packet, now, plan_block, previous_titles)
    except Exception as exc:
        brief = f"# Intel Brief - {now}\n\nCompose failed: {exc}\n"
        compose_in = 0
        compose_out = 0
    compose_duration = round(time.time() - compose_start, 1)
    total_duration = round(time.time() - run_start, 1)
    grand_in = tokens_in + compose_in
    grand_out = tokens_out + compose_out
    grand_total = grand_in + grand_out
    budget_pct = round(grand_total / token_budget * 100) if token_budget else 0
    cost_usd = 0.0 if use_chatgpt else (grand_in / 1_000_000 * 0.80) + (grand_out / 1_000_000 * 4.00)

    tool_rows = "".join(
        f"| {tool_name:<22} | {stats['calls']:>5} | {stats['chars']:>10,} | {stats['ms'] // stats['calls']:>5}ms |\n"
        for tool_name, stats in sorted(tool_stats.items(), key=lambda item: -item[1]["calls"])
    )

    state = _load_state()
    scan_num = state.get("scan_count", 0) + 1
    stats_block = f"""
---

## Run Stats

| Metric | Value |
|--------|-------|
| Tool calls | {total_calls} |
| Notes captured | {len(notes)} |
| Explore tokens in | {tokens_in:,} |
| Explore tokens out | {tokens_out:,} |
| Compose tokens in | {compose_in:,} |
| Compose tokens out | {compose_out:,} |
| **Total tokens** | **{grand_total:,}** |
| Budget | {token_budget:,} ({budget_pct}%) |
| Cost estimate | ${cost_usd:.4f} USD |
| Explore phase | {explore_duration}s |
| Compose phase | {compose_duration}s |
| **Total duration** | **{total_duration}s** |
| Model | {MODEL if use_chatgpt else _bedrock_model_id(MODEL) if use_bedrock else MODEL} |
| Scan # | {scan_num} |

### Tool Breakdown
| Tool | Calls | Chars | Avg Latency |
|------|-------|-------|-------------|
{tool_rows}"""

    full_output = brief + stats_block
    timestamp = datetime.now().strftime("%Y-%m-%d-%H%M")
    outfile = OUTPUT_DIR / f"{timestamp}-c{total_calls}-t{token_budget}.md"
    outfile.write_text(full_output, encoding="utf-8")

    print(f"  Tokens: {grand_total:,} ({budget_pct}% of budget) | Cost: ${cost_usd:.4f} | {total_duration}s")
    print(f"  Saved: {outfile}")

    _save_known_titles(notes)
    state["scan_count"] = scan_num
    state["last_scan"] = datetime.now().isoformat()
    _save_state(state)
    print(f"[{datetime.now().strftime('%H:%M:%S')}] Scout done (scan #{scan_num})")
    return full_output


def main() -> None:
    global MODEL
    parser = argparse.ArgumentParser(description="Autonomous intelligence scout")
    parser.add_argument("--loop", action="store_true", help="Run forever with interval")
    parser.add_argument("--interval", type=int, default=LOOP_INTERVAL_MINUTES, help=f"Loop interval in minutes (default: {LOOP_INTERVAL_MINUTES})")
    parser.add_argument("--model", type=str, default=MODEL, help=f"Model to use (default: {MODEL})")
    parser.add_argument("--tokens", type=int, default=15000, help="Total token budget. ~15k=quick, ~50k=deep, ~100k+=full research")
    parser.add_argument("--mission", type=str, default="", help="Override the default scan sequence with a custom mission")
    args = parser.parse_args()
    MODEL = args.model

    if args.loop:
        print(f"Scout looping every {args.interval} min | tokens={args.tokens}. Ctrl+C to stop.")
        while True:
            try:
                run_scout(token_budget=args.tokens, mission=args.mission)
                print(f"Sleeping {args.interval} minutes...")
                time.sleep(args.interval * 60)
            except KeyboardInterrupt:
                print("\nStopped.")
                break
    else:
        run_scout(token_budget=args.tokens, mission=args.mission)


if __name__ == "__main__":
    main()