validate_portfolio.py

#!/usr/bin/env python3
"""Lightweight portfolio validation checks."""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]

REQUIRED_ROOT_FILES = [
    "README.md",
    "PORTFOLIO_INDEX.md",
    "CASE_STUDY_TEMPLATE.md",
    "CONTRIBUTING.md",
    "LICENSE",
    ".gitignore",
    "docs/portfolio_principles.md",
    "docs/evidence_levels.md",
    "docs/audit_methodology.md",
    "docs/candidate_selection_criteria.md",
    "docs/expert_contact_policy.md",
    "docs/representation_policy.md",
    "docs/ai_operator_skill_map.md",
    "docs/future_targets.md",
    "backlog/candidate_papers.md",
    "backlog/candidate_repos.md",
    "backlog/scoring_rubric.md",
    "data/portfolio_cases.json",
]

REQUIRED_CASE_FILES = [
    "README.md",
    "case_study.md",
    "evidence.md",
    "workflow.md",
    "artifacts.md",
    "external_acknowledgment.md",
    "non_claims.md",
    "links.md",
]

EML_CASE_DIR = ROOT / "case_studies/001-eml-qualified-reproduction-audit"
EML_PUBLIC_REPO = "https://github.com/bankszach/eml-qualified-reproduction-audit"
EML_REQUIRED_FILES = [
    "executive_summary.md",
    "artifact_checklist.md",
]
EML_REPO_LINK_FILES = [
    "README.md",
    "case_study.md",
    "evidence.md",
    "workflow.md",
    "artifacts.md",
    "external_acknowledgment.md",
    "non_claims.md",
    "links.md",
    "executive_summary.md",
    "artifact_checklist.md",
]

UNSAFE_PHRASES = [
    "author " + "endorsed",
    "expert " + "endorsed",
    "signed " + "off",
    "certif" + "ied",
    "proved the paper " + "wrong",
    "refuted the " + "paper",
    "expert-" + "approved",
]

PRIVATE_TERMS = [
    "private " + "correspondence",
    "private " + "email",
    "author " + "email",
]
PRIVATE_QUOTE_PATTERN = re.compile(
    r"(" + "|".join(re.escape(term) for term in PRIVATE_TERMS) + r")\s*:\s*[\"'“”]",
    re.IGNORECASE,
)
PRIVATE_CORRESPONDENCE = "private " + "correspondence"
NOT_QUOTED = "not " + "quoted"
NOT_QUOTED_PUBLICLY = "not " + "quoted publicly"


def iter_text_files() -> list[Path]:
    skipped_dirs = {".git", "__pycache__"}
    files: list[Path] = []
    for path in ROOT.rglob("*"):
        if any(part in skipped_dirs for part in path.parts):
            continue
        if path.is_file() and path.suffix in {".md", ".py", ".json", ""}:
            files.append(path)
    return files


def main() -> int:
    errors: list[str] = []
    warnings: list[str] = []

    for rel in REQUIRED_ROOT_FILES:
        if not (ROOT / rel).exists():
            errors.append(f"missing required file: {rel}")

    cases_path = ROOT / "data/portfolio_cases.json"
    cases = []
    if cases_path.exists():
        try:
            cases = json.loads(cases_path.read_text(encoding="utf-8"))
        except json.JSONDecodeError as exc:
            errors.append(f"data/portfolio_cases.json does not parse: {exc}")

    if not isinstance(cases, list):
        errors.append("data/portfolio_cases.json must contain a list")
        cases = []

    for case in cases:
        case_id = str(case.get("id", "")).strip()
        slug = str(case.get("title", "")).lower().replace(" ", "-")
        if case_id == "001":
            case_dir = ROOT / "case_studies/001-eml-qualified-reproduction-audit"
        else:
            case_dir = ROOT / "case_studies" / f"{case_id}-{slug}"

        if not case_dir.exists():
            errors.append(f"missing case folder for case {case_id}: {case_dir.relative_to(ROOT)}")
            continue

        for filename in REQUIRED_CASE_FILES:
            if not (case_dir / filename).exists():
                errors.append(f"case {case_id} missing {filename}")

    for filename in EML_REQUIRED_FILES:
        if not (EML_CASE_DIR / filename).exists():
            errors.append(f"EML case missing {filename}")

    for filename in EML_REPO_LINK_FILES:
        path = EML_CASE_DIR / filename
        if path.exists() and EML_PUBLIC_REPO not in path.read_text(encoding="utf-8", errors="replace"):
            errors.append(f"EML case file missing public audit repo link: {path.relative_to(ROOT)}")

    placeholder = ROOT / "case_studies/002-placeholder/README.md"
    if not placeholder.exists():
        errors.append("missing placeholder case README")

    for path in iter_text_files():
        text = path.read_text(encoding="utf-8", errors="replace")
        lower_text = text.lower()
        rel = path.relative_to(ROOT)
        for phrase in UNSAFE_PHRASES:
            if phrase in lower_text:
                errors.append(f"unsafe claim language in {rel}: {phrase}")
        if PRIVATE_QUOTE_PATTERN.search(text):
            warnings.append(f"possible private-correspondence quote in {rel}")
        if (
            PRIVATE_CORRESPONDENCE in lower_text
            and NOT_QUOTED not in lower_text
            and NOT_QUOTED_PUBLICLY not in lower_text
        ):
            warnings.append(f"private-correspondence mention lacks not-quoted caveat in {rel}")

    print("Portfolio validation")
    print("====================")
    print(f"Root: {ROOT}")
    print(f"Cases: {len(cases)}")
    for case in cases:
        print(
            "- {id}: {title} | {status}".format(
                id=case.get("id", "unknown"),
                title=case.get("title", "untitled"),
                status=case.get("status", "unknown status"),
            )
        )

    if warnings:
        print("\nWarnings:")
        for warning in warnings:
            print(f"- {warning}")
    else:
        print("\nWarnings: none")

    if errors:
        print("\nErrors:")
        for error in errors:
            print(f"- {error}")
        return 1

    print("\nValidation passed.")
    return 0


if __name__ == "__main__":
    sys.exit(main())