Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 42 additions & 8 deletions adapters/batch_support.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
"""Batch estimate execution support for CLI and tests."""

from collections.abc import Callable
from dataclasses import dataclass
from dataclasses import dataclass, field
from io import StringIO
from pathlib import Path
from typing import Any

from rich import box
from rich.console import Console
Expand All @@ -13,7 +15,7 @@
from adapters.envelope import EstimatorResultEnvelope, build_estimator_envelope
from adapters.assets.geofence_geojson import GeofenceLoadError
from adapters.assets.obstacle_geojson import ObstacleLoadError
from adapters.io import InputLoadError, load_mission, load_vehicle
from adapters.io import InputDocument, InputLoadError, load_mission, load_vehicle
from adapters.assets.landing_zone_geojson import LandingZoneLoadError
from adapters.assets.terrain_grid import TerrainGridLoadError
from adapters.assets.wind_grid import WindGridLoadError
Expand All @@ -27,6 +29,21 @@
)
from estimator.environment.terrain import TerrainProvider
from schemas.batch import BatchManifest, BatchRun
from schemas.mission import MissionPlan
from schemas.vehicle import VehicleProfile


@dataclass
class _BatchLoadCaches:
"""Per-invocation caches so shared inputs parse once across runs."""

missions: dict[Path, tuple[MissionPlan, InputDocument]] = field(
default_factory=dict
)
vehicles: dict[Path, tuple[VehicleProfile, InputDocument]] = field(
default_factory=dict
)
assets: dict[Path, tuple[Any, InputDocument]] = field(default_factory=dict)

_BATCH_RUN_INPUT_ERRORS = (
InputLoadError,
Expand Down Expand Up @@ -90,14 +107,27 @@ def _status_label(
return _STATUS_LABELS.get(estimate.status, "ERROR")


def _run_estimate(run: BatchRun, *, engineering_only: bool) -> BatchRunResult:
def _run_estimate(
run: BatchRun,
*,
engineering_only: bool,
caches: _BatchLoadCaches | None = None,
) -> BatchRunResult:
caches = caches or _BatchLoadCaches()
mission_assets = MissionAssetBundle()
mission_model, mission_document = load_mission(run.mission)
vehicle_model, vehicle_document = load_vehicle(run.vehicle)
mission_key = run.mission.resolve(strict=False)
if mission_key not in caches.missions:
caches.missions[mission_key] = load_mission(run.mission)
mission_model, mission_document = caches.missions[mission_key]
vehicle_key = run.vehicle.resolve(strict=False)
if vehicle_key not in caches.vehicles:
caches.vehicles[vehicle_key] = load_vehicle(run.vehicle)
vehicle_model, vehicle_document = caches.vehicles[vehicle_key]
_populate_mission_assets(
mission_assets,
mission_model=mission_model,
mission_document=mission_document,
asset_cache=caches.assets,
)
result = try_estimate_mission_distance_time(
mission_model,
Expand Down Expand Up @@ -135,15 +165,19 @@ def _run_estimate(run: BatchRun, *, engineering_only: bool) -> BatchRunResult:
def run_batch_manifest(
manifest: BatchManifest,
*,
progress: Callable[[int, int], None] | None = None,
progress: Callable[[int, int, str], None] | None = None,
engineering_only: bool = False,
preloaded_missions: dict[Path, tuple[MissionPlan, InputDocument]] | None = None,
) -> list[BatchRunResult]:
"""Run all estimates in a validated batch manifest."""
results: list[BatchRunResult] = []
total = len(manifest.runs)
caches = _BatchLoadCaches(missions=dict(preloaded_missions or {}))
for index, run in enumerate(manifest.runs):
try:
results.append(_run_estimate(run, engineering_only=engineering_only))
results.append(
_run_estimate(run, engineering_only=engineering_only, caches=caches)
)
except _BATCH_RUN_INPUT_ERRORS as exc:
results.append(
BatchRunResult(
Expand All @@ -156,7 +190,7 @@ def run_batch_manifest(
)
)
if progress is not None:
progress(index + 1, total)
progress(index + 1, total, run.id)
return results


Expand Down
23 changes: 21 additions & 2 deletions adapters/cli_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from datetime import UTC, datetime
from math import isfinite
from pathlib import Path
from typing import TypeVar
from typing import Any, TypeVar

import typer

Expand Down Expand Up @@ -306,10 +306,22 @@ def _load_optional_asset(
*,
mission_path: Path,
loader: Callable[[Path], tuple[LoadedAssetT, InputDocument]],
cache: dict[Path, tuple[Any, InputDocument]] | None = None,
) -> tuple[LoadedAssetT | None, InputDocument | None]:
if path is None:
return None, None
return loader(_resolve_asset_path(path, mission_path=mission_path))
resolved = _resolve_asset_path(path, mission_path=mission_path)
if cache is None:
return loader(resolved)
key = resolved.resolve(strict=False)
if key not in cache:
cache[key] = loader(resolved)
value, document = cache[key]
# Zone lists are handed to the estimator per run; copy so one run can
# never see another's list object.
if isinstance(value, list):
value = list(value)
return value, document


def _status_for_failure_kind(kind: FailureKind) -> EstimateStatus:
Expand Down Expand Up @@ -395,37 +407,44 @@ def _populate_mission_assets(
*,
mission_model: MissionPlan,
mission_document: InputDocument,
asset_cache: dict[Path, tuple[Any, InputDocument]] | None = None,
) -> None:
mission_path = mission_document.path
bundle.terrain_provider, bundle.terrain_document = _load_optional_asset(
mission_model.assets.terrain_file,
mission_path=mission_path,
loader=load_terrain_grid,
cache=asset_cache,
)
bundle.population_provider, bundle.population_document = _load_optional_asset(
mission_model.assets.population_grid_file,
mission_path=mission_path,
loader=load_population_grid,
cache=asset_cache,
)
bundle.obstacle_provider, bundle.obstacle_document = _load_optional_asset(
mission_model.assets.obstacles_file,
mission_path=mission_path,
loader=load_obstacles,
cache=asset_cache,
)
bundle.wind_provider, bundle.wind_grid_document = _load_optional_asset(
mission_model.assets.wind_grid_file,
mission_path=mission_path,
loader=load_wind_grid,
cache=asset_cache,
)
bundle.geofences, bundle.geofence_document = _load_optional_asset(
mission_model.assets.geofences_file,
mission_path=mission_path,
loader=load_geofences,
cache=asset_cache,
)
bundle.landing_zones, bundle.landing_zone_document = _load_optional_asset(
mission_model.assets.landing_zones_file,
mission_path=mission_path,
loader=load_landing_zones,
cache=asset_cache,
)


Expand Down
27 changes: 20 additions & 7 deletions adapters/commands/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
)
from adapters.cli_support import OutputWriteError, _write_output
from adapters.envelope import OutputFormat
from adapters.io import InputLoadError, load_mission, load_vehicle
from adapters.io import InputDocument, InputLoadError, load_mission, load_vehicle
from adapters.preflight import (
check_file,
emit_preflight,
Expand All @@ -30,6 +30,7 @@
)
from adapters.progress import progress_reporter
from schemas.batch import BatchManifest
from schemas.mission import MissionPlan


BatchStdoutRenderer = Callable[[list[BatchRunResult]], str]
Expand All @@ -47,12 +48,21 @@
def _batch_protected_input_paths(
manifest: Path,
batch_manifest: BatchManifest,
) -> tuple[Path, ...]:
"""Resolve every file that a batch run may read before opening sidecars."""
) -> tuple[tuple[Path, ...], dict[Path, tuple[MissionPlan, InputDocument]]]:
"""Resolve every file a batch run may read before opening sidecars.

Missions are parsed here anyway to enumerate their asset paths, so the
parsed models are returned for the run loop to reuse instead of parsing
every mission a second time.
"""
protected = [manifest]
preloaded: dict[Path, tuple[MissionPlan, InputDocument]] = {}
for run in batch_manifest.runs:
protected.extend((run.mission, run.vehicle))
mission_model, _mission_document = load_mission(run.mission)
mission_key = run.mission.resolve(strict=False)
if mission_key not in preloaded:
preloaded[mission_key] = load_mission(run.mission)
mission_model, _mission_document = preloaded[mission_key]
for asset_path in mission_model.assets.model_dump().values():
if not isinstance(asset_path, Path):
continue
Expand All @@ -61,7 +71,7 @@ def _batch_protected_input_paths(
if asset_path.is_absolute()
else run.mission.parent / asset_path
)
return tuple(protected)
return tuple(protected), preloaded


def _validate_batch_output_paths(
Expand Down Expand Up @@ -174,7 +184,7 @@ def _run_batch_preflight(*, manifest: Path, as_json: bool) -> None:


def batch(
manifest: Path = typer.Argument(..., exists=True, readable=True, resolve_path=True),
manifest: Path = typer.Argument(..., resolve_path=True),
output_dir: Path | None = typer.Option(
None,
"--output-dir",
Expand Down Expand Up @@ -231,7 +241,9 @@ def batch(
err=True,
)
batch_manifest = load_batch_manifest(manifest)
protected_paths = _batch_protected_input_paths(manifest, batch_manifest)
protected_paths, preloaded_missions = _batch_protected_input_paths(
manifest, batch_manifest
)
_validate_batch_output_paths(
output_dir=output_dir,
output_format=format,
Expand All @@ -258,6 +270,7 @@ def batch(
batch_manifest,
progress=reporter,
engineering_only=engineering_only,
preloaded_missions=preloaded_missions,
)
_emit_batch_warnings(results)
_write_batch_file_outputs(
Expand Down
4 changes: 0 additions & 4 deletions adapters/commands/calibrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,11 @@ def _run_calibrate_preflight(
def calibrate(
vehicle: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to the base vehicle profile YAML file.",
),
traces: list[Path] = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="One or more flight-trace.v1 JSON files (from flight-log ingestion).",
),
Expand Down
2 changes: 0 additions & 2 deletions adapters/commands/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ def _run_compare_preflight(*, evidence: Path, as_json: bool) -> None:
def compare(
evidence: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to a sitl-evidence.v1 JSON bundle.",
),
Expand Down
2 changes: 1 addition & 1 deletion adapters/commands/convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def _run_convert_preflight(


def convert(
plan: Path = typer.Argument(..., exists=True, readable=True, resolve_path=True),
plan: Path = typer.Argument(..., resolve_path=True),
vehicle_profile: str | None = typer.Option(
None,
"--vehicle-profile",
Expand Down
16 changes: 9 additions & 7 deletions adapters/commands/estimate.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@
from adapters.preflight import (
check_file,
emit_preflight,
format_note_suffix,
is_json_format,
mission_asset_checks,
mission_block_notes,
)
from adapters.profile_markdown import render_profile_markdown
from adapters.sensitivity import render_sensitivity_markdown, run_sensitivity_sweep
Expand Down Expand Up @@ -301,9 +303,15 @@ def _run_estimate_preflight(
mission_check, mission_result = check_file(
role="mission", path_str=mission.name, loader=lambda: load_mission(mission)
)
if mission_check.ok and mission_result is not None:
mission_check = mission_check.model_copy(
update={"notes": mission_block_notes(mission_result[0])}
)
files.append(mission_check)
if mission_check.ok:
text_lines.append(f"mission: {mission.name}: OK")
text_lines.append(
f"mission: {mission.name}: OK{format_note_suffix(mission_check.notes)}"
)

vehicle_check, _ = check_file(
role="vehicle", path_str=vehicle.name, loader=lambda: load_vehicle(vehicle)
Expand Down Expand Up @@ -333,15 +341,11 @@ def _run_estimate_preflight(
def estimate(
mission: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to mission.v7 YAML file.",
),
vehicle: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to vehicle profile YAML file.",
),
Expand All @@ -368,8 +372,6 @@ def estimate(
calibration: Path | None = typer.Option(
None,
"--calibration",
exists=True,
readable=True,
resolve_path=True,
help=(
"Optional calibration-profile.v1 JSON to layer on the vehicle. "
Expand Down
2 changes: 0 additions & 2 deletions adapters/commands/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,6 @@ def load_and_build():
def export(
mission: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to mission.v7 YAML file.",
),
Expand Down
6 changes: 0 additions & 6 deletions adapters/commands/ingest_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@
def ingest_log(
log: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="ArduPilot DataFlash .log/.bin or PX4 .ulg file.",
),
Expand All @@ -38,16 +36,12 @@ def ingest_log(
mission: Path | None = typer.Option(
None,
"--mission",
exists=True,
readable=True,
resolve_path=True,
help="Paired mission file; requires --vehicle and embeds its SHA-256.",
),
vehicle: Path | None = typer.Option(
None,
"--vehicle",
exists=True,
readable=True,
resolve_path=True,
help="Paired vehicle file; requires --mission and embeds its SHA-256.",
),
Expand Down
2 changes: 0 additions & 2 deletions adapters/commands/propagate.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,6 @@ def _run_propagate_preflight(*, stochastic_file: Path, as_json: bool) -> None:
def propagate(
stochastic_file: Path = typer.Argument(
...,
exists=True,
readable=True,
resolve_path=True,
help="Path to stochastic.v2 diagnostic YAML file.",
),
Expand Down
Loading
Loading