diff --git a/.gitignore b/.gitignore index 0a9c3e1..8f0c15c 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ output/ # Build artifacts dist/ build/ +site/ *.egg-info/ # Test artifacts diff --git a/README.md b/README.md index c232b2f..e9454ae 100644 --- a/README.md +++ b/README.md @@ -48,29 +48,45 @@ DATA_DIR=./data ## 🚀 Quick Start -**📓 [Tutorial Notebook](notebooks/ONC_Data_Download_Tutorial.ipynb)** - The best way to get started with interactive examples. +For the guided beginner path, start with the +**[online documentation](https://spiffical.github.io/onc-hydrophone-data/)**. +An extended [tutorial notebook](notebooks/ONC_Data_Download_Tutorial.ipynb) is +also available for interactive exploration. ### Python API ```python -from onc_hydrophone_data.onc.common import load_config -from onc_hydrophone_data.data import HydrophoneDownloader +from datetime import datetime, timezone +from pathlib import Path + from onc_hydrophone_data.audio import SpectrogramGenerator +from onc_hydrophone_data.data import HydrophoneDownloader +from onc_hydrophone_data.onc.common import load_config -# Load credentials from .env file onc_token, data_dir = load_config() - -# Download spectrograms using intelligent sampling downloader = HydrophoneDownloader(onc_token, data_dir) -downloader.download_spectrograms_with_sampling_schedule( - deviceCode="ICLISTENHF6020", - start_date=(2021, 1, 1), - threshold_num=100 + +# Download a short, verified ONC audio range. +downloader.download_audio_for_range( + device_code="ICLISTENHF6324", + start_dt=datetime(2024, 4, 1, 12, 0, tzinfo=timezone.utc), + end_dt=datetime(2024, 4, 1, 12, 10, tzinfo=timezone.utc), ) -# Generate custom spectrograms from audio files -generator = SpectrogramGenerator(win_dur=2.0, overlap=0.75) -generator.process_directory("data/DEVICE/audio/", "output/spectrograms/") +# Generate PNG and MAT spectrograms locally from the downloaded audio. +audio_dir = Path(downloader.audio_path) +generator = SpectrogramGenerator( + win_dur=0.5, + overlap=0.75, + freq_lims=(20, 10_000), + crop_freq_lims=True, +) +generator.process_directory( + audio_dir, + audio_dir.parent / "custom_spectrograms", + save_plot=True, + save_mat=True, +) ``` ### Command Line @@ -81,11 +97,11 @@ python scripts/download_hydrophone_data.py # Download spectrograms with specific parameters python scripts/download_hydrophone_data.py --mode sampling \ - --device ICLISTENHF6020 --start-date 2021 1 1 --threshold 500 + --device ICLISTENHF6324 --start-date 2024 4 1 --threshold 500 # Include FLAC audio files python scripts/download_hydrophone_data.py --mode sampling \ - --device ICLISTENHF6020 --start-date 2021 1 1 --threshold 100 --download-audio + --device ICLISTENHF6324 --start-date 2024 4 1 --threshold 100 --download-audio # Generate custom spectrograms python scripts/generate_spectrograms.py --input-dir data/DEVICE/audio/ --win-dur 2.0 @@ -126,8 +142,8 @@ Downloads are organized in a clean, flat structure: ``` data/ -└── ICLISTENHF6020/ - └── sampling_2021-01-01_to_2021-01-31/ +└── ICLISTENHF6324/ + └── audio_range_2024-04-01_to_2024-04-01/ ├── onc_spectrograms/ # ONC-downloaded spectrograms (MAT/PNG) │ ├── *.mat # Spectrogram data files │ └── anomaly_report.txt # Any validation issues (if found) diff --git a/docs/audio_downloads.md b/docs/audio_downloads.md index d10472e..1c68fe5 100644 --- a/docs/audio_downloads.md +++ b/docs/audio_downloads.md @@ -7,6 +7,9 @@ without querying ONC again. ## Start with a short range +Run this setup first. The remaining examples on this page reuse `dl`, `DEVICE`, +`start`, and `end` from this block. + ```python from datetime import datetime, timezone @@ -16,10 +19,14 @@ from onc_hydrophone_data.onc.common import load_config onc_token, data_dir = load_config() dl = HydrophoneDownloader(onc_token, data_dir) +DEVICE = "ICLISTENHF6324" +start = datetime(2024, 4, 1, 12, 0, tzinfo=timezone.utc) +end = datetime(2024, 4, 1, 12, 10, tzinfo=timezone.utc) + dl.download_audio_for_range( - device_code="ICLISTENHF6324", - start_dt=datetime(2024, 4, 1, 12, 0, tzinfo=timezone.utc), - end_dt=datetime(2024, 4, 1, 12, 10, tzinfo=timezone.utc), + device_code=DEVICE, + start_dt=start, + end_dt=end, ) print("Audio directory:", dl.audio_path) @@ -70,10 +77,13 @@ To explore seasonal or long-term variation without downloading every file, request a uniform sample: ```python +sample_start = datetime(2024, 4, 1, tzinfo=timezone.utc) +sample_end = datetime(2024, 4, 8, tzinfo=timezone.utc) + result = dl.download_sampled_audio( device_code=DEVICE, - start_dt=start, - end_dt=end, + start_dt=sample_start, + end_dt=sample_end, total_audio_files=24, files_per_request=4, ) diff --git a/docs/custom_spectrograms.md b/docs/custom_spectrograms.md index 35d919e..be2f1ea 100644 --- a/docs/custom_spectrograms.md +++ b/docs/custom_spectrograms.md @@ -117,6 +117,12 @@ For many labeled events, one workflow can download the needed audio context, clip each event, and generate local spectrograms: ```python +from onc_hydrophone_data.data import HydrophoneDownloader +from onc_hydrophone_data.onc.common import load_config + +onc_token, data_dir = load_config() +dl = HydrophoneDownloader(onc_token, data_dir) + results = dl.create_custom_spectrograms_from_json( "custom_requests.json", save_mat=True, diff --git a/docs/downloads.md b/docs/downloads.md index c8244d6..7f7eeb2 100644 --- a/docs/downloads.md +++ b/docs/downloads.md @@ -17,7 +17,26 @@ Options](onc_spectrogram_options.md). ![Parallel ONC request pipeline](assets/parallel_pipeline.svg){: width="100%" } +## Shared setup for the examples + +Run this block once before the Python examples below: + +```python +from datetime import datetime, timezone + +from onc_hydrophone_data.data import HydrophoneDownloader +from onc_hydrophone_data.onc.common import load_config + +onc_token, data_dir = load_config() +dl = HydrophoneDownloader(onc_token, data_dir) + +DEVICE = "ICLISTENHF6324" +start = datetime(2024, 4, 1, 12, 0, tzinfo=timezone.utc) +end = datetime(2024, 4, 1, 14, 0, tzinfo=timezone.utc) +``` + ## Server-generated spectrograms for a range + ```python result = dl.download_spectrograms_for_range( device_code=DEVICE, @@ -28,6 +47,7 @@ result = dl.download_spectrograms_for_range( ``` ### Include matching audio + ```python result = dl.download_spectrograms_for_range( device_code=DEVICE, @@ -39,17 +59,22 @@ result = dl.download_spectrograms_for_range( ``` ## Sample uniformly across a range + ```python +sample_start = datetime(2024, 4, 1, tzinfo=timezone.utc) +sample_end = datetime(2024, 4, 8, tzinfo=timezone.utc) + result = dl.download_sampled_spectrograms( device_code=DEVICE, - start_dt=start, - end_dt=end, + start_dt=sample_start, + end_dt=sample_end, total_spectrograms=24, spectrograms_per_request=6, ) ``` ## Download around event timestamps + ```python events = [ datetime(2024, 4, 1, 12, 5, tzinfo=timezone.utc), @@ -76,6 +101,7 @@ result = dl.download_audio_for_range( ``` ## JSON/CSV request files + ```python results = dl.download_requests_from_json("/path/to/requests.json") results = dl.download_requests_from_csv("/path/to/requests.csv") diff --git a/docs/onc_spectrogram_options.md b/docs/onc_spectrogram_options.md index 5c032ac..9029978 100644 --- a/docs/onc_spectrogram_options.md +++ b/docs/onc_spectrogram_options.md @@ -55,10 +55,19 @@ specific downsampling guide recommends no more than one month per request. ### Python examples -Set a default for every request made by a downloader: +Run this setup once before the Python examples on this page. The two-hour range is +small enough for learning; begin with fewer windows for full-resolution output. ```python +from datetime import datetime, timezone + from onc_hydrophone_data.data import HydrophoneDownloader +from onc_hydrophone_data.onc.common import load_config + +ONC_TOKEN, DATA_DIR = load_config() +DEVICE = "ICLISTENHF6324" +start = datetime(2024, 4, 1, 12, 0, tzinfo=timezone.utc) +end = datetime(2024, 4, 1, 14, 0, tzinfo=timezone.utc) # Use ONC's compact, pre-generated one-minute MAT product by default. dl = HydrophoneDownloader( diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 74e9dea..2474de9 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -33,7 +33,8 @@ workflow. ## I cannot find the downloaded files -Print the active paths immediately after a download: +Print the active paths immediately after a download, using the same `dl` +instance created in the [Download Audio](audio_downloads.md) setup block: ```python print("Audio:", dl.audio_path) @@ -62,6 +63,8 @@ for long ranges. See **[Choose ONC Server Spectrograms](onc_spectrogram_options. Use the SciPy backend to separate backend installation from data problems: ```python +from onc_hydrophone_data.audio import SpectrogramGenerator + generator = SpectrogramGenerator(backend="scipy") ``` diff --git a/onc_hydrophone_data/data/downloader/onc_downloads.py b/onc_hydrophone_data/data/downloader/onc_downloads.py index d2add62..1e28ef4 100644 --- a/onc_hydrophone_data/data/downloader/onc_downloads.py +++ b/onc_hydrophone_data/data/downloader/onc_downloads.py @@ -170,7 +170,8 @@ def download_audio_files( False so rerunning a request resumes instead of wasting bandwidth. Returns: - Summary dict with files found/downloaded and extension used. + Summary dict with files found, downloaded during this call, skipped, + available locally after the call, errors, and the extension used. """ self.logger.info(f'Finding audio files for {deviceCode} from {start_time} to {end_time}') try: @@ -194,6 +195,7 @@ def download_audio_files( 'files_found': 0, 'files_downloaded': 0, 'files_skipped': 0, + 'files_available': 0, 'errors': 0, } @@ -240,7 +242,7 @@ def download_audio_files( attempts = {f: 0 for f in audio_files} max_attempts = 6 attempt_round = 0 - downloaded = len(existing_files) + downloaded = 0 errors = 0 if existing_files: @@ -302,15 +304,19 @@ def download_audio_files( time.sleep(5) self.logger.info( - f"{extension.upper()} files downloaded in {time.time() - download_start:.2f}s" + f"{extension.upper()} download pass completed in " + f"{time.time() - download_start:.2f}s " + f"({downloaded} downloaded, {len(existing_files)} skipped)" ) + available = len(existing_files) + downloaded summary.update({ - 'extension_used': extension if downloaded else None, + 'extension_used': extension if available else None, 'files_downloaded': downloaded, 'files_skipped': len(existing_files), + 'files_available': available, 'errors': errors, }) - if downloaded: + if available: return summary self.logger.warning( diff --git a/onc_hydrophone_data/utils/plotting.py b/onc_hydrophone_data/utils/plotting.py index 1d43d2e..2d951a2 100644 --- a/onc_hydrophone_data/utils/plotting.py +++ b/onc_hydrophone_data/utils/plotting.py @@ -449,14 +449,15 @@ def plot_availability_calendar( raise ValueError("plot_availability_calendar requires bin_size='day'") date_to_bin = {b['start'].date(): b for b in bins if b.get('start') is not None} - start_dt = availability.get('start') or bins[0]['start'] - end_dt = availability.get('end') or bins[-1]['end'] - if start_dt is None or end_dt is None: + first_bin_start = bins[0].get('start') + last_bin_start = bins[-1].get('start') + start_dt = availability.get('start') or first_bin_start + if start_dt is None or last_bin_start is None: print("Availability window is empty.") return None start_date = start_dt.date() - end_date = bins[-1]['start'].date() + end_date = last_bin_start.date() week0_start = start_date - timedelta(days=start_date.weekday()) total_days = (end_date - start_date).days num_weeks = ((end_date - week0_start).days // 7) + 1 diff --git a/pyproject.toml b/pyproject.toml index 1d0fb1c..18c260e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ where = ["."] include = ["onc_hydrophone_data*"] [tool.pytest.ini_options] -addopts = "-m 'not integration'" +addopts = ["-m", "not integration"] markers = [ "integration: makes live ONC API requests and may download data", ] diff --git a/scripts/generate_docs_figures.py b/scripts/generate_docs_figures.py index 016dfc0..41a2e4e 100644 --- a/scripts/generate_docs_figures.py +++ b/scripts/generate_docs_figures.py @@ -28,6 +28,9 @@ ) +LANCZOS = getattr(getattr(Image, "Resampling", Image), "LANCZOS") + + ROOT = Path(__file__).resolve().parents[1] OUTPUT_DIR = ROOT / "docs" / "assets" / "figures" ONC_AUDIO_URL = ( @@ -69,16 +72,18 @@ def _save_webp( ) -> None: """Save a compact WebP while keeping Matplotlib rendering deterministic.""" temporary_png = OUTPUT_DIR / f".{filename}.png" - fig.savefig(temporary_png, dpi=170, bbox_inches=bbox_inches) - with Image.open(temporary_png) as image: - image.thumbnail((1_800, 1_200), Image.Resampling.LANCZOS) - image.save( - OUTPUT_DIR / f"{filename}.webp", - format="WEBP", - quality=84, - method=6, - ) - temporary_png.unlink() + try: + fig.savefig(temporary_png, dpi=170, bbox_inches=bbox_inches) + with Image.open(temporary_png) as image: + image.thumbnail((1_800, 1_200), LANCZOS) + image.save( + OUTPUT_DIR / f"{filename}.webp", + format="WEBP", + quality=84, + method=6, + ) + finally: + temporary_png.unlink(missing_ok=True) def _download_onc_audio(destination: Path) -> None: @@ -87,8 +92,34 @@ def _download_onc_audio(destination: Path) -> None: ONC_AUDIO_URL, headers={"User-Agent": "onc-hydrophone-data documentation figure generator"}, ) - with urlopen(request, timeout=60) as response, destination.open("wb") as output: - shutil.copyfileobj(response, output) + partial_destination = destination.with_suffix(f"{destination.suffix}.part") + try: + with urlopen(request, timeout=60) as response: + status = getattr(response, "status", None) + if status is not None and not 200 <= status < 300: + raise RuntimeError( + f"ONC audio preview returned HTTP status {status}" + ) + + content_type = response.headers.get_content_type().lower() + if not ( + content_type.startswith("audio/") + or content_type == "application/octet-stream" + ): + raise RuntimeError( + "ONC audio preview returned unexpected content type " + f"{content_type!r}; expected audio" + ) + + with partial_destination.open("wb") as output: + shutil.copyfileobj(response, output) + + if partial_destination.stat().st_size == 0: + raise RuntimeError("ONC audio preview returned an empty response") + partial_destination.replace(destination) + except Exception: + partial_destination.unlink(missing_ok=True) + raise def generate_deployment_figures() -> None: @@ -251,23 +282,25 @@ def generate_spectrogram_figures(audio_path: Path) -> None: sample_rate, ) temporary_spectrogram = OUTPUT_DIR / ".example_local_spectrogram.png" - fig = generator.plot_spectrogram( - frequencies, - times, - power_db, - title="Humpback whale calls — Folger Passage — 2012-08-01 12:24 UTC", - save_path=temporary_spectrogram, - ) - plt.close(fig) - with Image.open(temporary_spectrogram) as image: - image.thumbnail((1_800, 1_200), Image.Resampling.LANCZOS) - image.save( - OUTPUT_DIR / "example_local_spectrogram.webp", - format="WEBP", - quality=84, - method=6, + try: + fig = generator.plot_spectrogram( + frequencies, + times, + power_db, + title="Humpback whale calls — Folger Passage — 2012-08-01 12:24 UTC", + save_path=temporary_spectrogram, ) - temporary_spectrogram.unlink() + plt.close(fig) + with Image.open(temporary_spectrogram) as image: + image.thumbnail((1_800, 1_200), LANCZOS) + image.save( + OUTPUT_DIR / "example_local_spectrogram.webp", + format="WEBP", + quality=84, + method=6, + ) + finally: + temporary_spectrogram.unlink(missing_ok=True) comparisons = [] for window_seconds in (0.032, 0.5): diff --git a/tests/test_docs_figure_generation.py b/tests/test_docs_figure_generation.py new file mode 100644 index 0000000..7057ebd --- /dev/null +++ b/tests/test_docs_figure_generation.py @@ -0,0 +1,59 @@ +import io +from email.message import Message +import importlib.util +from pathlib import Path + +import pytest + + +_SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "generate_docs_figures.py" +_SCRIPT_SPEC = importlib.util.spec_from_file_location( + "generate_docs_figures", + _SCRIPT_PATH, +) +assert _SCRIPT_SPEC is not None and _SCRIPT_SPEC.loader is not None +generate_docs_figures = importlib.util.module_from_spec(_SCRIPT_SPEC) +_SCRIPT_SPEC.loader.exec_module(generate_docs_figures) + + +class _FakeResponse(io.BytesIO): + def __init__(self, body: bytes, content_type: str, status: int = 200): + super().__init__(body) + self.status = status + self.headers = Message() + self.headers["Content-Type"] = content_type + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def test_download_onc_audio_rejects_non_audio_response(tmp_path, monkeypatch): + destination = tmp_path / "preview.mp3" + monkeypatch.setattr( + generate_docs_figures, + "urlopen", + lambda *args, **kwargs: _FakeResponse(b"Access denied", "text/html"), + ) + + with pytest.raises(RuntimeError, match="unexpected content type 'text/html'"): + generate_docs_figures._download_onc_audio(destination) + + assert not destination.exists() + assert not (tmp_path / "preview.mp3.part").exists() + + +def test_download_onc_audio_atomically_saves_audio(tmp_path, monkeypatch): + destination = tmp_path / "preview.mp3" + monkeypatch.setattr( + generate_docs_figures, + "urlopen", + lambda *args, **kwargs: _FakeResponse(b"real audio bytes", "audio/mp3"), + ) + + generate_docs_figures._download_onc_audio(destination) + + assert destination.read_bytes() == b"real audio bytes" + assert not (tmp_path / "preview.mp3.part").exists() diff --git a/tests/test_download_efficiency.py b/tests/test_download_efficiency.py index b637ea2..5733156 100644 --- a/tests/test_download_efficiency.py +++ b/tests/test_download_efficiency.py @@ -35,12 +35,46 @@ def test_audio_download_skips_existing_nonempty_file(tmp_path: Path): "2024-04-01T12:05:00Z", ) - assert summary["files_downloaded"] == 1 + assert summary["files_downloaded"] == 0 assert summary["files_skipped"] == 1 + assert summary["files_available"] == 1 + assert summary["extension_used"] == "flac" onc.getFile.assert_not_called() assert onc.outPath == "original" +def test_audio_download_counts_new_files_separately(tmp_path: Path): + audio_dir = tmp_path / "audio" + audio_dir.mkdir() + audio_name = "DEVICE_20240401T120000.000Z.flac" + + onc = MagicMock() + onc.outPath = "original" + onc.getListByDevice.return_value = {"files": [audio_name]} + downloader = SimpleNamespace( + onc=onc, + logger=MagicMock(), + audio_path=str(audio_dir), + max_workers=1, + _parse_timestamp_value=lambda value: value, + _build_request_windows=lambda start, end: [(start, end)], + ) + + summary = download_audio_files( + downloader, + "DEVICE", + "2024-04-01T12:00:00Z", + "2024-04-01T12:05:00Z", + ) + + assert summary["files_downloaded"] == 1 + assert summary["files_skipped"] == 0 + assert summary["files_available"] == 1 + assert summary["extension_used"] == "flac" + onc.getFile.assert_called_once_with(audio_name, overwrite=False) + assert onc.outPath == "original" + + def test_mat_requests_submit_concurrently_by_default(tmp_path: Path): class ConcurrentRequestManager: def __init__(self): diff --git a/tests/test_plot_layouts.py b/tests/test_plot_layouts.py index 3e324f3..9cae096 100644 --- a/tests/test_plot_layouts.py +++ b/tests/test_plot_layouts.py @@ -52,7 +52,8 @@ def _assert_legend_is_below_plot(fig, ax) -> None: renderer = fig.canvas.get_renderer() legend_bounds = fig.legends[0].get_window_extent(renderer) axes_bounds = ax.get_window_extent(renderer) - assert legend_bounds.y1 <= axes_bounds.y0 + tolerance_pixels = 2.0 + assert legend_bounds.y1 <= axes_bounds.y0 + tolerance_pixels def test_timeline_legend_does_not_cover_plot() -> None: @@ -81,3 +82,10 @@ def test_calendar_excludes_the_query_end_boundary() -> None: assert calendar_grid.shape[:2] == (7, 2) finally: plt.close(fig) + + +def test_calendar_handles_a_missing_last_bin_start() -> None: + availability = _daily_availability() + availability["bins"][-1]["start"] = None + + assert plot_availability_calendar(availability, show=False) is None