diff --git a/ui-react/src/lib/datasets/__tests__/ckan.test.ts b/ui-react/src/lib/datasets/__tests__/ckan.test.ts index 5bd98cb..a41497b 100644 --- a/ui-react/src/lib/datasets/__tests__/ckan.test.ts +++ b/ui-react/src/lib/datasets/__tests__/ckan.test.ts @@ -16,9 +16,11 @@ import { packageSpatialCoverage, packageTags, packagesMatchingVariables, + packagesMatchingVariableSubstring, packageTimePeriod, parseDate, resourceMatchesVariables, + resourceMatchesVariableSubstring, resourceStandardVariables, searchAllPackages, searchPackages, @@ -192,36 +194,16 @@ describe('showPackage', () => { describe('buildSearchQuery', () => { it('returns undefined when there is nothing to search on', () => { expect(buildSearchQuery({})).toBeUndefined(); - expect(buildSearchQuery({ variables: [] })).toBeUndefined(); + expect(buildSearchQuery({ name: '' })).toBeUndefined(); }); it('quotes a single name as a phrase', () => { expect(buildSearchQuery({ name: 'soil moisture' })).toBe('"soil moisture"'); }); - it('ORs multiple variables inside parentheses', () => { - expect(buildSearchQuery({ variables: ['precipitation', 'temperature'] })).toBe( - '("precipitation" OR "temperature")', - ); - }); - - it('does not parenthesise a lone variable', () => { - expect(buildSearchQuery({ variables: ['precipitation'] })).toBe('"precipitation"'); - }); - - it('ANDs a name together with the variable clause', () => { - expect(buildSearchQuery({ name: 'ethiopia', variables: ['a', 'b'] })).toBe( - '"ethiopia" AND ("a" OR "b")', - ); - }); - it('escapes embedded quotes so Solr does not see an unbalanced phrase', () => { expect(buildSearchQuery({ name: 'say "hi"' })).toBe('"say \\"hi\\""'); }); - - it('drops empty variable entries', () => { - expect(buildSearchQuery({ variables: ['', 'real'] })).toBe('"real"'); - }); }); // ─── Shaping helpers ────────────────────────────────────────────────────────── @@ -418,6 +400,102 @@ describe('packagesMatchingVariables', () => { }); }); +describe('resourceMatchesVariableSubstring', () => { + const annotated = { mint_standard_variables: 'groundwater__hydraulic_head' }; + + it('matches a fragment of the variable name', () => { + expect(resourceMatchesVariableSubstring(annotated, 'hydraulic')).toBe(true); + expect(resourceMatchesVariableSubstring(annotated, 'groundwater')).toBe(true); + }); + + it('matches across the underscores Solr tokenises on', () => { + // The whole point: `q` cannot find this, because Solr splits the name up. + expect(resourceMatchesVariableSubstring(annotated, 'water__hydraulic')).toBe(true); + }); + + it('ignores case on both sides', () => { + expect(resourceMatchesVariableSubstring(annotated, 'Hydraulic HEAD'.slice(0, 9))).toBe(true); + expect( + resourceMatchesVariableSubstring({ mint_standard_variables: 'Corpus_NLP' }, 'corpus_nlp'), + ).toBe(true); + }); + + it('does not match a term absent from the annotation', () => { + expect(resourceMatchesVariableSubstring(annotated, 'porosity')).toBe(false); + }); + + it('never matches an unannotated resource against a real term', () => { + expect(resourceMatchesVariableSubstring({ format: 'CSV' }, 'groundwater')).toBe(false); + }); + + it('passes everything through when the term is empty or blank', () => { + expect(resourceMatchesVariableSubstring({ format: 'CSV' }, '')).toBe(true); + expect(resourceMatchesVariableSubstring({ format: 'CSV' }, ' ')).toBe(true); + }); +}); + +describe('packagesMatchingVariableSubstring', () => { + const carrier: CkanPackage = { + name: 'capitan-reef-complex-aquifer-gam-files', + resources: [ + { id: 'r1', mint_standard_variables: 'groundwater__initial_head' }, + { id: 'r2', mint_standard_variables: 'aquifer__transmissivity' }, + { id: 'r3' }, + ], + }; + + /** Reads as a match in prose, carries no annotation. This is what `q` returned. */ + const falsePositive: CkanPackage = { + name: 'groundwater-initial-head-report', + notes: 'A report on groundwater initial head across the basin.', + resources: [{ id: 'r4', mint_standard_variables: '' }], + }; + + it('keeps a package whose resource carries a variable containing the term', () => { + expect(packagesMatchingVariableSubstring([carrier], 'head').map((p) => p.name)).toEqual([ + carrier.name, + ]); + }); + + it('drops the prose match that free-text search returned', () => { + expect(packagesMatchingVariableSubstring([falsePositive], 'groundwater')).toEqual([]); + }); + + it('finds an underscored name that free-text search cannot', () => { + // `corpus_nlp` returned 0 through Solr while 17 TACC datasets carried it. + const pkg: CkanPackage = { + name: 'nlp-corpus', + resources: [{ id: 'r5', mint_standard_variables: 'corpus_nlp' }], + }; + expect(packagesMatchingVariableSubstring([pkg], 'corpus_nlp')).toHaveLength(1); + }); + + it('narrows the kept package to the resources that carry the term', () => { + const [pkg] = packagesMatchingVariableSubstring([carrier], 'transmissivity'); + expect(pkg?.resources?.map((r) => r.id)).toEqual(['r2']); + }); + + it('keeps every resource whose annotation contains the term', () => { + const [pkg] = packagesMatchingVariableSubstring([carrier], 'a'); + expect(pkg?.resources?.map((r) => r.id)).toEqual(['r1', 'r2']); + }); + + it('leaves the input package untouched', () => { + packagesMatchingVariableSubstring([carrier], 'head'); + expect(carrier.resources).toHaveLength(3); + }); + + it('passes everything through when the term is empty', () => { + const all = [carrier, falsePositive]; + expect(packagesMatchingVariableSubstring(all, '')).toBe(all); + expect(packagesMatchingVariableSubstring(all, ' ')).toBe(all); + }); + + it('drops a package with no resources at all', () => { + expect(packagesMatchingVariableSubstring([{ name: 'empty' }], 'head')).toEqual([]); + }); +}); + describe('packageTags', () => { it('extracts tag names and drops empty ones', () => { expect(packageTags({ tags: [{ name: 'climate' }, { name: '' }, {}] })).toEqual(['climate']); diff --git a/ui-react/src/lib/datasets/__tests__/data-catalog-api.test.ts b/ui-react/src/lib/datasets/__tests__/data-catalog-api.test.ts new file mode 100644 index 0000000..f44ba30 --- /dev/null +++ b/ui-react/src/lib/datasets/__tests__/data-catalog-api.test.ts @@ -0,0 +1,142 @@ +/** + * Tests for the CKAN-backed data catalog client. + * + * The behaviour under test is what `/datasets/search` asks of it. Standard + * variable names cannot be searched through CKAN's `q` — Solr does not index + * `mint_standard_variables` and tokenises the names on `_` — so the client must + * read the catalog and match the annotation itself. See the header of ./ckan. + */ +import { describe, expect, it, beforeEach, afterEach } from 'vitest'; +import { http, HttpResponse } from 'msw'; +import { server } from '@/test/msw/server'; +import { searchDatasets } from '../data-catalog-api'; +import type { CkanPackage } from '../ckan'; + +const CKAN_HOST = 'https://ckan.example.org'; + +/** Every `package_search` URL the client asked for, in order. */ +let requestedUrls: URL[] = []; + +/** + * Serve `packages` from `package_search`, honouring `rows`/`start` so that a + * client which fails to page sees a truncated catalog, exactly as CKAN does. + */ +function stubCatalog(packages: CkanPackage[]) { + server.use( + http.get('*/api/3/action/package_search', ({ request }) => { + const url = new URL(request.url); + requestedUrls.push(url); + const start = Number(url.searchParams.get('start') ?? 0); + const rows = Number(url.searchParams.get('rows') ?? 100); + return HttpResponse.json({ + success: true, + result: { count: packages.length, results: packages.slice(start, start + rows) }, + }); + }), + ); +} + +/** A package carrying `variables` on its single resource. */ +function annotated(name: string, ...variables: string[]): CkanPackage { + return { + id: name, + name, + title: name, + resources: [{ id: `${name}-r1`, format: 'CSV', mint_standard_variables: variables.join(',') }], + }; +} + +beforeEach(() => { + requestedUrls = []; + window.__MINT_CONFIG__ = { DATA_CATALOG_API: CKAN_HOST } as never; +}); + +afterEach(() => { + delete (window as { __MINT_CONFIG__?: unknown }).__MINT_CONFIG__; +}); + +describe('searchDatasets — variable name search', () => { + const catalog: CkanPackage[] = [ + annotated('gam-heads', 'groundwater__initial_head'), + annotated('gam-transmissivity', 'aquifer__transmissivity'), + annotated('nlp-corpus', 'corpus_nlp'), + { + // Reads as groundwater in prose, carries no annotation at all. + id: 'groundwater-report', + name: 'groundwater-report', + title: 'Groundwater report', + notes: 'A report on groundwater across the basin.', + resources: [{ id: 'gr-r1', format: 'PDF' }], + }, + ]; + + it('matches the annotation rather than prose', async () => { + stubCatalog(catalog); + const results = await searchDatasets({ variableSubstring: 'groundwater' }); + expect(results.map((d) => d.id)).toEqual(['gam-heads']); + }); + + it('finds an underscored variable name that a Solr q cannot', async () => { + stubCatalog(catalog); + const results = await searchDatasets({ variableSubstring: 'corpus_nlp' }); + expect(results.map((d) => d.id)).toEqual(['nlp-corpus']); + }); + + it('never sends the variable term to CKAN as a query', async () => { + stubCatalog(catalog); + await searchDatasets({ variableSubstring: 'groundwater' }); + for (const url of requestedUrls) { + expect(url.searchParams.get('q')).toBeNull(); + } + }); + + it('matches on a fragment of the name, not just the whole name', async () => { + stubCatalog(catalog); + const results = await searchDatasets({ variableSubstring: 'transmiss' }); + expect(results.map((d) => d.id)).toEqual(['gam-transmissivity']); + }); + + it('ignores case', async () => { + stubCatalog(catalog); + const results = await searchDatasets({ variableSubstring: 'AQUIFER' }); + expect(results.map((d) => d.id)).toEqual(['gam-transmissivity']); + }); + + it('reports the variables the dataset actually carries', async () => { + stubCatalog([annotated('multi', 'soil__porosity', 'soil__water_content')]); + const [ds] = await searchDatasets({ variableSubstring: 'porosity' }); + expect(ds?.variables).toEqual(['soil__porosity', 'soil__water_content']); + }); + + it('lists the whole catalog when no term is given', async () => { + stubCatalog(catalog); + const results = await searchDatasets({}); + expect(results).toHaveLength(catalog.length); + }); +}); + +describe('searchDatasets — paging', () => { + /** More packages than CKAN's per-request default of 100. */ + const big: CkanPackage[] = Array.from({ length: 215 }, (_, i) => + annotated(`pkg-${i}`, i % 2 === 0 ? 'soil__porosity' : 'soil__water_content'), + ); + + it('reads past the 100-row default when listing every dataset', async () => { + stubCatalog(big); + const results = await searchDatasets({}); + expect(results).toHaveLength(215); + }); + + it('reads the whole catalog before matching variables, not just the first page', async () => { + stubCatalog(big); + const results = await searchDatasets({ variableSubstring: 'porosity' }); + expect(results).toHaveLength(108); + }); + + it('reads past the 100-row default for a name search too', async () => { + stubCatalog(big); + const results = await searchDatasets({ name: '*pkg*' }); + expect(results).toHaveLength(215); + expect(requestedUrls[0]?.searchParams.get('q')).toBe('"*pkg*"'); + }); +}); diff --git a/ui-react/src/lib/datasets/ckan.ts b/ui-react/src/lib/datasets/ckan.ts index 321690f..0f8b1e9 100644 --- a/ui-react/src/lib/datasets/ckan.ts +++ b/ui-react/src/lib/datasets/ckan.ts @@ -195,26 +195,18 @@ function formatBbox(bbox: BoundingBox): string { } /** - * Build a CKAN `q` string from a name and/or a set of standard variable names. - * Variables are OR-ed, since a dataset matching any requested variable is a - * candidate. Returns undefined when there is nothing to search on, which CKAN - * treats as "match everything". + * Build a CKAN `q` string from a dataset name. Returns undefined when there is + * nothing to search on, which CKAN treats as "match everything". + * + * Standard variable names deliberately have no place here. Solr does not index + * `mint_standard_variables`, so putting a variable name in `q` matches prose + * instead — which returned datasets carrying no such annotation, and nothing at + * all for a name holding an `_`. Match variables against the annotation + * instead: see `packagesMatchingVariables` and `packagesMatchingVariableSubstring`. */ -export function buildSearchQuery(opts: { - name?: string; - variables?: string[]; -}): string | undefined { - const terms: string[] = []; - if (opts.name) terms.push(quote(opts.name)); - - const variables = (opts.variables ?? []).filter(Boolean); - if (variables.length) { - const clause = variables.map(quote).join(' OR '); - terms.push(variables.length > 1 ? `(${clause})` : clause); - } - - if (!terms.length) return undefined; - return terms.join(' AND '); +export function buildSearchQuery(opts: { name?: string }): string | undefined { + if (!opts.name) return undefined; + return quote(opts.name); } /** Quote a term so Solr treats it as a phrase and does not choke on its syntax. */ @@ -282,6 +274,22 @@ export function resourceMatchesVariables(row: CkanResource, variables: string[]) return resourceStandardVariables(row).some((v) => variables.includes(v)); } +/** + * Does this resource carry a standard variable whose name contains `term`? + * + * The substring counterpart of `resourceMatchesVariables`, for the search box + * where a person types a fragment rather than picking a whole name. Matching is + * case-insensitive and runs against the raw annotation, so it spans the `_` and + * `~` that Solr would have split the name on. + * + * A blank term means "no variable filter" and matches everything. + */ +export function resourceMatchesVariableSubstring(row: CkanResource, term: string): boolean { + const needle = term.trim().toLowerCase(); + if (!needle) return true; + return resourceStandardVariables(row).some((v) => v.toLowerCase().includes(needle)); +} + /** * Keep only the packages that carry one of `variables`, narrowing each to the * resources that actually carry it. This is the standard-variable lookup: CKAN @@ -297,13 +305,43 @@ export function packagesMatchingVariables( ): CkanPackage[] { const wanted = variables.filter(Boolean); if (!wanted.length) return packages; + return narrowToMatchingResources(packages, (r) => resourceMatchesVariables(r, wanted)); +} - const matched: CkanPackage[] = []; +/** + * Keep only the packages carrying a standard variable whose name contains + * `term`, narrowing each to the resources that carry it. The substring + * counterpart of `packagesMatchingVariables`. + * + * A blank term passes everything through untouched. + */ +export function packagesMatchingVariableSubstring( + packages: CkanPackage[], + term: string, +): CkanPackage[] { + if (!term.trim()) return packages; + return narrowToMatchingResources(packages, (r) => resourceMatchesVariableSubstring(r, term)); +} + +/** + * Drop packages with no matching resource, and narrow the survivors to the + * resources that matched. Narrowing is not cosmetic: a package matches because + * *some* of its resources carry the variable, and binding the rest hands a model + * input files it cannot read (see #94 — one TACC package holds 35 resources, of + * which 1 is annotated). + * + * Copies rather than mutates, so callers keep the packages they passed in. + */ +function narrowToMatchingResources( + packages: CkanPackage[], + matches: (row: CkanResource) => boolean, +): CkanPackage[] { + const kept: CkanPackage[] = []; for (const pkg of packages) { - const resources = (pkg.resources ?? []).filter((r) => resourceMatchesVariables(r, wanted)); - if (resources.length) matched.push({ ...pkg, resources }); + const resources = (pkg.resources ?? []).filter(matches); + if (resources.length) kept.push({ ...pkg, resources }); } - return matched; + return kept; } export function packageTags(pkg: CkanPackage): string[] { diff --git a/ui-react/src/lib/datasets/data-catalog-api.ts b/ui-react/src/lib/datasets/data-catalog-api.ts index 7efc60c..66f8ea1 100644 --- a/ui-react/src/lib/datasets/data-catalog-api.ts +++ b/ui-react/src/lib/datasets/data-catalog-api.ts @@ -9,10 +9,12 @@ import { buildSearchQuery, cleanString, packageSpatialCoverage, + packagesMatchingVariableSubstring, packageTags, packageTimePeriod, parseDate, - searchPackages, + resourceStandardVariables, + searchAllPackages, showPackage, type CkanPackage, type CkanResource, @@ -42,7 +44,20 @@ function mapDataResource(row: CkanResource, parent?: CkanPackage): DataResource }; } -function mapDataset(pkg: CkanPackage, variables: string[] = []): Dataset { +/** + * The distinct standard variables carried by a package's resources, in the + * order first seen. Read off the resources rather than echoed back from the + * query, so it says what the dataset holds rather than what was asked for. + */ +function packageStandardVariables(pkg: CkanPackage): string[] { + const seen = new Set(); + for (const row of pkg.resources ?? []) { + for (const name of resourceStandardVariables(row)) seen.add(name); + } + return [...seen]; +} + +function mapDataset(pkg: CkanPackage): Dataset { const resources = pkg.resources ?? []; return { @@ -50,7 +65,7 @@ function mapDataset(pkg: CkanPackage, variables: string[] = []): Dataset { id: cleanString(pkg.name) || cleanString(pkg.id), name: cleanString(pkg.title) || cleanString(pkg.name), region: '', - variables, + variables: packageStandardVariables(pkg), // CKAN has no datatype field; resource formats are the closest analogue. datatype: cleanString(resources[0]?.format), time_period: packageTimePeriod(pkg), @@ -78,23 +93,25 @@ function mapDataset(pkg: CkanPackage, variables: string[] = []): Dataset { // --------------------------------------------------------------------------- /** - * Search datasets by name and/or standard variable names. + * Search datasets by name and/or a fragment of a standard variable name. * - * CKAN has no standard-variable vocabulary, so variable names are matched as - * free text against title, description and tags. + * The two halves are answered in different places. A dataset name goes to CKAN + * as `q`, which Solr can serve. A standard variable name cannot: the annotation + * lives on each resource in `mint_standard_variables`, which Solr neither + * indexes nor tokenises usefully, so it is matched here against the whole + * catalog. Reading every package is what makes that possible, and it is also + * what stops the plain listing from stopping at CKAN's 100-row default. */ export async function searchDatasets(params: DatasetQueryParameters): Promise { - const query = buildSearchQuery({ - ...(params.name ? { name: params.name } : {}), - ...(params.variables ? { variables: params.variables } : {}), - }); + const query = buildSearchQuery({ ...(params.name ? { name: params.name } : {}) }); - const packages = await searchPackages({ + const packages = await searchAllPackages({ ...(query ? { q: query } : {}), ...(params.spatialCoverage ? { boundingBox: params.spatialCoverage } : {}), }); - return packages.map((pkg) => mapDataset(pkg, params.variables ?? [])); + const matched = packagesMatchingVariableSubstring(packages, params.variableSubstring ?? ''); + return matched.map(mapDataset); } /** diff --git a/ui-react/src/lib/datasets/types.ts b/ui-react/src/lib/datasets/types.ts index 085c2d0..95898fd 100644 --- a/ui-react/src/lib/datasets/types.ts +++ b/ui-react/src/lib/datasets/types.ts @@ -60,7 +60,13 @@ export interface Dataset { /** Query parameter shape for the data catalog search endpoint. */ export interface DatasetQueryParameters { name?: string; - variables?: string[]; + /** + * A fragment of a standard variable name, as typed into the search box. Kept + * separate from an exact list of names because it is matched against the + * `mint_standard_variables` annotation client-side, never sent to CKAN as a + * query — Solr cannot answer it. + */ + variableSubstring?: string; spatialCoverage?: { xmin: number; xmax: number; diff --git a/ui-react/src/pages/datasets/DatasetsSearch.tsx b/ui-react/src/pages/datasets/DatasetsSearch.tsx index 7adc4ea..4b75c6e 100644 --- a/ui-react/src/pages/datasets/DatasetsSearch.tsx +++ b/ui-react/src/pages/datasets/DatasetsSearch.tsx @@ -51,11 +51,13 @@ export function DatasetsSearch() { const trimmed = text.trim(); // An empty term is a valid search: it lists every dataset in the catalog. + // A variable term is a bare substring — the client matches it against the + // `mint_standard_variables` annotation, so there is no wildcard to express. const params: DatasetQueryParameters = !trimmed ? {} : type === 'dataset_names' ? { name: `*${trimmed}*` } - : { variables: [`*${trimmed}*`] }; + : { variableSubstring: trimmed }; setLoading(true); setError(null);