fix(ui-react): search datasets by variable name against mint_standard_variables (#96)

ui-react/src/lib/datasets/__tests__/ckan.test.ts

@@ -16,9 +16,11 @@ import {
   packageSpatialCoverage,
   packageTags,
   packagesMatchingVariables,
+  packagesMatchingVariableSubstring,
   packageTimePeriod,
   parseDate,
   resourceMatchesVariables,
+  resourceMatchesVariableSubstring,
   resourceStandardVariables,
   searchAllPackages,
   searchPackages,
@@ -192,36 +194,16 @@ describe('showPackage', () => {
 describe('buildSearchQuery', () => {
   it('returns undefined when there is nothing to search on', () => {
     expect(buildSearchQuery({})).toBeUndefined();
-    expect(buildSearchQuery({ variables: [] })).toBeUndefined();
+    expect(buildSearchQuery({ name: '' })).toBeUndefined();
   });
 
   it('quotes a single name as a phrase', () => {
     expect(buildSearchQuery({ name: 'soil moisture' })).toBe('"soil moisture"');
   });
 
-  it('ORs multiple variables inside parentheses', () => {
-    expect(buildSearchQuery({ variables: ['precipitation', 'temperature'] })).toBe(
-      '("precipitation" OR "temperature")',
-    );
-  });
-
-  it('does not parenthesise a lone variable', () => {
-    expect(buildSearchQuery({ variables: ['precipitation'] })).toBe('"precipitation"');
-  });
-
-  it('ANDs a name together with the variable clause', () => {
-    expect(buildSearchQuery({ name: 'ethiopia', variables: ['a', 'b'] })).toBe(
-      '"ethiopia" AND ("a" OR "b")',
-    );
-  });
-
   it('escapes embedded quotes so Solr does not see an unbalanced phrase', () => {
     expect(buildSearchQuery({ name: 'say "hi"' })).toBe('"say \\"hi\\""');
   });
-
-  it('drops empty variable entries', () => {
-    expect(buildSearchQuery({ variables: ['', 'real'] })).toBe('"real"');
-  });
 });
 
 // ─── Shaping helpers ──────────────────────────────────────────────────────────
@@ -418,6 +400,102 @@ describe('packagesMatchingVariables', () => {
   });
 });
 
+describe('resourceMatchesVariableSubstring', () => {
+  const annotated = { mint_standard_variables: 'groundwater__hydraulic_head' };
+
+  it('matches a fragment of the variable name', () => {
+    expect(resourceMatchesVariableSubstring(annotated, 'hydraulic')).toBe(true);
+    expect(resourceMatchesVariableSubstring(annotated, 'groundwater')).toBe(true);
+  });
+
+  it('matches across the underscores Solr tokenises on', () => {
+    // The whole point: `q` cannot find this, because Solr splits the name up.
+    expect(resourceMatchesVariableSubstring(annotated, 'water__hydraulic')).toBe(true);
+  });
+
+  it('ignores case on both sides', () => {
+    expect(resourceMatchesVariableSubstring(annotated, 'Hydraulic HEAD'.slice(0, 9))).toBe(true);
+    expect(
+      resourceMatchesVariableSubstring({ mint_standard_variables: 'Corpus_NLP' }, 'corpus_nlp'),
+    ).toBe(true);
+  });
+
+  it('does not match a term absent from the annotation', () => {
+    expect(resourceMatchesVariableSubstring(annotated, 'porosity')).toBe(false);
+  });
+
+  it('never matches an unannotated resource against a real term', () => {
+    expect(resourceMatchesVariableSubstring({ format: 'CSV' }, 'groundwater')).toBe(false);
+  });
+
+  it('passes everything through when the term is empty or blank', () => {
+    expect(resourceMatchesVariableSubstring({ format: 'CSV' }, '')).toBe(true);
+    expect(resourceMatchesVariableSubstring({ format: 'CSV' }, '   ')).toBe(true);
+  });
+});
+
+describe('packagesMatchingVariableSubstring', () => {
+  const carrier: CkanPackage = {
+    name: 'capitan-reef-complex-aquifer-gam-files',
+    resources: [
+      { id: 'r1', mint_standard_variables: 'groundwater__initial_head' },
+      { id: 'r2', mint_standard_variables: 'aquifer__transmissivity' },
+      { id: 'r3' },
+    ],
+  };
+
+  /** Reads as a match in prose, carries no annotation. This is what `q` returned. */
+  const falsePositive: CkanPackage = {
+    name: 'groundwater-initial-head-report',
+    notes: 'A report on groundwater initial head across the basin.',
+    resources: [{ id: 'r4', mint_standard_variables: '' }],
+  };
+
+  it('keeps a package whose resource carries a variable containing the term', () => {
+    expect(packagesMatchingVariableSubstring([carrier], 'head').map((p) => p.name)).toEqual([
+      carrier.name,
+    ]);
+  });
+
+  it('drops the prose match that free-text search returned', () => {
+    expect(packagesMatchingVariableSubstring([falsePositive], 'groundwater')).toEqual([]);
+  });
+
+  it('finds an underscored name that free-text search cannot', () => {
+    // `corpus_nlp` returned 0 through Solr while 17 TACC datasets carried it.
+    const pkg: CkanPackage = {
+      name: 'nlp-corpus',
+      resources: [{ id: 'r5', mint_standard_variables: 'corpus_nlp' }],
+    };
+    expect(packagesMatchingVariableSubstring([pkg], 'corpus_nlp')).toHaveLength(1);
+  });
+
+  it('narrows the kept package to the resources that carry the term', () => {
+    const [pkg] = packagesMatchingVariableSubstring([carrier], 'transmissivity');
+    expect(pkg?.resources?.map((r) => r.id)).toEqual(['r2']);
+  });
+
+  it('keeps every resource whose annotation contains the term', () => {
+    const [pkg] = packagesMatchingVariableSubstring([carrier], 'a');
+    expect(pkg?.resources?.map((r) => r.id)).toEqual(['r1', 'r2']);
+  });
+
+  it('leaves the input package untouched', () => {
+    packagesMatchingVariableSubstring([carrier], 'head');
+    expect(carrier.resources).toHaveLength(3);
+  });
+
+  it('passes everything through when the term is empty', () => {
+    const all = [carrier, falsePositive];
+    expect(packagesMatchingVariableSubstring(all, '')).toBe(all);
+    expect(packagesMatchingVariableSubstring(all, '  ')).toBe(all);
+  });
+
+  it('drops a package with no resources at all', () => {
+    expect(packagesMatchingVariableSubstring([{ name: 'empty' }], 'head')).toEqual([]);
+  });
+});
+
 describe('packageTags', () => {
   it('extracts tag names and drops empty ones', () => {
     expect(packageTags({ tags: [{ name: 'climate' }, { name: '' }, {}] })).toEqual(['climate']);

ui-react/src/lib/datasets/__tests__/data-catalog-api.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Tests for the CKAN-backed data catalog client.
+ *
+ * The behaviour under test is what `/datasets/search` asks of it. Standard
+ * variable names cannot be searched through CKAN's `q` — Solr does not index
+ * `mint_standard_variables` and tokenises the names on `_` — so the client must
+ * read the catalog and match the annotation itself. See the header of ./ckan.
+ */
+import { describe, expect, it, beforeEach, afterEach } from 'vitest';
+import { http, HttpResponse } from 'msw';
+import { server } from '@/test/msw/server';
+import { searchDatasets } from '../data-catalog-api';
+import type { CkanPackage } from '../ckan';
+
+const CKAN_HOST = 'https://ckan.example.org';
+
+/** Every `package_search` URL the client asked for, in order. */
+let requestedUrls: URL[] = [];
+
+/**
+ * Serve `packages` from `package_search`, honouring `rows`/`start` so that a
+ * client which fails to page sees a truncated catalog, exactly as CKAN does.
+ */
+function stubCatalog(packages: CkanPackage[]) {
+  server.use(
+    http.get('*/api/3/action/package_search', ({ request }) => {
+      const url = new URL(request.url);
+      requestedUrls.push(url);
+      const start = Number(url.searchParams.get('start') ?? 0);
+      const rows = Number(url.searchParams.get('rows') ?? 100);
+      return HttpResponse.json({
+        success: true,
+        result: { count: packages.length, results: packages.slice(start, start + rows) },
+      });
+    }),
+  );
+}
+
+/** A package carrying `variables` on its single resource. */
+function annotated(name: string, ...variables: string[]): CkanPackage {
+  return {
+    id: name,
+    name,
+    title: name,
+    resources: [{ id: `${name}-r1`, format: 'CSV', mint_standard_variables: variables.join(',') }],
+  };
+}
+
+beforeEach(() => {
+  requestedUrls = [];
+  window.__MINT_CONFIG__ = { DATA_CATALOG_API: CKAN_HOST } as never;
+});
+
+afterEach(() => {
+  delete (window as { __MINT_CONFIG__?: unknown }).__MINT_CONFIG__;
+});
+
+describe('searchDatasets — variable name search', () => {
+  const catalog: CkanPackage[] = [
+    annotated('gam-heads', 'groundwater__initial_head'),
+    annotated('gam-transmissivity', 'aquifer__transmissivity'),
+    annotated('nlp-corpus', 'corpus_nlp'),
+    {
+      // Reads as groundwater in prose, carries no annotation at all.
+      id: 'groundwater-report',
+      name: 'groundwater-report',
+      title: 'Groundwater report',
+      notes: 'A report on groundwater across the basin.',
+      resources: [{ id: 'gr-r1', format: 'PDF' }],
+    },
+  ];
+
+  it('matches the annotation rather than prose', async () => {
+    stubCatalog(catalog);
+    const results = await searchDatasets({ variableSubstring: 'groundwater' });
+    expect(results.map((d) => d.id)).toEqual(['gam-heads']);
+  });
+
+  it('finds an underscored variable name that a Solr q cannot', async () => {
+    stubCatalog(catalog);
+    const results = await searchDatasets({ variableSubstring: 'corpus_nlp' });
+    expect(results.map((d) => d.id)).toEqual(['nlp-corpus']);
+  });
+
+  it('never sends the variable term to CKAN as a query', async () => {
+    stubCatalog(catalog);
+    await searchDatasets({ variableSubstring: 'groundwater' });
+    for (const url of requestedUrls) {
+      expect(url.searchParams.get('q')).toBeNull();
+    }
+  });
+
+  it('matches on a fragment of the name, not just the whole name', async () => {
+    stubCatalog(catalog);
+    const results = await searchDatasets({ variableSubstring: 'transmiss' });
+    expect(results.map((d) => d.id)).toEqual(['gam-transmissivity']);
+  });
+
+  it('ignores case', async () => {
+    stubCatalog(catalog);
+    const results = await searchDatasets({ variableSubstring: 'AQUIFER' });
+    expect(results.map((d) => d.id)).toEqual(['gam-transmissivity']);
+  });
+
+  it('reports the variables the dataset actually carries', async () => {
+    stubCatalog([annotated('multi', 'soil__porosity', 'soil__water_content')]);
+    const [ds] = await searchDatasets({ variableSubstring: 'porosity' });
+    expect(ds?.variables).toEqual(['soil__porosity', 'soil__water_content']);
+  });
+
+  it('lists the whole catalog when no term is given', async () => {
+    stubCatalog(catalog);
+    const results = await searchDatasets({});
+    expect(results).toHaveLength(catalog.length);
+  });
+});
+
+describe('searchDatasets — paging', () => {
+  /** More packages than CKAN's per-request default of 100. */
+  const big: CkanPackage[] = Array.from({ length: 215 }, (_, i) =>
+    annotated(`pkg-${i}`, i % 2 === 0 ? 'soil__porosity' : 'soil__water_content'),
+  );
+
+  it('reads past the 100-row default when listing every dataset', async () => {
+    stubCatalog(big);
+    const results = await searchDatasets({});
+    expect(results).toHaveLength(215);
+  });
+
+  it('reads the whole catalog before matching variables, not just the first page', async () => {
+    stubCatalog(big);
+    const results = await searchDatasets({ variableSubstring: 'porosity' });
+    expect(results).toHaveLength(108);
+  });
+
+  it('reads past the 100-row default for a name search too', async () => {
+    stubCatalog(big);
+    const results = await searchDatasets({ name: '*pkg*' });
+    expect(results).toHaveLength(215);
+    expect(requestedUrls[0]?.searchParams.get('q')).toBe('"*pkg*"');
+  });
+});