intent/evals/intent-discovery/corpus/tasks.ts at fb6bb782c4476e327eb25ad0cdec2b0e4c97fade · TanStack/intent · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import type {
  IntentDiscoveryCondition,
  PromptExplicitnessLevel,
} from './conditions'

const expectedSkillAreas = ['router', 'start', 'table-v9'] as const

export type ExpectedSkillArea = (typeof expectedSkillAreas)[number]

export type IntentDiscoveryFixture =
  'router-basic' | 'start-basic' | 'table-v9-basic'

export type IntentDiscoveryFailureClass =
  | 'strict-success'
  | 'no-discovery-attempt'
  | 'instruction-ignored'
  | 'wrong-surface'
  | 'command-unknown'
  | 'command-attempted-but-failed'
  | 'wrong-skill-selected'
  | 'late-load'
  | 'reference-only'
  | 'final-output-only'
  | 'context-saturation'
  | 'prompt-too-vague'
  | 'harness-error'

type IntentDiscoveryExpected = {
  strictInvocation: boolean
  correctSkillLoaded: boolean
  referenceOnly: boolean
  failureClass: IntentDiscoveryFailureClass
}

export type IntentDiscoveryTask = {
  id: string
  fixture: IntentDiscoveryFixture
  condition: IntentDiscoveryCondition
  explicitnessLevel: PromptExplicitnessLevel
  prompt: string
  expectedSkillAreas: Array<ExpectedSkillArea>
  expected: IntentDiscoveryExpected
}

export const tasks: Array<IntentDiscoveryTask> = [
  {
    id: 'router-current-intent-loads-router',
    fixture: 'router-basic',
    condition: 'current-intent',
    explicitnessLevel: 2,
    prompt: 'Add a route that loads user data before rendering the page.',
    expectedSkillAreas: ['router'],
    expected: {
      strictInvocation: true,
      correctSkillLoaded: true,
      referenceOnly: false,
      failureClass: 'strict-success',
    },
  },
  {
    id: 'router-plain-docs-reference-only',
    fixture: 'router-basic',
    condition: 'plain-docs',
    explicitnessLevel: 2,
    prompt: 'Add a route that loads user data before rendering the page.',
    expectedSkillAreas: ['router'],
    expected: {
      strictInvocation: false,
      correctSkillLoaded: false,
      referenceOnly: true,
      failureClass: 'reference-only',
    },
  },
  {
    id: 'table-v9-current-intent-loads-wrong-skill',
    fixture: 'table-v9-basic',
    condition: 'current-intent',
    explicitnessLevel: 2,
    prompt: 'Add a TanStack Table v9 column with sortable user roles.',
    expectedSkillAreas: ['table-v9'],
    expected: {
      strictInvocation: true,
      correctSkillLoaded: false,
      referenceOnly: false,
      failureClass: 'wrong-skill-selected',
    },
  },
]