chore: Remove leftover test logs (#1389) #465
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Braintrust Evals | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| types: | |
| - labeled | |
| - synchronize | |
| jobs: | |
| run-braintrust-evals: | |
| name: Run Braintrust Evals | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'braintrust-evals') | |
| env: | |
| BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} | |
| VOYAGE_API_KEY: ${{ secrets.TEST_MDB_MCP_VOYAGE_API_KEY }} | |
| EMBEDDING_PROVIDER_ENDPOINT: "https://api.voyageai.com/v1/embeddings" | |
| GIT_BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| # Limits concurrent evals to avoid slowdowns from simultaneous database or text/vector index creation per eval case | |
| EVAL_MAX_CONCURRENCY: 25 | |
| steps: | |
| - uses: GitHubSecurityLab/actions-permissions/monitor@v1 | |
| - uses: actions/checkout@v7 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@v5 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| # Braintrust push accepts Node 18–22 only | |
| # https://www.braintrust.dev/docs/evaluate/remote-evals#2-register-your-sandbox | |
| node-version: "22" | |
| cache: "pnpm" | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Push eval to Braintrust | |
| if: github.ref == 'refs/heads/main' | |
| run: pnpm run eval:push | |
| - name: Start local MongoDB | |
| run: pnpm run eval:db-start | |
| - name: Run Braintrust eval suite | |
| run: pnpm run eval:ci:run | |
| - name: Stop local MongoDB | |
| if: always() | |
| run: pnpm run eval:db-stop | |
| - name: Comment summary on PR | |
| if: github.event_name == 'pull_request' | |
| uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 | |
| with: | |
| hide_and_recreate: true | |
| hide_classify: "OUTDATED" | |
| path: .eval/ci-report.md |