|
| 1 | +import { |
| 2 | + generateTeamProjects, |
| 3 | + projectsTokenBudget, |
| 4 | + TeamProjectsUnusableError, |
| 5 | +} from '@/lib/team-pulse/projects'; |
| 6 | + |
| 7 | +/** |
| 8 | + * GLOOK-51. The Integrations team (14 developers) showed an empty Current |
| 9 | + * Projects section on every reload. The model HAD produced good output — it was |
| 10 | + * truncated mid-object by a flat tokenLimit(1500), JSON.parse threw, the catch |
| 11 | + * returned [], and service.ts cached that [] as a success. Because the lazy |
| 12 | + * top-up only re-runs when the column is NULL, one truncation blanked the |
| 13 | + * section permanently. |
| 14 | + * |
| 15 | + * The contract: a failure must be distinguishable from "this team genuinely has |
| 16 | + * no projects", because only the latter may be cached. |
| 17 | + */ |
| 18 | + |
| 19 | +const mockCreate = jest.fn(); |
| 20 | +jest.mock('@/lib/llm-provider', () => ({ |
| 21 | + getLLMClient: async () => ({ chat: { completions: { create: mockCreate } } }), |
| 22 | + LLM_MODEL: 'test-model', |
| 23 | + extraBodyProps: () => ({}), |
| 24 | + // Mirrors the real tokenLimit(): the openai provider gets |
| 25 | + // max_completion_tokens, everything else max_tokens. Hard-coding max_tokens |
| 26 | + // here would assert against a shape production may not use. |
| 27 | + tokenLimit: (n: number) => ({ max_tokens: n, max_completion_tokens: n }), |
| 28 | + promptTag: () => ({}), |
| 29 | + samplingParams: () => ({}), |
| 30 | +})); |
| 31 | +jest.mock('@/lib/prompt-loader', () => ({ loadPrompt: () => 'SYSTEM PROMPT' })); |
| 32 | + |
| 33 | +function input(teamSize: number) { |
| 34 | + const team_members = Array.from({ length: teamSize }, (_, i) => `dev${i}`); |
| 35 | + return { |
| 36 | + team_members, |
| 37 | + commits: [{ github_login: 'dev0', committed_at: '2026-09-10T00:00:00Z', commit_message: 'x', repo: 'r' }], |
| 38 | + jira_issues: [], |
| 39 | + in_flight_prs: [], |
| 40 | + in_flight_branches: [], |
| 41 | + } as any; |
| 42 | +} |
| 43 | + |
| 44 | +function reply(content: string, finish_reason = 'stop') { |
| 45 | + return { choices: [{ finish_reason, message: { content } }] }; |
| 46 | +} |
| 47 | + |
| 48 | +beforeEach(() => mockCreate.mockReset()); |
| 49 | + |
| 50 | +describe('projectsTokenBudget', () => { |
| 51 | + it('scales with team size instead of a flat cap', () => { |
| 52 | + // The flat 1500 is what truncated a 14-developer team. |
| 53 | + expect(projectsTokenBudget(14)).toBeGreaterThan(1500); |
| 54 | + expect(projectsTokenBudget(14)).toBeGreaterThan(projectsTokenBudget(3)); |
| 55 | + }); |
| 56 | + |
| 57 | + it('is bounded, and non-zero for an empty roster', () => { |
| 58 | + expect(projectsTokenBudget(1000)).toBeLessThanOrEqual(8000); |
| 59 | + expect(projectsTokenBudget(0)).toBeGreaterThan(0); |
| 60 | + }); |
| 61 | +}); |
| 62 | + |
| 63 | +describe('an unusable response RAISES rather than returning []', () => { |
| 64 | + it('raises on truncation, naming it as truncation', async () => { |
| 65 | + // The exact incident shape: good JSON, cut off mid-object. |
| 66 | + mockCreate.mockResolvedValue(reply( |
| 67 | + '{"projects":[{"name":"Datadog On-Call Migration","estimated_prs":20,', |
| 68 | + 'length', |
| 69 | + )); |
| 70 | + await expect(generateTeamProjects(input(14), 'Integrations')) |
| 71 | + .rejects.toThrow(TeamProjectsUnusableError); |
| 72 | + await expect(generateTeamProjects(input(14), 'Integrations')) |
| 73 | + .rejects.toThrow(/truncated[\s\S]*finish_reason=length/); |
| 74 | + }); |
| 75 | + |
| 76 | + it('raises on unparseable output that was NOT truncated', async () => { |
| 77 | + mockCreate.mockResolvedValue(reply('not json at all')); |
| 78 | + await expect(generateTeamProjects(input(5), 'Data')) |
| 79 | + .rejects.toThrow(/unparseable/); |
| 80 | + }); |
| 81 | + |
| 82 | + it('reports truncation distinctly from malformed JSON', async () => { |
| 83 | + // Both used to log "parse error", which sent the investigation looking at |
| 84 | + // the model instead of the token budget. |
| 85 | + mockCreate.mockResolvedValue(reply('{"projects":[{"name":"x",', 'length')); |
| 86 | + const truncated = await generateTeamProjects(input(14), 'T').catch((e) => e.message); |
| 87 | + mockCreate.mockResolvedValue(reply('garbage')); |
| 88 | + const malformed = await generateTeamProjects(input(14), 'T').catch((e) => e.message); |
| 89 | + expect(truncated).toMatch(/truncated/); |
| 90 | + expect(malformed).not.toMatch(/truncated/); |
| 91 | + }); |
| 92 | +}); |
| 93 | + |
| 94 | +describe('a genuine empty result is still a success', () => { |
| 95 | + it('returns [] without raising when the model reports no projects', async () => { |
| 96 | + // Must stay cacheable — otherwise every load pays for the LLM. |
| 97 | + mockCreate.mockResolvedValue(reply('{"projects":[]}')); |
| 98 | + await expect(generateTeamProjects(input(5), 'Small')) |
| 99 | + .resolves.toEqual({ projects: [], cacheable: true }); |
| 100 | + }); |
| 101 | + |
| 102 | + it('short-circuits with no LLM call when there is nothing to cluster', async () => { |
| 103 | + const empty = { team_members: ['a'], commits: [], jira_issues: [], in_flight_prs: [], in_flight_branches: [] } as any; |
| 104 | + await expect(generateTeamProjects(empty, 'Quiet')) |
| 105 | + .resolves.toEqual({ projects: [], cacheable: true }); |
| 106 | + expect(mockCreate).not.toHaveBeenCalled(); |
| 107 | + }); |
| 108 | +}); |
| 109 | + |
| 110 | +describe('a usable response is still parsed normally', () => { |
| 111 | + it('keeps projects whose developers are on the team', async () => { |
| 112 | + mockCreate.mockResolvedValue(reply(JSON.stringify({ |
| 113 | + projects: [ |
| 114 | + { name: 'Real', summary: 's', developers: ['dev0'], jira_count: 1, estimated_commits: 5, estimated_prs: 2 }, |
| 115 | + { name: 'Nobody', summary: 's', developers: ['stranger'], jira_count: 0 }, |
| 116 | + ], |
| 117 | + }))); |
| 118 | + const { projects: out } = await generateTeamProjects(input(3), 'T'); |
| 119 | + expect(out).toHaveLength(1); |
| 120 | + expect(out[0].name).toBe('Real'); |
| 121 | + // last_activity is overridden from real commit data. |
| 122 | + expect(out[0].last_activity).toBe('2026-09-10T00:00:00Z'); |
| 123 | + }); |
| 124 | + |
| 125 | + it('requests a literal budget for the incident team, not just "whatever the fn returns"', async () => { |
| 126 | + // Asserted as a literal on purpose: comparing against |
| 127 | + // projectsTokenBudget(14) compares the implementation to itself, and a |
| 128 | + // regression to the flat 1500 that caused GLOOK-51 would still pass. |
| 129 | + mockCreate.mockResolvedValue(reply('{"projects":[]}')); |
| 130 | + await generateTeamProjects(input(14), 'Integrations'); |
| 131 | + expect(projectsTokenBudget(14)).toBe(7600); |
| 132 | + expect(mockCreate.mock.calls[0][0].max_tokens).toBe(7600); |
| 133 | + expect(mockCreate.mock.calls[0][0].max_completion_tokens).toBe(7600); |
| 134 | + }); |
| 135 | +}); |
| 136 | + |
| 137 | +describe('an empty that the validator produced is served but NOT cached', () => { |
| 138 | + it('marks cacheable=false when the model named nobody on the team', async () => { |
| 139 | + // The model returned projects; every developer failed the teamSet filter |
| 140 | + // (e.g. display names instead of logins). Caching this would blank the |
| 141 | + // section permanently — GLOOK-51 arriving by a second route. Deliberately |
| 142 | + // not a throw: the empty may be correct, and raising buys a paid retry loop. |
| 143 | + mockCreate.mockResolvedValue(reply(JSON.stringify({ |
| 144 | + projects: [{ name: 'Real work', summary: 's', developers: ['Some Person'] }], |
| 145 | + }))); |
| 146 | + const res = await generateTeamProjects(input(3), 'T'); |
| 147 | + expect(res.projects).toEqual([]); |
| 148 | + expect(res.cacheable).toBe(false); |
| 149 | + }); |
| 150 | + |
| 151 | + it('still marks a genuinely empty model response as cacheable', async () => { |
| 152 | + mockCreate.mockResolvedValue(reply('{"projects":[]}')); |
| 153 | + const res = await generateTeamProjects(input(3), 'T'); |
| 154 | + expect(res).toEqual({ projects: [], cacheable: true }); |
| 155 | + }); |
| 156 | +}); |
0 commit comments