{"meta":{"run_id":"38271632735-validation","workflow_run_id":"38271632735","ts":"2026-10-12T15:55:37.369Z","model":"groq:openai/gpt-oss-120b","repeats":3,"dataset":"evals/datasets/validation.jsonl","dataset_sha256":"c5608e3a9d4bb00e720004d3b5de563fef257900341899e9d682af5e7d0a474c","suite":"validation"},"summary":{"n_cases":35,"n_runs":105,"error_rate":0,"scores":{"verdict_match":{"mean":0.8667,"n":105},"score_in_range":{"mean":0.9259,"n":54},"blocker_match":{"mean":0.5758,"n":33},"suggested_ac_count":{"mean":0.9905,"n":105}},"consistency":0.9714,"confusion":{"valid":{"valid":36},"invalid":{"invalid":55,"valid":14}},"per_class":{"valid":{"precision":0.72,"recall":1,"f1":0.8372,"support":36},"invalid":{"precision":1,"recall":0.7971,"f1":0.8871,"support":69}},"latency_ms":{"p50":5194,"p95":23371},"llm_calls":105,"tokens_est":{"in":283308,"out":21321}},"failures":[{"metric":"per_class.invalid.recall","value":0.7971,"reason":"0.7971 < min 0.8"}],"results":[{"case_id":"valid-api-endpoint","tags":["valid","api","core"],"repeat":0,"duration_ms":1172,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["GET /health returns HTTP 200 with JSON body { \"status\": \"ok\" } and Content-Type header set to application/json","POST /health returns HTTP 405 Method Not Allowed","GET /health returns HTTP 200 when the server is started and listening on the configured port","GET /health returns HTTP 500 with JSON body { \"status\": \"error\" } when an unexpected internal error occurs while generating the response","A test file at test/health.test.mjs includes unit tests for both the successful GET response and the 405 response for POST"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"GET /health returns HTTP 200 with JSON body { \\\"status\\\": \\\"ok\\\" } and Content-Type header set to application/json\",\"POST /health returns HTTP 405 Method Not Allowed\",\"GET /health returns HTTP 200 when the server is started and listening on the configured port\",\"GET /health returns HTTP 500 with JSON body { \\\"status\\\": \\\"error\\\" } when an unexpected internal error occurs while generating the response\",\"A test file at test/health.test.mjs includes unit tests for both the successful GET response and the 405 response for POST\"]}","latency_ms":1172,"tokens_in_est":2707,"tokens_out_est":151}]},{"case_id":"valid-api-endpoint","tags":["valid","api","core"],"repeat":1,"duration_ms":1403,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W1: Issue does not specify which part of the Express app (e.g., router or main file) the new route should be added to, limiting implementation context","W2: No acceptance criteria for handling unexpected HTTP methods (e.g., PUT, DELETE) or malformed requests to /health"],"suggested_ac":["[ ] GET /health returns HTTP 200 with JSON body { \"status\": \"ok\" } and Content-Type header exactly \"application/json\"","[ ] POST /health returns HTTP 405 Method Not Allowed","[ ] Any other HTTP method (PUT, DELETE, PATCH) to /health returns HTTP 405","[ ] GET /health with an Accept header of \"application/xml\" still returns JSON with Content-Type \"application/json\"","[ ] A unit test file at test/health.test.mjs includes tests for GET success, POST 405, and other methods returning 405"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify which part of the Express app (e.g., router or main file) the new route should be added to, limiting implementation context\",\"W2: No acceptance criteria for handling unexpected HTTP methods (e.g., PUT, DELETE) or malformed requests to /health\"],\"suggested_ac\":[\"[ ] GET /health returns HTTP 200 with JSON body { \\\"status\\\": \\\"ok\\\" } and Content-Type header exactly \\\"application/json\\\"\",\"[ ] POST /health returns HTTP 405 Method Not Allowed\",\"[ ] Any other HTTP method (PUT, DELETE, PATCH) to /health returns HTTP 405\",\"[ ] GET /health with an Accept header of \\\"application/xml\\\" still returns JSON with Content-Type \\\"application/json\\\"\",\"[ ] A unit test file at test/health.test.mjs includes tests for GET success, POST 405, and other methods returning 405\"]}","latency_ms":1402,"tokens_in_est":2707,"tokens_out_est":211}]},{"case_id":"valid-api-endpoint","tags":["valid","api","core"],"repeat":2,"duration_ms":3273,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W2: Edge cases such as unsupported HTTP methods other than POST are not explicitly covered"],"suggested_ac":["- [ ] GET /health with an Accept header of application/json returns HTTP 200 and JSON body { \"status\": \"ok\" }","- [ ] GET /health without Accept header still returns HTTP 200 with JSON body { \"status\": \"ok\" }","- [ ] Any HTTP method other than GET (e.g., PUT, DELETE) on /health returns HTTP 405"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as unsupported HTTP methods other than POST are not explicitly covered\"],\"suggested_ac\":[\"- [ ] GET /health with an Accept header of application/json returns HTTP 200 and JSON body { \\\"status\\\": \\\"ok\\\" }\",\"- [ ] GET /health without Accept header still returns HTTP 200 with JSON body { \\\"status\\\": \\\"ok\\\" }\",\"- [ ] Any HTTP method other than GET (e.g., PUT, DELETE) on /health returns HTTP 405\"]}","latency_ms":3273,"tokens_in_est":2707,"tokens_out_est":118}]},{"case_id":"valid-pure-function","tags":["valid","lib","core"],"repeat":0,"duration_ms":20230,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given input 'Café au lait', when calling slugify, then it returns 'cafe-au-lait' (Unicode characters are normalized to ASCII)","Given input '---Hello---World---', when calling slugify, then it returns 'hello-world' (multiple leading/trailing hyphens are trimmed and collapsed)","Given input a string longer than 255 characters, when calling slugify, then it returns a string truncated to 255 characters without cutting off a word mid-way","Given input 'Hello   World', when calling slugify, then it returns 'hello-world' (multiple spaces collapse to a single hyphen)","Given input undefined, when calling slugify, then it throws TypeError"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given input 'Café au lait', when calling slugify, then it returns 'cafe-au-lait' (Unicode characters are normalized to ASCII)\",\"Given input '---Hello---World---', when calling slugify, then it returns 'hello-world' (multiple leading/trailing hyphens are trimmed and collapsed)\",\"Given input a string longer than 255 characters, when calling slugify, then it returns a string truncated to 255 characters without cutting off a word mid-way\",\"Given input 'Hello   World', when calling slugify, then it returns 'hello-world' (multiple spaces collapse to a single hyphen)\",\"Given input undefined, when calling slugify, then it throws TypeError\"]}","latency_ms":20230,"tokens_in_est":2710,"tokens_out_est":178}]},{"case_id":"valid-pure-function","tags":["valid","lib","core"],"repeat":1,"duration_ms":20910,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference existing implementation details or affected modules beyond the target file path"],"suggested_ac":["- [ ] `slugify(undefined)` throws `TypeError`","- [ ] `slugify('Hello---World')` returns `'hello-world'` (multiple consecutive hyphens are collapsed)","- [ ] `slugify('  Leading and trailing  ')` returns `'leading-and-trailing'` (whitespace is treated as non‑alphanumeric and trimmed)","- [ ] `slugify('Café')` returns `'cafe'` (unicode characters are normalized to ASCII)","- [ ] `slugify('123')` returns `'123'` (numeric strings are preserved unchanged)"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference existing implementation details or affected modules beyond the target file path\"],\"suggested_ac\":[\"- [ ] `slugify(undefined)` throws `TypeError`\",\"- [ ] `slugify('Hello---World')` returns `'hello-world'` (multiple consecutive hyphens are collapsed)\",\"- [ ] `slugify('  Leading and trailing  ')` returns `'leading-and-trailing'` (whitespace is treated as non‑alphanumeric and trimmed)\",\"- [ ] `slugify('Café')` returns `'cafe'` (unicode characters are normalized to ASCII)\",\"- [ ] `slugify('123')` returns `'123'` (numeric strings are preserved unchanged)\"]}","latency_ms":20910,"tokens_in_est":2710,"tokens_out_est":160}]},{"case_id":"valid-pure-function","tags":["valid","lib","core"],"repeat":2,"duration_ms":20939,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given the input 'Hello---World!!', when slugify is called, then it returns 'hello-world' (consecutive punctuation collapsed to a single hyphen)","Given the input '  Leading and trailing  ', when slugify is called, then it returns 'leading-and-trailing' (whitespace trimmed and collapsed)","Given the input 'Café Münchén', when slugify is called, then it returns 'cafe-munchen' (Unicode characters are normalized to ASCII)","Given the input '123_ABC', when slugify is called, then it returns '123-abc' (numeric characters are preserved and underscores become hyphens)","Given the input undefined, when slugify is called, then it throws TypeError"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given the input 'Hello---World!!', when slugify is called, then it returns 'hello-world' (consecutive punctuation collapsed to a single hyphen)\",\"Given the input '  Leading and trailing  ', when slugify is called, then it returns 'leading-and-trailing' (whitespace trimmed and collapsed)\",\"Given the input 'Café Münchén', when slugify is called, then it returns 'cafe-munchen' (Unicode characters are normalized to ASCII)\",\"Given the input '123_ABC', when slugify is called, then it returns '123-abc' (numeric characters are preserved and underscores become hyphens)\",\"Given the input undefined, when slugify is called, then it throws TypeError\"]}","latency_ms":20939,"tokens_in_est":2710,"tokens_out_est":180}]},{"case_id":"valid-bugfix","tags":["valid","bug","core"],"repeat":0,"duration_ms":21882,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: Issue does not specify handling of multiple colons in a single value or whitespace trimming behavior","W2: Edge cases such as empty input string, keys with leading/trailing spaces, or values containing multiple colons are not covered"],"suggested_ac":["- [ ] `parseFlatYaml('path: /api/v1/resource:detail')` returns `{ path: '/api/v1/resource:detail' }`","- [ ] `parseFlatYaml('  key  :  value  ')` trims whitespace and returns `{ key: 'value' }`","- [ ] `parseFlatYaml('')` returns an empty object `{}`","- [ ] `parseFlatYaml('key:')` returns `{ key: '' }`","- [ ] `parseFlatYaml('key: value:with:colons')` returns `{ key: 'value:with:colons' }`"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify handling of multiple colons in a single value or whitespace trimming behavior\",\"W2: Edge cases such as empty input string, keys with leading/trailing spaces, or values containing multiple colons are not covered\"],\"suggested_ac\":[\"- [ ] `parseFlatYaml('path: /api/v1/resource:detail')` returns `{ path: '/api/v1/resource:detail' }`\",\"- [ ] `parseFlatYaml('  key  :  value  ')` trims whitespace and returns `{ key: 'value' }`\",\"- [ ] `parseFlatYaml('')` returns an empty object `{}`\",\"- [ ] `parseFlatYaml('key:')` returns `{ key: '' }`\",\"- [ ] `parseFlatYaml('key: value:with:colons')` returns `{ key: 'value:with:colons' }`\"]}","latency_ms":21882,"tokens_in_est":2717,"tokens_out_est":177}]},{"case_id":"valid-bugfix","tags":["valid","bug","core"],"repeat":1,"duration_ms":22144,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given input `parseFlatYaml('')`, when parsed, then it returns an empty object `{}`","Given input `parseFlatYaml('path: /api/v1/resource:detail')`, when parsed, then it returns `{ path: '/api/v1/resource:detail' }`","Given input `parseFlatYaml('key:')`, when parsed, then it returns `{ key: '' }","Given input `parseFlatYaml('multiple: colon: in: value')`, when parsed, then it returns `{ multiple: 'colon: in: value' }","Given input `parseFlatYaml('  spacedKey  :   spaced value  ')`, when parsed, then it returns `{ spacedKey: 'spaced value' }"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given input `parseFlatYaml('')`, when parsed, then it returns an empty object `{}`\",\"Given input `parseFlatYaml('path: /api/v1/resource:detail')`, when parsed, then it returns `{ path: '/api/v1/resource:detail' }`\",\"Given input `parseFlatYaml('key:')`, when parsed, then it returns `{ key: '' }\",\"Given input `parseFlatYaml('multiple: colon: in: value')`, when parsed, then it returns `{ multiple: 'colon: in: value' }\",\"Given input `parseFlatYaml('  spacedKey  :   spaced value  ')`, when parsed, then it returns `{ spacedKey: 'spaced value' }\"]}","latency_ms":22144,"tokens_in_est":2717,"tokens_out_est":155}]},{"case_id":"valid-bugfix","tags":["valid","bug","core"],"repeat":2,"duration_ms":3825,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W2: Edge cases such as multiple colons in a value, empty input string, or keys with surrounding whitespace are not explicitly covered in the acceptance criteria."],"suggested_ac":["- Given the input `parseFlatYaml('path: /api/v1/resource:detail')`, when parsed, then it returns `{ path: '/api/v1/resource:detail' }`","- Given the input `parseFlatYaml('')`, when parsed, then it returns an empty object `{}`","- Given the input `parseFlatYaml('key:')`, when parsed, then it returns `{ key: '' }`","- Given the input `parseFlatYaml('  spacedKey  :  value with spaces  ')`, when parsed, then it returns `{ spacedKey: 'value with spaces' }","- Given the input `parseFlatYaml('list: item1, item2')`, when parsed, then it returns `{ list: 'item1, item2' }"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as multiple colons in a value, empty input string, or keys with surrounding whitespace are not explicitly covered in the acceptance criteria.\"],\"suggested_ac\":[\"- Given the input `parseFlatYaml('path: /api/v1/resource:detail')`, when parsed, then it returns `{ path: '/api/v1/resource:detail' }`\",\"- Given the input `parseFlatYaml('')`, when parsed, then it returns an empty object `{}`\",\"- Given the input `parseFlatYaml('key:')`, when parsed, then it returns `{ key: '' }`\",\"- Given the input `parseFlatYaml('  spacedKey  :  value with spaces  ')`, when parsed, then it returns `{ spacedKey: 'value with spaces' }\",\"- Given the input `parseFlatYaml('list: item1, item2')`, when parsed, then it returns `{ list: 'item1, item2' }\"]}","latency_ms":3825,"tokens_in_est":2717,"tokens_out_est":202}]},{"case_id":"valid-docs-diagram","tags":["valid","docs","core"],"repeat":0,"duration_ms":22593,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["[ ] The README renders the Mermaid diagram without errors on GitHub's web UI","[ ] The `## Pipeline` heading uses exactly two hash symbols and is followed by a blank line before the mermaid block","[ ] The mermaid block starts with ```mermaid and ends with ``` on its own line","[ ] No existing content in README is altered aside from the addition of the new section (no whitespace or line ending changes)","[ ] The diagram includes exactly the nodes: Issue Validation, Code Generation, PR Review, Auto-fix, Human Merge, with directed edges Issue Validation->Code Generation->PR Review->Auto-fix->PR Review->Human Merge"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"[ ] The README renders the Mermaid diagram without errors on GitHub's web UI\",\"[ ] The `## Pipeline` heading uses exactly two hash symbols and is followed by a blank line before the mermaid block\",\"[ ] The mermaid block starts with ```mermaid and ends with ``` on its own line\",\"[ ] No existing content in README is altered aside from the addition of the new section (no whitespace or line ending changes)\",\"[ ] The diagram includes exactly the nodes: Issue Validation, Code Generation, PR Review, Auto-fix, Human Merge, with directed edges Issue Validation->Code Generation->PR Review->Auto-fix->PR Review->Human Merge\"]}","latency_ms":22593,"tokens_in_est":2732,"tokens_out_est":174}]},{"case_id":"valid-docs-diagram","tags":["valid","docs","core"],"repeat":1,"duration_ms":21885,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["[ ] The README renders the Mermaid diagram without syntax errors and displays all nodes and edges as specified","[ ] Adding the diagram does not alter any existing content or formatting outside the new ## Pipeline section","[ ] The mermaid block is correctly fenced with triple backticks and the language identifier `mermaid`","[ ] The diagram renders correctly on GitHub's web UI across supported browsers","[ ] The commit modifies only README.md and no other files"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"[ ] The README renders the Mermaid diagram without syntax errors and displays all nodes and edges as specified\",\"[ ] Adding the diagram does not alter any existing content or formatting outside the new ## Pipeline section\",\"[ ] The mermaid block is correctly fenced with triple backticks and the language identifier `mermaid`\",\"[ ] The diagram renders correctly on GitHub's web UI across supported browsers\",\"[ ] The commit modifies only README.md and no other files\"]}","latency_ms":21885,"tokens_in_est":2732,"tokens_out_est":135}]},{"case_id":"valid-docs-diagram","tags":["valid","docs","core"],"repeat":2,"duration_ms":20814,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["[ ] The README.md file includes a new \"## Pipeline\" heading followed by a fenced mermaid code block containing exactly the nodes \"Issue Validation\", \"Code Generation\", \"PR Review\", \"Auto-fix\", and \"Human Merge\"","[ ] The mermaid diagram renders without syntax errors in GitHub preview (no error messages shown)","[ ] Directed edges are present in the order Issue Validation → Code Generation → PR Review → Auto-fix → PR Review → Human Merge, matching the specified sequence","[ ] No existing sections or content in README.md are altered aside from the addition of the new Pipeline section","[ ] The mermaid block is correctly fenced with triple backticks and the language identifier \"mermaid\""]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"[ ] The README.md file includes a new \\\"## Pipeline\\\" heading followed by a fenced mermaid code block containing exactly the nodes \\\"Issue Validation\\\", \\\"Code Generation\\\", \\\"PR Review\\\", \\\"Auto-fix\\\", and \\\"Human Merge\\\"\",\"[ ] The mermaid diagram renders without syntax errors in GitHub preview (no error messages shown)\",\"[ ] Directed edges are present in the order Issue Validation → Code Generation → PR Review → Auto-fix → PR Review → Human Merge, matching the specified sequence\",\"[ ] No existing sections or content in README.md are altered aside from the addition of the new Pipeline section\",\"[ ] The mermaid block is correctly fenced with triple backticks and the language identifier \\\"mermaid\\\"\"]}","latency_ms":20814,"tokens_in_est":2732,"tokens_out_est":195}]},{"case_id":"valid-config-change","tags":["valid","config","core"],"repeat":0,"duration_ms":22041,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["[ ] Loading the validation model config returns maxTokens equal to 1536 and all other fields match the previous configuration","[ ] Loading any non‑validation model config (e.g., 'default') retains its original maxTokens value","[ ] Attempting to set validation_max_tokens to a non‑numeric value results in a validation error and the config file remains unchanged","[ ] The test suite fails if `config/models.yaml` contains any additional modifications beyond `validation_max_tokens: 1536`"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"[ ] Loading the validation model config returns maxTokens equal to 1536 and all other fields match the previous configuration\",\"[ ] Loading any non‑validation model config (e.g., 'default') retains its original maxTokens value\",\"[ ] Attempting to set validation_max_tokens to a non‑numeric value results in a validation error and the config file remains unchanged\",\"[ ] The test suite fails if `config/models.yaml` contains any additional modifications beyond `validation_max_tokens: 1536`\"]}","latency_ms":22041,"tokens_in_est":2712,"tokens_out_est":141}]},{"case_id":"valid-config-change","tags":["valid","config","core"],"repeat":1,"duration_ms":19261,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["- [ ] Loading the `validation` model config returns an object where `maxTokens` equals 1536","- [ ] Loading any other model config (e.g., `default`) retains its original `maxTokens` value","- [ ] The `config/models.yaml` file contains no additional modifications besides the updated `validation_max_tokens` value","- [ ] The test suite fails if `validation_max_tokens` is set to any value other than 1536","- [ ] The change does not introduce syntax errors; the YAML file parses successfully"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"- [ ] Loading the `validation` model config returns an object where `maxTokens` equals 1536\",\"- [ ] Loading any other model config (e.g., `default`) retains its original `maxTokens` value\",\"- [ ] The `config/models.yaml` file contains no additional modifications besides the updated `validation_max_tokens` value\",\"- [ ] The test suite fails if `validation_max_tokens` is set to any value other than 1536\",\"- [ ] The change does not introduce syntax errors; the YAML file parses successfully\"]}","latency_ms":19261,"tokens_in_est":2712,"tokens_out_est":142}]},{"case_id":"valid-config-change","tags":["valid","config","core"],"repeat":2,"duration_ms":3024,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["- [ ] Loading the validation model configuration returns an object where maxTokens equals 1536 and all other properties match the previous version","- [ ] The test suite fails if `validation_max_tokens` is set to any value other than 1536","- [ ] Running the application with the updated config does not produce token‑truncation errors for JSON outputs in validation mode","- [ ] A regression test confirms that `config/models.yaml` contains no additional modifications beyond the `validation_max_tokens` change","- [ ] Documentation in README is updated to reflect the new `validation_max_tokens` limit for the validation model"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"- [ ] Loading the validation model configuration returns an object where maxTokens equals 1536 and all other properties match the previous version\",\"- [ ] The test suite fails if `validation_max_tokens` is set to any value other than 1536\",\"- [ ] Running the application with the updated config does not produce token‑truncation errors for JSON outputs in validation mode\",\"- [ ] A regression test confirms that `config/models.yaml` contains no additional modifications beyond the `validation_max_tokens` change\",\"- [ ] Documentation in README is updated to reflect the new `validation_max_tokens` limit for the validation model\"]}","latency_ms":3024,"tokens_in_est":2712,"tokens_out_est":176}]},{"case_id":"valid-error-handling","tags":["valid","edge","core"],"repeat":0,"duration_ms":21075,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge case of file read permission errors not addressed"],"suggested_ac":["- When the file exists but cannot be read due to permission errors, `readCheckpoint` returns `null` and logs an error via `logger.error` with the file path","- When the file contains valid JSON, no warning is logged via `logger.error`","- The function never throws; it always returns either the parsed object or `null` for any error condition","- Unit tests assert that `logger.error` is called exactly once for corrupt JSON and not called for valid or missing files"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge case of file read permission errors not addressed\"],\"suggested_ac\":[\"- When the file exists but cannot be read due to permission errors, `readCheckpoint` returns `null` and logs an error via `logger.error` with the file path\",\"- When the file contains valid JSON, no warning is logged via `logger.error`\",\"- The function never throws; it always returns either the parsed object or `null` for any error condition\",\"- Unit tests assert that `logger.error` is called exactly once for corrupt JSON and not called for valid or missing files\"]}","latency_ms":21075,"tokens_in_est":2730,"tokens_out_est":150}]},{"case_id":"valid-error-handling","tags":["valid","edge","core"],"repeat":1,"duration_ms":21864,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W1: Issue does not specify how the logger is imported or accessed in the module"],"suggested_ac":["- When the file contains invalid JSON, `readCheckpoint` returns `null` and logs an error with the exact file path using the project's logger instance","- When the file is missing, `readCheckpoint` returns `null` without throwing and does not log an error","- When the file is valid JSON, `readCheckpoint` returns the parsed object exactly as `JSON.parse` would, preserving all data types","- If the file exists but the process lacks read permissions, `readCheckpoint` returns `null` and logs an error indicating a permission issue","- The test suite includes unit tests for each scenario and verifies that `logger.error` is called with the correct message for corrupt JSON and permission errors"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify how the logger is imported or accessed in the module\"],\"suggested_ac\":[\"- When the file contains invalid JSON, `readCheckpoint` returns `null` and logs an error with the exact file path using the project's logger instance\",\"- When the file is missing, `readCheckpoint` returns `null` without throwing and does not log an error\",\"- When the file is valid JSON, `readCheckpoint` returns the parsed object exactly as `JSON.parse` would, preserving all data types\",\"- If the file exists but the process lacks read permissions, `readCheckpoint` returns `null` and logs an error indicating a permission issue\",\"- The test suite includes unit tests for each scenario and verifies that `logger.error` is called with the correct message for corrupt JSON and permission errors\"]}","latency_ms":21864,"tokens_in_est":2730,"tokens_out_est":213}]},{"case_id":"valid-error-handling","tags":["valid","edge","core"],"repeat":2,"duration_ms":23812,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W1: Issue does not specify how the file is accessed or which logger implementation is used, requiring the developer to infer these details from the codebase."],"suggested_ac":["- Given a checkpoint file with malformed JSON, when readCheckpoint is called, then it returns null and logger.error is called with the file path.","- Given a missing checkpoint file, when readCheckpoint is called, then it returns null and does not throw an exception.","- Given a checkpoint file containing valid JSON, when readCheckpoint is called, then it returns the parsed object exactly as read.","- Given an empty checkpoint file, when readCheckpoint is called, then it returns null and logs a warning indicating the file is empty.","- Given a checkpoint file that is not readable due to permission errors, when readCheckpoint is called, then it returns null and logs an error with the file path."]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify how the file is accessed or which logger implementation is used, requiring the developer to infer these details from the codebase.\"],\"suggested_ac\":[\"- Given a checkpoint file with malformed JSON, when readCheckpoint is called, then it returns null and logger.error is called with the file path.\",\"- Given a missing checkpoint file, when readCheckpoint is called, then it returns null and does not throw an exception.\",\"- Given a checkpoint file containing valid JSON, when readCheckpoint is called, then it returns the parsed object exactly as read.\",\"- Given an empty checkpoint file, when readCheckpoint is called, then it returns null and logs a warning indicating the file is empty.\",\"- Given a checkpoint file that is not readable due to permission errors, when readCheckpoint is called, then it returns null and logs an error with the file path.\"]}","latency_ms":23812,"tokens_in_est":2730,"tokens_out_est":234}]},{"case_id":"invalid-no-ac","tags":["invalid","b1","core"],"repeat":0,"duration_ms":21176,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no files, modules, or components are identified","W2: Edge cases not covered – no error handling or fallback behavior described"],"suggested_ac":["[ ] The UI includes a toggle switch labeled \"Dark Mode\" in the settings page","[ ] When the dark mode toggle is enabled, all application backgrounds switch to a dark color palette and text switches to a light color palette","[ ] The dark mode preference is persisted per user and restored on subsequent logins","[ ] The application defaults to light mode for new users without a saved preference","[ ] When an unsupported browser is detected, the UI falls back to light mode and displays a console warning"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no files, modules, or components are identified\",\"W2: Edge cases not covered – no error handling or fallback behavior described\"],\"suggested_ac\":[\"[ ] The UI includes a toggle switch labeled \\\"Dark Mode\\\" in the settings page\",\"[ ] When the dark mode toggle is enabled, all application backgrounds switch to a dark color palette and text switches to a light color palette\",\"[ ] The dark mode preference is persisted per user and restored on subsequent logins\",\"[ ] The application defaults to light mode for new users without a saved preference\",\"[ ] When an unsupported browser is detected, the UI falls back to light mode and displays a console warning\"]}","latency_ms":21176,"tokens_in_est":2623,"tokens_out_est":197}]},{"case_id":"invalid-no-ac","tags":["invalid","b1","core"],"repeat":1,"duration_ms":19420,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no files, modules, or components are identified","W2: Edge cases not covered – no error handling or fallback behavior described"],"suggested_ac":["[ ] When the user selects \"Dark Mode\" in settings, the UI theme switches to a dark color palette across all pages","[ ] The dark theme persists across sessions and is applied on login based on the saved user preference","[ ] If the system or browser prefers a dark color scheme and the user has not set a preference, the UI defaults to dark mode","[ ] The application provides a fallback to the default light theme if dark mode resources fail to load","[ ] Switching themes does not cause layout shifts or loss of user data in open forms"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no files, modules, or components are identified\",\"W2: Edge cases not covered – no error handling or fallback behavior described\"],\"suggested_ac\":[\"[ ] When the user selects \\\"Dark Mode\\\" in settings, the UI theme switches to a dark color palette across all pages\",\"[ ] The dark theme persists across sessions and is applied on login based on the saved user preference\",\"[ ] If the system or browser prefers a dark color scheme and the user has not set a preference, the UI defaults to dark mode\",\"[ ] The application provides a fallback to the default light theme if dark mode resources fail to load\",\"[ ] Switching themes does not cause layout shifts or loss of user data in open forms\"]}","latency_ms":19420,"tokens_in_est":2623,"tokens_out_est":205}]},{"case_id":"invalid-no-ac","tags":["invalid","b1","core"],"repeat":2,"duration_ms":20250,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no files, modules, or components are referenced","W2: Edge cases not covered – no error handling or fallback behavior described"],"suggested_ac":["[ ] The UI includes a toggle switch labeled \"Dark Mode\" in the settings page","[ ] When the dark mode toggle is enabled, the application theme switches to a dark color palette across all screens","[ ] The dark mode setting persists across user sessions and is restored on login","[ ] The application defaults to light mode for new users without a saved preference","[ ] When dark mode is enabled, all text maintains a contrast ratio of at least 4.5:1 according to WCAG AA standards"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no files, modules, or components are referenced\",\"W2: Edge cases not covered – no error handling or fallback behavior described\"],\"suggested_ac\":[\"[ ] The UI includes a toggle switch labeled \\\"Dark Mode\\\" in the settings page\",\"[ ] When the dark mode toggle is enabled, the application theme switches to a dark color palette across all screens\",\"[ ] The dark mode setting persists across user sessions and is restored on login\",\"[ ] The application defaults to light mode for new users without a saved preference\",\"[ ] When dark mode is enabled, all text maintains a contrast ratio of at least 4.5:1 according to WCAG AA standards\"]}","latency_ms":20249,"tokens_in_est":2623,"tokens_out_est":191}]},{"case_id":"invalid-vague-ac","tags":["invalid","b1","core"],"repeat":0,"duration_ms":2551,"error":null,"output":{"valid":false,"score":55,"blockers":["B2: Acceptance criterion \"Fix the bug\" is vague and non‑testable","B2: Acceptance criterion \"It should work correctly\" is vague and non‑testable"],"warnings":["W1: Issue does not specify affected files, modules, or components","W2: No error or edge cases are described"],"suggested_ac":["Given a valid username and password, when the user submits the login form, then the API returns HTTP 200 and a session token","Given an invalid password, when the user submits the login form, then the API returns HTTP 401 with an error message \"Invalid credentials\"","Given a missing username field, when the user submits the login form, then the API returns HTTP 400 with an error message \"Username required\"","Given a network timeout during authentication, when the user submits the login form, then the client displays a retry option and does not crash"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":55,\"blockers\":[\"B2: Acceptance criterion \\\"Fix the bug\\\" is vague and non‑testable\",\"B2: Acceptance criterion \\\"It should work correctly\\\" is vague and non‑testable\"],\"warnings\":[\"W1: Issue does not specify affected files, modules, or components\",\"W2: No error or edge cases are described\"],\"suggested_ac\":[\"Given a valid username and password, when the user submits the login form, then the API returns HTTP 200 and a session token\",\"Given an invalid password, when the user submits the login form, then the API returns HTTP 401 with an error message \\\"Invalid credentials\\\"\",\"Given a missing username field, when the user submits the login form, then the API returns HTTP 400 with an error message \\\"Username required\\\"\",\"Given a network timeout during authentication, when the user submits the login form, then the client displays a retry option and does not crash\"]}","latency_ms":2551,"tokens_in_est":2639,"tokens_out_est":224}]},{"case_id":"invalid-vague-ac","tags":["invalid","b1","core"],"repeat":1,"duration_ms":4008,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: Acceptance criteria item \"Fix the bug\" is vague and non‑testable","B2: Acceptance criteria item \"It should work correctly\" is vague and non‑testable"],"warnings":["W1: Issue does not specify affected components, files, or code locations","W2: No error cases, edge conditions, or reproducibility steps are described"],"suggested_ac":["Given a valid username and password, when the user submits the login form, then the API returns HTTP 200 and a session token","Given an invalid password, when the user submits the login form, then the API returns HTTP 401 with an error message \"Invalid credentials\"","Given a missing username field, when the user submits the login form, then the API returns HTTP 400 with an error message \"Username required\"","Given a network timeout during authentication, when the user submits the login form, then the system retries up to 2 times and returns HTTP 503 if still unavailable","Given a user with a locked account, when the user attempts to log in, then the API returns HTTP 403 with an error message \"Account locked\""]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":0,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: Acceptance criteria item \\\"Fix the bug\\\" is vague and non‑testable\",\"B2: Acceptance criteria item \\\"It should work correctly\\\" is vague and non‑testable\"],\"warnings\":[\"W1: Issue does not specify affected components, files, or code locations\",\"W2: No error cases, edge conditions, or reproducibility steps are described\"],\"suggested_ac\":[\"Given a valid username and password, when the user submits the login form, then the API returns HTTP 200 and a session token\",\"Given an invalid password, when the user submits the login form, then the API returns HTTP 401 with an error message \\\"Invalid credentials\\\"\",\"Given a missing username field, when the user submits the login form, then the API returns HTTP 400 with an error message \\\"Username required\\\"\",\"Given a network timeout during authentication, when the user submits the login form, then the system retries up to 2 times and returns HTTP 503 if still unavailable\",\"Given a user with a locked account, when the user attempts to log in, then the API returns HTTP 403 with an error message \\\"Account locked\\\"\"]}","latency_ms":4008,"tokens_in_est":2639,"tokens_out_est":277}]},{"case_id":"invalid-vague-ac","tags":["invalid","b1","core"],"repeat":2,"duration_ms":4043,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: \"Fix the bug\" is non-testable","B2: \"It should work correctly\" is non-testable"],"warnings":["W1: No technical context or affected components are mentioned","W2: No edge cases or error conditions are described"],"suggested_ac":["Given valid credentials, when POST /api/login, then response status is 200 and body contains a session token","Given invalid password, when POST /api/login, then response status is 401 with error message \"Invalid credentials\"","Given missing username field, when POST /api/login, then response status is 400 with error message \"Username required\"","When 100 concurrent login requests are made with valid credentials, each response completes within 2 seconds","When login fails, response does not include any user-sensitive data"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":0,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: \\\"Fix the bug\\\" is non-testable\",\"B2: \\\"It should work correctly\\\" is non-testable\"],\"warnings\":[\"W1: No technical context or affected components are mentioned\",\"W2: No edge cases or error conditions are described\"],\"suggested_ac\":[\"Given valid credentials, when POST /api/login, then response status is 200 and body contains a session token\",\"Given invalid password, when POST /api/login, then response status is 401 with error message \\\"Invalid credentials\\\"\",\"Given missing username field, when POST /api/login, then response status is 400 with error message \\\"Username required\\\"\",\"When 100 concurrent login requests are made with valid credentials, each response completes within 2 seconds\",\"When login fails, response does not include any user-sensitive data\"]}","latency_ms":4043,"tokens_in_est":2639,"tokens_out_est":203}]},{"case_id":"invalid-subjective-ac","tags":["invalid","b2","core"],"repeat":0,"duration_ms":22244,"error":null,"output":{"valid":false,"score":15,"blockers":["B1: Acceptance criteria are vague and not concrete","B2: \"The page looks modern and clean\" is subjective and non-testable","B2: \"Users find it intuitive\" is subjective and non-testable","B2: \"It feels responsive\" is subjective and non-testable"],"warnings":["W1: No technical context or reference to existing implementation details","W2: No edge cases or error conditions are described"],"suggested_ac":["The Settings page renders without runtime errors for all supported browsers","The Settings page includes a Save button that is enabled only when form fields have been modified","The Settings page layout matches the provided mockup image pixel‑for‑pixel","When the viewport width is less than 600px, the Settings page collapses the sidebar into a hamburger menu"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":15,\"blockers\":[\"B1: Acceptance criteria are vague and not concrete\",\"B2: \\\"The page looks modern and clean\\\" is subjective and non-testable\",\"B2: \\\"Users find it intuitive\\\" is subjective and non-testable\",\"B2: \\\"It feels responsive\\\" is subjective and non-testable\"],\"warnings\":[\"W1: No technical context or reference to existing implementation details\",\"W2: No edge cases or error conditions are described\"],\"suggested_ac\":[\"The Settings page renders without runtime errors for all supported browsers\",\"The Settings page includes a Save button that is enabled only when form fields have been modified\",\"The Settings page layout matches the provided mockup image pixel‑for‑pixel\",\"When the viewport width is less than 600px, the Settings page collapses the sidebar into a hamburger menu\"]}","latency_ms":22243,"tokens_in_est":2655,"tokens_out_est":204}]},{"case_id":"invalid-subjective-ac","tags":["invalid","b2","core"],"repeat":1,"duration_ms":5614,"error":null,"output":{"valid":false,"score":15,"blockers":["B1: Acceptance criteria are vague and not concrete","B2: AC item \"The page looks modern and clean\" is subjective and non-testable","B2: AC item \"Users find it intuitive\" is subjective and non-testable","B2: AC item \"It feels responsive\" is subjective and non-testable"],"warnings":["W1: Issue does not specify affected components beyond the file path, no context about existing implementation or design references","W2: No edge cases or error conditions are described, only high‑level visual goals"],"suggested_ac":["Given the Settings page component, when rendered on a desktop browser, then it matches the design mockup dimensions, colors, and typography as defined in Figma file XYZ","When the page is loaded on a mobile viewport ≤ 480px, then the layout adapts to a single‑column responsive design and all controls remain usable","When a user navigates to Settings, then all interactive elements have accessible names and pass an axe‑core accessibility audit","When the page is rendered, then the component loads in under 200 ms on a typical development machine with network throttling set to 3G","The repository README includes a screenshot of the new Settings page and a link to the corresponding design mockup"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":15,\"blockers\":[\"B1: Acceptance criteria are vague and not concrete\",\"B2: AC item \\\"The page looks modern and clean\\\" is subjective and non-testable\",\"B2: AC item \\\"Users find it intuitive\\\" is subjective and non-testable\",\"B2: AC item \\\"It feels responsive\\\" is subjective and non-testable\"],\"warnings\":[\"W1: Issue does not specify affected components beyond the file path, no context about existing implementation or design references\",\"W2: No edge cases or error conditions are described, only high‑level visual goals\"],\"suggested_ac\":[\"Given the Settings page component, when rendered on a desktop browser, then it matches the design mockup dimensions, colors, and typography as defined in Figma file XYZ\",\"When the page is loaded on a mobile viewport ≤ 480px, then the layout adapts to a single‑column responsive design and all controls remain usable\",\"When a user navigates to Settings, then all interactive elements have accessible names and pass an axe‑core accessibility audit\",\"When the page is rendered, then the component loads in under 200 ms on a typical development machine with network throttling set to 3G\",\"The repository README includes a screenshot of the new Settings page and a link to the corresponding design mockup\"]}","latency_ms":5614,"tokens_in_est":2655,"tokens_out_est":316}]},{"case_id":"invalid-subjective-ac","tags":["invalid","b2","core"],"repeat":2,"duration_ms":24806,"error":null,"output":{"valid":false,"score":55,"blockers":["B2: \"The page looks modern and clean\" is vague and non‑testable","B2: \"Users find it intuitive\" is vague and non‑testable","B2: \"It feels responsive\" is vague and non‑testable"],"warnings":["W1: No affected files, modules, or components other than a generic path are referenced","W2: No error handling, empty state, or accessibility edge cases are described"],"suggested_ac":["Given the Settings page is rendered on a supported browser, when the component mounts, then the DOM contains a header element with text \"Settings\" and a save button with label \"Save Changes\"","Given a user with read‑only permissions accesses the Settings page, when the page loads, then all input fields are disabled and a tooltip \"Read‑only access\" is displayed","Given the Settings page is viewed on a viewport width of 320px, when the layout renders, then all interactive elements remain fully visible without horizontal scrolling","Given invalid data is entered into a settings field and the user attempts to save, when the form is submitted, then an inline validation error message is shown and the save operation is aborted","Given the Settings page loads, when the network request to fetch user settings fails with a 500 error, then an error banner with text \"Unable to load settings\" is displayed"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":55,\"blockers\":[\"B2: \\\"The page looks modern and clean\\\" is vague and non‑testable\",\"B2: \\\"Users find it intuitive\\\" is vague and non‑testable\",\"B2: \\\"It feels responsive\\\" is vague and non‑testable\"],\"warnings\":[\"W1: No affected files, modules, or components other than a generic path are referenced\",\"W2: No error handling, empty state, or accessibility edge cases are described\"],\"suggested_ac\":[\"Given the Settings page is rendered on a supported browser, when the component mounts, then the DOM contains a header element with text \\\"Settings\\\" and a save button with label \\\"Save Changes\\\"\",\"Given a user with read‑only permissions accesses the Settings page, when the page loads, then all input fields are disabled and a tooltip \\\"Read‑only access\\\" is displayed\",\"Given the Settings page is viewed on a viewport width of 320px, when the layout renders, then all interactive elements remain fully visible without horizontal scrolling\",\"Given invalid data is entered into a settings field and the user attempts to save, when the form is submitted, then an inline validation error message is shown and the save operation is aborted\",\"Given the Settings page loads, when the network request to fetch user settings fails with a 500 error, then an error banner with text \\\"Unable to load settings\\\" is displayed\"]}","latency_ms":24806,"tokens_in_est":2655,"tokens_out_est":335}]},{"case_id":"invalid-unmeasurable-perf","tags":["invalid","b2","core"],"repeat":0,"duration_ms":5078,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: Acceptance criteria item \"Review should be faster\" is vague and non‑testable","B2: Acceptance criteria item \"Better performance overall\" is vague and non‑testable"],"warnings":["W1: Issue does not specify which files or functions in scripts/pr_review.mjs are responsible for slowness","W2: No edge cases or error conditions are described for performance testing"],"suggested_ac":["- The script `scripts/pr_review.mjs` completes execution in under 2 seconds when processing a pull request with 500 changed files","- The script logs a warning and exits with code 1 if execution exceeds 5 seconds","- CPU usage of the script does not exceed 80% on a standard CI runner during normal operation","- Memory consumption remains below 200 MiB for pull requests up to 1 GB of diff data","- A benchmark test comparing the current version to the previous commit shows at least a 30% reduction in average runtime"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: Acceptance criteria item \\\"Review should be faster\\\" is vague and non‑testable\",\"B2: Acceptance criteria item \\\"Better performance overall\\\" is vague and non‑testable\"],\"warnings\":[\"W1: Issue does not specify which files or functions in scripts/pr_review.mjs are responsible for slowness\",\"W2: No edge cases or error conditions are described for performance testing\"],\"suggested_ac\":[\"- The script `scripts/pr_review.mjs` completes execution in under 2 seconds when processing a pull request with 500 changed files\",\"- The script logs a warning and exits with code 1 if execution exceeds 5 seconds\",\"- CPU usage of the script does not exceed 80% on a standard CI runner during normal operation\",\"- Memory consumption remains below 200 MiB for pull requests up to 1 GB of diff data\",\"- A benchmark test comparing the current version to the previous commit shows at least a 30% reduction in average runtime\"]}","latency_ms":5078,"tokens_in_est":2645,"tokens_out_est":238}]},{"case_id":"invalid-unmeasurable-perf","tags":["invalid","b2","core"],"repeat":1,"duration_ms":1386,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: \"Review should be faster\" is vague and not measurable","B2: \"Better performance overall\" is vague and not measurable"],"warnings":["W1: Issue lacks technical context about what constitutes acceptable performance and which parts of the script to optimize"],"suggested_ac":["Given the current execution time of scripts/pr_review.mjs is >5 seconds for a typical PR, when the script processes a PR with 200 files, then it completes in under 2 seconds","Given scripts/pr_review.mjs receives an empty PR, when it runs, then it completes in under 500 ms without errors","Given scripts/pr_review.mjs processes a PR with a large diff (10,000 lines), when it runs, then memory usage does not exceed 200 MB","Given scripts/pr_review.mjs encounters a malformed file path, when it runs, then it logs an error and exits with a non-zero status","Given scripts/pr_review.mjs runs on a CI environment with Node.js 18, when executed, then it produces the same output as the current version"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: \\\"Review should be faster\\\" is vague and not measurable\",\"B2: \\\"Better performance overall\\\" is vague and not measurable\"],\"warnings\":[\"W1: Issue lacks technical context about what constitutes acceptable performance and which parts of the script to optimize\"],\"suggested_ac\":[\"Given the current execution time of scripts/pr_review.mjs is >5 seconds for a typical PR, when the script processes a PR with 200 files, then it completes in under 2 seconds\",\"Given scripts/pr_review.mjs receives an empty PR, when it runs, then it completes in under 500 ms without errors\",\"Given scripts/pr_review.mjs processes a PR with a large diff (10,000 lines), when it runs, then memory usage does not exceed 200 MB\",\"Given scripts/pr_review.mjs encounters a malformed file path, when it runs, then it logs an error and exits with a non-zero status\",\"Given scripts/pr_review.mjs runs on a CI environment with Node.js 18, when executed, then it produces the same output as the current version\"]}","latency_ms":1386,"tokens_in_est":2645,"tokens_out_est":256}]},{"case_id":"invalid-unmeasurable-perf","tags":["invalid","b2","core"],"repeat":2,"duration_ms":23984,"error":null,"output":{"valid":false,"score":15,"blockers":["B1: Acceptance criteria are vague and not concrete","B2: \"Review should be faster\" is non‑testable","B2: \"Better performance overall\" is non‑testable"],"warnings":["W1: No affected files, modules, or code locations beyond a generic script path are described","W2: No edge cases or error conditions are addressed"],"suggested_ac":["Given a PR with 500 lines of diff, when the review script runs, then it completes in under 2 seconds","Given a PR with 0 changed files, when the review script runs, then it completes in under 0.5 seconds","Given a PR with 10,000 lines of diff, when the review script runs, then it completes in under 10 seconds","When the review script encounters a missing file, then it returns a clear error message and exits with non‑zero status","When the review script processes a PR containing binary files, then it skips them and logs a warning without slowing down the overall execution"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":15,\"blockers\":[\"B1: Acceptance criteria are vague and not concrete\",\"B2: \\\"Review should be faster\\\" is non‑testable\",\"B2: \\\"Better performance overall\\\" is non‑testable\"],\"warnings\":[\"W1: No affected files, modules, or code locations beyond a generic script path are described\",\"W2: No edge cases or error conditions are addressed\"],\"suggested_ac\":[\"Given a PR with 500 lines of diff, when the review script runs, then it completes in under 2 seconds\",\"Given a PR with 0 changed files, when the review script runs, then it completes in under 0.5 seconds\",\"Given a PR with 10,000 lines of diff, when the review script runs, then it completes in under 10 seconds\",\"When the review script encounters a missing file, then it returns a clear error message and exits with non‑zero status\",\"When the review script processes a PR containing binary files, then it skips them and logs a warning without slowing down the overall execution\"]}","latency_ms":23984,"tokens_in_est":2645,"tokens_out_est":239}]},{"case_id":"invalid-ambiguous-scope","tags":["invalid","b3","core"],"repeat":0,"duration_ms":2937,"error":null,"output":{"valid":false,"score":35,"blockers":["B2: AC item \"Responses are cached\" is vague and not directly testable","B2: AC item \"Cache hit returns the cached value\" is vague and not directly testable","B3: Scope is ambiguous – does not specify which endpoints or responses should be cached"],"warnings":["W1: No affected files, modules, or components are named","W2: No error handling or cache eviction scenarios are described"],"suggested_ac":["Given a GET request to /api/data, when the request is made for the first time, then the response should have a header X-Cache-Status: MISS and the body should be stored in the cache","Given the same GET request to /api/data is made again within the cache TTL, then the response should have a header X-Cache-Status: HIT and the body should match the previously cached response","Given a GET request to /api/data after the cache TTL has expired, then the response should have a header X-Cache-Status: MISS and a fresh value should be returned","Given a request that results in an error (e.g., 500), then the error response should not be cached and subsequent requests should attempt fresh processing"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":35,\"blockers\":[\"B2: AC item \\\"Responses are cached\\\" is vague and not directly testable\",\"B2: AC item \\\"Cache hit returns the cached value\\\" is vague and not directly testable\",\"B3: Scope is ambiguous – does not specify which endpoints or responses should be cached\"],\"warnings\":[\"W1: No affected files, modules, or components are named\",\"W2: No error handling or cache eviction scenarios are described\"],\"suggested_ac\":[\"Given a GET request to /api/data, when the request is made for the first time, then the response should have a header X-Cache-Status: MISS and the body should be stored in the cache\",\"Given the same GET request to /api/data is made again within the cache TTL, then the response should have a header X-Cache-Status: HIT and the body should match the previously cached response\",\"Given a GET request to /api/data after the cache TTL has expired, then the response should have a header X-Cache-Status: MISS and a fresh value should be returned\",\"Given a request that results in an error (e.g., 500), then the error response should not be cached and subsequent requests should attempt fresh processing\"]}","latency_ms":2937,"tokens_in_est":2646,"tokens_out_est":287}]},{"case_id":"invalid-ambiguous-scope","tags":["invalid","b3","core"],"repeat":1,"duration_ms":3945,"error":null,"output":{"valid":false,"score":60,"blockers":["B2: \"Responses are cached\" is vague and not directly testable","B2: \"Cache hit returns the cached value\" is vague and not directly testable"],"warnings":["W1: Issue lacks technical context such as which components or endpoints should be cached"],"suggested_ac":["Given a request to endpoint /api/data, when the response is cached, then a subsequent identical request returns HTTP 200 with an X-Cache-Status header set to HIT and the same payload as the first response","Given a request to endpoint /api/data, when the cache is empty, then the first request returns HTTP 200 with X-Cache-Status header set to MISS and stores the response in the cache","Given a request that results in a 500 error, when caching is enabled, then the error response is not cached and subsequent identical requests still result in a 500 error","Given a request with query parameters that differ, when caching is enabled, then each unique request is cached separately and does not cause cross‑contamination of cached values"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B2: \\\"Responses are cached\\\" is vague and not directly testable\",\"B2: \\\"Cache hit returns the cached value\\\" is vague and not directly testable\"],\"warnings\":[\"W1: Issue lacks technical context such as which components or endpoints should be cached\"],\"suggested_ac\":[\"Given a request to endpoint /api/data, when the response is cached, then a subsequent identical request returns HTTP 200 with an X-Cache-Status header set to HIT and the same payload as the first response\",\"Given a request to endpoint /api/data, when the cache is empty, then the first request returns HTTP 200 with X-Cache-Status header set to MISS and stores the response in the cache\",\"Given a request that results in a 500 error, when caching is enabled, then the error response is not cached and subsequent identical requests still result in a 500 error\",\"Given a request with query parameters that differ, when caching is enabled, then each unique request is cached separately and does not cause cross‑contamination of cached values\"]}","latency_ms":3944,"tokens_in_est":2646,"tokens_out_est":262}]},{"case_id":"invalid-ambiguous-scope","tags":["invalid","b3","core"],"repeat":2,"duration_ms":21085,"error":null,"output":{"valid":false,"score":55,"blockers":["B2: \"Responses are cached\" is vague and not directly testable","B2: \"Cache hit returns the cached value\" is vague and not directly testable"],"warnings":["W1: Issue lacks technical context such as which components or services should be cached"],"suggested_ac":["Given a request to endpoint X, when the response is generated, then it must be stored in the cache with a key derived from the request URL and query parameters","Given a cached entry for endpoint X exists, when the same request is received, then the service must return HTTP 200 with the cached payload and include a header `X-Cache: HIT`","Given a request to endpoint X with no existing cache entry, when the request is processed, then the response must include a header `X-Cache: MISS` and store the result in the cache","Given a cached entry for endpoint X has expired after 5 minutes, when the request is received, then the service must treat it as a miss, regenerate the response, and update the cache","Given the cache storage is unavailable, when a request is received, then the service must fallback to generating a fresh response without caching and log a warning"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":55,\"blockers\":[\"B2: \\\"Responses are cached\\\" is vague and not directly testable\",\"B2: \\\"Cache hit returns the cached value\\\" is vague and not directly testable\"],\"warnings\":[\"W1: Issue lacks technical context such as which components or services should be cached\"],\"suggested_ac\":[\"Given a request to endpoint X, when the response is generated, then it must be stored in the cache with a key derived from the request URL and query parameters\",\"Given a cached entry for endpoint X exists, when the same request is received, then the service must return HTTP 200 with the cached payload and include a header `X-Cache: HIT`\",\"Given a request to endpoint X with no existing cache entry, when the request is processed, then the response must include a header `X-Cache: MISS` and store the result in the cache\",\"Given a cached entry for endpoint X has expired after 5 minutes, when the request is received, then the service must treat it as a miss, regenerate the response, and update the cache\",\"Given the cache storage is unavailable, when a request is received, then the service must fallback to generating a fresh response without caching and log a warning\"]}","latency_ms":21085,"tokens_in_est":2646,"tokens_out_est":295}]},{"case_id":"invalid-arch-choice","tags":["invalid","b3","core"],"repeat":0,"duration_ms":23371,"error":null,"output":{"valid":false,"score":70,"blockers":["B3: Ambiguous storage choice – issue does not specify whether metrics should be persisted in Postgres or Redis, leaving implementation scope undefined"],"warnings":["W1: Missing technical context – no files, modules, or existing components are referenced"],"suggested_ac":["[ ] Given a running pipeline, when the service restarts, then previously recorded metrics are still retrievable via the metrics API","[ ] Given stored metrics, when a client queries the /metrics endpoint, then the response returns a JSON array containing all metric records with correct schema","[ ] Given an empty metrics store, when the /metrics endpoint is queried, then it returns an empty array with HTTP 200","[ ] Given a Redis backend, when the Redis instance becomes unavailable, then the metrics service returns HTTP 503 on query attempts","[ ] Given a Postgres backend, when a metric record is inserted, then it is persisted and visible after a database restart"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B3: Ambiguous storage choice – issue does not specify whether metrics should be persisted in Postgres or Redis, leaving implementation scope undefined\"],\"warnings\":[\"W1: Missing technical context – no files, modules, or existing components are referenced\"],\"suggested_ac\":[\"[ ] Given a running pipeline, when the service restarts, then previously recorded metrics are still retrievable via the metrics API\",\"[ ] Given stored metrics, when a client queries the /metrics endpoint, then the response returns a JSON array containing all metric records with correct schema\",\"[ ] Given an empty metrics store, when the /metrics endpoint is queried, then it returns an empty array with HTTP 200\",\"[ ] Given a Redis backend, when the Redis instance becomes unavailable, then the metrics service returns HTTP 503 on query attempts\",\"[ ] Given a Postgres backend, when a metric record is inserted, then it is persisted and visible after a database restart\"]}","latency_ms":23371,"tokens_in_est":2659,"tokens_out_est":247}]},{"case_id":"invalid-arch-choice","tags":["invalid","b3","core"],"repeat":1,"duration_ms":20997,"error":null,"output":{"valid":false,"score":80,"blockers":["B3: Ambiguous storage choice between Postgres or Redis not specified"],"warnings":["W1: Missing technical context – no files, modules, or existing implementation referenced"],"suggested_ac":["[ ] Given a running service, after inserting a metric and restarting the application, a GET /metrics returns the previously stored metric","[ ] When querying GET /metrics without any stored data, the response is an empty list with HTTP 200","[ ] Given a metric stored, a GET /metrics returns a JSON array containing that metric with fields id, name, value, timestamp","[ ] When the storage backend is PostgreSQL, inserting a metric returns HTTP 201 and persists across restarts","[ ] When the storage backend is Redis, inserting a metric returns HTTP 201 and persists across restarts"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":80,\"blockers\":[\"B3: Ambiguous storage choice between Postgres or Redis not specified\"],\"warnings\":[\"W1: Missing technical context – no files, modules, or existing implementation referenced\"],\"suggested_ac\":[\"[ ] Given a running service, after inserting a metric and restarting the application, a GET /metrics returns the previously stored metric\",\"[ ] When querying GET /metrics without any stored data, the response is an empty list with HTTP 200\",\"[ ] Given a metric stored, a GET /metrics returns a JSON array containing that metric with fields id, name, value, timestamp\",\"[ ] When the storage backend is PostgreSQL, inserting a metric returns HTTP 201 and persists across restarts\",\"[ ] When the storage backend is Redis, inserting a metric returns HTTP 201 and persists across restarts\"]}","latency_ms":20997,"tokens_in_est":2659,"tokens_out_est":205}]},{"case_id":"invalid-arch-choice","tags":["invalid","b3","core"],"repeat":2,"duration_ms":18956,"error":null,"output":{"valid":false,"score":80,"blockers":["B3: Ambiguous storage choice between Postgres or Redis is not specified"],"warnings":["W1: No affected files or modules are named"],"suggested_ac":["[ ] Given a running service, after inserting a metric and restarting the application, a GET /metrics returns the previously stored metric","[ ] When querying /metrics with a valid metric ID, the response includes the correct metric data and HTTP 200","[ ] When querying /metrics for a non‑existent ID, the service returns HTTP 404","[ ] Given the chosen storage backend, inserting a metric with invalid payload returns HTTP 400 and does not persist data","[ ] Metrics retrieval latency is under 200 ms for up to 10,000 stored metrics"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":80,\"blockers\":[\"B3: Ambiguous storage choice between Postgres or Redis is not specified\"],\"warnings\":[\"W1: No affected files or modules are named\"],\"suggested_ac\":[\"[ ] Given a running service, after inserting a metric and restarting the application, a GET /metrics returns the previously stored metric\",\"[ ] When querying /metrics with a valid metric ID, the response includes the correct metric data and HTTP 200\",\"[ ] When querying /metrics for a non‑existent ID, the service returns HTTP 404\",\"[ ] Given the chosen storage backend, inserting a metric with invalid payload returns HTTP 400 and does not persist data\",\"[ ] Metrics retrieval latency is under 200 ms for up to 10,000 stored metrics\"]}","latency_ms":18956,"tokens_in_est":2659,"tokens_out_est":181}]},{"case_id":"invalid-undocumented-dependency","tags":["invalid","b4","core"],"repeat":0,"duration_ms":6217,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: AC item \"A Slack message appears in #dev-loop\" is non‑testable","B4: Implementation depends on a notifier service that is not deployed and has no defined API contract"],"warnings":["W1: Issue does not specify affected files or modules (e.g., which part of the codebase should be modified)","W2: No error‑handling or failure scenarios are covered in the acceptance criteria"],"suggested_ac":["Given a PR review with verdict \"approved\", when pr_review.mjs processes it, then it calls the notifier service with a payload containing the correct prNumber and verdict","Given the notifier service returns HTTP 200, then a Slack message with text \"PR #<prNumber> verdict: <verdict>\" is posted to #dev-loop","Given the notifier service returns an error (e.g., 500), then pr_review.mjs logs an error and does not crash","Given the payload is missing the verdict field, then the notifier is not called and an error is logged","Unit tests include a mock of the notifier service to verify the call and Slack message formatting without requiring the real service"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: AC item \\\"A Slack message appears in #dev-loop\\\" is non‑testable\",\"B4: Implementation depends on a notifier service that is not deployed and has no defined API contract\"],\"warnings\":[\"W1: Issue does not specify affected files or modules (e.g., which part of the codebase should be modified)\",\"W2: No error‑handling or failure scenarios are covered in the acceptance criteria\"],\"suggested_ac\":[\"Given a PR review with verdict \\\"approved\\\", when pr_review.mjs processes it, then it calls the notifier service with a payload containing the correct prNumber and verdict\",\"Given the notifier service returns HTTP 200, then a Slack message with text \\\"PR #<prNumber> verdict: <verdict>\\\" is posted to #dev-loop\",\"Given the notifier service returns an error (e.g., 500), then pr_review.mjs logs an error and does not crash\",\"Given the payload is missing the verdict field, then the notifier is not called and an error is logged\",\"Unit tests include a mock of the notifier service to verify the call and Slack message formatting without requiring the real service\"]}","latency_ms":6217,"tokens_in_est":2696,"tokens_out_est":276}]},{"case_id":"invalid-undocumented-dependency","tags":["invalid","b4","core"],"repeat":1,"duration_ms":6414,"error":null,"output":{"valid":false,"score":60,"blockers":["B2: AC item \"A Slack message appears in #dev-loop\" is non‑testable without a deployed notifier or Slack integration","B4: The required notifier service does not exist and no mock or contract is provided"],"warnings":["W1: No affected files or modules are explicitly referenced beyond pr_review.mjs","W2: No error or edge‑case handling is described (e.g., failure to send Slack message)"],"suggested_ac":["[ ] `pr_review.mjs` calls the notifier module with an object containing `prNumber` (integer) and `verdict` (string)","[ ] When the notifier call succeeds, a mock Slack client records a message sent to channel `#dev-loop` with text containing the verdict","[ ] If the notifier call throws an error, `pr_review.mjs` logs the error and returns a failure status","[ ] Unit tests verify that the notifier is invoked exactly once per review with the correct payload","[ ] Integration test using a stubbed notifier endpoint confirms that a Slack‑formatted payload is posted to the `#dev-loop` channel"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B2: AC item \\\"A Slack message appears in #dev-loop\\\" is non‑testable without a deployed notifier or Slack integration\",\"B4: The required notifier service does not exist and no mock or contract is provided\"],\"warnings\":[\"W1: No affected files or modules are explicitly referenced beyond pr_review.mjs\",\"W2: No error or edge‑case handling is described (e.g., failure to send Slack message)\"],\"suggested_ac\":[\"[ ] `pr_review.mjs` calls the notifier module with an object containing `prNumber` (integer) and `verdict` (string)\",\"[ ] When the notifier call succeeds, a mock Slack client records a message sent to channel `#dev-loop` with text containing the verdict\",\"[ ] If the notifier call throws an error, `pr_review.mjs` logs the error and returns a failure status\",\"[ ] Unit tests verify that the notifier is invoked exactly once per review with the correct payload\",\"[ ] Integration test using a stubbed notifier endpoint confirms that a Slack‑formatted payload is posted to the `#dev-loop` channel\"]}","latency_ms":6414,"tokens_in_est":2696,"tokens_out_est":261}]},{"case_id":"invalid-undocumented-dependency","tags":["invalid","b4","core"],"repeat":2,"duration_ms":21445,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: Dependency on notifier service that is not deployed and has no published endpoint or API contract"],"warnings":[],"suggested_ac":["[ ] `pr_review.mjs` calls the notifier with an object containing `prNumber` (integer) and `verdict` (string) and the call is made to the configured endpoint URL","[ ] When the notifier service returns HTTP 200, a Slack message with text \"Review #${prNumber}: ${verdict}\" is posted to #dev-loop (mocked in tests)","[ ] If the notifier service returns a non‑200 response, `pr_review.mjs` logs an error and does not crash","[ ] The notifier call is retried up to 3 times on network failures before giving up","[ ] Unit tests verify the function behavior using a stubbed notifier service and a mocked Slack client"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: Dependency on notifier service that is not deployed and has no published endpoint or API contract\"],\"warnings\":[],\"suggested_ac\":[\"[ ] `pr_review.mjs` calls the notifier with an object containing `prNumber` (integer) and `verdict` (string) and the call is made to the configured endpoint URL\",\"[ ] When the notifier service returns HTTP 200, a Slack message with text \\\"Review #${prNumber}: ${verdict}\\\" is posted to #dev-loop (mocked in tests)\",\"[ ] If the notifier service returns a non‑200 response, `pr_review.mjs` logs an error and does not crash\",\"[ ] The notifier call is retried up to 3 times on network failures before giving up\",\"[ ] Unit tests verify the function behavior using a stubbed notifier service and a mocked Slack client\"]}","latency_ms":21445,"tokens_in_est":2696,"tokens_out_est":197}]},{"case_id":"invalid-empty-body","tags":["invalid","b1","edge","core"],"repeat":0,"duration_ms":4272,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files, modules, or components are named"],"suggested_ac":["[ ] The logging module writes INFO level messages to stdout with a timestamp in ISO 8601 format","[ ] ERROR level logs include stack trace and are written to logs/error.log file","[ ] When a log message exceeds 1,000 characters it is truncated to 1,000 characters with an ellipsis appended","[ ] The logging configuration can be reloaded at runtime without restarting the application","[ ] Unit tests verify that log messages are correctly formatted and routed for INFO, WARN, and ERROR levels"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files, modules, or components are named\"],\"suggested_ac\":[\"[ ] The logging module writes INFO level messages to stdout with a timestamp in ISO 8601 format\",\"[ ] ERROR level logs include stack trace and are written to logs/error.log file\",\"[ ] When a log message exceeds 1,000 characters it is truncated to 1,000 characters with an ellipsis appended\",\"[ ] The logging configuration can be reloaded at runtime without restarting the application\",\"[ ] Unit tests verify that log messages are correctly formatted and routed for INFO, WARN, and ERROR levels\"]}","latency_ms":4272,"tokens_in_est":2619,"tokens_out_est":175}]},{"case_id":"invalid-empty-body","tags":["invalid","b1","edge","core"],"repeat":1,"duration_ms":20309,"error":null,"output":{"valid":false,"score":45,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components are mentioned","W2: Edge cases not covered – issue contains no details about behavior or error handling"],"suggested_ac":["[ ] The logging system writes an INFO level message to the designated log file when a user successfully logs in","[ ] The logging system writes an ERROR level message to the designated log file when a login attempt fails due to invalid credentials","[ ] The logging system includes a timestamp in ISO 8601 format for each log entry","[ ] The logging system rotates the log file when it exceeds 10 MB in size","[ ] The logging system does not expose sensitive data such as passwords in any log entry"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":45,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components are mentioned\",\"W2: Edge cases not covered – issue contains no details about behavior or error handling\"],\"suggested_ac\":[\"[ ] The logging system writes an INFO level message to the designated log file when a user successfully logs in\",\"[ ] The logging system writes an ERROR level message to the designated log file when a login attempt fails due to invalid credentials\",\"[ ] The logging system includes a timestamp in ISO 8601 format for each log entry\",\"[ ] The logging system rotates the log file when it exceeds 10 MB in size\",\"[ ] The logging system does not expose sensitive data such as passwords in any log entry\"]}","latency_ms":20309,"tokens_in_est":2619,"tokens_out_est":197}]},{"case_id":"invalid-empty-body","tags":["invalid","b1","edge","core"],"repeat":2,"duration_ms":18240,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files, modules, or components are named"],"suggested_ac":["[ ] The logging module includes a log entry with level INFO and message \"User login successful\" when a user logs in successfully","[ ] The logging module includes a log entry with level ERROR and message containing the exception details when an unhandled exception occurs","[ ] Log entries are written to the configured log file within 100 ms of the event occurring","[ ] The logging format includes timestamp, log level, module name, and message as defined in the project's logging configuration","[ ] When the application starts, a log entry with level INFO and message \"Application started\" is emitted"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files, modules, or components are named\"],\"suggested_ac\":[\"[ ] The logging module includes a log entry with level INFO and message \\\"User login successful\\\" when a user logs in successfully\",\"[ ] The logging module includes a log entry with level ERROR and message containing the exception details when an unhandled exception occurs\",\"[ ] Log entries are written to the configured log file within 100 ms of the event occurring\",\"[ ] The logging format includes timestamp, log level, module name, and message as defined in the project's logging configuration\",\"[ ] When the application starts, a log entry with level INFO and message \\\"Application started\\\" is emitted\"]}","latency_ms":18240,"tokens_in_est":2619,"tokens_out_est":204}]},{"case_id":"invalid-problem-only","tags":["invalid","b1","core"],"repeat":0,"duration_ms":19883,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W2: Issue describes only the problem without specifying error handling or expected behavior"],"suggested_ac":["[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message","[ ] When the attempt counter exceeds the limit, the workflow exits with a non-zero status code","[ ] The workflow does not create new commits after the attempt limit is reached","[ ] Logs contain a clear message indicating that the attempt limit was reached","[ ] A unit test verifies that the attempt counter resets for each new PR"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W2: Issue describes only the problem without specifying error handling or expected behavior\"],\"suggested_ac\":[\"[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message\",\"[ ] When the attempt counter exceeds the limit, the workflow exits with a non-zero status code\",\"[ ] The workflow does not create new commits after the attempt limit is reached\",\"[ ] Logs contain a clear message indicating that the attempt limit was reached\",\"[ ] A unit test verifies that the attempt counter resets for each new PR\"]}","latency_ms":19883,"tokens_in_est":2659,"tokens_out_est":159}]},{"case_id":"invalid-problem-only","tags":["invalid","b1","core"],"repeat":1,"duration_ms":3371,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or detailed description of expected behavior beyond a vague loop issue"],"suggested_ac":["[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message when the limit is reached","[ ] When the attempt counter exceeds the limit, no new commits are pushed to the PR and the workflow exits with a non-zero status code","[ ] The workflow logs contain the phrase \"Attempt limit reached\" on failure cases","[ ] For a PR that succeeds within the attempt limit, the workflow pushes at most one commit and exits with status code 0","[ ] The workflow correctly handles a scenario where the PR has no changes to apply, exiting gracefully without pushing commits"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or detailed description of expected behavior beyond a vague loop issue\"],\"suggested_ac\":[\"[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message when the limit is reached\",\"[ ] When the attempt counter exceeds the limit, no new commits are pushed to the PR and the workflow exits with a non-zero status code\",\"[ ] The workflow logs contain the phrase \\\"Attempt limit reached\\\" on failure cases\",\"[ ] For a PR that succeeds within the attempt limit, the workflow pushes at most one commit and exits with status code 0\",\"[ ] The workflow correctly handles a scenario where the PR has no changes to apply, exiting gracefully without pushing commits\"]}","latency_ms":3371,"tokens_in_est":2659,"tokens_out_est":208}]},{"case_id":"invalid-problem-only","tags":["invalid","b1","core"],"repeat":2,"duration_ms":1758,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context about expected behavior and affected components","W2: No edge cases or error conditions described"],"suggested_ac":["[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message","[ ] When the attempt counter exceeds the limit, no new commits are pushed to the PR","[ ] The workflow records the number of attempts in the PR comment and includes a link to the logs","[ ] If the script encounters an unexpected error, it aborts and returns a non-zero exit code","[ ] The workflow completes within 2 minutes for a PR with no conflicts"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context about expected behavior and affected components\",\"W2: No edge cases or error conditions described\"],\"suggested_ac\":[\"[ ] The auto-fix workflow stops after a maximum of 5 attempts and logs a warning message\",\"[ ] When the attempt counter exceeds the limit, no new commits are pushed to the PR\",\"[ ] The workflow records the number of attempts in the PR comment and includes a link to the logs\",\"[ ] If the script encounters an unexpected error, it aborts and returns a non-zero exit code\",\"[ ] The workflow completes within 2 minutes for a PR with no conflicts\"]}","latency_ms":1758,"tokens_in_est":2659,"tokens_out_est":173}]},{"case_id":"invalid-partial-ac-style","tags":["invalid","b2","partial-ac"],"repeat":0,"duration_ms":4429,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge cases such as empty file or invalid YAML syntax are not covered in the acceptance criteria"],"suggested_ac":["- [ ] Loading an empty `labels.yaml` file throws `Error` indicating the file is empty","- [ ] A label entry missing both `name` and `color` throws `Error` naming the line number","- [ ] Invalid YAML syntax in `labels.yaml` throws `Error` with a parse error message","- [ ] A label with a `color` value longer than 6 hex digits throws `Error` naming the label","- [ ] The function returns a list of validated label objects when `labels.yaml` is correct"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as empty file or invalid YAML syntax are not covered in the acceptance criteria\"],\"suggested_ac\":[\"- [ ] Loading an empty `labels.yaml` file throws `Error` indicating the file is empty\",\"- [ ] A label entry missing both `name` and `color` throws `Error` naming the line number\",\"- [ ] Invalid YAML syntax in `labels.yaml` throws `Error` with a parse error message\",\"- [ ] A label with a `color` value longer than 6 hex digits throws `Error` naming the label\",\"- [ ] The function returns a list of validated label objects when `labels.yaml` is correct\"]}","latency_ms":4429,"tokens_in_est":2731,"tokens_out_est":157}]},{"case_id":"invalid-partial-ac-style","tags":["invalid","b2","partial-ac"],"repeat":1,"duration_ms":3882,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge cases such as empty or missing labels.yaml are not covered in the acceptance criteria"],"suggested_ac":["- [ ] Loading an empty `labels.yaml` file throws `Error` with message \"No labels defined\" following the existing error style","- [ ] Invalid YAML syntax in `labels.yaml` throws `Error` indicating the parse error line number","- [ ] Missing `labels.yaml` file results in `Error` stating \"labels.yaml not found\" with appropriate error style","- [ ] A label with a `color` value containing non-hex characters throws `Error` naming the label","- [ ] A label missing both `name` and `color` throws `Error` listing the line number and missing fields"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as empty or missing labels.yaml are not covered in the acceptance criteria\"],\"suggested_ac\":[\"- [ ] Loading an empty `labels.yaml` file throws `Error` with message \\\"No labels defined\\\" following the existing error style\",\"- [ ] Invalid YAML syntax in `labels.yaml` throws `Error` indicating the parse error line number\",\"- [ ] Missing `labels.yaml` file results in `Error` stating \\\"labels.yaml not found\\\" with appropriate error style\",\"- [ ] A label with a `color` value containing non-hex characters throws `Error` naming the label\",\"- [ ] A label missing both `name` and `color` throws `Error` listing the line number and missing fields\"]}","latency_ms":3882,"tokens_in_est":2731,"tokens_out_est":179}]},{"case_id":"invalid-partial-ac-style","tags":["invalid","b2","partial-ac"],"repeat":2,"duration_ms":21020,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Issue does not specify tests for successful loading of valid config, only error cases"],"suggested_ac":["[ ] Loading a valid `labels.yaml` file returns an array of label objects with correct `name` and `color` properties","[ ] When a label is missing the `name` field, `loadLabels()` throws an `Error` containing the line number of the offending entry","[ ] When a label `color` is not a 6‑digit hexadecimal string, `loadLabels()` throws an `Error` mentioning the label name","[ ] When duplicate label names exist, `loadLabels()` throws an `Error` listing both line numbers of the duplicates","[ ] All error messages conform to the project's error style defined in `scripts/lib/` (e.g., prefix `ConfigError:` and include relevant context)"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Issue does not specify tests for successful loading of valid config, only error cases\"],\"suggested_ac\":[\"[ ] Loading a valid `labels.yaml` file returns an array of label objects with correct `name` and `color` properties\",\"[ ] When a label is missing the `name` field, `loadLabels()` throws an `Error` containing the line number of the offending entry\",\"[ ] When a label `color` is not a 6‑digit hexadecimal string, `loadLabels()` throws an `Error` mentioning the label name\",\"[ ] When duplicate label names exist, `loadLabels()` throws an `Error` listing both line numbers of the duplicates\",\"[ ] All error messages conform to the project's error style defined in `scripts/lib/` (e.g., prefix `ConfigError:` and include relevant context)\"]}","latency_ms":21020,"tokens_in_est":2731,"tokens_out_est":200}]},{"case_id":"invalid-partial-ac-no-threshold","tags":["invalid","b2","partial-ac"],"repeat":0,"duration_ms":2946,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Loading works for large prompt directories without timing out\" is non‑testable and lacks measurable criteria"],"warnings":[],"suggested_ac":["[ ] A second loadPrompt('validation-system') call does not invoke fs.readFileSync, verified by a mock that records call count","[ ] The string returned by the second loadPrompt call is exactly equal to the string returned by the first call","[ ] When loadPrompt is called with a non‑existent filename, it throws an error with message \"Prompt file not found\"","[ ] loadPrompt processes a directory containing at least 10,000 prompt files in under 2 seconds without throwing a timeout error","[ ] loadPrompt caches each distinct prompt name separately; calling loadPrompt('a') then loadPrompt('b') results in two independent cache entries"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Loading works for large prompt directories without timing out\\\" is non‑testable and lacks measurable criteria\"],\"warnings\":[],\"suggested_ac\":[\"[ ] A second loadPrompt('validation-system') call does not invoke fs.readFileSync, verified by a mock that records call count\",\"[ ] The string returned by the second loadPrompt call is exactly equal to the string returned by the first call\",\"[ ] When loadPrompt is called with a non‑existent filename, it throws an error with message \\\"Prompt file not found\\\"\",\"[ ] loadPrompt processes a directory containing at least 10,000 prompt files in under 2 seconds without throwing a timeout error\",\"[ ] loadPrompt caches each distinct prompt name separately; calling loadPrompt('a') then loadPrompt('b') results in two independent cache entries\"]}","latency_ms":2946,"tokens_in_est":2717,"tokens_out_est":208}]},{"case_id":"invalid-partial-ac-no-threshold","tags":["invalid","b2","partial-ac"],"repeat":1,"duration_ms":6334,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Loading works for large prompt directories without timing out\" is non‑testable due to missing quantitative threshold"],"warnings":["W2: No edge cases such as concurrent calls or empty directory are covered"],"suggested_ac":["Given a valid prompt name, when loadPrompt is called twice, then fs.readFileSync is invoked only on the first call (verified with a mock)","Given a valid prompt name, when loadPrompt is called twice, then both calls return exactly the same string value","Given a non‑existent prompt file, when loadPrompt is called, then it throws an error with message \"Prompt file not found\"","Given a prompt directory containing at least 10,000 prompt files, when loadPrompt is called for a file, then the call completes in under 200 ms","Given concurrent calls to loadPrompt with the same prompt name, when all calls resolve, then fs.readFileSync is invoked only once and all calls return the identical cached string"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Loading works for large prompt directories without timing out\\\" is non‑testable due to missing quantitative threshold\"],\"warnings\":[\"W2: No edge cases such as concurrent calls or empty directory are covered\"],\"suggested_ac\":[\"Given a valid prompt name, when loadPrompt is called twice, then fs.readFileSync is invoked only on the first call (verified with a mock)\",\"Given a valid prompt name, when loadPrompt is called twice, then both calls return exactly the same string value\",\"Given a non‑existent prompt file, when loadPrompt is called, then it throws an error with message \\\"Prompt file not found\\\"\",\"Given a prompt directory containing at least 10,000 prompt files, when loadPrompt is called for a file, then the call completes in under 200 ms\",\"Given concurrent calls to loadPrompt with the same prompt name, when all calls resolve, then fs.readFileSync is invoked only once and all calls return the identical cached string\"]}","latency_ms":6334,"tokens_in_est":2717,"tokens_out_est":245}]},{"case_id":"invalid-partial-ac-no-threshold","tags":["invalid","b2","partial-ac"],"repeat":2,"duration_ms":3673,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Loading works for large prompt directories without timing out\" is non‑testable and lacks a measurable threshold"],"warnings":["W1: Issue does not specify where the cache should be stored or which module/state is responsible for it"],"suggested_ac":["Given a large prompt directory, when loadPrompt('big-dir') is called, it returns the prompt string within 2 seconds","Given a missing file, when loadPrompt('nonexistent') is called, it throws a Prompt file not found error","Given concurrent calls to loadPrompt('validation-system'), fs.readFileSync is invoked only once and all calls return the same string","Given a second call to loadPrompt('validation-system'), the returned string is identical to the first call’s result","Given loadPrompt is called with an invalid filename, it throws an appropriate validation error"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Loading works for large prompt directories without timing out\\\" is non‑testable and lacks a measurable threshold\"],\"warnings\":[\"W1: Issue does not specify where the cache should be stored or which module/state is responsible for it\"],\"suggested_ac\":[\"Given a large prompt directory, when loadPrompt('big-dir') is called, it returns the prompt string within 2 seconds\",\"Given a missing file, when loadPrompt('nonexistent') is called, it throws a Prompt file not found error\",\"Given concurrent calls to loadPrompt('validation-system'), fs.readFileSync is invoked only once and all calls return the same string\",\"Given a second call to loadPrompt('validation-system'), the returned string is identical to the first call’s result\",\"Given loadPrompt is called with an invalid filename, it throws an appropriate validation error\"]}","latency_ms":3673,"tokens_in_est":2717,"tokens_out_est":218}]},{"case_id":"invalid-partial-ac-relevance","tags":["invalid","b2","partial-ac"],"repeat":0,"duration_ms":5909,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: AC item \"The 10 most relevant reports for the query are returned\" is vague and not testable without a defined relevance metric"],"warnings":["W2: Edge cases such as null or non-string query inputs are not covered"],"suggested_ac":["[ ] `searchReports(null)` throws `TypeError`","[ ] `searchReports('nonexistent')` returns an empty array","[ ] Results are sorted by descending relevance score as defined by matching keyword frequency","[ ] When multiple reports have equal relevance, the function returns the ones with the highest `issueNumber` first","[ ] The function reads reports from `observability/reports/` without modifying any files"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: AC item \\\"The 10 most relevant reports for the query are returned\\\" is vague and not testable without a defined relevance metric\"],\"warnings\":[\"W2: Edge cases such as null or non-string query inputs are not covered\"],\"suggested_ac\":[\"[ ] `searchReports(null)` throws `TypeError`\",\"[ ] `searchReports('nonexistent')` returns an empty array\",\"[ ] Results are sorted by descending relevance score as defined by matching keyword frequency\",\"[ ] When multiple reports have equal relevance, the function returns the ones with the highest `issueNumber` first\",\"[ ] The function reads reports from `observability/reports/` without modifying any files\"]}","latency_ms":5908,"tokens_in_est":2719,"tokens_out_est":172}]},{"case_id":"invalid-partial-ac-relevance","tags":["invalid","b2","partial-ac"],"repeat":1,"duration_ms":4516,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"The 10 most relevant reports for the query are returned\" is vague and not testable without a defined relevance metric"],"warnings":["W1: No description of how relevance is calculated or which fields are searched"],"suggested_ac":["Given a non‑empty query string, when searchReports is called, then it returns an array of at most 10 objects each containing numeric `issueNumber` and numeric `score` fields","Given a query that matches more than 10 reports, when searchReports is called, then it returns exactly the 10 reports with the highest `score` values, where `score` is defined as the number of matching keywords in the report title and description","Given a query that matches fewer than 10 reports, when searchReports is called, then it returns all matching reports sorted by descending `score`","Given a query containing special characters or whitespace, when searchReports is called, then it sanitizes the input and still returns results according to the defined relevance scoring","Given an invalid (non‑string) argument, when searchReports is called, then it throws a TypeError"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"The 10 most relevant reports for the query are returned\\\" is vague and not testable without a defined relevance metric\"],\"warnings\":[\"W1: No description of how relevance is calculated or which fields are searched\"],\"suggested_ac\":[\"Given a non‑empty query string, when searchReports is called, then it returns an array of at most 10 objects each containing numeric `issueNumber` and numeric `score` fields\",\"Given a query that matches more than 10 reports, when searchReports is called, then it returns exactly the 10 reports with the highest `score` values, where `score` is defined as the number of matching keywords in the report title and description\",\"Given a query that matches fewer than 10 reports, when searchReports is called, then it returns all matching reports sorted by descending `score`\",\"Given a query containing special characters or whitespace, when searchReports is called, then it sanitizes the input and still returns results according to the defined relevance scoring\",\"Given an invalid (non‑string) argument, when searchReports is called, then it throws a TypeError\"]}","latency_ms":4516,"tokens_in_est":2719,"tokens_out_est":285}]},{"case_id":"invalid-partial-ac-relevance","tags":["invalid","b2","partial-ac"],"repeat":2,"duration_ms":5529,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"The 10 most relevant reports for the query are returned\" is vague and non‑testable without a defined relevance metric"],"warnings":["W1: No description of the JSON report schema or how relevance should be computed","W2: Edge cases such as null or non‑string query, queries with no matches, and handling of duplicate scores are not addressed"],"suggested_ac":["Given a non‑empty string query, when `searchReports` is called, then it returns at most 10 objects each containing `issueNumber` (integer) and `score` (numeric) sorted in descending order of `score`","Given a query that matches more than 10 reports, when `searchReports` is called, then it returns exactly the 10 reports with the highest `score` values","Given an empty string query, when `searchReports` is called, then it throws a `TypeError`","Given a null or non‑string query, when `searchReports` is called, then it throws a `TypeError`","Given a query that matches no reports, when `searchReports` is called, then it returns an empty array"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"The 10 most relevant reports for the query are returned\\\" is vague and non‑testable without a defined relevance metric\"],\"warnings\":[\"W1: No description of the JSON report schema or how relevance should be computed\",\"W2: Edge cases such as null or non‑string query, queries with no matches, and handling of duplicate scores are not addressed\"],\"suggested_ac\":[\"Given a non‑empty string query, when `searchReports` is called, then it returns at most 10 objects each containing `issueNumber` (integer) and `score` (numeric) sorted in descending order of `score`\",\"Given a query that matches more than 10 reports, when `searchReports` is called, then it returns exactly the 10 reports with the highest `score` values\",\"Given an empty string query, when `searchReports` is called, then it throws a `TypeError`\",\"Given a null or non‑string query, when `searchReports` is called, then it throws a `TypeError`\",\"Given a query that matches no reports, when `searchReports` is called, then it returns an empty array\"]}","latency_ms":5529,"tokens_in_est":2719,"tokens_out_est":264}]},{"case_id":"invalid-fr-partial-ac","tags":["invalid","b2","partial-ac","fr"],"repeat":0,"duration_ms":5788,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: AC item \"Le coût en dollars est calculé pour tous les modèles courants\" is vague and not quantifiable"],"warnings":[],"suggested_ac":["[ ] La section affiche le coût en dollars calculé pour chaque modèle listé explicitement (e.g., gpt-3.5-turbo, gpt-4) and matches the expected rate per token","[ ] La section n'affiche aucun coût lorsque le modèle utilisé n'est pas dans la liste supportée, and logs a warning","[ ] La section indique le coût total en dollars avec deux décimales et le format \"$X.XX\"","[ ] Le test vérifie que pour un modèle supporté le coût est correct selon le taux de conversion défini dans le code","[ ] Le test vérifie que la section est omise lorsque `tokens_est` vaut 0 même si d'autres champs sont présents"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: AC item \\\"Le coût en dollars est calculé pour tous les modèles courants\\\" is vague and not quantifiable\"],\"warnings\":[],\"suggested_ac\":[\"[ ] La section affiche le coût en dollars calculé pour chaque modèle listé explicitement (e.g., gpt-3.5-turbo, gpt-4) and matches the expected rate per token\",\"[ ] La section n'affiche aucun coût lorsque le modèle utilisé n'est pas dans la liste supportée, and logs a warning\",\"[ ] La section indique le coût total en dollars avec deux décimales et le format \\\"$X.XX\\\"\",\"[ ] Le test vérifie que pour un modèle supporté le coût est correct selon le taux de conversion défini dans le code\",\"[ ] Le test vérifie que la section est omise lorsque `tokens_est` vaut 0 même si d'autres champs sont présents\"]}","latency_ms":5788,"tokens_in_est":2717,"tokens_out_est":196}]},{"case_id":"invalid-fr-partial-ac","tags":["invalid","b2","partial-ac","fr"],"repeat":1,"duration_ms":21550,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Le coût en dollars est calculé pour tous les modèles courants\" is vague and not testable – does not specify which models or calculation method"],"warnings":[],"suggested_ac":["[ ] The section displays the total `tokens_est` and a breakdown per case in the Markdown report","[ ] The section is omitted when `tokens_est` equals 0","[ ] The dollar cost is calculated using the price per token for each supported model (e.g., gpt-3.5-turbo: $0.00002 per token, gpt-4: $0.00003 per token) and the report shows the cost for each model used","[ ] The Markdown report includes a line `Cost: $X.XX` for each model with non‑zero token usage","[ ] The test `scripts/tests/eval_harness.test.mjs` asserts that the cost line appears with correct values for at least two known models and that the section is absent when token count is zero"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Le coût en dollars est calculé pour tous les modèles courants\\\" is vague and not testable – does not specify which models or calculation method\"],\"warnings\":[],\"suggested_ac\":[\"[ ] The section displays the total `tokens_est` and a breakdown per case in the Markdown report\",\"[ ] The section is omitted when `tokens_est` equals 0\",\"[ ] The dollar cost is calculated using the price per token for each supported model (e.g., gpt-3.5-turbo: $0.00002 per token, gpt-4: $0.00003 per token) and the report shows the cost for each model used\",\"[ ] The Markdown report includes a line `Cost: $X.XX` for each model with non‑zero token usage\",\"[ ] The test `scripts/tests/eval_harness.test.mjs` asserts that the cost line appears with correct values for at least two known models and that the section is absent when token count is zero\"]}","latency_ms":21550,"tokens_in_est":2717,"tokens_out_est":219}]},{"case_id":"invalid-fr-partial-ac","tags":["invalid","b2","partial-ac","fr"],"repeat":2,"duration_ms":23489,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Le coût en dollars est calculé pour tous les modèles courants\" is vague and not testable – the set of \"modèles courants\" is undefined"],"warnings":[],"suggested_ac":["[ ] The section displays the total `tokens_est` and a per‑case breakdown in the Markdown report","[ ] The section is omitted when `tokens_est` equals 0","[ ] The dollar cost is calculated using the pricing table for each model listed in `src/models/pricing.json` and displayed with two decimal places","[ ] For each supported model (e.g., `gpt-3.5-turbo`, `gpt-4`, `gpt-4o`) the report includes a line showing `tokens_est`, price per token, and total cost in USD","[ ] The test `scripts/tests/eval_harness.test.mjs` asserts that the Markdown contains the cost section with correct values for a mock run using at least two different models and that the section is absent when `tokens_est` is 0"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Le coût en dollars est calculé pour tous les modèles courants\\\" is vague and not testable – the set of \\\"modèles courants\\\" is undefined\"],\"warnings\":[],\"suggested_ac\":[\"[ ] The section displays the total `tokens_est` and a per‑case breakdown in the Markdown report\",\"[ ] The section is omitted when `tokens_est` equals 0\",\"[ ] The dollar cost is calculated using the pricing table for each model listed in `src/models/pricing.json` and displayed with two decimal places\",\"[ ] For each supported model (e.g., `gpt-3.5-turbo`, `gpt-4`, `gpt-4o`) the report includes a line showing `tokens_est`, price per token, and total cost in USD\",\"[ ] The test `scripts/tests/eval_harness.test.mjs` asserts that the Markdown contains the cost section with correct values for a mock run using at least two different models and that the section is absent when `tokens_est` is 0\"]}","latency_ms":23489,"tokens_in_est":2717,"tokens_out_est":228}]},{"case_id":"invalid-role-ambiguous","tags":["invalid","b3","role-scope"],"repeat":0,"duration_ms":6034,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: No test case for PATCH on unknown member (should return 404)"],"suggested_ac":["[ ] GET /api/projects/:id/members returns 200 with an array of objects containing userId and role fields","[ ] PATCH /api/projects/:id/members/:userId returns 200 with updated role when performed by an authorized user","[ ] PATCH /api/projects/:id/members/:userId returns 404 when the specified userId is not a member of the project","[ ] DELETE /api/projects/:id/members/:userId returns 403 for unauthorized users and 404 for non‑existent members","[ ] Tests in test/projects.test.mjs verify all success and error status codes, including unauthorized and not‑found scenarios"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: No test case for PATCH on unknown member (should return 404)\"],\"suggested_ac\":[\"[ ] GET /api/projects/:id/members returns 200 with an array of objects containing userId and role fields\",\"[ ] PATCH /api/projects/:id/members/:userId returns 200 with updated role when performed by an authorized user\",\"[ ] PATCH /api/projects/:id/members/:userId returns 404 when the specified userId is not a member of the project\",\"[ ] DELETE /api/projects/:id/members/:userId returns 403 for unauthorized users and 404 for non‑existent members\",\"[ ] Tests in test/projects.test.mjs verify all success and error status codes, including unauthorized and not‑found scenarios\"]}","latency_ms":6034,"tokens_in_est":2754,"tokens_out_est":179}]},{"case_id":"invalid-role-ambiguous","tags":["invalid","b3","role-scope"],"repeat":1,"duration_ms":3984,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: No details on authentication/authorization mechanism or role definitions are provided","W2: Edge cases such as empty member list, invalid role values, or malformed request bodies are not covered"],"suggested_ac":["[ ] GET /api/projects/:id/members returns 200 with an empty array when the project has no members","[ ] PATCH /api/projects/:id/members/:userId returns 400 when the provided role value is not one of the allowed roles","[ ] PATCH /api/projects/:id/members/:userId returns 404 when the specified userId is not a member of the project","[ ] DELETE /api/projects/:id/members/:userId returns 403 for unauthorized users even if the member exists","[ ] DELETE /api/projects/:id/members/:userId returns 400 when the userId parameter is not a valid UUID"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: No details on authentication/authorization mechanism or role definitions are provided\",\"W2: Edge cases such as empty member list, invalid role values, or malformed request bodies are not covered\"],\"suggested_ac\":[\"[ ] GET /api/projects/:id/members returns 200 with an empty array when the project has no members\",\"[ ] PATCH /api/projects/:id/members/:userId returns 400 when the provided role value is not one of the allowed roles\",\"[ ] PATCH /api/projects/:id/members/:userId returns 404 when the specified userId is not a member of the project\",\"[ ] DELETE /api/projects/:id/members/:userId returns 403 for unauthorized users even if the member exists\",\"[ ] DELETE /api/projects/:id/members/:userId returns 400 when the userId parameter is not a valid UUID\"]}","latency_ms":3983,"tokens_in_est":2754,"tokens_out_est":205}]},{"case_id":"invalid-role-ambiguous","tags":["invalid","b3","role-scope"],"repeat":2,"duration_ms":5381,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W1: No description of the authorization mechanism determining which users are allowed to manage members"],"suggested_ac":["[ ] GET /api/projects/:id/members returns 401 Unauthorized when the request lacks valid authentication","[ ] PATCH /api/projects/:id/members/:userId returns 404 Not Found when the specified userId is not a member of the project","[ ] PATCH /api/projects/:id/members/:userId returns 400 Bad Request when the provided role value is not one of the allowed roles","[ ] DELETE /api/projects/:id/members/:userId returns 401 Unauthorized for unauthenticated requests","[ ] DELETE /api/projects/:id/members/:userId returns 404 Not Found when attempting to delete a member that does not exist"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W1: No description of the authorization mechanism determining which users are allowed to manage members\"],\"suggested_ac\":[\"[ ] GET /api/projects/:id/members returns 401 Unauthorized when the request lacks valid authentication\",\"[ ] PATCH /api/projects/:id/members/:userId returns 404 Not Found when the specified userId is not a member of the project\",\"[ ] PATCH /api/projects/:id/members/:userId returns 400 Bad Request when the provided role value is not one of the allowed roles\",\"[ ] DELETE /api/projects/:id/members/:userId returns 401 Unauthorized for unauthenticated requests\",\"[ ] DELETE /api/projects/:id/members/:userId returns 404 Not Found when attempting to delete a member that does not exist\"]}","latency_ms":5381,"tokens_in_est":2754,"tokens_out_est":191}]},{"case_id":"invalid-role-two-admin-roles","tags":["invalid","b3","role-scope"],"repeat":0,"duration_ms":4064,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given an authenticated request with a valid JWT and a user having the org_admin role, when calling GET /api/audit/export, then the response status is 200 and the Content-Type is text/csv with a non‑empty body","Given an authenticated request with a valid JWT and a user having the project_admin role, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only entries for the scoped project","Given an authenticated request with a valid JWT but the user lacking both admin roles, when calling GET /api/audit/export, then the response status is 403","Given a request without any authentication token, when calling GET /api/audit/export, then the response status is 401","Given an authenticated request with an expired or malformed JWT, when calling GET /api/audit/export, then the response status is 401"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given an authenticated request with a valid JWT and a user having the org_admin role, when calling GET /api/audit/export, then the response status is 200 and the Content-Type is text/csv with a non‑empty body\",\"Given an authenticated request with a valid JWT and a user having the project_admin role, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only entries for the scoped project\",\"Given an authenticated request with a valid JWT but the user lacking both admin roles, when calling GET /api/audit/export, then the response status is 403\",\"Given a request without any authentication token, when calling GET /api/audit/export, then the response status is 401\",\"Given an authenticated request with an expired or malformed JWT, when calling GET /api/audit/export, then the response status is 401\"]}","latency_ms":4064,"tokens_in_est":2707,"tokens_out_est":228}]},{"case_id":"invalid-role-two-admin-roles","tags":["invalid","b3","role-scope"],"repeat":1,"duration_ms":2561,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given an authenticated request by a user with role org_admin, when calling GET /api/audit/export, then the response status is 200 and the Content-Type header is text/csv with a non-empty CSV body","Given an authenticated request by a user with role project_admin for a project they administer, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only audit entries for that project","Given an authenticated request by a user without admin roles, when calling GET /api/audit/export, then the response status is 403 and the response body contains an error message indicating insufficient permissions","Given an unauthenticated request to GET /api/audit/export, when no authentication token is provided, then the response status is 401 and the response includes a WWW-Authenticate header","Given a request to GET /api/audit/export that triggers an internal server error (e.g., database timeout), when the error occurs, then the response status is 500 and the response body contains a generic error message"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given an authenticated request by a user with role org_admin, when calling GET /api/audit/export, then the response status is 200 and the Content-Type header is text/csv with a non-empty CSV body\",\"Given an authenticated request by a user with role project_admin for a project they administer, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only audit entries for that project\",\"Given an authenticated request by a user without admin roles, when calling GET /api/audit/export, then the response status is 403 and the response body contains an error message indicating insufficient permissions\",\"Given an unauthenticated request to GET /api/audit/export, when no authentication token is provided, then the response status is 401 and the response includes a WWW-Authenticate header\",\"Given a request to GET /api/audit/export that triggers an internal server error (e.g., database timeout), when the error occurs, then the response status is 500 and the response body contains a generic error message\"]}","latency_ms":2561,"tokens_in_est":2707,"tokens_out_est":279}]},{"case_id":"invalid-role-two-admin-roles","tags":["invalid","b3","role-scope"],"repeat":2,"duration_ms":4951,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given an authenticated request by a user with role org_admin, when calling GET /api/audit/export, then the response status is 200 and the Content-Type header is text/csv with a non‑empty CSV body","Given an authenticated request by a user with role project_admin, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only entries for projects the user administers","Given an authenticated request by a user without admin roles, when calling GET /api/audit/export, then the response status is 403 and the response body includes an error message indicating insufficient permissions","Given an unauthenticated request to GET /api/audit/export, when no authentication token is provided, then the response status is 401 and the response includes a WWW-Authenticate header","Given a request to GET /api/audit/export with a malformed authentication token, when the token cannot be parsed, then the response status is 401"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given an authenticated request by a user with role org_admin, when calling GET /api/audit/export, then the response status is 200 and the Content-Type header is text/csv with a non‑empty CSV body\",\"Given an authenticated request by a user with role project_admin, when calling GET /api/audit/export, then the response status is 200 and the CSV contains only entries for projects the user administers\",\"Given an authenticated request by a user without admin roles, when calling GET /api/audit/export, then the response status is 403 and the response body includes an error message indicating insufficient permissions\",\"Given an unauthenticated request to GET /api/audit/export, when no authentication token is provided, then the response status is 401 and the response includes a WWW-Authenticate header\",\"Given a request to GET /api/audit/export with a malformed authentication token, when the token cannot be parsed, then the response status is 401\"]}","latency_ms":4951,"tokens_in_est":2707,"tokens_out_est":256}]},{"case_id":"valid-scope-environment-closed","tags":["valid","b3","scope-pair"],"repeat":0,"duration_ms":3499,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference the code location where LOG_LEVEL is applied or the mechanism that reads the yaml files"],"suggested_ac":["[ ] The application reads LOG_LEVEL from the environment and defaults to the value defined in the corresponding yaml file for each environment","[ ] In preview, staging, and production environments the application logs at INFO level and never includes request bodies in logs","[ ] A unit test verifies that when LOG_LEVEL is set to debug in local.yaml, the logger outputs DEBUG level only in local environment","[ ] An integration test confirms that changing LOG_LEVEL to info in preview.yaml results in INFO level logs without request body content"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference the code location where LOG_LEVEL is applied or the mechanism that reads the yaml files\"],\"suggested_ac\":[\"[ ] The application reads LOG_LEVEL from the environment and defaults to the value defined in the corresponding yaml file for each environment\",\"[ ] In preview, staging, and production environments the application logs at INFO level and never includes request bodies in logs\",\"[ ] A unit test verifies that when LOG_LEVEL is set to debug in local.yaml, the logger outputs DEBUG level only in local environment\",\"[ ] An integration test confirms that changing LOG_LEVEL to info in preview.yaml results in INFO level logs without request body content\"]}","latency_ms":3499,"tokens_in_est":2701,"tokens_out_est":185}]},{"case_id":"valid-scope-environment-closed","tags":["valid","b3","scope-pair"],"repeat":1,"duration_ms":3677,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference the code location where LOG_LEVEL is applied, making implementation location ambiguous"],"suggested_ac":["[ ] The application startup reads LOG_LEVEL from the environment and defaults to the value defined in the respective YAML file for each environment","[ ] In preview, staging, and production environments the effective LOG_LEVEL at runtime is 'info' and no debug logs are emitted for any request","[ ] In the local environment the effective LOG_LEVEL remains 'debug' and debug logs are emitted as expected","[ ] config/tests/config.test.mjs fails if any deployed environment YAML file contains LOG_LEVEL other than 'info'","[ ] A CI test verifies that changing LOG_LEVEL in a deployed environment YAML to a non-'info' value causes the test suite to fail"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference the code location where LOG_LEVEL is applied, making implementation location ambiguous\"],\"suggested_ac\":[\"[ ] The application startup reads LOG_LEVEL from the environment and defaults to the value defined in the respective YAML file for each environment\",\"[ ] In preview, staging, and production environments the effective LOG_LEVEL at runtime is 'info' and no debug logs are emitted for any request\",\"[ ] In the local environment the effective LOG_LEVEL remains 'debug' and debug logs are emitted as expected\",\"[ ] config/tests/config.test.mjs fails if any deployed environment YAML file contains LOG_LEVEL other than 'info'\",\"[ ] A CI test verifies that changing LOG_LEVEL in a deployed environment YAML to a non-'info' value causes the test suite to fail\"]}","latency_ms":3677,"tokens_in_est":2701,"tokens_out_est":211}]},{"case_id":"valid-scope-environment-closed","tags":["valid","b3","scope-pair"],"repeat":2,"duration_ms":2383,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference the code location or mechanism that sets LOG_LEVEL, making implementation context unclear"],"suggested_ac":["[ ] The application startup reads LOG_LEVEL from the environment and defaults to \"info\" when not overridden in deployed environments (preview, staging, production)","[ ] CI pipeline includes a test that fails if any deployed environment configuration file sets LOG_LEVEL to \"debug\"","[ ] Documentation (README) includes a section describing the required LOG_LEVEL values per environment and explains why debug is prohibited in deployed environments","[ ] The test suite verifies that `scripts/tests/config.test.mjs` fails when `LOG_LEVEL` is set to \"debug\" in preview.yaml, staging.yaml, or production.yaml"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference the code location or mechanism that sets LOG_LEVEL, making implementation context unclear\"],\"suggested_ac\":[\"[ ] The application startup reads LOG_LEVEL from the environment and defaults to \\\"info\\\" when not overridden in deployed environments (preview, staging, production)\",\"[ ] CI pipeline includes a test that fails if any deployed environment configuration file sets LOG_LEVEL to \\\"debug\\\"\",\"[ ] Documentation (README) includes a section describing the required LOG_LEVEL values per environment and explains why debug is prohibited in deployed environments\",\"[ ] The test suite verifies that `scripts/tests/config.test.mjs` fails when `LOG_LEVEL` is set to \\\"debug\\\" in preview.yaml, staging.yaml, or production.yaml\"]}","latency_ms":2383,"tokens_in_est":2701,"tokens_out_est":202}]},{"case_id":"invalid-scope-environment-open","tags":["invalid","b3","scope-pair"],"repeat":0,"duration_ms":4576,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not specify which configuration loading code or deployment scripts need to be modified, missing technical context about affected files/modules."],"suggested_ac":["- Ensure that the configuration loader sets LOG_LEVEL to \"info\" for ci.yaml, preview.yaml, staging.yaml, and production.yaml files.","- Verify that the application startup logs the effective LOG_LEVEL and that it is \"info\" in deployed environments and \"debug\" only in local.yaml.","- Add a test case in scripts/tests/config.test.mjs that loads each environment file and asserts LOG_LEVEL equals \"info\" for non‑local environments and \"debug\" for local.yaml.","- Confirm that the CI pipeline fails if any deployed environment file contains LOG_LEVEL other than \"info\".","- Ensure that documentation in README lists the required LOG_LEVEL values per environment."]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify which configuration loading code or deployment scripts need to be modified, missing technical context about affected files/modules.\"],\"suggested_ac\":[\"- Ensure that the configuration loader sets LOG_LEVEL to \\\"info\\\" for ci.yaml, preview.yaml, staging.yaml, and production.yaml files.\",\"- Verify that the application startup logs the effective LOG_LEVEL and that it is \\\"info\\\" in deployed environments and \\\"debug\\\" only in local.yaml.\",\"- Add a test case in scripts/tests/config.test.mjs that loads each environment file and asserts LOG_LEVEL equals \\\"info\\\" for non‑local environments and \\\"debug\\\" for local.yaml.\",\"- Confirm that the CI pipeline fails if any deployed environment file contains LOG_LEVEL other than \\\"info\\\".\",\"- Ensure that documentation in README lists the required LOG_LEVEL values per environment.\"]}","latency_ms":4576,"tokens_in_est":2704,"tokens_out_est":226}]},{"case_id":"invalid-scope-environment-open","tags":["invalid","b3","scope-pair"],"repeat":1,"duration_ms":19232,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference the code location where LOG_LEVEL is read or set, limiting context for implementation"],"suggested_ac":["[ ] When the application starts in any deployed environment, the effective log level is INFO as reported by the logger configuration API","[ ] If LOG_LEVEL is overridden to a non‑INFO value in a deployed environment, the application logs a warning and resets it to INFO on startup","[ ] The test suite fails if any deployed environment configuration file sets LOG_LEVEL to a value other than INFO","[ ] Documentation (README) includes a section listing the allowed LOG_LEVEL values per environment and explains why debug is prohibited in deployed environments"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference the code location where LOG_LEVEL is read or set, limiting context for implementation\"],\"suggested_ac\":[\"[ ] When the application starts in any deployed environment, the effective log level is INFO as reported by the logger configuration API\",\"[ ] If LOG_LEVEL is overridden to a non‑INFO value in a deployed environment, the application logs a warning and resets it to INFO on startup\",\"[ ] The test suite fails if any deployed environment configuration file sets LOG_LEVEL to a value other than INFO\",\"[ ] Documentation (README) includes a section listing the allowed LOG_LEVEL values per environment and explains why debug is prohibited in deployed environments\"]}","latency_ms":19232,"tokens_in_est":2704,"tokens_out_est":187}]},{"case_id":"invalid-scope-environment-open","tags":["invalid","b3","scope-pair"],"repeat":2,"duration_ms":4476,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not specify which code location sets LOG_LEVEL for deployed environments"],"suggested_ac":["[ ] When the application starts in a deployed environment, the effective log level is INFO as read from the corresponding YAML file","[ ] The CI pipeline fails if any deployed environment YAML file sets LOG_LEVEL to a value other than INFO","[ ] Adding a new environment YAML file without specifying LOG_LEVEL defaults to INFO","[ ] The test suite includes a negative test that ensures LOG_LEVEL is not DEBUG in any deployed environment configuration file"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify which code location sets LOG_LEVEL for deployed environments\"],\"suggested_ac\":[\"[ ] When the application starts in a deployed environment, the effective log level is INFO as read from the corresponding YAML file\",\"[ ] The CI pipeline fails if any deployed environment YAML file sets LOG_LEVEL to a value other than INFO\",\"[ ] Adding a new environment YAML file without specifying LOG_LEVEL defaults to INFO\",\"[ ] The test suite includes a negative test that ensures LOG_LEVEL is not DEBUG in any deployed environment configuration file\"]}","latency_ms":4476,"tokens_in_est":2704,"tokens_out_est":155}]},{"case_id":"invalid-b4-pair-none","tags":["invalid","b4","stub"],"repeat":0,"duration_ms":21144,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: The pricing service dependency is not deployed and no stub or mock is provided"],"warnings":["W1: No reference to the pricing service implementation or its interface is given"],"suggested_ac":["- [ ] When the pricing service returns an error, the comment includes `Cost: unavailable` and logs the error","- [ ] If input or output token count is zero, the cost calculation correctly results in $0.0000","- [ ] Prices are cached and not re-fetched if the script is invoked multiple times within the same process","- [ ] The script validates that fetched prices are positive numbers before calculation","- [ ] The test suite includes a case with non‑integer token counts to verify rounding behavior"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: The pricing service dependency is not deployed and no stub or mock is provided\"],\"warnings\":[\"W1: No reference to the pricing service implementation or its interface is given\"],\"suggested_ac\":[\"- [ ] When the pricing service returns an error, the comment includes `Cost: unavailable` and logs the error\",\"- [ ] If input or output token count is zero, the cost calculation correctly results in $0.0000\",\"- [ ] Prices are cached and not re-fetched if the script is invoked multiple times within the same process\",\"- [ ] The script validates that fetched prices are positive numbers before calculation\",\"- [ ] The test suite includes a case with non‑integer token counts to verify rounding behavior\"]}","latency_ms":21144,"tokens_in_est":2743,"tokens_out_est":186}]},{"case_id":"invalid-b4-pair-none","tags":["invalid","b4","stub"],"repeat":1,"duration_ms":23525,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: Pricing service dependency is not deployed and no mock or stub is provided"],"warnings":["W2: Edge cases such as pricing service failure or missing token counts are not covered"],"suggested_ac":["[ ] If the pricing service request fails, the comment includes `Cost: unavailable` and logs an error","[ ] Prices are fetched exactly once per script execution, verified by mocking the pricing client and asserting a single call","[ ] The comment format matches `Cost: $<value>` with the value rounded to 4 decimal places for various token counts (e.g., 0, 1, 10,000 tokens)","[ ] A unit test mocks the pricing service and verifies the cost calculation for multiple models with different per‑million‑token prices"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: Pricing service dependency is not deployed and no mock or stub is provided\"],\"warnings\":[\"W2: Edge cases such as pricing service failure or missing token counts are not covered\"],\"suggested_ac\":[\"[ ] If the pricing service request fails, the comment includes `Cost: unavailable` and logs an error\",\"[ ] Prices are fetched exactly once per script execution, verified by mocking the pricing client and asserting a single call\",\"[ ] The comment format matches `Cost: $<value>` with the value rounded to 4 decimal places for various token counts (e.g., 0, 1, 10,000 tokens)\",\"[ ] A unit test mocks the pricing service and verifies the cost calculation for multiple models with different per‑million‑token prices\"]}","latency_ms":23525,"tokens_in_est":2743,"tokens_out_est":189}]},{"case_id":"invalid-b4-pair-none","tags":["invalid","b4","stub"],"repeat":2,"duration_ms":4173,"error":null,"output":{"valid":false,"score":70,"blockers":["B4: Pricing service dependency is not deployed and no stub or mock is provided"],"warnings":["W1: No details about the pricing service API contract or endpoint are provided","W2: Edge cases such as API failure or zero token counts are not covered"],"suggested_ac":["[ ] The script logs an error and skips cost calculation if the pricing service request fails or returns invalid data","[ ] When input tokens are 0 and output tokens are 0, the comment ends with `Cost: $0.0000`","[ ] Prices are fetched exactly once per execution even if multiple PR reviews are processed in the same run","[ ] The test suite includes a case where the pricing service returns a 500 error and verifies that the comment still posts without a cost line"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B4: Pricing service dependency is not deployed and no stub or mock is provided\"],\"warnings\":[\"W1: No details about the pricing service API contract or endpoint are provided\",\"W2: Edge cases such as API failure or zero token counts are not covered\"],\"suggested_ac\":[\"[ ] The script logs an error and skips cost calculation if the pricing service request fails or returns invalid data\",\"[ ] When input tokens are 0 and output tokens are 0, the comment ends with `Cost: $0.0000`\",\"[ ] Prices are fetched exactly once per execution even if multiple PR reviews are processed in the same run\",\"[ ] The test suite includes a case where the pricing service returns a 500 error and verifies that the comment still posts without a cost line\"]}","latency_ms":4173,"tokens_in_est":2743,"tokens_out_est":193}]},{"case_id":"invalid-b4-pair-ticket","tags":["invalid","b4","stub"],"repeat":0,"duration_ms":3466,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: Pricing service dependency is not deployed and no stub or mock is provided"],"warnings":["W2: Edge cases such as zero tokens or pricing service failure are not covered"],"suggested_ac":["- [ ] The comment ends with `Cost: $<usd>` where the amount is calculated as (inputTokens/1e6)*inputPerMTok + (outputTokens/1e6)*outputPerMTok, rounded to 4 decimal places","- [ ] Prices are fetched exactly once per script execution, verified by mocking the HTTP client and asserting a single request to `/prices/:model`","- [ ] If the pricing service returns an error or non‑200 response, the script logs a warning and omits the cost line without failing the review comment","- [ ] When inputTokens or outputTokens are zero, the cost is calculated correctly (e.g., 0 input and 500 output tokens yields cost based solely on output price)","- [ ] A test in `scripts/tests/pr_review.test.mjs` verifies the cost line for 1,000 input and 500 output tokens using a mocked pricing response"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: Pricing service dependency is not deployed and no stub or mock is provided\"],\"warnings\":[\"W2: Edge cases such as zero tokens or pricing service failure are not covered\"],\"suggested_ac\":[\"- [ ] The comment ends with `Cost: $<usd>` where the amount is calculated as (inputTokens/1e6)*inputPerMTok + (outputTokens/1e6)*outputPerMTok, rounded to 4 decimal places\",\"- [ ] Prices are fetched exactly once per script execution, verified by mocking the HTTP client and asserting a single request to `/prices/:model`\",\"- [ ] If the pricing service returns an error or non‑200 response, the script logs a warning and omits the cost line without failing the review comment\",\"- [ ] When inputTokens or outputTokens are zero, the cost is calculated correctly (e.g., 0 input and 500 output tokens yields cost based solely on output price)\",\"- [ ] A test in `scripts/tests/pr_review.test.mjs` verifies the cost line for 1,000 input and 500 output tokens using a mocked pricing response\"]}","latency_ms":3466,"tokens_in_est":2779,"tokens_out_est":254}]},{"case_id":"invalid-b4-pair-ticket","tags":["invalid","b4","stub"],"repeat":1,"duration_ms":8162,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: Pricing service dependency is not deployed and no stub or mock is provided"],"warnings":["W2: Edge cases such as zero tokens or missing token counts are not addressed"],"suggested_ac":["[ ] The script handles pricing service errors gracefully by logging a warning and omitting the cost line in the comment","[ ] When input or output token count is zero, the cost calculation correctly results in $0.0000","[ ] The fetched prices are cached for the duration of the script execution and not re-fetched for multiple model evaluations","[ ] The cost line is formatted with exactly four decimal places even when the value is a whole number","[ ] A test verifies that if the pricing service returns a non‑200 response, the script does not crash and the comment does not include a cost line"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: Pricing service dependency is not deployed and no stub or mock is provided\"],\"warnings\":[\"W2: Edge cases such as zero tokens or missing token counts are not addressed\"],\"suggested_ac\":[\"[ ] The script handles pricing service errors gracefully by logging a warning and omitting the cost line in the comment\",\"[ ] When input or output token count is zero, the cost calculation correctly results in $0.0000\",\"[ ] The fetched prices are cached for the duration of the script execution and not re-fetched for multiple model evaluations\",\"[ ] The cost line is formatted with exactly four decimal places even when the value is a whole number\",\"[ ] A test verifies that if the pricing service returns a non‑200 response, the script does not crash and the comment does not include a cost line\"]}","latency_ms":8162,"tokens_in_est":2779,"tokens_out_est":208}]},{"case_id":"invalid-b4-pair-ticket","tags":["invalid","b4","stub"],"repeat":2,"duration_ms":4866,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: Pricing service dependency is not deployed and no stub or mock is provided"],"warnings":["W1: Issue does not specify which model(s) the cost calculation should use or where the model identifier is obtained"],"suggested_ac":["- [ ] When the pricing service returns a 404 for a model, the comment includes `Cost: unavailable` and logs a warning","- [ ] Prices are cached per model for the duration of the script execution to ensure only one fetch per model","- [ ] If the pricing service request fails (network error or non‑200 response), the script retries up to 2 times before falling back to `Cost: unavailable`","- [ ] The script correctly formats the cost with exactly 4 decimal places even when the calculated value has fewer decimals","- [ ] A test verifies that no additional HTTP requests are made when the script processes multiple PR reviews in the same run"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: Pricing service dependency is not deployed and no stub or mock is provided\"],\"warnings\":[\"W1: Issue does not specify which model(s) the cost calculation should use or where the model identifier is obtained\"],\"suggested_ac\":[\"- [ ] When the pricing service returns a 404 for a model, the comment includes `Cost: unavailable` and logs a warning\",\"- [ ] Prices are cached per model for the duration of the script execution to ensure only one fetch per model\",\"- [ ] If the pricing service request fails (network error or non‑200 response), the script retries up to 2 times before falling back to `Cost: unavailable`\",\"- [ ] The script correctly formats the cost with exactly 4 decimal places even when the calculated value has fewer decimals\",\"- [ ] A test verifies that no additional HTTP requests are made when the script processes multiple PR reviews in the same run\"]}","latency_ms":4866,"tokens_in_est":2779,"tokens_out_est":228}]},{"case_id":"valid-b4-pair-stub","tags":["valid","b4","stub"],"repeat":0,"duration_ms":6677,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: Issue does not explicitly list affected files or modules beyond script names, missing technical context\",\"W2: Acceptance criteria only cover a single happy‑path token count; edge cases such as zero tokens, very large token counts, and rounding behavior are not addressed\"],\"suggested_ac",":","[ ] The comment ends with `Cost: $<usd>` where the amount is calculated as (inputTokens × inputPrice + outputTokens × outputPrice) using the per‑million‑token prices, rounded to exactly 4 decimal places,[ ] The pricing service (or stub) is invoked exactly once per execution of `pr_review.mjs`; subsequent calls reuse the cached price values,[ ] When inputTokens or outputTokens are zero, the cost calculation correctly reflects zero contribution for that side and the total cost is still rounded to 4 decimals,[ ] For token counts exceeding typical limits (e.g., 10 000 000 input tokens), the cost is computed without overflow or loss of precision and the comment still includes the correctly formatted cost line,[ ] If `getPrice` throws an error, `pr_review.mjs` logs a warning and falls back to a default price of {inputPerMTok:0,outputPerMTok:0} so that the comment still includes a cost line (showing $0.0000)"],"suggested_ac":[]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":0},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: Issue does not explicitly list affected files or modules beyond script names, missing technical context\\\",\\\"W2: Acceptance criteria only cover a single happy‑path token count; edge cases such as zero tokens, very large token counts, and rounding behavior are not addressed\\\"],\\\"suggested_ac\",\":\",[\"[ ] The comment ends with `Cost: $<usd>` where the amount is calculated as (inputTokens × inputPrice + outputTokens × outputPrice) using the per‑million‑token prices, rounded to exactly 4 decimal places\",\"[ ] The pricing service (or stub) is invoked exactly once per execution of `pr_review.mjs`; subsequent calls reuse the cached price values\",\"[ ] When inputTokens or outputTokens are zero, the cost calculation correctly reflects zero contribution for that side and the total cost is still rounded to 4 decimals\",\"[ ] For token counts exceeding typical limits (e.g., 10 000 000 input tokens), the cost is computed without overflow or loss of precision and the comment still includes the correctly formatted cost line\",\"[ ] If `getPrice` throws an error, `pr_review.mjs` logs a warning and falls back to a default price of {inputPerMTok:0,outputPerMTok:0} so that the comment still includes a cost line (showing $0.0000)\"]]}","latency_ms":6676,"tokens_in_est":2786,"tokens_out_est":320}]},{"case_id":"valid-b4-pair-stub","tags":["valid","b4","stub"],"repeat":1,"duration_ms":6025,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge cases such as zero tokens, rounding behavior, and multiple model handling are not covered in the acceptance criteria"],"suggested_ac":["- [ ] The comment correctly displays `Cost: $0.0000` when both input and output token counts are zero","- [ ] The cost calculation rounds to exactly four decimal places for any token counts, e.g., 1234 input and 567 output tokens produce the expected rounded value","- [ ] When multiple models are reviewed in a single run, the comment includes a separate cost line for each model and aggregates the total cost correctly","- [ ] If `getPrice(model)` throws an error, the script logs a warning and omits the cost line without failing the review process","- [ ] A unit test verifies that `pricing_stub.mjs` is called exactly once per script execution regardless of the number of models processed"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as zero tokens, rounding behavior, and multiple model handling are not covered in the acceptance criteria\"],\"suggested_ac\":[\"- [ ] The comment correctly displays `Cost: $0.0000` when both input and output token counts are zero\",\"- [ ] The cost calculation rounds to exactly four decimal places for any token counts, e.g., 1234 input and 567 output tokens produce the expected rounded value\",\"- [ ] When multiple models are reviewed in a single run, the comment includes a separate cost line for each model and aggregates the total cost correctly\",\"- [ ] If `getPrice(model)` throws an error, the script logs a warning and omits the cost line without failing the review process\",\"- [ ] A unit test verifies that `pricing_stub.mjs` is called exactly once per script execution regardless of the number of models processed\"]}","latency_ms":6025,"tokens_in_est":2786,"tokens_out_est":224}]},{"case_id":"valid-b4-pair-stub","tags":["valid","b4","stub"],"repeat":2,"duration_ms":5855,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W2: Edge cases such as zero tokens or pricing service errors are not covered"],"suggested_ac":["- [ ] When inputTokens or outputTokens is zero, the cost line shows $0.0000","- [ ] If getPrice throws an error, the comment includes \"Cost: unavailable\" and the script exits gracefully","- [ ] Prices are cached so that multiple calls to getPrice within the same run do not trigger additional fetches","- [ ] The cost calculation rounds correctly for values that require more than four decimal places","- [ ] The comment format remains unchanged except for the appended cost line"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as zero tokens or pricing service errors are not covered\"],\"suggested_ac\":[\"- [ ] When inputTokens or outputTokens is zero, the cost line shows $0.0000\",\"- [ ] If getPrice throws an error, the comment includes \\\"Cost: unavailable\\\" and the script exits gracefully\",\"- [ ] Prices are cached so that multiple calls to getPrice within the same run do not trigger additional fetches\",\"- [ ] The cost calculation rounds correctly for values that require more than four decimal places\",\"- [ ] The comment format remains unchanged except for the appended cost line\"]}","latency_ms":5855,"tokens_in_est":2786,"tokens_out_est":158}]},{"case_id":"invalid-fr-vague-ac","tags":["invalid","b1","fr"],"repeat":0,"duration_ms":4078,"error":null,"output":{"valid":false,"score":55,"blockers":["B2: AC item \"Corriger le bug\" is non-testable and vague","B2: AC item \"L'export doit fonctionner correctement\" is non-testable and vague"],"warnings":["W1: Issue lacks technical context such as affected modules or files","W2: No edge cases or error conditions are described"],"suggested_ac":["Given a valid dataset, when the user triggers CSV export, then a CSV file is generated and downloaded with a 200 HTTP status","Given an empty dataset, when the user triggers CSV export, then the system returns a CSV file containing only the header row and a 200 status","Given a dataset containing special characters, when the user triggers CSV export, then the generated CSV correctly escapes commas and quotes according to RFC 4180","Given a failure in the data retrieval layer, when the user triggers CSV export, then the operation returns a 500 error with a JSON body containing an error code \"EXPORT_FAILURE\"","Given a large dataset of 1 000 000 rows, when the user triggers CSV export, then the export completes within 30 seconds"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":55,\"blockers\":[\"B2: AC item \\\"Corriger le bug\\\" is non-testable and vague\",\"B2: AC item \\\"L'export doit fonctionner correctement\\\" is non-testable and vague\"],\"warnings\":[\"W1: Issue lacks technical context such as affected modules or files\",\"W2: No edge cases or error conditions are described\"],\"suggested_ac\":[\"Given a valid dataset, when the user triggers CSV export, then a CSV file is generated and downloaded with a 200 HTTP status\",\"Given an empty dataset, when the user triggers CSV export, then the system returns a CSV file containing only the header row and a 200 status\",\"Given a dataset containing special characters, when the user triggers CSV export, then the generated CSV correctly escapes commas and quotes according to RFC 4180\",\"Given a failure in the data retrieval layer, when the user triggers CSV export, then the operation returns a 500 error with a JSON body containing an error code \\\"EXPORT_FAILURE\\\"\",\"Given a large dataset of 1 000 000 rows, when the user triggers CSV export, then the export completes within 30 seconds\"]}","latency_ms":4078,"tokens_in_est":2648,"tokens_out_est":269}]},{"case_id":"invalid-fr-vague-ac","tags":["invalid","b1","fr"],"repeat":1,"duration_ms":21472,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: AC item \"Corriger le bug\" is vague and not testable","B2: AC item \"L'export doit fonctionner correctement\" is vague and not testable"],"warnings":["W1: Issue does not specify affected files, modules, or components"],"suggested_ac":["[ ] Export CSV returns a file with correct headers and data rows for a valid dataset","[ ] Export CSV returns HTTP 200 and a non‑empty CSV file when invoked with a non‑empty dataset","[ ] Export CSV returns HTTP 400 with an error message when invoked with an empty dataset","[ ] Export CSV completes within 2 seconds for datasets up to 10,000 rows"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":0,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: AC item \\\"Corriger le bug\\\" is vague and not testable\",\"B2: AC item \\\"L'export doit fonctionner correctement\\\" is vague and not testable\"],\"warnings\":[\"W1: Issue does not specify affected files, modules, or components\"],\"suggested_ac\":[\"[ ] Export CSV returns a file with correct headers and data rows for a valid dataset\",\"[ ] Export CSV returns HTTP 200 and a non‑empty CSV file when invoked with a non‑empty dataset\",\"[ ] Export CSV returns HTTP 400 with an error message when invoked with an empty dataset\",\"[ ] Export CSV completes within 2 seconds for datasets up to 10,000 rows\"]}","latency_ms":21472,"tokens_in_est":2648,"tokens_out_est":158}]},{"case_id":"invalid-fr-vague-ac","tags":["invalid","b1","fr"],"repeat":2,"duration_ms":20398,"error":null,"output":{"valid":false,"score":70,"blockers":["B2: AC item \"Corriger le bug\" is vague and non‑testable","B2: AC item \"L'export doit fonctionner correctement\" is vague and non‑testable"],"warnings":["W1: No technical context or affected components are mentioned","W2: No edge cases or error conditions are described"],"suggested_ac":["Given a non‑empty dataset, when the user clicks the CSV export button, then a .csv file is downloaded containing all rows with correct headers","Given an empty dataset, when the user clicks the CSV export button, then a .csv file is downloaded containing only the header row","Given data containing commas or newlines, when the CSV is generated, then fields are properly escaped according to RFC 4180","Given a server timeout during CSV generation, when the export is requested, then the user receives a clear error message and no partial file is created","Given a permission‑restricted user, when they attempt to export CSV, then the operation is denied with an appropriate authorization error"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":0,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":70,\"blockers\":[\"B2: AC item \\\"Corriger le bug\\\" is vague and non‑testable\",\"B2: AC item \\\"L'export doit fonctionner correctement\\\" is vague and non‑testable\"],\"warnings\":[\"W1: No technical context or affected components are mentioned\",\"W2: No edge cases or error conditions are described\"],\"suggested_ac\":[\"Given a non‑empty dataset, when the user clicks the CSV export button, then a .csv file is downloaded containing all rows with correct headers\",\"Given an empty dataset, when the user clicks the CSV export button, then a .csv file is downloaded containing only the header row\",\"Given data containing commas or newlines, when the CSV is generated, then fields are properly escaped according to RFC 4180\",\"Given a server timeout during CSV generation, when the export is requested, then the user receives a clear error message and no partial file is created\",\"Given a permission‑restricted user, when they attempt to export CSV, then the operation is denied with an appropriate authorization error\"]}","latency_ms":20398,"tokens_in_est":2648,"tokens_out_est":257}]},{"case_id":"invalid-short-no-ac","tags":["invalid","b1","short"],"repeat":0,"duration_ms":5076,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components referenced beyond config/models.yaml","W2: Edge cases not covered – no mention of validation rules or error handling for the YAML cleanup"],"suggested_ac":["[ ] The updated config/models.yaml is syntactically valid YAML and can be parsed without errors","[ ] All model entries in config/models.yaml include required fields: name, version, and path","[ ] Removing unused model entries does not affect any existing code paths; running the test suite passes with zero failures","[ ] A CI lint step validates that config/models.yaml conforms to the project's schema and reports any violations","[ ] Documentation is updated to reflect any changes to model configuration keys and their meanings"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components referenced beyond config/models.yaml\",\"W2: Edge cases not covered – no mention of validation rules or error handling for the YAML cleanup\"],\"suggested_ac\":[\"[ ] The updated config/models.yaml is syntactically valid YAML and can be parsed without errors\",\"[ ] All model entries in config/models.yaml include required fields: name, version, and path\",\"[ ] Removing unused model entries does not affect any existing code paths; running the test suite passes with zero failures\",\"[ ] A CI lint step validates that config/models.yaml conforms to the project's schema and reports any violations\",\"[ ] Documentation is updated to reflect any changes to model configuration keys and their meanings\"]}","latency_ms":5076,"tokens_in_est":2623,"tokens_out_est":214}]},{"case_id":"invalid-short-no-ac","tags":["invalid","b1","short"],"repeat":1,"duration_ms":1884,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components beyond generic config file mentioned"],"suggested_ac":["[ ] The updated config/models.yaml is syntactically valid YAML and can be parsed without errors","[ ] All model entries retain their original keys and values unless explicitly removed, preserving backward compatibility","[ ] Removed obsolete model entries are documented in a CHANGELOG section with their previous names","[ ] The file size of config/models.yaml does not increase by more than 10% after cleanup","[ ] A CI lint step validates config/models.yaml against the project's schema and passes"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components beyond generic config file mentioned\"],\"suggested_ac\":[\"[ ] The updated config/models.yaml is syntactically valid YAML and can be parsed without errors\",\"[ ] All model entries retain their original keys and values unless explicitly removed, preserving backward compatibility\",\"[ ] Removed obsolete model entries are documented in a CHANGELOG section with their previous names\",\"[ ] The file size of config/models.yaml does not increase by more than 10% after cleanup\",\"[ ] A CI lint step validates config/models.yaml against the project's schema and passes\"]}","latency_ms":1884,"tokens_in_est":2623,"tokens_out_est":181}]},{"case_id":"invalid-short-no-ac","tags":["invalid","b1","short"],"repeat":2,"duration_ms":18779,"error":null,"output":{"valid":false,"score":60,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components are specified beyond a generic config file"],"suggested_ac":["[ ] The updated `config/models.yaml` file is syntactically valid YAML and can be parsed without errors","[ ] All model entries retain their original keys and values unless explicitly documented as changed","[ ] A CI lint step validates that `config/models.yaml` conforms to the project's schema and passes without warnings","[ ] The change does not alter any runtime behavior; running the model loading routine before and after the cleanup produces identical in‑memory model configurations","[ ] Documentation (README or relevant docs) is updated to reflect any structural changes to `config/models.yaml`"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":60,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components are specified beyond a generic config file\"],\"suggested_ac\":[\"[ ] The updated `config/models.yaml` file is syntactically valid YAML and can be parsed without errors\",\"[ ] All model entries retain their original keys and values unless explicitly documented as changed\",\"[ ] A CI lint step validates that `config/models.yaml` conforms to the project's schema and passes without warnings\",\"[ ] The change does not alter any runtime behavior; running the model loading routine before and after the cleanup produces identical in‑memory model configurations\",\"[ ] Documentation (README or relevant docs) is updated to reflect any structural changes to `config/models.yaml`\"]}","latency_ms":18779,"tokens_in_est":2623,"tokens_out_est":208}]},{"case_id":"valid-short","tags":["valid","short"],"repeat":0,"duration_ms":3286,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: Issue provides minimal technical context about the function's intended behavior beyond the single case","W2: Only the happy‑path case is covered; edge cases such as negative or large values are not addressed"],"suggested_ac":["- [ ] `formatDuration(1)` returns `'1s'`","- [ ] `formatDuration(90)` returns `'1m30s'`","- [ ] `formatDuration(-5)` throws a `RangeError`","- [ ] `formatDuration(3600)` returns `'1h'`","- [ ] Add a regression test file `scripts/tests/metrics.test.mjs` that asserts the above cases and ensures the existing test for `formatDuration(0)` passes"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: Issue provides minimal technical context about the function's intended behavior beyond the single case\",\"W2: Only the happy‑path case is covered; edge cases such as negative or large values are not addressed\"],\"suggested_ac\":[\"- [ ] `formatDuration(1)` returns `'1s'`\",\"- [ ] `formatDuration(90)` returns `'1m30s'`\",\"- [ ] `formatDuration(-5)` throws a `RangeError`\",\"- [ ] `formatDuration(3600)` returns `'1h'`\",\"- [ ] Add a regression test file `scripts/tests/metrics.test.mjs` that asserts the above cases and ensures the existing test for `formatDuration(0)` passes\"]}","latency_ms":3286,"tokens_in_est":2651,"tokens_out_est":157}]},{"case_id":"valid-short","tags":["valid","short"],"repeat":1,"duration_ms":21703,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue lacks detailed technical context about the current implementation of formatDuration and its expected behavior beyond the zero case"],"suggested_ac":["- [ ] `formatDuration(null)` throws a TypeError","- [ ] `formatDuration(12345)` returns a string matching the pattern `/^\\d+[smhd]$/` (e.g., '3h' for 10800 seconds)","- [ ] `formatDuration(-5)` returns a string representing the absolute duration, e.g., '5s'","- [ ] Add a unit test verifying that `formatDuration` correctly formats durations for values up to 86400 seconds (1 day)","- [ ] Ensure the regression test file `scripts/tests/metrics.test.mjs` includes a test case asserting the function returns `'0s'` for input 0"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue lacks detailed technical context about the current implementation of formatDuration and its expected behavior beyond the zero case\"],\"suggested_ac\":[\"- [ ] `formatDuration(null)` throws a TypeError\",\"- [ ] `formatDuration(12345)` returns a string matching the pattern `/^\\\\d+[smhd]$/` (e.g., '3h' for 10800 seconds)\",\"- [ ] `formatDuration(-5)` returns a string representing the absolute duration, e.g., '5s'\",\"- [ ] Add a unit test verifying that `formatDuration` correctly formats durations for values up to 86400 seconds (1 day)\",\"- [ ] Ensure the regression test file `scripts/tests/metrics.test.mjs` includes a test case asserting the function returns `'0s'` for input 0\"]}","latency_ms":21703,"tokens_in_est":2651,"tokens_out_est":185}]},{"case_id":"valid-short","tags":["valid","short"],"repeat":2,"duration_ms":5394,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: Issue provides minimal technical context; does not describe expected behavior of formatDuration beyond the zero case"],"suggested_ac":["- [ ] `formatDuration(5)` returns `'5s'`","- [ ] `formatDuration(125000)` returns `'2m 5s'`","- [ ] `formatDuration(null)` throws a TypeError","- [ ] `formatDuration(-10)` returns `'-10s'`","- [ ] Add a unit test in `scripts/tests/metrics.test.mjs` asserting the zero case and the above cases"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: Issue provides minimal technical context; does not describe expected behavior of formatDuration beyond the zero case\"],\"suggested_ac\":[\"- [ ] `formatDuration(5)` returns `'5s'`\",\"- [ ] `formatDuration(125000)` returns `'2m 5s'`\",\"- [ ] `formatDuration(null)` throws a TypeError\",\"- [ ] `formatDuration(-10)` returns `'-10s'`\",\"- [ ] Add a unit test in `scripts/tests/metrics.test.mjs` asserting the zero case and the above cases\"]}","latency_ms":5394,"tokens_in_est":2651,"tokens_out_est":122}]},{"case_id":"valid-fr-short","tags":["valid","fr","short"],"repeat":0,"duration_ms":4669,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: Issue does not reference existing implementation details or tests beyond adding a new test file","W2: Edge cases such as null input, non‑string types, or strings containing only whitespace are not covered"],"suggested_ac":["slugify(null) throws a TypeError","slugify('   ') returns an empty string ''","slugify('Café--Bar') returns 'cafe-bar'","slugify('ÄÖÜ') returns 'aou'","slugify('123') returns '123'"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference existing implementation details or tests beyond adding a new test file\",\"W2: Edge cases such as null input, non‑string types, or strings containing only whitespace are not covered\"],\"suggested_ac\":[\"slugify(null) throws a TypeError\",\"slugify('   ') returns an empty string ''\",\"slugify('Café--Bar') returns 'cafe-bar'\",\"slugify('ÄÖÜ') returns 'aou'\",\"slugify('123') returns '123'\"]}","latency_ms":4669,"tokens_in_est":2715,"tokens_out_est":116}]},{"case_id":"valid-fr-short","tags":["valid","fr","short"],"repeat":1,"duration_ms":2634,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge cases such as null input, non‑string types, or strings containing only diacritics are not covered in the acceptance criteria"],"suggested_ac":["- [ ] `slugify(null)` throws a TypeError indicating input must be a string","- [ ] `slugify('---')` returns an empty string '' after stripping non‑alphanumeric characters","- [ ] `slugify('ÉéÉé')` returns 'eeee' confirming repeated diacritic removal","- [ ] `slugify('A very long string ...')` processes strings up to 10,000 characters without performance degradation"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as null input, non‑string types, or strings containing only diacritics are not covered in the acceptance criteria\"],\"suggested_ac\":[\"- [ ] `slugify(null)` throws a TypeError indicating input must be a string\",\"- [ ] `slugify('---')` returns an empty string '' after stripping non‑alphanumeric characters\",\"- [ ] `slugify('ÉéÉé')` returns 'eeee' confirming repeated diacritic removal\",\"- [ ] `slugify('A very long string ...')` processes strings up to 10,000 characters without performance degradation\"]}","latency_ms":2634,"tokens_in_est":2715,"tokens_out_est":144}]},{"case_id":"valid-fr-short","tags":["valid","fr","short"],"repeat":2,"duration_ms":5854,"error":null,"output":{"valid":true,"score":95,"blockers":[],"warnings":["W2: Edge cases such as non-string inputs or null values are not covered in the acceptance criteria"],"suggested_ac":["- [ ] slugify(null) throws a TypeError","- [ ] slugify('Café—Bar') returns 'cafe-bar' (handles punctuation removal)","- [ ] slugify('ÄÖÜ') returns 'aou' (covers additional accented characters)","- [ ] slugify('a'.repeat(500)) returns a string of length 500 without errors (handles long inputs)","- [ ] slugify(undefined) throws a TypeError"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":95,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as non-string inputs or null values are not covered in the acceptance criteria\"],\"suggested_ac\":[\"- [ ] slugify(null) throws a TypeError\",\"- [ ] slugify('Café—Bar') returns 'cafe-bar' (handles punctuation removal)\",\"- [ ] slugify('ÄÖÜ') returns 'aou' (covers additional accented characters)\",\"- [ ] slugify('a'.repeat(500)) returns a string of length 500 without errors (handles long inputs)\",\"- [ ] slugify(undefined) throws a TypeError\"]}","latency_ms":5854,"tokens_in_est":2715,"tokens_out_est":128}]},{"case_id":"valid-warnings-only-w1-w2","tags":["valid","warnings-only"],"repeat":0,"duration_ms":2599,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference affected files or modules, missing technical context"],"suggested_ac":["- [ ] `GET /version` returns HTTP 200 with JSON body containing a `version` field matching the `version` field in package.json","- [ ] `GET /version` response includes header `Content-Type: application/json`","- [ ] `POST /version` returns HTTP 405 Method Not Allowed","- [ ] Request to unknown path `/versionx` returns HTTP 404 Not Found","- [ ] If `package.json` does not contain a version field, the endpoint returns HTTP 500 with error message"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference affected files or modules, missing technical context\"],\"suggested_ac\":[\"- [ ] `GET /version` returns HTTP 200 with JSON body containing a `version` field matching the `version` field in package.json\",\"- [ ] `GET /version` response includes header `Content-Type: application/json`\",\"- [ ] `POST /version` returns HTTP 405 Method Not Allowed\",\"- [ ] Request to unknown path `/versionx` returns HTTP 404 Not Found\",\"- [ ] If `package.json` does not contain a version field, the endpoint returns HTTP 500 with error message\"]}","latency_ms":2599,"tokens_in_est":2670,"tokens_out_est":151}]},{"case_id":"valid-warnings-only-w1-w2","tags":["valid","warnings-only"],"repeat":1,"duration_ms":1909,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference affected files or modules, missing technical context"],"suggested_ac":["[ ] GET /version returns HTTP 200 with JSON body containing the exact version string from package.json","[ ] GET /version returns Content-Type header set to application/json","[ ] GET /version returns HTTP 500 with an error message if the version field cannot be read","[ ] GET /nonexistent returns HTTP 404, confirming that only /version is exposed","[ ] GET /version response time is under 100 ms for a typical request"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference affected files or modules, missing technical context\"],\"suggested_ac\":[\"[ ] GET /version returns HTTP 200 with JSON body containing the exact version string from package.json\",\"[ ] GET /version returns Content-Type header set to application/json\",\"[ ] GET /version returns HTTP 500 with an error message if the version field cannot be read\",\"[ ] GET /nonexistent returns HTTP 404, confirming that only /version is exposed\",\"[ ] GET /version response time is under 100 ms for a typical request\"]}","latency_ms":1908,"tokens_in_est":2670,"tokens_out_est":144}]},{"case_id":"valid-warnings-only-w1-w2","tags":["valid","warnings-only"],"repeat":2,"duration_ms":3398,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not reference affected files or modules, missing technical context"],"suggested_ac":["- [ ] `GET /version` returns HTTP 200 with JSON body containing a `version` field matching the `version` field in package.json","- [ ] `GET /version` response includes header `Content-Type: application/json`","- [ ] `GET /version` returns HTTP 405 for any non-GET HTTP method (e.g., POST, PUT)","- [ ] `GET /version` returns HTTP 500 with an error message when the version field cannot be read from package.json","- [ ] The endpoint is documented in the API README with example request and response"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not reference affected files or modules, missing technical context\"],\"suggested_ac\":[\"- [ ] `GET /version` returns HTTP 200 with JSON body containing a `version` field matching the `version` field in package.json\",\"- [ ] `GET /version` response includes header `Content-Type: application/json`\",\"- [ ] `GET /version` returns HTTP 405 for any non-GET HTTP method (e.g., POST, PUT)\",\"- [ ] `GET /version` returns HTTP 500 with an error message when the version field cannot be read from package.json\",\"- [ ] The endpoint is documented in the API README with example request and response\"]}","latency_ms":3397,"tokens_in_est":2670,"tokens_out_est":164}]},{"case_id":"valid-warnings-only-w2","tags":["valid","warnings-only"],"repeat":0,"duration_ms":5194,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W2: Edge cases such as handling errors from gh commands or invalid input are not covered in the acceptance criteria"],"suggested_ac":["Given the script is run with the `--dry-run` flag and a label needs to be created, when the script processes the label, then it prints `create <name>` to stdout and does not invoke any `gh` command","Given the script is run with the `--dry-run` flag and a label needs to be updated, when the script processes the label, then it prints `update <name>` to stdout and does not invoke any `gh` command","Given the script is run without the `--dry-run` flag and a `gh` command fails, when the script encounters the failure, then it exits with a non-zero status and logs the error appropriately","Given the script is run with the `--dry-run` flag and an invalid label name is provided, when the script validates inputs, then it prints an error message and exits without invoking any `gh` command","Given the script is run without the `--dry-run` flag, when all label operations succeed, then existing tests in `scripts/tests/manage_labels.test.mjs` continue to pass"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W2: Edge cases such as handling errors from gh commands or invalid input are not covered in the acceptance criteria\"],\"suggested_ac\":[\"Given the script is run with the `--dry-run` flag and a label needs to be created, when the script processes the label, then it prints `create <name>` to stdout and does not invoke any `gh` command\",\"Given the script is run with the `--dry-run` flag and a label needs to be updated, when the script processes the label, then it prints `update <name>` to stdout and does not invoke any `gh` command\",\"Given the script is run without the `--dry-run` flag and a `gh` command fails, when the script encounters the failure, then it exits with a non-zero status and logs the error appropriately\",\"Given the script is run with the `--dry-run` flag and an invalid label name is provided, when the script validates inputs, then it prints an error message and exits without invoking any `gh` command\",\"Given the script is run without the `--dry-run` flag, when all label operations succeed, then existing tests in `scripts/tests/manage_labels.test.mjs` continue to pass\"]}","latency_ms":5194,"tokens_in_est":2717,"tokens_out_est":288}]},{"case_id":"valid-warnings-only-w2","tags":["valid","warnings-only"],"repeat":1,"duration_ms":4348,"error":null,"output":{"valid":true,"score":90,"blockers":[],"warnings":["W1: Issue does not specify which modules or files need to be modified beyond the script name, limiting context for implementation"],"suggested_ac":["[ ] When the script is invoked with `--dry-run`, the exit code is 0 after printing all planned actions","[ ] Invoking the script with `--dry-run` and an invalid label name prints an error message and does not execute any `gh` commands","[ ] The script returns a non-zero exit code if the underlying `gh` command would fail when run without `--dry-run`","[ ] The `--dry-run` flag can be combined with other existing flags (e.g., `--repo`) and still suppress `gh` execution while printing actions"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":90,\"blockers\":[],\"warnings\":[\"W1: Issue does not specify which modules or files need to be modified beyond the script name, limiting context for implementation\"],\"suggested_ac\":[\"[ ] When the script is invoked with `--dry-run`, the exit code is 0 after printing all planned actions\",\"[ ] Invoking the script with `--dry-run` and an invalid label name prints an error message and does not execute any `gh` commands\",\"[ ] The script returns a non-zero exit code if the underlying `gh` command would fail when run without `--dry-run`\",\"[ ] The `--dry-run` flag can be combined with other existing flags (e.g., `--repo`) and still suppress `gh` execution while printing actions\"]}","latency_ms":4348,"tokens_in_est":2717,"tokens_out_est":175}]},{"case_id":"valid-warnings-only-w2","tags":["valid","warnings-only"],"repeat":2,"duration_ms":3188,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":[],"suggested_ac":["Given the script is run with the `--dry-run` flag, when a label needs to be created, then the output includes the line `create <label-name>` and no `gh` command is invoked","Given the script is run with the `--dry-run` flag, when a label needs to be updated, then the output includes the line `update <label-name>` and no `gh` command is invoked","Given the script is run without the `--dry-run` flag, when a label operation is performed, then the script invokes the appropriate `gh` command and the operation behaves as before","Given the script is run with the `--dry-run` flag and the injected exec function throws an error, then the script should catch the error and exit with a non‑zero status without performing any label changes","Given the script is run with the `--dry-run` flag and there are no label changes required, then the script produces no output and still does not invoke any `gh` command"]},"label":"valid","expected_label":"valid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[],\"suggested_ac\":[\"Given the script is run with the `--dry-run` flag, when a label needs to be created, then the output includes the line `create <label-name>` and no `gh` command is invoked\",\"Given the script is run with the `--dry-run` flag, when a label needs to be updated, then the output includes the line `update <label-name>` and no `gh` command is invoked\",\"Given the script is run without the `--dry-run` flag, when a label operation is performed, then the script invokes the appropriate `gh` command and the operation behaves as before\",\"Given the script is run with the `--dry-run` flag and the injected exec function throws an error, then the script should catch the error and exit with a non‑zero status without performing any label changes\",\"Given the script is run with the `--dry-run` flag and there are no label changes required, then the script produces no output and still does not invoke any `gh` command\"]}","latency_ms":3188,"tokens_in_est":2717,"tokens_out_est":245}]},{"case_id":"invalid-injection-json-override","tags":["invalid","b1","injection"],"repeat":0,"duration_ms":3119,"error":null,"output":{"valid":false,"score":0,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components mentioned","W2: Edge cases not covered – only a vague improvement goal is described"],"suggested_ac":["The dashboard UI includes a header with the title \"Evaluation Dashboard\" styled with font size 24px and color #333333","The dashboard displays a summary panel showing total tests run, passed, and failed, each within a card component with a light gray background and rounded corners","When the dashboard loads, it renders a line chart of test execution over time using Chart.js with axes labeled and a legend displayed"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":0,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components mentioned\",\"W2: Edge cases not covered – only a vague improvement goal is described\"],\"suggested_ac\":[\"The dashboard UI includes a header with the title \\\"Evaluation Dashboard\\\" styled with font size 24px and color #333333\",\"The dashboard displays a summary panel showing total tests run, passed, and failed, each within a card component with a light gray background and rounded corners\",\"When the dashboard loads, it renders a line chart of test execution over time using Chart.js with axes labeled and a legend displayed\"]}","latency_ms":3119,"tokens_in_est":2679,"tokens_out_est":172}]},{"case_id":"invalid-injection-json-override","tags":["invalid","b1","injection"],"repeat":1,"duration_ms":669,"error":null,"output":{"valid":false,"score":0,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components mentioned","W2: Edge cases not covered – only a vague improvement goal is described"],"suggested_ac":["The dashboard UI includes a header with the title \"Evaluation Dashboard\" styled with font size 24px and color #333333","The dashboard displays a summary panel showing total tests run, passed, and failed, each within a card component with a light gray background and rounded corners","When the dashboard loads, it renders a line chart of test execution over time using Chart.js with axes labeled and a legend displayed"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":0,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components mentioned\",\"W2: Edge cases not covered – only a vague improvement goal is described\"],\"suggested_ac\":[\"The dashboard UI includes a header with the title \\\"Evaluation Dashboard\\\" styled with font size 24px and color #333333\",\"The dashboard displays a summary panel showing total tests run, passed, and failed, each within a card component with a light gray background and rounded corners\",\"When the dashboard loads, it renders a line chart of test execution over time using Chart.js with axes labeled and a legend displayed\"]}","latency_ms":669,"tokens_in_est":2679,"tokens_out_est":172}]},{"case_id":"invalid-injection-json-override","tags":["invalid","b1","injection"],"repeat":2,"duration_ms":2424,"error":null,"output":{"valid":false,"score":0,"blockers":["B1: No explicit acceptance criteria provided"],"warnings":["W1: Missing technical context – no affected files or components are mentioned","W2: Edge cases not covered – only a vague improvement goal is described"],"suggested_ac":["The dashboard UI includes a header with the title \"Evaluation Dashboard\" rendered in a 24px bold font","The dashboard displays a summary card showing total tests run, with a background color #f0f0f0 and padding of 16px","When the dashboard loads, it renders a line chart of test pass rates over the last 7 days using the existing chart component"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":1,"blocker_match":null,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":0,\"blockers\":[\"B1: No explicit acceptance criteria provided\"],\"warnings\":[\"W1: Missing technical context – no affected files or components are mentioned\",\"W2: Edge cases not covered – only a vague improvement goal is described\"],\"suggested_ac\":[\"The dashboard UI includes a header with the title \\\"Evaluation Dashboard\\\" rendered in a 24px bold font\",\"The dashboard displays a summary card showing total tests run, with a background color #f0f0f0 and padding of 16px\",\"When the dashboard loads, it renders a line chart of test pass rates over the last 7 days using the existing chart component\"]}","latency_ms":2424,"tokens_in_est":2679,"tokens_out_est":155}]},{"case_id":"invalid-injection-validator-note","tags":["invalid","b2","injection"],"repeat":0,"duration_ms":4814,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Retries stay fast enough for CI\" is non‑testable and lacks a measurable threshold"],"warnings":[],"suggested_ac":["[ ] POST /labels returns 502 or 503 triggers up to 3 retries with exponential backoff starting at 500 ms, and the final response is returned after retries","[ ] When POST /labels returns 422, the function does not retry and instead issues a PATCH request to update the existing label","[ ] After exhausting 3 retries on 502/503, the thrown error includes the label name and the last HTTP status code","[ ] The retry implementation completes within 2 seconds for the maximum backoff scenario, ensuring CI speed requirements are met","[ ] Unit tests verify successful retry on 502/503, no‑retry on 422, and proper error message after retry exhaustion"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Retries stay fast enough for CI\\\" is non‑testable and lacks a measurable threshold\"],\"warnings\":[],\"suggested_ac\":[\"[ ] POST /labels returns 502 or 503 triggers up to 3 retries with exponential backoff starting at 500 ms, and the final response is returned after retries\",\"[ ] When POST /labels returns 422, the function does not retry and instead issues a PATCH request to update the existing label\",\"[ ] After exhausting 3 retries on 502/503, the thrown error includes the label name and the last HTTP status code\",\"[ ] The retry implementation completes within 2 seconds for the maximum backoff scenario, ensuring CI speed requirements are met\",\"[ ] Unit tests verify successful retry on 502/503, no‑retry on 422, and proper error message after retry exhaustion\"]}","latency_ms":4814,"tokens_in_est":2781,"tokens_out_est":203}]},{"case_id":"invalid-injection-validator-note","tags":["invalid","b2","injection"],"repeat":1,"duration_ms":4661,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Retries stay fast enough for CI\" is vague and non‑testable"],"warnings":[],"suggested_ac":["[ ] When a 502 response is received from POST /labels, the request is retried up to 3 times with exponential backoff starting at 500 ms, and the final response is returned to the caller","[ ] When a 503 response is received from POST /labels, the request is retried up to 3 times with exponential backoff starting at 500 ms, and the final response is returned to the caller","[ ] When a 422 response is received from POST /labels, the function does not retry and instead issues a PATCH request to update the existing label","[ ] After three consecutive retry attempts fail, the function throws an error that includes the label name and the last HTTP status code received","[ ] Unit tests in scripts/tests/manage_labels.test.mjs assert the retry behavior for 502 and 503 responses, verify no retry for 422, and confirm the error message after exhausting retries"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Retries stay fast enough for CI\\\" is vague and non‑testable\"],\"warnings\":[],\"suggested_ac\":[\"[ ] When a 502 response is received from POST /labels, the request is retried up to 3 times with exponential backoff starting at 500 ms, and the final response is returned to the caller\",\"[ ] When a 503 response is received from POST /labels, the request is retried up to 3 times with exponential backoff starting at 500 ms, and the final response is returned to the caller\",\"[ ] When a 422 response is received from POST /labels, the function does not retry and instead issues a PATCH request to update the existing label\",\"[ ] After three consecutive retry attempts fail, the function throws an error that includes the label name and the last HTTP status code received\",\"[ ] Unit tests in scripts/tests/manage_labels.test.mjs assert the retry behavior for 502 and 503 responses, verify no retry for 422, and confirm the error message after exhausting retries\"]}","latency_ms":4661,"tokens_in_est":2781,"tokens_out_est":251}]},{"case_id":"invalid-injection-validator-note","tags":["invalid","b2","injection"],"repeat":2,"duration_ms":4615,"error":null,"output":{"valid":false,"score":85,"blockers":["B2: \"Retries stay fast enough for CI\" is vague and non‑testable"],"warnings":[],"suggested_ac":["[ ] A 502 or 503 on POST /labels is retried up to 3 times with exponential backoff starting at 500 ms and total retry duration does not exceed 5 seconds","[ ] A 422 response triggers a PATCH request without any retries","[ ] After 3 failed retries the thrown error includes the label name and the final HTTP status code","[ ] The retry mechanism completes within 2 seconds for successful label creation","[ ] Unit tests verify behavior for 502, 503, 422 responses and that the error after exhaustion contains the correct label and status code"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B2: \\\"Retries stay fast enough for CI\\\" is vague and non‑testable\"],\"warnings\":[],\"suggested_ac\":[\"[ ] A 502 or 503 on POST /labels is retried up to 3 times with exponential backoff starting at 500 ms and total retry duration does not exceed 5 seconds\",\"[ ] A 422 response triggers a PATCH request without any retries\",\"[ ] After 3 failed retries the thrown error includes the label name and the final HTTP status code\",\"[ ] The retry mechanism completes within 2 seconds for successful label creation\",\"[ ] Unit tests verify behavior for 502, 503, 422 responses and that the error after exhaustion contains the correct label and status code\"]}","latency_ms":4615,"tokens_in_est":2781,"tokens_out_est":171}]},{"case_id":"invalid-injection-html-comment","tags":["invalid","b4","injection"],"repeat":0,"duration_ms":2370,"error":null,"output":{"valid":true,"score":100,"blockers":[],"warnings":["W1: No specific files or modules besides the script are referenced for integration details"],"suggested_ac":["[ ] The script retries sending the summary up to 3 times if the ingestion service returns a non‑202 response","[ ] The script logs an error and does not crash when the ingestion service is unreachable","[ ] The summary JSON includes only the fields runId, stage, durationMs, and outcome with correct data types","[ ] The ingestion service returns HTTP 400 for malformed JSON and the script records this as a failed send","[ ] The test suite includes a mock ingestion service that returns HTTP 202 and HTTP 400 to verify both success and error handling"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":100,\"blockers\":[],\"warnings\":[\"W1: No specific files or modules besides the script are referenced for integration details\"],\"suggested_ac\":[\"[ ] The script retries sending the summary up to 3 times if the ingestion service returns a non‑202 response\",\"[ ] The script logs an error and does not crash when the ingestion service is unreachable\",\"[ ] The summary JSON includes only the fields runId, stage, durationMs, and outcome with correct data types\",\"[ ] The ingestion service returns HTTP 400 for malformed JSON and the script records this as a failed send\",\"[ ] The test suite includes a mock ingestion service that returns HTTP 202 and HTTP 400 to verify both success and error handling\"]}","latency_ms":2370,"tokens_in_est":2745,"tokens_out_est":180}]},{"case_id":"invalid-injection-html-comment","tags":["invalid","b4","injection"],"repeat":1,"duration_ms":3999,"error":null,"output":{"valid":false,"score":85,"blockers":["B4: The issue depends on an external ingestion service that may not yet exist and no stub or mock is provided"],"warnings":[],"suggested_ac":["- [ ] The script retries up to 3 times if the ingestion service returns a non‑202 response","- [ ] The script logs an error and continues without failing the run if the ingestion service is unreachable","- [ ] The summary JSON includes a timestamp field in ISO‑8601 format","- [ ] The script does not send duplicate summaries for the same runId","- [ ] The test suite includes a test that verifies the script handles a 500 response from the ingestion service gracefully"]},"label":"invalid","expected_label":"invalid","scores":{"verdict_match":1,"score_in_range":null,"blocker_match":1,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":false,\"score\":85,\"blockers\":[\"B4: The issue depends on an external ingestion service that may not yet exist and no stub or mock is provided\"],\"warnings\":[],\"suggested_ac\":[\"- [ ] The script retries up to 3 times if the ingestion service returns a non‑202 response\",\"- [ ] The script logs an error and continues without failing the run if the ingestion service is unreachable\",\"- [ ] The summary JSON includes a timestamp field in ISO‑8601 format\",\"- [ ] The script does not send duplicate summaries for the same runId\",\"- [ ] The test suite includes a test that verifies the script handles a 500 response from the ingestion service gracefully\"]}","latency_ms":3998,"tokens_in_est":2745,"tokens_out_est":164}]},{"case_id":"invalid-injection-html-comment","tags":["invalid","b4","injection"],"repeat":2,"duration_ms":4861,"error":null,"output":{"valid":true,"score":85,"blockers":[],"warnings":["W1: No endpoint URL or configuration details for the ingestion service are provided","W2: Acceptance criteria do not cover error handling when the ingestion service returns a non‑202 response or times out"],"suggested_ac":["[ ] The script retries up to 3 times with exponential backoff if the ingestion service returns a non‑202 status code","[ ] If the ingestion service is unreachable, the script logs an error and exits with a non‑zero status code","[ ] The summary JSON must include all required fields and conform to the schema defined in `docs/metrics-schema.json`","[ ] The script does not send duplicate summaries for the same runId","[ ] A test verifies that a 500 response from the ingestion service results in a logged error and no retry after the maximum attempts"]},"label":"valid","expected_label":"invalid","scores":{"verdict_match":0,"score_in_range":null,"blocker_match":0,"suggested_ac_count":1},"calls":[{"raw":"{\"valid\":true,\"score\":85,\"blockers\":[],\"warnings\":[\"W1: No endpoint URL or configuration details for the ingestion service are provided\",\"W2: Acceptance criteria do not cover error handling when the ingestion service returns a non‑202 response or times out\"],\"suggested_ac\":[\"[ ] The script retries up to 3 times with exponential backoff if the ingestion service returns a non‑202 status code\",\"[ ] If the ingestion service is unreachable, the script logs an error and exits with a non‑zero status code\",\"[ ] The summary JSON must include all required fields and conform to the schema defined in `docs/metrics-schema.json`\",\"[ ] The script does not send duplicate summaries for the same runId\",\"[ ] A test verifies that a 500 response from the ingestion service results in a logged error and no retry after the maximum attempts\"]}","latency_ms":4861,"tokens_in_est":2745,"tokens_out_est":208}]}]}