{
"generatedAt": "2026-09-25T04:31:53.564Z",
"mode": "live",
"model": "gpt-6-astra",
"tokenizer": "estimate: characters / 4",
"caveats": [
"Offline mode measures INFORMATION CONTENT, not model skill: the raw-source variant is answered by regex extractors, so its accuracy is an upper bound a real model would not reach. Use --live to score a real model.",
"Token counts are estimates (characters / 4) applied identically to every variant; only the ratio between variants is claimed.",
"The compressed IR has a near-constant size, so the savings percentage grows with file size — read it together with the per-file table."
],
"totalLlmCalls": 267,
"elapsedMs": 461572,
"variants": {
"raw": {
"inputTokens": 279476,
"llmCalls": 87,
"elapsedMs": 154882,
"correct": 84,
"answered": 87,
"unresolved": 0,
"questions": 87,
"accuracyPct": 96.6
},
"ir": {
"inputTokens": 58549,
"llmCalls": 87,
"elapsedMs": 147007,
"correct": 82,
"answered": 82,
"unresolved": 5,
"questions": 87,
"accuracyPct": 94.3
},
"ir+raw": {
"inputTokens": 80340,
"llmCalls": 93,
"elapsedMs": 159332,
"correct": 87,
"answered": 87,
"unresolved": 0,
"questions": 87,
"accuracyPct": 100
}
},
"cases": [
{
"id": "01-http-api-server",
"file": "samples/http-api-server.js",
"note": "Inbound HTTP surface plus a shared-secret check. Nothing the IR summary flags.",
"sourceChars": 652,
"sourceLines": 27,
"sourceTokens": 163,
"irTokens": 272,
"ir": {
"mission": {
"target": "samples/http-api-server.js",
"objective": "security_refactor"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "medium"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [],
"summary": {
"size": 652,
"lines": 27,
"hasCrypto": true,
"hasFilesystem": false,
"hasNetwork": true
},
"facts": {
"ast": true,
"routes": [
"GET /health",
"POST /webhook",
"GET /version"
],
"exports": [
"app",
"start"
],
"imports": [
"express"
],
"env": [
"PORT",
"WEBHOOK_SECRET"
],
"operations": [
"inboundHttp",
"secretComparison"
],
"calls": {
"network": [
"app.get",
"app.listen",
"app.post"
]
},
"riskSignals": [
"secret_comparison:req.headers[\"x-hook-token\"] !== process.env.WEBHOOK_SECRET@11",
"secret_access:process.env.WEBHOOK_SECRET@11",
"inbound_http:app.listen@24"
]
}
},
"questions": [
{
"id": "endpoints",
"kind": "endpoints",
"prompt": "Which HTTP endpoints does this file expose? Answer as a list of \"METHOD /path\".",
"expected": [
"GET /health",
"POST /webhook",
"GET /version"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 265,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"POST /webhook",
"GET /version"
],
"correct": true,
"elapsedMs": 2760.271
},
"ir": {
"variant": "ir",
"inputTokens": 375,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"POST /webhook",
"GET /version"
],
"correct": true,
"elapsedMs": 2014.16
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 375,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"POST /webhook",
"GET /version"
],
"correct": true,
"elapsedMs": 1967.734
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 265,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1190.167
},
"ir": {
"variant": "ir",
"inputTokens": 375,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1457.573
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 375,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1288.685
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 258,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1583.692
},
"ir": {
"variant": "ir",
"inputTokens": 368,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1655.116
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 368,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1547.291
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 284,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 2279.143
},
"ir": {
"variant": "ir",
"inputTokens": 394,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1507.826
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 394,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 2459.367
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "medium",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 263,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 2014.307
},
"ir": {
"variant": "ir",
"inputTokens": 373,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 1026.691
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 373,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 2289.583
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"app",
"start"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 263,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"start"
],
"correct": true,
"elapsedMs": 2553.508
},
"ir": {
"variant": "ir",
"inputTokens": 373,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"start"
],
"correct": true,
"elapsedMs": 1490.612
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 373,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"start"
],
"correct": true,
"elapsedMs": 1275.873
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"PORT",
"WEBHOOK_SECRET"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 264,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"WEBHOOK_SECRET",
"PORT"
],
"correct": true,
"elapsedMs": 1543.461
},
"ir": {
"variant": "ir",
"inputTokens": 374,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"PORT",
"WEBHOOK_SECRET"
],
"correct": true,
"elapsedMs": 1643.554
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 374,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"PORT",
"WEBHOOK_SECRET"
],
"correct": true,
"elapsedMs": 1334.441
}
}
},
{
"id": "auth",
"kind": "auth_check",
"prompt": "Does this file verify a secret or token before acting? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 264,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 2343.35
},
"ir": {
"variant": "ir",
"inputTokens": 374,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1233.224
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 374,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1380.124
}
}
}
]
},
{
"id": "02-cache-writer",
"file": "samples/cache-writer.js",
"note": "Destructive filesystem work done through unlinkSync, which the IR forbidden list does not cover (it only looks for fs.rm).",
"sourceChars": 835,
"sourceLines": 33,
"sourceTokens": 209,
"irTokens": 287,
"ir": {
"mission": {
"target": "samples/cache-writer.js",
"objective": "filesystem_simplification"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "high"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [
"destructive_fs"
],
"summary": {
"size": 835,
"lines": 33,
"hasCrypto": false,
"hasFilesystem": true,
"hasNetwork": false
},
"facts": {
"ast": true,
"exports": [
"purge",
"purgeAll",
"saveEntry"
],
"imports": [
"fs",
"path"
],
"env": [
"CACHE_DIR"
],
"operations": [
"diskRead",
"diskWrite",
"diskDelete"
],
"calls": {
"filesystem": [
"fs.existsSync",
"fs.mkdirSync",
"fs.readdirSync",
"fs.unlinkSync",
"fs.writeFileSync"
]
},
"dangerous": [
"destructive_fs"
],
"riskSignals": [
"disk_write:fs.mkdirSync@10",
"disk_write:fs.writeFileSync@12",
"destructive_fs:fs.unlinkSync@19",
"destructive_fs:fs.unlinkSync@28"
]
}
},
"questions": [
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 303,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1961.233
},
"ir": {
"variant": "ir",
"inputTokens": 383,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1187.733
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 383,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1350.482
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 310,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1307.995
},
"ir": {
"variant": "ir",
"inputTokens": 390,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 2632.593
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 390,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1852.449
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [
"destructive_fs"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 329,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1580.193
},
"ir": {
"variant": "ir",
"inputTokens": 409,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 2215.084
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 409,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1500.606
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "high",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 308,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1974.294
},
"ir": {
"variant": "ir",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1516.858
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1996.329
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"purge",
"purgeAll",
"saveEntry"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 308,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"saveEntry",
"purge",
"purgeAll"
],
"correct": true,
"elapsedMs": 1845.424
},
"ir": {
"variant": "ir",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"purge",
"purgeAll",
"saveEntry"
],
"correct": true,
"elapsedMs": 1514.056
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"purge",
"purgeAll",
"saveEntry"
],
"correct": true,
"elapsedMs": 1957.791
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"CACHE_DIR"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 309,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"CACHE_DIR"
],
"correct": true,
"elapsedMs": 2104.331
},
"ir": {
"variant": "ir",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"CACHE_DIR"
],
"correct": true,
"elapsedMs": 2451.35
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 388,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"CACHE_DIR"
],
"correct": true,
"elapsedMs": 1335.638
}
}
},
{
"id": "endpoints",
"kind": "endpoints",
"prompt": "Which HTTP endpoints does this file expose? Answer as a list of \"METHOD /path\".",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 310,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1489.557
},
"ir": {
"variant": "ir",
"inputTokens": 389,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1546.333
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 389,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1203.923
}
}
}
]
},
{
"id": "03-status-client",
"file": "samples/status-client.js",
"note": "Plain outbound HTTP client: the case the IR summary flags correctly.",
"sourceChars": 688,
"sourceLines": 26,
"sourceTokens": 172,
"irTokens": 199,
"ir": {
"mission": {
"target": "samples/status-client.js",
"objective": "network_stability"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "medium"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [],
"summary": {
"size": 688,
"lines": 26,
"hasCrypto": false,
"hasFilesystem": false,
"hasNetwork": true
},
"facts": {
"ast": true,
"exports": [
"fetchStatus"
],
"imports": [
"axios"
],
"env": [
"STATUS_API_BASE",
"STATUS_TIMEOUT_MS"
],
"operations": [
"outboundHttp"
],
"calls": {
"network": [
"axios.get"
]
},
"riskSignals": [
"outbound_http:axios.get@14"
]
}
},
"questions": [
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 274,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1001.52
},
"ir": {
"variant": "ir",
"inputTokens": 301,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1571.498
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 301,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1166.097
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 267,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1168.931
},
"ir": {
"variant": "ir",
"inputTokens": 295,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1059.402
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 295,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1041.55
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 293,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1318.394
},
"ir": {
"variant": "ir",
"inputTokens": 320,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1824.88
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 320,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1323.53
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "medium",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 272,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 2587.052
},
"ir": {
"variant": "ir",
"inputTokens": 300,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 1170.551
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 300,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 974.266
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"fetchStatus"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 272,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"fetchStatus"
],
"correct": true,
"elapsedMs": 1424.293
},
"ir": {
"variant": "ir",
"inputTokens": 299,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"fetchStatus"
],
"correct": true,
"elapsedMs": 1533.272
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 299,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"fetchStatus"
],
"correct": true,
"elapsedMs": 1566.679
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"STATUS_API_BASE",
"STATUS_TIMEOUT_MS"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 272,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"STATUS_API_BASE",
"STATUS_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1509.3
},
"ir": {
"variant": "ir",
"inputTokens": 300,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"STATUS_API_BASE",
"STATUS_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1246.242
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 300,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"STATUS_API_BASE",
"STATUS_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1506.339
}
}
}
]
},
{
"id": "04-polynomial",
"file": "samples/polynomial.js",
"note": "Pure computation whose vocabulary (evaluate / evaluation) collides with the IR substring heuristics.",
"sourceChars": 890,
"sourceLines": 32,
"sourceTokens": 223,
"irTokens": 134,
"ir": {
"mission": {
"target": "samples/polynomial.js",
"objective": "general_optimization"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "low"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [],
"summary": {
"size": 890,
"lines": 32,
"hasCrypto": false,
"hasFilesystem": false,
"hasNetwork": false
},
"facts": {
"ast": true,
"exports": [
"derivative",
"evaluate",
"roots"
]
}
},
"questions": [
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "low",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 322,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "low",
"correct": true,
"elapsedMs": 1248.389
},
"ir": {
"variant": "ir",
"inputTokens": 234,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "low",
"correct": true,
"elapsedMs": 1490.897
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 234,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "low",
"correct": true,
"elapsedMs": 1103.85
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 342,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 967.834
},
"ir": {
"variant": "ir",
"inputTokens": 255,
"llmCalls": 1,
"escalated": false,
"resolved": false,
"basis": "live",
"missing": null,
"answer": null,
"correct": false,
"elapsedMs": 1600.974
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 736,
"llmCalls": 2,
"escalated": true,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 3800.36
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 317,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1101.627
},
"ir": {
"variant": "ir",
"inputTokens": 229,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1067.525
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 229,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1112.461
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 323,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1327.589
},
"ir": {
"variant": "ir",
"inputTokens": 236,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1029.779
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 236,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1463.593
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"derivative",
"evaluate",
"roots"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 321,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"evaluate",
"derivative",
"roots"
],
"correct": true,
"elapsedMs": 1187.945
},
"ir": {
"variant": "ir",
"inputTokens": 234,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"derivative",
"evaluate",
"roots"
],
"correct": true,
"elapsedMs": 1165.192
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 234,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"derivative",
"evaluate",
"roots"
],
"correct": true,
"elapsedMs": 1400.834
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 322,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1119
},
"ir": {
"variant": "ir",
"inputTokens": 234,
"llmCalls": 1,
"escalated": false,
"resolved": false,
"basis": "live",
"missing": null,
"answer": null,
"correct": false,
"elapsedMs": 1668.06
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 695,
"llmCalls": 2,
"escalated": true,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 2547.146
}
}
}
]
},
{
"id": "05-git-probe",
"file": "samples/git-probe.js",
"note": "Process spawning: the IR names child_process and process.exit but not the spawning call itself.",
"sourceChars": 594,
"sourceLines": 23,
"sourceTokens": 149,
"irTokens": 300,
"ir": {
"mission": {
"target": "samples/git-probe.js",
"objective": "general_optimization"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "critical"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [
"child_process",
"process_spawn",
"process.exit"
],
"summary": {
"size": 594,
"lines": 23,
"hasCrypto": false,
"hasFilesystem": false,
"hasNetwork": false
},
"facts": {
"ast": true,
"exports": [
"changedFiles",
"currentCommit"
],
"imports": [
"child_process"
],
"env": [
"REPO_DIR"
],
"operations": [
"processSpawn",
"childProcess",
"processExit"
],
"calls": {
"process": [
"execSync",
"process.exit",
"require(child_process)",
"spawnSync"
]
},
"dangerous": [
"child_process",
"process_spawn",
"process.exit"
],
"riskSignals": [
"child_process_import:require(child_process)@1",
"process_spawn:execSync@9",
"process_spawn:spawnSync@16",
"process_exit:process.exit@18"
]
}
},
"questions": [
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [
"child_process",
"process_spawn",
"process.exit"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 268,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit"
],
"correct": true,
"elapsedMs": 2370.037
},
"ir": {
"variant": "ir",
"inputTokens": 421,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit"
],
"correct": true,
"elapsedMs": 1109.286
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 421,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit"
],
"correct": true,
"elapsedMs": 3784.501
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "critical",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 247,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "low",
"correct": false,
"elapsedMs": 1851.491
},
"ir": {
"variant": "ir",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "critical",
"correct": true,
"elapsedMs": 1506.86
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "critical",
"correct": true,
"elapsedMs": 1070.629
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 242,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1510.201
},
"ir": {
"variant": "ir",
"inputTokens": 395,
"llmCalls": 1,
"escalated": false,
"resolved": false,
"basis": "live",
"missing": null,
"answer": null,
"correct": false,
"elapsedMs": 2474.628
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 942,
"llmCalls": 2,
"escalated": true,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 4359.301
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 249,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1140.668
},
"ir": {
"variant": "ir",
"inputTokens": 402,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1137.621
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 402,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1283.657
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"changedFiles",
"currentCommit"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 247,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"currentCommit",
"changedFiles"
],
"correct": true,
"elapsedMs": 1311.127
},
"ir": {
"variant": "ir",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"changedFiles",
"currentCommit"
],
"correct": true,
"elapsedMs": 2700.846
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"changedFiles",
"currentCommit"
],
"correct": true,
"elapsedMs": 1452.819
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"REPO_DIR"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 248,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPO_DIR"
],
"correct": true,
"elapsedMs": 1328.432
},
"ir": {
"variant": "ir",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPO_DIR"
],
"correct": true,
"elapsedMs": 1150.799
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 400,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPO_DIR"
],
"correct": true,
"elapsedMs": 849.311
}
}
}
]
},
{
"id": "06-token-signer",
"file": "samples/token-signer.js",
"note": "Secret handling with a constant-time comparison; the IR summary sees crypto but the risk label does not.",
"sourceChars": 649,
"sourceLines": 23,
"sourceTokens": 163,
"irTokens": 224,
"ir": {
"mission": {
"target": "samples/token-signer.js",
"objective": "security_refactor"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "medium"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [],
"summary": {
"size": 649,
"lines": 23,
"hasCrypto": true,
"hasFilesystem": false,
"hasNetwork": false
},
"facts": {
"ast": true,
"exports": [
"sign",
"verify"
],
"imports": [
"crypto"
],
"env": [
"SIGNING_KEY"
],
"operations": [
"cryptoUse",
"secretComparison"
],
"calls": {
"crypto": [
"crypto.createHmac",
"crypto.timingSafeEqual"
]
},
"riskSignals": [
"crypto_use:crypto.createHmac@10",
"secret_comparison:crypto.timingSafeEqual@20"
]
}
},
"questions": [
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "medium",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 262,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "low",
"correct": false,
"elapsedMs": 930.511
},
"ir": {
"variant": "ir",
"inputTokens": 324,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 963.32
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 324,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "medium",
"correct": true,
"elapsedMs": 822.701
}
}
},
{
"id": "auth",
"kind": "auth_check",
"prompt": "Does this file verify a secret or token before acting? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 263,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1210.481
},
"ir": {
"variant": "ir",
"inputTokens": 325,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1247.142
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 325,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 909.349
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 283,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1170.416
},
"ir": {
"variant": "ir",
"inputTokens": 345,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 2036.791
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 345,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1966.689
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 257,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1041.907
},
"ir": {
"variant": "ir",
"inputTokens": 319,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1124.984
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 319,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1351.67
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 264,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1131.414
},
"ir": {
"variant": "ir",
"inputTokens": 326,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 2521.024
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 326,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1552.822
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"SIGNING_KEY"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 262,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"SIGNING_KEY"
],
"correct": true,
"elapsedMs": 1944.178
},
"ir": {
"variant": "ir",
"inputTokens": 325,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"SIGNING_KEY"
],
"correct": true,
"elapsedMs": 1474.664
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 325,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"SIGNING_KEY"
],
"correct": true,
"elapsedMs": 1508.089
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"sign",
"verify"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 262,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"sign",
"verify"
],
"correct": true,
"elapsedMs": 2905.399
},
"ir": {
"variant": "ir",
"inputTokens": 324,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"sign",
"verify"
],
"correct": true,
"elapsedMs": 1732.949
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 324,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"sign",
"verify"
],
"correct": true,
"elapsedMs": 1552
}
}
}
]
},
{
"id": "07-order-service",
"file": "samples/order-service.js",
"note": "Larger, mixed-concern file (~120 lines). Sentinel-IR has a near-constant size, so this is where compression looks best and the loss is unchanged.",
"sourceChars": 3835,
"sourceLines": 138,
"sourceTokens": 959,
"irTokens": 527,
"ir": {
"mission": {
"target": "samples/order-service.js",
"objective": "security_refactor"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "high"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [
"destructive_fs"
],
"summary": {
"size": 3835,
"lines": 138,
"hasCrypto": true,
"hasFilesystem": true,
"hasNetwork": true
},
"facts": {
"ast": true,
"routes": [
"POST /orders",
"GET /orders/:id",
"POST /orders/:id/refund",
"DELETE /orders/:id"
],
"exports": [
"app",
"dropOrder",
"loadOrder",
"persistOrder",
"start",
"totalOf"
],
"imports": [
"axios",
"crypto",
"express",
"fs",
"path"
],
"env": [
"OPERATOR_TOKEN",
"ORDER_STATE_DIR",
"PAYMENT_API_BASE",
"PAYMENT_SIGNING_KEY",
"PAYMENT_TIMEOUT_MS",
"PORT"
],
"operations": [
"inboundHttp",
"outboundHttp",
"diskRead",
"diskWrite",
"diskDelete",
"cryptoUse",
"secretComparison"
],
"calls": {
"network": [
"app.delete",
"app.get",
"app.listen",
"app.post",
"axios.post"
],
"filesystem": [
"fs.existsSync",
"fs.mkdirSync",
"fs.readFileSync",
"fs.unlinkSync",
"fs.writeFileSync"
],
"crypto": [
"crypto.createHmac",
"crypto.randomBytes"
]
},
"dangerous": [
"destructive_fs"
],
"riskSignals": [
"disk_write:fs.mkdirSync@28",
"disk_write:fs.writeFileSync@29",
"destructive_fs:fs.unlinkSync@36",
"crypto_use:crypto.randomBytes@41",
"crypto_use:crypto.createHmac@45",
"outbound_http:axios.post@58",
"outbound_http:axios.post@67",
"secret_comparison:req.headers[\"x-operator-token\"] !== process.env.OPERATOR_TOKEN@72",
"secret_access:process.env.OPERATOR_TOKEN@72",
"inbound_http:app.listen@135"
]
}
},
"questions": [
{
"id": "endpoints",
"kind": "endpoints",
"prompt": "Which HTTP endpoints does this file expose? Answer as a list of \"METHOD /path\".",
"expected": [
"POST /orders",
"GET /orders/:id",
"POST /orders/:id/refund",
"DELETE /orders/:id"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1060,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"POST /orders",
"GET /orders/:id",
"POST /orders/:id/refund",
"DELETE /orders/:id"
],
"correct": true,
"elapsedMs": 1564.271
},
"ir": {
"variant": "ir",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"POST /orders",
"GET /orders/:id",
"POST /orders/:id/refund",
"DELETE /orders/:id"
],
"correct": true,
"elapsedMs": 1453.672
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"POST /orders",
"GET /orders/:id",
"POST /orders/:id/refund",
"DELETE /orders/:id"
],
"correct": true,
"elapsedMs": 1262.791
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1054,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1022.909
},
"ir": {
"variant": "ir",
"inputTokens": 623,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1100.193
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 623,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1496.367
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1060,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 826.301
},
"ir": {
"variant": "ir",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1214.955
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1379.939
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [
"destructive_fs"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1079,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1156.6
},
"ir": {
"variant": "ir",
"inputTokens": 648,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1410.751
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 648,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1233.1
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "high",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1059,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 2102.911
},
"ir": {
"variant": "ir",
"inputTokens": 628,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 994.424
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 628,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1661.882
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"app",
"dropOrder",
"loadOrder",
"persistOrder",
"start",
"totalOf"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1058,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"start",
"totalOf",
"loadOrder",
"persistOrder",
"dropOrder"
],
"correct": true,
"elapsedMs": 1464.576
},
"ir": {
"variant": "ir",
"inputTokens": 627,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"dropOrder",
"loadOrder",
"persistOrder",
"start",
"totalOf"
],
"correct": true,
"elapsedMs": 1518.18
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 627,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"dropOrder",
"loadOrder",
"persistOrder",
"start",
"totalOf"
],
"correct": true,
"elapsedMs": 1235.896
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"OPERATOR_TOKEN",
"ORDER_STATE_DIR",
"PAYMENT_API_BASE",
"PAYMENT_SIGNING_KEY",
"PAYMENT_TIMEOUT_MS",
"PORT"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1059,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"ORDER_STATE_DIR",
"PAYMENT_API_BASE",
"PAYMENT_SIGNING_KEY",
"PAYMENT_TIMEOUT_MS",
"OPERATOR_TOKEN",
"PORT"
],
"correct": true,
"elapsedMs": 1279.234
},
"ir": {
"variant": "ir",
"inputTokens": 628,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"OPERATOR_TOKEN",
"ORDER_STATE_DIR",
"PAYMENT_API_BASE",
"PAYMENT_SIGNING_KEY",
"PAYMENT_TIMEOUT_MS",
"PORT"
],
"correct": true,
"elapsedMs": 1586.727
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 628,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"OPERATOR_TOKEN",
"ORDER_STATE_DIR",
"PAYMENT_API_BASE",
"PAYMENT_SIGNING_KEY",
"PAYMENT_TIMEOUT_MS",
"PORT"
],
"correct": true,
"elapsedMs": 1509.556
}
}
},
{
"id": "auth",
"kind": "auth_check",
"prompt": "Does this file verify a secret or token before acting? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 1060,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1041.715
},
"ir": {
"variant": "ir",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1216.713
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 629,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1393.929
}
}
}
]
},
{
"id": "08-inventory-api",
"file": "samples/inventory-api.js",
"note": "Realistic ~390-line inventory service. First size class where the near-constant IR clearly beats raw source.",
"sourceChars": 11443,
"sourceLines": 393,
"sourceTokens": 2861,
"irTokens": 734,
"ir": {
"mission": {
"target": "samples/inventory-api.js",
"objective": "security_refactor"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "high"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [
"destructive_fs"
],
"summary": {
"size": 11443,
"lines": 393,
"hasCrypto": true,
"hasFilesystem": true,
"hasNetwork": true
},
"facts": {
"ast": true,
"routes": [
"GET /health",
"GET /items",
"GET /items/:sku",
"POST /items",
"PUT /items/:sku",
"PATCH /items/:sku/stock",
"DELETE /items/:sku",
"POST /pricing/refresh",
"GET /pricing/health",
"GET /reports/low-stock",
"GET /reports/movements/:sku"
],
"exports": [
"app",
"appendMovement",
"applyMovement",
"buildItem",
"dropItem",
"groupByCategory",
"isLowStock",
"listSkus",
"normalizeSku",
"pricingHealth",
"readItem",
"readMovements",
"refreshPrices",
"start",
"stockValueCents",
"validateItem",
"writeItem"
],
"imports": [
"axios",
"crypto",
"express",
"fs",
"path"
],
"env": [
"INVENTORY_DATA_DIR",
"INVENTORY_OPERATOR_TOKEN",
"INVENTORY_PORT",
"INVENTORY_SIGNING_KEY",
"LOW_STOCK_THRESHOLD",
"PRICING_API_BASE",
"PRICING_TIMEOUT_MS"
],
"operations": [
"inboundHttp",
"outboundHttp",
"diskRead",
"diskWrite",
"diskDelete",
"cryptoUse",
"secretComparison"
],
"calls": {
"network": [
"app.delete",
"app.get",
"app.listen",
"app.patch",
"app.post",
"app.put",
"axios.get",
"axios.post"
],
"filesystem": [
"fs.appendFileSync",
"fs.existsSync",
"fs.filter",
"fs.map",
"fs.mkdirSync",
"fs.readFileSync",
"fs.readdirSync",
"fs.slice",
"fs.sort",
"fs.split",
"fs.unlinkSync",
"fs.writeFileSync"
],
"crypto": [
"crypto.createHmac",
"crypto.randomBytes",
"crypto.timingSafeEqual"
]
},
"dangerous": [
"destructive_fs"
],
"riskSignals": [
"secret_access:process.env.INVENTORY_OPERATOR_TOKEN@21",
"disk_write:fs.mkdirSync@32",
"disk_write:fs.writeFileSync@58",
"destructive_fs:fs.unlinkSync@65",
"disk_write:fs.appendFileSync@81",
"crypto_use:crypto.createHmac@171",
"crypto_use:crypto.randomBytes@175",
"outbound_http:axios.post@181",
"outbound_http:axios.get@208",
"secret_comparison:presented.length !== OPERATOR_TOKEN.length@221",
"secret_comparison:crypto.timingSafeEqual@222",
"inbound_http:app.listen@371"
]
}
},
"questions": [
{
"id": "endpoints",
"kind": "endpoints",
"prompt": "Which HTTP endpoints does this file expose? Answer as a list of \"METHOD /path\".",
"expected": [
"GET /health",
"GET /items",
"GET /items/:sku",
"POST /items",
"PUT /items/:sku",
"PATCH /items/:sku/stock",
"DELETE /items/:sku",
"POST /pricing/refresh",
"GET /pricing/health",
"GET /reports/low-stock",
"GET /reports/movements/:sku"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2962,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"GET /items",
"GET /items/:sku",
"POST /items",
"PUT /items/:sku",
"PATCH /items/:sku/stock",
"DELETE /items/:sku",
"POST /pricing/refresh",
"GET /pricing/health",
"GET /reports/low-stock",
"GET /reports/movements/:sku"
],
"correct": true,
"elapsedMs": 2111.032
},
"ir": {
"variant": "ir",
"inputTokens": 836,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"GET /items",
"GET /items/:sku",
"POST /items",
"PUT /items/:sku",
"PATCH /items/:sku/stock",
"DELETE /items/:sku",
"POST /pricing/refresh",
"GET /pricing/health",
"GET /reports/low-stock",
"GET /reports/movements/:sku"
],
"correct": true,
"elapsedMs": 1835.92
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 836,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"GET /health",
"GET /items",
"GET /items/:sku",
"POST /items",
"PUT /items/:sku",
"PATCH /items/:sku/stock",
"DELETE /items/:sku",
"POST /pricing/refresh",
"GET /pricing/health",
"GET /reports/low-stock",
"GET /reports/movements/:sku"
],
"correct": true,
"elapsedMs": 2965.069
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2956,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 826.102
},
"ir": {
"variant": "ir",
"inputTokens": 830,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1250.391
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 830,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1494.55
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2962,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1347.117
},
"ir": {
"variant": "ir",
"inputTokens": 837,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1103.11
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 837,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1067.485
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [
"destructive_fs"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2981,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1619.59
},
"ir": {
"variant": "ir",
"inputTokens": 856,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 1065.727
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 856,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"destructive_fs"
],
"correct": true,
"elapsedMs": 963.999
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "high",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2961,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1701.355
},
"ir": {
"variant": "ir",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 907.385
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": true,
"elapsedMs": 1011.707
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"app",
"appendMovement",
"applyMovement",
"buildItem",
"dropItem",
"groupByCategory",
"isLowStock",
"listSkus",
"normalizeSku",
"pricingHealth",
"readItem",
"readMovements",
"refreshPrices",
"start",
"stockValueCents",
"validateItem",
"writeItem"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2960,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"start",
"readItem",
"writeItem",
"dropItem",
"listSkus",
"appendMovement",
"readMovements",
"validateItem",
"buildItem",
"applyMovement",
"isLowStock",
"stockValueCents",
"groupByCategory",
"refreshPrices",
"pricingHealth",
"normalizeSku"
],
"correct": true,
"elapsedMs": 2309.701
},
"ir": {
"variant": "ir",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"appendMovement",
"applyMovement",
"buildItem",
"dropItem",
"groupByCategory",
"isLowStock",
"listSkus",
"normalizeSku",
"pricingHealth",
"readItem",
"readMovements",
"refreshPrices",
"start",
"stockValueCents",
"validateItem",
"writeItem"
],
"correct": true,
"elapsedMs": 1796.424
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"app",
"appendMovement",
"applyMovement",
"buildItem",
"dropItem",
"groupByCategory",
"isLowStock",
"listSkus",
"normalizeSku",
"pricingHealth",
"readItem",
"readMovements",
"refreshPrices",
"start",
"stockValueCents",
"validateItem",
"writeItem"
],
"correct": true,
"elapsedMs": 1943.225
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"INVENTORY_DATA_DIR",
"INVENTORY_OPERATOR_TOKEN",
"INVENTORY_PORT",
"INVENTORY_SIGNING_KEY",
"LOW_STOCK_THRESHOLD",
"PRICING_API_BASE",
"PRICING_TIMEOUT_MS"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2961,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"INVENTORY_DATA_DIR",
"PRICING_API_BASE",
"PRICING_TIMEOUT_MS",
"INVENTORY_SIGNING_KEY",
"INVENTORY_OPERATOR_TOKEN",
"LOW_STOCK_THRESHOLD",
"INVENTORY_PORT"
],
"correct": true,
"elapsedMs": 1754.667
},
"ir": {
"variant": "ir",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"INVENTORY_DATA_DIR",
"INVENTORY_OPERATOR_TOKEN",
"INVENTORY_PORT",
"INVENTORY_SIGNING_KEY",
"LOW_STOCK_THRESHOLD",
"PRICING_API_BASE",
"PRICING_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1658.043
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 835,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"INVENTORY_DATA_DIR",
"INVENTORY_OPERATOR_TOKEN",
"INVENTORY_PORT",
"INVENTORY_SIGNING_KEY",
"LOW_STOCK_THRESHOLD",
"PRICING_API_BASE",
"PRICING_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1520.693
}
}
},
{
"id": "auth",
"kind": "auth_check",
"prompt": "Does this file verify a secret or token before acting? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 2962,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1439.513
},
"ir": {
"variant": "ir",
"inputTokens": 836,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1156.958
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 836,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 2278.904
}
}
}
]
},
{
"id": "09-report-worker",
"file": "samples/report-worker.js",
"note": "Realistic ~510-line batch worker with no HTTP surface: git shell-outs, gzip archiving and process.exit.",
"sourceChars": 15276,
"sourceLines": 511,
"sourceTokens": 3819,
"irTokens": 778,
"ir": {
"mission": {
"target": "samples/report-worker.js",
"objective": "security_refactor"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "critical"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"summary": {
"size": 15276,
"lines": 511,
"hasCrypto": true,
"hasFilesystem": true,
"hasNetwork": true
},
"facts": {
"ast": true,
"exports": [
"archive",
"averageBy",
"billingTotals",
"breachedAccounts",
"capacityHotspots",
"checksum",
"countDistinct",
"errorRate",
"gitBranch",
"gitDirty",
"gitRevision",
"groupBy",
"jobFiles",
"maxBy",
"normalizeJob",
"notify",
"percentile",
"processBatch",
"processJob",
"provenance",
"pruneArchive",
"readJob",
"render",
"renderCsv",
"renderJson",
"sumBy",
"summarize",
"upload",
"validateJob"
],
"imports": [
"axios",
"child_process",
"crypto",
"fs",
"os",
"path",
"zlib"
],
"env": [
"REPORT_ARCHIVE_DIR",
"REPORT_BATCH_SIZE",
"REPORT_OUTPUT_DIR",
"REPORT_QUEUE_DIR",
"REPORT_REPO_DIR",
"REPORT_RETENTION_DAYS",
"REPORT_UPLOAD_BASE",
"REPORT_UPLOAD_TIMEOUT_MS"
],
"operations": [
"outboundHttp",
"diskRead",
"diskWrite",
"diskDelete",
"processSpawn",
"childProcess",
"processExit",
"cryptoUse"
],
"calls": {
"network": [
"axios.post",
"axios.put"
],
"filesystem": [
"fs.appendFileSync",
"fs.existsSync",
"fs.filter",
"fs.map",
"fs.mkdirSync",
"fs.readFileSync",
"fs.readdirSync",
"fs.slice",
"fs.sort",
"fs.statSync",
"fs.unlinkSync",
"fs.writeFileSync"
],
"process": [
"execSync",
"process.exit",
"require(child_process)",
"spawnSync"
],
"crypto": [
"crypto.createHash"
]
},
"dangerous": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"riskSignals": [
"child_process_import:require(child_process)@15",
"disk_write:fs.mkdirSync@41",
"disk_write:fs.writeFileSync@67",
"disk_write:fs.writeFileSync@73",
"disk_write:fs.appendFileSync@79",
"destructive_fs:fs.unlinkSync@84",
"disk_write:fs.writeFileSync@92",
"destructive_fs:fs.unlinkSync@105",
"process_spawn:execSync@288",
"process_spawn:spawnSync@295",
"process_spawn:spawnSync@307",
"crypto_use:crypto.createHash@323",
"outbound_http:axios.put@335",
"outbound_http:axios.post@345",
"process_exit:process.exit@467",
"process_exit:process.exit@470",
"process_exit:process.exit@476"
]
}
},
"questions": [
{
"id": "endpoints",
"kind": "endpoints",
"prompt": "Which HTTP endpoints does this file expose? Answer as a list of \"METHOD /path\".",
"expected": [],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3920,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 1038.565
},
"ir": {
"variant": "ir",
"inputTokens": 880,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 2588.359
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 5583,
"llmCalls": 2,
"escalated": true,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [],
"correct": true,
"elapsedMs": 5116.52
}
}
},
{
"id": "disk",
"kind": "writes_disk",
"prompt": "Does this file write to disk? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3914,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1828.998
},
"ir": {
"variant": "ir",
"inputTokens": 874,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1573.9
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 874,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1062.352
}
}
},
{
"id": "network",
"kind": "network_io",
"prompt": "Does this file perform network I/O (inbound or outbound)? Answer true or false.",
"expected": true,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3921,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1431.792
},
"ir": {
"variant": "ir",
"inputTokens": 881,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 1185.265
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 881,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": true,
"correct": true,
"elapsedMs": 965.297
}
}
},
{
"id": "dangerous",
"kind": "dangerous_constructs",
"prompt": "Which dangerous constructs does this file use (eval, new Function, child_process, process_spawn, process.exit, destructive_fs)? Answer as a list.",
"expected": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3940,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"correct": true,
"elapsedMs": 1429.164
},
"ir": {
"variant": "ir",
"inputTokens": 900,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"correct": true,
"elapsedMs": 1658.4
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 900,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"child_process",
"process_spawn",
"process.exit",
"destructive_fs"
],
"correct": true,
"elapsedMs": 1430.297
}
}
},
{
"id": "risk",
"kind": "security_risk",
"prompt": "What is the security risk of this file (low, medium, high, critical)?",
"expected": "critical",
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3919,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "high",
"correct": false,
"elapsedMs": 1990.07
},
"ir": {
"variant": "ir",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "critical",
"correct": true,
"elapsedMs": 1254.722
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": "critical",
"correct": true,
"elapsedMs": 1136.52
}
}
},
{
"id": "exports",
"kind": "exported_functions",
"prompt": "Which symbols does this file export? Answer as a list of names.",
"expected": [
"archive",
"averageBy",
"billingTotals",
"breachedAccounts",
"capacityHotspots",
"checksum",
"countDistinct",
"errorRate",
"gitBranch",
"gitDirty",
"gitRevision",
"groupBy",
"jobFiles",
"maxBy",
"normalizeJob",
"notify",
"percentile",
"processBatch",
"processJob",
"provenance",
"pruneArchive",
"readJob",
"render",
"renderCsv",
"renderJson",
"sumBy",
"summarize",
"upload",
"validateJob"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3919,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"processBatch",
"processJob",
"validateJob",
"normalizeJob",
"render",
"renderCsv",
"renderJson",
"summarize",
"provenance",
"gitRevision",
"gitBranch",
"gitDirty",
"checksum",
"upload",
"notify",
"archive",
"pruneArchive",
"jobFiles",
"readJob",
"sumBy",
"averageBy",
"maxBy",
"percentile",
"groupBy",
"countDistinct",
"errorRate",
"breachedAccounts",
"billingTotals",
"capacityHotspots"
],
"correct": true,
"elapsedMs": 3050.818
},
"ir": {
"variant": "ir",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"archive",
"averageBy",
"billingTotals",
"breachedAccounts",
"capacityHotspots",
"checksum",
"countDistinct",
"errorRate",
"gitBranch",
"gitDirty",
"gitRevision",
"groupBy",
"jobFiles",
"maxBy",
"normalizeJob",
"notify",
"percentile",
"processBatch",
"processJob",
"provenance",
"pruneArchive",
"readJob",
"render",
"renderCsv",
"renderJson",
"sumBy",
"summarize",
"upload",
"validateJob"
],
"correct": true,
"elapsedMs": 2538.046
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"archive",
"averageBy",
"billingTotals",
"breachedAccounts",
"capacityHotspots",
"checksum",
"countDistinct",
"errorRate",
"gitBranch",
"gitDirty",
"gitRevision",
"groupBy",
"jobFiles",
"maxBy",
"normalizeJob",
"notify",
"percentile",
"processBatch",
"processJob",
"provenance",
"pruneArchive",
"readJob",
"render",
"renderCsv",
"renderJson",
"sumBy",
"summarize",
"upload",
"validateJob"
],
"correct": true,
"elapsedMs": 2973.813
}
}
},
{
"id": "env",
"kind": "env_vars",
"prompt": "Which environment variables does this file read? Answer as a list of names.",
"expected": [
"REPORT_ARCHIVE_DIR",
"REPORT_BATCH_SIZE",
"REPORT_OUTPUT_DIR",
"REPORT_QUEUE_DIR",
"REPORT_REPO_DIR",
"REPORT_RETENTION_DAYS",
"REPORT_UPLOAD_BASE",
"REPORT_UPLOAD_TIMEOUT_MS"
],
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3919,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPORT_QUEUE_DIR",
"REPORT_OUTPUT_DIR",
"REPORT_ARCHIVE_DIR",
"REPORT_UPLOAD_BASE",
"REPORT_UPLOAD_TIMEOUT_MS",
"REPORT_REPO_DIR",
"REPORT_BATCH_SIZE",
"REPORT_RETENTION_DAYS"
],
"correct": true,
"elapsedMs": 1706.221
},
"ir": {
"variant": "ir",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPORT_ARCHIVE_DIR",
"REPORT_BATCH_SIZE",
"REPORT_OUTPUT_DIR",
"REPORT_QUEUE_DIR",
"REPORT_REPO_DIR",
"REPORT_RETENTION_DAYS",
"REPORT_UPLOAD_BASE",
"REPORT_UPLOAD_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1674.335
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 879,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": [
"REPORT_ARCHIVE_DIR",
"REPORT_BATCH_SIZE",
"REPORT_OUTPUT_DIR",
"REPORT_QUEUE_DIR",
"REPORT_REPO_DIR",
"REPORT_RETENTION_DAYS",
"REPORT_UPLOAD_BASE",
"REPORT_UPLOAD_TIMEOUT_MS"
],
"correct": true,
"elapsedMs": 1706.24
}
}
},
{
"id": "auth",
"kind": "auth_check",
"prompt": "Does this file verify a secret or token before acting? Answer true or false.",
"expected": false,
"applicable": true,
"variants": {
"raw": {
"variant": "raw",
"inputTokens": 3920,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1299.041
},
"ir": {
"variant": "ir",
"inputTokens": 880,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 2001.732
},
"ir+raw": {
"variant": "ir+raw",
"inputTokens": 880,
"llmCalls": 1,
"escalated": false,
"resolved": true,
"basis": "live",
"missing": null,
"answer": false,
"correct": true,
"elapsedMs": 1801.28
}
}
}
]
},
{
"id": "10-analytics-kernel",
"file": "samples/analytics-kernel.js",
"note": "~950-line purely computational kernel: no I/O, no env, no network. The size class where substring heuristics used to invent risk.",
"sourceChars": 28743,
"sourceLines": 951,
"sourceTokens": 7186,
"irTokens": 498,
"ir": {
"mission": {
"target": "samples/analytics-kernel.js",
"objective": "general_optimization"
},
"world": {
"pressure": 0.5,
"confidence": 0.75,
"budget": 0.015,
"risk": "low"
},
"constraints": [
"preserve_api",
"avoid_breaking_changes"
],
"forbidden": [],
"summary": {
"size": 28743,
"lines": 951,
"hasCrypto": false,
"hasFilesystem": false,
"hasNetwork": false
},
"facts": {
"ast": true,
"exports": [
"addMatrices",
"assignClusters",
"autocorrelation",
"boundingBox",
"centroid",
"changePoints",
"clamp",
"compareSeries",
"confidenceInterval",
"convexHull",
"correlationMatrix",
"cosineSimilarity",
"covariance",
"cumulativeSum",
"describe",
"detectTrend",
"determinant",
"differences",
"entropy",
"erf",
"euclidean",
"exponentialSmoothing",
"forecast",
"giniImpurity",
"healthScore",
"histogram",
"identity",
"iqr",
"isNumber",
"kMeans",
"kurtosis",
"linearRegression",
"manhattan",
"mean",
"median",
"minMaxScale",
"minor",
"mode",
"movingAverage",
"multiply",
"normalCdf",
"normalPdf",
"normalizeWeights",
"outliers",
"pearson",
"percentile",
"polygonArea",
"polygonPerimeter",
"polynomialFit",
"predict",
"profileDataset",
"quartiles",
"range",
"rank",
"residuals",
"robustScale",
"round",
"safeDivide",
"scaleMatrix",
"seasonalIndices",
"shape",
"sign",
"silhouette",
"skewness",
"softmax",
"solve",
"spearman",
"stdDev",
"sum",
"summarizeMatrix",
"tScore",
"toNumbers",
"trace",
Náhled byl kvůli velikosti souboru zkrácen
Top comments (0)