{ "protocol_version": 1, "article_slug": "canary-ai-qa-defect-detection", "reviewed_on": "2026-10-08", "evidence_status": "proposed_protocol_not_executed", "not_a_vendor_sdk_schema": true, "sources": [ "https://www.runcanary.ai/blog/qa-bench-v0", "https://www.runcanary.ai/#install" ], "locales": { "en": { "title": "Canary AI QA: Test Defect Detection, Not Benchmark Scores", "cases": [ { "Defect": "Missing tenant ownership check", "Required assertion": "A cannot read B's record", "Useful negative control": "A can still read its own record" }, { "Defect": "Duplicate submission", "Required assertion": "One logical request creates one record", "Useful negative control": "Two different requests create two records" }, { "Defect": "Authorization only in UI", "Required assertion": "Direct backend write is denied", "Useful negative control": "Authorized role can write" }, { "Defect": "Failed step reported as success", "Required assertion": "User sees failure and stored state agrees", "Useful negative control": "Happy path persists correct state" }, { "Defect": "Destructive scope too broad", "Required assertion": "Delete affects only selected records", "Useful negative control": "Unselected records remain readable" }, { "Defect": "Expired session", "Required assertion": "No privileged write after expiry", "Useful negative control": "Valid session remains functional" } ], "steps": [ "Run the same starting application, PR context and time budget for each evaluated setup.", "Save plans, generated tests, executed commands, logs and destination state.", "Reproduce every claimed defect independently. Label infrastructure failures separately.", "Run each candidate test against both defective and repaired versions.", "Count confirmed detections, missed seeded defects, false positives on clean controls and reviewer minutes." ] }, "de": { "title": "Canary AI QA: Fehlererkennung statt Benchmark-Punkte prüfen", "cases": [ { "Defekt": "Fehlende Mandantenprüfung", "Erforderliche Assertion": "A kann B's Datensatz nicht lesen", "Nützliche Kontrolle": "A kann eigene Daten lesen" }, { "Defekt": "Doppelte Übermittlung", "Erforderliche Assertion": "Ein logischer Request erzeugt einen Datensatz", "Nützliche Kontrolle": "Zwei verschiedene Requests erzeugen zwei" }, { "Defekt": "Rechte nur in UI geprüft", "Erforderliche Assertion": "Direkter Backend-Schreibzugriff wird verweigert", "Nützliche Kontrolle": "Berechtigte Rolle kann schreiben" }, { "Defekt": "Fehler als Erfolg angezeigt", "Erforderliche Assertion": "Anzeige und gespeicherter Zustand zeigen Fehler", "Nützliche Kontrolle": "Normaler Ablauf speichert korrekt" }, { "Defekt": "Zu breites Löschen", "Erforderliche Assertion": "Nur ausgewählte Daten werden gelöscht", "Nützliche Kontrolle": "Andere Daten bleiben lesbar" }, { "Defekt": "Abgelaufene Session", "Erforderliche Assertion": "Kein privilegierter Schreibzugriff", "Nützliche Kontrolle": "Gültige Session funktioniert" } ], "steps": [ "Nutze dieselbe Ausgangsapp, PR-Kontext und Zeitgrenze für alle Konfigurationen.", "Sichere Pläne, Tests, Befehle, Logs und Zielzustand.", "Reproduziere gemeldete Defekte unabhängig und klassifiziere Infrastrukturfehler separat.", "Führe jeden Kandidatentest gegen fehlerhafte und reparierte Version aus.", "Zähle bestätigte Erkennungen, übersehene bekannte Defekte, Fehlalarme bei Kontrollen und Review-Minuten." ] }, "es": { "title": "Canary AI QA: medir defectos, no puntos de benchmark", "cases": [ { "Defecto": "Falta de comprobación de propietario", "Aserción necesaria": "A no puede leer registros de B", "Control útil": "A sigue leyendo los suyos" }, { "Defecto": "Envío duplicado", "Aserción necesaria": "Una solicitud lógica crea un registro", "Control útil": "Dos solicitudes distintas crean dos" }, { "Defecto": "Permisos solo en UI", "Aserción necesaria": "Escritura directa al backend denegada", "Control útil": "Rol autorizado puede escribir" }, { "Defecto": "Fallo mostrado como éxito", "Aserción necesaria": "UI y estado guardado indican fallo", "Control útil": "Camino normal persiste correctamente" }, { "Defecto": "Borrado demasiado amplio", "Aserción necesaria": "Solo borra registros seleccionados", "Control útil": "Los demás siguen accesibles" }, { "Defecto": "Sesión caducada", "Aserción necesaria": "Sin escritura privilegiada", "Control útil": "Sesión válida funciona" } ], "steps": [ "Usa igual aplicación inicial, contexto de PR y tiempo para cada configuración.", "Guarda planes, pruebas, comandos, logs y estado final.", "Reproduce defectos de forma independiente y separa fallos de infraestructura.", "Ejecuta cada prueba candidata en versión defectuosa y reparada.", "Cuenta detecciones confirmadas, defectos sembrados omitidos, falsos positivos y minutos de revisión." ] }, "zh": { "title": "Canary AI QA：验证缺陷检出，而不是排行榜分数", "cases": [ { "缺陷": "缺少租户所有权检查", "必须检查的断言": "A 不能读取 B 的记录", "有用的阴性对照": "A 仍能读取自己的记录" }, { "缺陷": "重复提交", "必须检查的断言": "一次逻辑请求只创建一个记录", "有用的阴性对照": "两次不同请求创建两个记录" }, { "缺陷": "仅 UI 检查授权", "必须检查的断言": "直接后端写入被拒绝", "有用的阴性对照": "授权角色能写入" }, { "缺陷": "失败步骤报告成功", "必须检查的断言": "用户看到失败且存储状态一致", "有用的阴性对照": "正常流程正确持久化" }, { "缺陷": "删除范围过大", "必须检查的断言": "仅影响选中记录", "有用的阴性对照": "未选记录仍可读取" }, { "缺陷": "会话过期", "必须检查的断言": "过期后无特权写入", "有用的阴性对照": "有效会话仍可操作" } ], "steps": [ "对每个配置固定初始应用、PR 上下文和时间预算。", "保存计划、生成测试、执行命令、日志与目标状态。", "独立复现每个发现，单独标记基础设施故障。", "每个候选测试同时在缺陷版和修复版运行。", "统计确认检出、遗漏的植入缺陷、干净对照误报和人工复核分钟。" ] } }, "run_configuration": { "product_versions": null, "model": null, "fixture_revision": null, "permissions": null, "time_budget_seconds": null, "cost_budget": null }, "trial_results": [], "report_fields": [ "attempt_id", "case_id", "observed_final_state", "acceptance_passed", "independent_evidence", "duration_seconds", "total_cost", "human_review_minutes", "unresolved_effects" ] }