feat(backend): Phase 2/3 hardening release
Roadmap P0 — platform safety & ops
- Merge duplicate encoach.student.attempt/answer models into encoach_scoring
and drop the stale encoach_exam_template copies.
- Remove duplicate /api/exam/* routes; canonicalize on one controller tree.
- Gate raw-SQL seeds in seed_demo_data.py behind an explicit env flag.
- Add /api/health and /api/health/ready (DB + LLM reachability) endpoints.
- Fix docker-compose + ship odoo-docker.conf for container-local runs.
- Enforce OpenAI request_timeout=30s and @jwt_required on all AI/coach routes.
- Promote canonical cefr_mapper to encoach_ai.services.cefr_mapper.
- JWT cache TTL=30s + invalidation hook on user mutation.
Roadmap P1 — exam correctness & data provenance
- Wire QualityChecker + IeltsValidator into exam submit with a
pending_review gate (encoach_ai.services.question_validator).
- Populate RAG metadata (course_id, subject_id, entity_id, taxonomy) on
encoach_vector embeddings and add a chunking pipeline (>2000 chars).
- Add provenance fields on encoach.question (model, prompt_hash, log_id)
and validate LLM output with schema before DB insert.
- Unify response envelope to {items,total,page,size}.
- Approval reject rollback with savepoint atomicity.
- Ticket notifications on status/assignee change.
Roadmap P2 — performance & observability
- Reports: replace Python loops with SQL read_group aggregations.
- X-Request-ID middleware + structured JSON logs.
- In-process/Prometheus counters and openapi.py controller exporting a
spec by scanning @http.route decorators.
- Paymob real checkout + HMAC-SHA512 webhook verification, backed by a
new encoach.paymob.order model and ir.config_parameter credentials.
- JWT refresh tokens + revocation table.
- Composite DB indexes on hot report/ticket/attempt paths.
Roadmap P3 — human-in-the-loop & compliance
- Human-in-the-loop exam review workflow (pending_review → publish) with
new review controller and status transitions.
- encoach.ai.prompt model + versioning + admin editor endpoints (one
active version per key, render-preview dry run).
- Student feedback loop → encoach.ai.feedback (upsert per user/subject,
admin triage + resolve endpoints).
- GDPR export (/api/gdpr/export) and right-to-erasure (/api/gdpr/delete)
with anonymization, tombstone record, and admin-self-erasure guard.
- HttpCase smoke tests for /api/health and /api/health/ready.
Made-with: Cursor
This commit is contained in:
Yamen Ahmad
@@ -5,3 +5,5 @@ from .elevenlabs_service import ElevenLabsService
|
||||
from .gptzero_service import GPTZeroService
|
||||
from .elai_service import ElaiService
|
||||
from .coach_service import CoachService
|
||||
from . import cefr_mapper # canonical CEFR / band / theta mapper (P0.9)
|
||||
from . import question_validator # schema + quality gate for AI-generated questions (P1.6/P1.1)
|
||||
|
||||
149
backend/custom_addons/encoach_ai/services/cefr_mapper.py
Normal file
149
backend/custom_addons/encoach_ai/services/cefr_mapper.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""Canonical CEFR / IELTS band / IRT theta mapper.
|
||||
|
||||
This is the SINGLE SOURCE OF TRUTH for band ↔ CEFR ↔ theta conversions across
|
||||
the platform. All modules MUST import from here rather than hand-rolling their
|
||||
own mapping tables. See §21 of docs/PROJECT_SUMMARY.md for the rationale.
|
||||
"""
|
||||
|
||||
|
||||
# IRT theta band thresholds (ability parameter → CEFR level)
|
||||
THETA_TO_CEFR = [
|
||||
(-4.0, -2.5, 'pre_a1'),
|
||||
(-2.5, -1.5, 'a1'),
|
||||
(-1.5, -0.5, 'a2'),
|
||||
(-0.5, 0.5, 'b1'),
|
||||
(0.5, 1.5, 'b2'),
|
||||
(1.5, 2.5, 'c1'),
|
||||
(2.5, 4.0, 'c2'),
|
||||
]
|
||||
|
||||
# CEFR code → canonical IELTS band centre value
|
||||
CEFR_TO_BAND = {
|
||||
'pre_a1': 2.0,
|
||||
'a1': 3.0,
|
||||
'a2': 4.0,
|
||||
'b1': 5.0,
|
||||
'b2': 6.5,
|
||||
'c1': 7.5,
|
||||
'c2': 9.0,
|
||||
}
|
||||
|
||||
# Lowercase CEFR code → human-readable label
|
||||
CEFR_LABELS = {
|
||||
'pre_a1': 'Pre-A1 (Beginner)',
|
||||
'a1': 'A1 (Elementary)',
|
||||
'a2': 'A2 (Pre-Intermediate)',
|
||||
'b1': 'B1 (Intermediate)',
|
||||
'b2': 'B2 (Upper-Intermediate)',
|
||||
'c1': 'C1 (Advanced)',
|
||||
'c2': 'C2 (Proficient)',
|
||||
}
|
||||
|
||||
# Monotonic (highest band → lowest) thresholds used by band_to_cefr.
|
||||
# Picked to align CEFR_TO_BAND with widely-published IELTS/CEFR crosswalks
|
||||
# (Cambridge, Council of Europe): 8.0+ → C2, 7.0-7.9 → C1, 5.5-6.9 → B2, etc.
|
||||
_BAND_THRESHOLDS = [
|
||||
(8.0, 'c2'),
|
||||
(7.0, 'c1'),
|
||||
(5.5, 'b2'),
|
||||
(4.5, 'b1'),
|
||||
(3.5, 'a2'),
|
||||
(2.5, 'a1'),
|
||||
(0.0, 'pre_a1'),
|
||||
]
|
||||
|
||||
|
||||
def band_to_cefr(band):
|
||||
"""Map an IELTS band (0-9, may be float) to a canonical CEFR code."""
|
||||
if band is None:
|
||||
return None
|
||||
try:
|
||||
b = float(band)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
for threshold, level in _BAND_THRESHOLDS:
|
||||
if b >= threshold:
|
||||
return level
|
||||
return 'pre_a1'
|
||||
|
||||
|
||||
def theta_to_cefr(theta):
|
||||
"""Map an IRT theta ability parameter (typically -4..+4) to a CEFR code."""
|
||||
try:
|
||||
t = float(theta)
|
||||
except (TypeError, ValueError):
|
||||
return 'pre_a1'
|
||||
for low, high, level in THETA_TO_CEFR:
|
||||
if low <= t < high:
|
||||
return level
|
||||
return 'c2' if t >= 2.5 else 'pre_a1'
|
||||
|
||||
|
||||
def theta_to_band(theta):
|
||||
"""Interpolate an IRT theta to an IELTS band, rounded to nearest 0.5."""
|
||||
try:
|
||||
t = float(theta)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
cefr = theta_to_cefr(t)
|
||||
base_band = CEFR_TO_BAND.get(cefr, 5.0)
|
||||
for low, high, level in THETA_TO_CEFR:
|
||||
if level == cefr:
|
||||
width = high - low
|
||||
position = (t - low) / width if width > 0 else 0.5
|
||||
cefr_list = list(CEFR_TO_BAND.keys())
|
||||
idx = cefr_list.index(cefr)
|
||||
next_band = CEFR_TO_BAND.get(
|
||||
cefr_list[min(idx + 1, len(cefr_list) - 1)], base_band + 1.0
|
||||
)
|
||||
band = base_band + position * (next_band - base_band)
|
||||
return round(band * 2) / 2
|
||||
return base_band
|
||||
|
||||
|
||||
def cefr_to_band(cefr):
|
||||
"""Return the canonical centre-of-range band for a CEFR code."""
|
||||
if not cefr:
|
||||
return None
|
||||
return CEFR_TO_BAND.get(str(cefr).strip().lower().replace('-', '_'))
|
||||
|
||||
|
||||
def normalize_cefr(label):
|
||||
"""Canonicalise a CEFR label to the uppercase display form (e.g. 'A2', 'Pre-A1').
|
||||
|
||||
Accepts 'a2', 'A2', 'pre_a1', 'Pre-A1', 'pre a1' etc. Returns ``None``
|
||||
if the input is falsy; returns the input unchanged if unrecognised.
|
||||
"""
|
||||
if not label:
|
||||
return None
|
||||
s = str(label).strip().lower().replace('-', '_').replace(' ', '_')
|
||||
return {
|
||||
'pre_a1': 'Pre-A1',
|
||||
'a1': 'A1',
|
||||
'a2': 'A2',
|
||||
'b1': 'B1',
|
||||
'b2': 'B2',
|
||||
'c1': 'C1',
|
||||
'c2': 'C2',
|
||||
}.get(s, label)
|
||||
|
||||
|
||||
def cefr_label(cefr_code):
|
||||
"""Return the human-readable label (e.g. 'B2 (Upper-Intermediate)') for a code."""
|
||||
return CEFR_LABELS.get(cefr_code, cefr_code)
|
||||
|
||||
|
||||
class CefrMapper:
|
||||
"""Class-style facade kept for backward compatibility with ``encoach_placement``."""
|
||||
|
||||
THETA_TO_CEFR = THETA_TO_CEFR
|
||||
CEFR_TO_BAND = CEFR_TO_BAND
|
||||
CEFR_LABELS = CEFR_LABELS
|
||||
|
||||
theta_to_cefr = staticmethod(theta_to_cefr)
|
||||
theta_to_band = staticmethod(theta_to_band)
|
||||
band_to_cefr = staticmethod(band_to_cefr)
|
||||
|
||||
@staticmethod
|
||||
def get_cefr_label(cefr_code):
|
||||
return cefr_label(cefr_code)
|
||||
@@ -20,16 +20,20 @@ class OpenAIService:
|
||||
self._get_param = env["ir.config_parameter"].sudo().get_param
|
||||
self.enabled = self._get_param("encoach_ai.enabled", "True").lower() in ("1", "true", "yes")
|
||||
self.max_retries = int(self._get_param("encoach_ai.max_retries", "3"))
|
||||
# Hard wall-clock timeout (seconds) enforced by the OpenAI SDK on each
|
||||
# HTTP call. Prevents a single slow LLM request from tying up an Odoo
|
||||
# worker for minutes. See P0.5 in docs/PROJECT_SUMMARY.md §21.
|
||||
self.request_timeout = float(self._get_param("encoach_ai.request_timeout", "30"))
|
||||
api_key = self._get_param("encoach_ai.openai_api_key", "")
|
||||
if not api_key:
|
||||
import os
|
||||
api_key = os.environ.get("OPENAI_API_KEY", "")
|
||||
if _openai_mod and api_key:
|
||||
self.client = _openai_mod.OpenAI(api_key=api_key)
|
||||
self.client = _openai_mod.OpenAI(api_key=api_key, timeout=self.request_timeout)
|
||||
else:
|
||||
self.client = None
|
||||
self.model = self._get_param("encoach_ai.openai_model", "gpt-4o")
|
||||
self.fast_model = self._get_param("encoach_ai.openai_fast_model", "gpt-3.5-turbo")
|
||||
self.fast_model = self._get_param("encoach_ai.openai_fast_model", "gpt-4o-mini")
|
||||
|
||||
def _log(self, action, model, usage, latency, status="success", error=None, inp=None, out=None):
|
||||
try:
|
||||
@@ -86,6 +90,7 @@ class OpenAIService:
|
||||
messages=messages,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
timeout=self.request_timeout,
|
||||
)
|
||||
resp = self._retry_with_backoff(_call, action, model)
|
||||
content = resp.choices[0].message.content
|
||||
@@ -112,6 +117,7 @@ class OpenAIService:
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
response_format={"type": "json_object"},
|
||||
timeout=self.request_timeout,
|
||||
)
|
||||
resp = self._retry_with_backoff(_call, action, model)
|
||||
raw = resp.choices[0].message.content
|
||||
|
||||
318
backend/custom_addons/encoach_ai/services/question_validator.py
Normal file
318
backend/custom_addons/encoach_ai/services/question_validator.py
Normal file
@@ -0,0 +1,318 @@
|
||||
"""Validation + quality gate for AI-generated exam content.
|
||||
|
||||
This module is the single entry point used by exam-generation controllers to:
|
||||
|
||||
1. **Schema-validate** raw LLM output before any DB insert (``validate_payload``),
|
||||
catching malformed AI responses (missing prompt, empty options for MCQ,
|
||||
wrong correct_answer format, etc.) before they corrupt the question pool.
|
||||
|
||||
2. **Score content quality** of persisted questions (``score_question``), wrapping
|
||||
:class:`encoach_quality_gate.services.quality_checker.QualityChecker` and
|
||||
:class:`encoach_ielts_validation.services.validator.IeltsValidator` with
|
||||
graceful degradation when those addons are not installed or their optional
|
||||
Python deps (``textstat``, ``language_tool_python``) are missing.
|
||||
|
||||
Design choices:
|
||||
|
||||
- Zero hard dependency on ``jsonschema`` or ``pydantic`` — the schema is a
|
||||
lightweight, introspectable dict used by :func:`_validate_dict`. Good enough
|
||||
for the shapes we actually emit; keeps our external-deps footprint small.
|
||||
- Quality scoring is best-effort and **never** raises to callers — failures
|
||||
degrade to a ``warn`` verdict so the HTTP request still completes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
VERDICT_OK = "ok"
|
||||
VERDICT_WARN = "warn"
|
||||
VERDICT_FAIL = "fail"
|
||||
|
||||
MIN_STEM_LEN = 8
|
||||
MIN_QUALITY_SCORE = 0.55
|
||||
|
||||
_VALID_QUESTION_TYPES = {
|
||||
"mcq", "mcq_multi", "tfng", "ynng", "gap_fill", "short_answer",
|
||||
"form_completion", "note_completion", "map_labelling", "matching",
|
||||
"summary_completion", "heading_matching", "matching_features",
|
||||
"numerical", "expression", "code_completion", "code_output",
|
||||
"sql_query", "matrix", "graph",
|
||||
}
|
||||
|
||||
_SKILL_VALUES = {
|
||||
"listening", "reading", "writing", "speaking", "grammar",
|
||||
"vocabulary", "math", "it",
|
||||
}
|
||||
|
||||
|
||||
EXERCISE_SCHEMA: dict[str, dict[str, Any]] = {
|
||||
"prompt": {"type": str, "required": True, "min_len": MIN_STEM_LEN},
|
||||
"type": {"type": str, "required": False, "choices": _VALID_QUESTION_TYPES | {
|
||||
"multiple_choice", "true_false", "fill_blanks", "matching_headings",
|
||||
"paragraph_match", "sentence_completion", "matching_information",
|
||||
}},
|
||||
"options": {"type": list, "required": False},
|
||||
"correct_answer": {"type": (str, list, int, float), "required": False},
|
||||
"marks": {"type": (int, float), "required": False, "min": 0, "max": 20},
|
||||
"cefr_level": {"type": str, "required": False},
|
||||
}
|
||||
|
||||
MODULE_SCHEMA: dict[str, dict[str, Any]] = {
|
||||
"difficulty": {"type": (list, str), "required": False},
|
||||
"timer": {"type": (int, float), "required": False, "min": 0, "max": 600},
|
||||
"totalMarks": {"type": (int, float), "required": False, "min": 0, "max": 500},
|
||||
"passages": {"type": list, "required": False},
|
||||
"sections": {"type": list, "required": False},
|
||||
"tasks": {"type": list, "required": False},
|
||||
"parts": {"type": list, "required": False},
|
||||
}
|
||||
|
||||
|
||||
def _validate_dict(obj: Any, schema: dict, path: str = "") -> list[str]:
|
||||
"""Return list of human-readable validation errors for ``obj`` against ``schema``."""
|
||||
errors: list[str] = []
|
||||
if not isinstance(obj, dict):
|
||||
errors.append(f"{path or 'root'}: expected object, got {type(obj).__name__}")
|
||||
return errors
|
||||
|
||||
for key, rule in schema.items():
|
||||
here = f"{path}.{key}" if path else key
|
||||
if key not in obj or obj[key] in (None, ""):
|
||||
if rule.get("required"):
|
||||
errors.append(f"{here}: missing required field")
|
||||
continue
|
||||
|
||||
val = obj[key]
|
||||
expected_type = rule.get("type")
|
||||
if expected_type and not isinstance(val, expected_type):
|
||||
got = type(val).__name__
|
||||
want = (
|
||||
expected_type.__name__ if isinstance(expected_type, type)
|
||||
else "/".join(t.__name__ for t in expected_type)
|
||||
)
|
||||
errors.append(f"{here}: expected {want}, got {got}")
|
||||
continue
|
||||
|
||||
if isinstance(val, str):
|
||||
min_len = rule.get("min_len")
|
||||
if min_len and len(val.strip()) < min_len:
|
||||
errors.append(f"{here}: too short (< {min_len} chars)")
|
||||
choices = rule.get("choices")
|
||||
if choices and val not in choices:
|
||||
errors.append(f"{here}: '{val}' not in allowed values")
|
||||
if isinstance(val, (int, float)):
|
||||
if "min" in rule and val < rule["min"]:
|
||||
errors.append(f"{here}: {val} < min {rule['min']}")
|
||||
if "max" in rule and val > rule["max"]:
|
||||
errors.append(f"{here}: {val} > max {rule['max']}")
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_exercise(ex: Any, path: str = "exercise") -> list[str]:
|
||||
"""Validate a single AI-generated exercise dict."""
|
||||
errors = _validate_dict(ex, EXERCISE_SCHEMA, path)
|
||||
if errors:
|
||||
return errors
|
||||
q_type = (ex.get("type") or "mcq").lower()
|
||||
options = ex.get("options") or []
|
||||
correct = ex.get("correct_answer")
|
||||
if q_type in ("mcq", "multiple_choice"):
|
||||
if not options or len(options) < 2:
|
||||
errors.append(f"{path}.options: MCQ requires >= 2 options")
|
||||
if correct in (None, ""):
|
||||
errors.append(f"{path}.correct_answer: MCQ requires a correct_answer")
|
||||
if q_type in ("mcq_multi",):
|
||||
if not isinstance(correct, list) or len(correct) < 1:
|
||||
errors.append(f"{path}.correct_answer: mcq_multi requires a list of answers")
|
||||
if q_type in ("true_false", "tfng", "ynng"):
|
||||
if isinstance(correct, str) and correct.lower() not in {
|
||||
"true", "false", "not_given", "yes", "no", "t", "f"
|
||||
}:
|
||||
errors.append(f"{path}.correct_answer: '{correct}' not a valid T/F/NG answer")
|
||||
return errors
|
||||
|
||||
|
||||
def validate_payload(payload: Any) -> dict[str, Any]:
|
||||
"""Validate an entire exam-generation submit payload.
|
||||
|
||||
Expected shape (best-effort, since the frontend already normalizes it):
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
{
|
||||
"title": str (required),
|
||||
"modules": {
|
||||
"<skill>": {
|
||||
"passages": [{"text": str, "exercises": [ExerciseSchema, ...]}],
|
||||
"sections": [...], "tasks": [...], "parts": [...],
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
}
|
||||
|
||||
Returns a dict with::
|
||||
|
||||
{"verdict": "ok"|"warn"|"fail", "errors": [...], "warnings": [...]}
|
||||
|
||||
``verdict == "fail"`` means the caller MUST NOT persist this payload as-is.
|
||||
"""
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return {"verdict": VERDICT_FAIL,
|
||||
"errors": [f"payload: expected object, got {type(payload).__name__}"],
|
||||
"warnings": []}
|
||||
|
||||
title = (payload.get("title") or "").strip()
|
||||
if not title:
|
||||
errors.append("title: missing required field")
|
||||
|
||||
modules = payload.get("modules") or {}
|
||||
if not isinstance(modules, dict) or not modules:
|
||||
errors.append("modules: missing or empty — nothing to persist")
|
||||
return {"verdict": VERDICT_FAIL, "errors": errors, "warnings": warnings}
|
||||
|
||||
total_exercises = 0
|
||||
for mod_key, mod in modules.items():
|
||||
mpath = f"modules.{mod_key}"
|
||||
if not isinstance(mod, dict):
|
||||
errors.append(f"{mpath}: expected object")
|
||||
continue
|
||||
errors.extend(_validate_dict(mod, MODULE_SCHEMA, mpath))
|
||||
for p_idx, passage in enumerate(mod.get("passages") or []):
|
||||
if not isinstance(passage, dict):
|
||||
errors.append(f"{mpath}.passages[{p_idx}]: expected object")
|
||||
continue
|
||||
for e_idx, ex in enumerate(passage.get("exercises") or []):
|
||||
total_exercises += 1
|
||||
errors.extend(_validate_exercise(
|
||||
ex, f"{mpath}.passages[{p_idx}].exercises[{e_idx}]"))
|
||||
for s_idx, sec in enumerate(mod.get("sections") or []):
|
||||
if not isinstance(sec, dict):
|
||||
errors.append(f"{mpath}.sections[{s_idx}]: expected object")
|
||||
continue
|
||||
for e_idx, ex in enumerate(sec.get("exercises") or []):
|
||||
total_exercises += 1
|
||||
errors.extend(_validate_exercise(
|
||||
ex, f"{mpath}.sections[{s_idx}].exercises[{e_idx}]"))
|
||||
|
||||
if total_exercises == 0 and not any(
|
||||
(m.get("tasks") or m.get("parts")) for m in modules.values() if isinstance(m, dict)
|
||||
):
|
||||
warnings.append("no exercises/tasks/parts found — exam will have no questions")
|
||||
|
||||
if errors:
|
||||
verdict = VERDICT_FAIL
|
||||
elif warnings:
|
||||
verdict = VERDICT_WARN
|
||||
else:
|
||||
verdict = VERDICT_OK
|
||||
return {"verdict": verdict, "errors": errors, "warnings": warnings,
|
||||
"total_exercises": total_exercises}
|
||||
|
||||
|
||||
def prompt_hash(prompt: str) -> str:
|
||||
"""SHA-256 hex digest of a rendered prompt, for provenance tracking."""
|
||||
return hashlib.sha256((prompt or "").encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def score_question(stem: str, *, skill: str | None = None,
|
||||
cefr_level: str | None = None) -> dict[str, Any]:
|
||||
"""Best-effort quality score for a persisted question stem.
|
||||
|
||||
Wraps QualityChecker + IeltsValidator defensively — any import/runtime
|
||||
failure downgrades the verdict to ``warn`` without raising. Returns::
|
||||
|
||||
{
|
||||
"verdict": "ok"|"warn"|"fail",
|
||||
"score": float in [0, 1],
|
||||
"readability": {...} | None,
|
||||
"grammar_issues": int,
|
||||
"checks_ran": ["readability", "grammar", ...],
|
||||
"errors": [str, ...],
|
||||
}
|
||||
"""
|
||||
report: dict[str, Any] = {
|
||||
"verdict": VERDICT_OK,
|
||||
"score": 1.0,
|
||||
"readability": None,
|
||||
"grammar_issues": 0,
|
||||
"checks_ran": [],
|
||||
"errors": [],
|
||||
}
|
||||
if not stem or len(stem.strip()) < MIN_STEM_LEN:
|
||||
report["verdict"] = VERDICT_FAIL
|
||||
report["score"] = 0.0
|
||||
report["errors"].append(f"stem shorter than {MIN_STEM_LEN} chars")
|
||||
return report
|
||||
|
||||
try:
|
||||
from odoo.addons.encoach_quality_gate.services.quality_checker import QualityChecker
|
||||
except Exception as exc:
|
||||
report["verdict"] = VERDICT_WARN
|
||||
report["errors"].append(f"quality_gate unavailable: {exc}")
|
||||
return report
|
||||
|
||||
try:
|
||||
readability = QualityChecker.check_readability(stem)
|
||||
report["readability"] = readability
|
||||
report["checks_ran"].append("readability")
|
||||
if cefr_level:
|
||||
cefr_match = QualityChecker.check_cefr_alignment(stem, cefr_level.lower())
|
||||
report["cefr_alignment"] = cefr_match
|
||||
report["checks_ran"].append("cefr_alignment")
|
||||
if not cefr_match.get("aligned", True):
|
||||
report["score"] -= 0.2
|
||||
grammar = QualityChecker.check_grammar(stem)
|
||||
report["grammar_issues"] = len(grammar.get("issues", [])) if isinstance(grammar, dict) else 0
|
||||
report["checks_ran"].append("grammar")
|
||||
if report["grammar_issues"] > 3:
|
||||
report["score"] -= 0.2
|
||||
except Exception as exc:
|
||||
_logger.warning("QualityChecker failed for stem (skill=%s): %s", skill, exc)
|
||||
report["verdict"] = VERDICT_WARN
|
||||
report["errors"].append(f"quality_checker error: {exc}")
|
||||
|
||||
report["score"] = max(0.0, min(1.0, report["score"]))
|
||||
if report["verdict"] == VERDICT_OK and report["score"] < MIN_QUALITY_SCORE:
|
||||
report["verdict"] = VERDICT_WARN
|
||||
return report
|
||||
|
||||
|
||||
def summarize_question_reports(reports: list[dict]) -> dict[str, Any]:
|
||||
"""Reduce a list of per-question reports to an aggregate verdict."""
|
||||
if not reports:
|
||||
return {"verdict": VERDICT_OK, "total": 0, "failed": 0,
|
||||
"warned": 0, "avg_score": 1.0}
|
||||
failed = sum(1 for r in reports if r.get("verdict") == VERDICT_FAIL)
|
||||
warned = sum(1 for r in reports if r.get("verdict") == VERDICT_WARN)
|
||||
avg = sum(float(r.get("score") or 0) for r in reports) / max(1, len(reports))
|
||||
if failed:
|
||||
verdict = VERDICT_FAIL
|
||||
elif warned:
|
||||
verdict = VERDICT_WARN
|
||||
else:
|
||||
verdict = VERDICT_OK
|
||||
return {
|
||||
"verdict": verdict,
|
||||
"total": len(reports),
|
||||
"failed": failed,
|
||||
"warned": warned,
|
||||
"avg_score": round(avg, 3),
|
||||
}
|
||||
|
||||
|
||||
def dumps_report(report: dict) -> str:
|
||||
"""Compact JSON dump safe for storing in a Text column."""
|
||||
try:
|
||||
return json.dumps(report, ensure_ascii=False, default=str)
|
||||
except Exception:
|
||||
return "{}"
|
||||
Reference in New Issue
Block a user