From 93c530eef224b287055d0bfbbc9d1a6aa4d3fe37 Mon Sep 17 00:00:00 2001 From: Yamen Ahmad Date: Sun, 19 Apr 2026 14:16:47 +0400 Subject: [PATCH] =?UTF-8?q?chore(ci,docs):=20GitHub=20Actions,=20ADRs,=20R?= =?UTF-8?q?EADME=20overhaul,=20=C2=A721=20Hardening=20Release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .github/workflows/ci.yml: two jobs — frontend (tsc --noEmit, lint, build, Playwright) and backend (Postgres 16 + odoo:19 --test-enable --test-tags encoach_api) — catches regressions before merge. - docs/adr/: start an Architecture Decision Record trail with 0001 canonical directory layout, 0002 JWT refresh flow, 0003 paginated response envelope, 0004 RAG metadata + chunking. - docs/PROJECT_SUMMARY.md §21 Hardening Release: full recap of the AI quality loop, compliance, Paymob, i18n, and CI work shipped in this drop, plus new DB tables, REST routes, frontend routes, verification results, and operator-facing configuration. - README.md refreshed for the v4 split-repo doctrine and the new feature surface. - new_project/DEPRECATED.md: formal retirement notice pointing at backend/ as the canonical tree. Made-with: Cursor --- .github/workflows/ci.yml | 109 ++++++++ README.md | 267 ++++++++++++------- docs/PROJECT_SUMMARY.md | 130 +++++++++ docs/adr/0000-template.md | 26 ++ docs/adr/0001-canonical-directory-layout.md | 52 ++++ docs/adr/0002-jwt-refresh-token-flow.md | 66 +++++ docs/adr/0003-paginated-response-envelope.md | 60 +++++ docs/adr/0004-rag-metadata-and-chunking.md | 67 +++++ docs/adr/README.md | 22 ++ new_project/DEPRECATED.md | 45 ++++ 10 files changed, 744 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 docs/adr/0000-template.md create mode 100644 docs/adr/0001-canonical-directory-layout.md create mode 100644 docs/adr/0002-jwt-refresh-token-flow.md create mode 100644 docs/adr/0003-paginated-response-envelope.md create mode 100644 docs/adr/0004-rag-metadata-and-chunking.md create mode 100644 docs/adr/README.md create mode 100644 new_project/DEPRECATED.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c538fccb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,109 @@ +name: CI + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # -------------------------------------------------------------------- + # Frontend: type-check, lint, build, and run Playwright smoke tests. + # -------------------------------------------------------------------- + frontend: + name: Frontend — lint + build + e2e + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Type check + run: npx tsc --noEmit -p tsconfig.app.json + + - name: Lint + run: npm run lint + + - name: Build + run: npm run build + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Run Playwright smoke tests + run: npm run test:e2e + env: + CI: "true" + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: frontend/playwright-report + retention-days: 7 + + # -------------------------------------------------------------------- + # Backend: run the Odoo HttpCase smoke suite via the official image. + # + # The image mounts our custom_addons tree and runs the test-tagged + # subset so we don't pay for a full install on every PR. + # -------------------------------------------------------------------- + backend: + name: Backend — Odoo HttpCase + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: odoo + POSTGRES_PASSWORD: odoo + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U odoo" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - name: Run Odoo HttpCase tests in a throwaway container + run: | + docker run --rm \ + --network host \ + -v "$GITHUB_WORKSPACE/backend/custom_addons:/mnt/extra-addons" \ + -e HOST=localhost \ + -e USER=odoo \ + -e PASSWORD=odoo \ + odoo:19 \ + --stop-after-init \ + --test-enable \ + --test-tags "encoach_api" \ + --log-level=warn \ + --without-demo=all \ + --init=encoach_api,encoach_core \ + -d encoach_ci || { + echo "::warning::Odoo HttpCase suite failed — see logs above." + exit 1 + } + env: + PGHOST: localhost + PGUSER: odoo + PGPASSWORD: odoo diff --git a/README.md b/README.md index f2bd2109..54cd4167 100644 --- a/README.md +++ b/README.md @@ -1,123 +1,190 @@ -# EnCoach Backend v2 — Odoo 19 Adaptive Learning Platform +# EnCoach — Adaptive AI Learning Platform -The Odoo 19 backend for the **EnCoach** adaptive learning and institutional LMS platform. Provides ~377 REST API endpoints serving a React 18 SPA frontend. +EnCoach is a smart learning environment for individual and collaborative learning, +fully integrated with AI and equipped with intelligent, professional-grade +exercises, assessments, and e-exams. -## System Overview +This repository hosts the full stack: -| Component | Detail | -|-----------|--------| -| **Framework** | Odoo 19 Community Edition | -| **Language** | Python 3.12+ | -| **Database** | PostgreSQL 16 | -| **LMS Foundation** | OpenEduCat Community (14 modules, ported to v19) | -| **Custom Modules** | 27 `encoach_*` modules | -| **AI/ML** | OpenAI GPT-4o, Whisper (local), AWS Polly, ELAI, GPTZero, FAISS | -| **Authentication** | JWT (PyJWT) | -| **Deployment** | Docker Compose | +- **Backend** — Odoo 19 + ~30 `encoach_*` modules (Python 3.12, PostgreSQL 16) +- **Frontend** — React 18 + Vite + TypeScript single-page app +- **AI layer** — OpenAI + pgvector RAG, quality-gate validation, IELTS validator +- **Ops** — Docker Compose, JWT auth with refresh tokens, Prometheus-compatible + metrics, dynamic OpenAPI 3.0 spec -## Module Architecture +> **Canonical trees:** `backend/` (all server code) and `frontend/` (all client +> code). The legacy `new_project/` directory is deprecated — see +> [`new_project/DEPRECATED.md`](new_project/DEPRECATED.md). -``` -new_project/ - custom_addons/ # 27 custom EnCoach modules - encoach_api/ # REST API controllers (~120 routes) - encoach_lms_api/ # OpenEduCat bridge API (~209 routes) - encoach_adaptive_api/ # Adaptive learning API (~41 routes) - encoach_resources/ # Resource library API (~7 routes) - encoach_core/ # Users, entities, roles, permissions - encoach_taxonomy/ # Subject > Domain > Topic taxonomy - encoach_adaptive/ # Proficiency, learning plans, mastery - encoach_adaptive_ai/ # AI for diagnostics, coaching, content - encoach_exam/ # AI-powered exam engine - encoach_ai/ # OpenAI service wrapper - encoach_ai_generation/ # AI exam content generation - encoach_ai_grading/ # AI grading (writing, speaking, math, IT) - encoach_ai_media/ # TTS (Polly), STT (Whisper), ELAI avatars - encoach_courseware/ # Chapters, materials, AI workbench - encoach_communication/ # Discussion boards, announcements, messaging - encoach_notification/ # Notification engine, rules, preferences - encoach_faq/ # FAQ categories and items - encoach_approval/ # Approval workflows, stages - encoach_assignment/ # Exam-wrapper assignments - encoach_classroom/ # Virtual classroom groups - encoach_stats/ # Session tracking, statistics - encoach_training/ # Training tips, walkthroughs, FAISS - encoach_subscription/ # Packages, Stripe/PayPal/Paymob - encoach_registration/ # User registration, batch import - encoach_ticket/ # Support tickets - encoach_sis/ # UTAS SIS integration - encoach_branding/ # Whitelabeling - openeducat_erp-19.0/ # 14 OpenEduCat community modules - openeducat_core/ # Courses, batches, students, faculty - openeducat_timetable/ # Timetable sessions - openeducat_attendance/ # Attendance management - openeducat_exam/ # Institutional exams, marksheets - openeducat_assignment/ # Course assignments, submissions - openeducat_admission/ # Admission registers, applications - openeducat_classroom/ # Physical classrooms - openeducat_facility/ # Facilities and assets - openeducat_fees/ # Fee plans, student fees - openeducat_library/ # Library media, movements, cards - openeducat_activity/ # Student activities - openeducat_parent/ # Parent-student relationships - openeducat_erp/ # ERP connector - theme_web_openeducat/ # Web theme -``` +--- -## Staging +## 1. Quickstart -| Service | URL | -|---------|-----| -| **Odoo Backend** | `http://5.189.151.117:8069` | -| **React Frontend** | `http://5.189.151.117:3000` | - -## Branching Workflow - -This repo is connected to the staging server via a Git post-receive hook. -**All deployment is automatic after code review approval.** - -1. Never push directly to `main` -- branch protection will block it. -2. Push your changes to a feature branch (e.g. `feature/full-backend-v1`). -3. Open a Pull Request on Gitea targeting `main`. -4. Request review from **devops (Talal)**. -5. Once approved and merged, the staging server rebuilds and redeploys automatically. - -The `.env` file is **not committed**. It lives only on the staging server at `/opt/encoach/backend-v2/.env`. - -## Local Development - -### Option A: Docker (easiest) +### Docker (recommended) ```bash docker compose up -d +# Odoo: http://localhost:8069 +# Frontend: cd frontend && npm install && npm run dev (http://localhost:8080) ``` -Open `http://localhost:8069`. First run: create a new database. +Create a new Odoo database on first visit, then install the `encoach_api` +module to pull in every `encoach_*` dependency. -Stop: `docker compose down` +### Source -### Option B: Source (venv + PostgreSQL) - -**Prerequisites:** macOS, Homebrew, Python 3.11+, PostgreSQL 15+ +Prerequisites: Python 3.12, PostgreSQL 16, Node 20, npm 10. ```bash -brew install postgresql@15 wkhtmltopdf -brew services start postgresql@15 -createuser -s $(whoami) +# Backend +./setup.sh # creates venv, installs requirements.txt +./run.sh # starts Odoo on :8069 -chmod +x setup.sh -./setup.sh -./run.sh +# Frontend +cd frontend +npm install +npm run dev # starts Vite on :8080, proxies /api → :8069 ``` -Open `http://localhost:8069`. +See [`MANUAL-RUN.md`](MANUAL-RUN.md) for a step-by-step walkthrough and +[`CONNECT-POSTGRES.md`](CONNECT-POSTGRES.md) for database wiring. -See [MANUAL-RUN.md](MANUAL-RUN.md) for step-by-step commands. +--- -## Related Documents +## 2. Repository layout + +``` +odoo19/ +├── backend/ +│ └── custom_addons/ +│ ├── encoach_api/ REST base + JWT + OpenAPI + metrics +│ ├── encoach_core/ Users, entities, roles, permissions +│ ├── encoach_taxonomy/ Subject → Domain → Topic +│ ├── encoach_ai/ OpenAI wrapper, cefr_mapper, validator +│ ├── encoach_ai_course/ AI course/exam generation pipelines +│ ├── encoach_ai_grading/ AI grading (writing/speaking/math/IT) +│ ├── encoach_ai_media/ TTS (Polly), STT (Whisper), ELAI +│ ├── encoach_vector/ pgvector store + RAG embeddings +│ ├── encoach_exam_template/ Canonical exam + student attempt models +│ ├── encoach_scoring/ Score computation, CEFR mapping +│ ├── encoach_quality_gate/ Automated content-quality checks +│ ├── encoach_ielts_validation/ IELTS-specific validators +│ ├── encoach_adaptive/ Adaptive engine, style matcher +│ ├── encoach_lms_api/ OpenEduCat bridge + LMS endpoints +│ ├── encoach_branding/ White-label config per entity +│ └── … (other encoach_* modules) +├── frontend/ +│ ├── src/ +│ │ ├── pages/ Route pages (React.lazy code-split) +│ │ ├── components/ Shared UI (shadcn/ui + custom) +│ │ ├── services/ Thin API wrappers over fetch +│ │ ├── hooks/queries/ React Query hooks + keys +│ │ ├── lib/api-client.ts Fetch + auto JWT refresh +│ │ └── types/ Shared DTO types +│ └── vite.config.ts manualChunks: react, query, charts, radix… +├── docs/ +│ ├── PROJECT_SUMMARY.md Release notes & architecture history +│ ├── adr/ Architecture Decision Records +│ └── ENCOACH_*.md SRS, workflows, user stories +├── docker-compose.yml +├── Dockerfile +├── requirements.txt Python deps (pgvector, textstat, etc.) +└── README.md You are here +``` + +--- + +## 3. Architecture at a glance + +``` +┌────────────────────┐ HTTPS/JWT ┌────────────────────┐ +│ React SPA (Vite) │ ───────────────► │ Odoo 19 + FastAPI-style │ +│ - React Query │ ◄─────────────── │ controllers (encoach_*) │ +│ - next-themes │ X-Request-Id └──────┬─────────────┘ +│ - shadcn/ui │ │ +└────────────────────┘ ▼ + ┌──────────────┐ + │ PostgreSQL │ + │ + pgvector │ + └──────┬───────┘ + │ + ▼ + ┌──────────────┐ + │ OpenAI, │ + │ Whisper, │ + │ Polly… │ + └──────────────┘ +``` + +Every request carries an `X-Request-Id` and emits structured JSON logs. +Prometheus-compatible counters are exposed at `/api/metrics`, and the live +OpenAPI 3.0 spec is at `/api/openapi.json`. + +--- + +## 4. Key conventions + +- **Canonical response envelope** — list endpoints return + `{ items: T[], total, page, size, data: T[] }` (see `encoach_api.controllers.base.paginated_envelope`). +- **CEFR mapping** — only `encoach_ai.services.cefr_mapper` is canonical. Do + not reintroduce local `band_to_cefr` copies. +- **JWT tokens** — short-lived access tokens (1h) + revocable refresh tokens + (7d). Only `access` tokens are accepted as Bearer credentials; refresh tokens + must go through `/api/auth/refresh`. See [`docs/adr/0002-jwt-refresh-token-flow.md`](docs/adr/0002-jwt-refresh-token-flow.md). +- **RAG metadata** — vector embeddings carry `course_id`, `subject_id`, + `entity_id`, `taxonomy`, `content_hash`. Chunking kicks in above 2 000 chars. +- **Frontend pagination** — `PaginatedResponse` exposes both `items` and + `data`. Read from `items` in new code. +- **Frontend theming** — tokens live in `frontend/src/index.css` (`:root` and + `.dark`). Always use `hsl(var(--token))` instead of raw hex. + +--- + +## 5. Health, observability, docs + +| Endpoint | Purpose | +|---|---| +| `GET /api/health` | Liveness (always 200 when server is up) | +| `GET /api/health/ready` | Readiness (DB + required config) | +| `GET /api/openapi.json` | Dynamic OpenAPI 3.0 spec generated from `@http.route` | +| `GET /api/metrics` | Prometheus-format counters per route | + +--- + +## 6. Deployment + +Staging and production both use Docker Compose. The staging server rebuilds +automatically from `main`; never force-push. See +[`INSTALL-ODOO-SUMMARY.md`](INSTALL-ODOO-SUMMARY.md) for bootstrap notes. + +| Service | Staging URL | +|---|---| +| Odoo backend | `http://5.189.151.117:8069` | +| React frontend | `http://5.189.151.117:3000` | + +The `.env` file is **never** committed. On staging it lives at +`/opt/encoach/backend-v2/.env`. + +--- + +## 7. Further reading | Document | Description | -|----------|-------------| -| `ENCOACH_ODOO19_BACKEND_SRS.md` | Backend SRS v3.0 (definitive) | -| `ENCOACH_UNIFIED_SRS.md` | Unified frontend + backend SRS v2.0 | -| `ENCOACH_PRODUCT_DESCRIPTION.md` | Non-technical product description | +|---|---| +| [`docs/PROJECT_SUMMARY.md`](docs/PROJECT_SUMMARY.md) | Release notes + architecture history | +| [`docs/adr/`](docs/adr/) | Architecture Decision Records (why we built it this way) | +| [`docs/ENCOACH_UNIFIED_SRS.md`](docs/ENCOACH_UNIFIED_SRS.md) | Unified frontend + backend SRS | +| [`docs/ENCOACH_ODOO19_BACKEND_SRS.md`](docs/ENCOACH_ODOO19_BACKEND_SRS.md) | Backend SRS v3.0 | +| [`docs/ENCOACH_WORKFLOWS_BACKEND_SRS.md`](docs/ENCOACH_WORKFLOWS_BACKEND_SRS.md) | Backend workflows | +| [`docs/ENCOACH_WORKFLOWS_FRONTEND_SRS.md`](docs/ENCOACH_WORKFLOWS_FRONTEND_SRS.md) | Frontend workflows | + +--- + +## 8. Contributing + +1. Branch from `main` — never push direct. Branch protection enforces it. +2. Run `npx tsc --noEmit -p tsconfig.app.json` (frontend) and the module smoke + tests before opening a PR. +3. Every architectural decision should be captured as an ADR under + [`docs/adr/`](docs/adr/). Copy `0000-template.md` to start one. +4. Open the PR against `main` and request review from devops (Talal). diff --git a/docs/PROJECT_SUMMARY.md b/docs/PROJECT_SUMMARY.md index 752446c1..1033de03 100644 --- a/docs/PROJECT_SUMMARY.md +++ b/docs/PROJECT_SUMMARY.md @@ -1319,3 +1319,133 @@ Network tab showed `/api/reports/filters 200`, `/api/reports/record?size=100 200 - **Both `encoach_exam_template` and `encoach_scoring` declare `encoach.student.attempt`.** The Reports controller only reads fields that exist on both definitions (`listening/reading/writing/speaking/overall_band`, `cefr_level`, `status`, `started_at`, and whichever of `completed_at` / `finished_at` is present — `_attempt_completed_at()` probes both). No field access assumes a specific addon version. - **In-progress attempts are excluded from the aggregates.** If you ever notice a student you expect to see missing from Student Performance or Stats Corporate, check their attempt status — `in_progress` attempts are deliberately skipped. They DO still appear in the Record page (which shows everything) unless you also pass `status` or `period` filters. + +--- + +## §21 — Hardening Release (Phase 2/3 completion) + +This section records the final wave of work that closed out all remaining items in the Phase 2 (performance & reliability) and Phase 3 (core AI features, accessibility, onboarding) roadmaps. After this release the platform is considered deployment-ready. + +### 21.1 What shipped + +The release is organised by capability, not file path. + +#### Human-in-the-loop AI quality loop (P3.3 → P3.5) + +| Stage | Where | What it does | +|---|---|---| +| 1. Quality gate | `encoach_exam_template.services.quality_checker` + `ielts_validator` | Runs after every AI exam submit; if scores fall below thresholds the exam is parked in `pending_review` instead of `published`. | +| 2. Review queue | `encoach_exam_template/controllers/review_workflow.py` + `frontend/src/pages/admin/ExamReview{Queue,Detail}.tsx` | Admin-only pages list pending exams, surface aggregate + per-question quality reports, and let a reviewer approve (`→ published`) or reject (`→ draft`) with audit notes. Writes to new `reviewed_by_id`, `reviewed_at`, `review_notes` fields. | +| 3. Prompt library | `encoach_ai.models.ai_prompt` + `controllers/prompt_controller.py` + `frontend/src/pages/admin/AIPromptEditor.tsx` | `encoach.ai.prompt` with a unique `(key, version)` key, one active row per key. Authors can add a new version, activate it (auto-deactivates prior), and dry-run render with sample variables. Renderer uses `str.format_map` against a safe dict. | +| 4. Student feedback | `encoach_ai.models.ai_feedback` + `controllers/feedback_controller.py` + `frontend/src/components/AIFeedbackButtons.tsx` + `frontend/src/pages/admin/AIFeedbackTriage.tsx` | Thumbs up/down on any AI artefact (`question`, `coach`, `explanation`, `translation`, `narrative`, `other`). Unique `(user, subject_type, subject_id)` constraint gives upsert semantics; thumbs-down requires a comment. Admin triage page lists/filters/resolves feedback (`open → acknowledged|fixed|dismissed`). | + +Together these four stages mean every AI output can be gated, reviewed, iterated, and measured — closing the loop from prompt → generation → delivery → feedback → next prompt revision. + +#### Compliance (P3.2) + +New `encoach_api/controllers/gdpr.py` + `models/gdpr_erasure.py`: + +- `GET /api/gdpr/export` — JSON dump of the calling user's profile, entity memberships, exam attempts, answers, AI feedback, AI calls, tickets, coaching sessions. +- `POST /api/gdpr/delete` — requires `{"confirm": true}`; anonymises the partner PII, deletes personal feedback + coaching transcripts, strips free-text PII from retained exam answers, nulls `user_id` on AI logs, deactivates the `res.users` row, and writes a tombstone to `encoach.gdpr.erasure.request` for audit. Admin accounts are blocked from self-erasure. +- `frontend/src/pages/PrivacyCenter.tsx` — routed at `/student/privacy`, `/teacher/privacy`, `/admin/privacy`; download-my-data button produces a timestamped JSON file; erase flow is gated behind a `type "DELETE" to confirm` alert dialog. + +#### Paymob real checkout (P2.6) + +New `encoach_lms_api/controllers/paymob.py` + `models/paymob_order.py`: + +- `POST /api/payments/paymob/checkout` runs the three-step Paymob Accept flow (auth → order → payment key) and returns the hosted iframe URL. Our `encoach.paymob.order` row is created *before* the external call so partial failures don't lose orders. +- `POST /api/payments/paymob/webhook` verifies the HMAC-SHA512 signature over Paymob's canonical field sequence (`amount_cents|created_at|currency|…|success`). Mismatches return 401; duplicates are idempotent; unknown orders return 200 to stop Paymob retrying. +- Credentials read at request time from `ir.config_parameter` (`encoach.paymob.api_key`, `hmac_secret`, `integration_id`, `iframe_id`) so operators can rotate without restarting Odoo. +- `GET /api/paymob-orders` now reads real rows instead of returning `[]`. + +#### Internationalisation (P3.1) + +- `frontend/src/i18n/` bootstraps `i18next` + `i18next-browser-languagedetector` with nested feature-scoped namespaces. +- Initial locales: `en` (source of truth, typed with `Translations` interface) and `ar` (full translation of the same keys). +- RTL handling: when the active language is in `RTL_LANGS`, `document.documentElement.dir` is flipped to `rtl` so Tailwind utilities and Radix primitives adapt. +- `frontend/src/components/LanguageToggle.tsx` added next to the theme toggle in both `AdminLmsLayout` and `RoleLayout` headers. +- Loaded via `main.tsx` side-effect import so every route picks it up. + +#### CI scaffolding (P3.8) + +- `.github/workflows/ci.yml` with two jobs: + - **frontend**: `tsc --noEmit` → `eslint` → `vite build` → Playwright smoke tests against the Vite preview server; uploads a Playwright HTML report on failure. + - **backend**: spins up Postgres 16 as a service and runs the Odoo 19 container with `--test-enable --test-tags encoach_api` against a throwaway `encoach_ci` database. +- `backend/custom_addons/encoach_api/tests/test_health.py` — first `HttpCase`: asserts `/api/health` returns 200 and `/api/health/ready` returns 200 or 503 (never 5xx). +- `frontend/playwright.config.ts` + `frontend/e2e/login.spec.ts` — the login page renders the sign-in form; `/` redirects unauthenticated traffic to `/login`. +- New npm scripts: `test:e2e`, `test:e2e:install`. + +### 21.2 New DB tables (summary) + +| Table | Purpose | Added in | +|---|---|---| +| `encoach.ai.prompt` | Versioned prompt templates (`key`, `version`, `is_active`, `content`). | P3.4 | +| `encoach.ai.feedback` | Student thumbs up/down on AI output, with triage status. | P3.5 | +| `encoach.gdpr.erasure.request` | Tombstone row per right-to-erasure request. | P3.2 | +| `encoach.paymob.order` | Full lifecycle of every Paymob checkout, including the verified HMAC. | P2.6 | + +All have per-group `ir.model.access.csv` entries: admin read/write, authenticated-user read for metadata, create-only on feedback. + +### 21.3 New REST endpoints + +Everything requires a valid JWT unless noted; write routes additionally check `base.group_system` where relevant. + +``` +GET /api/ai/prompts list keys (latest per key) +GET /api/ai/prompts/:key/versions full history for a key +GET /api/ai/prompts/:id one version (full content) +POST /api/ai/prompts create new version (admin) +POST /api/ai/prompts/:id/activate flip active pointer (admin) +POST /api/ai/prompts/:id/render dry-run render with sample vars + +POST /api/ai/feedback upsert (user) +GET /api/ai/feedback/summary up/down counts + my rating +GET /api/ai/feedback admin triage list +POST /api/ai/feedback/:id/resolve admin triage resolve + +GET /api/gdpr/export full data-subject export (self) +POST /api/gdpr/delete right-to-erasure (self, confirm=true) + +POST /api/payments/paymob/checkout start a checkout +POST /api/payments/paymob/webhook HMAC-verified callback (public) +GET /api/payments/paymob/orders list my orders +GET /api/paymob-orders admin + self order list (rewired) +``` + +### 21.4 Frontend surface (new routes) + +``` +/admin/exam/review-queue → ExamReviewQueue.tsx +/admin/exam/review/:examId → ExamReviewDetail.tsx +/admin/ai/prompts → AIPromptEditor.tsx +/admin/ai/feedback → AIFeedbackTriage.tsx +/student|teacher|admin/privacy → PrivacyCenter.tsx +``` + +Plus the `AIFeedbackButtons` primitive (drop-in `/` that handles the upsert, dialog, and comment prompt) and `LanguageToggle` in both main layouts. + +### 21.5 Verification + +- `python3 -m compileall -q backend/custom_addons/encoach_ai backend/custom_addons/encoach_api backend/custom_addons/encoach_lms_api` — clean. +- `npx tsc --noEmit -p tsconfig.app.json` — clean. +- `npm run build` — succeeds; bundle analyser shows the same split profile as the §20 release (initial JS ≈ 250 KB gzipped on the login path); new features are lazy-loaded chunks. +- New modules do not depend on optional addons: all cross-model reads use `if model in request.env` guards so missing addons degrade gracefully instead of 500-ing. + +### 21.6 Configuration operators need to set + +Before the Paymob flow will actually charge anything, set in **Settings → Technical → System Parameters**: + +``` +encoach.paymob.api_key = +encoach.paymob.hmac_secret = +encoach.paymob.integration_id = +encoach.paymob.iframe_id = +``` + +Until those are set, `POST /api/payments/paymob/checkout` returns 503 with a descriptive message instead of silently crashing. + +### 21.7 Deferred (known) + +- `P1.2` — blanket-sudo audit + entity-isolation `ir.rule`s. Needs a coordinated frontend + data migration pass; deferred to a follow-up release. Current security model relies on JWT identity + controller-level `has_group` checks. +- Broader i18n coverage. Only the keys listed in `src/i18n/locales/en.ts` are translated today — extending to every page is a rolling job that can happen in small PRs now that the plumbing is in place. +- Playwright coverage is intentionally minimal (login + redirect); it exists as a safety net, not as full end-to-end coverage. diff --git a/docs/adr/0000-template.md b/docs/adr/0000-template.md new file mode 100644 index 00000000..7930785f --- /dev/null +++ b/docs/adr/0000-template.md @@ -0,0 +1,26 @@ +# ADR {NUMBER}: {TITLE} + +- **Status:** Proposed | Accepted | Superseded by ADR-XXXX +- **Date:** YYYY-MM-DD +- **Deciders:** @handle1, @handle2 + +## Context + +What is the problem? What forces are at play (technical, business, social)? +Keep this short; link out to source files or tickets for depth. + +## Decision + +What did we decide to do? Use imperative voice. Be specific enough that a new +engineer can understand the scope by reading this section alone. + +## Consequences + +- Positive: … +- Negative / trade-offs: … +- Follow-up work: … + +## Alternatives considered + +- **Option A** — why it was rejected. +- **Option B** — why it was rejected. diff --git a/docs/adr/0001-canonical-directory-layout.md b/docs/adr/0001-canonical-directory-layout.md new file mode 100644 index 00000000..784d1204 --- /dev/null +++ b/docs/adr/0001-canonical-directory-layout.md @@ -0,0 +1,52 @@ +# ADR 0001: Canonical `backend/` and `frontend/` directory layout + +- **Status:** Accepted +- **Date:** 2026-04-08 +- **Deciders:** Platform team + +## Context + +The repository historically contained two overlapping copies of server code: + +- `backend/custom_addons/` — Odoo addons, actively developed. +- `new_project/custom_addons/` — older snapshot, intermittently used for + one-off scripts and diverging subtly from `backend/`. + +Both paths showed up in `odoo.conf` variants, deploy scripts, and developer +onboarding docs. Contributors routinely edited the wrong copy, shipped drift, +or rediscovered that fixes "weren't taking" because they landed in the stale +tree. + +The same ambiguity existed implicitly on the client side: several prototype +apps lived under `new_project/frontend/` in addition to the main +`frontend/` workspace. + +## Decision + +Declare a single canonical layout: + +- `backend/custom_addons/**` — the only Odoo addons tree. All deployments, + tests, and Docker images read from here. +- `frontend/**` — the only React/Vite workspace. +- `new_project/` — **deprecated**. A `DEPRECATED.md` is committed inside the + directory explaining the policy; no new files may be added. + +New `encoach_*` modules and frontend features MUST land under the canonical +paths. Existing imports, Dockerfiles, and docs were updated accordingly. + +## Consequences + +- Positive: one obvious place to work, deterministic CI, no drift. +- Positive: `PROJECT_SUMMARY.md` and this README become credible onboarding + material. +- Negative: scripts that hard-coded `new_project/` paths had to be migrated + (one-time cost, done). + +## Alternatives considered + +- **Delete `new_project/` outright.** Rejected for now because a few historic + tarballs and experiment scripts still reference it; leaving the tree with a + `DEPRECATED.md` marker lets us retire it in a later cleanup pass without + blocking the hardening release. +- **Rename `backend/` to `odoo/` to mirror Odoo's own layout.** Rejected + because `odoo/` is already used for the upstream Odoo source checkout. diff --git a/docs/adr/0002-jwt-refresh-token-flow.md b/docs/adr/0002-jwt-refresh-token-flow.md new file mode 100644 index 00000000..ec442e42 --- /dev/null +++ b/docs/adr/0002-jwt-refresh-token-flow.md @@ -0,0 +1,66 @@ +# ADR 0002: JWT access + refresh tokens with revocation ledger + +- **Status:** Accepted +- **Date:** 2026-04-09 +- **Deciders:** Platform team, Security + +## Context + +Originally `/api/login` issued a single long-lived JWT (24h+) stored in +`localStorage`. This gave us three problems: + +1. **No revocation.** A leaked token was valid until it expired; there was no + server-side way to invalidate it short of rotating the global JWT secret. +2. **Silent logouts.** When the token expired mid-session the browser just + started receiving 401s with no graceful recovery path. +3. **Surface area.** Every endpoint accepted the same kind of token, so a + token intended for a refresh use-case could be replayed as a full API + credential. + +## Decision + +Adopt a two-token flow: + +- **Access token** — 1 h TTL, stateless, carries `type: "access"`. Sent on + every request as `Authorization: Bearer …`. `validate_token()` in + `encoach_api.controllers.base` rejects tokens whose `type` is anything other + than `"access"`. +- **Refresh token** — 7 d TTL, carries `type: "refresh"` and a unique `jti`. + Every issued refresh token is logged in a new `encoach.jwt.token` Odoo model + (the revocation ledger) with fields for `user_id`, `issued_at`, `expires_at`, + `last_used_at`, `revoked`, `user_agent`, `remote_ip`. + +Endpoints: + +- `POST /api/login` — returns `access_token`, `refresh_token`, `expires_in`. +- `POST /api/auth/refresh` — validates the refresh token, revokes the old + ledger row (rotation), and issues a fresh access + refresh pair. +- `POST /api/logout` — revokes the supplied refresh token's ledger row. + +The frontend (`frontend/src/lib/api-client.ts`) handles rotation +transparently: on 401 it calls `/api/auth/refresh` once (coalesced across +concurrent requests) and retries the original request. If refresh fails, all +tokens are cleared and the user is redirected to `/login`. + +A cron (`encoach_api.data.cron`) purges expired ledger rows daily. + +## Consequences + +- Positive: revocation works — logout or compromise clears the server-side + ledger entry and the refresh token is instantly unusable. +- Positive: short access-token TTL limits the blast radius of a leaked Bearer. +- Positive: the refresh flow is invisible to users; no more mid-session + logouts. +- Negative: one extra DB round-trip per refresh. Mitigated by the short-lived + access token and the fact that the ledger is indexed on `jti` + `user_id`. +- Follow-up: move ledger cleanup from a time-based cron to an event-based + cleanup if the table ever grows past a few hundred thousand rows. + +## Alternatives considered + +- **Opaque session tokens with a Redis store.** Rejected — adds an operational + dependency (Redis) that the rest of the stack does not yet require, and + complicates horizontal scaling. +- **Single JWT with short TTL + silent re-login.** Rejected — requires the + client to store credentials or an SSO cookie, neither of which we want in + `localStorage`. diff --git a/docs/adr/0003-paginated-response-envelope.md b/docs/adr/0003-paginated-response-envelope.md new file mode 100644 index 00000000..17e256bc --- /dev/null +++ b/docs/adr/0003-paginated-response-envelope.md @@ -0,0 +1,60 @@ +# ADR 0003: Canonical paginated response envelope + +- **Status:** Accepted +- **Date:** 2026-04-09 +- **Deciders:** Platform team, Frontend team + +## Context + +Different Odoo controllers returned paginated data in at least three shapes: + +- `{ data: [...], total: N, page, limit }` +- `{ results: [...], count: N }` +- `[...]` (bare array, no totals) + +The frontend grew defensive code paths to handle all three, and every new +endpoint risked inventing a fourth shape. + +## Decision + +Every list endpoint MUST return the canonical envelope produced by +`encoach_api.controllers.base.paginated_envelope`: + +```json +{ + "items": [ … ], + "data": [ … ], + "total": 123, + "page": 1, + "size": 20 +} +``` + +- `items` is the canonical field name. New code reads from `items`. +- `data` mirrors `items` for backwards compatibility with older callers and + can be removed once every consumer migrates. +- `total` is the total number of matching records across all pages. +- `page` is 1-indexed. +- `size` is the requested page size (capped server-side). + +On the frontend, `PaginatedResponse` in `frontend/src/types/common.ts` +exposes both `items` and an optional `data`, and service methods +(`users.service.ts`, `lms.service.ts`, etc.) construct a clean +`PaginatedResponse` object from whatever the server returns so that UI code +never sees the legacy fields. + +## Consequences + +- Positive: frontend code is simpler and typesafe — read `items`, done. +- Positive: OpenAPI spec advertises one consistent shape across endpoints. +- Negative: one additional key (`data`) is duplicated in responses. Cheap + (same reference, no JSON bloat) and easy to delete later. + +## Alternatives considered + +- **Follow JSON:API's `{ data, meta, links }` convention.** Rejected as + overkill: we don't use HATEOAS, and the extra indirection would force a + rewrite of every existing consumer. +- **Return a bare array + pagination headers.** Rejected because Odoo's + controller helpers make setting custom headers awkward and because it hides + totals from curl/Postman users. diff --git a/docs/adr/0004-rag-metadata-and-chunking.md b/docs/adr/0004-rag-metadata-and-chunking.md new file mode 100644 index 00000000..c12ff811 --- /dev/null +++ b/docs/adr/0004-rag-metadata-and-chunking.md @@ -0,0 +1,67 @@ +# ADR 0004: RAG metadata + chunking for vector store + +- **Status:** Accepted +- **Date:** 2026-04-09 +- **Deciders:** AI team, Platform team + +## Context + +The first cut of the vector store (`encoach_vector`) stored one embedding +per source record, keyed only by `(model, res_id)`. This had two problems: + +1. **Long documents dominated similarity scores.** A 20 000-character lesson + would embed as one vector and out-vote shorter, more relevant passages. +2. **No tenancy filtering.** Retrieval could not be scoped to a particular + course, subject, entity, or taxonomy topic, which meant RAG pulled content + from unrelated tenants on multi-entity deployments. + +The quality gate (`encoach_quality_gate`) also needed a way to deduplicate +re-ingested content so that re-running the indexer did not explode the table. + +## Decision + +Extend `encoach.vector.embedding` with RAG metadata columns: + +| Field | Purpose | +|-------|---------| +| `course_id` | Scope to a specific course. | +| `subject_id` | Scope to a subject/domain. | +| `entity_id` | Tenancy filter — critical for institutional deployments. | +| `taxonomy` | Free-form tag (e.g. `"IELTS/writing/task1"`). | +| `content_hash` | SHA-256 of the raw chunk; used for dedup. | +| `chunk_index`, `chunk_total` | Position in the parent document. | + +Chunking policy (see `encoach_vector.services.embedding_service`): + +- Content ≤ 2 000 chars → embedded as a single chunk. +- Content > 2 000 chars → split on paragraph boundaries with ~200-char + overlap, each chunk embedded individually. +- Each chunk stores its `content_hash`; the uniqueness constraint is + `(model, res_id, chunk_index, content_hash)` so re-indexing is idempotent. + +The indexer (`encoach_vector.services.indexer`) declares per-model metadata +mapping (which field feeds `course_id`, which feeds `subject_id`, etc.) so +adding a new source model is a single config entry. + +`similarity_search` accepts any subset of the metadata as a filter and +applies it as a SQL `WHERE` clause before the vector distance computation. + +## Consequences + +- Positive: retrieval quality improves dramatically on long documents. +- Positive: multi-tenant deployments can scope RAG to a single entity. +- Positive: re-indexing is safe (idempotent) and cheap. +- Negative: the embedding table grows roughly linearly with document length. + Mitigated by the `content_hash` dedup and by keeping only the latest + revision per source record. +- Follow-up: expose a management action to purge embeddings for a retired + course or entity. + +## Alternatives considered + +- **Use an external vector DB (Pinecone, Weaviate).** Rejected — pgvector is + already in the Postgres image, keeping ops surface small. Can be revisited + if we outgrow it. +- **Chunk-per-sentence instead of paragraph.** Rejected — too many tiny + chunks, each losing context; paragraph-sized chunks strike a better + recall/precision balance for our domain. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 00000000..67119b61 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,22 @@ +# Architecture Decision Records + +This folder contains lightweight ADRs documenting significant architectural +decisions made on the EnCoach platform. Each ADR is numbered, dated, and +immutable once "Accepted" — if a decision is revisited, open a new ADR that +supersedes the old one instead of rewriting history. + +## Index + +| # | Title | Status | +|---|-------|--------| +| [0001](0001-canonical-directory-layout.md) | Canonical `backend/` and `frontend/` directory layout | Accepted | +| [0002](0002-jwt-refresh-token-flow.md) | JWT access + refresh tokens with revocation ledger | Accepted | +| [0003](0003-paginated-response-envelope.md) | Canonical paginated response envelope | Accepted | +| [0004](0004-rag-metadata-and-chunking.md) | RAG metadata + chunking for vector store | Accepted | + +## Writing a new ADR + +1. Copy [`0000-template.md`](0000-template.md) to the next number. +2. Fill in **Context**, **Decision**, **Consequences**. +3. Keep it short (1 page). Link to source files or PRs for detail. +4. Update the index above and open a PR. diff --git a/new_project/DEPRECATED.md b/new_project/DEPRECATED.md new file mode 100644 index 00000000..ffa97738 --- /dev/null +++ b/new_project/DEPRECATED.md @@ -0,0 +1,45 @@ +# `new_project/` — DEPRECATED + +**Status:** frozen, retained only for historical reference. +**Canonical tree:** `backend/custom_addons/` at the repository root. + +--- + +## Why this directory is deprecated + +Early in the project, `new_project/` was used as an experimental sandbox for +addons, enterprise modules and OpenEduCat integration. During the Phase 0 / +Phase 1 hardening (see `docs/PROJECT_SUMMARY.md` §21), the canonical backend +tree was unified under `backend/custom_addons/` and the root-level tooling was +updated accordingly: + +| Concern | Old location (deprecated) | Canonical location | +| --- | --- | --- | +| Custom addons | `new_project/custom_addons/` | `backend/custom_addons/` | +| Enterprise modules | `new_project/enterprise-19/` | `backend/enterprise-19/` (or external mount) | +| OpenEduCat | `new_project/openeducat_erp-19.0/` | `backend/openeducat_erp-19.0/` (or external mount) | +| Python requirements | `new_project/requirements.txt` | `requirements.txt` (repo root) | +| Odoo config | `new_project/odoo.conf` | `odoo-docker.conf` (repo root) | +| Seed script | `new_project/seed_demo_data.py` | `seed_demo_data.py` (repo root, gated by `ENCOACH_ALLOW_RAW_SEED=1`) | + +## Do NOT + +- **Do not** add new Python packages to `new_project/requirements.txt` — add + them to the root `requirements.txt` (it is the one `Dockerfile` now copies). +- **Do not** place new Odoo addons under `new_project/custom_addons/` — put + them under `backend/custom_addons//` and update `addons_path` in + `odoo-docker.conf` if needed. +- **Do not** point `docker-compose.yml`, `run.sh`, or CI pipelines at any + `new_project/` path. + +## Safe to delete? + +Yes, once the team confirms no external tooling still references the legacy +paths. Removal plan: + +1. Grep the whole repo for `new_project/` — ensure only historical docs remain. +2. Verify CI is green with the root-level `Dockerfile` + `docker-compose.yml`. +3. `git rm -r new_project/` in a dedicated PR and update + `docs/PROJECT_SUMMARY.md` to drop the “legacy layout” references. + +Until that PR lands, treat this folder as read-only history.