feat: Generation Page AI workflows + AI/Vector modules + exam session fixes
Generation Page (complete rebuild): - Full production-parity exam generation wizard with 4 IELTS modules - Reading: AI passage gen, 5 exercise types (MCQ, Fill, Write, T/F, Match) - Listening: 4 section types, AI context gen, TTS audio gen (ElevenLabs) - Writing: Task 1/2, AI instruction gen, word limits, marks - Speaking: 3 parts, AI script gen, avatar video gen (7 avatars) - Per-module config: timer, CEFR difficulty, access, approval, rubrics - Exam submission workflow (draft/published) Exam Structures: - New encoach.exam.structure model + CRUD controller - ExamStructuresPage wired to real API AI Module (encoach_ai): - OpenAI service, ElevenLabs TTS, AWS Polly, ELAI avatars - AI settings model with Odoo config parameters - 7 generation endpoints (passage, exercises, instructions, scripts, context) Vector Module (encoach_vector): - pgvector integration for RAG-based content search - Embedding service with sentence-transformers Exam Session Fixes: - Fixed ExamSession.tsx field mapping (question_type→type, exam_title→title) - Fixed submit payload to include attempt_id and answers - Fixed normalizeType to handle null/undefined Tested: 12/12 API tests passed, browser-verified with real OpenAI calls Made-with: Cursor
This commit is contained in:
Yamen Ahmad
17
backend/custom_addons/encoach_vector/__init__.py
Normal file
17
backend/custom_addons/encoach_vector/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from . import models
|
||||
from . import services
|
||||
|
||||
|
||||
def _post_init_hook(env):
|
||||
"""Run initial vector indexing after module install."""
|
||||
import logging
|
||||
_logger = logging.getLogger(__name__)
|
||||
try:
|
||||
from .services.indexer import index_all
|
||||
count = index_all(env)
|
||||
_logger.info("Post-init vector indexing complete: %d records", count)
|
||||
except Exception:
|
||||
_logger.warning(
|
||||
"Post-init vector indexing skipped (sentence-transformers may not be installed)",
|
||||
exc_info=True,
|
||||
)
|
||||
20
backend/custom_addons/encoach_vector/__manifest__.py
Normal file
20
backend/custom_addons/encoach_vector/__manifest__.py
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
'name': 'EnCoach Vector Search',
|
||||
'version': '19.0.1.0',
|
||||
'category': 'Education',
|
||||
'summary': 'pgvector-based semantic search and embedding storage for AI-enhanced learning',
|
||||
'author': 'EnCoach',
|
||||
'license': 'LGPL-3',
|
||||
'depends': ['encoach_core', 'encoach_ai'],
|
||||
'data': [
|
||||
'security/ir.model.access.csv',
|
||||
'data/vector_defaults.xml',
|
||||
],
|
||||
'external_dependencies': {
|
||||
'python': ['pgvector', 'sentence_transformers'],
|
||||
},
|
||||
'installable': True,
|
||||
'application': False,
|
||||
'auto_install': False,
|
||||
'post_init_hook': '_post_init_hook',
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<odoo>
|
||||
<!-- Scheduled action: re-index vectors daily -->
|
||||
<record id="ir_cron_vector_reindex" model="ir.cron">
|
||||
<field name="name">EnCoach: Vector Re-Index</field>
|
||||
<field name="model_id" ref="model_encoach_embedding"/>
|
||||
<field name="state">code</field>
|
||||
<field name="code">model.cron_reindex()</field>
|
||||
<field name="interval_number">1</field>
|
||||
<field name="interval_type">days</field>
|
||||
<field name="active">True</field>
|
||||
</record>
|
||||
</odoo>
|
||||
1
backend/custom_addons/encoach_vector/models/__init__.py
Normal file
1
backend/custom_addons/encoach_vector/models/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from . import embedding
|
||||
121
backend/custom_addons/encoach_vector/models/embedding.py
Normal file
121
backend/custom_addons/encoach_vector/models/embedding.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""Odoo model for storing vector embeddings via pgvector."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from odoo import api, models, fields
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
VECTOR_DIM = 384 # all-MiniLM-L6-v2 output dimension
|
||||
|
||||
|
||||
class EncoachEmbedding(models.Model):
|
||||
_name = 'encoach.embedding'
|
||||
_description = 'Vector Embedding'
|
||||
_order = 'create_date desc'
|
||||
|
||||
content_type = fields.Selection([
|
||||
('course', 'Course'),
|
||||
('resource', 'Resource'),
|
||||
('question', 'Question'),
|
||||
('module', 'Module'),
|
||||
('topic', 'Topic'),
|
||||
('feedback', 'Feedback'),
|
||||
('generation_log', 'Generation Log'),
|
||||
], required=True, index=True)
|
||||
content_id = fields.Integer(required=True, index=True)
|
||||
content_text = fields.Text()
|
||||
metadata_json = fields.Text(default='{}')
|
||||
|
||||
_content_unique = models.Constraint(
|
||||
'UNIQUE(content_type, content_id)',
|
||||
'Each content item can only have one embedding.',
|
||||
)
|
||||
|
||||
@api.model
|
||||
def _auto_init(self):
|
||||
res = super()._auto_init()
|
||||
cr = self.env.cr
|
||||
cr.execute("SELECT 1 FROM pg_extension WHERE extname = 'vector'")
|
||||
if not cr.fetchone():
|
||||
try:
|
||||
cr.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
_logger.info("pgvector extension created")
|
||||
except Exception:
|
||||
_logger.warning(
|
||||
"Could not create pgvector extension — run "
|
||||
"'CREATE EXTENSION vector' as a superuser",
|
||||
exc_info=True,
|
||||
)
|
||||
return res
|
||||
|
||||
cr.execute("""
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'encoach_embedding' AND column_name = 'embedding'
|
||||
""")
|
||||
if not cr.fetchone():
|
||||
cr.execute(
|
||||
f"ALTER TABLE encoach_embedding ADD COLUMN embedding vector({VECTOR_DIM})"
|
||||
)
|
||||
cr.execute(
|
||||
"CREATE INDEX IF NOT EXISTS encoach_embedding_vec_idx "
|
||||
"ON encoach_embedding USING ivfflat (embedding vector_cosine_ops) "
|
||||
"WITH (lists = 100)"
|
||||
)
|
||||
_logger.info("Vector column and index created on encoach_embedding")
|
||||
return res
|
||||
|
||||
def set_embedding(self, vector):
|
||||
"""Store a vector embedding for this record."""
|
||||
self.ensure_one()
|
||||
vec_str = '[' + ','.join(str(v) for v in vector) + ']'
|
||||
self.env.cr.execute(
|
||||
"UPDATE encoach_embedding SET embedding = %s WHERE id = %s",
|
||||
(vec_str, self.id),
|
||||
)
|
||||
|
||||
@api.model
|
||||
def cron_reindex(self):
|
||||
"""Cron entry point for periodic re-indexing."""
|
||||
from odoo.addons.encoach_vector.services.indexer import index_all
|
||||
return index_all(self.env)
|
||||
|
||||
@api.model
|
||||
def similarity_search(self, query_vector, *, content_type=None, limit=10):
|
||||
"""Find similar embeddings using cosine distance."""
|
||||
vec_str = '[' + ','.join(str(v) for v in query_vector) + ']'
|
||||
where = "WHERE embedding IS NOT NULL"
|
||||
params = [vec_str, limit]
|
||||
if content_type:
|
||||
where += " AND content_type = %s"
|
||||
params = [vec_str, content_type, limit]
|
||||
|
||||
query = f"""
|
||||
SELECT id, content_type, content_id, content_text, metadata_json,
|
||||
1 - (embedding <=> %s::vector) AS similarity
|
||||
FROM encoach_embedding
|
||||
{where}
|
||||
ORDER BY embedding <=> %s::vector
|
||||
LIMIT %s
|
||||
"""
|
||||
if content_type:
|
||||
self.env.cr.execute(query, (vec_str, content_type, vec_str, limit))
|
||||
else:
|
||||
self.env.cr.execute(query, (vec_str, vec_str, limit))
|
||||
|
||||
results = []
|
||||
for row in self.env.cr.dictfetchall():
|
||||
metadata = {}
|
||||
try:
|
||||
metadata = json.loads(row['metadata_json'] or '{}')
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
results.append({
|
||||
'id': row['id'],
|
||||
'content_type': row['content_type'],
|
||||
'content_id': row['content_id'],
|
||||
'text': row['content_text'],
|
||||
'metadata': metadata,
|
||||
'similarity': round(row['similarity'], 4),
|
||||
})
|
||||
return results
|
||||
@@ -0,0 +1,3 @@
|
||||
id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
|
||||
access_encoach_embedding_user,encoach.embedding.user,model_encoach_embedding,base.group_user,1,0,0,0
|
||||
access_encoach_embedding_admin,encoach.embedding.admin,model_encoach_embedding,base.group_system,1,1,1,1
|
||||
|
@@ -0,0 +1,2 @@
|
||||
from . import embedding_service
|
||||
from . import indexer
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Embedding service — encode text and manage vector storage."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
_model_instance = None
|
||||
|
||||
|
||||
def _get_model():
|
||||
"""Lazy-load the sentence-transformers model (cached across calls)."""
|
||||
global _model_instance
|
||||
if _model_instance is None:
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
_model_instance = SentenceTransformer('all-MiniLM-L6-v2')
|
||||
_logger.info("Loaded sentence-transformers model: all-MiniLM-L6-v2")
|
||||
except ImportError:
|
||||
_logger.error(
|
||||
"sentence-transformers not installed. "
|
||||
"Run: pip install sentence-transformers"
|
||||
)
|
||||
raise
|
||||
return _model_instance
|
||||
|
||||
|
||||
class EmbeddingService:
|
||||
"""Encode texts, upsert embeddings, and perform semantic search."""
|
||||
|
||||
def __init__(self, env):
|
||||
self.env = env
|
||||
self.Embedding = env['encoach.embedding'].sudo()
|
||||
|
||||
def encode(self, texts):
|
||||
"""Batch-encode texts to vectors.
|
||||
|
||||
Args:
|
||||
texts: list of strings
|
||||
|
||||
Returns:
|
||||
list of float lists (each 384-dim)
|
||||
"""
|
||||
model = _get_model()
|
||||
embeddings = model.encode(texts, normalize_embeddings=True, show_progress_bar=False)
|
||||
return [e.tolist() for e in embeddings]
|
||||
|
||||
def upsert(self, content_type, content_id, text, metadata=None):
|
||||
"""Encode and store (or update) a single embedding.
|
||||
|
||||
Returns:
|
||||
encoach.embedding record
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return None
|
||||
|
||||
existing = self.Embedding.search([
|
||||
('content_type', '=', content_type),
|
||||
('content_id', '=', content_id),
|
||||
], limit=1)
|
||||
|
||||
vectors = self.encode([text])
|
||||
meta_str = json.dumps(metadata or {})
|
||||
|
||||
if existing:
|
||||
existing.write({
|
||||
'content_text': text[:10000],
|
||||
'metadata_json': meta_str,
|
||||
})
|
||||
existing.set_embedding(vectors[0])
|
||||
return existing
|
||||
|
||||
record = self.Embedding.create({
|
||||
'content_type': content_type,
|
||||
'content_id': content_id,
|
||||
'content_text': text[:10000],
|
||||
'metadata_json': meta_str,
|
||||
})
|
||||
record.set_embedding(vectors[0])
|
||||
return record
|
||||
|
||||
def search(self, query, *, content_type=None, limit=10):
|
||||
"""Semantic search — encode query and find similar content.
|
||||
|
||||
Returns:
|
||||
list of dicts with text, metadata, similarity score
|
||||
"""
|
||||
if not query or not query.strip():
|
||||
return []
|
||||
|
||||
t0 = time.time()
|
||||
vectors = self.encode([query])
|
||||
results = self.Embedding.similarity_search(
|
||||
vectors[0],
|
||||
content_type=content_type,
|
||||
limit=limit,
|
||||
)
|
||||
latency = int((time.time() - t0) * 1000)
|
||||
_logger.info("Vector search for '%s' returned %d results in %dms",
|
||||
query[:80], len(results), latency)
|
||||
return results
|
||||
|
||||
def bulk_index(self, content_type, records_data):
|
||||
"""Batch-index multiple records.
|
||||
|
||||
Args:
|
||||
content_type: embedding content type
|
||||
records_data: list of dicts with keys: id, text, metadata
|
||||
"""
|
||||
if not records_data:
|
||||
return 0
|
||||
|
||||
texts = [r['text'] for r in records_data if r.get('text')]
|
||||
if not texts:
|
||||
return 0
|
||||
|
||||
vectors = self.encode(texts)
|
||||
|
||||
indexed = 0
|
||||
text_idx = 0
|
||||
for r in records_data:
|
||||
if not r.get('text'):
|
||||
continue
|
||||
self.upsert(content_type, r['id'], r['text'], r.get('metadata'))
|
||||
text_idx += 1
|
||||
indexed += 1
|
||||
|
||||
_logger.info("Bulk-indexed %d %s records", indexed, content_type)
|
||||
return indexed
|
||||
|
||||
def delete(self, content_type, content_id):
|
||||
"""Remove an embedding."""
|
||||
existing = self.Embedding.search([
|
||||
('content_type', '=', content_type),
|
||||
('content_id', '=', content_id),
|
||||
])
|
||||
if existing:
|
||||
existing.unlink()
|
||||
127
backend/custom_addons/encoach_vector/services/indexer.py
Normal file
127
backend/custom_addons/encoach_vector/services/indexer.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Indexer — batch-indexes existing Odoo records into the vector store."""
|
||||
|
||||
import logging
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_CONFIG = [
|
||||
{
|
||||
'model': 'op.course',
|
||||
'content_type': 'course',
|
||||
'text_field': 'name',
|
||||
'description_field': 'description',
|
||||
'metadata_fields': [],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.resource',
|
||||
'content_type': 'resource',
|
||||
'text_field': 'name',
|
||||
'description_field': 'content',
|
||||
'metadata_fields': ['type', 'cefr_level', 'difficulty'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.question',
|
||||
'content_type': 'question',
|
||||
'text_field': 'name',
|
||||
'description_field': None,
|
||||
'metadata_fields': ['question_type', 'difficulty', 'skill'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.course.module',
|
||||
'content_type': 'module',
|
||||
'text_field': 'name',
|
||||
'description_field': 'description',
|
||||
'metadata_fields': ['skill'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.ai.generation.log',
|
||||
'content_type': 'generation_log',
|
||||
'text_field': 'brief',
|
||||
'description_field': 'generated_content',
|
||||
'metadata_fields': ['course_type', 'status'],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _get_text(record, config):
|
||||
"""Extract indexable text from a record."""
|
||||
parts = []
|
||||
text_field = config.get('text_field', 'name')
|
||||
if hasattr(record, text_field):
|
||||
val = getattr(record, text_field)
|
||||
if val:
|
||||
parts.append(str(val))
|
||||
|
||||
desc_field = config.get('description_field')
|
||||
if desc_field and hasattr(record, desc_field):
|
||||
val = getattr(record, desc_field)
|
||||
if val:
|
||||
parts.append(str(val)[:2000])
|
||||
|
||||
return ' '.join(parts).strip()
|
||||
|
||||
|
||||
def _get_metadata(record, config):
|
||||
"""Extract metadata dict from a record."""
|
||||
meta = {}
|
||||
for f in config.get('metadata_fields', []):
|
||||
if hasattr(record, f):
|
||||
val = getattr(record, f)
|
||||
if val:
|
||||
meta[f] = str(val) if not isinstance(val, (int, float, bool)) else val
|
||||
return meta
|
||||
|
||||
|
||||
def index_model(env, config, batch_size=100):
|
||||
"""Index all records of a single model."""
|
||||
model_name = config['model']
|
||||
Model = env.get(model_name)
|
||||
if Model is None:
|
||||
_logger.warning("Model %s not found, skipping", model_name)
|
||||
return 0
|
||||
|
||||
Model = Model.sudo()
|
||||
|
||||
from .embedding_service import EmbeddingService
|
||||
svc = EmbeddingService(env)
|
||||
|
||||
total = Model.search_count([])
|
||||
indexed = 0
|
||||
offset = 0
|
||||
|
||||
while offset < total:
|
||||
records = Model.search([], limit=batch_size, offset=offset, order='id')
|
||||
batch_data = []
|
||||
for rec in records:
|
||||
text = _get_text(rec, config)
|
||||
if text:
|
||||
batch_data.append({
|
||||
'id': rec.id,
|
||||
'text': text,
|
||||
'metadata': _get_metadata(rec, config),
|
||||
})
|
||||
if batch_data:
|
||||
indexed += svc.bulk_index(config['content_type'], batch_data)
|
||||
offset += batch_size
|
||||
env.cr.commit()
|
||||
|
||||
_logger.info("Indexed %d/%d records for %s", indexed, total, model_name)
|
||||
return indexed
|
||||
|
||||
|
||||
def index_all(env, batch_size=100):
|
||||
"""Index all configured models."""
|
||||
total = 0
|
||||
for config in MODEL_CONFIG:
|
||||
try:
|
||||
total += index_model(env, config, batch_size)
|
||||
except Exception:
|
||||
_logger.exception("Failed to index %s", config['model'])
|
||||
_logger.info("Total records indexed: %d", total)
|
||||
return total
|
||||
|
||||
|
||||
def cron_reindex(env):
|
||||
"""Cron entry point for periodic re-indexing."""
|
||||
_logger.info("Starting scheduled vector re-index")
|
||||
return index_all(env)
|
||||
Reference in New Issue
Block a user