feat: initial backend codebase — EnCoach v3
Complete Odoo 19 backend with 25 custom addons: - encoach_core: user/entity/role management - encoach_api: REST API + JWT auth - encoach_ai: OpenAI integration, AI settings, generation - encoach_ai_course: AI-powered English & IELTS course generation - encoach_exam_template/session: exam creation, structures, sessions - encoach_scoring: AI auto-grading + manual approval - encoach_vector: pgvector RAG integration - encoach_adaptive: adaptive learning engine - encoach_placement: placement testing - encoach_taxonomy/resources: content taxonomy & resource management - Plus 14 more modules for courses, branding, portal, etc. Includes docs: user guide, generation report, developer workflow. Made-with: Cursor
This commit is contained in:
Yamen Ahmad
2
custom_addons/encoach_vector/services/__init__.py
Normal file
2
custom_addons/encoach_vector/services/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
from . import embedding_service
|
||||
from . import indexer
|
||||
139
custom_addons/encoach_vector/services/embedding_service.py
Normal file
139
custom_addons/encoach_vector/services/embedding_service.py
Normal file
@@ -0,0 +1,139 @@
|
||||
"""Embedding service — encode text and manage vector storage."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
_model_instance = None
|
||||
|
||||
|
||||
def _get_model():
|
||||
"""Lazy-load the sentence-transformers model (cached across calls)."""
|
||||
global _model_instance
|
||||
if _model_instance is None:
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
_model_instance = SentenceTransformer('all-MiniLM-L6-v2')
|
||||
_logger.info("Loaded sentence-transformers model: all-MiniLM-L6-v2")
|
||||
except ImportError:
|
||||
_logger.error(
|
||||
"sentence-transformers not installed. "
|
||||
"Run: pip install sentence-transformers"
|
||||
)
|
||||
raise
|
||||
return _model_instance
|
||||
|
||||
|
||||
class EmbeddingService:
|
||||
"""Encode texts, upsert embeddings, and perform semantic search."""
|
||||
|
||||
def __init__(self, env):
|
||||
self.env = env
|
||||
self.Embedding = env['encoach.embedding'].sudo()
|
||||
|
||||
def encode(self, texts):
|
||||
"""Batch-encode texts to vectors.
|
||||
|
||||
Args:
|
||||
texts: list of strings
|
||||
|
||||
Returns:
|
||||
list of float lists (each 384-dim)
|
||||
"""
|
||||
model = _get_model()
|
||||
embeddings = model.encode(texts, normalize_embeddings=True, show_progress_bar=False)
|
||||
return [e.tolist() for e in embeddings]
|
||||
|
||||
def upsert(self, content_type, content_id, text, metadata=None):
|
||||
"""Encode and store (or update) a single embedding.
|
||||
|
||||
Returns:
|
||||
encoach.embedding record
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return None
|
||||
|
||||
existing = self.Embedding.search([
|
||||
('content_type', '=', content_type),
|
||||
('content_id', '=', content_id),
|
||||
], limit=1)
|
||||
|
||||
vectors = self.encode([text])
|
||||
meta_str = json.dumps(metadata or {})
|
||||
|
||||
if existing:
|
||||
existing.write({
|
||||
'content_text': text[:10000],
|
||||
'metadata_json': meta_str,
|
||||
})
|
||||
existing.set_embedding(vectors[0])
|
||||
return existing
|
||||
|
||||
record = self.Embedding.create({
|
||||
'content_type': content_type,
|
||||
'content_id': content_id,
|
||||
'content_text': text[:10000],
|
||||
'metadata_json': meta_str,
|
||||
})
|
||||
record.set_embedding(vectors[0])
|
||||
return record
|
||||
|
||||
def search(self, query, *, content_type=None, limit=10):
|
||||
"""Semantic search — encode query and find similar content.
|
||||
|
||||
Returns:
|
||||
list of dicts with text, metadata, similarity score
|
||||
"""
|
||||
if not query or not query.strip():
|
||||
return []
|
||||
|
||||
t0 = time.time()
|
||||
vectors = self.encode([query])
|
||||
results = self.Embedding.similarity_search(
|
||||
vectors[0],
|
||||
content_type=content_type,
|
||||
limit=limit,
|
||||
)
|
||||
latency = int((time.time() - t0) * 1000)
|
||||
_logger.info("Vector search for '%s' returned %d results in %dms",
|
||||
query[:80], len(results), latency)
|
||||
return results
|
||||
|
||||
def bulk_index(self, content_type, records_data):
|
||||
"""Batch-index multiple records.
|
||||
|
||||
Args:
|
||||
content_type: embedding content type
|
||||
records_data: list of dicts with keys: id, text, metadata
|
||||
"""
|
||||
if not records_data:
|
||||
return 0
|
||||
|
||||
texts = [r['text'] for r in records_data if r.get('text')]
|
||||
if not texts:
|
||||
return 0
|
||||
|
||||
vectors = self.encode(texts)
|
||||
|
||||
indexed = 0
|
||||
text_idx = 0
|
||||
for r in records_data:
|
||||
if not r.get('text'):
|
||||
continue
|
||||
self.upsert(content_type, r['id'], r['text'], r.get('metadata'))
|
||||
text_idx += 1
|
||||
indexed += 1
|
||||
|
||||
_logger.info("Bulk-indexed %d %s records", indexed, content_type)
|
||||
return indexed
|
||||
|
||||
def delete(self, content_type, content_id):
|
||||
"""Remove an embedding."""
|
||||
existing = self.Embedding.search([
|
||||
('content_type', '=', content_type),
|
||||
('content_id', '=', content_id),
|
||||
])
|
||||
if existing:
|
||||
existing.unlink()
|
||||
127
custom_addons/encoach_vector/services/indexer.py
Normal file
127
custom_addons/encoach_vector/services/indexer.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Indexer — batch-indexes existing Odoo records into the vector store."""
|
||||
|
||||
import logging
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_CONFIG = [
|
||||
{
|
||||
'model': 'op.course',
|
||||
'content_type': 'course',
|
||||
'text_field': 'name',
|
||||
'description_field': 'description',
|
||||
'metadata_fields': [],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.resource',
|
||||
'content_type': 'resource',
|
||||
'text_field': 'name',
|
||||
'description_field': 'content',
|
||||
'metadata_fields': ['type', 'cefr_level', 'difficulty'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.question',
|
||||
'content_type': 'question',
|
||||
'text_field': 'name',
|
||||
'description_field': None,
|
||||
'metadata_fields': ['question_type', 'difficulty', 'skill'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.course.module',
|
||||
'content_type': 'module',
|
||||
'text_field': 'name',
|
||||
'description_field': 'description',
|
||||
'metadata_fields': ['skill'],
|
||||
},
|
||||
{
|
||||
'model': 'encoach.ai.generation.log',
|
||||
'content_type': 'generation_log',
|
||||
'text_field': 'brief',
|
||||
'description_field': 'generated_content',
|
||||
'metadata_fields': ['course_type', 'status'],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _get_text(record, config):
|
||||
"""Extract indexable text from a record."""
|
||||
parts = []
|
||||
text_field = config.get('text_field', 'name')
|
||||
if hasattr(record, text_field):
|
||||
val = getattr(record, text_field)
|
||||
if val:
|
||||
parts.append(str(val))
|
||||
|
||||
desc_field = config.get('description_field')
|
||||
if desc_field and hasattr(record, desc_field):
|
||||
val = getattr(record, desc_field)
|
||||
if val:
|
||||
parts.append(str(val)[:2000])
|
||||
|
||||
return ' '.join(parts).strip()
|
||||
|
||||
|
||||
def _get_metadata(record, config):
|
||||
"""Extract metadata dict from a record."""
|
||||
meta = {}
|
||||
for f in config.get('metadata_fields', []):
|
||||
if hasattr(record, f):
|
||||
val = getattr(record, f)
|
||||
if val:
|
||||
meta[f] = str(val) if not isinstance(val, (int, float, bool)) else val
|
||||
return meta
|
||||
|
||||
|
||||
def index_model(env, config, batch_size=100):
|
||||
"""Index all records of a single model."""
|
||||
model_name = config['model']
|
||||
Model = env.get(model_name)
|
||||
if Model is None:
|
||||
_logger.warning("Model %s not found, skipping", model_name)
|
||||
return 0
|
||||
|
||||
Model = Model.sudo()
|
||||
|
||||
from .embedding_service import EmbeddingService
|
||||
svc = EmbeddingService(env)
|
||||
|
||||
total = Model.search_count([])
|
||||
indexed = 0
|
||||
offset = 0
|
||||
|
||||
while offset < total:
|
||||
records = Model.search([], limit=batch_size, offset=offset, order='id')
|
||||
batch_data = []
|
||||
for rec in records:
|
||||
text = _get_text(rec, config)
|
||||
if text:
|
||||
batch_data.append({
|
||||
'id': rec.id,
|
||||
'text': text,
|
||||
'metadata': _get_metadata(rec, config),
|
||||
})
|
||||
if batch_data:
|
||||
indexed += svc.bulk_index(config['content_type'], batch_data)
|
||||
offset += batch_size
|
||||
env.cr.commit()
|
||||
|
||||
_logger.info("Indexed %d/%d records for %s", indexed, total, model_name)
|
||||
return indexed
|
||||
|
||||
|
||||
def index_all(env, batch_size=100):
|
||||
"""Index all configured models."""
|
||||
total = 0
|
||||
for config in MODEL_CONFIG:
|
||||
try:
|
||||
total += index_model(env, config, batch_size)
|
||||
except Exception:
|
||||
_logger.exception("Failed to index %s", config['model'])
|
||||
_logger.info("Total records indexed: %d", total)
|
||||
return total
|
||||
|
||||
|
||||
def cron_reindex(env):
|
||||
"""Cron entry point for periodic re-indexing."""
|
||||
_logger.info("Starting scheduled vector re-index")
|
||||
return index_all(env)
|
||||
Reference in New Issue
Block a user