Refactor document translation and enhance E2E testing and CI/CD

.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI Pipeline
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    name: Run Tests and Coverage
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+        cache: "pip"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pytest-cov
+
+    - name: Prepare Environment Variables
+      run: cp .env.example .env
+
+    - name: Run Pytest with Coverage
+      run: |
+        pytest --cov=app tests/ --cov-report=xml --cov-report=term
+
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        slug: your-org/translator
+      # Optional: Don't fail the build if codecov upload fails
+      continue-on-error: true
+
+  docker-build:
+    name: Test Docker Build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Build Docker Image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        push: false
+        tags: translator-api:test
+        cache-from: type=gha
+        cache-to: type=gha,mode=max

Dockerfile

@@ -14,7 +14,8 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 # Install python dependencies
 COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements.txt
 
 # Stage 2: Production
 FROM python:3.14.4-slim

app/core/config.py

@@ -10,17 +10,17 @@ class Settings(BaseSettings):
     POSTGRES_DB: str
     POSTGRES_HOST: str
     POSTGRES_PORT: str
-    DUCKLING_URL: str = "http://translator_duckling:8000/parse"
+    DUCKLING_URL: str 
 
     # LLM Settings
-    LLM_PROVIDER: str = "gemini" # gemini or ollama
-    GEMINI_API_KEY: str | None = None
-    OLLAMA_BASE_URL: str = "http://localhost:11434"
-    LLM_MODEL_NAME: str = "gemini-1.5-flash"
+    LLM_PROVIDER: str 
+    GEMINI_API_KEY: str 
+    OLLAMA_BASE_URL: str 
+    LLM_MODEL_NAME: str 
 
     # Authentication
-    API_USERNAME: str = "admin"
-    API_PASSWORD: str = "changeme"
+    API_USERNAME: str
+    API_PASSWORD: str 
 
     COMPLEXITY_THRESHOLD: int = 50
 

app/db/session.py

@@ -44,5 +44,5 @@ async def init_db():
 
             logger.info("Database tables verified/created.")
     except Exception as e:
-        logger.error(f"Failed to connect to the database or create tables: {e}")
+        logger.exception(f"Failed to connect to the database or create tables: {e}")
         raise e

app/document_translation/controller.py

@@ -14,6 +14,10 @@
     DocumentNode,
     is_ast_compatible,
 )
+from app.pipeline.translation import translate_json_with_llm
+from app.pipeline.complexity import calculate_complexity_score
+from app.core.config import settings
+from app.brands.service import BrandService
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +26,70 @@ def __init__(self, db: AsyncSession):
         self.db = db
         self.text_ctl = TextTranslationController(db)
 
+    async def _fetch_and_parse_document(self, document_url: str) -> tuple[DocumentNode | None, str, str | None]:
+        try:
+            parsed_url = urlparse(document_url)
+            filename = os.path.basename(parsed_url.path) or "document.json"
+        except Exception:
+            filename = "document.json"
+
+        try:
+            doc_data = await DocumentService.download_json(document_url)
+        except Exception as e:
+            return None, filename, f"Failed to fetch document: {str(e)}"
+
+        try:
+            root_node = json_to_ast(doc_data)
+            doc_node = DocumentNode(root_node, "json")
+            return doc_node, filename, None
+        except Exception as e:
+            return None, filename, f"Failed to parse document to AST: {str(e)}"
+
+    async def _process_local_node(
+        self, node, text: str, source_lang: str, target_lang: str, 
+        brand_uuid: str | None, domain_name: str | None, filename: str
+    ):
+        seg_payload = TranslationRequest(
+            text=text,
+            source_lang=source_lang,
+            target_lang=target_lang,
+        )
+        try:
+            res = await self.text_ctl.translate_text(
+                payload=seg_payload,
+                brand_uuid=brand_uuid,
+                domain_name=domain_name,
+                filename=filename,
+                property_name=node.path,
+            )
+            if "error" in res:
+                logger.warning("Failed to translate segment '%s' in path %s: %s", text[:30], node.path, res["error"])
+                node.translated_value = text
+            else:
+                node.translated_value = res.get("translation", text)
+        except Exception:
+            logger.exception("Error translating segment '%s'", text[:30])
+            node.translated_value = text
+
+    async def _process_llm_batch(
+        self, llm_batch: dict, translatable_nodes: list, source_lang: str, target_lang: str, brand_context: dict
+    ):
+        if not llm_batch:
+            return
+
+        try:
+            translated_batch = await translate_json_with_llm(
+                llm_batch, source_lang, target_lang, brand_context=brand_context
+            )
+            for node in translatable_nodes:
+                if node.path in translated_batch:
+                    node.translated_value = translated_batch[node.path]
+        except Exception:
+            logger.exception("Failed to batch translate JSON with LLM")
+            for node in translatable_nodes:
+                if node.path in llm_batch:
+                    node.translated_value = node.value
+
     async def translate_document(
         self,
         payload: DocumentTranslationRequest,
@@ -39,51 +107,39 @@ async def translate_document(
         if not is_in_supported_languages(source_lang, target_lang):
             return {"error": f"Language pair {source_lang}->{target_lang} is not supported"}
 
-        try:
-            parsed_url = urlparse(document_url)
-            filename = os.path.basename(parsed_url.path) or "document.json"
-        except Exception:
-            filename = "document.json"
-
-        try:
-            doc_data = await DocumentService.download_json(document_url)
-        except Exception as e:
-            return {"error": f"Failed to fetch document: {str(e)}"}
-
-        try:
-            root_node = json_to_ast(doc_data)
-            doc_node = DocumentNode(root_node, "json")
-        except Exception as e:
-            return {"error": f"Failed to parse document to AST: {str(e)}"}
+        doc_node, filename, err = await self._fetch_and_parse_document(document_url)
+        if err:
+            return {"error": err}
 
         translatable_nodes = collect_translatable_nodes(doc_node)
 
+        brand_service = BrandService(self.db)
+        brand_context = await brand_service.get_brand_context(brand_uuid) if brand_uuid else {}
+        glossary = brand_context.get("glossary", {}) if brand_context else {}
+        keywords = brand_context.get("keywords", []) if brand_context else []
+
+        llm_batch: dict[str, str] = {}
         for node in translatable_nodes:
-            seg_payload = TranslationRequest(
-                text=node.value,
-                source_lang=source_lang,
-                target_lang=target_lang,
+            text = node.value
+            text_lower = text.lower()
+
+            # Flattened complexity checks to reduce nesting
+            requires_llm = (
+                any(term.lower() in text_lower for term in glossary.keys()) or
+                any(kw.lower() in text_lower for kw in keywords) or
+                await calculate_complexity_score(text, brand_context) >= settings.COMPLEXITY_THRESHOLD
             )
-            try:
-                res = await self.text_ctl.translate_text(
-                    payload=seg_payload,
-                    brand_uuid=brand_uuid,
-                    domain_name=domain_name,
-                    filename=filename,
-                    property_name=node.path,
+
+            if requires_llm:
+                llm_batch[node.path] = text
+            else:
+                await self._process_local_node(
+                    node, text, source_lang, target_lang, brand_uuid, domain_name, filename
                 )
-                if "error" in res:
-                    logger.warning("Failed to translate segment '%s' in path %s: %s", node.value[:30], node.path, res["error"])
-                    node.translated_value = node.value
-                else:
-                    node.translated_value = res.get("translation", node.value)
-            except Exception:
-                logger.exception("Error translating segment '%s'", node.value[:30])
-                node.translated_value = node.value
 
-        # Reconstitute the document from AST
-        translated_document = doc_node.to_dict()
+        await self._process_llm_batch(llm_batch, translatable_nodes, source_lang, target_lang, brand_context)
 
+        translated_document = doc_node.to_dict()
         translated_ast_root = json_to_ast(translated_document)
         translated_doc_node = DocumentNode(translated_ast_root, "json")