From e538a0d503efdf686a2578800e08cd23d9c7674d Mon Sep 17 00:00:00 2001 From: Rui Batista Date: Tue, 6 Oct 2026 23:35:03 +0100 Subject: [PATCH] Add Crowdin translation sync workflow Adopt the Crowdin l10n infrastructure from the nvdaes add-on template so interface strings and documentation are translated in the community nvdaaddons Crowdin project. The workflow uploads the generated .pot and an .xliff derived from readme.md as source files, then imports back translations that reach the minimum completion threshold (50% by default), committing them to addon/locale and addon/doc. Schedules are staggered across the add-on repositories so their Crowdin syncs do not collide. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/checkTranslation.py | 144 ++++++++++++++++ .github/scripts/crowdinSync.ps1 | 227 ++++++++++++++++++++++++++ .github/scripts/languageMappings.json | 15 ++ .github/scripts/setOutputs.py | 21 +++ .github/workflows/crowdinL10n.yml | 66 ++++++++ 5 files changed, 473 insertions(+) create mode 100644 .github/scripts/checkTranslation.py create mode 100644 .github/scripts/crowdinSync.ps1 create mode 100644 .github/scripts/languageMappings.json create mode 100644 .github/scripts/setOutputs.py create mode 100644 .github/workflows/crowdinL10n.yml diff --git a/.github/scripts/checkTranslation.py b/.github/scripts/checkTranslation.py new file mode 100644 index 0000000..9d03dc6 --- /dev/null +++ b/.github/scripts/checkTranslation.py @@ -0,0 +1,144 @@ +# Copyright (C) 2026 NV Access Limited, Abdel +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. + +import sys +import os +from crowdin_api import CrowdinClient + + +def findFileId(client: CrowdinClient, projectId: int, baseTarget: str, searchExt: str) -> int | None: + """ + Iterates through all project files (using pagination) to find the ID of the source file matching the target name and extension. + + :param client: The Crowdin API client instance. + :param projectId: The ID of the Crowdin project. + :param base_target: The base name of the file (e.g., 'myAddon). + :param search_ext: The extension to look for (e.g., '.pot'). + :return: The file ID if found, otherwise None. + """ + offset = 0 + limit = 100 + + while True: + resp = client.source_files.list_files( + projectId=projectId, + limit=limit, + offset=offset, + ) + + data = resp["data"] + for f in data: + path_crowdin = f["data"]["path"].lower() + # Check if the path ends with addon_id.pot or addon_id.xliff. + if path_crowdin.endswith(f"{baseTarget}{searchExt}"): + fileId = f["data"]["id"] + print(f"DEBUG: Match found: {path_crowdin} (ID: {fileId})") + return fileId + + if len(data) < limit: + break + + offset += limit + + return None + + +def getScoreFromApi(fileNameToSearch: str, langId: str) -> float: + """ + Retrieves the translation progress score for a specific language and file. + Handles pagination for both file listing and language status. + + :param fileNameToSearch: The local path or name of the file to check. + :param langId: The language code (e.g., 'fr' or 'pt_BR'). + :return: The translation ratio between 0.0 and 1.0. + """ + token = os.environ.get("crowdinAuthToken") + projectIdEnv = os.environ.get("CROWDIN_PROJECT_ID") + + if not token or not projectIdEnv: + print("ERROR: Missing environment variables 'crowdinAuthToken' or 'CROWDIN_PROJECT_ID'.") + return 0.0 + + client = CrowdinClient(token=token) + projectId = int(projectIdEnv) + + try: + # Clean and prepare search patterns. + # Example: 'addon/locale/fr/LC_MESSAGES/myAddon.po' -> base_target: 'myAddon'. + baseTarget = fileNameToSearch.replace("\\", "/").split("/")[-1].rsplit(".", 1)[0].lower() + extTarget = fileNameToSearch.split(".")[-1].lower() + + # On Crowdin, the source for a .po file is usually a .pot file. + searchExt = ".pot" if extTarget == "po" else f".{extTarget}" + + print(f"DEBUG: Searching for source file: {baseTarget}{searchExt}") + + fileId = findFileId(client, projectId, baseTarget, searchExt) + + if fileId is None: + print(f"WARNING: File '{baseTarget}{searchExt}' not found on Crowdin.") + return 0.0 + + # Pagination for translation status (Progress). + offset = 0 + limit = 100 + + while True: + resp = client.translation_status.get_file_progress( + projectId=projectId, + fileId=fileId, + limit=limit, + offset=offset, + ) + + data = resp["data"] + for item in data: + langApi = item["data"]["languageId"] + + # Flexible matching (e.g., 'fr' will match 'fr' or 'fr-FR' from API). + # Also handles underscore to dash conversion for Crowdin compatibility + if langApi.lower().startswith(langId.lower().replace("_", "-")): + progress = float(item["data"]["translationProgress"]) + return progress + + # Check pagination total. + total = resp["pagination"]["totalCount"] + if offset + limit >= total: + break + offset += limit + + print(f"DEBUG: Language '{langId}' not found in progress list for this file.") + return 0.0 + + except Exception as e: + print(f"API ERROR: {e}") + return 0.0 + + +def main(): + if len(sys.argv) < 3: + print("Usage: python checkTranslation.py ") + sys.exit(2) + + input_file = sys.argv[1] + lang = sys.argv[2] + + score = getScoreFromApi(input_file, lang) + + # Output formatted for capture by the PowerShell script. + print(f"translationRatio={score}") + + # Identify extension to provide a specific score label. + ext = input_file.lower().split(".")[-1] + if ext == "md": + print(f"mdScore={score}") + elif ext == "xliff": + print(f"xliffScore={score}") + else: + # Default to poScore for .po and other localization files. + print(f"poScore={score}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/crowdinSync.ps1 b/.github/scripts/crowdinSync.ps1 new file mode 100644 index 0000000..8e7bb92 --- /dev/null +++ b/.github/scripts/crowdinSync.ps1 @@ -0,0 +1,227 @@ +#!/usr/bin/env pwsh +$ErrorActionPreference = 'Stop' + +# Git configuration for automated commits +git config user.name "github-actions[bot]" +git config user.email "github-actions[bot]@users.noreply.github.com" + +$rawAddonId = $env:ADDON_ID +if ([string]::IsNullOrWhiteSpace($rawAddonId)) { + Write-Error "Failed to get addon ID." + exit 1 +} +$addonId = $rawAddonId.Trim() + +# --- STEP 1: PREPARATION AND SOURCE UPDATE --- + +$xliffFile = "./$addonId.xliff" +$mdFile = "./readme.md" + +if (Test-Path $mdFile) { + if (Test-Path $xliffFile) { + $tempXliff = [System.IO.Path]::GetTempFileName() + try { + Copy-Item "$addonId.xliff" $tempXliff -Force + Write-Host "DEBUG: Updating XLIFF source based on readme.md..." + ./l10nUtil.exe md2xliff $mdFile $xliffFile -o $tempXliff + } + finally { + if (Test-Path $tempXliff) { + Remove-Item $tempXliff -Force + } + } + } + else { + Write-Host "DEBUG: XLIFF template not found. Creating new one from readme.md..." + ./l10nUtil.exe md2xliff $mdFile $xliffFile + } +} + +# Update POT file (addon interface) +uv run scons pot +$potFile = "$addonId.pot" + +# --- STEP 2: UPLOAD SOURCES TO CROWDIN --- + +if (Test-Path $potFile) { + Write-Host "DEBUG: Uploading updated POT source to Crowdin..." + ./l10nUtil.exe uploadSourceFile "$potFile" -c $env:L10N_UTIL_CONFIG +} + +if (Test-Path $xliffFile) { + Write-Host "DEBUG: Uploading updated XLIFF source to Crowdin..." + ./l10nUtil.exe uploadSourceFile "$xliffFile" -c $env:L10N_UTIL_CONFIG + + git add "$xliffFile" + git diff --staged --quiet + + if ($LASTEXITCODE -ne 0) { + git commit -m "Update $xliffFile for $addonId" + } +} + +# --- STEP 3: EXPORT AND PROCESS TRANSLATIONS --- + +Write-Host "DEBUG: Exporting translations from Crowdin..." +./l10nUtil.exe exportTranslations -o _addonL10n -c $env:L10N_UTIL_CONFIG + +# Ensure base directories exist +New-Item -ItemType Directory -Force -Path addon/locale | Out-Null +New-Item -ItemType Directory -Force -Path addon/doc | Out-Null + +# Load language mappings for Crowdin API calls +$languageMappings = Get-Content -Raw ".github/scripts/languageMappings.json" | ConvertFrom-Json + +foreach ($dir in Get-ChildItem -Path "_addonL10n/$addonId" -Directory) { + + $langCode = $dir.Name + + if ($langCode -eq "en") { + continue + } + + # --- Identify codes + + $crowdinLang = $null + + if ($languageMappings.PSObject.Properties.Name -contains $langCode) { + $crowdinLang = $languageMappings."$langCode" + } + + if (-not $crowdinLang) { + $crowdinLang = $langCode.Replace('_', '-') + } + + Write-Host "--- Processing Language: $langCode (Crowdin: $crowdinLang) ---" -ForegroundColor Cyan + + # Paths + + $remoteXliff = Join-Path $dir.FullName "$addonId.xliff" + $remotePo = Join-Path $dir.FullName "$addonId.po" + + $localMdDir = "addon/doc/$langCode" + $localMd = "$localMdDir/readme.md" + + $localPoPath = "addon/locale/$langCode/LC_MESSAGES/nvda.po" + + # --- 3.1 PO FILE PROCESSING --- + $poImported = $false + $scorePo = 0.0 + $threshold = $env:MIN_PERCENTAGE_TRANSLATED + + if (Test-Path $remotePo) { + + Write-Host "DEBUG: Evaluating Remote PO score..." + + $res = uv run python .github/scripts/checkTranslation.py "$addonId.po" $crowdinLang + + $scorePo = [double]( + ($res | Select-String "poScore=").ToString().Split("=")[1] + ) + + Write-Host "DEBUG: PO Score -> $scorePo" + + if ($scorePo -ge $threshold) { + + Write-Host "SUCCESS: Remote PO is above threshold. Importing to $localPoPath" + + New-Item -ItemType Directory -Force -Path (Split-Path $localPoPath) | Out-Null + + Move-Item $remotePo $localPoPath -Force + + $poImported = $true + } + else { + + Write-Host "WARNING: Remote PO score is below threshold ($threshold)." + } + } + + if (-not $poImported -and (Test-Path $localPoPath)) { + + Write-Host "ACTION: Uploading local legacy PO to Crowdin ($crowdinLang) as fallback." + + ./l10nUtil.exe uploadTranslationFile $crowdinLang "$addonId.po" $localPoPath -c $env:L10N_UTIL_CONFIG + } + + # --- 3.2 DOCUMENTATION PROCESSING (XLIFF ONLY) --- + + $scoreXliff = 0.0 + + if (Test-Path $remoteXliff) { + + Write-Host "DEBUG: Evaluating Remote XLIFF score..." + + $res = uv run python .github/scripts/checkTranslation.py "$addonId.xliff" $crowdinLang + + $scoreXliff = [double]( + ($res | Select-String "xliffScore=").ToString().Split("=")[1] + ) + } + else { + Write-Host "DEBUG: No remote XLIFF file found for this language." + } + + Write-Host "DEBUG: XLIFF Score -> $scoreXliff" + + $threshold = $env:MIN_PERCENTAGE_TRANSLATED + $docImported = $false + + if ($scoreXliff -ge $threshold) { + + if (!(Test-Path $localMdDir)) { + New-Item -ItemType Directory -Force -Path $localMdDir | Out-Null + } + + Write-Host "SUCCESS: Importing documentation from XLIFF ($langCode)..." + + ./l10nUtil.exe xliff2md $remoteXliff $localMd + + $docImported = $true + } + else { + + Write-Host "WARNING: Remote XLIFF score is below threshold ($threshold)." + } + + # No Markdown fallback upload anymore. + # XLIFF is now the single translation source in Crowdin. +} + +# --- STEP 4: COMMIT UPDATED TRANSLATIONS --- + +git add addon/locale addon/doc + +git diff --staged --quiet + +if ($LASTEXITCODE -ne 0) { + + git commit -m "Update translations for $addonId from Crowdin (Automatic Sync)" + + Write-Host "SUCCESS: Translations committed." +} +else { + + Write-Host "DEBUG: No changes in translations to commit." +} + +# Push all generated commits after successful Crowdin synchronization + +$pushOutput = git push 2>&1 + +$repository = $env:GITHUB_REPOSITORY + +Write-Host $pushOutput + +if ($LASTEXITCODE -ne 0) { + + Write-Host "ERROR: Failed to push commits to $repository." +} +elseif ($pushOutput -match "Everything up-to-date") { + + Write-Host "INFO: No new commits needed to be pushed." +} +else { + + Write-Host "SUCCESS: New commits successfully pushed to $repository." +} diff --git a/.github/scripts/languageMappings.json b/.github/scripts/languageMappings.json new file mode 100644 index 0000000..42b51d7 --- /dev/null +++ b/.github/scripts/languageMappings.json @@ -0,0 +1,15 @@ +{ + "af_ZA": "af", + "de_CH": "de-CH", + "es": "es-ES", + "es_CO": "es-CO", + "nb_NO": "nb", + "ne": "ne-NP", + "nn_NO": "nn-NO", + "pt_PT": "pt-PT", + "pt_BR": "pt-BR", + "sr": "sr-CS", + "zh_CN": "zh-CN", + "zh_HK": "zh-HK", + "zh_TW": "zh-TW" +} diff --git a/.github/scripts/setOutputs.py b/.github/scripts/setOutputs.py new file mode 100644 index 0000000..81385ae --- /dev/null +++ b/.github/scripts/setOutputs.py @@ -0,0 +1,21 @@ +# Copyright (C) 2025-2026 NV Access Limited, Noelia Ruiz Martínez +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. + +import os +import sys + +sys.path.insert(0, os.getcwd()) +import buildVars + + +def main(): + addonId = buildVars.addon_info["addon_name"] + name = "addonId" + value = addonId + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + _ = f.write(f"{name}={value}\n") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/crowdinL10n.yml b/.github/workflows/crowdinL10n.yml new file mode 100644 index 0000000..0f15ba6 --- /dev/null +++ b/.github/workflows/crowdinL10n.yml @@ -0,0 +1,66 @@ +name: Crowdin l10n + +on: + workflow_dispatch: + schedule: + # Every Monday at 03:00 UTC. + # If you add this workflow to several add-ons, give each repo a different + # day/hour so their Crowdin syncs do not collide. + - cron: '0 3 * * 1' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + crowdinAuthToken: ${{ secrets.CROWDIN_TOKEN }} + GH_TOKEN: ${{ github.token }} + CROWDIN_PROJECT_ID: ${{ vars.CROWDIN_PROJECT_ID || 780748 }} + L10N_UTIL_CONFIG: ${{ vars.L10N_UTIL_CONFIG || 'addon' }} + MIN_PERCENTAGE_TRANSLATED: ${{ vars.MIN_PERCENTAGE_TRANSLATED || 50 }} + +jobs: + crowdinSync: + runs-on: windows-latest + permissions: + contents: write + + steps: + - name: Random startup delay (0-5 minutes) + if: github.event_name == 'schedule' + shell: pwsh + run: | + $delaySeconds = Get-Random -Minimum 0 -Maximum 301 + Write-Host "Sleeping for $delaySeconds seconds..." + Start-Sleep -Seconds $delaySeconds + - name: Checkout add-on + uses: actions/checkout@v7 + with: + submodules: true + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version-file: ".python-version" + - name: Install uv + uses: astral-sh/setup-uv@v7 + - name: Install dependencies + run: uv sync + - name: Install gettext + run: | + choco install -y gettext + # Add gettext to PATH for current and future steps + $gettextPath = "C:\Program Files\gettext-iconv\bin" + echo "$gettextPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + - name: Get add-on info + id: getAddonInfo + shell: pwsh + run: uv run ./.github/scripts/setOutputs.py + - name: Download l10nUtil + run: | + gh release download --repo nvaccess/nvdaL10n --pattern "l10nUtil.exe" + - name: Download translations from Crowdin + if: ${{ env.crowdinAuthToken }} + shell: pwsh + env: + ADDON_ID: ${{ steps.getAddonInfo.outputs.addonId }} + run: ./.github/scripts/crowdinSync.ps1