diff --git a/.github/workflows/testCode.yaml b/.github/workflows/testCode.yml similarity index 100% rename from .github/workflows/testCode.yaml rename to .github/workflows/testCode.yml diff --git a/.github/workflows/transformDataToViews.yml b/.github/workflows/transformDataToViews.yml index 8c9dc4be2aa..21931e406d3 100644 --- a/.github/workflows/transformDataToViews.yml +++ b/.github/workflows/transformDataToViews.yml @@ -10,7 +10,13 @@ concurrency: jobs: transformAndPush: - runs-on: windows-latest + runs-on: ubuntu-latest + defaults: + run: + shell: bash + permissions: + contents: write + pull-requests: write env: # this is a git '--pretty=format' string # %h is SHA, %n is newline, @@ -24,38 +30,72 @@ jobs: uses: actions/checkout@v5 with: submodules: true - - name: Checkout views branch into separate folder + - name: Checkout addonstore-views repository uses: actions/checkout@v5 with: + repository: nvaccess/addonstore-views path: views - ref: views + ref: main + token: ${{ secrets.VIEWS_PUSH_TOKEN }} + - name: Install system deps for lxml + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libxml2-dev libxslt1-dev zlib1g-dev - name: Install the latest version of uv uses: astral-sh/setup-uv@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: x86 - name: Install requirements and run transformation run: | + set -euo pipefail uv sync - # empty the views git folder directory - Try { Remove-Item views/views/ -Recurse -ErrorAction stop } - Catch [System.Management.Automation.ItemNotFoundException] { $null } - uv run --directory transform python -m src.transform --loglevel ${{ env.logLevel }} nvdaAPIVersions.json ../addons ../views/views + + # generate transformed data from this repository's addons metadata + rm -rf generated/ + uv run --directory transform python -m src.transform --loglevel "${{ env.logLevel }}" nvdaAPIVersions.json ../addons ../generated + + # replace generated data directories in addonstore-views + rm -rf views/addons/ views/views/ + cp -R generated/addons views/addons + cp -R generated/views views/views env: logLevel: ${{ runner.debug && 'DEBUG' || 'INFO' }} - name: Copy files run: | - copy ./transform/nvdaAPIVersions.json ./views/nvdaAPIVersions.json - copy ./transform/docs/output.md ./views/output.md - copy ./readme.md ./views/readme.md - - name: Commit and push + set -euo pipefail + cp ./transform/nvdaAPIVersions.json ./views/nvdaAPIVersions.json + cp ./transform/docs/output.md ./views/output.md + cp ./README.md ./views/README.md + - name: Create PR and enable auto-merge + env: + GH_TOKEN: ${{ secrets.VIEWS_PUSH_TOKEN }} run: | + set -euo pipefail git log HEAD --pretty=format:"${{ env.COMMIT_FORMAT }}" -1 > commitMsg.txt cd views git config user.name github-actions git config user.email github-actions@github.com git add . - git commit -F ../commitMsg.txt - git push + if git diff --staged --quiet; then + echo "No generated changes; skipping PR creation." + exit 0 + fi + + branchName="transformViews${{ github.run_id }}-${{ github.run_attempt }}" + git checkout -b "$branchName" + # TODO: Temporarily suppress output during first commit as testing logs are too big. + # Remove quiet and gpg sign flags later. + git commit --quiet --no-gpg-sign -F ../commitMsg.txt + git push --set-upstream origin "$branchName" + + prUrl=$(gh pr create \ + --repo nvaccess/addonstore-views \ + --base main \ + --head "$branchName" \ + --title "[Automated] Transform add-on data to views" \ + --body-file ../commitMsg.txt) + + gh pr merge --auto --squash --delete-branch "$prUrl" diff --git a/.python-version b/.python-version index ad929f8eae9..89c353b46a3 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -cpython-3.13-windows-x86_64-none +cpython-3.13 diff --git a/README.md b/README.md index 88513669b9c..53d516ff375 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Add-on Store -The addon-datastore repository is a data pipeline of submitting, validating and transforming add-on data to views. -These views are hosted on the NV Access server and are available in the NVDA Add-on Store. +The addon-datastore repository is a data pipeline for submitting and validating add-on data. +Transformed add-on store data is produced with add-on files and symlinked views for hosting on the NV Access server. Please note: the NVDA project including the Add-on Store has a [Citizen and Contributor Code of Conduct](https://github.com/nvaccess/nvda/blob/master/CODE_OF_CONDUCT.md). NV Access expects that all contributors and other community members will read and abide by the rules set out in this document while participating in the project or contributing add-ons. diff --git a/docs/design/designOverview.md b/docs/design/designOverview.md index 751457b61d1..ebbd4a18c56 100644 --- a/docs/design/designOverview.md +++ b/docs/design/designOverview.md @@ -63,27 +63,34 @@ Aims: - While this is technically not necessary, it provides a good separation from implementation. If we wished to change our storage mechanism, we would not be breaking old versions of NVDA. - ## API data generation details -Triggered by a new commit to the `master` branch, [a GitHub workflow](../../.github/workflows/transformDataToViews.yml), [addon-datastore-transform](https://github.com/nvaccess/addon-datastore-transform), transforms the data into the required views. +Triggered by a new commit to the `master` branch, [a GitHub workflow](../../.github/workflows/transformDataToViews.yml), and the [transform](../../transform/) module, transforms the data into add-on files and projected views. For each NVDA API version and channel, the add-on metadata with the highest version number is written. -These views are then committed by the GitHub Action to the [views branch](https://github.com/nvaccess/addon-datastore/tree/views). +This transformed data is then committed by the GitHub Action to the [addonstore-views](https://github.com/nvaccess/addonstore-views) main branch. ### Data views -The following views will only be available on the [views branch](https://github.com/nvaccess/addon-datastore/tree/views) and located in a `views` folder. -Required transformations of the data: -- `/NVDA API Version/addon-1-ID/stable.json` -- `/NVDA API Version/addon-1-ID/beta.json` -- `/NVDA API Version/addon-2-ID/stable.json` + +The generated data is stored in two top-level folders: + +- `addons`: add-on data files by add-on version and language. +- `views`: compatibility and latest projections as relative symlinks into `addons`. + +The following projected views are available in the `views` folder. + +The views folder is expected to have the following structure: + +`/views////.json` Notes: + - 'NVDA API Version' will be something like '2019.3', there will be one folder for each NVDA API Version. - The `beta.json` and `stable.json` contain the information necessary for a store entry. - The contents for each add-on will include all the technical details required for NVDA to download, verify file integrity, and install. -- The file will include translations (if available) for the displayable metadata. +- View files are relative symlinks to files in `addons`. +- Files use translations (if available) for displayable metadata. This simplifies the processing on the hosting (E.G NV Access) server. @@ -116,16 +123,14 @@ Channel can be: all, dev, stable or beta. - Example: ### `GET` cacheHash + Returns a hash used for cache breaking. This hash will change whenever new add-on data is available. -The hash should match the latest commit hash of the [views branch](https://github.com/nvaccess/addon-datastore/commits/views). +For testing purposes, the hash should match the latest commit hash of the transformed [addonstore-views](https://github.com/nvaccess/addonstore-views) branch. - - Example return value: `"5fcf12f"` -### Legacy -This endpoint mirrors the legacy [get.php, from the addonFiles repository](https://github.com/nvaccess/addonFiles/blob/master/get.php). - #### `addonslist` end-point The `addonslist` parameter generates a list of list of add-ons using the same logic as the [latest end-point](#get-latest) for all channels. diff --git a/pyproject.toml b/pyproject.toml index b821bf5dbc1..86e4bf62fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,6 @@ reportMissingTypeStubs = false [tool.uv] default-groups = "all" -environments = ["sys_platform == 'win32'"] required-version = ">=0.8" [tool.setuptools] diff --git a/transform/README.md b/transform/README.md index 2bda2961462..8dfe18e491c 100644 --- a/transform/README.md +++ b/transform/README.md @@ -1,10 +1,13 @@ # Transforming data to views -This repository primarily exists to transform data from [nvaccess/addon-datastore:master](https://github.com/nvaccess/addon-datastore) to views located at [nvaccess/addon-datastore:views](https://github.com/nvaccess/addon-datastore/tree/views). +This module transforms add-on metadata into an output data layout with add-on files and views. +The output is designed to be published from a single branch. For each NVDA version that needs to be supported by the add-on store, an entry must be added to [`nvdaAPIVersions.json`](./nvdaAPIVersions.json). This includes patch versions. +This module should be run from linux, as symlinks are created for the server component. + ## Overview For each version of NVDA, the meta-data of the most recent (the highest version number) of each add-on is automatically diff --git a/transform/docs/output.md b/transform/docs/output.md index 4d5624bdfa8..787cb353932 100644 --- a/transform/docs/output.md +++ b/transform/docs/output.md @@ -1,28 +1,55 @@ +# Output + The output for running the transformation is described as follows. -This is written to a given directory, and removes existing json data from the directory in the process. +This is written to a given directory that must be new/empty; the transformation creates this directory and fails if it already exists. +Callers are responsible for deleting any previous output directory before running the transformation. ## Output file structure + The following subdirectories and files are created: -- `/NVDA API Version/addon-1-ID/stable.json` -- `/NVDA API Version/addon-1-ID/beta.json` -- `/NVDA API Version/addon-2-ID/stable.json` -eg: `/2020.3.0/nvdaOCR/stable.json` + +Translated add-on files: + +- `/addons/addon-1-ID//en.json` +- `/addons/addon-1-ID//ar.json` (when a translation exists) +- `/addons/addon-2-ID//en.json` + +Symlink end views which point to translated files: + +- `/views/en//addon-1-ID/stable.json` +- `/views/en//addon-1-ID/beta.json` +- `/views/en//addon-2-ID/stable.json` +- `/views/ar/addon-1-ID/stable.json` + +Examples: + +- `/addons/nvdaOCR/1.3.5/en.json` +- `/views/en/2020.3.0/nvdaOCR/stable.json` Where `NVDA API Version` may be: + - `2022.1.0`: A major release. - `2022.1.3`: A patch release. -The system differentiates patch releases from major releases to cater to the (very unlikely) event of requireing a breaking change or introduction to the NVDA add-on API. +The system differentiates patch releases from major releases to cater to the (very unlikely) event of requiring a breaking change or introduction to the NVDA add-on API. ## Output file data -Each addon file is the addon data taken from input that is the latest compatible version, with the given requirements `(NVDA API Version, addon-ID, stable|beta|dev)`. -The transformed data file content will be the same as the input. -The contents for each addon file includes all the technical details required for NVDA to download, verify file integrity, and install. + +Add-on files in `/addons` contain transformed add-on metadata by add-on version and language. +Views in `/views` are relative symlinks to files in `/addons`. + +For each required view `(language, NVDA API Version, addon-ID, channel)`, the view symlink points at a single file: + +- Prefer exact language translation +- Otherwise prefer language without locale (`pt_BR` -> `pt`) +- Otherwise fallback to `en` + +Each transformed add-on file includes all technical details required for NVDA to download, verify file integrity, and install. It also contains the information necessary for a store entry. -Later, translated versions will become available. ## Output notes + This structure simplifies the processing on the hosting (e.g. NV Access) server. -To fetch the latest add-ons for ``, the server can concatenate the appropriate JSON files that match a glob: `//*/stable.json`. -Similarly, to fetch the latest version of an add-on with `` for ``. The server can return the data at `///stable.json`. +To fetch the latest add-ons for ``, the server can concatenate the appropriate JSON files that match a glob: `/views///*/stable.json`. +Similarly, to fetch the latest version of an add-on with `` for ``, the server can return the data at `/views////stable.json`. Using the NV Access server as the endpoint for this is important in case the implementation has to change or be migrated away from GitHub for some reason. diff --git a/transform/src/tests/test_datastructures.py b/transform/src/tests/test_datastructures.py index 5abdeef5de3..79f0cef844c 100644 --- a/transform/src/tests/test_datastructures.py +++ b/transform/src/tests/test_datastructures.py @@ -25,7 +25,7 @@ def test_toStr_patch_optional(self): """Confirm that versions as string always include the patch number, 0 by default. Even if the patch isn't specified, it should be included - so that the output is consistent - e.g. /views/2021.1.3/stable.json""" + so that the output is consistent - e.g. /views/en/2021.1.3/addonId/stable.json""" self.assertEqual(str(MajorMinorPatch(13, 2)), "13.2.0") def test_fromDict(self): diff --git a/transform/src/tests/test_viewsGenerationSystem.py b/transform/src/tests/test_viewsGenerationSystem.py index 945002b10a5..8bdde06d0ce 100644 --- a/transform/src/tests/test_viewsGenerationSystem.py +++ b/transform/src/tests/test_viewsGenerationSystem.py @@ -45,6 +45,7 @@ class InputAddonVersion: class ExpectedAddonVersion: path: str addonVersion: str + targetPath: str | None = None def addonJson(path: str, channel: str, *, required: str, tested: str) -> InputAddonVersion: @@ -103,7 +104,7 @@ def write_addons(*addons: InputAddonVersion): for addon in addons: addonWritePath = os.path.join(DATA_DIR.INPUT.value, addon.path) Path(os.path.dirname(addonWritePath)).mkdir(parents=True, exist_ok=True) - with open(addonWritePath, "w") as addonFile: + with open(addonWritePath, "w", encoding="utf-8") as addonFile: addonFile.write(addon.addonDataBlob) @@ -203,6 +204,13 @@ def _assertAddonDataWritten(self, *expectedAddons: ExpectedAddonVersion): for expectedAddon in expectedAddons: fullPathToAddon = os.path.join(DATA_DIR.OUTPUT.value, expectedAddon.path) self.assertTrue(Path(fullPathToAddon).exists()) + if expectedAddon.targetPath is not None: + targetPath = os.path.join(DATA_DIR.OUTPUT.value, expectedAddon.targetPath) + self.assertTrue(os.path.islink(fullPathToAddon)) + self.assertEqual( + os.path.normpath(os.path.realpath(fullPathToAddon)), + os.path.normpath(os.path.realpath(targetPath)), + ) with open(fullPathToAddon, "r") as expectedAddonFile: addonData = json.load(expectedAddonFile) addonVersion = MajorMinorPatch(**addonData["addonVersionNumber"]) @@ -224,12 +232,141 @@ def test_output_file_structure_matches_expected(self): ) self.runTransformation() self._assertAddonDataWritten( - ExpectedAddonVersion("en/2020.2.0/oldNewAddon/stable.json", "2.1.0"), - ExpectedAddonVersion("en/2020.3.0/oldNewAddon/stable.json", "13.0.0"), # overrides 2.1.0 - ExpectedAddonVersion("en/2020.4.0/oldNewAddon/stable.json", "13.0.0"), - ExpectedAddonVersion("en/2020.4.0/betaStableAddon/stable.json", "0.0.1"), - ExpectedAddonVersion("en/2020.4.0/betaStableAddon/beta.json", "0.0.2"), - ExpectedAddonVersion("en/latest/betaStableAddon/beta.json", "0.0.2"), - ExpectedAddonVersion("en/latest/betaStableAddon/stable.json", "0.0.1"), - ExpectedAddonVersion("en/latest/oldNewAddon/stable.json", "13.0.0"), + ExpectedAddonVersion("addons/oldNewAddon/2.1.0/en.json", "2.1.0"), + ExpectedAddonVersion("addons/oldNewAddon/13.0.0/en.json", "13.0.0"), + ExpectedAddonVersion("addons/betaStableAddon/0.0.1/en.json", "0.0.1"), + ExpectedAddonVersion("addons/betaStableAddon/0.0.2/en.json", "0.0.2"), + ExpectedAddonVersion( + "views/en/2020.2.0/oldNewAddon/stable.json", + "2.1.0", + targetPath="addons/oldNewAddon/2.1.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2020.3.0/oldNewAddon/stable.json", + "13.0.0", + targetPath="addons/oldNewAddon/13.0.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2020.4.0/oldNewAddon/stable.json", + "13.0.0", + targetPath="addons/oldNewAddon/13.0.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2020.4.0/betaStableAddon/stable.json", + "0.0.1", + targetPath="addons/betaStableAddon/0.0.1/en.json", + ), + ExpectedAddonVersion( + "views/en/2020.4.0/betaStableAddon/beta.json", + "0.0.2", + targetPath="addons/betaStableAddon/0.0.2/en.json", + ), + ExpectedAddonVersion( + "views/en/latest/betaStableAddon/beta.json", + "0.0.2", + targetPath="addons/betaStableAddon/0.0.2/en.json", + ), + ExpectedAddonVersion( + "views/en/latest/betaStableAddon/stable.json", + "0.0.1", + targetPath="addons/betaStableAddon/0.0.1/en.json", + ), + ExpectedAddonVersion( + "views/en/latest/oldNewAddon/stable.json", + "13.0.0", + targetPath="addons/oldNewAddon/13.0.0/en.json", + ), + ) + + def test_translation_view_symlink_points_to_translated_addon_data(self): + """Confirms language-specific views symlink to translated addon data files.""" + write_addons( + InputAddonVersion( + "UIANotificationSwitch/2026.1.0.json", + """ +{ + "addonId": "UIANotificationSwitch", + "displayName": "UIA Notification Switch", + "description": "English description", + "channel": "stable", + "addonVersionNumber": { + "major": 2026, + "minor": 1, + "patch": 0 + }, + "minNVDAVersion": { + "major": 2019, + "minor": 1, + "patch": 0 + }, + "lastTestedVersion": { + "major": 2019, + "minor": 1, + "patch": 0 + }, + "translations": [ + { + "language": "ar", + "displayName": "مفتاح إشعارات UIA", + "description": "وصف عربي" + } + ] +} +""", + ), + ) + self.runTransformation() + self._assertAddonDataWritten( + ExpectedAddonVersion("addons/UIANotificationSwitch/2026.1.0/en.json", "2026.1.0"), + ExpectedAddonVersion("addons/UIANotificationSwitch/2026.1.0/ar.json", "2026.1.0"), + ExpectedAddonVersion( + "views/ar/2019.1.0/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/ar.json", + ), + ExpectedAddonVersion( + "views/ar/2019.1.1/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/ar.json", + ), + ExpectedAddonVersion( + "views/ar/2019.2.0/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/ar.json", + ), + ExpectedAddonVersion( + "views/ar/2019.2.1/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/ar.json", + ), + ExpectedAddonVersion( + "views/ar/latest/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/ar.json", + ), + ExpectedAddonVersion( + "views/en/2019.1.0/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2019.1.1/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2019.2.0/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/en.json", + ), + ExpectedAddonVersion( + "views/en/2019.2.1/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/en.json", + ), + ExpectedAddonVersion( + "views/en/latest/UIANotificationSwitch/stable.json", + "2026.1.0", + targetPath="addons/UIANotificationSwitch/2026.1.0/en.json", + ), ) diff --git a/transform/src/transform/transform.py b/transform/src/transform/transform.py index 7398b2dd72c..f6c9d8cf15d 100644 --- a/transform/src/transform/transform.py +++ b/transform/src/transform/transform.py @@ -5,6 +5,7 @@ import glob import json import logging +import os from pathlib import Path from collections.abc import Iterable from typing import Any @@ -79,18 +80,51 @@ def getLatestAddons( addonsForVersionChannel = latestAddons[nvdaAPIVersion.apiVer][addon.channel] if isAddonCompatible(addon, nvdaAPIVersion) and _isAddonNewer(addonsForVersionChannel, addon): addonsForVersionChannel[addon.addonId] = addon - log.error(f"added {addon.addonId} {addon.addonVersion}") + log.debug(f"added {addon.addonId} {addon.addonVersion}") else: - log.error(f"ignoring {addon.addonId} {addon.addonVersion}") + log.debug(f"ignoring {addon.addonId} {addon.addonVersion}") return latestAddons def writeAddons(addonDir: str, addons: WriteableAddons, supportedLanguages: set[str]) -> None: """ - Given a unique mapping of (nvdaAPIVersion, channel) -> addon, write the addons to file. + Given a unique mapping of (nvdaAPIVersion, channel) -> addon, write add-on files and views. + + The output structure is: + - addons/{addonId}/{addonVersion}/{language}.json + - views/{language}/{nvdaAPIVersion}/{addonId}/{channel}.json (relative symlink) + - views/{language}/latest/{addonId}/{channel}.json (relative symlink) + + Views are symlinked to add-on files to avoid duplicating data for each view projection. Throws a ValidationError and exits if writeable data does not match expected schema. """ + + def _getTranslatedAddonData( + baseAddonData: dict[str, Any], + addonTranslations: dict[str, dict[str, str]], + lang: str, + ) -> dict[str, object]: + translatedAddonData: dict[str, object] = baseAddonData.copy() + langWithoutLocale = lang.split("_")[0] + translation = addonTranslations.get(lang) + if translation is None: + translation = addonTranslations.get(langWithoutLocale) + if translation is not None: + if (translatedDisplayName := translation.get("displayName")) is not None: + translatedAddonData["displayName"] = translatedDisplayName + if (translatedDescription := translation.get("description")) is not None: + translatedAddonData["description"] = translatedDescription + return translatedAddonData + + def _createRelativeFileSymlink(*, targetPath: str, symlinkPath: str) -> None: + relativeTarget = os.path.relpath(targetPath, start=os.path.dirname(symlinkPath)) + symlink = Path(symlinkPath) + symlink.parent.mkdir(parents=True, exist_ok=True) + symlink.symlink_to(relativeTarget) + writtenLatestAddonForChannel: set[str] = set() + writtenTranslatedAddonPath: set[str] = set() + viewLanguages = {"en", *supportedLanguages} for nvdaAPIVersion in sorted(addons.keys(), reverse=True): # To generate the 'latest view', # check each api version, starting with the latest. @@ -100,18 +134,47 @@ def writeAddons(addonDir: str, addons: WriteableAddons, supportedLanguages: set[ for channel in addons[nvdaAPIVersion]: for addonName in addons[nvdaAPIVersion][channel]: addon = addons[nvdaAPIVersion][channel][addonName] - addonWritePath = f"{addonDir}/en/{str(nvdaAPIVersion)}/{addonName}" with open(addon.pathToData, "r", encoding="utf-8") as oldAddonFile: addonData: dict[str, Any] = json.load(oldAddonFile) if "translations" in addonData: del addonData["translations"] - Path(addonWritePath).mkdir(parents=True, exist_ok=True) - with open(f"{addonWritePath}/{channel}.json", "w") as newAddonFile: - validateJson(addonData, JSONSchemaPaths.ADDON_DATA) - json.dump(addonData, newAddonFile) - latestAddonWritePath = f"{addonDir}/en/latest/{addonName}" - Path(latestAddonWritePath).mkdir(parents=True, exist_ok=True) + addonVersion = str(addon.addonVersion) + translatedAddonDirPath = os.path.join(addonDir, "addons", addonName, addonVersion) + if translatedAddonDirPath not in writtenTranslatedAddonPath: + writtenTranslatedAddonPath.add(translatedAddonDirPath) + addonTranslations: dict[str, dict[str, str]] = { + t["language"]: t for t in addon.translations + } + translatedLanguages = {"en", *addonTranslations.keys()} + for lang in translatedLanguages: + translatedAddonData = _getTranslatedAddonData(addonData, addonTranslations, lang) + Path(translatedAddonDirPath).mkdir(parents=True, exist_ok=True) + with open(os.path.join(translatedAddonDirPath, f"{lang}.json"), "w") as newAddonFile: + validateJson(translatedAddonData, JSONSchemaPaths.ADDON_DATA) + json.dump(translatedAddonData, newAddonFile) + + addonTranslations: dict[str, dict[str, str]] = {t["language"]: t for t in addon.translations} + for lang in viewLanguages: + langWithoutLocale = lang.split("_")[0] + if lang in addonTranslations: + targetLanguage = lang + elif langWithoutLocale in addonTranslations: + targetLanguage = langWithoutLocale + else: + targetLanguage = "en" + + translatedAddonPath = os.path.join(translatedAddonDirPath, f"{targetLanguage}.json") + versionedViewPath = os.path.join( + addonDir, + "views", + lang, + str(nvdaAPIVersion), + addonName, + f"{channel}.json", + ) + _createRelativeFileSymlink(targetPath=translatedAddonPath, symlinkPath=versionedViewPath) + # paths are case insensitive # Identical add-on IDs may have different casing # due to legacy add-on submissions. @@ -121,40 +184,25 @@ def writeAddons(addonDir: str, addons: WriteableAddons, supportedLanguages: set[ if addLatest: log.error(f"Latest version: {addonName} {channel} {nvdaAPIVersion}") writtenLatestAddonForChannel.add(caseInsensitiveLatestAddonForChannel) - with open(f"{latestAddonWritePath}/{channel}.json", "w") as latestAddonFile: - json.dump(addonData, latestAddonFile) + for lang in viewLanguages: + langWithoutLocale = lang.split("_")[0] + if lang in addonTranslations: + targetLanguage = lang + elif langWithoutLocale in addonTranslations: + targetLanguage = langWithoutLocale + else: + targetLanguage = "en" - addonTranslations: dict[str, dict[str, str]] = {t["language"]: t for t in addon.translations} - translatedAddonData: dict[str, object] = addonData.copy() - for lang in supportedLanguages: - addonWritePath = f"{addonDir}/{lang}/{str(nvdaAPIVersion)}/{addonName}" - langWithoutLocale = lang.split("_")[0] - if lang in addonTranslations: - # update with translated version - translatedAddonData["displayName"] = addonTranslations[lang]["displayName"] - translatedAddonData["description"] = addonTranslations[lang]["description"] - elif langWithoutLocale in addonTranslations: - # fallback to lang without locale translated version - translatedAddonData["displayName"] = addonTranslations[langWithoutLocale][ - "displayName" - ] - translatedAddonData["description"] = addonTranslations[langWithoutLocale][ - "description" - ] - else: - # fallback and update to english - translatedAddonData["displayName"] = addonData["displayName"] - translatedAddonData["description"] = addonData["description"] - Path(addonWritePath).mkdir(parents=True, exist_ok=True) - with open(f"{addonWritePath}/{channel}.json", "w") as newAddonFile: - validateJson(translatedAddonData, JSONSchemaPaths.ADDON_DATA) - json.dump(translatedAddonData, newAddonFile) - if addLatest: - latestAddonWritePath = f"{addonDir}/{lang}/latest/{addonName}" - Path(latestAddonWritePath).mkdir(parents=True, exist_ok=True) - with open(f"{latestAddonWritePath}/{channel}.json", "w") as newAddonFile: - validateJson(translatedAddonData, JSONSchemaPaths.ADDON_DATA) - json.dump(translatedAddonData, newAddonFile) + translatedAddonPath = os.path.join(translatedAddonDirPath, f"{targetLanguage}.json") + latestViewPath = os.path.join( + addonDir, + "views", + lang, + "latest", + addonName, + f"{channel}.json", + ) + _createRelativeFileSymlink(targetPath=translatedAddonPath, symlinkPath=latestViewPath) def readAddons(addonDir: str) -> Iterable[Addon]: @@ -163,7 +211,7 @@ def readAddons(addonDir: str) -> Iterable[Addon]: Works as a generator to minimize memory usage, as such, each use of iteration should call readAddons. Skips addons and logs errors if the naming schema or json schema do not match what is expected. """ - for fileName in glob.glob(f"{addonDir}/**/*.json"): + for fileName in glob.glob(os.path.join(addonDir, "**", "*.json")): with open(fileName, "r", encoding="utf-8") as addonFile: addonData = json.load(addonFile) try: diff --git a/uv.lock b/uv.lock index 86cb1eda01a..dac70119749 100644 --- a/uv.lock +++ b/uv.lock @@ -1,30 +1,24 @@ version = 1 revision = 3 requires-python = "==3.13.*" -resolution-markers = [ - "sys_platform == 'win32'", -] -supported-markers = [ - "sys_platform == 'win32'", -] [[package]] name = "addon-datastore" source = { editable = "." } dependencies = [ - { name = "configobj", marker = "sys_platform == 'win32'" }, - { name = "jsonschema", marker = "sys_platform == 'win32'" }, - { name = "requests", marker = "sys_platform == 'win32'" }, + { name = "configobj" }, + { name = "jsonschema" }, + { name = "requests" }, ] [package.dev-dependencies] lint = [ - { name = "pre-commit", marker = "sys_platform == 'win32'" }, - { name = "pyright", marker = "sys_platform == 'win32'" }, - { name = "ruff", marker = "sys_platform == 'win32'" }, + { name = "pre-commit" }, + { name = "pyright" }, + { name = "ruff" }, ] unit-tests = [ - { name = "unittest-xml-reporting", marker = "sys_platform == 'win32'" }, + { name = "unittest-xml-reporting" }, ] [package.metadata] @@ -127,10 +121,10 @@ name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'win32'" }, - { name = "jsonschema-specifications", marker = "sys_platform == 'win32'" }, - { name = "referencing", marker = "sys_platform == 'win32'" }, - { name = "rpds-py", marker = "sys_platform == 'win32'" }, + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ @@ -142,7 +136,7 @@ name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "referencing", marker = "sys_platform == 'win32'" }, + { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ @@ -183,11 +177,11 @@ name = "pre-commit" version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cfgv", marker = "sys_platform == 'win32'" }, - { name = "identify", marker = "sys_platform == 'win32'" }, - { name = "nodeenv", marker = "sys_platform == 'win32'" }, - { name = "pyyaml", marker = "sys_platform == 'win32'" }, - { name = "virtualenv", marker = "sys_platform == 'win32'" }, + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } wheels = [ @@ -199,8 +193,8 @@ name = "pyright" version = "1.1.408" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nodeenv", marker = "sys_platform == 'win32'" }, - { name = "typing-extensions", marker = "sys_platform == 'win32'" }, + { name = "nodeenv" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/74/b2/5db700e52554b8f025faa9c3c624c59f1f6c8841ba81ab97641b54322f16/pyright-1.1.408.tar.gz", hash = "sha256:f28f2321f96852fa50b5829ea492f6adb0e6954568d1caa3f3af3a5f555eb684", size = 4400578, upload-time = "2026-01-08T08:07:38.795Z" } wheels = [ @@ -212,8 +206,8 @@ name = "python-discovery" version = "1.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "sys_platform == 'win32'" }, - { name = "platformdirs", marker = "sys_platform == 'win32'" }, + { name = "filelock" }, + { name = "platformdirs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/90/bcce6b46823c9bec1757c964dc37ed332579be512e17a30e9698095dcae4/python_discovery-1.2.0.tar.gz", hash = "sha256:7d33e350704818b09e3da2bd419d37e21e7c30db6e0977bb438916e06b41b5b1", size = 58055, upload-time = "2026-03-19T01:43:08.248Z" } wheels = [ @@ -236,8 +230,8 @@ name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "sys_platform == 'win32'" }, - { name = "rpds-py", marker = "sys_platform == 'win32'" }, + { name = "attrs" }, + { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -249,10 +243,10 @@ name = "requests" version = "2.33.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi", marker = "sys_platform == 'win32'" }, - { name = "charset-normalizer", marker = "sys_platform == 'win32'" }, - { name = "idna", marker = "sys_platform == 'win32'" }, - { name = "urllib3", marker = "sys_platform == 'win32'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } wheels = [ @@ -297,7 +291,7 @@ name = "unittest-xml-reporting" version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lxml", marker = "sys_platform == 'win32'" }, + { name = "lxml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/6b/5847d0e6e95d08e056f23b3f8cd95bede2d3ade10a1c1a9d5b50916454e1/unittest_xml_reporting-4.0.0.tar.gz", hash = "sha256:bfa1ed65e9e6f33c161d04470d89050458cfb65a5a5d0358834ef7ce037d9136", size = 43152, upload-time = "2026-01-07T15:50:58.983Z" } wheels = [ @@ -318,10 +312,10 @@ name = "virtualenv" version = "21.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "distlib", marker = "sys_platform == 'win32'" }, - { name = "filelock", marker = "sys_platform == 'win32'" }, - { name = "platformdirs", marker = "sys_platform == 'win32'" }, - { name = "python-discovery", marker = "sys_platform == 'win32'" }, + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, ] sdist = { url = "https://files.pythonhosted.org/packages/aa/92/58199fe10049f9703c2666e809c4f686c54ef0a68b0f6afccf518c0b1eb9/virtualenv-21.2.0.tar.gz", hash = "sha256:1720dc3a62ef5b443092e3f499228599045d7fea4c79199770499df8becf9098", size = 5840618, upload-time = "2026-03-09T17:24:38.013Z" } wheels = [