feat: support localization (#309)

.gitattributes

@@ -0,0 +1,4 @@
+# Auto detect text files and perform LF normalization
+* text=auto
+
+**/Translations/*.txt text working-tree-encoding=UTF-16LE-BOM eol=crlf

.github/workflows/build.yaml

@@ -45,6 +45,9 @@ concurrency:
     group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
     cancel-in-progress: true
 
+env:
+  TRANSLATION_FILES: package/Interface/Translations/
+
 jobs:
     check-changes:
         name: Check for changes in PRs
@@ -100,6 +103,56 @@ jobs:
                   echo "should-build=true" >> $GITHUB_OUTPUT
                   echo "hlsl-should-build=true" >> $GITHUB_OUTPUT
 
+    translate:
+      name: translate plugin
+      runs-on: ubuntu-latest
+      env:
+        ISO: .github/workflows/iso.json
+        TRANSLATION_CACHE: locales/
+        WHITELIST_FILES: CommunityShaders_english.txt
+      steps:
+          - uses: actions/checkout@v4
+            with:
+              fetch-depth: 0
+
+          - uses: actions/setup-node@v4
+            with:
+              node-version: 20
+
+          - name: pull files from Lokalise
+            env:
+              LOKALISE_TOKEN: ${{ secrets.LOKALISE_API_TOKEN }}
+              PROJECT_ID: ${{ vars.PROJECT_ID }}
+              FILE_FORMAT: json
+              GITHUB_REF_NAME: ${{ github.ref_name }}
+            run: |
+              node .github/workflows/lokalise_download.mjs
+
+          - name: convert json to utf-16-le-bom txt
+            run: |
+              for file in $TRANSLATION_CACHE/*.json; do
+                if [[ -f "$file" ]]; then
+                  iso_code=$(basename "$file" .json)
+                  language=$(jq -r --arg iso "$iso_code" '.iso_lang[$iso]' $ISO)
+                  if [[ -z "$language" || "$language" == "null" ]]; then
+                    echo "Warning: ISO code '$iso_code' not found. Skipping $file." >&2
+                    continue
+                  fi
+                  output_file="$TRANSLATION_FILES/CommunityShaders_${language}.txt"
+                  if [[ "$(basename "$output_file")" != "$WHITELIST_FILES" ]]; then
+                    printf "\xFF\xFE" > "$output_file"
+                    jq -r 'to_entries | .[] | "\(.key)\t\(.value)"' "$file" \
+                      | sed 's/$/\r/' | iconv -f UTF-8 -t UTF-16LE >> "$output_file"
+                  fi
+                fi
+              done
+
+          - name: Upload translations
+            uses: actions/upload-artifact@v4
+            with:
+              name: translations
+              path: ${{ env.TRANSLATION_FILES }}
+
     cpp-build:
         needs: [check-changes]
         if: >
@@ -128,6 +181,12 @@ jobs:
               with:
                   arch: x64
 
+            - name: Download translations
+              uses: actions/download-artifact@v4
+              with:
+                name: translations
+                path: ${{ env.TRANSLATION_FILES }}
+
             - name: Get MSVC version
               id: msvc_version
               shell: pwsh

.github/workflows/iso.json

@@ -0,0 +1,24 @@
+{
+  "lang_iso": {
+    "english": "en",
+    "french": "fr",
+    "italian": "it",
+    "german": "de",
+    "spanish": "es",
+    "polish": "pl",
+    "chinese": "zh_CN",
+    "russian": "ru",
+    "japanese": "ja"
+  },
+  "iso_lang": {
+    "en": "english",
+    "fr": "french",
+    "it": "italian",
+    "de": "german",
+    "es": "spanish",
+    "pl": "polish",
+    "zh_CN": "chinese",
+    "ru": "russian",
+    "ja": "japanese"
+  }
+}

.github/workflows/lokalise.yaml

@@ -0,0 +1,45 @@
+name: push to Lokalise
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      ISO: .github/workflows/iso.json
+      TRANSLATION_FILES: package/Interface/Translations/
+      TRANSLATION_CACHE: locales/
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: convert utf-16-le-bom txt to json
+        run: |
+          mkdir -p $TRANSLATION_CACHE
+          for file in "$TRANSLATION_FILES"/*.txt; do
+            if [[ -f "$file" ]]; then
+              language=$(basename "$file" | sed -E 's/CommunityShaders_(.*)\.txt/\1/')
+              iso_code=$(jq -r --arg lang "$language" '.lang_iso[$lang]' "$ISO")
+              if [[ -z "$iso_code" || "$iso_code" == "null" ]]; then
+                echo "Warning: Language '$language' not found. Skipping $file." >&2
+                continue
+              fi
+              iconv -f UTF-16LE -t UTF-8 "$file" \
+                | sed '1s/^\xEF\xBB\xBF//' \
+                | grep -vE '^(;|\s*$)' \
+                | jq -Rn '[ inputs | split("\t") | { (.[0]): .[1] } ] | add' \
+                > "$TRANSLATION_CACHE/$iso_code.json"
+            fi
+          done
+
+      - name: push files to Lokalise
+        uses: lokalise/[email protected]
+        with:
+          api_token: ${{ secrets.LOKALISE_API_TOKEN }}
+          project_id: ${{ vars.PROJECT_ID }}
+          file_format: json
+          flat_naming: true
+          translations_path: |
+            ${{ env.TRANSLATION_CACHE }}
+          base_lang: en

.github/workflows/lokalise_download.mjs

@@ -0,0 +1,117 @@
+import { execFile } from 'node:child_process';
+import { setTimeout } from 'node:timers/promises';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+const DEFAULT_MAX_RETRIES = 5;
+const DEFAULT_SLEEP = 1;    // in seconds
+const MAX_SLEEP = 60;       // exponential backoff limit
+const MAX_TOTAL = 300;      // total time limit in seconds
+const DEFAULT_TIMEOUT = 120; // download command timeout (sec)
+
+const envOr = (name, def) => process.env[name] ?? def;
+const parseIntEnv = (value, def) => isNaN(parseInt(value)) ? def : parseInt(value);
+const parseBoolEnv = (value) => ['true','1','yes'].includes(String(value).toLowerCase());
+
+const config = {
+    token:           envOr('LOKALISE_TOKEN', ''),
+    projectId:       envOr('PROJECT_ID', ''),
+    fileFormat:      envOr('FILE_FORMAT', ''),
+    refName:         envOr('GITHUB_REF_NAME',''),
+    skipIncludeTags: parseBoolEnv(envOr('SKIP_INCLUDE_TAGS', false)),
+    maxRetries:      parseIntEnv(envOr('MAX_RETRIES', DEFAULT_MAX_RETRIES), DEFAULT_MAX_RETRIES),
+    sleepTime:       parseIntEnv(envOr('SLEEP_TIME', DEFAULT_SLEEP), DEFAULT_SLEEP),
+    downloadTimeout: parseIntEnv(envOr('DOWNLOAD_TIMEOUT', DEFAULT_TIMEOUT), DEFAULT_TIMEOUT),
+    otherParams:     envOr('CLI_ADD_PARAMS', '').trim(),
+};
+
+if (!config.token) {
+    console.error("Error: Missing LOKALISE_TOKEN env var.");
+    process.exit(1);
+}
+if (!config.projectId) {
+    console.error("Error: Missing PROJECT_ID env var.");
+    process.exit(1);
+}
+if (!config.fileFormat) {
+    console.error("Error: Missing FILE_FORMAT env var.");
+    process.exit(1);
+}
+
+const isRateLimit = (txt) => txt.includes("API request error 429");
+const isNoKeysError = (txt) => txt.includes("API request error 406");
+
+async function installLokaliseCLI(timeoutSec) {
+    console.log("Installing Lokalise CLI using the official installer script...");
+    const installerUrl = "https://raw.githubusercontent.com/lokalise/lokalise-cli-2-go/master/install.sh";
+
+    await execFileAsync('sh', ['-c', `curl -sfL ${installerUrl} | sh`], { timeout: timeoutSec * 1000 });
+
+    console.log("Lokalise CLI installed successfully.");
+}
+
+async function executeDownload(cmdPath, args, timeoutSec) {
+    const { stdout, stderr } = await execFileAsync(cmdPath, args, { timeout: timeoutSec * 1000 });
+    const output = stdout + stderr;
+    if (isNoKeysError(output))   throw new Error(output);
+    if (isRateLimit(output))     throw new Error(output);
+    return output;
+}
+
+async function downloadFiles(cfg) {
+    console.log("Starting download from Lokalise...");
+
+    const args = [
+        `--token=${cfg.token}`,
+        `--project-id=${cfg.projectId}`,
+        'file', 'download',
+        `--format=${cfg.fileFormat}`,
+        '--original-filenames=true',
+        `--directory-prefix=/`,
+    ];
+
+    if (!config.skipIncludeTags && config.refName) {
+        args.push(`--include-tags=${config.refName}`);
+    }
+
+    let attempt = 1, sleepTime = cfg.sleepTime;
+    const startTime = Date.now();
+
+    while (attempt <= cfg.maxRetries) {
+        console.log(`Attempt ${attempt} of ${cfg.maxRetries}...`);
+
+        try {
+            await executeDownload('./bin/lokalise2', args, cfg.downloadTimeout);
+            return;
+        } catch (err) {
+            const msg = err.message || String(err);
+            if (isNoKeysError(msg)) {
+                throw new Error("No keys found for export with current settings.");
+            }
+            if (isRateLimit(msg)) {
+                const elapsed = (Date.now() - startTime) / 1000;
+                if (elapsed >= MAX_TOTAL) {
+                    throw new Error(`Max total time exceeded after ${attempt} attempts.`);
+                }
+                console.warn(`Rate-limited. Retrying in ${sleepTime}s...`);
+                await setTimeout(sleepTime * 1000);
+                sleepTime = Math.min(sleepTime * 2, MAX_SLEEP);
+            } else {
+                console.error("Unexpected error:", msg);
+            }
+        }
+        attempt++;
+    }
+    throw new Error(`Failed to download files after ${cfg.maxRetries} attempts.`);
+}
+
+try {
+    await installLokaliseCLI(DEFAULT_TIMEOUT);
+    await downloadFiles(config);
+    console.log("Successfully downloaded files.");
+    process.exit(0);
+} catch (err) {
+    console.error("Error:", err.message || err);
+    process.exit(1);
+}