Complete application localization across all supported languages

scripts/export-locales.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Export static web and desktop labels from the application's Fluent catalog."""
+
+import argparse
+import html
+import json
+import re
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+ROOT = Path(__file__).resolve().parents[1]
+XML_LANG = '{http://www.w3.org/XML/1998/namespace}lang'
+KEYS = {
+    'title': 'common.open-cad-studio-web-app',
+    'loading': 'common.loading',
+    'comment': 'common.a-cad-application-for-2d-3d-drawing-and-design',
+    'summary': 'common.value-2d-3d-cad-application-for-dwg-and-dxf-drawings',
+    'description': 'common.create-and-edit-technical-drawings-in-dwg-and-dxf-formats',
+    'drawing': 'common.drawing',
+    'design': 'common.design',
+    'engineering': 'common.engineering',
+    'Launch': 'common.launch-open-cad-studio',
+    'InstallerDescription': 'common.open-cad-studio-installer',
+    'Installer': 'common.installer',
+    'DwgDrawing': 'common.dwg-drawing',
+    'DxfDrawing': 'common.dxf-drawing',
+    'DesktopShortcut': 'common.desktop-shortcut',
+    'DesktopShortcutDescription': 'common.add-a-shortcut-to-open-cad-studio-on-the-desktop',
+    'DowngradeError': 'common.newer-version-already-installed',
+    'Open': 'common.open',
+    'Desktop': 'common.desktop',
+}
+
+
+def labels(path):
+    # Only static attributes are exported; the app resolves dynamic Fluent messages.
+    catalog = {}
+    group = key = None
+    for line in path.read_text().splitlines():
+        if match := re.match(r'^([A-Za-z][\w-]*)\s*=', line):
+            group, key = match[1], None
+        elif match := re.match(r'^    \.([A-Za-z][\w-]*)\s*=\s*(.*)', line):
+            key = f'{group}.{match[1]}'
+            catalog[key] = match[2]
+        elif key and line.startswith('        '):
+            catalog[key] += '\n' + line.strip()
+
+    def resolve(key, seen=()):
+        if key in seen:
+            raise ValueError(f'{path}: cyclic reference {key}')
+        text = re.sub(r'\{\s*([A-Za-z][\w-]*\.[A-Za-z][\w-]*)\s*\}',
+                      lambda match: resolve(match[1], (*seen, key)), catalog[key])
+        if re.search(r'[{}]|__ocs_', text):
+            raise ValueError(f'{path}: {key} is not a static label')
+        return text
+
+    return {name: resolve(key) for name, key in KEYS.items()}
+
+
+def outputs():
+    localized = {path.parent.name: labels(path)
+                 for path in sorted((ROOT / 'locales').glob('*/opencadstudio.ftl'))}
+    english = localized['en-US']
+    yield ROOT / 'web/locale-labels.json', json.dumps(
+        {locale: {key: text[key] for key in ('title', 'loading')}
+         for locale, text in localized.items()}, ensure_ascii=False, indent=2) + '\n'
+
+    page = ROOT / 'web-app.html'
+    content = re.sub(r'<title>[^<]*</title>',
+                     f'<title>{html.escape(english["title"])}</title>', page.read_text())
+    content = re.sub(r'(<div class="sub"[^>]*>)[^<]*(</div>)',
+                     lambda match: match[1] + html.escape(english['loading']) + match[2], content)
+    yield page, content
+
+    desktop = ROOT / 'packaging/OpenCADStudio.desktop'
+    lines = [line for line in desktop.read_text().splitlines()
+             if not re.match(r'^(Comment|Keywords)(\[|=)', line)]
+    for locale, text in localized.items():
+        # Generic language fallbacks cover regional variants; Chinese has two scripts.
+        suffix = '' if locale == 'en-US' else '[' + (
+            locale.replace('-', '_') if locale.startswith('zh-') else locale.split('-')[0]) + ']'
+        lines.append(f'Comment{suffix}={text["comment"]}')
+        lines.append(f'Keywords{suffix}=CAD;DWG;DXF;{text["drawing"]};{text["design"]};{text["engineering"]};2D;3D;')
+    yield desktop, '\n'.join(lines) + '\n'
+
+    metainfo = ROOT / 'packaging/io.github.HakanSeven12.OpenCadStudio.metainfo.xml'
+    component = ET.fromstring(metainfo.read_text())
+    for child in list(component):
+        if child.tag in ('summary', 'description', 'keywords'):
+            component.remove(child)
+    index = list(component).index(component.find('name')) + 1
+    description = ET.Element('description')
+    keywords = ET.Element('keywords')
+    for word in ('CAD', 'DWG', 'DXF'):
+        ET.SubElement(keywords, 'keyword').text = word
+    for locale, text in localized.items():
+        attributes = {} if locale == 'en-US' else {XML_LANG: locale}
+        summary = ET.Element('summary', attributes)
+        summary.text = text['summary']
+        component.insert(index, summary)
+        index += 1
+        ET.SubElement(description, 'p', attributes).text = text['description']
+        for key in ('drawing', 'design', 'engineering'):
+            ET.SubElement(keywords, 'keyword', attributes).text = text[key]
+    component.insert(index, description)
+    component.insert(index + 1, keywords)
+    ET.indent(component, space='  ')
+    yield metainfo, '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(component, encoding='unicode') + '\n'
+
+    # The release MSI uses the existing en-US culture.
+    installer = ET.Element('WixLocalization', {
+        'xmlns': 'http://schemas.microsoft.com/wix/2006/localization',
+        'Culture': 'en-US', 'Codepage': '1252',
+    })
+    for key, text in english.items():
+        if key[0].isupper():
+            ET.SubElement(installer, 'String', {'Id': key}).text = text
+    ET.SubElement(installer, 'String', {'Id': 'ApplicationDescription'}).text = english['summary']
+    ET.indent(installer, space='  ')
+    yield ROOT / 'packaging/windows/strings.en-US.wxl', '<?xml version="1.0" encoding="UTF-8"?>\n' + ET.tostring(installer, encoding='unicode') + '\n'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--check', action='store_true', help='Fail if generated labels need updating.')
+    args = parser.parse_args()
+    stale = []
+    for path, content in outputs():
+        if path.exists() and path.read_text() == content:
+            continue
+        if args.check:
+            stale.append(str(path.relative_to(ROOT)))
+        else:
+            path.write_text(content)
+    if stale:
+        parser.exit(1, 'Run python3 scripts/export-locales.py: ' + ', '.join(stale) + '\n')
+
+
+if __name__ == '__main__':
+    main()

scripts/test_locales.py

@@ -0,0 +1,38 @@
+"""Check literal translation lookups. Fluent syntax and rendering are tested in i18n.rs."""
+
+import json
+from pathlib import Path
+import re
+
+ROOT = Path(__file__).resolve().parents[1]
+STRING = r'"(?:[^"\\]|\\.)*"'
+lookup_text = (ROOT / 'src/locale_catalog.rs').read_text()
+lookup = {}
+for match in re.finditer(r'(' + STRING + r')\s*=>\s*Some\(\(\s*"([\w-]+)"\s*,\s*"([\w-]+)"\s*,?\s*\)\)', lookup_text):
+    source = json.loads(match[1])
+    assert source not in lookup, f'Duplicate source lookup: {source}'
+    lookup[source] = (match[2], match[3])
+assert len(lookup) > 3000
+
+keys = set()
+group = None
+for line in (ROOT / 'locales/en-US/opencadstudio.ftl').read_text().splitlines():
+    if match := re.match(r'^([A-Za-z][\w-]*)\s*=', line):
+        group = match[1]
+    elif match := re.match(r'^    \.([A-Za-z][\w-]*)\s*=', line):
+        keys.add((group, match[1]))
+assert not set(lookup.values()) - keys, f'Missing Fluent targets: {set(lookup.values()) - keys}'
+
+missing = []
+for path in sorted((ROOT / 'src').rglob('*.rs')):
+    text = path.read_text()
+    for match in re.finditer(r'\b(?:t|tf)!\(\s*(' + STRING + ')', text, re.S):
+        # Rust permits escaped line continuations and literal newlines.
+        literal = re.sub(r'\\\n\s*', '', match[1])
+        literal = re.sub(r'\\u\{([0-9a-fA-F_]+)\}', lambda m: json.dumps(chr(int(m[1].replace('_', ''), 16)))[1:-1], literal)
+        source = json.loads(literal, strict=False)
+        if source not in lookup and re.search(r'[A-Za-z]{2}', source):
+            line = text[:match.start()].count('\n') + 1
+            missing.append(f'{path.relative_to(ROOT)}:{line}: {source!r}')
+assert not missing, '\n'.join(missing[:30])
+print(f'{len(lookup)} source lookups and literal translation calls passed')

scripts/test_site.py

@@ -12,6 +12,9 @@ from urllib.parse import urlsplit
 import xml.etree.ElementTree as ET
 
 ROOT = Path(__file__).resolve().parents[1]
+subprocess.run(['python3', 'scripts/test_locales.py'], cwd=ROOT, check=True)
+subprocess.run(['python3', 'scripts/export-locales.py', '--check'], cwd=ROOT, check=True)
+subprocess.run(['node', 'scripts/test_web_locales.cjs'], cwd=ROOT, check=True)
 
 
 class Page(HTMLParser):

scripts/test_web_locales.cjs

@@ -0,0 +1,47 @@
+// Run with node scripts/test_web_locales.cjs.
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const vm = require('node:vm');
+const path = require('node:path');
+const root = path.resolve(__dirname, '..');
+const page = fs.readFileSync(path.join(root, 'web-app.html'), 'utf8');
+const script = [...page.matchAll(/<script>([\s\S]*?)<\/script>/g)].map(match => match[1]).join('\n');
+const labels = JSON.parse(fs.readFileSync(path.join(root, 'web/locale-labels.json'), 'utf8'));
+
+async function visit(languages, saved, blocked = false) {
+  const caption = { textContent: '' };
+  const splash = { querySelector: () => caption, classList: { add() {} }, remove() {} };
+  const document = { documentElement: {}, getElementById: () => splash, querySelector: () => ({}) };
+  vm.runInNewContext(script, {
+    document, navigator: { languages, language: languages[0] }, Intl,
+    localStorage: { getItem(key) {
+      assert.equal(key, 'opencadstudio.settings');
+      if (blocked) throw new Error('Storage disabled');
+      return saved;
+    } },
+    fetch: async url => { assert.equal(url, 'locale-labels.json'); return { json: async () => labels }; },
+    setTimeout() {},
+  });
+  await new Promise(setImmediate);
+  return { document, caption };
+}
+
+(async () => {
+  const cases = Object.keys(labels).map(locale => [[locale], undefined, locale]);
+  cases.push(
+    [['en-US'], JSON.stringify({ settings: { language: 'tr-TR' } }), 'tr-TR'],
+    [['fr-CA'], JSON.stringify({ settings: { language: 'system' } }), 'fr-FR'],
+    [['zh-HK'], undefined, 'zh-TW'], [['zh-Hans-SG'], undefined, 'zh-CN'],
+    [['sv-SE', 'de-AT'], '{invalid', 'de-DE'], [['sv-SE'], undefined, 'en-US'],
+  );
+  for (const [languages, saved, expected] of cases) {
+    const { document, caption } = await visit(languages, saved);
+    assert.equal(document.documentElement.lang, expected);
+    assert.equal(document.documentElement.dir, expected === 'ar-SA' ? 'rtl' : 'ltr');
+    assert.equal(document.title, labels[expected].title);
+    assert.equal(caption.textContent, labels[expected].loading);
+  }
+  const blocked = await visit(['ja-JP'], undefined, true);
+  assert.equal(blocked.document.documentElement.lang, 'ja-JP');
+  console.log('Web loading labels, saved language, regional fallbacks and blocked storage passed');
+})().catch(error => { console.error(error); process.exitCode = 1; });