Release 260111

selfdrive/ui/translations/README.md

@@ -0,0 +1,71 @@
+# Multilanguage
+
+[![languages](https://raw.githubusercontent.com/commaai/openpilot/badges/translation_badge.svg)](#)
+
+## Contributing
+
+Before getting started, make sure you have set up the openpilot Ubuntu development environment by reading the [tools README.md](/tools/README.md).
+
+### Policy
+
+Most of the languages supported by openpilot come from and are maintained by the community via pull requests. A pull request likely to be merged is one that [fixes a translation or adds missing translations.](https://github.com/commaai/openpilot/blob/master/selfdrive/ui/translations/README.md#improving-an-existing-language)
+
+We also generally merge pull requests adding support for a new language if there are community members willing to maintain it. Maintaining a language is ensuring quality and completion of translations before each openpilot release.
+
+comma may remove or hide language support from releases depending on translation quality and completeness.
+
+### Adding a New Language
+
+openpilot provides a few tools to help contributors manage their translations and to ensure quality. To get started:
+
+1. Add your new language to [languages.json](/selfdrive/ui/translations/languages.json) with the appropriate [language code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) and the localized language name (Traditional Chinese is `中文（繁體）`).
+2. Generate the XML translation file (`*.ts`):
+   ```shell
+   selfdrive/ui/update_translations.py
+   ```
+3. Edit the translation file, marking each translation as completed:
+   ```shell
+   linguist selfdrive/ui/translations/your_language_file.ts
+   ```
+4. View your finished translations by compiling and starting the UI, then find it in the language selector:
+   ```shell
+   scons -j$(nproc) selfdrive/ui && selfdrive/ui/ui
+   ```
+5. Read [Checking the UI](#checking-the-ui) to double-check your translations fit in the UI.
+
+### Improving an Existing Language
+
+Follow step 3. above, you can review existing translations and add missing ones. Once you're done, just open a pull request to openpilot.
+
+### Checking the UI
+Different languages use varying space to convey the same message, so it's a good idea to double-check that your translations do not overlap and fit into each widget. Start the UI (step 4. above) and view each page, making adjustments to translations as needed.
+
+#### To view offroad alerts:
+
+With the UI started, you can view the offroad alerts with:
+```shell
+selfdrive/ui/tests/cycle_offroad_alerts.py
+```
+
+### Updating the UI
+
+Any time you edit source code in the UI, you need to update the translations to ensure the line numbers and contexts are up to date (first step above).
+
+### Testing
+
+openpilot has a few unit tests to make sure all translations are up-to-date and that all strings are wrapped in a translation marker. They are run in CI, but you can also run them locally.
+
+Tests translation files up to date:
+
+```shell
+selfdrive/ui/tests/test_translations.py
+```
+
+Tests all static source strings are wrapped:
+
+```shell
+selfdrive/ui/tests/create_test_translations.sh && selfdrive/ui/tests/test_translations
+```
+
+---
+![multilanguage_onroad](https://user-images.githubusercontent.com/25857203/178912800-2c798af8-78e3-498e-9e19-35906e0bafff.png)

selfdrive/ui/translations/auto_translate.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import pathlib
+import xml.etree.ElementTree as ET
+from typing import cast
+
+import requests
+
+TRANSLATIONS_DIR = pathlib.Path(__file__).resolve().parent
+TRANSLATIONS_LANGUAGES = TRANSLATIONS_DIR / "languages.json"
+
+OPENAI_MODEL = "gpt-4"
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+OPENAI_PROMPT = "You are a professional translator from English to {language} (ISO 639 language code). " + \
+                "The following sentence or word is in the GUI of a software called openpilot, translate it accordingly."
+
+
+def get_language_files(languages: list[str] = None) -> dict[str, pathlib.Path]:
+  files = {}
+
+  with open(TRANSLATIONS_LANGUAGES) as fp:
+    language_dict = json.load(fp)
+
+    for filename in language_dict.values():
+      path = TRANSLATIONS_DIR / f"{filename}.ts"
+      language = path.stem.split("main_")[1]
+
+      if languages is None or language in languages:
+        files[language] = path
+
+  return files
+
+
+def translate_phrase(text: str, language: str) -> str:
+  response = requests.post(
+    "https://api.openai.com/v1/chat/completions",
+    json={
+      "model": OPENAI_MODEL,
+      "messages": [
+        {
+          "role": "system",
+          "content": OPENAI_PROMPT.format(language=language),
+        },
+        {
+          "role": "user",
+          "content": text,
+        },
+      ],
+      "temperature": 0.8,
+      "max_tokens": 1024,
+      "top_p": 1,
+    },
+    headers={
+      "Authorization": f"Bearer {OPENAI_API_KEY}",
+      "Content-Type": "application/json",
+    },
+  )
+
+  if 400 <= response.status_code < 600:
+    raise requests.HTTPError(f'Error {response.status_code}: {response.json()}', response=response)
+
+  data = response.json()
+
+  return cast(str, data["choices"][0]["message"]["content"])
+
+
+def translate_file(path: pathlib.Path, language: str, all_: bool) -> None:
+  tree = ET.parse(path)
+
+  root = tree.getroot()
+
+  for context in root.findall("./context"):
+    name = context.find("name")
+    if name is None:
+      raise ValueError("name not found")
+
+    print(f"Context: {name.text}")
+
+    for message in context.findall("./message"):
+      source = message.find("source")
+      translation = message.find("translation")
+
+      if source is None or translation is None:
+        raise ValueError("source or translation not found")
+
+      if not all_ and translation.attrib.get("type") != "unfinished":
+        continue
+
+      llm_translation = translate_phrase(cast(str, source.text), language)
+
+      print(f"Source: {source.text}\n" +
+            f"Current translation: {translation.text}\n" +
+            f"LLM translation: {llm_translation}")
+
+      translation.text = llm_translation
+
+  with path.open("w", encoding="utf-8") as fp:
+    fp.write('<?xml version="1.0" encoding="utf-8"?>\n' +
+             '<!DOCTYPE TS>\n' +
+             ET.tostring(root, encoding="utf-8").decode())
+
+
+def main():
+  arg_parser = argparse.ArgumentParser("Auto translate")
+
+  group = arg_parser.add_mutually_exclusive_group(required=True)
+  group.add_argument("-a", "--all-files", action="store_true", help="Translate all files")
+  group.add_argument("-f", "--file", nargs="+", help="Translate the selected files. (Example: -f fr de)")
+
+  arg_parser.add_argument("-t", "--all-translations", action="store_true", default=False, help="Translate all sections. (Default: only unfinished)")
+
+  args = arg_parser.parse_args()
+
+  if OPENAI_API_KEY is None:
+    print("OpenAI API key is missing. (Hint: use `export OPENAI_API_KEY=YOUR-KEY` before you run the script).\n" +
+          "If you don't have one go to: https://beta.openai.com/account/api-keys.")
+    exit(1)
+
+  files = get_language_files(None if args.all_files else args.file)
+
+  if args.file:
+    missing_files = set(args.file) - set(files)
+    if len(missing_files):
+      print(f"No language files found: {missing_files}")
+      exit(1)
+
+  print(f"Translation mode: {'all' if args.all_translations else 'only unfinished'}. Files: {list(files)}")
+
+  for lang, path in files.items():
+    print(f"Translate {lang} ({path})")
+    translate_file(path, lang, args.all_translations)
+
+
+if __name__ == "__main__":
+  main()

selfdrive/ui/translations/create_badges.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+import json
+import os
+import requests
+import xml.etree.ElementTree as ET
+
+from openpilot.common.basedir import BASEDIR
+from openpilot.selfdrive.ui.tests.test_translations import UNFINISHED_TRANSLATION_TAG
+from openpilot.selfdrive.ui.update_translations import LANGUAGES_FILE, TRANSLATIONS_DIR
+
+TRANSLATION_TAG = "<translation"
+BADGE_HEIGHT = 20 + 8
+SHIELDS_URL = "https://img.shields.io/badge"
+
+if __name__ == "__main__":
+  with open(LANGUAGES_FILE) as f:
+    translation_files = json.load(f)
+
+  badge_svg = []
+  max_badge_width = 0  # keep track of max width to set parent element
+  for idx, (name, file) in enumerate(translation_files.items()):
+    with open(os.path.join(TRANSLATIONS_DIR, f"{file}.ts")) as tr_f:
+      tr_file = tr_f.read()
+
+    total_translations = 0
+    unfinished_translations = 0
+    for line in tr_file.splitlines():
+      if TRANSLATION_TAG in line:
+        total_translations += 1
+      if UNFINISHED_TRANSLATION_TAG in line:
+        unfinished_translations += 1
+
+    percent_finished = int(100 - (unfinished_translations / total_translations * 100.))
+    color = f"rgb{(94, 188, 0) if percent_finished == 100 else (248, 255, 50) if percent_finished > 90 else (204, 55, 27)}"
+
+    # Download badge
+    badge_label = f"LANGUAGE {name}"
+    badge_message = f"{percent_finished}% complete"
+    if unfinished_translations != 0:
+      badge_message += f" ({unfinished_translations} unfinished)"
+
+    r = requests.get(f"{SHIELDS_URL}/{badge_label}-{badge_message}-{color}", timeout=10)
+    assert r.status_code == 200, "Error downloading badge"
+    content_svg = r.content.decode("utf-8")
+
+    xml = ET.fromstring(content_svg)
+    assert "width" in xml.attrib
+    max_badge_width = max(max_badge_width, int(xml.attrib["width"]))
+
+    # Make tag ids in each badge unique to combine them into one svg
+    for tag in ("r", "s"):
+      content_svg = content_svg.replace(f'id="{tag}"', f'id="{tag}{idx}"')
+      content_svg = content_svg.replace(f'"url(#{tag})"', f'"url(#{tag}{idx})"')
+
+    badge_svg.extend([f'<g transform="translate(0, {idx * BADGE_HEIGHT})">', content_svg, "</g>"])
+
+  badge_svg.insert(0, '<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" ' +
+                   f'height="{len(translation_files) * BADGE_HEIGHT}" width="{max_badge_width}">')
+  badge_svg.append("</svg>")
+
+  with open(os.path.join(BASEDIR, "translation_badge.svg"), "w") as badge_f:
+    badge_f.write("\n".join(badge_svg))

selfdrive/ui/translations/languages.json

@@ -0,0 +1,14 @@
+{
+  "English": "main_en",
+  "Deutsch": "main_de",
+  "Français": "main_fr",
+  "Português": "main_pt-BR",
+  "Español": "main_es",
+  "Türkçe": "main_tr",
+  "العربية": "main_ar",
+  "ไทย": "main_th",
+  "中文（繁體）": "main_zh-CHT",
+  "中文（简体）": "main_zh-CHS",
+  "한국어": "main_ko",
+  "日本語": "main_ja"
+}

selfdrive/ui/translations/main_en.ts

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE TS>
+<TS version="2.1" language="en_US">
+<context>
+    <name>InputDialog</name>
+    <message numerus="yes">
+        <source>Need at least %n character(s)!</source>
+        <translation>
+            <numerusform>Need at least %n character!</numerusform>
+            <numerusform>Need at least %n characters!</numerusform>
+        </translation>
+    </message>
+</context>
+<context>
+    <name>QObject</name>
+    <message numerus="yes">
+        <source>%n minute(s) ago</source>
+        <translation>
+            <numerusform>%n minute ago</numerusform>
+            <numerusform>%n minutes ago</numerusform>
+        </translation>
+    </message>
+    <message numerus="yes">
+        <source>%n hour(s) ago</source>
+        <translation>
+            <numerusform>%n hour ago</numerusform>
+            <numerusform>%n hours ago</numerusform>
+        </translation>
+    </message>
+    <message numerus="yes">
+        <source>%n day(s) ago</source>
+        <translation>
+            <numerusform>%n day ago</numerusform>
+            <numerusform>%n days ago</numerusform>
+        </translation>
+    </message>
+</context>
+</TS>