Fixes #448 : Improve legal code language selection/fallback

i18n/tests/test_utils.py

@@ -13,8 +13,8 @@
 # First-party/Local
 from i18n.utils import (
     active_translation,
-    get_default_language_for_jurisdiction_deed,
-    get_default_language_for_jurisdiction_naive,
+    get_default_language_for_jurisdiction_deed_ux,
+    get_default_language_for_jurisdiction_legal_code,
     get_jurisdiction_name,
     get_pofile_creation_date,
     get_pofile_path,
@@ -164,35 +164,35 @@ class I18NTest(TestCase):
     def test_get_language_for_jurisdiction_deed(self):
         # "be" jurisdiction default is "fr"
         self.assertEqual(
-            "fr", get_default_language_for_jurisdiction_deed("be")
+            "fr", get_default_language_for_jurisdiction_deed_ux("be")
         )
         # "am" jurisdiction default is "hy"
         # the "hy" translation is incomplete so we return the global default
         # https://github.com/creativecommons/cc-legal-tools-app/issues/444
         self.assertEqual(
-            "en", get_default_language_for_jurisdiction_deed("am")
+            "en", get_default_language_for_jurisdiction_deed_ux("am")
         )
         # "xx" is an invalid jurisdiction
         # return global default ("en")
         self.assertEqual(
-            "en", get_default_language_for_jurisdiction_deed("xx")
+            "en", get_default_language_for_jurisdiction_deed_ux("xx")
         )
 
     def test_get_language_for_jurisdiction_legal_code(self):
         # "be" jurisdiction default is "fr"
         self.assertEqual(
-            "fr", get_default_language_for_jurisdiction_naive("be")
+            "fr", get_default_language_for_jurisdiction_legal_code("be")
         )
         # "xx" is an invalid jurisdiction
         # return global default ("en")
         self.assertEqual(
-            "en", get_default_language_for_jurisdiction_naive("xx")
+            "en", get_default_language_for_jurisdiction_legal_code("xx")
         )
 
 
 class TranslationTest(TestCase):
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=["LANGUAGE_CODE"],
+        LANGUAGES_AVAILABLE_DEEDS_UX=["LANGUAGE_CODE"],
         LEGAL_CODE_LOCALE_PATH="LOCALE_DIRS",
     )
     def test_get_translation_object_specified_translated(self):
@@ -219,7 +219,7 @@ def test_get_translation_object_specified_translated(self):
         self.assertEqual(translation_object, result)
 
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=["LANGUAGE_DEFAULT"],
+        LANGUAGES_AVAILABLE_DEEDS_UX=["LANGUAGE_DEFAULT"],
         LEGAL_CODE_LOCALE_PATH="LOCALE_DIRS",
     )
     def test_get_translation_object_default_translated(self):
@@ -246,7 +246,7 @@ def test_get_translation_object_default_translated(self):
         self.assertEqual(translation_object, result)
 
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=[],
+        LANGUAGES_AVAILABLE_DEEDS_UX=[],
         LEGAL_CODE_LOCALE_PATH="LOCALE_DIRS",
     )
     def test_get_translation_object_untranslated(self):

i18n/utils.py

@@ -89,11 +89,11 @@ def get_translation_object(
 
     # Add a fallback to the standard Django translation for this language. This
     # gets us the non-legal-code parts of the pages.
-    if language_code in settings.LANGUAGES_MOSTLY_TRANSLATED:
+    if language_code in settings.LANGUAGES_AVAILABLE_DEEDS_UX:
         tool_translation_object.add_fallback(
             translation.trans_real.translation(language_code)
         )
-    elif language_default in settings.LANGUAGES_MOSTLY_TRANSLATED:
+    elif language_default in settings.LANGUAGES_AVAILABLE_DEEDS_UX:
         tool_translation_object.add_fallback(
             translation.trans_real.translation(language_default)
         )
@@ -253,21 +253,68 @@ def map_legacy_to_django_language_code(legacy_language_code: str) -> str:
     return django_language_code
 
 
-def get_default_language_for_jurisdiction_deed(jurisdiction_code):
+def get_default_language_for_jurisdiction_deed_ux(jurisdiction_code):
     default_language = DEFAULT_JURISDICTION_LANGUAGES.get(
         jurisdiction_code, settings.LANGUAGE_CODE
     )
-    if default_language in settings.LANGUAGES_MOSTLY_TRANSLATED:
+    if default_language in settings.LANGUAGES_AVAILABLE_DEEDS_UX:
         return default_language
     else:
         return settings.LANGUAGE_CODE
 
-
-def get_default_language_for_jurisdiction_naive(jurisdiction_code):
-    return DEFAULT_JURISDICTION_LANGUAGES.get(
+def get_default_language_for_jurisdiction_legal_code(jurisdiction_code):
+    default_language = DEFAULT_JURISDICTION_LANGUAGES.get(
         jurisdiction_code, settings.LANGUAGE_CODE
     )
+    if default_language in settings.LANGUAGES_AVAILABLE_LEGAL_CODE:
+        return default_language
+    else:
+        return settings.LANGUAGE_CODE
 
+def get_legal_code_pofiles():
+    legal_code_pofiles = []
+    for locale_name in os.listdir(settings.LEGAL_CODE_LOCALE_PATH):
+        language_code = translation.to_language(locale_name)
+        pofile_path = get_pofile_path(
+            locale_or_legalcode="locale",
+            language_code=language_code,
+            translation_domain="django",
+        )
+        if not os.path.isfile(pofile_path):  # pragma: no cover
+            continue
+        legal_code_pofiles.append([language_code, pofile_path])
+    legal_code_pofiles.sort(key=lambda x: x[0])
+    return legal_code_pofiles
+
+def load_legal_code_translations():
+    """
+    Process Deed & UX translations (store information on all and track those
+    that meet or exceed the TRANSLATION_THRESHOLD).
+    """
+    legal_code_po_file_info = {}
+    languages_available_legal_code = []
+    for language_code, pofile_path in get_legal_code_pofiles():
+        pofile_obj = polib.pofile(pofile_path)
+        percent_translated = pofile_obj.percent_translated()
+        legal_code_po_file_info[language_code] = {
+            "percent_translated": percent_translated,
+            "creation_date": get_pofile_creation_date(pofile_obj),
+            "revision_date": get_pofile_revision_date(pofile_obj),
+            "metadata": pofile_obj.metadata,
+        }
+        update_lang_info(language_code)
+        if (
+            percent_translated < settings.TRANSLATION_THRESHOLD
+            and language_code != settings.LANGUAGE_CODE
+        ):
+            continue
+        languages_available_legal_code.append(language_code)
+    legal_code_po_file_info = dict(sorted(legal_code_po_file_info.items()))
+    # Add global settings
+    settings.DEEDS_UX_PO_FILE_INFO = legal_code_po_file_info
+    settings.LANGUAGES_AVAILABLE_LEGAL_CODE = sorted(
+        list(set(languages_available_legal_code))
+    )
 
 def get_jurisdiction_name(category, unit, version, jurisdiction_code):
     # For details on nomenclature for unported licenses, see:
@@ -310,7 +357,7 @@ def load_deeds_ux_translations():
     that meet or exceed the TRANSLATION_THRESHOLD).
     """
     deeds_ux_po_file_info = {}
-    languages_mostly_translated = []
+    languages_available_deeds_ux = []
     for language_code, pofile_path in get_deeds_ux_pofiles():
         pofile_obj = polib.pofile(pofile_path)
         percent_translated = pofile_obj.percent_translated()
@@ -326,12 +373,12 @@ def load_deeds_ux_translations():
             and language_code != settings.LANGUAGE_CODE
         ):
             continue
-        languages_mostly_translated.append(language_code)
+        languages_available_deeds_ux.append(language_code)
     deeds_ux_po_file_info = dict(sorted(deeds_ux_po_file_info.items()))
     # Add global settings
     settings.DEEDS_UX_PO_FILE_INFO = deeds_ux_po_file_info
-    settings.LANGUAGES_MOSTLY_TRANSLATED = sorted(
-        list(set(languages_mostly_translated))
+    settings.LANGUAGES_AVAILABLE_DEEDS_UX = sorted(
+        list(set(languages_available_deeds_ux))
     )
 
 

legal_tools/management/commands/publish.py

@@ -18,7 +18,7 @@
 # First-party/Local
 from i18n import DEFAULT_CSV_FILE
 from i18n.utils import (
-    get_default_language_for_jurisdiction_deed,
+    get_default_language_for_jurisdiction_deed_ux,
     write_transstats_csv,
 )
 from legal_tools.git_utils import commit_and_push_changes, setup_local_branch
@@ -433,7 +433,7 @@ def distill_lists(self):
 
         arguments = []
         for category in ["licenses", "publicdomain"]:
-            for language_code in settings.LANGUAGES_MOSTLY_TRANSLATED:
+            for language_code in settings.LANGUAGES_AVAILABLE_DEEDS_UX:
                 arguments.append((output_dir, category, language_code))
         self.pool.starmap(save_list, arguments)
 
@@ -469,7 +469,7 @@ def distill_legal_tools(self):
                     (output_dir, legal_code, self.options["apache_only"])
                 )
             for tool in tools:
-                for language_code in settings.LANGUAGES_MOSTLY_TRANSLATED:
+                for language_code in settings.LANGUAGES_AVAILABLE_DEEDS_UX:
                     deed_arguments.append(
                         (
                             output_dir,
@@ -487,7 +487,7 @@ def distill_legal_tools(self):
                         default_languages_deeds[tool.version] = {}
                     default_languages_deeds[tool.version][
                         tool.jurisdiction_code
-                    ] = get_default_language_for_jurisdiction_deed(
+                    ] = get_default_language_for_jurisdiction_deed_ux(
                         tool.jurisdiction_code,
                     )
 

legal_tools/models.py

@@ -12,8 +12,8 @@
 # First-party/Local
 from i18n import LANGMAP_DJANGO_TO_PCRE
 from i18n.utils import (
-    get_default_language_for_jurisdiction_deed,
-    get_default_language_for_jurisdiction_naive,
+    get_default_language_for_jurisdiction_deed_ux,
+    get_default_language_for_jurisdiction_legal_code,
     get_jurisdiction_name,
     get_pofile_path,
     get_translation_object,
@@ -249,7 +249,7 @@ def get_publish_files(self):
         language_code = self.language_code
         tool = self.tool
         juris_code = tool.jurisdiction_code
-        language_default = get_default_language_for_jurisdiction_naive(
+        language_default = get_default_language_for_jurisdiction_legal_code(
             juris_code
         )
         filename = f"legalcode.{self.language_code}.html"
@@ -348,7 +348,7 @@ def translation_domain(self):
         return self.tool.resource_slug
 
     def get_translation_object(self):
-        language_default = get_default_language_for_jurisdiction_naive(
+        language_default = get_default_language_for_jurisdiction_legal_code(
             self.tool.jurisdiction_code
         )
         return get_translation_object(
@@ -554,7 +554,7 @@ def get_metadata(self):
         """
         Return a dictionary with the metadata for this tool.
         """
-        language_default = get_default_language_for_jurisdiction_deed(
+        language_default = get_default_language_for_jurisdiction_deed_ux(
             self.jurisdiction_code
         )
         data = {}
@@ -601,7 +601,7 @@ def get_publish_files(self, language_code):
            correctly
         """
         juris_code = self.jurisdiction_code
-        language_default = get_default_language_for_jurisdiction_deed(
+        language_default = get_default_language_for_jurisdiction_deed_ux(
             juris_code
         )
         filename = f"deed.{language_code}.html"

legal_tools/tests/test_views.py

@@ -11,7 +11,7 @@
 from django.utils.translation.trans_real import DjangoTranslation
 
 # First-party/Local
-from i18n.utils import get_default_language_for_jurisdiction_deed
+from i18n.utils import get_default_language_for_jurisdiction_deed_ux
 from legal_tools.models import UNITS_LICENSES, LegalCode, Tool, build_path
 from legal_tools.rdf_utils import (
     convert_https_to_http,
@@ -417,7 +417,7 @@ def test_get_category_and_category_title_category_publicdomain(self):
         self.assertEqual(category, "publicdomain")
         self.assertEqual(category_title, "Public Domain")
 
-    @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"])
+    @override_settings(LANGUAGES_AVAILABLE_DEEDS_UX=["x1", "x2"])
     def test_get_deed_rel_path_mostly_translated_language_code(self):
         expected_deed_rel_path = "deed.x1"
         deed_rel_path = get_deed_rel_path(
@@ -428,7 +428,7 @@ def test_get_deed_rel_path_mostly_translated_language_code(self):
         )
         self.assertEqual(expected_deed_rel_path, deed_rel_path)
 
-    @override_settings(LANGUAGES_MOSTLY_TRANSLATED=["x1", "x2"])
+    @override_settings(LANGUAGES_AVAILABLE_DEEDS_UX=["x1", "x2"])
     def test_get_deed_rel_path_less_translated_language_code(self):
         expected_deed_rel_path = "deed.x2"
         deed_rel_path = get_deed_rel_path(
@@ -441,7 +441,7 @@ def test_get_deed_rel_path_less_translated_language_code(self):
 
     @override_settings(
         LANGUAGE_CODE="x1",
-        LANGUAGES_MOSTLY_TRANSLATED=[],
+        LANGUAGES_AVAILABLE_DEEDS_UX=[],
     )
     def test_get_deed_rel_path_less_translated_language_default(self):
         expected_deed_rel_path = "deed.x1"
@@ -461,7 +461,7 @@ def test_get_legal_code_replaced_rel_path_cache_miss(self):
         )
         path_start = "/licenses/by/3.0"
         language_code = "en"
-        language_default = get_default_language_for_jurisdiction_deed(None)
+        language_default = get_default_language_for_jurisdiction_deed_ux(None)
 
         cache.clear()
         self.assertFalse(cache.has_key("by-4.0--en-replaced_deed_str"))
@@ -909,7 +909,7 @@ def test_view_legal_code_identifying_jurisdiction_default_language(self):
         self.assertEqual(lc, context["legal_code"])
 
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=["ar", settings.LANGUAGE_CODE],
+        LANGUAGES_AVAILABLE_DEEDS_UX=["ar", settings.LANGUAGE_CODE],
     )
     def test_view_legal_code_40(self):
         tool = ToolFactory(
@@ -949,7 +949,7 @@ def test_view_legal_code_40(self):
                 self.assertContains(rsp, 'dir="rtl"')
 
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=["es", settings.LANGUAGE_CODE],
+        LANGUAGES_AVAILABLE_DEEDS_UX=["es", settings.LANGUAGE_CODE],
     )
     def test_view_legal_code_30_default_translated(self):
         language_code = "ast"
@@ -973,7 +973,7 @@ def test_view_legal_code_30_default_translated(self):
         self.assertContains(rsp, 'dir="ltr"')
 
     @override_settings(
-        LANGUAGES_MOSTLY_TRANSLATED=[settings.LANGUAGE_CODE],
+        LANGUAGES_AVAILABLE_DEEDS_UX=[settings.LANGUAGE_CODE],
     )
     def test_view_legal_code_30_default_untranslated(self):
         language_code = "ast"

legal_tools/utils.py

@@ -16,7 +16,7 @@
 from i18n import UNIT_NAMES
 from i18n.utils import (
     active_translation,
-    get_default_language_for_jurisdiction_naive,
+    get_default_language_for_jurisdiction_legal_code,
     get_jurisdiction_name,
     get_translation_object,
     map_legacy_to_django_language_code,
@@ -145,7 +145,7 @@ def parse_legal_code_filename(filename):
     if jurisdiction:
         language_code = (
             language_code
-            or get_default_language_for_jurisdiction_naive(jurisdiction)
+            or get_default_language_for_jurisdiction_legal_code(jurisdiction)
         )
     else:
         language_code = language_code or settings.LANGUAGE_CODE
@@ -511,7 +511,7 @@ def update_title(options):
                 # Translate title using legal code translation domain for legal
                 # code that is in Transifex (ex. CC0, Licenses 4.0)
                 slug = f"{unit}_{version}".replace(".", "")
-                language_default = get_default_language_for_jurisdiction_naive(
+                language_default = get_default_language_for_jurisdiction_legal_code(
                     jurisdiction
                 )
                 current_translation = get_translation_object(