feat: pybabel extractor for YAML files

Author: matthiasschaub

ohsome_quality_api/utils/helper.py

@@ -118,22 +118,6 @@ def write_geojson(outfile: str, geojson_object: Feature | FeatureCollection) ->
         logging.info("Output file written:\t" + str(outfile))
 
 
-def flatten_sequence(input_seq: dict | list | tuple | set) -> list:
-    """Returns the given input sequence as a list.
-
-    If the input is a dict, it returns all values.
-    """
-    output = []
-    if isinstance(input_seq, dict):
-        input_seq = input_seq.values()
-    for val in input_seq:
-        if isinstance(val, dict | list | tuple | set):
-            output += flatten_sequence(val)
-        else:
-            output.append(val)
-    return output
-
-
 def get_project_root() -> Path:
     """Get root of the Python project."""
     return Path(__file__).resolve().parent.parent.parent.resolve()

ohsome_quality_api/utils/pybabel_yaml_extractor.py

@@ -0,0 +1,61 @@
+from typing import Iterator
+
+import yaml
+
+
+def flatten_sequence(input_seq: dict | list | tuple | set) -> list:
+    """Returns the given input sequence as a list.
+
+    If the input is a dict, it returns all values.
+    """
+    output = []
+    if isinstance(input_seq, dict):
+        input_seq = input_seq.values()
+    for val in input_seq:
+        if isinstance(val, dict | list | tuple | set):
+            output += flatten_sequence(val)
+        else:
+            output.append(val)
+    return output
+
+
+def filter_dictonary_by_keys(dictonary: dict, whitelist: list | tuple) -> dict:
+    if not isinstance(dictonary, dict):
+        return dictonary
+    result = {}
+    for key, value in dictonary.items():
+        if key in whitelist:
+            if isinstance(value, dict):
+                result[key] = filter_dictonary_by_keys(value, whitelist)
+            else:
+                result[key] = value
+        elif isinstance(value, dict):
+            filtered_value = filter_dictonary_by_keys(value, whitelist)
+            if filtered_value:
+                result[key] = filtered_value
+    return result
+
+
+def pybabel_yaml_extractor(
+    fileobj,
+    keywords,
+    comment_tags,
+    options: dict,
+) -> Iterator[str]:
+    """Method for extracting localizable text from YAML files.
+
+    Usage:
+        pybabel_yaml_extractor(fileobj, options={"keys": "name,description"})
+
+    Complies with the following interface:
+    https://babel.readthedocs.io/en/latest/messages.html#writing-extraction-methods
+    """
+    if "keys" in options.keys():
+        keys = options["keys"].split(",")
+    else:
+        raise ValueError("Options should contain non-empty `keys` item.")
+    content = yaml.safe_load(fileobj)
+    filtered = filter_dictonary_by_keys(content, keys)
+    flattened = flatten_sequence(filtered)
+    for string in flattened:
+        yield string

tests/unittests/test_helper.py

@@ -13,7 +13,6 @@
 )
 from ohsome_quality_api.utils.helper import (
     camel_to_hyphen,
-    flatten_sequence,
     get_class_from_key,
     get_project_root,
     hyphen_to_camel,
@@ -46,18 +45,6 @@ def test_name_to_class(self):
                 get_class_from_key(class_type="indicator", key=indicator_name)
             )
 
-    # TODO: add tests for other input than dict
-    def test_flatten_seq(self):
-        input_seq = {
-            "regions": {"default": "ogc_fid"},
-            "gadm": {
-                "default": "uid",  # ISO 3166-1 alpha-3 country code
-                "other": (("name_1", "name_2"), ("id_1", "id_2")),
-            },
-        }
-        output_seq = ["ogc_fid", "uid", "name_1", "name_2", "id_1", "id_2"]
-        self.assertListEqual(flatten_sequence(input_seq), output_seq)
-
     def test_json_serialize_valid_input_datetime(self):
         self.assertIsInstance(json_serialize(datetime.datetime.now()), str)
 

tests/unittests/test_utils_pybabel_yaml_extractor.py

@@ -0,0 +1,93 @@
+import pytest
+import yaml
+
+from ohsome_quality_api.utils.pybabel_yaml_extractor import (
+    filter_dictonary_by_keys,
+    flatten_sequence,
+    pybabel_yaml_extractor,
+)
+
+
+def test_flatten_seq():
+    input_seq = {
+        "regions": {"default": "ogc_fid"},
+        "gadm": {
+            "default": "uid",  # ISO 3166-1 alpha-3 country code
+            "other": (("name_1", "name_2"), ("id_1", "id_2")),
+        },
+    }
+    output_seq = ["ogc_fid", "uid", "name_1", "name_2", "id_1", "id_2"]
+    assert flatten_sequence(input_seq) == output_seq
+
+
+def test_filter_dictonary_by_keys():
+    input = {
+        "regions": {"default": "ogc_fid"},
+        "gadm": {
+            "default": "uid",  # ISO 3166-1 alpha-3 country code
+            "other": (("name_1", "name_2"), ("id_1", "id_2")),
+        },
+    }
+    output = filter_dictonary_by_keys(input, ["other"])
+    assert output == {"gadm": {"other": (("name_1", "name_2"), ("id_1", "id_2"))}}
+
+
+def test_no_keys(tmp_path):
+    path = tmp_path / "foo.yaml"
+    with open(path, "w") as file:
+        file.write(
+            yaml.dump(
+                {
+                    "name": "Silenthand Olleander",
+                    "race": "Human",
+                    "traits": ["ONE_HAND", "ONE_EYE"],
+                }
+            )
+        )
+    options = {}
+    with open(path, "r") as file:
+        with pytest.raises(ValueError):
+            list(pybabel_yaml_extractor(file, None, None, options))
+
+
+def test_flat(tmp_path):
+    path = tmp_path / "foo.yaml"
+    with open(path, "w") as file:
+        file.write(
+            yaml.dump(
+                {
+                    "name": "Silenthand Olleander",
+                    "race": "Human",
+                    "traits": ["ONE_HAND", "ONE_EYE"],
+                }
+            )
+        )
+    options = {"keys": "name,traits"}
+    with open(path, "r") as file:
+        result = list(pybabel_yaml_extractor(file, None, None, options))
+    assert result == ["Silenthand Olleander", "ONE_HAND", "ONE_EYE"]
+
+
+def test_nested(tmp_path):
+    path = tmp_path / "foo.yaml"
+    with open(path, "w") as file:
+        file.write(
+            yaml.dump(
+                {
+                    "hero": {
+                        "hp": 34,
+                        "sp": 8,
+                        "level": 4,
+                    },
+                    "orc": {
+                        "hp": 12,
+                        "sp": 0,
+                        "level": 2,
+                    },
+                }
+            )
+        )
+    options = {"keys": "hp"}
+    with open(path, "r") as file:
+        result = list(pybabel_yaml_extractor(file, None, None, options))
+    assert result == [34, 12]