From 9b3d19877209359337ea7cff879e3bf9dbe32acb Mon Sep 17 00:00:00 2001 From: Yiyuan Chen Date: Tue, 11 Aug 2026 04:42:02 +0000 Subject: [PATCH] fix(stats): Only strip known global prefixes in strip_namespace Previously, the CSV importer blindly stripped everything before the first colon, which aggressively dropped custom namespaces (like oecd:). This updates strip_namespace and strip_namespace_series to strictly target known prefixes (dcid:, dcs:, schema:). --- simple/stats/data.py | 8 ++++++-- simple/tests/stats/data_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/simple/stats/data.py b/simple/stats/data.py index e9c5524e7..364c84690 100644 --- a/simple/stats/data.py +++ b/simple/stats/data.py @@ -63,6 +63,7 @@ _DCS_PREFIX = "dcs:" _NAMESPACE_DELIMITER = ':' +_GLOBAL_PREFIXES = ("dcid:", "dcs:", "schema:") @dataclass @@ -591,7 +592,10 @@ def strip_namespace(v: str) -> str: Strips namespaces from dcids. e.g. 'dcid:country/USA' -> 'country/USA' """ - return v[v.find(_NAMESPACE_DELIMITER) + 1:] + for prefix in _GLOBAL_PREFIXES: + if v.startswith(prefix): + return v[len(prefix):] + return v def strip_namespace_series(series: pd.Series) -> pd.Series: @@ -603,7 +607,7 @@ def strip_namespace_series(series: pd.Series) -> pd.Series: Returns: Series with namespaces stripped """ - return series.str.split(_NAMESPACE_DELIMITER, n=1).str[-1] + return series.str.replace(r'^(dcid:|dcs:|schema:)', '', regex=True) def filter_invalid_observation_values(df: pd.DataFrame) -> pd.DataFrame: diff --git a/simple/tests/stats/data_test.py b/simple/tests/stats/data_test.py index 83b529fcb..7cb3698ff 100644 --- a/simple/tests/stats/data_test.py +++ b/simple/tests/stats/data_test.py @@ -14,7 +14,11 @@ import unittest +import pandas as pd + from stats.data import _get_flattened_dataclass_field_names +from stats.data import strip_namespace +from stats.data import strip_namespace_series from stats.data import Event from stats.data import McfNode from stats.data import Observation @@ -204,3 +208,23 @@ def test_get_flattened_dataclass_field_names(self): ] self.assertListEqual(_get_flattened_dataclass_field_names(Observation), expected) + + def test_strip_namespace(self): + self.assertEqual(strip_namespace("dcid:country/USA"), "country/USA") + self.assertEqual(strip_namespace("dcs:measuredValue"), "measuredValue") + self.assertEqual(strip_namespace("schema:Thing"), "Thing") + # Custom namespace should NOT be stripped + self.assertEqual(strip_namespace("oecd:Annual_Average_Wage"), "oecd:Annual_Average_Wage") + # No namespace + self.assertEqual(strip_namespace("country/USA"), "country/USA") + + def test_strip_namespace_series(self): + series = pd.Series([ + "dcid:country/USA", "dcs:measuredValue", "schema:Thing", + "oecd:Annual_Average_Wage", "country/USA" + ]) + expected = pd.Series([ + "country/USA", "measuredValue", "Thing", + "oecd:Annual_Average_Wage", "country/USA" + ]) + pd.testing.assert_series_equal(strip_namespace_series(series), expected)