feat(helpers): sync dlt_helper and expanded test suite (Sprint 2-3)

tests/test_helpers.py

@@ -27,6 +27,12 @@ from helpers.gws_helper import (
     DataCleansingEngine,
     generate_synthetic_gws_waste_data
 )
+from helpers.dlt_helper import (
+    DLTAPIExtractorOperator,
+    flatten_json_record,
+    normalize_api_response,
+    generate_synthetic_dlt_telemetry
+)
 
 
 class TestStorageHelper:
@@ -201,3 +207,90 @@ class TestGwsHelper:
         assert "volume_sampah_kg" in result["columns"]
         assert os.path.exists(result["local_path"])
         os.remove(result["local_path"])
+
+
+class TestDltHelper:
+    def test_flatten_json_record(self):
+        nested = {
+            "station_id": "SPKU_01",
+            "location": {"city": "Jakarta", "coords": {"lat": -6.2, "lon": 106.8}},
+            "readings": {"pm25": 42.5, "pm10": 60.1},
+            "tags": ["ambient", "urban"]
+        }
+        flat = flatten_json_record(nested)
+        assert flat["station_id"] == "SPKU_01"
+        assert flat["location_city"] == "Jakarta"
+        assert flat["location_coords_lat"] == -6.2
+        assert flat["location_coords_lon"] == 106.8
+        assert flat["readings_pm25"] == 42.5
+        assert flat["readings_pm10"] == 60.1
+        assert "urban" in flat["tags"]
+
+    def test_normalize_api_response_dict_and_list(self):
+        # Case A: Dict with "data" key
+        payload = {
+            "status": "success",
+            "data": [
+                {"id": 1, "metrics": {"value": 100}},
+                {"id": 2, "metrics": {"value": 200}}
+            ]
+        }
+        records = normalize_api_response(payload, unnest_mode="auto_flatten")
+        assert len(records) == 2
+        assert records[0]["metrics_value"] == 100
+        assert records[1]["metrics_value"] == 200
+
+        # Case B: Top-level list
+        list_payload = [{"device": "A", "val": 10}, {"device": "B", "val": 20}]
+        rec_list = normalize_api_response(list_payload, unnest_mode="auto_flatten")
+        assert len(rec_list) == 2
+        assert rec_list[0]["device"] == "A"
+
+    def test_generate_synthetic_dlt_telemetry(self):
+        synth = generate_synthetic_dlt_telemetry("https://api.spku.dlh.id/v1/telemetry", limit=3)
+        assert synth["status"] == "success"
+        assert len(synth["data"]) == 3
+        assert "station_id" in synth["data"][0]
+        assert "telemetry" in synth["data"][0]
+
+    def test_dlt_api_extractor_operator_execution(self):
+        config = {
+            "endpoint_url": "https://api.spku.dlh.id/v1/telemetry/hourly",
+            "http_method": "GET",
+            "pagination": {
+                "strategy": "page_number",
+                "page_param": "page",
+                "limit_param": "limit",
+                "page_size": 4,
+                "max_pages": 1
+            },
+            "incremental": {
+                "cursor_column": "measured_at",
+                "lookback_window_hours": 3
+            },
+            "unnest_mode": "auto_flatten",
+            "landing_zone": {
+                "bucket": "dlh-landing-test",
+                "prefix": "test_landing/api"
+            },
+            "bronze_zone": {
+                "bucket": "dlh-bronze-test",
+                "prefix": "test_bronze/api",
+                "format": "parquet"
+            }
+        }
+        operator = DLTAPIExtractorOperator(
+            task_id="test_dlt_api_op",
+            bronze_config=config,
+            mock_on_network_error=True
+        )
+        result = operator.execute(context={"execution_date": datetime(2026, 9, 28, 12, 0, 0)})
+
+        assert result["status"] == "success"
+        assert result["rows_extracted"] >= 1
+        assert "landing_uri" in result
+        assert "bronze_uri" in result
+        assert "telemetry_pm25" in result["columns"] or "pm25" in str(result["columns"])
+        assert "_dlt_load_id" in result["columns"]
+        assert "_ingested_at" in result["columns"]
+