feat: initial commit for merica-pipeline delivery
This commit is contained in:
MERICA Admin
@@ -0,0 +1,248 @@
|
||||
"""
|
||||
Unit tests for MericaAlertHook: multi-channel alerting, governance enrichment,
|
||||
and anti-alert fatigue throttling engine.
|
||||
"""
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from helpers.merica_alert_hook import MericaAlertHook, _ALERT_SUPPRESSION_CACHE
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_alert_cache():
|
||||
"""Clear suppression cache before and after every test."""
|
||||
MericaAlertHook.clear_suppression_cache()
|
||||
yield
|
||||
MericaAlertHook.clear_suppression_cache()
|
||||
|
||||
|
||||
class TestMericaAlertHookChannels:
|
||||
"""Test individual dispatch channels for Telegram, Slack, Discord, SITOMAT."""
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_webhook_success(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
|
||||
result = MericaAlertHook.send_webhook("http://example.com/webhook", {"test": "data"})
|
||||
assert result is True
|
||||
mock_urlopen.assert_called_once()
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_webhook_failure(self, mock_urlopen):
|
||||
mock_urlopen.side_effect = Exception("Connection refused")
|
||||
result = MericaAlertHook.send_webhook("http://example.com/webhook", {"test": "data"})
|
||||
assert result is False
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_telegram_alert(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
|
||||
result = MericaAlertHook.send_telegram_alert(
|
||||
bot_token="bot123456:ABC-DEF",
|
||||
chat_id="-100987654321",
|
||||
message="<b>Test Alert</b>"
|
||||
)
|
||||
assert result is True
|
||||
args, kwargs = mock_urlopen.call_args
|
||||
req = args[0]
|
||||
assert "api.telegram.org/botbot123456:ABC-DEF/sendMessage" in req.full_url
|
||||
assert b"-100987654321" in req.data
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_slack_alert(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
|
||||
result = MericaAlertHook.send_slack_alert(
|
||||
webhook_url="https://hooks.slack.com/services/T00/B00/X00",
|
||||
payload={"text": "Test Slack"}
|
||||
)
|
||||
assert result is True
|
||||
args, kwargs = mock_urlopen.call_args
|
||||
req = args[0]
|
||||
assert "hooks.slack.com" in req.full_url
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_discord_alert(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 204
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
|
||||
result = MericaAlertHook.send_discord_alert(
|
||||
webhook_url="https://discord.com/api/webhooks/123/abc",
|
||||
payload={"content": "Test Discord"}
|
||||
)
|
||||
assert result is True
|
||||
|
||||
@patch("urllib.request.urlopen")
|
||||
def test_send_sitomat_alert(self, mock_urlopen):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
||||
|
||||
result = MericaAlertHook.send_sitomat_alert(
|
||||
endpoint_url="http://sitomat:8080/api/v1/lakehouse/webhook",
|
||||
incident_payload={"incidentType": "TASK_FAILURE"},
|
||||
api_key="secret-api-key"
|
||||
)
|
||||
assert result is True
|
||||
args, kwargs = mock_urlopen.call_args
|
||||
req = args[0]
|
||||
assert req.headers.get("X-api-key") == "secret-api-key" or req.headers.get("Authorization") == "Bearer secret-api-key"
|
||||
|
||||
|
||||
class TestAntiAlertFatigue:
|
||||
"""Test suppression and throttling of duplicate/cascading incident alerts."""
|
||||
|
||||
def test_anti_alert_fatigue_throttling(self):
|
||||
cache_key = "dag_permits:task_silver:TASK_FAILURE"
|
||||
|
||||
# Panggilan 1: Belum pernah terlihat, harus lolos (suppressed = False)
|
||||
is_suppressed, count = MericaAlertHook.check_and_record_suppression(cache_key, window_seconds=300)
|
||||
assert is_suppressed is False
|
||||
assert count == 1
|
||||
|
||||
# Panggilan 2: Terjadi dalam 300s window, harus dibatasi (suppressed = True)
|
||||
is_suppressed, count = MericaAlertHook.check_and_record_suppression(cache_key, window_seconds=300)
|
||||
assert is_suppressed is True
|
||||
assert count == 2
|
||||
|
||||
# Panggilan 3: Masih dalam window
|
||||
is_suppressed, count = MericaAlertHook.check_and_record_suppression(cache_key, window_seconds=300)
|
||||
assert is_suppressed is True
|
||||
assert count == 3
|
||||
|
||||
# Reset cache manual
|
||||
MericaAlertHook.clear_suppression_cache()
|
||||
is_suppressed, count = MericaAlertHook.check_and_record_suppression(cache_key, window_seconds=300)
|
||||
assert is_suppressed is False
|
||||
assert count == 1
|
||||
|
||||
|
||||
class TestGovernanceEnrichment:
|
||||
"""Test governance metadata extraction and severity routing."""
|
||||
|
||||
def test_context_extraction_tier1_p0(self):
|
||||
mock_dag = MagicMock()
|
||||
mock_dag.dag_id = "gold_analytics_pipeline"
|
||||
mock_dag.params = {
|
||||
"domain": "TataLingkunganPerizinan",
|
||||
"owner": "dlh_data_engineering_group",
|
||||
"tier": "Tier.Tier1"
|
||||
}
|
||||
|
||||
mock_ti = MagicMock()
|
||||
mock_ti.task_id = "dbt_gold_mart"
|
||||
mock_ti.try_number = 2
|
||||
mock_ti.max_tries = 3
|
||||
mock_ti.duration = 42.5
|
||||
mock_ti.log_url = "http://localhost:8088/log/123"
|
||||
|
||||
context = {
|
||||
"dag": mock_dag,
|
||||
"task_instance": mock_ti,
|
||||
"exception": Exception("Table locked by concurrent vacuum process")
|
||||
}
|
||||
|
||||
data = MericaAlertHook.extract_context(context)
|
||||
assert data["dag_id"] == "gold_analytics_pipeline"
|
||||
assert data["task_id"] == "dbt_gold_mart"
|
||||
assert data["try_number"] == 2
|
||||
assert data["domain"] == "TataLingkunganPerizinan"
|
||||
assert data["tier"] == "Tier.Tier1"
|
||||
assert "P0 (CRITICAL)" in data["severity"]
|
||||
assert data["color_hex"] == "#E01E5A"
|
||||
assert "Table locked" in data["error_msg"]
|
||||
|
||||
def test_context_extraction_tier2_p1(self):
|
||||
mock_dag = MagicMock()
|
||||
mock_dag.dag_id = "silver_cleaning_pipeline"
|
||||
mock_dag.params = {"tier": "Tier.Tier2"}
|
||||
|
||||
mock_ti = MagicMock()
|
||||
mock_ti.task_id = "cleanse_task"
|
||||
mock_ti.try_number = 1
|
||||
mock_ti.duration = 10.0
|
||||
|
||||
context = {
|
||||
"dag": mock_dag,
|
||||
"task_instance": mock_ti,
|
||||
"exception": Exception("Syntax error")
|
||||
}
|
||||
|
||||
data = MericaAlertHook.extract_context(context)
|
||||
assert "P1 (WARNING)" in data["severity"]
|
||||
assert data["color_hex"] == "#ECB22E"
|
||||
|
||||
|
||||
class TestAirflowLifecycleCallbacks:
|
||||
"""Test failure, SLA miss, and quality breach lifecycle handlers."""
|
||||
|
||||
@patch("helpers.merica_alert_hook.MericaAlertHook.send_telegram_alert")
|
||||
@patch("helpers.merica_alert_hook.MericaAlertHook.send_slack_alert")
|
||||
def test_on_failure_callback_dispatches_channels(self, mock_slack, mock_tg, monkeypatch):
|
||||
monkeypatch.setenv("ALERT_TELEGRAM_BOT_TOKEN", "mock_bot_token")
|
||||
monkeypatch.setenv("ALERT_TELEGRAM_CHAT_ID", "mock_chat_id")
|
||||
monkeypatch.setenv("ALERT_SLACK_WEBHOOK_URL", "https://hooks.slack.com/mock")
|
||||
|
||||
mock_slack.return_value = True
|
||||
mock_tg.return_value = True
|
||||
|
||||
mock_dag = MagicMock()
|
||||
mock_dag.dag_id = "permits_medallion"
|
||||
mock_dag.params = {"tier": "Tier.Tier1", "domain": "Perizinan", "owner": "DE"}
|
||||
|
||||
mock_ti = MagicMock()
|
||||
mock_ti.task_id = "quality_gate_silver"
|
||||
mock_ti.duration = 15.0
|
||||
|
||||
context = {
|
||||
"dag": mock_dag,
|
||||
"task_instance": mock_ti,
|
||||
"exception": Exception("Check constraint violated: status must be in set")
|
||||
}
|
||||
|
||||
results = MericaAlertHook.on_failure_callback(context)
|
||||
assert results.get("telegram") is True
|
||||
assert results.get("slack") is True
|
||||
mock_tg.assert_called_once()
|
||||
mock_slack.assert_called_once()
|
||||
|
||||
@patch("helpers.merica_alert_hook.MericaAlertHook.dispatch_alert")
|
||||
def test_sla_miss_callback(self, mock_dispatch):
|
||||
mock_dispatch.return_value = {"telegram": True}
|
||||
mock_dag = MagicMock()
|
||||
mock_dag.dag_id = "sla_test_dag"
|
||||
|
||||
res = MericaAlertHook.sla_miss_callback(
|
||||
dag=mock_dag,
|
||||
task_list=["task_1"],
|
||||
blocking_task_list=["task_1"],
|
||||
slas=[30],
|
||||
blocking_tis=[]
|
||||
)
|
||||
assert res == {"telegram": True}
|
||||
mock_dispatch.assert_called_once()
|
||||
call_args = mock_dispatch.call_args[0]
|
||||
assert call_args[0] == "SLA_MISS"
|
||||
assert call_args[1]["dag_id"] == "sla_test_dag"
|
||||
|
||||
@patch("helpers.merica_alert_hook.MericaAlertHook.dispatch_alert")
|
||||
def test_on_quality_gate_breach(self, mock_dispatch):
|
||||
mock_dispatch.return_value = {"slack": True}
|
||||
context = {
|
||||
"dag": MagicMock(dag_id="spku_iot_pipeline", params={"tier": "Tier.Tier2"}),
|
||||
"task_instance": MagicMock(task_id="ge_pm25_check")
|
||||
}
|
||||
failed_rules = [{"rule": "expect_column_values_to_be_between", "column": "pm25"}]
|
||||
|
||||
res = MericaAlertHook.on_quality_gate_breach(context, failed_rules, 75.0)
|
||||
assert res == {"slack": True}
|
||||
call_args = mock_dispatch.call_args[0]
|
||||
assert call_args[0] == "QUALITY_GATE_BREACH"
|
||||
assert "75.0%" in call_args[1]["error_msg"]
|
||||
assert "pm25" in call_args[1]["error_msg"]
|
||||
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Unit Tests for GreatExpectationsLakehouseOperator
|
||||
Verifies declarative rules (not_null, unique, in_set, row_count),
|
||||
custom SQL assertions, error handling, and XCom payload formatting.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure airflow/dags directory is on python path
|
||||
DAGS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dags"))
|
||||
if DAGS_DIR not in sys.path:
|
||||
sys.path.insert(0, DAGS_DIR)
|
||||
|
||||
from helpers.ge_lakehouse_operator import GreatExpectationsLakehouseOperator
|
||||
|
||||
try:
|
||||
from airflow.exceptions import AirflowException
|
||||
except ImportError:
|
||||
class AirflowException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TestGreatExpectationsLakehouseOperator:
|
||||
|
||||
@patch("helpers.ge_lakehouse_operator.create_engine")
|
||||
def test_declarative_rules_all_pass(self, mock_create_engine):
|
||||
mock_conn = MagicMock()
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value.__enter__.return_value = mock_conn
|
||||
mock_create_engine.return_value = mock_engine
|
||||
|
||||
# Mock query outputs:
|
||||
# 1. row count = 10 (valid for min 1)
|
||||
# 2. null count = 0 (valid for not_null)
|
||||
# 3. in_set invalid count = 0 (valid for in_set)
|
||||
mock_conn.execute.return_value.scalar.side_effect = [10, 0, 0]
|
||||
|
||||
op = GreatExpectationsLakehouseOperator(
|
||||
task_id="test_ge_pass",
|
||||
expectation_suite="permits_test_suite",
|
||||
table_name="bronze_lakehouse.permits",
|
||||
rules=[
|
||||
{"expect": "table_row_count_to_be_between", "min_value": 1},
|
||||
{"expect": "column_values_to_not_be_null", "column": "permit_id"},
|
||||
{"expect": "column_values_to_be_in_set", "column": "status", "allowed_values": ["APPROVED", "REJECTED"]}
|
||||
],
|
||||
database_url="sqlite:///:memory:"
|
||||
)
|
||||
|
||||
res = op.execute(context={})
|
||||
assert res["status"] == "PASSED"
|
||||
assert res["overall_success"] is True
|
||||
assert res["total_evaluations"] == 3
|
||||
assert res["successful_evaluations"] == 3
|
||||
assert res["success_percent"] == 100.0
|
||||
|
||||
@patch("helpers.ge_lakehouse_operator.create_engine")
|
||||
def test_custom_sql_query_pass(self, mock_create_engine):
|
||||
mock_conn = MagicMock()
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value.__enter__.return_value = mock_conn
|
||||
mock_create_engine.return_value = mock_engine
|
||||
|
||||
# Query returns 0 (expected 0)
|
||||
mock_conn.execute.return_value.scalar.return_value = 0
|
||||
|
||||
op = GreatExpectationsLakehouseOperator(
|
||||
task_id="test_custom_sql",
|
||||
expectation_suite="permits_sql_suite",
|
||||
queries=[
|
||||
{
|
||||
"name": "check_no_negative_amounts",
|
||||
"sql": "SELECT count(*) FROM table WHERE amount < 0",
|
||||
"expected": 0,
|
||||
"error_message": "Found negative amounts!"
|
||||
}
|
||||
],
|
||||
database_url="sqlite:///:memory:"
|
||||
)
|
||||
|
||||
res = op.execute(context={})
|
||||
assert res["status"] == "PASSED"
|
||||
assert res["overall_success"] is True
|
||||
assert len(res["details"]) == 1
|
||||
assert res["details"][0]["observed_value"] == 0
|
||||
|
||||
@patch("helpers.ge_lakehouse_operator.create_engine")
|
||||
def test_fail_fast_raises_airflow_exception(self, mock_create_engine):
|
||||
mock_conn = MagicMock()
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value.__enter__.return_value = mock_conn
|
||||
mock_create_engine.return_value = mock_engine
|
||||
|
||||
# Mock: 3 null values found in permit_id
|
||||
mock_conn.execute.return_value.scalar.return_value = 3
|
||||
|
||||
op = GreatExpectationsLakehouseOperator(
|
||||
task_id="test_fail_fast",
|
||||
expectation_suite="permits_fail_suite",
|
||||
table_name="bronze_lakehouse.permits",
|
||||
rules=[
|
||||
{"expect": "column_values_to_not_be_null", "column": "permit_id"}
|
||||
],
|
||||
fail_on_error=True,
|
||||
database_url="sqlite:///:memory:"
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as excinfo:
|
||||
op.execute(context={})
|
||||
assert "Great Expectations Quality Gate GAGAL" in str(excinfo.value)
|
||||
|
||||
@patch("helpers.ge_lakehouse_operator.create_engine")
|
||||
def test_fail_on_error_false_does_not_raise(self, mock_create_engine):
|
||||
mock_conn = MagicMock()
|
||||
mock_engine = MagicMock()
|
||||
mock_engine.connect.return_value.__enter__.return_value = mock_conn
|
||||
mock_create_engine.return_value = mock_engine
|
||||
|
||||
# Mock: 2 invalid records
|
||||
mock_conn.execute.return_value.scalar.return_value = 2
|
||||
|
||||
op = GreatExpectationsLakehouseOperator(
|
||||
task_id="test_warn_only",
|
||||
expectation_suite="permits_warn_suite",
|
||||
table_name="silver.permits",
|
||||
rules=[
|
||||
{"expect": "column_values_to_be_in_set", "column": "status", "allowed_values": ["APPROVED"]}
|
||||
],
|
||||
fail_on_error=False,
|
||||
database_url="sqlite:///:memory:"
|
||||
)
|
||||
|
||||
res = op.execute(context={})
|
||||
assert res["status"] == "FAILED"
|
||||
assert res["overall_success"] is False
|
||||
assert res["unsuccessful_evaluations"] == 1
|
||||
assert res["total_unexpected_records"] == 2
|
||||
@@ -0,0 +1,203 @@
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import pytest
|
||||
import pandas as pd
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
# Ensure airflow/dags directory is on python path
|
||||
DAGS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dags"))
|
||||
if DAGS_DIR not in sys.path:
|
||||
sys.path.insert(0, DAGS_DIR)
|
||||
|
||||
from helpers.storage_helper import storage_manager, build_partitioned_path, StorageManager
|
||||
from helpers.api_helper import (
|
||||
GenericAPIExtractorOperator,
|
||||
create_resilient_session,
|
||||
generate_synthetic_telemetry
|
||||
)
|
||||
from helpers.sling_helper import (
|
||||
SlingDBIngestionOperator,
|
||||
get_checkpoint_value,
|
||||
set_checkpoint_value,
|
||||
generate_synthetic_permits_data
|
||||
)
|
||||
from helpers.gws_helper import (
|
||||
GWSExtractorOperator,
|
||||
DataCleansingEngine,
|
||||
generate_synthetic_gws_waste_data
|
||||
)
|
||||
|
||||
|
||||
class TestStorageHelper:
|
||||
def test_build_partitioned_path(self):
|
||||
fixed_dt = datetime(2026, 9, 28, 10, 0, 0)
|
||||
path = build_partitioned_path("api/spku_api", "telemetry.json", dt=fixed_dt)
|
||||
assert path == "api/spku_api/year=2026/month=09/day=28/telemetry.json"
|
||||
|
||||
def test_save_json_local_fallback(self):
|
||||
test_payload = {"service": "merica", "metric": "air_quality", "value": 42}
|
||||
result = storage_manager.save_json(
|
||||
data=test_payload,
|
||||
bucket="dlh-bronze-test",
|
||||
key="test/sample.json"
|
||||
)
|
||||
assert result["s3_uri"] == "s3://dlh-bronze-test/test/sample.json"
|
||||
assert os.path.exists(result["local_path"])
|
||||
with open(result["local_path"], "r") as f:
|
||||
loaded = json.load(f)
|
||||
assert loaded["value"] == 42
|
||||
os.remove(result["local_path"])
|
||||
|
||||
def test_save_dataframe_csv(self):
|
||||
df = pd.DataFrame({"col1": [1, 2], "col2": ["A", "B"]})
|
||||
result = storage_manager.save_dataframe(
|
||||
df=df,
|
||||
bucket="dlh-bronze-test",
|
||||
key="test/table.csv",
|
||||
target_format="csv"
|
||||
)
|
||||
assert result["format"] == "csv"
|
||||
assert result["rows_written"] == 2
|
||||
assert os.path.exists(result["local_path"])
|
||||
read_df = pd.read_csv(result["local_path"])
|
||||
assert len(read_df) == 2
|
||||
os.remove(result["local_path"])
|
||||
|
||||
|
||||
class TestApiHelper:
|
||||
def test_resilient_session(self):
|
||||
session = create_resilient_session(max_retries=2, backoff_factor=1.0)
|
||||
assert session is not None
|
||||
assert "http://" in session.adapters
|
||||
assert "https://" in session.adapters
|
||||
|
||||
def test_generate_synthetic_telemetry(self):
|
||||
telemetry = generate_synthetic_telemetry("SPKU_01", "2026-09-28T00:00:00Z", "2026-09-28T03:00:00Z")
|
||||
assert telemetry["status"] == "success"
|
||||
assert telemetry["station_id"] == "SPKU_01"
|
||||
assert len(telemetry["readings"]) > 0
|
||||
assert "pm25" in telemetry["readings"][0]
|
||||
|
||||
def test_api_extractor_operator_execution(self):
|
||||
config = {
|
||||
"source_type": "api",
|
||||
"connection_id": "conn_spku_api_service",
|
||||
"api_endpoint": "https://api.spku.dlh.id/v1/stasiun/spku/data-realtime",
|
||||
"lookback_hours": 3,
|
||||
"target": {
|
||||
"bucket": "dlh-bronze-test",
|
||||
"prefix": "test_api/spku"
|
||||
}
|
||||
}
|
||||
operator = GenericAPIExtractorOperator(
|
||||
task_id="test_api_op",
|
||||
bronze_config=config,
|
||||
mock_on_network_error=True
|
||||
)
|
||||
result = operator.execute(context={"execution_date": datetime(2026, 9, 28, 12, 0, 0)})
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["record_count"] >= 1
|
||||
assert "window_start" in result
|
||||
assert "window_end" in result
|
||||
assert os.path.exists(result["local_path"])
|
||||
os.remove(result["local_path"])
|
||||
|
||||
|
||||
class TestSlingHelper:
|
||||
def test_checkpoint_state(self):
|
||||
key = "unit_test_checkpoint"
|
||||
set_checkpoint_value(key, "2026-09-28 10:00:00")
|
||||
val = get_checkpoint_value(key)
|
||||
assert val == "2026-09-28 10:00:00"
|
||||
state_file = f"/tmp/{key}.state"
|
||||
if os.path.exists(state_file):
|
||||
os.remove(state_file)
|
||||
|
||||
def test_generate_synthetic_permits(self):
|
||||
df = generate_synthetic_permits_data(row_count=4)
|
||||
assert len(df) == 4
|
||||
assert "permit_number" in df.columns
|
||||
assert "status" in df.columns
|
||||
|
||||
def test_sling_db_ingestion_operator_execution(self):
|
||||
config = {
|
||||
"source_type": "database",
|
||||
"connection_id": "conn_postgres_source_db",
|
||||
"table_name": "public.environmental_permits",
|
||||
"ingestion_mode": "incremental_watermark",
|
||||
"watermark_column": "watermark_ts",
|
||||
"target": {
|
||||
"bucket": "dlh-bronze-test",
|
||||
"format": "csv",
|
||||
"prefix": "test_db/permits"
|
||||
}
|
||||
}
|
||||
operator = SlingDBIngestionOperator(
|
||||
task_id="test_sling_op",
|
||||
bronze_config=config
|
||||
)
|
||||
result = operator.execute(context={"execution_date": datetime(2026, 9, 28, 12, 0, 0)})
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["rows_extracted"] > 0
|
||||
assert result["checkpoint_new"] is not None
|
||||
assert os.path.exists(result["local_path"])
|
||||
os.remove(result["local_path"])
|
||||
|
||||
|
||||
class TestGwsHelper:
|
||||
def test_data_cleansing_headers(self):
|
||||
assert DataCleansingEngine.clean_header("Total Volume (Kg) ") == "total_volume_kg"
|
||||
assert DataCleansingEngine.clean_header("Nama TPS / Lokasi") == "nama_tps_lokasi"
|
||||
assert DataCleansingEngine.clean_header("Biaya Retribusi (Rp)") == "biaya_retribusi_rp"
|
||||
|
||||
def test_data_cleansing_numeric_units(self):
|
||||
assert DataCleansingEngine.clean_numeric_value("1,500 kg") == 1500
|
||||
assert DataCleansingEngine.clean_numeric_value("Rp 250,000") == 250000
|
||||
assert DataCleansingEngine.clean_numeric_value("25.5 Ton") == 25.5
|
||||
assert DataCleansingEngine.clean_numeric_value("Jakarta Timur") == "Jakarta Timur"
|
||||
|
||||
def test_data_cleansing_dataframe_sanitization(self):
|
||||
raw_df = generate_synthetic_gws_waste_data()
|
||||
cleaned_df = DataCleansingEngine.sanitize_dataframe(raw_df)
|
||||
|
||||
# Headers sanitized
|
||||
assert "wilayah_kecamatan" in cleaned_df.columns
|
||||
assert "volume_sampah_kg" in cleaned_df.columns
|
||||
assert "biaya_retribusi_rp" in cleaned_df.columns
|
||||
|
||||
# Merged cells forward-filled (first column had empty strings)
|
||||
assert cleaned_df["wilayah_kecamatan"].isna().sum() == 0
|
||||
assert cleaned_df["wilayah_kecamatan"].iloc[1] == "Jakarta Timur"
|
||||
assert cleaned_df["wilayah_kecamatan"].iloc[4] == "Jakarta Selatan"
|
||||
|
||||
# Numeric values cleaned
|
||||
assert cleaned_df["volume_sampah_kg"].iloc[0] == 1250
|
||||
assert cleaned_df["biaya_retribusi_rp"].iloc[0] == 250000
|
||||
|
||||
def test_gws_extractor_operator_execution(self):
|
||||
config = {
|
||||
"source_type": "gws",
|
||||
"connection_id": "conn_gws_service_account",
|
||||
"spreadsheet_id": "1OSRugXZvfjoY9M0yySKT-F3L4LTZXa40J0knU3_4Hik",
|
||||
"sheet_range": "Sheet1!A1:Z1000",
|
||||
"target": {
|
||||
"bucket": "dlh-bronze-test",
|
||||
"format": "csv",
|
||||
"prefix": "test_gws/waste_reports"
|
||||
}
|
||||
}
|
||||
operator = GWSExtractorOperator(
|
||||
task_id="test_gws_op",
|
||||
bronze_config=config,
|
||||
auto_clean=True
|
||||
)
|
||||
result = operator.execute(context={"execution_date": datetime(2026, 9, 28, 12, 0, 0)})
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["rows_extracted"] == 5
|
||||
assert "volume_sampah_kg" in result["columns"]
|
||||
assert os.path.exists(result["local_path"])
|
||||
os.remove(result["local_path"])
|
||||
@@ -0,0 +1,88 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
# Ensure airflow/dags directory is on python path
|
||||
DAGS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dags"))
|
||||
if DAGS_DIR not in sys.path:
|
||||
sys.path.insert(0, DAGS_DIR)
|
||||
|
||||
from helpers.base_lakehouse_operator import BaseLakehouseOperator
|
||||
from helpers.duckdb_operator import DuckDBIcebergOperator
|
||||
from helpers.pyiceberg_operator import PyIcebergTransformationOperator
|
||||
from helpers.dbt_trino_operator import DbtTrinoIcebergOperator
|
||||
from helpers.pyspark_operator import PySparkIcebergOperator
|
||||
|
||||
|
||||
class TestLakehouseOperators(unittest.TestCase):
|
||||
def test_base_lakehouse_operator_initialization(self):
|
||||
op = BaseLakehouseOperator(
|
||||
task_id="test_base_task",
|
||||
inlets=["s3://dlh-bronze/test_data.parquet"],
|
||||
outlets=["iceberg://dlh_lakehouse.silver.test_data"],
|
||||
pipeline_id="test_pipeline",
|
||||
layer="silver",
|
||||
engine="generic"
|
||||
)
|
||||
self.assertEqual(op.task_id, "test_base_task")
|
||||
self.assertEqual(op.inlets, ["s3://dlh-bronze/test_data.parquet"])
|
||||
self.assertEqual(op.outlets, ["iceberg://dlh_lakehouse.silver.test_data"])
|
||||
self.assertEqual(op.layer, "silver")
|
||||
self.assertEqual(op.engine, "generic")
|
||||
|
||||
def test_duckdb_iceberg_operator_initialization(self):
|
||||
op = DuckDBIcebergOperator(
|
||||
task_id="silver_duckdb_task",
|
||||
source_parquet_path="s3://dlh-bronze/environmental_permits.parquet",
|
||||
target_table_name="dlh_lakehouse.silver.environmental_permits",
|
||||
primary_keys=["id", "updated_at"]
|
||||
)
|
||||
self.assertEqual(op.task_id, "silver_duckdb_task")
|
||||
self.assertEqual(op.inlets, ["s3://dlh-bronze/environmental_permits.parquet"])
|
||||
self.assertEqual(op.outlets, ["dlh_lakehouse.silver.environmental_permits"])
|
||||
self.assertEqual(op.layer, "silver")
|
||||
self.assertEqual(op.engine, "duckdb")
|
||||
self.assertIn("PARTITION BY id", op.transformation_sql)
|
||||
|
||||
def test_pyiceberg_operator_initialization(self):
|
||||
op = PyIcebergTransformationOperator(
|
||||
task_id="silver_pyiceberg_task",
|
||||
source_parquet_path="s3://dlh-bronze/waste_reports.parquet",
|
||||
target_table_name="dlh_lakehouse.silver.waste_reports",
|
||||
write_mode="overwrite"
|
||||
)
|
||||
self.assertEqual(op.task_id, "silver_pyiceberg_task")
|
||||
self.assertEqual(op.inlets, ["s3://dlh-bronze/waste_reports.parquet"])
|
||||
self.assertEqual(op.outlets, ["dlh_lakehouse.silver.waste_reports"])
|
||||
self.assertEqual(op.layer, "silver")
|
||||
self.assertEqual(op.engine, "pyiceberg")
|
||||
self.assertEqual(op.write_mode, "overwrite")
|
||||
|
||||
def test_dbt_trino_operator_initialization(self):
|
||||
op = DbtTrinoIcebergOperator(
|
||||
task_id="gold_dbt_task",
|
||||
select_models="gold_permits_summary_daily",
|
||||
layer="gold"
|
||||
)
|
||||
self.assertEqual(op.task_id, "gold_dbt_task")
|
||||
self.assertEqual(op.select_models, "gold_permits_summary_daily")
|
||||
self.assertEqual(op.layer, "gold")
|
||||
self.assertEqual(op.engine, "dbt_trino")
|
||||
self.assertEqual(op.outlets, ["gold.gold_permits_summary_daily"])
|
||||
|
||||
def test_pyspark_operator_initialization(self):
|
||||
op = PySparkIcebergOperator(
|
||||
task_id="silver_spark_task",
|
||||
source_parquet_path="s3://dlh-bronze/spku_realtime.parquet",
|
||||
target_table_name="dlh_lakehouse.silver.spku_realtime"
|
||||
)
|
||||
self.assertEqual(op.task_id, "silver_spark_task")
|
||||
self.assertEqual(op.inlets, ["s3://dlh-bronze/spku_realtime.parquet"])
|
||||
self.assertEqual(op.outlets, ["dlh_lakehouse.silver.spku_realtime"])
|
||||
self.assertEqual(op.layer, "silver")
|
||||
self.assertEqual(op.engine, "pyspark")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Unit tests for SitomatCallbackOperator (F16 Pipeline Hub Integration).
|
||||
Menggunakan unittest.mock standar tanpa dependensi eksternal tambahan.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
# Ensure airflow/dags directory is on python path
|
||||
DAGS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "dags"))
|
||||
if DAGS_DIR not in sys.path:
|
||||
sys.path.insert(0, DAGS_DIR)
|
||||
|
||||
from helpers.sitomat_callback_operator import SitomatCallbackOperator, create_sitomat_session
|
||||
|
||||
|
||||
class TestSitomatCallbackOperator:
|
||||
|
||||
def test_invalid_layer_raises_value_error(self):
|
||||
with pytest.raises(ValueError, match="Layer 'platinum' tidak valid"):
|
||||
SitomatCallbackOperator(
|
||||
task_id="test_invalid_layer",
|
||||
run_id="run-123",
|
||||
layer="platinum",
|
||||
artifact_ref="table.name"
|
||||
)
|
||||
|
||||
def test_mock_mode_execution(self):
|
||||
operator = SitomatCallbackOperator(
|
||||
task_id="test_mock",
|
||||
run_id="run-001",
|
||||
layer="bronze",
|
||||
artifact_ref="iceberg.bronze.permits",
|
||||
checksum="sha256-mock",
|
||||
mock_mode=True
|
||||
)
|
||||
result = operator.execute()
|
||||
assert result["status"] == "mock_success"
|
||||
assert result["acknowledged"] is True
|
||||
assert result["layer"] == "bronze"
|
||||
assert result["runId"] == "run-001"
|
||||
|
||||
@patch("helpers.sitomat_callback_operator.create_sitomat_session")
|
||||
def test_successful_bronze_callback(self, mock_create_session):
|
||||
mock_session = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 201
|
||||
mock_response.json.return_value = {"success": True, "versionId": "v1-uuid", "layer": "bronze"}
|
||||
mock_session.post.return_value = mock_response
|
||||
mock_create_session.return_value = mock_session
|
||||
|
||||
sitomat_url = "http://sitomat.bpkp.go.id:3000"
|
||||
run_id = "run-20261005-001"
|
||||
|
||||
operator = SitomatCallbackOperator(
|
||||
task_id="test_bronze_callback",
|
||||
run_id=run_id,
|
||||
layer="bronze",
|
||||
artifact_ref="lakehouse.bronze.permits_cdc",
|
||||
checksum="sha256-111",
|
||||
sitomat_base_url=sitomat_url,
|
||||
api_key="secret-pipeline-key",
|
||||
metadata_payload={"record_count": 8, "ge_status": "PASSED"},
|
||||
mock_mode=False
|
||||
)
|
||||
|
||||
result = operator.execute()
|
||||
assert result["status"] == "success"
|
||||
assert result["statusCode"] == 201
|
||||
assert result["response"]["versionId"] == "v1-uuid"
|
||||
mock_session.post.assert_called_once()
|
||||
|
||||
@patch("helpers.sitomat_callback_operator.create_sitomat_session")
|
||||
def test_idempotent_replay_conflict_409(self, mock_create_session):
|
||||
mock_session = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 409
|
||||
mock_response.text = "conflict_artifact: run already registered"
|
||||
mock_session.post.return_value = mock_response
|
||||
mock_create_session.return_value = mock_session
|
||||
|
||||
sitomat_url = "http://sitomat.bpkp.go.id:3000"
|
||||
run_id = "run-20261005-002"
|
||||
|
||||
operator = SitomatCallbackOperator(
|
||||
task_id="test_conflict_silver",
|
||||
run_id=run_id,
|
||||
layer="silver",
|
||||
artifact_ref="lakehouse.silver.permits_clean",
|
||||
derived_from_version_id="v1-uuid",
|
||||
checksum="sha256-222",
|
||||
sitomat_base_url=sitomat_url,
|
||||
mock_mode=False
|
||||
)
|
||||
|
||||
result = operator.execute()
|
||||
assert result["status"] == "conflict_acknowledged"
|
||||
assert result["statusCode"] == 409
|
||||
assert "conflict_artifact" in result["response"]
|
||||
|
||||
@patch("helpers.sitomat_callback_operator.create_sitomat_session")
|
||||
def test_connection_error_graceful_handling(self, mock_create_session):
|
||||
import requests
|
||||
mock_session = MagicMock()
|
||||
mock_session.post.side_effect = requests.exceptions.ConnectionError("Connection refused to Sitomat")
|
||||
mock_create_session.return_value = mock_session
|
||||
|
||||
operator = SitomatCallbackOperator(
|
||||
task_id="test_conn_error",
|
||||
run_id="run-unreachable",
|
||||
layer="gold",
|
||||
artifact_ref="lakehouse.gold.mart",
|
||||
sitomat_base_url="http://192.0.2.1:9999",
|
||||
mock_mode=False
|
||||
)
|
||||
|
||||
result = operator.execute()
|
||||
assert result["status"] == "connection_warning"
|
||||
assert result["acknowledged"] is False
|
||||
assert "Connection refused" in result["error"]
|
||||
Reference in New Issue
Block a user