merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(quality): emit Great Expectations test case results directly to OpenMetadata table test suites

Browse Source
This commit is contained in:
syahril
2026-10-19 03:21:06 +07:00
parent b125f65351
commit abce291520
6 changed files with 1098 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
metadata/e2e_medallion_permits_flow.json
+445
View File
@@ -0,0 +1,445 @@
{
"id": "e2e_medallion_permits_flow",
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
"schedule": "0 */1 * * *",
"owner_id": "local-admin",
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-9377",
"type": "customNode",
"position": {
"x": 80,
"y": 80
},
"data": {
"label": "Source Database",
"type": "source_db",
"subtitle": "Postgres / MySQL / Oracle",
"isSource": true,
"configured": true,
"config": {
"conn": "Postgres",
"conn_id": "conn_postgres_source_db",
"table": "permits",
"table_name": "public.stasiun_spku",
"target_type": "minio_s3"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-0405",
"type": "customNode",
"position": {
"x": 270,
"y": 80
},
"data": {
"label": "GE Quality Check",
"type": "ge_quality",
"subtitle": "Great Expectations",
"configured": true,
"config": {
"ge_scope_level": "all",
"ge_table_name": "staging_environmental_permits",
"test_table_row_count": true,
"test_table_schema_match": true,
"test_table_table_not_empty": true,
"table_min_rows": 1,
"table_max_rows": 1000000,
"ge_col_table": "stasiun_spku",
"ge_target_column": "stasiun_id",
"ge_column_test_type": "not_null",
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"expected": 0,
"error_message": "Unexpected rows detected in table",
"suite": "bronze",
"fail": "Stop DAG"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-1406",
"type": "customNode",
"position": {
"x": 460,
"y": 80
},
"data": {
"label": "DuckDB Transform",
"type": "duckdb_transform",
"subtitle": "In-process SQL (<512MB)",
"configured": true,
"config": {
"layer": "silver",
"source_parquet_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
"source_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
"target_table": "dlh_lakehouse.silver.environmental_permits",
"table_name": "dlh_lakehouse.silver.environmental_permits",
"primary_keys": "id, updated_at",
"query": "SELECT * FROM source_tbl",
"sql_query": "SELECT * FROM source_parquet",
"transformation_sql": "SELECT permit_id, permit_number, company_name, business_type, location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-2426",
"type": "customNode",
"position": {
"x": 650,
"y": 80
},
"data": {
"label": "GE Quality Check",
"type": "ge_quality",
"subtitle": "Great Expectations",
"configured": true,
"config": {
"ge_scope_level": "all",
"ge_table_name": "bronze_environmental_permits",
"test_table_row_count": true,
"test_table_schema_match": true,
"test_table_table_not_empty": true,
"table_min_rows": 1,
"table_max_rows": 1000000,
"ge_col_table": "stasiun_spku",
"ge_target_column": "stasiun_id",
"ge_column_test_type": "not_null",
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"expected": 0,
"error_message": "Unexpected rows detected in table",
"suite": "bronze",
"fail": "Stop DAG"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-3473",
"type": "customNode",
"position": {
"x": 840,
"y": 80
},
"data": {
"label": "dbt Transform",
"type": "dbt_transform",
"subtitle": "dbt-postgres",
"configured": true,
"config": {
"dbt_layer": "silver",
"dbt_model": "stg_spku_realtime",
"sel": "silver",
"tgt": "postgres",
"trino_catalog": "iceberg",
"sql_query_preview": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type",
"query": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-4489",
"type": "customNode",
"position": {
"x": 1030,
"y": 80
},
"data": {
"label": "GE Quality Check",
"type": "ge_quality",
"subtitle": "Great Expectations",
"configured": true,
"config": {
"ge_scope_level": "all",
"ge_table_name": "silver_environmental_compliance",
"test_table_row_count": true,
"test_table_schema_match": true,
"test_table_table_not_empty": true,
"table_min_rows": 1,
"table_max_rows": 1000000,
"ge_col_table": "stasiun_spku",
"ge_target_column": "stasiun_id",
"ge_column_test_type": "not_null",
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"expected": 0,
"error_message": "Unexpected rows detected in table",
"suite": "bronze",
"fail": "Stop DAG"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-5491",
"type": "customNode",
"position": {
"x": 1220,
"y": 80
},
"data": {
"label": "dbt Transform",
"type": "dbt_transform",
"subtitle": "dbt-postgres",
"configured": true,
"config": {
"dbt_layer": "silver",
"dbt_model": "stg_spku_realtime",
"sel": "silver",
"tgt": "postgres",
"trino_catalog": "iceberg",
"sql_query_preview": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC",
"query": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-6528",
"type": "customNode",
"position": {
"x": 1410,
"y": 80
},
"data": {
"label": "GE Quality Check",
"type": "ge_quality",
"subtitle": "Great Expectations",
"configured": true,
"config": {
"ge_scope_level": "all",
"ge_table_name": "gold_environmental_risk_summary",
"test_table_row_count": true,
"test_table_schema_match": true,
"test_table_table_not_empty": true,
"table_min_rows": 1,
"table_max_rows": 1000000,
"ge_col_table": "stasiun_spku",
"ge_target_column": "stasiun_id",
"ge_column_test_type": "not_null",
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"expected": 0,
"error_message": "Unexpected rows detected in table",
"suite": "bronze",
"fail": "Stop DAG"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-7512",
"type": "customNode",
"position": {
"x": 1600,
"y": 80
},
"data": {
"label": "Sitomat Callback",
"type": "sitomat_callback",
"subtitle": "Status callback",
"configured": true,
"config": {
"layer": "bronze",
"artifact_ref": "lakehouse.bronze.dataset_cdc",
"checksum": "sha256-verified-pipeline-pass",
"sitomat_base_url": "http://localhost:3000",
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
"ge_status": "PASSED",
"url": "/sitomat/callback",
"stage": "bronze"
},
"status": "warn",
"errorMessage": "Disconnected from workflow",
"missingFields": [],
"hasError": false,
"hasWarning": true
},
"width": 96,
"height": 96,
"selected": true,
"dragging": false
}
],
"edges": [
{
"source": "node-9377",
"sourceHandle": null,
"target": "node-0405",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-9377-node-0405"
},
{
"source": "node-0405",
"sourceHandle": null,
"target": "node-1406",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-0405-node-1406"
},
{
"source": "node-1406",
"sourceHandle": null,
"target": "node-2426",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-1406-node-2426"
},
{
"source": "node-2426",
"sourceHandle": null,
"target": "node-3473",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-2426-node-3473"
},
{
"source": "node-3473",
"sourceHandle": null,
"target": "node-4489",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-3473-node-4489"
},
{
"source": "node-4489",
"sourceHandle": null,
"target": "node-5491",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-4489-node-5491"
},
{
"source": "node-5491",
"sourceHandle": null,
"target": "node-6528",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-5491-node-6528"
},
{
"source": "node-6528",
"sourceHandle": null,
"target": "node-7512",
"targetHandle": null,
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-6528-node-7512"
}
],
"notes": [],
"is_draft": false,
"status": "deployed",
"updated_at": "2026-10-18T17:53:01.689174Z"
}
metadata/example_permits_medallion_dbt_postgres.draft.json
+304
View File
@@ -0,0 +1,304 @@
{
"id": "example_permits_medallion_dbt_postgres",
"description": "Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver -> dbt-postgres Gold Daily Mart",
"schedule": "0 8 * * *",
"owner_id": "local-admin",
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-bronze-src",
"type": "customNode",
"position": {
"x": 60,
"y": 60
},
"data": {
"label": "Bronze DB Source (CDC Landing)",
"type": "source_db",
"configured": true,
"config": {
"source_type": "database",
"connection_id": "conn_postgres_source_db",
"table_name": "public.environmental_permits",
"target_table": "bronze_lakehouse.environmental_permits_cdc"
}
},
"width": 96,
"height": 96
},
{
"id": "node-ge-bronze",
"type": "customNode",
"position": {
"x": 250,
"y": 60
},
"data": {
"label": "GE Quality Check (Bronze Suite)",
"type": "ge_quality",
"configured": true,
"config": {
"expectation_suite": "permits_bronze_suite",
"table_name": "bronze_lakehouse.environmental_permits_cdc",
"rules": [
{
"type": "table_row_count_to_be_between",
"min_value": 1
},
{
"type": "column_values_to_not_be_null",
"column": "permit_id"
}
],
"queries": [
{
"name": "check_bronze_null_permit_id",
"sql": "SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id IS NULL",
"expected": 0,
"error_message": "Null permit_id detected in Bronze"
}
]
}
},
"width": 96,
"height": 96
},
{
"id": "node-sitomat-bronze",
"type": "customNode",
"position": {
"x": 440,
"y": 60
},
"data": {
"label": "Sitomat Bronze Callback",
"type": "sitomat_callback",
"configured": true,
"config": {
"layer": "bronze",
"artifact_ref": "dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc",
"checksum": "auto-sha256"
}
},
"width": 96,
"height": 96,
"selected": false,
"dragging": false
},
{
"id": "node-silver-dbt",
"type": "customNode",
"position": {
"x": 630,
"y": 60
},
"data": {
"label": "dbt Silver Cleanse (dbt-postgres)",
"type": "dbt_transform",
"configured": true,
"config": {
"dbt_layer": "silver",
"dbt_model": "silver_environmental_permits_clean",
"target_table": "dlh_lakehouse.silver.silver_environmental_permits_clean"
}
},
"width": 96,
"height": 96
},
{
"id": "node-ge-silver",
"type": "customNode",
"position": {
"x": 820,
"y": 60
},
"data": {
"label": "GE Quality Check (Silver Suite)",
"type": "ge_quality",
"configured": true,
"config": {
"expectation_suite": "permits_silver_suite",
"table_name": "silver.silver_environmental_permits_clean",
"rules": [
{
"type": "column_values_to_not_be_null",
"column": "permit_id"
},
{
"type": "column_values_to_be_in_set",
"column": "permit_status",
"allowed_set": [
"APPROVED",
"IN_REVIEW",
"REJECTED"
]
}
],
"queries": [
{
"name": "check_invalid_status_count",
"sql": "SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
"expected": 0,
"error_message": "Invalid status detected in Silver"
}
]
}
},
"width": 96,
"height": 96
},
{
"id": "node-sitomat-silver",
"type": "customNode",
"position": {
"x": 1010,
"y": 60
},
"data": {
"label": "Sitomat Silver Callback",
"type": "sitomat_callback",
"configured": true,
"config": {
"layer": "silver",
"artifact_ref": "dlh_lakehouse.silver.silver_environmental_permits_clean",
"checksum": "auto-sha256"
}
},
"width": 96,
"height": 96
},
{
"id": "node-gold-dbt",
"type": "customNode",
"position": {
"x": 1200,
"y": 60
},
"data": {
"label": "dbt Gold Mart (dbt-postgres)",
"type": "dbt_transform",
"configured": true,
"config": {
"dbt_layer": "gold",
"dbt_model": "gold_permits_summary_daily",
"target_table": "dlh_lakehouse.gold.gold_permits_summary_daily"
}
},
"width": 96,
"height": 96
},
{
"id": "node-sitomat-gold",
"type": "customNode",
"position": {
"x": 1390,
"y": 60
},
"data": {
"label": "Sitomat Gold Callback (Data Product)",
"type": "sitomat_callback",
"configured": true,
"config": {
"layer": "gold",
"artifact_ref": "dlh_lakehouse.gold.gold_permits_summary_daily",
"checksum": "auto-sha256"
}
},
"width": 96,
"height": 96
}
],
"edges": [
{
"id": "e1-src-ge_bronze",
"source": "node-bronze-src",
"target": "node-ge-bronze",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e2-ge_bronze-sitomat_bronze",
"source": "node-ge-bronze",
"target": "node-sitomat-bronze",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e3-sitomat_bronze-silver",
"source": "node-sitomat-bronze",
"target": "node-silver-dbt",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e4-silver-ge_silver",
"source": "node-silver-dbt",
"target": "node-ge-silver",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e5-ge_silver-sitomat_silver",
"source": "node-ge-silver",
"target": "node-sitomat-silver",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e6-sitomat_silver-gold",
"source": "node-sitomat-silver",
"target": "node-gold-dbt",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
},
{
"id": "e7-gold-sitomat_gold",
"source": "node-gold-dbt",
"target": "node-sitomat-gold",
"type": "removableEdge",
"style": {
"stroke": "var(--muted)",
"strokeWidth": 2
},
"data": {}
}
],
"notes": [
{
"id": "note-7937",
"text": "New note\nDouble-click to edit this note.",
"x": 190,
"y": -10,
"w": 300,
"h": 200,
"c": 0
}
],
"is_draft": true,
"status": "draft",
"updated_at": "2026-10-18T09:45:33.620665Z"
}
metadata/ingestion_spku_realtime_lakehouse.draft.json
+46
View File
@@ -0,0 +1,46 @@
{
"id": "ingestion_spku_realtime_lakehouse",
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
"schedule": "0 */1 * * *",
"owner_id": "local-admin",
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-7560",
"type": "customNode",
"position": {
"x": 80,
"y": 80
},
"data": {
"label": "dbt Transform",
"type": "dbt_transform",
"subtitle": "dbt-postgres",
"configured": true,
"config": {
"dbt_layer": "silver",
"dbt_model": "my_custom_model_silver",
"sel": "my_custom_model_silver",
"tgt": "postgres",
"trino_catalog": "iceberg",
"sql_query_preview": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}",
"query": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}"
},
"status": "ok",
"errorMessage": "Configured & Valid",
"missingFields": [],
"hasError": false,
"hasWarning": false
},
"width": 96,
"height": 96,
"selected": true,
"dragging": false
}
],
"edges": [],
"notes": [],
"is_draft": true,
"status": "draft",
"updated_at": "2026-10-18T17:54:28.688466Z"
}
metadata/rbac_mappings.json
+27
View File
@@ -0,0 +1,27 @@
{
"editor_roles": [
"data_engineer",
"pranata_komputer_ahli"
],
"editor_groups": [
"/Data Engineers",
"/DLH-Developers"
],
"viewer_roles": [
"merica_viewer",
"auditor"
],
"viewer_groups": [
"/Pimpinan Unit Kerja",
"/Auditor Internal BPKP"
],
"admin_roles": [
"merica_admin",
"superadmin"
],
"admin_groups": [
"/Platform Administrators"
],
"updated_at": "2026-10-18T20:17:09.212620+00:00",
"updated_by": "admin"
}