{ "id": "e2e_medallion_permits_flow", "description": "Dynamic Data Lakehouse Pipeline via DLH Studio", "schedule": "0 */1 * * *", "owner_id": "local-admin", "owner_name": "Administrator Lokal (Break-Glass)", "nodes": [ { "id": "node-9377", "type": "customNode", "position": { "x": 80, "y": 80 }, "data": { "label": "Source Database", "type": "source_db", "subtitle": "Postgres / MySQL / Oracle", "isSource": true, "configured": true, "config": { "conn": "Postgres", "conn_id": "conn_postgres_source_db", "table": "permits", "table_name": "public.stasiun_spku", "target_type": "minio_s3" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-0405", "type": "customNode", "position": { "x": 270, "y": 80 }, "data": { "label": "GE Quality Check", "type": "ge_quality", "subtitle": "Great Expectations", "configured": true, "config": { "ge_scope_level": "all", "ge_table_name": "staging_environmental_permits", "test_table_row_count": true, "test_table_schema_match": true, "test_table_table_not_empty": true, "table_min_rows": 1, "table_max_rows": 1000000, "ge_col_table": "stasiun_spku", "ge_target_column": "stasiun_id", "ge_column_test_type": "not_null", "sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "expected": 0, "error_message": "Unexpected rows detected in table", "suite": "bronze", "fail": "Stop DAG" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-1406", "type": "customNode", "position": { "x": 460, "y": 80 }, "data": { "label": "DuckDB Transform", "type": "duckdb_transform", "subtitle": "In-process SQL (<512MB)", "configured": true, "config": { "layer": "silver", "source_parquet_path": "s3://dlh-bronze/db_cdc/environmental_permits/", "source_path": "s3://dlh-bronze/db_cdc/environmental_permits/", "target_table": "dlh_lakehouse.silver.environmental_permits", "table_name": "dlh_lakehouse.silver.environmental_permits", "primary_keys": "id, updated_at", "query": "SELECT * FROM source_tbl", "sql_query": "SELECT * FROM source_parquet", "transformation_sql": "SELECT permit_id, permit_number, company_name, business_type, location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-2426", "type": "customNode", "position": { "x": 650, "y": 80 }, "data": { "label": "GE Quality Check", "type": "ge_quality", "subtitle": "Great Expectations", "configured": true, "config": { "ge_scope_level": "all", "ge_table_name": "bronze_environmental_permits", "test_table_row_count": true, "test_table_schema_match": true, "test_table_table_not_empty": true, "table_min_rows": 1, "table_max_rows": 1000000, "ge_col_table": "stasiun_spku", "ge_target_column": "stasiun_id", "ge_column_test_type": "not_null", "sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "expected": 0, "error_message": "Unexpected rows detected in table", "suite": "bronze", "fail": "Stop DAG" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-3473", "type": "customNode", "position": { "x": 840, "y": 80 }, "data": { "label": "dbt Transform", "type": "dbt_transform", "subtitle": "dbt-postgres", "configured": true, "config": { "dbt_layer": "silver", "dbt_model": "stg_spku_realtime", "sel": "silver", "tgt": "postgres", "trino_catalog": "iceberg", "sql_query_preview": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type", "query": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-4489", "type": "customNode", "position": { "x": 1030, "y": 80 }, "data": { "label": "GE Quality Check", "type": "ge_quality", "subtitle": "Great Expectations", "configured": true, "config": { "ge_scope_level": "all", "ge_table_name": "silver_environmental_compliance", "test_table_row_count": true, "test_table_schema_match": true, "test_table_table_not_empty": true, "table_min_rows": 1, "table_max_rows": 1000000, "ge_col_table": "stasiun_spku", "ge_target_column": "stasiun_id", "ge_column_test_type": "not_null", "sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "expected": 0, "error_message": "Unexpected rows detected in table", "suite": "bronze", "fail": "Stop DAG" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-5491", "type": "customNode", "position": { "x": 1220, "y": 80 }, "data": { "label": "dbt Transform", "type": "dbt_transform", "subtitle": "dbt-postgres", "configured": true, "config": { "dbt_layer": "silver", "dbt_model": "stg_spku_realtime", "sel": "silver", "tgt": "postgres", "trino_catalog": "iceberg", "sql_query_preview": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC", "query": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-6528", "type": "customNode", "position": { "x": 1410, "y": 80 }, "data": { "label": "GE Quality Check", "type": "ge_quality", "subtitle": "Great Expectations", "configured": true, "config": { "ge_scope_level": "all", "ge_table_name": "gold_environmental_risk_summary", "test_table_row_count": true, "test_table_schema_match": true, "test_table_table_not_empty": true, "table_min_rows": 1, "table_max_rows": 1000000, "ge_col_table": "stasiun_spku", "ge_target_column": "stasiun_id", "ge_column_test_type": "not_null", "sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "expected": 0, "error_message": "Unexpected rows detected in table", "suite": "bronze", "fail": "Stop DAG" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-7512", "type": "customNode", "position": { "x": 1600, "y": 80 }, "data": { "label": "Sitomat Callback", "type": "sitomat_callback", "subtitle": "Status callback", "configured": true, "config": { "layer": "bronze", "artifact_ref": "lakehouse.bronze.dataset_cdc", "checksum": "sha256-verified-pipeline-pass", "sitomat_base_url": "http://localhost:3000", "openmetadata_fqn": "dlh_airflow_service.pipeline.task", "ge_status": "PASSED", "url": "/sitomat/callback", "stage": "bronze" }, "status": "warn", "errorMessage": "Disconnected from workflow", "missingFields": [], "hasError": false, "hasWarning": true }, "width": 96, "height": 96, "selected": true, "dragging": false } ], "edges": [ { "source": "node-9377", "sourceHandle": null, "target": "node-0405", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-9377-node-0405" }, { "source": "node-0405", "sourceHandle": null, "target": "node-1406", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-0405-node-1406" }, { "source": "node-1406", "sourceHandle": null, "target": "node-2426", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-1406-node-2426" }, { "source": "node-2426", "sourceHandle": null, "target": "node-3473", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-2426-node-3473" }, { "source": "node-3473", "sourceHandle": null, "target": "node-4489", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-3473-node-4489" }, { "source": "node-4489", "sourceHandle": null, "target": "node-5491", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-4489-node-5491" }, { "source": "node-5491", "sourceHandle": null, "target": "node-6528", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-5491-node-6528" }, { "source": "node-6528", "sourceHandle": null, "target": "node-7512", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-6528-node-7512" } ], "notes": [], "is_draft": false, "status": "deployed", "updated_at": "2026-10-18T17:53:01.689174Z" }