445 lines
15 KiB
JSON
445 lines
15 KiB
JSON
{
|
|
"id": "e2e_medallion_permits_flow",
|
|
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
|
"schedule": "0 */1 * * *",
|
|
"owner_id": "local-admin",
|
|
"owner_name": "Administrator Lokal (Break-Glass)",
|
|
"nodes": [
|
|
{
|
|
"id": "node-9377",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 80,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "Source Database",
|
|
"type": "source_db",
|
|
"subtitle": "Postgres / MySQL / Oracle",
|
|
"isSource": true,
|
|
"configured": true,
|
|
"config": {
|
|
"conn": "Postgres",
|
|
"conn_id": "conn_postgres_source_db",
|
|
"table": "permits",
|
|
"table_name": "public.stasiun_spku",
|
|
"target_type": "minio_s3"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-0405",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 270,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "GE Quality Check",
|
|
"type": "ge_quality",
|
|
"subtitle": "Great Expectations",
|
|
"configured": true,
|
|
"config": {
|
|
"ge_scope_level": "all",
|
|
"ge_table_name": "staging_environmental_permits",
|
|
"test_table_row_count": true,
|
|
"test_table_schema_match": true,
|
|
"test_table_table_not_empty": true,
|
|
"table_min_rows": 1,
|
|
"table_max_rows": 1000000,
|
|
"ge_col_table": "stasiun_spku",
|
|
"ge_target_column": "stasiun_id",
|
|
"ge_column_test_type": "not_null",
|
|
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"expected": 0,
|
|
"error_message": "Unexpected rows detected in table",
|
|
"suite": "bronze",
|
|
"fail": "Stop DAG"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-1406",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 460,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "DuckDB Transform",
|
|
"type": "duckdb_transform",
|
|
"subtitle": "In-process SQL (<512MB)",
|
|
"configured": true,
|
|
"config": {
|
|
"layer": "silver",
|
|
"source_parquet_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
|
"source_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
|
"target_table": "dlh_lakehouse.silver.environmental_permits",
|
|
"table_name": "dlh_lakehouse.silver.environmental_permits",
|
|
"primary_keys": "id, updated_at",
|
|
"query": "SELECT * FROM source_tbl",
|
|
"sql_query": "SELECT * FROM source_parquet",
|
|
"transformation_sql": "SELECT permit_id, permit_number, company_name, business_type, location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-2426",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 650,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "GE Quality Check",
|
|
"type": "ge_quality",
|
|
"subtitle": "Great Expectations",
|
|
"configured": true,
|
|
"config": {
|
|
"ge_scope_level": "all",
|
|
"ge_table_name": "bronze_environmental_permits",
|
|
"test_table_row_count": true,
|
|
"test_table_schema_match": true,
|
|
"test_table_table_not_empty": true,
|
|
"table_min_rows": 1,
|
|
"table_max_rows": 1000000,
|
|
"ge_col_table": "stasiun_spku",
|
|
"ge_target_column": "stasiun_id",
|
|
"ge_column_test_type": "not_null",
|
|
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"expected": 0,
|
|
"error_message": "Unexpected rows detected in table",
|
|
"suite": "bronze",
|
|
"fail": "Stop DAG"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-3473",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 840,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "dbt Transform",
|
|
"type": "dbt_transform",
|
|
"subtitle": "dbt-postgres",
|
|
"configured": true,
|
|
"config": {
|
|
"dbt_layer": "silver",
|
|
"dbt_model": "stg_spku_realtime",
|
|
"sel": "silver",
|
|
"tgt": "postgres",
|
|
"trino_catalog": "iceberg",
|
|
"sql_query_preview": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type",
|
|
"query": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-4489",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 1030,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "GE Quality Check",
|
|
"type": "ge_quality",
|
|
"subtitle": "Great Expectations",
|
|
"configured": true,
|
|
"config": {
|
|
"ge_scope_level": "all",
|
|
"ge_table_name": "silver_environmental_compliance",
|
|
"test_table_row_count": true,
|
|
"test_table_schema_match": true,
|
|
"test_table_table_not_empty": true,
|
|
"table_min_rows": 1,
|
|
"table_max_rows": 1000000,
|
|
"ge_col_table": "stasiun_spku",
|
|
"ge_target_column": "stasiun_id",
|
|
"ge_column_test_type": "not_null",
|
|
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"expected": 0,
|
|
"error_message": "Unexpected rows detected in table",
|
|
"suite": "bronze",
|
|
"fail": "Stop DAG"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-5491",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 1220,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "dbt Transform",
|
|
"type": "dbt_transform",
|
|
"subtitle": "dbt-postgres",
|
|
"configured": true,
|
|
"config": {
|
|
"dbt_layer": "silver",
|
|
"dbt_model": "stg_spku_realtime",
|
|
"sel": "silver",
|
|
"tgt": "postgres",
|
|
"trino_catalog": "iceberg",
|
|
"sql_query_preview": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC",
|
|
"query": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-6528",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 1410,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "GE Quality Check",
|
|
"type": "ge_quality",
|
|
"subtitle": "Great Expectations",
|
|
"configured": true,
|
|
"config": {
|
|
"ge_scope_level": "all",
|
|
"ge_table_name": "gold_environmental_risk_summary",
|
|
"test_table_row_count": true,
|
|
"test_table_schema_match": true,
|
|
"test_table_table_not_empty": true,
|
|
"table_min_rows": 1,
|
|
"table_max_rows": 1000000,
|
|
"ge_col_table": "stasiun_spku",
|
|
"ge_target_column": "stasiun_id",
|
|
"ge_column_test_type": "not_null",
|
|
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
|
"expected": 0,
|
|
"error_message": "Unexpected rows detected in table",
|
|
"suite": "bronze",
|
|
"fail": "Stop DAG"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": false,
|
|
"dragging": false
|
|
},
|
|
{
|
|
"id": "node-7512",
|
|
"type": "customNode",
|
|
"position": {
|
|
"x": 1600,
|
|
"y": 80
|
|
},
|
|
"data": {
|
|
"label": "Sitomat Callback",
|
|
"type": "sitomat_callback",
|
|
"subtitle": "Status callback",
|
|
"configured": true,
|
|
"config": {
|
|
"layer": "bronze",
|
|
"artifact_ref": "lakehouse.bronze.dataset_cdc",
|
|
"checksum": "sha256-verified-pipeline-pass",
|
|
"sitomat_base_url": "http://localhost:3000",
|
|
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
|
"ge_status": "PASSED",
|
|
"url": "/sitomat/callback",
|
|
"stage": "bronze"
|
|
},
|
|
"status": "warn",
|
|
"errorMessage": "Disconnected from workflow",
|
|
"missingFields": [],
|
|
"hasError": false,
|
|
"hasWarning": true
|
|
},
|
|
"width": 96,
|
|
"height": 96,
|
|
"selected": true,
|
|
"dragging": false
|
|
}
|
|
],
|
|
"edges": [
|
|
{
|
|
"source": "node-9377",
|
|
"sourceHandle": null,
|
|
"target": "node-0405",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-9377-node-0405"
|
|
},
|
|
{
|
|
"source": "node-0405",
|
|
"sourceHandle": null,
|
|
"target": "node-1406",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-0405-node-1406"
|
|
},
|
|
{
|
|
"source": "node-1406",
|
|
"sourceHandle": null,
|
|
"target": "node-2426",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-1406-node-2426"
|
|
},
|
|
{
|
|
"source": "node-2426",
|
|
"sourceHandle": null,
|
|
"target": "node-3473",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-2426-node-3473"
|
|
},
|
|
{
|
|
"source": "node-3473",
|
|
"sourceHandle": null,
|
|
"target": "node-4489",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-3473-node-4489"
|
|
},
|
|
{
|
|
"source": "node-4489",
|
|
"sourceHandle": null,
|
|
"target": "node-5491",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-4489-node-5491"
|
|
},
|
|
{
|
|
"source": "node-5491",
|
|
"sourceHandle": null,
|
|
"target": "node-6528",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-5491-node-6528"
|
|
},
|
|
{
|
|
"source": "node-6528",
|
|
"sourceHandle": null,
|
|
"target": "node-7512",
|
|
"targetHandle": null,
|
|
"type": "removableEdge",
|
|
"style": {
|
|
"stroke": "var(--muted)",
|
|
"strokeWidth": 2
|
|
},
|
|
"data": {},
|
|
"id": "reactflow__edge-node-6528-node-7512"
|
|
}
|
|
],
|
|
"notes": [],
|
|
"is_draft": false,
|
|
"status": "deployed",
|
|
"updated_at": "2026-10-18T17:53:01.689174Z"
|
|
} |