feat(appliance): initial release of MERICA standalone appliance v1.0.0
This commit is contained in:
@@ -0,0 +1,445 @@
|
||||
{
|
||||
"id": "e2e_medallion_permits_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-9377",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source Database",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "permits",
|
||||
"table_name": "public.stasiun_spku",
|
||||
"target_type": "minio_s3"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0405",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "staging_environmental_permits",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-1406",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Transform",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
||||
"source_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
||||
"target_table": "dlh_lakehouse.silver.environmental_permits",
|
||||
"table_name": "dlh_lakehouse.silver.environmental_permits",
|
||||
"primary_keys": "id, updated_at",
|
||||
"query": "SELECT * FROM source_tbl",
|
||||
"sql_query": "SELECT * FROM source_parquet",
|
||||
"transformation_sql": "SELECT permit_id, permit_number, company_name, business_type, location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2426",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "bronze_environmental_permits",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3473",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Transform",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "stg_spku_realtime",
|
||||
"sel": "silver",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type",
|
||||
"query": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4489",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "silver_environmental_compliance",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-5491",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Transform",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "stg_spku_realtime",
|
||||
"sel": "silver",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC",
|
||||
"query": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6528",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "gold_environmental_risk_summary",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-7512",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1600,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Sitomat Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "bronze",
|
||||
"artifact_ref": "lakehouse.bronze.dataset_cdc",
|
||||
"checksum": "sha256-verified-pipeline-pass",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-9377",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0405",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9377-node-0405"
|
||||
},
|
||||
{
|
||||
"source": "node-0405",
|
||||
"sourceHandle": null,
|
||||
"target": "node-1406",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0405-node-1406"
|
||||
},
|
||||
{
|
||||
"source": "node-1406",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2426",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-1406-node-2426"
|
||||
},
|
||||
{
|
||||
"source": "node-2426",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3473",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2426-node-3473"
|
||||
},
|
||||
{
|
||||
"source": "node-3473",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4489",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3473-node-4489"
|
||||
},
|
||||
{
|
||||
"source": "node-4489",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5491",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-4489-node-5491"
|
||||
},
|
||||
{
|
||||
"source": "node-5491",
|
||||
"sourceHandle": null,
|
||||
"target": "node-6528",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5491-node-6528"
|
||||
},
|
||||
{
|
||||
"source": "node-6528",
|
||||
"sourceHandle": null,
|
||||
"target": "node-7512",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6528-node-7512"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-18T17:53:01.689174Z"
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
{
|
||||
"id": "e2e_modernized_ingestion_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-5630",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source Database (Environmental Permits)",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.environmental_permits",
|
||||
"table_name": "public.environmental_permits",
|
||||
"target_type": "minio_s3",
|
||||
"watermark_column": "updated_at",
|
||||
"ingestion_mode": "incremental_watermark"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6855",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source GWS (Waste Reports)",
|
||||
"type": "source_gws",
|
||||
"subtitle": "Sheets & Drive PDF",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"url": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||
"spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||
"sheet_range": "Sheet1",
|
||||
"sheet_name": "Sheet1",
|
||||
"folder_id": "1OSRugXZvfjoY9M0yySKT",
|
||||
"drive_folder_id": "1OSRugXZvfjoY9M0yySKT",
|
||||
"selected_filename": "Laporan_Sampah_Kabupaten_2026.xlsx",
|
||||
"header_row": 2
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-8156",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source REST API dlt (Sensor SPKU)",
|
||||
"type": "source_api_dlt",
|
||||
"subtitle": "Auto Un-nesting & Pagination",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"endpoint_url": "https://api.spku.dlh.id/v1/telemetry",
|
||||
"http_method": "GET",
|
||||
"auth_type": "none",
|
||||
"auth_token": "",
|
||||
"pagination_strategy": "page_number",
|
||||
"page_param": "page",
|
||||
"limit_param": "limit",
|
||||
"page_size": 100,
|
||||
"cursor_column": "measured_at",
|
||||
"lookback_hours": 3,
|
||||
"unnest_mode": "auto_flatten",
|
||||
"landing_bucket": "dlh-landing",
|
||||
"landing_prefix": "api/spku_hourly/",
|
||||
"bronze_table": "bronze_spku_telemetry",
|
||||
"url": "https://api.spku.dlh.id/v1/telemetry"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-9404",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Bronze Ingestion & Dedup",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet",
|
||||
"target_table": "bronze_environmental_permits",
|
||||
"table_name": "bronze_environmental_permits",
|
||||
"primary_keys": "permit_id",
|
||||
"query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
|
||||
"sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
|
||||
"transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0708",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Cleansed Permits Enriched",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "silver_permits_enriched",
|
||||
"sel": "silver_permits_enriched",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'",
|
||||
"query": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'",
|
||||
"test_configs": {
|
||||
"unique_columns": [
|
||||
"permit_id",
|
||||
"permit_number"
|
||||
],
|
||||
"not_null_columns": [
|
||||
"permit_id",
|
||||
"status",
|
||||
"updated_at"
|
||||
],
|
||||
"accepted_values": {},
|
||||
"enforce_contract": true
|
||||
},
|
||||
"unique_columns": [
|
||||
"permit_id",
|
||||
"permit_number"
|
||||
],
|
||||
"not_null_columns": [
|
||||
"permit_id",
|
||||
"status",
|
||||
"updated_at"
|
||||
],
|
||||
"enforce_contract": true,
|
||||
"use_cosmos": true,
|
||||
"transformation_engine": "cosmos"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2234",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Silver Compliance Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "table",
|
||||
"ge_table_name": "silver_permits_enriched",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "",
|
||||
"sql": "",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3694",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Regulatory Compliance Mart",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "gold_compliance_summary_mart",
|
||||
"sel": "gold_compliance_summary_mart",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
|
||||
"query": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
|
||||
"test_configs": {
|
||||
"unique_columns": [],
|
||||
"not_null_columns": [
|
||||
"permit_type",
|
||||
"status"
|
||||
],
|
||||
"accepted_values": {},
|
||||
"enforce_contract": false
|
||||
},
|
||||
"unique_columns": [],
|
||||
"not_null_columns": [
|
||||
"permit_type",
|
||||
"status"
|
||||
],
|
||||
"use_cosmos": true,
|
||||
"transformation_engine": "cosmos"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4977",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "SITOMAT Governance Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "gold",
|
||||
"artifact_ref": "lakehouse.gold.gold_compliance_summary_mart",
|
||||
"checksum": "sha256-verified-modernized-ingestion",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-5630",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5630-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-6855",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6855-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-8156",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-8156-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-9404",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0708",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9404-node-0708"
|
||||
},
|
||||
{
|
||||
"source": "node-0708",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2234",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0708-node-2234"
|
||||
},
|
||||
{
|
||||
"source": "node-2234",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3694",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2234-node-3694"
|
||||
},
|
||||
{
|
||||
"source": "node-3694",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4977",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3694-node-4977"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-21T08:59:34.908289Z"
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
{
|
||||
"id": "e2e_orders_master_medallion_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-9826",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Extract Master Facilities",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.ref_waste_facilities",
|
||||
"table_name": "public.ref_waste_facilities",
|
||||
"target_type": "minio_s3"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0827",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Master Facilities to Iceberg Bronze",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/ref_facilities/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/ref_facilities/*.parquet",
|
||||
"target_table": "bronze_ref_facilities",
|
||||
"table_name": "bronze_ref_facilities",
|
||||
"primary_keys": "facility_id",
|
||||
"query": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1",
|
||||
"sql_query": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1",
|
||||
"transformation_sql": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-1812",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Extract Raw Waste Orders",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.raw_waste_orders",
|
||||
"table_name": "public.raw_waste_orders",
|
||||
"target_type": "minio_s3"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2810",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Orders Staging Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "staging_waste_orders",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Orders to Iceberg Bronze",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/waste_orders/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/waste_orders/*.parquet",
|
||||
"target_table": "bronze_waste_orders",
|
||||
"table_name": "bronze_waste_orders",
|
||||
"primary_keys": "order_id",
|
||||
"query": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1",
|
||||
"sql_query": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1",
|
||||
"transformation_sql": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Orders Bronze Iceberg Quality",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "bronze_waste_orders",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-5809",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Waste Manifest Enriched",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "silver_waste_manifest_enriched",
|
||||
"sel": "silver_waste_manifest_enriched",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\nFROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities') }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'",
|
||||
"query": "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\nFROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities') }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Silver Compliance Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "silver_waste_manifest_enriched",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-7793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1600,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Regional Waste Revenue Mart",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "gold_regional_waste_metrics",
|
||||
"sel": "gold_regional_waste_metrics",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type, waste_category\nORDER BY total_waste_volume_ton DESC",
|
||||
"query": "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type, waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-8871",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1790,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Gold Mart Integrity Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "gold_regional_waste_metrics",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-9855",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1980,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "SITOMAT Governance Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "gold",
|
||||
"artifact_ref": "lakehouse.gold.gold_regional_waste_metrics",
|
||||
"checksum": "sha256-verified-waste-orders-pass",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-9826",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0827",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9826-node-0827"
|
||||
},
|
||||
{
|
||||
"source": "node-0827",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5809",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0827-node-5809"
|
||||
},
|
||||
{
|
||||
"source": "node-1812",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2810",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-1812-node-2810"
|
||||
},
|
||||
{
|
||||
"source": "node-2810",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2810-node-3793"
|
||||
},
|
||||
{
|
||||
"source": "node-3793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3793-node-4793"
|
||||
},
|
||||
{
|
||||
"source": "node-4793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5809",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-4793-node-5809"
|
||||
},
|
||||
{
|
||||
"source": "node-5809",
|
||||
"sourceHandle": null,
|
||||
"target": "node-6793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5809-node-6793"
|
||||
},
|
||||
{
|
||||
"source": "node-6793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-7793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6793-node-7793"
|
||||
},
|
||||
{
|
||||
"source": "node-7793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-8871",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-7793-node-8871"
|
||||
},
|
||||
{
|
||||
"source": "node-8871",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9855",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-8871-node-9855"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-18T18:28:07.599927Z"
|
||||
}
|
||||
Reference in New Issue
Block a user