{ "id": "e2e_modernized_ingestion_flow", "description": "Dynamic Data Lakehouse Pipeline via DLH Studio", "schedule": "0 */1 * * *", "owner_id": "local-admin", "owner_name": "Administrator Lokal (Break-Glass)", "nodes": [ { "id": "node-5630", "type": "customNode", "position": { "x": 80, "y": 80 }, "data": { "label": "Source Database (Environmental Permits)", "type": "source_db", "subtitle": "Postgres / MySQL / Oracle", "isSource": true, "configured": true, "config": { "conn": "Postgres", "conn_id": "conn_postgres_source_db", "table": "public.environmental_permits", "table_name": "public.environmental_permits", "target_type": "minio_s3", "watermark_column": "updated_at", "ingestion_mode": "incremental_watermark" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-6855", "type": "customNode", "position": { "x": 270, "y": 80 }, "data": { "label": "Source GWS (Waste Reports)", "type": "source_gws", "subtitle": "Sheets & Drive PDF", "isSource": true, "configured": true, "config": { "url": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026", "spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026", "sheet_range": "Sheet1", "sheet_name": "Sheet1", "folder_id": "1OSRugXZvfjoY9M0yySKT", "drive_folder_id": "1OSRugXZvfjoY9M0yySKT", "selected_filename": "Laporan_Sampah_Kabupaten_2026.xlsx", "header_row": 2 } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-8156", "type": "customNode", "position": { "x": 460, "y": 80 }, "data": { "label": "Source REST API dlt (Sensor SPKU)", "type": "source_api_dlt", "subtitle": "Auto Un-nesting & Pagination", "isSource": true, "configured": true, "config": { "endpoint_url": "https://api.spku.dlh.id/v1/telemetry", "http_method": "GET", "auth_type": "none", "auth_token": "", "pagination_strategy": "page_number", "page_param": "page", "limit_param": "limit", "page_size": 100, "cursor_column": "measured_at", "lookback_hours": 3, "unnest_mode": "auto_flatten", "landing_bucket": "dlh-landing", "landing_prefix": "api/spku_hourly/", "bronze_table": "bronze_spku_telemetry", "url": "https://api.spku.dlh.id/v1/telemetry" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-9404", "type": "customNode", "position": { "x": 650, "y": 80 }, "data": { "label": "DuckDB Bronze Ingestion & Dedup", "type": "duckdb_transform", "subtitle": "In-process SQL (<512MB)", "configured": true, "config": { "layer": "silver", "source_parquet_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet", "source_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet", "target_table": "bronze_environmental_permits", "table_name": "bronze_environmental_permits", "primary_keys": "permit_id", "query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1", "sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1", "transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-0708", "type": "customNode", "position": { "x": 840, "y": 80 }, "data": { "label": "dbt Silver Cleansed Permits Enriched", "type": "dbt_transform", "subtitle": "dbt-postgres", "configured": true, "config": { "dbt_layer": "silver", "dbt_model": "silver_permits_enriched", "sel": "silver_permits_enriched", "tgt": "postgres", "trino_catalog": "iceberg", "sql_query_preview": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'", "query": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'", "test_configs": { "unique_columns": [ "permit_id", "permit_number" ], "not_null_columns": [ "permit_id", "status", "updated_at" ], "accepted_values": {}, "enforce_contract": true }, "unique_columns": [ "permit_id", "permit_number" ], "not_null_columns": [ "permit_id", "status", "updated_at" ], "enforce_contract": true, "use_cosmos": true, "transformation_engine": "cosmos" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-2234", "type": "customNode", "position": { "x": 1030, "y": 80 }, "data": { "label": "GE Silver Compliance Quality Gate", "type": "ge_quality", "subtitle": "Great Expectations", "configured": true, "config": { "ge_scope_level": "table", "ge_table_name": "silver_permits_enriched", "test_table_row_count": true, "test_table_schema_match": true, "test_table_table_not_empty": true, "table_min_rows": 1, "table_max_rows": 1000000, "ge_col_table": "stasiun_spku", "ge_target_column": "stasiun_id", "ge_column_test_type": "not_null", "sql_assertion": "", "sql": "", "expected": 0, "error_message": "Unexpected rows detected in table", "suite": "bronze", "fail": "Stop DAG" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-3694", "type": "customNode", "position": { "x": 1220, "y": 80 }, "data": { "label": "dbt Gold Regulatory Compliance Mart", "type": "dbt_transform", "subtitle": "dbt-postgres", "configured": true, "config": { "dbt_layer": "gold", "dbt_model": "gold_compliance_summary_mart", "sel": "gold_compliance_summary_mart", "tgt": "postgres", "trino_catalog": "iceberg", "sql_query_preview": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status", "query": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status", "test_configs": { "unique_columns": [], "not_null_columns": [ "permit_type", "status" ], "accepted_values": {}, "enforce_contract": false }, "unique_columns": [], "not_null_columns": [ "permit_type", "status" ], "use_cosmos": true, "transformation_engine": "cosmos" } }, "width": 96, "height": 96, "selected": false, "dragging": false }, { "id": "node-4977", "type": "customNode", "position": { "x": 1410, "y": 80 }, "data": { "label": "SITOMAT Governance Callback", "type": "sitomat_callback", "subtitle": "Status callback", "configured": true, "config": { "layer": "gold", "artifact_ref": "lakehouse.gold.gold_compliance_summary_mart", "checksum": "sha256-verified-modernized-ingestion", "sitomat_base_url": "http://localhost:3000", "openmetadata_fqn": "dlh_airflow_service.pipeline.task", "ge_status": "PASSED", "url": "/sitomat/callback", "stage": "bronze" } }, "width": 96, "height": 96, "selected": true, "dragging": false } ], "edges": [ { "source": "node-5630", "sourceHandle": null, "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-5630-node-9404" }, { "source": "node-6855", "sourceHandle": null, "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-6855-node-9404" }, { "source": "node-8156", "sourceHandle": null, "target": "node-9404", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-8156-node-9404" }, { "source": "node-9404", "sourceHandle": null, "target": "node-0708", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-9404-node-0708" }, { "source": "node-0708", "sourceHandle": null, "target": "node-2234", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-0708-node-2234" }, { "source": "node-2234", "sourceHandle": null, "target": "node-3694", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-2234-node-3694" }, { "source": "node-3694", "sourceHandle": null, "target": "node-4977", "targetHandle": null, "type": "removableEdge", "style": { "stroke": "var(--muted)", "strokeWidth": 2 }, "data": {}, "id": "reactflow__edge-node-3694-node-4977" } ], "notes": [], "is_draft": false, "status": "deployed", "updated_at": "2026-10-21T08:59:34.908289Z" }