diff --git a/configs/e2e_modernized_ingestion_flow.yaml b/configs/e2e_modernized_ingestion_flow.yaml index 257b00d..8a9af37 100644 --- a/configs/e2e_modernized_ingestion_flow.yaml +++ b/configs/e2e_modernized_ingestion_flow.yaml @@ -8,7 +8,7 @@ governance: tier: Tier.Tier1 certification: Certification.Gold tasks: -- task_id: node-1048 +- task_id: node-2331 type: source_db label: Source Database (Environmental Permits) enabled: true @@ -21,13 +21,13 @@ tasks: target_type: minio_s3 watermark_column: updated_at ingestion_mode: incremental_watermark -- task_id: node-2264 +- task_id: node-3521 type: source_gws label: Source GWS (Waste Reports) enabled: true dependencies: [] config: - url: '' + url: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026 spreadsheet_id: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026 sheet_range: Sheet1 sheet_name: Sheet1 @@ -35,7 +35,7 @@ tasks: drive_folder_id: 1OSRugXZvfjoY9M0yySKT selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx header_row: 2 -- task_id: node-3310 +- task_id: node-4618 type: source_api_dlt label: Source REST API dlt (Sensor SPKU) enabled: true @@ -56,14 +56,14 @@ tasks: landing_prefix: api/spku_hourly/ bronze_table: bronze_spku_telemetry url: https://api.spku.dlh.id/v1/telemetry -- task_id: node-4363 +- task_id: node-5668 type: duckdb_transform label: DuckDB Bronze Ingestion & Dedup enabled: true dependencies: - - node-1048 - - node-2264 - - node-3310 + - node-2331 + - node-3521 + - node-4618 config: layer: silver source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet @@ -80,12 +80,12 @@ tasks: transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1 -- task_id: node-5408 +- task_id: node-6733 type: dbt_transform label: dbt Silver Cleansed Permits Enriched enabled: true dependencies: - - node-4363 + - node-5668 config: dbt_layer: silver dbt_model: silver_permits_enriched @@ -115,12 +115,12 @@ tasks: enforce_contract: true use_cosmos: true transformation_engine: cosmos -- task_id: node-6444 +- task_id: node-7939 type: ge_quality label: GE Silver Compliance Quality Gate enabled: true dependencies: - - node-5408 + - node-6733 config: ge_scope_level: table ge_table_name: silver_permits_enriched @@ -138,12 +138,12 @@ tasks: error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG -- task_id: node-7494 +- task_id: node-9072 type: dbt_transform label: dbt Gold Regulatory Compliance Mart enabled: true dependencies: - - node-6444 + - node-7939 config: dbt_layer: gold dbt_model: gold_compliance_summary_mart @@ -168,12 +168,12 @@ tasks: enforce_contract: false use_cosmos: true transformation_engine: cosmos -- task_id: node-8543 +- task_id: node-0230 type: sitomat_callback label: SITOMAT Governance Callback enabled: true dependencies: - - node-7494 + - node-9072 config: layer: gold artifact_ref: lakehouse.gold.gold_compliance_summary_mart diff --git a/metadata/e2e_modernized_ingestion_flow.json b/metadata/e2e_modernized_ingestion_flow.json index 58a1d4a..ca2f1ce 100644 --- a/metadata/e2e_modernized_ingestion_flow.json +++ b/metadata/e2e_modernized_ingestion_flow.json @@ -6,7 +6,7 @@ "owner_name": "Administrator Lokal (Break-Glass)", "nodes": [ { - "id": "node-1048", + "id": "node-2331", "type": "customNode", "position": { "x": 80, @@ -34,7 +34,7 @@ "dragging": false }, { - "id": "node-2264", + "id": "node-3521", "type": "customNode", "position": { "x": 270, @@ -47,7 +47,7 @@ "isSource": true, "configured": true, "config": { - "url": "", + "url": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026", "spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026", "sheet_range": "Sheet1", "sheet_name": "Sheet1", @@ -63,7 +63,7 @@ "dragging": false }, { - "id": "node-3310", + "id": "node-4618", "type": "customNode", "position": { "x": 460, @@ -99,7 +99,7 @@ "dragging": false }, { - "id": "node-4363", + "id": "node-5668", "type": "customNode", "position": { "x": 650, @@ -128,7 +128,7 @@ "dragging": false }, { - "id": "node-5408", + "id": "node-6733", "type": "customNode", "position": { "x": 840, @@ -180,7 +180,7 @@ "dragging": false }, { - "id": "node-6444", + "id": "node-7939", "type": "customNode", "position": { "x": 1030, @@ -216,7 +216,7 @@ "dragging": false }, { - "id": "node-7494", + "id": "node-9072", "type": "customNode", "position": { "x": 1220, @@ -256,7 +256,7 @@ "dragging": false }, { - "id": "node-8543", + "id": "node-0230", "type": "customNode", "position": { "x": 1410, @@ -286,9 +286,9 @@ ], "edges": [ { - "source": "node-1048", + "source": "node-2331", "sourceHandle": null, - "target": "node-4363", + "target": "node-5668", "targetHandle": null, "type": "removableEdge", "style": { @@ -296,12 +296,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-1048-node-4363" + "id": "reactflow__edge-node-2331-node-5668" }, { - "source": "node-2264", + "source": "node-3521", "sourceHandle": null, - "target": "node-4363", + "target": "node-5668", "targetHandle": null, "type": "removableEdge", "style": { @@ -309,12 +309,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-2264-node-4363" + "id": "reactflow__edge-node-3521-node-5668" }, { - "source": "node-3310", + "source": "node-4618", "sourceHandle": null, - "target": "node-4363", + "target": "node-5668", "targetHandle": null, "type": "removableEdge", "style": { @@ -322,12 +322,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-3310-node-4363" + "id": "reactflow__edge-node-4618-node-5668" }, { - "source": "node-4363", + "source": "node-5668", "sourceHandle": null, - "target": "node-5408", + "target": "node-6733", "targetHandle": null, "type": "removableEdge", "style": { @@ -335,12 +335,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-4363-node-5408" + "id": "reactflow__edge-node-5668-node-6733" }, { - "source": "node-5408", + "source": "node-6733", "sourceHandle": null, - "target": "node-6444", + "target": "node-7939", "targetHandle": null, "type": "removableEdge", "style": { @@ -348,12 +348,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-5408-node-6444" + "id": "reactflow__edge-node-6733-node-7939" }, { - "source": "node-6444", + "source": "node-7939", "sourceHandle": null, - "target": "node-7494", + "target": "node-9072", "targetHandle": null, "type": "removableEdge", "style": { @@ -361,12 +361,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-6444-node-7494" + "id": "reactflow__edge-node-7939-node-9072" }, { - "source": "node-7494", + "source": "node-9072", "sourceHandle": null, - "target": "node-8543", + "target": "node-0230", "targetHandle": null, "type": "removableEdge", "style": { @@ -374,11 +374,11 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-7494-node-8543" + "id": "reactflow__edge-node-9072-node-0230" } ], "notes": [], "is_draft": false, "status": "deployed", - "updated_at": "2026-10-19T09:42:04.272325Z" + "updated_at": "2026-10-19T10:22:32.393592Z" } \ No newline at end of file diff --git a/metadata/rbac_mappings.json b/metadata/rbac_mappings.json index ab08a24..bc79f3b 100644 --- a/metadata/rbac_mappings.json +++ b/metadata/rbac_mappings.json @@ -22,6 +22,6 @@ "admin_groups": [ "/Platform Administrators" ], - "updated_at": "2026-10-19T09:46:41.835103+00:00", + "updated_at": "2026-10-19T10:19:51.346970+00:00", "updated_by": "admin" } \ No newline at end of file