diff --git a/configs/e2e_modernized_ingestion_flow.yaml b/configs/e2e_modernized_ingestion_flow.yaml index 8004c59..a719305 100644 --- a/configs/e2e_modernized_ingestion_flow.yaml +++ b/configs/e2e_modernized_ingestion_flow.yaml @@ -8,7 +8,7 @@ governance: tier: Tier.Tier1 certification: Certification.Gold tasks: -- task_id: node-2331 +- task_id: node-7278 type: source_db label: Source Database (Environmental Permits) enabled: true @@ -21,7 +21,7 @@ tasks: target_type: minio_s3 watermark_column: updated_at ingestion_mode: incremental_watermark -- task_id: node-3521 +- task_id: node-8718 type: source_gws label: Source GWS (Waste Reports) enabled: true @@ -35,7 +35,7 @@ tasks: drive_folder_id: 1OSRugXZvfjoY9M0yySKT selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx header_row: 2 -- task_id: node-4618 +- task_id: node-0372 type: source_api_dlt label: Source REST API dlt (Sensor SPKU) enabled: true @@ -56,14 +56,14 @@ tasks: landing_prefix: api/spku_hourly/ bronze_table: bronze_spku_telemetry url: https://api.spku.dlh.id/v1/telemetry -- task_id: node-5668 +- task_id: node-2418 type: duckdb_transform label: DuckDB Bronze Ingestion & Dedup enabled: true dependencies: - - node-2331 - - node-3521 - - node-4618 + - node-7278 + - node-8718 + - node-0372 config: layer: silver source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet @@ -71,21 +71,24 @@ tasks: target_table: bronze_environmental_permits table_name: bronze_environmental_permits primary_keys: permit_id - query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, - issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') - QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1 - sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, - issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') - QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1 - transformation_sql: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, + query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') - QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1 -- task_id: node-6733 + QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) + = 1 + sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, + status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') + QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) + = 1 + transformation_sql: SELECT permit_id, permit_number, company_name, activity_type + AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') + QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) + = 1 +- task_id: node-4451 type: dbt_transform label: dbt Silver Cleansed Permits Enriched enabled: true dependencies: - - node-5668 + - node-2418 config: dbt_layer: silver dbt_model: silver_permits_enriched @@ -115,12 +118,12 @@ tasks: enforce_contract: true use_cosmos: true transformation_engine: cosmos -- task_id: node-7939 +- task_id: node-6068 type: ge_quality label: GE Silver Compliance Quality Gate enabled: true dependencies: - - node-6733 + - node-4451 config: ge_scope_level: table ge_table_name: silver_permits_enriched @@ -138,12 +141,12 @@ tasks: error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG -- task_id: node-9072 +- task_id: node-8030 type: dbt_transform label: dbt Gold Regulatory Compliance Mart enabled: true dependencies: - - node-7939 + - node-6068 config: dbt_layer: gold dbt_model: gold_compliance_summary_mart @@ -168,12 +171,12 @@ tasks: enforce_contract: false use_cosmos: true transformation_engine: cosmos -- task_id: node-0230 +- task_id: node-9428 type: sitomat_callback label: SITOMAT Governance Callback enabled: true dependencies: - - node-9072 + - node-8030 config: layer: gold artifact_ref: lakehouse.gold.gold_compliance_summary_mart diff --git a/metadata/e2e_modernized_ingestion_flow.json b/metadata/e2e_modernized_ingestion_flow.json index ca2f1ce..d062c9d 100644 --- a/metadata/e2e_modernized_ingestion_flow.json +++ b/metadata/e2e_modernized_ingestion_flow.json @@ -6,7 +6,7 @@ "owner_name": "Administrator Lokal (Break-Glass)", "nodes": [ { - "id": "node-2331", + "id": "node-7278", "type": "customNode", "position": { "x": 80, @@ -34,7 +34,7 @@ "dragging": false }, { - "id": "node-3521", + "id": "node-8718", "type": "customNode", "position": { "x": 270, @@ -63,7 +63,7 @@ "dragging": false }, { - "id": "node-4618", + "id": "node-0372", "type": "customNode", "position": { "x": 460, @@ -99,7 +99,7 @@ "dragging": false }, { - "id": "node-5668", + "id": "node-2418", "type": "customNode", "position": { "x": 650, @@ -117,9 +117,9 @@ "target_table": "bronze_environmental_permits", "table_name": "bronze_environmental_permits", "primary_keys": "permit_id", - "query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1", - "sql_query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1", - "transformation_sql": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1" + "query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1", + "sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1", + "transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1" } }, "width": 96, @@ -128,7 +128,7 @@ "dragging": false }, { - "id": "node-6733", + "id": "node-4451", "type": "customNode", "position": { "x": 840, @@ -180,7 +180,7 @@ "dragging": false }, { - "id": "node-7939", + "id": "node-6068", "type": "customNode", "position": { "x": 1030, @@ -216,7 +216,7 @@ "dragging": false }, { - "id": "node-9072", + "id": "node-8030", "type": "customNode", "position": { "x": 1220, @@ -256,7 +256,7 @@ "dragging": false }, { - "id": "node-0230", + "id": "node-9428", "type": "customNode", "position": { "x": 1410, @@ -286,9 +286,9 @@ ], "edges": [ { - "source": "node-2331", + "source": "node-7278", "sourceHandle": null, - "target": "node-5668", + "target": "node-2418", "targetHandle": null, "type": "removableEdge", "style": { @@ -296,12 +296,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-2331-node-5668" + "id": "reactflow__edge-node-7278-node-2418" }, { - "source": "node-3521", + "source": "node-8718", "sourceHandle": null, - "target": "node-5668", + "target": "node-2418", "targetHandle": null, "type": "removableEdge", "style": { @@ -309,12 +309,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-3521-node-5668" + "id": "reactflow__edge-node-8718-node-2418" }, { - "source": "node-4618", + "source": "node-0372", "sourceHandle": null, - "target": "node-5668", + "target": "node-2418", "targetHandle": null, "type": "removableEdge", "style": { @@ -322,12 +322,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-4618-node-5668" + "id": "reactflow__edge-node-0372-node-2418" }, { - "source": "node-5668", + "source": "node-2418", "sourceHandle": null, - "target": "node-6733", + "target": "node-4451", "targetHandle": null, "type": "removableEdge", "style": { @@ -335,12 +335,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-5668-node-6733" + "id": "reactflow__edge-node-2418-node-4451" }, { - "source": "node-6733", + "source": "node-4451", "sourceHandle": null, - "target": "node-7939", + "target": "node-6068", "targetHandle": null, "type": "removableEdge", "style": { @@ -348,12 +348,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-6733-node-7939" + "id": "reactflow__edge-node-4451-node-6068" }, { - "source": "node-7939", + "source": "node-6068", "sourceHandle": null, - "target": "node-9072", + "target": "node-8030", "targetHandle": null, "type": "removableEdge", "style": { @@ -361,12 +361,12 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-7939-node-9072" + "id": "reactflow__edge-node-6068-node-8030" }, { - "source": "node-9072", + "source": "node-8030", "sourceHandle": null, - "target": "node-0230", + "target": "node-9428", "targetHandle": null, "type": "removableEdge", "style": { @@ -374,11 +374,11 @@ "strokeWidth": 2 }, "data": {}, - "id": "reactflow__edge-node-9072-node-0230" + "id": "reactflow__edge-node-8030-node-9428" } ], "notes": [], "is_draft": false, "status": "deployed", - "updated_at": "2026-10-19T10:22:32.393592Z" + "updated_at": "2026-10-21T07:58:01.895850Z" } \ No newline at end of file