merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(e2e): update deployed e2e modernized pipeline configuration

Browse Source
This commit is contained in:
syahril
2026-10-21 15:00:31 +07:00
parent 8deb174221
commit 90c0a03b7e
2 changed files with 59 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/e2e_modernized_ingestion_flow.yaml
+26 -23
View File
@@ -8,7 +8,7 @@ governance:
tier: Tier.Tier1 tier: Tier.Tier1
certification: Certification.Gold certification: Certification.Gold
tasks: tasks:
- task_id: node-2331 - task_id: node-7278
type: source_db type: source_db
label: Source Database (Environmental Permits) label: Source Database (Environmental Permits)
enabled: true enabled: true
@@ -21,7 +21,7 @@ tasks:
target_type: minio_s3 target_type: minio_s3
watermark_column: updated_at watermark_column: updated_at
ingestion_mode: incremental_watermark ingestion_mode: incremental_watermark
- task_id: node-3521 - task_id: node-8718
type: source_gws type: source_gws
label: Source GWS (Waste Reports) label: Source GWS (Waste Reports)
enabled: true enabled: true
@@ -35,7 +35,7 @@ tasks:
drive_folder_id: 1OSRugXZvfjoY9M0yySKT drive_folder_id: 1OSRugXZvfjoY9M0yySKT
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
header_row: 2 header_row: 2
- task_id: node-4618 - task_id: node-0372
type: source_api_dlt type: source_api_dlt
label: Source REST API dlt (Sensor SPKU) label: Source REST API dlt (Sensor SPKU)
enabled: true enabled: true
@@ -56,14 +56,14 @@ tasks:
landing_prefix: api/spku_hourly/ landing_prefix: api/spku_hourly/
bronze_table: bronze_spku_telemetry bronze_table: bronze_spku_telemetry
url: https://api.spku.dlh.id/v1/telemetry url: https://api.spku.dlh.id/v1/telemetry
- task_id: node-5668 - task_id: node-2418
type: duckdb_transform type: duckdb_transform
label: DuckDB Bronze Ingestion & Dedup label: DuckDB Bronze Ingestion & Dedup
enabled: true enabled: true
dependencies: dependencies:
- node-2331 - node-7278
- node-3521 - node-8718
- node-4618 - node-0372
config: config:
layer: silver layer: silver
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
@@ -71,21 +71,24 @@ tasks:
target_table: bronze_environmental_permits target_table: bronze_environmental_permits
table_name: bronze_environmental_permits table_name: bronze_environmental_permits
primary_keys: permit_id primary_keys: permit_id
query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
transformation_sql: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1 QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
- task_id: node-6733 = 1
sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
= 1
transformation_sql: SELECT permit_id, permit_number, company_name, activity_type
AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
= 1
- task_id: node-4451
type: dbt_transform type: dbt_transform
label: dbt Silver Cleansed Permits Enriched label: dbt Silver Cleansed Permits Enriched
enabled: true enabled: true
dependencies: dependencies:
- node-5668 - node-2418
config: config:
dbt_layer: silver dbt_layer: silver
dbt_model: silver_permits_enriched dbt_model: silver_permits_enriched
@@ -115,12 +118,12 @@ tasks:
enforce_contract: true enforce_contract: true
use_cosmos: true use_cosmos: true
transformation_engine: cosmos transformation_engine: cosmos
- task_id: node-7939 - task_id: node-6068
type: ge_quality type: ge_quality
label: GE Silver Compliance Quality Gate label: GE Silver Compliance Quality Gate
enabled: true enabled: true
dependencies: dependencies:
- node-6733 - node-4451
config: config:
ge_scope_level: table ge_scope_level: table
ge_table_name: silver_permits_enriched ge_table_name: silver_permits_enriched
@@ -138,12 +141,12 @@ tasks:
error_message: Unexpected rows detected in table error_message: Unexpected rows detected in table
suite: bronze suite: bronze
fail: Stop DAG fail: Stop DAG
- task_id: node-9072 - task_id: node-8030
type: dbt_transform type: dbt_transform
label: dbt Gold Regulatory Compliance Mart label: dbt Gold Regulatory Compliance Mart
enabled: true enabled: true
dependencies: dependencies:
- node-7939 - node-6068
config: config:
dbt_layer: gold dbt_layer: gold
dbt_model: gold_compliance_summary_mart dbt_model: gold_compliance_summary_mart
@@ -168,12 +171,12 @@ tasks:
enforce_contract: false enforce_contract: false
use_cosmos: true use_cosmos: true
transformation_engine: cosmos transformation_engine: cosmos
- task_id: node-0230 - task_id: node-9428
type: sitomat_callback type: sitomat_callback
label: SITOMAT Governance Callback label: SITOMAT Governance Callback
enabled: true enabled: true
dependencies: dependencies:
- node-9072 - node-8030
config: config:
layer: gold layer: gold
artifact_ref: lakehouse.gold.gold_compliance_summary_mart artifact_ref: lakehouse.gold.gold_compliance_summary_mart
metadata/e2e_modernized_ingestion_flow.json
+33 -33
View File
@@ -6,7 +6,7 @@
"owner_name": "Administrator Lokal (Break-Glass)", "owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [ "nodes": [
{ {
"id": "node-2331", "id": "node-7278",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 80, "x": 80,
@@ -34,7 +34,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-3521", "id": "node-8718",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 270, "x": 270,
@@ -63,7 +63,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-4618", "id": "node-0372",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 460, "x": 460,
@@ -99,7 +99,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-5668", "id": "node-2418",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 650, "x": 650,
@@ -117,9 +117,9 @@
"target_table": "bronze_environmental_permits", "target_table": "bronze_environmental_permits",
"table_name": "bronze_environmental_permits", "table_name": "bronze_environmental_permits",
"primary_keys": "permit_id", "primary_keys": "permit_id",
"query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1", "query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
"sql_query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1", "sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
"transformation_sql": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1" "transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
} }
}, },
"width": 96, "width": 96,
@@ -128,7 +128,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-6733", "id": "node-4451",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 840, "x": 840,
@@ -180,7 +180,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-7939", "id": "node-6068",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 1030, "x": 1030,
@@ -216,7 +216,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-9072", "id": "node-8030",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 1220, "x": 1220,
@@ -256,7 +256,7 @@
"dragging": false "dragging": false
}, },
{ {
"id": "node-0230", "id": "node-9428",
"type": "customNode", "type": "customNode",
"position": { "position": {
"x": 1410, "x": 1410,
@@ -286,9 +286,9 @@
], ],
"edges": [ "edges": [
{ {
"source": "node-2331", "source": "node-7278",
"sourceHandle": null, "sourceHandle": null,
"target": "node-5668", "target": "node-2418",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -296,12 +296,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-2331-node-5668" "id": "reactflow__edge-node-7278-node-2418"
}, },
{ {
"source": "node-3521", "source": "node-8718",
"sourceHandle": null, "sourceHandle": null,
"target": "node-5668", "target": "node-2418",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -309,12 +309,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-3521-node-5668" "id": "reactflow__edge-node-8718-node-2418"
}, },
{ {
"source": "node-4618", "source": "node-0372",
"sourceHandle": null, "sourceHandle": null,
"target": "node-5668", "target": "node-2418",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -322,12 +322,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-4618-node-5668" "id": "reactflow__edge-node-0372-node-2418"
}, },
{ {
"source": "node-5668", "source": "node-2418",
"sourceHandle": null, "sourceHandle": null,
"target": "node-6733", "target": "node-4451",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -335,12 +335,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-5668-node-6733" "id": "reactflow__edge-node-2418-node-4451"
}, },
{ {
"source": "node-6733", "source": "node-4451",
"sourceHandle": null, "sourceHandle": null,
"target": "node-7939", "target": "node-6068",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -348,12 +348,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-6733-node-7939" "id": "reactflow__edge-node-4451-node-6068"
}, },
{ {
"source": "node-7939", "source": "node-6068",
"sourceHandle": null, "sourceHandle": null,
"target": "node-9072", "target": "node-8030",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -361,12 +361,12 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-7939-node-9072" "id": "reactflow__edge-node-6068-node-8030"
}, },
{ {
"source": "node-9072", "source": "node-8030",
"sourceHandle": null, "sourceHandle": null,
"target": "node-0230", "target": "node-9428",
"targetHandle": null, "targetHandle": null,
"type": "removableEdge", "type": "removableEdge",
"style": { "style": {
@@ -374,11 +374,11 @@
"strokeWidth": 2 "strokeWidth": 2
}, },
"data": {}, "data": {},
"id": "reactflow__edge-node-9072-node-0230" "id": "reactflow__edge-node-8030-node-9428"
} }
], ],
"notes": [], "notes": [],
"is_draft": false, "is_draft": false,
"status": "deployed", "status": "deployed",
"updated_at": "2026-10-19T10:22:32.393592Z" "updated_at": "2026-10-21T07:58:01.895850Z"
} }