test(e2e): update deployed e2e modernized ingestion pipeline metadata
This commit is contained in:
@@ -8,7 +8,7 @@ governance:
|
|||||||
tier: Tier.Tier1
|
tier: Tier.Tier1
|
||||||
certification: Certification.Gold
|
certification: Certification.Gold
|
||||||
tasks:
|
tasks:
|
||||||
- task_id: node-1048
|
- task_id: node-2331
|
||||||
type: source_db
|
type: source_db
|
||||||
label: Source Database (Environmental Permits)
|
label: Source Database (Environmental Permits)
|
||||||
enabled: true
|
enabled: true
|
||||||
@@ -21,13 +21,13 @@ tasks:
|
|||||||
target_type: minio_s3
|
target_type: minio_s3
|
||||||
watermark_column: updated_at
|
watermark_column: updated_at
|
||||||
ingestion_mode: incremental_watermark
|
ingestion_mode: incremental_watermark
|
||||||
- task_id: node-2264
|
- task_id: node-3521
|
||||||
type: source_gws
|
type: source_gws
|
||||||
label: Source GWS (Waste Reports)
|
label: Source GWS (Waste Reports)
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies: []
|
dependencies: []
|
||||||
config:
|
config:
|
||||||
url: ''
|
url: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026
|
||||||
spreadsheet_id: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026
|
spreadsheet_id: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026
|
||||||
sheet_range: Sheet1
|
sheet_range: Sheet1
|
||||||
sheet_name: Sheet1
|
sheet_name: Sheet1
|
||||||
@@ -35,7 +35,7 @@ tasks:
|
|||||||
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
|
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
|
||||||
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
|
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
|
||||||
header_row: 2
|
header_row: 2
|
||||||
- task_id: node-3310
|
- task_id: node-4618
|
||||||
type: source_api_dlt
|
type: source_api_dlt
|
||||||
label: Source REST API dlt (Sensor SPKU)
|
label: Source REST API dlt (Sensor SPKU)
|
||||||
enabled: true
|
enabled: true
|
||||||
@@ -56,14 +56,14 @@ tasks:
|
|||||||
landing_prefix: api/spku_hourly/
|
landing_prefix: api/spku_hourly/
|
||||||
bronze_table: bronze_spku_telemetry
|
bronze_table: bronze_spku_telemetry
|
||||||
url: https://api.spku.dlh.id/v1/telemetry
|
url: https://api.spku.dlh.id/v1/telemetry
|
||||||
- task_id: node-4363
|
- task_id: node-5668
|
||||||
type: duckdb_transform
|
type: duckdb_transform
|
||||||
label: DuckDB Bronze Ingestion & Dedup
|
label: DuckDB Bronze Ingestion & Dedup
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- node-1048
|
- node-2331
|
||||||
- node-2264
|
- node-3521
|
||||||
- node-3310
|
- node-4618
|
||||||
config:
|
config:
|
||||||
layer: silver
|
layer: silver
|
||||||
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
|
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
|
||||||
@@ -80,12 +80,12 @@ tasks:
|
|||||||
transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type,
|
transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type,
|
||||||
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
|
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
|
||||||
- task_id: node-5408
|
- task_id: node-6733
|
||||||
type: dbt_transform
|
type: dbt_transform
|
||||||
label: dbt Silver Cleansed Permits Enriched
|
label: dbt Silver Cleansed Permits Enriched
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- node-4363
|
- node-5668
|
||||||
config:
|
config:
|
||||||
dbt_layer: silver
|
dbt_layer: silver
|
||||||
dbt_model: silver_permits_enriched
|
dbt_model: silver_permits_enriched
|
||||||
@@ -115,12 +115,12 @@ tasks:
|
|||||||
enforce_contract: true
|
enforce_contract: true
|
||||||
use_cosmos: true
|
use_cosmos: true
|
||||||
transformation_engine: cosmos
|
transformation_engine: cosmos
|
||||||
- task_id: node-6444
|
- task_id: node-7939
|
||||||
type: ge_quality
|
type: ge_quality
|
||||||
label: GE Silver Compliance Quality Gate
|
label: GE Silver Compliance Quality Gate
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- node-5408
|
- node-6733
|
||||||
config:
|
config:
|
||||||
ge_scope_level: table
|
ge_scope_level: table
|
||||||
ge_table_name: silver_permits_enriched
|
ge_table_name: silver_permits_enriched
|
||||||
@@ -138,12 +138,12 @@ tasks:
|
|||||||
error_message: Unexpected rows detected in table
|
error_message: Unexpected rows detected in table
|
||||||
suite: bronze
|
suite: bronze
|
||||||
fail: Stop DAG
|
fail: Stop DAG
|
||||||
- task_id: node-7494
|
- task_id: node-9072
|
||||||
type: dbt_transform
|
type: dbt_transform
|
||||||
label: dbt Gold Regulatory Compliance Mart
|
label: dbt Gold Regulatory Compliance Mart
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- node-6444
|
- node-7939
|
||||||
config:
|
config:
|
||||||
dbt_layer: gold
|
dbt_layer: gold
|
||||||
dbt_model: gold_compliance_summary_mart
|
dbt_model: gold_compliance_summary_mart
|
||||||
@@ -168,12 +168,12 @@ tasks:
|
|||||||
enforce_contract: false
|
enforce_contract: false
|
||||||
use_cosmos: true
|
use_cosmos: true
|
||||||
transformation_engine: cosmos
|
transformation_engine: cosmos
|
||||||
- task_id: node-8543
|
- task_id: node-0230
|
||||||
type: sitomat_callback
|
type: sitomat_callback
|
||||||
label: SITOMAT Governance Callback
|
label: SITOMAT Governance Callback
|
||||||
enabled: true
|
enabled: true
|
||||||
dependencies:
|
dependencies:
|
||||||
- node-7494
|
- node-9072
|
||||||
config:
|
config:
|
||||||
layer: gold
|
layer: gold
|
||||||
artifact_ref: lakehouse.gold.gold_compliance_summary_mart
|
artifact_ref: lakehouse.gold.gold_compliance_summary_mart
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||||
"nodes": [
|
"nodes": [
|
||||||
{
|
{
|
||||||
"id": "node-1048",
|
"id": "node-2331",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 80,
|
"x": 80,
|
||||||
@@ -34,7 +34,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-2264",
|
"id": "node-3521",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 270,
|
"x": 270,
|
||||||
@@ -47,7 +47,7 @@
|
|||||||
"isSource": true,
|
"isSource": true,
|
||||||
"configured": true,
|
"configured": true,
|
||||||
"config": {
|
"config": {
|
||||||
"url": "",
|
"url": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||||
"spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
"spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||||
"sheet_range": "Sheet1",
|
"sheet_range": "Sheet1",
|
||||||
"sheet_name": "Sheet1",
|
"sheet_name": "Sheet1",
|
||||||
@@ -63,7 +63,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-3310",
|
"id": "node-4618",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 460,
|
"x": 460,
|
||||||
@@ -99,7 +99,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-4363",
|
"id": "node-5668",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 650,
|
"x": 650,
|
||||||
@@ -128,7 +128,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-5408",
|
"id": "node-6733",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 840,
|
"x": 840,
|
||||||
@@ -180,7 +180,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-6444",
|
"id": "node-7939",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 1030,
|
"x": 1030,
|
||||||
@@ -216,7 +216,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-7494",
|
"id": "node-9072",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 1220,
|
"x": 1220,
|
||||||
@@ -256,7 +256,7 @@
|
|||||||
"dragging": false
|
"dragging": false
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "node-8543",
|
"id": "node-0230",
|
||||||
"type": "customNode",
|
"type": "customNode",
|
||||||
"position": {
|
"position": {
|
||||||
"x": 1410,
|
"x": 1410,
|
||||||
@@ -286,9 +286,9 @@
|
|||||||
],
|
],
|
||||||
"edges": [
|
"edges": [
|
||||||
{
|
{
|
||||||
"source": "node-1048",
|
"source": "node-2331",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-4363",
|
"target": "node-5668",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -296,12 +296,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-1048-node-4363"
|
"id": "reactflow__edge-node-2331-node-5668"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-2264",
|
"source": "node-3521",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-4363",
|
"target": "node-5668",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -309,12 +309,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-2264-node-4363"
|
"id": "reactflow__edge-node-3521-node-5668"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-3310",
|
"source": "node-4618",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-4363",
|
"target": "node-5668",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -322,12 +322,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-3310-node-4363"
|
"id": "reactflow__edge-node-4618-node-5668"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-4363",
|
"source": "node-5668",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-5408",
|
"target": "node-6733",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -335,12 +335,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-4363-node-5408"
|
"id": "reactflow__edge-node-5668-node-6733"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-5408",
|
"source": "node-6733",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-6444",
|
"target": "node-7939",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -348,12 +348,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-5408-node-6444"
|
"id": "reactflow__edge-node-6733-node-7939"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-6444",
|
"source": "node-7939",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-7494",
|
"target": "node-9072",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -361,12 +361,12 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-6444-node-7494"
|
"id": "reactflow__edge-node-7939-node-9072"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": "node-7494",
|
"source": "node-9072",
|
||||||
"sourceHandle": null,
|
"sourceHandle": null,
|
||||||
"target": "node-8543",
|
"target": "node-0230",
|
||||||
"targetHandle": null,
|
"targetHandle": null,
|
||||||
"type": "removableEdge",
|
"type": "removableEdge",
|
||||||
"style": {
|
"style": {
|
||||||
@@ -374,11 +374,11 @@
|
|||||||
"strokeWidth": 2
|
"strokeWidth": 2
|
||||||
},
|
},
|
||||||
"data": {},
|
"data": {},
|
||||||
"id": "reactflow__edge-node-7494-node-8543"
|
"id": "reactflow__edge-node-9072-node-0230"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"notes": [],
|
"notes": [],
|
||||||
"is_draft": false,
|
"is_draft": false,
|
||||||
"status": "deployed",
|
"status": "deployed",
|
||||||
"updated_at": "2026-10-19T09:42:04.272325Z"
|
"updated_at": "2026-10-19T10:22:32.393592Z"
|
||||||
}
|
}
|
||||||
@@ -22,6 +22,6 @@
|
|||||||
"admin_groups": [
|
"admin_groups": [
|
||||||
"/Platform Administrators"
|
"/Platform Administrators"
|
||||||
],
|
],
|
||||||
"updated_at": "2026-10-19T09:46:41.835103+00:00",
|
"updated_at": "2026-10-19T10:19:51.346970+00:00",
|
||||||
"updated_by": "admin"
|
"updated_by": "admin"
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user