merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(e2e): update deployed e2e modernized pipeline configuration

Browse Source
This commit is contained in:
syahril
2026-10-21 15:00:31 +07:00
parent 8deb174221
commit 90c0a03b7e
2 changed files with 59 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
metadata/e2e_modernized_ingestion_flow.json
+33 -33
View File
@@ -6,7 +6,7 @@
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-2331",
"id": "node-7278",
"type": "customNode",
"position": {
"x": 80,
@@ -34,7 +34,7 @@
"dragging": false
},
{
"id": "node-3521",
"id": "node-8718",
"type": "customNode",
"position": {
"x": 270,
@@ -63,7 +63,7 @@
"dragging": false
},
{
"id": "node-4618",
"id": "node-0372",
"type": "customNode",
"position": {
"x": 460,
@@ -99,7 +99,7 @@
"dragging": false
},
{
"id": "node-5668",
"id": "node-2418",
"type": "customNode",
"position": {
"x": 650,
@@ -117,9 +117,9 @@
"target_table": "bronze_environmental_permits",
"table_name": "bronze_environmental_permits",
"primary_keys": "permit_id",
"query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1",
"sql_query": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1",
"transformation_sql": "SELECT id AS permit_id, permit_number, company_name, permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1"
"query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
"sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
"transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
}
},
"width": 96,
@@ -128,7 +128,7 @@
"dragging": false
},
{
"id": "node-6733",
"id": "node-4451",
"type": "customNode",
"position": {
"x": 840,
@@ -180,7 +180,7 @@
"dragging": false
},
{
"id": "node-7939",
"id": "node-6068",
"type": "customNode",
"position": {
"x": 1030,
@@ -216,7 +216,7 @@
"dragging": false
},
{
"id": "node-9072",
"id": "node-8030",
"type": "customNode",
"position": {
"x": 1220,
@@ -256,7 +256,7 @@
"dragging": false
},
{
"id": "node-0230",
"id": "node-9428",
"type": "customNode",
"position": {
"x": 1410,
@@ -286,9 +286,9 @@
],
"edges": [
{
"source": "node-2331",
"source": "node-7278",
"sourceHandle": null,
"target": "node-5668",
"target": "node-2418",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -296,12 +296,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-2331-node-5668"
"id": "reactflow__edge-node-7278-node-2418"
},
{
"source": "node-3521",
"source": "node-8718",
"sourceHandle": null,
"target": "node-5668",
"target": "node-2418",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -309,12 +309,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-3521-node-5668"
"id": "reactflow__edge-node-8718-node-2418"
},
{
"source": "node-4618",
"source": "node-0372",
"sourceHandle": null,
"target": "node-5668",
"target": "node-2418",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -322,12 +322,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-4618-node-5668"
"id": "reactflow__edge-node-0372-node-2418"
},
{
"source": "node-5668",
"source": "node-2418",
"sourceHandle": null,
"target": "node-6733",
"target": "node-4451",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -335,12 +335,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-5668-node-6733"
"id": "reactflow__edge-node-2418-node-4451"
},
{
"source": "node-6733",
"source": "node-4451",
"sourceHandle": null,
"target": "node-7939",
"target": "node-6068",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -348,12 +348,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-6733-node-7939"
"id": "reactflow__edge-node-4451-node-6068"
},
{
"source": "node-7939",
"source": "node-6068",
"sourceHandle": null,
"target": "node-9072",
"target": "node-8030",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -361,12 +361,12 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-7939-node-9072"
"id": "reactflow__edge-node-6068-node-8030"
},
{
"source": "node-9072",
"source": "node-8030",
"sourceHandle": null,
"target": "node-0230",
"target": "node-9428",
"targetHandle": null,
"type": "removableEdge",
"style": {
@@ -374,11 +374,11 @@
"strokeWidth": 2
},
"data": {},
"id": "reactflow__edge-node-9072-node-0230"
"id": "reactflow__edge-node-8030-node-9428"
}
],
"notes": [],
"is_draft": false,
"status": "deployed",
"updated_at": "2026-10-19T10:22:32.393592Z"
"updated_at": "2026-10-21T07:58:01.895850Z"
}