feat(pipelines): sync spku realtime pipeline config, bronze model, and rbac mappings

This commit is contained in:
2026-10-19 14:54:32 +07:00
parent ab5ba1d7a7
commit f82726db40
6 changed files with 72 additions and 47 deletions
@@ -0,0 +1,21 @@
pipeline_id: ingestion_spku_realtime_lakehouse
description: Dynamic Data Lakehouse Pipeline via DLH Studio
schedule: 0 */1 * * *
mode: flexible
governance:
domain: Lingkungan Hidup & Kehutanan
owner: de_team_dlh
tier: Tier.Tier1
certification: Certification.Gold
tasks:
- task_id: node-0579
type: source_db
label: Source Database
enabled: true
dependencies: []
config:
conn: Postgres
conn_id: conn_postgres_source_db
table: permits
table_name: public.stasiun_spku
target_type: minio_s3
@@ -0,0 +1,4 @@
-- dbt Bronze View Model: stasiun_spku
{{ config(materialized='view') }}
SELECT * FROM {{ source('public', 'stasiun_spku') }}
+2
View File
@@ -29,3 +29,5 @@ sources:
- name: raw_cems_telemetry_logs - name: raw_cems_telemetry_logs
description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
pipeline pipeline
- name: stasiun_spku
description: Auto-registered source table 'stasiun_spku' from canvas pipeline
@@ -1,46 +0,0 @@
{
"id": "ingestion_spku_realtime_lakehouse",
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
"schedule": "0 */1 * * *",
"owner_id": "local-admin",
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-7560",
"type": "customNode",
"position": {
"x": 80,
"y": 80
},
"data": {
"label": "dbt Transform",
"type": "dbt_transform",
"subtitle": "dbt-postgres",
"configured": true,
"config": {
"dbt_layer": "silver",
"dbt_model": "my_custom_model_silver",
"sel": "my_custom_model_silver",
"tgt": "postgres",
"trino_catalog": "iceberg",
"sql_query_preview": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}",
"query": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}"
},
"status": "ok",
"errorMessage": "Configured & Valid",
"missingFields": [],
"hasError": false,
"hasWarning": false
},
"width": 96,
"height": 96,
"selected": true,
"dragging": false
}
],
"edges": [],
"notes": [],
"is_draft": true,
"status": "draft",
"updated_at": "2026-10-18T17:54:28.688466Z"
}
@@ -0,0 +1,44 @@
{
"id": "ingestion_spku_realtime_lakehouse",
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
"schedule": "0 */1 * * *",
"owner_id": "local-admin",
"owner_name": "Administrator Lokal (Break-Glass)",
"nodes": [
{
"id": "node-0579",
"type": "customNode",
"position": {
"x": -20,
"y": 50
},
"data": {
"label": "Source Database",
"type": "source_db",
"subtitle": "Postgres / MySQL / Oracle",
"isSource": true,
"configured": true,
"config": {
"conn": "Postgres",
"conn_id": "conn_postgres_source_db",
"table": "permits",
"table_name": "public.stasiun_spku",
"target_type": "minio_s3"
}
},
"width": 96,
"height": 96,
"selected": true,
"positionAbsolute": {
"x": -20,
"y": 50
},
"dragging": false
}
],
"edges": [],
"notes": [],
"is_draft": false,
"status": "deployed",
"updated_at": "2026-10-19T03:57:43.892615Z"
}
+1 -1
View File
@@ -22,6 +22,6 @@
"admin_groups": [ "admin_groups": [
"/Platform Administrators" "/Platform Administrators"
], ],
"updated_at": "2026-10-18T20:42:26.680284+00:00", "updated_at": "2026-10-19T07:47:22.327195+00:00",
"updated_by": "admin" "updated_by": "admin"
} }