feat(pipelines): sync spku realtime pipeline config, bronze model, and rbac mappings
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
pipeline_id: ingestion_spku_realtime_lakehouse
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
governance:
|
||||
domain: Lingkungan Hidup & Kehutanan
|
||||
owner: de_team_dlh
|
||||
tier: Tier.Tier1
|
||||
certification: Certification.Gold
|
||||
tasks:
|
||||
- task_id: node-0579
|
||||
type: source_db
|
||||
label: Source Database
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: permits
|
||||
table_name: public.stasiun_spku
|
||||
target_type: minio_s3
|
||||
@@ -0,0 +1,4 @@
|
||||
-- dbt Bronze View Model: stasiun_spku
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT * FROM {{ source('public', 'stasiun_spku') }}
|
||||
@@ -29,3 +29,5 @@ sources:
|
||||
- name: raw_cems_telemetry_logs
|
||||
description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
|
||||
pipeline
|
||||
- name: stasiun_spku
|
||||
description: Auto-registered source table 'stasiun_spku' from canvas pipeline
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
{
|
||||
"id": "ingestion_spku_realtime_lakehouse",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-7560",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Transform",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "my_custom_model_silver",
|
||||
"sel": "my_custom_model_silver",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}",
|
||||
"query": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}"
|
||||
},
|
||||
"status": "ok",
|
||||
"errorMessage": "Configured & Valid",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": false
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
"notes": [],
|
||||
"is_draft": true,
|
||||
"status": "draft",
|
||||
"updated_at": "2026-10-18T17:54:28.688466Z"
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"id": "ingestion_spku_realtime_lakehouse",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-0579",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": -20,
|
||||
"y": 50
|
||||
},
|
||||
"data": {
|
||||
"label": "Source Database",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "permits",
|
||||
"table_name": "public.stasiun_spku",
|
||||
"target_type": "minio_s3"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"positionAbsolute": {
|
||||
"x": -20,
|
||||
"y": 50
|
||||
},
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-19T03:57:43.892615Z"
|
||||
}
|
||||
@@ -22,6 +22,6 @@
|
||||
"admin_groups": [
|
||||
"/Platform Administrators"
|
||||
],
|
||||
"updated_at": "2026-10-18T20:42:26.680284+00:00",
|
||||
"updated_at": "2026-10-19T07:47:22.327195+00:00",
|
||||
"updated_by": "admin"
|
||||
}
|
||||
Reference in New Issue
Block a user