feat(pipelines): sync spku realtime pipeline config, bronze model, and rbac mappings
This commit is contained in:
@@ -0,0 +1,21 @@
|
|||||||
|
pipeline_id: ingestion_spku_realtime_lakehouse
|
||||||
|
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||||
|
schedule: 0 */1 * * *
|
||||||
|
mode: flexible
|
||||||
|
governance:
|
||||||
|
domain: Lingkungan Hidup & Kehutanan
|
||||||
|
owner: de_team_dlh
|
||||||
|
tier: Tier.Tier1
|
||||||
|
certification: Certification.Gold
|
||||||
|
tasks:
|
||||||
|
- task_id: node-0579
|
||||||
|
type: source_db
|
||||||
|
label: Source Database
|
||||||
|
enabled: true
|
||||||
|
dependencies: []
|
||||||
|
config:
|
||||||
|
conn: Postgres
|
||||||
|
conn_id: conn_postgres_source_db
|
||||||
|
table: permits
|
||||||
|
table_name: public.stasiun_spku
|
||||||
|
target_type: minio_s3
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
-- dbt Bronze View Model: stasiun_spku
|
||||||
|
{{ config(materialized='view') }}
|
||||||
|
|
||||||
|
SELECT * FROM {{ source('public', 'stasiun_spku') }}
|
||||||
@@ -29,3 +29,5 @@ sources:
|
|||||||
- name: raw_cems_telemetry_logs
|
- name: raw_cems_telemetry_logs
|
||||||
description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
|
description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
|
||||||
pipeline
|
pipeline
|
||||||
|
- name: stasiun_spku
|
||||||
|
description: Auto-registered source table 'stasiun_spku' from canvas pipeline
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
{
|
|
||||||
"id": "ingestion_spku_realtime_lakehouse",
|
|
||||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
|
||||||
"schedule": "0 */1 * * *",
|
|
||||||
"owner_id": "local-admin",
|
|
||||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
|
||||||
"nodes": [
|
|
||||||
{
|
|
||||||
"id": "node-7560",
|
|
||||||
"type": "customNode",
|
|
||||||
"position": {
|
|
||||||
"x": 80,
|
|
||||||
"y": 80
|
|
||||||
},
|
|
||||||
"data": {
|
|
||||||
"label": "dbt Transform",
|
|
||||||
"type": "dbt_transform",
|
|
||||||
"subtitle": "dbt-postgres",
|
|
||||||
"configured": true,
|
|
||||||
"config": {
|
|
||||||
"dbt_layer": "silver",
|
|
||||||
"dbt_model": "my_custom_model_silver",
|
|
||||||
"sel": "my_custom_model_silver",
|
|
||||||
"tgt": "postgres",
|
|
||||||
"trino_catalog": "iceberg",
|
|
||||||
"sql_query_preview": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}",
|
|
||||||
"query": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}"
|
|
||||||
},
|
|
||||||
"status": "ok",
|
|
||||||
"errorMessage": "Configured & Valid",
|
|
||||||
"missingFields": [],
|
|
||||||
"hasError": false,
|
|
||||||
"hasWarning": false
|
|
||||||
},
|
|
||||||
"width": 96,
|
|
||||||
"height": 96,
|
|
||||||
"selected": true,
|
|
||||||
"dragging": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"edges": [],
|
|
||||||
"notes": [],
|
|
||||||
"is_draft": true,
|
|
||||||
"status": "draft",
|
|
||||||
"updated_at": "2026-10-18T17:54:28.688466Z"
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
{
|
||||||
|
"id": "ingestion_spku_realtime_lakehouse",
|
||||||
|
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||||
|
"schedule": "0 */1 * * *",
|
||||||
|
"owner_id": "local-admin",
|
||||||
|
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"id": "node-0579",
|
||||||
|
"type": "customNode",
|
||||||
|
"position": {
|
||||||
|
"x": -20,
|
||||||
|
"y": 50
|
||||||
|
},
|
||||||
|
"data": {
|
||||||
|
"label": "Source Database",
|
||||||
|
"type": "source_db",
|
||||||
|
"subtitle": "Postgres / MySQL / Oracle",
|
||||||
|
"isSource": true,
|
||||||
|
"configured": true,
|
||||||
|
"config": {
|
||||||
|
"conn": "Postgres",
|
||||||
|
"conn_id": "conn_postgres_source_db",
|
||||||
|
"table": "permits",
|
||||||
|
"table_name": "public.stasiun_spku",
|
||||||
|
"target_type": "minio_s3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"width": 96,
|
||||||
|
"height": 96,
|
||||||
|
"selected": true,
|
||||||
|
"positionAbsolute": {
|
||||||
|
"x": -20,
|
||||||
|
"y": 50
|
||||||
|
},
|
||||||
|
"dragging": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"edges": [],
|
||||||
|
"notes": [],
|
||||||
|
"is_draft": false,
|
||||||
|
"status": "deployed",
|
||||||
|
"updated_at": "2026-10-19T03:57:43.892615Z"
|
||||||
|
}
|
||||||
@@ -22,6 +22,6 @@
|
|||||||
"admin_groups": [
|
"admin_groups": [
|
||||||
"/Platform Administrators"
|
"/Platform Administrators"
|
||||||
],
|
],
|
||||||
"updated_at": "2026-10-18T20:42:26.680284+00:00",
|
"updated_at": "2026-10-19T07:47:22.327195+00:00",
|
||||||
"updated_by": "admin"
|
"updated_by": "admin"
|
||||||
}
|
}
|
||||||
Reference in New Issue
Block a user