From f82726db40cd288a807e67be2ea8224e0028a11d Mon Sep 17 00:00:00 2001 From: syahril Date: Mon, 19 Oct 2026 14:54:32 +0700 Subject: [PATCH] feat(pipelines): sync spku realtime pipeline config, bronze model, and rbac mappings --- .../ingestion_spku_realtime_lakehouse.yaml | 21 +++++++++ dbt/models/bronze/bronze_stasiun_spku.sql | 4 ++ dbt/models/sources.yml | 2 + ...gestion_spku_realtime_lakehouse.draft.json | 46 ------------------- .../ingestion_spku_realtime_lakehouse.json | 44 ++++++++++++++++++ metadata/rbac_mappings.json | 2 +- 6 files changed, 72 insertions(+), 47 deletions(-) create mode 100644 configs/ingestion_spku_realtime_lakehouse.yaml create mode 100644 dbt/models/bronze/bronze_stasiun_spku.sql delete mode 100644 metadata/ingestion_spku_realtime_lakehouse.draft.json create mode 100644 metadata/ingestion_spku_realtime_lakehouse.json diff --git a/configs/ingestion_spku_realtime_lakehouse.yaml b/configs/ingestion_spku_realtime_lakehouse.yaml new file mode 100644 index 0000000..9bf933c --- /dev/null +++ b/configs/ingestion_spku_realtime_lakehouse.yaml @@ -0,0 +1,21 @@ +pipeline_id: ingestion_spku_realtime_lakehouse +description: Dynamic Data Lakehouse Pipeline via DLH Studio +schedule: 0 */1 * * * +mode: flexible +governance: + domain: Lingkungan Hidup & Kehutanan + owner: de_team_dlh + tier: Tier.Tier1 + certification: Certification.Gold +tasks: +- task_id: node-0579 + type: source_db + label: Source Database + enabled: true + dependencies: [] + config: + conn: Postgres + conn_id: conn_postgres_source_db + table: permits + table_name: public.stasiun_spku + target_type: minio_s3 diff --git a/dbt/models/bronze/bronze_stasiun_spku.sql b/dbt/models/bronze/bronze_stasiun_spku.sql new file mode 100644 index 0000000..8901345 --- /dev/null +++ b/dbt/models/bronze/bronze_stasiun_spku.sql @@ -0,0 +1,4 @@ +-- dbt Bronze View Model: stasiun_spku +{{ config(materialized='view') }} + +SELECT * FROM {{ source('public', 'stasiun_spku') }} diff --git a/dbt/models/sources.yml b/dbt/models/sources.yml index baeeb43..92e5023 100644 --- a/dbt/models/sources.yml +++ b/dbt/models/sources.yml @@ -29,3 +29,5 @@ sources: - name: raw_cems_telemetry_logs description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas pipeline + - name: stasiun_spku + description: Auto-registered source table 'stasiun_spku' from canvas pipeline diff --git a/metadata/ingestion_spku_realtime_lakehouse.draft.json b/metadata/ingestion_spku_realtime_lakehouse.draft.json deleted file mode 100644 index 32bb582..0000000 --- a/metadata/ingestion_spku_realtime_lakehouse.draft.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "ingestion_spku_realtime_lakehouse", - "description": "Dynamic Data Lakehouse Pipeline via DLH Studio", - "schedule": "0 */1 * * *", - "owner_id": "local-admin", - "owner_name": "Administrator Lokal (Break-Glass)", - "nodes": [ - { - "id": "node-7560", - "type": "customNode", - "position": { - "x": 80, - "y": 80 - }, - "data": { - "label": "dbt Transform", - "type": "dbt_transform", - "subtitle": "dbt-postgres", - "configured": true, - "config": { - "dbt_layer": "silver", - "dbt_model": "my_custom_model_silver", - "sel": "my_custom_model_silver", - "tgt": "postgres", - "trino_catalog": "iceberg", - "sql_query_preview": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}", - "query": "SELECT stasiun_id, LOWER(nama_stasiun) AS nama_stasiun, ispu_val FROM {{ ref('bronze_spku_raw') }}" - }, - "status": "ok", - "errorMessage": "Configured & Valid", - "missingFields": [], - "hasError": false, - "hasWarning": false - }, - "width": 96, - "height": 96, - "selected": true, - "dragging": false - } - ], - "edges": [], - "notes": [], - "is_draft": true, - "status": "draft", - "updated_at": "2026-10-18T17:54:28.688466Z" -} \ No newline at end of file diff --git a/metadata/ingestion_spku_realtime_lakehouse.json b/metadata/ingestion_spku_realtime_lakehouse.json new file mode 100644 index 0000000..bc10d03 --- /dev/null +++ b/metadata/ingestion_spku_realtime_lakehouse.json @@ -0,0 +1,44 @@ +{ + "id": "ingestion_spku_realtime_lakehouse", + "description": "Dynamic Data Lakehouse Pipeline via DLH Studio", + "schedule": "0 */1 * * *", + "owner_id": "local-admin", + "owner_name": "Administrator Lokal (Break-Glass)", + "nodes": [ + { + "id": "node-0579", + "type": "customNode", + "position": { + "x": -20, + "y": 50 + }, + "data": { + "label": "Source Database", + "type": "source_db", + "subtitle": "Postgres / MySQL / Oracle", + "isSource": true, + "configured": true, + "config": { + "conn": "Postgres", + "conn_id": "conn_postgres_source_db", + "table": "permits", + "table_name": "public.stasiun_spku", + "target_type": "minio_s3" + } + }, + "width": 96, + "height": 96, + "selected": true, + "positionAbsolute": { + "x": -20, + "y": 50 + }, + "dragging": false + } + ], + "edges": [], + "notes": [], + "is_draft": false, + "status": "deployed", + "updated_at": "2026-10-19T03:57:43.892615Z" +} \ No newline at end of file diff --git a/metadata/rbac_mappings.json b/metadata/rbac_mappings.json index e748519..e7df077 100644 --- a/metadata/rbac_mappings.json +++ b/metadata/rbac_mappings.json @@ -22,6 +22,6 @@ "admin_groups": [ "/Platform Administrators" ], - "updated_at": "2026-10-18T20:42:26.680284+00:00", + "updated_at": "2026-10-19T07:47:22.327195+00:00", "updated_by": "admin" } \ No newline at end of file