feat: initial commit for merica-pipeline delivery

dbt/models/silver/silver_environmental_permits_clean.sql

@@ -0,0 +1,38 @@
+{{ config(
+    materialized='incremental',
+    unique_key='permit_id',
+    schema='silver'
+) }}
+
+WITH cdc_raw AS (
+    SELECT
+        CAST(permit_id AS VARCHAR) AS permit_id,
+        TRIM(applicant_name) AS applicant_name,
+        LOWER(permit_type) AS permit_type,
+        CAST(status AS VARCHAR) AS permit_status,
+        CAST(issued_date AS TIMESTAMP) AS issued_date,
+        CAST(updated_at AS TIMESTAMP) AS updated_at,
+        _sling_op,
+        CAST(_sling_ts AS TIMESTAMP) AS _sling_ts,
+        ROW_NUMBER() OVER (
+            PARTITION BY permit_id 
+            ORDER BY _sling_ts DESC
+        ) AS row_num
+    FROM {{ source('bronze_lakehouse', 'environmental_permits_cdc') }}
+    {% if is_incremental() %}
+        WHERE _sling_ts > (SELECT MAX(_sling_ts) FROM {{ this }})
+    {% endif %}
+)
+
+SELECT
+    permit_id,
+    applicant_name,
+    permit_type,
+    permit_status,
+    issued_date,
+    updated_at,
+    _sling_ts,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM cdc_raw
+WHERE row_num = 1 
+  AND _sling_op != 'D'  -- Buang record yang di-DELETE

dbt/models/silver/silver_gws_waste_reports_clean.sql

@@ -0,0 +1,22 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_sheets AS (
+    SELECT
+        TRIM(CAST(lokasi_tps AS VARCHAR)) AS lokasi_tps,
+        TRIM(CAST(kecamatan AS VARCHAR)) AS kecamatan,
+        CAST(volume_sampah_m3 AS NUMERIC(10,2)) AS volume_sampah_m3,
+        CAST(tanggal_laporan AS DATE) AS tanggal_laporan
+    FROM {{ source('bronze_lakehouse', 'gws_waste_reports') }}
+    WHERE lokasi_tps IS NOT NULL AND lokasi_tps != ''
+)
+
+SELECT
+    lokasi_tps,
+    kecamatan,
+    volume_sampah_m3,
+    tanggal_laporan,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_sheets

dbt/models/silver/silver_spku_sensor_clean.sql

@@ -0,0 +1,17 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_json AS (
+    SELECT json_payload FROM {{ source('bronze_lakehouse', 'spku_api_raw') }}
+)
+
+SELECT
+    (json_payload->>'stasiun_id')::VARCHAR AS stasiun_id,
+    (json_payload->>'nama_stasiun')::VARCHAR AS nama_stasiun,
+    (json_payload->>'pm25')::NUMERIC(8,2) AS pm25_value,
+    (json_payload->>'pm10')::NUMERIC(8,2) AS pm10_value,
+    (json_payload->>'recorded_at')::TIMESTAMP AS recorded_at,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_json

dbt/models/silver/stg_spku_realtime.sql

@@ -0,0 +1,10 @@
+-- dbt Silver Model: stg_spku_realtime
+{{ config(materialized='table', schema='silver') }}
+
+SELECT
+    stasiun_id,
+    LOWER(TRIM(nama_stasiun)) AS nama_stasiun_clean,
+    CAST(ispu_val AS FLOAT) AS ispu_value,
+    created_at
+FROM {{ source('bronze', 'spku_api_raw') }}
+WHERE ispu_val IS NOT NULL