feat: initial commit for merica-pipeline delivery

2026-10-08 19:33:15 +07:00
commit 42fedab752
46 changed files with 5484 additions and 0 deletions
@@ -0,0 +1,25 @@
+name: 'dlh_transformation'
+version: '1.0.0'
+config-version: 2
+
+profile: 'dlh_postgres'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  dlh_transformation:
+    silver:
+      +materialized: table
+      +schema: silver
+    gold:
+      +materialized: table
+      +schema: gold
@@ -0,0 +1,7 @@
+{% macro generate_schema_name(custom_schema_name, node) -%}
+    {%- if custom_schema_name is none -%}
+        {{ target.schema }}
+    {%- else -%}
+        {{ custom_schema_name | trim }}
+    {%- endif -%}
+{%- endmacro %}
@@ -0,0 +1,12 @@
+-- dbt Gold Model: fct_spku_daily_mart
+{{ config(materialized='table', schema='gold') }}
+
+SELECT
+    stasiun_id,
+    nama_stasiun_clean,
+    AVG(ispu_value) AS avg_daily_ispu,
+    MAX(ispu_value) AS max_daily_ispu,
+    COUNT(*) AS total_readings,
+    CURRENT_DATE AS mart_date
+FROM {{ ref('stg_spku_realtime') }}
+GROUP BY stasiun_id, nama_stasiun_clean
@@ -0,0 +1,16 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    stasiun_id,
+    nama_stasiun,
+    CAST(recorded_at AS DATE) AS observation_date,
+    AVG(pm25_value) AS avg_pm25,
+    MAX(pm25_value) AS max_pm25,
+    AVG(pm10_value) AS avg_pm10,
+    MAX(pm10_value) AS max_pm10,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_spku_sensor_clean') }}
+GROUP BY 1, 2, 3
@@ -0,0 +1,13 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    CAST(issued_date AS DATE) AS permit_date,
+    permit_type,
+    permit_status,
+    COUNT(permit_id) AS total_permits,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_environmental_permits_clean') }}
+GROUP BY 1, 2, 3
@@ -0,0 +1,14 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    kecamatan,
+    tanggal_laporan,
+    COUNT(lokasi_tps) AS total_tps,
+    SUM(volume_sampah_m3) AS total_volume_sampah_m3,
+    AVG(volume_sampah_m3) AS avg_volume_per_tps_m3,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_gws_waste_reports_clean') }}
+GROUP BY 1, 2
@@ -0,0 +1,38 @@
+{{ config(
+    materialized='incremental',
+    unique_key='permit_id',
+    schema='silver'
+) }}
+
+WITH cdc_raw AS (
+    SELECT
+        CAST(permit_id AS VARCHAR) AS permit_id,
+        TRIM(applicant_name) AS applicant_name,
+        LOWER(permit_type) AS permit_type,
+        CAST(status AS VARCHAR) AS permit_status,
+        CAST(issued_date AS TIMESTAMP) AS issued_date,
+        CAST(updated_at AS TIMESTAMP) AS updated_at,
+        _sling_op,
+        CAST(_sling_ts AS TIMESTAMP) AS _sling_ts,
+        ROW_NUMBER() OVER (
+            PARTITION BY permit_id 
+            ORDER BY _sling_ts DESC
+        ) AS row_num
+    FROM {{ source('bronze_lakehouse', 'environmental_permits_cdc') }}
+    {% if is_incremental() %}
+        WHERE _sling_ts > (SELECT MAX(_sling_ts) FROM {{ this }})
+    {% endif %}
+)
+
+SELECT
+    permit_id,
+    applicant_name,
+    permit_type,
+    permit_status,
+    issued_date,
+    updated_at,
+    _sling_ts,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM cdc_raw
+WHERE row_num = 1 
+  AND _sling_op != 'D'  -- Buang record yang di-DELETE
@@ -0,0 +1,22 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_sheets AS (
+    SELECT
+        TRIM(CAST(lokasi_tps AS VARCHAR)) AS lokasi_tps,
+        TRIM(CAST(kecamatan AS VARCHAR)) AS kecamatan,
+        CAST(volume_sampah_m3 AS NUMERIC(10,2)) AS volume_sampah_m3,
+        CAST(tanggal_laporan AS DATE) AS tanggal_laporan
+    FROM {{ source('bronze_lakehouse', 'gws_waste_reports') }}
+    WHERE lokasi_tps IS NOT NULL AND lokasi_tps != ''
+)
+
+SELECT
+    lokasi_tps,
+    kecamatan,
+    volume_sampah_m3,
+    tanggal_laporan,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_sheets
@@ -0,0 +1,17 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_json AS (
+    SELECT json_payload FROM {{ source('bronze_lakehouse', 'spku_api_raw') }}
+)
+
+SELECT
+    (json_payload->>'stasiun_id')::VARCHAR AS stasiun_id,
+    (json_payload->>'nama_stasiun')::VARCHAR AS nama_stasiun,
+    (json_payload->>'pm25')::NUMERIC(8,2) AS pm25_value,
+    (json_payload->>'pm10')::NUMERIC(8,2) AS pm10_value,
+    (json_payload->>'recorded_at')::TIMESTAMP AS recorded_at,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_json
@@ -0,0 +1,10 @@
+-- dbt Silver Model: stg_spku_realtime
+{{ config(materialized='table', schema='silver') }}
+
+SELECT
+    stasiun_id,
+    LOWER(TRIM(nama_stasiun)) AS nama_stasiun_clean,
+    CAST(ispu_val AS FLOAT) AS ispu_value,
+    created_at
+FROM {{ source('bronze', 'spku_api_raw') }}
+WHERE ispu_val IS NOT NULL
@@ -0,0 +1,19 @@
+version: 2
+
+sources:
+  - name: bronze_lakehouse
+    schema: bronze_lakehouse
+    tables:
+      - name: environmental_permits_cdc
+        description: "Tabel landing Bronze staging untuk Change Data Capture (CDC) Perizinan Lingkungan"
+      - name: gws_waste_reports
+        description: "Tabel landing Bronze staging laporan penanganan sampah Google Workspace"
+      - name: spku_api_raw
+        description: "Tabel landing Bronze staging raw JSON payload sensor kualitas udara SPKU"
+
+  - name: bronze
+    schema: bronze_lakehouse
+    tables:
+      - name: environmental_permits_cdc
+      - name: gws_waste_reports
+      - name: spku_api_raw
@@ -0,0 +1,12 @@
+dlh_postgres:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST', 'postgres') }}"
+      port: 5432
+      user: "{{ env_var('POSTGRES_USER', 'dlh_user') }}"
+      pass: "{{ env_var('POSTGRES_PASSWORD', 'dlh_password') }}"
+      dbname: "{{ env_var('POSTGRES_DB', 'dlh_lakehouse') }}"
+      schema: public
+      threads: 4