first commit

2026-10-09 02:35:14 +07:00
commit 4ca7a9a58e
108 changed files with 23035 additions and 0 deletions
@@ -0,0 +1,56 @@
+# =============================================================================
+# eBdesk SiCABE — dbt Project Configuration
+# =============================================================================
+# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
+# Pipeline: Bronze → Silver → Gold
+# =============================================================================
+
+name: "sicabe"
+version: "1.0.0"
+config-version: 2
+require-dbt-version: [">=1.8.0", "<2.0.0"]
+
+profile: "trino"
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  sicabe:
+    bronze_to_silver:
+      +materialized: table
+      +schema: silver
+      +database: iceberg
+    silver_to_gold:
+      +materialized: table
+      +schema: gold
+      +database: iceberg
+    incremental:
+      +materialized: incremental
+      +schema: bronze
+      +database: iceberg
+      +unique_key: "id"
+      +incremental_strategy: merge
+      # ───────────────────────────────────────────────────────────────────────
+      # WAJIB: relasi sementara harus berupa TABEL, bukan VIEW.
+      #
+      # Katalog Iceberg Trino di stack ini bertipe `nessie`, dan Trino TIDAK
+      # mendukung view di sana. Tanpa setelan ini, setiap run incremental gagal:
+      #   TrinoUserError(NOT_SUPPORTED):
+      #     "createView is not supported for Iceberg Nessie catalogs"
+      #
+      # Sebabnya: untuk strategi `merge`/`append`, dbt-trino membuat relasi
+      # sementara sebagai VIEW. `views_enabled: false` membuatnya memakai tabel.
+      # Dibuktikan 2026-09-22: MERGE (1 rows) berhasil untuk baris baru maupun
+      # baris yang diubah (upsert). Lihat reviews/sicabe-15-transform-service.md.
+      # ───────────────────────────────────────────────────────────────────────
+      +views_enabled: false
@@ -0,0 +1,15 @@
+-- =============================================================================
+-- eBdesk SiCABE — dbt Macro: generate_schema_name
+-- =============================================================================
+-- Override default schema naming agar +schema menghasilkan nama persis
+-- (bukan {default}_{custom}). Contoh: +schema: gold → schema "gold"
+-- =============================================================================
+
+{% macro generate_schema_name(custom_schema_name, node) -%}
+    {%- set default_schema = target.schema -%}
+    {%- if custom_schema_name is none -%}
+        {{ default_schema }}
+    {%- else -%}
+        {{ custom_schema_name | trim }}
+    {%- endif -%}
+{%- endmacro %}
@@ -0,0 +1,36 @@
+-- =============================================================================
+-- eBdesk SiCABE — dbt Model: Bronze → Silver
+-- =============================================================================
+-- Transformasi data mentah (bronze.orders) menjadi data bersih (silver.orders):
+--   - Normalisasi tipe data
+--   - Filter data valid (harga > 0, stok >= 0)
+--   - Standarisasi format tanggal
+-- =============================================================================
+
+WITH bronze_orders AS (
+    SELECT
+        id,
+        CAST(order_date AS DATE) AS order_date,
+        komoditas,
+        CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
+        CAST(stok_kg AS BIGINT) AS stok_kg,
+        region,
+        created_at,
+        updated_at
+    FROM {{ source('iceberg', 'orders') }}
+)
+
+SELECT
+    id,
+    order_date,
+    komoditas,
+    harga_per_kg,
+    stok_kg,
+    region,
+    -- Nilai total per transaksi
+    ROUND(harga_per_kg * stok_kg, 2) AS total_value,
+    created_at,
+    updated_at
+FROM bronze_orders
+WHERE harga_per_kg > 0
+  AND stok_kg >= 0
@@ -0,0 +1,29 @@
+-- =============================================================================
+-- eBdesk SiCABE — dbt Model: Incremental (CDC)
+-- =============================================================================
+-- Incremental update untuk data bronze.orders:
+--   - Hanya memproses baris baru/berubah (berdasarkan updated_at)
+--   - Materialized: incremental dengan unique_key = id
+-- =============================================================================
+
+{{ config(
+    materialized='incremental',
+    unique_key='id',
+    incremental_strategy='merge'
+) }}
+
+SELECT
+    id,
+    CAST(order_date AS DATE) AS order_date,
+    komoditas,
+    CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
+    CAST(stok_kg AS BIGINT) AS stok_kg,
+    region,
+    created_at,
+    updated_at
+FROM {{ source('iceberg', 'orders') }}
+
+{% if is_incremental() %}
+-- Hanya ambil data yang baru/berubah sejak run terakhir
+WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }})
+{% endif %}
@@ -0,0 +1,35 @@
+-- =============================================================================
+-- eBdesk SiCABE — dbt Model: Silver → Gold
+-- =============================================================================
+-- Agregasi data bersih (silver.orders) menjadi data analitik (gold.orders):
+--   - Rata-rata harga per komoditas per hari
+--   - Total stok per komoditas
+--   - Ranking komoditas berdasarkan nilai transaksi
+-- =============================================================================
+
+WITH silver_orders AS (
+    SELECT
+        id,
+        order_date,
+        komoditas,
+        harga_per_kg,
+        stok_kg,
+        region,
+        total_value
+    FROM {{ ref('orders_silver') }}
+)
+
+SELECT
+    order_date,
+    komoditas,
+    region,
+    COUNT(*) AS jumlah_transaksi,
+    ROUND(AVG(harga_per_kg), 2) AS avg_harga_per_kg,
+    ROUND(MIN(harga_per_kg), 2) AS min_harga_per_kg,
+    ROUND(MAX(harga_per_kg), 2) AS max_harga_per_kg,
+    SUM(stok_kg) AS total_stok_kg,
+    ROUND(SUM(total_value), 2) AS total_nilai,
+    -- Harga rata-rata tertimbang (weighted average)
+    ROUND(SUM(harga_per_kg * stok_kg) / NULLIF(SUM(stok_kg), 0), 2) AS weighted_avg_harga
+FROM silver_orders
+GROUP BY order_date, komoditas, region
@@ -0,0 +1,32 @@
+# =============================================================================
+# eBdesk SiCABE — dbt Sources
+# =============================================================================
+# Mendefinisikan sumber data di Iceberg (via Trino catalog)
+# =============================================================================
+
+version: 2
+
+sources:
+  - name: iceberg
+    database: iceberg
+    schema: bronze
+    tables:
+      - name: orders
+        description: "Data mentah orders dari PostgreSQL (via Sling)"
+        columns:
+          - name: id
+            description: "Primary key"
+          - name: order_date
+            description: "Tanggal order"
+          - name: komoditas
+            description: "Nama komoditas"
+          - name: harga_per_kg
+            description: "Harga per kilogram"
+          - name: stok_kg
+            description: "Stok dalam kilogram"
+          - name: region
+            description: "Wilayah"
+          - name: created_at
+            description: "Waktu dibuat"
+          - name: updated_at
+            description: "Waktu diupdate"