feat(appliance): initial release of MERICA standalone appliance v1.0.0

2026-10-21 17:16:46 +07:00
commit 851be0cda7
144 changed files with 31135 additions and 0 deletions
@@ -0,0 +1,28 @@
+name: 'dlh_transformation'
+version: '1.0.0'
+config-version: 2
+
+profile: 'dlh_postgres'
+
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:
+  - "target"
+  - "dbt_packages"
+
+models:
+  dlh_transformation:
+    bronze:
+      +materialized: view
+      +schema: bronze
+    silver:
+      +materialized: table
+      +schema: silver
+    gold:
+      +materialized: table
+      +schema: gold
@@ -0,0 +1,7 @@
+{% macro generate_schema_name(custom_schema_name, node) -%}
+    {%- if custom_schema_name is none -%}
+        {{ target.schema }}
+    {%- else -%}
+        {{ custom_schema_name | trim }}
+    {%- endif -%}
+{%- endmacro %}
@@ -0,0 +1,20 @@
+-- dbt Bronze View Model: environmental_permits
+{{ config(materialized='view') }}
+
+SELECT 
+    permit_id,
+    permit_number,
+    applicant_name,
+    company_name,
+    business_type,
+    activity_type,
+    activity_type AS permit_type,
+    location,
+    status,
+    environmental_score,
+    issued_date,
+    expiry_date,
+    created_at,
+    updated_at
+FROM {{ source('public', 'environmental_permits') }}
+
@@ -0,0 +1,4 @@
+-- dbt Bronze View Model: raw_cems_telemetry_logs
+{{ config(materialized='view') }}
+
+SELECT * FROM {{ source('public', 'raw_cems_telemetry_logs') }}
@@ -0,0 +1,4 @@
+-- dbt Bronze View Model: ref_emission_sources
+{{ config(materialized='view') }}
+
+SELECT * FROM {{ source('public', 'ref_emission_sources') }}
@@ -0,0 +1,15 @@
+-- dbt Bronze Ref Facilities Model
+{{ config(materialized='view') }}
+
+SELECT 
+    facility_id,
+    facility_name,
+    operator_name,
+    license_number,
+    region_zone,
+    facility_type,
+    max_daily_capacity_ton,
+    facility_status,
+    compliance_rating,
+    created_at
+FROM {{ source('public', 'ref_waste_facilities') }}
@@ -0,0 +1,4 @@
+-- dbt Bronze View Model: stasiun_spku
+{{ config(materialized='view') }}
+
+SELECT * FROM {{ source('public', 'stasiun_spku') }}
@@ -0,0 +1,16 @@
+-- dbt Bronze Raw Waste Orders Model
+{{ config(materialized='view') }}
+
+SELECT 
+    order_id,
+    manifest_code,
+    facility_id,
+    generator_company,
+    waste_category,
+    volume_ton,
+    tariff_per_ton,
+    transport_fee,
+    order_status,
+    order_date,
+    created_at
+FROM {{ source('public', 'raw_waste_orders') }}
@@ -0,0 +1,12 @@
+-- dbt Gold Model: fct_spku_daily_mart
+{{ config(materialized='table', schema='gold') }}
+
+SELECT
+    stasiun_id,
+    nama_stasiun_clean,
+    AVG(ispu_value) AS avg_daily_ispu,
+    MAX(ispu_value) AS max_daily_ispu,
+    COUNT(*) AS total_readings,
+    CURRENT_DATE AS mart_date
+FROM {{ ref('stg_spku_realtime') }}
+GROUP BY stasiun_id, nama_stasiun_clean
@@ -0,0 +1,9 @@
+-- dbt Gold Regulatory Compliance Mart
+SELECT 
+  permit_type,
+  status,
+  COUNT(permit_id) AS total_permits,
+  COUNT(DISTINCT company_name) AS unique_companies,
+  CURRENT_TIMESTAMP AS mart_refreshed_at
+FROM {{ ref('silver_permits_enriched') }}
+GROUP BY permit_type, status
@@ -0,0 +1,21 @@
+-- dbt Gold Industrial Emission Compliance & Load Mart
+SELECT 
+  region_zone,
+  industry_sector,
+  COUNT(log_id) AS total_telemetry_events,
+  COUNT(DISTINCT source_id) AS active_chimneys_count,
+  ROUND(AVG(so2_mg_nm3), 2) AS avg_so2_concentration,
+  ROUND(AVG(nox_mg_nm3), 2) AS avg_nox_concentration,
+  ROUND(AVG(pm_dust_mg_nm3), 2) AS avg_pm_concentration,
+  ROUND(SUM(so2_mass_rate_kg_h) / 1000.0, 3) AS total_so2_emission_ton,
+  ROUND(SUM(nox_mass_rate_kg_h) / 1000.0, 3) AS total_nox_emission_ton,
+  ROUND(SUM(pm_mass_rate_kg_h) / 1000.0, 3) AS total_pm_emission_ton,
+  COUNT(CASE WHEN compliance_status = 'EXCEEDING_LIMIT' THEN 1 END) AS total_violations_count,
+  ROUND(
+    (COUNT(CASE WHEN compliance_status = 'COMPLIANT' THEN 1 END)::NUMERIC / NULLIF(COUNT(log_id), 0)) * 100.0,
+    2
+  ) AS compliance_rate_percent,
+  CURRENT_TIMESTAMP AS mart_refreshed_at
+FROM {{ ref('silver_cems_emission_enriched') }}
+GROUP BY region_zone, industry_sector
+ORDER BY total_violations_count DESC, total_so2_emission_ton DESC
@@ -0,0 +1,16 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    stasiun_id,
+    nama_stasiun,
+    CAST(recorded_at AS DATE) AS observation_date,
+    AVG(pm25_value) AS avg_pm25,
+    MAX(pm25_value) AS max_pm25,
+    AVG(pm10_value) AS avg_pm10,
+    MAX(pm10_value) AS max_pm10,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_spku_sensor_clean') }}
+GROUP BY 1, 2, 3
@@ -0,0 +1,13 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    CAST(issued_date AS DATE) AS permit_date,
+    permit_type,
+    permit_status,
+    COUNT(permit_id) AS total_permits,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_environmental_permits_clean') }}
+GROUP BY 1, 2, 3
@@ -0,0 +1,16 @@
+-- dbt Gold Regional Waste Revenue & Compliance Mart
+SELECT 
+  region_zone,
+  facility_type,
+  waste_category,
+  COUNT(order_id) AS total_orders,
+  COUNT(DISTINCT facility_id) AS active_facilities_count,
+  COUNT(DISTINCT generator_company) AS active_generators_count,
+  ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,
+  ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,
+  ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,
+  COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,
+  CURRENT_TIMESTAMP AS mart_refreshed_at
+FROM {{ ref('silver_waste_manifest_enriched') }}
+GROUP BY region_zone, facility_type, waste_category
+ORDER BY total_waste_volume_ton DESC
@@ -0,0 +1,14 @@
+{{ config(
+    materialized='table',
+    schema='gold'
+) }}
+
+SELECT
+    kecamatan,
+    tanggal_laporan,
+    COUNT(lokasi_tps) AS total_tps,
+    SUM(volume_sampah_m3) AS total_volume_sampah_m3,
+    AVG(volume_sampah_m3) AS avg_volume_per_tps_m3,
+    CURRENT_TIMESTAMP AS refreshed_at_gold
+FROM {{ ref('silver_gws_waste_reports_clean') }}
+GROUP BY 1, 2
@@ -0,0 +1,12 @@
+version: 2
+models:
+- name: gold_compliance_summary_mart
+  description: dbt Model gold_compliance_summary_mart auto-registered with Shift-Left
+    quality assertions
+  columns:
+  - name: permit_type
+    tests:
+    - not_null
+  - name: status
+    tests:
+    - not_null
@@ -0,0 +1,34 @@
+version: 2
+models:
+- name: silver_permits_enriched
+  description: dbt Model silver_permits_enriched auto-registered with Shift-Left quality
+    assertions
+  config:
+    contract:
+      enforced: true
+  columns:
+  - name: company_name
+    data_type: character varying(255)
+  - name: dbt_validated_at
+    data_type: timestamp with time zone
+  - name: issued_date
+    data_type: date
+  - name: permit_id
+    tests:
+    - unique
+    - not_null
+    data_type: character varying(50)
+  - name: permit_number
+    tests:
+    - unique
+    data_type: character varying(50)
+  - name: permit_type
+    data_type: character varying(50)
+  - name: status
+    tests:
+    - not_null
+    data_type: character varying(50)
+  - name: updated_at
+    tests:
+    - not_null
+    data_type: timestamp with time zone
@@ -0,0 +1,31 @@
+-- dbt Silver Cleansed CEMS Telemetry Enriched with Master Sources
+SELECT 
+  t.log_id,
+  t.telemetry_timestamp,
+  t.source_id,
+  s.facility_name,
+  s.industry_sector,
+  s.chimney_code,
+  s.region_zone,
+  t.so2_mg_nm3,
+  s.max_so2_limit_mg_nm3,
+  t.nox_mg_nm3,
+  s.max_nox_limit_mg_nm3,
+  t.pm_dust_mg_nm3,
+  s.max_pm_limit_mg_nm3,
+  t.co_mg_nm3,
+  t.flue_gas_flow_m3_s,
+  t.flue_temp_celsius,
+  ROUND((t.so2_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS so2_mass_rate_kg_h,
+  ROUND((t.nox_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS nox_mass_rate_kg_h,
+  ROUND((t.pm_dust_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS pm_mass_rate_kg_h,
+  CASE 
+    WHEN t.so2_mg_nm3 > s.max_so2_limit_mg_nm3 OR t.nox_mg_nm3 > s.max_nox_limit_mg_nm3 OR t.pm_dust_mg_nm3 > s.max_pm_limit_mg_nm3 
+    THEN 'EXCEEDING_LIMIT'
+    ELSE 'COMPLIANT'
+  END AS compliance_status,
+  t.sensor_status
+FROM {{ ref('bronze_raw_cems_telemetry_logs') }} t
+JOIN {{ ref('bronze_ref_emission_sources') }} s
+  ON t.source_id = s.source_id
+WHERE t.sensor_status = 'VALID'
@@ -0,0 +1,38 @@
+{{ config(
+    materialized='incremental',
+    unique_key='permit_id',
+    schema='silver'
+) }}
+
+WITH cdc_raw AS (
+    SELECT
+        CAST(permit_id AS VARCHAR) AS permit_id,
+        TRIM(applicant_name) AS applicant_name,
+        LOWER(permit_type) AS permit_type,
+        CAST(status AS VARCHAR) AS permit_status,
+        CAST(issued_date AS TIMESTAMP) AS issued_date,
+        CAST(updated_at AS TIMESTAMP) AS updated_at,
+        _sling_op,
+        CAST(_sling_ts AS TIMESTAMP) AS _sling_ts,
+        ROW_NUMBER() OVER (
+            PARTITION BY permit_id 
+            ORDER BY _sling_ts DESC
+        ) AS row_num
+    FROM {{ source('bronze_lakehouse', 'environmental_permits_cdc') }}
+    {% if is_incremental() %}
+        WHERE _sling_ts > (SELECT MAX(_sling_ts) FROM {{ this }})
+    {% endif %}
+)
+
+SELECT
+    permit_id,
+    applicant_name,
+    permit_type,
+    permit_status,
+    issued_date,
+    updated_at,
+    _sling_ts,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM cdc_raw
+WHERE row_num = 1 
+  AND _sling_op != 'D'  -- Buang record yang di-DELETE
@@ -0,0 +1,22 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_sheets AS (
+    SELECT
+        TRIM(CAST(lokasi_tps AS VARCHAR)) AS lokasi_tps,
+        TRIM(CAST(kecamatan AS VARCHAR)) AS kecamatan,
+        CAST(volume_sampah_m3 AS NUMERIC(10,2)) AS volume_sampah_m3,
+        CAST(tanggal_laporan AS DATE) AS tanggal_laporan
+    FROM {{ source('bronze_lakehouse', 'gws_waste_reports') }}
+    WHERE lokasi_tps IS NOT NULL AND lokasi_tps != ''
+)
+
+SELECT
+    lokasi_tps,
+    kecamatan,
+    volume_sampah_m3,
+    tanggal_laporan,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_sheets
@@ -0,0 +1,12 @@
+-- dbt Silver Cleansed Environmental Permits
+SELECT 
+  permit_id,
+  permit_number,
+  company_name,
+  permit_type,
+  status,
+  issued_date,
+  updated_at,
+  CURRENT_TIMESTAMP AS dbt_validated_at
+FROM {{ ref('bronze_environmental_permits') }}
+WHERE status != 'REVISED'
@@ -0,0 +1,17 @@
+{{ config(
+    materialized='table',
+    schema='silver'
+) }}
+
+WITH raw_json AS (
+    SELECT json_payload FROM {{ source('bronze_lakehouse', 'spku_api_raw') }}
+)
+
+SELECT
+    (json_payload->>'stasiun_id')::VARCHAR AS stasiun_id,
+    (json_payload->>'nama_stasiun')::VARCHAR AS nama_stasiun,
+    (json_payload->>'pm25')::NUMERIC(8,2) AS pm25_value,
+    (json_payload->>'pm10')::NUMERIC(8,2) AS pm10_value,
+    (json_payload->>'recorded_at')::TIMESTAMP AS recorded_at,
+    CURRENT_TIMESTAMP AS ingested_at_silver
+FROM raw_json
@@ -0,0 +1,27 @@
+-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities
+SELECT 
+  o.order_id,
+  o.manifest_code,
+  o.order_date,
+  o.generator_company,
+  o.waste_category,
+  o.volume_ton,
+  o.tariff_per_ton,
+  o.transport_fee,
+  ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,
+  o.order_status,
+  f.facility_id,
+  f.facility_name,
+  f.region_zone,
+  f.facility_type,
+  f.compliance_rating,
+  f.facility_status,
+  CASE 
+    WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'
+    WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'
+    ELSE 'CAPACITY_EXCEEDED_WARNING'
+  END AS environmental_compliance_status
+FROM {{ ref('bronze_waste_orders') }} o
+LEFT JOIN {{ ref('bronze_ref_facilities') }} f
+  ON o.facility_id = f.facility_id
+WHERE o.order_status != 'CANCELLED'
@@ -0,0 +1,10 @@
+-- dbt Silver Model: stg_spku_realtime
+{{ config(materialized='table', schema='silver') }}
+
+SELECT
+    stasiun_id,
+    LOWER(TRIM(nama_stasiun)) AS nama_stasiun_clean,
+    CAST(ispu_val AS FLOAT) AS ispu_value,
+    created_at
+FROM {{ source('bronze', 'spku_api_raw') }}
+WHERE ispu_val IS NOT NULL
@@ -0,0 +1,36 @@
+version: 2
+sources:
+- name: bronze_lakehouse
+  schema: bronze_lakehouse
+  tables:
+  - name: environmental_permits_cdc
+    description: Tabel landing Bronze staging untuk Change Data Capture (CDC) Perizinan
+      Lingkungan
+  - name: gws_waste_reports
+    description: Tabel landing Bronze staging laporan penanganan sampah Google Workspace
+  - name: spku_api_raw
+    description: Tabel landing Bronze staging raw JSON payload sensor kualitas udara
+      SPKU
+- name: bronze
+  schema: bronze_lakehouse
+  tables:
+  - name: environmental_permits_cdc
+  - name: gws_waste_reports
+  - name: spku_api_raw
+- name: public
+  schema: public
+  tables:
+  - name: ref_waste_facilities
+    description: Master dataset fasilitas pengolahan limbah regional
+  - name: raw_waste_orders
+    description: Tabel transaksi manifest pengiriman dan penerimaan limbah
+  - name: ref_emission_sources
+    description: Auto-registered source table 'ref_emission_sources' from canvas pipeline
+  - name: raw_cems_telemetry_logs
+    description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
+      pipeline
+  - name: stasiun_spku
+    description: Auto-registered source table 'stasiun_spku' from canvas pipeline
+  - name: environmental_permits
+    description: Auto-registered source table 'environmental_permits' from canvas
+      pipeline
@@ -0,0 +1,12 @@
+dlh_postgres:
+  target: dev
+  outputs:
+    dev:
+      type: postgres
+      host: "{{ env_var('POSTGRES_HOST', 'postgres') }}"
+      port: 5432
+      user: "{{ env_var('POSTGRES_USER', 'dlh_user') }}"
+      pass: "{{ env_var('POSTGRES_PASSWORD', 'dlh_password') }}"
+      dbname: "{{ env_var('POSTGRES_DB', 'dlh_lakehouse') }}"
+      schema: public
+      threads: 4