feat(pipeline): add e2e cems industrial emissions medallion pipeline and dbt models

2026-10-19 03:50:17 +07:00
parent abce291520
commit b37ffc52ae
6 changed files with 842 additions and 26 deletions
@@ -0,0 +1,21 @@
+-- dbt Gold Industrial Emission Compliance & Load Mart
+SELECT 
+  region_zone,
+  industry_sector,
+  COUNT(log_id) AS total_telemetry_events,
+  COUNT(DISTINCT source_id) AS active_chimneys_count,
+  ROUND(AVG(so2_mg_nm3), 2) AS avg_so2_concentration,
+  ROUND(AVG(nox_mg_nm3), 2) AS avg_nox_concentration,
+  ROUND(AVG(pm_dust_mg_nm3), 2) AS avg_pm_concentration,
+  ROUND(SUM(so2_mass_rate_kg_h) / 1000.0, 3) AS total_so2_emission_ton,
+  ROUND(SUM(nox_mass_rate_kg_h) / 1000.0, 3) AS total_nox_emission_ton,
+  ROUND(SUM(pm_mass_rate_kg_h) / 1000.0, 3) AS total_pm_emission_ton,
+  COUNT(CASE WHEN compliance_status = 'EXCEEDING_LIMIT' THEN 1 END) AS total_violations_count,
+  ROUND(
+    (COUNT(CASE WHEN compliance_status = 'COMPLIANT' THEN 1 END)::NUMERIC / NULLIF(COUNT(log_id), 0)) * 100.0,
+    2
+  ) AS compliance_rate_percent,
+  CURRENT_TIMESTAMP AS mart_refreshed_at
+FROM {{ ref('silver_cems_emission_enriched') }}
+GROUP BY region_zone, industry_sector
+ORDER BY total_violations_count DESC, total_so2_emission_ton DESC
@@ -0,0 +1,31 @@
+-- dbt Silver Cleansed CEMS Telemetry Enriched with Master Sources
+SELECT 
+  t.log_id,
+  t.telemetry_timestamp,
+  t.source_id,
+  s.facility_name,
+  s.industry_sector,
+  s.chimney_code,
+  s.region_zone,
+  t.so2_mg_nm3,
+  s.max_so2_limit_mg_nm3,
+  t.nox_mg_nm3,
+  s.max_nox_limit_mg_nm3,
+  t.pm_dust_mg_nm3,
+  s.max_pm_limit_mg_nm3,
+  t.co_mg_nm3,
+  t.flue_gas_flow_m3_s,
+  t.flue_temp_celsius,
+  ROUND((t.so2_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS so2_mass_rate_kg_h,
+  ROUND((t.nox_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS nox_mass_rate_kg_h,
+  ROUND((t.pm_dust_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS pm_mass_rate_kg_h,
+  CASE 
+    WHEN t.so2_mg_nm3 > s.max_so2_limit_mg_nm3 OR t.nox_mg_nm3 > s.max_nox_limit_mg_nm3 OR t.pm_dust_mg_nm3 > s.max_pm_limit_mg_nm3 
+    THEN 'EXCEEDING_LIMIT'
+    ELSE 'COMPLIANT'
+  END AS compliance_status,
+  t.sensor_status
+FROM {{ ref('bronze_cems_telemetry_logs') }} t
+JOIN {{ ref('bronze_ref_emission_sources') }} s
+  ON t.source_id = s.source_id
+WHERE t.sensor_status = 'VALID'
@@ -1,27 +1,31 @@
 version: 2
-
 sources:
-  - name: bronze_lakehouse
-    schema: bronze_lakehouse
-    tables:
-      - name: environmental_permits_cdc
-        description: "Tabel landing Bronze staging untuk Change Data Capture (CDC) Perizinan Lingkungan"
-      - name: gws_waste_reports
-        description: "Tabel landing Bronze staging laporan penanganan sampah Google Workspace"
-      - name: spku_api_raw
-        description: "Tabel landing Bronze staging raw JSON payload sensor kualitas udara SPKU"
-
-  - name: bronze
-    schema: bronze_lakehouse
-    tables:
-      - name: environmental_permits_cdc
-      - name: gws_waste_reports
-      - name: spku_api_raw
-
-  - name: public
-    schema: public
-    tables:
-      - name: ref_waste_facilities
-        description: "Master dataset fasilitas pengolahan limbah regional"
-      - name: raw_waste_orders
-        description: "Tabel transaksi manifest pengiriman dan penerimaan limbah"
+- name: bronze_lakehouse
+  schema: bronze_lakehouse
+  tables:
+  - name: environmental_permits_cdc
+    description: Tabel landing Bronze staging untuk Change Data Capture (CDC) Perizinan
+      Lingkungan
+  - name: gws_waste_reports
+    description: Tabel landing Bronze staging laporan penanganan sampah Google Workspace
+  - name: spku_api_raw
+    description: Tabel landing Bronze staging raw JSON payload sensor kualitas udara
+      SPKU
+- name: bronze
+  schema: bronze_lakehouse
+  tables:
+  - name: environmental_permits_cdc
+  - name: gws_waste_reports
+  - name: spku_api_raw
+- name: public
+  schema: public
+  tables:
+  - name: ref_waste_facilities
+    description: Master dataset fasilitas pengolahan limbah regional
+  - name: raw_waste_orders
+    description: Tabel transaksi manifest pengiriman dan penerimaan limbah
+  - name: ref_emission_sources
+    description: Auto-registered source table 'ref_emission_sources' from canvas pipeline
+  - name: raw_cems_telemetry_logs
+    description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
+      pipeline