feat(pipeline): deploy e2e_cems_industrial_emissions_flow from MERICA Canvas UI

configs/e2e_cems_industrial_emissions_flow.yaml

@@ -8,7 +8,7 @@ governance:
   tier: Tier.Tier1
   certification: Certification.Gold
 tasks:
-- task_id: node-1975
+- task_id: node-6716
   type: source_db
   label: Extract Master Emission Sources
   enabled: true
@@ -19,12 +19,12 @@ tasks:
     table: public.ref_emission_sources
     table_name: public.ref_emission_sources
     target_type: minio_s3
-- task_id: node-3042
+- task_id: node-7780
   type: duckdb_transform
   label: DuckDB Master to Bronze Iceberg
   enabled: true
   dependencies:
-  - node-1975
+  - node-6716
   config:
     layer: silver
     source_parquet_path: s3://dlh-bronze/staging/ref_emission_sources/*.parquet
@@ -47,7 +47,7 @@ tasks:
       region_zone, license_status, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_emission_sources/*.parquet')
       QUALIFY ROW_NUMBER() OVER (PARTITION BY source_id ORDER BY created_at DESC)
       = 1
-- task_id: node-4059
+- task_id: node-8779
   type: source_db
   label: Extract Raw CEMS Telemetry
   enabled: true
@@ -58,12 +58,12 @@ tasks:
     table: public.raw_cems_telemetry_logs
     table_name: public.raw_cems_telemetry_logs
     target_type: minio_s3
-- task_id: node-5075
+- task_id: node-9796
   type: ge_quality
   label: GE Telemetry Staging Check
   enabled: true
   dependencies:
-  - node-4059
+  - node-8779
   config:
     ge_scope_level: all
     ge_table_name: staging_cems_telemetry
@@ -75,43 +75,45 @@ tasks:
     ge_col_table: stasiun_spku
     ge_target_column: stasiun_id
     ge_column_test_type: not_null
-    sql_assertion: ''
-    sql: ''
+    sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
+      'IN_REVIEW', 'REJECTED')
+    sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
+      'REJECTED')
     expected: 0
     error_message: Unexpected rows detected in table
     suite: bronze
     fail: Stop DAG
-- task_id: node-6110
+- task_id: node-0811
   type: duckdb_transform
   label: DuckDB Telemetry to Bronze Iceberg
   enabled: true
   dependencies:
-  - node-5075
+  - node-9796
   config:
     layer: silver
-    source_parquet_path: s3://dlh-bronze/staging/raw_cems_telemetry/*.parquet
-    source_path: s3://dlh-bronze/staging/raw_cems_telemetry/*.parquet
+    source_parquet_path: s3://dlh-bronze/staging/raw_cems_telemetry_logs/*.parquet
+    source_path: s3://dlh-bronze/staging/raw_cems_telemetry_logs/*.parquet
     target_table: bronze_cems_telemetry_logs
     table_name: bronze_cems_telemetry_logs
     primary_keys: log_id
     query: SELECT log_id, source_id, telemetry_timestamp, so2_mg_nm3, nox_mg_nm3,
       pm_dust_mg_nm3, co_mg_nm3, flue_gas_flow_m3_s, flue_temp_celsius, sensor_status,
-      created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry/*.parquet')
+      created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry_logs/*.parquet')
       QUALIFY ROW_NUMBER() OVER (PARTITION BY log_id ORDER BY created_at DESC) = 1
     sql_query: SELECT log_id, source_id, telemetry_timestamp, so2_mg_nm3, nox_mg_nm3,
       pm_dust_mg_nm3, co_mg_nm3, flue_gas_flow_m3_s, flue_temp_celsius, sensor_status,
-      created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry/*.parquet')
+      created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry_logs/*.parquet')
       QUALIFY ROW_NUMBER() OVER (PARTITION BY log_id ORDER BY created_at DESC) = 1
     transformation_sql: SELECT log_id, source_id, telemetry_timestamp, so2_mg_nm3,
       nox_mg_nm3, pm_dust_mg_nm3, co_mg_nm3, flue_gas_flow_m3_s, flue_temp_celsius,
-      sensor_status, created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry/*.parquet')
+      sensor_status, created_at FROM read_parquet('s3://dlh-bronze/staging/raw_cems_telemetry_logs/*.parquet')
       QUALIFY ROW_NUMBER() OVER (PARTITION BY log_id ORDER BY created_at DESC) = 1
-- task_id: node-7127
+- task_id: node-1818
   type: ge_quality
   label: GE Telemetry Bronze Quality
   enabled: true
   dependencies:
-  - node-6110
+  - node-0811
   config:
     ge_scope_level: all
     ge_table_name: bronze_cems_telemetry_logs
@@ -123,19 +125,21 @@ tasks:
     ge_col_table: stasiun_spku
     ge_target_column: stasiun_id
     ge_column_test_type: not_null
-    sql_assertion: ''
-    sql: ''
+    sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
+      'IN_REVIEW', 'REJECTED')
+    sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
+      'REJECTED')
     expected: 0
     error_message: Unexpected rows detected in table
     suite: bronze
     fail: Stop DAG
-- task_id: node-8143
+- task_id: node-2864
   type: dbt_transform
   label: dbt Silver CEMS Enriched
   enabled: true
   dependencies:
-  - node-3042
-  - node-7127
+  - node-7780
+  - node-1818
   config:
     dbt_layer: silver
     dbt_model: silver_cems_emission_enriched
@@ -170,12 +174,12 @@ tasks:
       \  t.sensor_status\nFROM {{ ref('bronze_cems_telemetry_logs') }} t\nJOIN {{\
       \ ref('bronze_ref_emission_sources') }} s\n  ON t.source_id = s.source_id\n\
       WHERE t.sensor_status = 'VALID'"
-- task_id: node-9175
+- task_id: node-3898
   type: ge_quality
   label: GE Silver Compliance Quality Gate
   enabled: true
   dependencies:
-  - node-8143
+  - node-2864
   config:
     ge_scope_level: all
     ge_table_name: silver_cems_emission_enriched
@@ -187,18 +191,20 @@ tasks:
     ge_col_table: stasiun_spku
     ge_target_column: stasiun_id
     ge_column_test_type: not_null
-    sql_assertion: ''
-    sql: ''
+    sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
+      'IN_REVIEW', 'REJECTED')
+    sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
+      'REJECTED')
     expected: 0
     error_message: Unexpected rows detected in table
     suite: bronze
     fail: Stop DAG
-- task_id: node-0193
+- task_id: node-5020
   type: dbt_transform
   label: dbt Gold Industrial Compliance Mart
   enabled: true
   dependencies:
-  - node-9175
+  - node-3898
   config:
     dbt_layer: gold
     dbt_model: gold_industry_emission_compliance_mart
@@ -231,12 +237,12 @@ tasks:
       \  CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_cems_emission_enriched')\
       \ }}\nGROUP BY region_zone, industry_sector\nORDER BY total_violations_count\
       \ DESC, total_so2_emission_ton DESC"
-- task_id: node-1241
+- task_id: node-6105
   type: ge_quality
   label: GE Gold Compliance Mart Gate
   enabled: true
   dependencies:
-  - node-0193
+  - node-5020
   config:
     ge_scope_level: all
     ge_table_name: gold_industry_emission_compliance_mart
@@ -248,18 +254,20 @@ tasks:
     ge_col_table: stasiun_spku
     ge_target_column: stasiun_id
     ge_column_test_type: not_null
-    sql_assertion: ''
-    sql: ''
+    sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
+      'IN_REVIEW', 'REJECTED')
+    sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
+      'REJECTED')
     expected: 0
     error_message: Unexpected rows detected in table
     suite: bronze
     fail: Stop DAG
-- task_id: node-2409
+- task_id: node-7152
   type: sitomat_callback
   label: SITOMAT Governance Callback
   enabled: true
   dependencies:
-  - node-1241
+  - node-6105
   config:
     layer: gold
     artifact_ref: lakehouse.gold.gold_industry_emission_compliance_mart