feat(appliance): initial release of MERICA standalone appliance v1.0.0
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
pipeline_id: e2e_medallion_permits_flow
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
tasks:
|
||||
- task_id: node-9377
|
||||
type: source_db
|
||||
label: Source Database
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: permits
|
||||
table_name: public.stasiun_spku
|
||||
target_type: minio_s3
|
||||
- task_id: node-0405
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-9377
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: staging_environmental_permits
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-1406
|
||||
type: duckdb_transform
|
||||
label: DuckDB Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-0405
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/db_cdc/environmental_permits/
|
||||
source_path: s3://dlh-bronze/db_cdc/environmental_permits/
|
||||
target_table: dlh_lakehouse.silver.environmental_permits
|
||||
table_name: dlh_lakehouse.silver.environmental_permits
|
||||
primary_keys: id, updated_at
|
||||
query: SELECT * FROM source_tbl
|
||||
sql_query: SELECT * FROM source_parquet
|
||||
transformation_sql: SELECT permit_id, permit_number, company_name, business_type,
|
||||
location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
- task_id: node-2426
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-1406
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: bronze_environmental_permits
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-3473
|
||||
type: dbt_transform
|
||||
label: dbt Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2426
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: stg_spku_realtime
|
||||
sel: silver
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Silver Cleansed Model with Master Table Enrichment\n\
|
||||
SELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n\
|
||||
\ m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n\
|
||||
\ ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n\
|
||||
\ p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type\
|
||||
\ = m.business_type"
|
||||
query: "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n \
|
||||
\ p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n\
|
||||
\ m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n\
|
||||
\ ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n\
|
||||
\ p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type\
|
||||
\ = m.business_type"
|
||||
- task_id: node-4489
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-3473
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: silver_environmental_compliance
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-5491
|
||||
type: dbt_transform
|
||||
label: dbt Transform
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-4489
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: stg_spku_realtime
|
||||
sel: silver
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n\
|
||||
\ business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n\
|
||||
\ COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE\
|
||||
\ WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score),\
|
||||
\ 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2)\
|
||||
\ AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance')\
|
||||
\ }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score\
|
||||
\ DESC"
|
||||
query: "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n\
|
||||
\ category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status\
|
||||
\ = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED'\
|
||||
\ THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS\
|
||||
\ avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n\
|
||||
\ CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance')\
|
||||
\ }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score\
|
||||
\ DESC"
|
||||
- task_id: node-6528
|
||||
type: ge_quality
|
||||
label: GE Quality Check
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-5491
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: gold_environmental_risk_summary
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-7512
|
||||
type: sitomat_callback
|
||||
label: Sitomat Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-6528
|
||||
config:
|
||||
layer: bronze
|
||||
artifact_ref: lakehouse.bronze.dataset_cdc
|
||||
checksum: sha256-verified-pipeline-pass
|
||||
sitomat_base_url: http://localhost:3000
|
||||
openmetadata_fqn: dlh_airflow_service.pipeline.task
|
||||
ge_status: PASSED
|
||||
url: /sitomat/callback
|
||||
stage: bronze
|
||||
@@ -0,0 +1,189 @@
|
||||
pipeline_id: e2e_modernized_ingestion_flow
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
governance:
|
||||
domain: Lingkungan Hidup & Kehutanan
|
||||
owner: de_team_dlh
|
||||
tier: Tier.Tier1
|
||||
certification: Certification.Gold
|
||||
tasks:
|
||||
- task_id: node-5630
|
||||
type: source_db
|
||||
label: Source Database (Environmental Permits)
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: public.environmental_permits
|
||||
table_name: public.environmental_permits
|
||||
target_type: minio_s3
|
||||
watermark_column: updated_at
|
||||
ingestion_mode: incremental_watermark
|
||||
- task_id: node-6855
|
||||
type: source_gws
|
||||
label: Source GWS (Waste Reports)
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
url: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026
|
||||
spreadsheet_id: gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026
|
||||
sheet_range: Sheet1
|
||||
sheet_name: Sheet1
|
||||
folder_id: 1OSRugXZvfjoY9M0yySKT
|
||||
drive_folder_id: 1OSRugXZvfjoY9M0yySKT
|
||||
selected_filename: Laporan_Sampah_Kabupaten_2026.xlsx
|
||||
header_row: 2
|
||||
- task_id: node-8156
|
||||
type: source_api_dlt
|
||||
label: Source REST API dlt (Sensor SPKU)
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
endpoint_url: https://api.spku.dlh.id/v1/telemetry
|
||||
http_method: GET
|
||||
auth_type: none
|
||||
auth_token: ''
|
||||
pagination_strategy: page_number
|
||||
page_param: page
|
||||
limit_param: limit
|
||||
page_size: 100
|
||||
cursor_column: measured_at
|
||||
lookback_hours: 3
|
||||
unnest_mode: auto_flatten
|
||||
landing_bucket: dlh-landing
|
||||
landing_prefix: api/spku_hourly/
|
||||
bronze_table: bronze_spku_telemetry
|
||||
url: https://api.spku.dlh.id/v1/telemetry
|
||||
- task_id: node-9404
|
||||
type: duckdb_transform
|
||||
label: DuckDB Bronze Ingestion & Dedup
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-5630
|
||||
- node-6855
|
||||
- node-8156
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
|
||||
source_path: s3://dlh-bronze/staging/environmental_permits/*.parquet
|
||||
target_table: bronze_environmental_permits
|
||||
table_name: bronze_environmental_permits
|
||||
primary_keys: permit_id
|
||||
query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
|
||||
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
|
||||
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
transformation_sql: SELECT permit_id, permit_number, company_name, activity_type
|
||||
AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC)
|
||||
= 1
|
||||
- task_id: node-0708
|
||||
type: dbt_transform
|
||||
label: dbt Silver Cleansed Permits Enriched
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-9404
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: silver_permits_enriched
|
||||
sel: silver_permits_enriched
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n\
|
||||
\ permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n\
|
||||
\ updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }}\nWHERE status != 'REVISED'"
|
||||
query: "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n\
|
||||
\ permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n\
|
||||
\ updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits')\
|
||||
\ }}\nWHERE status != 'REVISED'"
|
||||
test_configs:
|
||||
unique_columns: &id001
|
||||
- permit_id
|
||||
- permit_number
|
||||
not_null_columns: &id002
|
||||
- permit_id
|
||||
- status
|
||||
- updated_at
|
||||
accepted_values: {}
|
||||
enforce_contract: true
|
||||
unique_columns: *id001
|
||||
not_null_columns: *id002
|
||||
enforce_contract: true
|
||||
use_cosmos: true
|
||||
transformation_engine: cosmos
|
||||
- task_id: node-2234
|
||||
type: ge_quality
|
||||
label: GE Silver Compliance Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-0708
|
||||
config:
|
||||
ge_scope_level: table
|
||||
ge_table_name: silver_permits_enriched
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: ''
|
||||
sql: ''
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-3694
|
||||
type: dbt_transform
|
||||
label: dbt Gold Regulatory Compliance Mart
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2234
|
||||
config:
|
||||
dbt_layer: gold
|
||||
dbt_model: gold_compliance_summary_mart
|
||||
sel: gold_compliance_summary_mart
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n\
|
||||
\ status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name)\
|
||||
\ AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\
|
||||
\ }}\nGROUP BY permit_type, status"
|
||||
query: "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n\
|
||||
\ COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n\
|
||||
\ CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched')\
|
||||
\ }}\nGROUP BY permit_type, status"
|
||||
test_configs:
|
||||
unique_columns: &id003 []
|
||||
not_null_columns: &id004
|
||||
- permit_type
|
||||
- status
|
||||
accepted_values: {}
|
||||
enforce_contract: false
|
||||
unique_columns: *id003
|
||||
not_null_columns: *id004
|
||||
use_cosmos: true
|
||||
transformation_engine: cosmos
|
||||
- task_id: node-4977
|
||||
type: sitomat_callback
|
||||
label: SITOMAT Governance Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-3694
|
||||
config:
|
||||
layer: gold
|
||||
artifact_ref: lakehouse.gold.gold_compliance_summary_mart
|
||||
checksum: sha256-verified-modernized-ingestion
|
||||
sitomat_base_url: http://localhost:3000
|
||||
openmetadata_fqn: dlh_airflow_service.pipeline.task
|
||||
ge_status: PASSED
|
||||
url: /sitomat/callback
|
||||
stage: bronze
|
||||
@@ -0,0 +1,256 @@
|
||||
pipeline_id: e2e_orders_master_medallion_flow
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
governance:
|
||||
domain: Lingkungan Hidup & Kehutanan
|
||||
owner: de_team_dlh
|
||||
owner_name: Data Engineering DLH
|
||||
tier: Tier.Tier1
|
||||
certification: Certification.Gold
|
||||
tasks:
|
||||
- task_id: node-9826
|
||||
type: source_db
|
||||
label: Extract Master Facilities
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: public.ref_waste_facilities
|
||||
table_name: public.ref_waste_facilities
|
||||
target_type: minio_s3
|
||||
- task_id: node-0827
|
||||
type: duckdb_transform
|
||||
label: DuckDB Master Facilities to Iceberg Bronze
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-9826
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
|
||||
source_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
|
||||
target_table: bronze_ref_facilities
|
||||
table_name: bronze_ref_facilities
|
||||
primary_keys: facility_id
|
||||
query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
|
||||
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
|
||||
sql_query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
|
||||
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
|
||||
transformation_sql: SELECT facility_id, facility_name, operator_name, license_number,
|
||||
region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating,
|
||||
created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC)
|
||||
= 1
|
||||
- task_id: node-1812
|
||||
type: source_db
|
||||
label: Extract Raw Waste Orders
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: public.raw_waste_orders
|
||||
table_name: public.raw_waste_orders
|
||||
target_type: minio_s3
|
||||
- task_id: node-2810
|
||||
type: ge_quality
|
||||
label: GE Orders Staging Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-1812
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: staging_waste_orders
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE order_status NOT IN ('COMPLETED', 'PENDING', 'CANCELLED', 'IN_TRANSIT')
|
||||
sql: SELECT count(*) FROM public.raw_waste_orders WHERE order_status NOT IN ('COMPLETED', 'PENDING', 'CANCELLED', 'IN_TRANSIT')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-3793
|
||||
type: duckdb_transform
|
||||
label: DuckDB Orders to Iceberg Bronze
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2810
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/waste_orders/*.parquet
|
||||
source_path: s3://dlh-bronze/staging/waste_orders/*.parquet
|
||||
target_table: bronze_waste_orders
|
||||
table_name: bronze_waste_orders
|
||||
primary_keys: order_id
|
||||
query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
|
||||
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
|
||||
sql_query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
|
||||
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
|
||||
transformation_sql: SELECT order_id, manifest_code, facility_id, generator_company,
|
||||
waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date,
|
||||
created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) =
|
||||
1
|
||||
- task_id: node-4793
|
||||
type: ge_quality
|
||||
label: GE Orders Bronze Iceberg Quality
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-3793
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: bronze_waste_orders
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE volume_ton <= 0
|
||||
sql: SELECT count(*) FROM public.raw_waste_orders WHERE volume_ton <= 0
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-5809
|
||||
type: dbt_transform
|
||||
label: dbt Silver Waste Manifest Enriched
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-0827
|
||||
- node-4793
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: silver_waste_manifest_enriched
|
||||
sel: silver_waste_manifest_enriched
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Silver Cleansed Manifest Orders Enriched with Master\
|
||||
\ Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n\
|
||||
\ o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n\
|
||||
\ o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee,\
|
||||
\ 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n\
|
||||
\ f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n\
|
||||
\ CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
|
||||
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
|
||||
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
|
||||
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
|
||||
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
query: "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\n\
|
||||
SELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n\
|
||||
\ o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n\
|
||||
\ ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n\
|
||||
\ o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n\
|
||||
\ f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n\
|
||||
\ WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
|
||||
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
|
||||
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
|
||||
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
|
||||
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
- task_id: node-6793
|
||||
type: ge_quality
|
||||
label: GE Silver Compliance Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-5809
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: silver_waste_manifest_enriched
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE facility_id IS NULL
|
||||
sql: SELECT count(*) FROM public.raw_waste_orders WHERE facility_id IS NULL
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-7793
|
||||
type: dbt_transform
|
||||
label: dbt Gold Regional Waste Revenue Mart
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-6793
|
||||
config:
|
||||
dbt_layer: gold
|
||||
dbt_model: gold_regional_waste_metrics
|
||||
sel: gold_regional_waste_metrics
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT\
|
||||
\ \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS\
|
||||
\ total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n\
|
||||
\ COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton),\
|
||||
\ 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
|
||||
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
|
||||
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
|
||||
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
|
||||
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
|
||||
\ waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
query: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n\
|
||||
\ facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n \
|
||||
\ COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT\
|
||||
\ generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2)\
|
||||
\ AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
|
||||
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
|
||||
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
|
||||
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
|
||||
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
|
||||
\ waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
- task_id: node-8871
|
||||
type: ge_quality
|
||||
label: GE Gold Mart Integrity Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-7793
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: gold_regional_waste_metrics
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
sql_assertion: SELECT count(*) FROM public.ref_waste_facilities WHERE max_daily_capacity_ton <= 0
|
||||
sql: SELECT count(*) FROM public.ref_waste_facilities WHERE max_daily_capacity_ton <= 0
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-9855
|
||||
type: sitomat_callback
|
||||
label: SITOMAT Governance Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-8871
|
||||
config:
|
||||
layer: gold
|
||||
artifact_ref: lakehouse.gold.gold_regional_waste_metrics
|
||||
checksum: sha256-verified-waste-orders-pass
|
||||
sitomat_base_url: http://localhost:3000
|
||||
openmetadata_fqn: dlh_airflow_service.pipeline.task
|
||||
ge_status: PASSED
|
||||
url: /sitomat/callback
|
||||
stage: bronze
|
||||
mock_mode: true
|
||||
@@ -0,0 +1,28 @@
|
||||
name: 'dlh_transformation'
|
||||
version: '1.0.0'
|
||||
config-version: 2
|
||||
|
||||
profile: 'dlh_postgres'
|
||||
|
||||
model-paths: ["models"]
|
||||
analysis-paths: ["analyses"]
|
||||
test-paths: ["tests"]
|
||||
seed-paths: ["seeds"]
|
||||
macro-paths: ["macros"]
|
||||
snapshot-paths: ["snapshots"]
|
||||
|
||||
clean-targets:
|
||||
- "target"
|
||||
- "dbt_packages"
|
||||
|
||||
models:
|
||||
dlh_transformation:
|
||||
bronze:
|
||||
+materialized: view
|
||||
+schema: bronze
|
||||
silver:
|
||||
+materialized: table
|
||||
+schema: silver
|
||||
gold:
|
||||
+materialized: table
|
||||
+schema: gold
|
||||
@@ -0,0 +1,7 @@
|
||||
{% macro generate_schema_name(custom_schema_name, node) -%}
|
||||
{%- if custom_schema_name is none -%}
|
||||
{{ target.schema }}
|
||||
{%- else -%}
|
||||
{{ custom_schema_name | trim }}
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,20 @@
|
||||
-- dbt Bronze View Model: environmental_permits
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT
|
||||
permit_id,
|
||||
permit_number,
|
||||
applicant_name,
|
||||
company_name,
|
||||
business_type,
|
||||
activity_type,
|
||||
activity_type AS permit_type,
|
||||
location,
|
||||
status,
|
||||
environmental_score,
|
||||
issued_date,
|
||||
expiry_date,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM {{ source('public', 'environmental_permits') }}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- dbt Bronze View Model: raw_cems_telemetry_logs
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT * FROM {{ source('public', 'raw_cems_telemetry_logs') }}
|
||||
@@ -0,0 +1,4 @@
|
||||
-- dbt Bronze View Model: ref_emission_sources
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT * FROM {{ source('public', 'ref_emission_sources') }}
|
||||
@@ -0,0 +1,15 @@
|
||||
-- dbt Bronze Ref Facilities Model
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT
|
||||
facility_id,
|
||||
facility_name,
|
||||
operator_name,
|
||||
license_number,
|
||||
region_zone,
|
||||
facility_type,
|
||||
max_daily_capacity_ton,
|
||||
facility_status,
|
||||
compliance_rating,
|
||||
created_at
|
||||
FROM {{ source('public', 'ref_waste_facilities') }}
|
||||
@@ -0,0 +1,4 @@
|
||||
-- dbt Bronze View Model: stasiun_spku
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT * FROM {{ source('public', 'stasiun_spku') }}
|
||||
@@ -0,0 +1,16 @@
|
||||
-- dbt Bronze Raw Waste Orders Model
|
||||
{{ config(materialized='view') }}
|
||||
|
||||
SELECT
|
||||
order_id,
|
||||
manifest_code,
|
||||
facility_id,
|
||||
generator_company,
|
||||
waste_category,
|
||||
volume_ton,
|
||||
tariff_per_ton,
|
||||
transport_fee,
|
||||
order_status,
|
||||
order_date,
|
||||
created_at
|
||||
FROM {{ source('public', 'raw_waste_orders') }}
|
||||
@@ -0,0 +1,12 @@
|
||||
-- dbt Gold Model: fct_spku_daily_mart
|
||||
{{ config(materialized='table', schema='gold') }}
|
||||
|
||||
SELECT
|
||||
stasiun_id,
|
||||
nama_stasiun_clean,
|
||||
AVG(ispu_value) AS avg_daily_ispu,
|
||||
MAX(ispu_value) AS max_daily_ispu,
|
||||
COUNT(*) AS total_readings,
|
||||
CURRENT_DATE AS mart_date
|
||||
FROM {{ ref('stg_spku_realtime') }}
|
||||
GROUP BY stasiun_id, nama_stasiun_clean
|
||||
@@ -0,0 +1,9 @@
|
||||
-- dbt Gold Regulatory Compliance Mart
|
||||
SELECT
|
||||
permit_type,
|
||||
status,
|
||||
COUNT(permit_id) AS total_permits,
|
||||
COUNT(DISTINCT company_name) AS unique_companies,
|
||||
CURRENT_TIMESTAMP AS mart_refreshed_at
|
||||
FROM {{ ref('silver_permits_enriched') }}
|
||||
GROUP BY permit_type, status
|
||||
@@ -0,0 +1,21 @@
|
||||
-- dbt Gold Industrial Emission Compliance & Load Mart
|
||||
SELECT
|
||||
region_zone,
|
||||
industry_sector,
|
||||
COUNT(log_id) AS total_telemetry_events,
|
||||
COUNT(DISTINCT source_id) AS active_chimneys_count,
|
||||
ROUND(AVG(so2_mg_nm3), 2) AS avg_so2_concentration,
|
||||
ROUND(AVG(nox_mg_nm3), 2) AS avg_nox_concentration,
|
||||
ROUND(AVG(pm_dust_mg_nm3), 2) AS avg_pm_concentration,
|
||||
ROUND(SUM(so2_mass_rate_kg_h) / 1000.0, 3) AS total_so2_emission_ton,
|
||||
ROUND(SUM(nox_mass_rate_kg_h) / 1000.0, 3) AS total_nox_emission_ton,
|
||||
ROUND(SUM(pm_mass_rate_kg_h) / 1000.0, 3) AS total_pm_emission_ton,
|
||||
COUNT(CASE WHEN compliance_status = 'EXCEEDING_LIMIT' THEN 1 END) AS total_violations_count,
|
||||
ROUND(
|
||||
(COUNT(CASE WHEN compliance_status = 'COMPLIANT' THEN 1 END)::NUMERIC / NULLIF(COUNT(log_id), 0)) * 100.0,
|
||||
2
|
||||
) AS compliance_rate_percent,
|
||||
CURRENT_TIMESTAMP AS mart_refreshed_at
|
||||
FROM {{ ref('silver_cems_emission_enriched') }}
|
||||
GROUP BY region_zone, industry_sector
|
||||
ORDER BY total_violations_count DESC, total_so2_emission_ton DESC
|
||||
@@ -0,0 +1,16 @@
|
||||
{{ config(
|
||||
materialized='table',
|
||||
schema='gold'
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
stasiun_id,
|
||||
nama_stasiun,
|
||||
CAST(recorded_at AS DATE) AS observation_date,
|
||||
AVG(pm25_value) AS avg_pm25,
|
||||
MAX(pm25_value) AS max_pm25,
|
||||
AVG(pm10_value) AS avg_pm10,
|
||||
MAX(pm10_value) AS max_pm10,
|
||||
CURRENT_TIMESTAMP AS refreshed_at_gold
|
||||
FROM {{ ref('silver_spku_sensor_clean') }}
|
||||
GROUP BY 1, 2, 3
|
||||
@@ -0,0 +1,13 @@
|
||||
{{ config(
|
||||
materialized='table',
|
||||
schema='gold'
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
CAST(issued_date AS DATE) AS permit_date,
|
||||
permit_type,
|
||||
permit_status,
|
||||
COUNT(permit_id) AS total_permits,
|
||||
CURRENT_TIMESTAMP AS refreshed_at_gold
|
||||
FROM {{ ref('silver_environmental_permits_clean') }}
|
||||
GROUP BY 1, 2, 3
|
||||
@@ -0,0 +1,16 @@
|
||||
-- dbt Gold Regional Waste Revenue & Compliance Mart
|
||||
SELECT
|
||||
region_zone,
|
||||
facility_type,
|
||||
waste_category,
|
||||
COUNT(order_id) AS total_orders,
|
||||
COUNT(DISTINCT facility_id) AS active_facilities_count,
|
||||
COUNT(DISTINCT generator_company) AS active_generators_count,
|
||||
ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,
|
||||
ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,
|
||||
ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,
|
||||
COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,
|
||||
CURRENT_TIMESTAMP AS mart_refreshed_at
|
||||
FROM {{ ref('silver_waste_manifest_enriched') }}
|
||||
GROUP BY region_zone, facility_type, waste_category
|
||||
ORDER BY total_waste_volume_ton DESC
|
||||
@@ -0,0 +1,14 @@
|
||||
{{ config(
|
||||
materialized='table',
|
||||
schema='gold'
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
kecamatan,
|
||||
tanggal_laporan,
|
||||
COUNT(lokasi_tps) AS total_tps,
|
||||
SUM(volume_sampah_m3) AS total_volume_sampah_m3,
|
||||
AVG(volume_sampah_m3) AS avg_volume_per_tps_m3,
|
||||
CURRENT_TIMESTAMP AS refreshed_at_gold
|
||||
FROM {{ ref('silver_gws_waste_reports_clean') }}
|
||||
GROUP BY 1, 2
|
||||
@@ -0,0 +1,12 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: gold_compliance_summary_mart
|
||||
description: dbt Model gold_compliance_summary_mart auto-registered with Shift-Left
|
||||
quality assertions
|
||||
columns:
|
||||
- name: permit_type
|
||||
tests:
|
||||
- not_null
|
||||
- name: status
|
||||
tests:
|
||||
- not_null
|
||||
@@ -0,0 +1,34 @@
|
||||
version: 2
|
||||
models:
|
||||
- name: silver_permits_enriched
|
||||
description: dbt Model silver_permits_enriched auto-registered with Shift-Left quality
|
||||
assertions
|
||||
config:
|
||||
contract:
|
||||
enforced: true
|
||||
columns:
|
||||
- name: company_name
|
||||
data_type: character varying(255)
|
||||
- name: dbt_validated_at
|
||||
data_type: timestamp with time zone
|
||||
- name: issued_date
|
||||
data_type: date
|
||||
- name: permit_id
|
||||
tests:
|
||||
- unique
|
||||
- not_null
|
||||
data_type: character varying(50)
|
||||
- name: permit_number
|
||||
tests:
|
||||
- unique
|
||||
data_type: character varying(50)
|
||||
- name: permit_type
|
||||
data_type: character varying(50)
|
||||
- name: status
|
||||
tests:
|
||||
- not_null
|
||||
data_type: character varying(50)
|
||||
- name: updated_at
|
||||
tests:
|
||||
- not_null
|
||||
data_type: timestamp with time zone
|
||||
@@ -0,0 +1,31 @@
|
||||
-- dbt Silver Cleansed CEMS Telemetry Enriched with Master Sources
|
||||
SELECT
|
||||
t.log_id,
|
||||
t.telemetry_timestamp,
|
||||
t.source_id,
|
||||
s.facility_name,
|
||||
s.industry_sector,
|
||||
s.chimney_code,
|
||||
s.region_zone,
|
||||
t.so2_mg_nm3,
|
||||
s.max_so2_limit_mg_nm3,
|
||||
t.nox_mg_nm3,
|
||||
s.max_nox_limit_mg_nm3,
|
||||
t.pm_dust_mg_nm3,
|
||||
s.max_pm_limit_mg_nm3,
|
||||
t.co_mg_nm3,
|
||||
t.flue_gas_flow_m3_s,
|
||||
t.flue_temp_celsius,
|
||||
ROUND((t.so2_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS so2_mass_rate_kg_h,
|
||||
ROUND((t.nox_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS nox_mass_rate_kg_h,
|
||||
ROUND((t.pm_dust_mg_nm3 * t.flue_gas_flow_m3_s * 3600.0) / 1000000.0, 3) AS pm_mass_rate_kg_h,
|
||||
CASE
|
||||
WHEN t.so2_mg_nm3 > s.max_so2_limit_mg_nm3 OR t.nox_mg_nm3 > s.max_nox_limit_mg_nm3 OR t.pm_dust_mg_nm3 > s.max_pm_limit_mg_nm3
|
||||
THEN 'EXCEEDING_LIMIT'
|
||||
ELSE 'COMPLIANT'
|
||||
END AS compliance_status,
|
||||
t.sensor_status
|
||||
FROM {{ ref('bronze_raw_cems_telemetry_logs') }} t
|
||||
JOIN {{ ref('bronze_ref_emission_sources') }} s
|
||||
ON t.source_id = s.source_id
|
||||
WHERE t.sensor_status = 'VALID'
|
||||
@@ -0,0 +1,38 @@
|
||||
{{ config(
|
||||
materialized='incremental',
|
||||
unique_key='permit_id',
|
||||
schema='silver'
|
||||
) }}
|
||||
|
||||
WITH cdc_raw AS (
|
||||
SELECT
|
||||
CAST(permit_id AS VARCHAR) AS permit_id,
|
||||
TRIM(applicant_name) AS applicant_name,
|
||||
LOWER(permit_type) AS permit_type,
|
||||
CAST(status AS VARCHAR) AS permit_status,
|
||||
CAST(issued_date AS TIMESTAMP) AS issued_date,
|
||||
CAST(updated_at AS TIMESTAMP) AS updated_at,
|
||||
_sling_op,
|
||||
CAST(_sling_ts AS TIMESTAMP) AS _sling_ts,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY permit_id
|
||||
ORDER BY _sling_ts DESC
|
||||
) AS row_num
|
||||
FROM {{ source('bronze_lakehouse', 'environmental_permits_cdc') }}
|
||||
{% if is_incremental() %}
|
||||
WHERE _sling_ts > (SELECT MAX(_sling_ts) FROM {{ this }})
|
||||
{% endif %}
|
||||
)
|
||||
|
||||
SELECT
|
||||
permit_id,
|
||||
applicant_name,
|
||||
permit_type,
|
||||
permit_status,
|
||||
issued_date,
|
||||
updated_at,
|
||||
_sling_ts,
|
||||
CURRENT_TIMESTAMP AS ingested_at_silver
|
||||
FROM cdc_raw
|
||||
WHERE row_num = 1
|
||||
AND _sling_op != 'D' -- Buang record yang di-DELETE
|
||||
@@ -0,0 +1,22 @@
|
||||
{{ config(
|
||||
materialized='table',
|
||||
schema='silver'
|
||||
) }}
|
||||
|
||||
WITH raw_sheets AS (
|
||||
SELECT
|
||||
TRIM(CAST(lokasi_tps AS VARCHAR)) AS lokasi_tps,
|
||||
TRIM(CAST(kecamatan AS VARCHAR)) AS kecamatan,
|
||||
CAST(volume_sampah_m3 AS NUMERIC(10,2)) AS volume_sampah_m3,
|
||||
CAST(tanggal_laporan AS DATE) AS tanggal_laporan
|
||||
FROM {{ source('bronze_lakehouse', 'gws_waste_reports') }}
|
||||
WHERE lokasi_tps IS NOT NULL AND lokasi_tps != ''
|
||||
)
|
||||
|
||||
SELECT
|
||||
lokasi_tps,
|
||||
kecamatan,
|
||||
volume_sampah_m3,
|
||||
tanggal_laporan,
|
||||
CURRENT_TIMESTAMP AS ingested_at_silver
|
||||
FROM raw_sheets
|
||||
@@ -0,0 +1,12 @@
|
||||
-- dbt Silver Cleansed Environmental Permits
|
||||
SELECT
|
||||
permit_id,
|
||||
permit_number,
|
||||
company_name,
|
||||
permit_type,
|
||||
status,
|
||||
issued_date,
|
||||
updated_at,
|
||||
CURRENT_TIMESTAMP AS dbt_validated_at
|
||||
FROM {{ ref('bronze_environmental_permits') }}
|
||||
WHERE status != 'REVISED'
|
||||
@@ -0,0 +1,17 @@
|
||||
{{ config(
|
||||
materialized='table',
|
||||
schema='silver'
|
||||
) }}
|
||||
|
||||
WITH raw_json AS (
|
||||
SELECT json_payload FROM {{ source('bronze_lakehouse', 'spku_api_raw') }}
|
||||
)
|
||||
|
||||
SELECT
|
||||
(json_payload->>'stasiun_id')::VARCHAR AS stasiun_id,
|
||||
(json_payload->>'nama_stasiun')::VARCHAR AS nama_stasiun,
|
||||
(json_payload->>'pm25')::NUMERIC(8,2) AS pm25_value,
|
||||
(json_payload->>'pm10')::NUMERIC(8,2) AS pm10_value,
|
||||
(json_payload->>'recorded_at')::TIMESTAMP AS recorded_at,
|
||||
CURRENT_TIMESTAMP AS ingested_at_silver
|
||||
FROM raw_json
|
||||
@@ -0,0 +1,27 @@
|
||||
-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities
|
||||
SELECT
|
||||
o.order_id,
|
||||
o.manifest_code,
|
||||
o.order_date,
|
||||
o.generator_company,
|
||||
o.waste_category,
|
||||
o.volume_ton,
|
||||
o.tariff_per_ton,
|
||||
o.transport_fee,
|
||||
ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,
|
||||
o.order_status,
|
||||
f.facility_id,
|
||||
f.facility_name,
|
||||
f.region_zone,
|
||||
f.facility_type,
|
||||
f.compliance_rating,
|
||||
f.facility_status,
|
||||
CASE
|
||||
WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'
|
||||
WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'
|
||||
ELSE 'CAPACITY_EXCEEDED_WARNING'
|
||||
END AS environmental_compliance_status
|
||||
FROM {{ ref('bronze_waste_orders') }} o
|
||||
LEFT JOIN {{ ref('bronze_ref_facilities') }} f
|
||||
ON o.facility_id = f.facility_id
|
||||
WHERE o.order_status != 'CANCELLED'
|
||||
@@ -0,0 +1,10 @@
|
||||
-- dbt Silver Model: stg_spku_realtime
|
||||
{{ config(materialized='table', schema='silver') }}
|
||||
|
||||
SELECT
|
||||
stasiun_id,
|
||||
LOWER(TRIM(nama_stasiun)) AS nama_stasiun_clean,
|
||||
CAST(ispu_val AS FLOAT) AS ispu_value,
|
||||
created_at
|
||||
FROM {{ source('bronze', 'spku_api_raw') }}
|
||||
WHERE ispu_val IS NOT NULL
|
||||
@@ -0,0 +1,36 @@
|
||||
version: 2
|
||||
sources:
|
||||
- name: bronze_lakehouse
|
||||
schema: bronze_lakehouse
|
||||
tables:
|
||||
- name: environmental_permits_cdc
|
||||
description: Tabel landing Bronze staging untuk Change Data Capture (CDC) Perizinan
|
||||
Lingkungan
|
||||
- name: gws_waste_reports
|
||||
description: Tabel landing Bronze staging laporan penanganan sampah Google Workspace
|
||||
- name: spku_api_raw
|
||||
description: Tabel landing Bronze staging raw JSON payload sensor kualitas udara
|
||||
SPKU
|
||||
- name: bronze
|
||||
schema: bronze_lakehouse
|
||||
tables:
|
||||
- name: environmental_permits_cdc
|
||||
- name: gws_waste_reports
|
||||
- name: spku_api_raw
|
||||
- name: public
|
||||
schema: public
|
||||
tables:
|
||||
- name: ref_waste_facilities
|
||||
description: Master dataset fasilitas pengolahan limbah regional
|
||||
- name: raw_waste_orders
|
||||
description: Tabel transaksi manifest pengiriman dan penerimaan limbah
|
||||
- name: ref_emission_sources
|
||||
description: Auto-registered source table 'ref_emission_sources' from canvas pipeline
|
||||
- name: raw_cems_telemetry_logs
|
||||
description: Auto-registered source table 'raw_cems_telemetry_logs' from canvas
|
||||
pipeline
|
||||
- name: stasiun_spku
|
||||
description: Auto-registered source table 'stasiun_spku' from canvas pipeline
|
||||
- name: environmental_permits
|
||||
description: Auto-registered source table 'environmental_permits' from canvas
|
||||
pipeline
|
||||
@@ -0,0 +1,12 @@
|
||||
dlh_postgres:
|
||||
target: dev
|
||||
outputs:
|
||||
dev:
|
||||
type: postgres
|
||||
host: "{{ env_var('POSTGRES_HOST', 'postgres') }}"
|
||||
port: 5432
|
||||
user: "{{ env_var('POSTGRES_USER', 'dlh_user') }}"
|
||||
pass: "{{ env_var('POSTGRES_PASSWORD', 'dlh_password') }}"
|
||||
dbname: "{{ env_var('POSTGRES_DB', 'dlh_lakehouse') }}"
|
||||
schema: public
|
||||
threads: 4
|
||||
@@ -0,0 +1,445 @@
|
||||
{
|
||||
"id": "e2e_medallion_permits_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-9377",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source Database",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "permits",
|
||||
"table_name": "public.stasiun_spku",
|
||||
"target_type": "minio_s3"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0405",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "staging_environmental_permits",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-1406",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Transform",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
||||
"source_path": "s3://dlh-bronze/db_cdc/environmental_permits/",
|
||||
"target_table": "dlh_lakehouse.silver.environmental_permits",
|
||||
"table_name": "dlh_lakehouse.silver.environmental_permits",
|
||||
"primary_keys": "id, updated_at",
|
||||
"query": "SELECT * FROM source_tbl",
|
||||
"sql_query": "SELECT * FROM source_parquet",
|
||||
"transformation_sql": "SELECT permit_id, permit_number, company_name, business_type, location, status, environmental_score, created_at, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2426",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "bronze_environmental_permits",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3473",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Transform",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "stg_spku_realtime",
|
||||
"sel": "silver",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type",
|
||||
"query": "-- dbt Silver Cleansed Model with Master Table Enrichment\nSELECT \n p.permit_id,\n p.permit_number,\n p.company_name,\n p.business_type,\n m.category_name,\n m.risk_level,\n m.compliance_weight,\n p.environmental_score,\n ROUND(p.environmental_score * m.compliance_weight, 2) AS weighted_compliance_score,\n p.status,\n p.location,\n p.updated_at\nFROM {{ ref('bronze_environmental_permits') }} p\nLEFT JOIN {{ ref('ref_business_categories') }} m \n ON p.business_type = m.business_type"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4489",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "silver_environmental_compliance",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-5491",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Transform",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "stg_spku_realtime",
|
||||
"sel": "silver",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC",
|
||||
"query": "-- dbt Gold Aggregated Risk Mart\nSELECT \n risk_level,\n business_type,\n category_name,\n COUNT(permit_id) AS total_permits,\n COUNT(CASE WHEN status = 'ACTIVE' THEN 1 END) AS active_permits,\n COUNT(CASE WHEN status = 'SUSPENDED' THEN 1 END) AS suspended_permits,\n ROUND(AVG(environmental_score), 2) AS avg_environmental_score,\n ROUND(AVG(weighted_compliance_score), 2) AS avg_weighted_score,\n CURRENT_TIMESTAMP AS calculated_at\nFROM {{ ref('silver_environmental_compliance') }}\nGROUP BY risk_level, business_type, category_name\nORDER BY avg_weighted_score DESC"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6528",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Quality Check",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "gold_environmental_risk_summary",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-7512",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1600,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Sitomat Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "bronze",
|
||||
"artifact_ref": "lakehouse.bronze.dataset_cdc",
|
||||
"checksum": "sha256-verified-pipeline-pass",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
},
|
||||
"status": "warn",
|
||||
"errorMessage": "Disconnected from workflow",
|
||||
"missingFields": [],
|
||||
"hasError": false,
|
||||
"hasWarning": true
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-9377",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0405",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9377-node-0405"
|
||||
},
|
||||
{
|
||||
"source": "node-0405",
|
||||
"sourceHandle": null,
|
||||
"target": "node-1406",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0405-node-1406"
|
||||
},
|
||||
{
|
||||
"source": "node-1406",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2426",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-1406-node-2426"
|
||||
},
|
||||
{
|
||||
"source": "node-2426",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3473",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2426-node-3473"
|
||||
},
|
||||
{
|
||||
"source": "node-3473",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4489",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3473-node-4489"
|
||||
},
|
||||
{
|
||||
"source": "node-4489",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5491",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-4489-node-5491"
|
||||
},
|
||||
{
|
||||
"source": "node-5491",
|
||||
"sourceHandle": null,
|
||||
"target": "node-6528",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5491-node-6528"
|
||||
},
|
||||
{
|
||||
"source": "node-6528",
|
||||
"sourceHandle": null,
|
||||
"target": "node-7512",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6528-node-7512"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-18T17:53:01.689174Z"
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
{
|
||||
"id": "e2e_modernized_ingestion_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-5630",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source Database (Environmental Permits)",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.environmental_permits",
|
||||
"table_name": "public.environmental_permits",
|
||||
"target_type": "minio_s3",
|
||||
"watermark_column": "updated_at",
|
||||
"ingestion_mode": "incremental_watermark"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6855",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source GWS (Waste Reports)",
|
||||
"type": "source_gws",
|
||||
"subtitle": "Sheets & Drive PDF",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"url": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||
"spreadsheet_id": "gws_sheet_1OSRugXZvfjoY9M0yySKT_sampah_2026",
|
||||
"sheet_range": "Sheet1",
|
||||
"sheet_name": "Sheet1",
|
||||
"folder_id": "1OSRugXZvfjoY9M0yySKT",
|
||||
"drive_folder_id": "1OSRugXZvfjoY9M0yySKT",
|
||||
"selected_filename": "Laporan_Sampah_Kabupaten_2026.xlsx",
|
||||
"header_row": 2
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-8156",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Source REST API dlt (Sensor SPKU)",
|
||||
"type": "source_api_dlt",
|
||||
"subtitle": "Auto Un-nesting & Pagination",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"endpoint_url": "https://api.spku.dlh.id/v1/telemetry",
|
||||
"http_method": "GET",
|
||||
"auth_type": "none",
|
||||
"auth_token": "",
|
||||
"pagination_strategy": "page_number",
|
||||
"page_param": "page",
|
||||
"limit_param": "limit",
|
||||
"page_size": 100,
|
||||
"cursor_column": "measured_at",
|
||||
"lookback_hours": 3,
|
||||
"unnest_mode": "auto_flatten",
|
||||
"landing_bucket": "dlh-landing",
|
||||
"landing_prefix": "api/spku_hourly/",
|
||||
"bronze_table": "bronze_spku_telemetry",
|
||||
"url": "https://api.spku.dlh.id/v1/telemetry"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-9404",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Bronze Ingestion & Dedup",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/environmental_permits/*.parquet",
|
||||
"target_table": "bronze_environmental_permits",
|
||||
"table_name": "bronze_environmental_permits",
|
||||
"primary_keys": "permit_id",
|
||||
"query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
|
||||
"sql_query": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1",
|
||||
"transformation_sql": "SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0708",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Cleansed Permits Enriched",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "silver_permits_enriched",
|
||||
"sel": "silver_permits_enriched",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'",
|
||||
"query": "-- dbt Silver Cleansed Environmental Permits\nSELECT \n permit_id,\n permit_number,\n company_name,\n permit_type,\n status,\n issued_date,\n updated_at,\n CURRENT_TIMESTAMP AS dbt_validated_at\nFROM {{ ref('bronze_environmental_permits') }}\nWHERE status != 'REVISED'",
|
||||
"test_configs": {
|
||||
"unique_columns": [
|
||||
"permit_id",
|
||||
"permit_number"
|
||||
],
|
||||
"not_null_columns": [
|
||||
"permit_id",
|
||||
"status",
|
||||
"updated_at"
|
||||
],
|
||||
"accepted_values": {},
|
||||
"enforce_contract": true
|
||||
},
|
||||
"unique_columns": [
|
||||
"permit_id",
|
||||
"permit_number"
|
||||
],
|
||||
"not_null_columns": [
|
||||
"permit_id",
|
||||
"status",
|
||||
"updated_at"
|
||||
],
|
||||
"enforce_contract": true,
|
||||
"use_cosmos": true,
|
||||
"transformation_engine": "cosmos"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2234",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Silver Compliance Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "table",
|
||||
"ge_table_name": "silver_permits_enriched",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "",
|
||||
"sql": "",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3694",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Regulatory Compliance Mart",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "gold_compliance_summary_mart",
|
||||
"sel": "gold_compliance_summary_mart",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
|
||||
"query": "-- dbt Gold Regulatory Compliance Mart\nSELECT \n permit_type,\n status,\n COUNT(permit_id) AS total_permits,\n COUNT(DISTINCT company_name) AS unique_companies,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_permits_enriched') }}\nGROUP BY permit_type, status",
|
||||
"test_configs": {
|
||||
"unique_columns": [],
|
||||
"not_null_columns": [
|
||||
"permit_type",
|
||||
"status"
|
||||
],
|
||||
"accepted_values": {},
|
||||
"enforce_contract": false
|
||||
},
|
||||
"unique_columns": [],
|
||||
"not_null_columns": [
|
||||
"permit_type",
|
||||
"status"
|
||||
],
|
||||
"use_cosmos": true,
|
||||
"transformation_engine": "cosmos"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4977",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "SITOMAT Governance Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "gold",
|
||||
"artifact_ref": "lakehouse.gold.gold_compliance_summary_mart",
|
||||
"checksum": "sha256-verified-modernized-ingestion",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-5630",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5630-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-6855",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6855-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-8156",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9404",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-8156-node-9404"
|
||||
},
|
||||
{
|
||||
"source": "node-9404",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0708",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9404-node-0708"
|
||||
},
|
||||
{
|
||||
"source": "node-0708",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2234",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0708-node-2234"
|
||||
},
|
||||
{
|
||||
"source": "node-2234",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3694",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2234-node-3694"
|
||||
},
|
||||
{
|
||||
"source": "node-3694",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4977",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3694-node-4977"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-21T08:59:34.908289Z"
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
{
|
||||
"id": "e2e_orders_master_medallion_flow",
|
||||
"description": "Dynamic Data Lakehouse Pipeline via DLH Studio",
|
||||
"schedule": "0 */1 * * *",
|
||||
"owner_id": "local-admin",
|
||||
"owner_name": "Administrator Lokal (Break-Glass)",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "node-9826",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 80,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Extract Master Facilities",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.ref_waste_facilities",
|
||||
"table_name": "public.ref_waste_facilities",
|
||||
"target_type": "minio_s3"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-0827",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 270,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Master Facilities to Iceberg Bronze",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/ref_facilities/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/ref_facilities/*.parquet",
|
||||
"target_table": "bronze_ref_facilities",
|
||||
"table_name": "bronze_ref_facilities",
|
||||
"primary_keys": "facility_id",
|
||||
"query": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1",
|
||||
"sql_query": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1",
|
||||
"transformation_sql": "SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-1812",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 460,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "Extract Raw Waste Orders",
|
||||
"type": "source_db",
|
||||
"subtitle": "Postgres / MySQL / Oracle",
|
||||
"isSource": true,
|
||||
"configured": true,
|
||||
"config": {
|
||||
"conn": "Postgres",
|
||||
"conn_id": "conn_postgres_source_db",
|
||||
"table": "public.raw_waste_orders",
|
||||
"table_name": "public.raw_waste_orders",
|
||||
"target_type": "minio_s3"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-2810",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 650,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Orders Staging Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "staging_waste_orders",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-3793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 840,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "DuckDB Orders to Iceberg Bronze",
|
||||
"type": "duckdb_transform",
|
||||
"subtitle": "In-process SQL (<512MB)",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "silver",
|
||||
"source_parquet_path": "s3://dlh-bronze/staging/waste_orders/*.parquet",
|
||||
"source_path": "s3://dlh-bronze/staging/waste_orders/*.parquet",
|
||||
"target_table": "bronze_waste_orders",
|
||||
"table_name": "bronze_waste_orders",
|
||||
"primary_keys": "order_id",
|
||||
"query": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1",
|
||||
"sql_query": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1",
|
||||
"transformation_sql": "SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-4793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1030,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Orders Bronze Iceberg Quality",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "bronze_waste_orders",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-5809",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1220,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Silver Waste Manifest Enriched",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "silver",
|
||||
"dbt_model": "silver_waste_manifest_enriched",
|
||||
"sel": "silver_waste_manifest_enriched",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\nFROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities') }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'",
|
||||
"query": "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\nFROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities') }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-6793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1410,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Silver Compliance Quality Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "silver_waste_manifest_enriched",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-7793",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1600,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "dbt Gold Regional Waste Revenue Mart",
|
||||
"type": "dbt_transform",
|
||||
"subtitle": "dbt-postgres",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"dbt_layer": "gold",
|
||||
"dbt_model": "gold_regional_waste_metrics",
|
||||
"sel": "gold_regional_waste_metrics",
|
||||
"tgt": "postgres",
|
||||
"trino_catalog": "iceberg",
|
||||
"sql_query_preview": "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type, waste_category\nORDER BY total_waste_volume_ton DESC",
|
||||
"query": "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END) AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type, waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-8871",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1790,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "GE Gold Mart Integrity Gate",
|
||||
"type": "ge_quality",
|
||||
"subtitle": "Great Expectations",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"ge_scope_level": "all",
|
||||
"ge_table_name": "gold_regional_waste_metrics",
|
||||
"test_table_row_count": true,
|
||||
"test_table_schema_match": true,
|
||||
"test_table_table_not_empty": true,
|
||||
"table_min_rows": 1,
|
||||
"table_max_rows": 1000000,
|
||||
"ge_col_table": "stasiun_spku",
|
||||
"ge_target_column": "stasiun_id",
|
||||
"ge_column_test_type": "not_null",
|
||||
"sql_assertion": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"sql": "SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')",
|
||||
"expected": 0,
|
||||
"error_message": "Unexpected rows detected in table",
|
||||
"suite": "bronze",
|
||||
"fail": "Stop DAG"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": false,
|
||||
"dragging": false
|
||||
},
|
||||
{
|
||||
"id": "node-9855",
|
||||
"type": "customNode",
|
||||
"position": {
|
||||
"x": 1980,
|
||||
"y": 80
|
||||
},
|
||||
"data": {
|
||||
"label": "SITOMAT Governance Callback",
|
||||
"type": "sitomat_callback",
|
||||
"subtitle": "Status callback",
|
||||
"configured": true,
|
||||
"config": {
|
||||
"layer": "gold",
|
||||
"artifact_ref": "lakehouse.gold.gold_regional_waste_metrics",
|
||||
"checksum": "sha256-verified-waste-orders-pass",
|
||||
"sitomat_base_url": "http://localhost:3000",
|
||||
"openmetadata_fqn": "dlh_airflow_service.pipeline.task",
|
||||
"ge_status": "PASSED",
|
||||
"url": "/sitomat/callback",
|
||||
"stage": "bronze"
|
||||
}
|
||||
},
|
||||
"width": 96,
|
||||
"height": 96,
|
||||
"selected": true,
|
||||
"dragging": false
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source": "node-9826",
|
||||
"sourceHandle": null,
|
||||
"target": "node-0827",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-9826-node-0827"
|
||||
},
|
||||
{
|
||||
"source": "node-0827",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5809",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-0827-node-5809"
|
||||
},
|
||||
{
|
||||
"source": "node-1812",
|
||||
"sourceHandle": null,
|
||||
"target": "node-2810",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-1812-node-2810"
|
||||
},
|
||||
{
|
||||
"source": "node-2810",
|
||||
"sourceHandle": null,
|
||||
"target": "node-3793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-2810-node-3793"
|
||||
},
|
||||
{
|
||||
"source": "node-3793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-4793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-3793-node-4793"
|
||||
},
|
||||
{
|
||||
"source": "node-4793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-5809",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-4793-node-5809"
|
||||
},
|
||||
{
|
||||
"source": "node-5809",
|
||||
"sourceHandle": null,
|
||||
"target": "node-6793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-5809-node-6793"
|
||||
},
|
||||
{
|
||||
"source": "node-6793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-7793",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-6793-node-7793"
|
||||
},
|
||||
{
|
||||
"source": "node-7793",
|
||||
"sourceHandle": null,
|
||||
"target": "node-8871",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-7793-node-8871"
|
||||
},
|
||||
{
|
||||
"source": "node-8871",
|
||||
"sourceHandle": null,
|
||||
"target": "node-9855",
|
||||
"targetHandle": null,
|
||||
"type": "removableEdge",
|
||||
"style": {
|
||||
"stroke": "var(--muted)",
|
||||
"strokeWidth": 2
|
||||
},
|
||||
"data": {},
|
||||
"id": "reactflow__edge-node-8871-node-9855"
|
||||
}
|
||||
],
|
||||
"notes": [],
|
||||
"is_draft": false,
|
||||
"status": "deployed",
|
||||
"updated_at": "2026-10-18T18:28:07.599927Z"
|
||||
}
|
||||
Reference in New Issue
Block a user