merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
a5d766489adf73a6c8089730d42dfffd33ad2e69
merica-pipelines/configs/e2e_orders_master_medallion_flow.yaml
T

262 lines
11 KiB
YAML
Raw Normal View History

feat(pipeline): add e2e orders master medallion flow and dbt models
2026-10-19 01:53:47 +07:00
pipeline_id: e2e_orders_master_medallion_flow
description: Dynamic Data Lakehouse Pipeline via DLH Studio
schedule: 0 */1 * * *
mode: flexible
tasks:
- task_id: node-9826
type: source_db
label: Extract Master Facilities
enabled: true
dependencies: []
config:
conn: Postgres
conn_id: conn_postgres_source_db
table: public.ref_waste_facilities
table_name: public.ref_waste_facilities
target_type: minio_s3
- task_id: node-0827
type: duckdb_transform
label: DuckDB Master Facilities to Iceberg Bronze
enabled: true
dependencies:
- node-9826
config:
layer: silver
source_parquet_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
source_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
target_table: bronze_ref_facilities
table_name: bronze_ref_facilities
primary_keys: facility_id
query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
sql_query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
transformation_sql: SELECT facility_id, facility_name, operator_name, license_number,
region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating,
created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC)
= 1
- task_id: node-1812
type: source_db
label: Extract Raw Waste Orders
enabled: true
dependencies: []
config:
conn: Postgres
conn_id: conn_postgres_source_db
table: public.raw_waste_orders
table_name: public.raw_waste_orders
target_type: minio_s3
- task_id: node-2810
type: ge_quality
label: GE Orders Staging Quality Gate
enabled: true
dependencies:
- node-1812
config:
ge_scope_level: all
ge_table_name: staging_waste_orders
test_table_row_count: true
test_table_schema_match: true
test_table_table_not_empty: true
table_min_rows: 1
table_max_rows: 1000000
ge_col_table: stasiun_spku
ge_target_column: stasiun_id
ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
'IN_REVIEW', 'REJECTED')
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-3793
type: duckdb_transform
label: DuckDB Orders to Iceberg Bronze
enabled: true
dependencies:
- node-2810
config:
layer: silver
source_parquet_path: s3://dlh-bronze/staging/waste_orders/*.parquet
source_path: s3://dlh-bronze/staging/waste_orders/*.parquet
target_table: bronze_waste_orders
table_name: bronze_waste_orders
primary_keys: order_id
query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
sql_query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
transformation_sql: SELECT order_id, manifest_code, facility_id, generator_company,
waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date,
created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) =
1
- task_id: node-4793
type: ge_quality
label: GE Orders Bronze Iceberg Quality
enabled: true
dependencies:
- node-3793
config:
ge_scope_level: all
ge_table_name: bronze_waste_orders
test_table_row_count: true
test_table_schema_match: true
test_table_table_not_empty: true
table_min_rows: 1
table_max_rows: 1000000
ge_col_table: stasiun_spku
ge_target_column: stasiun_id
ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
'IN_REVIEW', 'REJECTED')
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-5809
type: dbt_transform
label: dbt Silver Waste Manifest Enriched
enabled: true
dependencies:
- node-0827
- node-4793
config:
dbt_layer: silver
dbt_model: silver_waste_manifest_enriched
sel: silver_waste_manifest_enriched
tgt: postgres
trino_catalog: iceberg
sql_query_preview: "-- dbt Silver Cleansed Manifest Orders Enriched with Master\
\ Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n\
\ o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n\
\ o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee,\
\ 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n\
\ f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n\
\ CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
query: "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\n\
SELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n\
\ o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n\
\ ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n\
\ o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n\
\ f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n\
\ WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
- task_id: node-6793
type: ge_quality
label: GE Silver Compliance Quality Gate
enabled: true
dependencies:
- node-5809
config:
ge_scope_level: all
ge_table_name: silver_waste_manifest_enriched
test_table_row_count: true
test_table_schema_match: true
test_table_table_not_empty: true
table_min_rows: 1
table_max_rows: 1000000
ge_col_table: stasiun_spku
ge_target_column: stasiun_id
ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
'IN_REVIEW', 'REJECTED')
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-7793
type: dbt_transform
label: dbt Gold Regional Waste Revenue Mart
enabled: true
dependencies:
- node-6793
config:
dbt_layer: gold
dbt_model: gold_regional_waste_metrics
sel: gold_regional_waste_metrics
tgt: postgres
trino_catalog: iceberg
sql_query_preview: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT\
\ \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS\
\ total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n\
\ COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton),\
\ 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
\ waste_category\nORDER BY total_waste_volume_ton DESC"
query: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n\
\ facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n \
\ COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT\
\ generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2)\
\ AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
\ waste_category\nORDER BY total_waste_volume_ton DESC"
- task_id: node-8871
type: ge_quality
label: GE Gold Mart Integrity Gate
enabled: true
dependencies:
- node-7793
config:
ge_scope_level: all
ge_table_name: gold_regional_waste_metrics
test_table_row_count: true
test_table_schema_match: true
test_table_table_not_empty: true
table_min_rows: 1
table_max_rows: 1000000
ge_col_table: stasiun_spku
ge_target_column: stasiun_id
ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
'IN_REVIEW', 'REJECTED')
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0
error_message: Unexpected rows detected in table
suite: bronze
fail: Stop DAG
- task_id: node-9855
type: sitomat_callback
label: SITOMAT Governance Callback
enabled: true
dependencies:
- node-8871
config:
layer: gold
artifact_ref: lakehouse.gold.gold_regional_waste_metrics
checksum: sha256-verified-waste-orders-pass
sitomat_base_url: http://localhost:3000
openmetadata_fqn: dlh_airflow_service.pipeline.task
ge_status: PASSED
url: /sitomat/callback
stage: bronze
Reference in New Issue Copy Permalink