feat(pipeline): add e2e orders master medallion flow and dbt models
This commit is contained in:
@@ -0,0 +1,261 @@
|
||||
pipeline_id: e2e_orders_master_medallion_flow
|
||||
description: Dynamic Data Lakehouse Pipeline via DLH Studio
|
||||
schedule: 0 */1 * * *
|
||||
mode: flexible
|
||||
tasks:
|
||||
- task_id: node-9826
|
||||
type: source_db
|
||||
label: Extract Master Facilities
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: public.ref_waste_facilities
|
||||
table_name: public.ref_waste_facilities
|
||||
target_type: minio_s3
|
||||
- task_id: node-0827
|
||||
type: duckdb_transform
|
||||
label: DuckDB Master Facilities to Iceberg Bronze
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-9826
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
|
||||
source_path: s3://dlh-bronze/staging/ref_facilities/*.parquet
|
||||
target_table: bronze_ref_facilities
|
||||
table_name: bronze_ref_facilities
|
||||
primary_keys: facility_id
|
||||
query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
|
||||
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
|
||||
sql_query: SELECT facility_id, facility_name, operator_name, license_number, region_zone,
|
||||
facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1
|
||||
transformation_sql: SELECT facility_id, facility_name, operator_name, license_number,
|
||||
region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating,
|
||||
created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC)
|
||||
= 1
|
||||
- task_id: node-1812
|
||||
type: source_db
|
||||
label: Extract Raw Waste Orders
|
||||
enabled: true
|
||||
dependencies: []
|
||||
config:
|
||||
conn: Postgres
|
||||
conn_id: conn_postgres_source_db
|
||||
table: public.raw_waste_orders
|
||||
table_name: public.raw_waste_orders
|
||||
target_type: minio_s3
|
||||
- task_id: node-2810
|
||||
type: ge_quality
|
||||
label: GE Orders Staging Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-1812
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: staging_waste_orders
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-3793
|
||||
type: duckdb_transform
|
||||
label: DuckDB Orders to Iceberg Bronze
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-2810
|
||||
config:
|
||||
layer: silver
|
||||
source_parquet_path: s3://dlh-bronze/staging/waste_orders/*.parquet
|
||||
source_path: s3://dlh-bronze/staging/waste_orders/*.parquet
|
||||
target_table: bronze_waste_orders
|
||||
table_name: bronze_waste_orders
|
||||
primary_keys: order_id
|
||||
query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
|
||||
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
|
||||
sql_query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category,
|
||||
volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at
|
||||
FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY
|
||||
ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1
|
||||
transformation_sql: SELECT order_id, manifest_code, facility_id, generator_company,
|
||||
waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date,
|
||||
created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet')
|
||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) =
|
||||
1
|
||||
- task_id: node-4793
|
||||
type: ge_quality
|
||||
label: GE Orders Bronze Iceberg Quality
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-3793
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: bronze_waste_orders
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-5809
|
||||
type: dbt_transform
|
||||
label: dbt Silver Waste Manifest Enriched
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-0827
|
||||
- node-4793
|
||||
config:
|
||||
dbt_layer: silver
|
||||
dbt_model: silver_waste_manifest_enriched
|
||||
sel: silver_waste_manifest_enriched
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Silver Cleansed Manifest Orders Enriched with Master\
|
||||
\ Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n\
|
||||
\ o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n\
|
||||
\ o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee,\
|
||||
\ 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n\
|
||||
\ f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n\
|
||||
\ CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
|
||||
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
|
||||
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
|
||||
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
|
||||
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
query: "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\n\
|
||||
SELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n\
|
||||
\ o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n\
|
||||
\ ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n\
|
||||
\ o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n\
|
||||
\ f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n\
|
||||
\ WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\
|
||||
\ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\
|
||||
\ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\
|
||||
FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\
|
||||
\ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'"
|
||||
- task_id: node-6793
|
||||
type: ge_quality
|
||||
label: GE Silver Compliance Quality Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-5809
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: silver_waste_manifest_enriched
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-7793
|
||||
type: dbt_transform
|
||||
label: dbt Gold Regional Waste Revenue Mart
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-6793
|
||||
config:
|
||||
dbt_layer: gold
|
||||
dbt_model: gold_regional_waste_metrics
|
||||
sel: gold_regional_waste_metrics
|
||||
tgt: postgres
|
||||
trino_catalog: iceberg
|
||||
sql_query_preview: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT\
|
||||
\ \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS\
|
||||
\ total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n\
|
||||
\ COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton),\
|
||||
\ 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
|
||||
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
|
||||
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
|
||||
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
|
||||
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
|
||||
\ waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
query: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n\
|
||||
\ facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n \
|
||||
\ COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT\
|
||||
\ generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2)\
|
||||
\ AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\
|
||||
\ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\
|
||||
\ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\
|
||||
\ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\
|
||||
\ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\
|
||||
\ waste_category\nORDER BY total_waste_volume_ton DESC"
|
||||
- task_id: node-8871
|
||||
type: ge_quality
|
||||
label: GE Gold Mart Integrity Gate
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-7793
|
||||
config:
|
||||
ge_scope_level: all
|
||||
ge_table_name: gold_regional_waste_metrics
|
||||
test_table_row_count: true
|
||||
test_table_schema_match: true
|
||||
test_table_table_not_empty: true
|
||||
table_min_rows: 1
|
||||
table_max_rows: 1000000
|
||||
ge_col_table: stasiun_spku
|
||||
ge_target_column: stasiun_id
|
||||
ge_column_test_type: not_null
|
||||
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED',
|
||||
'IN_REVIEW', 'REJECTED')
|
||||
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
|
||||
'REJECTED')
|
||||
expected: 0
|
||||
error_message: Unexpected rows detected in table
|
||||
suite: bronze
|
||||
fail: Stop DAG
|
||||
- task_id: node-9855
|
||||
type: sitomat_callback
|
||||
label: SITOMAT Governance Callback
|
||||
enabled: true
|
||||
dependencies:
|
||||
- node-8871
|
||||
config:
|
||||
layer: gold
|
||||
artifact_ref: lakehouse.gold.gold_regional_waste_metrics
|
||||
checksum: sha256-verified-waste-orders-pass
|
||||
sitomat_base_url: http://localhost:3000
|
||||
openmetadata_fqn: dlh_airflow_service.pipeline.task
|
||||
ge_status: PASSED
|
||||
url: /sitomat/callback
|
||||
stage: bronze
|
||||
Reference in New Issue
Block a user