pipeline_id: e2e_orders_master_medallion_flow description: Dynamic Data Lakehouse Pipeline via DLH Studio schedule: 0 */1 * * * mode: flexible tasks: - task_id: node-9826 type: source_db label: Extract Master Facilities enabled: true dependencies: [] config: conn: Postgres conn_id: conn_postgres_source_db table: public.ref_waste_facilities table_name: public.ref_waste_facilities target_type: minio_s3 - task_id: node-0827 type: duckdb_transform label: DuckDB Master Facilities to Iceberg Bronze enabled: true dependencies: - node-9826 config: layer: silver source_parquet_path: s3://dlh-bronze/staging/ref_facilities/*.parquet source_path: s3://dlh-bronze/staging/ref_facilities/*.parquet target_table: bronze_ref_facilities table_name: bronze_ref_facilities primary_keys: facility_id query: SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1 sql_query: SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1 transformation_sql: SELECT facility_id, facility_name, operator_name, license_number, region_zone, facility_type, max_daily_capacity_ton, facility_status, compliance_rating, created_at FROM read_parquet('s3://dlh-bronze/staging/ref_facilities/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY facility_id ORDER BY created_at DESC) = 1 - task_id: node-1812 type: source_db label: Extract Raw Waste Orders enabled: true dependencies: [] config: conn: Postgres conn_id: conn_postgres_source_db table: public.raw_waste_orders table_name: public.raw_waste_orders target_type: minio_s3 - task_id: node-2810 type: ge_quality label: GE Orders Staging Quality Gate enabled: true dependencies: - node-1812 config: ge_scope_level: all ge_table_name: staging_waste_orders test_table_row_count: true test_table_schema_match: true test_table_table_not_empty: true table_min_rows: 1 table_max_rows: 1000000 ge_col_table: stasiun_spku ge_target_column: stasiun_id sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE order_status NOT IN ('COMPLETED', 'PENDING', 'CANCELLED', 'IN_TRANSIT') sql: SELECT count(*) FROM public.raw_waste_orders WHERE order_status NOT IN ('COMPLETED', 'PENDING', 'CANCELLED', 'IN_TRANSIT') expected: 0 error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG - task_id: node-3793 type: duckdb_transform label: DuckDB Orders to Iceberg Bronze enabled: true dependencies: - node-2810 config: layer: silver source_parquet_path: s3://dlh-bronze/staging/waste_orders/*.parquet source_path: s3://dlh-bronze/staging/waste_orders/*.parquet target_table: bronze_waste_orders table_name: bronze_waste_orders primary_keys: order_id query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1 sql_query: SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1 transformation_sql: SELECT order_id, manifest_code, facility_id, generator_company, waste_category, volume_ton, tariff_per_ton, transport_fee, order_status, order_date, created_at FROM read_parquet('s3://dlh-bronze/staging/waste_orders/*.parquet') QUALIFY ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY created_at DESC) = 1 - task_id: node-4793 type: ge_quality label: GE Orders Bronze Iceberg Quality enabled: true dependencies: - node-3793 config: ge_scope_level: all ge_table_name: bronze_waste_orders test_table_row_count: true test_table_schema_match: true test_table_table_not_empty: true table_min_rows: 1 table_max_rows: 1000000 ge_col_table: stasiun_spku ge_target_column: stasiun_id sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE volume_ton <= 0 sql: SELECT count(*) FROM public.raw_waste_orders WHERE volume_ton <= 0 expected: 0 error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG - task_id: node-5809 type: dbt_transform label: dbt Silver Waste Manifest Enriched enabled: true dependencies: - node-0827 - node-4793 config: dbt_layer: silver dbt_model: silver_waste_manifest_enriched sel: silver_waste_manifest_enriched tgt: postgres trino_catalog: iceberg sql_query_preview: "-- dbt Silver Cleansed Manifest Orders Enriched with Master\ \ Facilities\nSELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n\ \ o.generator_company,\n o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n\ \ o.transport_fee,\n ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee,\ \ 2) AS total_order_amount,\n o.order_status,\n f.facility_id,\n f.facility_name,\n\ \ f.region_zone,\n f.facility_type,\n f.compliance_rating,\n f.facility_status,\n\ \ CASE \n WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\ \ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\ \ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\ FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\ \ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'" query: "-- dbt Silver Cleansed Manifest Orders Enriched with Master Facilities\n\ SELECT \n o.order_id,\n o.manifest_code,\n o.order_date,\n o.generator_company,\n\ \ o.waste_category,\n o.volume_ton,\n o.tariff_per_ton,\n o.transport_fee,\n\ \ ROUND((o.volume_ton * o.tariff_per_ton) + o.transport_fee, 2) AS total_order_amount,\n\ \ o.order_status,\n f.facility_id,\n f.facility_name,\n f.region_zone,\n\ \ f.facility_type,\n f.compliance_rating,\n f.facility_status,\n CASE \n\ \ WHEN f.facility_status = 'ACTIVE' AND o.volume_ton <= f.max_daily_capacity_ton\ \ THEN 'COMPLIANT'\n WHEN f.facility_status != 'ACTIVE' THEN 'UNLICENSED_FACILITY_WARNING'\n\ \ ELSE 'CAPACITY_EXCEEDED_WARNING'\n END AS environmental_compliance_status\n\ FROM {{ ref('bronze_waste_orders') }} o\nLEFT JOIN {{ ref('bronze_ref_facilities')\ \ }} f\n ON o.facility_id = f.facility_id\nWHERE o.order_status != 'CANCELLED'" - task_id: node-6793 type: ge_quality label: GE Silver Compliance Quality Gate enabled: true dependencies: - node-5809 config: ge_scope_level: all ge_table_name: silver_waste_manifest_enriched test_table_row_count: true test_table_schema_match: true test_table_table_not_empty: true table_min_rows: 1 table_max_rows: 1000000 ge_col_table: stasiun_spku ge_target_column: stasiun_id sql_assertion: SELECT count(*) FROM public.raw_waste_orders WHERE facility_id IS NULL sql: SELECT count(*) FROM public.raw_waste_orders WHERE facility_id IS NULL expected: 0 error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG - task_id: node-7793 type: dbt_transform label: dbt Gold Regional Waste Revenue Mart enabled: true dependencies: - node-6793 config: dbt_layer: gold dbt_model: gold_regional_waste_metrics sel: gold_regional_waste_metrics tgt: postgres trino_catalog: iceberg sql_query_preview: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT\ \ \n region_zone,\n facility_type,\n waste_category,\n COUNT(order_id) AS\ \ total_orders,\n COUNT(DISTINCT facility_id) AS active_facilities_count,\n\ \ COUNT(DISTINCT generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton),\ \ 2) AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\ \ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\ \ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\ \ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\ \ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\ \ waste_category\nORDER BY total_waste_volume_ton DESC" query: "-- dbt Gold Regional Waste Revenue & Compliance Mart\nSELECT \n region_zone,\n\ \ facility_type,\n waste_category,\n COUNT(order_id) AS total_orders,\n \ \ COUNT(DISTINCT facility_id) AS active_facilities_count,\n COUNT(DISTINCT\ \ generator_company) AS active_generators_count,\n ROUND(SUM(volume_ton), 2)\ \ AS total_waste_volume_ton,\n ROUND(SUM(total_order_amount), 2) AS total_gross_revenue,\n\ \ ROUND(AVG(total_order_amount / NULLIF(volume_ton, 0)), 2) AS avg_revenue_per_ton,\n\ \ COUNT(CASE WHEN environmental_compliance_status = 'COMPLIANT' THEN 1 END)\ \ AS compliant_orders_count,\n CURRENT_TIMESTAMP AS mart_refreshed_at\nFROM\ \ {{ ref('silver_waste_manifest_enriched') }}\nGROUP BY region_zone, facility_type,\ \ waste_category\nORDER BY total_waste_volume_ton DESC" - task_id: node-8871 type: ge_quality label: GE Gold Mart Integrity Gate enabled: true dependencies: - node-7793 config: ge_scope_level: all ge_table_name: gold_regional_waste_metrics test_table_row_count: true test_table_schema_match: true test_table_table_not_empty: true table_min_rows: 1 table_max_rows: 1000000 ge_col_table: stasiun_spku ge_target_column: stasiun_id sql_assertion: SELECT count(*) FROM public.ref_waste_facilities WHERE max_daily_capacity_ton <= 0 sql: SELECT count(*) FROM public.ref_waste_facilities WHERE max_daily_capacity_ton <= 0 expected: 0 error_message: Unexpected rows detected in table suite: bronze fail: Stop DAG - task_id: node-9855 type: sitomat_callback label: SITOMAT Governance Callback enabled: true dependencies: - node-8871 config: layer: gold artifact_ref: lakehouse.gold.gold_regional_waste_metrics checksum: sha256-verified-waste-orders-pass sitomat_base_url: http://localhost:3000 openmetadata_fqn: dlh_airflow_service.pipeline.task ge_status: PASSED url: /sitomat/callback stage: bronze mock_mode: true