{ "id": "example_permits_medallion_dbt_postgres", "description": "Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver -> dbt-postgres Gold Daily Mart", "schedule": "0 8 * * *", "nodes": [ { "id": "node-bronze-src", "type": "customNode", "position": { "x": 50, "y": 200 }, "data": { "label": "Bronze DB Source (CDC Landing)", "type": "source_db", "configured": true, "config": { "source_type": "database", "connection_id": "conn_postgres_source_db", "table_name": "public.environmental_permits", "target_table": "bronze_lakehouse.environmental_permits_cdc" } } }, { "id": "node-ge-bronze", "type": "customNode", "position": { "x": 300, "y": 200 }, "data": { "label": "GE Quality Check (Bronze Suite)", "type": "ge_quality", "configured": true, "config": { "expectation_suite": "permits_bronze_suite", "table_name": "bronze_lakehouse.environmental_permits_cdc", "rules": [ { "type": "table_row_count_to_be_between", "min_value": 1 }, { "type": "column_values_to_not_be_null", "column": "permit_id" } ], "queries": [ { "name": "check_bronze_null_permit_id", "sql": "SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id IS NULL", "expected": 0, "error_message": "Null permit_id detected in Bronze" } ] } } }, { "id": "node-sitomat-bronze", "type": "customNode", "position": { "x": 550, "y": 200 }, "data": { "label": "Sitomat Bronze Callback", "type": "sitomat_callback", "configured": true, "config": { "layer": "bronze", "artifact_ref": "dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc", "checksum": "auto-sha256" } } }, { "id": "node-silver-dbt", "type": "customNode", "position": { "x": 800, "y": 200 }, "data": { "label": "dbt Silver Cleanse (dbt-postgres)", "type": "dbt_transform", "configured": true, "config": { "dbt_layer": "silver", "dbt_model": "silver_environmental_permits_clean", "target_table": "dlh_lakehouse.silver.silver_environmental_permits_clean" } } }, { "id": "node-ge-silver", "type": "customNode", "position": { "x": 1050, "y": 200 }, "data": { "label": "GE Quality Check (Silver Suite)", "type": "ge_quality", "configured": true, "config": { "expectation_suite": "permits_silver_suite", "table_name": "silver.silver_environmental_permits_clean", "rules": [ { "type": "column_values_to_not_be_null", "column": "permit_id" }, { "type": "column_values_to_be_in_set", "column": "permit_status", "allowed_set": [ "APPROVED", "IN_REVIEW", "REJECTED" ] } ], "queries": [ { "name": "check_invalid_status_count", "sql": "SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')", "expected": 0, "error_message": "Invalid status detected in Silver" } ] } } }, { "id": "node-sitomat-silver", "type": "customNode", "position": { "x": 1300, "y": 200 }, "data": { "label": "Sitomat Silver Callback", "type": "sitomat_callback", "configured": true, "config": { "layer": "silver", "artifact_ref": "dlh_lakehouse.silver.silver_environmental_permits_clean", "checksum": "auto-sha256" } } }, { "id": "node-gold-dbt", "type": "customNode", "position": { "x": 1550, "y": 200 }, "data": { "label": "dbt Gold Mart (dbt-postgres)", "type": "dbt_transform", "configured": true, "config": { "dbt_layer": "gold", "dbt_model": "gold_permits_summary_daily", "target_table": "dlh_lakehouse.gold.gold_permits_summary_daily" } } }, { "id": "node-sitomat-gold", "type": "customNode", "position": { "x": 1800, "y": 200 }, "data": { "label": "Sitomat Gold Callback (Data Product)", "type": "sitomat_callback", "configured": true, "config": { "layer": "gold", "artifact_ref": "dlh_lakehouse.gold.gold_permits_summary_daily", "checksum": "auto-sha256" } } } ], "edges": [ { "id": "e1-src-ge_bronze", "source": "node-bronze-src", "target": "node-ge-bronze" }, { "id": "e2-ge_bronze-sitomat_bronze", "source": "node-ge-bronze", "target": "node-sitomat-bronze" }, { "id": "e3-sitomat_bronze-silver", "source": "node-sitomat-bronze", "target": "node-silver-dbt" }, { "id": "e4-silver-ge_silver", "source": "node-silver-dbt", "target": "node-ge-silver" }, { "id": "e5-ge_silver-sitomat_silver", "source": "node-ge-silver", "target": "node-sitomat-silver" }, { "id": "e6-sitomat_silver-gold", "source": "node-sitomat-silver", "target": "node-gold-dbt" }, { "id": "e7-gold-sitomat_gold", "source": "node-gold-dbt", "target": "node-sitomat-gold" } ], "is_draft": false, "status": "deployed" }