pipeline_id: example_permits_medallion_dbt_postgres
description: 'Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze
  CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver ->
  dbt-postgres Gold Daily Mart'
schedule: 0 8 * * *
owner: de_team_dlh
tags:
- dlh
- medallion
- dbt_postgres
- great_expectations
mode: flexible
governance:
  domain: TataLingkunganPerizinan
  owner: dlh_data_engineering_group
  tier: Tier.Tier1
  retention: P3Y
  certification: Gold Certified
alerting:
  enabled: true
  min_severity: P1
  coalescing_window_seconds: 300
  channels:
    telegram:
      enabled: true
      bot_token_secret_key: ALERT_TELEGRAM_BOT_TOKEN
      chat_id_secret_key: ALERT_TELEGRAM_CHAT_ID
    slack:
      enabled: true
      webhook_url_secret_key: ALERT_SLACK_WEBHOOK_URL
    discord:
      enabled: false
      webhook_url_secret_key: ALERT_DISCORD_WEBHOOK_URL
    sitomat:
      enabled: true
      endpoint_url: http://sitomat-backend:8080/api/v1/lakehouse/webhook
  triggers:
    on_failure: true
    on_ge_quality_breach: true
    on_sla_miss: true
    on_success: false
tasks:
- task_id: bronze_permits_cdc_task
  type: source_db
  label: Bronze DB Source (CDC Landing)
  dependencies: []
  config:
    source_type: database
    connection_id: conn_postgres_source_db
    table_name: public.environmental_permits
    target:
      bucket: dlh-bronze
      format: parquet
- task_id: quality_check_bronze_task
  type: ge_quality
  label: GE Quality Check (Bronze Suite)
  dependencies:
  - bronze_permits_cdc_task
  config:
    expectation_suite: permits_bronze_suite
    table_name: bronze_lakehouse.environmental_permits_cdc
    rules:
    - type: table_row_count_to_be_between
      min_value: 1
    - type: column_values_to_not_be_null
      column: permit_id
    queries:
    - name: check_bronze_null_permit_id
      sql: SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id
        IS NULL
      expected: 0
      error_message: Null permit_id detected in Bronze
- task_id: sitomat_register_bronze_task
  type: sitomat_callback
  label: Sitomat Bronze Callback
  dependencies:
  - quality_check_bronze_task
  config:
    layer: bronze
    artifact_ref: dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc
    checksum: auto-sha256
- task_id: silver_transformation_dbt_postgres_task
  type: dbt_transform
  label: dbt Silver Cleanse (dbt-postgres)
  dependencies:
  - sitomat_register_bronze_task
  config:
    dbt_layer: silver
    dbt_model: silver_environmental_permits_clean
    select_models: silver_environmental_permits_clean
- task_id: quality_check_silver_task
  type: ge_quality
  label: GE Quality Check (Silver Suite)
  dependencies:
  - silver_transformation_dbt_postgres_task
  config:
    expectation_suite: permits_silver_suite
    table_name: silver.silver_environmental_permits_clean
    rules:
    - type: column_values_to_not_be_null
      column: permit_id
    - type: column_values_to_be_in_set
      column: permit_status
      allowed_set:
      - APPROVED
      - IN_REVIEW
      - REJECTED
    queries:
    - name: check_invalid_status_count
      sql: SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status
        NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')
      expected: 0
      error_message: Invalid status detected in Silver
- task_id: sitomat_register_silver_task
  type: sitomat_callback
  label: Sitomat Silver Callback
  dependencies:
  - quality_check_silver_task
  config:
    layer: silver
    artifact_ref: dlh_lakehouse.silver.silver_environmental_permits_clean
    checksum: auto-sha256
- task_id: gold_transformation_dbt_postgres_task
  type: dbt_transform
  label: dbt Gold Mart (dbt-postgres)
  dependencies:
  - sitomat_register_silver_task
  config:
    dbt_layer: gold
    dbt_model: gold_permits_summary_daily
    select_models: gold_permits_summary_daily
- task_id: sitomat_register_gold_task
  type: sitomat_callback
  label: Sitomat Gold Callback (Data Product)
  dependencies:
  - gold_transformation_dbt_postgres_task
  config:
    layer: gold
    artifact_ref: dlh_lakehouse.gold.gold_permits_summary_daily
    checksum: auto-sha256