pipeline_id: ingestion_db_environmental_permits_cdc
description: "CDC Ingestion for Environmental Permits (Perizinan Lingkungan) Postgres DB -> Bronze -> dbt Silver -> dbt Gold"
schedule: "0 */2 * * *"
owner: de_team_dlh
tags:
  - dlh
  - cdc
  - permits
mode: medallion

bronze:
  source_type: database
  connection_id: conn_postgres_source_db
  table_name: public.environmental_permits
  target:
    bucket: dlh-bronze
    format: parquet

silver:
  engine: duckdb
  quality_check:
    expectation_suite: permits_bronze_suite
  transformation:
    engine: duckdb
    source_parquet_path: s3://dlh-bronze/environmental_permits.parquet
    target_table: dlh_lakehouse.silver.environmental_permits
    primary_keys:
      - id
      - updated_at
    select_models: silver_environmental_permits_clean

gold:
  engine: dbt
  transformation:
    engine: dbt
    select_models: gold_permits_summary_daily