38 lines
901 B
YAML
38 lines
901 B
YAML
pipeline_id: ingestion_db_environmental_permits_cdc
|
|
description: "CDC Ingestion for Environmental Permits (Perizinan Lingkungan) Postgres DB -> Bronze -> dbt Silver -> dbt Gold"
|
|
schedule: "0 */2 * * *"
|
|
owner: de_team_dlh
|
|
tags:
|
|
- dlh
|
|
- cdc
|
|
- permits
|
|
mode: medallion
|
|
|
|
bronze:
|
|
source_type: database
|
|
connection_id: conn_postgres_source_db
|
|
table_name: public.environmental_permits
|
|
target:
|
|
bucket: dlh-bronze
|
|
format: parquet
|
|
|
|
silver:
|
|
engine: duckdb
|
|
quality_check:
|
|
expectation_suite: permits_bronze_suite
|
|
transformation:
|
|
engine: duckdb
|
|
source_parquet_path: s3://dlh-bronze/environmental_permits.parquet
|
|
target_table: dlh_lakehouse.silver.environmental_permits
|
|
primary_keys:
|
|
- id
|
|
- updated_at
|
|
select_models: silver_environmental_permits_clean
|
|
|
|
gold:
|
|
engine: dbt
|
|
transformation:
|
|
engine: dbt
|
|
select_models: gold_permits_summary_daily
|
|
|