141 lines
4.1 KiB
YAML
141 lines
4.1 KiB
YAML
pipeline_id: example_permits_medallion_dbt_postgres
|
|
description: 'Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze
|
|
CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver ->
|
|
dbt-postgres Gold Daily Mart'
|
|
schedule: 0 8 * * *
|
|
owner: de_team_dlh
|
|
tags:
|
|
- dlh
|
|
- medallion
|
|
- dbt_postgres
|
|
- great_expectations
|
|
mode: flexible
|
|
governance:
|
|
domain: TataLingkunganPerizinan
|
|
owner: dlh_data_engineering_group
|
|
tier: Tier.Tier1
|
|
retention: P3Y
|
|
certification: Gold Certified
|
|
alerting:
|
|
enabled: true
|
|
min_severity: P1
|
|
coalescing_window_seconds: 300
|
|
channels:
|
|
telegram:
|
|
enabled: true
|
|
bot_token_secret_key: ALERT_TELEGRAM_BOT_TOKEN
|
|
chat_id_secret_key: ALERT_TELEGRAM_CHAT_ID
|
|
slack:
|
|
enabled: true
|
|
webhook_url_secret_key: ALERT_SLACK_WEBHOOK_URL
|
|
discord:
|
|
enabled: false
|
|
webhook_url_secret_key: ALERT_DISCORD_WEBHOOK_URL
|
|
sitomat:
|
|
enabled: true
|
|
endpoint_url: http://sitomat-backend:8080/api/v1/lakehouse/webhook
|
|
triggers:
|
|
on_failure: true
|
|
on_ge_quality_breach: true
|
|
on_sla_miss: true
|
|
on_success: false
|
|
tasks:
|
|
- task_id: bronze_permits_cdc_task
|
|
type: source_db
|
|
label: Bronze DB Source (CDC Landing)
|
|
dependencies: []
|
|
config:
|
|
source_type: database
|
|
connection_id: conn_postgres_source_db
|
|
table_name: public.environmental_permits
|
|
target:
|
|
bucket: dlh-bronze
|
|
format: parquet
|
|
- task_id: quality_check_bronze_task
|
|
type: ge_quality
|
|
label: GE Quality Check (Bronze Suite)
|
|
dependencies:
|
|
- bronze_permits_cdc_task
|
|
config:
|
|
expectation_suite: permits_bronze_suite
|
|
table_name: bronze_lakehouse.environmental_permits_cdc
|
|
rules:
|
|
- type: table_row_count_to_be_between
|
|
min_value: 1
|
|
- type: column_values_to_not_be_null
|
|
column: permit_id
|
|
queries:
|
|
- name: check_bronze_null_permit_id
|
|
sql: SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id
|
|
IS NULL
|
|
expected: 0
|
|
error_message: Null permit_id detected in Bronze
|
|
- task_id: sitomat_register_bronze_task
|
|
type: sitomat_callback
|
|
label: Sitomat Bronze Callback
|
|
dependencies:
|
|
- quality_check_bronze_task
|
|
config:
|
|
layer: bronze
|
|
artifact_ref: dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc
|
|
checksum: auto-sha256
|
|
- task_id: silver_transformation_dbt_postgres_task
|
|
type: dbt_transform
|
|
label: dbt Silver Cleanse (dbt-postgres)
|
|
dependencies:
|
|
- sitomat_register_bronze_task
|
|
config:
|
|
dbt_layer: silver
|
|
dbt_model: silver_environmental_permits_clean
|
|
select_models: silver_environmental_permits_clean
|
|
- task_id: quality_check_silver_task
|
|
type: ge_quality
|
|
label: GE Quality Check (Silver Suite)
|
|
dependencies:
|
|
- silver_transformation_dbt_postgres_task
|
|
config:
|
|
expectation_suite: permits_silver_suite
|
|
table_name: silver.silver_environmental_permits_clean
|
|
rules:
|
|
- type: column_values_to_not_be_null
|
|
column: permit_id
|
|
- type: column_values_to_be_in_set
|
|
column: permit_status
|
|
allowed_set:
|
|
- APPROVED
|
|
- IN_REVIEW
|
|
- REJECTED
|
|
queries:
|
|
- name: check_invalid_status_count
|
|
sql: SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status
|
|
NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')
|
|
expected: 0
|
|
error_message: Invalid status detected in Silver
|
|
- task_id: sitomat_register_silver_task
|
|
type: sitomat_callback
|
|
label: Sitomat Silver Callback
|
|
dependencies:
|
|
- quality_check_silver_task
|
|
config:
|
|
layer: silver
|
|
artifact_ref: dlh_lakehouse.silver.silver_environmental_permits_clean
|
|
checksum: auto-sha256
|
|
- task_id: gold_transformation_dbt_postgres_task
|
|
type: dbt_transform
|
|
label: dbt Gold Mart (dbt-postgres)
|
|
dependencies:
|
|
- sitomat_register_silver_task
|
|
config:
|
|
dbt_layer: gold
|
|
dbt_model: gold_permits_summary_daily
|
|
select_models: gold_permits_summary_daily
|
|
- task_id: sitomat_register_gold_task
|
|
type: sitomat_callback
|
|
label: Sitomat Gold Callback (Data Product)
|
|
dependencies:
|
|
- gold_transformation_dbt_postgres_task
|
|
config:
|
|
layer: gold
|
|
artifact_ref: dlh_lakehouse.gold.gold_permits_summary_daily
|
|
checksum: auto-sha256
|