feat: initial commit for merica-pipeline delivery

2026-10-08 19:33:15 +07:00
commit 42fedab752
46 changed files with 5484 additions and 0 deletions
@@ -0,0 +1,140 @@
+pipeline_id: example_permits_medallion_dbt_postgres
+description: 'Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze
+  CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver ->
+  dbt-postgres Gold Daily Mart'
+schedule: 0 8 * * *
+owner: de_team_dlh
+tags:
+- dlh
+- medallion
+- dbt_postgres
+- great_expectations
+mode: flexible
+governance:
+  domain: TataLingkunganPerizinan
+  owner: dlh_data_engineering_group
+  tier: Tier.Tier1
+  retention: P3Y
+  certification: Gold Certified
+alerting:
+  enabled: true
+  min_severity: P1
+  coalescing_window_seconds: 300
+  channels:
+    telegram:
+      enabled: true
+      bot_token_secret_key: ALERT_TELEGRAM_BOT_TOKEN
+      chat_id_secret_key: ALERT_TELEGRAM_CHAT_ID
+    slack:
+      enabled: true
+      webhook_url_secret_key: ALERT_SLACK_WEBHOOK_URL
+    discord:
+      enabled: false
+      webhook_url_secret_key: ALERT_DISCORD_WEBHOOK_URL
+    sitomat:
+      enabled: true
+      endpoint_url: http://sitomat-backend:8080/api/v1/lakehouse/webhook
+  triggers:
+    on_failure: true
+    on_ge_quality_breach: true
+    on_sla_miss: true
+    on_success: false
+tasks:
+- task_id: bronze_permits_cdc_task
+  type: source_db
+  label: Bronze DB Source (CDC Landing)
+  dependencies: []
+  config:
+    source_type: database
+    connection_id: conn_postgres_source_db
+    table_name: public.environmental_permits
+    target:
+      bucket: dlh-bronze
+      format: parquet
+- task_id: quality_check_bronze_task
+  type: ge_quality
+  label: GE Quality Check (Bronze Suite)
+  dependencies:
+  - bronze_permits_cdc_task
+  config:
+    expectation_suite: permits_bronze_suite
+    table_name: bronze_lakehouse.environmental_permits_cdc
+    rules:
+    - type: table_row_count_to_be_between
+      min_value: 1
+    - type: column_values_to_not_be_null
+      column: permit_id
+    queries:
+    - name: check_bronze_null_permit_id
+      sql: SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id
+        IS NULL
+      expected: 0
+      error_message: Null permit_id detected in Bronze
+- task_id: sitomat_register_bronze_task
+  type: sitomat_callback
+  label: Sitomat Bronze Callback
+  dependencies:
+  - quality_check_bronze_task
+  config:
+    layer: bronze
+    artifact_ref: dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc
+    checksum: auto-sha256
+- task_id: silver_transformation_dbt_postgres_task
+  type: dbt_transform
+  label: dbt Silver Cleanse (dbt-postgres)
+  dependencies:
+  - sitomat_register_bronze_task
+  config:
+    dbt_layer: silver
+    dbt_model: silver_environmental_permits_clean
+    select_models: silver_environmental_permits_clean
+- task_id: quality_check_silver_task
+  type: ge_quality
+  label: GE Quality Check (Silver Suite)
+  dependencies:
+  - silver_transformation_dbt_postgres_task
+  config:
+    expectation_suite: permits_silver_suite
+    table_name: silver.silver_environmental_permits_clean
+    rules:
+    - type: column_values_to_not_be_null
+      column: permit_id
+    - type: column_values_to_be_in_set
+      column: permit_status
+      allowed_set:
+      - APPROVED
+      - IN_REVIEW
+      - REJECTED
+    queries:
+    - name: check_invalid_status_count
+      sql: SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status
+        NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')
+      expected: 0
+      error_message: Invalid status detected in Silver
+- task_id: sitomat_register_silver_task
+  type: sitomat_callback
+  label: Sitomat Silver Callback
+  dependencies:
+  - quality_check_silver_task
+  config:
+    layer: silver
+    artifact_ref: dlh_lakehouse.silver.silver_environmental_permits_clean
+    checksum: auto-sha256
+- task_id: gold_transformation_dbt_postgres_task
+  type: dbt_transform
+  label: dbt Gold Mart (dbt-postgres)
+  dependencies:
+  - sitomat_register_silver_task
+  config:
+    dbt_layer: gold
+    dbt_model: gold_permits_summary_daily
+    select_models: gold_permits_summary_daily
+- task_id: sitomat_register_gold_task
+  type: sitomat_callback
+  label: Sitomat Gold Callback (Data Product)
+  dependencies:
+  - gold_transformation_dbt_postgres_task
+  config:
+    layer: gold
+    artifact_ref: dlh_lakehouse.gold.gold_permits_summary_daily
+    checksum: auto-sha256