feat: initial commit for merica-pipeline delivery

2026-10-08 19:33:15 +07:00
commit 42fedab752
46 changed files with 5484 additions and 0 deletions
@@ -0,0 +1,140 @@
+pipeline_id: example_permits_medallion_dbt_postgres
+description: 'Example Use Case: Environmental Permits Ingestion (Postgres DB) -> Bronze
+  CDC Landing -> GE Quality Bronze -> dbt-postgres Silver -> GE Quality Silver ->
+  dbt-postgres Gold Daily Mart'
+schedule: 0 8 * * *
+owner: de_team_dlh
+tags:
+- dlh
+- medallion
+- dbt_postgres
+- great_expectations
+mode: flexible
+governance:
+  domain: TataLingkunganPerizinan
+  owner: dlh_data_engineering_group
+  tier: Tier.Tier1
+  retention: P3Y
+  certification: Gold Certified
+alerting:
+  enabled: true
+  min_severity: P1
+  coalescing_window_seconds: 300
+  channels:
+    telegram:
+      enabled: true
+      bot_token_secret_key: ALERT_TELEGRAM_BOT_TOKEN
+      chat_id_secret_key: ALERT_TELEGRAM_CHAT_ID
+    slack:
+      enabled: true
+      webhook_url_secret_key: ALERT_SLACK_WEBHOOK_URL
+    discord:
+      enabled: false
+      webhook_url_secret_key: ALERT_DISCORD_WEBHOOK_URL
+    sitomat:
+      enabled: true
+      endpoint_url: http://sitomat-backend:8080/api/v1/lakehouse/webhook
+  triggers:
+    on_failure: true
+    on_ge_quality_breach: true
+    on_sla_miss: true
+    on_success: false
+tasks:
+- task_id: bronze_permits_cdc_task
+  type: source_db
+  label: Bronze DB Source (CDC Landing)
+  dependencies: []
+  config:
+    source_type: database
+    connection_id: conn_postgres_source_db
+    table_name: public.environmental_permits
+    target:
+      bucket: dlh-bronze
+      format: parquet
+- task_id: quality_check_bronze_task
+  type: ge_quality
+  label: GE Quality Check (Bronze Suite)
+  dependencies:
+  - bronze_permits_cdc_task
+  config:
+    expectation_suite: permits_bronze_suite
+    table_name: bronze_lakehouse.environmental_permits_cdc
+    rules:
+    - type: table_row_count_to_be_between
+      min_value: 1
+    - type: column_values_to_not_be_null
+      column: permit_id
+    queries:
+    - name: check_bronze_null_permit_id
+      sql: SELECT count(*) FROM bronze_lakehouse.environmental_permits_cdc WHERE permit_id
+        IS NULL
+      expected: 0
+      error_message: Null permit_id detected in Bronze
+- task_id: sitomat_register_bronze_task
+  type: sitomat_callback
+  label: Sitomat Bronze Callback
+  dependencies:
+  - quality_check_bronze_task
+  config:
+    layer: bronze
+    artifact_ref: dlh_lakehouse.bronze_lakehouse.environmental_permits_cdc
+    checksum: auto-sha256
+- task_id: silver_transformation_dbt_postgres_task
+  type: dbt_transform
+  label: dbt Silver Cleanse (dbt-postgres)
+  dependencies:
+  - sitomat_register_bronze_task
+  config:
+    dbt_layer: silver
+    dbt_model: silver_environmental_permits_clean
+    select_models: silver_environmental_permits_clean
+- task_id: quality_check_silver_task
+  type: ge_quality
+  label: GE Quality Check (Silver Suite)
+  dependencies:
+  - silver_transformation_dbt_postgres_task
+  config:
+    expectation_suite: permits_silver_suite
+    table_name: silver.silver_environmental_permits_clean
+    rules:
+    - type: column_values_to_not_be_null
+      column: permit_id
+    - type: column_values_to_be_in_set
+      column: permit_status
+      allowed_set:
+      - APPROVED
+      - IN_REVIEW
+      - REJECTED
+    queries:
+    - name: check_invalid_status_count
+      sql: SELECT count(*) FROM silver.silver_environmental_permits_clean WHERE permit_status
+        NOT IN ('APPROVED', 'IN_REVIEW', 'REJECTED')
+      expected: 0
+      error_message: Invalid status detected in Silver
+- task_id: sitomat_register_silver_task
+  type: sitomat_callback
+  label: Sitomat Silver Callback
+  dependencies:
+  - quality_check_silver_task
+  config:
+    layer: silver
+    artifact_ref: dlh_lakehouse.silver.silver_environmental_permits_clean
+    checksum: auto-sha256
+- task_id: gold_transformation_dbt_postgres_task
+  type: dbt_transform
+  label: dbt Gold Mart (dbt-postgres)
+  dependencies:
+  - sitomat_register_silver_task
+  config:
+    dbt_layer: gold
+    dbt_model: gold_permits_summary_daily
+    select_models: gold_permits_summary_daily
+- task_id: sitomat_register_gold_task
+  type: sitomat_callback
+  label: Sitomat Gold Callback (Data Product)
+  dependencies:
+  - gold_transformation_dbt_postgres_task
+  config:
+    layer: gold
+    artifact_ref: dlh_lakehouse.gold.gold_permits_summary_daily
+    checksum: auto-sha256
@@ -0,0 +1,55 @@
+pipeline_id: ingestion_api_spku_air_quality
+description: >-
+  REST API Ingestion for SPKU Realtime Air Quality Data -> MinIO S3 Bronze ->
+  DuckDB Iceberg Silver -> GE Quality Suite -> dbt Trino Gold Mart
+schedule: 0 4 * * *
+mode: flexible
+tasks:
+  - task_id: bronze_spku_api_source_task
+    type: source_api
+    label: Bronze SPKU API Source
+    dependencies: []
+    config:
+      source_type: api
+      connection_id: conn_spku_api_service
+      api_endpoint: https://api.spku.dlh.id/v1/stasiun/spku/data-realtime
+      http_method: GET
+      target:
+        bucket: dlh-bronze
+        format: json
+  - task_id: bronze_spku_minio_storage_task
+    type: minio_storage
+    label: MinIO Bronze Storage
+    dependencies:
+      - bronze_spku_api_source_task
+    config:
+      bucket: dlh-bronze
+      format: json
+  - task_id: silver_spku_duckdb_iceberg_task
+    type: duckdb_transform
+    label: DuckDB Silver Iceberg Transform
+    dependencies:
+      - bronze_spku_minio_storage_task
+    config:
+      source_parquet_path: s3://dlh-bronze/spku_realtime.parquet
+      target_table: dlh_lakehouse.silver.spku_realtime
+      primary_keys:
+        - stasiun_id
+        - waktu
+      select_models: stg_spku_realtime
+  - task_id: quality_spku_ge_suite_task
+    type: ge_quality
+    label: GE Quality (spku_api_suite)
+    dependencies:
+      - silver_spku_duckdb_iceberg_task
+    config:
+      expectation_suite: spku_api_suite
+  - task_id: gold_spku_dbt_mart_task
+    type: dbt_transform
+    label: dbt Gold Mart (fct_spku_daily_mart)
+    dependencies:
+      - quality_spku_ge_suite_task
+    config:
+      dbt_layer: gold
+      dbt_model: fct_spku_daily_mart
+
@@ -0,0 +1,37 @@
+pipeline_id: ingestion_db_environmental_permits_cdc
+description: "CDC Ingestion for Environmental Permits (Perizinan Lingkungan) Postgres DB -> Bronze -> dbt Silver -> dbt Gold"
+schedule: "0 */2 * * *"
+owner: de_team_dlh
+tags:
+  - dlh
+  - cdc
+  - permits
+mode: medallion
+
+bronze:
+  source_type: database
+  connection_id: conn_postgres_source_db
+  table_name: public.environmental_permits
+  target:
+    bucket: dlh-bronze
+    format: parquet
+
+silver:
+  engine: duckdb
+  quality_check:
+    expectation_suite: permits_bronze_suite
+  transformation:
+    engine: duckdb
+    source_parquet_path: s3://dlh-bronze/environmental_permits.parquet
+    target_table: dlh_lakehouse.silver.environmental_permits
+    primary_keys:
+      - id
+      - updated_at
+    select_models: silver_environmental_permits_clean
+
+gold:
+  engine: dbt
+  transformation:
+    engine: dbt
+    select_models: gold_permits_summary_daily
+
@@ -0,0 +1,36 @@
+pipeline_id: ingestion_gws_waste_reports
+description: "Google Workspace Spreadsheet Extractor for Waste Reports -> Pandas Cleansing -> Trino Iceberg Target -> dbt Gold Mart"
+schedule: "0 4 * * *"
+owner: de_team_dlh
+tags:
+  - dlh
+  - gws
+  - waste
+mode: medallion
+
+bronze:
+  source_type: gws
+  connection_id: conn_gws_service_account
+  spreadsheet_id: "1OSRugXZvfjoY9M0yySKT-F3L4LTZXa40J0knU3_4Hik"
+  sheet_range: "Sheet1!A1:Z1000"
+  target:
+    bucket: dlh-bronze
+    format: csv
+
+silver:
+  engine: pyiceberg
+  quality_check:
+    expectation_suite: waste_gws_suite
+  transformation:
+    engine: pyiceberg
+    source_parquet_path: s3://dlh-bronze/waste_reports.parquet
+    target_table: dlh_lakehouse.silver.waste_reports
+    write_mode: overwrite
+    select_models: stg_waste_reports
+
+gold:
+  engine: dbt
+  transformation:
+    engine: dbt
+    select_models: fct_waste_daily_mart
+