feat: initial commit for merica-pipeline delivery

2026-10-08 19:33:15 +07:00
commit 42fedab752
46 changed files with 5484 additions and 0 deletions
@@ -0,0 +1,37 @@
+pipeline_id: ingestion_db_environmental_permits_cdc
+description: "CDC Ingestion for Environmental Permits (Perizinan Lingkungan) Postgres DB -> Bronze -> dbt Silver -> dbt Gold"
+schedule: "0 */2 * * *"
+owner: de_team_dlh
+tags:
+  - dlh
+  - cdc
+  - permits
+mode: medallion
+
+bronze:
+  source_type: database
+  connection_id: conn_postgres_source_db
+  table_name: public.environmental_permits
+  target:
+    bucket: dlh-bronze
+    format: parquet
+
+silver:
+  engine: duckdb
+  quality_check:
+    expectation_suite: permits_bronze_suite
+  transformation:
+    engine: duckdb
+    source_parquet_path: s3://dlh-bronze/environmental_permits.parquet
+    target_table: dlh_lakehouse.silver.environmental_permits
+    primary_keys:
+      - id
+      - updated_at
+    select_models: silver_environmental_permits_clean
+
+gold:
+  engine: dbt
+  transformation:
+    engine: dbt
+    select_models: gold_permits_summary_daily
+