feat: initial commit for merica-pipeline delivery

2026-10-08 19:33:15 +07:00
commit 42fedab752
46 changed files with 5484 additions and 0 deletions
@@ -0,0 +1,36 @@
+pipeline_id: ingestion_gws_waste_reports
+description: "Google Workspace Spreadsheet Extractor for Waste Reports -> Pandas Cleansing -> Trino Iceberg Target -> dbt Gold Mart"
+schedule: "0 4 * * *"
+owner: de_team_dlh
+tags:
+  - dlh
+  - gws
+  - waste
+mode: medallion
+
+bronze:
+  source_type: gws
+  connection_id: conn_gws_service_account
+  spreadsheet_id: "1OSRugXZvfjoY9M0yySKT-F3L4LTZXa40J0knU3_4Hik"
+  sheet_range: "Sheet1!A1:Z1000"
+  target:
+    bucket: dlh-bronze
+    format: csv
+
+silver:
+  engine: pyiceberg
+  quality_check:
+    expectation_suite: waste_gws_suite
+  transformation:
+    engine: pyiceberg
+    source_parquet_path: s3://dlh-bronze/waste_reports.parquet
+    target_table: dlh_lakehouse.silver.waste_reports
+    write_mode: overwrite
+    select_models: stg_waste_reports
+
+gold:
+  engine: dbt
+  transformation:
+    engine: dbt
+    select_models: fct_waste_daily_mart
+