pipeline_id: ingestion_gws_waste_reports
description: "Google Workspace Spreadsheet Extractor for Waste Reports -> Pandas Cleansing -> Trino Iceberg Target -> dbt Gold Mart"
schedule: "0 4 * * *"
owner: de_team_dlh
tags:
  - dlh
  - gws
  - waste
mode: medallion

bronze:
  source_type: gws
  connection_id: conn_gws_service_account
  spreadsheet_id: "1OSRugXZvfjoY9M0yySKT-F3L4LTZXa40J0knU3_4Hik"
  sheet_range: "Sheet1!A1:Z1000"
  target:
    bucket: dlh-bronze
    format: csv

silver:
  engine: pyiceberg
  quality_check:
    expectation_suite: waste_gws_suite
  transformation:
    engine: pyiceberg
    source_parquet_path: s3://dlh-bronze/waste_reports.parquet
    target_table: dlh_lakehouse.silver.waste_reports
    write_mode: overwrite
    select_models: stg_waste_reports

gold:
  engine: dbt
  transformation:
    engine: dbt
    select_models: fct_waste_daily_mart