37 lines
880 B
YAML
37 lines
880 B
YAML
|
pipeline_id: ingestion_gws_waste_reports
|
||
|
|
description: "Google Workspace Spreadsheet Extractor for Waste Reports -> Pandas Cleansing -> Trino Iceberg Target -> dbt Gold Mart"
|
||
|
|
schedule: "0 4 * * *"
|
||
|
|
owner: de_team_dlh
|
||
|
|
tags:
|
||
|
|
- dlh
|
||
|
|
- gws
|
||
|
|
- waste
|
||
|
|
mode: medallion
|
||
|
|
|
||
|
|
bronze:
|
||
|
|
source_type: gws
|
||
|
|
connection_id: conn_gws_service_account
|
||
|
|
spreadsheet_id: "1OSRugXZvfjoY9M0yySKT-F3L4LTZXa40J0knU3_4Hik"
|
||
|
|
sheet_range: "Sheet1!A1:Z1000"
|
||
|
|
target:
|
||
|
|
bucket: dlh-bronze
|
||
|
|
format: csv
|
||
|
|
|
||
|
|
silver:
|
||
|
|
engine: pyiceberg
|
||
|
|
quality_check:
|
||
|
|
expectation_suite: waste_gws_suite
|
||
|
|
transformation:
|
||
|
|
engine: pyiceberg
|
||
|
|
source_parquet_path: s3://dlh-bronze/waste_reports.parquet
|
||
|
|
target_table: dlh_lakehouse.silver.waste_reports
|
||
|
|
write_mode: overwrite
|
||
|
|
select_models: stg_waste_reports
|
||
|
|
|
||
|
|
gold:
|
||
|
|
engine: dbt
|
||
|
|
transformation:
|
||
|
|
engine: dbt
|
||
|
|
select_models: fct_waste_daily_mart
|
||
|
|
|