merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(dag): add sys.path relative import and correct duckdb partition column

Browse Source
This commit is contained in:
syahril
2026-10-21 14:55:57 +07:00
parent 6a34dabf3e
commit 8deb174221
2 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/e2e_modernized_ingestion_flow.yaml
+6 -6
View File
@@ -71,15 +71,15 @@ tasks:
target_table: bronze_environmental_permits
table_name: bronze_environmental_permits
primary_keys: permit_id
query: SELECT id AS permit_id, permit_number, company_name, permit_type, status,
query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
sql_query: SELECT id AS permit_id, permit_number, company_name, permit_type, status,
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type,
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
transformation_sql: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
- task_id: node-6733
type: dbt_transform
label: dbt Silver Cleansed Permits Enriched