fix(dag): add sys.path relative import and correct duckdb partition column

This commit is contained in:
2026-10-21 14:55:57 +07:00
parent 6a34dabf3e
commit 8deb174221
2 changed files with 12 additions and 6 deletions
+6 -6
View File
@@ -71,15 +71,15 @@ tasks:
target_table: bronze_environmental_permits target_table: bronze_environmental_permits
table_name: bronze_environmental_permits table_name: bronze_environmental_permits
primary_keys: permit_id primary_keys: permit_id
query: SELECT id AS permit_id, permit_number, company_name, permit_type, status, query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1 QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
sql_query: SELECT id AS permit_id, permit_number, company_name, permit_type, status, sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1 QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type, transformation_sql: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet') status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1 QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
- task_id: node-6733 - task_id: node-6733
type: dbt_transform type: dbt_transform
label: dbt Silver Cleansed Permits Enriched label: dbt Silver Cleansed Permits Enriched
+6
View File
@@ -1,9 +1,15 @@
import os import os
import sys
import glob import glob
import yaml import yaml
from datetime import datetime, timedelta from datetime import datetime, timedelta
from airflow import DAG from airflow import DAG
# Ensure bundle/dags directory is on sys.path for relative helper imports
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir not in sys.path:
sys.path.insert(0, current_dir)
try: try:
from airflow.providers.standard.operators.bash import BashOperator from airflow.providers.standard.operators.bash import BashOperator
from airflow.providers.standard.operators.python import PythonOperator from airflow.providers.standard.operators.python import PythonOperator