fix(dag): add sys.path relative import and correct duckdb partition column
This commit is contained in:
@@ -71,15 +71,15 @@ tasks:
|
|||||||
target_table: bronze_environmental_permits
|
target_table: bronze_environmental_permits
|
||||||
table_name: bronze_environmental_permits
|
table_name: bronze_environmental_permits
|
||||||
primary_keys: permit_id
|
primary_keys: permit_id
|
||||||
query: SELECT id AS permit_id, permit_number, company_name, permit_type, status,
|
query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
|
||||||
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
|
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
|
||||||
sql_query: SELECT id AS permit_id, permit_number, company_name, permit_type, status,
|
sql_query: SELECT permit_id, permit_number, company_name, activity_type AS permit_type, status,
|
||||||
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
|
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
|
||||||
transformation_sql: SELECT id AS permit_id, permit_number, company_name, permit_type,
|
transformation_sql: SELECT permit_id, permit_number, company_name, activity_type AS permit_type,
|
||||||
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
status, issued_date, updated_at FROM read_parquet('s3://dlh-bronze/staging/environmental_permits/*.parquet')
|
||||||
QUALIFY ROW_NUMBER() OVER (PARTITION BY id ORDER BY updated_at DESC) = 1
|
QUALIFY ROW_NUMBER() OVER (PARTITION BY permit_id ORDER BY updated_at DESC) = 1
|
||||||
- task_id: node-6733
|
- task_id: node-6733
|
||||||
type: dbt_transform
|
type: dbt_transform
|
||||||
label: dbt Silver Cleansed Permits Enriched
|
label: dbt Silver Cleansed Permits Enriched
|
||||||
|
|||||||
@@ -1,9 +1,15 @@
|
|||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
import glob
|
import glob
|
||||||
import yaml
|
import yaml
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from airflow import DAG
|
from airflow import DAG
|
||||||
|
|
||||||
|
# Ensure bundle/dags directory is on sys.path for relative helper imports
|
||||||
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
if current_dir not in sys.path:
|
||||||
|
sys.path.insert(0, current_dir)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from airflow.providers.standard.operators.bash import BashOperator
|
from airflow.providers.standard.operators.bash import BashOperator
|
||||||
from airflow.providers.standard.operators.python import PythonOperator
|
from airflow.providers.standard.operators.python import PythonOperator
|
||||||
|
|||||||
Reference in New Issue
Block a user