merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pipeline): enable dynamic table extraction, dbt bronze models, and valid quality gate assertions

Browse Source
This commit is contained in:
syahril
2026-10-19 02:07:39 +07:00
parent a5d766489a
commit fa1065c670
7 changed files with 84 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dags/helpers/dbt_trino_operator.py
+3 -1
View File
@@ -73,10 +73,12 @@ class DbtTrinoIcebergOperator(BaseLakehouseOperator):
return preferred
candidates = [
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "dbt")),
"/opt/airflow/dag_bundles/merica-pipeline/tracking_repo/dbt",
"/opt/airflow/dag_bundles/merica-pipeline/dbt",
"/opt/airflow/pipeline-bundles/dbt",
"/opt/airflow/dag_bundles/dlh-pipeline-bundle/dbt",
"/opt/airflow/dag_bundles/merica-pipeline-bundles/dbt",
"/opt/airflow/dag_bundles/merica-pipeline/dbt",
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "merica-pipeline", "dbt")),
os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "merica-pipeline-bundles", "dbt")),
"/opt/airflow/dbt/dlh_transformation",
dags/helpers/sling_helper.py
+31 -8
View File
@@ -132,21 +132,30 @@ class SlingDBIngestionOperator(BaseOperator):
def execute(self, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
cfg = self.bronze_config
source_cfg = cfg.get("source_config", {}) if "source_config" in cfg else cfg
target_cfg = cfg.get("target", {})
bucket = target_cfg.get("bucket", "dlh-bronze")
target_format = target_cfg.get("format", "parquet")
prefix = target_cfg.get("prefix", "db/permits")
table_name = cfg.get("table_name", "public.environmental_permits")
conn_id = cfg.get("connection_id", "conn_postgres_source_db")
table_name = source_cfg.get("table_name") or source_cfg.get("table") or cfg.get("table_name") or "public.environmental_permits"
conn_id = source_cfg.get("conn_id") or source_cfg.get("connection_id") or cfg.get("connection_id", "conn_postgres_source_db")
clean_table = table_name.split(".")[-1]
# Dynamic prefix resolution matching downstream DuckDB / Iceberg paths
if "ref_waste_facilities" in table_name or "ref_facilities" in table_name:
prefix = target_cfg.get("prefix") or "staging/ref_facilities"
elif "raw_waste_orders" in table_name or "waste_orders" in table_name:
prefix = target_cfg.get("prefix") or "staging/waste_orders"
else:
prefix = target_cfg.get("prefix") or f"staging/{clean_table}"
exec_dt = datetime.utcnow()
clean_table = table_name.replace(".", "_")
checkpoint_key = f"merica_checkpoint_{clean_table}"
last_checkpoint = get_checkpoint_value(checkpoint_key, default="1970-01-01 00:00:00")
self.log.info(
"Executing DB Ingestion on '%s' (mode: %s, last checkpoint: %s)",
table_name, self.ingestion_mode, last_checkpoint
"Executing DB Ingestion on '%s' (mode: %s, last checkpoint: %s, prefix: %s)",
table_name, self.ingestion_mode, last_checkpoint, prefix
)
df = None
@@ -178,10 +187,16 @@ class SlingDBIngestionOperator(BaseOperator):
self.log.info("Running Native Database Extraction Engine.")
db_url = resolve_db_connection_url(conn_id)
try:
from sqlalchemy import create_engine
from sqlalchemy import create_engine, inspect
engine = create_engine(db_url, connect_args={"connect_timeout": 5})
if self.ingestion_mode == "incremental_watermark":
# Check whether watermark column exists in the table
insp = inspect(engine)
schema_name = table_name.split(".")[0] if "." in table_name else "public"
pure_tbl = table_name.split(".")[-1]
cols = [c["name"] for c in insp.get_columns(pure_tbl, schema=schema_name)]
if self.ingestion_mode == "incremental_watermark" and self.watermark_column in cols:
query = f"SELECT * FROM {table_name} WHERE {self.watermark_column} > '{last_checkpoint}' ORDER BY {self.watermark_column} ASC LIMIT {self.batch_size}"
else:
query = f"SELECT * FROM {table_name} LIMIT {self.batch_size}"
@@ -217,6 +232,14 @@ class SlingDBIngestionOperator(BaseOperator):
key=storage_key,
target_format=target_format
)
# Also save flat key in prefix directory so glob patterns like s3://dlh-bronze/staging/ref_facilities/*.parquet match directly
flat_key = f"{prefix.strip('/')}/{filename}"
storage_manager.save_dataframe(
df=df,
bucket=bucket,
key=flat_key,
target_format=target_format
)
# 5. Sync to PostgreSQL Bronze CDC staging table for dbt-postgres medallion pipelines
try: