feat(appliance): initial release of MERICA standalone appliance v1.0.0
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
"""Services package initialization."""
|
||||
from app.services.metadb_service import save_connection_to_metadb
|
||||
from app.services.dbt_service import sync_dbt_models_from_nodes
|
||||
from app.services.pipeline_parser import (
|
||||
parse_yaml_to_pipeline_dict,
|
||||
list_all_pipelines,
|
||||
save_pipeline_draft,
|
||||
deploy_pipeline_dag,
|
||||
delete_pipeline_files,
|
||||
)
|
||||
|
||||
from app.services.openmetadata_service import (
|
||||
sync_pipeline_to_openmetadata,
|
||||
register_openmetadata_lineage,
|
||||
auto_sync_canvas_lineage,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"save_connection_to_metadb",
|
||||
"sync_dbt_models_from_nodes",
|
||||
"parse_yaml_to_pipeline_dict",
|
||||
"list_all_pipelines",
|
||||
"save_pipeline_draft",
|
||||
"deploy_pipeline_dag",
|
||||
"delete_pipeline_files",
|
||||
"sync_pipeline_to_openmetadata",
|
||||
"register_openmetadata_lineage",
|
||||
"auto_sync_canvas_lineage",
|
||||
]
|
||||
@@ -0,0 +1,331 @@
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import yaml
|
||||
from typing import List, Dict, Any, Optional
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def infer_data_type(col_name: str) -> str:
|
||||
"""Infer standard Lakehouse data type for dbt contract enforcement."""
|
||||
name = col_name.lower().strip()
|
||||
if name.endswith("_date") or name == "date":
|
||||
return "date"
|
||||
if name.endswith("_at") or "timestamp" in name or name.endswith("_time"):
|
||||
return "timestamp with time zone"
|
||||
if any(k in name for k in ["score", "rate", "amount", "percent", "price", "val", "total", "count"]):
|
||||
return "numeric"
|
||||
if name.endswith("_id") or name.endswith("_number") or name == "id" or "code" in name or "status" in name or "type" in name:
|
||||
return "character varying(50)"
|
||||
return "character varying(255)"
|
||||
|
||||
|
||||
def generate_model_schema_dict(
|
||||
model_name: str,
|
||||
test_configs: Optional[Dict[str, Any]] = None,
|
||||
sql_query: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate dbt schema dictionary for a model, configuring tests and contract enforcement."""
|
||||
clean_name = model_name.replace(".sql", "")
|
||||
cfg = test_configs or {}
|
||||
|
||||
unique_cols = cfg.get("unique_columns") or cfg.get("unique_cols") or []
|
||||
if isinstance(unique_cols, str):
|
||||
unique_cols = [c.strip() for c in unique_cols.split(",") if c.strip()]
|
||||
|
||||
not_null_cols = cfg.get("not_null_columns") or cfg.get("not_null_cols") or []
|
||||
if isinstance(not_null_cols, str):
|
||||
not_null_cols = [c.strip() for c in not_null_cols.split(",") if c.strip()]
|
||||
|
||||
accepted_values = cfg.get("accepted_values") or {}
|
||||
relationships = cfg.get("relationships") or {}
|
||||
enforce_contract = cfg.get("enforce_contract", False)
|
||||
column_types = cfg.get("column_types") or {}
|
||||
|
||||
# Collect all configured columns
|
||||
all_cols = set(unique_cols) | set(not_null_cols) | set(accepted_values.keys()) | set(relationships.keys()) | set(column_types.keys())
|
||||
|
||||
# If contract is enforced, extract all output columns from SQL query to satisfy dbt contract rules
|
||||
effective_sql = sql_query or cfg.get("sql_query") or cfg.get("query") or ""
|
||||
if enforce_contract and effective_sql:
|
||||
m = re.search(r'SELECT\s+(.*?)\s+FROM', effective_sql, re.IGNORECASE | re.DOTALL)
|
||||
if m:
|
||||
for raw_col in m.group(1).split(","):
|
||||
raw_col = raw_col.strip()
|
||||
if not raw_col:
|
||||
continue
|
||||
if " as " in raw_col.lower():
|
||||
col_id = re.split(r'\s+as\s+', raw_col, flags=re.IGNORECASE)[-1].strip().strip('"').strip("'")
|
||||
else:
|
||||
col_id = raw_col.split(".")[-1].strip().strip('"').strip("'")
|
||||
if col_id and re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', col_id):
|
||||
all_cols.add(col_id)
|
||||
|
||||
columns_list = []
|
||||
for col in sorted(all_cols):
|
||||
col_tests = []
|
||||
if col in unique_cols:
|
||||
col_tests.append("unique")
|
||||
if col in not_null_cols:
|
||||
col_tests.append("not_null")
|
||||
if col in accepted_values and accepted_values[col]:
|
||||
vals = accepted_values[col]
|
||||
if isinstance(vals, str):
|
||||
vals = [v.strip() for v in vals.split(",") if v.strip()]
|
||||
col_tests.append({"accepted_values": {"values": vals}})
|
||||
if col in relationships and relationships[col]:
|
||||
rel_info = relationships[col]
|
||||
if isinstance(rel_info, dict):
|
||||
col_tests.append({"relationships": {
|
||||
"to": rel_info.get("to", "ref('upstream')"),
|
||||
"field": rel_info.get("field", "id")
|
||||
}})
|
||||
|
||||
col_dict: Dict[str, Any] = {"name": col}
|
||||
if col_tests:
|
||||
col_dict["tests"] = col_tests
|
||||
if col in column_types:
|
||||
col_dict["data_type"] = column_types[col]
|
||||
elif enforce_contract:
|
||||
col_dict["data_type"] = infer_data_type(col)
|
||||
columns_list.append(col_dict)
|
||||
|
||||
model_entry: Dict[str, Any] = {
|
||||
"name": clean_name,
|
||||
"description": f"dbt Model {clean_name} auto-registered with Shift-Left quality assertions"
|
||||
}
|
||||
|
||||
if enforce_contract:
|
||||
model_entry["config"] = {"contract": {"enforced": True}}
|
||||
if columns_list:
|
||||
model_entry["columns"] = columns_list
|
||||
|
||||
return model_entry
|
||||
|
||||
|
||||
def compile_dbt_model(
|
||||
model_name: str,
|
||||
layer: str = "silver",
|
||||
sql_query: str = "",
|
||||
test_configs: Optional[Dict[str, Any]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Perform pre-flight compilation check for dbt models:
|
||||
- Verifies Jinja delimiters pairing.
|
||||
- Extracts {{ ref('...') }} and {{ source('...', '...') }}.
|
||||
- Compiles Jinja refs into simulated physical lakehouse identifiers.
|
||||
- Validates SQL structure (SELECT/WITH statements, parentheses matching).
|
||||
- Generates corresponding dbt schema.yml definition with quality tests.
|
||||
"""
|
||||
clean_name = (model_name or "model").replace(".sql", "")
|
||||
errors: List[str] = []
|
||||
warnings: List[str] = []
|
||||
|
||||
if not sql_query or not sql_query.strip():
|
||||
errors.append("SQL query cannot be empty.")
|
||||
|
||||
# 1. Jinja delimiter syntax checking
|
||||
open_brackets = len(re.findall(r'\{\{', sql_query))
|
||||
close_brackets = len(re.findall(r'\}\}', sql_query))
|
||||
if open_brackets != close_brackets:
|
||||
errors.append(f"Mismatched Jinja expression delimiters: {open_brackets} '{{{{' vs {close_brackets} '}}}}'.")
|
||||
|
||||
open_blocks = len(re.findall(r'\{%', sql_query))
|
||||
close_blocks = len(re.findall(r'%\}', sql_query))
|
||||
if open_blocks != close_blocks:
|
||||
errors.append(f"Mismatched Jinja block delimiters: {open_blocks} '{{%' vs {close_blocks} '%}}'.")
|
||||
|
||||
# 2. Extract references & sources
|
||||
extracted_refs = re.findall(r"ref\(\s*['\"]([^'\"]+)['\"]\s*\)", sql_query)
|
||||
extracted_sources = re.findall(r"source\(\s*['\"]([^'\"]+)['\"]\s*,\s*['\"]([^'\"]+)['\"]\s*\)", sql_query)
|
||||
|
||||
# 3. Compile simulated SQL (substituting Jinja with lakehouse catalogue paths)
|
||||
catalog = os.getenv("ICEBERG_CATALOG_NAME", "dlh_lakehouse")
|
||||
compiled_sql = sql_query
|
||||
|
||||
for ref in extracted_refs:
|
||||
escaped_ref = re.escape(ref)
|
||||
replacement = f"{catalog}.{layer}.{ref}" if not ref.startswith("bronze") else f"{catalog}.bronze.{ref}"
|
||||
pattern = re.compile(r"\{\{\s*ref\(\s*['\"]" + escaped_ref + r"['\"]\s*\)\s*\}\}")
|
||||
compiled_sql = pattern.sub(replacement, compiled_sql)
|
||||
|
||||
for src_schema, src_tbl in extracted_sources:
|
||||
escaped_schema = re.escape(src_schema)
|
||||
escaped_tbl = re.escape(src_tbl)
|
||||
replacement = f"{catalog}.bronze.{src_tbl}"
|
||||
pattern = re.compile(r"\{\{\s*source\(\s*['\"]" + escaped_schema + r"['\"]\s*,\s*['\"]" + escaped_tbl + r"['\"]\s*\)\s*\}\}")
|
||||
compiled_sql = pattern.sub(replacement, compiled_sql)
|
||||
|
||||
# Clean up dbt config() macro in compiled preview
|
||||
compiled_sql = re.sub(r"\{\{\s*config\([^)]*\)\s*\}\}", "-- [dbt config applied]", compiled_sql)
|
||||
|
||||
# 4. Basic SQL structural validation
|
||||
clean_sql = re.sub(r'--.*', '', compiled_sql).strip()
|
||||
if clean_sql:
|
||||
if not re.search(r'\b(select|with)\b', clean_sql, re.IGNORECASE):
|
||||
warnings.append("SQL query does not start with SELECT or WITH clause; ensure it produces a valid dataset.")
|
||||
if clean_sql.count('(') != clean_sql.count(')'):
|
||||
errors.append("Unbalanced parentheses '(' and ')' detected in SQL query.")
|
||||
|
||||
# 5. Generate Schema YAML testing representation
|
||||
schema_dict = generate_model_schema_dict(clean_name, test_configs, sql_query)
|
||||
wrapper = {"version": 2, "models": [schema_dict]}
|
||||
schema_yml_str = yaml.dump(wrapper, sort_keys=False, default_flow_style=False)
|
||||
|
||||
is_valid = len(errors) == 0
|
||||
return {
|
||||
"status": "success" if is_valid else "error",
|
||||
"valid": is_valid,
|
||||
"model_name": clean_name,
|
||||
"layer": layer,
|
||||
"compiled_sql": compiled_sql.strip(),
|
||||
"schema_yml": schema_yml_str,
|
||||
"extracted_refs": extracted_refs,
|
||||
"extracted_sources": [{"schema": s[0], "table": s[1]} for s in extracted_sources],
|
||||
"warnings": warnings,
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
|
||||
def sync_dbt_models_from_nodes(nodes: List[Dict[str, Any]]) -> List[str]:
|
||||
"""Sync dbt SQL model files, schema.yml (with tests), and sources.yml physically in both dbt target directories."""
|
||||
synced_files = []
|
||||
|
||||
target_dbt_dirs = [settings.DBT_DIR]
|
||||
if hasattr(settings, "DEFAULT_DBT_DIR") and settings.DEFAULT_DBT_DIR and settings.DEFAULT_DBT_DIR != settings.DBT_DIR:
|
||||
target_dbt_dirs.append(settings.DEFAULT_DBT_DIR)
|
||||
|
||||
for base_dbt_dir in target_dbt_dirs:
|
||||
try:
|
||||
models_dir = os.path.join(base_dbt_dir, "models")
|
||||
os.makedirs(models_dir, exist_ok=True)
|
||||
|
||||
models_by_layer: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
# 1. Sync dbt SQL models
|
||||
for node in nodes:
|
||||
node_type = node.get("data", {}).get("type", "")
|
||||
cfg = node.get("data", {}).get("config", {})
|
||||
if node_type == "dbt_transform":
|
||||
layer = (cfg.get("dbt_layer") or cfg.get("sel") or "silver").lower()
|
||||
model_name = cfg.get("dbt_model") or cfg.get("sel") or node.get("data", {}).get("label", "").lower().replace(" ", "_")
|
||||
if not model_name:
|
||||
continue
|
||||
if not model_name.endswith(".sql"):
|
||||
model_name += ".sql"
|
||||
|
||||
target_dir = os.path.join(models_dir, layer)
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
sql_content = cfg.get("query") or cfg.get("sql_query_preview") or f"-- dbt Model: {model_name}\nSELECT * FROM {{{{ ref('bronze_raw') }}}}"
|
||||
model_path = os.path.join(target_dir, model_name)
|
||||
with open(model_path, "w", encoding="utf-8") as f:
|
||||
f.write(sql_content)
|
||||
logger.info("Auto-synced dbt model file to %s", model_path)
|
||||
synced_files.append(model_path)
|
||||
|
||||
# Collect for schema.yml tests
|
||||
test_cfgs = cfg.get("test_configs") or cfg.get("quality_tests")
|
||||
if not test_cfgs and any(k in cfg for k in ["unique_columns", "not_null_columns", "accepted_values", "enforce_contract"]):
|
||||
test_cfgs = {
|
||||
"unique_columns": cfg.get("unique_columns") or [],
|
||||
"not_null_columns": cfg.get("not_null_columns") or [],
|
||||
"accepted_values": cfg.get("accepted_values") or {},
|
||||
"enforce_contract": cfg.get("enforce_contract") or False,
|
||||
}
|
||||
if test_cfgs:
|
||||
model_entry = generate_model_schema_dict(model_name, test_cfgs, sql_content)
|
||||
models_by_layer.setdefault(layer, []).append(model_entry)
|
||||
|
||||
# 2. Auto-sync schema.yml per layer if test_configs are present
|
||||
for layer, m_list in models_by_layer.items():
|
||||
if not m_list:
|
||||
continue
|
||||
schema_file = os.path.join(models_dir, layer, "schema.yml")
|
||||
existing_schema: Dict[str, Any] = {"version": 2, "models": []}
|
||||
if os.path.exists(schema_file):
|
||||
try:
|
||||
with open(schema_file, "r", encoding="utf-8") as sf:
|
||||
existing_schema = yaml.safe_load(sf) or {"version": 2, "models": []}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
existing_models = {m.get("name"): m for m in existing_schema.get("models", [])}
|
||||
for new_m in m_list:
|
||||
existing_models[new_m["name"]] = new_m
|
||||
existing_schema["models"] = list(existing_models.values())
|
||||
|
||||
with open(schema_file, "w", encoding="utf-8") as sf:
|
||||
yaml.safe_dump(existing_schema, sf, sort_keys=False, default_flow_style=False)
|
||||
logger.info("Auto-synced dbt schema tests to %s", schema_file)
|
||||
synced_files.append(schema_file)
|
||||
|
||||
# 3. Auto-sync sources.yml from source_db nodes
|
||||
sources_file = os.path.join(models_dir, "sources.yml")
|
||||
source_tables_by_schema = {}
|
||||
for node in nodes:
|
||||
node_type = node.get("data", {}).get("type", "")
|
||||
cfg = node.get("data", {}).get("config", {})
|
||||
if node_type in ["source_db", "database"]:
|
||||
tbl = cfg.get("table_name", "")
|
||||
if tbl:
|
||||
parts = tbl.split(".")
|
||||
s_name = parts[0] if len(parts) > 1 else "public"
|
||||
t_name = parts[1] if len(parts) > 1 else parts[0]
|
||||
source_tables_by_schema.setdefault(s_name, set()).add(t_name)
|
||||
|
||||
if source_tables_by_schema and os.path.exists(sources_file):
|
||||
try:
|
||||
with open(sources_file, "r", encoding="utf-8") as sf:
|
||||
src_data = yaml.safe_load(sf) or {}
|
||||
|
||||
sources_list = src_data.get("sources", [])
|
||||
modified = False
|
||||
for schema_name, tbl_set in source_tables_by_schema.items():
|
||||
schema_entry = next((s for s in sources_list if s.get("schema") == schema_name), None)
|
||||
if not schema_entry:
|
||||
schema_entry = {"name": schema_name, "schema": schema_name, "tables": []}
|
||||
sources_list.append(schema_entry)
|
||||
modified = True
|
||||
|
||||
existing_tables = {t.get("name") for t in schema_entry.get("tables", [])}
|
||||
for t in tbl_set:
|
||||
if t not in existing_tables:
|
||||
schema_entry.setdefault("tables", []).append({
|
||||
"name": t,
|
||||
"description": f"Auto-registered source table '{t}' from canvas pipeline"
|
||||
})
|
||||
modified = True
|
||||
|
||||
if modified:
|
||||
src_data["sources"] = sources_list
|
||||
with open(sources_file, "w", encoding="utf-8") as sf:
|
||||
yaml.safe_dump(src_data, sf, sort_keys=False, default_flow_style=False)
|
||||
logger.info("Auto-synced sources.yml at %s", sources_file)
|
||||
except Exception as se:
|
||||
logger.warning("Auto-sync sources.yml warning: %s", se)
|
||||
|
||||
# 4. Auto-sync bronze view models from source_db tables so downstream models can ref() them
|
||||
bronze_dir = os.path.join(models_dir, "bronze")
|
||||
os.makedirs(bronze_dir, exist_ok=True)
|
||||
for schema_name, tbl_set in source_tables_by_schema.items():
|
||||
for t_name in tbl_set:
|
||||
m_name = f"bronze_{t_name}.sql" if not t_name.startswith("bronze_") else f"{t_name}.sql"
|
||||
m_path = os.path.join(bronze_dir, m_name)
|
||||
if not os.path.exists(m_path):
|
||||
b_content = (
|
||||
f"-- dbt Bronze View Model: {t_name}\n"
|
||||
f"{{{{ config(materialized='view') }}}}\n\n"
|
||||
f"SELECT * FROM {{{{ source('{schema_name}', '{t_name}') }}}}\n"
|
||||
)
|
||||
with open(m_path, "w", encoding="utf-8") as bf:
|
||||
bf.write(b_content)
|
||||
logger.info("Auto-synced bronze dbt model to %s", m_path)
|
||||
synced_files.append(m_path)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning("dbt Model Sync Note: %s", e)
|
||||
|
||||
return synced_files
|
||||
@@ -0,0 +1,43 @@
|
||||
import logging
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def save_connection_to_metadb(
|
||||
conn_id: str,
|
||||
conn_type: str,
|
||||
host: str,
|
||||
port: int,
|
||||
schema: str,
|
||||
login: str,
|
||||
password: str = ""
|
||||
) -> bool:
|
||||
"""Persist custom connection to Airflow metadata database if Airflow & SQLAlchemy are available."""
|
||||
try:
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
engine = create_engine(settings.AIRFLOW_DATABASE_URL)
|
||||
Session = sessionmaker(bind=engine)
|
||||
session = Session()
|
||||
|
||||
from airflow.models import Connection
|
||||
existing = session.query(Connection).filter(Connection.conn_id == conn_id).first()
|
||||
if not existing:
|
||||
c = Connection(
|
||||
conn_id=conn_id,
|
||||
conn_type=conn_type,
|
||||
host=host,
|
||||
port=port,
|
||||
schema=schema,
|
||||
login=login,
|
||||
password=password
|
||||
)
|
||||
session.add(c)
|
||||
session.commit()
|
||||
logger.info("Connection '%s' saved to Airflow MetaDB", conn_id)
|
||||
session.close()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning("MetaDB Connection Save Note: %s", e)
|
||||
return False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,773 @@
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import yaml
|
||||
import glob
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from app.core.config import settings
|
||||
from app.services.metadb_service import save_connection_to_metadb
|
||||
from app.services.dbt_service import sync_dbt_models_from_nodes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_yaml_to_pipeline_dict(cfg: dict, filename: str) -> dict:
|
||||
"""Parse an Airflow YAML config file into UI Canvas compatible node/edge structure."""
|
||||
pipeline_id = cfg.get("pipeline_id") or filename.replace(".yaml", "").replace(".yml", "")
|
||||
description = cfg.get("description", "Airflow Configured Pipeline")
|
||||
schedule = cfg.get("schedule", "0 */1 * * *")
|
||||
mode = cfg.get("mode", "medallion")
|
||||
|
||||
nodes = []
|
||||
edges = []
|
||||
|
||||
if mode == "flexible" or "tasks" in cfg:
|
||||
tasks = cfg.get("tasks", [])
|
||||
for idx, t in enumerate(tasks):
|
||||
node_id = str(t.get("task_id", f"node_{idx}"))
|
||||
n_type = t.get("type") or t.get("operator_type") or "source_db"
|
||||
label = t.get("label") or node_id
|
||||
nodes.append({
|
||||
"id": node_id,
|
||||
"type": "customNode",
|
||||
"position": {"x": 80 + idx * 260, "y": 160},
|
||||
"data": {
|
||||
"label": label,
|
||||
"type": n_type,
|
||||
"configured": True,
|
||||
"config": t.get("config", {})
|
||||
}
|
||||
})
|
||||
for dep_id in t.get("dependencies", []):
|
||||
edges.append({
|
||||
"id": f"e_{dep_id}_{node_id}",
|
||||
"source": str(dep_id),
|
||||
"target": str(node_id)
|
||||
})
|
||||
else:
|
||||
# Medallion mode parsing
|
||||
bronze = cfg.get("bronze", {})
|
||||
silver = cfg.get("silver", {})
|
||||
gold = cfg.get("gold", {})
|
||||
|
||||
src_type = bronze.get("source_type", "database")
|
||||
node_src_type = "source_db"
|
||||
if src_type == "gws":
|
||||
node_src_type = "source_gws"
|
||||
elif src_type == "api":
|
||||
node_src_type = "source_api"
|
||||
|
||||
nodes.append({
|
||||
"id": "node-bronze-src",
|
||||
"type": "customNode",
|
||||
"position": {"x": 80, "y": 160},
|
||||
"data": {
|
||||
"label": f"Bronze {src_type.upper()} Source",
|
||||
"type": node_src_type,
|
||||
"configured": True,
|
||||
"config": bronze
|
||||
}
|
||||
})
|
||||
|
||||
nodes.append({
|
||||
"id": "node-bronze-storage",
|
||||
"type": "customNode",
|
||||
"position": {"x": 340, "y": 160},
|
||||
"data": {
|
||||
"label": "MinIO Bronze Storage",
|
||||
"type": "minio_storage",
|
||||
"configured": True,
|
||||
"config": bronze.get("target", {})
|
||||
}
|
||||
})
|
||||
edges.append({"id": "e1-2", "source": "node-bronze-src", "target": "node-bronze-storage"})
|
||||
|
||||
sil_trans = silver.get("transformation", {})
|
||||
if sil_trans:
|
||||
nodes.append({
|
||||
"id": "node-silver-dbt",
|
||||
"type": "customNode",
|
||||
"position": {"x": 600, "y": 160},
|
||||
"data": {
|
||||
"label": f"dbt Silver Model ({sil_trans.get('select_models', 'silver')})",
|
||||
"type": "dbt_transform",
|
||||
"configured": True,
|
||||
"config": {"dbt_layer": "silver", "dbt_model": sil_trans.get("select_models", "")}
|
||||
}
|
||||
})
|
||||
edges.append({"id": "e2-3", "source": "node-bronze-storage", "target": "node-silver-dbt"})
|
||||
|
||||
sil_ge = silver.get("quality_check", {})
|
||||
if sil_ge:
|
||||
prev_id = "node-silver-dbt" if sil_trans else "node-bronze-storage"
|
||||
nodes.append({
|
||||
"id": "node-silver-ge",
|
||||
"type": "customNode",
|
||||
"position": {"x": 860, "y": 160},
|
||||
"data": {
|
||||
"label": f"GE Quality ({sil_ge.get('expectation_suite', 'default')})",
|
||||
"type": "ge_quality",
|
||||
"configured": True,
|
||||
"config": sil_ge
|
||||
}
|
||||
})
|
||||
edges.append({"id": "e3-4", "source": prev_id, "target": "node-silver-ge"})
|
||||
|
||||
gold_trans = gold.get("transformation", {})
|
||||
if gold_trans:
|
||||
prev_id = "node-silver-ge" if sil_ge else ("node-silver-dbt" if sil_trans else "node-bronze-storage")
|
||||
nodes.append({
|
||||
"id": "node-gold-dbt",
|
||||
"type": "customNode",
|
||||
"position": {"x": 1120, "y": 160},
|
||||
"data": {
|
||||
"label": f"dbt Gold Mart ({gold_trans.get('select_models', 'gold')})",
|
||||
"type": "dbt_transform",
|
||||
"configured": True,
|
||||
"config": {"dbt_layer": "gold", "dbt_model": gold_trans.get("select_models", "")}
|
||||
}
|
||||
})
|
||||
edges.append({"id": "e4-5", "source": prev_id, "target": "node-gold-dbt"})
|
||||
|
||||
return {
|
||||
"id": pipeline_id,
|
||||
"description": description,
|
||||
"schedule": schedule,
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"is_draft": False,
|
||||
"status": "deployed"
|
||||
}
|
||||
|
||||
|
||||
def get_metadata_path(filename: str) -> str:
|
||||
"""Resolve existing metadata file path, checking METADATA_DIR then CONFIGS_DIR."""
|
||||
meta_dir = getattr(settings, "METADATA_DIR", settings.CONFIGS_DIR)
|
||||
p = os.path.join(meta_dir, filename)
|
||||
if os.path.exists(p):
|
||||
return p
|
||||
cfg_p = os.path.join(settings.CONFIGS_DIR, filename)
|
||||
if os.path.exists(cfg_p):
|
||||
return cfg_p
|
||||
return p
|
||||
|
||||
|
||||
def get_target_metadata_path(filename: str) -> str:
|
||||
"""Resolve target path for writing metadata file."""
|
||||
meta_dir = getattr(settings, "METADATA_DIR", settings.CONFIGS_DIR)
|
||||
return os.path.join(meta_dir, filename)
|
||||
|
||||
|
||||
def get_pipeline_by_id(pipeline_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Retrieve single pipeline dictionary by ID, searching JSON drafts, deployed JSON, or YAML."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
draft_path = get_metadata_path(f"{norm_id}.draft.json")
|
||||
if os.path.exists(draft_path):
|
||||
try:
|
||||
with open(draft_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["id"] = norm_id
|
||||
data["status"] = "draft"
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning("Error reading draft JSON %s: %s", draft_path, e)
|
||||
|
||||
json_path = get_metadata_path(f"{norm_id}.json")
|
||||
if os.path.exists(json_path):
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["id"] = norm_id
|
||||
data["status"] = data.get("status", "deployed")
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.warning("Error reading metadata JSON %s: %s", json_path, e)
|
||||
|
||||
yaml_path = os.path.join(settings.CONFIGS_DIR, f"{norm_id}.yaml")
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
with open(yaml_path, "r", encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
if isinstance(cfg, dict):
|
||||
return parse_yaml_to_pipeline_dict(cfg, f"{norm_id}.yaml")
|
||||
except Exception as e:
|
||||
logger.warning("Error reading YAML %s: %s", yaml_path, e)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def list_all_pipelines() -> List[Dict[str, Any]]:
|
||||
"""Retrieve all pipeline configs (JSON drafts/metadata and deployed YAMLs)."""
|
||||
os.makedirs(settings.CONFIGS_DIR, exist_ok=True)
|
||||
if hasattr(settings, "METADATA_DIR"):
|
||||
os.makedirs(settings.METADATA_DIR, exist_ok=True)
|
||||
|
||||
pipelines_map: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
# 1. Load JSON files (saved drafts and UI canvas snapshots)
|
||||
scan_dirs = []
|
||||
if hasattr(settings, "METADATA_DIR") and os.path.exists(settings.METADATA_DIR):
|
||||
scan_dirs.append(settings.METADATA_DIR)
|
||||
container_meta = "/app/merica-pipelines/metadata"
|
||||
if container_meta not in scan_dirs and os.path.exists(container_meta):
|
||||
scan_dirs.append(container_meta)
|
||||
if settings.CONFIGS_DIR not in scan_dirs and os.path.exists(settings.CONFIGS_DIR):
|
||||
scan_dirs.append(settings.CONFIGS_DIR)
|
||||
|
||||
for s_dir in scan_dirs:
|
||||
for file_name in sorted(os.listdir(s_dir)):
|
||||
file_path = os.path.join(s_dir, file_name)
|
||||
if file_name.endswith(".json") and not ".version." in file_name:
|
||||
if file_name in ["rbac_mappings.json", "package.json", "tsconfig.json"]:
|
||||
continue
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
if "nodes" not in data and not file_name.endswith(".draft.json") and "tasks" not in data and "pipeline_id" not in data:
|
||||
continue
|
||||
p_id = data.get("id") or data.get("pipeline_id") or file_name.replace(".draft.json", "").replace(".json", "")
|
||||
data["id"] = p_id
|
||||
if file_name.endswith(".draft.json") or data.get("is_draft"):
|
||||
data["status"] = "draft"
|
||||
else:
|
||||
data["status"] = "deployed"
|
||||
pipelines_map[p_id] = data
|
||||
except Exception as e:
|
||||
logger.warning("Error reading JSON file %s: %s", file_name, e)
|
||||
|
||||
# 2. Load YAML files (real Airflow DAG configs)
|
||||
yaml_scan_dirs = []
|
||||
if os.path.exists(settings.CONFIGS_DIR):
|
||||
yaml_scan_dirs.append(settings.CONFIGS_DIR)
|
||||
container_cfg = "/app/merica-pipelines/configs"
|
||||
if container_cfg not in yaml_scan_dirs and os.path.exists(container_cfg):
|
||||
yaml_scan_dirs.append(container_cfg)
|
||||
for b_dir in sorted(glob.glob("/opt/airflow/dag_bundles/*/tracking_repo/configs")):
|
||||
if b_dir not in yaml_scan_dirs and os.path.exists(b_dir):
|
||||
yaml_scan_dirs.append(b_dir)
|
||||
|
||||
for y_dir in yaml_scan_dirs:
|
||||
for file_name in sorted(os.listdir(y_dir)):
|
||||
file_path = os.path.join(y_dir, file_name)
|
||||
if file_name.endswith(".yaml") or file_name.endswith(".yml"):
|
||||
p_id = file_name.replace(".yaml", "").replace(".yml", "")
|
||||
existing = pipelines_map.get(p_id)
|
||||
should_load_yaml = False
|
||||
if existing is None:
|
||||
should_load_yaml = True
|
||||
elif existing.get("status") == "deployed":
|
||||
json_path = get_metadata_path(f"{p_id}.json")
|
||||
if os.path.exists(json_path) and os.path.getmtime(file_path) > os.path.getmtime(json_path):
|
||||
should_load_yaml = True
|
||||
|
||||
if should_load_yaml:
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
if isinstance(cfg, dict):
|
||||
parsed_data = parse_yaml_to_pipeline_dict(cfg, file_name)
|
||||
pipelines_map[p_id] = parsed_data
|
||||
except Exception as e:
|
||||
logger.warning("Error parsing YAML config %s: %s", file_name, e)
|
||||
|
||||
return list(pipelines_map.values())
|
||||
|
||||
|
||||
def save_pipeline_draft(
|
||||
pipeline_id: str,
|
||||
description: Optional[str],
|
||||
schedule: Optional[str],
|
||||
nodes: List[Dict[str, Any]],
|
||||
edges: List[Dict[str, Any]],
|
||||
notes: Optional[List[Dict[str, Any]]] = None,
|
||||
owner_id: Optional[str] = None,
|
||||
owner_name: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Save an in-progress pipeline design as a .draft.json file."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
target_meta_dir = getattr(settings, "METADATA_DIR", settings.CONFIGS_DIR)
|
||||
os.makedirs(target_meta_dir, exist_ok=True)
|
||||
|
||||
json_path = os.path.join(target_meta_dir, f"{norm_id}.draft.json")
|
||||
existing = get_pipeline_by_id(norm_id) if os.path.exists(json_path) else None
|
||||
resolved_owner_id = owner_id or (existing.get("owner_id") if existing else None)
|
||||
resolved_owner_name = owner_name or (existing.get("owner_name") if existing else None)
|
||||
resolved_notes = notes if notes is not None else (existing.get("notes", []) if existing else [])
|
||||
|
||||
draft_data = {
|
||||
"id": norm_id,
|
||||
"description": description or "Dynamic Lakehouse Pipeline Draft",
|
||||
"schedule": schedule or "0 */1 * * *",
|
||||
"owner_id": resolved_owner_id,
|
||||
"owner_name": resolved_owner_name,
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"notes": resolved_notes,
|
||||
"is_draft": True,
|
||||
"status": "draft",
|
||||
"updated_at": datetime.utcnow().isoformat() + "Z"
|
||||
}
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(draft_data, f, indent=2)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"pipeline_id": norm_id,
|
||||
"filename": f"{norm_id}.draft.json",
|
||||
"message": f"Draft pipeline '{norm_id}' saved successfully."
|
||||
}
|
||||
|
||||
|
||||
def duplicate_pipeline(
|
||||
pipeline_id: str,
|
||||
new_pipeline_id: str,
|
||||
owner_id: Optional[str] = None,
|
||||
owner_name: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Duplicate a pipeline with a new ID.
|
||||
Per Decision #4: The cloner becomes the owner, and the status is set to draft.
|
||||
"""
|
||||
source = get_pipeline_by_id(pipeline_id)
|
||||
if not source:
|
||||
raise ValueError(f"Source pipeline '{pipeline_id}' not found.")
|
||||
|
||||
norm_new_id = new_pipeline_id.lower().replace(" ", "_")
|
||||
return save_pipeline_draft(
|
||||
pipeline_id=norm_new_id,
|
||||
description=f"Copy of {source.get('description', '')}".strip(),
|
||||
schedule=source.get("schedule", "0 */1 * * *"),
|
||||
nodes=source.get("nodes", []),
|
||||
edges=source.get("edges", []),
|
||||
notes=source.get("notes", []),
|
||||
owner_id=owner_id,
|
||||
owner_name=owner_name
|
||||
)
|
||||
|
||||
|
||||
def build_semantic_task_id(node: Dict[str, Any], seen_ids: Optional[set] = None) -> str:
|
||||
"""Transform random ReactFlow IDs (node_179...) into semantic Airflow task IDs."""
|
||||
data = node.get("data", {})
|
||||
cfg = data.get("config", {})
|
||||
raw_id = str(node.get("id", "task"))
|
||||
|
||||
if not raw_id.startswith("node_"):
|
||||
candidate = re.sub(r'[^a-zA-Z0-9_.-]', '_', raw_id).lower()
|
||||
if seen_ids is not None:
|
||||
seen_ids.add(candidate)
|
||||
return candidate
|
||||
|
||||
op_type = (data.get("type") or "task").lower()
|
||||
label = (data.get("label") or "").lower()
|
||||
|
||||
layer = "bronze" if any(k in op_type for k in ["source", "bronze", "sling", "cdc", "gws", "api"]) else (
|
||||
"silver" if any(k in op_type for k in ["silver", "iceberg", "duckdb", "pyiceberg"]) else (
|
||||
"gold" if any(k in op_type for k in ["gold", "mart"]) else (
|
||||
"quality" if any(k in op_type for k in ["ge", "quality"]) else "task"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
entity = ""
|
||||
for f in ["table_name", "target_table", "dbt_model", "dataset_name"]:
|
||||
if cfg.get(f):
|
||||
entity = re.sub(r'[^a-zA-Z0-9]', '', str(cfg[f]).split(".")[-1].replace("_clean", "").replace("_raw", "")).lower()
|
||||
break
|
||||
if not entity and label:
|
||||
entity = re.sub(r'[^a-zA-Z0-9]', '_', label).strip('_').lower().replace("source_", "").replace("dbt_", "").replace("_task", "")
|
||||
if not entity:
|
||||
entity = op_type.replace("source_", "").replace("_transform", "")
|
||||
|
||||
engine = "cdc" if "cdc" in op_type or "sling" in op_type or "db" in op_type else (
|
||||
"gws" if "gws" in op_type or "sheet" in op_type else (
|
||||
"api" if "api" in op_type else (
|
||||
"duckdb" if "duckdb" in op_type else (
|
||||
"pyiceberg" if "pyiceberg" in op_type else (
|
||||
"spark" if "spark" in op_type else (
|
||||
"dbt" if "dbt" in op_type else "op"
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
s_id = f"{layer}_{entity}_{engine}_task"
|
||||
s_id = re.sub(r'_+', '_', s_id).strip('_')
|
||||
|
||||
if seen_ids is not None:
|
||||
base = s_id
|
||||
seq = 2
|
||||
while s_id in seen_ids:
|
||||
s_id = f"{base}_{seq}"
|
||||
seq += 1
|
||||
seen_ids.add(s_id)
|
||||
|
||||
return s_id
|
||||
|
||||
|
||||
def deploy_pipeline_dag(
|
||||
pipeline_id: str,
|
||||
description: Optional[str],
|
||||
schedule: Optional[str],
|
||||
nodes: List[Dict[str, Any]],
|
||||
edges: List[Dict[str, Any]],
|
||||
spec_yaml: Optional[str] = "",
|
||||
notes: Optional[List[Dict[str, Any]]] = None,
|
||||
owner_id: Optional[str] = None,
|
||||
owner_name: Optional[str] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Compile and deploy pipeline: sync dbt models, persist custom connections, write JSON and YAML."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
os.makedirs(settings.CONFIGS_DIR, exist_ok=True)
|
||||
|
||||
# Sync dbt SQL model files physically if dbt nodes exist
|
||||
sync_dbt_models_from_nodes(nodes)
|
||||
|
||||
# Automatically persist custom database connections if present in nodes
|
||||
for node in nodes:
|
||||
cfg = node.get("data", {}).get("config", {})
|
||||
if cfg.get("conn_id") == "custom_db" or cfg.get("custom_host"):
|
||||
conn_id = cfg.get("custom_conn_id") or f"conn_custom_{norm_id}"
|
||||
save_connection_to_metadb(
|
||||
conn_id=conn_id,
|
||||
conn_type=cfg.get("custom_db_type", "postgres"),
|
||||
host=cfg.get("custom_host", "localhost"),
|
||||
port=int(cfg.get("custom_port", 5432)),
|
||||
schema=cfg.get("custom_database", "public"),
|
||||
login=cfg.get("custom_user", "postgres"),
|
||||
password=cfg.get("custom_password", "")
|
||||
)
|
||||
|
||||
# Build semantic node ID mapping
|
||||
seen_ids = set()
|
||||
node_id_map = {}
|
||||
for node in nodes:
|
||||
raw_id = str(node.get("id"))
|
||||
node_id_map[raw_id] = build_semantic_task_id(node, seen_ids)
|
||||
|
||||
# 1. Save JSON metadata for UI load/edit (marking as deployed)
|
||||
json_path = get_target_metadata_path(f"{norm_id}.json")
|
||||
existing = get_pipeline_by_id(norm_id) if os.path.exists(json_path) else None
|
||||
resolved_owner_id = owner_id or (existing.get("owner_id") if existing else None)
|
||||
resolved_owner_name = owner_name or (existing.get("owner_name") if existing else None)
|
||||
resolved_notes = notes if notes is not None else (existing.get("notes", []) if existing else [])
|
||||
|
||||
pipeline_data = {
|
||||
"id": norm_id,
|
||||
"description": description or "Dynamic Lakehouse Pipeline",
|
||||
"schedule": schedule or "0 */1 * * *",
|
||||
"owner_id": resolved_owner_id,
|
||||
"owner_name": resolved_owner_name,
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
"notes": resolved_notes,
|
||||
"is_draft": False,
|
||||
"status": "deployed",
|
||||
"updated_at": datetime.utcnow().isoformat() + "Z"
|
||||
}
|
||||
with open(json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(pipeline_data, f, indent=2)
|
||||
|
||||
# Remove draft file if it existed
|
||||
for draft_d in [getattr(settings, "METADATA_DIR", None), settings.CONFIGS_DIR]:
|
||||
if draft_d:
|
||||
d_p = os.path.join(draft_d, f"{norm_id}.draft.json")
|
||||
if os.path.exists(d_p):
|
||||
os.remove(d_p)
|
||||
|
||||
# 2. Sanitize and Save YAML specification for Airflow DAG Processor
|
||||
yaml_path = os.path.join(settings.CONFIGS_DIR, f"{norm_id}.yaml")
|
||||
if spec_yaml:
|
||||
try:
|
||||
parsed_spec = yaml.safe_load(spec_yaml)
|
||||
if isinstance(parsed_spec, dict) and "tasks" in parsed_spec:
|
||||
for t in parsed_spec["tasks"]:
|
||||
raw_tid = str(t.get("task_id"))
|
||||
if raw_tid in node_id_map:
|
||||
t["task_id"] = node_id_map[raw_tid]
|
||||
# Remap dependencies to semantic task IDs
|
||||
t["dependencies"] = [node_id_map.get(str(d), str(d)) for d in t.get("dependencies", [])]
|
||||
yaml_content = yaml.dump(parsed_spec, sort_keys=False)
|
||||
else:
|
||||
yaml_content = spec_yaml
|
||||
except Exception:
|
||||
yaml_content = spec_yaml
|
||||
else:
|
||||
yaml_content = yaml.dump(pipeline_data, sort_keys=False)
|
||||
|
||||
with open(yaml_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_content)
|
||||
|
||||
# 3. Auto-Register pipeline & lineage to OpenMetadata 2.0 with Dynamic Governance
|
||||
try:
|
||||
from app.services.openmetadata_service import sync_pipeline_to_openmetadata, auto_sync_canvas_lineage
|
||||
gov_dict = {}
|
||||
if "parsed_spec" in locals() and isinstance(parsed_spec, dict):
|
||||
gov_dict = parsed_spec.get("governance", {})
|
||||
elif isinstance(pipeline_data, dict):
|
||||
gov_dict = pipeline_data.get("governance", {})
|
||||
if not gov_dict.get("owner") and resolved_owner_id:
|
||||
gov_dict["owner"] = resolved_owner_id
|
||||
if not gov_dict.get("owner_name") and resolved_owner_name:
|
||||
gov_dict["owner_name"] = resolved_owner_name
|
||||
|
||||
sync_pipeline_to_openmetadata(norm_id, description or "", nodes=nodes, governance=gov_dict)
|
||||
auto_sync_canvas_lineage(norm_id, nodes, edges)
|
||||
except Exception as e:
|
||||
logger.warning("OpenMetadata Auto-Registration Note: %s", e)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"pipeline_id": norm_id,
|
||||
"filename": f"{norm_id}.yaml",
|
||||
"message": f"Pipeline '{norm_id}' successfully generated & saved to {yaml_path}"
|
||||
}
|
||||
|
||||
|
||||
def delete_pipeline_files(pipeline_id: str) -> bool:
|
||||
"""Delete all artifacts (json, draft.json, yaml, version snapshots) associated with a pipeline."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
target_dirs = []
|
||||
if hasattr(settings, "METADATA_DIR") and settings.METADATA_DIR:
|
||||
target_dirs.append(settings.METADATA_DIR)
|
||||
if settings.CONFIGS_DIR not in target_dirs:
|
||||
target_dirs.append(settings.CONFIGS_DIR)
|
||||
|
||||
deleted = False
|
||||
for d in target_dirs:
|
||||
if not os.path.exists(d):
|
||||
continue
|
||||
json_path = os.path.join(d, f"{norm_id}.json")
|
||||
draft_path = os.path.join(d, f"{norm_id}.draft.json")
|
||||
yaml_path = os.path.join(d, f"{norm_id}.yaml")
|
||||
|
||||
for p in (json_path, draft_path, yaml_path):
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
deleted = True
|
||||
|
||||
# Delete version snapshots
|
||||
for file_name in os.listdir(d):
|
||||
if file_name.startswith(f"{norm_id}.version.") and file_name.endswith(".json"):
|
||||
v_path = os.path.join(d, file_name)
|
||||
os.remove(v_path)
|
||||
deleted = True
|
||||
|
||||
return deleted
|
||||
|
||||
|
||||
def publish_pipeline_version(
|
||||
pipeline_id: str,
|
||||
version_tag: str,
|
||||
changelog: Optional[str] = "",
|
||||
bundle_name: Optional[str] = "dlh-core-ingestion",
|
||||
nodes: Optional[List[Dict[str, Any]]] = None,
|
||||
edges: Optional[List[Dict[str, Any]]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Publish a permanent semantic version snapshot for Airflow 3 DAG Bundle & UI Time Machine."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
os.makedirs(settings.CONFIGS_DIR, exist_ok=True)
|
||||
if hasattr(settings, "METADATA_DIR"):
|
||||
os.makedirs(settings.METADATA_DIR, exist_ok=True)
|
||||
|
||||
clean_tag = version_tag.strip().lower()
|
||||
if not clean_tag.startswith("v"):
|
||||
clean_tag = f"v{clean_tag}"
|
||||
|
||||
json_path = get_metadata_path(f"{norm_id}.json")
|
||||
draft_path = get_metadata_path(f"{norm_id}.draft.json")
|
||||
|
||||
# 1. Resolve pipeline topology
|
||||
pipeline_data = {}
|
||||
if nodes is not None and edges is not None:
|
||||
pipeline_data = {
|
||||
"id": norm_id,
|
||||
"description": f"Published version {clean_tag}",
|
||||
"schedule": "0 */1 * * *",
|
||||
"nodes": nodes,
|
||||
"edges": edges,
|
||||
}
|
||||
elif os.path.exists(json_path):
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
pipeline_data = json.load(f)
|
||||
elif os.path.exists(draft_path):
|
||||
with open(draft_path, "r", encoding="utf-8") as f:
|
||||
pipeline_data = json.load(f)
|
||||
else:
|
||||
pipeline_data = {
|
||||
"id": norm_id,
|
||||
"description": f"Published version {clean_tag}",
|
||||
"schedule": "0 */1 * * *",
|
||||
"nodes": [],
|
||||
"edges": [],
|
||||
}
|
||||
|
||||
# Update metadata
|
||||
pipeline_data["version"] = clean_tag
|
||||
pipeline_data["bundle_name"] = bundle_name or "dlh-core-ingestion"
|
||||
pipeline_data["changelog"] = changelog or "Released new version via MERICA Studio"
|
||||
pipeline_data["published_at"] = datetime.utcnow().isoformat() + "Z"
|
||||
pipeline_data["is_draft"] = False
|
||||
pipeline_data["status"] = "deployed"
|
||||
|
||||
# Sync dbt models
|
||||
if "nodes" in pipeline_data and pipeline_data["nodes"]:
|
||||
sync_dbt_models_from_nodes(pipeline_data["nodes"])
|
||||
|
||||
# 2. Save version snapshot file in METADATA_DIR
|
||||
snapshot_filename = f"{norm_id}.version.{clean_tag}.json"
|
||||
snapshot_path = get_target_metadata_path(snapshot_filename)
|
||||
with open(snapshot_path, "w", encoding="utf-8") as f:
|
||||
json.dump(pipeline_data, f, indent=2)
|
||||
|
||||
# 3. Update active JSON metadata in METADATA_DIR
|
||||
target_json_path = get_target_metadata_path(f"{norm_id}.json")
|
||||
with open(target_json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(pipeline_data, f, indent=2)
|
||||
|
||||
# Remove draft if exists in either dir
|
||||
for draft_d in [getattr(settings, "METADATA_DIR", None), settings.CONFIGS_DIR]:
|
||||
if draft_d:
|
||||
d_p = os.path.join(draft_d, f"{norm_id}.draft.json")
|
||||
if os.path.exists(d_p):
|
||||
os.remove(d_p)
|
||||
|
||||
# 4. Generate/Update YAML DAG Specification for Airflow 3 in CONFIGS_DIR
|
||||
yaml_path = os.path.join(settings.CONFIGS_DIR, f"{norm_id}.yaml")
|
||||
yaml_header = (
|
||||
f"# ============================================================\n"
|
||||
f"# MERICA Studio Data Lakehouse Pipeline\n"
|
||||
f"# Airflow 3 DAG Bundle: {pipeline_data['bundle_name']}\n"
|
||||
f"# Pinned Version: {clean_tag} | Published: {pipeline_data['published_at']}\n"
|
||||
f"# Changelog: {pipeline_data['changelog']}\n"
|
||||
f"# ============================================================\n"
|
||||
)
|
||||
yaml_body = yaml.dump(pipeline_data, sort_keys=False)
|
||||
with open(yaml_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_header + yaml_body)
|
||||
|
||||
return {
|
||||
"status": "published",
|
||||
"pipeline_id": norm_id,
|
||||
"version": clean_tag,
|
||||
"bundle_name": pipeline_data["bundle_name"],
|
||||
"changelog": pipeline_data["changelog"],
|
||||
"published_at": pipeline_data["published_at"],
|
||||
"snapshot_filename": snapshot_filename,
|
||||
"nodes_count": len(pipeline_data.get("nodes", [])),
|
||||
"edges_count": len(pipeline_data.get("edges", [])),
|
||||
"message": f"Pipeline '{norm_id}' version {clean_tag} published and pinned to bundle '{pipeline_data['bundle_name']}'."
|
||||
}
|
||||
|
||||
|
||||
def list_pipeline_versions(pipeline_id: str) -> Dict[str, Any]:
|
||||
"""Retrieve all published historical version snapshots for a pipeline."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
os.makedirs(settings.CONFIGS_DIR, exist_ok=True)
|
||||
if hasattr(settings, "METADATA_DIR"):
|
||||
os.makedirs(settings.METADATA_DIR, exist_ok=True)
|
||||
|
||||
current_version = None
|
||||
current_bundle = "dlh-core-ingestion"
|
||||
json_path = get_metadata_path(f"{norm_id}.json")
|
||||
if os.path.exists(json_path):
|
||||
try:
|
||||
with open(json_path, "r", encoding="utf-8") as f:
|
||||
cur_data = json.load(f)
|
||||
current_version = cur_data.get("version")
|
||||
current_bundle = cur_data.get("bundle_name", "dlh-core-ingestion")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
versions = []
|
||||
prefix = f"{norm_id}.version."
|
||||
seen_snapshots = set()
|
||||
scan_dirs = []
|
||||
if hasattr(settings, "METADATA_DIR") and settings.METADATA_DIR:
|
||||
scan_dirs.append(settings.METADATA_DIR)
|
||||
if settings.CONFIGS_DIR not in scan_dirs:
|
||||
scan_dirs.append(settings.CONFIGS_DIR)
|
||||
|
||||
for s_dir in scan_dirs:
|
||||
if not os.path.exists(s_dir):
|
||||
continue
|
||||
for file_name in sorted(os.listdir(s_dir)):
|
||||
if file_name.startswith(prefix) and file_name.endswith(".json") and file_name not in seen_snapshots:
|
||||
seen_snapshots.add(file_name)
|
||||
file_path = os.path.join(s_dir, file_name)
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
snap = json.load(f)
|
||||
versions.append({
|
||||
"version": snap.get("version", "v1.0.0"),
|
||||
"pipeline_id": norm_id,
|
||||
"bundle_name": snap.get("bundle_name", current_bundle),
|
||||
"changelog": snap.get("changelog", ""),
|
||||
"published_at": snap.get("published_at", ""),
|
||||
"nodes_count": len(snap.get("nodes", [])),
|
||||
"edges_count": len(snap.get("edges", [])),
|
||||
"snapshot_filename": file_name
|
||||
})
|
||||
except Exception as e:
|
||||
logger.warning("Error reading version snapshot %s: %s", file_name, e)
|
||||
|
||||
# Sort newest published first
|
||||
versions.sort(key=lambda x: x.get("published_at", ""), reverse=True)
|
||||
|
||||
return {
|
||||
"pipeline_id": norm_id,
|
||||
"current_version": current_version,
|
||||
"bundle_name": current_bundle,
|
||||
"versions": versions
|
||||
}
|
||||
|
||||
|
||||
def revert_pipeline_version(pipeline_id: str, version_tag: str) -> Dict[str, Any]:
|
||||
"""Revert the active canvas pipeline to a specific past historical version snapshot."""
|
||||
norm_id = pipeline_id.lower().replace(" ", "_")
|
||||
clean_tag = version_tag.strip().lower()
|
||||
if not clean_tag.startswith("v"):
|
||||
clean_tag = f"v{clean_tag}"
|
||||
|
||||
snapshot_filename = f"{norm_id}.version.{clean_tag}.json"
|
||||
snapshot_path = get_metadata_path(snapshot_filename)
|
||||
if not os.path.exists(snapshot_path):
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Version snapshot '{snapshot_filename}' not found for pipeline '{norm_id}'."
|
||||
}
|
||||
|
||||
with open(snapshot_path, "r", encoding="utf-8") as f:
|
||||
reverted_data = json.load(f)
|
||||
|
||||
# Update active JSON file in METADATA_DIR
|
||||
target_json_path = get_target_metadata_path(f"{norm_id}.json")
|
||||
with open(target_json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(reverted_data, f, indent=2)
|
||||
|
||||
# Recompile active YAML DAG file in CONFIGS_DIR
|
||||
yaml_path = os.path.join(settings.CONFIGS_DIR, f"{norm_id}.yaml")
|
||||
yaml_header = (
|
||||
f"# ============================================================\n"
|
||||
f"# MERICA Studio Data Lakehouse Pipeline (Reverted)\n"
|
||||
f"# Airflow 3 DAG Bundle: {reverted_data.get('bundle_name', 'dlh-core-ingestion')}\n"
|
||||
f"# Pinned Version: {clean_tag} | Reverted at: {datetime.utcnow().isoformat()}Z\n"
|
||||
f"# ============================================================\n"
|
||||
)
|
||||
yaml_body = yaml.dump(reverted_data, sort_keys=False)
|
||||
with open(yaml_path, "w", encoding="utf-8") as f:
|
||||
f.write(yaml_header + yaml_body)
|
||||
|
||||
return {
|
||||
"status": "reverted",
|
||||
"pipeline_id": norm_id,
|
||||
"active_version": clean_tag,
|
||||
"pipeline_data": reverted_data,
|
||||
"message": f"Successfully reverted pipeline '{norm_id}' to version {clean_tag}."
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user