feat(governance): dynamic OpenMetadata registration with domains, owners, tiers, and certification (zero hardcoding)
This commit is contained in:
+161
-15
@@ -42,11 +42,19 @@ def notify_openmetadata_status(dag_id: str, execution_status: str, task_names: l
|
||||
token = None
|
||||
om_hosts = [os.getenv("OPENMETADATA_HOST", "http://openmetadata:8585"), "http://localhost:8585", "http://127.0.0.1:8585"]
|
||||
active_host = None
|
||||
om_user = os.getenv("OPENMETADATA_USERNAME", "admin@open-metadata.org")
|
||||
om_pass = os.getenv("OPENMETADATA_PASSWORD", "admin")
|
||||
if om_pass == "admin":
|
||||
b64_pass = "YWRtaW4="
|
||||
else:
|
||||
import base64
|
||||
b64_pass = base64.b64encode(om_pass.encode("utf-8")).decode("utf-8")
|
||||
|
||||
for h in om_hosts:
|
||||
try:
|
||||
login_req = urllib.request.Request(
|
||||
f"{h}/api/v1/users/login",
|
||||
data=json.dumps({"email": "admin@open-metadata.org", "password": "YWRtaW4="}).encode("utf-8"),
|
||||
data=json.dumps({"email": om_user, "password": b64_pass}).encode("utf-8"),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
@@ -60,11 +68,15 @@ def notify_openmetadata_status(dag_id: str, execution_status: str, task_names: l
|
||||
if not token or not active_host:
|
||||
return
|
||||
|
||||
fqn = f"dlh_airflow_service.{dag_id}"
|
||||
pipeline_svc = os.getenv("OPENMETADATA_PIPELINE_SERVICE_NAME", "dlh_airflow_service")
|
||||
airflow_host = os.getenv("OPENMETADATA_AIRFLOW_HOST", "http://merica-airflow-webserver:8080")
|
||||
fqn = f"{pipeline_svc}.{dag_id}"
|
||||
if not task_names:
|
||||
task_names = ["bronze_ingestion_task", "silver_transformation_task", "gold_data_mart_task"]
|
||||
|
||||
now_ms = int(time.time() * 1000)
|
||||
dag = context.get('dag') if context else None
|
||||
dag_params = getattr(dag, 'params', {}) if dag else {}
|
||||
dag_run = context.get('dag_run') if context else None
|
||||
dag_start = int(dag_run.start_date.timestamp() * 1000) if (dag_run and getattr(dag_run, 'start_date', None)) else (now_ms - 60000)
|
||||
dag_end = int(dag_run.end_date.timestamp() * 1000) if (dag_run and getattr(dag_run, 'end_date', None)) else now_ms
|
||||
@@ -111,16 +123,23 @@ def notify_openmetadata_status(dag_id: str, execution_status: str, task_names: l
|
||||
"taskStatus": task_statuses
|
||||
}
|
||||
|
||||
# Auto-provision pipelineService if not already created
|
||||
# Dynamic Governance Resolution (Zero Hardcoding: Context > Env > Defaults)
|
||||
gov_domain = (dag_params.get('domain') if isinstance(dag_params, dict) else None) or os.getenv("GOVERNANCE_DOMAIN", "Lingkungan Hidup & Kehutanan")
|
||||
gov_owner_team = (dag_params.get('owner') if isinstance(dag_params, dict) else None) or os.getenv("GOVERNANCE_OWNER_TEAM", "de_team_dlh")
|
||||
gov_owner_name = os.getenv("GOVERNANCE_OWNER_NAME", "Data Engineering DLH")
|
||||
gov_tier = (dag_params.get('tier') if isinstance(dag_params, dict) else None) or os.getenv("GOVERNANCE_TIER", "Tier.Tier1")
|
||||
gov_cert = (dag_params.get('certification') if isinstance(dag_params, dict) else None) or os.getenv("GOVERNANCE_CERTIFICATION", "Certification.Gold")
|
||||
|
||||
# 1. Auto-provision pipelineService if not already created
|
||||
try:
|
||||
svc_payload = {
|
||||
"name": "dlh_airflow_service",
|
||||
"displayName": "DLH Airflow Service",
|
||||
"name": pipeline_svc,
|
||||
"displayName": pipeline_svc.replace("_", " ").title(),
|
||||
"serviceType": "Airflow",
|
||||
"connection": {
|
||||
"config": {
|
||||
"type": "Airflow",
|
||||
"hostPort": "http://merica-airflow-webserver:8080"
|
||||
"hostPort": airflow_host
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -135,25 +154,146 @@ def notify_openmetadata_status(dag_id: str, execution_status: str, task_names: l
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Auto-provision pipeline entity in OpenMetadata
|
||||
# 2. Auto-provision Domain in OpenMetadata if missing
|
||||
domain_id = None
|
||||
domain_name = None
|
||||
if gov_domain:
|
||||
try:
|
||||
dom_slug = gov_domain.replace(" ", "_").replace("&", "").replace("__", "_").strip("_")
|
||||
try:
|
||||
d_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/domains/name/{dom_slug}",
|
||||
headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
with urllib.request.urlopen(d_req, timeout=3) as d_res:
|
||||
d_data = json.loads(d_res.read().decode("utf-8"))
|
||||
domain_id = d_data.get("id")
|
||||
domain_name = d_data.get("name")
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
dom_payload = {
|
||||
"name": dom_slug,
|
||||
"displayName": gov_domain,
|
||||
"domainType": "Aggregate",
|
||||
"description": f"Domain {gov_domain} managed by MERICA"
|
||||
}
|
||||
c_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/domains",
|
||||
data=json.dumps(dom_payload).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
with urllib.request.urlopen(c_req, timeout=3) as c_res:
|
||||
c_data = json.loads(c_res.read().decode("utf-8"))
|
||||
domain_id = c_data.get("id")
|
||||
domain_name = c_data.get("name")
|
||||
except Exception as e:
|
||||
print(f"Domain provisioning note: {e}")
|
||||
|
||||
# 3. Auto-provision Team in OpenMetadata if missing
|
||||
owner_id = None
|
||||
if gov_owner_team:
|
||||
try:
|
||||
team_slug = gov_owner_team.replace(" ", "_").lower()
|
||||
try:
|
||||
t_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/teams/name/{team_slug}",
|
||||
headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
with urllib.request.urlopen(t_req, timeout=3) as t_res:
|
||||
t_data = json.loads(t_res.read().decode("utf-8"))
|
||||
owner_id = t_data.get("id")
|
||||
except urllib.error.HTTPError as he:
|
||||
if he.code == 404:
|
||||
team_payload = {
|
||||
"name": team_slug,
|
||||
"displayName": gov_owner_name or gov_owner_team,
|
||||
"teamType": "Group",
|
||||
"description": f"Team {gov_owner_name or gov_owner_team} managed by MERICA"
|
||||
}
|
||||
c_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/teams",
|
||||
data=json.dumps(team_payload).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
method="POST"
|
||||
)
|
||||
with urllib.request.urlopen(c_req, timeout=3) as c_res:
|
||||
c_data = json.loads(c_res.read().decode("utf-8"))
|
||||
owner_id = c_data.get("id")
|
||||
except Exception as e:
|
||||
print(f"Team provisioning note: {e}")
|
||||
|
||||
# 4. Auto-provision pipeline entity in OpenMetadata with Governance attributes
|
||||
pipe_id = None
|
||||
try:
|
||||
pipe_payload = {
|
||||
"name": dag_id,
|
||||
"displayName": dag_id.replace("_", " ").title(),
|
||||
"service": "dlh_airflow_service",
|
||||
"service": pipeline_svc,
|
||||
"description": f"Managed Airflow Pipeline {dag_id}",
|
||||
"tasks": [{"name": str(t), "displayName": str(t), "taskType": "custom_task"} for t in (task_names or [])]
|
||||
}
|
||||
if domain_name:
|
||||
pipe_payload["domains"] = [domain_name]
|
||||
if owner_id:
|
||||
pipe_payload["owners"] = [{"id": owner_id, "type": "team"}]
|
||||
if gov_tier:
|
||||
pipe_payload["tags"] = [{"tagFQN": gov_tier, "source": "Classification", "labelType": "Manual", "state": "Confirmed"}]
|
||||
|
||||
pipe_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/pipelines",
|
||||
data=json.dumps(pipe_payload).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
|
||||
method="PUT"
|
||||
)
|
||||
with urllib.request.urlopen(pipe_req, timeout=4) as _:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
with urllib.request.urlopen(pipe_req, timeout=4) as p_resp:
|
||||
p_data = json.loads(p_resp.read().decode("utf-8"))
|
||||
pipe_id = p_data.get("id")
|
||||
except Exception as e:
|
||||
print(f"Pipeline PUT note: {e}")
|
||||
|
||||
# 5. Patch Governance attributes (Domains, Owners, Tier, Certification)
|
||||
try:
|
||||
if not pipe_id:
|
||||
fetch_r = urllib.request.Request(
|
||||
f"{active_host}/api/v1/pipelines/name/{fqn}",
|
||||
headers={"Authorization": f"Bearer {token}"}
|
||||
)
|
||||
with urllib.request.urlopen(fetch_r, timeout=3) as f_res:
|
||||
pipe_id = json.loads(f_res.read().decode("utf-8")).get("id")
|
||||
|
||||
if pipe_id:
|
||||
patch_ops = []
|
||||
if domain_id:
|
||||
patch_ops.append({"op": "add", "path": "/domains", "value": [{"id": domain_id, "type": "domain"}]})
|
||||
if owner_id:
|
||||
patch_ops.append({"op": "add", "path": "/owners", "value": [{"id": owner_id, "type": "team"}]})
|
||||
if gov_tier:
|
||||
patch_ops.append({"op": "add", "path": "/tags", "value": [{"tagFQN": gov_tier, "source": "Classification", "labelType": "Manual", "state": "Confirmed"}]})
|
||||
if gov_cert:
|
||||
patch_ops.append({
|
||||
"op": "add",
|
||||
"path": "/certification",
|
||||
"value": {
|
||||
"tagLabel": {
|
||||
"tagFQN": gov_cert,
|
||||
"source": "Classification",
|
||||
"labelType": "Manual",
|
||||
"state": "Confirmed"
|
||||
}
|
||||
}
|
||||
})
|
||||
if patch_ops:
|
||||
patch_req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/pipelines/{pipe_id}",
|
||||
data=json.dumps(patch_ops).encode("utf-8"),
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json-patch+json"},
|
||||
method="PATCH"
|
||||
)
|
||||
with urllib.request.urlopen(patch_req, timeout=4) as _:
|
||||
pass
|
||||
except Exception as e:
|
||||
print(f"Pipeline Governance PATCH note: {e}")
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{active_host}/api/v1/pipelines/{fqn}/status",
|
||||
@@ -203,17 +343,23 @@ def build_dag_from_yaml(config_file: str) -> DAG:
|
||||
dag_id = re.sub(r'[^a-zA-Z0-9_.-]', '_', str(raw_id).strip()).lower()
|
||||
dag_id = re.sub(r'_+', '_', dag_id).strip('_')
|
||||
|
||||
default_domain = os.getenv('GOVERNANCE_DOMAIN', 'Lingkungan Hidup & Kehutanan')
|
||||
default_owner = os.getenv('GOVERNANCE_OWNER_TEAM', 'de_team_dlh')
|
||||
default_tier = os.getenv('GOVERNANCE_TIER', 'Tier.Tier1')
|
||||
default_certification = os.getenv('GOVERNANCE_CERTIFICATION', 'Certification.Gold')
|
||||
|
||||
schedule = cfg.get('schedule', '@daily')
|
||||
owner = cfg.get('owner', 'de_team_dlh')
|
||||
owner = cfg.get('owner') or default_owner
|
||||
tags = cfg.get('tags', ['dlh', 'medallion'])
|
||||
mode = cfg.get('mode', 'medallion')
|
||||
|
||||
governance = cfg.get('governance', {})
|
||||
alert_cfg = cfg.get('alerting', {})
|
||||
dag_params = {
|
||||
'domain': governance.get('domain', cfg.get('domain', 'Lingkungan Hidup & Kehutanan')),
|
||||
'domain': governance.get('domain', cfg.get('domain', default_domain)),
|
||||
'owner': governance.get('owner', owner),
|
||||
'tier': governance.get('tier', cfg.get('tier', 'Tier.Tier2')),
|
||||
'tier': governance.get('tier', cfg.get('tier', default_tier)),
|
||||
'certification': governance.get('certification', cfg.get('certification', default_certification)),
|
||||
'alerting': alert_cfg
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user