merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(governance): dynamic OpenMetadata registration with domains, owners, tiers, and certification (zero hardcoding)

Browse Source
This commit is contained in:
syahril
2026-10-19 02:40:31 +07:00
parent d52768f6bf
commit b125f65351
5 changed files with 200 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dags/helpers/base_lakehouse_operator.py
+17 -5
View File
@@ -93,7 +93,8 @@ class BaseLakehouseOperator(BaseOperator):
logger.debug("OpenMetadata host/token unavailable; skipping lineage emission.")
return
pipeline_fqn = f"dlh_airflow_service.{dag_id}"
pipeline_svc = os.getenv("OPENMETADATA_PIPELINE_SERVICE_NAME", "dlh_airflow_service")
pipeline_fqn = f"{pipeline_svc}.{dag_id}"
pipeline_id_uuid = self._resolve_entity_uuid(active_host, token, "pipeline", pipeline_fqn)
# Emit lineage for each inlet -> outlet pair
@@ -141,13 +142,16 @@ class BaseLakehouseOperator(BaseOperator):
e.g. 's3://dlh-bronze/db_cdc/environmental_permits' -> 'dlh_s3_storage.dlh-bronze.db_cdc.environmental_permits'
e.g. 'silver.environmental_permits_clean' -> 'dlh_postgres_service.dlh_lakehouse.silver.environmental_permits_clean'
"""
storage_svc = os.getenv("OPENMETADATA_STORAGE_SERVICE_NAME", "dlh_minio_s3")
db_svc = os.getenv("OPENMETADATA_DB_SERVICE_NAME", "dlh_postgres_service")
db_name = os.getenv("POSTGRES_DB", "dlh_lakehouse")
clean = dataset_str.strip()
if clean.startswith("s3://"):
parts = clean.replace("s3://", "").strip("/").split("/")
return f"dlh_minio_s3.{'.'.join(parts)}"
return f"{storage_svc}.{'.'.join(parts)}"
elif "." in clean:
return f"dlh_postgres_service.dlh_lakehouse.{clean}"
return f"dlh_postgres_service.dlh_lakehouse.public.{clean}"
return f"{db_svc}.{db_name}.{clean}"
return f"{db_svc}.{db_name}.public.{clean}"
def _get_openmetadata_auth(self) -> tuple:
"""Authenticate and get access token."""
@@ -156,11 +160,19 @@ class BaseLakehouseOperator(BaseOperator):
"http://localhost:8585",
"http://127.0.0.1:8585"
]
om_user = os.getenv("OPENMETADATA_USERNAME", "admin@open-metadata.org")
om_pass = os.getenv("OPENMETADATA_PASSWORD", "admin")
if om_pass == "admin":
b64_pass = "YWRtaW4="
else:
import base64
b64_pass = base64.b64encode(om_pass.encode("utf-8")).decode("utf-8")
for h in hosts:
try:
login_req = urllib.request.Request(
f"{h}/api/v1/users/login",
data=json.dumps({"email": "admin@open-metadata.org", "password": "YWRtaW4="}).encode("utf-8"),
data=json.dumps({"email": om_user, "password": b64_pass}).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST"
)
dags/helpers/ge_lakehouse_operator.py
+11 -2
View File
@@ -352,9 +352,17 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator):
try:
# Login to get token
om_user = os.getenv("OPENMETADATA_USERNAME", "admin@open-metadata.org")
om_pass = os.getenv("OPENMETADATA_PASSWORD", "admin")
if om_pass == "admin":
b64_pass = "YWRtaW4="
else:
import base64
b64_pass = base64.b64encode(om_pass.encode("utf-8")).decode("utf-8")
login_req = urllib.request.Request(
f"{om_host}/api/v1/users/login",
data=json.dumps({"email": "admin@open-metadata.org", "password": "YWRtaW4="}).encode("utf-8"),
data=json.dumps({"email": om_user, "password": b64_pass}).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST"
)
@@ -364,7 +372,8 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator):
if not token:
return
fqn = f"dlh_airflow_service.{self.dag_id or 'lakehouse_pipeline'}.{self.task_id}"
pipeline_svc = os.getenv("OPENMETADATA_PIPELINE_SERVICE_NAME", "dlh_airflow_service")
fqn = f"{pipeline_svc}.{self.dag_id or 'lakehouse_pipeline'}.{self.task_id}"
payload = {
"timestamp": int(time.time() * 1000),
"testCaseStatus": "Success" if summary["overall_success"] else "Failed",
dags/helpers/merica_alert_hook.py
+5 -2
View File
@@ -293,6 +293,9 @@ class MericaAlertHook:
results: Dict[str, bool] = {}
om_public_url = os.getenv("OPENMETADATA_PUBLIC_URL", os.getenv("OPENMETADATA_HOST", "http://localhost:14585"))
pipeline_svc = os.getenv("OPENMETADATA_PIPELINE_SERVICE_NAME", "dlh_airflow_service")
# 1. Telegram Dispatch
tg_bot_token = os.getenv("ALERT_TELEGRAM_BOT_TOKEN")
tg_chat_id = os.getenv("ALERT_TELEGRAM_CHAT_ID")
@@ -313,7 +316,7 @@ class MericaAlertHook:
f"👤 <b>Owner:</b> <code>{ctx_data['owner']}</code> | <b>Tier:</b> <code>{ctx_data['tier']}</code>\n\n"
f"⚠️ <b>Root Cause:</b>\n<code>{ctx_data['error_msg']}</code>\n\n"
f"🔗 <a href='{ctx_data['log_url']}'>Airflow Log</a> | "
f"<a href='http://localhost:8585/pipeline/dlh_airflow_service.{ctx_data['dag_id']}/lineage'>OpenMetadata Lineage</a>"
f"<a href='{om_public_url}/pipeline/{pipeline_svc}.{ctx_data['dag_id']}/lineage'>OpenMetadata Lineage</a>"
)
results["telegram"] = cls.send_telegram_alert(tg_bot_token, tg_chat_id, tg_message)
@@ -354,7 +357,7 @@ class MericaAlertHook:
"type": "actions",
"elements": [
{"type": "button", "text": {"type": "plain_text", "text": "View Airflow Log"}, "url": ctx_data["log_url"]},
{"type": "button", "text": {"type": "plain_text", "text": "OpenMetadata Lineage"}, "url": f"http://localhost:8585/pipeline/dlh_airflow_service.{ctx_data['dag_id']}/lineage"}
{"type": "button", "text": {"type": "plain_text", "text": "OpenMetadata Lineage"}, "url": f"{om_public_url}/pipeline/{pipeline_svc}.{ctx_data['dag_id']}/lineage"}
]
}
]