feat(omd): auto-provision pipelineService, pipeline, and missing table entities during execution

2026-10-19 02:19:13 +07:00
parent 6de3c213af
commit d52768f6bf
2 changed files with 77 additions and 0 deletions
@@ -111,6 +111,50 @@ def notify_openmetadata_status(dag_id: str, execution_status: str, task_names: l
            "taskStatus": task_statuses
        }

+        # Auto-provision pipelineService if not already created
+        try:
+            svc_payload = {
+                "name": "dlh_airflow_service",
+                "displayName": "DLH Airflow Service",
+                "serviceType": "Airflow",
+                "connection": {
+                    "config": {
+                        "type": "Airflow",
+                        "hostPort": "http://merica-airflow-webserver:8080"
+                    }
+                }
+            }
+            svc_req = urllib.request.Request(
+                f"{active_host}/api/v1/services/pipelineServices",
+                data=json.dumps(svc_payload).encode("utf-8"),
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                method="POST"
+            )
+            with urllib.request.urlopen(svc_req, timeout=3) as _:
+                pass
+        except Exception:
+            pass
+
+        # Auto-provision pipeline entity in OpenMetadata
+        try:
+            pipe_payload = {
+                "name": dag_id,
+                "displayName": dag_id.replace("_", " ").title(),
+                "service": "dlh_airflow_service",
+                "description": f"Managed Airflow Pipeline {dag_id}",
+                "tasks": [{"name": str(t), "displayName": str(t), "taskType": "custom_task"} for t in (task_names or [])]
+            }
+            pipe_req = urllib.request.Request(
+                f"{active_host}/api/v1/pipelines",
+                data=json.dumps(pipe_payload).encode("utf-8"),
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                method="PUT"
+            )
+            with urllib.request.urlopen(pipe_req, timeout=4) as _:
+                pass
+        except Exception:
+            pass
+
        req = urllib.request.Request(
            f"{active_host}/api/v1/pipelines/{fqn}/status",
            data=json.dumps(status_payload).encode("utf-8"),