feat(quality): emit Great Expectations test case results directly to OpenMetadata table test suites

dags/helpers/ge_lakehouse_operator.py

@@ -372,12 +372,78 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator):
             if not token:
                 return
 
-            pipeline_svc = os.getenv("OPENMETADATA_PIPELINE_SERVICE_NAME", "dlh_airflow_service")
-            fqn = f"{pipeline_svc}.{self.dag_id or 'lakehouse_pipeline'}.{self.task_id}"
+            db_svc = os.getenv("OPENMETADATA_DB_SERVICE_NAME", "dlh_postgres_service")
+            db_name = os.getenv("POSTGRES_DB", "dlh_lakehouse")
+            raw_tbl = str(self.table_name or getattr(self, "ge_table_name", None) or "table")
+            parts = raw_tbl.split(".")
+            if len(parts) == 2:
+                schema = parts[0]
+                tbl_name = parts[1]
+            else:
+                tbl_name = parts[-1]
+                if tbl_name.startswith("gold_"):
+                    schema = "gold"
+                elif tbl_name.startswith("silver_"):
+                    schema = "silver"
+                elif tbl_name.startswith("bronze_"):
+                    schema = "bronze"
+                elif "permit" in tbl_name.lower():
+                    schema = "bronze_lakehouse"
+                else:
+                    schema = "bronze"
+            table_fqn = f"{db_svc}.{db_name}.{schema}.{tbl_name}"
+
+            # 1. Ensure Table Test Suite
+            suite_payload = {
+                "name": f"{table_fqn}.testSuite",
+                "displayName": f"Quality Suite for {tbl_name}",
+                "basicEntityReference": table_fqn
+            }
+            suite_req = urllib.request.Request(
+                f"{om_host}/api/v1/dataQuality/testSuites/basic",
+                data=json.dumps(suite_payload).encode("utf-8"),
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                method="POST"
+            )
+            try:
+                with urllib.request.urlopen(suite_req, timeout=3) as _:
+                    pass
+            except Exception:
+                pass
+
+            # 2. Ensure Test Case on Table EntityLink
+            tc_name = re.sub(r'[^a-zA-Z0-9_.-]', '_', f"{self.task_id}_quality").lower()
+            case_payload = {
+                "name": tc_name,
+                "displayName": f"GE Gate {self.task_id}",
+                "description": f"Great Expectations suite '{self.expectation_suite}' on {table_fqn}",
+                "entityLink": f"<#E::table::{table_fqn}>",
+                "testDefinition": "tableCustomSQLQuery",
+                "parameterValues": [
+                    {"name": "sqlExpression", "value": str(self.sql_assertion or getattr(self, "sql", "SELECT 1"))}
+                ]
+            }
+            tc_req = urllib.request.Request(
+                f"{om_host}/api/v1/dataQuality/testCases",
+                data=json.dumps(case_payload).encode("utf-8"),
+                headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
+                method="POST"
+            )
+            try:
+                with urllib.request.urlopen(tc_req, timeout=3) as _:
+                    pass
+            except Exception:
+                pass
+
+            # 3. Post Test Case Result to OpenMetadata Table Test Case
+            tc_fqn = f"{table_fqn}.{tc_name}"
             payload = {
                 "timestamp": int(time.time() * 1000),
                 "testCaseStatus": "Success" if summary["overall_success"] else "Failed",
                 "result": f"Suite '{self.expectation_suite}': {summary['successful_evaluations']}/{summary['total_evaluations']} rules passed.",
+                "passedRows": summary.get("successful_evaluations", 1),
+                "failedRows": summary.get("total_unexpected_records", 0),
+                "passedRowsPercentage": float(summary.get("success_percent", 100.0)),
                 "testResultValue": [
                     {"name": "evaluated_count", "value": str(summary["total_evaluations"])},
                     {"name": "unexpected_count", "value": str(summary["total_unexpected_records"])},
@@ -386,16 +452,15 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator):
             }
 
             req = urllib.request.Request(
-                f"{om_host}/api/v1/dataQuality/testCases/testCaseResult",
+                f"{om_host}/api/v1/dataQuality/testCases/testCaseResults/{urllib.parse.quote(tc_fqn, safe='')}",
                 data=json.dumps(payload).encode("utf-8"),
                 headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"},
-                method="PUT"
+                method="POST"
             )
             try:
-                with urllib.request.urlopen(req, timeout=3) as resp:
-                    logger.info("✅ Data Quality metric emitted to OpenMetadata: %s", payload["testCaseStatus"])
+                with urllib.request.urlopen(req, timeout=4) as resp:
+                    logger.info("✅ Table Data Quality metric emitted to OpenMetadata: %s (%s)", tc_fqn, payload["testCaseStatus"])
             except Exception:
-                # Fallback non-blocking
                 pass
         except Exception as e:
             logger.debug("OpenMetadata DQ emission note: %s", e)