merica-dlh/merica-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pipeline): remove placeholder dummy SQL assertion from GE quality nodes

Browse Source
This commit is contained in:
syahril
2026-10-19 03:58:21 +07:00
parent b37ffc52ae
commit 13835055b1
2 changed files with 9 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
configs/e2e_cems_industrial_emissions_flow.yaml
+8 -16
View File
@@ -75,10 +75,8 @@ tasks:
ge_col_table: stasiun_spku ge_col_table: stasiun_spku
ge_target_column: stasiun_id ge_target_column: stasiun_id
ge_column_test_type: not_null ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', sql_assertion: ''
'IN_REVIEW', 'REJECTED') sql: ''
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0 expected: 0
error_message: Unexpected rows detected in table error_message: Unexpected rows detected in table
suite: bronze suite: bronze
@@ -125,10 +123,8 @@ tasks:
ge_col_table: stasiun_spku ge_col_table: stasiun_spku
ge_target_column: stasiun_id ge_target_column: stasiun_id
ge_column_test_type: not_null ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', sql_assertion: ''
'IN_REVIEW', 'REJECTED') sql: ''
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0 expected: 0
error_message: Unexpected rows detected in table error_message: Unexpected rows detected in table
suite: bronze suite: bronze
@@ -191,10 +187,8 @@ tasks:
ge_col_table: stasiun_spku ge_col_table: stasiun_spku
ge_target_column: stasiun_id ge_target_column: stasiun_id
ge_column_test_type: not_null ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', sql_assertion: ''
'IN_REVIEW', 'REJECTED') sql: ''
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0 expected: 0
error_message: Unexpected rows detected in table error_message: Unexpected rows detected in table
suite: bronze suite: bronze
@@ -254,10 +248,8 @@ tasks:
ge_col_table: stasiun_spku ge_col_table: stasiun_spku
ge_target_column: stasiun_id ge_target_column: stasiun_id
ge_column_test_type: not_null ge_column_test_type: not_null
sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', sql_assertion: ''
'IN_REVIEW', 'REJECTED') sql: ''
sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW',
'REJECTED')
expected: 0 expected: 0
error_message: Unexpected rows detected in table error_message: Unexpected rows detected in table
suite: bronze suite: bronze
dags/helpers/ge_lakehouse_operator.py
+1 -1
View File
@@ -163,7 +163,7 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator):
expected = q_item.get("expected", 0) expected = q_item.get("expected", 0)
err_msg = q_item.get("error_message") or f"Query assertion '{q_name}' failed." err_msg = q_item.get("error_message") or f"Query assertion '{q_name}' failed."
if not sql: if not sql or not str(sql).strip() or "silver.table" in str(sql).lower():
continue continue
eval_res = self._evaluate_sql_query(conn, q_name, sql, expected, err_msg) eval_res = self._evaluate_sql_query(conn, q_name, sql, expected, err_msg)