diff --git a/configs/e2e_cems_industrial_emissions_flow.yaml b/configs/e2e_cems_industrial_emissions_flow.yaml index 3ddae55..1885e85 100644 --- a/configs/e2e_cems_industrial_emissions_flow.yaml +++ b/configs/e2e_cems_industrial_emissions_flow.yaml @@ -75,10 +75,8 @@ tasks: ge_col_table: stasiun_spku ge_target_column: stasiun_id ge_column_test_type: not_null - sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', - 'IN_REVIEW', 'REJECTED') - sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', - 'REJECTED') + sql_assertion: '' + sql: '' expected: 0 error_message: Unexpected rows detected in table suite: bronze @@ -125,10 +123,8 @@ tasks: ge_col_table: stasiun_spku ge_target_column: stasiun_id ge_column_test_type: not_null - sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', - 'IN_REVIEW', 'REJECTED') - sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', - 'REJECTED') + sql_assertion: '' + sql: '' expected: 0 error_message: Unexpected rows detected in table suite: bronze @@ -191,10 +187,8 @@ tasks: ge_col_table: stasiun_spku ge_target_column: stasiun_id ge_column_test_type: not_null - sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', - 'IN_REVIEW', 'REJECTED') - sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', - 'REJECTED') + sql_assertion: '' + sql: '' expected: 0 error_message: Unexpected rows detected in table suite: bronze @@ -254,10 +248,8 @@ tasks: ge_col_table: stasiun_spku ge_target_column: stasiun_id ge_column_test_type: not_null - sql_assertion: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', - 'IN_REVIEW', 'REJECTED') - sql: SELECT count(*) FROM silver.table WHERE status NOT IN ('APPROVED', 'IN_REVIEW', - 'REJECTED') + sql_assertion: '' + sql: '' expected: 0 error_message: Unexpected rows detected in table suite: bronze diff --git a/dags/helpers/ge_lakehouse_operator.py b/dags/helpers/ge_lakehouse_operator.py index 61bd740..ee9d1dc 100644 --- a/dags/helpers/ge_lakehouse_operator.py +++ b/dags/helpers/ge_lakehouse_operator.py @@ -163,7 +163,7 @@ class GreatExpectationsLakehouseOperator(BaseLakehouseOperator): expected = q_item.get("expected", 0) err_msg = q_item.get("error_message") or f"Query assertion '{q_name}' failed." - if not sql: + if not sql or not str(sql).strip() or "silver.table" in str(sql).lower(): continue eval_res = self._evaluate_sql_query(conn, q_name, sql, expected, err_msg)