first commit
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "cdc_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "cdc-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders"
|
||||
},
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "gold_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "gold-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders"
|
||||
},
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "id"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "stok_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 100000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Great Expectations Configuration
|
||||
# =============================================================================
|
||||
# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
|
||||
# Validasi kualitas data di layer Gold sebelum dikonsumsi aplikasi.
|
||||
# =============================================================================
|
||||
|
||||
config_version: 4.0
|
||||
|
||||
datasources:
|
||||
trino_iceberg:
|
||||
class_name: Datasource
|
||||
execution_engine:
|
||||
class_name: SqlAlchemyExecutionEngine
|
||||
connection_string: "trino://{{ env_var('GE_TRINO_USER', 'admin') }}:{{ env_var('GE_TRINO_PASSWORD', '') }}@{{ env_var('GE_TRINO_HOST', 'trino-coordinator') }}:8080/iceberg/gold"
|
||||
data_connectors:
|
||||
default_inferred_data_connector_name:
|
||||
class_name: InferredAssetSqlDataConnector
|
||||
include_schema_name: true
|
||||
introspection_directives:
|
||||
schema_name: gold
|
||||
|
||||
expectations_store:
|
||||
class_name: ExpectationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: expectations/
|
||||
|
||||
validations_store:
|
||||
class_name: ValidationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: validations/
|
||||
|
||||
data_docs_sites:
|
||||
local:
|
||||
class_name: SiteBuilder
|
||||
show_how_to_buttons: true
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: uncommitted/data_docs/local_site/
|
||||
site_index_builder:
|
||||
class_name: DefaultSiteIndexBuilder
|
||||
@@ -0,0 +1,18 @@
|
||||
# =============================================================================
|
||||
# Konfigurasi koneksi OpenMetadata untuk GE Action (dibaca oleh ometa_config_handler)
|
||||
# =============================================================================
|
||||
# Direktori ini berisi "config.yml" karena action membacanya sebagai TEMPLATE Jinja:
|
||||
# create_jinja_environment(<dir>) + render_template(env) → <dir>/config.yml
|
||||
# `env('VAR')` disediakan oleh handler, sehingga token TIDAK ditulis di repo.
|
||||
#
|
||||
# Dipakai oleh: metadata/great_expectations/action.py (OpenMetadata >= 2.0)
|
||||
# OpenMetadataValidationAction(config_file_path="/opt/airflow/great_expectations/om", ...)
|
||||
# Di OpenMetadata 1.13.x modul action-nya berbeda (generasi 1xx) dan dihapus di
|
||||
# 2.0; rincian migrasi ada di deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# Jalankan dengan OM_JWT_TOKEN tersedia di environment container.
|
||||
# =============================================================================
|
||||
hostPort: "{{ env('OM_API_URL') }}"
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "{{ env('OM_JWT_TOKEN') }}"
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Jalankan Great Expectations DAN kirim hasilnya ke OpenMetadata
|
||||
# =============================================================================
|
||||
# Jalur A (GE -> OpenMetadata): hasil GE tidak berhenti di log Airflow.
|
||||
# Action resmi OpenMetadata (metadata.great_expectations.action) dipasang pada
|
||||
# checkpoint GE 1.x, sehingga setiap kali checkpoint dijalankan, hasilnya dikirim
|
||||
# ke OpenMetadata dan muncul sebagai Data Quality Test Suite/Test Case dengan
|
||||
# platform "GreatExpectations".
|
||||
#
|
||||
# Catatan versi: OM 1.13.x memakai modul action generasi lama (kelas berakhiran
|
||||
# "1xx"); di OM 2.0 modul itu dihapus dan kelasnya kini bernama
|
||||
# OpenMetadataValidationAction di metadata.great_expectations.action.
|
||||
# Rincian migrasinya: deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# ⚠️ Sejak OM 2.0 action MENOLAK batch yang bukan dari database relasional:
|
||||
# `_get_checkpoint_batch_spec()` melempar ValueError bila batch_spec tidak
|
||||
# memuat kunci SQL (table_name/schema_name/query/temp_table_schema_name).
|
||||
# Karena itu ekspektasi dijalankan di Trino lewat DATASOURCE SQL — bukan atas
|
||||
# pandas dataframe — sehingga batch_spec memuat schema_name + table_name dan
|
||||
# action dapat menemukan tabel OpenMetadata-nya.
|
||||
#
|
||||
# Dijalankan DI DALAM container Airflow (butuh great_expectations + metadata):
|
||||
# docker exec -e OM_JWT_TOKEN=<token> -e DBT_TRINO_PASSWORD=<pw> \
|
||||
# sicabe-airflow-webserver python /opt/airflow/great_expectations/validate_to_openmetadata.py
|
||||
#
|
||||
# Exit code: 0 = semua ekspektasi lulus; 1 = ADA yang gagal (task Airflow akan
|
||||
# FAILED) — hasilnya TETAP terkirim ke OpenMetadata sebelum exit.
|
||||
# =============================================================================
|
||||
import os
|
||||
import sys
|
||||
|
||||
import great_expectations as gx
|
||||
import trino
|
||||
from great_expectations.core.expectation_suite import ExpectationSuite
|
||||
|
||||
from metadata.great_expectations.action import OpenMetadataValidationAction
|
||||
OM_CONFIG_DIR = os.getenv("OM_CONFIG_DIR", "/opt/airflow/great_expectations/om")
|
||||
# OM_API_URL = alamat OM dari DALAM jaringan container (bukan OPENMETADATA_URL yang
|
||||
# dipakai dari host). Dipakai juga oleh template config action (om/config.yml).
|
||||
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
|
||||
DATABASE_SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
|
||||
DATABASE_NAME = os.getenv("OM_DATABASE_NAME", "iceberg")
|
||||
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
|
||||
|
||||
# (schema, tabel, daftar ekspektasi)
|
||||
CHECKS = [
|
||||
("bronze", "orders", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="harga_per_kg", min_value=0, max_value=1000000),
|
||||
]),
|
||||
("silver", "orders_silver", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_value", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
("gold", "orders_gold", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="avg_harga_per_kg", min_value=0, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_nilai", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
def _trino_connection():
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return trino.dbapi.connect(
|
||||
host=os.getenv("DBT_TRINO_HOST", "trino-coordinator"),
|
||||
port=int(os.getenv("DBT_TRINO_PORT", "8443")),
|
||||
http_scheme="https",
|
||||
user=user,
|
||||
catalog=DATABASE_NAME,
|
||||
auth=trino.auth.BasicAuthentication(user, password),
|
||||
verify=TRINO_CERT,
|
||||
)
|
||||
|
||||
|
||||
def _trino_sqlalchemy_url():
|
||||
"""URL SQLAlchemy untuk datasource SQL Great Expectations (tanpa password).
|
||||
|
||||
Password TIDAK ditulis di URL — dikirim lewat objek auth pada connect_args,
|
||||
sehingga tidak muncul di connection string, log, atau pesan error.
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
host = os.getenv("DBT_TRINO_HOST", "trino-coordinator")
|
||||
port = os.getenv("DBT_TRINO_PORT", "8443")
|
||||
return f"trino://{user}@{host}:{port}/{DATABASE_NAME}"
|
||||
|
||||
|
||||
def _trino_connect_args():
|
||||
"""Argumen koneksi untuk dialect SQLAlchemy Trino.
|
||||
|
||||
Penting: dialect Trino TIDAK membaca `http_scheme`/`verify` dari query URL —
|
||||
keduanya harus diberikan sebagai connect_args. `verify` juga harus berupa
|
||||
path sertifikat (bukan JSON string seperti bila lewat query URL).
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return {
|
||||
"http_scheme": "https",
|
||||
"verify": TRINO_CERT,
|
||||
"auth": trino.auth.BasicAuthentication(user, password),
|
||||
}
|
||||
|
||||
|
||||
def _assert_table_not_empty(schema, table):
|
||||
"""Fail-fast bila tabel kosong: tidak ada yang bermakna untuk divalidasi."""
|
||||
row = None
|
||||
with _trino_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"SELECT count(*) FROM {schema}.{table}")
|
||||
row = cursor.fetchone()
|
||||
if not row or not row[0]:
|
||||
raise RuntimeError(f"Tabel {schema}.{table} kosong — tidak ada yang divalidasi")
|
||||
|
||||
|
||||
def _build_checkpoint(context, schema, table, expectations):
|
||||
"""Satu checkpoint per tabel: suite + batch definition SQL + action OM."""
|
||||
_assert_table_not_empty(schema, table)
|
||||
|
||||
suite_name = f"{schema}_{table}_suite"
|
||||
data_source = context.data_sources.add_or_update_sql(
|
||||
name=f"trino_{schema}_{table}",
|
||||
connection_string=_trino_sqlalchemy_url(),
|
||||
kwargs={"connect_args": _trino_connect_args()},
|
||||
)
|
||||
asset = data_source.add_table_asset(name=table, table_name=table, schema_name=schema)
|
||||
batch_definition = asset.add_batch_definition_whole_table(f"bd_{schema}_{table}")
|
||||
|
||||
suite = ExpectationSuite(name=suite_name)
|
||||
for expectation in expectations:
|
||||
suite.add_expectation(expectation)
|
||||
context.suites.add_or_update(suite)
|
||||
|
||||
# ValidationDefinition memerlukan BatchDefinition (bukan Batch) + suite.
|
||||
validation_definition = context.validation_definitions.add(
|
||||
gx.ValidationDefinition(name=f"vd_{schema}_{table}", data=batch_definition, suite=suite)
|
||||
)
|
||||
|
||||
action = OpenMetadataValidationAction(
|
||||
config_file_path=OM_CONFIG_DIR,
|
||||
database_service_name=DATABASE_SERVICE,
|
||||
database_name=DATABASE_NAME,
|
||||
schema_name=schema,
|
||||
table_name=table,
|
||||
)
|
||||
checkpoint = context.checkpoints.add(
|
||||
gx.Checkpoint(
|
||||
name=f"cp_{schema}_{table}",
|
||||
validation_definitions=[validation_definition],
|
||||
actions=[action],
|
||||
)
|
||||
)
|
||||
# Batch "whole table" dijalankan di Trino (tanpa batch_parameters).
|
||||
return checkpoint
|
||||
|
||||
|
||||
def _ensure_om_token():
|
||||
"""Token OpenMetadata untuk action: pakai OM_JWT_TOKEN, atau login sendiri.
|
||||
|
||||
Login sendiri membuat skrip aman dipakai pada jadwal (DAG) — tidak bergantung
|
||||
pada token sementara yang ditempel di environment.
|
||||
"""
|
||||
if os.getenv("OM_JWT_TOKEN"):
|
||||
return True
|
||||
# Template config action membaca OM_API_URL → pastikan selalu terisi.
|
||||
os.environ.setdefault("OM_API_URL", OM_API_URL)
|
||||
email = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
|
||||
password = os.getenv("OPENMETADATA_ADMIN_PASSWORD")
|
||||
import base64
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
# DUA kandidat, MAKSIMAL DUA percobaan (jangan diulang dalam loop — akun admin bisa
|
||||
# terkunci): password .env, lalu default 'admin' yang dipakai OM 2.0.x sampai
|
||||
# bootstrap step 7 menulis hash bcrypt. Sama seperti ingest-openmetadata.sh.
|
||||
candidates = [c for c in dict.fromkeys([password or "", "admin"]) if c]
|
||||
err_last = ""
|
||||
for pw in candidates:
|
||||
body = json.dumps({"email": email,
|
||||
"password": base64.b64encode(pw.encode()).decode()}).encode()
|
||||
req = urllib.request.Request(f"{OM_API_URL}/v1/users/login", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
token = json.load(urllib.request.urlopen(req, timeout=20)).get("accessToken", "")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
err_last = str(exc)
|
||||
token = ""
|
||||
if token:
|
||||
os.environ["OM_JWT_TOKEN"] = token
|
||||
return True
|
||||
print(f"ERROR: login ke OpenMetadata gagal: {err_last or 'tanpa token'}")
|
||||
print(" dicoba: password .env + 'admin' — pastikan: ./bootstrap.sh --step 7")
|
||||
return False
|
||||
|
||||
|
||||
def _suite_exists_in_om(schema: str, table: str):
|
||||
"""Apakah test suite untuk tabel ini ADA di OpenMetadata?
|
||||
|
||||
Mengembalikan nama suite bila ada, False bila tidak, None bila tidak dapat
|
||||
diverifikasi.
|
||||
|
||||
Dua hal penting yang membuat versi pertama fungsi ini SALAH:
|
||||
1. `checkpoint.run()` hanya melaporkan status CHECKPOINT (validasi expectation),
|
||||
bukan status pengiriman action — jadi tidak boleh disimpulkan dari situ.
|
||||
2. Action OpenMetadataValidationAction menamai test suite menurut FQN TABEL:
|
||||
<service>.<database>.<schema>.<table>.testSuite
|
||||
BUKAN menurut nama ExpectationSuite GE (`<schema>_<table>_suite`). Memeriksa
|
||||
nama GE-nya menghasilkan false negative: DQ sudah ada di OM tapi dilaporkan
|
||||
tidak ada.
|
||||
"""
|
||||
token = os.environ.get("OM_JWT_TOKEN", "")
|
||||
if not token:
|
||||
return None
|
||||
import json
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
f"{OM_API_URL}/v1/dataQuality/testSuites?limit=200",
|
||||
headers={"Authorization": f"Bearer {token}"})
|
||||
try:
|
||||
d = json.load(urllib.request.urlopen(req, timeout=20))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[GE] verifikasi ke OM gagal: {exc}")
|
||||
return None
|
||||
names = {s.get("name") for s in (d.get("data") or [])}
|
||||
table_fqn = f"{DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}"
|
||||
for expected in (f"{table_fqn}.testSuite", table_fqn):
|
||||
if expected in names:
|
||||
return expected
|
||||
# toleran terhadap variasi penamaan: suite apa pun yang berawalan FQN tabel ini
|
||||
for n in names:
|
||||
if n and n.startswith(table_fqn):
|
||||
return n
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
only = None
|
||||
if len(sys.argv) > 2 and sys.argv[1] == "--table":
|
||||
only = sys.argv[2] # format: schema.table
|
||||
|
||||
if not _ensure_om_token():
|
||||
print("ERROR: token OpenMetadata tidak tersedia (set OM_JWT_TOKEN, atau "
|
||||
"OPENMETADATA_ADMIN_PASSWORD untuk login otomatis)")
|
||||
return 1
|
||||
|
||||
checks = [c for c in CHECKS if only is None or f"{c[0]}.{c[1]}" == only]
|
||||
if not checks:
|
||||
print(f"ERROR: tidak ada pemeriksaan untuk '{only}'")
|
||||
return 1
|
||||
|
||||
context = gx.get_context(mode="ephemeral")
|
||||
checkpoints = [_build_checkpoint(context, s, t, e) for s, t, e in checks]
|
||||
|
||||
failed = 0
|
||||
not_in_om = 0
|
||||
for (schema, table, _expectations), checkpoint in zip(checks, checkpoints):
|
||||
result = checkpoint.run()
|
||||
success = bool(getattr(result, "success", False))
|
||||
in_om = _suite_exists_in_om(schema, table)
|
||||
if in_om is None:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success}; "
|
||||
f"keberadaan di OpenMetadata TIDAK dapat diverifikasi")
|
||||
elif in_om:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"→ test suite '{in_om}' ADA di OpenMetadata")
|
||||
else:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"TAPI belum ada test suite untuk {DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}")
|
||||
print("[GE] penyebab tersering: entitas TABEL-nya belum ada di katalog OM.")
|
||||
print("[GE] urutan yang benar: ./scripts/ingest-openmetadata.sh --all DULU,")
|
||||
print("[GE] baru DAG -- karena action GE menempel ke entitas tabel di OM.")
|
||||
not_in_om += 1
|
||||
if not success:
|
||||
failed += 1
|
||||
|
||||
total = len(checkpoints)
|
||||
print(f"[GE] ringkasan: {total - failed}/{total} checkpoint lulus"
|
||||
+ (f"; {not_in_om} suite TIDAK ADA di OpenMetadata" if not_in_om else ""))
|
||||
# PENTING: `success` hanyalah status CHECKPOINT, bukan status pengiriman. Pesan lama
|
||||
# ("success=True → hasil dikirim ke OpenMetadata") menyesatkan persis karena itu.
|
||||
# Suite yang tidak ada di OM juga membuat task gagal, supaya penyebabnya terlihat
|
||||
# di sini alih-alih muncul sebagai kegagalan gate beberapa langkah kemudian.
|
||||
return 1 if (failed or not_in_om) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user