first commit
This commit is contained in:
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — seed contoh Data Quality NATIVE OpenMetadata (Jalur B)
|
||||
# =============================================================================
|
||||
# Mengapa skrip ini ada:
|
||||
# Test case native TIDAK bisa diandalkan lewat TestRunner SDK yang berjalan di
|
||||
# klien: jalur itu memakai koneksi TERSIMPAN yang kehilangan secret sehingga
|
||||
# selalu berakhir `Aborted` dengan `401 Access Denied` (bug upstream, masih ada
|
||||
# di OM 2.0.2 — kredensial sudah dipastikan valid).
|
||||
# Jalur yang bekerja: definisi test case dibuat lewat REST API, lalu eksekusi
|
||||
# DIPICU OpenMetadata lewat pipeline service client (plugin → Airflow). Koneksi
|
||||
# Trino dibawa EKSPLISIT di `sourceConfig.serviceConnections`, sehingga tidak
|
||||
# bergantung pada koneksi tersimpan.
|
||||
#
|
||||
# Idempoten: test case diperiksa berdasarkan nama; pipeline diperiksa berdasarkan
|
||||
# nama. Menjalankan ulang tidak menggandakan apa pun.
|
||||
#
|
||||
# Dipanggil oleh: scripts/seed-native-dq.sh (dari host) — atau manual:
|
||||
# docker exec -e OM_SERVICE=trino_sicabe ... sicabe-airflow-webserver \
|
||||
# python /opt/airflow/scripts/seed-om-native-dq.py [--trigger]
|
||||
#
|
||||
# Exit code: 0 = semua langkah yang diminta berhasil.
|
||||
# =============================================================================
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
|
||||
ADMIN_EMAIL = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
|
||||
SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
|
||||
CATALOG = os.getenv("OM_DATABASE_NAME", "iceberg")
|
||||
TABLE_FQN = os.getenv("OM_DQ_SEED_TABLE", f"{SERVICE}.{CATALOG}.gold.orders_gold")
|
||||
PIPELINE_NAME = os.getenv("OM_DQ_SEED_PIPELINE", "sicabe_dq_gold_native")
|
||||
SCHEDULE = os.getenv("OM_DQ_SEED_SCHEDULE", "0 3 * * *")
|
||||
TRINO_HOSTPORT = os.getenv("OM_TRINO_HOSTPORT", "trino-coordinator:8443")
|
||||
TRINO_USER = os.getenv("DBT_TRINO_USER", "svc_om")
|
||||
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
|
||||
|
||||
# Contoh test native. `parameterValues` HARUS memakai nama parameter definisi
|
||||
# (tableRowCountToBeBetween: minValue/maxValue) dan columnValuesToBeNotNull
|
||||
# tidak punya parameter sama sekali — kolomnya ditentukan lewat entityLink.
|
||||
TEST_CASES = [
|
||||
{
|
||||
"name": "native_row_count_between",
|
||||
"testDefinition": "tableRowCountToBeBetween",
|
||||
"entityLink": f"<#E::table::{TABLE_FQN}>",
|
||||
"parameterValues": [
|
||||
{"name": "minValue", "value": "1"},
|
||||
{"name": "maxValue", "value": "1000000"},
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "native_komoditas_not_null",
|
||||
"testDefinition": "columnValuesToBeNotNull",
|
||||
"entityLink": f"<#E::table::{TABLE_FQN}::columns::komoditas>",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _request(method, path, payload=None, token=None):
|
||||
"""Panggil REST API OpenMetadata; kembalikan (status, body terurai)."""
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
data = json.dumps(payload).encode() if payload is not None else None
|
||||
req = urllib.request.Request(f"{OM_API_URL}{path}", data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
raw = resp.read()
|
||||
return resp.status, (json.loads(raw) if raw else {})
|
||||
except urllib.error.HTTPError as exc:
|
||||
raw = exc.read()[:300]
|
||||
try:
|
||||
return exc.code, json.loads(raw)
|
||||
except Exception: # noqa: BLE001
|
||||
return exc.code, raw.decode(errors="replace")
|
||||
|
||||
|
||||
def login():
|
||||
"""Token admin: password dikirim base64 (wajib sejak OM 2.0.x).
|
||||
|
||||
Dua kandidat: OPENMETADATA_ADMIN_PASSWORD, lalu 'admin' — default yang dipakai
|
||||
OM 2.0.x sampai bootstrap step 7 menyetel hash-nya dari .env. MAKSIMAL DUA
|
||||
percobaan dan tidak pernah diulang dalam loop: akun admin OM bisa terkunci.
|
||||
"""
|
||||
last = "tidak ada kandidat password"
|
||||
for password in (os.getenv("OPENMETADATA_ADMIN_PASSWORD"), "admin"):
|
||||
if not password:
|
||||
continue
|
||||
body = {"email": ADMIN_EMAIL,
|
||||
"password": base64.b64encode(password.encode()).decode()}
|
||||
status, res = _request("POST", "/v1/users/login", body)
|
||||
if status == 200 and res.get("accessToken"):
|
||||
return res["accessToken"]
|
||||
last = f"HTTP {status}: {str(res)[:120]}"
|
||||
raise RuntimeError(f"login gagal (password .env dan 'admin'): {last}")
|
||||
|
||||
|
||||
def ensure_test_cases(token):
|
||||
"""Buat test case native yang belum ada. Kembalikan jumlah yang dibuat."""
|
||||
status, page = _request("GET", "/v1/dataQuality/testCases?limit=200", token=token)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"daftar test case tidak terbaca (HTTP {status})")
|
||||
existing = {tc.get("name") for tc in page.get("data", [])}
|
||||
|
||||
created = 0
|
||||
for spec in TEST_CASES:
|
||||
if spec["name"] in existing:
|
||||
print(f" [SKIP] test case sudah ada: {spec['name']}")
|
||||
continue
|
||||
status, res = _request("POST", "/v1/dataQuality/testCases", spec, token=token)
|
||||
if status in (200, 201):
|
||||
created += 1
|
||||
print(f" [OK] test case dibuat: {spec['name']}")
|
||||
else:
|
||||
raise RuntimeError(f"gagal membuat {spec['name']} (HTTP {status}): {str(res)[:200]}")
|
||||
return created
|
||||
|
||||
|
||||
def _pipeline_payload(suite_id):
|
||||
"""Payload pipeline TestSuite dengan koneksi Trino EKSPLISIT (anti-401)."""
|
||||
return {
|
||||
"name": PIPELINE_NAME,
|
||||
"pipelineType": "TestSuite",
|
||||
"service": {"id": suite_id, "type": "testSuite"},
|
||||
"sourceConfig": {"config": {
|
||||
"type": "TestSuite",
|
||||
"entityFullyQualifiedName": TABLE_FQN,
|
||||
"serviceConnections": [{
|
||||
"serviceName": SERVICE,
|
||||
"serviceConnection": {"config": {
|
||||
"type": "Trino", "scheme": "trino", "hostPort": TRINO_HOSTPORT,
|
||||
"username": TRINO_USER,
|
||||
"authType": {"password": os.getenv("TRINO_SVC_OM_PASSWORD", "")},
|
||||
"catalog": CATALOG, "verify": TRINO_CERT,
|
||||
}},
|
||||
}],
|
||||
}},
|
||||
"airflowConfig": {"scheduleInterval": SCHEDULE},
|
||||
}
|
||||
|
||||
|
||||
def ensure_pipeline(token):
|
||||
"""Buat pipeline TestSuite bila belum ada. Kembalikan (id, dibuat_baru)."""
|
||||
status, suites = _request("GET", "/v1/dataQuality/testSuites?limit=50", token=token)
|
||||
if status != 200:
|
||||
raise RuntimeError(f"daftar suite tidak terbaca (HTTP {status})")
|
||||
suite = next((s for s in suites.get("data", []) if TABLE_FQN.rsplit(".", 1)[0] in s.get("name", "")), None)
|
||||
if suite is None:
|
||||
raise RuntimeError(f"test suite untuk '{TABLE_FQN}' belum ada — jalankan ingestion dulu")
|
||||
|
||||
status, page = _request("GET", "/v1/services/ingestionPipelines?limit=50", token=token)
|
||||
for pipe in (page.get("data") or []) if status == 200 else []:
|
||||
if pipe.get("name") == PIPELINE_NAME:
|
||||
print(f" [SKIP] pipeline sudah ada: {PIPELINE_NAME} (id {pipe['id']})")
|
||||
return pipe["id"], False
|
||||
|
||||
status, res = _request("POST", "/v1/services/ingestionPipelines",
|
||||
_pipeline_payload(suite["id"]), token=token)
|
||||
if status not in (200, 201):
|
||||
raise RuntimeError(f"gagal membuat pipeline (HTTP {status}): {str(res)[:200]}")
|
||||
print(f" [OK] pipeline dibuat: {PIPELINE_NAME} (id {res['id']})")
|
||||
return res["id"], True
|
||||
|
||||
|
||||
def deploy_and_trigger(token, pipeline_id):
|
||||
"""Deploy lalu trigger pipeline; trigger diulang sekali bila DAG belum siap."""
|
||||
import time
|
||||
|
||||
status, res = _request("POST", f"/v1/services/ingestionPipelines/deploy/{pipeline_id}", token=token)
|
||||
print(f" [{'OK' if status == 200 else 'GAGAL'}] deploy -> HTTP {status}")
|
||||
if status != 200:
|
||||
raise RuntimeError(f"deploy gagal: {str(res)[:200]}")
|
||||
|
||||
for attempt in (1, 2):
|
||||
status, res = _request("POST", f"/v1/services/ingestionPipelines/trigger/{pipeline_id}", token=token)
|
||||
print(f" [{'OK' if status == 200 else 'GAGAL'}] trigger -> HTTP {status} (percobaan {attempt})")
|
||||
if status == 200:
|
||||
return
|
||||
# DAG baru dibuat: dag-processor bisa belum memparsingnya.
|
||||
time.sleep(20)
|
||||
raise RuntimeError("trigger gagal setelah 2 percobaan")
|
||||
|
||||
|
||||
def main():
|
||||
trigger = "--trigger" in sys.argv
|
||||
print(f"[seed-native-dq] tabel={TABLE_FQN} pipeline={PIPELINE_NAME}")
|
||||
token = login()
|
||||
print(f" [OK] login {ADMIN_EMAIL}")
|
||||
|
||||
created = ensure_test_cases(token)
|
||||
pipeline_id, new_pipeline = ensure_pipeline(token)
|
||||
|
||||
if trigger:
|
||||
deploy_and_trigger(token, pipeline_id)
|
||||
print(f"[seed-native-dq] selesai: {created} test case baru, pipeline "
|
||||
f"{'baru' if new_pipeline else 'sudah ada'}"
|
||||
f"{', deploy+trigger dijalankan' if trigger else ''}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"ERROR: {exc}")
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user