first commit
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
# =============================================================================
|
||||
# Apache Airflow — Configuration (Airflow 3.3.x)
|
||||
# Referensi arsitektur: bab 4.5.1 (Apache Airflow — Orchestrator)
|
||||
# =============================================================================
|
||||
# ⚠️ Airflow 3.3 memakai SimpleAuthManager (bukan FabAuthManager).
|
||||
# Config [api] auth_backends dan [webserver] auth_backend dari Airflow 2.x
|
||||
# TIDAK berlaku lagi — dihapus agar task runner bisa autentikasi internal.
|
||||
# =============================================================================
|
||||
|
||||
[core]
|
||||
executor = LocalExecutor
|
||||
simple_auth_manager_users = admin:Admin
|
||||
simple_auth_manager_passwords_file = /opt/airflow/simple_auth_manager_passwords.json
|
||||
parallelism = 32
|
||||
max_active_tasks_per_dag = 16
|
||||
dagbag_import_timeout = 30
|
||||
load_examples = False
|
||||
default_timezone = Asia/Jakarta
|
||||
|
||||
[database]
|
||||
# ⚠️ ConfigParser does NOT expand ${VAR} syntax — this is a placeholder only.
|
||||
# The actual connection string is set via AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
# environment variable in docker-compose.yml.
|
||||
sql_alchemy_conn = postgresql+psycopg2://airflow:PLACEHOLDER@postgres:5432/airflow
|
||||
sql_alchemy_pool_enabled = True
|
||||
sql_alchemy_pool_size = 10
|
||||
sql_alchemy_max_overflow = 20
|
||||
|
||||
[webserver]
|
||||
# base_url internal (container) — task runner connect ke sini
|
||||
base_url = http://airflow-webserver:8080
|
||||
default_ui_timezone = Asia/Jakarta
|
||||
expose_config = True
|
||||
|
||||
[scheduler]
|
||||
parsing_processes = 4
|
||||
max_threads = 4
|
||||
scheduler_heartbeat_sec = 5
|
||||
|
||||
[dag_processor]
|
||||
# Airflow 3.x: dag_dir_list_interval dipindah dari [scheduler] ke sini
|
||||
# (menghindari DeprecationWarning saat start).
|
||||
refresh_interval = 30
|
||||
|
||||
[logging]
|
||||
remote_logging = False
|
||||
log_level = INFO
|
||||
|
||||
[email]
|
||||
email_backend = airflow.utils.email.send_email_smtp
|
||||
|
||||
[smtp]
|
||||
smtp_host = localhost
|
||||
smtp_port = 25
|
||||
smtp_timeout = 30
|
||||
smtp_mail_from = airflow@sicabe.local
|
||||
@@ -0,0 +1,96 @@
|
||||
"""sitomat — DAG-3 (gold build) — TAHAP A: CONNECTIVITY SMOKE DAG (Airflow 3.x).
|
||||
|
||||
HONESTY (baca dulu):
|
||||
DAG ini HANYA memvalidasi konektivitas sitomat <-> Airflow (validate DAG,
|
||||
trigger run, poll status). Ia TIDAK membangun gold dan **TIDAK** memanggil
|
||||
callback `register_gold` — jadi **tidak ada artefak gold yang diklaim**.
|
||||
|
||||
Tahap B akan menggantinya dengan transform nyata: baca silver (via Trino),
|
||||
tulis gold, lalu callback ke sitomat. Lihat README.md di folder yang sama.
|
||||
|
||||
Airflow 3.x: dataclass import publik adalah `airflow.sdk` (bukan
|
||||
`airflow.decorators`/`airflow.models`), dan `schedule_interval` sudah dihapus
|
||||
-> gunakan `schedule=None` untuk DAG yang hanya dipicu lewat REST.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import pendulum
|
||||
from airflow.sdk import dag, get_current_context, task
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Kunci conf yang WAJIB dikirim sitomat saat trigger (lihat F16 `flattenTriggerConf`).
|
||||
REQUIRED_CONF_KEYS = (
|
||||
"correlationId",
|
||||
"dataProductBuildId",
|
||||
"dataProductId",
|
||||
"useCaseId",
|
||||
)
|
||||
|
||||
|
||||
@dag(
|
||||
dag_id="sitomat_gold_build", # samakan dengan env `GOLD_DAG_ID` di sitomat
|
||||
description="sitomat F16 DAG-3 — connectivity smoke (tidak membangun gold)",
|
||||
schedule=None, # Airflow 3: hanya dipicu lewat REST (POST /api/v2/.../dagRuns)
|
||||
start_date=pendulum.datetime(2026, 1, 1, tz="UTC"),
|
||||
catchup=False,
|
||||
tags=["sitomat", "f16", "connectivity"],
|
||||
)
|
||||
def sitomat_gold_build():
|
||||
@task
|
||||
def parse_conf() -> dict:
|
||||
"""Baca `dag_run.conf` dari sitomat dan validasi kunci wajib."""
|
||||
ctx = get_current_context()
|
||||
conf = dict((ctx["dag_run"].conf or {})) if ctx else {}
|
||||
|
||||
missing = [key for key in REQUIRED_CONF_KEYS if not conf.get(key)]
|
||||
if missing:
|
||||
# Gagal jujur (BUKAN sukses palsu): conf tidak lengkap.
|
||||
raise ValueError(f"conf sitomat tidak lengkap, hilang: {missing}")
|
||||
|
||||
silver_refs = [ref for ref in str(conf.get("silverRefs", "")).split(",") if ref]
|
||||
log.info(
|
||||
"sitomat conf OK: correlationId=%s build=%s product=%s useCase=%s "
|
||||
"sources=%s silverRefs=%d",
|
||||
conf.get("correlationId"),
|
||||
conf.get("dataProductBuildId"),
|
||||
conf.get("dataProductId"),
|
||||
conf.get("useCaseId"),
|
||||
conf.get("sourceCount"),
|
||||
len(silver_refs),
|
||||
)
|
||||
if not silver_refs:
|
||||
log.warning(
|
||||
"silverRefs kosong — Tahap B (build nyata) belum bisa membaca sumber."
|
||||
)
|
||||
return {
|
||||
"correlationId": conf.get("correlationId"),
|
||||
"dataProductBuildId": conf.get("dataProductBuildId"),
|
||||
"silverRefs": silver_refs,
|
||||
}
|
||||
|
||||
@task
|
||||
def connectivity_report(payload: dict) -> None:
|
||||
"""TAHAP A: sengaja hanya melaporkan. TIDAK menulis gold.
|
||||
|
||||
Tahap B (ganti task ini):
|
||||
1. baca `payload["silverRefs"]` via Trino (koneksi `trino_default`),
|
||||
2. rakit & tulis tabel gold (Iceberg/Nessie),
|
||||
3. POST ke sitomat:
|
||||
{SITOMAT_BASE_URL}/data-products/builds/{dataProductBuildId}/gold
|
||||
header: `X-API-Key: <kunci scope pipeline>`
|
||||
body: {"goldRef": ..., "checksum": ..., "rowCount": ...}
|
||||
Simpan URL & kunci sebagai Airflow Variable/Connection (JANGAN hardcode).
|
||||
"""
|
||||
log.info(
|
||||
"sitomat DAG-3 connectivity OK untuk build=%s (Tahap A: tanpa gold)",
|
||||
payload.get("dataProductBuildId"),
|
||||
)
|
||||
|
||||
connectivity_report(parse_conf())
|
||||
|
||||
|
||||
sitomat_gold_build()
|
||||
@@ -0,0 +1,344 @@
|
||||
"""
|
||||
DAG-3 PRODUKSI — bangun GOLD untuk sebuah Data Product, lalu lapor ke sitomat.
|
||||
|
||||
Fitur F16/F15 (ADR-0004..0008). sitomat TIDAK memindahkan byte: DAG ini yang
|
||||
membangun artefak di data lake (Iceberg lewat Trino) dan memanggil callback.
|
||||
|
||||
Kontrak
|
||||
-------
|
||||
Trigger conf (dikirim F15 `createGoldBuildTriggerPort`; SEMUA nilai STRING):
|
||||
correlationId, dataProductBuildId, dataProductId, useCaseId,
|
||||
datasetVersionIds (dipisah koma), silverRefs (dipisah koma; posisinya
|
||||
sejajar dengan datasetVersionIds, slot kosong = sumber tanpa ref),
|
||||
sourceCount
|
||||
|
||||
Callback WAJIB (tanpa ini build sitomat tetap 'building'):
|
||||
POST {SITOMAT_BASE_URL}/data-products/builds/{dataProductBuildId}/gold
|
||||
Header: X-API-Key: <kunci F09-A scope `pipeline`>
|
||||
Body: {"goldRef": "...", "checksum": "sha256:...", "rowCount": N}
|
||||
|
||||
Konfigurasi — Airflow Variable dulu, lalu env, lalu default:
|
||||
sitomat_base_url SITOMAT_BASE_URL http://172.27.228.103:13000
|
||||
sitomat_api_key SITOMAT_API_KEY (WAJIB) kunci scope `pipeline`
|
||||
trino_url TRINO_URL https://sicabe-trino:8443
|
||||
trino_user TRINO_USER svc_seed
|
||||
trino_password TRINO_PASSWORD (WAJIB)
|
||||
gold_target GOLD_TARGET iceberg.gold.dp_<dataProductId>
|
||||
gold_sql GOLD_SQL opsional; bila kosong dirakit dari silverRefs
|
||||
gold_tls_verify GOLD_TLS_VERIFY false (sertifikat internal self-signed)
|
||||
checksum_max_rows CHECKSUM_MAX_ROWS 100000 (0 = tanpa batas)
|
||||
|
||||
**Presedensi:** `silverRefs` dari conf adalah jalur NYATA dan menang; `gold_sql`
|
||||
= **fallback** untuk dataset yang belum menyimpan `artifact_ref`.
|
||||
|
||||
**Kegagalan DILAPORKAN, bukan menggantung:** build NYATA yang sumbernya belum
|
||||
punya `artifact_ref` (atau tanpa sumber sama sekali) **GAGAL dengan sengaja** —
|
||||
DAG menolak membangun produk parsial dan task `report_failure` memanggil
|
||||
`POST …/fail`, sehingga build/produk/use case menjadi `failed`. Hanya run
|
||||
DIAGNOSTIK (`bun run check:airflow`) yang berhenti diam tanpa callback.
|
||||
|
||||
Cakupan OPA: stack ini memakai allowlist per-tabel (`default allow := false`).
|
||||
Menulis ke tabel gold BARU mensyaratkan policy OPA memberi grup `data-engineer`
|
||||
akses pada schema gold/silver (lihat README). Bila belum, set `GOLD_TARGET` ke
|
||||
tabel yang sudah diizinkan (mis. `iceberg.gold.orders_gold`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
from airflow.sdk import dag, get_current_context, task
|
||||
|
||||
# Sertifikat internal self-signed: jangan banjiri log dengan peringatan.
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
DEFAULT_BASE_URL = "http://172.27.228.103:13000"
|
||||
DEFAULT_TRINO_URL = "https://sicabe-trino:8443"
|
||||
DEFAULT_TRINO_USER = "svc_seed"
|
||||
DEFAULT_CHECKSUM_MAX_ROWS = 100000
|
||||
|
||||
# Nama identifier SQL yang aman; apa pun di luar ini DITOLAK (anti-injeksi).
|
||||
IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
# ── Pengaturan (Airflow Variable → env → default) ────────────────────────────
|
||||
|
||||
|
||||
def _variable_get(key: str) -> str | None:
|
||||
"""Baca Airflow Variable `key`; `None` bila tidak ada / tidak tersedia."""
|
||||
Variable = None
|
||||
try:
|
||||
from airflow.sdk import Variable # Airflow 3 SDK
|
||||
except Exception: # pragma: no cover - tergantung versi Airflow
|
||||
try:
|
||||
from airflow.models import Variable # fallback klasik
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
return Variable.get(key)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def setting(name: str, default: str | None = None) -> str | None:
|
||||
"""Airflow Variable `name` (huruf kecil) → env `NAME` (huruf besar) → default."""
|
||||
value = os.environ.get(name.upper())
|
||||
if value:
|
||||
return value
|
||||
variable = _variable_get(name.lower())
|
||||
return variable if variable else default
|
||||
|
||||
|
||||
# ── Util murni ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def qualified(identifier: str) -> str:
|
||||
"""Validasi `tabel` / `schema.tabel` / `catalog.schema.tabel`. Raise bila bukan."""
|
||||
raw = identifier.strip().strip('"').strip("`")
|
||||
parts = [part.strip().strip('"').strip("`") for part in raw.split(".")]
|
||||
if not 1 <= len(parts) <= 3 or not all(IDENTIFIER.match(part) for part in parts):
|
||||
raise ValueError(f"nama tabel tidak valid: {identifier!r}")
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
def safe_suffix(value: str) -> str:
|
||||
"""Jadikan nilai bebas menjadi sufiks identifier yang aman."""
|
||||
cleaned = re.sub(r"[^A-Za-z0-9_]", "_", value or "x")
|
||||
if not cleaned or cleaned[0].isdigit():
|
||||
cleaned = f"t_{cleaned}"
|
||||
return cleaned
|
||||
|
||||
|
||||
def _as_int(value) -> int:
|
||||
"""Integer non-negatif dari conf (semua nilai conf adalah STRING)."""
|
||||
try:
|
||||
return max(0, int(str(value).strip()))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def target_and_sql(conf: dict, sources: list[str]) -> tuple[str, str]:
|
||||
"""(target_fqn, sql_ctas) dari conf + pengaturan.
|
||||
|
||||
`GOLD_SQL` (Variable `gold_sql`) adalah escape hatch yang disengaja: dipakai
|
||||
bila `silverRefs` kosong (mis. versi silver lama belum menyimpan `artifact_ref`)
|
||||
ATAU untuk mengganti transform. Tanpa keduanya, tidak ada yang bisa dibangun.
|
||||
"""
|
||||
configured = setting("gold_target")
|
||||
target = qualified(
|
||||
configured
|
||||
or f"iceberg.gold.dp_{safe_suffix(str(conf.get('dataProductId', 'x')))}"
|
||||
)
|
||||
explicit = setting("gold_sql")
|
||||
# PRESEDENSI: sumber dari conf (`silverRefs`) adalah jalur NYATA dan menang;
|
||||
# `gold_sql` hanya FALLBACK operator untuk dataset yang belum punya
|
||||
# `artifact_ref`. (Kebalikannya membuat override global menelan lineage.)
|
||||
if sources:
|
||||
select = " UNION ALL ".join(f"SELECT * FROM {source}" for source in sources)
|
||||
elif explicit:
|
||||
select = explicit.strip().rstrip(";")
|
||||
else:
|
||||
raise ValueError("tidak ada silverRefs dan tidak ada gold_sql — tidak ada yang dibangun")
|
||||
return target, f"CREATE OR REPLACE TABLE {target} AS {select}"
|
||||
|
||||
|
||||
# ── Klien Trino REST minimal (/v1/statement + paging nextUri) ─────────────────
|
||||
|
||||
|
||||
class Trino:
|
||||
"""Klien Trino HTTPS berbatas; tidak ada dependensi di luar `requests`."""
|
||||
|
||||
def __init__(self, url: str, user: str, password: str | None, verify: bool) -> None:
|
||||
self.url = url.rstrip("/")
|
||||
self.session = requests.Session()
|
||||
self.session.verify = verify
|
||||
self.session.auth = (user, password)
|
||||
self.session.headers.update({"X-Trino-User": user, "Content-Type": "text/plain"})
|
||||
|
||||
def pages(self, sql: str):
|
||||
"""Iterasi (columns, rows) per halaman; raise bila Trino melaporkan error."""
|
||||
response = self.session.post(f"{self.url}/v1/statement", data=sql, timeout=300)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
for _ in range(10_000): # pengaman halaman, bukan batas logis
|
||||
error = payload.get("error")
|
||||
if error:
|
||||
raise RuntimeError(f"trino error: {error.get('message', error)}")
|
||||
columns = [col.get("name") for col in payload.get("columns", [])]
|
||||
yield columns, payload.get("data", [])
|
||||
next_uri = payload.get("nextUri")
|
||||
if not next_uri:
|
||||
return
|
||||
payload = self.session.get(next_uri, timeout=300).json()
|
||||
raise RuntimeError("trino: terlalu banyak halaman tanpa akhir")
|
||||
|
||||
def execute(self, sql: str) -> None:
|
||||
"""Jalankan statement tanpa peduli baris (mis. CTAS)."""
|
||||
for _ in self.pages(sql):
|
||||
pass
|
||||
|
||||
def scalar(self, sql: str):
|
||||
"""Nilai pertama baris pertama."""
|
||||
for _columns, rows in self.pages(sql):
|
||||
if rows:
|
||||
return rows[0][0]
|
||||
return None
|
||||
|
||||
def checksum(self, table: str, max_rows: int) -> str:
|
||||
"""sha256 deterministik atas header + baris (JSON ringkas), opsional dibatasi."""
|
||||
digest = hashlib.sha256()
|
||||
header_written = False
|
||||
seen = 0
|
||||
for columns, rows in self.pages(f"SELECT * FROM {table}"):
|
||||
if not header_written and columns:
|
||||
digest.update((",".join(columns) + "\n").encode("utf-8"))
|
||||
header_written = True
|
||||
for row in rows:
|
||||
if max_rows and seen >= max_rows:
|
||||
return f"sha256:{digest.hexdigest()} (bounded:{max_rows})"
|
||||
digest.update(
|
||||
(json.dumps(row, separators=(",", ":"), default=str) + "\n").encode("utf-8")
|
||||
)
|
||||
seen += 1
|
||||
return f"sha256:{digest.hexdigest()}"
|
||||
|
||||
|
||||
def trino_client() -> Trino:
|
||||
password = setting("trino_password")
|
||||
if not password:
|
||||
raise RuntimeError("trino_password belum diatur (Airflow Variable atau TRINO_PASSWORD)")
|
||||
return Trino(
|
||||
setting("trino_url", DEFAULT_TRINO_URL) or DEFAULT_TRINO_URL,
|
||||
setting("trino_user", DEFAULT_TRINO_USER) or DEFAULT_TRINO_USER,
|
||||
password,
|
||||
str(setting("gold_tls_verify", "false")).lower() in {"1", "true", "yes"},
|
||||
)
|
||||
|
||||
|
||||
# ── DAG ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dag(
|
||||
dag_id="sitomat_gold_build_prod",
|
||||
schedule=None,
|
||||
catchup=False,
|
||||
tags=["sitomat", "f16", "gold"],
|
||||
doc_md=__doc__,
|
||||
)
|
||||
def sitomat_gold_build_prod():
|
||||
@task
|
||||
def build_gold() -> dict:
|
||||
"""CTAS gold dari silverRefs; kembalikan identitas artefak."""
|
||||
conf = get_current_context()["dag_run"].conf or {}
|
||||
build_id = str(conf.get("dataProductBuildId", "")).strip()
|
||||
# Run DIAGNOSTIK (`bun run check:airflow`) tidak punya build nyata.
|
||||
diagnostic = not build_id or build_id == "airflow-check"
|
||||
|
||||
source_count = _as_int(conf.get("sourceCount"))
|
||||
sources = [
|
||||
qualified(raw)
|
||||
for raw in str(conf.get("silverRefs") or "").split(",")
|
||||
if raw.strip()
|
||||
]
|
||||
missing = max(0, source_count - len(sources))
|
||||
explicit = setting("gold_sql")
|
||||
|
||||
# Run diagnostik tanpa sumber: tidak ada yang dibangun, tidak ada callback.
|
||||
if diagnostic and not sources and not explicit:
|
||||
print("[sitomat] run diagnostik tanpa sumber — tidak membangun, tidak callback")
|
||||
return {"built": False}
|
||||
|
||||
# PRODUK PARSIAL DITOLAK. Sumber yang belum punya `artifact_ref` TIDAK
|
||||
# boleh diam-diam dihilangkan (itu menghasilkan gold yang salah). Untuk
|
||||
# build NYATA kita GAGAL di sini, sehingga task `report_failure`
|
||||
# melaporkannya ke sitomat — bukan menggantung `building` selamanya.
|
||||
if missing > 0 and not explicit:
|
||||
raise ValueError(
|
||||
f"build {build_id}: {missing} dari {source_count} sumber tidak punya "
|
||||
"artifact_ref — menolak membangun produk parsial"
|
||||
)
|
||||
|
||||
target, sql = target_and_sql(conf, sources)
|
||||
client = trino_client()
|
||||
print(f"[sitomat] membangun {target} dari {len(sources)} sumber")
|
||||
client.execute(sql)
|
||||
|
||||
row_count = int(client.scalar(f"SELECT count(*) FROM {target}") or 0)
|
||||
checksum = client.checksum(target, int(setting("checksum_max_rows", str(DEFAULT_CHECKSUM_MAX_ROWS))))
|
||||
print(f"[sitomat] selesai: {target} rows={row_count} checksum={checksum}")
|
||||
return {
|
||||
"built": True,
|
||||
"goldRef": target,
|
||||
"rowCount": row_count,
|
||||
"checksum": checksum,
|
||||
"buildId": str(conf.get("dataProductBuildId", "")),
|
||||
}
|
||||
|
||||
@task
|
||||
def register_gold(build: dict) -> None:
|
||||
"""Laporkan artefak ke sitomat (callback `register_gold`)."""
|
||||
if not build.get("built"):
|
||||
return
|
||||
base = (setting("sitomat_base_url", DEFAULT_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
|
||||
api_key = setting("sitomat_api_key")
|
||||
if not api_key:
|
||||
raise RuntimeError("sitomat_api_key belum diatur (Airflow Variable atau SITOMAT_API_KEY)")
|
||||
url = f"{base}/data-products/builds/{build['buildId']}/gold"
|
||||
response = requests.post(
|
||||
url,
|
||||
json={
|
||||
"goldRef": build["goldRef"],
|
||||
"checksum": build["checksum"],
|
||||
"rowCount": build["rowCount"],
|
||||
},
|
||||
headers={"X-API-Key": api_key},
|
||||
timeout=60,
|
||||
)
|
||||
if response.status_code >= 300:
|
||||
raise RuntimeError(
|
||||
f"callback sitomat gagal: HTTP {response.status_code} {response.text[:300]}"
|
||||
)
|
||||
print(f"[sitomat] callback OK ({response.status_code}): {response.text[:200]}")
|
||||
|
||||
@task(trigger_rule="one_failed")
|
||||
def report_failure() -> None:
|
||||
"""Laporkan kegagalan ke sitomat agar build TIDAK menggantung `building`.
|
||||
|
||||
Best-effort: kegagalan pelaporan tidak boleh menutupi error asli DAG.
|
||||
"""
|
||||
context = get_current_context()
|
||||
conf = context["dag_run"].conf or {}
|
||||
build_id = str(conf.get("dataProductBuildId", "")).strip()
|
||||
# Run diagnostik (`bun run check:airflow`) tidak punya build nyata.
|
||||
if not build_id or build_id == "airflow-check":
|
||||
print("[sitomat] tidak ada build nyata untuk dilaporkan (mode konektivitas)")
|
||||
return
|
||||
base = (setting("sitomat_base_url", DEFAULT_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
|
||||
api_key = setting("sitomat_api_key")
|
||||
if not api_key:
|
||||
print("[sitomat] sitomat_api_key tidak diatur — kegagalan TIDAK dilaporkan")
|
||||
return
|
||||
cause = f"DAG-3 sitomat_gold_build_prod failed (run {context['dag_run'].run_id})"
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{base}/data-products/builds/{build_id}/fail",
|
||||
json={"cause": cause},
|
||||
headers={"X-API-Key": api_key},
|
||||
timeout=60,
|
||||
)
|
||||
print(f"[sitomat] failure callback -> HTTP {response.status_code}")
|
||||
except Exception as error: # noqa: BLE001 - never mask the original failure
|
||||
print(f"[sitomat] failure callback GAGAL (diabaikan): {error}")
|
||||
|
||||
built = build_gold()
|
||||
registered = register_gold(built)
|
||||
# `one_failed` + kedua upstream: lapor bila SALAH SATU task gagal.
|
||||
[built, registered] >> report_failure()
|
||||
|
||||
|
||||
sitomat_gold_build_prod()
|
||||
@@ -0,0 +1,213 @@
|
||||
"""
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Contoh DAG Pipeline End-to-End
|
||||
#
|
||||
# Referensi arsitektur:
|
||||
# bab 5.4 (Ingestion → Processing: Pipeline Integration)
|
||||
# bab 8.1 (Skenario End-to-End: Batch Harian)
|
||||
# bab 8.2 (Skenario End-to-End: CDC Real-Time)
|
||||
# =============================================================================
|
||||
|
||||
Alur:
|
||||
1. [Ingestion] Sling tarik data dari PostgreSQL → Bronze (Iceberg via Nessie)
|
||||
2. [Processing] dbt via Trino: Bronze → Silver → Gold
|
||||
3. [Governance] Great Expectations validasi output pipeline DI ICEBERG
|
||||
|
||||
Prerequisites (dibuat oleh bootstrap.sh):
|
||||
- Airflow connection: minio_s3 (object storage)
|
||||
- Sling env: ~/.sling/env.yaml (koneksi POSTGRES_DB + ICEBERG)
|
||||
- dbt project: /opt/airflow/dbt/sicabe/ (bind-mount dari config/airflow/dbt) dengan model bronze_to_silver, silver_to_gold, incremental
|
||||
- Tabel sumber: public.orders di database sicabe_data (PostgreSQL)
|
||||
- Kredensial Trino service account: env DBT_TRINO_USER / DBT_TRINO_PASSWORD
|
||||
|
||||
Catatan penting (jangan diubah tanpa alasan):
|
||||
- Task dbt memakai `env=DBT_TASK_ENV, append_env=True`. Tanpa `append_env=True`,
|
||||
`env=` MENGGANTI seluruh environment sehingga PATH & HOME hilang dan `dbt`
|
||||
(terpasang di ~/.local/bin) tidak ditemukan: "dbt: command not found".
|
||||
- Validator Great Expectations membaca tabel Iceberg LEWAT TRINO (HTTPS 8443,
|
||||
service account), bukan tabel sumber di PostgreSQL — supaya hasil "LULUS"
|
||||
benar-benar membuktikan output pipeline (bronze/gold), bukan data mentah.
|
||||
GE tidak punya datasource Trino → dipakai Pandas datasource dari hasil query.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import os
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.providers.standard.operators.bash import BashOperator
|
||||
from airflow.providers.standard.operators.python import PythonOperator
|
||||
|
||||
|
||||
def _run_ge_to_om(table):
|
||||
"""Jalankan validasi Great Expectations DAN kirim hasilnya ke OpenMetadata.
|
||||
|
||||
Runner adalah SATU sumber logika validasi (dipakai juga di luar DAG) dan
|
||||
memakai action resmi OpenMetadata untuk GE 1.x, sehingga hasilnya muncul
|
||||
sebagai Data Quality Test Suite/Test Case di OpenMetadata — TERMASUK ketika
|
||||
GAGAL. Sebelumnya hasil GE hanya berhenti di log Airflow.
|
||||
|
||||
Fail-fast tetap dipertahankan: exit code non-nol → task FAILED.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
script = os.getenv(
|
||||
"GE_OM_RUNNER", "/opt/airflow/great_expectations/validate_to_openmetadata.py"
|
||||
)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, script, "--table", table], capture_output=True, text=True
|
||||
)
|
||||
for stream in (proc.stdout, proc.stderr):
|
||||
if stream:
|
||||
print(stream[-2000:])
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Data quality check GAGAL untuk {table} (hasil tetap dikirim ke OpenMetadata)"
|
||||
)
|
||||
|
||||
|
||||
def validate_gold_data():
|
||||
"""Validasi GOLD (output akhir pipeline) — `gold.orders_gold` → hasil ke OM."""
|
||||
_run_ge_to_om("gold.orders_gold")
|
||||
|
||||
|
||||
def validate_bronze_data():
|
||||
"""Validasi BRONZE (hasil ingestion Sling) — `bronze.orders` → hasil ke OM."""
|
||||
_run_ge_to_om("bronze.orders")
|
||||
|
||||
|
||||
# Env untuk task dbt: autentikasi Trino via SERVICE ACCOUNT (HTTP Basic) langsung
|
||||
# ke HTTPS 8443 (lihat config/airflow/dbt/profiles.yml). Password dari env
|
||||
# DBT_TRINO_PASSWORD (di-set docker-compose untuk service Airflow).
|
||||
DBT_TASK_ENV = {
|
||||
"DBT_TRINO_USER": os.getenv("DBT_TRINO_USER", "svc_dbt"),
|
||||
"DBT_TRINO_PASSWORD": os.getenv("DBT_TRINO_PASSWORD", ""),
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Default Arguments
|
||||
# =============================================================================
|
||||
default_args = {
|
||||
"owner": "engineer",
|
||||
"depends_on_past": False,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 2,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
"execution_timeout": timedelta(hours=2),
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# DAG — Batch Harian (bab 8.1)
|
||||
# =============================================================================
|
||||
BATCH_DAG_ID = "sicabe_batch_daily"
|
||||
BATCH_SCHEDULE = "0 2 * * *" # Setiap hari jam 02:00
|
||||
|
||||
with DAG(
|
||||
dag_id=BATCH_DAG_ID,
|
||||
default_args=default_args,
|
||||
description="SiCABE: Batch harian — Ingestion → Processing → Governance",
|
||||
schedule=BATCH_SCHEDULE,
|
||||
start_date=datetime(2026, 1, 1),
|
||||
catchup=False,
|
||||
tags=["sicabe", "batch", "bronze", "silver", "gold"],
|
||||
) as dag:
|
||||
|
||||
# ─── Step 1: Ingestion — Tarik data ke Bronze (Iceberg) ──────────────
|
||||
ingest_to_bronze = BashOperator(
|
||||
task_id="ingest_to_bronze",
|
||||
bash_command=(
|
||||
'sling run --src-conn POSTGRES_DB '
|
||||
'--src-stream public.orders '
|
||||
'--tgt-conn ICEBERG '
|
||||
'--tgt-object bronze.orders '
|
||||
'--mode full-refresh '
|
||||
'--stdout'
|
||||
),
|
||||
)
|
||||
|
||||
# ─── Step 2: Processing — Transform Bronze → Silver ──────────────────
|
||||
bronze_to_silver = BashOperator(
|
||||
task_id="bronze_to_silver",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select bronze_to_silver --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 3: Processing — Transform Silver → Gold ────────────────────
|
||||
silver_to_gold = BashOperator(
|
||||
task_id="silver_to_gold",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select silver_to_gold --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 4: Governance — Validasi GOLD di Iceberg ───────────────────
|
||||
validate_gold = PythonOperator(
|
||||
task_id="validate_gold",
|
||||
python_callable=validate_gold_data,
|
||||
)
|
||||
|
||||
# ─── Dependency Chain ────────────────────────────────────────────────
|
||||
ingest_to_bronze >> bronze_to_silver >> silver_to_gold >> validate_gold
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# DAG — CDC Real-Time (bab 8.2)
|
||||
# =============================================================================
|
||||
CDC_DAG_ID = "sicabe_cdc_realtime"
|
||||
CDC_SCHEDULE = "*/15 * * * *" # Setiap 15 menit
|
||||
|
||||
with DAG(
|
||||
dag_id=CDC_DAG_ID,
|
||||
default_args=default_args,
|
||||
description="SiCABE: CDC real-time — Incremental ingestion + processing",
|
||||
schedule=CDC_SCHEDULE,
|
||||
start_date=datetime(2026, 1, 1),
|
||||
catchup=False,
|
||||
tags=["sicabe", "cdc", "incremental", "realtime"],
|
||||
) as dag:
|
||||
|
||||
# ─── Step 1: CDC Ingestion via Sling ─────────────────────────────────
|
||||
cdc_ingest = BashOperator(
|
||||
task_id="cdc_ingest",
|
||||
bash_command=(
|
||||
'sling run --src-conn POSTGRES_DB '
|
||||
'--src-stream public.orders '
|
||||
'--tgt-conn ICEBERG '
|
||||
'--tgt-object bronze.orders '
|
||||
'--mode incremental '
|
||||
'--primary-key id '
|
||||
'--update-key updated_at '
|
||||
'--stdout'
|
||||
),
|
||||
)
|
||||
|
||||
# ─── Step 2: dbt incremental update ──────────────────────────────────
|
||||
dbt_incremental = BashOperator(
|
||||
task_id="dbt_incremental",
|
||||
bash_command=(
|
||||
'cd /opt/airflow/dbt/sicabe && '
|
||||
'export DBT_PROFILES_DIR=/opt/airflow/dbt && '
|
||||
'dbt run --select incremental --profile trino --target prod'
|
||||
),
|
||||
env=DBT_TASK_ENV,
|
||||
append_env=True,
|
||||
)
|
||||
|
||||
# ─── Step 3: Validasi BRONZE di Iceberg ──────────────────────────────
|
||||
cdc_validate = PythonOperator(
|
||||
task_id="cdc_validate",
|
||||
python_callable=validate_bronze_data,
|
||||
)
|
||||
|
||||
cdc_ingest >> dbt_incremental >> cdc_validate
|
||||
@@ -0,0 +1,58 @@
|
||||
# =============================================================================
|
||||
# SAMPLE — Lineage otomatis via operator SQL (tanpa inlets/outlets)
|
||||
# =============================================================================
|
||||
# Tujuan : membuktikan bahwa task yang memakai operator SQL memperoleh lineage
|
||||
# tabel SECARA OTOMATIS dari teks SQL, tanpa perlu menuliskan
|
||||
# inlets/outlets.
|
||||
#
|
||||
# Alur lineage yang diharapkan:
|
||||
# iceberg.bronze.orders ---> iceberg.silver.orders_clean
|
||||
#
|
||||
# Cara kerja:
|
||||
# Provider OpenLineage mem-parsing SQL task (lihat
|
||||
# airflow.providers.openlineage.utils.sql_hook_lineage) dan mengekstrak
|
||||
# tabel input/output, lalu mengirimnya ke OpenMetadata.
|
||||
#
|
||||
# Prasyarat di Airflow:
|
||||
# 1. Koneksi `trino_ol` (type trino) menunjuk ke Trino koordinator.
|
||||
# 2. Transport OpenLineage aktif + shim eventTime (lihat shim/ol_time_shim.py).
|
||||
# 3. AIRFLOW__OPENLINEAGE__EXECUTE_IN_THREAD=true (tanpa ini emisi HANG).
|
||||
#
|
||||
# Catatan: DAG ini tidak memakai inlets/outlets secara sengaja.
|
||||
# =============================================================================
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator
|
||||
|
||||
CONN_ID = "trino_ol"
|
||||
|
||||
CLEAN_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS iceberg.silver.orders_clean AS
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region
|
||||
FROM iceberg.bronze.orders
|
||||
WHERE id IS NOT NULL
|
||||
"""
|
||||
|
||||
with DAG(
|
||||
dag_id="sample_sql_lineage",
|
||||
description="SAMPLE: lineage otomatis via operator SQL (bronze.orders -> silver.orders_clean)",
|
||||
start_date=datetime(2026, 1, 1),
|
||||
schedule=None, # dijalankan manual
|
||||
catchup=False,
|
||||
tags=["sample", "lineage", "openlineage", "sql"],
|
||||
) as dag:
|
||||
|
||||
# Lineage bronze.orders -> silver.orders_clean diekstrak otomatis dari SQL ini.
|
||||
clean_orders = SQLExecuteQueryOperator(
|
||||
task_id="clean_orders",
|
||||
conn_id=CONN_ID,
|
||||
sql=CLEAN_SQL,
|
||||
)
|
||||
@@ -0,0 +1,9 @@
|
||||
"""
|
||||
This file has been generated from dag_runner.j2
|
||||
"""
|
||||
from airflow import DAG
|
||||
from openmetadata_managed_apis.workflows import workflow_factory
|
||||
|
||||
workflow = workflow_factory.WorkflowFactory.create("/opt/airflow/dag_generated_configs/sicabe_dq_gold_native.json")
|
||||
workflow.generate_dag(globals())
|
||||
dag = workflow.get_dag()
|
||||
@@ -0,0 +1,39 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Profiles (Trino Connection)
|
||||
# =============================================================================
|
||||
# Service/pipeline memakai SERVICE ACCOUNT via HTTP Basic (dbt-trino method
|
||||
# 'ldap' = username/password Basic). Koneksi LANGSUNG ke Trino HTTPS
|
||||
# (self-signed) → cert: false (lewati verifikasi TLS) untuk dev.
|
||||
# Password dari env DBT_TRINO_PASSWORD.
|
||||
# =============================================================================
|
||||
|
||||
trino:
|
||||
target: prod
|
||||
outputs:
|
||||
prod:
|
||||
type: trino
|
||||
method: ldap
|
||||
user: "{{ env_var('DBT_TRINO_USER', 'svc_dbt') }}"
|
||||
password: "{{ env_var('DBT_TRINO_PASSWORD', '') }}"
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
host: "{{ env_var('DBT_TRINO_HOST', 'trino-coordinator') }}"
|
||||
port: 8443
|
||||
threads: 4
|
||||
http_scheme: https
|
||||
cert: false
|
||||
suppress_cert_warning: true
|
||||
|
||||
dev:
|
||||
type: trino
|
||||
method: ldap
|
||||
user: "{{ env_var('DBT_TRINO_USER', 'svc_dbt') }}"
|
||||
password: "{{ env_var('DBT_TRINO_PASSWORD', '') }}"
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
host: "{{ env_var('DBT_TRINO_HOST', 'localhost') }}"
|
||||
port: 12443
|
||||
threads: 2
|
||||
http_scheme: https
|
||||
cert: false
|
||||
suppress_cert_warning: true
|
||||
@@ -0,0 +1,56 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Project Configuration
|
||||
# =============================================================================
|
||||
# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
|
||||
# Pipeline: Bronze → Silver → Gold
|
||||
# =============================================================================
|
||||
|
||||
name: "sicabe"
|
||||
version: "1.0.0"
|
||||
config-version: 2
|
||||
require-dbt-version: [">=1.8.0", "<2.0.0"]
|
||||
|
||||
profile: "trino"
|
||||
|
||||
model-paths: ["models"]
|
||||
analysis-paths: ["analyses"]
|
||||
test-paths: ["tests"]
|
||||
seed-paths: ["seeds"]
|
||||
macro-paths: ["macros"]
|
||||
snapshot-paths: ["snapshots"]
|
||||
|
||||
target-path: "target"
|
||||
clean-targets:
|
||||
- "target"
|
||||
- "dbt_packages"
|
||||
|
||||
models:
|
||||
sicabe:
|
||||
bronze_to_silver:
|
||||
+materialized: table
|
||||
+schema: silver
|
||||
+database: iceberg
|
||||
silver_to_gold:
|
||||
+materialized: table
|
||||
+schema: gold
|
||||
+database: iceberg
|
||||
incremental:
|
||||
+materialized: incremental
|
||||
+schema: bronze
|
||||
+database: iceberg
|
||||
+unique_key: "id"
|
||||
+incremental_strategy: merge
|
||||
# ───────────────────────────────────────────────────────────────────────
|
||||
# WAJIB: relasi sementara harus berupa TABEL, bukan VIEW.
|
||||
#
|
||||
# Katalog Iceberg Trino di stack ini bertipe `nessie`, dan Trino TIDAK
|
||||
# mendukung view di sana. Tanpa setelan ini, setiap run incremental gagal:
|
||||
# TrinoUserError(NOT_SUPPORTED):
|
||||
# "createView is not supported for Iceberg Nessie catalogs"
|
||||
#
|
||||
# Sebabnya: untuk strategi `merge`/`append`, dbt-trino membuat relasi
|
||||
# sementara sebagai VIEW. `views_enabled: false` membuatnya memakai tabel.
|
||||
# Dibuktikan 2026-09-22: MERGE (1 rows) berhasil untuk baris baru maupun
|
||||
# baris yang diubah (upsert). Lihat reviews/sicabe-15-transform-service.md.
|
||||
# ───────────────────────────────────────────────────────────────────────
|
||||
+views_enabled: false
|
||||
@@ -0,0 +1,15 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Macro: generate_schema_name
|
||||
-- =============================================================================
|
||||
-- Override default schema naming agar +schema menghasilkan nama persis
|
||||
-- (bukan {default}_{custom}). Contoh: +schema: gold → schema "gold"
|
||||
-- =============================================================================
|
||||
|
||||
{% macro generate_schema_name(custom_schema_name, node) -%}
|
||||
{%- set default_schema = target.schema -%}
|
||||
{%- if custom_schema_name is none -%}
|
||||
{{ default_schema }}
|
||||
{%- else -%}
|
||||
{{ custom_schema_name | trim }}
|
||||
{%- endif -%}
|
||||
{%- endmacro %}
|
||||
@@ -0,0 +1,36 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Bronze → Silver
|
||||
-- =============================================================================
|
||||
-- Transformasi data mentah (bronze.orders) menjadi data bersih (silver.orders):
|
||||
-- - Normalisasi tipe data
|
||||
-- - Filter data valid (harga > 0, stok >= 0)
|
||||
-- - Standarisasi format tanggal
|
||||
-- =============================================================================
|
||||
|
||||
WITH bronze_orders AS (
|
||||
SELECT
|
||||
id,
|
||||
CAST(order_date AS DATE) AS order_date,
|
||||
komoditas,
|
||||
CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
|
||||
CAST(stok_kg AS BIGINT) AS stok_kg,
|
||||
region,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM {{ source('iceberg', 'orders') }}
|
||||
)
|
||||
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region,
|
||||
-- Nilai total per transaksi
|
||||
ROUND(harga_per_kg * stok_kg, 2) AS total_value,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM bronze_orders
|
||||
WHERE harga_per_kg > 0
|
||||
AND stok_kg >= 0
|
||||
@@ -0,0 +1,29 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Incremental (CDC)
|
||||
-- =============================================================================
|
||||
-- Incremental update untuk data bronze.orders:
|
||||
-- - Hanya memproses baris baru/berubah (berdasarkan updated_at)
|
||||
-- - Materialized: incremental dengan unique_key = id
|
||||
-- =============================================================================
|
||||
|
||||
{{ config(
|
||||
materialized='incremental',
|
||||
unique_key='id',
|
||||
incremental_strategy='merge'
|
||||
) }}
|
||||
|
||||
SELECT
|
||||
id,
|
||||
CAST(order_date AS DATE) AS order_date,
|
||||
komoditas,
|
||||
CAST(harga_per_kg AS DOUBLE) AS harga_per_kg,
|
||||
CAST(stok_kg AS BIGINT) AS stok_kg,
|
||||
region,
|
||||
created_at,
|
||||
updated_at
|
||||
FROM {{ source('iceberg', 'orders') }}
|
||||
|
||||
{% if is_incremental() %}
|
||||
-- Hanya ambil data yang baru/berubah sejak run terakhir
|
||||
WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }})
|
||||
{% endif %}
|
||||
@@ -0,0 +1,35 @@
|
||||
-- =============================================================================
|
||||
-- eBdesk SiCABE — dbt Model: Silver → Gold
|
||||
-- =============================================================================
|
||||
-- Agregasi data bersih (silver.orders) menjadi data analitik (gold.orders):
|
||||
-- - Rata-rata harga per komoditas per hari
|
||||
-- - Total stok per komoditas
|
||||
-- - Ranking komoditas berdasarkan nilai transaksi
|
||||
-- =============================================================================
|
||||
|
||||
WITH silver_orders AS (
|
||||
SELECT
|
||||
id,
|
||||
order_date,
|
||||
komoditas,
|
||||
harga_per_kg,
|
||||
stok_kg,
|
||||
region,
|
||||
total_value
|
||||
FROM {{ ref('orders_silver') }}
|
||||
)
|
||||
|
||||
SELECT
|
||||
order_date,
|
||||
komoditas,
|
||||
region,
|
||||
COUNT(*) AS jumlah_transaksi,
|
||||
ROUND(AVG(harga_per_kg), 2) AS avg_harga_per_kg,
|
||||
ROUND(MIN(harga_per_kg), 2) AS min_harga_per_kg,
|
||||
ROUND(MAX(harga_per_kg), 2) AS max_harga_per_kg,
|
||||
SUM(stok_kg) AS total_stok_kg,
|
||||
ROUND(SUM(total_value), 2) AS total_nilai,
|
||||
-- Harga rata-rata tertimbang (weighted average)
|
||||
ROUND(SUM(harga_per_kg * stok_kg) / NULLIF(SUM(stok_kg), 0), 2) AS weighted_avg_harga
|
||||
FROM silver_orders
|
||||
GROUP BY order_date, komoditas, region
|
||||
@@ -0,0 +1,32 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dbt Sources
|
||||
# =============================================================================
|
||||
# Mendefinisikan sumber data di Iceberg (via Trino catalog)
|
||||
# =============================================================================
|
||||
|
||||
version: 2
|
||||
|
||||
sources:
|
||||
- name: iceberg
|
||||
database: iceberg
|
||||
schema: bronze
|
||||
tables:
|
||||
- name: orders
|
||||
description: "Data mentah orders dari PostgreSQL (via Sling)"
|
||||
columns:
|
||||
- name: id
|
||||
description: "Primary key"
|
||||
- name: order_date
|
||||
description: "Tanggal order"
|
||||
- name: komoditas
|
||||
description: "Nama komoditas"
|
||||
- name: harga_per_kg
|
||||
description: "Harga per kilogram"
|
||||
- name: stok_kg
|
||||
description: "Stok dalam kilogram"
|
||||
- name: region
|
||||
description: "Wilayah"
|
||||
- name: created_at
|
||||
description: "Waktu dibuat"
|
||||
- name: updated_at
|
||||
description: "Waktu diupdate"
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "cdc_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "cdc-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "bronze.orders"
|
||||
},
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"name": "gold_quality_checkpoint",
|
||||
"config_version": 4.0,
|
||||
"template_name": null,
|
||||
"run_name_template": "gold-quality-%Y%m%d-%H%M%S",
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders",
|
||||
"batch_spec_passthrough": {}
|
||||
},
|
||||
"action_list": [
|
||||
{
|
||||
"name": "store_validation_result",
|
||||
"action": {
|
||||
"class_name": "StoreValidationResultAction"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "update_data_docs",
|
||||
"action": {
|
||||
"class_name": "UpdateDataDocsAction"
|
||||
}
|
||||
}
|
||||
],
|
||||
"evaluation_parameters": {},
|
||||
"runtime_configuration": {},
|
||||
"validations": [
|
||||
{
|
||||
"batch_request": {
|
||||
"datasource_name": "trino_iceberg",
|
||||
"data_connector_name": "default_inferred_data_connector_name",
|
||||
"data_asset_name": "gold.orders"
|
||||
},
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"action_list": []
|
||||
}
|
||||
],
|
||||
"profilers": [],
|
||||
"ge_cloud_id": null,
|
||||
"expectation_suite_ge_cloud_id": null
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"expectation_suite_name": "bronze_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "id"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"expectation_suite_name": "gold_orders_suite",
|
||||
"ge_cloud_id": null,
|
||||
"meta": {
|
||||
"created_by": "sicabe-bootstrap",
|
||||
"created_at": "2026-08-31"
|
||||
},
|
||||
"expectations": [
|
||||
{
|
||||
"expectation_type": "expect_table_row_count_to_be_between",
|
||||
"kwargs": {
|
||||
"min_value": 1,
|
||||
"max_value": 100000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_not_be_null",
|
||||
"kwargs": {
|
||||
"column": "komoditas"
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "harga_per_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 1000000
|
||||
},
|
||||
"meta": {}
|
||||
},
|
||||
{
|
||||
"expectation_type": "expect_column_values_to_be_between",
|
||||
"kwargs": {
|
||||
"column": "stok_kg",
|
||||
"min_value": 0,
|
||||
"max_value": 100000000
|
||||
},
|
||||
"meta": {}
|
||||
}
|
||||
],
|
||||
"data_asset_type": null
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Great Expectations Configuration
|
||||
# =============================================================================
|
||||
# Referensi arsitektur: bab 8.1 (Batch Harian), bab 8.2 (CDC Real-Time)
|
||||
# Validasi kualitas data di layer Gold sebelum dikonsumsi aplikasi.
|
||||
# =============================================================================
|
||||
|
||||
config_version: 4.0
|
||||
|
||||
datasources:
|
||||
trino_iceberg:
|
||||
class_name: Datasource
|
||||
execution_engine:
|
||||
class_name: SqlAlchemyExecutionEngine
|
||||
connection_string: "trino://{{ env_var('GE_TRINO_USER', 'admin') }}:{{ env_var('GE_TRINO_PASSWORD', '') }}@{{ env_var('GE_TRINO_HOST', 'trino-coordinator') }}:8080/iceberg/gold"
|
||||
data_connectors:
|
||||
default_inferred_data_connector_name:
|
||||
class_name: InferredAssetSqlDataConnector
|
||||
include_schema_name: true
|
||||
introspection_directives:
|
||||
schema_name: gold
|
||||
|
||||
expectations_store:
|
||||
class_name: ExpectationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: expectations/
|
||||
|
||||
validations_store:
|
||||
class_name: ValidationsStore
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: validations/
|
||||
|
||||
data_docs_sites:
|
||||
local:
|
||||
class_name: SiteBuilder
|
||||
show_how_to_buttons: true
|
||||
store_backend:
|
||||
class_name: TupleFilesystemStoreBackend
|
||||
base_directory: uncommitted/data_docs/local_site/
|
||||
site_index_builder:
|
||||
class_name: DefaultSiteIndexBuilder
|
||||
@@ -0,0 +1,18 @@
|
||||
# =============================================================================
|
||||
# Konfigurasi koneksi OpenMetadata untuk GE Action (dibaca oleh ometa_config_handler)
|
||||
# =============================================================================
|
||||
# Direktori ini berisi "config.yml" karena action membacanya sebagai TEMPLATE Jinja:
|
||||
# create_jinja_environment(<dir>) + render_template(env) → <dir>/config.yml
|
||||
# `env('VAR')` disediakan oleh handler, sehingga token TIDAK ditulis di repo.
|
||||
#
|
||||
# Dipakai oleh: metadata/great_expectations/action.py (OpenMetadata >= 2.0)
|
||||
# OpenMetadataValidationAction(config_file_path="/opt/airflow/great_expectations/om", ...)
|
||||
# Di OpenMetadata 1.13.x modul action-nya berbeda (generasi 1xx) dan dihapus di
|
||||
# 2.0; rincian migrasi ada di deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# Jalankan dengan OM_JWT_TOKEN tersedia di environment container.
|
||||
# =============================================================================
|
||||
hostPort: "{{ env('OM_API_URL') }}"
|
||||
authProvider: openmetadata
|
||||
securityConfig:
|
||||
jwtToken: "{{ env('OM_JWT_TOKEN') }}"
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Jalankan Great Expectations DAN kirim hasilnya ke OpenMetadata
|
||||
# =============================================================================
|
||||
# Jalur A (GE -> OpenMetadata): hasil GE tidak berhenti di log Airflow.
|
||||
# Action resmi OpenMetadata (metadata.great_expectations.action) dipasang pada
|
||||
# checkpoint GE 1.x, sehingga setiap kali checkpoint dijalankan, hasilnya dikirim
|
||||
# ke OpenMetadata dan muncul sebagai Data Quality Test Suite/Test Case dengan
|
||||
# platform "GreatExpectations".
|
||||
#
|
||||
# Catatan versi: OM 1.13.x memakai modul action generasi lama (kelas berakhiran
|
||||
# "1xx"); di OM 2.0 modul itu dihapus dan kelasnya kini bernama
|
||||
# OpenMetadataValidationAction di metadata.great_expectations.action.
|
||||
# Rincian migrasinya: deploy/output/merge-om20x-rollback.md.
|
||||
#
|
||||
# ⚠️ Sejak OM 2.0 action MENOLAK batch yang bukan dari database relasional:
|
||||
# `_get_checkpoint_batch_spec()` melempar ValueError bila batch_spec tidak
|
||||
# memuat kunci SQL (table_name/schema_name/query/temp_table_schema_name).
|
||||
# Karena itu ekspektasi dijalankan di Trino lewat DATASOURCE SQL — bukan atas
|
||||
# pandas dataframe — sehingga batch_spec memuat schema_name + table_name dan
|
||||
# action dapat menemukan tabel OpenMetadata-nya.
|
||||
#
|
||||
# Dijalankan DI DALAM container Airflow (butuh great_expectations + metadata):
|
||||
# docker exec -e OM_JWT_TOKEN=<token> -e DBT_TRINO_PASSWORD=<pw> \
|
||||
# sicabe-airflow-webserver python /opt/airflow/great_expectations/validate_to_openmetadata.py
|
||||
#
|
||||
# Exit code: 0 = semua ekspektasi lulus; 1 = ADA yang gagal (task Airflow akan
|
||||
# FAILED) — hasilnya TETAP terkirim ke OpenMetadata sebelum exit.
|
||||
# =============================================================================
|
||||
import os
|
||||
import sys
|
||||
|
||||
import great_expectations as gx
|
||||
import trino
|
||||
from great_expectations.core.expectation_suite import ExpectationSuite
|
||||
|
||||
from metadata.great_expectations.action import OpenMetadataValidationAction
|
||||
OM_CONFIG_DIR = os.getenv("OM_CONFIG_DIR", "/opt/airflow/great_expectations/om")
|
||||
# OM_API_URL = alamat OM dari DALAM jaringan container (bukan OPENMETADATA_URL yang
|
||||
# dipakai dari host). Dipakai juga oleh template config action (om/config.yml).
|
||||
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
|
||||
DATABASE_SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
|
||||
DATABASE_NAME = os.getenv("OM_DATABASE_NAME", "iceberg")
|
||||
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
|
||||
|
||||
# (schema, tabel, daftar ekspektasi)
|
||||
CHECKS = [
|
||||
("bronze", "orders", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="harga_per_kg", min_value=0, max_value=1000000),
|
||||
]),
|
||||
("silver", "orders_silver", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="id"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_value", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
("gold", "orders_gold", [
|
||||
gx.expectations.ExpectTableRowCountToBeBetween(min_value=1, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToNotBeNull(column="komoditas"),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="avg_harga_per_kg", min_value=0, max_value=1000000),
|
||||
gx.expectations.ExpectColumnValuesToBeBetween(
|
||||
column="total_nilai", min_value=0, max_value=1000000000000),
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
def _trino_connection():
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return trino.dbapi.connect(
|
||||
host=os.getenv("DBT_TRINO_HOST", "trino-coordinator"),
|
||||
port=int(os.getenv("DBT_TRINO_PORT", "8443")),
|
||||
http_scheme="https",
|
||||
user=user,
|
||||
catalog=DATABASE_NAME,
|
||||
auth=trino.auth.BasicAuthentication(user, password),
|
||||
verify=TRINO_CERT,
|
||||
)
|
||||
|
||||
|
||||
def _trino_sqlalchemy_url():
|
||||
"""URL SQLAlchemy untuk datasource SQL Great Expectations (tanpa password).
|
||||
|
||||
Password TIDAK ditulis di URL — dikirim lewat objek auth pada connect_args,
|
||||
sehingga tidak muncul di connection string, log, atau pesan error.
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
host = os.getenv("DBT_TRINO_HOST", "trino-coordinator")
|
||||
port = os.getenv("DBT_TRINO_PORT", "8443")
|
||||
return f"trino://{user}@{host}:{port}/{DATABASE_NAME}"
|
||||
|
||||
|
||||
def _trino_connect_args():
|
||||
"""Argumen koneksi untuk dialect SQLAlchemy Trino.
|
||||
|
||||
Penting: dialect Trino TIDAK membaca `http_scheme`/`verify` dari query URL —
|
||||
keduanya harus diberikan sebagai connect_args. `verify` juga harus berupa
|
||||
path sertifikat (bukan JSON string seperti bila lewat query URL).
|
||||
"""
|
||||
user = os.getenv("DBT_TRINO_USER", "svc_dbt")
|
||||
password = os.getenv("DBT_TRINO_PASSWORD")
|
||||
if not password:
|
||||
raise RuntimeError("DBT_TRINO_PASSWORD tidak diset")
|
||||
return {
|
||||
"http_scheme": "https",
|
||||
"verify": TRINO_CERT,
|
||||
"auth": trino.auth.BasicAuthentication(user, password),
|
||||
}
|
||||
|
||||
|
||||
def _assert_table_not_empty(schema, table):
|
||||
"""Fail-fast bila tabel kosong: tidak ada yang bermakna untuk divalidasi."""
|
||||
row = None
|
||||
with _trino_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(f"SELECT count(*) FROM {schema}.{table}")
|
||||
row = cursor.fetchone()
|
||||
if not row or not row[0]:
|
||||
raise RuntimeError(f"Tabel {schema}.{table} kosong — tidak ada yang divalidasi")
|
||||
|
||||
|
||||
def _build_checkpoint(context, schema, table, expectations):
|
||||
"""Satu checkpoint per tabel: suite + batch definition SQL + action OM."""
|
||||
_assert_table_not_empty(schema, table)
|
||||
|
||||
suite_name = f"{schema}_{table}_suite"
|
||||
data_source = context.data_sources.add_or_update_sql(
|
||||
name=f"trino_{schema}_{table}",
|
||||
connection_string=_trino_sqlalchemy_url(),
|
||||
kwargs={"connect_args": _trino_connect_args()},
|
||||
)
|
||||
asset = data_source.add_table_asset(name=table, table_name=table, schema_name=schema)
|
||||
batch_definition = asset.add_batch_definition_whole_table(f"bd_{schema}_{table}")
|
||||
|
||||
suite = ExpectationSuite(name=suite_name)
|
||||
for expectation in expectations:
|
||||
suite.add_expectation(expectation)
|
||||
context.suites.add_or_update(suite)
|
||||
|
||||
# ValidationDefinition memerlukan BatchDefinition (bukan Batch) + suite.
|
||||
validation_definition = context.validation_definitions.add(
|
||||
gx.ValidationDefinition(name=f"vd_{schema}_{table}", data=batch_definition, suite=suite)
|
||||
)
|
||||
|
||||
action = OpenMetadataValidationAction(
|
||||
config_file_path=OM_CONFIG_DIR,
|
||||
database_service_name=DATABASE_SERVICE,
|
||||
database_name=DATABASE_NAME,
|
||||
schema_name=schema,
|
||||
table_name=table,
|
||||
)
|
||||
checkpoint = context.checkpoints.add(
|
||||
gx.Checkpoint(
|
||||
name=f"cp_{schema}_{table}",
|
||||
validation_definitions=[validation_definition],
|
||||
actions=[action],
|
||||
)
|
||||
)
|
||||
# Batch "whole table" dijalankan di Trino (tanpa batch_parameters).
|
||||
return checkpoint
|
||||
|
||||
|
||||
def _ensure_om_token():
|
||||
"""Token OpenMetadata untuk action: pakai OM_JWT_TOKEN, atau login sendiri.
|
||||
|
||||
Login sendiri membuat skrip aman dipakai pada jadwal (DAG) — tidak bergantung
|
||||
pada token sementara yang ditempel di environment.
|
||||
"""
|
||||
if os.getenv("OM_JWT_TOKEN"):
|
||||
return True
|
||||
# Template config action membaca OM_API_URL → pastikan selalu terisi.
|
||||
os.environ.setdefault("OM_API_URL", OM_API_URL)
|
||||
email = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
|
||||
password = os.getenv("OPENMETADATA_ADMIN_PASSWORD")
|
||||
import base64
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
# DUA kandidat, MAKSIMAL DUA percobaan (jangan diulang dalam loop — akun admin bisa
|
||||
# terkunci): password .env, lalu default 'admin' yang dipakai OM 2.0.x sampai
|
||||
# bootstrap step 7 menulis hash bcrypt. Sama seperti ingest-openmetadata.sh.
|
||||
candidates = [c for c in dict.fromkeys([password or "", "admin"]) if c]
|
||||
err_last = ""
|
||||
for pw in candidates:
|
||||
body = json.dumps({"email": email,
|
||||
"password": base64.b64encode(pw.encode()).decode()}).encode()
|
||||
req = urllib.request.Request(f"{OM_API_URL}/v1/users/login", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
token = json.load(urllib.request.urlopen(req, timeout=20)).get("accessToken", "")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
err_last = str(exc)
|
||||
token = ""
|
||||
if token:
|
||||
os.environ["OM_JWT_TOKEN"] = token
|
||||
return True
|
||||
print(f"ERROR: login ke OpenMetadata gagal: {err_last or 'tanpa token'}")
|
||||
print(" dicoba: password .env + 'admin' — pastikan: ./bootstrap.sh --step 7")
|
||||
return False
|
||||
|
||||
|
||||
def _suite_exists_in_om(schema: str, table: str):
|
||||
"""Apakah test suite untuk tabel ini ADA di OpenMetadata?
|
||||
|
||||
Mengembalikan nama suite bila ada, False bila tidak, None bila tidak dapat
|
||||
diverifikasi.
|
||||
|
||||
Dua hal penting yang membuat versi pertama fungsi ini SALAH:
|
||||
1. `checkpoint.run()` hanya melaporkan status CHECKPOINT (validasi expectation),
|
||||
bukan status pengiriman action — jadi tidak boleh disimpulkan dari situ.
|
||||
2. Action OpenMetadataValidationAction menamai test suite menurut FQN TABEL:
|
||||
<service>.<database>.<schema>.<table>.testSuite
|
||||
BUKAN menurut nama ExpectationSuite GE (`<schema>_<table>_suite`). Memeriksa
|
||||
nama GE-nya menghasilkan false negative: DQ sudah ada di OM tapi dilaporkan
|
||||
tidak ada.
|
||||
"""
|
||||
token = os.environ.get("OM_JWT_TOKEN", "")
|
||||
if not token:
|
||||
return None
|
||||
import json
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
f"{OM_API_URL}/v1/dataQuality/testSuites?limit=200",
|
||||
headers={"Authorization": f"Bearer {token}"})
|
||||
try:
|
||||
d = json.load(urllib.request.urlopen(req, timeout=20))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[GE] verifikasi ke OM gagal: {exc}")
|
||||
return None
|
||||
names = {s.get("name") for s in (d.get("data") or [])}
|
||||
table_fqn = f"{DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}"
|
||||
for expected in (f"{table_fqn}.testSuite", table_fqn):
|
||||
if expected in names:
|
||||
return expected
|
||||
# toleran terhadap variasi penamaan: suite apa pun yang berawalan FQN tabel ini
|
||||
for n in names:
|
||||
if n and n.startswith(table_fqn):
|
||||
return n
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
only = None
|
||||
if len(sys.argv) > 2 and sys.argv[1] == "--table":
|
||||
only = sys.argv[2] # format: schema.table
|
||||
|
||||
if not _ensure_om_token():
|
||||
print("ERROR: token OpenMetadata tidak tersedia (set OM_JWT_TOKEN, atau "
|
||||
"OPENMETADATA_ADMIN_PASSWORD untuk login otomatis)")
|
||||
return 1
|
||||
|
||||
checks = [c for c in CHECKS if only is None or f"{c[0]}.{c[1]}" == only]
|
||||
if not checks:
|
||||
print(f"ERROR: tidak ada pemeriksaan untuk '{only}'")
|
||||
return 1
|
||||
|
||||
context = gx.get_context(mode="ephemeral")
|
||||
checkpoints = [_build_checkpoint(context, s, t, e) for s, t, e in checks]
|
||||
|
||||
failed = 0
|
||||
not_in_om = 0
|
||||
for (schema, table, _expectations), checkpoint in zip(checks, checkpoints):
|
||||
result = checkpoint.run()
|
||||
success = bool(getattr(result, "success", False))
|
||||
in_om = _suite_exists_in_om(schema, table)
|
||||
if in_om is None:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success}; "
|
||||
f"keberadaan di OpenMetadata TIDAK dapat diverifikasi")
|
||||
elif in_om:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"→ test suite '{in_om}' ADA di OpenMetadata")
|
||||
else:
|
||||
print(f"[GE] {checkpoint.name}: checkpoint success={success} "
|
||||
f"TAPI belum ada test suite untuk {DATABASE_SERVICE}.{DATABASE_NAME}.{schema}.{table}")
|
||||
print("[GE] penyebab tersering: entitas TABEL-nya belum ada di katalog OM.")
|
||||
print("[GE] urutan yang benar: ./scripts/ingest-openmetadata.sh --all DULU,")
|
||||
print("[GE] baru DAG -- karena action GE menempel ke entitas tabel di OM.")
|
||||
not_in_om += 1
|
||||
if not success:
|
||||
failed += 1
|
||||
|
||||
total = len(checkpoints)
|
||||
print(f"[GE] ringkasan: {total - failed}/{total} checkpoint lulus"
|
||||
+ (f"; {not_in_om} suite TIDAK ADA di OpenMetadata" if not_in_om else ""))
|
||||
# PENTING: `success` hanyalah status CHECKPOINT, bukan status pengiriman. Pesan lama
|
||||
# ("success=True → hasil dikirim ke OpenMetadata") menyesatkan persis karena itu.
|
||||
# Suite yang tidak ada di OM juga membuat task gagal, supaya penyebabnya terlihat
|
||||
# di sini alih-alih muncul sebagai kegagalan gate beberapa langkah kemudian.
|
||||
return 1 if (failed or not_in_om) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user