platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
subroto
2026-10-09 02:35:14 +07:00
commit 4ca7a9a58e
108 changed files with 23035 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/scripts/airflow/entrypoint.sh
Executable
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — Airflow Entrypoint Wrapper
# =============================================================================
# 1. Install sling CLI (binary, not available via pip)
# 2. Delegate ke Airflow entrypoint asli untuk service start
# =============================================================================
set -euo pipefail
# ─── Percayai sertifikat TLS Trino (dev, self-signed) ───────────────────────
# Sebagian klien MEMVERIFIKASI TLS dan tidak punya opsi mematikannya — contohnya
# Great Expectations / trino client (`verify=True`) dan DQ native OpenMetadata.
# Alih-alih `append` ke certifi (dgn guard marker) — yang MEMBUAT certifi tetap
# memegang cert LAMA setelah cert Trino di-regenerasi — kita bangun ULANG bundle
# CA (certifi + cert Trino) setiap start, lalu arahkan REQUESTS_CA_BUNDLE /
# SSL_CERT_FILE ke sana. Verifikasi TLS TETAP aktif.
CA_SRC="/etc/ssl/trino/trino.crt"
SICABE_CA="/tmp/sicabe-ca.pem"
if [[ -f "${CA_SRC}" ]]; then
CERTIFI_BUNDLE="$(python -c 'import certifi;print(certifi.where())' 2>/dev/null || true)"
if [[ -n "${CERTIFI_BUNDLE}" && -f "${CERTIFI_BUNDLE}" ]]; then
if cat "${CERTIFI_BUNDLE}" "${CA_SRC}" > "${SICABE_CA}" 2>/dev/null; then
export REQUESTS_CA_BUNDLE="${SICABE_CA}"
export SSL_CERT_FILE="${SICABE_CA}"
echo "[SiCABE] CA bundle (certifi + Trino) → ${SICABE_CA} (selalu dibangun ulang)."
else
echo "[SiCABE] WARNING: gagal membangun CA bundle — lanjut."
fi
fi
fi
# ─── Install sling CLI jika belum ada (NON-BLOCKING, fail-fast) ─────────────
# Install ke ~/.local/bin (writable oleh user airflow, sudah di PATH)
if ! command -v sling &>/dev/null; then
echo "[SiCABE] Installing sling CLI (timeout 15s)..."
SLING_VER="1.5.23"
mkdir -p "${HOME}/.local/bin"
if curl -fL --max-time 15 --connect-timeout 5 \
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VER}/sling_linux_amd64.tar.gz" \
-o /tmp/sling.tar.gz 2>/dev/null && tar -xzf /tmp/sling.tar.gz -C /tmp/ 2>/dev/null && [[ -f /tmp/sling ]]; then
chmod +x /tmp/sling
mv /tmp/sling "${HOME}/.local/bin/sling"
echo "[SiCABE] Sling CLI v${SLING_VER} installed: $(sling --version 2>&1)"
else
echo "[SiCABE] WARNING: sling CLI download gagal — lanjut tanpa sling (DAG akan warning)."
fi
rm -rf /tmp/sling.tar.gz /tmp/sling 2>/dev/null || true
fi
# ─── Copy dbt project ke home airflow (writable) ────────────────────────────
# Mount /opt/airflow/dbt adalah read-only (host bind). dbt perlu menulis
# target/, logs/, dbt_packages/. Copy ke ~/dbt yang writable oleh airflow.
if [[ -d /opt/airflow/dbt/sicabe && ! -d "${HOME}/dbt/sicabe" ]]; then
echo "[SiCABE] Copying dbt project ke ${HOME}/dbt..."
mkdir -p "${HOME}/dbt"
cp -r /opt/airflow/dbt/sicabe "${HOME}/dbt/sicabe"
cp /opt/airflow/dbt/profiles.yml "${HOME}/dbt/profiles.yml"
echo "[SiCABE] dbt project siap di ${HOME}/dbt/sicabe"
fi
# ─── Copy Great Expectations ke home airflow (writable) ─────────────────────
if [[ -d /opt/airflow/great_expectations && ! -d "${HOME}/great_expectations" ]]; then
echo "[SiCABE] Copying Great Expectations ke ${HOME}/great_expectations..."
cp -r /opt/airflow/great_expectations "${HOME}/great_expectations"
echo "[SiCABE] Great Expectations siap di ${HOME}/great_expectations"
fi
# ─── LINK sling env.yaml ke ~/.sling (SELALU ikut file yang di-mount) ───────
# WAJIB symlink, bukan copy: bootstrap men-regenerasi /opt/airflow/sling/env.yaml
# SETELAH container start (host-aware), sehingga salinan hasil `cp` menjadi STALE
# (gejala: Sling "lookup postgres ... server misbehaving"). Symlink → perubahan
# langsung terbaca tanpa restart. ~/.sling/ tetap writable untuk .sling.db.
# NON-FATAL: jangan sampai crash-loop container (set -e).
# Dibuat TANPA syarat berkas ada: bila env.yaml belum ada saat start (bootstrap jalan
# SETELAH container), symlink "menggantung" lalu VALID begitu berkas muncul. Dulu ada
# guard `[[ -f … ]]` → symlink tak dibuat → Sling "could not find connection POSTGRES_DB".
mkdir -p "${HOME}/.sling"
rm -f "${HOME}/.sling/env.yaml" 2>/dev/null || true
ln -s /opt/airflow/sling/env.yaml "${HOME}/.sling/env.yaml" \
&& echo "[SiCABE] Sling env.yaml -> symlink ${HOME}/.sling/env.yaml" \
|| echo "[SiCABE] WARNING: gagal symlink sling env.yaml — lanjut."
# ─── dbt-trino http_scheme ──────────────────────────────────────────────────
# Sejak image di-build dari paket resmi (tanpa patch), http_scheme sudah HTTPS
# lewat config/airflow/dbt/profiles.yml — tidak ada patch yang perlu dibalik.
# ─── Delegasikan ke entrypoint asli Airflow ─────────────────────────────────
# Airflow official entrypoint ada di /entrypoint
if [[ -x /entrypoint ]]; then
exec /entrypoint "$@"
elif [[ -x /usr/bin/docker-entrypoint.sh ]]; then
exec /usr/bin/docker-entrypoint.sh "$@"
else
echo "[SiCABE] WARNING: Airflow entrypoint tidak ditemukan, menjalankan langsung..."
exec "$@"
fi
deploy/scripts/airflow/run-om-native-dq.py
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python
# =============================================================================
# eBdesk SiCABE — JALUR B: Data Quality NATIVE OpenMetadata (OM yang menjalankan)
# =============================================================================
# Bedanya dengan Jalur A (Great Expectations):
# A: GE yang MENJALANKAN ekspektasi → hasilnya DIKIRIM ke OM
# (test case ber-platform "GreatExpectations").
# B: OPENMETADATA yang MENJALANKAN test (definisi bawaan OM:
# tableRowCountToBeBetween, columnValuesToBeNotNull, columnValuesToBeBetween)
# → hasilnya langsung tersimpan di OM (platform "OpenMetadata").
#
# Catatan implementasi (dua hal yang harus begini, sudah diverifikasi):
# 1. HARUS lewat `TestRunner.for_table`/`from_yaml` — `TestRunner(...)` mentah
# tidak memuat entity tabel sehingga run() gagal "Table entity not provided".
# 2. Koneksi data HARUS dibawa eksplisit lewat workflow (serviceConnection +
# verify=<sertifikat Trino>). Bila mengandalkan koneksi yang TERSIMPAN di OM,
# eksekusi gagal 401 'Invalid credentials' pada OM 1.13.1. `metadata ingest`
# CLI juga bukan jalur eksekusi TestSuite (registry source-nya terpisah).
#
# Jalankan DI DALAM container Airflow:
# docker exec -e OPENMETADATA_ADMIN_PASSWORD=... \
# sicabe-airflow-webserver python /opt/airflow/scripts/run-om-native-dq.py
#
# Exit code: 0 = semua test lulus; 1 = ada test gagal/aborted.
# =============================================================================
import base64
import json
import os
import sys
import urllib.request
from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
OpenMetadataConnection,
)
from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
OpenMetadataJWTClientConfig,
)
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.sdk.data_quality.runner import TestRunner
from metadata.sdk.data_quality.tests import (
ColumnValuesToBeBetween,
ColumnValuesToBeNotNull,
TableRowCountToBeBetween,
)
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
CATALOG = os.getenv("OM_DATABASE_NAME", "iceberg")
TRINO_HOSTPORT = os.getenv("OM_TRINO_HOSTPORT", "trino-coordinator:8443")
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
# Workflow TestSuite: membawa koneksi Trino eksplisit (username + password +
# verify=sertifikat) supaya eksekusi terautentikasi dan TLS tetap DIVERIFIKASI.
WORKFLOW_TEMPLATE = """
source:
type: TestSuite
serviceName: {service}
sourceConfig:
config:
type: TestSuite
entityFullyQualifiedName: {table}
# WAJIB 'serviceConnections' (plural, list) — DQ source membaca field ini via
# _load_yaml_service_connections(). Memakai 'serviceConnection' (tunggal)
# diabaikan, sehingga eksekusi jatuh ke koneksi TERSIMPAN di OM yang
# kehilangan secret → HTTP 401 'Invalid credentials' (lihat TODO upstream
# open-metadata#21259). Dengan bentuk ini koneksi dari YAML yang dipakai.
serviceConnections:
- serviceName: {service}
serviceConnection:
config:
type: Trino
scheme: trino
hostPort: {hostport}
username: svc_om
authType:
password: "{password}"
catalog: {catalog}
verify: {cert}
sink:
type: metadata-rest
config: {{}}
workflowConfig:
loggerLevel: INFO
openMetadataServerConfig:
hostPort: {om_api}
authProvider: openmetadata
securityConfig:
jwtToken: "{token}"
"""
# Test NATIVE (dijalankan OM) — definisi bawaan OM, bukan ekspektasi GE.
TABLES = {
f"{SERVICE}.{CATALOG}.bronze.orders": [
TableRowCountToBeBetween(min_count=1, max_count=1000000),
ColumnValuesToBeNotNull(column="id"),
ColumnValuesToBeNotNull(column="komoditas"),
ColumnValuesToBeBetween(column="harga_per_kg", min_value=0, max_value=1000000),
],
f"{SERVICE}.{CATALOG}.silver.orders_silver": [
TableRowCountToBeBetween(min_count=1, max_count=1000000),
ColumnValuesToBeNotNull(column="id"),
ColumnValuesToBeBetween(column="total_value", min_value=0, max_value=1000000000000),
],
f"{SERVICE}.{CATALOG}.gold.orders_gold": [
TableRowCountToBeBetween(min_count=1, max_count=1000000),
ColumnValuesToBeNotNull(column="komoditas"),
ColumnValuesToBeBetween(column="avg_harga_per_kg", min_value=0, max_value=1000000),
ColumnValuesToBeBetween(column="total_nilai", min_value=0, max_value=1000000000000),
],
}
def _token():
"""Token OM: dari OM_JWT_TOKEN, atau login pakai password admin.
Dua kandidat: OPENMETADATA_ADMIN_PASSWORD lalu 'admin' — default yang dipakai
OM 2.0.x sampai bootstrap step 7 menyetel hash-nya dari .env. MAKSIMAL DUA
percobaan dan tidak diulang dalam loop: akun admin OM bisa terkunci.
Catatan: 401 pada kandidat pertama adalah hal NORMAL (mis. step 7 belum jalan),
jadi urlopen-nya dibungkus try/except — tanpa itu HTTPError 401 menggagalkan
seluruh skrip.
"""
if os.getenv("OM_JWT_TOKEN"):
return os.environ["OM_JWT_TOKEN"]
email = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
for password in (os.getenv("OPENMETADATA_ADMIN_PASSWORD"), "admin"):
if not password:
continue
body = json.dumps({
"email": email,
"password": base64.b64encode(password.encode()).decode(),
}).encode()
req = urllib.request.Request(f"{OM_API_URL}/v1/users/login", data=body,
headers={"Content-Type": "application/json"})
try:
tok = json.load(urllib.request.urlopen(req, timeout=20)).get("accessToken")
except Exception: # noqa: BLE001 — kandidat pertama gagal = hal biasa
tok = None
if tok:
return tok
return None
def main():
token = _token()
trino_password = os.getenv("TRINO_SVC_OM_PASSWORD")
if not token:
print("ERROR: token OpenMetadata tidak tersedia (OM_JWT_TOKEN / "
"OPENMETADATA_ADMIN_PASSWORD)")
return 1
if not trino_password:
print("ERROR: TRINO_SVC_OM_PASSWORD tidak diset — koneksi Trino tidak bisa dibuat")
return 1
client = OpenMetadata(
OpenMetadataConnection(
hostPort=OM_API_URL,
authProvider="openmetadata",
securityConfig=OpenMetadataJWTClientConfig(jwtToken=token),
)
)
failed = 0
total = 0
for table_fqn, tests in TABLES.items():
print(f"[DQ-native] tabel {table_fqn} — {len(tests)} test")
workflow = WORKFLOW_TEMPLATE.format(
service=SERVICE, hostport=TRINO_HOSTPORT, password=trino_password,
catalog=CATALOG, cert=TRINO_CERT, table=table_fqn, om_api=OM_API_URL,
token=token,
)
# use_connection_from_yaml=True → pakai koneksi dari workflow (bukan yang
# tersimpan di OM), sehingga autentikasi Trino bekerja.
runner = TestRunner.from_yaml(
yaml_string=workflow, use_connection_from_yaml=True, client=client
)
for test in tests:
runner.add_test(test)
for result in runner.run():
total += 1
status = str(getattr(result.testCaseResult, "testCaseStatus", None))
name = getattr(result, "name", None) or getattr(result, "fqn", "test")
print(f" - {name}: {status.split('.')[-1]}")
if status.split(".")[-1] != "Success":
failed += 1
print(f"[DQ-native] ringkasan: {total - failed}/{total} test lulus")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())
deploy/scripts/airflow/seed-om-native-dq.py
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python
# =============================================================================
# eBdesk SiCABE — seed contoh Data Quality NATIVE OpenMetadata (Jalur B)
# =============================================================================
# Mengapa skrip ini ada:
# Test case native TIDAK bisa diandalkan lewat TestRunner SDK yang berjalan di
# klien: jalur itu memakai koneksi TERSIMPAN yang kehilangan secret sehingga
# selalu berakhir `Aborted` dengan `401 Access Denied` (bug upstream, masih ada
# di OM 2.0.2 — kredensial sudah dipastikan valid).
# Jalur yang bekerja: definisi test case dibuat lewat REST API, lalu eksekusi
# DIPICU OpenMetadata lewat pipeline service client (plugin → Airflow). Koneksi
# Trino dibawa EKSPLISIT di `sourceConfig.serviceConnections`, sehingga tidak
# bergantung pada koneksi tersimpan.
#
# Idempoten: test case diperiksa berdasarkan nama; pipeline diperiksa berdasarkan
# nama. Menjalankan ulang tidak menggandakan apa pun.
#
# Dipanggil oleh: scripts/seed-native-dq.sh (dari host) — atau manual:
# docker exec -e OM_SERVICE=trino_sicabe ... sicabe-airflow-webserver \
# python /opt/airflow/scripts/seed-om-native-dq.py [--trigger]
#
# Exit code: 0 = semua langkah yang diminta berhasil.
# =============================================================================
import base64
import json
import os
import sys
import urllib.error
import urllib.request
OM_API_URL = os.getenv("OM_API_URL", "http://openmetadata-server:8585/api")
ADMIN_EMAIL = os.getenv("OPENMETADATA_ADMIN_EMAIL", "admin@open-metadata.org")
SERVICE = os.getenv("OM_DATABASE_SERVICE", "trino_sicabe")
CATALOG = os.getenv("OM_DATABASE_NAME", "iceberg")
TABLE_FQN = os.getenv("OM_DQ_SEED_TABLE", f"{SERVICE}.{CATALOG}.gold.orders_gold")
PIPELINE_NAME = os.getenv("OM_DQ_SEED_PIPELINE", "sicabe_dq_gold_native")
SCHEDULE = os.getenv("OM_DQ_SEED_SCHEDULE", "0 3 * * *")
TRINO_HOSTPORT = os.getenv("OM_TRINO_HOSTPORT", "trino-coordinator:8443")
TRINO_USER = os.getenv("DBT_TRINO_USER", "svc_om")
TRINO_CERT = os.getenv("TRINO_CA_BUNDLE", "/etc/ssl/trino/trino.crt")
# Contoh test native. `parameterValues` HARUS memakai nama parameter definisi
# (tableRowCountToBeBetween: minValue/maxValue) dan columnValuesToBeNotNull
# tidak punya parameter sama sekali — kolomnya ditentukan lewat entityLink.
TEST_CASES = [
{
"name": "native_row_count_between",
"testDefinition": "tableRowCountToBeBetween",
"entityLink": f"<#E::table::{TABLE_FQN}>",
"parameterValues": [
{"name": "minValue", "value": "1"},
{"name": "maxValue", "value": "1000000"},
],
},
{
"name": "native_komoditas_not_null",
"testDefinition": "columnValuesToBeNotNull",
"entityLink": f"<#E::table::{TABLE_FQN}::columns::komoditas>",
},
]
def _request(method, path, payload=None, token=None):
"""Panggil REST API OpenMetadata; kembalikan (status, body terurai)."""
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
data = json.dumps(payload).encode() if payload is not None else None
req = urllib.request.Request(f"{OM_API_URL}{path}", data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=60) as resp:
raw = resp.read()
return resp.status, (json.loads(raw) if raw else {})
except urllib.error.HTTPError as exc:
raw = exc.read()[:300]
try:
return exc.code, json.loads(raw)
except Exception: # noqa: BLE001
return exc.code, raw.decode(errors="replace")
def login():
"""Token admin: password dikirim base64 (wajib sejak OM 2.0.x).
Dua kandidat: OPENMETADATA_ADMIN_PASSWORD, lalu 'admin' — default yang dipakai
OM 2.0.x sampai bootstrap step 7 menyetel hash-nya dari .env. MAKSIMAL DUA
percobaan dan tidak pernah diulang dalam loop: akun admin OM bisa terkunci.
"""
last = "tidak ada kandidat password"
for password in (os.getenv("OPENMETADATA_ADMIN_PASSWORD"), "admin"):
if not password:
continue
body = {"email": ADMIN_EMAIL,
"password": base64.b64encode(password.encode()).decode()}
status, res = _request("POST", "/v1/users/login", body)
if status == 200 and res.get("accessToken"):
return res["accessToken"]
last = f"HTTP {status}: {str(res)[:120]}"
raise RuntimeError(f"login gagal (password .env dan 'admin'): {last}")
def ensure_test_cases(token):
"""Buat test case native yang belum ada. Kembalikan jumlah yang dibuat."""
status, page = _request("GET", "/v1/dataQuality/testCases?limit=200", token=token)
if status != 200:
raise RuntimeError(f"daftar test case tidak terbaca (HTTP {status})")
existing = {tc.get("name") for tc in page.get("data", [])}
created = 0
for spec in TEST_CASES:
if spec["name"] in existing:
print(f" [SKIP] test case sudah ada: {spec['name']}")
continue
status, res = _request("POST", "/v1/dataQuality/testCases", spec, token=token)
if status in (200, 201):
created += 1
print(f" [OK] test case dibuat: {spec['name']}")
else:
raise RuntimeError(f"gagal membuat {spec['name']} (HTTP {status}): {str(res)[:200]}")
return created
def _pipeline_payload(suite_id):
"""Payload pipeline TestSuite dengan koneksi Trino EKSPLISIT (anti-401)."""
return {
"name": PIPELINE_NAME,
"pipelineType": "TestSuite",
"service": {"id": suite_id, "type": "testSuite"},
"sourceConfig": {"config": {
"type": "TestSuite",
"entityFullyQualifiedName": TABLE_FQN,
"serviceConnections": [{
"serviceName": SERVICE,
"serviceConnection": {"config": {
"type": "Trino", "scheme": "trino", "hostPort": TRINO_HOSTPORT,
"username": TRINO_USER,
"authType": {"password": os.getenv("TRINO_SVC_OM_PASSWORD", "")},
"catalog": CATALOG, "verify": TRINO_CERT,
}},
}],
}},
"airflowConfig": {"scheduleInterval": SCHEDULE},
}
def ensure_pipeline(token):
"""Buat pipeline TestSuite bila belum ada. Kembalikan (id, dibuat_baru)."""
status, suites = _request("GET", "/v1/dataQuality/testSuites?limit=50", token=token)
if status != 200:
raise RuntimeError(f"daftar suite tidak terbaca (HTTP {status})")
suite = next((s for s in suites.get("data", []) if TABLE_FQN.rsplit(".", 1)[0] in s.get("name", "")), None)
if suite is None:
raise RuntimeError(f"test suite untuk '{TABLE_FQN}' belum ada — jalankan ingestion dulu")
status, page = _request("GET", "/v1/services/ingestionPipelines?limit=50", token=token)
for pipe in (page.get("data") or []) if status == 200 else []:
if pipe.get("name") == PIPELINE_NAME:
print(f" [SKIP] pipeline sudah ada: {PIPELINE_NAME} (id {pipe['id']})")
return pipe["id"], False
status, res = _request("POST", "/v1/services/ingestionPipelines",
_pipeline_payload(suite["id"]), token=token)
if status not in (200, 201):
raise RuntimeError(f"gagal membuat pipeline (HTTP {status}): {str(res)[:200]}")
print(f" [OK] pipeline dibuat: {PIPELINE_NAME} (id {res['id']})")
return res["id"], True
def deploy_and_trigger(token, pipeline_id):
"""Deploy lalu trigger pipeline; trigger diulang sekali bila DAG belum siap."""
import time
status, res = _request("POST", f"/v1/services/ingestionPipelines/deploy/{pipeline_id}", token=token)
print(f" [{'OK' if status == 200 else 'GAGAL'}] deploy -> HTTP {status}")
if status != 200:
raise RuntimeError(f"deploy gagal: {str(res)[:200]}")
for attempt in (1, 2):
status, res = _request("POST", f"/v1/services/ingestionPipelines/trigger/{pipeline_id}", token=token)
print(f" [{'OK' if status == 200 else 'GAGAL'}] trigger -> HTTP {status} (percobaan {attempt})")
if status == 200:
return
# DAG baru dibuat: dag-processor bisa belum memparsingnya.
time.sleep(20)
raise RuntimeError("trigger gagal setelah 2 percobaan")
def main():
trigger = "--trigger" in sys.argv
print(f"[seed-native-dq] tabel={TABLE_FQN} pipeline={PIPELINE_NAME}")
token = login()
print(f" [OK] login {ADMIN_EMAIL}")
created = ensure_test_cases(token)
pipeline_id, new_pipeline = ensure_pipeline(token)
if trigger:
deploy_and_trigger(token, pipeline_id)
print(f"[seed-native-dq] selesai: {created} test case baru, pipeline "
f"{'baru' if new_pipeline else 'sudah ada'}"
f"{', deploy+trigger dijalankan' if trigger else ''}")
return 0
if __name__ == "__main__":
try:
sys.exit(main())
except Exception as exc: # noqa: BLE001
print(f"ERROR: {exc}")
sys.exit(1)