#!/usr/bin/env bash # ============================================================================= # eBdesk SiCABE — seed contoh Data Quality NATIVE OpenMetadata (Jalur B) # ============================================================================= # Membuat contoh test case native + pipeline TestSuite yang membawa koneksi Trino # eksplisit, lalu (opsional) deploy + trigger supaya hasilnya masuk ke OM. # # Mengapa lewat pipeline, bukan TestRunner SDK: lihat header # scripts/airflow/seed-om-native-dq.py dan docs/sicabe-install-phases-detail.md # §8.10 — jalur SDK memakai koneksi tersimpan yang kehilangan secret (bug # upstream) sehingga selalu `Aborted`. # # Usage: # ./scripts/seed-native-dq.sh # buat definisi + pipeline saja # ./scripts/seed-native-dq.sh --trigger # sekaligus deploy + trigger # # Idempoten: test case & pipeline yang sudah ada dilewati. # Dijalankan dari host; pekerjaannya dieksekusi DI DALAM container Airflow. # # PRASYARAT: entitas tabel `trino_sicabe.iceberg.gold.orders_gold` SUDAH ada di OM # (setelah DAG + `ingest-openmetadata.sh --all`). Bila belum → worker gagal # `HTTP 404: table instance ... not found`. # ============================================================================= set -euo pipefail # ⚠️ JALUR B TERBLOKIR UPSTREAM — banner ini disengaja. # DQ native OpenMetadata (Jalur B) memakai koneksi tersimpan yang kehilangan secret saat # dibangun runner, sehingga test SELALU berakhir `Aborted` (docs/sicabe-install-phases-detail.md # §8.10). Karena itu skrip ini TIDAK dipanggil bootstrap.sh / seed-sample-data.sh / # ingest-openmetadata.sh: ia opt-in untuk eksperimen. Gate fase 6 memperlakukannya sebagai # SKIP, bukan FAIL. Untuk DQ yang benar-benar bekerja, pakai Jalur A (Great Expectations) # pada task validate_gold / cdc_validate di DAG sicabe_batch_daily. echo "⚠️ seed-native-dq.sh = Jalur B (DQ native OM) yang TERBLOKIR bug upstream (§8.10)." echo " Hasil test akan 'Aborted'. Untuk DQ yang bekerja: task validate_gold/cdc_validate (Jalur A)." SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEPLOY_DIR="$(dirname "${SCRIPT_DIR}")" # shellcheck source=../lib/common.sh . "${DEPLOY_DIR}/lib/common.sh" 2>/dev/null || true # shellcheck source=../config/environment.defaults.sh . "${DEPLOY_DIR}/config/environment.defaults.sh" 2>/dev/null || true # .env memuat password admin OM + password service account Trino. # Multi-host: dukung --env-file / SICABE_ENV_FILE seperti skrip lain. sicabe_load_env "${DEPLOY_DIR}" "$@" || exit 1 AIRFLOW_CONTAINER="${AIRFLOW_CONTAINER:-sicabe-airflow-webserver}" WORKER="/opt/airflow/scripts/seed-om-native-dq.py" # Multi-host: Airflow ada di host `control`; `docker exec` bersifat lokal. sicabe_require_local_container "${AIRFLOW_CONTAINER}" "Airflow webserver" "control" || exit $? for var in OPENMETADATA_ADMIN_PASSWORD TRINO_SVC_OM_PASSWORD; do if [[ -z "${!var:-}" ]]; then echo "GAGAL: ${var} kosong — periksa ${DEPLOY_DIR}/.env" >&2 exit 1 fi done echo "== seed contoh DQ native ==" # Buang `--env-file ` dari argumen yang diteruskan ke worker Python # (sudah dipakai sicabe_load_env; worker tak mengenalnya). WORKER_ARGS=(); _snq_prev="" for _snq_a in "$@"; do if [[ "${_snq_prev}" == "--env-file" ]]; then _snq_prev=""; continue; fi if [[ "${_snq_a}" == "--env-file" ]]; then _snq_prev="--env-file"; continue; fi WORKER_ARGS+=("${_snq_a}") done docker exec \ -e OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD}" \ -e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD}" \ "${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}" || { echo " (catatan) Bila error 404 'table instance ... not found' → tabel gold belum ada di OM." >&2 echo " Jalankan DAG + ingest-openmetadata dulu, atau pakai Jalur A (GE di DAG)." >&2 exit 1 }