70 lines
3.4 KiB
Bash
Executable File
70 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# eBdesk SiCABE — seed contoh Data Quality NATIVE OpenMetadata (Jalur B)
|
|
# =============================================================================
|
|
# Membuat contoh test case native + pipeline TestSuite yang membawa koneksi Trino
|
|
# eksplisit, lalu (opsional) deploy + trigger supaya hasilnya masuk ke OM.
|
|
#
|
|
# Mengapa lewat pipeline, bukan TestRunner SDK: lihat header
|
|
# scripts/airflow/seed-om-native-dq.py dan docs/sicabe-install-phases-detail.md
|
|
# §8.10 — jalur SDK memakai koneksi tersimpan yang kehilangan secret (bug
|
|
# upstream) sehingga selalu `Aborted`.
|
|
#
|
|
# Usage:
|
|
# ./scripts/seed-native-dq.sh # buat definisi + pipeline saja
|
|
# ./scripts/seed-native-dq.sh --trigger # sekaligus deploy + trigger
|
|
#
|
|
# Idempoten: test case & pipeline yang sudah ada dilewati.
|
|
# Dijalankan dari host; pekerjaannya dieksekusi DI DALAM container Airflow.
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
# ⚠️ JALUR B TERBLOKIR UPSTREAM — banner ini disengaja.
|
|
# DQ native OpenMetadata (Jalur B) memakai koneksi tersimpan yang kehilangan secret saat
|
|
# dibangun runner, sehingga test SELALU berakhir `Aborted` (docs/sicabe-install-phases-detail.md
|
|
# §8.10). Karena itu skrip ini TIDAK dipanggil bootstrap.sh / seed-sample-data.sh /
|
|
# ingest-openmetadata.sh: ia opt-in untuk eksperimen. Gate fase 6 memperlakukannya sebagai
|
|
# SKIP, bukan FAIL. Untuk DQ yang benar-benar bekerja, pakai Jalur A (Great Expectations)
|
|
# pada task validate_gold / cdc_validate di DAG sicabe_batch_daily.
|
|
echo "⚠️ seed-native-dq.sh = Jalur B (DQ native OM) yang TERBLOKIR bug upstream (§8.10)."
|
|
echo " Hasil test akan 'Aborted'. Untuk DQ yang bekerja: task validate_gold/cdc_validate (Jalur A)."
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
DEPLOY_DIR="$(dirname "${SCRIPT_DIR}")"
|
|
|
|
# shellcheck source=../lib/common.sh
|
|
. "${DEPLOY_DIR}/lib/common.sh" 2>/dev/null || true
|
|
# shellcheck source=../config/environment.defaults.sh
|
|
. "${DEPLOY_DIR}/config/environment.defaults.sh" 2>/dev/null || true
|
|
|
|
# .env memuat password admin OM + password service account Trino.
|
|
# Multi-host: dukung --env-file / SICABE_ENV_FILE seperti skrip lain.
|
|
sicabe_load_env "${DEPLOY_DIR}" "$@" || exit 1
|
|
|
|
AIRFLOW_CONTAINER="${AIRFLOW_CONTAINER:-sicabe-airflow-webserver}"
|
|
WORKER="/opt/airflow/scripts/seed-om-native-dq.py"
|
|
|
|
# Multi-host: Airflow ada di host `control`; `docker exec` bersifat lokal.
|
|
sicabe_require_local_container "${AIRFLOW_CONTAINER}" "Airflow webserver" "control" || exit $?
|
|
|
|
for var in OPENMETADATA_ADMIN_PASSWORD TRINO_SVC_OM_PASSWORD; do
|
|
if [[ -z "${!var:-}" ]]; then
|
|
echo "GAGAL: ${var} kosong — periksa ${DEPLOY_DIR}/.env" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "== seed contoh DQ native =="
|
|
# Buang `--env-file <berkas>` dari argumen yang diteruskan ke worker Python
|
|
# (sudah dipakai sicabe_load_env; worker tak mengenalnya).
|
|
WORKER_ARGS=(); _snq_prev=""
|
|
for _snq_a in "$@"; do
|
|
if [[ "${_snq_prev}" == "--env-file" ]]; then _snq_prev=""; continue; fi
|
|
if [[ "${_snq_a}" == "--env-file" ]]; then _snq_prev="--env-file"; continue; fi
|
|
WORKER_ARGS+=("${_snq_a}")
|
|
done
|
|
docker exec \
|
|
-e OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD}" \
|
|
-e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD}" \
|
|
"${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}"
|