sicabe-core/deploy/scripts/seed-native-dq.sh

#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — seed contoh Data Quality NATIVE OpenMetadata (Jalur B)
# =============================================================================
# Membuat contoh test case native + pipeline TestSuite yang membawa koneksi Trino
# eksplisit, lalu (opsional) deploy + trigger supaya hasilnya masuk ke OM.
#
# Mengapa lewat pipeline, bukan TestRunner SDK: lihat header
# scripts/airflow/seed-om-native-dq.py dan docs/sicabe-install-phases-detail.md
# §8.10 — jalur SDK memakai koneksi tersimpan yang kehilangan secret (bug
# upstream) sehingga selalu `Aborted`.
#
# Usage:
#   ./scripts/seed-native-dq.sh              # buat definisi + pipeline saja
#   ./scripts/seed-native-dq.sh --trigger    # sekaligus deploy + trigger
#
# Idempoten: test case & pipeline yang sudah ada dilewati.
# Dijalankan dari host; pekerjaannya dieksekusi DI DALAM container Airflow.
#
# PRASYARAT: entitas tabel `trino_sicabe.iceberg.gold.orders_gold` SUDAH ada di OM
#   (setelah DAG + `ingest-openmetadata.sh --all`). Bila belum → worker gagal
#   `HTTP 404: table instance ... not found`.
# =============================================================================
set -euo pipefail

# ⚠️ JALUR B TERBLOKIR UPSTREAM — banner ini disengaja.
# DQ native OpenMetadata (Jalur B) memakai koneksi tersimpan yang kehilangan secret saat
# dibangun runner, sehingga test SELALU berakhir `Aborted` (docs/sicabe-install-phases-detail.md
# §8.10). Karena itu skrip ini TIDAK dipanggil bootstrap.sh / seed-sample-data.sh /
# ingest-openmetadata.sh: ia opt-in untuk eksperimen. Gate fase 6 memperlakukannya sebagai
# SKIP, bukan FAIL. Untuk DQ yang benar-benar bekerja, pakai Jalur A (Great Expectations)
# pada task validate_gold / cdc_validate di DAG sicabe_batch_daily.
echo "⚠️  seed-native-dq.sh = Jalur B (DQ native OM) yang TERBLOKIR bug upstream (§8.10)."
echo "    Hasil test akan 'Aborted'. Untuk DQ yang bekerja: task validate_gold/cdc_validate (Jalur A)."

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEPLOY_DIR="$(dirname "${SCRIPT_DIR}")"

# shellcheck source=../lib/common.sh
. "${DEPLOY_DIR}/lib/common.sh" 2>/dev/null || true
# shellcheck source=../config/environment.defaults.sh
. "${DEPLOY_DIR}/config/environment.defaults.sh" 2>/dev/null || true

# .env memuat password admin OM + password service account Trino.
# Multi-host: dukung --env-file / SICABE_ENV_FILE seperti skrip lain.
sicabe_load_env "${DEPLOY_DIR}" "$@" || exit 1

AIRFLOW_CONTAINER="${AIRFLOW_CONTAINER:-sicabe-airflow-webserver}"
WORKER="/opt/airflow/scripts/seed-om-native-dq.py"

# Multi-host: Airflow ada di host `control`; `docker exec` bersifat lokal.
sicabe_require_local_container "${AIRFLOW_CONTAINER}" "Airflow webserver" "control" || exit $?

for var in OPENMETADATA_ADMIN_PASSWORD TRINO_SVC_OM_PASSWORD; do
  if [[ -z "${!var:-}" ]]; then
    echo "GAGAL: ${var} kosong — periksa ${DEPLOY_DIR}/.env" >&2
    exit 1
  fi
done

echo "== seed contoh DQ native =="
# Buang `--env-file <berkas>` dari argumen yang diteruskan ke worker Python
# (sudah dipakai sicabe_load_env; worker tak mengenalnya).
WORKER_ARGS=(); _snq_prev=""
for _snq_a in "$@"; do
  if [[ "${_snq_prev}" == "--env-file" ]]; then _snq_prev=""; continue; fi
  if [[ "${_snq_a}" == "--env-file" ]]; then _snq_prev="--env-file"; continue; fi
  WORKER_ARGS+=("${_snq_a}")
done
docker exec \
  -e OPENMETADATA_ADMIN_PASSWORD="${OPENMETADATA_ADMIN_PASSWORD}" \
  -e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD}" \
  "${AIRFLOW_CONTAINER}" python "${WORKER}" "${WORKER_ARGS[@]+"${WORKER_ARGS[@]}"}" || {
  echo "  (catatan) Bila error 404 'table instance ... not found' → tabel gold belum ada di OM." >&2
  echo "  Jalankan DAG + ingest-openmetadata dulu, atau pakai Jalur A (GE di DAG)." >&2
  exit 1
}