85 lines
4.1 KiB
Bash
85 lines
4.1 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# eBdesk SiCABE — Orkestrasi alur data E2E (satu perintah) — jalankan DI CONTROL.
|
||
|
|
#
|
||
|
|
# Menghilangkan langkah manual berulang pada cold start. Dua "chicken-egg" yang
|
||
|
|
# ditangani otomatis:
|
||
|
|
# (a) aksi Great Expectations menempel ke ENTITAS tabel di OM, tetapi
|
||
|
|
# `gold.orders_gold` DIBUAT oleh DAG → DAG #1 gagal di GE (wajar).
|
||
|
|
# (b) lineage dbt dibaca dari ARTEFAK (`manifest.json`+`catalog.json`+`run_results.json`
|
||
|
|
# di /tmp/dbt/target) yang DIBUAT oleh `dbt docs generate` — DAG hanya `dbt run`
|
||
|
|
# (tanpa catalog.json), dan artefak HILANG bila container Airflow di-recreate.
|
||
|
|
#
|
||
|
|
# Urutan: ingest metadata → DAG #1 (data) → dbt docs generate → ingest --all →
|
||
|
|
# DAG #2 (GE) → verify-phase6.
|
||
|
|
#
|
||
|
|
# PRASYARAT (sekali, DI HOST MASING-MASING — di luar skrip ini):
|
||
|
|
# (state) ./scripts/seed-source-postgres.sh --env-file generated/host-state/.env
|
||
|
|
# (compute) ./scripts/seed-starrocks.sh --env-file generated/host-compute/.env
|
||
|
|
# DQ yang bekerja = Jalur A (Great Expectations pada task validate_gold/cdc_validate di DAG).
|
||
|
|
# seed-native-dq.sh = Jalur B (DQ native OM) opt-in & TERBLOKIR bug upstream §8.10 —
|
||
|
|
# JANGAN dipakai di alur ini (menempel ke entitas gold yang baru ada setelah DAG+ingest).
|
||
|
|
#
|
||
|
|
# Pemakaian: ./scripts/run-e2e.sh [--env-file <path>] [--dag <nama>]
|
||
|
|
# =============================================================================
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
|
cd "${SCRIPT_DIR}"
|
||
|
|
# shellcheck disable=SC1091
|
||
|
|
source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true
|
||
|
|
|
||
|
|
ENV_FILE="${SCRIPT_DIR}/generated/host-control/.env"
|
||
|
|
DAG="${SICABE_DAG:-sicabe_batch_daily}"
|
||
|
|
WEB="${SICABE_AIRFLOW_WEB:-sicabe-airflow-webserver}"
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--env-file) ENV_FILE="${2:-}"; shift 2 ;;
|
||
|
|
--dag) DAG="${2:-}"; shift 2 ;;
|
||
|
|
-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||
|
|
*) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
[[ -f "${ENV_FILE}" ]] || ENV_FILE="${SCRIPT_DIR}/.env"
|
||
|
|
[[ -f "${ENV_FILE}" ]] || { echo "env file tak ditemukan: ${ENV_FILE}" >&2; exit 2; }
|
||
|
|
|
||
|
|
# Password service account dbt (untuk `dbt docs generate` di dalam container).
|
||
|
|
DBT_PW="$(sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' "${ENV_FILE}" | head -1)"
|
||
|
|
DBT_USER="${DBT_TRINO_USER:-svc_dbt}"
|
||
|
|
|
||
|
|
step() { echo; echo -e "\033[1m▶ $*\033[0m"; }
|
||
|
|
|
||
|
|
echo -e "\033[1mAlur data E2E (SiCABE)\033[0m"
|
||
|
|
echo " env-file : ${ENV_FILE}"
|
||
|
|
echo " DAG : ${DAG}"
|
||
|
|
echo " Prasyarat: seed sudah dijalankan di tiap host (state/compute)."
|
||
|
|
|
||
|
|
step "1/6 ingest metadata (service + entitas Trino; TANPA dbt — artefak belum ada pra-DAG)"
|
||
|
|
./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" \
|
||
|
|
|| echo " PERHATIAN: ingest metadata gagal (mis. tabel belum ada pra-DAG) — lanjut; di-ingest ulang di step 4."
|
||
|
|
|
||
|
|
step "2/6 DAG #1 — buat data (bronze→silver→gold). GE boleh gagal: entitas gold belum di OM."
|
||
|
|
docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}" \
|
||
|
|
|| echo " PERHATIAN: DAG #1 gagal (biasanya di GE karena gold belum di OM) — lanjut; wajar pada cold start."
|
||
|
|
|
||
|
|
step "3/6 artefak dbt (manifest+catalog+run_results) untuk lineage"
|
||
|
|
echo " (wajib: DAG hanya 'dbt run' → catalog.json absen; artefak hilang bila container di-recreate)"
|
||
|
|
docker exec \
|
||
|
|
-e DBT_TRINO_USER="${DBT_USER}" -e DBT_TRINO_PASSWORD="${DBT_PW}" \
|
||
|
|
-e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=/tmp/dbt/target \
|
||
|
|
-e DBT_LOG_PATH=/tmp/dbt/logs -w /opt/airflow/dbt/sicabe \
|
||
|
|
"${WEB}" bash -lc 'dbt docs generate --profile trino --target prod' \
|
||
|
|
|| echo " PERHATIAN: dbt docs generate gagal — lineage dbt mungkin terlewat."
|
||
|
|
|
||
|
|
step "4/6 ingest ulang — entitas gold + lineage dbt (artefak sudah ada)"
|
||
|
|
./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" --all
|
||
|
|
|
||
|
|
step "5/6 DAG #2 — GE kini menemukan entitas gold di OM (target: hijau)"
|
||
|
|
docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}"
|
||
|
|
|
||
|
|
step "6/6 verifikasi governance (target FAIL=0)"
|
||
|
|
./scripts/verify-phase6-governance.sh --env-file "${ENV_FILE}"
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo -e "\033[1;32m✓ Alur E2E selesai.\033[0m"
|