sync dari sicabe-doc/deploy @ bc1a6e6

2026-10-11 13:59:47 +07:00
parent 4ca7a9a58e
commit cbabe96061
29 changed files with 1210 additions and 650 deletions
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# =============================================================================
+# eBdesk SiCABE — Orkestrasi alur data E2E (satu perintah) — jalankan DI CONTROL.
+#
+# Menghilangkan langkah manual berulang pada cold start. Dua "chicken-egg" yang
+# ditangani otomatis:
+#   (a) aksi Great Expectations menempel ke ENTITAS tabel di OM, tetapi
+#       `gold.orders_gold` DIBUAT oleh DAG → DAG #1 gagal di GE (wajar).
+#   (b) lineage dbt dibaca dari ARTEFAK (`manifest.json`+`catalog.json`+`run_results.json`
+#       di /tmp/dbt/target) yang DIBUAT oleh `dbt docs generate` — DAG hanya `dbt run`
+#       (tanpa catalog.json), dan artefak HILANG bila container Airflow di-recreate.
+#
+# Urutan: ingest metadata → DAG #1 (data) → dbt docs generate → ingest --all →
+#         DAG #2 (GE) → verify-phase6.
+#
+# PRASYARAT (sekali, DI HOST MASING-MASING — di luar skrip ini):
+#   (state)   ./scripts/seed-source-postgres.sh --env-file generated/host-state/.env
+#   (compute) ./scripts/seed-starrocks.sh       --env-file generated/host-compute/.env
+# DQ yang bekerja = Jalur A (Great Expectations pada task validate_gold/cdc_validate di DAG).
+# seed-native-dq.sh = Jalur B (DQ native OM) opt-in & TERBLOKIR bug upstream §8.10 —
+# JANGAN dipakai di alur ini (menempel ke entitas gold yang baru ada setelah DAG+ingest).
+#
+# Pemakaian:  ./scripts/run-e2e.sh [--env-file <path>] [--dag <nama>]
+# =============================================================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "${SCRIPT_DIR}"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/lib/common.sh" 2>/dev/null || true
+
+ENV_FILE="${SCRIPT_DIR}/generated/host-control/.env"
+DAG="${SICABE_DAG:-sicabe_batch_daily}"
+WEB="${SICABE_AIRFLOW_WEB:-sicabe-airflow-webserver}"
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --env-file) ENV_FILE="${2:-}"; shift 2 ;;
+    --dag)      DAG="${2:-}"; shift 2 ;;
+    -h|--help)  sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    *) echo "Argumen tidak dikenal: $1" >&2; exit 2 ;;
+  esac
+done
+[[ -f "${ENV_FILE}" ]] || ENV_FILE="${SCRIPT_DIR}/.env"
+[[ -f "${ENV_FILE}" ]] || { echo "env file tak ditemukan: ${ENV_FILE}" >&2; exit 2; }
+
+# Password service account dbt (untuk `dbt docs generate` di dalam container).
+DBT_PW="$(sed -n 's/^TRINO_SVC_DBT_PASSWORD=//p' "${ENV_FILE}" | head -1)"
+DBT_USER="${DBT_TRINO_USER:-svc_dbt}"
+
+step() { echo; echo -e "\033[1m▶ $*\033[0m"; }
+
+echo -e "\033[1mAlur data E2E (SiCABE)\033[0m"
+echo "  env-file : ${ENV_FILE}"
+echo "  DAG      : ${DAG}"
+echo "  Prasyarat: seed sudah dijalankan di tiap host (state/compute)."
+
+step "1/6 ingest metadata (service + entitas Trino; TANPA dbt — artefak belum ada pra-DAG)"
+./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" \
+  || echo "  PERHATIAN: ingest metadata gagal (mis. tabel belum ada pra-DAG) — lanjut; di-ingest ulang di step 4."
+
+step "2/6 DAG #1 — buat data (bronze→silver→gold). GE boleh gagal: entitas gold belum di OM."
+docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}" \
+  || echo "  PERHATIAN: DAG #1 gagal (biasanya di GE karena gold belum di OM) — lanjut; wajar pada cold start."
+
+step "3/6 artefak dbt (manifest+catalog+run_results) untuk lineage"
+echo "  (wajib: DAG hanya 'dbt run' → catalog.json absen; artefak hilang bila container di-recreate)"
+docker exec \
+  -e DBT_TRINO_USER="${DBT_USER}" -e DBT_TRINO_PASSWORD="${DBT_PW}" \
+  -e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=/tmp/dbt/target \
+  -e DBT_LOG_PATH=/tmp/dbt/logs -w /opt/airflow/dbt/sicabe \
+  "${WEB}" bash -lc 'dbt docs generate --profile trino --target prod' \
+  || echo "  PERHATIAN: dbt docs generate gagal — lineage dbt mungkin terlewat."
+
+step "4/6 ingest ulang — entitas gold + lineage dbt (artefak sudah ada)"
+./scripts/ingest-openmetadata.sh --env-file "${ENV_FILE}" --all
+
+step "5/6 DAG #2 — GE kini menemukan entitas gold di OM (target: hijau)"
+docker exec -e SICABE_DAG_RETRIES=0 "${WEB}" airflow dags test "${DAG}"
+
+step "6/6 verifikasi governance (target FAIL=0)"
+./scripts/verify-phase6-governance.sh --env-file "${ENV_FILE}"
+
+echo
+echo -e "\033[1;32m✓ Alur E2E selesai.\033[0m"