191 lines
9.1 KiB
Bash
191 lines
9.1 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# eBdesk SiCABE — Jalankan agent ingestion OpenMetadata
|
||
|
|
# =============================================================================
|
||
|
|
# Mengapa script ini ada:
|
||
|
|
# Membuat SERVICE di OpenMetadata hanya menyimpan konfigurasi koneksi. Metadata
|
||
|
|
# (database → schema → tabel → kolom) baru terisi setelah AGENT dijalankan, dan
|
||
|
|
# halaman Explore membaca hasilnya dari search index. Tanpa agent: Explore kosong.
|
||
|
|
#
|
||
|
|
# LINEAGE (bronze→silver→gold) TIDAK dihasilkan oleh ingestion metadata: lineage
|
||
|
|
# berasal dari definisi transformasi, jadi diambil dari ARTEFAK dbt lewat workflow
|
||
|
|
# terpisah (`ingest-dbt.yaml`). Keduanya dijalankan oleh `--all`.
|
||
|
|
#
|
||
|
|
# Script memakai Ingestion Framework yang sudah ada di container Airflow
|
||
|
|
# (`metadata` CLI) — tidak perlu memasang apa pun.
|
||
|
|
#
|
||
|
|
# ⚠️ Cakupan metadata mengikuti ACL identitas agent (workflow memakai `svc_om`,
|
||
|
|
# role `data-catalog` = baca-saja seluruh tabel).
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# ./scripts/ingest-openmetadata.sh # metadata (Trino) saja
|
||
|
|
# ./scripts/ingest-openmetadata.sh --all # metadata + lineage dbt
|
||
|
|
# ./scripts/ingest-openmetadata.sh --dbt-only # lineage dbt saja
|
||
|
|
# ./scripts/ingest-openmetadata.sh --workflow config/openmetadata/ingest-dbt.yaml
|
||
|
|
# =============================================================================
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||
|
|
# shellcheck source=../lib/common.sh
|
||
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
||
|
|
# shellcheck source=../config/environment.defaults.sh
|
||
|
|
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||
|
|
sicabe_load_env "${SCRIPT_DIR}" "$@"
|
||
|
|
|
||
|
|
OM_URL="${OPENMETADATA_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||
|
|
OM_ADMIN_EMAIL="${OPENMETADATA_ADMIN_EMAIL:-admin@open-metadata.org}"
|
||
|
|
OM_ADMIN_PASS="${OPENMETADATA_ADMIN_PASSWORD:-openmetadata123}"
|
||
|
|
SERVICE_NAME="${OPENMETADATA_TRINO_SERVICE:-trino_sicabe}"
|
||
|
|
AIRFLOW_CONTAINER="${AIRFLOW_INGEST_CONTAINER:-sicabe-airflow-webserver}"
|
||
|
|
REMOTE_DIR="/tmp/om-ingest"
|
||
|
|
DBT_PROJECT_DIR="${DBT_PROJECT_DIR:-/opt/airflow/dbt/sicabe}"
|
||
|
|
|
||
|
|
# Di multi-host, pakai workflow TER-RENDER dari bundle host (hostname → alamat host);
|
||
|
|
# sumber `config/**` masih memakai nama container (single-host). `SICABE_BUNDLE_DIR`
|
||
|
|
# diset sicabe_load_env(): single-host = deploy/ (config sumber), multi-host =
|
||
|
|
# generated/host-<x>/ (config ter-render).
|
||
|
|
CONFIG_DIR="${SICABE_BUNDLE_DIR:-${SCRIPT_DIR}}/config"
|
||
|
|
WF_METADATA="${CONFIG_DIR}/openmetadata/ingest-trino.yaml"
|
||
|
|
WF_DBT="${CONFIG_DIR}/openmetadata/ingest-dbt.yaml"
|
||
|
|
MODE="metadata"
|
||
|
|
CUSTOM_WORKFLOW=""
|
||
|
|
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--env-file) shift 2 ;; # sudah dipakai sicabe_load_env() di atas
|
||
|
|
--all) MODE="all"; shift ;;
|
||
|
|
--dbt-only) MODE="dbt"; shift ;;
|
||
|
|
--workflow) CUSTOM_WORKFLOW="$2"; shift 2 ;;
|
||
|
|
*) echo "Argumen tidak dikenal: $1"; exit 1 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
# ── Helper ──────────────────────────────────────────────────────────────────
|
||
|
|
om_ws() { # om_ws <method> <path> [body]
|
||
|
|
local method="$1" path="$2" body="${3:-}"
|
||
|
|
if [[ -n "${body}" ]]; then
|
||
|
|
curl -s --max-time 30 -X "${method}" "${OM_URL}${path}" \
|
||
|
|
-H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" -d "${body}" 2>/dev/null
|
||
|
|
else
|
||
|
|
curl -s --max-time 30 -X "${method}" "${OM_URL}${path}" -H "Authorization: Bearer ${TOKEN}" 2>/dev/null
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
run_workflow() { # run_workflow <path-workflow> <label>
|
||
|
|
local wf="$1" label="$2" remote
|
||
|
|
remote="${REMOTE_DIR}/$(basename "${wf}")"
|
||
|
|
if [[ ! -f "${wf}" ]]; then
|
||
|
|
echo -e "${ERR} Workflow tidak ditemukan: ${wf}"
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
echo -e "${INFO} [${label}] menyalin ${wf##*/} ke container..."
|
||
|
|
docker exec "${AIRFLOW_CONTAINER}" mkdir -p "${REMOTE_DIR}"
|
||
|
|
docker cp "${wf}" "${AIRFLOW_CONTAINER}:${remote}" >/dev/null
|
||
|
|
|
||
|
|
echo -e "${INFO} [${label}] menjalankan ingestion..."
|
||
|
|
local out
|
||
|
|
out="$(docker exec -e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD:-}" -e OM_JWT_TOKEN="${TOKEN}" \
|
||
|
|
"${AIRFLOW_CONTAINER}" metadata ingest -c "${remote}" 2>&1)"
|
||
|
|
echo "${out}" | sed 's/\x1b\[[0-9;]*m//g' |
|
||
|
|
grep -iE "Processed records|Errors:|Success %|Workflow Success|Error running node|ERROR" | tail -8
|
||
|
|
if ! echo "${out}" | grep -q "Workflow Success %: 100.0"; then
|
||
|
|
echo -e "${ERR} [${label}] ingestion tidak 100% sukses — periksa keluaran di atas."
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
|
||
|
|
# ── Prasyarat ───────────────────────────────────────────────────────────────
|
||
|
|
for f in "${WF_METADATA}" "${WF_DBT}"; do
|
||
|
|
[[ -f "${f}" ]] || { echo -e "${ERR} Workflow tidak ditemukan: ${f}"; exit 1; }
|
||
|
|
done
|
||
|
|
if ! docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${AIRFLOW_CONTAINER}$"; then
|
||
|
|
echo -e "${ERR} Container '${AIRFLOW_CONTAINER}' tidak berjalan (butuh Ingestion Framework)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${INFO} Login ke OpenMetadata (${OM_ADMIN_EMAIL})..."
|
||
|
|
# Dua kandidat, MAKSIMAL DUA percobaan — jangan diulang dalam loop (akun admin bisa
|
||
|
|
# terkunci): password .env, lalu default 'admin' yang dipakai OM 2.0.x sampai
|
||
|
|
# bootstrap step 7 menyetel hash-nya. `curl -s` TANPA -f, agar kegagalan HTTP tidak
|
||
|
|
# mematikan skrip di bawah `set -e` — penanganannya ada di blok if berikutnya.
|
||
|
|
TOKEN=""
|
||
|
|
for TRY_PASS in "${OM_ADMIN_PASS}" "admin"; do
|
||
|
|
[[ -n "${TRY_PASS}" ]] || continue
|
||
|
|
PW64="$(printf '%s' "${TRY_PASS}" | base64)"
|
||
|
|
TOKEN="$(curl -s -X POST "${OM_URL}/api/v1/users/login" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"email\":\"${OM_ADMIN_EMAIL}\",\"password\":\"${PW64}\"}" 2>/dev/null |
|
||
|
|
python3 -c "import sys,json; print(json.load(sys.stdin).get('accessToken',''))" 2>/dev/null || true)"
|
||
|
|
[[ -n "${TOKEN}" ]] && break
|
||
|
|
done
|
||
|
|
if [[ -z "${TOKEN}" ]]; then
|
||
|
|
echo -e "${ERR} Gagal login ke OpenMetadata di ${OM_URL} (dicoba: password .env + 'admin')"
|
||
|
|
echo " pastikan admin sudah diset: ./bootstrap.sh --step 7"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Token diperoleh."
|
||
|
|
|
||
|
|
RC=0
|
||
|
|
if [[ -n "${CUSTOM_WORKFLOW}" ]]; then
|
||
|
|
run_workflow "${CUSTOM_WORKFLOW}" "custom" || RC=1
|
||
|
|
else
|
||
|
|
[[ "${MODE}" == "metadata" || "${MODE}" == "all" ]] && { run_workflow "${WF_METADATA}" "metadata" || RC=1; }
|
||
|
|
if [[ "${MODE}" == "dbt" || "${MODE}" == "all" ]]; then
|
||
|
|
# Artefak dbt wajib ada di container (hilang bila container di-recreate,
|
||
|
|
# karena berada di layer container, bukan bind-mount).
|
||
|
|
# Artefak dbt ditulis ke DBT_TARGET_PATH (default /tmp/dbt/target — lihat
|
||
|
|
# docker-compose.yml). DULU diperiksa di ${DBT_PROJECT_DIR}/target, yaitu langsung
|
||
|
|
# di dalam bind-mount repo yang TIDAK bisa ditulis container (uid 50000 vs uid host),
|
||
|
|
# sehingga pemeriksaan ini selalu melaporkan "belum ada" walaupun artefaknya ada.
|
||
|
|
DBT_TARGET_PATH="${DBT_TARGET_PATH:-/tmp/dbt/target}"
|
||
|
|
if ! docker exec "${AIRFLOW_CONTAINER}" test -f "${DBT_TARGET_PATH}/manifest.json" 2>/dev/null; then
|
||
|
|
echo -e "${WARN} Artefak dbt belum ada di container (${DBT_TARGET_PATH})."
|
||
|
|
echo -e "${INFO} Buat dulu (butuh kredensial dbt). Perhatikan DBT_TARGET_PATH/DBT_LOG_PATH:"
|
||
|
|
echo " docker exec -e DBT_TRINO_USER=svc_dbt -e DBT_TRINO_PASSWORD=\$TRINO_SVC_DBT_PASSWORD \\"
|
||
|
|
echo " -e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=${DBT_TARGET_PATH} \\"
|
||
|
|
echo " -e DBT_LOG_PATH=/tmp/dbt/logs -w ${DBT_PROJECT_DIR} ${AIRFLOW_CONTAINER} \\"
|
||
|
|
echo " bash -lc 'dbt docs generate --profile trino --target prod'"
|
||
|
|
RC=1
|
||
|
|
else
|
||
|
|
run_workflow "${WF_DBT}" "lineage dbt" || RC=1
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Bukti hasil ─────────────────────────────────────────────────────────────
|
||
|
|
echo -e "${INFO} Memverifikasi hasil di OpenMetadata..."
|
||
|
|
SUMMARY="$(om_ws GET "/api/v1/tables?service=${SERVICE_NAME}&fields=columns&limit=50" |
|
||
|
|
python3 -c "
|
||
|
|
import sys,json
|
||
|
|
try:
|
||
|
|
d=json.load(sys.stdin).get('data',[])
|
||
|
|
print('{0}|{1}'.format(len(d), sum(len(t.get('columns') or []) for t in d)))
|
||
|
|
except Exception: print('0|0')" 2>/dev/null)"
|
||
|
|
TABLES="${SUMMARY%%|*}"; COLUMNS="${SUMMARY##*|}"
|
||
|
|
if [[ "${TABLES}" =~ ^[0-9]+$ ]] && [[ ${TABLES} -gt 0 ]]; then
|
||
|
|
echo -e "${OK} ${TABLES} tabel ter-indeks (${COLUMNS} kolom) — muncul di Explore."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Belum ada tabel ter-indeks untuk '${SERVICE_NAME}'."
|
||
|
|
RC=1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Lineage (ditampilkan bila workflow dbt dijalankan)
|
||
|
|
if [[ "${MODE}" == "dbt" || "${MODE}" == "all" ]]; then
|
||
|
|
EDGES="$(om_ws GET "/api/v1/lineage/getLineage?fqn=${SERVICE_NAME}.iceberg.bronze.orders&upstreamDepth=0&downstreamDepth=3" |
|
||
|
|
python3 -c "
|
||
|
|
import sys,json
|
||
|
|
try:
|
||
|
|
d=json.load(sys.stdin)
|
||
|
|
print(len(d.get('downstreamEdges') or []))
|
||
|
|
except Exception: print('0')" 2>/dev/null)"
|
||
|
|
if [[ "${EDGES}" =~ ^[0-9]+$ ]] && [[ ${EDGES} -gt 0 ]]; then
|
||
|
|
echo -e "${OK} Lineage terbentuk: ${EDGES} relasi downstream dari bronze.orders."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Lineage belum terbentuk (0 relasi) — periksa keluaran ingestion dbt."
|
||
|
|
RC=1
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
exit ${RC}
|