first commit
This commit is contained in:
Executable
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Jalankan agent ingestion OpenMetadata
|
||||
# =============================================================================
|
||||
# Mengapa script ini ada:
|
||||
# Membuat SERVICE di OpenMetadata hanya menyimpan konfigurasi koneksi. Metadata
|
||||
# (database → schema → tabel → kolom) baru terisi setelah AGENT dijalankan, dan
|
||||
# halaman Explore membaca hasilnya dari search index. Tanpa agent: Explore kosong.
|
||||
#
|
||||
# LINEAGE (bronze→silver→gold) TIDAK dihasilkan oleh ingestion metadata: lineage
|
||||
# berasal dari definisi transformasi, jadi diambil dari ARTEFAK dbt lewat workflow
|
||||
# terpisah (`ingest-dbt.yaml`). Keduanya dijalankan oleh `--all`.
|
||||
#
|
||||
# Script memakai Ingestion Framework yang sudah ada di container Airflow
|
||||
# (`metadata` CLI) — tidak perlu memasang apa pun.
|
||||
#
|
||||
# ⚠️ Cakupan metadata mengikuti ACL identitas agent (workflow memakai `svc_om`,
|
||||
# role `data-catalog` = baca-saja seluruh tabel).
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/ingest-openmetadata.sh # metadata (Trino) saja
|
||||
# ./scripts/ingest-openmetadata.sh --all # metadata + lineage dbt
|
||||
# ./scripts/ingest-openmetadata.sh --dbt-only # lineage dbt saja
|
||||
# ./scripts/ingest-openmetadata.sh --workflow config/openmetadata/ingest-dbt.yaml
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# shellcheck source=../lib/common.sh
|
||||
source "${SCRIPT_DIR}/lib/common.sh"
|
||||
# shellcheck source=../config/environment.defaults.sh
|
||||
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||||
sicabe_load_env "${SCRIPT_DIR}" "$@"
|
||||
|
||||
OM_URL="${OPENMETADATA_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||||
OM_ADMIN_EMAIL="${OPENMETADATA_ADMIN_EMAIL:-admin@open-metadata.org}"
|
||||
OM_ADMIN_PASS="${OPENMETADATA_ADMIN_PASSWORD:-openmetadata123}"
|
||||
SERVICE_NAME="${OPENMETADATA_TRINO_SERVICE:-trino_sicabe}"
|
||||
AIRFLOW_CONTAINER="${AIRFLOW_INGEST_CONTAINER:-sicabe-airflow-webserver}"
|
||||
REMOTE_DIR="/tmp/om-ingest"
|
||||
DBT_PROJECT_DIR="${DBT_PROJECT_DIR:-/opt/airflow/dbt/sicabe}"
|
||||
|
||||
# Di multi-host, pakai workflow TER-RENDER dari bundle host (hostname → alamat host);
|
||||
# sumber `config/**` masih memakai nama container (single-host). `SICABE_BUNDLE_DIR`
|
||||
# diset sicabe_load_env(): single-host = deploy/ (config sumber), multi-host =
|
||||
# generated/host-<x>/ (config ter-render).
|
||||
CONFIG_DIR="${SICABE_BUNDLE_DIR:-${SCRIPT_DIR}}/config"
|
||||
WF_METADATA="${CONFIG_DIR}/openmetadata/ingest-trino.yaml"
|
||||
WF_DBT="${CONFIG_DIR}/openmetadata/ingest-dbt.yaml"
|
||||
MODE="metadata"
|
||||
CUSTOM_WORKFLOW=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--env-file) shift 2 ;; # sudah dipakai sicabe_load_env() di atas
|
||||
--all) MODE="all"; shift ;;
|
||||
--dbt-only) MODE="dbt"; shift ;;
|
||||
--workflow) CUSTOM_WORKFLOW="$2"; shift 2 ;;
|
||||
*) echo "Argumen tidak dikenal: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Helper ──────────────────────────────────────────────────────────────────
|
||||
om_ws() { # om_ws <method> <path> [body]
|
||||
local method="$1" path="$2" body="${3:-}"
|
||||
if [[ -n "${body}" ]]; then
|
||||
curl -s --max-time 30 -X "${method}" "${OM_URL}${path}" \
|
||||
-H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" -d "${body}" 2>/dev/null
|
||||
else
|
||||
curl -s --max-time 30 -X "${method}" "${OM_URL}${path}" -H "Authorization: Bearer ${TOKEN}" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
run_workflow() { # run_workflow <path-workflow> <label>
|
||||
local wf="$1" label="$2" remote
|
||||
remote="${REMOTE_DIR}/$(basename "${wf}")"
|
||||
if [[ ! -f "${wf}" ]]; then
|
||||
echo -e "${ERR} Workflow tidak ditemukan: ${wf}"
|
||||
return 1
|
||||
fi
|
||||
echo -e "${INFO} [${label}] menyalin ${wf##*/} ke container..."
|
||||
docker exec "${AIRFLOW_CONTAINER}" mkdir -p "${REMOTE_DIR}"
|
||||
docker cp "${wf}" "${AIRFLOW_CONTAINER}:${remote}" >/dev/null
|
||||
|
||||
echo -e "${INFO} [${label}] menjalankan ingestion..."
|
||||
local out
|
||||
out="$(docker exec -e TRINO_SVC_OM_PASSWORD="${TRINO_SVC_OM_PASSWORD:-}" -e OM_JWT_TOKEN="${TOKEN}" \
|
||||
"${AIRFLOW_CONTAINER}" metadata ingest -c "${remote}" 2>&1)"
|
||||
echo "${out}" | sed 's/\x1b\[[0-9;]*m//g' |
|
||||
grep -iE "Processed records|Errors:|Success %|Workflow Success|Error running node|ERROR" | tail -8
|
||||
if ! echo "${out}" | grep -q "Workflow Success %: 100.0"; then
|
||||
echo -e "${ERR} [${label}] ingestion tidak 100% sukses — periksa keluaran di atas."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Prasyarat ───────────────────────────────────────────────────────────────
|
||||
for f in "${WF_METADATA}" "${WF_DBT}"; do
|
||||
[[ -f "${f}" ]] || { echo -e "${ERR} Workflow tidak ditemukan: ${f}"; exit 1; }
|
||||
done
|
||||
if ! docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${AIRFLOW_CONTAINER}$"; then
|
||||
echo -e "${ERR} Container '${AIRFLOW_CONTAINER}' tidak berjalan (butuh Ingestion Framework)."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${INFO} Login ke OpenMetadata (${OM_ADMIN_EMAIL})..."
|
||||
# Dua kandidat, MAKSIMAL DUA percobaan — jangan diulang dalam loop (akun admin bisa
|
||||
# terkunci): password .env, lalu default 'admin' yang dipakai OM 2.0.x sampai
|
||||
# bootstrap step 7 menyetel hash-nya. `curl -s` TANPA -f, agar kegagalan HTTP tidak
|
||||
# mematikan skrip di bawah `set -e` — penanganannya ada di blok if berikutnya.
|
||||
TOKEN=""
|
||||
for TRY_PASS in "${OM_ADMIN_PASS}" "admin"; do
|
||||
[[ -n "${TRY_PASS}" ]] || continue
|
||||
PW64="$(printf '%s' "${TRY_PASS}" | base64)"
|
||||
TOKEN="$(curl -s -X POST "${OM_URL}/api/v1/users/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"email\":\"${OM_ADMIN_EMAIL}\",\"password\":\"${PW64}\"}" 2>/dev/null |
|
||||
python3 -c "import sys,json; print(json.load(sys.stdin).get('accessToken',''))" 2>/dev/null || true)"
|
||||
[[ -n "${TOKEN}" ]] && break
|
||||
done
|
||||
if [[ -z "${TOKEN}" ]]; then
|
||||
echo -e "${ERR} Gagal login ke OpenMetadata di ${OM_URL} (dicoba: password .env + 'admin')"
|
||||
echo " pastikan admin sudah diset: ./bootstrap.sh --step 7"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${OK} Token diperoleh."
|
||||
|
||||
RC=0
|
||||
if [[ -n "${CUSTOM_WORKFLOW}" ]]; then
|
||||
run_workflow "${CUSTOM_WORKFLOW}" "custom" || RC=1
|
||||
else
|
||||
[[ "${MODE}" == "metadata" || "${MODE}" == "all" ]] && { run_workflow "${WF_METADATA}" "metadata" || RC=1; }
|
||||
if [[ "${MODE}" == "dbt" || "${MODE}" == "all" ]]; then
|
||||
# Artefak dbt wajib ada di container (hilang bila container di-recreate,
|
||||
# karena berada di layer container, bukan bind-mount).
|
||||
# Artefak dbt ditulis ke DBT_TARGET_PATH (default /tmp/dbt/target — lihat
|
||||
# docker-compose.yml). DULU diperiksa di ${DBT_PROJECT_DIR}/target, yaitu langsung
|
||||
# di dalam bind-mount repo yang TIDAK bisa ditulis container (uid 50000 vs uid host),
|
||||
# sehingga pemeriksaan ini selalu melaporkan "belum ada" walaupun artefaknya ada.
|
||||
DBT_TARGET_PATH="${DBT_TARGET_PATH:-/tmp/dbt/target}"
|
||||
if ! docker exec "${AIRFLOW_CONTAINER}" test -f "${DBT_TARGET_PATH}/manifest.json" 2>/dev/null; then
|
||||
echo -e "${WARN} Artefak dbt belum ada di container (${DBT_TARGET_PATH})."
|
||||
echo -e "${INFO} Buat dulu (butuh kredensial dbt). Perhatikan DBT_TARGET_PATH/DBT_LOG_PATH:"
|
||||
echo " docker exec -e DBT_TRINO_USER=svc_dbt -e DBT_TRINO_PASSWORD=\$TRINO_SVC_DBT_PASSWORD \\"
|
||||
echo " -e DBT_PROFILES_DIR=/opt/airflow/dbt -e DBT_TARGET_PATH=${DBT_TARGET_PATH} \\"
|
||||
echo " -e DBT_LOG_PATH=/tmp/dbt/logs -w ${DBT_PROJECT_DIR} ${AIRFLOW_CONTAINER} \\"
|
||||
echo " bash -lc 'dbt docs generate --profile trino --target prod'"
|
||||
RC=1
|
||||
else
|
||||
run_workflow "${WF_DBT}" "lineage dbt" || RC=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Bukti hasil ─────────────────────────────────────────────────────────────
|
||||
echo -e "${INFO} Memverifikasi hasil di OpenMetadata..."
|
||||
SUMMARY="$(om_ws GET "/api/v1/tables?service=${SERVICE_NAME}&fields=columns&limit=50" |
|
||||
python3 -c "
|
||||
import sys,json
|
||||
try:
|
||||
d=json.load(sys.stdin).get('data',[])
|
||||
print('{0}|{1}'.format(len(d), sum(len(t.get('columns') or []) for t in d)))
|
||||
except Exception: print('0|0')" 2>/dev/null)"
|
||||
TABLES="${SUMMARY%%|*}"; COLUMNS="${SUMMARY##*|}"
|
||||
if [[ "${TABLES}" =~ ^[0-9]+$ ]] && [[ ${TABLES} -gt 0 ]]; then
|
||||
echo -e "${OK} ${TABLES} tabel ter-indeks (${COLUMNS} kolom) — muncul di Explore."
|
||||
else
|
||||
echo -e "${WARN} Belum ada tabel ter-indeks untuk '${SERVICE_NAME}'."
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# Lineage (ditampilkan bila workflow dbt dijalankan)
|
||||
if [[ "${MODE}" == "dbt" || "${MODE}" == "all" ]]; then
|
||||
EDGES="$(om_ws GET "/api/v1/lineage/getLineage?fqn=${SERVICE_NAME}.iceberg.bronze.orders&upstreamDepth=0&downstreamDepth=3" |
|
||||
python3 -c "
|
||||
import sys,json
|
||||
try:
|
||||
d=json.load(sys.stdin)
|
||||
print(len(d.get('downstreamEdges') or []))
|
||||
except Exception: print('0')" 2>/dev/null)"
|
||||
if [[ "${EDGES}" =~ ^[0-9]+$ ]] && [[ ${EDGES} -gt 0 ]]; then
|
||||
echo -e "${OK} Lineage terbentuk: ${EDGES} relasi downstream dari bronze.orders."
|
||||
else
|
||||
echo -e "${WARN} Lineage belum terbentuk (0 relasi) — periksa keluaran ingestion dbt."
|
||||
RC=1
|
||||
fi
|
||||
fi
|
||||
|
||||
exit ${RC}
|
||||
Reference in New Issue
Block a user