sync dari sicabe-doc/deploy @ bc1a6e6
This commit is contained in:
@@ -109,6 +109,170 @@ for pair in "MINIO_IMAGE:quay.io/minio/minio" "MINIO_MC_IMAGE:quay.io/minio/mc"
|
||||
fi
|
||||
done
|
||||
|
||||
# ─── 0e. Komponen EKSTERNAL ber-`address` (skenario P3) ──────────────────────
|
||||
# `components.<c>.address` = komponen dikelola SISTEM LAIN → TIDAK dijalankan cluster
|
||||
# (tak muncul di services host mana pun) + endpoint-nya = alamat itu. Diuji via cluster.p3.yml.
|
||||
p3_problems="$(python3 - "${DEPLOY_DIR}" <<'PY'
|
||||
import sys
|
||||
sys.path.insert(0, sys.argv[1] + "/lib/cluster")
|
||||
import inventory
|
||||
d = inventory.apply_preset(inventory.load(sys.argv[1] + "/cluster.p3.yml"))
|
||||
pl = inventory.plan(d)
|
||||
svc = {s for h in pl.values() for s in h["services"]}
|
||||
p = []
|
||||
for c in ("keycloak", "minio", "nessie"):
|
||||
if c in svc:
|
||||
p.append(c + " masih dijalankan")
|
||||
if not inventory.external_address(d, c):
|
||||
p.append(c + ": address hilang")
|
||||
if "openldap" not in pl.get("control", {}).get("services", []):
|
||||
p.append("openldap tak di control")
|
||||
print("; ".join(p))
|
||||
PY
|
||||
)"
|
||||
if [[ -z "${p3_problems}" ]]; then
|
||||
ok "P3: components.<c>.address → eksternal tak dijalankan, openldap lokal di control"
|
||||
else
|
||||
bad "P3: perilaku components.<c>.address salah — ${p3_problems}"
|
||||
fi
|
||||
|
||||
# ─── 0f. Deteksi keystore Trino BEDA antar-host ──────────────────────────────
|
||||
# Keystore Trino harus identik di semua host (satu --generate + --sync). Skrip ini
|
||||
# mendeteksi divergensi (mis. --generate dijalankan di >1 host).
|
||||
cks="${DEPLOY_DIR}/scripts/check-trino-keystore.sh"
|
||||
if [[ -x "${cks}" ]] && bash -n "${cks}" 2>/dev/null \
|
||||
&& grep -q -- '--inventory' "${cks}" && grep -q -- '--expect' "${cks}" \
|
||||
&& grep -q 'keystore.p12' "${cks}" && grep -q 'password.db' "${cks}" \
|
||||
&& grep -q 'svc_dbt' "${cks}"; then
|
||||
ok "check-trino-keystore.sh: deteksi keystore + password.db antar-host (--inventory/--expect)"
|
||||
else
|
||||
bad "check-trino-keystore.sh hilang/tidak lengkap (divergensi keystore/password.db tak terdeteksi)"
|
||||
fi
|
||||
# Fungsional (bila bundle host sudah ada): cetak sha256 & hormati --expect.
|
||||
_first_bundle="$(ls -d "${DEPLOY_DIR}"/generated/host-* 2>/dev/null | head -1)"
|
||||
if [[ -n "${_first_bundle}" ]]; then
|
||||
_bh="$(basename "${_first_bundle}" | sed 's/^host-//')"
|
||||
_sha="$("${cks}" --host "${_bh}" 2>/dev/null | sed -n 's/^ keystore : sha256=//p')"
|
||||
if [[ "${_sha}" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
ok "check-trino-keystore.sh: sha256 host '${_bh}' terbaca"
|
||||
"${cks}" --host "${_bh}" --expect "${_sha}" >/dev/null 2>&1 \
|
||||
&& ok "check-trino-keystore.sh: --expect cocok → exit 0" \
|
||||
|| bad "check-trino-keystore.sh: --expect cocok seharusnya exit 0"
|
||||
else
|
||||
bad "check-trino-keystore.sh: sha256 tidak terbaca untuk host '${_bh}'"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── 0g. .env auto-generate memuat SEMUA var WAJIB compose ───────────────────
|
||||
# Regresi nyata: STARROCKS_ROOT_PASSWORD (dipakai `${VAR:?}` di compose) TIDAK dibuat
|
||||
# install.sh → `docker compose --env-file .env` (tanpa shell env) gagal
|
||||
# "required variable ... is missing a value".
|
||||
_required="$(grep -oE '\$\{[A-Z_][A-Z0-9_]*:\?' "${DEPLOY_DIR}/docker-compose.yml" | sed 's/\${//;s/:?//' | sort -u)"
|
||||
_envvars="$(awk '/^[[:space:]]*cat > .*\/\.env" <<EOF/{f=1;next} f&&/^EOF$/{exit} f' "${DEPLOY_DIR}/install.sh" \
|
||||
| grep -oE '^[A-Z_][A-Z0-9_]*=' | tr -d '=' | sort -u)"
|
||||
_miss_env=""
|
||||
for _v in ${_required}; do grep -qx "${_v}" <<<"${_envvars}" || _miss_env+=" ${_v}"; done
|
||||
if [[ -z "${_miss_env}" ]]; then
|
||||
ok ".env auto-generate memuat semua var wajib compose (${_required//$'\n'/ })"
|
||||
else
|
||||
bad "var wajib compose TIDAK di-generate ke .env:${_miss_env}"
|
||||
fi
|
||||
|
||||
# ─── 0h. Orkestrasi alur data E2E (run-e2e.sh) ───────────────────────────────
|
||||
e2e="${DEPLOY_DIR}/scripts/run-e2e.sh"
|
||||
if [[ -x "${e2e}" ]] && bash -n "${e2e}" 2>/dev/null \
|
||||
&& grep -q 'ingest-openmetadata.sh' "${e2e}" \
|
||||
&& grep -q 'airflow dags test' "${e2e}" \
|
||||
&& grep -q 'verify-phase6-governance.sh' "${e2e}"; then
|
||||
ok "run-e2e.sh: orkestrasi alur data E2E (ingest→DAG→ingest→DAG→verify)"
|
||||
else
|
||||
bad "run-e2e.sh hilang/tidak lengkap"
|
||||
fi
|
||||
|
||||
# ─── 0i. Prometheus: target MinIO EKSTERNAL → alamat:port publik ─────────────
|
||||
# Prometheus tak mendukung substitusi env, dan render-config hanya memetakan komponen
|
||||
# milik host lain — komponen EKSTERNAL (components.minio.address) harus di-rewrite
|
||||
# generate.py, kalau tidak target tetap `minio:9000` → scrape gagal dari host prometheus.
|
||||
_tmp_prom="$(mktemp -d)"; _prom_ok=""
|
||||
if [[ -f "${DEPLOY_DIR}/.env" ]]; then
|
||||
cp "${DEPLOY_DIR}/.env" "${_tmp_prom}/e.env"
|
||||
printf '\nMINIO_INTERNAL_PORT=12900\n' >> "${_tmp_prom}/e.env"
|
||||
if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${DEPLOY_DIR}/cluster.p3.yml" "${DEPLOY_DIR}/docker-compose.yml" \
|
||||
"${_tmp_prom}/e.env" "${_tmp_prom}/out" "${DEPLOY_DIR}/scripts/render-config.sh" \
|
||||
>/dev/null 2>&1; then
|
||||
_pyml="${_tmp_prom}/out/host-control/config/prometheus/prometheus.yml"
|
||||
if grep -q 'targets: \["100.93.54.108:12900"\]' "${_pyml}" 2>/dev/null \
|
||||
&& ! grep -q 'targets: \["minio:9000"\]' "${_pyml}" 2>/dev/null; then
|
||||
_prom_ok=1
|
||||
fi
|
||||
fi
|
||||
if [[ -n "${_prom_ok}" ]]; then
|
||||
ok "prometheus: target MinIO eksternal → 100.93.54.108:12900 (bukan minio:9000)"
|
||||
else
|
||||
bad "prometheus: target MinIO eksternal salah (masih minio:9000?)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── 0j. Verifikasi Keycloak lengkap (verify-keycloak.sh) ────────────────────
|
||||
vk="${DEPLOY_DIR}/scripts/verify-keycloak.sh"
|
||||
if [[ -x "${vk}" ]] && bash -n "${vk}" 2>/dev/null \
|
||||
&& grep -q 'client_credentials' "${vk}" && grep -q 'redirectUris' "${vk}" \
|
||||
&& grep -q 'publicClient' "${vk}" && grep -q 'origin host' "${vk}" \
|
||||
&& grep -q 'kc_auth_probe' "${vk}" && grep -q 'password masih DEFAULT' "${vk}"; then
|
||||
ok "verify-keycloak.sh: realm/roles/client(redirectUri origin-host,secret,publicClient)/user/password-default + fungsional"
|
||||
else
|
||||
bad "verify-keycloak.sh hilang/tidak lengkap"
|
||||
fi
|
||||
|
||||
# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_HOST (bukan literal `keycloak`) ──
|
||||
# Kls bug sama dgn target MinIO Prometheus: Trino server-ke-server ke Keycloak. Bila
|
||||
# Keycloak EKSTERNAL (P3), literal `keycloak:` tak resolve → penukaran token gagal →
|
||||
# callback `/oauth2/callback` balas "Authentication response could not be verified".
|
||||
_tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties"
|
||||
if grep -q 'token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'jwks-url=http://${ENV:KEYCLOAK_HOST}:' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}' "${_tc}" 2>/dev/null \
|
||||
&& grep -q 'KEYCLOAK_HOST: ${KEYCLOAK_HOST' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
|
||||
&& grep -q 'TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \
|
||||
&& ! grep -qE 'token-url=http://keycloak:' "${_tc}" 2>/dev/null; then
|
||||
ok "trino oauth2: token-url/jwks-url pakai KEYCLOAK_HOST + state-key tetap"
|
||||
else
|
||||
bad "trino oauth2: token-url/jwks-url literal 'keycloak:' atau state-key belum diset"
|
||||
fi
|
||||
|
||||
# ─── 0l. StarRocks OIDC: redirect_url = alamat browser (bukan localhost) ──────
|
||||
if ! grep -qE "redirect_url' = 'http://localhost" "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
|
||||
&& grep -q 'sr_redirect_url=' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null \
|
||||
&& grep -q 'bootstrap_browser_addr' "${DEPLOY_DIR}/bootstrap.sh" 2>/dev/null; then
|
||||
ok "StarRocks OIDC: redirect_url pakai alamat browser (bukan localhost literal)"
|
||||
else
|
||||
bad "StarRocks OIDC: redirect_url masih localhost literal"
|
||||
fi
|
||||
|
||||
# ─── 0m. Tujuan `--sync` BEBAS nama repo (cermin $HOME, bukan ~/sicabe-doc) ───
|
||||
# Regresi: dest default dipatok `~/sicabe-doc/deploy` → repo distribusi bersih
|
||||
# (`sicabe-core`, isinya hanya deploy/) salah tujuan & membuat salinan kedua
|
||||
# `~/sicabe-doc/deploy`. Nilai harus DITURUNKAN dari lokasi script.
|
||||
dest_probe="$(HOME=/home/u bash -c '
|
||||
# shellcheck disable=SC1091
|
||||
source "'"${DEPLOY_DIR}"'/lib/common.sh" >/dev/null 2>&1 || exit 9
|
||||
for d in /home/u/sicabe-core/deploy /home/u/sicabe-doc/deploy /opt/apps/sicabe-core/deploy; do
|
||||
printf "%s|" "$(sicabe_default_dest "$d")"
|
||||
done' 2>/dev/null)"
|
||||
if [[ "${dest_probe}" == "~/sicabe-core/deploy|~/sicabe-doc/deploy|~/sicabe-core/deploy|" ]]; then
|
||||
ok "--sync: tujuan default mencerminkan nama repo (sicabe-core → ~/sicabe-core/deploy)"
|
||||
else
|
||||
bad "--sync: sicabe_default_dest salah → '${dest_probe}'"
|
||||
fi
|
||||
if ! grep -q 'dest="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/install.sh" \
|
||||
&& ! grep -q 'DEST="~/sicabe-doc/deploy"' "${DEPLOY_DIR}/scripts/check-trino-keystore.sh" \
|
||||
&& grep -q 'sicabe_default_dest' "${DEPLOY_DIR}/install.sh"; then
|
||||
ok "--sync: tidak ada lagi path repo ter-hardcode (~/sicabe-doc/deploy)"
|
||||
else
|
||||
bad "--sync: masih ada path repo ter-hardcode (~/sicabe-doc/deploy)"
|
||||
fi
|
||||
|
||||
# ─── 1. generate ────────────────────────────────────────────────────────────
|
||||
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
|
||||
|
||||
Reference in New Issue
Block a user