platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync dari sicabe-doc/deploy @ bc1a6e6

Browse Source
This commit is contained in:
subroto
2026-10-11 13:59:47 +07:00
parent 4ca7a9a58e
commit cbabe96061
29 changed files with 1210 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/bootstrap.sh
+120 -19
View File
@@ -94,6 +94,16 @@ ldap_endpoint() {
fi
}
# Alamat BROWSER-reachable komponen (tempat user memakai UI di browser): nilai <host_var>
# bila berupa ALAMAT (mengandung titik); kalau tidak → SICABE_HOST_ADDRESS (multi-host);
# kalau tak ada → localhost (single-host). Dipakai redirect_url OIDC StarRocks & origin
# redirectUris Keycloak (JANGAN nama container/hostname internal — browser tak bisa resolve).
bootstrap_browser_addr() { # <host_var>
local v="${!1:-}"
if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; return; fi
echo "${SICABE_HOST_ADDRESS:-localhost}"
}
# =============================================================================
# STEP 1: Keycloak — Realm, Client, User, Role (bab 5.1)
# =============================================================================
@@ -218,6 +228,28 @@ bootstrap_keycloak() {
["grafana"]="${GRAFANA_PUBLIC_URL:-}"
)
# Origin HOST (skema://alamat:port) — tempat browser MEMBUKA komponen. WAJIB ada di
# redirectUris, kalau tidak login via ALAMAT (mis. https://100.123.123.71:12443) ditolak
# "Invalid parameter: redirect_uri" (redirectUris hanya localhost). Sinkron dgn
# scripts/verify-keycloak.sh.
# <client>="<scheme> <host_var> <port_var> <default_port>"
declare -A CLIENT_ORIGIN=(
["trino"]="https TRINO_HOST TRINO_HTTPS_PORT 12443"
["grafana"]="http GRAFANA_HOST GRAFANA_PORT 12300"
["openmetadata"]="http OPENMETADATA_HOST OPENMETADATA_PORT 12085"
["airflow"]="http AIRFLOW_HOST AIRFLOW_PORT 12082"
["starrocks"]="http STARROCKS_FE_HOST STARROCKS_HTTP_PORT 12030"
)
_comp_addr() { # <host_var> → alamat efektif (IP/alamat, atau host ini bila co-located)
local v="${!1:-}"
if [[ "${v}" == *.* && "${v}" != "127.0.0.1" ]]; then echo "${v}"; else echo "${SICABE_HOST_ADDRESS:-localhost}"; fi
}
_client_origin() { # <client> → skema://alamat:port (atau kosong)
local spec="${CLIENT_ORIGIN[$1]:-}"; [[ -n "${spec}" ]] || { echo ""; return; }
local s hv pv dp; read -r s hv pv dp <<< "${spec}"
printf '%s://%s:%s' "${s}" "$(_comp_addr "${hv}")" "${!pv:-${dp}}"
}
for client_id in "${!CLIENTS[@]}"; do
local entry="${CLIENTS[$client_id]}"
IFS="|" read -r desc secret <<< "${entry}"
@@ -260,25 +292,38 @@ bootstrap_keycloak() {
# yang membuat login browser lewat alamat publik ditolak:
# Invalid parameter: redirect_uri
if [[ -n "${client_uuid}" ]]; then
local _payload
local _payload _host_origin
_host_origin="$(_client_origin "${client_id}")"
_payload=$(curl -sf -H "Authorization: Bearer ${token}" \
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" \
| PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" python3 -c '
| PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" HOST_ORIGIN="${_host_origin}" \
EXTRA_ORIGINS="${SICABE_CLIENT_REDIRECT_ORIGINS:-}" python3 -c '
import json, os, sys
c = json.load(sys.stdin)
pub = (os.environ.get("PUBLIC_ORIGIN") or "").rstrip("/")
host = (os.environ.get("HOST_ORIGIN") or "").rstrip("/")
redirects = set(c.get("redirectUris") or []) | {"http://localhost:*", "https://localhost:*"}
origins = set(c.get("webOrigins") or []) | {"http://localhost:*", "https://localhost:*"}
if host:
redirects.add(host + "/*")
origins.add(host)
if pub:
redirects.add(pub + "/*")
origins.add(pub)
# Origin client dari CLUSTER (host existing tak tahu alamat host cluster) — daftar
# spasi-dipisah, mis. SICABE_CLIENT_REDIRECT_ORIGINS="https://100.123.123.71:12443 ...".
for o in (os.environ.get("EXTRA_ORIGINS") or "").split():
o = o.rstrip("/")
if o:
redirects.add(o + "/*")
origins.add(o)
print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origins)}))
' 2>/dev/null || echo "")
if [[ -n "${_payload}" && -n "${CLIENT_PUBLIC_URL[$client_id]:-}" ]]; then
if [[ -n "${_payload}" ]]; then
curl -sf -X PUT -H "Authorization: Bearer ${token}" \
-H "Content-Type: application/json" -d "${_payload}" \
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" >/dev/null 2>&1 || true
echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (${CLIENT_PUBLIC_URL[$client_id]})."
echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (origin host ${_host_origin:-—}; publik ${CLIENT_PUBLIC_URL[$client_id]:-—})."
fi
fi
@@ -312,7 +357,8 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin
# StarRocks FE mengirim redirect internal (http://starrocks-fe:8030/api/oauth2)
# saat membangun URL OAuth2 → WAJIB terdaftar, jika tidak Keycloak menolak
# dengan "Invalid parameter: redirect_uri". Pola host dipakai untuk callback browser.
local sr_client_id sr_client_json
local sr_client_id sr_client_json sr_fe_browser_addr
sr_fe_browser_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)"
sr_client_id=$(curl -sf -H "Authorization: Bearer ${token}" \
"${kc_base}/admin/realms/${realm}/clients?clientId=starrocks" | \
python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['id'] if d else '')" 2>/dev/null || echo "")
@@ -322,7 +368,7 @@ print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origin
python3 -c "
import sys,json
c=json.load(sys.stdin)
extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*']
extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*','http://${sr_fe_browser_addr}:${STARROCKS_HTTP_PORT:-12030}/*']
c['redirectUris']=sorted(set((c.get('redirectUris') or [])+extra))
print(json.dumps(c))
" 2>/dev/null || true)
@@ -451,12 +497,20 @@ bootstrap_minio() {
local mc_endpoint="${S3_ENDPOINT:-http://minio:9000}"
local mc_user="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
local mc_pass="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
# Nama bucket = config (MINIO_BUCKET_ICEBERG_DATA), bukan hardcoded.
local bucket="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
local mc_image="quay.io/minio/mc:${MINIO_MC_VERSION:-RELEASE.2025-08-13T08-35-41Z}"
local local_minio_ep="http://minio:9000"
# Penantian via host hanya relevan untuk MinIO lokal (S3 existing tidak di localhost).
if [[ "${mc_endpoint}" == "${local_minio_ep}" ]]; then
# Pemilik storage ⇔ container MinIO ada di bundle host INI (single-host → selalu
# lokal, krena SICABE_HOST_NAME kosong). Bila storage dikelola SISTEM LAIN
# (mis. sisi existing P1/P3), host ini HANYA MEMVERIFIKASI bucket — pembuatan
# bucket adalah tugas sisi PEMILIK storage. Ini menghindari cluster "menciptakan"
# bucket di storage pihak lain tanpa otoritas.
local mc_mode="create"
sicabe_host_expects_container sicabe-minio || mc_mode="verify"
# Penantian via host hanya relevan untuk MinIO lokal milik host ini.
if [[ "${mc_mode}" == "create" ]]; then
echo -e "${INFO} Menunggu MinIO siap..."
for i in $(seq 1 15); do
if curl -sf "http://localhost:${MINIO_PORT:-12900}/minio/health/live" >/dev/null 2>&1; then
@@ -467,11 +521,11 @@ bootstrap_minio() {
sleep 2
done
else
echo -e "${INFO} Object storage eksternal (${mc_endpoint}) — lewati penantian host."
echo -e "${INFO} Object storage dikelola sistem lain (${mc_endpoint}) — mode VERIFIKASI bucket."
fi
if [[ -z "${mc_pass}" ]]; then
echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati pembuatan bucket."
echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati cek bucket."
return 0
fi
@@ -488,12 +542,13 @@ bootstrap_minio() {
local out
out=$(docker run --rm -i \
--network "${SICABE_NETWORK:-sicabe-network}" \
-e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" -e BUCKET="${bucket}" \
-e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" \
-e BUCKET="${bucket}" -e MODE="${mc_mode}" \
--entrypoint sh "${mc_image}" -c '
if ! mc alias set probe "$S3E" "$S3U" "$S3P" >/dev/null 2>&1; then echo ALIAS_FAIL; exit 0; fi
if mc ls "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_OK
elif mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED
else echo BUCKET_FAIL; fi' 2>&1)
elif [ "$MODE" = "create" ] && mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED
else echo BUCKET_MISSING; fi' 2>&1)
case "${out}" in
*ALIAS_FAIL*)
@@ -507,9 +562,14 @@ else echo BUCKET_FAIL; fi' 2>&1)
*BUCKET_CREATED*)
echo -e "${OK} Bucket '${bucket}' dibuat."
;;
*)
echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:"
echo " ./scripts/verify-phase2-storage.sh"
*BUCKET_MISSING*)
if [[ "${mc_mode}" == "verify" ]]; then
echo -e "${WARN} Bucket '${bucket}' belum ada — storage dikelola sistem lain; buat di sisi PEMILIK storage."
echo " Bucket = MINIO_BUCKET_ICEBERG_DATA (config); konfirmasi ke admin storage existing."
else
echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:"
echo " ./scripts/verify-phase2-storage.sh"
fi
return 0
;;
esac
@@ -719,6 +779,12 @@ bootstrap_starrocks() {
# ── 2. Security integration OIDC (Keycloak) ─────────────────────────────
echo -e "${INFO} [2/6] Security integration OIDC..."
# redirect_url = URL yang dibuka BROWSER (host FE + port HTTP publik) — bukan localhost/
# nama container. Keycloak memakai nilai ini LITERAL sebagai redirect_uri.
local sr_redirect_addr sr_redirect_url
sr_redirect_addr="$(bootstrap_browser_addr STARROCKS_FE_HOST)"
sr_redirect_url="http://${sr_redirect_addr}:${STARROCKS_HTTP_PORT:-12030}/api/oauth2"
echo -e "${INFO} redirect_url OIDC StarRocks: ${sr_redirect_url}"
if ! sr_mysql "SHOW SECURITY INTEGRATIONS;" | grep -qi "sr_oidc"; then
# redirect_url dipakai LITERAL sebagai redirect_uri & harus reachable dari
# browser (DBeaver membuka browser ke sini). JANGAN pakai hostname internal.
@@ -728,7 +794,7 @@ bootstrap_starrocks() {
'token_server_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token',
'client_id' = 'starrocks',
'client_secret' = '${STARROCKS_CLIENT_SECRET}',
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
'redirect_url' = '${sr_redirect_url}',
'jwks_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/certs',
'principal_field' = 'preferred_username',
'required_issuer' = '${kc_base}/realms/${SICABE_REALM:-sicabe}',
@@ -756,7 +822,7 @@ bootstrap_starrocks() {
'ldap_cache_refresh_interval' = '900'
);" && echo -e "${OK} Group provider 'sr_ldap' dibuat/disinkronkan." || echo -e "${WARN} CREATE GROUP PROVIDER gagal."
sr_mysql "ALTER SECURITY INTEGRATION sr_oidc SET (
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
'redirect_url' = '${sr_redirect_url}',
'group_provider' = 'sr_ldap',
'permitted_groups' = 'data-admin,data-engineer,data-analyst,data-viewer',
'auth_server_url' = '${kc_base}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth',
@@ -1504,6 +1570,27 @@ step_tiers() {
esac
}
# Container yang menandai sebuah step BERJALAN di host ini. Dipakai mode `--host`
# untuk MELEWATI step yang komponennya dikelola SISTEM LAIN (tidak ada di bundle host
# ini) — mis. keycloak/nessie existing pada skenario P1/P3. Tanpa ini, `--host control`
# tetap mencoba bootstrap Keycloak meski Keycloak ada di host existing.
# Step 2 (MinIO) sengaja DIPERKECUALIKAN oleh pemanggil: bila storage dikelola sistem
# lain, step itu tetap jalan dalam MODE VERIFIKASI (bukan skip).
step_container() {
case "$1" in
1) echo sicabe-keycloak ;;
2) echo sicabe-minio ;;
3) echo sicabe-openldap ;;
4) echo sicabe-trino-coordinator ;;
5) echo sicabe-starrocks-fe ;;
6) echo sicabe-airflow-webserver ;;
7) echo sicabe-openmetadata ;;
8) echo sicabe-grafana ;;
9) echo sicabe-dremio ;;
*) echo "" ;;
esac
}
# =============================================================================
main() {
local START_STEP=1
@@ -1562,6 +1649,9 @@ main() {
# sicabe_load_env(): tanpa ini sub-script menimpa endpoint host dengan deploy/.env
# (nilai single-host) → menyentuh endpoint yang salah di multi-host.
export SICABE_ENV_FILE="${henv}"
# Bundle host ini — dipakai sicabe_host_expects_container() agar tahu komponen mana
# yang dijalankan host ini (untuk melewati step milik sistem lain, lihat loop step).
export SICABE_BUNDLE_DIR="$(dirname "${henv}")"
# Generator menulis daftar tier dengan pemisah KOMA (SICABE_HOST_TIERS=
# identity,policy,pipeline,governance); pencocokan di bawah memakai spasi.
HOST_TIERS="${SICABE_HOST_TIERS//,/ }"
@@ -1624,6 +1714,17 @@ main() {
echo ""
continue
fi
# Komponen dikelola SISTEM LAIN (tak ada di bundle host ini) → step tak berlaku.
# Step 2 (MinIO) DIPERKECUALIKAN: tetap jalan untuk memverifikasi bucket existing.
if [[ -n "${HOST_NAME}" && "${step}" != "2" ]]; then
local _step_ctr
_step_ctr="$(step_container "${step}")"
if [[ -n "${_step_ctr}" ]] && ! sicabe_host_expects_container "${_step_ctr}"; then
echo -e "${INFO} [skip] step ${step} — '${_step_ctr}' dikelola sistem lain di host ini"
echo ""
continue
fi
fi
case ${step} in
1) bootstrap_keycloak ;;
2) bootstrap_minio ;;