1656 lines
78 KiB
Bash
1656 lines
78 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# eBdesk SiCABE — Bootstrap: Inisialisasi & Integrasi Lintas Lapisan
|
||
|
|
# =============================================================================
|
||
|
|
# Script ini menjalankan inisialisasi dan integrasi antar komponen setelah
|
||
|
|
# semua container berhasil dideploy oleh install.sh.
|
||
|
|
#
|
||
|
|
# Sesuai urutan di dokumen (Bab 5 — Integrasi & Konfigurasi Lintas Lapisan):
|
||
|
|
# 5.1 Alur Otentikasi Terpusat (Keycloak + OpenLDAP)
|
||
|
|
# 5.2 Storage Layer: Koneksi Komponen ke MinIO + Nessie
|
||
|
|
# 5.3 Storage → Processing: Koneksi Query Engine ke Nessie (+ OPA authz)
|
||
|
|
# 5.3b Storage → Processing: Dremio (query engine kedua) + source Nessie
|
||
|
|
# 5.4 Ingestion → Processing: Pipeline Integration
|
||
|
|
# 5.5 Governance: Observability Integration
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# ./bootstrap.sh # Jalankan semua langkah
|
||
|
|
# ./bootstrap.sh --step 1 # Jalankan step tertentu
|
||
|
|
# ./bootstrap.sh --status # Cek status bootstrap
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
cd "${SCRIPT_DIR}"
|
||
|
|
|
||
|
|
# ─── Shared Library ──────────────────────────────────────────────────────────
|
||
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
||
|
|
|
||
|
|
# ─── Load Environment ───────────────────────────────────────────────────────
|
||
|
|
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||
|
|
if [[ -f "${SCRIPT_DIR}/.env" ]]; then
|
||
|
|
set -a; source "${SCRIPT_DIR}/.env"; set +a
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ─── Fungsi Bantuan ──────────────────────────────────────────────────────────
|
||
|
|
usage_bootstrap() {
|
||
|
|
echo -e "${BOLD}Usage:${NC}"
|
||
|
|
echo " $0 # Bootstrap semua langkah"
|
||
|
|
echo " $0 --step N # Jalankan step tertentu"
|
||
|
|
echo " $0 --host <nama> [--dry-run] # Hanya step milik host itu (endpoint dari"
|
||
|
|
echo " # generated/host-<nama>/.env)"
|
||
|
|
echo " $0 --status # Cek status bootstrap"
|
||
|
|
echo " $0 --help # Bantuan"
|
||
|
|
echo ""
|
||
|
|
echo "Steps:"
|
||
|
|
echo " 1 = Keycloak: Realm, Client, User, Role"
|
||
|
|
echo " 2 = MinIO: Bucket"
|
||
|
|
echo " 3 = OpenLDAP: Seed user + grup (group provider)"
|
||
|
|
echo " 4 = Trino: Verifikasi koneksi Iceberg + Nessie + OPA"
|
||
|
|
echo " 5 = StarRocks: Storage volume + OIDC + LDAP group provider + External Catalog Nessie"
|
||
|
|
echo " 6 = Airflow: Connections + Example DAG"
|
||
|
|
echo " 7 = OpenMetadata: Konektor (Trino, Airflow)"
|
||
|
|
echo " 8 = Grafana: Datasource + Dashboard"
|
||
|
|
echo " 9 = Dremio: First-run admin + Source Nessie"
|
||
|
|
exit 0
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Fungsi HTTP Helper ──────────────────────────────────────────────────────
|
||
|
|
kc_admin() {
|
||
|
|
curl -sf -u "${KEYCLOAK_ADMIN:-admin}:${KEYCLOAK_ADMIN_PASSWORD}" "$@"
|
||
|
|
}
|
||
|
|
|
||
|
|
kc_post() {
|
||
|
|
curl -sf -X POST -u "${KEYCLOAK_ADMIN:-admin}:${KEYCLOAK_ADMIN_PASSWORD}" \
|
||
|
|
-H "Content-Type: application/json" "$@"
|
||
|
|
}
|
||
|
|
|
||
|
|
kc_get() {
|
||
|
|
curl -sf -u "${KEYCLOAK_ADMIN:-admin}:${KEYCLOAK_ADMIN_PASSWORD}" "$@"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Helper LDAP: container lokal (docker exec) vs LDAP EXISTING (dari host) ──
|
||
|
|
# Dipakai step 3. Bila container `sicabe-openldap` tidak berjalan, perintah LDAP
|
||
|
|
# dijalankan langsung dari host ke ${LDAP_HOST}:${LDAP_PORT} — sehingga bootstrap
|
||
|
|
# tetap bisa memakai/seed LDAP yang sudah ada (mode IAM existing).
|
||
|
|
ldap_is_local_container() {
|
||
|
|
docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${LDAP_CONTAINER_NAME:-sicabe-openldap}$"
|
||
|
|
}
|
||
|
|
|
||
|
|
ldap_run() {
|
||
|
|
if ldap_is_local_container; then
|
||
|
|
docker exec -i "${LDAP_CONTAINER_NAME:-sicabe-openldap}" "$@"
|
||
|
|
else
|
||
|
|
"$@"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
ldap_endpoint() {
|
||
|
|
if ldap_is_local_container; then
|
||
|
|
echo "ldap://localhost"
|
||
|
|
else
|
||
|
|
echo "ldap://${LDAP_HOST:-localhost}:${LDAP_PORT:-389}"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 1: Keycloak — Realm, Client, User, Role (bab 5.1)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_keycloak() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 1/9: Keycloak — Realm, Client, User, Role ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.1.1.5-4.1.1.8"
|
||
|
|
|
||
|
|
local kc_base="${KEYCLOAK_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}"
|
||
|
|
|
||
|
|
# Tunggu Keycloak siap
|
||
|
|
echo -e "${INFO} Menunggu Keycloak siap..."
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
if curl -sf "${kc_base}/health/ready" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Keycloak siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 30 ]]; then
|
||
|
|
echo -e "${ERR} Keycloak tidak siap dalam 60 detik."
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
sleep 2
|
||
|
|
done
|
||
|
|
|
||
|
|
# Dapatkan token admin
|
||
|
|
local token
|
||
|
|
token=$(curl -sf -X POST "${kc_base}/realms/master/protocol/openid-connect/token" \
|
||
|
|
-H "Content-Type: application/x-www-form-urlencoded" \
|
||
|
|
-d "client_id=admin-cli" \
|
||
|
|
-d "username=${KEYCLOAK_ADMIN:-admin}" \
|
||
|
|
-d "password=${KEYCLOAK_ADMIN_PASSWORD}" \
|
||
|
|
-d "grant_type=password" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))") || true
|
||
|
|
|
||
|
|
echo -e "${INFO} Token admin diperoleh."
|
||
|
|
|
||
|
|
# ── Buat Realm (jika belum ada) ──────────────────────────────────────────────
|
||
|
|
local realm="${SICABE_REALM:-sicabe}"
|
||
|
|
local realm_exists
|
||
|
|
realm_exists=$(curl -sf -o /dev/null -w "%{http_code}" \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}" 2>/dev/null || echo "404")
|
||
|
|
|
||
|
|
if [[ "${realm_exists}" == "200" ]]; then
|
||
|
|
echo -e "${OK} Realm '${realm}' sudah ada."
|
||
|
|
else
|
||
|
|
echo -e "${INFO} Membuat realm '${realm}'..."
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"realm\": \"${realm}\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"displayName\": \"eBdesk SiCABE\",
|
||
|
|
\"sslRequired\": \"none\",
|
||
|
|
\"registrationAllowed\": false,
|
||
|
|
\"loginWithEmailAllowed\": true,
|
||
|
|
\"resetPasswordAllowed\": true,
|
||
|
|
\"accessTokenLifespan\": 3600
|
||
|
|
}" "${kc_base}/admin/realms" >/dev/null
|
||
|
|
echo -e "${OK} Realm '${realm}' dibuat."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── sslRequired: samakan dengan kebijakan deployment ─────────────────────────
|
||
|
|
# Keycloak default `external` menuntut HTTPS untuk permintaan dari alamat NON-lokal.
|
||
|
|
# Alamat Tailscale 100.64.0.0/10 (CGNAT) TERMASUK non-lokal (isLocal = loopback +
|
||
|
|
# RFC1918 + link-local + fc00::/7), jadi admin console di http://<tailscale>:12080
|
||
|
|
# gagal "HTTPS required". Keycloak 26 tidak punya opsi server-level untuk ini.
|
||
|
|
# Dijalankan untuk realm `master` juga — admin console ada di sana, dan dulu realm ini
|
||
|
|
# tidak pernah disentuh sama sekali (sisa default `external`).
|
||
|
|
local ssl_required="${KEYCLOAK_SSL_REQUIRED:-none}"
|
||
|
|
for r in master "${realm}"; do
|
||
|
|
if curl -sf -X PUT -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"sslRequired\": \"${ssl_required}\"}" \
|
||
|
|
"${kc_base}/admin/realms/${r}" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Realm '${r}' sslRequired=${ssl_required}."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Gagal menyetel sslRequired realm '${r}' (lanjut)."
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# ── Buat Roles ──────────────────────────────────────────────────────────────
|
||
|
|
echo -e "${INFO} Membuat roles..."
|
||
|
|
for role in admin engineer analyst viewer; do
|
||
|
|
local role_exists
|
||
|
|
role_exists=$(curl -sf -o /dev/null -w "%{http_code}" \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/roles/${role}" 2>/dev/null || echo "404")
|
||
|
|
if [[ "${role_exists}" != "200" ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"name\": \"${role}\", \"description\": \"Role: ${role}\"}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/roles" >/dev/null
|
||
|
|
echo -e "${OK} Role '${role}' dibuat."
|
||
|
|
else
|
||
|
|
echo -e "${OK} Role '${role}' sudah ada."
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
# ── Buat Clients (setiap komponen di setiap lapisan) ─────────────────────────
|
||
|
|
echo -e "${INFO} Membuat OIDC clients untuk setiap komponen..."
|
||
|
|
# ⚠️ Client secrets di sini WAJIB sinkron dengan file konfigurasi
|
||
|
|
# komponen terkait (trino/catalog/iceberg.properties, starrocks, dll).
|
||
|
|
# Di development, nilai default sudah cocok.
|
||
|
|
# Di production, override lewat .env dan update file config komponen.
|
||
|
|
declare -A CLIENTS
|
||
|
|
CLIENTS=(
|
||
|
|
["trino"]="Trino — Processing:Query Engine|${TRINO_CLIENT_SECRET}"
|
||
|
|
["starrocks"]="StarRocks — Serving:OLAP Engine|${STARROCKS_CLIENT_SECRET}"
|
||
|
|
["airflow"]="Airflow — Ingestion:Orchestrator|${AIRFLOW_CLIENT_SECRET}"
|
||
|
|
["openmetadata"]="OpenMetadata — Governance:Metadata Platform|${OPENMETADATA_CLIENT_SECRET}"
|
||
|
|
["grafana"]="Grafana — Governance:Observability|${GRAFANA_CLIENT_SECRET}"
|
||
|
|
)
|
||
|
|
|
||
|
|
# URL PUBLIK tiap aplikasi (dari .env — diisi generator; mode none = alamat host
|
||
|
|
# pemilik). Dipakai untuk redirectUris/webOrigins, karena browser yang dipakai
|
||
|
|
# operator TIDAK selalu localhost.
|
||
|
|
declare -A CLIENT_PUBLIC_URL
|
||
|
|
CLIENT_PUBLIC_URL=(
|
||
|
|
["trino"]="${TRINO_PUBLIC_URL:-}"
|
||
|
|
["starrocks"]=""
|
||
|
|
["airflow"]="${AIRFLOW_PUBLIC_URL:-}"
|
||
|
|
["openmetadata"]="${OPENMETADATA_PUBLIC_URL:-}"
|
||
|
|
["grafana"]="${GRAFANA_PUBLIC_URL:-}"
|
||
|
|
)
|
||
|
|
|
||
|
|
for client_id in "${!CLIENTS[@]}"; do
|
||
|
|
local entry="${CLIENTS[$client_id]}"
|
||
|
|
IFS="|" read -r desc secret <<< "${entry}"
|
||
|
|
# Cek apakah client sudah ada
|
||
|
|
local client_exists
|
||
|
|
client_exists=$(curl -sf \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients?clientId=${client_id}" | \
|
||
|
|
python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
if [[ "${client_exists}" -gt 0 ]]; then
|
||
|
|
echo -e "${OK} Client '${client_id}' sudah ada."
|
||
|
|
else
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"clientId\": \"${client_id}\",
|
||
|
|
\"name\": \"${desc}\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"protocol\": \"openid-connect\",
|
||
|
|
\"clientAuthenticatorType\": \"client-secret\",
|
||
|
|
\"secret\": \"${secret}\",
|
||
|
|
\"standardFlowEnabled\": true,
|
||
|
|
\"serviceAccountsEnabled\": true,
|
||
|
|
\"publicClient\": false,
|
||
|
|
\"redirectUris\": [\"http://localhost:*\", \"https://localhost:*\"],
|
||
|
|
\"webOrigins\": [\"http://localhost:*\", \"https://localhost:*\"]
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/clients" >/dev/null
|
||
|
|
echo -e "${OK} Client '${client_id}' (${desc}) dibuat."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Tambah audience mapper 'account' untuk semua client (dibutuhkan Trino OIDC)
|
||
|
|
local client_uuid
|
||
|
|
client_uuid=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients?clientId=${client_id}" | \
|
||
|
|
python3 -c "import sys,json; print(json.load(sys.stdin)[0]['id'])" 2>/dev/null || echo "")
|
||
|
|
# Sinkronkan redirectUris/webOrigins: DIGABUNG dengan yang sudah ada (tidak menimpa),
|
||
|
|
# jadi aman dijalankan berulang. Ini juga memperbaiki client LAMA yang dibuat saat
|
||
|
|
# URL publik belum dikonfigurasi (dulu hanya "localhost" dan tidak pernah di-update),
|
||
|
|
# yang membuat login browser lewat alamat publik ditolak:
|
||
|
|
# Invalid parameter: redirect_uri
|
||
|
|
if [[ -n "${client_uuid}" ]]; then
|
||
|
|
local _payload
|
||
|
|
_payload=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" \
|
||
|
|
| PUBLIC_ORIGIN="${CLIENT_PUBLIC_URL[$client_id]:-}" python3 -c '
|
||
|
|
import json, os, sys
|
||
|
|
c = json.load(sys.stdin)
|
||
|
|
pub = (os.environ.get("PUBLIC_ORIGIN") or "").rstrip("/")
|
||
|
|
redirects = set(c.get("redirectUris") or []) | {"http://localhost:*", "https://localhost:*"}
|
||
|
|
origins = set(c.get("webOrigins") or []) | {"http://localhost:*", "https://localhost:*"}
|
||
|
|
if pub:
|
||
|
|
redirects.add(pub + "/*")
|
||
|
|
origins.add(pub)
|
||
|
|
print(json.dumps({"redirectUris": sorted(redirects), "webOrigins": sorted(origins)}))
|
||
|
|
' 2>/dev/null || echo "")
|
||
|
|
if [[ -n "${_payload}" && -n "${CLIENT_PUBLIC_URL[$client_id]:-}" ]]; then
|
||
|
|
curl -sf -X PUT -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" -d "${_payload}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}" >/dev/null 2>&1 || true
|
||
|
|
echo -e "${OK} Client '${client_id}' redirectUris disinkronkan (${CLIENT_PUBLIC_URL[$client_id]})."
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ -n "${client_uuid}" ]]; then
|
||
|
|
local mapper_name="audience-account"
|
||
|
|
local mapper_exists
|
||
|
|
mapper_exists=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients/${client_uuid}/protocol-mappers/models" | \
|
||
|
|
python3 -c "import sys,json; mappers=json.load(sys.stdin); print(any(m['name']=='${mapper_name}' for m in mappers))" 2>/dev/null || echo "False")
|
||
|
|
if [[ "${mapper_exists}" != "True" ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"name\": \"${mapper_name}\",
|
||
|
|
\"protocol\": \"openid-connect\",
|
||
|
|
\"protocolMapper\": \"oidc-audience-mapper\",
|
||
|
|
\"config\": {
|
||
|
|
\"included.client.audience\": \"account\",
|
||
|
|
\"id.token.claim\": \"false\",
|
||
|
|
\"access.token.claim\": \"true\"
|
||
|
|
}
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/clients/${client_uuid}/protocol-mappers/models" >/dev/null
|
||
|
|
echo -e "${OK} Audience mapper '${mapper_name}' untuk '${client_id}' ditambahkan."
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
unset IFS
|
||
|
|
done
|
||
|
|
|
||
|
|
# ── StarRocks OIDC: sinkronkan redirectUris (idempoten) ────────────────────
|
||
|
|
# StarRocks FE mengirim redirect internal (http://starrocks-fe:8030/api/oauth2)
|
||
|
|
# saat membangun URL OAuth2 → WAJIB terdaftar, jika tidak Keycloak menolak
|
||
|
|
# dengan "Invalid parameter: redirect_uri". Pola host dipakai untuk callback browser.
|
||
|
|
local sr_client_id sr_client_json
|
||
|
|
sr_client_id=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients?clientId=starrocks" | \
|
||
|
|
python3 -c "import sys,json; d=json.load(sys.stdin); print(d[0]['id'] if d else '')" 2>/dev/null || echo "")
|
||
|
|
if [[ -n "${sr_client_id}" ]]; then
|
||
|
|
sr_client_json=$(curl -sf -H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients/${sr_client_id}" | \
|
||
|
|
python3 -c "
|
||
|
|
import sys,json
|
||
|
|
c=json.load(sys.stdin)
|
||
|
|
extra=['http://starrocks-fe:8030/api/oauth2','http://localhost:${STARROCKS_HTTP_PORT:-12030}/*']
|
||
|
|
c['redirectUris']=sorted(set((c.get('redirectUris') or [])+extra))
|
||
|
|
print(json.dumps(c))
|
||
|
|
" 2>/dev/null || true)
|
||
|
|
if [[ -n "${sr_client_json}" ]]; then
|
||
|
|
curl -sf -X PUT -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "${sr_client_json}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/clients/${sr_client_id}" >/dev/null \
|
||
|
|
&& echo -e "${OK} Client 'starrocks' redirectUris disinkronkan (redirect internal FE)."
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── Buat Users ──────────────────────────────────────────────────────────────
|
||
|
|
echo -e "${INFO} Membuat users..."
|
||
|
|
# Admin user
|
||
|
|
local admin_exists
|
||
|
|
admin_exists=$(curl -sf \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/users?username=admin" | \
|
||
|
|
python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
if [[ "${admin_exists}" -eq 0 ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"username\": \"admin\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"email\": \"admin@sicabe.local\",
|
||
|
|
\"firstName\": \"Administrator\",
|
||
|
|
\"lastName\": \"SiCABE\",
|
||
|
|
\"credentials\": [{\"type\": \"password\", \"value\": \"${KEYCLOAK_ADMIN_PASSWORD}\", \"temporary\": false}],
|
||
|
|
\"realmRoles\": [\"admin\"]
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/users" >/dev/null
|
||
|
|
echo -e "${OK} User 'admin' dibuat."
|
||
|
|
else
|
||
|
|
echo -e "${OK} User 'admin' sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Data Engineer user
|
||
|
|
local engineer_exists
|
||
|
|
engineer_exists=$(curl -sf \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/users?username=engineer" | \
|
||
|
|
python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
if [[ "${engineer_exists}" -eq 0 ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"username\": \"engineer\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"email\": \"engineer@sicabe.local\",
|
||
|
|
\"firstName\": \"Data\",
|
||
|
|
\"lastName\": \"Engineer\",
|
||
|
|
\"credentials\": [{\"type\": \"password\", \"value\": \"engineer123\", \"temporary\": false}],
|
||
|
|
\"realmRoles\": [\"engineer\"]
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/users" >/dev/null
|
||
|
|
echo -e "${OK} User 'engineer' dibuat."
|
||
|
|
else
|
||
|
|
echo -e "${OK} User 'engineer' sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Data Analyst user
|
||
|
|
local analyst_exists
|
||
|
|
analyst_exists=$(curl -sf \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/users?username=analyst" | \
|
||
|
|
python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
if [[ "${analyst_exists}" -eq 0 ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"username\": \"analyst\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"email\": \"analyst@sicabe.local\",
|
||
|
|
\"firstName\": \"Data\",
|
||
|
|
\"lastName\": \"Analyst\",
|
||
|
|
\"credentials\": [{\"type\": \"password\", \"value\": \"analyst123\", \"temporary\": false}],
|
||
|
|
\"realmRoles\": [\"analyst\"]
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/users" >/dev/null
|
||
|
|
echo -e "${OK} User 'analyst' dibuat."
|
||
|
|
else
|
||
|
|
echo -e "${OK} User 'analyst' sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Viewer user
|
||
|
|
local viewer_exists
|
||
|
|
viewer_exists=$(curl -sf \
|
||
|
|
-H "Authorization: Bearer ${token}" \
|
||
|
|
"${kc_base}/admin/realms/${realm}/users?username=viewer" | \
|
||
|
|
python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
if [[ "${viewer_exists}" -eq 0 ]]; then
|
||
|
|
curl -sf -X POST -H "Authorization: Bearer ${token}" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{
|
||
|
|
\"username\": \"viewer\",
|
||
|
|
\"enabled\": true,
|
||
|
|
\"email\": \"viewer@sicabe.local\",
|
||
|
|
\"firstName\": \"Viewer\",
|
||
|
|
\"lastName\": \"SiCABE\",
|
||
|
|
\"credentials\": [{\"type\": \"password\", \"value\": \"viewer123\", \"temporary\": false}],
|
||
|
|
\"realmRoles\": [\"viewer\"]
|
||
|
|
}" "${kc_base}/admin/realms/${realm}/users" >/dev/null
|
||
|
|
echo -e "${OK} User 'viewer' dibuat."
|
||
|
|
else
|
||
|
|
echo -e "${OK} User 'viewer' sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 1 selesai — Keycloak terkonfigurasi.${NC}"
|
||
|
|
echo -e " URL Admin Console: ${CYAN}${KEYCLOAK_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}/admin/${realm}/console${NC}"
|
||
|
|
echo -e " Login: admin / ${KEYCLOAK_ADMIN_PASSWORD}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 2: MinIO — Bucket, Policy, Service Account (bab 5.2)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_minio() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 2/9: Object storage — Bucket ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.2.1.7-4.2.1.8"
|
||
|
|
|
||
|
|
# Berfungsi untuk MinIO lokal MAUPUN object storage existing: endpoint &
|
||
|
|
# kredensial diambil dari S3_* (lihat config/environment.defaults.sh §2.9).
|
||
|
|
local mc_endpoint="${S3_ENDPOINT:-http://minio:9000}"
|
||
|
|
local mc_user="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
|
||
|
|
local mc_pass="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
|
||
|
|
local bucket="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
|
||
|
|
local mc_image="quay.io/minio/mc:${MINIO_MC_VERSION:-RELEASE.2025-08-13T08-35-41Z}"
|
||
|
|
local local_minio_ep="http://minio:9000"
|
||
|
|
|
||
|
|
# Penantian via host hanya relevan untuk MinIO lokal (S3 existing tidak di localhost).
|
||
|
|
if [[ "${mc_endpoint}" == "${local_minio_ep}" ]]; then
|
||
|
|
echo -e "${INFO} Menunggu MinIO siap..."
|
||
|
|
for i in $(seq 1 15); do
|
||
|
|
if curl -sf "http://localhost:${MINIO_PORT:-12900}/minio/health/live" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} MinIO siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
[[ $i -eq 15 ]] && echo -e "${WARN} MinIO lokal belum siap — dicoba lewat container."
|
||
|
|
sleep 2
|
||
|
|
done
|
||
|
|
else
|
||
|
|
echo -e "${INFO} Object storage eksternal (${mc_endpoint}) — lewati penantian host."
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ -z "${mc_pass}" ]]; then
|
||
|
|
echo -e "${WARN} Kredensial storage kosong (S3_SECRET_KEY / MINIO_ROOT_PASSWORD) — lewati pembuatan bucket."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Bucket diuji/dibuat dari container pada jaringan compose: S3_ENDPOINT umumnya
|
||
|
|
# hanya resolve di jaringan itu (berlaku sama untuk MinIO lokal & S3 existing).
|
||
|
|
if ! docker image inspect "${mc_image}" >/dev/null 2>&1; then
|
||
|
|
echo -e "${INFO} Menarik image mc ${mc_image}..."
|
||
|
|
if ! docker pull "${mc_image}" >/dev/null 2>&1; then
|
||
|
|
echo -e "${WARN} Gagal menarik image mc — buat bucket '${bucket}' manual lalu jalankan gate fase 2."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
local out
|
||
|
|
out=$(docker run --rm -i \
|
||
|
|
--network "${SICABE_NETWORK:-sicabe-network}" \
|
||
|
|
-e S3E="${mc_endpoint}" -e S3U="${mc_user}" -e S3P="${mc_pass}" -e BUCKET="${bucket}" \
|
||
|
|
--entrypoint sh "${mc_image}" -c '
|
||
|
|
if ! mc alias set probe "$S3E" "$S3U" "$S3P" >/dev/null 2>&1; then echo ALIAS_FAIL; exit 0; fi
|
||
|
|
if mc ls "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_OK
|
||
|
|
elif mc mb "probe/$BUCKET" >/dev/null 2>&1; then echo BUCKET_CREATED
|
||
|
|
else echo BUCKET_FAIL; fi' 2>&1)
|
||
|
|
|
||
|
|
case "${out}" in
|
||
|
|
*ALIAS_FAIL*)
|
||
|
|
echo -e "${WARN} Tidak bisa autentikasi ke ${mc_endpoint} dari container."
|
||
|
|
echo " Cek S3_ENDPOINT + S3_ACCESS_KEY/S3_SECRET_KEY; untuk S3 existing buat bucket '${bucket}' manual."
|
||
|
|
return 0
|
||
|
|
;;
|
||
|
|
*BUCKET_OK*)
|
||
|
|
echo -e "${OK} Bucket '${bucket}' sudah ada."
|
||
|
|
;;
|
||
|
|
*BUCKET_CREATED*)
|
||
|
|
echo -e "${OK} Bucket '${bucket}' dibuat."
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
echo -e "${WARN} Bucket '${bucket}' belum bisa dipastikan — buat manual, lalu:"
|
||
|
|
echo " ./scripts/verify-phase2-storage.sh"
|
||
|
|
return 0
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 2 selesai — object storage terkonfigurasi.${NC}"
|
||
|
|
echo " Endpoint: ${mc_endpoint} | Bucket: ${bucket}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 3: OpenLDAP — Seed user + grup (sumber grup Trino & StarRocks)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_openldap() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 3/9: OpenLDAP — Seed user + grup ═══${NC}"
|
||
|
|
echo -e "${INFO} LDAP = source of truth grup untuk Trino & StarRocks group provider"
|
||
|
|
|
||
|
|
local ldap_container="${LDAP_CONTAINER_NAME:-sicabe-openldap}"
|
||
|
|
local admin_dn="${LDAP_ADMIN_DN:-cn=admin,dc=sicabe,dc=local}"
|
||
|
|
local base_dn="${LDAP_BASE_DN:-dc=sicabe,dc=local}"
|
||
|
|
local ldap_url
|
||
|
|
ldap_url="$(ldap_endpoint)"
|
||
|
|
|
||
|
|
if ldap_is_local_container; then
|
||
|
|
echo -e "${INFO} Mode LDAP lokal (container ${ldap_container})."
|
||
|
|
else
|
||
|
|
echo -e "${INFO} Mode LDAP EXISTING — target ${ldap_url}"
|
||
|
|
if ! command -v ldapsearch >/dev/null 2>&1; then
|
||
|
|
echo -e "${WARN} ldapsearch tidak ada di host. Install ldap-utils lalu ulangi step 3."
|
||
|
|
echo -e "${WARN} Melewati seed LDAP."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${INFO} Menunggu LDAP siap..."
|
||
|
|
for i in $(seq 1 15); do
|
||
|
|
if ldap_run ldapsearch -x -H "${ldap_url}" \
|
||
|
|
-b "${base_dn}" -D "${admin_dn}" -w "${LDAP_ADMIN_PASSWORD}" \
|
||
|
|
"(objectClass=*)" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} LDAP siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 15 ]]; then
|
||
|
|
echo -e "${WARN} LDAP belum siap. Lanjutkan..."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 2
|
||
|
|
done
|
||
|
|
|
||
|
|
# Seed idempotent: cek apakah OU users sudah ada (pakai -LLL agar filter tidak
|
||
|
|
# ikut tercetak — jika tidak, grep "ou=users" selalu cocok = false positive).
|
||
|
|
local has_users
|
||
|
|
has_users=$(ldap_run ldapsearch -LLL -x -H "${ldap_url}" \
|
||
|
|
-b "${base_dn}" -D "${admin_dn}" -w "${LDAP_ADMIN_PASSWORD}" \
|
||
|
|
"(ou=users)" dn 2>/dev/null | grep -c "^dn: ou=users" || true)
|
||
|
|
if [[ "${has_users}" -eq 0 ]]; then
|
||
|
|
ldap_run ldapadd -x -H "${ldap_url}" \
|
||
|
|
-D "${admin_dn}" -w "${LDAP_ADMIN_PASSWORD}" \
|
||
|
|
< "${SCRIPT_DIR}/config/openldap/seed.ldif" >/dev/null 2>&1 \
|
||
|
|
&& echo -e "${OK} Seed LDAP (user + grup) dibuat." \
|
||
|
|
|| echo -e "${WARN} Seed LDAP gagal (mungkin sudah ada)."
|
||
|
|
else
|
||
|
|
echo -e "${OK} Seed LDAP sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 3 selesai — LDAP terkonfigurasi.${NC}"
|
||
|
|
echo " Endpoint: ${ldap_url}"
|
||
|
|
echo " Base DN: ${base_dn}"
|
||
|
|
echo " Groups: data-admin, data-engineer, data-analyst, data-viewer"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 4: Trino — Verifikasi Koneksi Iceberg + Nessie + OPA (bab 5.3)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_trino() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 4/9: Trino — Verifikasi Iceberg + Nessie + OPA ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.3.1.3 & 4.3.1.8"
|
||
|
|
|
||
|
|
# Endpoint dari sisi HOST — dapat diarahkan ke layanan EXISTING lewat .env:
|
||
|
|
# TRINO_URL / OPA_URL / NESSIE_HOST+NESSIE_PORT (lihat config/environment.defaults.sh)
|
||
|
|
local trino_url="${TRINO_URL:-https://localhost:${TRINO_HTTPS_PORT:-12443}}"
|
||
|
|
local nessie_url="http://${NESSIE_HOST:-localhost}:${NESSIE_PORT:-12120}"
|
||
|
|
local opa_url="${OPA_URL:-http://localhost:${OPA_PORT:-12181}}"
|
||
|
|
|
||
|
|
# Tunggu Trino siap
|
||
|
|
echo -e "${INFO} Menunggu Trino siap..."
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
if curl -skf "${trino_url}/v1/info" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Trino siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 30 ]]; then
|
||
|
|
echo -e "${WARN} Trino belum siap."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 2
|
||
|
|
done
|
||
|
|
|
||
|
|
# Verifikasi Nessie (catalog) reachable
|
||
|
|
if curl -sf "${nessie_url}/api/v2/config" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Nessie (catalog) reachable di ${nessie_url}."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Nessie tidak reachable di ${nessie_url}."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Verifikasi OPA (decision point authz) reachable
|
||
|
|
if curl -sf -X POST "${opa_url}/v1/data/trino/allow" \
|
||
|
|
-H "Content-Type: application/json" -d '{"input":{}}' >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} OPA (authz) reachable di ${opa_url}."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} OPA tidak reachable di ${opa_url}."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Verifikasi catalog 'iceberg' via service account (HTTP Basic) langsung ke HTTPS
|
||
|
|
local result
|
||
|
|
result=$(curl -skf -X POST "${trino_url}/v1/statement" \
|
||
|
|
-u "svc_seed:${TRINO_SVC_SEED_PASSWORD}" \
|
||
|
|
-H "Content-Type: text/plain" \
|
||
|
|
-d "SHOW CATALOGS" 2>/dev/null || true)
|
||
|
|
if echo "${result}" | grep -q "iceberg"; then
|
||
|
|
echo -e "${OK} Catalog 'iceberg' terdeteksi di Trino (service account)."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Catalog belum terverifikasi via service account (cek auth/endpoint)."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 4 selesai — Trino terverifikasi.${NC}"
|
||
|
|
echo " URL: https://localhost:${TRINO_HTTPS_PORT:-12443}"
|
||
|
|
echo " Catalog: iceberg (via Nessie) | Authz: OPA"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 5: StarRocks — External Catalog + OIDC (bab 5.3 lanjutan)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_starrocks() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 5/9: StarRocks — shared-data + OIDC + LDAP + External Catalog ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.4.1.3-4.4.1.4"
|
||
|
|
|
||
|
|
local sr_port="${STARROCKS_FE_PORT:-12930}"
|
||
|
|
# Base Keycloak yang dijangkau HOST ini — owner + port TERPUBLIKASI (samakan dgn
|
||
|
|
# verify-phase4 KC_BASE). single-host → localhost:12080; cluster → <owner>:12080.
|
||
|
|
# Sebelumnya pakai KEYCLOAK_URL (default localhost) → issuer salah saat keycloak existing.
|
||
|
|
local kc_base
|
||
|
|
kc_base="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)"
|
||
|
|
# Alamat Keycloak server→server (dari container StarRocks): service/owner + port container.
|
||
|
|
local kc_internal="http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}"
|
||
|
|
|
||
|
|
sr_mysql() {
|
||
|
|
mysql -h 127.0.0.1 -P "${sr_port}" -u root -p"${STARROCKS_ROOT_PASSWORD}" -e "$1" 2>/dev/null
|
||
|
|
}
|
||
|
|
|
||
|
|
# Tunggu StarRocks FE siap
|
||
|
|
echo -e "${INFO} Menunggu StarRocks FE siap..."
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
if sr_mysql "SELECT 1;" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} StarRocks FE siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 30 ]]; then
|
||
|
|
echo -e "${WARN} StarRocks FE belum siap atau MySQL client tidak tersedia."
|
||
|
|
print_starrocks_sql
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
|
||
|
|
if ! command -v mysql &>/dev/null; then
|
||
|
|
echo -e "${WARN} MySQL client tidak ditemukan."
|
||
|
|
echo " docker run --rm --network=sicabe-network mysql:8 mysql -hstarrocks-fe -P9030 -uroot -p..."
|
||
|
|
print_starrocks_sql
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 1. Storage volume S3 (shared-data) ──────────────────────────────────
|
||
|
|
echo -e "${INFO} [1/6] Storage volume S3 (MinIO)..."
|
||
|
|
if ! sr_mysql "SHOW STORAGE VOLUMES;" | grep -qi "s3_volume"; then
|
||
|
|
sr_mysql "CREATE STORAGE VOLUME s3_volume
|
||
|
|
TYPE = S3
|
||
|
|
LOCATIONS = ('s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/starrocks/')
|
||
|
|
PROPERTIES (
|
||
|
|
'enabled' = 'true',
|
||
|
|
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
|
||
|
|
'aws.s3.region' = '${S3_REGION:-us-east-1}',
|
||
|
|
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
|
||
|
|
'aws.s3.secret_key' = '${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}',
|
||
|
|
'aws.s3.use_instance_profile' = 'false',
|
||
|
|
'aws.s3.use_aws_sdk_default_behavior' = 'false',
|
||
|
|
'aws.s3.enable_path_style_access' = 'true'
|
||
|
|
);" && echo -e "${OK} Storage volume 's3_volume' dibuat." || echo -e "${WARN} CREATE STORAGE VOLUME gagal."
|
||
|
|
else
|
||
|
|
# Volume sudah ADA → SINKRONKAN properti: S3_ENDPOINT/kredensial bisa berubah
|
||
|
|
# (mis. diarahkan ke MinIO existing / port non-standar). Tanpa ini endpoint LAMA
|
||
|
|
# tertinggal → verify-phase4 FAIL "aws.s3.endpoint … ≠ S3_ENDPOINT".
|
||
|
|
if sr_mysql "ALTER STORAGE VOLUME s3_volume SET (
|
||
|
|
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
|
||
|
|
'aws.s3.region' = '${S3_REGION:-us-east-1}',
|
||
|
|
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
|
||
|
|
'aws.s3.secret_key' = '${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}'
|
||
|
|
);" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Storage volume 's3_volume' ada — properti disinkronkan (endpoint/kredensial)."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Storage volume 's3_volume' ada tetapi ALTER gagal — endpoint mungkin masih lama."
|
||
|
|
echo -e "${WARN} Perbaiki manual: DROP STORAGE VOLUME s3_volume; lalu jalankan ulang step 5."
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
sr_mysql "SET s3_volume AS DEFAULT STORAGE VOLUME;" >/dev/null 2>&1 && echo -e "${OK} s3_volume = DEFAULT STORAGE VOLUME"
|
||
|
|
|
||
|
|
# ── 2. Security integration OIDC (Keycloak) ─────────────────────────────
|
||
|
|
echo -e "${INFO} [2/6] Security integration OIDC..."
|
||
|
|
if ! sr_mysql "SHOW SECURITY INTEGRATIONS;" | grep -qi "sr_oidc"; then
|
||
|
|
# redirect_url dipakai LITERAL sebagai redirect_uri & harus reachable dari
|
||
|
|
# browser (DBeaver membuka browser ke sini). JANGAN pakai hostname internal.
|
||
|
|
sr_mysql "CREATE SECURITY INTEGRATION sr_oidc PROPERTIES (
|
||
|
|
'type' = 'authentication_oauth2',
|
||
|
|
'auth_server_url' = '${kc_base}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth',
|
||
|
|
'token_server_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token',
|
||
|
|
'client_id' = 'starrocks',
|
||
|
|
'client_secret' = '${STARROCKS_CLIENT_SECRET}',
|
||
|
|
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
|
||
|
|
'jwks_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/certs',
|
||
|
|
'principal_field' = 'preferred_username',
|
||
|
|
'required_issuer' = '${kc_base}/realms/${SICABE_REALM:-sicabe}',
|
||
|
|
'required_audience' = 'starrocks'
|
||
|
|
);" && echo -e "${OK} Security integration 'sr_oidc' dibuat." || echo -e "${WARN} CREATE SECURITY INTEGRATION gagal."
|
||
|
|
else
|
||
|
|
echo -e "${OK} Security integration 'sr_oidc' sudah ada."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 3. Group provider LDAP (OpenLDAP) ───────────────────────────────────
|
||
|
|
echo -e "${INFO} [3/6] Group provider LDAP..."
|
||
|
|
# ALTER GROUP PROVIDER TIDAK didukung (uji: syntax error) → DROP+CREATE agar
|
||
|
|
# ldap_conn_url/dn selalu mengikuti env saat re-run (mis. LDAP existing).
|
||
|
|
sr_mysql "DROP GROUP PROVIDER sr_ldap;" >/dev/null 2>&1 || true
|
||
|
|
sr_mysql "CREATE GROUP PROVIDER sr_ldap PROPERTIES (
|
||
|
|
'type' = 'ldap',
|
||
|
|
'ldap_conn_url' = '${LDAP_URL:-ldap://openldap:389}',
|
||
|
|
'ldap_bind_root_dn' = '${LDAP_ADMIN_DN:-cn=admin,dc=sicabe,dc=local}',
|
||
|
|
'ldap_bind_root_pwd' = '${LDAP_ADMIN_PASSWORD}',
|
||
|
|
'ldap_bind_base_dn' = '${LDAP_BASE_DN:-dc=sicabe,dc=local}',
|
||
|
|
'ldap_group_filter' = '(objectClass=groupOfNames)',
|
||
|
|
'ldap_group_identifier_attr' = 'cn',
|
||
|
|
'ldap_group_member_attr' = 'member',
|
||
|
|
'ldap_user_search_attr' = 'uid',
|
||
|
|
'ldap_cache_refresh_interval' = '900'
|
||
|
|
);" && echo -e "${OK} Group provider 'sr_ldap' dibuat/disinkronkan." || echo -e "${WARN} CREATE GROUP PROVIDER gagal."
|
||
|
|
sr_mysql "ALTER SECURITY INTEGRATION sr_oidc SET (
|
||
|
|
'redirect_url' = 'http://localhost:${STARROCKS_HTTP_PORT:-12030}/api/oauth2',
|
||
|
|
'group_provider' = 'sr_ldap',
|
||
|
|
'permitted_groups' = 'data-admin,data-engineer,data-analyst,data-viewer',
|
||
|
|
'auth_server_url' = '${kc_base}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth',
|
||
|
|
'token_server_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token',
|
||
|
|
'jwks_url' = '${kc_internal}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/certs',
|
||
|
|
'required_issuer' = '${kc_base}/realms/${SICABE_REALM:-sicabe}',
|
||
|
|
'client_secret' = '${STARROCKS_CLIENT_SECRET}'
|
||
|
|
);" >/dev/null 2>&1 && echo -e "${OK} sr_oidc disinkronkan (issuer/secret/endpoint + group provider)."
|
||
|
|
|
||
|
|
# ── 4. Roles + GRANT ke EXTERNAL GROUP ──────────────────────────────────
|
||
|
|
echo -e "${INFO} [4/6] Roles + GRANT ke EXTERNAL GROUP..."
|
||
|
|
for role in sr_admin sr_engineer sr_analyst sr_viewer; do
|
||
|
|
sr_mysql "CREATE ROLE ${role};" >/dev/null 2>&1 || true
|
||
|
|
done
|
||
|
|
for pair in "sr_admin:data-admin" "sr_engineer:data-engineer" "sr_analyst:data-analyst" "sr_viewer:data-viewer"; do
|
||
|
|
local role="${pair%%:*}" grp="${pair##*:}"
|
||
|
|
sr_mysql "GRANT ${role} TO EXTERNAL GROUP \`${grp}\`;" >/dev/null 2>&1 \
|
||
|
|
&& echo -e "${OK} GRANT ${role} → EXTERNAL GROUP ${grp}" \
|
||
|
|
|| echo -e "${WARN} GRANT ${role} → ${grp} (mungkin sudah ada)"
|
||
|
|
done
|
||
|
|
# Catatan: activate_all_roles_on_login & authentication_chain diterapkan di
|
||
|
|
# langkah [5] oleh scripts/starrocks-reapply-dynamic-config.sh (satu tempat).
|
||
|
|
|
||
|
|
# ── 5. Dynamic config (auth chain + role on login) ──────────────────────
|
||
|
|
# TIDAK bisa ditaruh di fe.conf: `sr_oidc` baru ada setelah SQL di atas,
|
||
|
|
# sedangkan fe.conf dibaca saat FE boot (saat integrasi belum ada → error).
|
||
|
|
# Karena itu diterapkan lewat skrip terpisah yang JUGA bisa dijalankan ulang
|
||
|
|
# setiap kali FE restart: scripts/starrocks-reapply-dynamic-config.sh
|
||
|
|
echo -e "${INFO} [5/6] Dynamic config (auth chain OIDC)..."
|
||
|
|
local dc_out
|
||
|
|
if dc_out=$("${SCRIPT_DIR}/scripts/starrocks-reapply-dynamic-config.sh" 2>&1); then
|
||
|
|
echo -e "${OK} authentication_chain memuat ${STARROCKS_OIDC_INTEGRATION:-sr_oidc} + activate_all_roles_on_login"
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Dynamic config belum diterapkan — jalankan manual:"
|
||
|
|
echo "${dc_out}" | sed 's/^/ /'
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 6. External Catalog Iceberg via Nessie ──────────────────────────────
|
||
|
|
echo -e "${INFO} [6/6] External Catalog Iceberg via Nessie..."
|
||
|
|
sr_mysql "DROP CATALOG IF EXISTS iceberg_catalog;" >/dev/null 2>&1 || true
|
||
|
|
sr_mysql "CREATE EXTERNAL CATALOG iceberg_catalog
|
||
|
|
PROPERTIES (
|
||
|
|
'type' = 'iceberg',
|
||
|
|
'iceberg.catalog.type' = 'rest',
|
||
|
|
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
|
||
|
|
'iceberg.catalog.warehouse' = 'warehouse',
|
||
|
|
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
|
||
|
|
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
|
||
|
|
'aws.s3.access_key' = '${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
|
||
|
|
'aws.s3.secret_key' = '${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}',
|
||
|
|
'aws.s3.enable_path_style_access' = 'true',
|
||
|
|
'aws.s3.region' = '${S3_REGION:-us-east-1}'
|
||
|
|
);" 2>/dev/null && echo -e "${OK} External Catalog 'iceberg_catalog' dibuat." || \
|
||
|
|
echo -e "${WARN} External Catalog gagal/mungkin sudah ada."
|
||
|
|
sr_mysql "SHOW CATALOGS;" | grep -q "iceberg_catalog" && echo -e "${OK} Catalog 'iceberg_catalog' terverifikasi."
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 5 selesai — StarRocks terkonfigurasi.${NC}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
print_starrocks_sql() {
|
||
|
|
cat <<EOF
|
||
|
|
Jalankan SQL berikut di StarRocks (setelah FE siap), via:
|
||
|
|
docker exec -it sicabe-starrocks-fe mysql -h 127.0.0.1 -P 9030 -u root -p
|
||
|
|
|
||
|
|
-- 1) Storage volume S3 (shared-data)
|
||
|
|
CREATE STORAGE VOLUME s3_volume TYPE=S3
|
||
|
|
LOCATIONS=('s3://${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}/starrocks/')
|
||
|
|
PROPERTIES ('enabled'='true','aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}',
|
||
|
|
'aws.s3.region'='${S3_REGION:-us-east-1}',
|
||
|
|
'aws.s3.access_key'='${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}',
|
||
|
|
'aws.s3.secret_key'='${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}',
|
||
|
|
'aws.s3.use_instance_profile'='false',
|
||
|
|
'aws.s3.use_aws_sdk_default_behavior'='false',
|
||
|
|
'aws.s3.enable_path_style_access'='true');
|
||
|
|
SET s3_volume AS DEFAULT STORAGE VOLUME;
|
||
|
|
|
||
|
|
-- 2) OIDC + LDAP group provider
|
||
|
|
CREATE SECURITY INTEGRATION sr_oidc PROPERTIES ('type'='authentication_oauth2', ...);
|
||
|
|
CREATE GROUP PROVIDER sr_ldap PROPERTIES ('type'='ldap', ...);
|
||
|
|
ALTER SECURITY INTEGRATION sr_oidc SET ('group_provider'='sr_ldap','permitted_groups'='data-admin,...');
|
||
|
|
ADMIN SET FRONTEND CONFIG ("authentication_chain" = "native,sr_oidc");
|
||
|
|
GRANT sr_admin TO EXTERNAL GROUP \`data-admin\`;
|
||
|
|
|
||
|
|
-- 3) External catalog Iceberg via Nessie
|
||
|
|
CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
|
||
|
|
'type'='iceberg','iceberg.catalog.type'='rest',
|
||
|
|
'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}/iceberg/',
|
||
|
|
'iceberg.catalog.warehouse'='warehouse',
|
||
|
|
'aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}', ...);
|
||
|
|
EOF
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 6: Airflow — Connections + Example DAG (bab 5.4)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_airflow() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 6/9: Airflow — Connections + Example DAG ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.5.1.5-4.5.1.9"
|
||
|
|
|
||
|
|
local airflow_url="http://localhost:${AIRFLOW_PORT:-12082}"
|
||
|
|
|
||
|
|
# Tunggu Airflow siap (Airflow 3.x: /api/v2/monitor/health)
|
||
|
|
echo -e "${INFO} Menunggu Airflow siap..."
|
||
|
|
for i in $(seq 1 30); do
|
||
|
|
health=$(curl -sf "${airflow_url}/api/v2/monitor/health" 2>/dev/null | python3 -c "import json,sys;d=json.load(sys.stdin);print(d.get('metadatabase',{}).get('status',''))" 2>/dev/null)
|
||
|
|
if [[ "${health}" == "healthy" ]]; then
|
||
|
|
echo -e "${OK} Airflow siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 30 ]]; then
|
||
|
|
echo -e "${WARN} Airflow belum siap."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 3
|
||
|
|
done
|
||
|
|
|
||
|
|
# Airflow 3.x menghapus /api/v1/security/login.
|
||
|
|
# Setup connections via Airflow CLI dalam container.
|
||
|
|
echo -e "${INFO} Setup connections via Airflow CLI..."
|
||
|
|
if docker exec sicabe-airflow-webserver airflow connections list >/dev/null 2>&1; then
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections delete 'minio_s3' 2>/dev/null || true
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections add 'minio_s3' \
|
||
|
|
--conn-type 'aws' \
|
||
|
|
--conn-login "${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}" \
|
||
|
|
--conn-password "${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}" \
|
||
|
|
--conn-extra "{\"endpoint_url\": \"${S3_ENDPOINT:-http://minio:9000}\", \"region_name\": \"${S3_REGION:-us-east-1}\"}"
|
||
|
|
echo -e "${OK} Connection 'minio_s3' dibuat."
|
||
|
|
|
||
|
|
# 'trino_default' SENGAJA TIDAK dibuat (keputusan review Fase 5): tidak ada DAG
|
||
|
|
# yang memakainya (task memakai dbt HTTPS 8443 + Sling), sedangkan versi lama
|
||
|
|
# menunjuk http://trino:8080 tanpa kredensial → selalu 403 (lihat §7.6).
|
||
|
|
# Baris berikut menyapu sisa connection lama agar instalasi konvergen.
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections delete 'trino_default' 2>/dev/null || true
|
||
|
|
|
||
|
|
# ── OpenLineage → OpenMetadata (eksperimen experiments/lineage-airflow-om) ─
|
||
|
|
# Konfigurasi transport disimpan di KONEKSI Airflow (bukan .env) supaya token
|
||
|
|
# lineage TIDAK ikut tersebar di berkas/plaintext env. Dua koneksi:
|
||
|
|
# openlineage_config : extra = {"transport": {...}} — endpoint OM +
|
||
|
|
# shim `eventTime` (ol_time_shim.ZuluHttpTransport)
|
||
|
|
# openmetadata_lineage_api : password = token bot `lineage-bot` (UNLIMITED)
|
||
|
|
# `auth.type = airflow_connection_api_key` akan diganti provider menjadi
|
||
|
|
# {"type":"api_key","apiKey":<password koneksi openmetadata_lineage_api>}.
|
||
|
|
# Provider tetap INERT sampai AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID diisi —
|
||
|
|
# lihat catatan di config/environment.defaults.sh.
|
||
|
|
OM_BASE="${OPENMETADATA_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||
|
|
OM_LOGIN_EMAIL="${OPENMETADATA_ADMIN_EMAIL:-admin@open-metadata.org}"
|
||
|
|
OM_LOGIN_PASS="${OPENMETADATA_ADMIN_PASSWORD:-openmetadata123}"
|
||
|
|
OM_PASS64="$(printf '%s' "${OM_LOGIN_PASS}" | base64)"
|
||
|
|
# JANGAN pakai `curl -sf` di dalam assignment ini. Dengan `set -euo pipefail`, curl
|
||
|
|
# yang gagal (mis. login ditolak karena password admin OpenMetadata berbeda dari
|
||
|
|
# .env — OM 2.0.x sering memakai 'admin') membuat ASSIGNMENT-nya gagal, dan itu
|
||
|
|
# MEMATIKAN seluruh step 6 diam-diam. Akibatnya blok penyalinan DAG di bawah tidak
|
||
|
|
# pernah jalan dan gate fase 5 melaporkan "DAG 'sicabe_batch_daily' tidak ditemukan"
|
||
|
|
# tanpa sebab yang jelas. `curl -s` (tanpa -f) + `|| true` menyerahkan kasus itu ke
|
||
|
|
# jalur WARN yang memang sudah disediakan di bawah.
|
||
|
|
OM_ADMIN_TOKEN=""
|
||
|
|
local _om_pw _om_b64
|
||
|
|
for _om_pw in "${OM_LOGIN_PASS}" "admin"; do
|
||
|
|
[[ -n "${_om_pw}" ]] || continue
|
||
|
|
_om_b64="$(printf '%s' "${_om_pw}" | base64)"
|
||
|
|
OM_ADMIN_TOKEN="$(curl -s -X POST "${OM_BASE}/api/v1/users/login" \
|
||
|
|
-H 'Content-Type: application/json' \
|
||
|
|
-d "{\"email\":\"${OM_LOGIN_EMAIL}\",\"password\":\"${_om_b64}\"}" 2>/dev/null |
|
||
|
|
python3 -c "import sys,json; print(json.load(sys.stdin).get('accessToken',''))" 2>/dev/null || true)"
|
||
|
|
[[ -n "${OM_ADMIN_TOKEN}" ]] && break
|
||
|
|
done
|
||
|
|
if [[ -n "${OM_ADMIN_TOKEN}" ]]; then
|
||
|
|
BOT_ID="$(curl -s "${OM_BASE}/api/v1/users?limit=100" -H "Authorization: Bearer ${OM_ADMIN_TOKEN}" 2>/dev/null |
|
||
|
|
python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['id'] for u in d.get('data',[]) if u.get('isBot') or u.get('name')=='lineage-bot'), ''))" 2>/dev/null)"
|
||
|
|
BOT_TOKEN=""
|
||
|
|
if [[ -n "${BOT_ID}" ]]; then
|
||
|
|
BOT_TOKEN="$(curl -s -X POST "${OM_BASE}/api/v1/users/generateToken" \
|
||
|
|
-H "Authorization: Bearer ${OM_ADMIN_TOKEN}" -H 'Content-Type: application/json' \
|
||
|
|
-d "{\"JWTTokenExpiry\":\"Unlimited\",\"id\":\"${BOT_ID}\"}" 2>/dev/null |
|
||
|
|
python3 -c "import sys,json; print(json.load(sys.stdin).get('JWTToken',''))" 2>/dev/null)"
|
||
|
|
fi
|
||
|
|
if [[ -n "${BOT_TOKEN}" ]]; then
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections delete 'openmetadata_lineage_api' 2>/dev/null || true
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections add 'openmetadata_lineage_api' \
|
||
|
|
--conn-type 'generic' --conn-password "${BOT_TOKEN}" >/dev/null
|
||
|
|
OL_TRANSPORT="{\"transport\":{\"type\":\"ol_time_shim.ZuluHttpTransport\",\"url\":\"http://${OPENMETADATA_HOST:-openmetadata-server}:8585/api/v1/openlineage/\",\"endpoint\":\"lineage\",\"auth\":{\"type\":\"airflow_connection_api_key\",\"conn_id\":\"openmetadata_lineage_api\"}}}"
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections delete 'openlineage_config' 2>/dev/null || true
|
||
|
|
docker exec sicabe-airflow-webserver airflow connections add 'openlineage_config' \
|
||
|
|
--conn-type 'generic' --conn-extra "${OL_TRANSPORT}" >/dev/null
|
||
|
|
echo -e "${OK} Connections OpenLineage dibuat (openlineage_config + openmetadata_lineage_api)."
|
||
|
|
echo -e "${INFO} Aktifkan: set AIRFLOW_OPENLINEAGE_CONFIG_CONN_ID=openlineage_config lalu recreat webserver+scheduler."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Token bot 'lineage-bot' tidak diperoleh — OpenLineage dibiarkan inert."
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Login OpenMetadata gagal — OpenLineage dibiarkan inert."
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Gagal akses Airflow CLI."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Copy contoh DAG
|
||
|
|
echo -e "${INFO} Menyalin contoh DAG..."
|
||
|
|
if docker exec sicabe-airflow-webserver ls /opt/airflow/dags/ >/dev/null 2>&1; then
|
||
|
|
docker cp "${SCRIPT_DIR}/config/airflow/dags/example_sicabe_pipeline.py" \
|
||
|
|
sicabe-airflow-webserver:/opt/airflow/dags/ 2>/dev/null && \
|
||
|
|
echo -e "${OK} Contoh DAG disalin." || \
|
||
|
|
echo -e "${WARN} Gagal menyalin DAG. Salin manual: docker cp config/airflow/dags/ sicabe-airflow-webserver:/opt/airflow/dags/"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Aktifkan DAG contoh. Default Airflow adalah PAUSED, sehingga pipeline harian tidak
|
||
|
|
# akan jalan sampai diaktifkan. Dilakukan di sini (skrip provisioning, yang memang
|
||
|
|
# memiliki DAG-nya) supaya instalasi baru langsung siap. Ada retry singkat karena
|
||
|
|
# dagbag bisa belum memuat berkas yang baru disalin.
|
||
|
|
local _i _up=0
|
||
|
|
for _i in 1 2 3 4 5 6; do
|
||
|
|
if docker exec sicabe-airflow-webserver airflow dags unpause sicabe_batch_daily >/dev/null 2>&1; then
|
||
|
|
_up=1; break
|
||
|
|
fi
|
||
|
|
sleep 5
|
||
|
|
done
|
||
|
|
if [[ ${_up} -eq 1 ]]; then
|
||
|
|
echo -e "${OK} DAG 'sicabe_batch_daily' diaktifkan (unpause)."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Gagal mengaktifkan DAG 'sicabe_batch_daily' — jalankan manual:"
|
||
|
|
echo -e "${WARN} docker exec sicabe-airflow-webserver airflow dags unpause sicabe_batch_daily"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Seed SUMBER contoh pipeline (PostgreSQL sicabe_data.public.orders).
|
||
|
|
# Idempoten. Tanpa ini, DAG sicabe_batch_daily gagal di task ingest_to_bronze
|
||
|
|
# dengan pesan database sicabe_data tidak ditemukan.
|
||
|
|
echo -e "${INFO} Menyiapkan sumber contoh pipeline (sicabe_data.public.orders)..."
|
||
|
|
if [[ -x "${SCRIPT_DIR}/scripts/seed-source-postgres.sh" ]]; then
|
||
|
|
local seed_rc=0
|
||
|
|
"${SCRIPT_DIR}/scripts/seed-source-postgres.sh" >/dev/null || seed_rc=$?
|
||
|
|
if [[ ${seed_rc} -eq 0 ]]; then
|
||
|
|
echo -e "${OK} Sumber contoh siap (sicabe_data.public.orders)."
|
||
|
|
elif [[ ${seed_rc} -eq 3 ]]; then
|
||
|
|
echo -e "${WARN} Sumber contoh ada DI HOST LAIN (PostgreSQL) — jalankan DI host itu:"
|
||
|
|
echo -e "${WARN} ./scripts/seed-source-postgres.sh --env-file generated/host-state/.env"
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Seeder sumber gagal (kode ${seed_rc}). Jalankan manual: ./scripts/seed-source-postgres.sh"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
echo -e "${WARN} scripts/seed-source-postgres.sh tidak ada / tidak executable."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 6 selesai — Airflow terkonfigurasi.${NC}"
|
||
|
|
echo " URL: http://localhost:${AIRFLOW_PORT:-12082}"
|
||
|
|
echo " Login: admin / ${AIRFLOW_ADMIN_PASSWORD}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
|
# Generate file kredensial runtime (JANGAN di-commit ke git):
|
||
|
|
# - config/airflow/sling/env.yaml (kredensial Sling: PostgreSQL, Nessie, MinIO)
|
||
|
|
# - config/airflow/simple_auth_manager_passwords.json (password admin Airflow)
|
||
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
|
generate_runtime_credentials() {
|
||
|
|
echo -e "${INFO} Generate file kredensial runtime (sling + airflow passwords)..."
|
||
|
|
|
||
|
|
# Multi-host: tulis ke BUNDLE HOST, jangan ke config/ bersama. Endpoint di file
|
||
|
|
# ini khusus host tersebut; menulisnya ke config/ bersama membuat seluruh deploy
|
||
|
|
# (termasuk single-host) menunjuk alamat host itu. Bug ini NYATA: saat menguji
|
||
|
|
# `bootstrap.sh --host state`, deploy/config/airflow/sling/env.yaml tertimpa
|
||
|
|
# alamat contoh 10.20.0.x (berkas itu gitignored, jadi tidak terlihat git status).
|
||
|
|
local cfg_base="${SCRIPT_DIR}/config"
|
||
|
|
if [[ -n "${HOST_NAME:-}" ]]; then
|
||
|
|
cfg_base="${SCRIPT_DIR}/generated/host-${HOST_NAME}/config"
|
||
|
|
echo -e "${INFO} Mode host '${HOST_NAME}': kredensial runtime → ${cfg_base}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 1. Generate sling env.yaml (target: Iceberg REST via Nessie) ───────────
|
||
|
|
# Sling TIDAK mendukung catalog_type=nessie → pakai Iceberg REST Nessie
|
||
|
|
# (http://nessie:19120/iceberg/), tanpa auth (Nessie terbuka di network internal).
|
||
|
|
local sling_dir="${cfg_base}/airflow/sling"
|
||
|
|
mkdir -p "${sling_dir}"
|
||
|
|
# Guard: Docker membuat DIRECTORY bila sumber bind-mount absen saat `up` → buang & buat file.
|
||
|
|
sicabe_ensure_file_mount "${sling_dir}/env.yaml" 644
|
||
|
|
cat > "${sling_dir}/env.yaml" <<EOF
|
||
|
|
# =============================================================================
|
||
|
|
# eBdesk SiCABE — Sling Environment Configuration (AUTO-GENERATED)
|
||
|
|
# JANGAN commit ke git — di-generate oleh bootstrap.sh (generate_runtime_credentials)
|
||
|
|
# Catatan: Sling TIDAK mendukung catalog_type=nessie → pakai Iceberg REST Nessie.
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
connections:
|
||
|
|
POSTGRES_DB:
|
||
|
|
type: postgres
|
||
|
|
host: ${POSTGRES_HOST:-postgres}
|
||
|
|
port: 5432
|
||
|
|
database: sicabe_data
|
||
|
|
user: sicabe_admin
|
||
|
|
password: ${POSTGRES_PASSWORD}
|
||
|
|
|
||
|
|
ICEBERG:
|
||
|
|
type: iceberg
|
||
|
|
catalog_type: rest
|
||
|
|
catalog_name: iceberg
|
||
|
|
schema: main
|
||
|
|
rest_uri: ${NESSIE_URL:-http://nessie:19120}/iceberg/
|
||
|
|
rest_warehouse: warehouse
|
||
|
|
s3_endpoint: ${S3_ENDPOINT:-http://minio:9000}
|
||
|
|
s3_access_key_id: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
||
|
|
s3_secret_access_key: ${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD}}
|
||
|
|
s3_region: ${S3_REGION:-us-east-1}
|
||
|
|
EOF
|
||
|
|
chmod 644 "${sling_dir}/env.yaml"
|
||
|
|
echo -e "${OK} sling/env.yaml di-generate."
|
||
|
|
|
||
|
|
# ── 2. Generate simple_auth_manager_passwords.json ─────────────────────────
|
||
|
|
# SimpleAuthManager membuka file dengan mode r+ → wajib writable (chmod 666)
|
||
|
|
local pw_file="${cfg_base}/airflow/simple_auth_manager_passwords.json"
|
||
|
|
# Guard: SimpleAuthManager gagal (IsADirectoryError) bila path berupa direktori sisa `up`.
|
||
|
|
sicabe_ensure_file_mount "${pw_file}" 666
|
||
|
|
cat > "${pw_file}" <<EOF
|
||
|
|
{"admin": "${AIRFLOW_ADMIN_PASSWORD}"}
|
||
|
|
EOF
|
||
|
|
chmod 666 "${pw_file}"
|
||
|
|
echo -e "${OK} simple_auth_manager_passwords.json di-generate."
|
||
|
|
|
||
|
|
# ── 3. Generate Trino password.db (service account, HTTP Basic) ────────────
|
||
|
|
# Pra-syarat: htpasswd (httpd-tools). Trino memakai authenticator 'file'
|
||
|
|
# (bcrypt). Bind-mount file harus ada sebelum Trino start.
|
||
|
|
if ! command -v htpasswd &>/dev/null; then
|
||
|
|
echo -e "${ERR} htpasswd tidak ditemukan (dibutuhkan untuk password service account Trino)."
|
||
|
|
echo " Install: Debian/Ubuntu → sudo apt-get install -y apache2-utils"
|
||
|
|
echo " RHEL/Fedora → sudo dnf install -y httpd-tools"
|
||
|
|
echo " macOS → brew install httpd"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
local trino_pw_file="${cfg_base}/trino/password.db"
|
||
|
|
# Guard: Docker membuat DIRECTORY bila sumber bind-mount absen → buang dir sisa & buat file.
|
||
|
|
sicabe_ensure_file_mount "${trino_pw_file}" 644
|
||
|
|
: > "${trino_pw_file}" # kosongkan (idempoten: jangan menumpuk baris saat re-run)
|
||
|
|
local pair tuser tpass
|
||
|
|
for pair in \
|
||
|
|
"svc_seed:${TRINO_SVC_SEED_PASSWORD:?}" \
|
||
|
|
"svc_dbt:${TRINO_SVC_DBT_PASSWORD:?}" \
|
||
|
|
"svc_airflow:${TRINO_SVC_AIRFLOW_PASSWORD:?}" \
|
||
|
|
"svc_om:${TRINO_SVC_OM_PASSWORD:?}"; do
|
||
|
|
tuser="${pair%%:*}"; tpass="${pair#*:}"
|
||
|
|
htpasswd -nbB -C 10 "${tuser}" "${tpass}" | sed '/^$/d' >> "${trino_pw_file}"
|
||
|
|
done
|
||
|
|
chmod 644 "${trino_pw_file}"
|
||
|
|
echo -e "${OK} Trino password.db di-generate: $(cut -d: -f1 "${trino_pw_file}" | paste -sd, -)"
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 7: OpenMetadata — Konektor (Trino, Airflow) (bab 5.5)
|
||
|
|
# =============================================================================
|
||
|
|
# Daftarkan service/konektor ke OpenMetadata — kegagalan TIDAK disembunyikan.
|
||
|
|
# Sebelumnya `curl -sf … || echo "mungkin sudah ada"` menutupi error nyata
|
||
|
|
# (mis. field 'verifySSL' yang tidak dikenal skema koneksi OM 2.0.2) sehingga konektor tidak
|
||
|
|
# pernah terbuat padahal laporannya tampak sukses.
|
||
|
|
om_register_service() { # <path> <json> <label>
|
||
|
|
local path="$1" json="$2" label="$3" resp name id patch_body patch_resp
|
||
|
|
resp="$(curl -s -X POST "${om_url}${path}" \
|
||
|
|
-H "Authorization: Bearer ${om_token}" \
|
||
|
|
-H "Content-Type: application/json" -d "${json}" 2>/dev/null)"
|
||
|
|
if [[ "${resp}" == *'"fullyQualifiedName"'* ]]; then
|
||
|
|
echo -e "${OK} ${label} didaftarkan."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
if [[ "${resp}" != *"already exist"* ]]; then
|
||
|
|
echo -e "${WARN} ${label} GAGAL: $(echo "${resp}" | tr -d '\n' | cut -c1-170)"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Sudah ada → POST TIDAK meng-update apa pun. Ini berbahaya begitu endpoint
|
||
|
|
# berubah (mis. Trino dipisah coordinator/worker, atau penempatan multi-host):
|
||
|
|
# konfigurasi tersimpan di OM menjadi BASI dan gate akan melaporkan "menyimpang".
|
||
|
|
# Karena itu script ini adalah sumber kebenaran → sinkronkan blok `connection`.
|
||
|
|
name="$(printf '%s' "${json}" | python3 -c "
|
||
|
|
import sys,json
|
||
|
|
print(json.load(sys.stdin).get('name',''))" 2>/dev/null)"
|
||
|
|
id="$(curl -s -X GET "${om_url}${path}/name/${name}" \
|
||
|
|
-H "Authorization: Bearer ${om_token}" 2>/dev/null | python3 -c "
|
||
|
|
import sys,json
|
||
|
|
print(json.load(sys.stdin).get('id',''))" 2>/dev/null)"
|
||
|
|
if [[ -z "${id}" ]]; then
|
||
|
|
echo -e "${WARN} ${label} sudah ada, tetapi id-nya tidak ditemukan — konfigurasi tidak disinkronkan."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
patch_body="$(printf '%s' "${json}" | python3 -c "
|
||
|
|
import sys,json
|
||
|
|
j=json.load(sys.stdin)
|
||
|
|
print(json.dumps([{'op':'replace','path':'/connection','value':j.get('connection',{})}]))" 2>/dev/null)"
|
||
|
|
patch_resp="$(curl -s -X PATCH "${om_url}${path}/${id}" \
|
||
|
|
-H "Authorization: Bearer ${om_token}" \
|
||
|
|
-H "Content-Type: application/json-patch+json" \
|
||
|
|
-d "${patch_body}" 2>/dev/null)"
|
||
|
|
if [[ "${patch_resp}" == *'"fullyQualifiedName"'* ]]; then
|
||
|
|
echo -e "${OK} ${label} sudah ada — konfigurasi koneksi disinkronkan."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} ${label} sudah ada, sinkronisasi koneksi GAGAL: $(echo "${patch_resp}" | tr -d '\n' | cut -c1-170)"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
bootstrap_openmetadata() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 7/9: OpenMetadata — Konektor (Trino, Airflow) ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.6.1.4-4.6.1.8"
|
||
|
|
|
||
|
|
local om_url="${OPENMETADATA_URL:-http://localhost:${OPENMETADATA_PORT:-12085}}"
|
||
|
|
|
||
|
|
# Tunggu OpenMetadata siap
|
||
|
|
echo -e "${INFO} Menunggu OpenMetadata siap..."
|
||
|
|
for i in $(seq 1 60); do
|
||
|
|
if curl -sf "${om_url}/" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} OpenMetadata siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 60 ]]; then
|
||
|
|
echo -e "${WARN} OpenMetadata belum siap dalam 2 menit. Lanjutkan..."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 2
|
||
|
|
done
|
||
|
|
|
||
|
|
# Dapatkan token OpenMetadata
|
||
|
|
# OpenMetadata 2.0.x: password pada payload login HARUS di-encode Base64
|
||
|
|
# (tanpa itu dijawab 400 "Password needs to be encoded in Base-64."); email
|
||
|
|
# admin default admin@open-metadata.org. OM 2.0.x TIDAK lagi membaca env
|
||
|
|
# OPENMETADATA_ADMIN_PASSWORD saat bootstrap → default 'admin' sampai diganti
|
||
|
|
# lewat UI, dan password pengganti WAJIB memenuhi kebijakan 2.0.x
|
||
|
|
# (min 8 + huruf besar + kecil + digit + spesial) — lihat install.sh.
|
||
|
|
local om_admin_email="admin@open-metadata.org"
|
||
|
|
local om_admin_pass="${OPENMETADATA_ADMIN_PASSWORD:-admin}"
|
||
|
|
|
||
|
|
# Best-effort: set hash password admin langsung di DB memakai skema 2.0.x
|
||
|
|
# (kredensial ada di user_entity.authenticationMechanism.config.password;
|
||
|
|
# field `password` tingkat atas pada 2.0.x kosong). Butuh modul bcrypt di HOST
|
||
|
|
# — image OM 2.0.2 tidak lagi menyediakan python3. Bila gagal, lewati: login
|
||
|
|
# tetap dicoba dengan password .env lalu 'admin' di bawah.
|
||
|
|
# Hash bcrypt: coba modul python bcrypt dulu, lalu `htpasswd -nbB` yang sudah menjadi
|
||
|
|
# prasyarat Trino (httpd-tools) sehingga biasanya TERSEDIA. Ini penting: bila keduanya
|
||
|
|
# tidak ada, dulu blok ini dilewati TANPA pesan apa pun, sehingga OM tetap memakai
|
||
|
|
# password default 'admin' dan gate fase 6 (yang saat itu hanya mencoba password .env)
|
||
|
|
# gagal dengan pesan yang menyesatkan.
|
||
|
|
local om_hash=""
|
||
|
|
om_hash=$(python3 -c "import bcrypt;print(bcrypt.hashpw(b'${om_admin_pass}',bcrypt.gensalt(rounds=12,prefix=b'2a')).decode())" 2>/dev/null) || true
|
||
|
|
if [[ -z "${om_hash}" ]] && command -v htpasswd &>/dev/null; then
|
||
|
|
om_hash="$(htpasswd -nbB -C 12 admin "${om_admin_pass}" 2>/dev/null | cut -d: -f2)" || true
|
||
|
|
[[ -n "${om_hash}" ]] && echo -e "${INFO} Hash password dibuat lewat htpasswd (modul python bcrypt tidak tersedia)."
|
||
|
|
fi
|
||
|
|
if [[ -z "${om_hash}" ]]; then
|
||
|
|
echo -e "${WARN} Tidak bisa membuat hash bcrypt (python bcrypt DAN htpasswd tidak tersedia)."
|
||
|
|
echo -e "${WARN} Password admin OpenMetadata dibiarkan default; login memakai 'admin'."
|
||
|
|
else
|
||
|
|
docker exec sicabe-postgres psql -U openmetadata -d openmetadata_db -c \
|
||
|
|
"UPDATE user_entity SET json=jsonb_set(json::jsonb,'{authenticationMechanism,config,password}','\"${om_hash}\"')::json WHERE name='admin';" >/dev/null 2>&1 \
|
||
|
|
&& echo -e "${OK} Password admin OpenMetadata di-set (${om_admin_email})." \
|
||
|
|
|| echo -e "${WARN} UPDATE hash password admin gagal — login akan dicoba dengan .env lalu 'admin'."
|
||
|
|
fi
|
||
|
|
|
||
|
|
local om_token=""
|
||
|
|
local try_pass om_pass_b64
|
||
|
|
for try_pass in "${om_admin_pass}" "admin"; do
|
||
|
|
om_pass_b64=$(printf '%s' "${try_pass}" | base64)
|
||
|
|
om_token=$(curl -sf -X POST "${om_url}/api/v1/users/login" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"email\":\"${om_admin_email}\",\"password\":\"${om_pass_b64}\"}" \
|
||
|
|
2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin).get('accessToken',''))") || true
|
||
|
|
if [[ -n "${om_token}" ]]; then
|
||
|
|
echo -e "${OK} Login OpenMetadata berhasil (password: ${try_pass})."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
if [[ -z "${om_token}" ]]; then
|
||
|
|
echo -e "${WARN} Gagal login ke OpenMetadata."
|
||
|
|
echo -e "${INFO} Jalankan konfigurasi konektor manual dari UI OpenMetadata."
|
||
|
|
echo " URL: ${om_url}"
|
||
|
|
echo " Login: ${om_admin_email} / (default: admin)"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Daftarkan konektor Trino (Processing Layer)
|
||
|
|
# ⚠️ OpenMetadata 2.0.x (diperiksa di image 2.0.2) TIDAK menyediakan konektor
|
||
|
|
# Iceberg standalone: modul metadata.ingestion.source.database.iceberg tidak ada
|
||
|
|
# dan paket ingestion tidak punya extra 'iceberg'. Iceberg tables di-surface
|
||
|
|
# melalui query engine (Trino) — lihat docs:
|
||
|
|
# https://docs.open-metadata.org/v2.0.x/connectors/database/iceberg
|
||
|
|
# ⚠️ Skema koneksi Trino di OM 2.0.2 TIDAK mengenal field 'verifySSL' maupun
|
||
|
|
# 'sslConfig' → mengirimnya membuat pembuatan service GAGAL
|
||
|
|
# ("unrecognized field"). Jadi field itu TIDAK dikirim; trust TLS diatur di
|
||
|
|
# sisi OM bila nanti menjalankan ingestion (bukan bagian deploy ini).
|
||
|
|
echo -e "${INFO} Mendaftarkan konektor Trino (Iceberg via Nessie)..."
|
||
|
|
om_register_service "/api/v1/services/databaseServices" "{
|
||
|
|
\"name\": \"trino_sicabe\",
|
||
|
|
\"serviceType\": \"Trino\",
|
||
|
|
\"description\": \"Trino — Query Engine untuk Iceberg via Nessie (Storage Layer)\",
|
||
|
|
\"connection\": {
|
||
|
|
\"config\": {
|
||
|
|
\"type\": \"Trino\",
|
||
|
|
\"scheme\": \"trino\",
|
||
|
|
\"hostPort\": \"trino-coordinator:8443\",
|
||
|
|
\"username\": \"svc_om\",
|
||
|
|
\"authType\": { \"password\": \"${TRINO_SVC_OM_PASSWORD}\" },
|
||
|
|
\"catalog\": \"iceberg\",
|
||
|
|
\"databaseSchema\": \"bronze\",
|
||
|
|
\"supportsMetadataExtraction\": true,
|
||
|
|
\"supportsProfiler\": true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}" "Konektor Trino 'trino_sicabe'"
|
||
|
|
|
||
|
|
# Daftarkan konektor Airflow (Ingestion Layer)
|
||
|
|
echo -e "${INFO} Mendaftarkan konektor Airflow..."
|
||
|
|
om_register_service "/api/v1/services/pipelineServices" "{
|
||
|
|
\"name\": \"airflow_sicabe\",
|
||
|
|
\"serviceType\": \"Airflow\",
|
||
|
|
\"description\": \"Apache Airflow — Pipeline Orchestrator (Ingestion Layer)\",
|
||
|
|
\"connection\": {
|
||
|
|
\"config\": {
|
||
|
|
\"type\": \"Airflow\",
|
||
|
|
\"hostPort\": \"http://airflow-webserver:8080\",
|
||
|
|
\"supportsMetadataExtraction\": true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}" "Konektor Airflow 'airflow_sicabe'"
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 7 selesai — OpenMetadata terkonfigurasi.${NC}"
|
||
|
|
echo " URL: ${om_url}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 8: Grafana — Datasource + Dashboard (bab 5.5)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_grafana() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 8/9: Grafana — Datasource + Dashboard ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.6.3.2-4.6.3.6"
|
||
|
|
|
||
|
|
local grafana_url="${GRAFANA_URL:-http://localhost:${GRAFANA_PORT:-12300}}"
|
||
|
|
|
||
|
|
# Tunggu Grafana siap
|
||
|
|
echo -e "${INFO} Menunggu Grafana siap..."
|
||
|
|
for i in $(seq 1 20); do
|
||
|
|
if curl -sf "${grafana_url}/api/health" >/dev/null 2>&1; then
|
||
|
|
echo -e "${OK} Grafana siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 20 ]]; then
|
||
|
|
echo -e "${WARN} Grafana belum siap."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 3
|
||
|
|
done
|
||
|
|
|
||
|
|
# Dapatkan akses Grafana API
|
||
|
|
# ⚠️ Grafana 13.x: endpoint /api/auth/login deprecated (401).
|
||
|
|
# Gunakan Basic Auth langsung (terbukti bekerja untuk /api/*).
|
||
|
|
local gf_basic_auth
|
||
|
|
gf_basic_auth="admin:${GRAFANA_ADMIN_PASSWORD:-grafana123}"
|
||
|
|
|
||
|
|
# Verifikasi akses via Basic Auth
|
||
|
|
local user_check
|
||
|
|
user_check=$(curl -sf -u "${gf_basic_auth}" \
|
||
|
|
"${grafana_url}/api/user" 2>/dev/null) || true
|
||
|
|
if [[ -z "${user_check}" ]]; then
|
||
|
|
echo -e "${WARN} Gagal autentikasi ke Grafana API."
|
||
|
|
echo -e "${INFO} Grafana datasource sudah diprovisioning via file config."
|
||
|
|
echo " URL: ${grafana_url}"
|
||
|
|
echo " Login: admin / ${GRAFANA_ADMIN_PASSWORD:-grafana123}"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Autentikasi Grafana berhasil (Basic Auth)."
|
||
|
|
|
||
|
|
# Verifikasi datasource sudah terprovisioning
|
||
|
|
local ds_check
|
||
|
|
ds_check=$(curl -sf -u "${gf_basic_auth}" \
|
||
|
|
"${grafana_url}/api/datasources" 2>/dev/null)
|
||
|
|
if echo "${ds_check}" | grep -q "Prometheus"; then
|
||
|
|
echo -e "${OK} Datasource Prometheus terverifikasi."
|
||
|
|
fi
|
||
|
|
if echo "${ds_check}" | grep -q "OpenMetadata"; then
|
||
|
|
echo -e "${OK} Datasource OpenMetadata terverifikasi."
|
||
|
|
fi
|
||
|
|
if echo "${ds_check}" | grep -q "MinIO"; then
|
||
|
|
echo -e "${OK} Datasource MinIO Metrics terverifikasi."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 8 selesai — Grafana terkonfigurasi.${NC}"
|
||
|
|
echo " URL: ${grafana_url}"
|
||
|
|
echo " Login: admin / ${GRAFANA_ADMIN_PASSWORD:-grafana123}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STEP 9: Dremio — First-run admin + Source Nessie (bab 5.3 lanjutan)
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_dremio() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══ STEP 9/9: Dremio — First-run admin + Source Nessie ═══${NC}"
|
||
|
|
echo -e "${INFO} Referensi: bab 4.3 lanjutan (query engine kedua di atas Nessie)"
|
||
|
|
|
||
|
|
# Dremio bersifat OPT-IN (profile `dremio`) → dilewati bila container tidak ada.
|
||
|
|
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -qx 'sicabe-dremio'; then
|
||
|
|
echo -e "${WARN} Container sicabe-dremio tidak berjalan — langkah dilewati."
|
||
|
|
echo -e "${INFO} Aktifkan dengan: docker compose --profile dremio up -d dremio"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
local dremio_url="${DREMIO_URL:-http://localhost:${DREMIO_UI_PORT:-12947}}"
|
||
|
|
local dremio_user="${DREMIO_USER:-admin}"
|
||
|
|
local dremio_pass="${DREMIO_PASSWORD:-dremio_admin_DevOnly1}"
|
||
|
|
local dremio_email="${DREMIO_ADMIN_EMAIL:-admin@sicabe.local}"
|
||
|
|
|
||
|
|
# Tunggu Dremio siap (server up → kode HTTP apa pun selain 000)
|
||
|
|
echo -e "${INFO} Menunggu Dremio siap..."
|
||
|
|
for i in $(seq 1 60); do
|
||
|
|
local http_code
|
||
|
|
http_code=$(curl -s -o /dev/null -w "%{http_code}" "${dremio_url}/" 2>/dev/null) || http_code="000"
|
||
|
|
if [[ "${http_code}" != "000" ]]; then
|
||
|
|
echo -e "${OK} Dremio siap."
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
if [[ $i -eq 60 ]]; then
|
||
|
|
echo -e "${WARN} Dremio belum siap dalam 3 menit. Lanjutkan..."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
sleep 3
|
||
|
|
done
|
||
|
|
|
||
|
|
# ── 1. Pastikan admin first-run ada (idempoten) ─────────────────────────
|
||
|
|
# Dremio OSS belum punya admin saat pertama start. Bila belum ada, buat via
|
||
|
|
# REST API first-run (PUT-only); 400 = sudah pernah dibuat.
|
||
|
|
local login_code
|
||
|
|
login_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
|
|
-X POST "${dremio_url}/apiv2/login" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"userName\":\"${dremio_user}\",\"password\":\"${dremio_pass}\"}" \
|
||
|
|
2>/dev/null) || login_code="000"
|
||
|
|
|
||
|
|
if [[ "${login_code}" == "200" ]]; then
|
||
|
|
echo -e "${OK} Admin '${dremio_user}' sudah ada."
|
||
|
|
else
|
||
|
|
echo -e "${INFO} Admin belum ada (login → HTTP ${login_code}). Membuat first user..."
|
||
|
|
# Dremio mewajibkan password ≥8 karakter, mengandung huruf DAN angka.
|
||
|
|
local create_code create_msg
|
||
|
|
create_code=$(curl -s -o /tmp/dremio_firstuser.json -w "%{http_code}" \
|
||
|
|
-X PUT "${dremio_url}/apiv2/bootstrap/firstuser" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-H "Authorization: _dremionull" \
|
||
|
|
-d "{\"userName\":\"${dremio_user}\",\"firstName\":\"Dremio\",\"lastName\":\"Admin\",\"email\":\"${dremio_email}\",\"password\":\"${dremio_pass}\"}" \
|
||
|
|
2>/dev/null) || create_code="000"
|
||
|
|
create_msg=$(python3 -c "import json;print(json.load(open('/tmp/dremio_firstuser.json')).get('errorMessage',''))" 2>/dev/null || true)
|
||
|
|
case "${create_code}" in
|
||
|
|
200)
|
||
|
|
echo -e "${OK} Admin '${dremio_user}' dibuat (first-run)."
|
||
|
|
;;
|
||
|
|
400)
|
||
|
|
if echo "${create_msg}" | grep -qi "already"; then
|
||
|
|
echo -e "${OK} First user sudah ada (400)."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Pembuatan admin ditolak: ${create_msg:-HTTP 400}"
|
||
|
|
fi
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
echo -e "${WARN} Gagal membuat admin Dremio (HTTP ${create_code})."
|
||
|
|
echo " Buat manual via UI: ${dremio_url} (form first-run)."
|
||
|
|
return 0
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 2. Verifikasi login ─────────────────────────────────────────────────
|
||
|
|
local verify_code
|
||
|
|
verify_code=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
|
|
-X POST "${dremio_url}/apiv2/login" \
|
||
|
|
-H "Content-Type: application/json" \
|
||
|
|
-d "{\"userName\":\"${dremio_user}\",\"password\":\"${dremio_pass}\"}" \
|
||
|
|
2>/dev/null) || verify_code="000"
|
||
|
|
if [[ "${verify_code}" != "200" ]]; then
|
||
|
|
echo -e "${WARN} Login Dremio gagal (HTTP ${verify_code})."
|
||
|
|
echo " Set DREMIO_USER/DREMIO_PASSWORD di .env bila admin berbeda."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ── 3. Registrasi source Nessie (idempoten) ─────────────────────────────
|
||
|
|
echo -e "${INFO} Mendaftarkan source Nessie (via REST API)..."
|
||
|
|
if DREMIO_URL="${dremio_url}" DREMIO_USER="${dremio_user}" DREMIO_PASSWORD="${dremio_pass}" \
|
||
|
|
MINIO_ROOT_USER="${MINIO_ROOT_USER:-minioadmin}" MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \
|
||
|
|
"${SCRIPT_DIR}/scripts/dremio/register-nessie-source.sh"; then
|
||
|
|
echo -e "${OK} Source Nessie terdaftar."
|
||
|
|
else
|
||
|
|
echo -e "${WARN} Registrasi source Nessie gagal (lihat pesan di atas)."
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} ${GREEN}Step 9 selesai — Dremio terkonfigurasi.${NC}"
|
||
|
|
echo " URL: ${dremio_url}"
|
||
|
|
echo " Login: ${dremio_user} / ${dremio_pass}"
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# STATUS BOOTSTRAP
|
||
|
|
# =============================================================================
|
||
|
|
bootstrap_status() {
|
||
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo -e "${BOLD} STATUS BOOTSTRAP eBdesk SiCABE${NC}"
|
||
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
for svc in keycloak openldap minio nessie opa trino starrocks-fe starrocks-cn dremio airflow-webserver openmetadata grafana; do
|
||
|
|
local container_name="sicabe-${svc}"
|
||
|
|
if docker ps --format "{{.Names}}" | grep -q "${container_name}"; then
|
||
|
|
echo -e "${GREEN}✅ ${container_name} — RUNNING${NC}"
|
||
|
|
else
|
||
|
|
if docker ps -a --format "{{.Names}}" | grep -q "${container_name}"; then
|
||
|
|
echo -e "${RED}❌ ${container_name} — STOPPED${NC}"
|
||
|
|
else
|
||
|
|
echo -e "${YELLOW}⬜ ${container_name} — NOT FOUND${NC}"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Endpoints:${NC}"
|
||
|
|
echo -e " Keycloak: ${CYAN}${KEYCLOAK_URL:-http://localhost:${KEYCLOAK_PORT:-12080}}/admin/${SICABE_REALM:-sicabe}/console${NC}"
|
||
|
|
echo -e " OpenLDAP: ${CYAN}ldap://localhost:${OPENLDAP_PORT:-12389}${NC}"
|
||
|
|
echo -e " MinIO Console: ${CYAN}http://localhost:${MINIO_CONSOLE_PORT:-12901}${NC}"
|
||
|
|
echo -e " Nessie: ${CYAN}http://localhost:${NESSIE_PORT:-12120}${NC}"
|
||
|
|
echo -e " OPA: ${CYAN}http://localhost:${OPA_PORT:-12181}${NC}"
|
||
|
|
echo -e " Trino: ${CYAN}https://localhost:${TRINO_HTTPS_PORT:-12443}${NC} (API + JDBC, HTTPS-only)"
|
||
|
|
echo -e " Trino Web UI: ${CYAN}https://localhost:${TRINO_HTTPS_PORT:-12443}${NC} (OAuth2, self-signed)"
|
||
|
|
echo -e " StarRocks FE: ${CYAN}localhost:${STARROCKS_FE_PORT:-12930}${NC} (MySQL protocol)"
|
||
|
|
echo -e " Dremio: ${CYAN}http://localhost:${DREMIO_UI_PORT:-12947}${NC}"
|
||
|
|
echo -e " Airflow: ${CYAN}http://localhost:${AIRFLOW_PORT:-12082}${NC}"
|
||
|
|
echo -e " OpenMetadata: ${CYAN}http://localhost:${OPENMETADATA_PORT:-12085}${NC}"
|
||
|
|
echo -e " Grafana: ${CYAN}http://localhost:${GRAFANA_PORT:-12300}${NC}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
# MAIN
|
||
|
|
|
||
|
|
# ─── Multi-host (Fase 2b): step mana milik tier mana ─────────────────────────
|
||
|
|
# Dipakai `--host <nama>`: hanya menjalankan step yang relevan bagi host itu,
|
||
|
|
# memakai endpoint dari generated/host-<nama>/.env.
|
||
|
|
step_tiers() {
|
||
|
|
case "$1" in
|
||
|
|
1|3) echo "identity" ;; # Keycloak realm/user, OpenLDAP seed
|
||
|
|
2) echo "state" ;; # bucket MinIO
|
||
|
|
4|5) echo "coord engine" ;; # Trino & StarRocks (katalog + registry CN)
|
||
|
|
6) echo "pipeline" ;; # Airflow connections + DAG
|
||
|
|
7|8) echo "governance" ;; # OpenMetadata connector, Grafana datasource
|
||
|
|
9) echo "engine" ;; # Dremio (opt-in)
|
||
|
|
*) echo "" ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# =============================================================================
|
||
|
|
main() {
|
||
|
|
local START_STEP=1
|
||
|
|
local END_STEP=9
|
||
|
|
local HOST_NAME=""
|
||
|
|
local DRY_RUN=0
|
||
|
|
local HOST_TIERS=""
|
||
|
|
|
||
|
|
if [[ $# -gt 0 ]]; then
|
||
|
|
case "$1" in
|
||
|
|
--help|-h) usage_bootstrap ;;
|
||
|
|
--status) bootstrap_status; exit 0 ;;
|
||
|
|
--step)
|
||
|
|
if [[ $# -lt 2 ]]; then
|
||
|
|
echo -e "${ERR} Sebutkan step (1-9)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
START_STEP="$2"
|
||
|
|
END_STEP="$2"
|
||
|
|
[[ "${3:-}" == "--dry-run" ]] && DRY_RUN=1
|
||
|
|
;;
|
||
|
|
--host)
|
||
|
|
if [[ $# -lt 2 ]]; then
|
||
|
|
echo -e "${ERR} Sebutkan nama host (lihat: install.sh --generate)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
HOST_NAME="$2"
|
||
|
|
shift
|
||
|
|
[[ "${2:-}" == "--dry-run" ]] && DRY_RUN=1
|
||
|
|
;;
|
||
|
|
--dry-run) DRY_RUN=1 ;;
|
||
|
|
*)
|
||
|
|
echo -e "${ERR} Argumen tidak dikenal: $1"
|
||
|
|
usage_bootstrap
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Rencana step: default SELURUH rentang (perilaku lama). Dalam mode host, disaring
|
||
|
|
# ke step yang relevan bagi host itu.
|
||
|
|
local PLAN_STEPS
|
||
|
|
PLAN_STEPS="$(seq "${START_STEP}" "${END_STEP}" | paste -sd' ' -)"
|
||
|
|
if [[ -n "${HOST_NAME}" ]]; then
|
||
|
|
# Pakai .env hasil generate untuk host ini (rahasia + endpoint yang sudah
|
||
|
|
# diarahkan ke ALAMAT HOST, lihat Fase 1/2a). Tanpa ini, step akan menembak
|
||
|
|
# localhost/12xxx milik host tempat script dijalankan.
|
||
|
|
PLAN_STEPS=""
|
||
|
|
local henv="${SCRIPT_DIR}/generated/host-${HOST_NAME}/.env"
|
||
|
|
if [[ ! -f "${henv}" ]]; then
|
||
|
|
echo -e "${ERR} Belum ada ${henv}"
|
||
|
|
echo " Jalankan dulu: ./install.sh --generate"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
set -a; source "${henv}"; set +a
|
||
|
|
# Wariskan berkas env HOST ini ke sub-script (seed-*/ingest-*/reapply) lewat helper
|
||
|
|
# sicabe_load_env(): tanpa ini sub-script menimpa endpoint host dengan deploy/.env
|
||
|
|
# (nilai single-host) → menyentuh endpoint yang salah di multi-host.
|
||
|
|
export SICABE_ENV_FILE="${henv}"
|
||
|
|
# Generator menulis daftar tier dengan pemisah KOMA (SICABE_HOST_TIERS=
|
||
|
|
# identity,policy,pipeline,governance); pencocokan di bawah memakai spasi.
|
||
|
|
HOST_TIERS="${SICABE_HOST_TIERS//,/ }"
|
||
|
|
echo -e "${INFO} Mode host '${HOST_NAME}' — tiers=[${HOST_TIERS}] (endpoint dari ${henv})"
|
||
|
|
local s tier want
|
||
|
|
for s in $(seq "${START_STEP}" "${END_STEP}"); do
|
||
|
|
want=0
|
||
|
|
for tier in $(step_tiers "${s}"); do
|
||
|
|
[[ " ${HOST_TIERS} " == *" ${tier} "* ]] && want=1
|
||
|
|
done
|
||
|
|
# Dremio dimatikan di inventory (enabled:false) -> EXTERNAL_SERVICES memuatnya
|
||
|
|
if [[ "${s}" == "9" && " ${EXTERNAL_SERVICES:-} " == *" dremio "* ]]; then
|
||
|
|
want=0
|
||
|
|
fi
|
||
|
|
[[ ${want} -eq 1 ]] && PLAN_STEPS+=" ${s}"
|
||
|
|
done
|
||
|
|
echo -e "${INFO} Step untuk host ini:${PLAN_STEPS:- (tidak ada)}"
|
||
|
|
if [[ -z "${PLAN_STEPS// /}" ]]; then
|
||
|
|
echo -e "${WARN} Tidak ada step bootstrap untuk host ini."
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ ${DRY_RUN} -eq 1 ]]; then
|
||
|
|
echo -e "${INFO} (dry-run) step yang akan dijalankan: ${PLAN_STEPS}"
|
||
|
|
echo -e "${INFO} (dry-run) tidak ada yang dijalankan."
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${BOLD}${BLUE}"
|
||
|
|
echo "╔═══════════════════════════════════════════════════════╗"
|
||
|
|
echo "║ eBdesk SiCABE — Bootstrap & Integrasi ║"
|
||
|
|
echo "║ Inisialisasi Komponen & Konfigurasi Lintas Lapisan ║"
|
||
|
|
echo "╚═══════════════════════════════════════════════════════╝"
|
||
|
|
echo -e "${NC}"
|
||
|
|
echo -e "${INFO} Memulai bootstrap step ${START_STEP} sampai ${END_STEP}..."
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Validasi container sudah running
|
||
|
|
echo -e "${INFO} Memastikan container sudah berjalan..."
|
||
|
|
docker ps --format "{{.Names}}" | grep -q "sicabe-" || {
|
||
|
|
echo -e "${ERR} Tidak ada container SiCABE yang berjalan."
|
||
|
|
echo " Jalankan ./install.sh terlebih dahulu."
|
||
|
|
exit 1
|
||
|
|
}
|
||
|
|
echo -e "${OK} Container terdeteksi."
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Kredensial runtime dibuat DULU (sling env.yaml, airflow passwords, trino password.db).
|
||
|
|
# SENGAJA sebelum loop step: bila sebuah step gagal (set -euo pipefail → skrip abort),
|
||
|
|
# berkas runtime tetap ada. Dulu dipanggil SETELAH loop → env.yaml hilang bila step gagal
|
||
|
|
# (bug nyata: verify-phase5 "config/airflow/sling/env.yaml tidak ada").
|
||
|
|
generate_runtime_credentials
|
||
|
|
|
||
|
|
# Jalankan step
|
||
|
|
for step in $(seq "${START_STEP}" "${END_STEP}"); do
|
||
|
|
# Jalankan hanya step dalam rencana (mode host menyaringnya).
|
||
|
|
if [[ " ${PLAN_STEPS} " != *" ${step} "* ]]; then
|
||
|
|
echo -e "${INFO} [skip] step ${step} bukan milik host '${HOST_NAME}' (tier: $(step_tiers "${step}"))"
|
||
|
|
echo ""
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
case ${step} in
|
||
|
|
1) bootstrap_keycloak ;;
|
||
|
|
2) bootstrap_minio ;;
|
||
|
|
3) bootstrap_openldap ;;
|
||
|
|
4) bootstrap_trino ;;
|
||
|
|
5) bootstrap_starrocks ;;
|
||
|
|
6) bootstrap_airflow ;;
|
||
|
|
7) bootstrap_openmetadata ;;
|
||
|
|
8) bootstrap_grafana ;;
|
||
|
|
9) bootstrap_dremio ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
echo -e "${BOLD}${GREEN}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} ✅ BOOTSTRAP SELESAI${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} Jalankan ./test-verification.sh untuk verifikasi${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} Gate kontrak per fase (opsional, jalankan berurutan):${NC}"
|
||
|
|
echo " ./scripts/verify-phase1-iam.sh # IAM: Keycloak + OpenLDAP (fase 1)"
|
||
|
|
echo " ./scripts/verify-phase2-storage.sh # Storage & catalog: MinIO + Nessie (fase 2)"
|
||
|
|
echo " ./scripts/verify-phase3-processing.sh # Processing: Trino + OPA (fase 3)"
|
||
|
|
echo " ./scripts/verify-phase4-serving.sh # Serving: StarRocks shared-data (fase 4)"
|
||
|
|
echo " ./scripts/verify-phase5-ingestion.sh # Ingestion: Airflow + connections (fase 5)"
|
||
|
|
echo " ./scripts/verify-phase6-governance.sh # Governance: OpenMetadata + Grafana (fase 6)"
|
||
|
|
echo " Setelah restart StarRocks FE, pulihkan dynamic config (auth chain OIDC):"
|
||
|
|
echo " ./scripts/starrocks-reapply-dynamic-config.sh"
|
||
|
|
echo -e "${BOLD}${GREEN}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
}
|
||
|
|
|
||
|
|
main "$@"
|