998 lines
45 KiB
Bash
998 lines
45 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# eBdesk SiCABE — Installer
|
||
|
|
# =============================================================================
|
||
|
|
# Sesuai urutan deployment berdasarkan Dependency Graph (bab 3.1 & 12.2):
|
||
|
|
# Phase 1: Governance (IAM) → Keycloak
|
||
|
|
# Phase 2: Storage → MinIO → Nessie
|
||
|
|
# Phase 3: Processing → Trino
|
||
|
|
# Phase 4: Serving → StarRocks
|
||
|
|
# Phase 5: Ingestion → Airflow
|
||
|
|
# Phase 6: Governance (Lan) → OpenMetadata → Grafana
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# ./install.sh # Deploy semua phase
|
||
|
|
# ./install.sh --phase 1 # Deploy hanya phase tertentu
|
||
|
|
# ./install.sh --phase 1-3 # Deploy phase 1 sampai 3
|
||
|
|
# ./install.sh --status # Cek status semua komponen
|
||
|
|
# ./install.sh --destroy # Hentikan dan hapus semua container
|
||
|
|
# ./install.sh --help # Tampilkan bantuan
|
||
|
|
# =============================================================================
|
||
|
|
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
# ─── Base Directory ─────────────────────────────────────────────────────────
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
cd "${SCRIPT_DIR}"
|
||
|
|
|
||
|
|
# ─── Shared Library ──────────────────────────────────────────────────────────
|
||
|
|
source "${SCRIPT_DIR}/lib/common.sh"
|
||
|
|
|
||
|
|
# ─── Color & Styling (overrides untuk install.sh) ────────────────────────────
|
||
|
|
STEP="${BOLD}[${NC}"
|
||
|
|
|
||
|
|
# ─── Generate .env jika belum ada ──────────────────────────────────────────
|
||
|
|
# KECUALI mode multi-host (--host): rahasia untuk host itu sudah ada di
|
||
|
|
# generated/host-<x>/.env. Membuat .env acak di server ini BERBAHAYA karena rahasianya
|
||
|
|
# akan BEDA dari host lain (client secret OIDC, password DB, service account), dan
|
||
|
|
# berkas itu akan dipakai diam-diam oleh pemanggilan TANPA --host (bootstrap.sh,
|
||
|
|
# test-verification.sh, gate fase) sehingga muncul kegagalan login yang sulit dilacak.
|
||
|
|
HOST_MODE_ARG=0
|
||
|
|
ENV_WAS_CREATED=0
|
||
|
|
for _arg in "$@"; do
|
||
|
|
[[ "${_arg}" == "--host" ]] && HOST_MODE_ARG=1
|
||
|
|
done
|
||
|
|
if [[ ${HOST_MODE_ARG} -eq 1 && ! -f "${SCRIPT_DIR}/.env" ]]; then
|
||
|
|
echo -e "${INFO} Mode --host: .env lokal TIDAK dibuat (memakai generated/host-<x>/.env)"
|
||
|
|
fi
|
||
|
|
if [[ ${HOST_MODE_ARG} -eq 0 && ! -f "${SCRIPT_DIR}/.env" ]]; then
|
||
|
|
echo -e "${INFO} Membuat .env dengan password acak..."
|
||
|
|
echo -e "${WARN} Password tersimpan di .env — JANGAN commit file ini ke git!"
|
||
|
|
# PERHATIAN: Password PostgreSQL hanya diproses SAAT PERTAMA container dibuat.
|
||
|
|
# Jika sudah pernah deploy lalu mengubah password di .env, jalankan:
|
||
|
|
# docker compose down -v && docker volume rm sicabe_postgres-data
|
||
|
|
# untuk menghapus volume lama.
|
||
|
|
cat > "${SCRIPT_DIR}/.env" <<EOF
|
||
|
|
# eBdesk SiCABE — Environment File (auto-generated by install.sh)
|
||
|
|
# Override nilai di bawah sesuai kebutuhan production.
|
||
|
|
# PostgreSQL: 1 instance shared untuk semua service (beda database)
|
||
|
|
POSTGRES_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
KEYCLOAK_DB_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
NESSIE_DB_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
AIRFLOW_DB_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
OPENMETADATA_DB_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
KEYCLOAK_ADMIN_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
MINIO_ROOT_PASSWORD=$(openssl rand -hex 16)
|
||
|
|
AIRFLOW_ADMIN_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
AIRFLOW_SECRET_KEY=$(openssl rand -hex 32)
|
||
|
|
TRINO_SVC_SEED_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
TRINO_SVC_DBT_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
TRINO_SVC_AIRFLOW_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
TRINO_SVC_OM_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
GRAFANA_ADMIN_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
# OM 2.0.x menegakkan kebijakan password (min 8 karakter + huruf besar + huruf
|
||
|
|
# kecil + digit + karakter spesial). `openssl rand -hex` hanya menghasilkan
|
||
|
|
# 0-9a-f sehingga TIDAK memenuhi kebijakan itu — password harus dirakit agar
|
||
|
|
# keempat kelas karakter dijamin ada.
|
||
|
|
OPENMETADATA_ADMIN_PASSWORD="$(openssl rand -hex 12)Aa1!"
|
||
|
|
LDAP_ADMIN_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
LDAP_CONFIG_PASSWORD=$(openssl rand -hex 8)
|
||
|
|
|
||
|
|
# OIDC Client Secrets (harus sinkron dengan config .properties/.yaml files)
|
||
|
|
TRINO_CLIENT_SECRET=$(openssl rand -hex 16)
|
||
|
|
STARROCKS_CLIENT_SECRET=$(openssl rand -hex 16)
|
||
|
|
AIRFLOW_CLIENT_SECRET=$(openssl rand -hex 16)
|
||
|
|
OPENMETADATA_CLIENT_SECRET=$(openssl rand -hex 16)
|
||
|
|
GRAFANA_CLIENT_SECRET=$(openssl rand -hex 16)
|
||
|
|
EOF
|
||
|
|
chmod 600 "${SCRIPT_DIR}/.env"
|
||
|
|
echo -e "${OK} .env dibuat dengan password acak."
|
||
|
|
ENV_WAS_CREATED=1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ─── Load Default Environment ───────────────────────────────────────────────
|
||
|
|
source "${SCRIPT_DIR}/config/environment.defaults.sh"
|
||
|
|
|
||
|
|
# ─── Load .env jika ada ─────────────────────────────────────────────────────
|
||
|
|
if [[ -f "${SCRIPT_DIR}/.env" ]]; then
|
||
|
|
set -a
|
||
|
|
source "${SCRIPT_DIR}/.env"
|
||
|
|
set +a
|
||
|
|
echo -e "${INFO} Loaded overrides from .env"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ─── Fungsi Bantuan ──────────────────────────────────────────────────────────
|
||
|
|
usage() {
|
||
|
|
echo -e "${BOLD}Usage:${NC}"
|
||
|
|
echo " $0 # Deploy semua phase (1-6)"
|
||
|
|
echo " $0 --phase N # Deploy phase tertentu (1-6)"
|
||
|
|
echo " $0 --phase N-M # Deploy range phase (contoh: 1-3)"
|
||
|
|
echo " $0 --status [--host <nama>] # Cek status container (--host: pakai bundle host)"
|
||
|
|
echo " $0 --logs [service] # Lihat logs service (atau semua)"
|
||
|
|
echo " $0 --destroy # Hentikan dan hapus semua container"
|
||
|
|
echo " $0 --reset-ldap # Reset volume OpenLDAP (seed ulang user/grup)"
|
||
|
|
echo " $0 --restart [service] # Restart service tertentu"
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Multi-host (Fase 1):${NC}"
|
||
|
|
echo " $0 --generate [--inventory F] [--blueprint a3|b5]"
|
||
|
|
echo " # Hasilkan generated/host-<x>/ dari cluster.yml"
|
||
|
|
echo " $0 --host <nama> [--dry-run] # Jalankan compose untuk satu host (di host itu!)"
|
||
|
|
echo " $0 --preflight [--inventory F]"
|
||
|
|
echo " # Cek keterjangkauan lintas host (tanpa deploy)"
|
||
|
|
echo " $0 --sync [--host <nama>] [--dest DIR] [--user U] [--delete] [--dry-run]"
|
||
|
|
echo " # Salin deploy/ (termasuk .env) ke host lain (dari control)"
|
||
|
|
echo " $0 --help # Tampilkan bantuan ini"
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Phase:${NC}"
|
||
|
|
echo " 1 = Keycloak (Governance: IAM)"
|
||
|
|
echo " 2 = MinIO + Nessie (Storage)"
|
||
|
|
echo " 3 = Trino (Processing)"
|
||
|
|
echo " 4 = StarRocks (Serving)"
|
||
|
|
echo " 5 = Airflow (Ingestion)"
|
||
|
|
echo " 6 = OpenMetadata + Grafana (Governance: Metadata & Observability)"
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Komponen existing (dikelola sistem lain):${NC}"
|
||
|
|
echo " Arahkan endpoint di .env → container komponen itu TIDAK dinyalakan:"
|
||
|
|
echo " KEYCLOAK_URL · LDAP_URL · S3_ENDPOINT · NESSIE_URL · TRINO_URL · OPA_URL"
|
||
|
|
echo " Override eksplisit (mis. Postgres milik DBA):"
|
||
|
|
echo " EXTERNAL_SERVICES=\"keycloak postgres\" LOCAL_SERVICES=\"minio\""
|
||
|
|
exit 0
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Fungsi Prasyarat ─────────────────────────────────────────────────────────
|
||
|
|
check_prerequisites() {
|
||
|
|
echo -e "${INFO} Memeriksa prasyarat..."
|
||
|
|
|
||
|
|
# Docker
|
||
|
|
if ! command -v docker &>/dev/null; then
|
||
|
|
echo -e "${ERR} Docker tidak ditemukan. Install Docker terlebih dahulu."
|
||
|
|
echo " https://docs.docker.com/engine/install/"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Docker: $(docker --version)"
|
||
|
|
|
||
|
|
# Docker Compose
|
||
|
|
if command -v docker-compose &>/dev/null; then
|
||
|
|
DOCKER_COMPOSE="docker-compose"
|
||
|
|
elif docker compose version &>/dev/null; then
|
||
|
|
DOCKER_COMPOSE="docker compose"
|
||
|
|
else
|
||
|
|
echo -e "${ERR} Docker Compose tidak ditemukan."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Docker Compose: $(${DOCKER_COMPOSE} version 2>/dev/null || docker compose version)"
|
||
|
|
|
||
|
|
# mc (MinIO client) — opsional untuk bootstrap
|
||
|
|
if command -v mc &>/dev/null; then
|
||
|
|
echo -e "${OK} MinIO Client (mc): tersedia"
|
||
|
|
else
|
||
|
|
echo -e "${WARN} MinIO Client (mc) tidak ditemukan. Beberapa fitur bootstrap mungkin terbatas."
|
||
|
|
fi
|
||
|
|
|
||
|
|
# OpenSSL
|
||
|
|
if ! command -v openssl &>/dev/null; then
|
||
|
|
echo -e "${ERR} OpenSSL tidak ditemukan. Install openssl."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# curl
|
||
|
|
if ! command -v curl &>/dev/null; then
|
||
|
|
echo -e "${ERR} curl tidak ditemukan."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo -e "${OK} curl tersedia"
|
||
|
|
|
||
|
|
# htpasswd (httpd-tools) — generate password.db service account Trino
|
||
|
|
if ! command -v htpasswd &>/dev/null; then
|
||
|
|
echo -e "${ERR} htpasswd tidak ditemukan (dibutuhkan untuk password service account Trino)."
|
||
|
|
echo " Install: Debian/Ubuntu → sudo apt-get install -y apache2-utils"
|
||
|
|
echo " RHEL/Fedora → sudo dnf install -y httpd-tools"
|
||
|
|
echo " macOS → brew install httpd"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
echo -e "${OK} htpasswd: tersedia"
|
||
|
|
|
||
|
|
# python3 (untuk JSON parsing di bootstrap.sh & test-verification.sh)
|
||
|
|
if ! command -v python3 &>/dev/null; then
|
||
|
|
echo -e "${WARN} python3 tidak ditemukan. Beberapa fungsi bootstrap dan test mungkin terbatas."
|
||
|
|
echo " Install python3 atau pastikan tersedia di PATH."
|
||
|
|
else
|
||
|
|
echo -e "${OK} python3: tersedia"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Pastikan direktori config ada
|
||
|
|
for dir in config/keycloak config/trino/catalog config/starrocks config/grafana/datasources config/grafana/dashboards config/airflow config/nessie config/opa config/openldap; do
|
||
|
|
if [[ ! -d "${SCRIPT_DIR}/${dir}" ]]; then
|
||
|
|
echo -e "${WARN} Direktori ${dir} tidak ditemukan, membuat..."
|
||
|
|
mkdir -p "${SCRIPT_DIR}/${dir}"
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
echo -e "${OK} Semua prasyarat terpenuhi."
|
||
|
|
echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Generate password.db service account Trino (bcrypt) ─────────────────────
|
||
|
|
# Pra-syarat: htpasswd. Harus dijalankan SEBELUM Trino start (bind-mount file
|
||
|
|
# harus ada; jika tidak, Docker membuat DIRECTORY di path tsb).
|
||
|
|
generate_trino_password_db() {
|
||
|
|
local pw_file="${SCRIPT_DIR}/config/trino/password.db"
|
||
|
|
# Wajib FILE (bukan direktori): Docker membuat DIRECTORY bila sumber bind-mount
|
||
|
|
# tidak ada saat `up`. `sicabe_ensure_file_mount` membuang sisa direktori & membuat file.
|
||
|
|
sicabe_ensure_file_mount "${pw_file}" 644
|
||
|
|
local pair user pass
|
||
|
|
for pair in \
|
||
|
|
"svc_seed:${TRINO_SVC_SEED_PASSWORD:?}" \
|
||
|
|
"svc_dbt:${TRINO_SVC_DBT_PASSWORD:?}" \
|
||
|
|
"svc_airflow:${TRINO_SVC_AIRFLOW_PASSWORD:?}" \
|
||
|
|
"svc_om:${TRINO_SVC_OM_PASSWORD:?}"; do
|
||
|
|
user="${pair%%:*}"; pass="${pair#*:}"
|
||
|
|
htpasswd -nbB -C 10 "${user}" "${pass}" | sed '/^$/d' >> "${pw_file}"
|
||
|
|
done
|
||
|
|
chmod 644 "${pw_file}"
|
||
|
|
echo -e "${OK} password.db di-generate: $(cut -d: -f1 "${pw_file}" | paste -sd, -)"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Deteksi komponen EXISTING (dikelola sistem lain) ────────────────────────
|
||
|
|
# Prinsip: docker-compose.yml hanya untuk komponen yang MEMANG dijalankan di sini.
|
||
|
|
# Komponen yang endpoint-nya sudah diarahkan ke luar tidak dinyalakan, sehingga
|
||
|
|
# tidak ada container kembar / bentrok port dengan sistem yang sudah ada.
|
||
|
|
# Lihat config/environment.defaults.sh untuk tabel variabelnya.
|
||
|
|
|
||
|
|
# Nilai "lokal" (sesuai docker-compose.yml) untuk tiap komponen.
|
||
|
|
local_endpoint_of() {
|
||
|
|
case "$1" in
|
||
|
|
keycloak) echo "http://localhost:${KEYCLOAK_PORT:-12080}" ;;
|
||
|
|
openldap) echo "ldap://openldap:389" ;;
|
||
|
|
minio) echo "http://minio:9000" ;;
|
||
|
|
nessie) echo "http://nessie:19120" ;;
|
||
|
|
trino) echo "https://localhost:${TRINO_HTTPS_PORT:-12443}" ;;
|
||
|
|
opa) echo "http://localhost:${OPA_PORT:-12181}" ;;
|
||
|
|
*) echo "" ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# Endpoint yang sedang dikonfigurasi (.env / defaults).
|
||
|
|
configured_endpoint_of() {
|
||
|
|
case "$1" in
|
||
|
|
keycloak) echo "${KEYCLOAK_URL:-}" ;;
|
||
|
|
openldap) echo "${LDAP_URL:-}" ;;
|
||
|
|
minio) echo "${S3_ENDPOINT:-}" ;;
|
||
|
|
nessie) echo "${NESSIE_URL:-}" ;;
|
||
|
|
trino) echo "${TRINO_URL:-}" ;;
|
||
|
|
opa) echo "${OPA_URL:-}" ;;
|
||
|
|
*) echo "" ;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# Daftar service yang dikelola EKSTERNAL (spasi-separated, unik).
|
||
|
|
# Nama LOGIS -> nama service sebenarnya.
|
||
|
|
#
|
||
|
|
# Filter komponen eksternal mencocokkan NAMA SERVICE (`starrocks-fe`, `trino-worker`,
|
||
|
|
# …). Tanpa pemetaan ini, menulis nama logis di EXTERNAL_SERVICES/LOCAL_SERVICES
|
||
|
|
# diam-diam TIDAK berefek: `EXTERNAL_SERVICES=starrocks` tidak melewati apa pun, dan
|
||
|
|
# `LOCAL_SERVICES=starrocks` tidak memaksa apa pun. Dulu hanya `trino` yang istimewa,
|
||
|
|
# sehingga orang yang menulis `starrocks` merasa setelannya diabaikan.
|
||
|
|
expand_service_names() { # <nama-logis|nama-service>...
|
||
|
|
local n out=""
|
||
|
|
for n in "$@"; do
|
||
|
|
case "${n}" in
|
||
|
|
trino) out+=" trino-coordinator trino-worker " ;;
|
||
|
|
starrocks) out+=" starrocks-fe starrocks-cn " ;;
|
||
|
|
airflow) out+=" airflow-init airflow-webserver airflow-scheduler airflow-dag-processor " ;;
|
||
|
|
openmetadata) out+=" openmetadata-server openmetadata-opensearch " ;;
|
||
|
|
*) out+=" ${n} " ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
printf '%s' "${out# }"
|
||
|
|
}
|
||
|
|
|
||
|
|
external_services() {
|
||
|
|
local out=" $(expand_service_names ${EXTERNAL_SERVICES:-}) " svc cfg local_val
|
||
|
|
for svc in keycloak openldap minio nessie trino opa; do
|
||
|
|
cfg="$(configured_endpoint_of "${svc}")"
|
||
|
|
local_val="$(local_endpoint_of "${svc}")"
|
||
|
|
if [[ -n "${cfg}" && -n "${local_val}" && "${cfg}" != "${local_val}" ]]; then
|
||
|
|
out+=" ${svc} "
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
# S3 eksternal ⇒ minio-init (pembuat bucket lokal) tidak berlaku.
|
||
|
|
if [[ "${out}" == *" minio "* ]]; then
|
||
|
|
out+=" minio-init "
|
||
|
|
fi
|
||
|
|
# Trino kini DUA service (coordinator + worker): nama logis `trino` dipetakan ke
|
||
|
|
# keduanya agar deteksi eksternal benar-benar melewati container-nya.
|
||
|
|
if [[ "${out}" == *" trino "* ]]; then
|
||
|
|
out=" $(echo "${out}" | sed 's/ trino / /') "
|
||
|
|
out+=" trino-coordinator trino-worker "
|
||
|
|
fi
|
||
|
|
# StarRocks: FE & CN dikelola sistem lain bila hostnya bukan localhost.
|
||
|
|
if [[ -n "${STARROCKS_HOST:-}" && "${STARROCKS_HOST}" != "localhost" && "${STARROCKS_HOST}" != "127.0.0.1" ]]; then
|
||
|
|
out+=" starrocks-fe starrocks-cn "
|
||
|
|
fi
|
||
|
|
# Airflow: seluruh service fase 5 dikelola sistem lain bila hostnya bukan localhost.
|
||
|
|
if [[ -n "${AIRFLOW_HOST:-}" && "${AIRFLOW_HOST}" != "localhost" && "${AIRFLOW_HOST}" != "127.0.0.1" ]]; then
|
||
|
|
out+=" airflow-init airflow-webserver airflow-dag-processor airflow-scheduler "
|
||
|
|
fi
|
||
|
|
# OpenMetadata / Grafana: dikelola sistem lain bila URL host-nya bukan localhost.
|
||
|
|
if [[ -n "${OPENMETADATA_URL:-}" && "${OPENMETADATA_URL}" != "http://localhost:${OPENMETADATA_PORT:-12085}" ]]; then
|
||
|
|
out+=" openmetadata-opensearch openmetadata-migrate openmetadata-server "
|
||
|
|
fi
|
||
|
|
if [[ -n "${GRAFANA_URL:-}" && "${GRAFANA_URL}" != "http://localhost:${GRAFANA_PORT:-12300}" ]]; then
|
||
|
|
out+=" grafana "
|
||
|
|
fi
|
||
|
|
# LOCAL_SERVICES: paksa komponen tertentu tetap dijalankan lokal.
|
||
|
|
# Nama logis (`trino`, `starrocks`, `airflow`, `openmetadata`) diuraikan oleh
|
||
|
|
# expand_service_names() supaya cocok dengan nama service yang sebenarnya.
|
||
|
|
local s
|
||
|
|
for s in $(expand_service_names ${LOCAL_SERVICES:-}); do
|
||
|
|
out=" $(echo "${out}" | sed "s/ ${s} / /g") "
|
||
|
|
done
|
||
|
|
# PENTING (regresi e099b95): JANGAN pakai `grep -v '^$'` di sini. Saat TIDAK ada
|
||
|
|
# service eksternal, input hanya baris kosong → grep tidak menemukan baris cocok →
|
||
|
|
# exit 1 → dengan `set -euo pipefail` assignment `EXT_SVCS="$(external_services)"`
|
||
|
|
# MEMATIKAN install.sh tepat setelah banner (tanpa pesan error). `awk 'NF'` setara
|
||
|
|
# (buang baris kosong) tetapi SELALU exit 0.
|
||
|
|
echo "${out}" | tr ' ' '\n' | awk 'NF' | sort -u | paste -sd' ' -
|
||
|
|
}
|
||
|
|
|
||
|
|
is_external() {
|
||
|
|
[[ " $(external_services) " == *" $1 "* ]]
|
||
|
|
}
|
||
|
|
|
||
|
|
# Service yang harus dinyalakan untuk sebuah fase:
|
||
|
|
# kandidat dari profil compose DIKURANGI komponen eksternal.
|
||
|
|
phase_services() {
|
||
|
|
local phase="$1" svc out=""
|
||
|
|
local ext=" $(external_services) "
|
||
|
|
while read -r svc; do
|
||
|
|
[[ -z "${svc}" ]] && continue
|
||
|
|
if [[ "${ext}" == *" ${svc} "* ]]; then
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
out+=" ${svc}"
|
||
|
|
done < <(${DOCKER_COMPOSE} -f "${SCRIPT_DIR}/docker-compose.yml" --profile "phase${phase}" config --services 2>/dev/null)
|
||
|
|
echo "${out# }"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Naikkan container satu fase.
|
||
|
|
# - Tanpa komponen eksternal: perilaku lama (`--profile phaseN up -d`) supaya
|
||
|
|
# depends_on tetap mengatur urutan + menunggu healthcheck.
|
||
|
|
# - Dengan komponen eksternal: hanya service lokal, `--no-deps` agar depends_on
|
||
|
|
# tidak menyeret service yang dikelola sistem lain. Postgres (bila lokal)
|
||
|
|
# dinaikkan lebih dulu supaya dependennya tidak menabrak DB yang belum siap.
|
||
|
|
deploy_phase_compose() {
|
||
|
|
local phase="$1" svcs
|
||
|
|
svcs="$(phase_services "${phase}")"
|
||
|
|
|
||
|
|
# Dir bind-mount yang absen di git (config/airflow/sling hanya env.yaml gitignored):
|
||
|
|
# bila tidak ada, Docker (root) membuatnya sebagai ROOT → operator non-root gagal
|
||
|
|
# menulis env.yaml lewat bootstrap. Pastikan ada & user-owned sebelum `up`.
|
||
|
|
mkdir -p "${SCRIPT_DIR}/config/airflow/sling"
|
||
|
|
|
||
|
|
if [[ -z "$(external_services)" ]]; then
|
||
|
|
${DOCKER_COMPOSE} -f "${SCRIPT_DIR}/docker-compose.yml" --profile "phase${phase}" up -d
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ -z "${svcs}" ]]; then
|
||
|
|
echo -e "${INFO} Semua komponen fase ${phase} dikelola eksternal — tidak ada container yang dinyalakan."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
echo -e "${INFO} Mode existing — container fase ${phase} yang dinyalakan: ${BOLD}${svcs}${NC}"
|
||
|
|
if [[ " ${svcs} " == *" postgres "* ]]; then
|
||
|
|
${DOCKER_COMPOSE} -f "${SCRIPT_DIR}/docker-compose.yml" --profile all up -d --no-deps postgres
|
||
|
|
wait_for_container "sicabe-postgres" 60
|
||
|
|
fi
|
||
|
|
# shellcheck disable=SC2086
|
||
|
|
${DOCKER_COMPOSE} -f "${SCRIPT_DIR}/docker-compose.yml" --profile all up -d --no-deps ${svcs}
|
||
|
|
}
|
||
|
|
|
||
|
|
# Tunggu komponen: bila eksternal, tunggu endpoint eksternalnya (bukan localhost).
|
||
|
|
wait_component() {
|
||
|
|
local svc="$1" local_url="$2" timeout="$3" ext_url="${4:-}"
|
||
|
|
if is_external "${svc}"; then
|
||
|
|
if [[ "${ext_url}" == http* ]]; then
|
||
|
|
echo -e "${INFO} ${svc} dikelola eksternal — menunggu ${ext_url}"
|
||
|
|
wait_for_service "${svc} (eksternal)" "${ext_url}" "${timeout}"
|
||
|
|
else
|
||
|
|
echo -e "${INFO} ${svc} dikelola eksternal — lewati penantian (diverifikasi gate fase)"
|
||
|
|
fi
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
wait_for_service "${svc}" "${local_url}" "${timeout}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Fungsi Deploy per Phase ─────────────────────────────────────────────────
|
||
|
|
deploy_phase() {
|
||
|
|
local phase="$1"
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
|
||
|
|
case "${phase}" in
|
||
|
|
1)
|
||
|
|
echo -e "${STEP} Phase 1${NC} — Governance (IAM): Keycloak"
|
||
|
|
deploy_phase_compose 1
|
||
|
|
wait_component "keycloak" "http://localhost:${KEYCLOAK_PORT:-12080}/health/ready" 120 \
|
||
|
|
"${KEYCLOAK_URL:-}/health/ready"
|
||
|
|
if is_external openldap; then
|
||
|
|
echo -e "${INFO} OpenLDAP dikelola eksternal (LDAP_URL=${LDAP_URL}) — lewati penantian container"
|
||
|
|
else
|
||
|
|
wait_for_container "sicabe-openldap" 60
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Phase 1 selesai — Keycloak ${KEYCLOAK_URL}, OpenLDAP ${LDAP_URL}"
|
||
|
|
;;
|
||
|
|
2)
|
||
|
|
echo -e "${STEP} Phase 2${NC} — Storage: MinIO + Nessie"
|
||
|
|
deploy_phase_compose 2
|
||
|
|
wait_component "minio" "http://localhost:${MINIO_PORT:-12900}/minio/health/live" 60
|
||
|
|
wait_component "nessie" "http://localhost:${NESSIE_PORT:-12120}/api/v2/config" 90 \
|
||
|
|
"http://${NESSIE_HOST:-localhost}:${NESSIE_PORT:-12120}/api/v2/config"
|
||
|
|
echo -e "${OK} Phase 2 selesai — object storage ${S3_ENDPOINT}, Nessie ${NESSIE_URL}"
|
||
|
|
;;
|
||
|
|
3)
|
||
|
|
echo -e "${STEP} Phase 3${NC} — Processing: Trino + OPA"
|
||
|
|
if is_external trino; then
|
||
|
|
echo -e "${INFO} Trino dikelola eksternal (${TRINO_URL}) — password.db lokal tidak dibuat/dipakai"
|
||
|
|
else
|
||
|
|
generate_trino_password_db
|
||
|
|
fi
|
||
|
|
deploy_phase_compose 3
|
||
|
|
wait_component "trino" "https://localhost:${TRINO_HTTPS_PORT:-12443}/v1/info" 90 \
|
||
|
|
"${TRINO_URL:-}/v1/info"
|
||
|
|
echo -e "${OK} Phase 3 selesai — Trino ${TRINO_URL} (OAuth2 + Basic) · OPA ${OPA_URL}"
|
||
|
|
;;
|
||
|
|
4)
|
||
|
|
echo -e "${STEP} Phase 4${NC} — Serving: StarRocks ${BOLD}${NC}"
|
||
|
|
deploy_phase_compose 4
|
||
|
|
if is_external starrocks-fe; then
|
||
|
|
echo -e "${INFO} StarRocks dikelola eksternal — lewati penantian start"
|
||
|
|
else
|
||
|
|
echo -e "${INFO} StarRocks FE membutuhkan waktu ~2 menit untuk start..."
|
||
|
|
sleep 30
|
||
|
|
fi
|
||
|
|
echo -e "${OK} Phase 4 selesai — StarRocks FE :${STARROCKS_FE_PORT:-12930}, CN (shared-data)"
|
||
|
|
;;
|
||
|
|
5)
|
||
|
|
echo -e "${STEP} Phase 5${NC} — Ingestion: Airflow ${BOLD}${NC}"
|
||
|
|
# SimpleAuthManager membuka file ini dengan mode a+ → wajib ADA & writable (666).
|
||
|
|
# File ini TIDAK di-track git (lihat .gitignore) → generate dari .env di sini
|
||
|
|
# agar Docker tidak membuat bind-mount file ini sebagai DIRECTORY (fresh clone).
|
||
|
|
if is_external airflow-webserver; then
|
||
|
|
echo -e "${INFO} Airflow dikelola eksternal — lewati pembuatan file auth lokal"
|
||
|
|
else
|
||
|
|
local pw_file="${SCRIPT_DIR}/config/airflow/simple_auth_manager_passwords.json"
|
||
|
|
sicabe_ensure_file_mount "${pw_file}" 666
|
||
|
|
printf '{"admin": "%s"}\n' "${AIRFLOW_ADMIN_PASSWORD:?AIRFLOW_ADMIN_PASSWORD belum diset di .env}" > "${pw_file}"
|
||
|
|
chmod 666 "${pw_file}"
|
||
|
|
fi
|
||
|
|
deploy_phase_compose 5
|
||
|
|
wait_component "airflow-webserver" "http://localhost:${AIRFLOW_PORT:-12082}/api/v2/monitor/health" 120
|
||
|
|
echo -e "${OK} Phase 5 selesai — Airflow siap di http://localhost:${AIRFLOW_PORT:-12082}"
|
||
|
|
;;
|
||
|
|
6)
|
||
|
|
echo -e "${STEP} Phase 6${NC} — Governance (Lanjutan): OpenMetadata + Grafana ${BOLD}${NC}"
|
||
|
|
deploy_phase_compose 6
|
||
|
|
wait_component "openmetadata-server" "http://localhost:${OPENMETADATA_PORT:-12085}/health" 180
|
||
|
|
wait_component "grafana" "http://localhost:${GRAFANA_PORT:-12300}/api/health" 60
|
||
|
|
echo -e "${OK} Phase 6 selesai — OpenMetadata :${OPENMETADATA_PORT:-12085}, Grafana :${GRAFANA_PORT:-12300}"
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
echo -e "${ERR} Phase tidak valid: ${phase}. Gunakan 1-6."
|
||
|
|
exit 1
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Status ──────────────────────────────────────────────────────────────────
|
||
|
|
show_status() {
|
||
|
|
# `--status [--host <nama>]`: dengan --host memakai bundle host itu
|
||
|
|
# (compose + .env hasil generate) sehingga endpoint menunjuk ALAMAT HOST,
|
||
|
|
# bukan localhost (perbaikan bug multi-host).
|
||
|
|
local st_host="${1:-}"
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
if [[ -n "${st_host}" ]]; then
|
||
|
|
compose_file="${SCRIPT_DIR}/generated/host-${st_host}/docker-compose.yml"
|
||
|
|
local henv="${SCRIPT_DIR}/generated/host-${st_host}/.env"
|
||
|
|
if [[ ! -f "${compose_file}" || ! -f "${henv}" ]]; then
|
||
|
|
echo -e "${ERR} Host '${st_host}' belum digenerate (${compose_file})."
|
||
|
|
echo " Jalankan dulu: ./install.sh --generate"
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
set -a; source "${henv}"; set +a
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo -e "${BOLD} STATUS KOMPONEN eBdesk SiCABE${NC}"
|
||
|
|
[[ -n "${st_host}" ]] && echo -e "${BOLD} (host: ${st_host})${NC}"
|
||
|
|
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
if ! ${DOCKER_COMPOSE} -f "${compose_file}" ps &>/dev/null; then
|
||
|
|
echo -e "${INFO} Tidak ada container SiCABE yang berjalan."
|
||
|
|
return
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${BOLD}Container Status:${NC}"
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" ps --all 2>/dev/null || \
|
||
|
|
docker ps --filter "name=sicabe-" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Endpoint Access:${NC}"
|
||
|
|
# Alamat dasar: alamat host (mode --host) atau localhost (single-host).
|
||
|
|
local ea="localhost"
|
||
|
|
[[ -n "${st_host}" ]] && ea="${SICABE_HOST_ADDRESS:-localhost}"
|
||
|
|
# `*_PUBLIC_URL` (diisi generator) MENANG bila ada. Untuk layanan tanpa URL publik,
|
||
|
|
# ambil HOST PEMILIK dari variabel endpoint bundle (mode --host), jatuh ke ea.
|
||
|
|
local h_kc="$ea" h_minio="$ea" h_sr="$ea" h_afs="$ea" h_om="$ea" h_ldap="$ea" h_opa="$ea" h_nessie="$ea"
|
||
|
|
if [[ -n "${st_host}" ]]; then
|
||
|
|
h_kc="${KEYCLOAK_HOST:-$ea}"
|
||
|
|
h_minio="${MINIO_HOST:-$ea}"
|
||
|
|
h_sr="${STARROCKS_FE_HOST:-$ea}"
|
||
|
|
h_afs="${AIRFLOW_WEBSERVER_HOST:-$ea}"
|
||
|
|
h_om="${OPENMETADATA_HOST:-$ea}"
|
||
|
|
if [[ -n "${NESSIE_URL:-}" ]]; then h_nessie="${NESSIE_URL#*://}"; h_nessie="${h_nessie%%[:/]*}"; fi
|
||
|
|
if [[ -n "${OPA_URL:-}" ]]; then h_opa="${OPA_URL#*://}"; h_opa="${h_opa%%[:/]*}"; fi
|
||
|
|
if [[ -n "${LDAP_URL:-}" ]]; then h_ldap="${LDAP_URL#*://}"; h_ldap="${h_ldap%%[:/]*}"; fi
|
||
|
|
fi
|
||
|
|
echo -e " Keycloak ${CYAN}${KEYCLOAK_PUBLIC_URL:-http://${h_kc}:${KEYCLOAK_PORT:-12080}}${NC}"
|
||
|
|
echo -e " OpenLDAP ${CYAN}ldap://${h_ldap}:${OPENLDAP_PORT:-12389}${NC}"
|
||
|
|
echo -e " MinIO API ${CYAN}http://${h_minio}:${MINIO_PORT:-12900}${NC}"
|
||
|
|
echo -e " MinIO Console ${CYAN}${MINIO_CONSOLE_PUBLIC_URL:-http://${h_minio}:${MINIO_CONSOLE_PORT:-12901}}${NC}"
|
||
|
|
echo -e " Nessie ${CYAN}http://${h_nessie}:${NESSIE_PORT:-12120}${NC}"
|
||
|
|
echo -e " Trino ${CYAN}${TRINO_PUBLIC_URL:-https://${ea}:${TRINO_HTTPS_PORT:-12443}}${NC} (OAuth2 + Basic, HTTPS-only)"
|
||
|
|
echo -e " OPA ${CYAN}http://${h_opa}:${OPA_PORT:-12181}${NC}"
|
||
|
|
echo -e " StarRocks FE ${CYAN}${h_sr}:${STARROCKS_FE_PORT:-12930}${NC} (MySQL protocol)"
|
||
|
|
echo -e " Dremio ${CYAN}http://${ea}:${DREMIO_UI_PORT:-12947}${NC}"
|
||
|
|
echo -e " Airflow ${CYAN}${AIRFLOW_PUBLIC_URL:-http://${h_afs}:${AIRFLOW_PORT:-12082}}${NC}"
|
||
|
|
echo -e " OpenMetadata ${CYAN}${OPENMETADATA_PUBLIC_URL:-http://${h_om}:${OPENMETADATA_PORT:-12085}}${NC}"
|
||
|
|
echo -e " Grafana ${CYAN}${GRAFANA_PUBLIC_URL:-http://${ea}:${GRAFANA_PORT:-12300}}${NC}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Logs ─────────────────────────────────────────────────────────────────────
|
||
|
|
show_logs() {
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
if [[ $# -eq 0 ]]; then
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" logs --tail=50 -f
|
||
|
|
else
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" logs --tail=100 -f "$@"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Destroy ─────────────────────────────────────────────────────────────────
|
||
|
|
destroy() {
|
||
|
|
echo -e "${YELLOW}${BOLD}⚠️ PERINGATAN: Ini akan menghapus semua container dan volume!${NC}"
|
||
|
|
echo -e "${YELLOW}Data di volume akan dihapus jika menggunakan --volumes.${NC}"
|
||
|
|
echo ""
|
||
|
|
echo -n "Lanjutkan? (y/N): "
|
||
|
|
read -r confirm
|
||
|
|
if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then
|
||
|
|
echo -e "${INFO} Dibatalkan."
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
# destroy() tidak memanggil check_prerequisites → DOCKER_COMPOSE bisa belum
|
||
|
|
# di-set. Deteksi di sini agar tidak "unbound variable" (set -u).
|
||
|
|
if [[ -z "${DOCKER_COMPOSE:-}" ]]; then
|
||
|
|
if docker compose version &>/dev/null; then
|
||
|
|
DOCKER_COMPOSE="docker compose"
|
||
|
|
elif docker-compose version &>/dev/null; then
|
||
|
|
DOCKER_COMPOSE="docker-compose"
|
||
|
|
else
|
||
|
|
echo -e "${ERR} Docker Compose tidak ditemukan."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
echo -e "${INFO} Menghentikan container..."
|
||
|
|
# Semua service di balik `profiles:` → WAJIB `--profile all`, jika tidak
|
||
|
|
# `down` tidak menyentuh container apa pun (no-op).
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" --profile all down --remove-orphans 2>/dev/null || true
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -n "Hapus semua volume data? (y/N): "
|
||
|
|
read -r confirm_vol
|
||
|
|
if [[ "${confirm_vol}" == "y" || "${confirm_vol}" == "Y" ]]; then
|
||
|
|
echo -e "${INFO} Menghapus volume..."
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" --profile all down -v 2>/dev/null || true
|
||
|
|
docker volume prune -f --filter "label=com.docker.compose.project=sicabe" 2>/dev/null || true
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${OK} Destroy selesai."
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Reset LDAP ──────────────────────────────────────────────────────────────
|
||
|
|
# Menghapus volume OpenLDAP agar config/openldap/seed.ldif di-apply ulang oleh
|
||
|
|
# bootstrap.sh. Diperlukan saat nama user/grup berubah, karena seeding di-skip
|
||
|
|
# bila OU users sudah ada (idempotent).
|
||
|
|
reset_ldap() {
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
local project="${COMPOSE_PROJECT_NAME:-sicabe}"
|
||
|
|
|
||
|
|
if [[ -z "${DOCKER_COMPOSE:-}" ]]; then
|
||
|
|
if docker compose version &>/dev/null; then DOCKER_COMPOSE="docker compose"
|
||
|
|
elif docker-compose version &>/dev/null; then DOCKER_COMPOSE="docker-compose"
|
||
|
|
else echo -e "${ERR} Docker Compose tidak ditemukan."; exit 1; fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -n "Reset OpenLDAP (hapus volume user+group, seed ulang via bootstrap)? (y/N): "
|
||
|
|
read -r confirm
|
||
|
|
if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then
|
||
|
|
echo -e "${INFO} Dibatalkan."
|
||
|
|
exit 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo -e "${INFO} Menghentikan OpenLDAP..."
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" --profile all stop openldap 2>/dev/null || true
|
||
|
|
|
||
|
|
echo -e "${INFO} Menghapus volume ${project}_openldap-data & ${project}_openldap-config..."
|
||
|
|
docker volume rm "${project}_openldap-data" "${project}_openldap-config" 2>/dev/null || true
|
||
|
|
|
||
|
|
echo -e "${INFO} Menjalankan ulang OpenLDAP..."
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" --profile all up -d openldap 2>/dev/null || true
|
||
|
|
|
||
|
|
echo -e "${OK} OpenLDAP di-reset. Jalankan './bootstrap.sh' (Step 3) untuk seed ulang."
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Main ────────────────────────────────────────────────────────────────────
|
||
|
|
main() {
|
||
|
|
local MODE="all"
|
||
|
|
local PHASE_START=1
|
||
|
|
local PHASE_END=6
|
||
|
|
local EXTRA_ARGS=()
|
||
|
|
|
||
|
|
if [[ $# -gt 0 ]]; then
|
||
|
|
# ── Perintah multi-host (Fase 1) — ditangani lebih dulu ──────────────────
|
||
|
|
# Catatan: `DOCKER_COMPOSE` baru di-set oleh check_prerequisites(), jadi perintah
|
||
|
|
# yang butuh Docker memanggilnya lebih dulu; `--generate` justru TIDAK butuh Docker
|
||
|
|
# (hanya python3+PyYAML) sehingga tidak dipaksa.
|
||
|
|
case "${1:-}" in
|
||
|
|
--generate)
|
||
|
|
local inv="${SCRIPT_DIR}/cluster.yml" bp=""
|
||
|
|
shift
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--inventory) inv="${2:-}"; shift 2 ;;
|
||
|
|
--blueprint) bp="${2:-}"; shift 2 ;;
|
||
|
|
*) echo -e "${ERR} Argumen tidak dikenal untuk --generate: $1"; exit 1 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
if ! command -v python3 &>/dev/null; then
|
||
|
|
echo -e "${ERR} python3 wajib untuk --generate."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
if ! python3 -c 'import yaml' &>/dev/null; then
|
||
|
|
echo -e "${ERR} PyYAML wajib untuk --generate (pip install pyyaml)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
generate_hosts "${inv}" "${bp}" && exit 0
|
||
|
|
exit 1
|
||
|
|
;;
|
||
|
|
--preflight)
|
||
|
|
local pinv="${SCRIPT_DIR}/cluster.yml"
|
||
|
|
shift
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--inventory) pinv="${2:-}"; shift 2 ;;
|
||
|
|
*) echo -e "${ERR} Argumen tidak dikenal untuk --preflight: $1"; exit 1 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
if [[ ! -f "${pinv}" ]]; then
|
||
|
|
echo -e "${ERR} Inventory tidak ditemukan: ${pinv}"
|
||
|
|
echo " Mulai dari contoh: cp cluster.example.yml cluster.yml"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
python3 "${SCRIPT_DIR}/lib/cluster/preflight.py" "${pinv}" \
|
||
|
|
--compose "${SCRIPT_DIR}/docker-compose.yml" --deploy "${SCRIPT_DIR}"
|
||
|
|
exit $?
|
||
|
|
;;
|
||
|
|
--sync)
|
||
|
|
sync_hosts "$@"
|
||
|
|
exit $?
|
||
|
|
;;
|
||
|
|
--host)
|
||
|
|
local hname="${2:-}" hdry=0
|
||
|
|
if [[ -z "${hname}" ]]; then
|
||
|
|
echo -e "${ERR} Sebutkan nama host (lihat: --generate)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
shift 2
|
||
|
|
[[ "${1:-}" == "--dry-run" ]] && hdry=1
|
||
|
|
check_prerequisites # menyetel DOCKER_COMPOSE + memvalidasi Docker
|
||
|
|
run_host "${hname}" "${hdry}" && exit 0
|
||
|
|
exit 1
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
|
||
|
|
|
||
|
|
case "$1" in
|
||
|
|
--help|-h) usage ;;
|
||
|
|
--status)
|
||
|
|
check_prerequisites
|
||
|
|
local st_host=""
|
||
|
|
[[ "${2:-}" == "--host" ]] && st_host="${3:-}"
|
||
|
|
show_status "${st_host}"; exit 0 ;;
|
||
|
|
--logs) check_prerequisites; shift; show_logs "$@"; exit 0 ;;
|
||
|
|
--destroy) destroy; exit 0 ;;
|
||
|
|
--reset-ldap) reset_ldap; exit 0 ;;
|
||
|
|
--restart)
|
||
|
|
check_prerequisites
|
||
|
|
local compose_file="${SCRIPT_DIR}/docker-compose.yml"
|
||
|
|
if [[ $# -lt 2 ]]; then
|
||
|
|
echo -e "${ERR} Sebutkan service yang akan direstart."
|
||
|
|
echo " $0 --restart keycloak"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
${DOCKER_COMPOSE} -f "${compose_file}" restart "${@:2}"
|
||
|
|
exit $?
|
||
|
|
;;
|
||
|
|
--phase)
|
||
|
|
if [[ $# -lt 2 ]]; then
|
||
|
|
echo -e "${ERR} Sebutkan phase (1-6 atau range 1-3)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
MODE="phase"
|
||
|
|
if [[ "$2" == *-* ]]; then
|
||
|
|
PHASE_START="${2%-*}"
|
||
|
|
PHASE_END="${2#*-}"
|
||
|
|
else
|
||
|
|
PHASE_START="$2"
|
||
|
|
PHASE_END="$2"
|
||
|
|
fi
|
||
|
|
shift 2
|
||
|
|
EXTRA_ARGS=("$@")
|
||
|
|
;;
|
||
|
|
*)
|
||
|
|
echo -e "${ERR} Argumen tidak dikenal: $1"
|
||
|
|
usage
|
||
|
|
;;
|
||
|
|
esac
|
||
|
|
fi
|
||
|
|
|
||
|
|
# ─── Mulai Deployment ──────────────────────────────────────────────────────
|
||
|
|
echo -e "${BOLD}${BLUE}"
|
||
|
|
echo "╔═══════════════════════════════════════════════════════╗"
|
||
|
|
echo "║ eBdesk SiCABE — Deployment Script ║"
|
||
|
|
echo "║ Arsitektur 5 Lapisan — Urutan Berdasar Dependensi ║"
|
||
|
|
echo "╚═══════════════════════════════════════════════════════╝"
|
||
|
|
echo -e "${NC}"
|
||
|
|
|
||
|
|
# Komponen yang dikelola sistem lain → container-nya TIDAK dinyalakan.
|
||
|
|
local EXT_SVCS
|
||
|
|
EXT_SVCS="$(external_services)"
|
||
|
|
if [[ -n "${EXT_SVCS}" ]]; then
|
||
|
|
echo -e "${INFO} Komponen dikelola EKSTERNAL: ${BOLD}${EXT_SVCS}${NC}"
|
||
|
|
echo " Container komponen tsb tidak dinyalakan; endpoint diambil dari .env."
|
||
|
|
echo " Verifikasi setelah bootstrap: ./scripts/verify-phase1-iam.sh dst."
|
||
|
|
echo ""
|
||
|
|
fi
|
||
|
|
|
||
|
|
check_prerequisites
|
||
|
|
|
||
|
|
if [[ "${MODE}" == "all" ]]; then
|
||
|
|
echo -e "${BOLD}Mode: Full Deployment (Semua Phase 1-6)${NC}"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Phase 1: Governance (IAM)
|
||
|
|
echo -e "${STEP}╔═══ Phase 1/6: Governance (IAM) — Keycloak ═══╗${NC}"
|
||
|
|
deploy_phase 1
|
||
|
|
|
||
|
|
# Phase 2: Storage
|
||
|
|
echo -e "${STEP}╔═══ Phase 2/6: Storage — MinIO → Nessie ═══╗${NC}"
|
||
|
|
deploy_phase 2
|
||
|
|
|
||
|
|
# Phase 3: Processing
|
||
|
|
echo -e "${STEP}╔═══ Phase 3/6: Processing — Trino ═══╗${NC}"
|
||
|
|
deploy_phase 3
|
||
|
|
|
||
|
|
# Phase 4: Serving
|
||
|
|
echo -e "${STEP}╔═══ Phase 4/6: Serving — StarRocks ═══╗${NC}"
|
||
|
|
deploy_phase 4
|
||
|
|
|
||
|
|
# Phase 5: Ingestion
|
||
|
|
echo -e "${STEP}╔═══ Phase 5/6: Ingestion — Airflow ═══╗${NC}"
|
||
|
|
deploy_phase 5
|
||
|
|
|
||
|
|
# Phase 6: Governance (Lanjutan)
|
||
|
|
echo -e "${STEP}╔═══ Phase 6/6: Governance (Lan) — OpenMetadata → Grafana ═══╗${NC}"
|
||
|
|
deploy_phase 6
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}${GREEN}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} ✅ SEMUA PHASE SELESAI${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} Jalankan ./bootstrap.sh untuk inisialisasi & integrasi${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN} Jalankan ./test-verification.sh untuk verifikasi${NC}"
|
||
|
|
echo -e "${BOLD}${GREEN}═══════════════════════════════════════════════════════${NC}"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
elif [[ "${MODE}" == "phase" ]]; then
|
||
|
|
echo -e "${BOLD}Mode: Phase ${PHASE_START} sampai ${PHASE_END}${NC}"
|
||
|
|
for p in $(seq "${PHASE_START}" "${PHASE_END}"); do
|
||
|
|
deploy_phase "${p}"
|
||
|
|
done
|
||
|
|
echo -e "${OK} Phase ${PHASE_START}-${PHASE_END} selesai!"
|
||
|
|
fi
|
||
|
|
|
||
|
|
show_status
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Multi-host (Fase 1, R10 §4.5) ───────────────────────────────────────────
|
||
|
|
# Membangkitkan artefak deploy per host ke `generated/host-<x>/`.
|
||
|
|
generate_hosts() {
|
||
|
|
local inv="$1" blueprint="$2" args=()
|
||
|
|
if [[ ! -f "${inv}" ]]; then
|
||
|
|
echo -e "${ERR} Inventory tidak ditemukan: ${inv}"
|
||
|
|
echo " Mulai dari contoh: cp cluster.example.yml cluster.yml (lalu sesuaikan address)"
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
# Rahasia untuk SEMUA host diambil dari .env mesin ini (keputusan S1: satu .env
|
||
|
|
# bersama). Bila .env baru saja dibuat otomatis, rahasianya BARU dan harus menjadi
|
||
|
|
# satu-satunya sumber: distribusikan generated/ ke semua host, dan JANGAN membuat
|
||
|
|
# .env terpisah di tiap host (client secret OIDC akan tidak cocok -> login gagal).
|
||
|
|
if [[ "${ENV_WAS_CREATED:-0}" -eq 1 ]]; then
|
||
|
|
echo -e "${WARN} .env BARU SAJA dibuat di mesin ini (password acak)."
|
||
|
|
echo " .env ini adalah sumber rahasia untuk SEMUA host."
|
||
|
|
echo " → distribusikan hasil generated/ (di dalamnya rahasia sudah tertanam) ke tiap host;"
|
||
|
|
echo " → JANGAN membuat .env terpisah di host lain (OIDC client secret harus identik)."
|
||
|
|
fi
|
||
|
|
[[ -n "${blueprint}" ]] && args+=("${blueprint}")
|
||
|
|
echo -e "${INFO} Membangkitkan artefak per host dari ${inv}..."
|
||
|
|
python3 "${SCRIPT_DIR}/lib/cluster/generate.py" render \
|
||
|
|
"${inv}" "${SCRIPT_DIR}/docker-compose.yml" "${SCRIPT_DIR}/.env" \
|
||
|
|
"${SCRIPT_DIR}/generated" "${SCRIPT_DIR}/scripts/render-config.sh" "${args[@]}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Menjalankan compose untuk SATU host dari hasil generate.
|
||
|
|
run_host() {
|
||
|
|
# Catatan bash: JANGAN menggabung `local name="$1" ... dir="...${name}"` dalam satu
|
||
|
|
# baris — dengan `set -u`, bash meng-expand seluruh kata sebelum assignment sehingga
|
||
|
|
# `${name}` masih unbound ("name: unbound variable"). Pisahkan per baris.
|
||
|
|
local name="$1"
|
||
|
|
local dry="$2"
|
||
|
|
local dir="${SCRIPT_DIR}/generated/host-${name}"
|
||
|
|
if [[ ! -d "${dir}" ]]; then
|
||
|
|
echo -e "${ERR} Host '${name}' belum digenerate (tidak ada ${dir})."
|
||
|
|
echo " Jalankan dulu: $0 --generate [--inventory cluster.yml]"
|
||
|
|
return 1
|
||
|
|
fi
|
||
|
|
# Guard: berkas runtime bind-mount WAJIB FILE (Docker membuat DIRECTORY bila sumber
|
||
|
|
# hilang → mis. Airflow IsADirectoryError). Isi ditulis setelah .env host di-source.
|
||
|
|
sicabe_ensure_file_mount "${dir}/config/trino/password.db" 644
|
||
|
|
echo -e "${WARN} Jalankan ini HANYA di host '${name}'."
|
||
|
|
echo -e "${WARN} Pilot berdampingan di host yang SAMA tidak cukup dipisah network/port:"
|
||
|
|
echo -e "${WARN} RAM tidak cukup untuk dua stack — matikan stack lama lebih dulu (README)."
|
||
|
|
# ── PRECEDENCE (bug nyata yang dilaporkan dari server) ──────────────────────
|
||
|
|
# Compose memberi prioritas pada variabel **SHELL** di atas `--env-file`. Skrip ini
|
||
|
|
# sudah men-source config/environment.defaults.sh, yang mengekspor
|
||
|
|
# POSTGRES_HOST=postgres (default single-host) dan kawan-kawan. Akibatnya nilai dari
|
||
|
|
# bundle host KALAH, container mencari hostname `postgres` yang hanya ada di host
|
||
|
|
# lain -> java.net.UnknownHostException: postgres, walaupun --env-file sudah benar.
|
||
|
|
# Karena itu .env host di-source lebih dulu (set -a) supaya nilai host yang menang.
|
||
|
|
set -a
|
||
|
|
# shellcheck disable=SC1091
|
||
|
|
. "${dir}/.env"
|
||
|
|
set +a
|
||
|
|
|
||
|
|
# Berkas runtime bind-mount WAJIB FILE (Docker membuat DIRECTORY bila sumber hilang →
|
||
|
|
# mis. Airflow SimpleAuthManager IsADirectoryError). Bundle normal sudah memuatnya;
|
||
|
|
# jaring pengaman ini menulis ulang isi dari .env host.
|
||
|
|
local _pwp="${dir}/config/airflow/simple_auth_manager_passwords.json"
|
||
|
|
sicabe_ensure_file_mount "${_pwp}" 666
|
||
|
|
printf '{"admin": "%s"}\n' "${AIRFLOW_ADMIN_PASSWORD:?AIRFLOW_ADMIN_PASSWORD tidak ada di ${dir}/.env}" > "${_pwp}"
|
||
|
|
chmod 666 "${_pwp}"
|
||
|
|
|
||
|
|
# ── INGRESS (T-20) ─────────────────────────────────────────────────────────
|
||
|
|
# Service `traefik` berada di profile `edge`. Bila bundle host ini memuatnya
|
||
|
|
# (artinya tls.mode != none), profile itu WAJIB ikut: tanpa itu Traefik tidak pernah
|
||
|
|
# start sehingga semua rute publik tidak ada — walaupun label sudah terpasang di
|
||
|
|
# tiap layanan dan sertifikat sudah bisa diterbitkan. Dideteksi dari BERKAS bundle,
|
||
|
|
# bukan dari variabel, supaya tidak bisa meleset antara mode none dan edge.
|
||
|
|
local profiles="--profile all"
|
||
|
|
if grep -qE '^ traefik:' "${dir}/docker-compose.yml" 2>/dev/null; then
|
||
|
|
profiles="--profile all --profile edge"
|
||
|
|
fi
|
||
|
|
|
||
|
|
if [[ "${dry}" == "1" ]]; then
|
||
|
|
echo -e "${INFO} (dry-run) perintah yang akan dijalankan:"
|
||
|
|
echo " ${DOCKER_COMPOSE} -f ${dir}/docker-compose.yml --env-file ${dir}/.env ${profiles} up -d"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
${DOCKER_COMPOSE} -f "${dir}/docker-compose.yml" --env-file "${dir}/.env" ${profiles} up -d
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Multi-host: distribusi bundle ke host lain (Fase 1) ─────────────────────
|
||
|
|
# Menyalin direktori deploy/ (TERMASUK .env + generated/) ke host lain via rsync
|
||
|
|
# over SSH. Dijalankan di host CONTROL (master installer). Target = semua host di
|
||
|
|
# cluster.yml KECUALI host ini (dideteksi dari kecocokan alamat lokal) — atau satu
|
||
|
|
# host saja via --host. Aman diulang (rsync idempoten).
|
||
|
|
sync_hosts() {
|
||
|
|
local inv="${SCRIPT_DIR}/cluster.yml" dest="~/sicabe-doc/deploy" user="${USER:-root}" \
|
||
|
|
only_host="" do_delete=0 dry=0
|
||
|
|
shift
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--inventory) inv="${2:-}"; shift 2 ;;
|
||
|
|
--dest) dest="${2:-}"; shift 2 ;;
|
||
|
|
--user) user="${2:-}"; shift 2 ;;
|
||
|
|
--host) only_host="${2:-}"; shift 2 ;;
|
||
|
|
--delete) do_delete=1; shift ;;
|
||
|
|
--dry-run) dry=1; shift ;;
|
||
|
|
*) echo -e "${ERR} Argumen tidak dikenal untuk --sync: $1"; exit 1 ;;
|
||
|
|
esac
|
||
|
|
done
|
||
|
|
|
||
|
|
if [[ ! -f "${inv}" ]]; then
|
||
|
|
echo -e "${ERR} Inventory tidak ditemukan: ${inv}"
|
||
|
|
echo " Mulai dari contoh: cp cluster.example.yml cluster.yml"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
if ! command -v rsync &>/dev/null; then
|
||
|
|
echo -e "${ERR} rsync wajib terpasang (mis. sudo apt-get install -y rsync)."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
local pairs
|
||
|
|
pairs="$(python3 "${SCRIPT_DIR}/lib/cluster/inventory.py" hosts "${inv}" 2>/dev/null)" || {
|
||
|
|
echo -e "${ERR} Gagal membaca inventory (${inv})."; exit 1; }
|
||
|
|
if [[ -z "${pairs}" ]]; then
|
||
|
|
echo -e "${ERR} Inventory tidak memuat host."; exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Tentukan host INI dari kecocokan alamat lokal; bila tak cocok → sinkron ke SEMUA.
|
||
|
|
local self="" lip n a
|
||
|
|
lip="$(hostname -I 2>/dev/null || true)"
|
||
|
|
while read -r n a; do
|
||
|
|
[[ -z "${n}" ]] && continue
|
||
|
|
[[ " ${lip} " == *" ${a} "* ]] && self="${n}"
|
||
|
|
done <<< "${pairs}"
|
||
|
|
|
||
|
|
# Kumpulkan target: kecuali host ini; hormati --host.
|
||
|
|
local targets=()
|
||
|
|
while read -r n a; do
|
||
|
|
[[ -z "${n}" ]] && continue
|
||
|
|
[[ -n "${self}" && "${n}" == "${self}" ]] && continue
|
||
|
|
[[ -n "${only_host}" && "${n}" != "${only_host}" ]] && continue
|
||
|
|
targets+=("${n}|${a}")
|
||
|
|
done <<< "${pairs}"
|
||
|
|
|
||
|
|
echo -e "${BOLD}${BLUE}Distribusi deploy/ ke host lain${NC}"
|
||
|
|
echo " sumber : ${SCRIPT_DIR}/"
|
||
|
|
echo " tujuan : ${user}@<host>:${dest}"
|
||
|
|
echo " host ini : ${self:-<tak dikenal — sinkron ke semua>}"
|
||
|
|
[[ ${do_delete} -eq 1 ]] && echo " mode : --delete (hapus berkas tujuan yang tak ada di sumber)"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
if [[ ${#targets[@]} -eq 0 ]]; then
|
||
|
|
echo -e "${WARN} Tidak ada host tujuan (host ini=${self:-?}; --host=${only_host:-—})."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
local t name addr okc=0 failc=0
|
||
|
|
for t in "${targets[@]}"; do
|
||
|
|
name="${t%%|*}"; addr="${t##*|}"
|
||
|
|
echo -e "${STEP} → ${name} (${user}@${addr}:${dest})${NC}"
|
||
|
|
local rargs=(-az --human-readable -e "ssh -o ConnectTimeout=8" --exclude '.git' --exclude 'output')
|
||
|
|
[[ ${do_delete} -eq 1 ]] && rargs+=(--delete)
|
||
|
|
[[ ${dry} -eq 1 ]] && rargs+=(--dry-run)
|
||
|
|
# Buat direktori tujuan bila belum ada (`~` diekspansi oleh shell REMOTE).
|
||
|
|
rargs+=(--rsync-path "mkdir -p ${dest} && rsync")
|
||
|
|
if rsync "${rargs[@]}" "${SCRIPT_DIR}/" "${user}@${addr}:${dest}/"; then
|
||
|
|
echo -e " ${OK} ${name}: tersinkron"
|
||
|
|
okc=$((okc + 1))
|
||
|
|
else
|
||
|
|
echo -e " ${ERR} ${name}: GAGAL — periksa SSH/rsync ke ${addr}" >&2
|
||
|
|
failc=$((failc + 1))
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo -e "${BOLD}Ringkasan${NC}"
|
||
|
|
echo " berhasil: ${okc} · gagal: ${failc}"
|
||
|
|
echo " Langkah berikutnya DI tiap host tujuan:"
|
||
|
|
echo " ./scripts/prefetch-images.sh --host <nama> && ./install.sh --host <nama>"
|
||
|
|
[[ ${failc} -eq 0 ]]
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Jalankan ─────────────────────────────────────────────────────────────────
|
||
|
|
main "$@"
|