From 9f454b6e8e66d0b34f52fc4dc75a71c88a0bc81b Mon Sep 17 00:00:00 2001 From: Subroto Date: Sun, 11 Oct 2026 18:13:03 +0700 Subject: [PATCH] sync dari sicabe-doc/deploy @ e8f8fde --- deploy/bootstrap.sh | 8 ++- deploy/cluster.example.yml | 16 +++++ .../trino/config-coordinator.properties | 13 ++-- deploy/docker-compose.yml | 8 ++- deploy/lib/cluster/generate.py | 47 +++++++++++-- deploy/lib/cluster/inventory.py | 41 +++++++++++ deploy/lib/common.sh | 11 +++ deploy/scripts/test-generate.sh | 68 ++++++++++++++++--- deploy/scripts/verify-keycloak.sh | 2 +- deploy/scripts/verify-phase1-iam.sh | 2 +- deploy/scripts/verify-phase3-processing.sh | 2 +- deploy/scripts/verify-phase4-serving.sh | 2 +- 12 files changed, 191 insertions(+), 29 deletions(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 94ed9e1..4755c93 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -713,9 +713,11 @@ bootstrap_starrocks() { # verify-phase4 KC_BASE). single-host → localhost:12080; cluster → :12080. # Sebelumnya pakai KEYCLOAK_URL (default localhost) → issuer salah saat keycloak existing. local kc_base - kc_base="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)" - # Alamat Keycloak server→server (dari container StarRocks): service/owner + port container. - local kc_internal="http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}" + kc_base="$(sicabe_keycloak_url)" + # Alamat Keycloak server→server (dari container StarRocks). KEYCLOAK_INTERNAL_URL bila + # diset (Keycloak EKSTERNAL ber-URL penuh, mis. https://sso.example.com), selain itu + # dibangun dari HOST:PORT (mis. 100.93.54.108:12080). + local kc_internal="${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}" sr_mysql() { mysql -h 127.0.0.1 -P "${sr_port}" -u root -p"${STARROCKS_ROOT_PASSWORD}" -e "$1" 2>/dev/null diff --git a/deploy/cluster.example.yml b/deploy/cluster.example.yml index 5fdc632..5fa3f96 100644 --- a/deploy/cluster.example.yml +++ b/deploy/cluster.example.yml @@ -62,6 +62,22 @@ components: # StarRocks FE ke host control bila compute sudah terlalu padat. # starrocks-fe: { host: control } + # ── Komponen EKSTERNAL (dikelola sistem lain) ─────────────────────────────── + # Tandai dengan `address` — maka komponen TIDAK dijalankan di cluster ini dan + # koneksi diarahkan ke alamat itu. Host = alamat IP/hostname biasa: + # keycloak: { address: 100.93.54.108 } + # URL boleh berupa **URL PENUH** (skema + host + port). Skema & port dihormati + # apa adanya (port default per skema, mis. https→443, boleh dihilangkan): + # keycloak: { address: https://sso.example.com } + # minio: { address: https://minio.corp:9000 } + # nessie: { address: https://nessie.corp } + # openldap: { address: ldaps://ldap.corp:636 } + # opa: { address: https://opa.corp } + # Efek URL penuh: `*_HOST` = host saja; `*_URL`/`*_PUBLIC_URL` = URL itu; + # server→server OIDC diarahkan lewat KEYCLOAK_INTERNAL_URL (Trino/Grafana); + # `https://…` ⇒ KEYCLOAK_SSL_REQUIRED=external. `host` dan `address` tak boleh + # bersamaan (host = dijalankan cluster; address = dikelola pihak lain). + # Dremio default NONAKTIF. Bila diaktifkan: taruh di host bertier `engine` # (butuh RAM ~4 GB lebih — cek kapasitas host). dremio: { enabled: false } diff --git a/deploy/config/trino/config-coordinator.properties b/deploy/config/trino/config-coordinator.properties index 33509bf..f3c0db7 100644 --- a/deploy/config/trino/config-coordinator.properties +++ b/deploy/config/trino/config-coordinator.properties @@ -41,17 +41,18 @@ http-server.process-forwarded=${ENV:TRINO_PROCESS_FORWARDED} # single-host (localhost) dan multi-host tanpa TLS (http://:12080). # Kalau di-hardcode, issuer di klaster multi-host tidak cocok dan token ditolak. http-server.authentication.oauth2.issuer=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe -# Matikan OIDC discovery — Trino tidak bisa mengakses URL publik dari dalam container. -# Endpoint di-set manual: browser pakai URL publik (auth-url); server-ke-server (token/jwks) -# pakai KEYCLOAK_HOST:INTERNAL_PORT — di multi-host dgn Keycloak EKSTERNAL ini = ALAMAT host -# pemilik (mis. 100.93.54.108:12080). JANGAN literal `keycloak` (nama container, tak resolve +# Server-ke-server (token/jwks) pakai KEYCLOAK_INTERNAL_URL — URL penuh +# (skema+host+port). Di multi-host dgn Keycloak EKSTERNAL berdomain mis. +# `https://sso.example.com`, nilainya persis itu (skema+port dihormati). Selain itu +# compose mengisi default `http://${KEYCLOAK_HOST}:${KEYCLOAK_INTERNAL_PORT}` +# (mis. 100.93.54.108:12080). JANGAN literal `keycloak` (nama container, tak resolve # di host lain → penukaran token gagal → "Authentication response could not be verified"). http-server.authentication.oauth2.oidc.discovery=false # Endpoint browser-facing (auth redirect) — harus URL publik juga http-server.authentication.oauth2.auth-url=${ENV:KEYCLOAK_PUBLIC_URL}/realms/sicabe/protocol/openid-connect/auth # Endpoint server-side (token exchange & JWKS) — reachable dari container -http-server.authentication.oauth2.token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/token -http-server.authentication.oauth2.jwks-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}/realms/sicabe/protocol/openid-connect/certs +http-server.authentication.oauth2.token-url=${ENV:KEYCLOAK_INTERNAL_URL}/realms/sicabe/protocol/openid-connect/token +http-server.authentication.oauth2.jwks-url=${ENV:KEYCLOAK_INTERNAL_URL}/realms/sicabe/protocol/openid-connect/certs # Kunci HMAC tetap untuk `state` OAuth2 (default Trino = acak tiap coordinator start → # sesi login batal saat restart). Nilai acak-per-instalasi dari .env. http-server.authentication.oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 0242530..cd85323 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -489,6 +489,10 @@ services: # EKSTERNAL: ini = ALAMAT host pemilik (mis. 100.93.54.108) — bukan nama container # `keycloak` yang tak bisa di-resolve → penukaran token gagal ("could not be verified"). KEYCLOAK_HOST: ${KEYCLOAK_HOST:-keycloak} + # URL server→server LENGKAP (skema+host+port) untuk token/jwks. Default dibangun + # dari HOST:PORT; generator menimpanya bila Keycloak EKSTERNAL ber-URL penuh + # (mis. https://sso.example.com) — lihat config-coordinator.properties. + KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}} # Kunci HMAC tetap untuk state OAuth2 Trino (default Trino acak tiap start). TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY:-trino-oauth-state-dev-only} healthcheck: @@ -1361,8 +1365,8 @@ services: GF_AUTH_GENERIC_OAUTH_CLIENT_ID: grafana GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "${GRAFANA_CLIENT_SECRET:?err}" GF_AUTH_GENERIC_OAUTH_AUTH_URL: http://localhost:${KEYCLOAK_PORT:-12080}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/auth - GF_AUTH_GENERIC_OAUTH_TOKEN_URL: http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token - GF_AUTH_GENERIC_OAUTH_API_URL: http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/userinfo + GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token" + GF_AUTH_GENERIC_OAUTH_API_URL: "${KEYCLOAK_INTERNAL_URL:-http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}}/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/userinfo" GF_AUTH_GENERIC_OAUTH_SCOPES: openid profile email GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: contains(roles[*], 'admin') && 'Admin' || 'Viewer' GF_AUTH_DISABLE_LOGIN_FORM: "false" diff --git a/deploy/lib/cluster/generate.py b/deploy/lib/cluster/generate.py index 71053c4..131166d 100644 --- a/deploy/lib/cluster/generate.py +++ b/deploy/lib/cluster/generate.py @@ -96,6 +96,30 @@ PUBLIC_URL_ENDPOINTS = { "MINIO_CONSOLE_PUBLIC_URL": ("minio", "http://{addr}:12901"), } + +def _ext_url(ext: str, template: str, port: str = "") -> str: + """URL untuk komponen EKSTERNAL dari `components..address`. + + Bila operator menulis **URL penuh** (mengandung `://`, mis. + `https://sso.example.com`) → pakai APA ADANYA: skema + host + port dihormati + (default port per skema, mis. https→443, TIDAK ditulis bila default). Bila hanya + host/IP polos (perilaku lama) → susun dari `template` + `port` seperti sebelumnya. + """ + scheme, host, p = inventory.parse_address(ext) + if scheme: + # Buang port bila = default skema (https→443), supaya `https://host`, bukan + # `https://host:443` (ekuivalen, tapi lebih bersih & cocok utk issuer/URL publik). + show = p and p != inventory._DEFAULT_SCHEME_PORT.get(scheme) + return f"{scheme}://{host}" + (f":{p}" if show else "") + if port and "{port}" in template: + return template.format(addr=host, port=(p if p else port)) + return template.format(addr=host) + + +def _keycloak_external_scheme(doc: dict) -> str: + """Skema URL Keycloak eksternal ('' bila bukan URL penuh). Untuk sslRequired.""" + return inventory.parse_address(inventory.external_address(doc, "keycloak") or "")[0] + # SENGAJA TIDAK diarahkan ke IP: KEYCLOAK_URL / TRINO_URL dan var `*_URL` lain yang # bersifat identitas (issuer OIDC harus stabil — lihat R10 §4.10.1: pakai DNS # internal, bukan IP). Itu milik operator dan harus sudah benar di `cluster.yml` @@ -195,7 +219,7 @@ def _is_topology_var(line: str) -> bool: """ key = line.split("=", 1)[0].strip() owned = { - "EXTERNAL_SERVICES", "LOCAL_SERVICES", "KEYCLOAK_SSL_REQUIRED", + "EXTERNAL_SERVICES", "LOCAL_SERVICES", "KEYCLOAK_SSL_REQUIRED", "KEYCLOAK_INTERNAL_URL", "KC_PROXY_HEADERS", "TRINO_PROCESS_FORWARDED", "CLUSTER_DOMAIN", "TRAEFIK_BIND_ADDR", *ENDPOINT_VARS, *URL_ENDPOINT_VARS, *inventory.PUBLIC_URL_SERVICES, # KEYCLOAK_PUBLIC_URL, GRAFANA_PUBLIC_URL, … @@ -458,7 +482,10 @@ def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]: for var, component in ENDPOINT_VARS.items(): ext = inventory.external_address(doc, component) if ext: - lines.append(f"{var}={ext}") # EKSTERNAL (pihak lain) → alamat tetap + # EKSTERNAL → HOST saja (buang skema/port bila operator menulis URL penuh, + # mis. `https://sso.example.com` → `sso.example.com`). URL lengkapnya + # ditangani `*_URL` / KEYCLOAK_INTERNAL_URL, bukan var `*_HOST`. + lines.append(f"{var}={inventory.parse_address(ext)[1]}") continue owners = placement.get(component) if not owners: @@ -467,6 +494,13 @@ def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]: lines.append(f"{var}={component}") # SE-HOST → nama service else: lines.append(f"{var}={plan[owners[0]]['address']}") # LINTAS host → alamat + # Keycloak: URL server→server (token/jwks/Grafana). Hanya ditulis bila Keycloak + # EKSTERNAL memakai URL PENUH (mis. https://sso.example.com) agar skema+port + # dihormati; selain itu compose/script memakai default + # `http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}`. + kc_ext = inventory.external_address(doc, "keycloak") + if kc_ext and inventory.parse_address(kc_ext)[0]: + lines.append(f"KEYCLOAK_INTERNAL_URL={_ext_url(kc_ext, '')}") disabled = [n for n, v in inventory.enabled_components(doc).items() if v is False] if disabled: # Komponen `enabled: false` = tidak dijalankan di host mana pun → installer @@ -498,7 +532,7 @@ def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]: for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items(): ext = inventory.external_address(doc, component) if ext: - resolved[var] = tmpl.format(addr=ext) # eksternal → alamat tetap + resolved[var] = _ext_url(ext, tmpl) # eksternal → URL penuh dihormati continue owners = placement.get(component) if not owners: @@ -523,8 +557,11 @@ def tls_policy_lines(doc: dict) -> list[str]: tls.mode `edge`/`full` -> `external` (ada Traefik/TLS di depan) """ edge = inventory.tls_mode(doc) != "none" + # Keycloak EKSTERNAL ber-URL `https://…` juga menuntut sslRequired=external (permintaan + # dari alamat non-lokal lewat TLS di depannya). + ext_https = _keycloak_external_scheme(doc) == "https" return [ - f"KEYCLOAK_SSL_REQUIRED={'external' if edge else 'none'}", + f"KEYCLOAK_SSL_REQUIRED={'external' if (edge or ext_https) else 'none'}", # Di belakang Traefik, Keycloak WAJIB memercayai X-Forwarded-* — tanpa itu # permintaan HTTPS dari proxy dianggap HTTP (redirect salah / tuntutan HTTPS). # Sengaja KOSONG saat tanpa proxy: memasangnya membuat Keycloak memercayai @@ -665,7 +702,7 @@ def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[ for var, (component, template) in URL_ENDPOINT_VARS.items(): ext = inventory.external_address(doc, component) if ext: - lines.append(f"{var}={template.format(addr=ext, port=_endpoint_port(var, env_path))}") + lines.append(f"{var}={_ext_url(ext, template, _endpoint_port(var, env_path))}") continue owners = placement.get(component) if not owners: diff --git a/deploy/lib/cluster/inventory.py b/deploy/lib/cluster/inventory.py index 57db6b2..6d7b83c 100644 --- a/deploy/lib/cluster/inventory.py +++ b/deploy/lib/cluster/inventory.py @@ -145,6 +145,47 @@ def external_address(doc: dict, service: str) -> str | None: return a or None +# Port default per skema — dipakai parse_address() bila operator menulis URL penuh +# TANPA port (mis. `https://sso.example.com` → port 443). +_DEFAULT_SCHEME_PORT = { + "https": 443, "http": 80, "ldaps": 636, "ldap": 389, + "postgres": 5432, "postgresql": 5432, +} + + +def parse_address(addr: str) -> tuple[str, str, int | None]: + """Pisah alamat komponen → (skema, host, port). + + Menerima dua bentuk: + - **URL penuh** (skema eksplisit): `https://sso.example.com` → ('https','sso.example.com',443) + `http://minio.corp:9000` → ('http','minio.corp',9000) + - **host/IP polos** (perilaku lama): `sso.example.com` → ('','sso.example.com',None) + `10.20.0.10` → ('','10.20.0.10',None) + + Skema non-standar diterima apa adanya; path (`/foo`) dibuang. + """ + a = (addr or "").strip() + scheme = "" + if "://" in a: + scheme, a = a.split("://", 1) + scheme = scheme.strip().lower() + a = a.split("/", 1)[0] # buang path bila ada + host, sep, port_s = a.rpartition(":") + # host:port HANYA bila tepat satu ':' dan port numerik — melindungi IPv6 + # (mis. `2001:db8::1`) agar tak salah dipecah. + if sep and port_s.isdigit() and a.count(":") == 1: + host, port = host, int(port_s) + else: + host, port = a, _DEFAULT_SCHEME_PORT.get(scheme) + return scheme, host, port + + +def external_host(doc: dict, service: str) -> str | None: + """HOST saja (tanpa skema/port) dari alamat eksternal — untuk var `*_HOST`.""" + a = external_address(doc, service) + return parse_address(a)[1] if a else None + + def resolve(doc: dict) -> dict[str, list[str]]: """Komponen → daftar host (tier `engine` bisa >1 host). Komponen EKSTERNAL (ber-`address`) TIDAK masuk hasil — ia bukan milik host mana pun.""" diff --git a/deploy/lib/common.sh b/deploy/lib/common.sh index 53ab5b7..a041763 100644 --- a/deploy/lib/common.sh +++ b/deploy/lib/common.sh @@ -188,6 +188,17 @@ sicabe_local_url() { # [url_var] echo "${_slu_s}://localhost:${_slu_p}" } +# ─── Helper: URL Keycloak yang REACHABLE dari host ini ────────────────────── +# Prioritas: KEYCLOAK_INTERNAL_URL (URL penuh skema+host+port, mis. +# `https://sso.example.com` — generator menulisnya bila Keycloak EKSTERNAL ber-URL +# penuh) → KEYCLOAK_URL eksplisit non-localhost (override operator) → host+port +# (perilaku lama: nama service bila co-located, alamat pemilik bila lintas host). +# Dipakai gate/verifikasi & bootstrap agar tak perlu set KEYCLOAK_URL manual. +sicabe_keycloak_url() { + if [[ -n "${KEYCLOAK_INTERNAL_URL:-}" ]]; then printf '%s' "${KEYCLOAK_INTERNAL_URL}"; return 0; fi + sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL +} + # ─── Helper: HOST endpoint efektif (sadar EXTERNAL & multi-host) ──────────── # Urutan: (1) bila NON-localhost (override operator / komponen EXTERNAL); # (2) multi-host (SICABE_HOST_NAME) → ; diff --git a/deploy/scripts/test-generate.sh b/deploy/scripts/test-generate.sh index 88ac9ee..11c98a9 100755 --- a/deploy/scripts/test-generate.sh +++ b/deploy/scripts/test-generate.sh @@ -89,9 +89,9 @@ grep -q '${MINIO_INTERNAL_PORT:-9000}' "${dc}" || miss="${miss} MINIO" hard="$(grep -nE '\$\{(POSTGRES_HOST|KEYCLOAK_HOST|MINIO_HOST)[^}]*\}:(5432|8080|9000)' "${dc}" || true)" [[ -z "${hard}" ]] && ok "docker-compose: tak ada port internal literal di var *_HOST" \ || bad "docker-compose: masih ada port literal: ${hard}" -grep -q '${ENV:KEYCLOAK_INTERNAL_PORT}' "${DEPLOY_DIR}/config/trino/config-coordinator.properties" \ - && ok "Trino config: token/jwks-url pakai \${ENV:KEYCLOAK_INTERNAL_PORT}" \ - || bad "Trino config: port Keycloak masih literal 8080" +grep -q '${ENV:KEYCLOAK_INTERNAL_URL}' "${DEPLOY_DIR}/config/trino/config-coordinator.properties" \ + && ok "Trino config: token/jwks-url pakai \${ENV:KEYCLOAK_INTERNAL_URL} (skema+host+port)" \ + || bad "Trino config: URL Keycloak belum terparameter (token/jwks literal?)" grep -q ':{port}' "${DEPLOY_DIR}/lib/cluster/generate.py" \ && ok "generate.py: URL_ENDPOINT_VARS memakai {port} (dari .env)" \ || bad "generate.py: URL_ENDPOINT_VARS masih port literal" @@ -225,20 +225,20 @@ else bad "verify-keycloak.sh hilang/tidak lengkap" fi -# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_HOST (bukan literal `keycloak`) ── +# ─── 0k. Trino OAuth2: token/jwks pakai KEYCLOAK_INTERNAL_URL (bukan literal) ── # Kls bug sama dgn target MinIO Prometheus: Trino server-ke-server ke Keycloak. Bila # Keycloak EKSTERNAL (P3), literal `keycloak:` tak resolve → penukaran token gagal → # callback `/oauth2/callback` balas "Authentication response could not be verified". _tc="${DEPLOY_DIR}/config/trino/config-coordinator.properties" -if grep -q 'token-url=http://${ENV:KEYCLOAK_HOST}:${ENV:KEYCLOAK_INTERNAL_PORT}' "${_tc}" 2>/dev/null \ - && grep -q 'jwks-url=http://${ENV:KEYCLOAK_HOST}:' "${_tc}" 2>/dev/null \ +if grep -q 'token-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${_tc}" 2>/dev/null \ + && grep -q 'jwks-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${_tc}" 2>/dev/null \ && grep -q 'oauth2.state-key=${ENV:TRINO_OAUTH_STATE_KEY}' "${_tc}" 2>/dev/null \ - && grep -q 'KEYCLOAK_HOST: ${KEYCLOAK_HOST' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \ + && grep -q 'KEYCLOAK_INTERNAL_URL: ${KEYCLOAK_INTERNAL_URL' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \ && grep -q 'TRINO_OAUTH_STATE_KEY: ${TRINO_OAUTH_STATE_KEY' "${DEPLOY_DIR}/docker-compose.yml" 2>/dev/null \ && ! grep -qE 'token-url=http://keycloak:' "${_tc}" 2>/dev/null; then - ok "trino oauth2: token-url/jwks-url pakai KEYCLOAK_HOST + state-key tetap" + ok "trino oauth2: token/jwks pakai KEYCLOAK_INTERNAL_URL + state-key tetap" else - bad "trino oauth2: token-url/jwks-url literal 'keycloak:' atau state-key belum diset" + bad "trino oauth2: token/jwks belum pakai KEYCLOAK_INTERNAL_URL atau state-key belum diset" fi # ─── 0l. StarRocks OIDC: redirect_url = alamat browser (bukan localhost) ────── @@ -307,6 +307,56 @@ else bad "storage: konfigurasi bucket/warehouse belum lengkap →${wh_bad}" fi +# ─── 0o. Komponen EKSTERNAL ber-URL penuh (domain + skema) ──────────────────── +# Client existing (mis. Keycloak) bisa sudah berdomain `https://sso.example.com`. +# Alamat itu WAJIB didukung: skema+port dihormati (tak ditempeli http://…:12xxx) dan +# server→server diarahkan lewat KEYCLOAK_INTERNAL_URL (token/jwks Trino & Grafana). +_uo=1 +grep -q 'def parse_address' "${DEPLOY_DIR}/lib/cluster/inventory.py" || _uo=0 +grep -q 'def _ext_url' "${DEPLOY_DIR}/lib/cluster/generate.py" || _uo=0 +grep -q 'token-url=${ENV:KEYCLOAK_INTERNAL_URL}' "${DEPLOY_DIR}/config/trino/config-coordinator.properties" || _uo=0 +grep -q 'KEYCLOAK_INTERNAL_URL' "${DEPLOY_DIR}/docker-compose.yml" || _uo=0 +_uo_parse="$(python3 - "${DEPLOY_DIR}" <<'PY' +import importlib.util, sys +d = sys.argv[1] +s = importlib.util.spec_from_file_location("inv", f"{d}/lib/cluster/inventory.py") +inv = importlib.util.module_from_spec(s); s.loader.exec_module(inv) +cases = {"https://sso.example.com": ("https", "sso.example.com", 443), + "http://minio.corp:9000": ("http", "minio.corp", 9000), + "https://sso.example.com:8443": ("https", "sso.example.com", 8443), + "sso.example.com": ("", "sso.example.com", None), + "10.20.0.10": ("", "10.20.0.10", None)} +print("1" if all(inv.parse_address(k) == v for k, v in cases.items()) else "0") +PY +)" +cat > "${OUT}/inv-url.yml" <<'YML' +version: 1 +tls: { mode: none } +hosts: + control: { address: 10.20.0.10, tiers: [identity, policy, pipeline, governance] } + compute: { address: 10.20.0.11, tiers: [coord, engine] } + state: { address: 10.20.0.12, tiers: [state] } +components: + keycloak: { address: https://sso.example.com } +YML +_uo_gen=0 +if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ + "${OUT}/inv-url.yml" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ + "${OUT}/url" "${DEPLOY_DIR}/scripts/render-config.sh" > "${OUT}/uri.log" 2>&1; then + _ue="${OUT}/url/host-control/.env" + if grep -qx 'KEYCLOAK_HOST=sso.example.com' "${_ue}" 2>/dev/null \ + && grep -qx 'KEYCLOAK_PUBLIC_URL=https://sso.example.com' "${_ue}" 2>/dev/null \ + && grep -qx 'KEYCLOAK_INTERNAL_URL=https://sso.example.com' "${_ue}" 2>/dev/null \ + && grep -qx 'KEYCLOAK_SSL_REQUIRED=external' "${_ue}" 2>/dev/null; then + _uo_gen=1 + fi +fi +if [[ "${_uo}" == "1" && "${_uo_parse}" == "1" && "${_uo_gen}" == "1" ]]; then + ok "eksternal URL penuh: domain https dihormati + KEYCLOAK_INTERNAL_URL + sslRequired=external" +else + bad "eksternal URL penuh belum didukung (wire=${_uo} parse=${_uo_parse} gen=${_uo_gen})" +fi + # ─── 1. generate ──────────────────────────────────────────────────────────── if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \ "${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \ diff --git a/deploy/scripts/verify-keycloak.sh b/deploy/scripts/verify-keycloak.sh index bba6fef..7c535f3 100755 --- a/deploy/scripts/verify-keycloak.sh +++ b/deploy/scripts/verify-keycloak.sh @@ -33,7 +33,7 @@ source "${SCRIPT_DIR}/lib/common.sh" source "${SCRIPT_DIR}/config/environment.defaults.sh" sicabe_load_env "${SCRIPT_DIR}" "$@" || exit 1 -KC="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)" +KC="$(sicabe_keycloak_url)" REALM="${SICABE_REALM:-sicabe}" REQUIRED_CLIENTS=("trino" "starrocks" "airflow" "openmetadata" "grafana") diff --git a/deploy/scripts/verify-phase1-iam.sh b/deploy/scripts/verify-phase1-iam.sh index d819547..4fb3d30 100755 --- a/deploy/scripts/verify-phase1-iam.sh +++ b/deploy/scripts/verify-phase1-iam.sh @@ -35,7 +35,7 @@ sicabe_load_env "${SCRIPT_DIR}" "$@" # Config RELEVAN: mode host (--env-file) = BUNDLE host (ter-render), bukan deploy/config/ sumber. if [[ -n "${SICABE_BUNDLE_DIR:-}" ]]; then CONFIG_DIR="${SICABE_BUNDLE_DIR}/config"; else CONFIG_DIR="${SCRIPT_DIR}/config"; fi -KC="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)" +KC="$(sicabe_keycloak_url)" REALM="${SICABE_REALM:-sicabe}" LDAP_HOST="${LDAP_HOST:-localhost}" LDAP_PORT="${LDAP_PORT:-389}" diff --git a/deploy/scripts/verify-phase3-processing.sh b/deploy/scripts/verify-phase3-processing.sh index efc7fea..b7ba960 100755 --- a/deploy/scripts/verify-phase3-processing.sh +++ b/deploy/scripts/verify-phase3-processing.sh @@ -131,7 +131,7 @@ trino_exec_oidc() { _trino_stmt "$2" -H "Authorization: Bearer $1"; } # oidc_token → access token (client 'trino', scope openid) oidc_token() { curl -s --max-time 20 -X POST \ - "$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token" \ + "$(sicabe_keycloak_url)/realms/${SICABE_REALM:-sicabe}/protocol/openid-connect/token" \ -d grant_type=password -d client_id=trino -d "client_secret=${TRINO_CLIENT_SECRET}" \ -d "username=$1" -d "password=$2" -d scope=openid 2>/dev/null | python3 -c "import sys,json;print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null diff --git a/deploy/scripts/verify-phase4-serving.sh b/deploy/scripts/verify-phase4-serving.sh index 18fe58c..c2f7da1 100755 --- a/deploy/scripts/verify-phase4-serving.sh +++ b/deploy/scripts/verify-phase4-serving.sh @@ -53,7 +53,7 @@ BUCKET="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}" S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}" S3_REGION="${S3_REGION:-us-east-1}" LDAP_URL="${LDAP_URL:-ldap://openldap:389}" -KC_BASE="$(sicabe_local_url http KEYCLOAK_HOST "${KEYCLOAK_PORT:-12080}" KEYCLOAK_URL)" +KC_BASE="$(sicabe_keycloak_url)" REALM="${SICABE_REALM:-sicabe}" GROUP_PROVIDER_NAME="${STARROCKS_GROUP_PROVIDER:-sr_ldap}" OIDC_NAME="${STARROCKS_OIDC_INTEGRATION:-sr_oidc}"