sync @ 88a1d74
This commit is contained in:
+3
-3
@@ -872,7 +872,7 @@ bootstrap_starrocks() {
|
||||
PROPERTIES (
|
||||
'type' = 'iceberg',
|
||||
'iceberg.catalog.type' = 'rest',
|
||||
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
|
||||
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}${NESSIE_ICEBERG_PREFIX:-/iceberg}/',
|
||||
'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
|
||||
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
|
||||
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
|
||||
@@ -915,7 +915,7 @@ print_starrocks_sql() {
|
||||
-- 3) External catalog Iceberg via Nessie
|
||||
CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
|
||||
'type'='iceberg','iceberg.catalog.type'='rest',
|
||||
'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}/iceberg/',
|
||||
'iceberg.catalog.uri'='${NESSIE_URL:-http://nessie:19120}${NESSIE_ICEBERG_PREFIX:-/iceberg}/',
|
||||
'iceberg.catalog.warehouse'='${ICEBERG_WAREHOUSE:-warehouse}',
|
||||
'aws.s3.endpoint'='${S3_ENDPOINT:-http://minio:9000}', ...);
|
||||
EOF
|
||||
@@ -1124,7 +1124,7 @@ connections:
|
||||
catalog_type: rest
|
||||
catalog_name: iceberg
|
||||
schema: main
|
||||
rest_uri: ${NESSIE_URL:-http://nessie:19120}/iceberg/
|
||||
rest_uri: ${NESSIE_URL:-http://nessie:19120}${NESSIE_ICEBERG_PREFIX:-/iceberg}/
|
||||
rest_warehouse: ${ICEBERG_WAREHOUSE:-warehouse}
|
||||
s3_endpoint: ${S3_ENDPOINT:-http://minio:9000}
|
||||
s3_access_key_id: ${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}
|
||||
|
||||
@@ -91,6 +91,9 @@ export MINIO_BUCKET_ICEBERG_DATA="${MINIO_BUCKET_ICEBERG_DATA:-iceberg-data}"
|
||||
# StarRocks/OM/Sling external catalog. Ganti SATU nilai ini bila client memakai
|
||||
# nama/path berbeda (mis. `wh`) — lalu `--generate --sync`.
|
||||
export ICEBERG_WAREHOUSE="${ICEBERG_WAREHOUSE:-warehouse}"
|
||||
# Prefix REST Iceberg pada Nessie (dipakai konsumen REST: Sling, StarRocks, gate fase 2).
|
||||
# Nessie menyajikannya di `/iceberg` (tetap); ubah bila layanan existing memakai prefix lain.
|
||||
export NESSIE_ICEBERG_PREFIX="${NESSIE_ICEBERG_PREFIX:-/iceberg}"
|
||||
|
||||
# Apache Nessie (Phase 2 — Storage: Metadata Catalog)
|
||||
export NESSIE_VERSION="${NESSIE_VERSION:-0.108.4-java}"
|
||||
|
||||
@@ -743,6 +743,10 @@ def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None:
|
||||
Di sini diganti `<alamat-eksternal>:<MINIO_INTERNAL_PORT>`.
|
||||
|
||||
MinIO se-host (single-host / P1) → tak diubah (`minio:9000`, satu jaringan Docker).
|
||||
|
||||
Address boleh ber-URL penuh (`https://minio.corp:9000`): target ditulis `host:port`
|
||||
(TANPA skema, tanpa port dobel) dan `scheme:` job minio disamakan (https) agar scrape
|
||||
tak menembak http ke port TLS.
|
||||
"""
|
||||
yml = host_dir / "config" / "prometheus" / "prometheus.yml"
|
||||
if not yml.is_file():
|
||||
@@ -750,9 +754,31 @@ def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None:
|
||||
ext = inventory.external_address(doc, "minio")
|
||||
if not ext:
|
||||
return
|
||||
port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000"
|
||||
scheme, host, p = inventory.parse_address(ext)
|
||||
if scheme:
|
||||
target = f"{host}:{p}" if p else host
|
||||
eff_scheme = scheme
|
||||
else:
|
||||
# alamat polos (IP/hostname) → port internal, perilaku lama; skema ikut S3_ENDPOINT
|
||||
port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000"
|
||||
target = f"{host}:{port}"
|
||||
eff_scheme = ((_source_env_value(env_path, "S3_ENDPOINT") or "").split("://", 1)[0]
|
||||
or "http")
|
||||
txt = yml.read_text(encoding="utf-8")
|
||||
new = txt.replace("minio:9000", f"{ext}:{port}")
|
||||
out: list[str] = []
|
||||
in_minio = False
|
||||
for line in txt.splitlines():
|
||||
s = line.strip()
|
||||
if s.startswith("- job_name:"):
|
||||
in_minio = s.split(":", 1)[1].strip() in ("minio", "minio-bucket")
|
||||
if "minio:9000" in line:
|
||||
line = line.replace("minio:9000", target)
|
||||
if in_minio and s == "scheme: http" and eff_scheme != "http":
|
||||
line = line.replace("scheme: http", f"scheme: {eff_scheme}")
|
||||
out.append(line)
|
||||
new = "\n".join(out)
|
||||
if txt.endswith("\n"):
|
||||
new += "\n"
|
||||
if new != txt:
|
||||
yml.write_text(new, encoding="utf-8")
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ ensure_external_catalog() {
|
||||
sr_sql "CREATE EXTERNAL CATALOG iceberg_catalog PROPERTIES (
|
||||
'type' = 'iceberg',
|
||||
'iceberg.catalog.type' = 'rest',
|
||||
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}/iceberg/',
|
||||
'iceberg.catalog.uri' = '${NESSIE_URL:-http://nessie:19120}${NESSIE_ICEBERG_PREFIX:-/iceberg}/',
|
||||
'iceberg.catalog.warehouse' = '${ICEBERG_WAREHOUSE:-warehouse}',
|
||||
'iceberg.catalog.rest.nested-namespace-enabled' = 'true',
|
||||
'aws.s3.endpoint' = '${S3_ENDPOINT:-http://minio:9000}',
|
||||
|
||||
@@ -437,6 +437,68 @@ else
|
||||
bad "Trino OAuth2: realm masih literal 'sicabe' atau SICABE_REALM tak diteruskan"
|
||||
fi
|
||||
|
||||
# ─── 0t. Prefix REST Iceberg Nessie terparameter (NESSIE_ICEBERG_PREFIX) ──────
|
||||
# Regresi: path `/iceberg` dipatok → client yang Nessie-nya memakai prefix lain
|
||||
# (REST Iceberg di path berbeda) FAIL di gate fase 2 / Sling / StarRocks.
|
||||
_np=1
|
||||
grep -q 'export NESSIE_ICEBERG_PREFIX=' "${DEPLOY_DIR}/config/environment.defaults.sh" || _np=0
|
||||
grep -q 'ICEBERG_PREFIX="${NESSIE_ICEBERG_PREFIX:-/iceberg}"' "${DEPLOY_DIR}/scripts/verify-phase2-storage.sh" || _np=0
|
||||
grep -q '${NESSIE_HOST_API}${ICEBERG_PREFIX}/v1/config' "${DEPLOY_DIR}/scripts/verify-phase2-storage.sh" || _np=0
|
||||
grep -q '${NESSIE_ICEBERG_PREFIX:-/iceberg}/' "${DEPLOY_DIR}/bootstrap.sh" || _np=0
|
||||
grep -q '${NESSIE_ICEBERG_PREFIX:-/iceberg}/' "${DEPLOY_DIR}/scripts/seed-starrocks.sh" || _np=0
|
||||
grep -q 'NESSIE_URL:-http://nessie:19120}/iceberg/' "${DEPLOY_DIR}/bootstrap.sh" && _np=0
|
||||
grep -q 'NESSIE_URL:-http://nessie:19120}/iceberg/' "${DEPLOY_DIR}/scripts/seed-starrocks.sh" && _np=0
|
||||
if [[ "${_np}" == "1" ]]; then
|
||||
ok "prefix REST Iceberg Nessie terparameter (NESSIE_ICEBERG_PREFIX, default /iceberg)"
|
||||
else
|
||||
bad "prefix REST Iceberg Nessie masih literal '/iceberg' di salah satu tempat"
|
||||
fi
|
||||
|
||||
# ─── 0u. Prometheus: MinIO EKSTERNAL ber-URL penuh → target host:port + scheme ─
|
||||
# Regresi: rewrite menempel port ke address MENTAH → `https://host:9000:9000` (invalid)
|
||||
# dan scheme job tetap `http` → scrape menembak http ke port TLS.
|
||||
_up_inv="${OUT}/inv-minio-url.yml"
|
||||
cat > "${_up_inv}" <<'YML'
|
||||
version: 1
|
||||
tls: { mode: none }
|
||||
hosts:
|
||||
control: { address: 10.20.0.10, tiers: [identity, policy, pipeline, governance] }
|
||||
compute: { address: 10.20.0.11, tiers: [coord, engine] }
|
||||
state: { address: 10.20.0.12, tiers: [state] }
|
||||
components:
|
||||
minio: { address: https://minio.corp:9000 }
|
||||
YML
|
||||
_up=0
|
||||
if python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${_up_inv}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
|
||||
"${OUT}/uminio" "${DEPLOY_DIR}/scripts/render-config.sh" > "${OUT}/uminio.log" 2>&1; then
|
||||
_upy="${OUT}/uminio/host-control/config/prometheus/prometheus.yml"
|
||||
if grep -q 'targets: \["minio.corp:9000"\]' "${_upy}" 2>/dev/null \
|
||||
&& ! grep -q 'minio.corp:9000:9000' "${_upy}" 2>/dev/null \
|
||||
&& grep -A2 'job_name: minio$' "${_upy}" 2>/dev/null | grep -q 'scheme: https'; then
|
||||
_up=1
|
||||
fi
|
||||
fi
|
||||
if [[ "${_up}" == "1" ]]; then
|
||||
ok "prometheus: MinIO eksternal URL penuh → target host:port + scheme https"
|
||||
else
|
||||
bad "prometheus: target/scheme MinIO URL salah (port dobel atau scheme http)"
|
||||
fi
|
||||
|
||||
# ─── 0v. verify-phase2: pesan FAIL menampilkan kode HTTP + penyebab ──────────
|
||||
# Tanpa ini, FAIL hanya "tidak merespon" → tak terlihat apakah 404 (path absen /
|
||||
# Iceberg REST belum aktif), 401 (auth), 500 (config), atau 000 (tak terjangkau).
|
||||
_v2f="${DEPLOY_DIR}/scripts/verify-phase2-storage.sh"
|
||||
_vh=1
|
||||
grep -q 'http_code() {' "${_v2f}" || _vh=0
|
||||
grep -q 'code_hint() {' "${_v2f}" || _vh=0
|
||||
grep -qF 'tidak merespon [HTTP ${code} — $(code_hint "${code}")]' "${_v2f}" || _vh=0
|
||||
if [[ "${_vh}" == "1" ]]; then
|
||||
ok "verify-phase2: pesan FAIL menyertakan kode HTTP + penyebab (404/401/500/000)"
|
||||
else
|
||||
bad "verify-phase2: pesan FAIL belum menampilkan kode HTTP + penyebab"
|
||||
fi
|
||||
|
||||
# ─── 1. generate ────────────────────────────────────────────────────────────
|
||||
if ! python3 "${DEPLOY_DIR}/lib/cluster/generate.py" render \
|
||||
"${INVENTORY}" "${DEPLOY_DIR}/docker-compose.yml" "${DEPLOY_DIR}/.env" \
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
#
|
||||
# Yang diperiksa:
|
||||
# A. Object storage : endpoint terjangkau, bucket ada, tulis+baca berhasil
|
||||
# B. Nessie : /api/v2/config, default branch, endpoint Iceberg REST /iceberg/v1/config
|
||||
# B. Nessie : /api/v2/config, default branch, endpoint Iceberg REST <prefix>/v1/config
|
||||
# (prefix dari NESSIE_ICEBERG_PREFIX, default /iceberg)
|
||||
# C. Version store : database `nessie` + tabel Nessie di PostgreSQL (JDBC2)
|
||||
# D. Kontrak & antarmuka:
|
||||
# D1 (milik Phase 2) : warehouse & endpoint S3 pada Nessie (compose) → FAIL bila salah
|
||||
@@ -42,6 +43,7 @@ sicabe_load_env "${SCRIPT_DIR}" "$@"
|
||||
if [[ -n "${SICABE_BUNDLE_DIR:-}" ]]; then CONFIG_DIR="${SICABE_BUNDLE_DIR}/config"; else CONFIG_DIR="${SCRIPT_DIR}/config"; fi
|
||||
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://minio:9000}"
|
||||
S3_SCHEME="${S3_ENDPOINT%%://*}" # http | https — dipakai uji kesehatan host
|
||||
S3_REGION="${S3_REGION:-us-east-1}"
|
||||
S3_ACCESS_KEY="${S3_ACCESS_KEY:-${MINIO_ROOT_USER:-minioadmin}}"
|
||||
S3_SECRET_KEY="${S3_SECRET_KEY:-${MINIO_ROOT_PASSWORD:-}}"
|
||||
@@ -55,6 +57,8 @@ NESSIE_PORT="${NESSIE_PORT:-12120}"
|
||||
# 19120). Selebihnya (nama service `nessie:19120` / single-host) → host:port terbit (NESSIE_PORT).
|
||||
NESSIE_HOST_API="$(sicabe_host_url NESSIE_URL NESSIE_HOST "${NESSIE_PORT:-12120}")"
|
||||
NESSIE_DB="${NESSIE_DB:-nessie}"
|
||||
# Prefix REST Iceberg Nessie (mis. `/iceberg`); ubah bila layanan existing memakai prefix lain.
|
||||
ICEBERG_PREFIX="${NESSIE_ICEBERG_PREFIX:-/iceberg}"
|
||||
|
||||
# Mode --deep: ikut menguji KONSUMEN (Trino) — komponennya harus sudah ada (fase 3+).
|
||||
DEEP=0
|
||||
@@ -79,6 +83,20 @@ fail() { echo -e " ${FAIL} $1"; FAIL_N=$((FAIL_N + 1)); FAIL_MSGS+=("$1"); }
|
||||
warn() { echo -e " ${WARN} $1"; WARN_N=$((WARN_N + 1)); }
|
||||
skip() { echo -e " ${YELLOW}[SKIP]${NC} $1"; SKIP_N=$((SKIP_N + 1)); }
|
||||
|
||||
# Kode HTTP untuk <url> (000 = koneksi gagal/timeout). Dipakai agar pesan FAIL
|
||||
# menampilkan PENYEBAB (404/401/500/000), bukan hanya "tidak merespon".
|
||||
http_code() { curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$1" 2>/dev/null || echo 000; }
|
||||
code_hint() {
|
||||
case "$1" in
|
||||
000) echo "tidak terjangkau (koneksi/DNS/timeout) — cek port, alamat, jaringan";;
|
||||
401|403) echo "butuh autentikasi — Nessie existing mungkin memakai bearer token";;
|
||||
404) echo "path tidak ada — versi Nessie lama / Iceberg REST belum aktif (butuh warehouse+object store) / proxy tak merutekan path ini";;
|
||||
429) echo "rate-limited oleh proxy/edge";;
|
||||
500|502|503|504) echo "error di server/proxy — cek konfigurasi Nessie (warehouse/object store) atau proxy";;
|
||||
*) echo "HTTP $1";;
|
||||
esac
|
||||
}
|
||||
|
||||
container_running() {
|
||||
docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^$1$"
|
||||
}
|
||||
@@ -107,14 +125,14 @@ check_object_storage() {
|
||||
# S3_ENDPOINT umumnya hanya terjangkau dari jaringan Docker (mis.
|
||||
# http://minio:9000) → uji keterjangkauan dilakukan dari container (uji mc).
|
||||
# Di host kita cek port publik MinIO saja (informatif, bukan gate).
|
||||
local host_ep="$(sicabe_local_url http MINIO_HOST "${MINIO_PORT:-12900}")"
|
||||
local host_ep="$(sicabe_local_url "${S3_SCHEME:-http}" MINIO_HOST "${MINIO_PORT:-12900}")"
|
||||
local host_code
|
||||
host_code=$(curl -sk -o /dev/null -w "%{http_code}" --max-time 8 \
|
||||
"${host_ep}/minio/health/live" 2>/dev/null || true)
|
||||
if [[ "${host_code}" == "200" ]]; then
|
||||
pass "Storage terjangkau dari host: ${host_ep}"
|
||||
else
|
||||
warn "Storage tidak terjangkau dari host (${host_ep}) — normal bila memakai S3 eksternal"
|
||||
warn "Storage tidak terjangkau dari host (${host_ep}) — normal bila memakai S3 eksternal [HTTP ${host_code:-000} — $(code_hint "${host_code:-000}")]"
|
||||
fi
|
||||
|
||||
if ! docker image inspect "${MC_IMAGE}" >/dev/null 2>&1; then
|
||||
@@ -163,10 +181,11 @@ fi')
|
||||
check_nessie() {
|
||||
echo -e "${BOLD}B. Nessie — ${NESSIE_URL} (API host: ${NESSIE_HOST_API})${NC}"
|
||||
|
||||
local cfg
|
||||
local cfg code
|
||||
cfg=$(curl -sf --max-time 10 "${NESSIE_HOST_API}/api/v2/config" 2>/dev/null || echo "")
|
||||
if [[ -z "${cfg}" ]]; then
|
||||
fail "Nessie REST /api/v2/config tidak merespon di ${NESSIE_HOST_API}"
|
||||
code="$(http_code "${NESSIE_HOST_API}/api/v2/config")"
|
||||
fail "Nessie REST /api/v2/config tidak merespon di ${NESSIE_HOST_API} [HTTP ${code} — $(code_hint "${code}")]"
|
||||
return 1
|
||||
fi
|
||||
pass "Nessie REST v2 aktif"
|
||||
@@ -177,11 +196,12 @@ print(' defaultBranch =', d.get('defaultBranch'))
|
||||
print(' specVersion =', d.get('specVersion'))" 2>/dev/null || true
|
||||
|
||||
local iceberg_cfg
|
||||
iceberg_cfg=$(curl -sf --max-time 10 "${NESSIE_HOST_API}/iceberg/v1/config" 2>/dev/null || echo "")
|
||||
iceberg_cfg=$(curl -sf --max-time 10 "${NESSIE_HOST_API}${ICEBERG_PREFIX}/v1/config" 2>/dev/null || echo "")
|
||||
if [[ -n "${iceberg_cfg}" ]]; then
|
||||
pass "Endpoint Iceberg REST (/iceberg) aktif (dipakai Sling & StarRocks)"
|
||||
pass "Endpoint Iceberg REST (${ICEBERG_PREFIX}) aktif (dipakai Sling & StarRocks)"
|
||||
else
|
||||
fail "Endpoint Iceberg REST ${NESSIE_HOST_API}/iceberg tidak merespon"
|
||||
code="$(http_code "${NESSIE_HOST_API}${ICEBERG_PREFIX}/v1/config")"
|
||||
fail "Endpoint Iceberg REST ${NESSIE_HOST_API}${ICEBERG_PREFIX} tidak merespon [HTTP ${code} — $(code_hint "${code}")]"
|
||||
fi
|
||||
|
||||
local branch
|
||||
@@ -189,7 +209,7 @@ print(' specVersion =', d.get('specVersion'))" 2>/dev/null || true
|
||||
if [[ "${branch}" == "200" ]]; then
|
||||
pass "Branch 'main' tersedia"
|
||||
else
|
||||
warn "Branch 'main' tidak merespon 200 (HTTP ${branch}) — katalog mungkin masih kosong"
|
||||
warn "Branch 'main' tidak merespon 200 [HTTP ${branch} — $(code_hint "${branch}")] — katalog mungkin masih kosong"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
@@ -459,7 +459,7 @@ test_storage_nessie() {
|
||||
|
||||
# Health via API port (management 9000 tidak dipublikasikan)
|
||||
check_url_scoped "Nessie API config" sicabe-nessie "${nessie_url}/api/v2/config" 60
|
||||
check_url_scoped "Nessie Iceberg REST config" sicabe-nessie "${nessie_url}/iceberg/v1/config" 60
|
||||
check_url_scoped "Nessie Iceberg REST config" sicabe-nessie "${nessie_url}${NESSIE_ICEBERG_PREFIX:-/iceberg}/v1/config" 60
|
||||
|
||||
require_component sicabe-nessie \
|
||||
"Nessie branch 'main'" \
|
||||
|
||||
Reference in New Issue
Block a user