platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
subroto
2026-10-09 02:35:14 +07:00
commit 4ca7a9a58e
108 changed files with 23035 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/lib/cluster/generate.py
+899
View File
@@ -0,0 +1,899 @@
#!/usr/bin/env python3
"""generate.py — render artefak deploy per host dari docker-compose.yml + cluster.yml.
Fase 1 R10 §4.5 (**Varian A: render, bukan overlay asli Compose**). Alasan: merge
overlay tidak dapat MENGHAPUS service, sedangkan kita perlu menyisakan hanya
service milik satu host.
Yang dihasilkan `generated/host-<x>/`:
1. `docker-compose.yml` — hanya service milik host itu.
2. `.env` — endpoint menunjuk ALAMAT HOST pemilik komponen (B1) + seluruh isi
`.env` sumber (rahasia wajib identik lintas host, S1 R10 §4.12).
3. `config/` — hasil `render-config.sh` (hostname container → alamat host), karena
berkas di `config/**` tidak dapat melakukan ekspansi env saat runtime.
`depends_on` lintas host DIBUANG (penggantinya `preflight`, Fase 2); intra-host
dipertahankan supaya urutan + healthcheck tetap berlaku di dalam satu host.
Hasil di `generated/` bersifat ARTEFAK (gitignored) — jangan disunting tangan.
Perintah:
generate.py render <inventory> <base-compose> <base-env> <out-base> <render-config.sh>
"""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import inventory # noqa: E402 (modul sebelah, sengaja)
try:
import yaml
except ImportError: # pragma: no cover
sys.stderr.write("PyYAML wajib tersedia\n")
raise SystemExit(2)
# Variabel endpoint (B1) → komponen pemiliknya. Dipakai untuk mengarahkan
# container ke ALAMAT HOST, bukan nama container (nama container hanya resolve
# di host yang sama).
ENDPOINT_VARS = {
"POSTGRES_HOST": "postgres",
"MINIO_HOST": "minio",
"KEYCLOAK_HOST": "keycloak",
"OPENMETADATA_HOST": "openmetadata-server",
"AIRFLOW_WEBSERVER_HOST": "airflow-webserver",
"STARROCKS_FE_HOST": "starrocks-fe",
"STARROCKS_CN_HOST": "starrocks-cn",
"OPENSEARCH_HOST": "openmetadata-opensearch",
# Var SISI-HOST yang belum ada padanannya, dipakai `scripts/verify-phase*` (mis. Trino
# & Grafana hanya punya *_PUBLIC_URL). Nilainya = alamat host pemilik. CATATAN:
# STARROCKS_HOST/AIRFLOW_HOST sengaja TIDAK ditulis di sini — skrip memakainya sebagai
# penanda komponen EXTERNAL (default `localhost`); mengisinya dengan alamat host akan
# salah menandai komponen lokal sebagai external.
"TRINO_HOST": "trino-coordinator",
# Dipakai dbt (profiles.yml: env_var DBT_TRINO_HOST) — lintas-host butuh ALAMAT IP.
"DBT_TRINO_HOST": "trino-coordinator",
"GRAFANA_HOST": "grafana",
"LDAP_HOST": "openldap",
"NESSIE_HOST": "nessie",
}
# Sebagian konsumen memakai bentuk URL (bukan `*_HOST`), dan berkas config yang
# dibaca container TIDAK dapat ekspansi env → URL-nya harus ikut diarahkan di sini,
# kalau tidak Trino/Sling/StarRocks lintas host akan mencari nama container yang
# hanya ada di host lain. (Komponen pemilik tetap diambil dari inventory.)
URL_ENDPOINT_VARS = {
"S3_ENDPOINT": ("minio", "http://{addr}:{port}"),
"NESSIE_URL": ("nessie", "http://{addr}:{port}"),
"LDAP_URL": ("openldap", "ldap://{addr}:{port}"),
"OPA_URL": ("opa", "http://{addr}:{port}"),
}
# Port container untuk var URL di atas — bisa ditimpa di .env (mis. layanan EXISTING
# berport non-standar). Default = port standar → perilaku lama tidak berubah.
URL_ENDPOINT_PORTS = {
"S3_ENDPOINT": ("MINIO_INTERNAL_PORT", 9000),
"NESSIE_URL": ("NESSIE_INTERNAL_PORT", 19120),
"LDAP_URL": ("OPENLDAP_INTERNAL_PORT", 389),
"OPA_URL": ("OPA_INTERNAL_PORT", 8181),
}
# URL BROWSER-FACING saat tls.mode = none (tanpa proxy/TLS): dialamatkan ke HOST yang
# MENJALANKAN komponen, bukan ke localhost maupun ke IP host lokal. Nilainya diambil
# dari host pemilik sehingga SAMA di semua bundle — wajib, karena `iss`/issuer OIDC
# harus satu nilai; kalau tiap host menulis alamatnya sendiri, validator akan tolak.
# Port di sini mengikuti skema 12xxx (sama seperti URL_ENDPOINT_VARS di bawah).
PUBLIC_URL_ENDPOINTS = {
"KEYCLOAK_PUBLIC_URL": ("keycloak", "http://{addr}:12080"),
"GRAFANA_PUBLIC_URL": ("grafana", "http://{addr}:12300"),
"OPENMETADATA_PUBLIC_URL": ("openmetadata-server", "http://{addr}:12085"),
"AIRFLOW_PUBLIC_URL": ("airflow-webserver", "http://{addr}:12082"),
"TRINO_PUBLIC_URL": ("trino-coordinator", "https://{addr}:12443"),
"MINIO_CONSOLE_PUBLIC_URL": ("minio", "http://{addr}:12901"),
}
# SENGAJA TIDAK diarahkan ke IP: KEYCLOAK_URL / TRINO_URL dan var `*_URL` lain yang
# bersifat identitas (issuer OIDC harus stabil — lihat R10 §4.10.1: pakai DNS
# internal, bukan IP). Itu milik operator dan harus sudah benar di `cluster.yml`
# (domain) / `.env`.
# Hostname container yang dikenali render-config.sh → nama komponen inventory.
RENDER_HOSTS = (
"postgres", "minio", "nessie", "keycloak", "openldap", "opa",
"trino-coordinator", "trino-worker", "starrocks-fe", "starrocks-cn",
"airflow-webserver", "openmetadata-server", "openmetadata-opensearch",
"prometheus", "grafana", "dremio", "statsd-exporter", "trino-jmx-exporter",
)
# Catatan: co-location ditangani render_config_map() — komponen SE-HOST TIDAK ditulis ke
# alamat host (biar tetap nama container, satu jaringan Docker). Maka daftar ini boleh
# lengkap; yang penting hostname-nya ada di KNOWN_HOSTS render-config.sh.
# node_exporter = PRA-SYARAT HOST (dipasang di luar installer, port 9100). Semua
# host di cluster.yml dianggap menjalankannya; generate.py menulis daftar target
# file_sd (job `node`) ke bundle host pemilik Prometheus.
NODE_EXPORTER_PORT = 9100
def load_compose(path: str) -> dict:
with open(path, encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
def keep_only(services: dict, wanted: set[str]) -> dict:
return {name: spec for name, spec in services.items() if name in wanted}
def prune_depends_on(services: dict) -> list[str]:
"""Buang `depends_on` lintas host; kembalikan daftar yang dibuang."""
dropped: list[str] = []
for name, spec in services.items():
deps = spec.get("depends_on")
if deps is None:
continue
if isinstance(deps, dict):
gone = set(deps) - set(services)
kept: object = {k: v for k, v in deps.items() if k in services}
else:
gone = set(deps) - set(services)
kept = [d for d in deps if d in services]
dropped += [f"{name} → {g}" for g in sorted(gone)]
if kept:
spec["depends_on"] = kept
else:
spec.pop("depends_on", None)
return dropped
def used_volumes(services: dict) -> set[str]:
used: set[str] = set()
for spec in services.values():
for vol in spec.get("volumes", []) or []:
if isinstance(vol, str):
src = vol.split(":", 1)[0]
elif isinstance(vol, dict):
src = vol.get("source", "")
else:
continue
if src and not src.startswith((".", "/", "~")):
used.add(src)
return used
def build_compose(base: dict, wanted: set[str]) -> tuple[dict, list[str]]:
services = keep_only(base["services"], wanted)
dropped = prune_depends_on(services)
declared = base.get("volumes") or {}
out: dict = {"services": services}
if base.get("name"):
out["name"] = base["name"]
out["volumes"] = {n: s for n, s in declared.items() if n in used_volumes(services)}
for key in ("networks", "configs", "secrets"):
if key in base:
out[key] = base[key]
return out, dropped
def write_compose(path: Path, doc: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "w", encoding="utf-8") as fh:
fh.write("# DIBANGKITKAN OTOMATIS — jangan disunting tangan.\n")
fh.write("# Sumber: docker-compose.yml + cluster.yml · regenerasi: ./install.sh --generate\n")
yaml.safe_dump(doc, fh, sort_keys=False, default_flow_style=False, allow_unicode=True)
def _is_topology_var(line: str) -> bool:
"""Baris .env yang menjadi milik INVENTORY (bukan isi .env sumber).
Kalau tidak disaring, `.env` sumber ikut menyalin `EXTERNAL_SERVICES`/
`*_HOST` milik single-host → di dotenv yang MENANG adalah yang TERAKHIR,
sehingga nilai dari inventory bisa hilang diam-diam (bug yang ditemukan saat
uji: `EXTERNAL_SERVICES` muncul dua kali).
"""
key = line.split("=", 1)[0].strip()
owned = {
"EXTERNAL_SERVICES", "LOCAL_SERVICES", "KEYCLOAK_SSL_REQUIRED",
"KC_PROXY_HEADERS", "TRINO_PROCESS_FORWARDED", "CLUSTER_DOMAIN", "TRAEFIK_BIND_ADDR",
*ENDPOINT_VARS, *URL_ENDPOINT_VARS,
*inventory.PUBLIC_URL_SERVICES, # KEYCLOAK_PUBLIC_URL, GRAFANA_PUBLIC_URL, …
}
return key in owned or key.startswith("SICABE_HOST_")
def render_env(base_env: Path, out_env: Path, host: str, spec: dict, endpoints: list[str]) -> None:
lines: list[str] = []
if base_env.is_file():
for line in base_env.read_text(encoding="utf-8").splitlines():
if _is_topology_var(line):
continue # ditentukan inventory, bukan .env sumber
lines.append(line)
else:
lines.append("# (tidak ada .env sumber — rahasia wajib dilengkapi manual)")
lines += [
"",
"# ── Ditambahkan generator multi-host (Fase 1) — variabel topologi ──",
f"SICABE_HOST_NAME={host}",
f"SICABE_HOST_ADDRESS={spec['address']}",
f"SICABE_HOST_TIERS={','.join(spec['tiers'])}",
]
lines += endpoints
out_env.parent.mkdir(parents=True, exist_ok=True)
out_env.write_text("\n".join(lines) + "\n", encoding="utf-8")
# Port KONTAINER yang harus terjangkau lintas host (Fase 2). Variabel `*_HOST`
# hanya memuat NAMA host (tanpa port), jadi port diambil dari referensi compose =
# port container (5432, 8080, …). Selama port container itu tidak terbuka di alamat
# host, container di host lain tidak bisa menjangkau — jadi generator menambahkannya.
#
# Dikunci ke ALAMAT HOST (mis. `10.20.0.12:5432:5432`), bukan `0.0.0.0`, supaya hanya
# terjangkau lewat NIC privat host itu (R10 §4.8). Ini SENGAJA hanya di artefak
# multi-host: instalasi single-host tidak pernah membuka port internal ini, dan
# `docker-compose.yml` tetap byte-identik (invarian Fase 0 terjaga).
# Eksposur untuk manusia/klien tetap memakai port host `12xxx` yang sudah ada.
INTERNAL_PORTS: dict[str, list[int]] = {
"postgres": [5432],
"minio": [9000],
"nessie": [19120],
"keycloak": [8080],
"openldap": [389],
"opa": [8181],
# 8080 = worker mendaftar (discovery) · 8443 = klien lintas host (OM ingestion,
# task dbt, native DQ) — keduanya dibutuhkan begitu coordinator & worker beda host.
"trino-coordinator": [8080, 8443],
"starrocks-fe": [9030, 8030],
"starrocks-cn": [9050, 8040, 8060],
"openmetadata-opensearch": [9200],
"openmetadata-server": [8585],
"airflow-webserver": [8080],
# Sidecar JMX di host `coord`: Prometheus (di host `governance`) men-scrape 9404
# LINTAS host → portnya WAJIB diterbitkan di alamat host pemilik.
"trino-jmx-exporter": [9404],
}
def _reachable_on_host_port(entry, cport: int, address: str) -> bool:
"""Apakah `entry` benar-benar membuka port container `cport` DI alamat host?
Jebakan yang sempat lolos: `12080:8080` memetakan port host 12080 → container
8080. Itu TIDAK membuat port 8080 terjangkau di host, sehingga container di host
lain tetap tidak bisa menjangkau. Yang dihitung hanya pemetaan yang port host-nya
SAMA dengan port container (mis. `addr:8080:8080`).
"""
if isinstance(entry, str):
parts = entry.split(":")
if len(parts) == 2:
host_ip, host_port = None, parts[0]
elif len(parts) == 3:
host_ip, host_port = parts[0], parts[1]
else:
return False
if host_port != str(cport):
return False
return host_ip in (None, "", address)
if isinstance(entry, dict):
published = entry.get("published")
return (
entry.get("target") == cport
and str(published) == str(cport)
and entry.get("host_ip") in (None, "", address)
)
return False
def tier_implied_components(placement: dict, plan: dict) -> set[str]:
"""Komponen yang PASTI dikonsumsi lintas host walau tidak terlihat di env/config.
`trino-coordinator` selalu dipakai host bertier `pipeline` (task dbt Airflow) dan
`governance` (ingestion OpenMetadata). Referensinya tidak dapat ditemukan dengan
pemindaian env/config karena `ingest-trino.yaml` **disalin ke container saat
runtime** (docker cp), bukan di-mount — celah ini ditemukan saat menguji preflight.
"""
needed: set[str] = set()
owners = placement.get("trino-coordinator") or []
if not owners:
return needed
owner = owners[0]
for host, spec in plan.items():
if host == owner:
continue
if {"pipeline", "governance"} & set(spec["tiers"]):
needed.add("trino-coordinator")
break
return needed
def config_referenced_components(deploy_dir: Path, base: dict, plan: dict,
placement: dict, known: set[str]) -> set[str]:
"""Komponen yang dirujuk dari file CONFIG yang di-mount oleh service di host lain.
Deteksi berbasis environment saja tidak cukup: Trino tidak memakai `*_HOST` untuk
OAuth2, melainkan `config/trino/config-coordinator.properties`
(`http://keycloak:8080/...`). Tanpa pemindaian ini, port Keycloak tidak dibuka dan
Trino di host lain gagal mengambil JWKS.
"""
import re
needed: set[str] = set()
for host, spec in plan.items():
for svc in spec["services"]:
for vol in base["services"].get(svc, {}).get("volumes") or []:
src = vol.split(":", 1)[0] if isinstance(vol, str) else vol.get("source", "")
if not src.startswith("./config/"):
continue
path = deploy_dir / src[2:]
files = list(path.rglob("*")) if path.is_dir() else [path]
for f in files:
if not f.is_file():
continue
try:
text = f.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
for comp in known:
if re.search(r"(?<![\w.-])" + re.escape(comp) + r"(?![\w-])", text):
owners = placement.get(comp) or []
if owners and host not in owners:
needed.add(comp)
return needed
def cross_host_components(base: dict, placement: dict, plan: dict) -> set[str]:
"""Komponen yang PUNYA konsumen di host lain → perlu port internal terbuka.
Tanpa perhitungan ini generator membuka port untuk semua komponen, dan di
blueprint a3 itu langsung menabrak: `keycloak` dan `airflow-webserver` sama-sama
meminta `<control>:8080` (dua service, satu port host) → `compose up` gagal
'port is already allocated'. Padahal `airflow-webserver` hanya dikonsumsi
OpenMetadata yang berada di host yang sama → tidak perlu port host sama sekali.
Bonus keamanan: port internal yang tidak dibutuhkan tidak pernah dibuka.
"""
var_to_component = {v: c for v, c in ENDPOINT_VARS.items()}
var_to_component.update({v: c for v, (c, _t) in URL_ENDPOINT_VARS.items()})
needed: set[str] = set()
for host, spec in plan.items():
for svc in spec["services"]:
source = base["services"].get(svc, {})
text = json.dumps(source.get("environment") or {})
for var, component in var_to_component.items():
if var not in text:
continue
owners = placement.get(component) or []
if owners and host not in owners:
needed.add(component)
return needed
def required_cross_host(base: dict, deploy_dir: Path, plan: dict, placement: dict) -> set[str]:
"""Gabungan SEMUA alasan sebuah komponen butuh port lintas host.
Satu fungsi supaya generator dan preflight TIDAK bisa berbeda pendapat — memisahkan
perhitungan ini sempat membuat preflight melewatkan trino-coordinator begitu aturan
baru ditambahkan ke generator.
"""
known = set(base["services"]) & set(INTERNAL_PORTS)
return (
cross_host_components(base, placement, plan)
| config_referenced_components(deploy_dir, base, plan, placement, known)
| tier_implied_components(placement, plan)
)
def add_internal_ports(services: dict, address: str, needed: set[str]) -> list[str]:
"""Tambahkan binding port internal ke alamat host; kembalikan yang ditambahkan."""
added: list[str] = []
for name, spec in services.items():
if name not in needed:
continue
want = INTERNAL_PORTS.get(name)
if not want:
continue
ports = spec.setdefault("ports", [])
for cport in want:
if any(_reachable_on_host_port(e, cport, address) for e in ports):
continue
ports.append(f"{address}:{cport}:{cport}")
added.append(f"{name}:{cport}")
return added
def component_placements(doc: dict) -> dict[str, list[str]]:
return inventory.resolve(doc)
def extra_bind_dirs(base: dict) -> list[str]:
"""Bind-mount relatif selain `config/` yang harus ikut disalin ke tiap host.
Contoh: `./scripts/airflow:/opt/airflow/scripts:ro`. Tanpa disalin, direktori
generated tidak mandiri — Docker akan membuat direktori KOSONG di host tujuan
(skrip Airflow hilang diam-diam). Ditemukan oleh test-generate.sh.
"""
found: set[str] = set()
for spec in base["services"].values():
for vol in spec.get("volumes", []) or []:
if isinstance(vol, str):
src = vol.split(":", 1)[0]
elif isinstance(vol, dict):
src = vol.get("source", "")
else:
continue
if src.startswith("./") and not src.startswith("./config/"):
found.add(src[2:])
return sorted(found)
def copy_extra_binds(deploy_dir: Path, host_dir: Path, paths: list[str]) -> list[str]:
"""Salin bind relatif non-config (mis. `scripts/airflow`) ke direktori host."""
import shutil
copied: list[str] = []
for rel in paths:
src = deploy_dir / rel
if not src.exists():
continue
dst = host_dir / rel
dst.parent.mkdir(parents=True, exist_ok=True)
if src.is_dir():
shutil.copytree(src, dst, dirs_exist_ok=True)
else:
shutil.copy2(src, dst)
copied.append(rel)
return copied
def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]:
"""Baris `VAR=alamat|nama` untuk host ini.
SE-HOST → **nama service** (satu jaringan Docker). Port container TIDAK diterbitkan
di host (mis. OM 8585), jadi menulis alamat host justru "connection refused"
(regresi: Airflow/OpenLineage → `http://<ip-control>:8585/api`). Beda host → alamat
host pemilik (port internal diterbitkan generator untuk itu).
"""
placement = component_placements(doc)
lines: list[str] = []
for var, component in ENDPOINT_VARS.items():
owners = placement.get(component)
if not owners:
continue
if host in owners:
lines.append(f"{var}={component}") # SE-HOST → nama service
else:
lines.append(f"{var}={plan[owners[0]]['address']}") # LINTAS host → alamat
disabled = [n for n, v in inventory.enabled_components(doc).items() if v is False]
if disabled:
# Komponen `enabled: false` = tidak dijalankan di host mana pun → installer
# harus melewatinya (install.sh: EXTERNAL_SERVICES menang atas auto-deteksi).
lines.append('EXTERNAL_SERVICES="%s"' % " ".join(disabled))
return lines
def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]:
"""Baris `*_PUBLIC_URL=...` — URL yang dilihat BROWSER (issuer OIDC, redirect, UI),
bukan lalu lintas container.
tls.mode != none : `https://<label>.<domain>` (inventory.public_urls).
tls.mode == none : `http(s)://<alamat HOST PEMILIK>:<port>`. Sebelumnya mode ini
mengembalikan daftar kosong, sehingga .env host memakai default
`http://localhost:<port>` — dan browser di host lain dibuang ke
localhost (issuer OIDC pun localhost). Sekarang alamatnya diambil
dari host yang menjalankan komponen, jadi akses lintas host benar
tanpa perlu nilai berbeda per bundle.
"""
by_domain = inventory.public_urls(doc)
if by_domain:
return [f"{var}={url}" for var, url in sorted(by_domain.items())]
if plan is None:
return []
placement = component_placements(doc)
resolved: dict[str, str] = {}
for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items():
owners = placement.get(component)
if not owners:
continue
# Sengaja memakai alamat PEMILIK (bukan host lokal) dan urutan yang pasti,
# supaya nilainya identik di setiap bundle.
resolved[var] = tmpl.format(addr=plan[sorted(owners)[0]]["address"])
return [f"{var}={url}" for var, url in sorted(resolved.items())]
def tls_policy_lines(doc: dict) -> list[str]:
"""Kebijakan keamanan turunan `tls.mode` (saat ini hanya realm Keycloak).
Keycloak DEFAULT `sslRequired=external` menuntut HTTPS untuk permintaan dari alamat
NON-lokal. Alamat Tailscale `100.64.0.0/10` (CGNAT) TERMASUK non-lokal bagi Keycloak
(`isLocal` = loopback + RFC1918 + link-local + `fc00::/7`), sehingga admin console di
`http://<alamat-tailscale>:12080` gagal dengan "HTTPS required" — dan Keycloak 26
TIDAK punya opsi server-level untuk ini (tidak ada `KC_HTTPS_REQUIRED`), hanya per
realm. Karena itu nilainya diteruskan ke bootstrap.sh.
tls.mode `none` -> `none` (HTTP polos; instalasi dev/simulasi)
tls.mode `edge`/`full` -> `external` (ada Traefik/TLS di depan)
"""
edge = inventory.tls_mode(doc) != "none"
return [
f"KEYCLOAK_SSL_REQUIRED={'external' if edge else 'none'}",
# Di belakang Traefik, Keycloak WAJIB memercayai X-Forwarded-* — tanpa itu
# permintaan HTTPS dari proxy dianggap HTTP (redirect salah / tuntutan HTTPS).
# Sengaja KOSONG saat tanpa proxy: memasangnya membuat Keycloak memercayai
# header yang bisa dipalsukan klien.
f"KC_PROXY_HEADERS={'xforwarded' if edge else ''}",
# Trino: X-Forwarded-Proto supaya skema publik benar saat di belakang proxy.
f"TRINO_PROCESS_FORWARDED={'true' if edge else 'false'}",
]
# Rute untuk komponen publik yang TIDAK berjalan di host ingress. Traefik hanya melihat
# container di host-nya sendiri, jadi komponen di host lain dirutekan lewat berkas
# dinamis (provider file) ke PORT HOST-nya (12xxx), bukan port container.
# Nilai: (label rute, URL, butuh serverstransport-insecure?)
REMOTE_ROUTE_TARGETS = {
"keycloak": ("kc", "http://{addr}:12080", False),
"grafana": ("grafana", "http://{addr}:12300", False),
"openmetadata-server": ("om", "http://{addr}:12085", False),
"airflow-webserver": ("airflow", "http://{addr}:12082", False),
"trino-coordinator": ("trino", "https://{addr}:12443", True),
"minio": ("minio", "http://{addr}:12901", False),
}
def ingress_host(doc: dict, plan: dict) -> str:
"""Host yang menjalankan ingress Traefik (T-20).
Default: host yang menjalankan `keycloak` (tier identity — pada preset a3 itu
`control`), supaya tidak perlu setelan tambahan. Bisa dipatok eksplisit lewat
`tls.ingress_host`. Kosong bila tls.mode `none` (memang tanpa ingress).
"""
if inventory.tls_mode(doc) == "none":
return ""
explicit = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
if explicit:
return explicit if explicit in plan else ""
owners = sorted(component_placements(doc).get("keycloak") or [])
if owners:
return owners[0]
return sorted(plan)[0] if plan else ""
def remote_route_doc(doc: dict, plan: dict) -> dict | None:
"""Isi berkas dinamis untuk rute komponen yang ada DI HOST LAIN.
Mengembalikan None bila tidak ada rute jarak jauh (semua komponen publik memang
berada di host ingress).
"""
ing = ingress_host(doc, plan)
if not ing:
return None
domain = inventory.cluster_domain(doc)
placement = component_placements(doc)
routers: dict = {}
services: dict = {}
for component, (label, tmpl, needs_insecure) in REMOTE_ROUTE_TARGETS.items():
owners = sorted(placement.get(component) or [])
if not owners or ing in owners:
continue # sudah ditangani label container di host ini
owner = owners[0]
router = {
"rule": f"Host(`{label}.{domain}`)",
"entryPoints": ["websecure"],
"service": label,
"tls": {
"certResolver": "le",
"domains": [{"main": domain, "sans": [f"*.{domain}"]}],
},
}
lb: dict = {"servers": [{"url": tmpl.format(addr=plan[owner]["address"])}]}
if needs_insecure:
lb["serversTransport"] = "trino-insecure"
routers[label] = router
services[label] = {"loadBalancer": lb}
if not routers:
return None
# serversTransport wajib ikut di berkas ini: berkas dinamis yang dibangkitkan
# MENGGANTIKAN salinan dasar, jadi definisinya harus ada di sini juga.
return {
"http": {
"serversTransports": {"trino-insecure": {"insecureSkipVerify": True}},
"routers": routers,
"services": services,
}
}
def node_targets_doc(plan: dict) -> list[dict]:
"""Berkas target file_sd untuk job `node`: satu entri `<alamat>:9100` per host.
node_exporter adalah PRA-SYARAT HOST (di luar installer), sehingga SEMUA host di
cluster.yml dianggap menjalankannya di port 9100. Label `host` disalin ke
`instance` oleh relabel_configs di prometheus.yml (panel menampilkan nama host).
"""
return [
{"targets": [f"{spec['address']}:{NODE_EXPORTER_PORT}"], "labels": {"host": name}}
for name, spec in sorted(plan.items())
]
def ingress_lines(doc: dict, host: str, plan: dict) -> list[str]:
"""Variabel ingress untuk .env host (T-20).
`CLUSTER_DOMAIN` dipakai label rute Traefik (Host(`<label>.<domain>`)), dan
`TRAEFIK_BIND_ADDR` membatasi port 80/443 ke alamat host ini — bukan semua antarmuka.
Hanya ditulis saat ada TLS/ingress; tanpa itu Traefik memang tidak dijalankan.
"""
if inventory.tls_mode(doc) == "none":
return []
if host != ingress_host(doc, plan):
return []
return [
f"CLUSTER_DOMAIN={inventory.cluster_domain(doc)}",
f"TRAEFIK_BIND_ADDR={plan[host]['address']}",
]
def _endpoint_port(var, env_path):
"""Port container untuk var URL (dari .env, fallback default URL_ENDPOINT_PORTS)."""
spec = URL_ENDPOINT_PORTS.get(var)
if not spec:
return ""
env_name, default = spec
val = _source_env_value(env_path, env_name) if env_path else None
return val if val else str(default)
def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[str]:
"""Baris `VAR=<url>` untuk konsumen bentuk URL.
SE-HOST → **nama service** (mis. `http://nessie:19120`, `http://opa:8181`);
LINTAS host → alamat IP pemilik (port internalnya diterbitkan generator).
Konsisten dengan endpoint_lines(): port container tak diterbitkan di host.
Port diambil dari .env (mis. NESSIE_INTERNAL_PORT) bila di-override.
"""
placement = component_placements(doc)
lines: list[str] = []
for var, (component, template) in URL_ENDPOINT_VARS.items():
owners = placement.get(component)
if not owners:
continue
addr = component if host in owners else plan[owners[0]]["address"]
lines.append(f"{var}={template.format(addr=addr, port=_endpoint_port(var, env_path))}")
return lines
def render_config_map(doc: dict, host: str, plan: dict) -> list[str]:
"""Peta hostname container → alamat, untuk render-config.sh (khusus host ini).
HANYA komponen yang dimiliki HOST LAIN yang dipetakan. Komponen SE-HOST dibiarkan
memakai NAMA container (satu jaringan Docker) — menulisnya ke alamat host justru
RUSAK: port container sering tak diterbitkan di host. Regresi nyata: `prometheus`
(se-host dengan grafana) ditulis `http://<ip-host>:9090` di datasources.yaml →
Grafana "connection refused"; `statsd-exporter` (se-host dengan prometheus) serupa.
"""
placement = component_placements(doc)
args: list[str] = []
for component in RENDER_HOSTS:
owners = placement.get(component)
if not owners:
continue
if host in owners:
continue # SE-HOST → biarkan nama container
args += ["--map", f"{component}={plan[owners[0]]['address']}"]
return args
def cmd_render(argv: list[str]) -> int:
if len(argv) < 7:
sys.stderr.write(__doc__ or "")
return 2
inv_path, compose_path, env_path, out_base, renderer = argv[2:7]
preset_override = argv[7] if len(argv) > 7 else ""
doc = inventory.load(inv_path)
if preset_override:
doc["preset"] = preset_override
doc = inventory.apply_preset(doc)
errors = inventory.validate(doc)
if errors:
sys.stderr.write("cluster.yml tidak valid — jalankan 'inventory.py validate' untuk detail:\n")
for e in errors:
sys.stderr.write(f" ✗ {e}\n")
return 1
base = load_compose(compose_path)
extra_binds = extra_bind_dirs(base)
plan = inventory.plan(doc)
out_root = Path(out_base)
placement = component_placements(doc)
needed = required_cross_host(base, Path(compose_path).parent, plan, placement)
total_dropped: dict[str, list[str]] = {}
port_internal_all: set[str] = set()
# Trino cert/keystore: dibangkitkan SEKALI (SAN = ALAMAT HOST semua host), lalu
# disalin IDENTIK ke tiap bundle. WAJIB sekali — kalau tiap host beda keypair, cert
# klien (OM ingestion, dll.) tak cocok dgn cert server → verifikasi TLS gagal.
#
# DIPERSISTENKAN di deploy/.trino-keystore/ (gitignored): kalau cert dibuat BARU tiap
# `--generate`, server Trino yang belum di-recreate menyajikan cert LAMA → klien
# (GE/OM) "self-signed certificate". Stabil kecuali daftar SAN (alamat host) berubah.
_ks_cache = Path(__file__).resolve().parents[2] / ".trino-keystore"
_ks_cache.mkdir(parents=True, exist_ok=True)
_trino_ks_src = _ks_cache
_san = ",".join(f"IP:{plan[h]['address']}" for h in sorted(plan) if plan[h].get("address"))
_ks_script = Path(renderer).resolve().parent / "generate-trino-keystore.sh"
_marker = _ks_cache / "san.txt"
_need_regen = (not (_ks_cache / "keystore.p12").is_file()
or not _marker.is_file()
or _marker.read_text(encoding="utf-8").strip() != _san)
if _san and _ks_script.is_file() and _need_regen:
subprocess.run([str(_ks_script), "--out", str(_trino_ks_src), "--san-extra", _san],
capture_output=True, text=True)
try:
_marker.write_text(_san, encoding="utf-8")
except OSError:
pass
for host, spec in plan.items():
host_dir = out_root / f"host-{host}"
# 1) compose
# Ingress: setiap host menjalankan Traefik-nya sendiri saat ada TLS/proxy
# (satu sertifikat per host — acme.json tidak bisa dibagi antar host).
# Tanpa TLS, service traefik memang tidak ikut.
wanted = set(spec["services"])
# Ingress satu host: HANYA host ingress yang menjalankan Traefik. Ini juga
# menghilangkan perebutan record TXT DNS-01 (dulu tiap host menerbitkan
# wildcard yang sama) dan menekan pemakaian batas sertifikat Let's Encrypt.
ing = ingress_host(doc, plan)
if ing and host == ing:
wanted.add("traefik")
compose_out, dropped = build_compose(base, wanted)
# Fase 2: buka port internal ke alamat host ini agar komponen di host lain
# dapat menjangkau (pengganti ketiadaan satu bridge network lintas host).
added_ports = add_internal_ports(compose_out["services"], spec["address"], needed)
port_internal_all.update(added_ports)
write_compose(host_dir / "docker-compose.yml", compose_out)
total_dropped[host] = dropped
# 2) .env
render_env(
Path(env_path), host_dir / ".env", host, spec,
endpoint_lines(doc, host, plan)
+ url_endpoint_lines(doc, host, plan, env_path)
+ public_url_lines(doc, plan)
+ tls_policy_lines(doc)
+ ingress_lines(doc, host, plan),
)
# 3) config ter-render
# 3a) bind relatif non-config (mis. scripts/airflow) → bundle mandiri
copied = copy_extra_binds(Path(compose_path).parent, host_dir, extra_binds)
# 3b) config ter-render melalui render-config.sh
# Bersihkan target berkas runtime yang mungkin berwujud DIREKTORI (sisa `up`
# Docker saat sumber bind-mount belum ada). Tanpa ini render-config GAGAL
# ("tulis gagal") dan generator berhenti SEBELUM menulis berkas runtime.
for _rel in ("airflow/simple_auth_manager_passwords.json",
"trino/password.db"):
_t = host_dir / "config" / _rel
if _t.is_dir():
shutil.rmtree(_t)
rc = subprocess.run(
[renderer, "--out", str(host_dir / "config"), *render_config_map(doc, host, plan)],
capture_output=True, text=True,
)
# Berkas dinamis rute komponen di host lain (hanya di host ingress).
# Ditulis SETELAH render-config karena berkas ini menggantikan salinan dasar.
remote = remote_route_doc(doc, plan)
if remote is not None and host == ing:
(host_dir / "config" / "traefik").mkdir(parents=True, exist_ok=True)
with open(host_dir / "config" / "traefik" / "dynamic.yml", "w", encoding="utf-8") as fh:
fh.write("# DIBANGKITKAN OTOMATIS oleh install.sh --generate — jangan disunting tangan.\n")
fh.write("# Ingress satu host: rute untuk komponen yang TIDAK ada di host ini.\n")
yaml.safe_dump(remote, fh, sort_keys=False, allow_unicode=True)
# Target node_exporter (job `node`, file_sd) — hanya di host pemilik
# Prometheus. Ditulis SETELAH render-config karena menggantikan salinan dasar.
if host in (placement.get("prometheus") or []):
tdir = host_dir / "config" / "prometheus" / "targets"
tdir.mkdir(parents=True, exist_ok=True)
with open(tdir / "node.yml", "w", encoding="utf-8") as fh:
fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n")
fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n")
yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True)
if rc.returncode != 0:
sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}")
return 1
# Salin cert/keystore Trino (dibangkitkan SEKALI dgn SAN IP semua host) ke bundle
# ini — keypair/cert IDENTIK lintas host agar verifikasi TLS server↔klien lolos.
if (_trino_ks_src / "keystore.p12").is_file():
(host_dir / "config" / "trino" / "certs").mkdir(parents=True, exist_ok=True)
shutil.copy2(_trino_ks_src / "keystore.p12",
host_dir / "config" / "trino" / "keystore.p12")
shutil.copy2(_trino_ks_src / "certs" / "trino.crt",
host_dir / "config" / "trino" / "certs" / "trino.crt")
# Berkas runtime gitignored (Airflow auth, Trino password.db) tidak ikut
# render-config → tulis eksplisit agar compose tak me-mount sumber hilang (DIRECTORY).
_write_bundle_runtime_files(host_dir, Path(env_path))
print(f"host-{host:9s} {spec['address']:15s} services={len(spec['services'])} "
f"depends_on_dibuang={len(dropped)} bind_disalin={len(copied)} "
f"port_internal={len(added_ports)}")
print(f"komponen butuh port lintas host: {', '.join(sorted(needed)) or '—'}")
if port_internal_all:
print(f"port internal dibuka: {len(port_internal_all)} ("
+ ", ".join(sorted(port_internal_all)) + ")")
# ringkasan untuk di-review
for host, dropped in total_dropped.items():
for item in dropped:
print(f" [host-{host}] depends_on lintas host dibuang: {item}")
print(json.dumps({h: len(s["services"]) for h, s in plan.items()}, sort_keys=True))
return 0
def _source_env_value(env_path, key):
"""Ambil satu nilai dari .env sumber (kutip luar dibuang). None bila tak ada."""
try:
text = Path(env_path).read_text(encoding="utf-8")
except OSError:
return None
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
name, _, value = line.partition("=")
if name.strip() != key:
continue
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
return value
return None
def _write_bundle_runtime_files(host_dir, env_path):
"""Tulis berkas runtime yang TIDAK di-track git ke bundle host.
compose generated me-mount path ini sebagai FILE. Bila file tidak ada, Docker
membuat DIRECTORY di host → service gagal (Airflow SimpleAuthManager:
IsADirectoryError; Trino: password.db bukan file). ``render-config`` tidak
menyalinnya (gitignored), jadi harus ditulis eksplisit di sini.
"""
cfg = host_dir / "config"
# Dir bind-mount yang ABSEN di git (config/airflow/sling hanya memuat env.yaml yang
# gitignored) → Docker (daemon root) membuatnya sebagai ROOT saat `up` → operator
# non-root tak bisa menulis env.yaml (bootstrap gagal). Pastikan ada & user-owned.
(cfg / "airflow" / "sling").mkdir(parents=True, exist_ok=True)
# 1) Airflow SimpleAuthManager — {"admin": <password>} (mode a+ → wajib writable)
pw_json = cfg / "airflow" / "simple_auth_manager_passwords.json"
pw_json.parent.mkdir(parents=True, exist_ok=True)
if pw_json.is_dir():
shutil.rmtree(pw_json)
admin_pw = _source_env_value(env_path, "AIRFLOW_ADMIN_PASSWORD")
if admin_pw is not None:
pw_json.write_text(json.dumps({"admin": admin_pw}) + "\n", encoding="utf-8")
elif not pw_json.is_file():
pw_json.write_text("{}\n", encoding="utf-8")
pw_json.chmod(0o666)
# 2) Trino password.db — byte-identik dari sumber bila ada; kalau tidak, FILE kosong
# (yang penting BUKAN direktori; bootstrap mengisinya).
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
dst_db = cfg / "trino" / "password.db"
dst_db.parent.mkdir(parents=True, exist_ok=True)
if dst_db.is_dir():
shutil.rmtree(dst_db)
if src_db.is_file():
shutil.copy2(src_db, dst_db)
elif not dst_db.is_file():
dst_db.write_text("", encoding="utf-8")
def main(argv: list[str]) -> int:
if len(argv) < 2:
sys.stderr.write(__doc__ or "")
return 2
if argv[1] == "render":
return cmd_render(argv)
sys.stderr.write(f"perintah tak dikenal: {argv[1]}\n")
return 2
if __name__ == "__main__":
raise SystemExit(main(sys.argv))
deploy/lib/cluster/inventory.py
+365
View File
@@ -0,0 +1,365 @@
#!/usr/bin/env python3
"""inventory.py — cluster.yml: parse, validasi, resolusi penempatan (Fase 1 multi-host).
Ini otak Fase 1 (R10 §4.3/§4.5, R11 §3–§5). Satu implementasi otoritatif dipakai
oleh `lib/cluster/generate.sh` dan `install.sh`; tidak ada logika penempatan yang
diduplikasi di shell.
Model:
- Tier (7): state, identity, policy, coord, engine, pipeline, governance.
Tiap service compose dipetakan ke tepat satu tier (TIER_SERVICES).
- Preset (a3/b5) hanya mengisi DEFAULT `hosts.<h>.tiers`; komponen diturunkan
dari tier-nya. Override eksplisit `components.<c>.host` menang.
- Tier `engine` boleh >1 host (scale-out): komponennya muncul di SEMUA host
bertier engine. Tier lain tepat 1 host.
Perintah:
inventory.py validate <cluster.yml> # exit 0 = valid; cetak ringkasan/error
inventory.py hosts <cluster.yml> # daftar host
inventory.py plan <cluster.yml> # JSON: host -> {address, tiers, services}
inventory.py components <cluster.yml> # komponen -> host (untuk dokumentasi)
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
try:
import yaml
except ImportError: # pragma: no cover
sys.stderr.write("PyYAML wajib tersedia (python3 -c 'import yaml')\n")
raise SystemExit(2)
# ─── Taksonomi (R11 §3.1) ───────────────────────────────────────────────────
# Setiap service compose milik TEPAT SATU tier.
TIER_SERVICES: dict[str, list[str]] = {
"state": ["postgres", "minio", "minio-init", "openmetadata-opensearch"],
"identity": ["keycloak", "openldap", "nessie"],
"policy": ["opa"],
"coord": ["trino-coordinator", "starrocks-fe", "trino-jmx-exporter"],
"engine": ["trino-worker", "starrocks-cn", "dremio"],
"pipeline": [
"airflow-volume-init",
"airflow-init",
"airflow-webserver",
"airflow-dag-processor",
"airflow-scheduler",
],
# `prometheus` menyusul R11 (belum ada di taksonomi 7-tier). Ditempatkan di
# `governance`: pada blueprint A maupun B ia berada di host yang sama, jadi
# granularitas tier tidak mengubah penempatan.
# `statsd-exporter` (M2 observabilitas) ikut `governance` bersama Prometheus;
# pada a3/b5 tier ini se-host dengan `pipeline` (Airflow), sehingga UDP
# StatsD Airflow→statsd-exporter tetap lokal.
"governance": ["openmetadata-migrate", "openmetadata-server", "grafana", "prometheus", "statsd-exporter"],
}
VALID_TIERS = tuple(TIER_SERVICES)
# ─── TLS & URL publik (langkah 1: hanya skema/var; BELUM mengaktifkan TLS) ──
# `none` = perilaku sekarang (http, IP+port 12xxx). `edge` = browser memakai nama
# domain lewat ingress, bidang internal tetap plaintext. `full` = seluruh segmen TLS.
# Default `none` supaya TANPA cluster.yml perilaku single-host tidak berubah.
TLS_MODES = ("none", "edge", "full")
# Layanan yang punya URL BROWSER-FACING → label subdomain di bawah `domain`.
# Dipakai juga oleh ingress (step 2) sebagai daftar rute Host(), jadi nama di sini
# adalah satu-satunya sumber kebenaran penamaan.
PUBLIC_URL_SERVICES: dict[str, str] = {
"KEYCLOAK_PUBLIC_URL": "kc",
"GRAFANA_PUBLIC_URL": "grafana",
"OPENMETADATA_PUBLIC_URL": "om",
"AIRFLOW_PUBLIC_URL": "airflow",
"TRINO_PUBLIC_URL": "trino",
"MINIO_CONSOLE_PUBLIC_URL": "minio",
}
# Blueprint: default tiers per host (R11 §4.1/§4.2).
PRESETS: dict[str, dict[str, list[str]]] = {
"a3": {
"control": ["identity", "policy", "pipeline", "governance"],
"compute": ["coord", "engine"],
"state": ["state"],
},
"b5": {
"control": ["identity", "policy", "pipeline", "governance"],
"coord": ["coord"],
"engine-1": ["engine"],
"engine-2": ["engine"],
"state": ["state"],
},
}
# Komponen stateful: tidak boleh berpindah host tanpa migrasi volume (R10 R7).
PINNED = ("postgres", "minio", "openmetadata-opensearch", "starrocks-fe", "starrocks-cn", "dremio")
def tier_of(service: str) -> str | None:
"""Tier pemilik sebuah service compose."""
for tier, services in TIER_SERVICES.items():
if service in services:
return tier
return None
def load(path: str) -> dict:
try:
with open(path, encoding="utf-8") as fh:
return yaml.safe_load(fh) or {}
except yaml.YAMLError as exc:
raise SystemExit(f"cluster.yml bukan YAML yang sah: {exc}") from exc
def apply_preset(doc: dict) -> dict:
"""Isi `hosts` dari preset bila belum didefinisikan eksplisit.
Preset tak dikenal TIDAK menghentikan program di sini — biarkan `validate()`
yang melaporkannya (dulu ini KeyError → traceback, bukan pesan yang berguna).
"""
preset = doc.get("preset") or "none"
if preset not in PRESETS:
return doc
hosts = doc.setdefault("hosts", {})
for name, tiers in PRESETS[preset].items():
hosts.setdefault(name, {})
hosts[name].setdefault("tiers", list(tiers))
return doc
def resolve(doc: dict) -> dict[str, list[str]]:
"""Komponen → daftar host (tier `engine` bisa >1 host)."""
hosts = doc.get("hosts") or {}
tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs}
explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()}
placement: dict[str, list[str]] = {}
for service, tier in tiers_by_service.items():
if service in explicit and explicit[service]:
placement[service] = [explicit[service]]
continue
owners = [h for h, spec in hosts.items() if tier in (spec.get("tiers") or [])]
if owners:
placement[service] = sorted(owners)
return placement
def tls_mode(doc: dict) -> str:
"""Mode TLS dari inventory (`none` bila tidak ditulis)."""
return (doc.get("tls") or {}).get("mode") or "none"
def cluster_domain(doc: dict) -> str:
return ((doc.get("cluster") or {}).get("domain") or "").strip()
def public_urls(doc: dict) -> dict[str, str]:
"""URL browser-facing turunan dari tls.mode + domain.
Kosong bila mode `none` — dalam mode itu pemakaian tetap IP+port 12xxx seperti
sekarang, sehingga perilaku single-host TIDAK berubah.
"""
mode = tls_mode(doc)
domain = cluster_domain(doc)
if mode == "none" or not domain:
return {}
scheme = "https" if mode in ("edge", "full") else "http"
return {var: f"{scheme}://{label}.{domain}" for var, label in PUBLIC_URL_SERVICES.items()}
def enabled_components(doc: dict) -> dict[str, bool]:
return {
name: bool(spec.get("enabled", True))
for name, spec in (doc.get("components") or {}).items()
}
def validate(doc: dict) -> list[str]:
"""Validasi R10 §4.3. Mengembalikan daftar error (kosong = valid)."""
errors: list[str] = []
preset = doc.get("preset") or "none"
if preset not in ("none", *PRESETS):
errors.append(f"preset '{preset}' tidak dikenal (pakai: none|{'|'.join(PRESETS)})")
# TLS & domain (langkah 1)
mode = (doc.get("tls") or {}).get("mode") or "none"
if mode not in TLS_MODES:
errors.append(f"tls.mode '{mode}' tidak dikenal (pakai: {'|'.join(TLS_MODES)})")
domain = ((doc.get("cluster") or {}).get("domain") or "").strip()
if mode != "none":
if not domain:
errors.append(f"tls.mode '{mode}' memerlukan cluster.domain (mis. sicabe.com)")
elif "://" in domain or "/" in domain or "." not in domain:
errors.append(f"cluster.domain '{domain}' tidak sah — isi nama domain saja "
f"(mis. sicabe.com), tanpa skema atau garis miring")
# Ingress satu host (T-20): opsional — default host yang menjalankan keycloak.
ing = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
if ing:
if mode == "none":
errors.append("tls.ingress_host tidak bermakna bila tls.mode 'none' "
"(tanpa Traefik/TLS) — hapus atau ubah tls.mode")
elif ing not in (doc.get("hosts") or {}):
errors.append(f"tls.ingress_host '{ing}' bukan nama host yang dikenal "
f"(host: {', '.join(sorted((doc.get('hosts') or {})))})")
hosts = doc.get("hosts") or {}
if not hosts:
errors.append("'hosts' kosong — inventory harus mendefinisikan minimal satu host")
return errors
# (f) tiap host punya address + tier yang sah
for name, spec in hosts.items():
if not spec.get("address"):
errors.append(f"host '{name}': 'address' wajib diisi")
tiers = spec.get("tiers") or []
if not tiers:
errors.append(f"host '{name}': minimal satu tier")
for tier in tiers:
if tier not in VALID_TIERS:
errors.append(f"host '{name}': tier '{tier}' tidak dikenal (sah: {', '.join(VALID_TIERS)})")
# (d) tier state tepat 1 host · R11 aturan 2 (state selalu host sendiri)
state_hosts = [h for h, s in hosts.items() if "state" in (s.get("tiers") or [])]
if len(state_hosts) != 1:
errors.append(
f"tier 'state' harus di TEPAT 1 host, ditemukan {len(state_hosts)}: "
f"{state_hosts or '—'} (I/O-bound & pinned, R11 §3.2)"
)
# (3) tier coord tepat 1 host (OSS Trino = 1 coordinator)
coord_hosts = [h for h, s in hosts.items() if "coord" in (s.get("tiers") or [])]
if len(coord_hosts) != 1:
errors.append(f"tier 'coord' harus di TEPAT 1 host, ditemukan {len(coord_hosts)}: {coord_hosts or '—'}")
# engine: boleh >1 (scale-out) — hanya info bila tidak ada
if not [h for h, s in hosts.items() if "engine" in (s.get("tiers") or [])]:
errors.append("tidak ada host bertier 'engine' — tidak ada tempat eksekusi Trino/StarRocks")
# anti-affinity R11 §3.2: state ⊥ engine
for h, s in hosts.items():
t = set(s.get("tiers") or [])
if "state" in t and "engine" in t:
errors.append(f"host '{h}': tier 'state' dan 'engine' tidak boleh satu host (anti-affinity)")
# (a) komponen menunjuk host yang ada · (c) endpoint hanya ke komponen enabled
components = doc.get("components") or {}
for name, spec in components.items():
host = spec.get("host")
if host and host not in hosts:
errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'")
if tier_of(name) is None and name not in ("dremio",):
errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)")
# (c) tiap komponen harus bisa ditempatkan
placement = resolve(doc)
for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs):
if service not in placement:
errors.append(
f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier "
f"'{tier_of(service)}' dan tidak ada components.{service}.host"
)
# stateful: cegah pindah host tanpa migrasi (peringatan, bukan error)
return errors
def plan(doc: dict) -> dict[str, dict]:
"""host → {address, tiers, services} untuk generator."""
hosts = doc.get("hosts") or {}
placement = resolve(doc)
enabled = enabled_components(doc)
result: dict[str, dict] = {}
for name, spec in hosts.items():
result[name] = {
"address": spec.get("address"),
"tiers": spec.get("tiers") or [],
"services": [],
}
for service, owners in placement.items():
if enabled.get(service) is False:
continue
for owner in owners:
if owner in result:
result[owner]["services"].append(service)
for name in result:
result[name]["services"].sort()
return result
def warnings(doc: dict) -> list[str]:
out: list[str] = []
placement = resolve(doc)
for service in PINNED:
hosts = placement.get(service) or []
if len(hosts) > 1:
out.append(f"'{service}' stateful berada di >1 host {hosts} — butuh volume terpisah per host")
return out
def cmd_validate(path: str) -> int:
doc = apply_preset(load(path))
errors = validate(doc)
warns = warnings(doc)
if errors:
print(f"cluster.yml TIDAK valid ({len(errors)} masalah):")
for e in errors:
print(f" ✗ {e}")
return 1
p = plan(doc)
print(f"cluster.yml valid — preset '{doc.get('preset') or 'none'}'")
for host, spec in p.items():
print(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}] "
f"services={len(spec['services'])}")
for w in warns:
print(f" ⚠ {w}")
return 0
def cmd_plan(path: str) -> int:
doc = apply_preset(load(path))
errors = validate(doc)
if errors:
sys.stderr.write("cluster.yml tidak valid — jalankan 'validate' untuk detail\n")
return 1
print(json.dumps(plan(doc), indent=2, sort_keys=True))
return 0
def cmd_components(path: str) -> int:
doc = apply_preset(load(path))
for service, owners in sorted(resolve(doc).items()):
print(f"{service:24s} -> {','.join(owners)}")
return 0
def cmd_hosts(path: str) -> int:
doc = apply_preset(load(path))
for name, spec in (doc.get("hosts") or {}).items():
print(f"{name} {spec.get('address')}")
return 0
def main(argv: list[str]) -> int:
if len(argv) < 3:
sys.stderr.write(__doc__ or "")
return 2
command, path = argv[1], argv[2]
if not Path(path).is_file():
sys.stderr.write(f"tidak ditemukan: {path}\n")
return 2
commands = {
"validate": cmd_validate,
"plan": cmd_plan,
"components": cmd_components,
"hosts": cmd_hosts,
}
if command not in commands:
sys.stderr.write(f"perintah tak dikenal: {command}\n")
return 2
return commands[command](path)
if __name__ == "__main__":
raise SystemExit(main(sys.argv))
deploy/lib/cluster/preflight.py
+233
View File
@@ -0,0 +1,233 @@
#!/usr/bin/env python3
"""preflight.py — verifikasi kesiapan lintas host SEBELUM deploy (Fase 2, R10 §4.7).
Pengganti `depends_on` lintas host: generator MEMBUANG `depends_on` yang menyeberang
host (Compose tidak bisa menyalakan service di host lain), sehingga tidak ada lagi yang
menjamin urutan/ketersediaan antar host. Preflight menutup celah itu dengan memeriksa
**dari sisi jaringan**: apakah tiap port yang dibutuhkan benar-benar terjangkau di
alamat host pemiliknya.
Yang diperiksa:
1. Inventory valid (aturan penempatan R10 §4.3 / R11 §3.2).
2. Port internal yang dibutuhkan lintas host (perhitungan yang SAMA dengan generator —
diimpor, bukan disalin) benar-benar terbuka di alamat host pemiliknya.
3. TLS Trino coordinator (`8443`) menyelesaikan handshake.
4. Selisih waktu antar host (header `Date` HTTP) — JWT/`notBefore` sensitif, jadi
skew besar = kegagalan login yang sulit dilacak.
Pemakaian:
preflight.py <cluster.yml> [--compose docker-compose.yml] [--deploy DIR] [--timeout N]
Keluaran mengikuti konvensi gate repo: [PASS]/[FAIL]/[WARN]/[SKIP] + Ringkasan + recap.
Exit 1 bila ada FAIL.
"""
from __future__ import annotations
import json
import socket
import ssl
import sys
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import generate # noqa: E402
import inventory # noqa: E402
GRN = "\033[32m"; RED = "\033[31m"; YLW = "\033[33m"; BLD = "\033[1m"; NC = "\033[0m"
class Report:
def __init__(self) -> None:
self.passed = 0
self.failed = 0
self.warned = 0
self.skipped = 0
self.fail_msgs: list[str] = []
def ok(self, msg: str) -> None:
print(f" {GRN}[PASS]{NC} {msg}"); self.passed += 1
def bad(self, msg: str) -> None:
print(f" {RED}[FAIL]{NC} {msg}"); self.failed += 1; self.fail_msgs.append(msg)
def warn(self, msg: str) -> None:
print(f" {YLW}[WARN]{NC} {msg}"); self.warned += 1
def skip(self, msg: str) -> None:
print(f" {YLW}[SKIP]{NC} {msg}"); self.skipped += 1
def recap(self) -> int:
print()
print(f"{BLD}Ringkasan:{NC} {self.passed} PASS · {self.failed} FAIL · "
f"{self.warned} WARN · {self.skipped} SKIP")
if self.fail_msgs:
print(f"\n{BLD}Yang gagal ({len(self.fail_msgs)}):{NC}")
for i, m in enumerate(self.fail_msgs, 1):
print(f" {i}. {m}")
return 1 if self.failed else 0
def tcp_open(host: str, port: int, timeout: float) -> tuple[bool, str]:
try:
with socket.create_connection((host, port), timeout=timeout):
return True, ""
except socket.timeout:
return False, "timeout"
except ConnectionRefusedError:
return False, "connection refused (port tertutup)"
except OSError as exc:
return False, str(exc)
def tls_handshake(host: str, port: int, timeout: float) -> tuple[bool, str]:
"""Handshake TLS tanpa verifikasi sertifikat (deploy ini self-signed).
Yang diuji hanya: TLS tersedia dan handshake selesai. Verifikasi nama/host
sengaja tidak dilakukan di sini — itu urusan klien masing-masing
(mis. OM memakai `verify: /etc/ssl/trino/trino.crt`).
"""
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
try:
with socket.create_connection((host, port), timeout=timeout) as raw:
with ctx.wrap_socket(raw, server_hostname=host) as tls:
_ = tls.version()
return True, ""
except OSError as exc:
return False, str(exc)
def http_date(host: str, port: int, timeout: float) -> datetime | None:
"""Ambil header `Date` dari respons HTTP paling sederhana (tanpa dependensi)."""
try:
with socket.create_connection((host, port), timeout=timeout) as sock:
sock.sendall(f"HEAD / HTTP/1.0\r\nHost: {host}\r\n\r\n".encode())
data = sock.recv(2048).decode("latin-1", errors="ignore")
except OSError:
return None
for line in data.splitlines():
if line.lower().startswith("date:"):
try:
return parsedate_to_datetime(line.split(":", 1)[1].strip())
except (ValueError, TypeError):
return None
return None
def clock_skew(host: str, port: int, timeout: float, report: Report) -> None:
remote = http_date(host, port, timeout)
if remote is None:
return
if remote.tzinfo is None:
remote = remote.replace(tzinfo=timezone.utc)
skew = abs((datetime.now(timezone.utc) - remote).total_seconds())
if skew > 60:
report.warn(f"{host}: selisih waktu {skew:.0f}s (>60s) — JWT/OIDC sensitif; "
f"pastikan NTP aktif di host itu")
else:
report.ok(f"{host}: selisih waktu {skew:.0f}s (aman)")
def main(argv: list[str]) -> int:
if len(argv) < 2:
sys.stderr.write(__doc__ or "")
return 2
inv_path = argv[1]
compose_path = "docker-compose.yml"
deploy_dir = Path(".")
timeout = 3.0
rest = argv[2:]
while rest:
flag = rest.pop(0)
if flag == "--compose" and rest:
compose_path = rest.pop(0)
elif flag == "--deploy" and rest:
deploy_dir = Path(rest.pop(0))
elif flag == "--timeout" and rest:
timeout = float(rest.pop(0))
else:
sys.stderr.write(f"argumen tak dikenal: {flag}\n")
return 2
report = Report()
print(f"{BLD}Preflight lintas host{NC}")
print(f" inventory : {inv_path}")
print(f" compose : {compose_path}")
print(f" {YLW}preflight hanya MEMERIKSA — tidak menyalakan apa pun. Urutan start "
f"antar host tetap manual (belum ada orkestrator).{NC}")
print()
# ── 1. inventory ────────────────────────────────────────────────────────
print(f"{BLD}A. Inventory{NC}")
doc = inventory.apply_preset(inventory.load(inv_path))
errors = inventory.validate(doc)
if errors:
for e in errors:
report.bad(f"inventory: {e}")
return report.recap()
plan = inventory.plan(doc)
report.ok(f"inventory valid — preset '{doc.get('preset') or 'none'}' · {len(plan)} host")
for host, spec in plan.items():
report.ok(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}]")
# ── 2. port internal yang dibutuhkan lintas host ─────────────────────────
print(f"\n{BLD}B. Port lintas host{NC}")
base = generate.load_compose(compose_path)
placement = inventory.resolve(doc)
needed = generate.required_cross_host(base, deploy_dir, plan, placement)
if not needed:
report.skip("tidak ada komponen yang dikonsumsi dari host lain")
for component in sorted(needed):
owners = placement.get(component) or []
ports = generate.INTERNAL_PORTS.get(component) or []
if len(owners) > 1:
report.skip(f"{component}: di {len(owners)} host (scale-out) — "
f"periksa manual {owners}")
continue
if not owners:
report.bad(f"{component}: tidak ada host pemilik"); continue
address = plan[owners[0]]["address"]
for port in ports:
open_ok, why = tcp_open(address, port, timeout)
if open_ok:
report.ok(f"{component} {address}:{port} terjangkau")
else:
report.bad(f"{component} {address}:{port} TIDAK terjangkau ({why})")
# ── 3. TLS Trino coordinator ─────────────────────────────────────────────
print(f"\n{BLD}C. TLS Trino coordinator{NC}")
trino_owners = placement.get("trino-coordinator") or []
if trino_owners:
addr = plan[trino_owners[0]]["address"]
handshake_ok, why = tls_handshake(addr, 8443, timeout)
if handshake_ok:
report.ok(f"TLS handshake coordinator {addr}:8443 berhasil")
else:
report.bad(f"TLS handshake coordinator {addr}:8443 gagal ({why})")
clock_skew(addr, 8443, timeout, report)
else:
report.skip("tidak ada trino-coordinator di inventory")
# ── 4. waktu antar host (lewat layanan HTTP) ─────────────────────────────
print(f"\n{BLD}D. Waktu antar host{NC}")
checked = 0
for component in ("keycloak", "nessie", "opa"):
owners = placement.get(component) or []
port = {"keycloak": 8080, "nessie": 19120, "opa": 8181}[component]
if len(owners) != 1:
continue
addr = plan[owners[0]]["address"]
if tcp_open(addr, port, timeout)[0]:
clock_skew(addr, port, timeout, report)
checked += 1
if checked == 0:
report.skip("tidak ada layanan HTTP yang terjangkau untuk memeriksa waktu "
"(pastikan NTP aktif di semua host)")
return report.recap()
if __name__ == "__main__":
raise SystemExit(main(sys.argv))
deploy/lib/common.sh
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env bash
# =============================================================================
# eBdesk SiCABE — Common Library (shared by install.sh, bootstrap.sh, test-verification.sh)
# =============================================================================
# ─── Color & Styling ─────────────────────────────────────────────────────────
export RED='\033[0;31m'
export GREEN='\033[0;32m'
export YELLOW='\033[1;33m'
export BLUE='\033[0;34m'
export CYAN='\033[0;36m'
export BOLD='\033[1m'
export NC='\033[0m'
export INFO="${CYAN}[INFO]${NC}"
export OK="${GREEN}[OK]${NC}"
export WARN="${YELLOW}[WARN]${NC}"
export ERR="${RED}[ERROR]${NC}"
export STEP="${BOLD}[STEP]${NC}"
export PASS="${GREEN}[PASS]${NC}"
export FAIL="${RED}[FAIL]${NC}"
# ─── Named Constants (Timeouts) ─────────────────────────────────────────────-
# Default max wait times in seconds (override via env var)
export WAIT_SHORT="${WAIT_SHORT:-30}" # ~30s — quick services
export WAIT_MEDIUM="${WAIT_MEDIUM:-60}" # ~60s — most services
export WAIT_LONG="${WAIT_LONG:-120}" # ~120s — slow services (Keycloak, StarRocks)
export WAIT_EXTRA="${WAIT_EXTRA:-180}" # ~180s — OpenMetadata
export SLEEP_STANDARD="${SLEEP_STANDARD:-2}" # polling interval
export SLEEP_SLOW="${SLEEP_SLOW:-5}" # slow polling (StarRocks)
# ─── Helper: Wait for HTTP endpoint ─────────────────────────────────────────
# Usage: wait_for_service "name" "http://url/health" [max_attempts]
# Selalu return 0 (non-fatal): readiness wait saja. Kegagalan nyata ditangkap
# test-verification.sh. Return non-zero akan mematikan script di bawah `set -e`.
wait_for_service() {
local name="$1"
local url="$2"
local max_attempts="${3:-${WAIT_MEDIUM}}"
local attempt=0
local sleep_time="${SLEEP_STANDARD}"
echo -e "${INFO} Menunggu ${name} siap (maks ${max_attempts}s)..."
while [[ ${attempt} -lt ${max_attempts} ]]; do
if curl -skf "${url}" >/dev/null 2>&1; then
echo -e "${OK} ${name} siap!"
return 0
fi
attempt=$((attempt + sleep_time))
sleep "${sleep_time}"
done
echo -e "${WARN} ${name} belum siap dalam ${max_attempts}s. Lanjutkan... (cek nanti: curl ${url})"
return 0
}
# ─── Helper: Wait for container readiness via Docker inspect ────────────────
# Usage: wait_for_container "container_name" [max_attempts]
wait_for_container() {
local container="$1"
local max_attempts="${2:-${WAIT_MEDIUM}}"
local attempt=0
local sleep_time="${SLEEP_STANDARD}"
echo -e "${INFO} Menunggu container ${container} running..."
while [[ ${attempt} -lt ${max_attempts} ]]; do
if docker ps --format "{{.Names}}" | grep -q "${container}"; then
echo -e "${OK} Container ${container} running."
return 0
fi
attempt=$((attempt + sleep_time))
sleep "${sleep_time}"
done
echo -e "${WARN} Container ${container} belum running dalam ${max_attempts}s."
return 1
}
# ─── Helper: JSON field extractor ───────────────────────────────────────────
# Usage: extract_json "field_name" < <json_string>
# Requires python3
extract_json() {
local field="$1"
python3 -c "import sys,json; print(json.load(sys.stdin).get('${field}',''))" 2>/dev/null || echo ""
}
# ─── Helper: Check if a command exists ──────────────────────────────────────
check_command() {
local cmd="$1"
local name="${2:-$1}"
if command -v "${cmd}" &>/dev/null; then
echo -e "${OK} ${name}: tersedia ($(command -v "${cmd}"))"
return 0
else
echo -e "${WARN} ${name} tidak ditemukan."
return 1
fi
}
# ─── Helper: Print section header ───────────────────────────────────────────
print_section() {
local title="$1"
echo ""
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
echo -e "${BOLD} ${title}${NC}"
echo -e "${BOLD}${BLUE}═══════════════════════════════════════════════════════${NC}"
echo ""
}
# ─── Helper: Muat berkas env (dukung mode multi-host) ───────────────────────
# Prioritas sumber env: --env-file <berkas> > $SICABE_ENV_FILE > <base>/.env
#
# KENAPA: di multi-host, endpoint (POSTGRES_HOST, TRINO_URL, KEYCLOAK_URL, …) BERBEDA
# per host dan ada di `generated/host-<x>/.env` — BUKAN di `deploy/.env`. Tanpa ini
# skrip memakai nilai single-host (localhost) dan memeriksa/menyentuh endpoint salah.
#
# Dipakai: `sicabe_load_env "${SCRIPT_DIR}" "$@"` — GANTI blok `source .env` inline.
# - `--env-file <berkas>` dipilih (relatif dihitung dari <base>).
# - `$SICABE_ENV_FILE` dihormati agar sub-script yang dipanggil `bootstrap.sh --host`
# otomatis mewarisi env host tanpa menimpa dengan deploy/.env.
# - `source` berjalan di shell PEMANGGIL (fungsi tidak membuat subshell), jadi
# variabel tetap terlihat setelah fungsi kembali.
sicabe_load_env() {
local _sicabe_base="$1"; shift
local _sicabe_override="${SICABE_ENV_FILE:-}" _sicabe_prev="" _sicabe_a
for _sicabe_a in "$@"; do
[[ "${_sicabe_prev}" == "--env-file" ]] && _sicabe_override="${_sicabe_a}"
_sicabe_prev="${_sicabe_a}"
done
local _sicabe_envf="${_sicabe_override:-${_sicabe_base}/.env}"
[[ "${_sicabe_envf}" != /* ]] && _sicabe_envf="${_sicabe_base}/${_sicabe_envf}"
if [[ ! -f "${_sicabe_envf}" ]]; then
if [[ -n "${_sicabe_override}" ]]; then
echo -e "${ERR} env-file tidak ditemukan: ${_sicabe_envf}" >&2
return 1
fi
return 0
fi
set -a
# shellcheck disable=SC1090
source "${_sicabe_envf}"
set +a
# Direktori bundle (config + docker-compose.yml + .env) — dipakai helper host-aware
# untuk tahu container mana milik host ini (lihat sicabe_host_expects_container).
SICABE_BUNDLE_DIR="$(dirname "${_sicabe_envf}")"
[[ -n "${_sicabe_override}" ]] && echo -e "${INFO} env-file: ${_sicabe_envf} (mode host)" >&2
return 0
}
# ─── Helper: URL endpoint efektif (sadar EXTERNAL & multi-host) ─────────────
# Urutan: (1) <url_var> eksplisit NON-localhost (komponen EXTERNAL) → pakai apa adanya;
# (2) multi-host (bundle memuat SICABE_HOST_NAME) → scheme://<host_var>:<port>;
# (3) selain itu → scheme://localhost:<port> (single-host).
# Dipakai `scripts/verify-phase*` agar endpoint menunjuk ALAMAT HOST di multi-host,
# tanpa merusak dukungan komponen existing (single-host tetap localhost; external
# tetap memakai URL yang diberikan operator).
sicabe_local_url() { # <scheme> <host_var> <port> [url_var]
local _slu_s="$1" _slu_hv="$2" _slu_p="$3" _slu_uv="${4:-}" _slu_u="" _slu_h=""
[[ -n "${_slu_uv}" ]] && _slu_u="${!_slu_uv:-}"
if [[ -n "${_slu_u}" && "${_slu_u}" != *localhost* && "${_slu_u}" != *127.0.0.1* ]]; then
echo "${_slu_u}"; return 0
fi
_slu_h="${!_slu_hv:-}"
# Remote ⟺ nilai berupa ALAMAT (mengandung titik: IP pemilik, mis. 100.112.76.124).
# Nama layanan tanpa titik (mis. `airflow-webserver`, `keycloak`, `openmetadata-server`)
# = komponen CO-LOCATED → dari host dijangkau via localhost. Uji LAMA ("≠ localhost")
# keliru menganggap nama layanan sebagai remote → `http://airflow-webserver:12082`
# (nama container + port HOST) yang tak bisa diakses dari host.
if [[ -n "${SICABE_HOST_NAME:-}" && "${_slu_h}" == *.* && "${_slu_h}" != "127.0.0.1" ]]; then
echo "${_slu_s}://${_slu_h}:${_slu_p}"; return 0
fi
echo "${_slu_s}://localhost:${_slu_p}"
}
# ─── Helper: HOST endpoint efektif (sadar EXTERNAL & multi-host) ────────────
# Urutan: (1) <fallback_var> bila NON-localhost (override operator / komponen EXTERNAL);
# (2) multi-host (SICABE_HOST_NAME) → <host_var>;
# (3) selain itu → localhost.
# Dipakai untuk variabel yang di skrip SEKALIGUS menjadi penanda external
# (mis. STARROCKS_HOST/AIRFLOW_HOST): penanda tetap memakai nilai operator, sedangkan
# koneksi memakai host efektif ini.
sicabe_local_host() { # <host_var> [fallback_var]
local _slh_hv="$1" _slh_fb="${2:-}" _slh_f=""
if [[ -n "${_slh_fb}" ]]; then
_slh_f="${!_slh_fb:-}"
# Remote = ALAMAT (mengandung titik). Nama layanan tanpa titik = co-located → localhost.
if [[ "${_slh_f}" == *.* && "${_slh_f}" != "127.0.0.1" ]]; then
echo "${_slh_f}"; return 0
fi
fi
local _slh_h="${!_slh_hv:-}"
if [[ -n "${SICABE_HOST_NAME:-}" && "${_slh_h}" == *.* && "${_slh_h}" != "127.0.0.1" ]]; then
echo "${_slh_h}"; return 0
fi
echo "localhost"
}
# ─── Helper: apakah container milik HOST INI? ───────────────────────────────
# Single-host (SICABE_HOST_NAME kosong) → selalu true (semua lokal).
# Multi-host (dijalankan dgn `--env-file generated/host-<x>/.env`) → true hanya bila
# container ada di bundle host ini. Dipakai gate agar container host LAIN di-SKIP
# (bukan FAIL) — sama seperti mode host di test-verification.sh.
# Daftar container di-cache (1× `docker compose config` per proses).
sicabe_host_expects_container() { # <nama-container>
[[ -z "${SICABE_HOST_NAME:-}" ]] && return 0
local _shc_dir="${SICABE_BUNDLE_DIR:-.}"
if [[ -z "${SICABE_BUNDLE_CONTAINERS+x}" ]]; then
SICABE_BUNDLE_CONTAINERS="$(docker compose -f "${_shc_dir}/docker-compose.yml" \
--env-file "${_shc_dir}/.env" --profile all config 2>/dev/null \
| python3 -c "import sys,yaml;d=yaml.safe_load(sys.stdin) or {};print('\n'.join((s.get('container_name') or '') for s in (d.get('services') or {}).values()))" \
2>/dev/null || true)"
fi
printf '%s\n' "${SICABE_BUNDLE_CONTAINERS}" | grep -qx "$1"
}
# ─── Helper: wajibkan container LOKAL (kalau bukan → instruksi + kode 3) ─────
# Dipakai skrip seed/reapply yang `docker exec` container tertentu. Di multi-host,
# bila container itu milik host LAIN, skrip harus dijalankan DI host pemiliknya.
sicabe_require_local_container() { # <container> <nama-komponen> <host-contoh>
sicabe_host_expects_container "$1" && return 0
echo -e "${WARN} '${1}' (${2}) berjalan di host LAIN — skrip ini harus dijalankan DI host itu." >&2
echo -e "${WARN} contoh: ./scripts/${0##*/} --env-file generated/host-${3}/.env (di host ${3})" >&2
return 3
}
# ─── Helper: pastikan sumber bind-mount berupa FILE (bukan direktori) ─────────
# Docker membuat DIRECTORY bila sumber bind-mount sebuah file TIDAK ADA saat `up`
# (bind mount ke path non-existent di host). Akibatnya service gagal, mis.
# Airflow SimpleAuthManager: IsADirectoryError '/opt/airflow/simple_auth_manager_pa...'.
# Bila path sudah berupa direktori (sisa `up` sebelumnya) → buang; lalu pastikan ada file.
sicabe_ensure_file_mount() { # <path> [mode]
local path="$1" mode="${2:-644}"
if [[ -d "${path}" ]]; then rm -rf "${path}"; fi
[[ -e "${path}" ]] || : > "${path}"
chmod "${mode}" "${path}" 2>/dev/null || true
}