366 lines
14 KiB
Python
366 lines
14 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""inventory.py — cluster.yml: parse, validasi, resolusi penempatan (Fase 1 multi-host).
|
||
|
|
|
||
|
|
Ini otak Fase 1 (R10 §4.3/§4.5, R11 §3–§5). Satu implementasi otoritatif dipakai
|
||
|
|
oleh `lib/cluster/generate.sh` dan `install.sh`; tidak ada logika penempatan yang
|
||
|
|
diduplikasi di shell.
|
||
|
|
|
||
|
|
Model:
|
||
|
|
- Tier (7): state, identity, policy, coord, engine, pipeline, governance.
|
||
|
|
Tiap service compose dipetakan ke tepat satu tier (TIER_SERVICES).
|
||
|
|
- Preset (a3/b5) hanya mengisi DEFAULT `hosts.<h>.tiers`; komponen diturunkan
|
||
|
|
dari tier-nya. Override eksplisit `components.<c>.host` menang.
|
||
|
|
- Tier `engine` boleh >1 host (scale-out): komponennya muncul di SEMUA host
|
||
|
|
bertier engine. Tier lain tepat 1 host.
|
||
|
|
|
||
|
|
Perintah:
|
||
|
|
inventory.py validate <cluster.yml> # exit 0 = valid; cetak ringkasan/error
|
||
|
|
inventory.py hosts <cluster.yml> # daftar host
|
||
|
|
inventory.py plan <cluster.yml> # JSON: host -> {address, tiers, services}
|
||
|
|
inventory.py components <cluster.yml> # komponen -> host (untuk dokumentasi)
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
try:
|
||
|
|
import yaml
|
||
|
|
except ImportError: # pragma: no cover
|
||
|
|
sys.stderr.write("PyYAML wajib tersedia (python3 -c 'import yaml')\n")
|
||
|
|
raise SystemExit(2)
|
||
|
|
|
||
|
|
# ─── Taksonomi (R11 §3.1) ───────────────────────────────────────────────────
|
||
|
|
# Setiap service compose milik TEPAT SATU tier.
|
||
|
|
TIER_SERVICES: dict[str, list[str]] = {
|
||
|
|
"state": ["postgres", "minio", "minio-init", "openmetadata-opensearch"],
|
||
|
|
"identity": ["keycloak", "openldap", "nessie"],
|
||
|
|
"policy": ["opa"],
|
||
|
|
"coord": ["trino-coordinator", "starrocks-fe", "trino-jmx-exporter"],
|
||
|
|
"engine": ["trino-worker", "starrocks-cn", "dremio"],
|
||
|
|
"pipeline": [
|
||
|
|
"airflow-volume-init",
|
||
|
|
"airflow-init",
|
||
|
|
"airflow-webserver",
|
||
|
|
"airflow-dag-processor",
|
||
|
|
"airflow-scheduler",
|
||
|
|
],
|
||
|
|
# `prometheus` menyusul R11 (belum ada di taksonomi 7-tier). Ditempatkan di
|
||
|
|
# `governance`: pada blueprint A maupun B ia berada di host yang sama, jadi
|
||
|
|
# granularitas tier tidak mengubah penempatan.
|
||
|
|
# `statsd-exporter` (M2 observabilitas) ikut `governance` bersama Prometheus;
|
||
|
|
# pada a3/b5 tier ini se-host dengan `pipeline` (Airflow), sehingga UDP
|
||
|
|
# StatsD Airflow→statsd-exporter tetap lokal.
|
||
|
|
"governance": ["openmetadata-migrate", "openmetadata-server", "grafana", "prometheus", "statsd-exporter"],
|
||
|
|
}
|
||
|
|
|
||
|
|
VALID_TIERS = tuple(TIER_SERVICES)
|
||
|
|
|
||
|
|
# ─── TLS & URL publik (langkah 1: hanya skema/var; BELUM mengaktifkan TLS) ──
|
||
|
|
# `none` = perilaku sekarang (http, IP+port 12xxx). `edge` = browser memakai nama
|
||
|
|
# domain lewat ingress, bidang internal tetap plaintext. `full` = seluruh segmen TLS.
|
||
|
|
# Default `none` supaya TANPA cluster.yml perilaku single-host tidak berubah.
|
||
|
|
TLS_MODES = ("none", "edge", "full")
|
||
|
|
|
||
|
|
# Layanan yang punya URL BROWSER-FACING → label subdomain di bawah `domain`.
|
||
|
|
# Dipakai juga oleh ingress (step 2) sebagai daftar rute Host(), jadi nama di sini
|
||
|
|
# adalah satu-satunya sumber kebenaran penamaan.
|
||
|
|
PUBLIC_URL_SERVICES: dict[str, str] = {
|
||
|
|
"KEYCLOAK_PUBLIC_URL": "kc",
|
||
|
|
"GRAFANA_PUBLIC_URL": "grafana",
|
||
|
|
"OPENMETADATA_PUBLIC_URL": "om",
|
||
|
|
"AIRFLOW_PUBLIC_URL": "airflow",
|
||
|
|
"TRINO_PUBLIC_URL": "trino",
|
||
|
|
"MINIO_CONSOLE_PUBLIC_URL": "minio",
|
||
|
|
}
|
||
|
|
|
||
|
|
# Blueprint: default tiers per host (R11 §4.1/§4.2).
|
||
|
|
PRESETS: dict[str, dict[str, list[str]]] = {
|
||
|
|
"a3": {
|
||
|
|
"control": ["identity", "policy", "pipeline", "governance"],
|
||
|
|
"compute": ["coord", "engine"],
|
||
|
|
"state": ["state"],
|
||
|
|
},
|
||
|
|
"b5": {
|
||
|
|
"control": ["identity", "policy", "pipeline", "governance"],
|
||
|
|
"coord": ["coord"],
|
||
|
|
"engine-1": ["engine"],
|
||
|
|
"engine-2": ["engine"],
|
||
|
|
"state": ["state"],
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
# Komponen stateful: tidak boleh berpindah host tanpa migrasi volume (R10 R7).
|
||
|
|
PINNED = ("postgres", "minio", "openmetadata-opensearch", "starrocks-fe", "starrocks-cn", "dremio")
|
||
|
|
|
||
|
|
|
||
|
|
def tier_of(service: str) -> str | None:
|
||
|
|
"""Tier pemilik sebuah service compose."""
|
||
|
|
for tier, services in TIER_SERVICES.items():
|
||
|
|
if service in services:
|
||
|
|
return tier
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def load(path: str) -> dict:
|
||
|
|
try:
|
||
|
|
with open(path, encoding="utf-8") as fh:
|
||
|
|
return yaml.safe_load(fh) or {}
|
||
|
|
except yaml.YAMLError as exc:
|
||
|
|
raise SystemExit(f"cluster.yml bukan YAML yang sah: {exc}") from exc
|
||
|
|
|
||
|
|
|
||
|
|
def apply_preset(doc: dict) -> dict:
|
||
|
|
"""Isi `hosts` dari preset bila belum didefinisikan eksplisit.
|
||
|
|
|
||
|
|
Preset tak dikenal TIDAK menghentikan program di sini — biarkan `validate()`
|
||
|
|
yang melaporkannya (dulu ini KeyError → traceback, bukan pesan yang berguna).
|
||
|
|
"""
|
||
|
|
preset = doc.get("preset") or "none"
|
||
|
|
if preset not in PRESETS:
|
||
|
|
return doc
|
||
|
|
hosts = doc.setdefault("hosts", {})
|
||
|
|
for name, tiers in PRESETS[preset].items():
|
||
|
|
hosts.setdefault(name, {})
|
||
|
|
hosts[name].setdefault("tiers", list(tiers))
|
||
|
|
return doc
|
||
|
|
|
||
|
|
|
||
|
|
def resolve(doc: dict) -> dict[str, list[str]]:
|
||
|
|
"""Komponen → daftar host (tier `engine` bisa >1 host)."""
|
||
|
|
hosts = doc.get("hosts") or {}
|
||
|
|
tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs}
|
||
|
|
explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()}
|
||
|
|
|
||
|
|
placement: dict[str, list[str]] = {}
|
||
|
|
for service, tier in tiers_by_service.items():
|
||
|
|
if service in explicit and explicit[service]:
|
||
|
|
placement[service] = [explicit[service]]
|
||
|
|
continue
|
||
|
|
owners = [h for h, spec in hosts.items() if tier in (spec.get("tiers") or [])]
|
||
|
|
if owners:
|
||
|
|
placement[service] = sorted(owners)
|
||
|
|
return placement
|
||
|
|
|
||
|
|
|
||
|
|
def tls_mode(doc: dict) -> str:
|
||
|
|
"""Mode TLS dari inventory (`none` bila tidak ditulis)."""
|
||
|
|
return (doc.get("tls") or {}).get("mode") or "none"
|
||
|
|
|
||
|
|
|
||
|
|
def cluster_domain(doc: dict) -> str:
|
||
|
|
return ((doc.get("cluster") or {}).get("domain") or "").strip()
|
||
|
|
|
||
|
|
|
||
|
|
def public_urls(doc: dict) -> dict[str, str]:
|
||
|
|
"""URL browser-facing turunan dari tls.mode + domain.
|
||
|
|
|
||
|
|
Kosong bila mode `none` — dalam mode itu pemakaian tetap IP+port 12xxx seperti
|
||
|
|
sekarang, sehingga perilaku single-host TIDAK berubah.
|
||
|
|
"""
|
||
|
|
mode = tls_mode(doc)
|
||
|
|
domain = cluster_domain(doc)
|
||
|
|
if mode == "none" or not domain:
|
||
|
|
return {}
|
||
|
|
scheme = "https" if mode in ("edge", "full") else "http"
|
||
|
|
return {var: f"{scheme}://{label}.{domain}" for var, label in PUBLIC_URL_SERVICES.items()}
|
||
|
|
|
||
|
|
|
||
|
|
def enabled_components(doc: dict) -> dict[str, bool]:
|
||
|
|
return {
|
||
|
|
name: bool(spec.get("enabled", True))
|
||
|
|
for name, spec in (doc.get("components") or {}).items()
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def validate(doc: dict) -> list[str]:
|
||
|
|
"""Validasi R10 §4.3. Mengembalikan daftar error (kosong = valid)."""
|
||
|
|
errors: list[str] = []
|
||
|
|
|
||
|
|
preset = doc.get("preset") or "none"
|
||
|
|
if preset not in ("none", *PRESETS):
|
||
|
|
errors.append(f"preset '{preset}' tidak dikenal (pakai: none|{'|'.join(PRESETS)})")
|
||
|
|
|
||
|
|
# TLS & domain (langkah 1)
|
||
|
|
mode = (doc.get("tls") or {}).get("mode") or "none"
|
||
|
|
if mode not in TLS_MODES:
|
||
|
|
errors.append(f"tls.mode '{mode}' tidak dikenal (pakai: {'|'.join(TLS_MODES)})")
|
||
|
|
domain = ((doc.get("cluster") or {}).get("domain") or "").strip()
|
||
|
|
if mode != "none":
|
||
|
|
if not domain:
|
||
|
|
errors.append(f"tls.mode '{mode}' memerlukan cluster.domain (mis. sicabe.com)")
|
||
|
|
elif "://" in domain or "/" in domain or "." not in domain:
|
||
|
|
errors.append(f"cluster.domain '{domain}' tidak sah — isi nama domain saja "
|
||
|
|
f"(mis. sicabe.com), tanpa skema atau garis miring")
|
||
|
|
# Ingress satu host (T-20): opsional — default host yang menjalankan keycloak.
|
||
|
|
ing = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
|
||
|
|
if ing:
|
||
|
|
if mode == "none":
|
||
|
|
errors.append("tls.ingress_host tidak bermakna bila tls.mode 'none' "
|
||
|
|
"(tanpa Traefik/TLS) — hapus atau ubah tls.mode")
|
||
|
|
elif ing not in (doc.get("hosts") or {}):
|
||
|
|
errors.append(f"tls.ingress_host '{ing}' bukan nama host yang dikenal "
|
||
|
|
f"(host: {', '.join(sorted((doc.get('hosts') or {})))})")
|
||
|
|
|
||
|
|
hosts = doc.get("hosts") or {}
|
||
|
|
if not hosts:
|
||
|
|
errors.append("'hosts' kosong — inventory harus mendefinisikan minimal satu host")
|
||
|
|
return errors
|
||
|
|
|
||
|
|
# (f) tiap host punya address + tier yang sah
|
||
|
|
for name, spec in hosts.items():
|
||
|
|
if not spec.get("address"):
|
||
|
|
errors.append(f"host '{name}': 'address' wajib diisi")
|
||
|
|
tiers = spec.get("tiers") or []
|
||
|
|
if not tiers:
|
||
|
|
errors.append(f"host '{name}': minimal satu tier")
|
||
|
|
for tier in tiers:
|
||
|
|
if tier not in VALID_TIERS:
|
||
|
|
errors.append(f"host '{name}': tier '{tier}' tidak dikenal (sah: {', '.join(VALID_TIERS)})")
|
||
|
|
|
||
|
|
# (d) tier state tepat 1 host · R11 aturan 2 (state selalu host sendiri)
|
||
|
|
state_hosts = [h for h, s in hosts.items() if "state" in (s.get("tiers") or [])]
|
||
|
|
if len(state_hosts) != 1:
|
||
|
|
errors.append(
|
||
|
|
f"tier 'state' harus di TEPAT 1 host, ditemukan {len(state_hosts)}: "
|
||
|
|
f"{state_hosts or '—'} (I/O-bound & pinned, R11 §3.2)"
|
||
|
|
)
|
||
|
|
|
||
|
|
# (3) tier coord tepat 1 host (OSS Trino = 1 coordinator)
|
||
|
|
coord_hosts = [h for h, s in hosts.items() if "coord" in (s.get("tiers") or [])]
|
||
|
|
if len(coord_hosts) != 1:
|
||
|
|
errors.append(f"tier 'coord' harus di TEPAT 1 host, ditemukan {len(coord_hosts)}: {coord_hosts or '—'}")
|
||
|
|
|
||
|
|
# engine: boleh >1 (scale-out) — hanya info bila tidak ada
|
||
|
|
if not [h for h, s in hosts.items() if "engine" in (s.get("tiers") or [])]:
|
||
|
|
errors.append("tidak ada host bertier 'engine' — tidak ada tempat eksekusi Trino/StarRocks")
|
||
|
|
|
||
|
|
# anti-affinity R11 §3.2: state ⊥ engine
|
||
|
|
for h, s in hosts.items():
|
||
|
|
t = set(s.get("tiers") or [])
|
||
|
|
if "state" in t and "engine" in t:
|
||
|
|
errors.append(f"host '{h}': tier 'state' dan 'engine' tidak boleh satu host (anti-affinity)")
|
||
|
|
|
||
|
|
# (a) komponen menunjuk host yang ada · (c) endpoint hanya ke komponen enabled
|
||
|
|
components = doc.get("components") or {}
|
||
|
|
for name, spec in components.items():
|
||
|
|
host = spec.get("host")
|
||
|
|
if host and host not in hosts:
|
||
|
|
errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'")
|
||
|
|
if tier_of(name) is None and name not in ("dremio",):
|
||
|
|
errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)")
|
||
|
|
|
||
|
|
# (c) tiap komponen harus bisa ditempatkan
|
||
|
|
placement = resolve(doc)
|
||
|
|
for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs):
|
||
|
|
if service not in placement:
|
||
|
|
errors.append(
|
||
|
|
f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier "
|
||
|
|
f"'{tier_of(service)}' dan tidak ada components.{service}.host"
|
||
|
|
)
|
||
|
|
|
||
|
|
# stateful: cegah pindah host tanpa migrasi (peringatan, bukan error)
|
||
|
|
return errors
|
||
|
|
|
||
|
|
|
||
|
|
def plan(doc: dict) -> dict[str, dict]:
|
||
|
|
"""host → {address, tiers, services} untuk generator."""
|
||
|
|
hosts = doc.get("hosts") or {}
|
||
|
|
placement = resolve(doc)
|
||
|
|
enabled = enabled_components(doc)
|
||
|
|
|
||
|
|
result: dict[str, dict] = {}
|
||
|
|
for name, spec in hosts.items():
|
||
|
|
result[name] = {
|
||
|
|
"address": spec.get("address"),
|
||
|
|
"tiers": spec.get("tiers") or [],
|
||
|
|
"services": [],
|
||
|
|
}
|
||
|
|
for service, owners in placement.items():
|
||
|
|
if enabled.get(service) is False:
|
||
|
|
continue
|
||
|
|
for owner in owners:
|
||
|
|
if owner in result:
|
||
|
|
result[owner]["services"].append(service)
|
||
|
|
for name in result:
|
||
|
|
result[name]["services"].sort()
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def warnings(doc: dict) -> list[str]:
|
||
|
|
out: list[str] = []
|
||
|
|
placement = resolve(doc)
|
||
|
|
for service in PINNED:
|
||
|
|
hosts = placement.get(service) or []
|
||
|
|
if len(hosts) > 1:
|
||
|
|
out.append(f"'{service}' stateful berada di >1 host {hosts} — butuh volume terpisah per host")
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_validate(path: str) -> int:
|
||
|
|
doc = apply_preset(load(path))
|
||
|
|
errors = validate(doc)
|
||
|
|
warns = warnings(doc)
|
||
|
|
if errors:
|
||
|
|
print(f"cluster.yml TIDAK valid ({len(errors)} masalah):")
|
||
|
|
for e in errors:
|
||
|
|
print(f" ✗ {e}")
|
||
|
|
return 1
|
||
|
|
p = plan(doc)
|
||
|
|
print(f"cluster.yml valid — preset '{doc.get('preset') or 'none'}'")
|
||
|
|
for host, spec in p.items():
|
||
|
|
print(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}] "
|
||
|
|
f"services={len(spec['services'])}")
|
||
|
|
for w in warns:
|
||
|
|
print(f" ⚠ {w}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_plan(path: str) -> int:
|
||
|
|
doc = apply_preset(load(path))
|
||
|
|
errors = validate(doc)
|
||
|
|
if errors:
|
||
|
|
sys.stderr.write("cluster.yml tidak valid — jalankan 'validate' untuk detail\n")
|
||
|
|
return 1
|
||
|
|
print(json.dumps(plan(doc), indent=2, sort_keys=True))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_components(path: str) -> int:
|
||
|
|
doc = apply_preset(load(path))
|
||
|
|
for service, owners in sorted(resolve(doc).items()):
|
||
|
|
print(f"{service:24s} -> {','.join(owners)}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_hosts(path: str) -> int:
|
||
|
|
doc = apply_preset(load(path))
|
||
|
|
for name, spec in (doc.get("hosts") or {}).items():
|
||
|
|
print(f"{name} {spec.get('address')}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv: list[str]) -> int:
|
||
|
|
if len(argv) < 3:
|
||
|
|
sys.stderr.write(__doc__ or "")
|
||
|
|
return 2
|
||
|
|
command, path = argv[1], argv[2]
|
||
|
|
if not Path(path).is_file():
|
||
|
|
sys.stderr.write(f"tidak ditemukan: {path}\n")
|
||
|
|
return 2
|
||
|
|
commands = {
|
||
|
|
"validate": cmd_validate,
|
||
|
|
"plan": cmd_plan,
|
||
|
|
"components": cmd_components,
|
||
|
|
"hosts": cmd_hosts,
|
||
|
|
}
|
||
|
|
if command not in commands:
|
||
|
|
sys.stderr.write(f"perintah tak dikenal: {command}\n")
|
||
|
|
return 2
|
||
|
|
return commands[command](path)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(main(sys.argv))
|