#!/usr/bin/env python3 """inventory.py — cluster.yml: parse, validasi, resolusi penempatan (Fase 1 multi-host). Ini otak Fase 1 (R10 §4.3/§4.5, R11 §3–§5). Satu implementasi otoritatif dipakai oleh `lib/cluster/generate.sh` dan `install.sh`; tidak ada logika penempatan yang diduplikasi di shell. Model: - Tier (7): state, identity, policy, coord, engine, pipeline, governance. Tiap service compose dipetakan ke tepat satu tier (TIER_SERVICES). - Preset (a3/b5) hanya mengisi DEFAULT `hosts..tiers`; komponen diturunkan dari tier-nya. Override eksplisit `components..host` menang. - Tier `engine` boleh >1 host (scale-out): komponennya muncul di SEMUA host bertier engine. Tier lain tepat 1 host. Perintah: inventory.py validate # exit 0 = valid; cetak ringkasan/error inventory.py hosts # daftar host inventory.py plan # JSON: host -> {address, tiers, services} inventory.py components # komponen -> host (untuk dokumentasi) """ from __future__ import annotations import json import sys from pathlib import Path try: import yaml except ImportError: # pragma: no cover sys.stderr.write("PyYAML wajib tersedia (python3 -c 'import yaml')\n") raise SystemExit(2) # ─── Taksonomi (R11 §3.1) ─────────────────────────────────────────────────── # Setiap service compose milik TEPAT SATU tier. TIER_SERVICES: dict[str, list[str]] = { "state": ["postgres", "minio", "minio-init", "openmetadata-opensearch"], "identity": ["keycloak", "openldap", "nessie"], "policy": ["opa"], "coord": ["trino-coordinator", "starrocks-fe", "trino-jmx-exporter"], "engine": ["trino-worker", "starrocks-cn", "dremio"], "pipeline": [ "airflow-volume-init", "airflow-init", "airflow-webserver", "airflow-dag-processor", "airflow-scheduler", ], # `prometheus` menyusul R11 (belum ada di taksonomi 7-tier). Ditempatkan di # `governance`: pada blueprint A maupun B ia berada di host yang sama, jadi # granularitas tier tidak mengubah penempatan. # `statsd-exporter` (M2 observabilitas) ikut `governance` bersama Prometheus; # pada a3/b5 tier ini se-host dengan `pipeline` (Airflow), sehingga UDP # StatsD Airflow→statsd-exporter tetap lokal. "governance": ["openmetadata-migrate", "openmetadata-server", "grafana", "prometheus", "statsd-exporter"], } VALID_TIERS = tuple(TIER_SERVICES) # ─── TLS & URL publik (langkah 1: hanya skema/var; BELUM mengaktifkan TLS) ── # `none` = perilaku sekarang (http, IP+port 12xxx). `edge` = browser memakai nama # domain lewat ingress, bidang internal tetap plaintext. `full` = seluruh segmen TLS. # Default `none` supaya TANPA cluster.yml perilaku single-host tidak berubah. TLS_MODES = ("none", "edge", "full") # Layanan yang punya URL BROWSER-FACING → label subdomain di bawah `domain`. # Dipakai juga oleh ingress (step 2) sebagai daftar rute Host(), jadi nama di sini # adalah satu-satunya sumber kebenaran penamaan. PUBLIC_URL_SERVICES: dict[str, str] = { "KEYCLOAK_PUBLIC_URL": "kc", "GRAFANA_PUBLIC_URL": "grafana", "OPENMETADATA_PUBLIC_URL": "om", "AIRFLOW_PUBLIC_URL": "airflow", "TRINO_PUBLIC_URL": "trino", "MINIO_CONSOLE_PUBLIC_URL": "minio", } # Blueprint: default tiers per host (R11 §4.1/§4.2). PRESETS: dict[str, dict[str, list[str]]] = { "a3": { "control": ["identity", "policy", "pipeline", "governance"], "compute": ["coord", "engine"], "state": ["state"], }, "b5": { "control": ["identity", "policy", "pipeline", "governance"], "coord": ["coord"], "engine-1": ["engine"], "engine-2": ["engine"], "state": ["state"], }, } # Komponen stateful: tidak boleh berpindah host tanpa migrasi volume (R10 R7). PINNED = ("postgres", "minio", "openmetadata-opensearch", "starrocks-fe", "starrocks-cn", "dremio") def tier_of(service: str) -> str | None: """Tier pemilik sebuah service compose.""" for tier, services in TIER_SERVICES.items(): if service in services: return tier return None def load(path: str) -> dict: try: with open(path, encoding="utf-8") as fh: return yaml.safe_load(fh) or {} except yaml.YAMLError as exc: raise SystemExit(f"cluster.yml bukan YAML yang sah: {exc}") from exc def apply_preset(doc: dict) -> dict: """Isi `hosts` dari preset bila belum didefinisikan eksplisit. Preset tak dikenal TIDAK menghentikan program di sini — biarkan `validate()` yang melaporkannya (dulu ini KeyError → traceback, bukan pesan yang berguna). """ preset = doc.get("preset") or "none" if preset not in PRESETS: return doc hosts = doc.setdefault("hosts", {}) for name, tiers in PRESETS[preset].items(): hosts.setdefault(name, {}) hosts[name].setdefault("tiers", list(tiers)) return doc def external_components(doc: dict) -> set[str]: """Komponen yang dikelola SISTEM LAIN: ditandai `components..address`. Komponen ini TIDAK dijalankan cluster (tak masuk `placement`/`plan`); endpoint-nya diarahkan ke alamat itu di SEMUA host (lihat generate.endpoint_lines). Berguna untuk IAM/objek-storage milik pihak lain, termasuk di host/IP yang berbeda satu per komponen. Turunan: `minio-init` (pembuat bucket MinIO **LOKAL** via `mc`) IKUT eksternal bila `minio` eksternal — kalau tidak, ia nyasar ke bundle `state` padahal `minio` absen. """ ext = {n for n, s in (doc.get("components") or {}).items() if (s.get("address") or "").strip()} if "minio" in ext: ext.add("minio-init") return ext def external_address(doc: dict, service: str) -> str | None: """Alamat eksternal komponen (components..address) bila ada, selain itu None.""" spec = (doc.get("components") or {}).get(service) or {} a = (spec.get("address") or "").strip() return a or None # Port default per skema — dipakai parse_address() bila operator menulis URL penuh # TANPA port (mis. `https://sso.example.com` → port 443). _DEFAULT_SCHEME_PORT = { "https": 443, "http": 80, "ldaps": 636, "ldap": 389, "postgres": 5432, "postgresql": 5432, } def parse_address(addr: str) -> tuple[str, str, int | None]: """Pisah alamat komponen → (skema, host, port). Menerima dua bentuk: - **URL penuh** (skema eksplisit): `https://sso.example.com` → ('https','sso.example.com',443) `http://minio.corp:9000` → ('http','minio.corp',9000) - **host/IP polos** (perilaku lama): `sso.example.com` → ('','sso.example.com',None) `10.20.0.10` → ('','10.20.0.10',None) Skema non-standar diterima apa adanya; path (`/foo`) dibuang. """ a = (addr or "").strip() scheme = "" if "://" in a: scheme, a = a.split("://", 1) scheme = scheme.strip().lower() a = a.split("/", 1)[0] # buang path bila ada host, sep, port_s = a.rpartition(":") # host:port HANYA bila tepat satu ':' dan port numerik — melindungi IPv6 # (mis. `2001:db8::1`) agar tak salah dipecah. if sep and port_s.isdigit() and a.count(":") == 1: host, port = host, int(port_s) else: host, port = a, _DEFAULT_SCHEME_PORT.get(scheme) return scheme, host, port def external_host(doc: dict, service: str) -> str | None: """HOST saja (tanpa skema/port) dari alamat eksternal — untuk var `*_HOST`.""" a = external_address(doc, service) return parse_address(a)[1] if a else None def resolve(doc: dict) -> dict[str, list[str]]: """Komponen → daftar host (tier `engine` bisa >1 host). Komponen EKSTERNAL (ber-`address`) TIDAK masuk hasil — ia bukan milik host mana pun.""" hosts = doc.get("hosts") or {} tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs} explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()} external = external_components(doc) placement: dict[str, list[str]] = {} for service, tier in tiers_by_service.items(): if service in external: continue # eksternal (pihak lain) → bukan milik host mana pun if service in explicit and explicit[service]: placement[service] = [explicit[service]] continue owners = [h for h, spec in hosts.items() if tier in (spec.get("tiers") or [])] if owners: placement[service] = sorted(owners) return placement def tls_mode(doc: dict) -> str: """Mode TLS dari inventory (`none` bila tidak ditulis).""" return (doc.get("tls") or {}).get("mode") or "none" def cluster_domain(doc: dict) -> str: return ((doc.get("cluster") or {}).get("domain") or "").strip() def public_urls(doc: dict) -> dict[str, str]: """URL browser-facing turunan dari tls.mode + domain. Kosong bila mode `none` — dalam mode itu pemakaian tetap IP+port 12xxx seperti sekarang, sehingga perilaku single-host TIDAK berubah. """ mode = tls_mode(doc) domain = cluster_domain(doc) if mode == "none" or not domain: return {} scheme = "https" if mode in ("edge", "full") else "http" return {var: f"{scheme}://{label}.{domain}" for var, label in PUBLIC_URL_SERVICES.items()} def enabled_components(doc: dict) -> dict[str, bool]: return { name: bool(spec.get("enabled", True)) for name, spec in (doc.get("components") or {}).items() } def validate(doc: dict) -> list[str]: """Validasi R10 §4.3. Mengembalikan daftar error (kosong = valid).""" errors: list[str] = [] preset = doc.get("preset") or "none" if preset not in ("none", *PRESETS): errors.append(f"preset '{preset}' tidak dikenal (pakai: none|{'|'.join(PRESETS)})") # TLS & domain (langkah 1) mode = (doc.get("tls") or {}).get("mode") or "none" if mode not in TLS_MODES: errors.append(f"tls.mode '{mode}' tidak dikenal (pakai: {'|'.join(TLS_MODES)})") domain = ((doc.get("cluster") or {}).get("domain") or "").strip() if mode != "none": if not domain: errors.append(f"tls.mode '{mode}' memerlukan cluster.domain (mis. sicabe.com)") elif "://" in domain or "/" in domain or "." not in domain: errors.append(f"cluster.domain '{domain}' tidak sah — isi nama domain saja " f"(mis. sicabe.com), tanpa skema atau garis miring") # Ingress satu host (T-20): opsional — default host yang menjalankan keycloak. ing = ((doc.get("tls") or {}).get("ingress_host") or "").strip() if ing: if mode == "none": errors.append("tls.ingress_host tidak bermakna bila tls.mode 'none' " "(tanpa Traefik/TLS) — hapus atau ubah tls.mode") elif ing not in (doc.get("hosts") or {}): errors.append(f"tls.ingress_host '{ing}' bukan nama host yang dikenal " f"(host: {', '.join(sorted((doc.get('hosts') or {})))})") hosts = doc.get("hosts") or {} if not hosts: errors.append("'hosts' kosong — inventory harus mendefinisikan minimal satu host") return errors # (f) tiap host punya address + tier yang sah for name, spec in hosts.items(): if not spec.get("address"): errors.append(f"host '{name}': 'address' wajib diisi") tiers = spec.get("tiers") or [] if not tiers: errors.append(f"host '{name}': minimal satu tier") for tier in tiers: if tier not in VALID_TIERS: errors.append(f"host '{name}': tier '{tier}' tidak dikenal (sah: {', '.join(VALID_TIERS)})") # (d) tier state tepat 1 host · R11 aturan 2 (state selalu host sendiri) state_hosts = [h for h, s in hosts.items() if "state" in (s.get("tiers") or [])] if len(state_hosts) != 1: errors.append( f"tier 'state' harus di TEPAT 1 host, ditemukan {len(state_hosts)}: " f"{state_hosts or '—'} (I/O-bound & pinned, R11 §3.2)" ) # (3) tier coord tepat 1 host (OSS Trino = 1 coordinator) coord_hosts = [h for h, s in hosts.items() if "coord" in (s.get("tiers") or [])] if len(coord_hosts) != 1: errors.append(f"tier 'coord' harus di TEPAT 1 host, ditemukan {len(coord_hosts)}: {coord_hosts or '—'}") # engine: boleh >1 (scale-out) — hanya info bila tidak ada if not [h for h, s in hosts.items() if "engine" in (s.get("tiers") or [])]: errors.append("tidak ada host bertier 'engine' — tidak ada tempat eksekusi Trino/StarRocks") # anti-affinity R11 §3.2: state ⊥ engine for h, s in hosts.items(): t = set(s.get("tiers") or []) if "state" in t and "engine" in t: errors.append(f"host '{h}': tier 'state' dan 'engine' tidak boleh satu host (anti-affinity)") # (a) komponen menunjuk host yang ada · (c) endpoint hanya ke komponen enabled components = doc.get("components") or {} for name, spec in components.items(): host = spec.get("host") addr = (spec.get("address") or "").strip() if host and host not in hosts: errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'") if host and addr: errors.append(f"komponen '{name}': 'host' dan 'address' tak boleh bersamaan " f"(host = dijalankan cluster; address = dikelola sistem lain)") if tier_of(name) is None and name not in ("dremio",): errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)") # (c) tiap komponen harus bisa ditempatkan placement = resolve(doc) external_all = external_components(doc) for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs): if service not in placement and service not in external_all: errors.append( f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier " f"'{tier_of(service)}', tidak ada components.{service}.host, " f"dan bukan eksternal (components.{service}.address)" ) # stateful: cegah pindah host tanpa migrasi (peringatan, bukan error) return errors def plan(doc: dict) -> dict[str, dict]: """host → {address, tiers, services} untuk generator.""" hosts = doc.get("hosts") or {} placement = resolve(doc) enabled = enabled_components(doc) result: dict[str, dict] = {} for name, spec in hosts.items(): result[name] = { "address": spec.get("address"), "tiers": spec.get("tiers") or [], "services": [], } for service, owners in placement.items(): if enabled.get(service) is False: continue for owner in owners: if owner in result: result[owner]["services"].append(service) for name in result: result[name]["services"].sort() return result def warnings(doc: dict) -> list[str]: out: list[str] = [] placement = resolve(doc) for service in PINNED: hosts = placement.get(service) or [] if len(hosts) > 1: out.append(f"'{service}' stateful berada di >1 host {hosts} — butuh volume terpisah per host") return out def cmd_validate(path: str) -> int: doc = apply_preset(load(path)) errors = validate(doc) warns = warnings(doc) if errors: print(f"cluster.yml TIDAK valid ({len(errors)} masalah):") for e in errors: print(f" ✗ {e}") return 1 p = plan(doc) print(f"cluster.yml valid — preset '{doc.get('preset') or 'none'}'") for host, spec in p.items(): print(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}] " f"services={len(spec['services'])}") for w in warns: print(f" ⚠ {w}") return 0 def cmd_plan(path: str) -> int: doc = apply_preset(load(path)) errors = validate(doc) if errors: sys.stderr.write("cluster.yml tidak valid — jalankan 'validate' untuk detail\n") return 1 print(json.dumps(plan(doc), indent=2, sort_keys=True)) return 0 def cmd_components(path: str) -> int: doc = apply_preset(load(path)) for service, owners in sorted(resolve(doc).items()): print(f"{service:24s} -> {','.join(owners)}") for service in sorted(external_components(doc)): print(f"{service:24s} -> (eksternal {external_address(doc, service)})") return 0 def cmd_hosts(path: str) -> int: doc = apply_preset(load(path)) for name, spec in (doc.get("hosts") or {}).items(): print(f"{name} {spec.get('address')}") return 0 def main(argv: list[str]) -> int: if len(argv) < 3: sys.stderr.write(__doc__ or "") return 2 command, path = argv[1], argv[2] if not Path(path).is_file(): sys.stderr.write(f"tidak ditemukan: {path}\n") return 2 commands = { "validate": cmd_validate, "plan": cmd_plan, "components": cmd_components, "hosts": cmd_hosts, } if command not in commands: sys.stderr.write(f"perintah tak dikenal: {command}\n") return 2 return commands[command](path) if __name__ == "__main__": raise SystemExit(main(sys.argv))