first commit
This commit is contained in:
@@ -0,0 +1,899 @@
|
||||
#!/usr/bin/env python3
|
||||
"""generate.py — render artefak deploy per host dari docker-compose.yml + cluster.yml.
|
||||
|
||||
Fase 1 R10 §4.5 (**Varian A: render, bukan overlay asli Compose**). Alasan: merge
|
||||
overlay tidak dapat MENGHAPUS service, sedangkan kita perlu menyisakan hanya
|
||||
service milik satu host.
|
||||
|
||||
Yang dihasilkan `generated/host-<x>/`:
|
||||
1. `docker-compose.yml` — hanya service milik host itu.
|
||||
2. `.env` — endpoint menunjuk ALAMAT HOST pemilik komponen (B1) + seluruh isi
|
||||
`.env` sumber (rahasia wajib identik lintas host, S1 R10 §4.12).
|
||||
3. `config/` — hasil `render-config.sh` (hostname container → alamat host), karena
|
||||
berkas di `config/**` tidak dapat melakukan ekspansi env saat runtime.
|
||||
|
||||
`depends_on` lintas host DIBUANG (penggantinya `preflight`, Fase 2); intra-host
|
||||
dipertahankan supaya urutan + healthcheck tetap berlaku di dalam satu host.
|
||||
|
||||
Hasil di `generated/` bersifat ARTEFAK (gitignored) — jangan disunting tangan.
|
||||
|
||||
Perintah:
|
||||
generate.py render <inventory> <base-compose> <base-env> <out-base> <render-config.sh>
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import inventory # noqa: E402 (modul sebelah, sengaja)
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
sys.stderr.write("PyYAML wajib tersedia\n")
|
||||
raise SystemExit(2)
|
||||
|
||||
# Variabel endpoint (B1) → komponen pemiliknya. Dipakai untuk mengarahkan
|
||||
# container ke ALAMAT HOST, bukan nama container (nama container hanya resolve
|
||||
# di host yang sama).
|
||||
ENDPOINT_VARS = {
|
||||
"POSTGRES_HOST": "postgres",
|
||||
"MINIO_HOST": "minio",
|
||||
"KEYCLOAK_HOST": "keycloak",
|
||||
"OPENMETADATA_HOST": "openmetadata-server",
|
||||
"AIRFLOW_WEBSERVER_HOST": "airflow-webserver",
|
||||
"STARROCKS_FE_HOST": "starrocks-fe",
|
||||
"STARROCKS_CN_HOST": "starrocks-cn",
|
||||
"OPENSEARCH_HOST": "openmetadata-opensearch",
|
||||
# Var SISI-HOST yang belum ada padanannya, dipakai `scripts/verify-phase*` (mis. Trino
|
||||
# & Grafana hanya punya *_PUBLIC_URL). Nilainya = alamat host pemilik. CATATAN:
|
||||
# STARROCKS_HOST/AIRFLOW_HOST sengaja TIDAK ditulis di sini — skrip memakainya sebagai
|
||||
# penanda komponen EXTERNAL (default `localhost`); mengisinya dengan alamat host akan
|
||||
# salah menandai komponen lokal sebagai external.
|
||||
"TRINO_HOST": "trino-coordinator",
|
||||
# Dipakai dbt (profiles.yml: env_var DBT_TRINO_HOST) — lintas-host butuh ALAMAT IP.
|
||||
"DBT_TRINO_HOST": "trino-coordinator",
|
||||
"GRAFANA_HOST": "grafana",
|
||||
"LDAP_HOST": "openldap",
|
||||
"NESSIE_HOST": "nessie",
|
||||
}
|
||||
|
||||
# Sebagian konsumen memakai bentuk URL (bukan `*_HOST`), dan berkas config yang
|
||||
# dibaca container TIDAK dapat ekspansi env → URL-nya harus ikut diarahkan di sini,
|
||||
# kalau tidak Trino/Sling/StarRocks lintas host akan mencari nama container yang
|
||||
# hanya ada di host lain. (Komponen pemilik tetap diambil dari inventory.)
|
||||
URL_ENDPOINT_VARS = {
|
||||
"S3_ENDPOINT": ("minio", "http://{addr}:{port}"),
|
||||
"NESSIE_URL": ("nessie", "http://{addr}:{port}"),
|
||||
"LDAP_URL": ("openldap", "ldap://{addr}:{port}"),
|
||||
"OPA_URL": ("opa", "http://{addr}:{port}"),
|
||||
}
|
||||
|
||||
# Port container untuk var URL di atas — bisa ditimpa di .env (mis. layanan EXISTING
|
||||
# berport non-standar). Default = port standar → perilaku lama tidak berubah.
|
||||
URL_ENDPOINT_PORTS = {
|
||||
"S3_ENDPOINT": ("MINIO_INTERNAL_PORT", 9000),
|
||||
"NESSIE_URL": ("NESSIE_INTERNAL_PORT", 19120),
|
||||
"LDAP_URL": ("OPENLDAP_INTERNAL_PORT", 389),
|
||||
"OPA_URL": ("OPA_INTERNAL_PORT", 8181),
|
||||
}
|
||||
|
||||
# URL BROWSER-FACING saat tls.mode = none (tanpa proxy/TLS): dialamatkan ke HOST yang
|
||||
# MENJALANKAN komponen, bukan ke localhost maupun ke IP host lokal. Nilainya diambil
|
||||
# dari host pemilik sehingga SAMA di semua bundle — wajib, karena `iss`/issuer OIDC
|
||||
# harus satu nilai; kalau tiap host menulis alamatnya sendiri, validator akan tolak.
|
||||
# Port di sini mengikuti skema 12xxx (sama seperti URL_ENDPOINT_VARS di bawah).
|
||||
PUBLIC_URL_ENDPOINTS = {
|
||||
"KEYCLOAK_PUBLIC_URL": ("keycloak", "http://{addr}:12080"),
|
||||
"GRAFANA_PUBLIC_URL": ("grafana", "http://{addr}:12300"),
|
||||
"OPENMETADATA_PUBLIC_URL": ("openmetadata-server", "http://{addr}:12085"),
|
||||
"AIRFLOW_PUBLIC_URL": ("airflow-webserver", "http://{addr}:12082"),
|
||||
"TRINO_PUBLIC_URL": ("trino-coordinator", "https://{addr}:12443"),
|
||||
"MINIO_CONSOLE_PUBLIC_URL": ("minio", "http://{addr}:12901"),
|
||||
}
|
||||
|
||||
# SENGAJA TIDAK diarahkan ke IP: KEYCLOAK_URL / TRINO_URL dan var `*_URL` lain yang
|
||||
# bersifat identitas (issuer OIDC harus stabil — lihat R10 §4.10.1: pakai DNS
|
||||
# internal, bukan IP). Itu milik operator dan harus sudah benar di `cluster.yml`
|
||||
# (domain) / `.env`.
|
||||
|
||||
# Hostname container yang dikenali render-config.sh → nama komponen inventory.
|
||||
RENDER_HOSTS = (
|
||||
"postgres", "minio", "nessie", "keycloak", "openldap", "opa",
|
||||
"trino-coordinator", "trino-worker", "starrocks-fe", "starrocks-cn",
|
||||
"airflow-webserver", "openmetadata-server", "openmetadata-opensearch",
|
||||
"prometheus", "grafana", "dremio", "statsd-exporter", "trino-jmx-exporter",
|
||||
)
|
||||
# Catatan: co-location ditangani render_config_map() — komponen SE-HOST TIDAK ditulis ke
|
||||
# alamat host (biar tetap nama container, satu jaringan Docker). Maka daftar ini boleh
|
||||
# lengkap; yang penting hostname-nya ada di KNOWN_HOSTS render-config.sh.
|
||||
|
||||
# node_exporter = PRA-SYARAT HOST (dipasang di luar installer, port 9100). Semua
|
||||
# host di cluster.yml dianggap menjalankannya; generate.py menulis daftar target
|
||||
# file_sd (job `node`) ke bundle host pemilik Prometheus.
|
||||
NODE_EXPORTER_PORT = 9100
|
||||
|
||||
|
||||
def load_compose(path: str) -> dict:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return yaml.safe_load(fh) or {}
|
||||
|
||||
|
||||
def keep_only(services: dict, wanted: set[str]) -> dict:
|
||||
return {name: spec for name, spec in services.items() if name in wanted}
|
||||
|
||||
|
||||
def prune_depends_on(services: dict) -> list[str]:
|
||||
"""Buang `depends_on` lintas host; kembalikan daftar yang dibuang."""
|
||||
dropped: list[str] = []
|
||||
for name, spec in services.items():
|
||||
deps = spec.get("depends_on")
|
||||
if deps is None:
|
||||
continue
|
||||
if isinstance(deps, dict):
|
||||
gone = set(deps) - set(services)
|
||||
kept: object = {k: v for k, v in deps.items() if k in services}
|
||||
else:
|
||||
gone = set(deps) - set(services)
|
||||
kept = [d for d in deps if d in services]
|
||||
dropped += [f"{name} → {g}" for g in sorted(gone)]
|
||||
if kept:
|
||||
spec["depends_on"] = kept
|
||||
else:
|
||||
spec.pop("depends_on", None)
|
||||
return dropped
|
||||
|
||||
|
||||
def used_volumes(services: dict) -> set[str]:
|
||||
used: set[str] = set()
|
||||
for spec in services.values():
|
||||
for vol in spec.get("volumes", []) or []:
|
||||
if isinstance(vol, str):
|
||||
src = vol.split(":", 1)[0]
|
||||
elif isinstance(vol, dict):
|
||||
src = vol.get("source", "")
|
||||
else:
|
||||
continue
|
||||
if src and not src.startswith((".", "/", "~")):
|
||||
used.add(src)
|
||||
return used
|
||||
|
||||
|
||||
def build_compose(base: dict, wanted: set[str]) -> tuple[dict, list[str]]:
|
||||
services = keep_only(base["services"], wanted)
|
||||
dropped = prune_depends_on(services)
|
||||
declared = base.get("volumes") or {}
|
||||
out: dict = {"services": services}
|
||||
if base.get("name"):
|
||||
out["name"] = base["name"]
|
||||
out["volumes"] = {n: s for n, s in declared.items() if n in used_volumes(services)}
|
||||
for key in ("networks", "configs", "secrets"):
|
||||
if key in base:
|
||||
out[key] = base[key]
|
||||
return out, dropped
|
||||
|
||||
|
||||
def write_compose(path: Path, doc: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
fh.write("# DIBANGKITKAN OTOMATIS — jangan disunting tangan.\n")
|
||||
fh.write("# Sumber: docker-compose.yml + cluster.yml · regenerasi: ./install.sh --generate\n")
|
||||
yaml.safe_dump(doc, fh, sort_keys=False, default_flow_style=False, allow_unicode=True)
|
||||
|
||||
|
||||
def _is_topology_var(line: str) -> bool:
|
||||
"""Baris .env yang menjadi milik INVENTORY (bukan isi .env sumber).
|
||||
|
||||
Kalau tidak disaring, `.env` sumber ikut menyalin `EXTERNAL_SERVICES`/
|
||||
`*_HOST` milik single-host → di dotenv yang MENANG adalah yang TERAKHIR,
|
||||
sehingga nilai dari inventory bisa hilang diam-diam (bug yang ditemukan saat
|
||||
uji: `EXTERNAL_SERVICES` muncul dua kali).
|
||||
"""
|
||||
key = line.split("=", 1)[0].strip()
|
||||
owned = {
|
||||
"EXTERNAL_SERVICES", "LOCAL_SERVICES", "KEYCLOAK_SSL_REQUIRED",
|
||||
"KC_PROXY_HEADERS", "TRINO_PROCESS_FORWARDED", "CLUSTER_DOMAIN", "TRAEFIK_BIND_ADDR",
|
||||
*ENDPOINT_VARS, *URL_ENDPOINT_VARS,
|
||||
*inventory.PUBLIC_URL_SERVICES, # KEYCLOAK_PUBLIC_URL, GRAFANA_PUBLIC_URL, …
|
||||
}
|
||||
return key in owned or key.startswith("SICABE_HOST_")
|
||||
|
||||
|
||||
def render_env(base_env: Path, out_env: Path, host: str, spec: dict, endpoints: list[str]) -> None:
|
||||
lines: list[str] = []
|
||||
if base_env.is_file():
|
||||
for line in base_env.read_text(encoding="utf-8").splitlines():
|
||||
if _is_topology_var(line):
|
||||
continue # ditentukan inventory, bukan .env sumber
|
||||
lines.append(line)
|
||||
else:
|
||||
lines.append("# (tidak ada .env sumber — rahasia wajib dilengkapi manual)")
|
||||
lines += [
|
||||
"",
|
||||
"# ── Ditambahkan generator multi-host (Fase 1) — variabel topologi ──",
|
||||
f"SICABE_HOST_NAME={host}",
|
||||
f"SICABE_HOST_ADDRESS={spec['address']}",
|
||||
f"SICABE_HOST_TIERS={','.join(spec['tiers'])}",
|
||||
]
|
||||
lines += endpoints
|
||||
out_env.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_env.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
# Port KONTAINER yang harus terjangkau lintas host (Fase 2). Variabel `*_HOST`
|
||||
# hanya memuat NAMA host (tanpa port), jadi port diambil dari referensi compose =
|
||||
# port container (5432, 8080, …). Selama port container itu tidak terbuka di alamat
|
||||
# host, container di host lain tidak bisa menjangkau — jadi generator menambahkannya.
|
||||
#
|
||||
# Dikunci ke ALAMAT HOST (mis. `10.20.0.12:5432:5432`), bukan `0.0.0.0`, supaya hanya
|
||||
# terjangkau lewat NIC privat host itu (R10 §4.8). Ini SENGAJA hanya di artefak
|
||||
# multi-host: instalasi single-host tidak pernah membuka port internal ini, dan
|
||||
# `docker-compose.yml` tetap byte-identik (invarian Fase 0 terjaga).
|
||||
# Eksposur untuk manusia/klien tetap memakai port host `12xxx` yang sudah ada.
|
||||
INTERNAL_PORTS: dict[str, list[int]] = {
|
||||
"postgres": [5432],
|
||||
"minio": [9000],
|
||||
"nessie": [19120],
|
||||
"keycloak": [8080],
|
||||
"openldap": [389],
|
||||
"opa": [8181],
|
||||
# 8080 = worker mendaftar (discovery) · 8443 = klien lintas host (OM ingestion,
|
||||
# task dbt, native DQ) — keduanya dibutuhkan begitu coordinator & worker beda host.
|
||||
"trino-coordinator": [8080, 8443],
|
||||
"starrocks-fe": [9030, 8030],
|
||||
"starrocks-cn": [9050, 8040, 8060],
|
||||
"openmetadata-opensearch": [9200],
|
||||
"openmetadata-server": [8585],
|
||||
"airflow-webserver": [8080],
|
||||
# Sidecar JMX di host `coord`: Prometheus (di host `governance`) men-scrape 9404
|
||||
# LINTAS host → portnya WAJIB diterbitkan di alamat host pemilik.
|
||||
"trino-jmx-exporter": [9404],
|
||||
}
|
||||
|
||||
|
||||
def _reachable_on_host_port(entry, cport: int, address: str) -> bool:
|
||||
"""Apakah `entry` benar-benar membuka port container `cport` DI alamat host?
|
||||
|
||||
Jebakan yang sempat lolos: `12080:8080` memetakan port host 12080 → container
|
||||
8080. Itu TIDAK membuat port 8080 terjangkau di host, sehingga container di host
|
||||
lain tetap tidak bisa menjangkau. Yang dihitung hanya pemetaan yang port host-nya
|
||||
SAMA dengan port container (mis. `addr:8080:8080`).
|
||||
"""
|
||||
if isinstance(entry, str):
|
||||
parts = entry.split(":")
|
||||
if len(parts) == 2:
|
||||
host_ip, host_port = None, parts[0]
|
||||
elif len(parts) == 3:
|
||||
host_ip, host_port = parts[0], parts[1]
|
||||
else:
|
||||
return False
|
||||
if host_port != str(cport):
|
||||
return False
|
||||
return host_ip in (None, "", address)
|
||||
if isinstance(entry, dict):
|
||||
published = entry.get("published")
|
||||
return (
|
||||
entry.get("target") == cport
|
||||
and str(published) == str(cport)
|
||||
and entry.get("host_ip") in (None, "", address)
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def tier_implied_components(placement: dict, plan: dict) -> set[str]:
|
||||
"""Komponen yang PASTI dikonsumsi lintas host walau tidak terlihat di env/config.
|
||||
|
||||
`trino-coordinator` selalu dipakai host bertier `pipeline` (task dbt Airflow) dan
|
||||
`governance` (ingestion OpenMetadata). Referensinya tidak dapat ditemukan dengan
|
||||
pemindaian env/config karena `ingest-trino.yaml` **disalin ke container saat
|
||||
runtime** (docker cp), bukan di-mount — celah ini ditemukan saat menguji preflight.
|
||||
"""
|
||||
needed: set[str] = set()
|
||||
owners = placement.get("trino-coordinator") or []
|
||||
if not owners:
|
||||
return needed
|
||||
owner = owners[0]
|
||||
for host, spec in plan.items():
|
||||
if host == owner:
|
||||
continue
|
||||
if {"pipeline", "governance"} & set(spec["tiers"]):
|
||||
needed.add("trino-coordinator")
|
||||
break
|
||||
return needed
|
||||
|
||||
|
||||
def config_referenced_components(deploy_dir: Path, base: dict, plan: dict,
|
||||
placement: dict, known: set[str]) -> set[str]:
|
||||
"""Komponen yang dirujuk dari file CONFIG yang di-mount oleh service di host lain.
|
||||
|
||||
Deteksi berbasis environment saja tidak cukup: Trino tidak memakai `*_HOST` untuk
|
||||
OAuth2, melainkan `config/trino/config-coordinator.properties`
|
||||
(`http://keycloak:8080/...`). Tanpa pemindaian ini, port Keycloak tidak dibuka dan
|
||||
Trino di host lain gagal mengambil JWKS.
|
||||
"""
|
||||
import re
|
||||
|
||||
needed: set[str] = set()
|
||||
for host, spec in plan.items():
|
||||
for svc in spec["services"]:
|
||||
for vol in base["services"].get(svc, {}).get("volumes") or []:
|
||||
src = vol.split(":", 1)[0] if isinstance(vol, str) else vol.get("source", "")
|
||||
if not src.startswith("./config/"):
|
||||
continue
|
||||
path = deploy_dir / src[2:]
|
||||
files = list(path.rglob("*")) if path.is_dir() else [path]
|
||||
for f in files:
|
||||
if not f.is_file():
|
||||
continue
|
||||
try:
|
||||
text = f.read_text(encoding="utf-8", errors="ignore")
|
||||
except OSError:
|
||||
continue
|
||||
for comp in known:
|
||||
if re.search(r"(?<![\w.-])" + re.escape(comp) + r"(?![\w-])", text):
|
||||
owners = placement.get(comp) or []
|
||||
if owners and host not in owners:
|
||||
needed.add(comp)
|
||||
return needed
|
||||
|
||||
|
||||
def cross_host_components(base: dict, placement: dict, plan: dict) -> set[str]:
|
||||
"""Komponen yang PUNYA konsumen di host lain → perlu port internal terbuka.
|
||||
|
||||
Tanpa perhitungan ini generator membuka port untuk semua komponen, dan di
|
||||
blueprint a3 itu langsung menabrak: `keycloak` dan `airflow-webserver` sama-sama
|
||||
meminta `<control>:8080` (dua service, satu port host) → `compose up` gagal
|
||||
'port is already allocated'. Padahal `airflow-webserver` hanya dikonsumsi
|
||||
OpenMetadata yang berada di host yang sama → tidak perlu port host sama sekali.
|
||||
|
||||
Bonus keamanan: port internal yang tidak dibutuhkan tidak pernah dibuka.
|
||||
"""
|
||||
var_to_component = {v: c for v, c in ENDPOINT_VARS.items()}
|
||||
var_to_component.update({v: c for v, (c, _t) in URL_ENDPOINT_VARS.items()})
|
||||
needed: set[str] = set()
|
||||
for host, spec in plan.items():
|
||||
for svc in spec["services"]:
|
||||
source = base["services"].get(svc, {})
|
||||
text = json.dumps(source.get("environment") or {})
|
||||
for var, component in var_to_component.items():
|
||||
if var not in text:
|
||||
continue
|
||||
owners = placement.get(component) or []
|
||||
if owners and host not in owners:
|
||||
needed.add(component)
|
||||
return needed
|
||||
|
||||
|
||||
def required_cross_host(base: dict, deploy_dir: Path, plan: dict, placement: dict) -> set[str]:
|
||||
"""Gabungan SEMUA alasan sebuah komponen butuh port lintas host.
|
||||
|
||||
Satu fungsi supaya generator dan preflight TIDAK bisa berbeda pendapat — memisahkan
|
||||
perhitungan ini sempat membuat preflight melewatkan trino-coordinator begitu aturan
|
||||
baru ditambahkan ke generator.
|
||||
"""
|
||||
known = set(base["services"]) & set(INTERNAL_PORTS)
|
||||
return (
|
||||
cross_host_components(base, placement, plan)
|
||||
| config_referenced_components(deploy_dir, base, plan, placement, known)
|
||||
| tier_implied_components(placement, plan)
|
||||
)
|
||||
|
||||
|
||||
def add_internal_ports(services: dict, address: str, needed: set[str]) -> list[str]:
|
||||
"""Tambahkan binding port internal ke alamat host; kembalikan yang ditambahkan."""
|
||||
added: list[str] = []
|
||||
for name, spec in services.items():
|
||||
if name not in needed:
|
||||
continue
|
||||
want = INTERNAL_PORTS.get(name)
|
||||
if not want:
|
||||
continue
|
||||
ports = spec.setdefault("ports", [])
|
||||
for cport in want:
|
||||
if any(_reachable_on_host_port(e, cport, address) for e in ports):
|
||||
continue
|
||||
ports.append(f"{address}:{cport}:{cport}")
|
||||
added.append(f"{name}:{cport}")
|
||||
return added
|
||||
|
||||
|
||||
def component_placements(doc: dict) -> dict[str, list[str]]:
|
||||
return inventory.resolve(doc)
|
||||
|
||||
|
||||
def extra_bind_dirs(base: dict) -> list[str]:
|
||||
"""Bind-mount relatif selain `config/` yang harus ikut disalin ke tiap host.
|
||||
|
||||
Contoh: `./scripts/airflow:/opt/airflow/scripts:ro`. Tanpa disalin, direktori
|
||||
generated tidak mandiri — Docker akan membuat direktori KOSONG di host tujuan
|
||||
(skrip Airflow hilang diam-diam). Ditemukan oleh test-generate.sh.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
for spec in base["services"].values():
|
||||
for vol in spec.get("volumes", []) or []:
|
||||
if isinstance(vol, str):
|
||||
src = vol.split(":", 1)[0]
|
||||
elif isinstance(vol, dict):
|
||||
src = vol.get("source", "")
|
||||
else:
|
||||
continue
|
||||
if src.startswith("./") and not src.startswith("./config/"):
|
||||
found.add(src[2:])
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def copy_extra_binds(deploy_dir: Path, host_dir: Path, paths: list[str]) -> list[str]:
|
||||
"""Salin bind relatif non-config (mis. `scripts/airflow`) ke direktori host."""
|
||||
import shutil
|
||||
|
||||
copied: list[str] = []
|
||||
for rel in paths:
|
||||
src = deploy_dir / rel
|
||||
if not src.exists():
|
||||
continue
|
||||
dst = host_dir / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if src.is_dir():
|
||||
shutil.copytree(src, dst, dirs_exist_ok=True)
|
||||
else:
|
||||
shutil.copy2(src, dst)
|
||||
copied.append(rel)
|
||||
return copied
|
||||
|
||||
|
||||
def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]:
|
||||
"""Baris `VAR=alamat|nama` untuk host ini.
|
||||
|
||||
SE-HOST → **nama service** (satu jaringan Docker). Port container TIDAK diterbitkan
|
||||
di host (mis. OM 8585), jadi menulis alamat host justru "connection refused"
|
||||
(regresi: Airflow/OpenLineage → `http://<ip-control>:8585/api`). Beda host → alamat
|
||||
host pemilik (port internal diterbitkan generator untuk itu).
|
||||
"""
|
||||
placement = component_placements(doc)
|
||||
lines: list[str] = []
|
||||
for var, component in ENDPOINT_VARS.items():
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
if host in owners:
|
||||
lines.append(f"{var}={component}") # SE-HOST → nama service
|
||||
else:
|
||||
lines.append(f"{var}={plan[owners[0]]['address']}") # LINTAS host → alamat
|
||||
disabled = [n for n, v in inventory.enabled_components(doc).items() if v is False]
|
||||
if disabled:
|
||||
# Komponen `enabled: false` = tidak dijalankan di host mana pun → installer
|
||||
# harus melewatinya (install.sh: EXTERNAL_SERVICES menang atas auto-deteksi).
|
||||
lines.append('EXTERNAL_SERVICES="%s"' % " ".join(disabled))
|
||||
return lines
|
||||
|
||||
|
||||
def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]:
|
||||
"""Baris `*_PUBLIC_URL=...` — URL yang dilihat BROWSER (issuer OIDC, redirect, UI),
|
||||
bukan lalu lintas container.
|
||||
|
||||
tls.mode != none : `https://<label>.<domain>` (inventory.public_urls).
|
||||
tls.mode == none : `http(s)://<alamat HOST PEMILIK>:<port>`. Sebelumnya mode ini
|
||||
mengembalikan daftar kosong, sehingga .env host memakai default
|
||||
`http://localhost:<port>` — dan browser di host lain dibuang ke
|
||||
localhost (issuer OIDC pun localhost). Sekarang alamatnya diambil
|
||||
dari host yang menjalankan komponen, jadi akses lintas host benar
|
||||
tanpa perlu nilai berbeda per bundle.
|
||||
"""
|
||||
by_domain = inventory.public_urls(doc)
|
||||
if by_domain:
|
||||
return [f"{var}={url}" for var, url in sorted(by_domain.items())]
|
||||
if plan is None:
|
||||
return []
|
||||
|
||||
placement = component_placements(doc)
|
||||
resolved: dict[str, str] = {}
|
||||
for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items():
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
# Sengaja memakai alamat PEMILIK (bukan host lokal) dan urutan yang pasti,
|
||||
# supaya nilainya identik di setiap bundle.
|
||||
resolved[var] = tmpl.format(addr=plan[sorted(owners)[0]]["address"])
|
||||
return [f"{var}={url}" for var, url in sorted(resolved.items())]
|
||||
|
||||
|
||||
def tls_policy_lines(doc: dict) -> list[str]:
|
||||
"""Kebijakan keamanan turunan `tls.mode` (saat ini hanya realm Keycloak).
|
||||
|
||||
Keycloak DEFAULT `sslRequired=external` menuntut HTTPS untuk permintaan dari alamat
|
||||
NON-lokal. Alamat Tailscale `100.64.0.0/10` (CGNAT) TERMASUK non-lokal bagi Keycloak
|
||||
(`isLocal` = loopback + RFC1918 + link-local + `fc00::/7`), sehingga admin console di
|
||||
`http://<alamat-tailscale>:12080` gagal dengan "HTTPS required" — dan Keycloak 26
|
||||
TIDAK punya opsi server-level untuk ini (tidak ada `KC_HTTPS_REQUIRED`), hanya per
|
||||
realm. Karena itu nilainya diteruskan ke bootstrap.sh.
|
||||
|
||||
tls.mode `none` -> `none` (HTTP polos; instalasi dev/simulasi)
|
||||
tls.mode `edge`/`full` -> `external` (ada Traefik/TLS di depan)
|
||||
"""
|
||||
edge = inventory.tls_mode(doc) != "none"
|
||||
return [
|
||||
f"KEYCLOAK_SSL_REQUIRED={'external' if edge else 'none'}",
|
||||
# Di belakang Traefik, Keycloak WAJIB memercayai X-Forwarded-* — tanpa itu
|
||||
# permintaan HTTPS dari proxy dianggap HTTP (redirect salah / tuntutan HTTPS).
|
||||
# Sengaja KOSONG saat tanpa proxy: memasangnya membuat Keycloak memercayai
|
||||
# header yang bisa dipalsukan klien.
|
||||
f"KC_PROXY_HEADERS={'xforwarded' if edge else ''}",
|
||||
# Trino: X-Forwarded-Proto supaya skema publik benar saat di belakang proxy.
|
||||
f"TRINO_PROCESS_FORWARDED={'true' if edge else 'false'}",
|
||||
]
|
||||
|
||||
|
||||
# Rute untuk komponen publik yang TIDAK berjalan di host ingress. Traefik hanya melihat
|
||||
# container di host-nya sendiri, jadi komponen di host lain dirutekan lewat berkas
|
||||
# dinamis (provider file) ke PORT HOST-nya (12xxx), bukan port container.
|
||||
# Nilai: (label rute, URL, butuh serverstransport-insecure?)
|
||||
REMOTE_ROUTE_TARGETS = {
|
||||
"keycloak": ("kc", "http://{addr}:12080", False),
|
||||
"grafana": ("grafana", "http://{addr}:12300", False),
|
||||
"openmetadata-server": ("om", "http://{addr}:12085", False),
|
||||
"airflow-webserver": ("airflow", "http://{addr}:12082", False),
|
||||
"trino-coordinator": ("trino", "https://{addr}:12443", True),
|
||||
"minio": ("minio", "http://{addr}:12901", False),
|
||||
}
|
||||
|
||||
|
||||
def ingress_host(doc: dict, plan: dict) -> str:
|
||||
"""Host yang menjalankan ingress Traefik (T-20).
|
||||
|
||||
Default: host yang menjalankan `keycloak` (tier identity — pada preset a3 itu
|
||||
`control`), supaya tidak perlu setelan tambahan. Bisa dipatok eksplisit lewat
|
||||
`tls.ingress_host`. Kosong bila tls.mode `none` (memang tanpa ingress).
|
||||
"""
|
||||
if inventory.tls_mode(doc) == "none":
|
||||
return ""
|
||||
explicit = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
|
||||
if explicit:
|
||||
return explicit if explicit in plan else ""
|
||||
owners = sorted(component_placements(doc).get("keycloak") or [])
|
||||
if owners:
|
||||
return owners[0]
|
||||
return sorted(plan)[0] if plan else ""
|
||||
|
||||
|
||||
def remote_route_doc(doc: dict, plan: dict) -> dict | None:
|
||||
"""Isi berkas dinamis untuk rute komponen yang ada DI HOST LAIN.
|
||||
|
||||
Mengembalikan None bila tidak ada rute jarak jauh (semua komponen publik memang
|
||||
berada di host ingress).
|
||||
"""
|
||||
ing = ingress_host(doc, plan)
|
||||
if not ing:
|
||||
return None
|
||||
domain = inventory.cluster_domain(doc)
|
||||
placement = component_placements(doc)
|
||||
routers: dict = {}
|
||||
services: dict = {}
|
||||
for component, (label, tmpl, needs_insecure) in REMOTE_ROUTE_TARGETS.items():
|
||||
owners = sorted(placement.get(component) or [])
|
||||
if not owners or ing in owners:
|
||||
continue # sudah ditangani label container di host ini
|
||||
owner = owners[0]
|
||||
router = {
|
||||
"rule": f"Host(`{label}.{domain}`)",
|
||||
"entryPoints": ["websecure"],
|
||||
"service": label,
|
||||
"tls": {
|
||||
"certResolver": "le",
|
||||
"domains": [{"main": domain, "sans": [f"*.{domain}"]}],
|
||||
},
|
||||
}
|
||||
lb: dict = {"servers": [{"url": tmpl.format(addr=plan[owner]["address"])}]}
|
||||
if needs_insecure:
|
||||
lb["serversTransport"] = "trino-insecure"
|
||||
routers[label] = router
|
||||
services[label] = {"loadBalancer": lb}
|
||||
if not routers:
|
||||
return None
|
||||
# serversTransport wajib ikut di berkas ini: berkas dinamis yang dibangkitkan
|
||||
# MENGGANTIKAN salinan dasar, jadi definisinya harus ada di sini juga.
|
||||
return {
|
||||
"http": {
|
||||
"serversTransports": {"trino-insecure": {"insecureSkipVerify": True}},
|
||||
"routers": routers,
|
||||
"services": services,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def node_targets_doc(plan: dict) -> list[dict]:
|
||||
"""Berkas target file_sd untuk job `node`: satu entri `<alamat>:9100` per host.
|
||||
|
||||
node_exporter adalah PRA-SYARAT HOST (di luar installer), sehingga SEMUA host di
|
||||
cluster.yml dianggap menjalankannya di port 9100. Label `host` disalin ke
|
||||
`instance` oleh relabel_configs di prometheus.yml (panel menampilkan nama host).
|
||||
"""
|
||||
return [
|
||||
{"targets": [f"{spec['address']}:{NODE_EXPORTER_PORT}"], "labels": {"host": name}}
|
||||
for name, spec in sorted(plan.items())
|
||||
]
|
||||
|
||||
|
||||
def ingress_lines(doc: dict, host: str, plan: dict) -> list[str]:
|
||||
"""Variabel ingress untuk .env host (T-20).
|
||||
|
||||
`CLUSTER_DOMAIN` dipakai label rute Traefik (Host(`<label>.<domain>`)), dan
|
||||
`TRAEFIK_BIND_ADDR` membatasi port 80/443 ke alamat host ini — bukan semua antarmuka.
|
||||
Hanya ditulis saat ada TLS/ingress; tanpa itu Traefik memang tidak dijalankan.
|
||||
"""
|
||||
if inventory.tls_mode(doc) == "none":
|
||||
return []
|
||||
if host != ingress_host(doc, plan):
|
||||
return []
|
||||
return [
|
||||
f"CLUSTER_DOMAIN={inventory.cluster_domain(doc)}",
|
||||
f"TRAEFIK_BIND_ADDR={plan[host]['address']}",
|
||||
]
|
||||
|
||||
|
||||
def _endpoint_port(var, env_path):
|
||||
"""Port container untuk var URL (dari .env, fallback default URL_ENDPOINT_PORTS)."""
|
||||
spec = URL_ENDPOINT_PORTS.get(var)
|
||||
if not spec:
|
||||
return ""
|
||||
env_name, default = spec
|
||||
val = _source_env_value(env_path, env_name) if env_path else None
|
||||
return val if val else str(default)
|
||||
|
||||
|
||||
def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[str]:
|
||||
"""Baris `VAR=<url>` untuk konsumen bentuk URL.
|
||||
|
||||
SE-HOST → **nama service** (mis. `http://nessie:19120`, `http://opa:8181`);
|
||||
LINTAS host → alamat IP pemilik (port internalnya diterbitkan generator).
|
||||
Konsisten dengan endpoint_lines(): port container tak diterbitkan di host.
|
||||
Port diambil dari .env (mis. NESSIE_INTERNAL_PORT) bila di-override.
|
||||
"""
|
||||
placement = component_placements(doc)
|
||||
lines: list[str] = []
|
||||
for var, (component, template) in URL_ENDPOINT_VARS.items():
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
addr = component if host in owners else plan[owners[0]]["address"]
|
||||
lines.append(f"{var}={template.format(addr=addr, port=_endpoint_port(var, env_path))}")
|
||||
return lines
|
||||
|
||||
|
||||
def render_config_map(doc: dict, host: str, plan: dict) -> list[str]:
|
||||
"""Peta hostname container → alamat, untuk render-config.sh (khusus host ini).
|
||||
|
||||
HANYA komponen yang dimiliki HOST LAIN yang dipetakan. Komponen SE-HOST dibiarkan
|
||||
memakai NAMA container (satu jaringan Docker) — menulisnya ke alamat host justru
|
||||
RUSAK: port container sering tak diterbitkan di host. Regresi nyata: `prometheus`
|
||||
(se-host dengan grafana) ditulis `http://<ip-host>:9090` di datasources.yaml →
|
||||
Grafana "connection refused"; `statsd-exporter` (se-host dengan prometheus) serupa.
|
||||
"""
|
||||
placement = component_placements(doc)
|
||||
args: list[str] = []
|
||||
for component in RENDER_HOSTS:
|
||||
owners = placement.get(component)
|
||||
if not owners:
|
||||
continue
|
||||
if host in owners:
|
||||
continue # SE-HOST → biarkan nama container
|
||||
args += ["--map", f"{component}={plan[owners[0]]['address']}"]
|
||||
return args
|
||||
|
||||
|
||||
def cmd_render(argv: list[str]) -> int:
|
||||
if len(argv) < 7:
|
||||
sys.stderr.write(__doc__ or "")
|
||||
return 2
|
||||
inv_path, compose_path, env_path, out_base, renderer = argv[2:7]
|
||||
preset_override = argv[7] if len(argv) > 7 else ""
|
||||
doc = inventory.load(inv_path)
|
||||
if preset_override:
|
||||
doc["preset"] = preset_override
|
||||
doc = inventory.apply_preset(doc)
|
||||
errors = inventory.validate(doc)
|
||||
if errors:
|
||||
sys.stderr.write("cluster.yml tidak valid — jalankan 'inventory.py validate' untuk detail:\n")
|
||||
for e in errors:
|
||||
sys.stderr.write(f" ✗ {e}\n")
|
||||
return 1
|
||||
|
||||
base = load_compose(compose_path)
|
||||
extra_binds = extra_bind_dirs(base)
|
||||
plan = inventory.plan(doc)
|
||||
out_root = Path(out_base)
|
||||
placement = component_placements(doc)
|
||||
needed = required_cross_host(base, Path(compose_path).parent, plan, placement)
|
||||
total_dropped: dict[str, list[str]] = {}
|
||||
port_internal_all: set[str] = set()
|
||||
|
||||
# Trino cert/keystore: dibangkitkan SEKALI (SAN = ALAMAT HOST semua host), lalu
|
||||
# disalin IDENTIK ke tiap bundle. WAJIB sekali — kalau tiap host beda keypair, cert
|
||||
# klien (OM ingestion, dll.) tak cocok dgn cert server → verifikasi TLS gagal.
|
||||
#
|
||||
# DIPERSISTENKAN di deploy/.trino-keystore/ (gitignored): kalau cert dibuat BARU tiap
|
||||
# `--generate`, server Trino yang belum di-recreate menyajikan cert LAMA → klien
|
||||
# (GE/OM) "self-signed certificate". Stabil kecuali daftar SAN (alamat host) berubah.
|
||||
_ks_cache = Path(__file__).resolve().parents[2] / ".trino-keystore"
|
||||
_ks_cache.mkdir(parents=True, exist_ok=True)
|
||||
_trino_ks_src = _ks_cache
|
||||
_san = ",".join(f"IP:{plan[h]['address']}" for h in sorted(plan) if plan[h].get("address"))
|
||||
_ks_script = Path(renderer).resolve().parent / "generate-trino-keystore.sh"
|
||||
_marker = _ks_cache / "san.txt"
|
||||
_need_regen = (not (_ks_cache / "keystore.p12").is_file()
|
||||
or not _marker.is_file()
|
||||
or _marker.read_text(encoding="utf-8").strip() != _san)
|
||||
if _san and _ks_script.is_file() and _need_regen:
|
||||
subprocess.run([str(_ks_script), "--out", str(_trino_ks_src), "--san-extra", _san],
|
||||
capture_output=True, text=True)
|
||||
try:
|
||||
_marker.write_text(_san, encoding="utf-8")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
for host, spec in plan.items():
|
||||
host_dir = out_root / f"host-{host}"
|
||||
# 1) compose
|
||||
# Ingress: setiap host menjalankan Traefik-nya sendiri saat ada TLS/proxy
|
||||
# (satu sertifikat per host — acme.json tidak bisa dibagi antar host).
|
||||
# Tanpa TLS, service traefik memang tidak ikut.
|
||||
wanted = set(spec["services"])
|
||||
# Ingress satu host: HANYA host ingress yang menjalankan Traefik. Ini juga
|
||||
# menghilangkan perebutan record TXT DNS-01 (dulu tiap host menerbitkan
|
||||
# wildcard yang sama) dan menekan pemakaian batas sertifikat Let's Encrypt.
|
||||
ing = ingress_host(doc, plan)
|
||||
if ing and host == ing:
|
||||
wanted.add("traefik")
|
||||
compose_out, dropped = build_compose(base, wanted)
|
||||
# Fase 2: buka port internal ke alamat host ini agar komponen di host lain
|
||||
# dapat menjangkau (pengganti ketiadaan satu bridge network lintas host).
|
||||
added_ports = add_internal_ports(compose_out["services"], spec["address"], needed)
|
||||
port_internal_all.update(added_ports)
|
||||
write_compose(host_dir / "docker-compose.yml", compose_out)
|
||||
total_dropped[host] = dropped
|
||||
# 2) .env
|
||||
render_env(
|
||||
Path(env_path), host_dir / ".env", host, spec,
|
||||
endpoint_lines(doc, host, plan)
|
||||
+ url_endpoint_lines(doc, host, plan, env_path)
|
||||
+ public_url_lines(doc, plan)
|
||||
+ tls_policy_lines(doc)
|
||||
+ ingress_lines(doc, host, plan),
|
||||
)
|
||||
# 3) config ter-render
|
||||
# 3a) bind relatif non-config (mis. scripts/airflow) → bundle mandiri
|
||||
copied = copy_extra_binds(Path(compose_path).parent, host_dir, extra_binds)
|
||||
# 3b) config ter-render melalui render-config.sh
|
||||
# Bersihkan target berkas runtime yang mungkin berwujud DIREKTORI (sisa `up`
|
||||
# Docker saat sumber bind-mount belum ada). Tanpa ini render-config GAGAL
|
||||
# ("tulis gagal") dan generator berhenti SEBELUM menulis berkas runtime.
|
||||
for _rel in ("airflow/simple_auth_manager_passwords.json",
|
||||
"trino/password.db"):
|
||||
_t = host_dir / "config" / _rel
|
||||
if _t.is_dir():
|
||||
shutil.rmtree(_t)
|
||||
rc = subprocess.run(
|
||||
[renderer, "--out", str(host_dir / "config"), *render_config_map(doc, host, plan)],
|
||||
capture_output=True, text=True,
|
||||
)
|
||||
# Berkas dinamis rute komponen di host lain (hanya di host ingress).
|
||||
# Ditulis SETELAH render-config karena berkas ini menggantikan salinan dasar.
|
||||
remote = remote_route_doc(doc, plan)
|
||||
if remote is not None and host == ing:
|
||||
(host_dir / "config" / "traefik").mkdir(parents=True, exist_ok=True)
|
||||
with open(host_dir / "config" / "traefik" / "dynamic.yml", "w", encoding="utf-8") as fh:
|
||||
fh.write("# DIBANGKITKAN OTOMATIS oleh install.sh --generate — jangan disunting tangan.\n")
|
||||
fh.write("# Ingress satu host: rute untuk komponen yang TIDAK ada di host ini.\n")
|
||||
yaml.safe_dump(remote, fh, sort_keys=False, allow_unicode=True)
|
||||
# Target node_exporter (job `node`, file_sd) — hanya di host pemilik
|
||||
# Prometheus. Ditulis SETELAH render-config karena menggantikan salinan dasar.
|
||||
if host in (placement.get("prometheus") or []):
|
||||
tdir = host_dir / "config" / "prometheus" / "targets"
|
||||
tdir.mkdir(parents=True, exist_ok=True)
|
||||
with open(tdir / "node.yml", "w", encoding="utf-8") as fh:
|
||||
fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n")
|
||||
fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n")
|
||||
yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True)
|
||||
if rc.returncode != 0:
|
||||
sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}")
|
||||
return 1
|
||||
# Salin cert/keystore Trino (dibangkitkan SEKALI dgn SAN IP semua host) ke bundle
|
||||
# ini — keypair/cert IDENTIK lintas host agar verifikasi TLS server↔klien lolos.
|
||||
if (_trino_ks_src / "keystore.p12").is_file():
|
||||
(host_dir / "config" / "trino" / "certs").mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(_trino_ks_src / "keystore.p12",
|
||||
host_dir / "config" / "trino" / "keystore.p12")
|
||||
shutil.copy2(_trino_ks_src / "certs" / "trino.crt",
|
||||
host_dir / "config" / "trino" / "certs" / "trino.crt")
|
||||
# Berkas runtime gitignored (Airflow auth, Trino password.db) tidak ikut
|
||||
# render-config → tulis eksplisit agar compose tak me-mount sumber hilang (DIRECTORY).
|
||||
_write_bundle_runtime_files(host_dir, Path(env_path))
|
||||
print(f"host-{host:9s} {spec['address']:15s} services={len(spec['services'])} "
|
||||
f"depends_on_dibuang={len(dropped)} bind_disalin={len(copied)} "
|
||||
f"port_internal={len(added_ports)}")
|
||||
|
||||
print(f"komponen butuh port lintas host: {', '.join(sorted(needed)) or '—'}")
|
||||
if port_internal_all:
|
||||
print(f"port internal dibuka: {len(port_internal_all)} ("
|
||||
+ ", ".join(sorted(port_internal_all)) + ")")
|
||||
|
||||
# ringkasan untuk di-review
|
||||
for host, dropped in total_dropped.items():
|
||||
for item in dropped:
|
||||
print(f" [host-{host}] depends_on lintas host dibuang: {item}")
|
||||
print(json.dumps({h: len(s["services"]) for h, s in plan.items()}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def _source_env_value(env_path, key):
|
||||
"""Ambil satu nilai dari .env sumber (kutip luar dibuang). None bila tak ada."""
|
||||
try:
|
||||
text = Path(env_path).read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
return None
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
name, _, value = line.partition("=")
|
||||
if name.strip() != key:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
||||
value = value[1:-1]
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _write_bundle_runtime_files(host_dir, env_path):
|
||||
"""Tulis berkas runtime yang TIDAK di-track git ke bundle host.
|
||||
|
||||
compose generated me-mount path ini sebagai FILE. Bila file tidak ada, Docker
|
||||
membuat DIRECTORY di host → service gagal (Airflow SimpleAuthManager:
|
||||
IsADirectoryError; Trino: password.db bukan file). ``render-config`` tidak
|
||||
menyalinnya (gitignored), jadi harus ditulis eksplisit di sini.
|
||||
"""
|
||||
cfg = host_dir / "config"
|
||||
# Dir bind-mount yang ABSEN di git (config/airflow/sling hanya memuat env.yaml yang
|
||||
# gitignored) → Docker (daemon root) membuatnya sebagai ROOT saat `up` → operator
|
||||
# non-root tak bisa menulis env.yaml (bootstrap gagal). Pastikan ada & user-owned.
|
||||
(cfg / "airflow" / "sling").mkdir(parents=True, exist_ok=True)
|
||||
# 1) Airflow SimpleAuthManager — {"admin": <password>} (mode a+ → wajib writable)
|
||||
pw_json = cfg / "airflow" / "simple_auth_manager_passwords.json"
|
||||
pw_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
if pw_json.is_dir():
|
||||
shutil.rmtree(pw_json)
|
||||
admin_pw = _source_env_value(env_path, "AIRFLOW_ADMIN_PASSWORD")
|
||||
if admin_pw is not None:
|
||||
pw_json.write_text(json.dumps({"admin": admin_pw}) + "\n", encoding="utf-8")
|
||||
elif not pw_json.is_file():
|
||||
pw_json.write_text("{}\n", encoding="utf-8")
|
||||
pw_json.chmod(0o666)
|
||||
# 2) Trino password.db — byte-identik dari sumber bila ada; kalau tidak, FILE kosong
|
||||
# (yang penting BUKAN direktori; bootstrap mengisinya).
|
||||
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
|
||||
dst_db = cfg / "trino" / "password.db"
|
||||
dst_db.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst_db.is_dir():
|
||||
shutil.rmtree(dst_db)
|
||||
if src_db.is_file():
|
||||
shutil.copy2(src_db, dst_db)
|
||||
elif not dst_db.is_file():
|
||||
dst_db.write_text("", encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
sys.stderr.write(__doc__ or "")
|
||||
return 2
|
||||
if argv[1] == "render":
|
||||
return cmd_render(argv)
|
||||
sys.stderr.write(f"perintah tak dikenal: {argv[1]}\n")
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""inventory.py — cluster.yml: parse, validasi, resolusi penempatan (Fase 1 multi-host).
|
||||
|
||||
Ini otak Fase 1 (R10 §4.3/§4.5, R11 §3–§5). Satu implementasi otoritatif dipakai
|
||||
oleh `lib/cluster/generate.sh` dan `install.sh`; tidak ada logika penempatan yang
|
||||
diduplikasi di shell.
|
||||
|
||||
Model:
|
||||
- Tier (7): state, identity, policy, coord, engine, pipeline, governance.
|
||||
Tiap service compose dipetakan ke tepat satu tier (TIER_SERVICES).
|
||||
- Preset (a3/b5) hanya mengisi DEFAULT `hosts.<h>.tiers`; komponen diturunkan
|
||||
dari tier-nya. Override eksplisit `components.<c>.host` menang.
|
||||
- Tier `engine` boleh >1 host (scale-out): komponennya muncul di SEMUA host
|
||||
bertier engine. Tier lain tepat 1 host.
|
||||
|
||||
Perintah:
|
||||
inventory.py validate <cluster.yml> # exit 0 = valid; cetak ringkasan/error
|
||||
inventory.py hosts <cluster.yml> # daftar host
|
||||
inventory.py plan <cluster.yml> # JSON: host -> {address, tiers, services}
|
||||
inventory.py components <cluster.yml> # komponen -> host (untuk dokumentasi)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
sys.stderr.write("PyYAML wajib tersedia (python3 -c 'import yaml')\n")
|
||||
raise SystemExit(2)
|
||||
|
||||
# ─── Taksonomi (R11 §3.1) ───────────────────────────────────────────────────
|
||||
# Setiap service compose milik TEPAT SATU tier.
|
||||
TIER_SERVICES: dict[str, list[str]] = {
|
||||
"state": ["postgres", "minio", "minio-init", "openmetadata-opensearch"],
|
||||
"identity": ["keycloak", "openldap", "nessie"],
|
||||
"policy": ["opa"],
|
||||
"coord": ["trino-coordinator", "starrocks-fe", "trino-jmx-exporter"],
|
||||
"engine": ["trino-worker", "starrocks-cn", "dremio"],
|
||||
"pipeline": [
|
||||
"airflow-volume-init",
|
||||
"airflow-init",
|
||||
"airflow-webserver",
|
||||
"airflow-dag-processor",
|
||||
"airflow-scheduler",
|
||||
],
|
||||
# `prometheus` menyusul R11 (belum ada di taksonomi 7-tier). Ditempatkan di
|
||||
# `governance`: pada blueprint A maupun B ia berada di host yang sama, jadi
|
||||
# granularitas tier tidak mengubah penempatan.
|
||||
# `statsd-exporter` (M2 observabilitas) ikut `governance` bersama Prometheus;
|
||||
# pada a3/b5 tier ini se-host dengan `pipeline` (Airflow), sehingga UDP
|
||||
# StatsD Airflow→statsd-exporter tetap lokal.
|
||||
"governance": ["openmetadata-migrate", "openmetadata-server", "grafana", "prometheus", "statsd-exporter"],
|
||||
}
|
||||
|
||||
VALID_TIERS = tuple(TIER_SERVICES)
|
||||
|
||||
# ─── TLS & URL publik (langkah 1: hanya skema/var; BELUM mengaktifkan TLS) ──
|
||||
# `none` = perilaku sekarang (http, IP+port 12xxx). `edge` = browser memakai nama
|
||||
# domain lewat ingress, bidang internal tetap plaintext. `full` = seluruh segmen TLS.
|
||||
# Default `none` supaya TANPA cluster.yml perilaku single-host tidak berubah.
|
||||
TLS_MODES = ("none", "edge", "full")
|
||||
|
||||
# Layanan yang punya URL BROWSER-FACING → label subdomain di bawah `domain`.
|
||||
# Dipakai juga oleh ingress (step 2) sebagai daftar rute Host(), jadi nama di sini
|
||||
# adalah satu-satunya sumber kebenaran penamaan.
|
||||
PUBLIC_URL_SERVICES: dict[str, str] = {
|
||||
"KEYCLOAK_PUBLIC_URL": "kc",
|
||||
"GRAFANA_PUBLIC_URL": "grafana",
|
||||
"OPENMETADATA_PUBLIC_URL": "om",
|
||||
"AIRFLOW_PUBLIC_URL": "airflow",
|
||||
"TRINO_PUBLIC_URL": "trino",
|
||||
"MINIO_CONSOLE_PUBLIC_URL": "minio",
|
||||
}
|
||||
|
||||
# Blueprint: default tiers per host (R11 §4.1/§4.2).
|
||||
PRESETS: dict[str, dict[str, list[str]]] = {
|
||||
"a3": {
|
||||
"control": ["identity", "policy", "pipeline", "governance"],
|
||||
"compute": ["coord", "engine"],
|
||||
"state": ["state"],
|
||||
},
|
||||
"b5": {
|
||||
"control": ["identity", "policy", "pipeline", "governance"],
|
||||
"coord": ["coord"],
|
||||
"engine-1": ["engine"],
|
||||
"engine-2": ["engine"],
|
||||
"state": ["state"],
|
||||
},
|
||||
}
|
||||
|
||||
# Komponen stateful: tidak boleh berpindah host tanpa migrasi volume (R10 R7).
|
||||
PINNED = ("postgres", "minio", "openmetadata-opensearch", "starrocks-fe", "starrocks-cn", "dremio")
|
||||
|
||||
|
||||
def tier_of(service: str) -> str | None:
|
||||
"""Tier pemilik sebuah service compose."""
|
||||
for tier, services in TIER_SERVICES.items():
|
||||
if service in services:
|
||||
return tier
|
||||
return None
|
||||
|
||||
|
||||
def load(path: str) -> dict:
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
return yaml.safe_load(fh) or {}
|
||||
except yaml.YAMLError as exc:
|
||||
raise SystemExit(f"cluster.yml bukan YAML yang sah: {exc}") from exc
|
||||
|
||||
|
||||
def apply_preset(doc: dict) -> dict:
|
||||
"""Isi `hosts` dari preset bila belum didefinisikan eksplisit.
|
||||
|
||||
Preset tak dikenal TIDAK menghentikan program di sini — biarkan `validate()`
|
||||
yang melaporkannya (dulu ini KeyError → traceback, bukan pesan yang berguna).
|
||||
"""
|
||||
preset = doc.get("preset") or "none"
|
||||
if preset not in PRESETS:
|
||||
return doc
|
||||
hosts = doc.setdefault("hosts", {})
|
||||
for name, tiers in PRESETS[preset].items():
|
||||
hosts.setdefault(name, {})
|
||||
hosts[name].setdefault("tiers", list(tiers))
|
||||
return doc
|
||||
|
||||
|
||||
def resolve(doc: dict) -> dict[str, list[str]]:
|
||||
"""Komponen → daftar host (tier `engine` bisa >1 host)."""
|
||||
hosts = doc.get("hosts") or {}
|
||||
tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs}
|
||||
explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()}
|
||||
|
||||
placement: dict[str, list[str]] = {}
|
||||
for service, tier in tiers_by_service.items():
|
||||
if service in explicit and explicit[service]:
|
||||
placement[service] = [explicit[service]]
|
||||
continue
|
||||
owners = [h for h, spec in hosts.items() if tier in (spec.get("tiers") or [])]
|
||||
if owners:
|
||||
placement[service] = sorted(owners)
|
||||
return placement
|
||||
|
||||
|
||||
def tls_mode(doc: dict) -> str:
|
||||
"""Mode TLS dari inventory (`none` bila tidak ditulis)."""
|
||||
return (doc.get("tls") or {}).get("mode") or "none"
|
||||
|
||||
|
||||
def cluster_domain(doc: dict) -> str:
|
||||
return ((doc.get("cluster") or {}).get("domain") or "").strip()
|
||||
|
||||
|
||||
def public_urls(doc: dict) -> dict[str, str]:
|
||||
"""URL browser-facing turunan dari tls.mode + domain.
|
||||
|
||||
Kosong bila mode `none` — dalam mode itu pemakaian tetap IP+port 12xxx seperti
|
||||
sekarang, sehingga perilaku single-host TIDAK berubah.
|
||||
"""
|
||||
mode = tls_mode(doc)
|
||||
domain = cluster_domain(doc)
|
||||
if mode == "none" or not domain:
|
||||
return {}
|
||||
scheme = "https" if mode in ("edge", "full") else "http"
|
||||
return {var: f"{scheme}://{label}.{domain}" for var, label in PUBLIC_URL_SERVICES.items()}
|
||||
|
||||
|
||||
def enabled_components(doc: dict) -> dict[str, bool]:
|
||||
return {
|
||||
name: bool(spec.get("enabled", True))
|
||||
for name, spec in (doc.get("components") or {}).items()
|
||||
}
|
||||
|
||||
|
||||
def validate(doc: dict) -> list[str]:
|
||||
"""Validasi R10 §4.3. Mengembalikan daftar error (kosong = valid)."""
|
||||
errors: list[str] = []
|
||||
|
||||
preset = doc.get("preset") or "none"
|
||||
if preset not in ("none", *PRESETS):
|
||||
errors.append(f"preset '{preset}' tidak dikenal (pakai: none|{'|'.join(PRESETS)})")
|
||||
|
||||
# TLS & domain (langkah 1)
|
||||
mode = (doc.get("tls") or {}).get("mode") or "none"
|
||||
if mode not in TLS_MODES:
|
||||
errors.append(f"tls.mode '{mode}' tidak dikenal (pakai: {'|'.join(TLS_MODES)})")
|
||||
domain = ((doc.get("cluster") or {}).get("domain") or "").strip()
|
||||
if mode != "none":
|
||||
if not domain:
|
||||
errors.append(f"tls.mode '{mode}' memerlukan cluster.domain (mis. sicabe.com)")
|
||||
elif "://" in domain or "/" in domain or "." not in domain:
|
||||
errors.append(f"cluster.domain '{domain}' tidak sah — isi nama domain saja "
|
||||
f"(mis. sicabe.com), tanpa skema atau garis miring")
|
||||
# Ingress satu host (T-20): opsional — default host yang menjalankan keycloak.
|
||||
ing = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
|
||||
if ing:
|
||||
if mode == "none":
|
||||
errors.append("tls.ingress_host tidak bermakna bila tls.mode 'none' "
|
||||
"(tanpa Traefik/TLS) — hapus atau ubah tls.mode")
|
||||
elif ing not in (doc.get("hosts") or {}):
|
||||
errors.append(f"tls.ingress_host '{ing}' bukan nama host yang dikenal "
|
||||
f"(host: {', '.join(sorted((doc.get('hosts') or {})))})")
|
||||
|
||||
hosts = doc.get("hosts") or {}
|
||||
if not hosts:
|
||||
errors.append("'hosts' kosong — inventory harus mendefinisikan minimal satu host")
|
||||
return errors
|
||||
|
||||
# (f) tiap host punya address + tier yang sah
|
||||
for name, spec in hosts.items():
|
||||
if not spec.get("address"):
|
||||
errors.append(f"host '{name}': 'address' wajib diisi")
|
||||
tiers = spec.get("tiers") or []
|
||||
if not tiers:
|
||||
errors.append(f"host '{name}': minimal satu tier")
|
||||
for tier in tiers:
|
||||
if tier not in VALID_TIERS:
|
||||
errors.append(f"host '{name}': tier '{tier}' tidak dikenal (sah: {', '.join(VALID_TIERS)})")
|
||||
|
||||
# (d) tier state tepat 1 host · R11 aturan 2 (state selalu host sendiri)
|
||||
state_hosts = [h for h, s in hosts.items() if "state" in (s.get("tiers") or [])]
|
||||
if len(state_hosts) != 1:
|
||||
errors.append(
|
||||
f"tier 'state' harus di TEPAT 1 host, ditemukan {len(state_hosts)}: "
|
||||
f"{state_hosts or '—'} (I/O-bound & pinned, R11 §3.2)"
|
||||
)
|
||||
|
||||
# (3) tier coord tepat 1 host (OSS Trino = 1 coordinator)
|
||||
coord_hosts = [h for h, s in hosts.items() if "coord" in (s.get("tiers") or [])]
|
||||
if len(coord_hosts) != 1:
|
||||
errors.append(f"tier 'coord' harus di TEPAT 1 host, ditemukan {len(coord_hosts)}: {coord_hosts or '—'}")
|
||||
|
||||
# engine: boleh >1 (scale-out) — hanya info bila tidak ada
|
||||
if not [h for h, s in hosts.items() if "engine" in (s.get("tiers") or [])]:
|
||||
errors.append("tidak ada host bertier 'engine' — tidak ada tempat eksekusi Trino/StarRocks")
|
||||
|
||||
# anti-affinity R11 §3.2: state ⊥ engine
|
||||
for h, s in hosts.items():
|
||||
t = set(s.get("tiers") or [])
|
||||
if "state" in t and "engine" in t:
|
||||
errors.append(f"host '{h}': tier 'state' dan 'engine' tidak boleh satu host (anti-affinity)")
|
||||
|
||||
# (a) komponen menunjuk host yang ada · (c) endpoint hanya ke komponen enabled
|
||||
components = doc.get("components") or {}
|
||||
for name, spec in components.items():
|
||||
host = spec.get("host")
|
||||
if host and host not in hosts:
|
||||
errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'")
|
||||
if tier_of(name) is None and name not in ("dremio",):
|
||||
errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)")
|
||||
|
||||
# (c) tiap komponen harus bisa ditempatkan
|
||||
placement = resolve(doc)
|
||||
for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs):
|
||||
if service not in placement:
|
||||
errors.append(
|
||||
f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier "
|
||||
f"'{tier_of(service)}' dan tidak ada components.{service}.host"
|
||||
)
|
||||
|
||||
# stateful: cegah pindah host tanpa migrasi (peringatan, bukan error)
|
||||
return errors
|
||||
|
||||
|
||||
def plan(doc: dict) -> dict[str, dict]:
|
||||
"""host → {address, tiers, services} untuk generator."""
|
||||
hosts = doc.get("hosts") or {}
|
||||
placement = resolve(doc)
|
||||
enabled = enabled_components(doc)
|
||||
|
||||
result: dict[str, dict] = {}
|
||||
for name, spec in hosts.items():
|
||||
result[name] = {
|
||||
"address": spec.get("address"),
|
||||
"tiers": spec.get("tiers") or [],
|
||||
"services": [],
|
||||
}
|
||||
for service, owners in placement.items():
|
||||
if enabled.get(service) is False:
|
||||
continue
|
||||
for owner in owners:
|
||||
if owner in result:
|
||||
result[owner]["services"].append(service)
|
||||
for name in result:
|
||||
result[name]["services"].sort()
|
||||
return result
|
||||
|
||||
|
||||
def warnings(doc: dict) -> list[str]:
|
||||
out: list[str] = []
|
||||
placement = resolve(doc)
|
||||
for service in PINNED:
|
||||
hosts = placement.get(service) or []
|
||||
if len(hosts) > 1:
|
||||
out.append(f"'{service}' stateful berada di >1 host {hosts} — butuh volume terpisah per host")
|
||||
return out
|
||||
|
||||
|
||||
def cmd_validate(path: str) -> int:
|
||||
doc = apply_preset(load(path))
|
||||
errors = validate(doc)
|
||||
warns = warnings(doc)
|
||||
if errors:
|
||||
print(f"cluster.yml TIDAK valid ({len(errors)} masalah):")
|
||||
for e in errors:
|
||||
print(f" ✗ {e}")
|
||||
return 1
|
||||
p = plan(doc)
|
||||
print(f"cluster.yml valid — preset '{doc.get('preset') or 'none'}'")
|
||||
for host, spec in p.items():
|
||||
print(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}] "
|
||||
f"services={len(spec['services'])}")
|
||||
for w in warns:
|
||||
print(f" ⚠ {w}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_plan(path: str) -> int:
|
||||
doc = apply_preset(load(path))
|
||||
errors = validate(doc)
|
||||
if errors:
|
||||
sys.stderr.write("cluster.yml tidak valid — jalankan 'validate' untuk detail\n")
|
||||
return 1
|
||||
print(json.dumps(plan(doc), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_components(path: str) -> int:
|
||||
doc = apply_preset(load(path))
|
||||
for service, owners in sorted(resolve(doc).items()):
|
||||
print(f"{service:24s} -> {','.join(owners)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_hosts(path: str) -> int:
|
||||
doc = apply_preset(load(path))
|
||||
for name, spec in (doc.get("hosts") or {}).items():
|
||||
print(f"{name} {spec.get('address')}")
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 3:
|
||||
sys.stderr.write(__doc__ or "")
|
||||
return 2
|
||||
command, path = argv[1], argv[2]
|
||||
if not Path(path).is_file():
|
||||
sys.stderr.write(f"tidak ditemukan: {path}\n")
|
||||
return 2
|
||||
commands = {
|
||||
"validate": cmd_validate,
|
||||
"plan": cmd_plan,
|
||||
"components": cmd_components,
|
||||
"hosts": cmd_hosts,
|
||||
}
|
||||
if command not in commands:
|
||||
sys.stderr.write(f"perintah tak dikenal: {command}\n")
|
||||
return 2
|
||||
return commands[command](path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""preflight.py — verifikasi kesiapan lintas host SEBELUM deploy (Fase 2, R10 §4.7).
|
||||
|
||||
Pengganti `depends_on` lintas host: generator MEMBUANG `depends_on` yang menyeberang
|
||||
host (Compose tidak bisa menyalakan service di host lain), sehingga tidak ada lagi yang
|
||||
menjamin urutan/ketersediaan antar host. Preflight menutup celah itu dengan memeriksa
|
||||
**dari sisi jaringan**: apakah tiap port yang dibutuhkan benar-benar terjangkau di
|
||||
alamat host pemiliknya.
|
||||
|
||||
Yang diperiksa:
|
||||
1. Inventory valid (aturan penempatan R10 §4.3 / R11 §3.2).
|
||||
2. Port internal yang dibutuhkan lintas host (perhitungan yang SAMA dengan generator —
|
||||
diimpor, bukan disalin) benar-benar terbuka di alamat host pemiliknya.
|
||||
3. TLS Trino coordinator (`8443`) menyelesaikan handshake.
|
||||
4. Selisih waktu antar host (header `Date` HTTP) — JWT/`notBefore` sensitif, jadi
|
||||
skew besar = kegagalan login yang sulit dilacak.
|
||||
|
||||
Pemakaian:
|
||||
preflight.py <cluster.yml> [--compose docker-compose.yml] [--deploy DIR] [--timeout N]
|
||||
Keluaran mengikuti konvensi gate repo: [PASS]/[FAIL]/[WARN]/[SKIP] + Ringkasan + recap.
|
||||
Exit 1 bila ada FAIL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
import ssl
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import generate # noqa: E402
|
||||
import inventory # noqa: E402
|
||||
|
||||
GRN = "\033[32m"; RED = "\033[31m"; YLW = "\033[33m"; BLD = "\033[1m"; NC = "\033[0m"
|
||||
|
||||
|
||||
class Report:
|
||||
def __init__(self) -> None:
|
||||
self.passed = 0
|
||||
self.failed = 0
|
||||
self.warned = 0
|
||||
self.skipped = 0
|
||||
self.fail_msgs: list[str] = []
|
||||
|
||||
def ok(self, msg: str) -> None:
|
||||
print(f" {GRN}[PASS]{NC} {msg}"); self.passed += 1
|
||||
|
||||
def bad(self, msg: str) -> None:
|
||||
print(f" {RED}[FAIL]{NC} {msg}"); self.failed += 1; self.fail_msgs.append(msg)
|
||||
|
||||
def warn(self, msg: str) -> None:
|
||||
print(f" {YLW}[WARN]{NC} {msg}"); self.warned += 1
|
||||
|
||||
def skip(self, msg: str) -> None:
|
||||
print(f" {YLW}[SKIP]{NC} {msg}"); self.skipped += 1
|
||||
|
||||
def recap(self) -> int:
|
||||
print()
|
||||
print(f"{BLD}Ringkasan:{NC} {self.passed} PASS · {self.failed} FAIL · "
|
||||
f"{self.warned} WARN · {self.skipped} SKIP")
|
||||
if self.fail_msgs:
|
||||
print(f"\n{BLD}Yang gagal ({len(self.fail_msgs)}):{NC}")
|
||||
for i, m in enumerate(self.fail_msgs, 1):
|
||||
print(f" {i}. {m}")
|
||||
return 1 if self.failed else 0
|
||||
|
||||
|
||||
def tcp_open(host: str, port: int, timeout: float) -> tuple[bool, str]:
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout):
|
||||
return True, ""
|
||||
except socket.timeout:
|
||||
return False, "timeout"
|
||||
except ConnectionRefusedError:
|
||||
return False, "connection refused (port tertutup)"
|
||||
except OSError as exc:
|
||||
return False, str(exc)
|
||||
|
||||
|
||||
def tls_handshake(host: str, port: int, timeout: float) -> tuple[bool, str]:
|
||||
"""Handshake TLS tanpa verifikasi sertifikat (deploy ini self-signed).
|
||||
|
||||
Yang diuji hanya: TLS tersedia dan handshake selesai. Verifikasi nama/host
|
||||
sengaja tidak dilakukan di sini — itu urusan klien masing-masing
|
||||
(mis. OM memakai `verify: /etc/ssl/trino/trino.crt`).
|
||||
"""
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout) as raw:
|
||||
with ctx.wrap_socket(raw, server_hostname=host) as tls:
|
||||
_ = tls.version()
|
||||
return True, ""
|
||||
except OSError as exc:
|
||||
return False, str(exc)
|
||||
|
||||
|
||||
def http_date(host: str, port: int, timeout: float) -> datetime | None:
|
||||
"""Ambil header `Date` dari respons HTTP paling sederhana (tanpa dependensi)."""
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=timeout) as sock:
|
||||
sock.sendall(f"HEAD / HTTP/1.0\r\nHost: {host}\r\n\r\n".encode())
|
||||
data = sock.recv(2048).decode("latin-1", errors="ignore")
|
||||
except OSError:
|
||||
return None
|
||||
for line in data.splitlines():
|
||||
if line.lower().startswith("date:"):
|
||||
try:
|
||||
return parsedate_to_datetime(line.split(":", 1)[1].strip())
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def clock_skew(host: str, port: int, timeout: float, report: Report) -> None:
|
||||
remote = http_date(host, port, timeout)
|
||||
if remote is None:
|
||||
return
|
||||
if remote.tzinfo is None:
|
||||
remote = remote.replace(tzinfo=timezone.utc)
|
||||
skew = abs((datetime.now(timezone.utc) - remote).total_seconds())
|
||||
if skew > 60:
|
||||
report.warn(f"{host}: selisih waktu {skew:.0f}s (>60s) — JWT/OIDC sensitif; "
|
||||
f"pastikan NTP aktif di host itu")
|
||||
else:
|
||||
report.ok(f"{host}: selisih waktu {skew:.0f}s (aman)")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) < 2:
|
||||
sys.stderr.write(__doc__ or "")
|
||||
return 2
|
||||
inv_path = argv[1]
|
||||
compose_path = "docker-compose.yml"
|
||||
deploy_dir = Path(".")
|
||||
timeout = 3.0
|
||||
rest = argv[2:]
|
||||
while rest:
|
||||
flag = rest.pop(0)
|
||||
if flag == "--compose" and rest:
|
||||
compose_path = rest.pop(0)
|
||||
elif flag == "--deploy" and rest:
|
||||
deploy_dir = Path(rest.pop(0))
|
||||
elif flag == "--timeout" and rest:
|
||||
timeout = float(rest.pop(0))
|
||||
else:
|
||||
sys.stderr.write(f"argumen tak dikenal: {flag}\n")
|
||||
return 2
|
||||
|
||||
report = Report()
|
||||
print(f"{BLD}Preflight lintas host{NC}")
|
||||
print(f" inventory : {inv_path}")
|
||||
print(f" compose : {compose_path}")
|
||||
print(f" {YLW}preflight hanya MEMERIKSA — tidak menyalakan apa pun. Urutan start "
|
||||
f"antar host tetap manual (belum ada orkestrator).{NC}")
|
||||
print()
|
||||
|
||||
# ── 1. inventory ────────────────────────────────────────────────────────
|
||||
print(f"{BLD}A. Inventory{NC}")
|
||||
doc = inventory.apply_preset(inventory.load(inv_path))
|
||||
errors = inventory.validate(doc)
|
||||
if errors:
|
||||
for e in errors:
|
||||
report.bad(f"inventory: {e}")
|
||||
return report.recap()
|
||||
plan = inventory.plan(doc)
|
||||
report.ok(f"inventory valid — preset '{doc.get('preset') or 'none'}' · {len(plan)} host")
|
||||
for host, spec in plan.items():
|
||||
report.ok(f" {host:9s} {spec['address']:15s} tiers=[{','.join(spec['tiers'])}]")
|
||||
|
||||
# ── 2. port internal yang dibutuhkan lintas host ─────────────────────────
|
||||
print(f"\n{BLD}B. Port lintas host{NC}")
|
||||
base = generate.load_compose(compose_path)
|
||||
placement = inventory.resolve(doc)
|
||||
needed = generate.required_cross_host(base, deploy_dir, plan, placement)
|
||||
if not needed:
|
||||
report.skip("tidak ada komponen yang dikonsumsi dari host lain")
|
||||
for component in sorted(needed):
|
||||
owners = placement.get(component) or []
|
||||
ports = generate.INTERNAL_PORTS.get(component) or []
|
||||
if len(owners) > 1:
|
||||
report.skip(f"{component}: di {len(owners)} host (scale-out) — "
|
||||
f"periksa manual {owners}")
|
||||
continue
|
||||
if not owners:
|
||||
report.bad(f"{component}: tidak ada host pemilik"); continue
|
||||
address = plan[owners[0]]["address"]
|
||||
for port in ports:
|
||||
open_ok, why = tcp_open(address, port, timeout)
|
||||
if open_ok:
|
||||
report.ok(f"{component} {address}:{port} terjangkau")
|
||||
else:
|
||||
report.bad(f"{component} {address}:{port} TIDAK terjangkau ({why})")
|
||||
|
||||
# ── 3. TLS Trino coordinator ─────────────────────────────────────────────
|
||||
print(f"\n{BLD}C. TLS Trino coordinator{NC}")
|
||||
trino_owners = placement.get("trino-coordinator") or []
|
||||
if trino_owners:
|
||||
addr = plan[trino_owners[0]]["address"]
|
||||
handshake_ok, why = tls_handshake(addr, 8443, timeout)
|
||||
if handshake_ok:
|
||||
report.ok(f"TLS handshake coordinator {addr}:8443 berhasil")
|
||||
else:
|
||||
report.bad(f"TLS handshake coordinator {addr}:8443 gagal ({why})")
|
||||
clock_skew(addr, 8443, timeout, report)
|
||||
else:
|
||||
report.skip("tidak ada trino-coordinator di inventory")
|
||||
|
||||
# ── 4. waktu antar host (lewat layanan HTTP) ─────────────────────────────
|
||||
print(f"\n{BLD}D. Waktu antar host{NC}")
|
||||
checked = 0
|
||||
for component in ("keycloak", "nessie", "opa"):
|
||||
owners = placement.get(component) or []
|
||||
port = {"keycloak": 8080, "nessie": 19120, "opa": 8181}[component]
|
||||
if len(owners) != 1:
|
||||
continue
|
||||
addr = plan[owners[0]]["address"]
|
||||
if tcp_open(addr, port, timeout)[0]:
|
||||
clock_skew(addr, port, timeout, report)
|
||||
checked += 1
|
||||
if checked == 0:
|
||||
report.skip("tidak ada layanan HTTP yang terjangkau untuk memeriksa waktu "
|
||||
"(pastikan NTP aktif di semua host)")
|
||||
|
||||
return report.recap()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Reference in New Issue
Block a user