1027 lines
46 KiB
Python
1027 lines
46 KiB
Python
#!/usr/bin/env python3
|
|
"""generate.py — render artefak deploy per host dari docker-compose.yml + cluster.yml.
|
|
|
|
Fase 1 R10 §4.5 (**Varian A: render, bukan overlay asli Compose**). Alasan: merge
|
|
overlay tidak dapat MENGHAPUS service, sedangkan kita perlu menyisakan hanya
|
|
service milik satu host.
|
|
|
|
Yang dihasilkan `generated/host-<x>/`:
|
|
1. `docker-compose.yml` — hanya service milik host itu.
|
|
2. `.env` — endpoint menunjuk ALAMAT HOST pemilik komponen (B1) + seluruh isi
|
|
`.env` sumber (rahasia wajib identik lintas host, S1 R10 §4.12).
|
|
3. `config/` — hasil `render-config.sh` (hostname container → alamat host), karena
|
|
berkas di `config/**` tidak dapat melakukan ekspansi env saat runtime.
|
|
|
|
`depends_on` lintas host DIBUANG (penggantinya `preflight`, Fase 2); intra-host
|
|
dipertahankan supaya urutan + healthcheck tetap berlaku di dalam satu host.
|
|
|
|
Hasil di `generated/` bersifat ARTEFAK (gitignored) — jangan disunting tangan.
|
|
|
|
Perintah:
|
|
generate.py render <inventory> <base-compose> <base-env> <out-base> <render-config.sh>
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
import inventory # noqa: E402 (modul sebelah, sengaja)
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError: # pragma: no cover
|
|
sys.stderr.write("PyYAML wajib tersedia\n")
|
|
raise SystemExit(2)
|
|
|
|
# Variabel endpoint (B1) → komponen pemiliknya. Dipakai untuk mengarahkan
|
|
# container ke ALAMAT HOST, bukan nama container (nama container hanya resolve
|
|
# di host yang sama).
|
|
ENDPOINT_VARS = {
|
|
"POSTGRES_HOST": "postgres",
|
|
"MINIO_HOST": "minio",
|
|
"KEYCLOAK_HOST": "keycloak",
|
|
"OPENMETADATA_HOST": "openmetadata-server",
|
|
"AIRFLOW_WEBSERVER_HOST": "airflow-webserver",
|
|
"STARROCKS_FE_HOST": "starrocks-fe",
|
|
"STARROCKS_CN_HOST": "starrocks-cn",
|
|
"OPENSEARCH_HOST": "openmetadata-opensearch",
|
|
# Var SISI-HOST yang belum ada padanannya, dipakai `scripts/verify-phase*` (mis. Trino
|
|
# & Grafana hanya punya *_PUBLIC_URL). Nilainya = alamat host pemilik. CATATAN:
|
|
# STARROCKS_HOST/AIRFLOW_HOST sengaja TIDAK ditulis di sini — skrip memakainya sebagai
|
|
# penanda komponen EXTERNAL (default `localhost`); mengisinya dengan alamat host akan
|
|
# salah menandai komponen lokal sebagai external.
|
|
"TRINO_HOST": "trino-coordinator",
|
|
# Dipakai dbt (profiles.yml: env_var DBT_TRINO_HOST) — lintas-host butuh ALAMAT IP.
|
|
"DBT_TRINO_HOST": "trino-coordinator",
|
|
"GRAFANA_HOST": "grafana",
|
|
"LDAP_HOST": "openldap",
|
|
"NESSIE_HOST": "nessie",
|
|
}
|
|
|
|
# Sebagian konsumen memakai bentuk URL (bukan `*_HOST`), dan berkas config yang
|
|
# dibaca container TIDAK dapat ekspansi env → URL-nya harus ikut diarahkan di sini,
|
|
# kalau tidak Trino/Sling/StarRocks lintas host akan mencari nama container yang
|
|
# hanya ada di host lain. (Komponen pemilik tetap diambil dari inventory.)
|
|
URL_ENDPOINT_VARS = {
|
|
"S3_ENDPOINT": ("minio", "http://{addr}:{port}"),
|
|
"NESSIE_URL": ("nessie", "http://{addr}:{port}"),
|
|
"LDAP_URL": ("openldap", "ldap://{addr}:{port}"),
|
|
"OPA_URL": ("opa", "http://{addr}:{port}"),
|
|
}
|
|
|
|
# Port container untuk var URL di atas — bisa ditimpa di .env (mis. layanan EXISTING
|
|
# berport non-standar). Default = port standar → perilaku lama tidak berubah.
|
|
URL_ENDPOINT_PORTS = {
|
|
"S3_ENDPOINT": ("MINIO_INTERNAL_PORT", 9000),
|
|
"NESSIE_URL": ("NESSIE_INTERNAL_PORT", 19120),
|
|
"LDAP_URL": ("OPENLDAP_INTERNAL_PORT", 389),
|
|
"OPA_URL": ("OPA_INTERNAL_PORT", 8181),
|
|
}
|
|
|
|
# URL BROWSER-FACING saat tls.mode = none (tanpa proxy/TLS): dialamatkan ke HOST yang
|
|
# MENJALANKAN komponen, bukan ke localhost maupun ke IP host lokal. Nilainya diambil
|
|
# dari host pemilik sehingga SAMA di semua bundle — wajib, karena `iss`/issuer OIDC
|
|
# harus satu nilai; kalau tiap host menulis alamatnya sendiri, validator akan tolak.
|
|
# Port di sini mengikuti skema 12xxx (sama seperti URL_ENDPOINT_VARS di bawah).
|
|
PUBLIC_URL_ENDPOINTS = {
|
|
"KEYCLOAK_PUBLIC_URL": ("keycloak", "http://{addr}:12080"),
|
|
"GRAFANA_PUBLIC_URL": ("grafana", "http://{addr}:12300"),
|
|
"OPENMETADATA_PUBLIC_URL": ("openmetadata-server", "http://{addr}:12085"),
|
|
"AIRFLOW_PUBLIC_URL": ("airflow-webserver", "http://{addr}:12082"),
|
|
"TRINO_PUBLIC_URL": ("trino-coordinator", "https://{addr}:12443"),
|
|
"MINIO_CONSOLE_PUBLIC_URL": ("minio", "http://{addr}:12901"),
|
|
}
|
|
|
|
|
|
def _ext_url(ext: str, template: str, port: str = "") -> str:
|
|
"""URL untuk komponen EKSTERNAL dari `components.<c>.address`.
|
|
|
|
Bila operator menulis **URL penuh** (mengandung `://`, mis.
|
|
`https://sso.example.com`) → pakai APA ADANYA: skema + host + port dihormati
|
|
(default port per skema, mis. https→443, TIDAK ditulis bila default). Bila hanya
|
|
host/IP polos (perilaku lama) → susun dari `template` + `port` seperti sebelumnya.
|
|
"""
|
|
scheme, host, p = inventory.parse_address(ext)
|
|
if scheme:
|
|
# Buang port bila = default skema (https→443), supaya `https://host`, bukan
|
|
# `https://host:443` (ekuivalen, tapi lebih bersih & cocok utk issuer/URL publik).
|
|
show = p and p != inventory._DEFAULT_SCHEME_PORT.get(scheme)
|
|
return f"{scheme}://{host}" + (f":{p}" if show else "")
|
|
if port and "{port}" in template:
|
|
return template.format(addr=host, port=(p if p else port))
|
|
return template.format(addr=host)
|
|
|
|
|
|
def _keycloak_external_scheme(doc: dict) -> str:
|
|
"""Skema URL Keycloak eksternal ('' bila bukan URL penuh). Untuk sslRequired."""
|
|
return inventory.parse_address(inventory.external_address(doc, "keycloak") or "")[0]
|
|
|
|
# SENGAJA TIDAK diarahkan ke IP: KEYCLOAK_URL / TRINO_URL dan var `*_URL` lain yang
|
|
# bersifat identitas (issuer OIDC harus stabil — lihat R10 §4.10.1: pakai DNS
|
|
# internal, bukan IP). Itu milik operator dan harus sudah benar di `cluster.yml`
|
|
# (domain) / `.env`.
|
|
|
|
# Hostname container yang dikenali render-config.sh → nama komponen inventory.
|
|
RENDER_HOSTS = (
|
|
"postgres", "minio", "nessie", "keycloak", "openldap", "opa",
|
|
"trino-coordinator", "trino-worker", "starrocks-fe", "starrocks-cn",
|
|
"airflow-webserver", "openmetadata-server", "openmetadata-opensearch",
|
|
"prometheus", "grafana", "dremio", "statsd-exporter", "trino-jmx-exporter",
|
|
)
|
|
# Catatan: co-location ditangani render_config_map() — komponen SE-HOST TIDAK ditulis ke
|
|
# alamat host (biar tetap nama container, satu jaringan Docker). Maka daftar ini boleh
|
|
# lengkap; yang penting hostname-nya ada di KNOWN_HOSTS render-config.sh.
|
|
|
|
# node_exporter = PRA-SYARAT HOST (dipasang di luar installer, port 9100). Semua
|
|
# host di cluster.yml dianggap menjalankannya; generate.py menulis daftar target
|
|
# file_sd (job `node`) ke bundle host pemilik Prometheus.
|
|
NODE_EXPORTER_PORT = 9100
|
|
|
|
|
|
def load_compose(path: str) -> dict:
|
|
with open(path, encoding="utf-8") as fh:
|
|
return yaml.safe_load(fh) or {}
|
|
|
|
|
|
def keep_only(services: dict, wanted: set[str]) -> dict:
|
|
return {name: spec for name, spec in services.items() if name in wanted}
|
|
|
|
|
|
def prune_depends_on(services: dict) -> list[str]:
|
|
"""Buang `depends_on` lintas host; kembalikan daftar yang dibuang."""
|
|
dropped: list[str] = []
|
|
for name, spec in services.items():
|
|
deps = spec.get("depends_on")
|
|
if deps is None:
|
|
continue
|
|
if isinstance(deps, dict):
|
|
gone = set(deps) - set(services)
|
|
kept: object = {k: v for k, v in deps.items() if k in services}
|
|
else:
|
|
gone = set(deps) - set(services)
|
|
kept = [d for d in deps if d in services]
|
|
dropped += [f"{name} → {g}" for g in sorted(gone)]
|
|
if kept:
|
|
spec["depends_on"] = kept
|
|
else:
|
|
spec.pop("depends_on", None)
|
|
return dropped
|
|
|
|
|
|
def used_volumes(services: dict) -> set[str]:
|
|
used: set[str] = set()
|
|
for spec in services.values():
|
|
for vol in spec.get("volumes", []) or []:
|
|
if isinstance(vol, str):
|
|
src = vol.split(":", 1)[0]
|
|
elif isinstance(vol, dict):
|
|
src = vol.get("source", "")
|
|
else:
|
|
continue
|
|
if src and not src.startswith((".", "/", "~")):
|
|
used.add(src)
|
|
return used
|
|
|
|
|
|
def build_compose(base: dict, wanted: set[str]) -> tuple[dict, list[str]]:
|
|
services = keep_only(base["services"], wanted)
|
|
dropped = prune_depends_on(services)
|
|
declared = base.get("volumes") or {}
|
|
out: dict = {"services": services}
|
|
if base.get("name"):
|
|
out["name"] = base["name"]
|
|
out["volumes"] = {n: s for n, s in declared.items() if n in used_volumes(services)}
|
|
for key in ("networks", "configs", "secrets"):
|
|
if key in base:
|
|
out[key] = base[key]
|
|
return out, dropped
|
|
|
|
|
|
def write_compose(path: Path, doc: dict) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write("# DIBANGKITKAN OTOMATIS — jangan disunting tangan.\n")
|
|
fh.write("# Sumber: docker-compose.yml + cluster.yml · regenerasi: ./install.sh --generate\n")
|
|
yaml.safe_dump(doc, fh, sort_keys=False, default_flow_style=False, allow_unicode=True)
|
|
|
|
|
|
def _is_topology_var(line: str) -> bool:
|
|
"""Baris .env yang menjadi milik INVENTORY (bukan isi .env sumber).
|
|
|
|
Kalau tidak disaring, `.env` sumber ikut menyalin `EXTERNAL_SERVICES`/
|
|
`*_HOST` milik single-host → di dotenv yang MENANG adalah yang TERAKHIR,
|
|
sehingga nilai dari inventory bisa hilang diam-diam (bug yang ditemukan saat
|
|
uji: `EXTERNAL_SERVICES` muncul dua kali).
|
|
"""
|
|
key = line.split("=", 1)[0].strip()
|
|
owned = {
|
|
"EXTERNAL_SERVICES", "LOCAL_SERVICES", "KEYCLOAK_SSL_REQUIRED", "KEYCLOAK_INTERNAL_URL",
|
|
"KC_PROXY_HEADERS", "TRINO_PROCESS_FORWARDED", "CLUSTER_DOMAIN", "TRAEFIK_BIND_ADDR",
|
|
*ENDPOINT_VARS, *URL_ENDPOINT_VARS,
|
|
*inventory.PUBLIC_URL_SERVICES, # KEYCLOAK_PUBLIC_URL, GRAFANA_PUBLIC_URL, …
|
|
}
|
|
return key in owned or key.startswith("SICABE_HOST_")
|
|
|
|
|
|
def render_env(base_env: Path, out_env: Path, host: str, spec: dict, endpoints: list[str]) -> None:
|
|
lines: list[str] = []
|
|
if base_env.is_file():
|
|
for line in base_env.read_text(encoding="utf-8").splitlines():
|
|
if _is_topology_var(line):
|
|
continue # ditentukan inventory, bukan .env sumber
|
|
lines.append(line)
|
|
else:
|
|
lines.append("# (tidak ada .env sumber — rahasia wajib dilengkapi manual)")
|
|
lines += [
|
|
"",
|
|
"# ── Ditambahkan generator multi-host (Fase 1) — variabel topologi ──",
|
|
f"SICABE_HOST_NAME={host}",
|
|
f"SICABE_HOST_ADDRESS={spec['address']}",
|
|
f"SICABE_HOST_TIERS={','.join(spec['tiers'])}",
|
|
]
|
|
lines += endpoints
|
|
out_env.parent.mkdir(parents=True, exist_ok=True)
|
|
out_env.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
# Port KONTAINER yang harus terjangkau lintas host (Fase 2). Variabel `*_HOST`
|
|
# hanya memuat NAMA host (tanpa port), jadi port diambil dari referensi compose =
|
|
# port container (5432, 8080, …). Selama port container itu tidak terbuka di alamat
|
|
# host, container di host lain tidak bisa menjangkau — jadi generator menambahkannya.
|
|
#
|
|
# Dikunci ke ALAMAT HOST (mis. `10.20.0.12:5432:5432`), bukan `0.0.0.0`, supaya hanya
|
|
# terjangkau lewat NIC privat host itu (R10 §4.8). Ini SENGAJA hanya di artefak
|
|
# multi-host: instalasi single-host tidak pernah membuka port internal ini, dan
|
|
# `docker-compose.yml` tetap byte-identik (invarian Fase 0 terjaga).
|
|
# Eksposur untuk manusia/klien tetap memakai port host `12xxx` yang sudah ada.
|
|
INTERNAL_PORTS: dict[str, list[int]] = {
|
|
"postgres": [5432],
|
|
"minio": [9000],
|
|
"nessie": [19120],
|
|
"keycloak": [8080],
|
|
"openldap": [389],
|
|
"opa": [8181],
|
|
# 8080 = worker mendaftar (discovery) · 8443 = klien lintas host (OM ingestion,
|
|
# task dbt, native DQ) — keduanya dibutuhkan begitu coordinator & worker beda host.
|
|
"trino-coordinator": [8080, 8443],
|
|
"starrocks-fe": [9030, 8030],
|
|
"starrocks-cn": [9050, 8040, 8060],
|
|
"openmetadata-opensearch": [9200],
|
|
"openmetadata-server": [8585],
|
|
"airflow-webserver": [8080],
|
|
# Sidecar JMX di host `coord`: Prometheus (di host `governance`) men-scrape 9404
|
|
# LINTAS host → portnya WAJIB diterbitkan di alamat host pemilik.
|
|
"trino-jmx-exporter": [9404],
|
|
}
|
|
|
|
|
|
def _reachable_on_host_port(entry, cport: int, address: str) -> bool:
|
|
"""Apakah `entry` benar-benar membuka port container `cport` DI alamat host?
|
|
|
|
Jebakan yang sempat lolos: `12080:8080` memetakan port host 12080 → container
|
|
8080. Itu TIDAK membuat port 8080 terjangkau di host, sehingga container di host
|
|
lain tetap tidak bisa menjangkau. Yang dihitung hanya pemetaan yang port host-nya
|
|
SAMA dengan port container (mis. `addr:8080:8080`).
|
|
"""
|
|
if isinstance(entry, str):
|
|
parts = entry.split(":")
|
|
if len(parts) == 2:
|
|
host_ip, host_port = None, parts[0]
|
|
elif len(parts) == 3:
|
|
host_ip, host_port = parts[0], parts[1]
|
|
else:
|
|
return False
|
|
if host_port != str(cport):
|
|
return False
|
|
return host_ip in (None, "", address)
|
|
if isinstance(entry, dict):
|
|
published = entry.get("published")
|
|
return (
|
|
entry.get("target") == cport
|
|
and str(published) == str(cport)
|
|
and entry.get("host_ip") in (None, "", address)
|
|
)
|
|
return False
|
|
|
|
|
|
def tier_implied_components(placement: dict, plan: dict) -> set[str]:
|
|
"""Komponen yang PASTI dikonsumsi lintas host walau tidak terlihat di env/config.
|
|
|
|
`trino-coordinator` selalu dipakai host bertier `pipeline` (task dbt Airflow) dan
|
|
`governance` (ingestion OpenMetadata). Referensinya tidak dapat ditemukan dengan
|
|
pemindaian env/config karena `ingest-trino.yaml` **disalin ke container saat
|
|
runtime** (docker cp), bukan di-mount — celah ini ditemukan saat menguji preflight.
|
|
"""
|
|
needed: set[str] = set()
|
|
owners = placement.get("trino-coordinator") or []
|
|
if not owners:
|
|
return needed
|
|
owner = owners[0]
|
|
for host, spec in plan.items():
|
|
if host == owner:
|
|
continue
|
|
if {"pipeline", "governance"} & set(spec["tiers"]):
|
|
needed.add("trino-coordinator")
|
|
break
|
|
return needed
|
|
|
|
|
|
def config_referenced_components(deploy_dir: Path, base: dict, plan: dict,
|
|
placement: dict, known: set[str]) -> set[str]:
|
|
"""Komponen yang dirujuk dari file CONFIG yang di-mount oleh service di host lain.
|
|
|
|
Deteksi berbasis environment saja tidak cukup: Trino tidak memakai `*_HOST` untuk
|
|
OAuth2, melainkan `config/trino/config-coordinator.properties`
|
|
(`http://keycloak:8080/...`). Tanpa pemindaian ini, port Keycloak tidak dibuka dan
|
|
Trino di host lain gagal mengambil JWKS.
|
|
"""
|
|
import re
|
|
|
|
needed: set[str] = set()
|
|
for host, spec in plan.items():
|
|
for svc in spec["services"]:
|
|
for vol in base["services"].get(svc, {}).get("volumes") or []:
|
|
src = vol.split(":", 1)[0] if isinstance(vol, str) else vol.get("source", "")
|
|
if not src.startswith("./config/"):
|
|
continue
|
|
path = deploy_dir / src[2:]
|
|
files = list(path.rglob("*")) if path.is_dir() else [path]
|
|
for f in files:
|
|
if not f.is_file():
|
|
continue
|
|
try:
|
|
text = f.read_text(encoding="utf-8", errors="ignore")
|
|
except OSError:
|
|
continue
|
|
for comp in known:
|
|
if re.search(r"(?<![\w.-])" + re.escape(comp) + r"(?![\w-])", text):
|
|
owners = placement.get(comp) or []
|
|
if owners and host not in owners:
|
|
needed.add(comp)
|
|
return needed
|
|
|
|
|
|
def cross_host_components(base: dict, placement: dict, plan: dict) -> set[str]:
|
|
"""Komponen yang PUNYA konsumen di host lain → perlu port internal terbuka.
|
|
|
|
Tanpa perhitungan ini generator membuka port untuk semua komponen, dan di
|
|
blueprint a3 itu langsung menabrak: `keycloak` dan `airflow-webserver` sama-sama
|
|
meminta `<control>:8080` (dua service, satu port host) → `compose up` gagal
|
|
'port is already allocated'. Padahal `airflow-webserver` hanya dikonsumsi
|
|
OpenMetadata yang berada di host yang sama → tidak perlu port host sama sekali.
|
|
|
|
Bonus keamanan: port internal yang tidak dibutuhkan tidak pernah dibuka.
|
|
"""
|
|
var_to_component = {v: c for v, c in ENDPOINT_VARS.items()}
|
|
var_to_component.update({v: c for v, (c, _t) in URL_ENDPOINT_VARS.items()})
|
|
needed: set[str] = set()
|
|
for host, spec in plan.items():
|
|
for svc in spec["services"]:
|
|
source = base["services"].get(svc, {})
|
|
text = json.dumps(source.get("environment") or {})
|
|
for var, component in var_to_component.items():
|
|
if var not in text:
|
|
continue
|
|
owners = placement.get(component) or []
|
|
if owners and host not in owners:
|
|
needed.add(component)
|
|
return needed
|
|
|
|
|
|
def required_cross_host(base: dict, deploy_dir: Path, plan: dict, placement: dict) -> set[str]:
|
|
"""Gabungan SEMUA alasan sebuah komponen butuh port lintas host.
|
|
|
|
Satu fungsi supaya generator dan preflight TIDAK bisa berbeda pendapat — memisahkan
|
|
perhitungan ini sempat membuat preflight melewatkan trino-coordinator begitu aturan
|
|
baru ditambahkan ke generator.
|
|
"""
|
|
known = set(base["services"]) & set(INTERNAL_PORTS)
|
|
return (
|
|
cross_host_components(base, placement, plan)
|
|
| config_referenced_components(deploy_dir, base, plan, placement, known)
|
|
| tier_implied_components(placement, plan)
|
|
)
|
|
|
|
|
|
def add_internal_ports(services: dict, address: str, needed: set[str]) -> list[str]:
|
|
"""Tambahkan binding port internal ke alamat host; kembalikan yang ditambahkan."""
|
|
added: list[str] = []
|
|
for name, spec in services.items():
|
|
if name not in needed:
|
|
continue
|
|
want = INTERNAL_PORTS.get(name)
|
|
if not want:
|
|
continue
|
|
ports = spec.setdefault("ports", [])
|
|
for cport in want:
|
|
if any(_reachable_on_host_port(e, cport, address) for e in ports):
|
|
continue
|
|
ports.append(f"{address}:{cport}:{cport}")
|
|
added.append(f"{name}:{cport}")
|
|
return added
|
|
|
|
|
|
def component_placements(doc: dict) -> dict[str, list[str]]:
|
|
return inventory.resolve(doc)
|
|
|
|
|
|
def extra_bind_dirs(base: dict) -> list[str]:
|
|
"""Bind-mount relatif selain `config/` yang harus ikut disalin ke tiap host.
|
|
|
|
Contoh: `./scripts/airflow:/opt/airflow/scripts:ro`. Tanpa disalin, direktori
|
|
generated tidak mandiri — Docker akan membuat direktori KOSONG di host tujuan
|
|
(skrip Airflow hilang diam-diam). Ditemukan oleh test-generate.sh.
|
|
"""
|
|
found: set[str] = set()
|
|
for spec in base["services"].values():
|
|
for vol in spec.get("volumes", []) or []:
|
|
if isinstance(vol, str):
|
|
src = vol.split(":", 1)[0]
|
|
elif isinstance(vol, dict):
|
|
src = vol.get("source", "")
|
|
else:
|
|
continue
|
|
if src.startswith("./") and not src.startswith("./config/"):
|
|
found.add(src[2:])
|
|
return sorted(found)
|
|
|
|
|
|
def copy_extra_binds(deploy_dir: Path, host_dir: Path, paths: list[str]) -> list[str]:
|
|
"""Salin bind relatif non-config (mis. `scripts/airflow`) ke direktori host."""
|
|
import shutil
|
|
|
|
copied: list[str] = []
|
|
for rel in paths:
|
|
src = deploy_dir / rel
|
|
if not src.exists():
|
|
continue
|
|
dst = host_dir / rel
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
if src.is_dir():
|
|
shutil.copytree(src, dst, dirs_exist_ok=True)
|
|
else:
|
|
shutil.copy2(src, dst)
|
|
copied.append(rel)
|
|
return copied
|
|
|
|
|
|
def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]:
|
|
"""Baris `VAR=alamat|nama` untuk host ini.
|
|
|
|
SE-HOST → **nama service** (satu jaringan Docker). Port container TIDAK diterbitkan
|
|
di host (mis. OM 8585), jadi menulis alamat host justru "connection refused"
|
|
(regresi: Airflow/OpenLineage → `http://<ip-control>:8585/api`). Beda host → alamat
|
|
host pemilik (port internal diterbitkan generator untuk itu).
|
|
"""
|
|
placement = component_placements(doc)
|
|
lines: list[str] = []
|
|
for var, component in ENDPOINT_VARS.items():
|
|
ext = inventory.external_address(doc, component)
|
|
if ext:
|
|
# EKSTERNAL → HOST saja (buang skema/port bila operator menulis URL penuh,
|
|
# mis. `https://sso.example.com` → `sso.example.com`). URL lengkapnya
|
|
# ditangani `*_URL` / KEYCLOAK_INTERNAL_URL, bukan var `*_HOST`.
|
|
lines.append(f"{var}={inventory.parse_address(ext)[1]}")
|
|
continue
|
|
owners = placement.get(component)
|
|
if not owners:
|
|
continue
|
|
if host in owners:
|
|
lines.append(f"{var}={component}") # SE-HOST → nama service
|
|
else:
|
|
lines.append(f"{var}={plan[owners[0]]['address']}") # LINTAS host → alamat
|
|
# Keycloak: URL server→server (token/jwks/Grafana). Hanya ditulis bila Keycloak
|
|
# EKSTERNAL memakai URL PENUH (mis. https://sso.example.com) agar skema+port
|
|
# dihormati; selain itu compose/script memakai default
|
|
# `http://${KEYCLOAK_HOST:-keycloak}:${KEYCLOAK_INTERNAL_PORT:-8080}`.
|
|
kc_ext = inventory.external_address(doc, "keycloak")
|
|
if kc_ext and inventory.parse_address(kc_ext)[0]:
|
|
lines.append(f"KEYCLOAK_INTERNAL_URL={_ext_url(kc_ext, '')}")
|
|
disabled = [n for n, v in inventory.enabled_components(doc).items() if v is False]
|
|
if disabled:
|
|
# Komponen `enabled: false` = tidak dijalankan di host mana pun → installer
|
|
# harus melewatinya (install.sh: EXTERNAL_SERVICES menang atas auto-deteksi).
|
|
lines.append('EXTERNAL_SERVICES="%s"' % " ".join(disabled))
|
|
return lines
|
|
|
|
|
|
def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]:
|
|
"""Baris `*_PUBLIC_URL=...` — URL yang dilihat BROWSER (issuer OIDC, redirect, UI),
|
|
bukan lalu lintas container.
|
|
|
|
tls.mode != none : `https://<label>.<domain>` (inventory.public_urls).
|
|
tls.mode == none : `http(s)://<alamat HOST PEMILIK>:<port>`. Sebelumnya mode ini
|
|
mengembalikan daftar kosong, sehingga .env host memakai default
|
|
`http://localhost:<port>` — dan browser di host lain dibuang ke
|
|
localhost (issuer OIDC pun localhost). Sekarang alamatnya diambil
|
|
dari host yang menjalankan komponen, jadi akses lintas host benar
|
|
tanpa perlu nilai berbeda per bundle.
|
|
"""
|
|
by_domain = inventory.public_urls(doc)
|
|
if by_domain:
|
|
return [f"{var}={url}" for var, url in sorted(by_domain.items())]
|
|
if plan is None:
|
|
return []
|
|
|
|
placement = component_placements(doc)
|
|
resolved: dict[str, str] = {}
|
|
for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items():
|
|
ext = inventory.external_address(doc, component)
|
|
if ext:
|
|
resolved[var] = _ext_url(ext, tmpl) # eksternal → URL penuh dihormati
|
|
continue
|
|
owners = placement.get(component)
|
|
if not owners:
|
|
continue
|
|
# Sengaja memakai alamat PEMILIK (bukan host lokal) dan urutan yang pasti,
|
|
# supaya nilainya identik di setiap bundle.
|
|
resolved[var] = tmpl.format(addr=plan[sorted(owners)[0]]["address"])
|
|
return [f"{var}={url}" for var, url in sorted(resolved.items())]
|
|
|
|
|
|
def tls_policy_lines(doc: dict) -> list[str]:
|
|
"""Kebijakan keamanan turunan `tls.mode` (saat ini hanya realm Keycloak).
|
|
|
|
Keycloak DEFAULT `sslRequired=external` menuntut HTTPS untuk permintaan dari alamat
|
|
NON-lokal. Alamat Tailscale `100.64.0.0/10` (CGNAT) TERMASUK non-lokal bagi Keycloak
|
|
(`isLocal` = loopback + RFC1918 + link-local + `fc00::/7`), sehingga admin console di
|
|
`http://<alamat-tailscale>:12080` gagal dengan "HTTPS required" — dan Keycloak 26
|
|
TIDAK punya opsi server-level untuk ini (tidak ada `KC_HTTPS_REQUIRED`), hanya per
|
|
realm. Karena itu nilainya diteruskan ke bootstrap.sh.
|
|
|
|
tls.mode `none` -> `none` (HTTP polos; instalasi dev/simulasi)
|
|
tls.mode `edge`/`full` -> `external` (ada Traefik/TLS di depan)
|
|
"""
|
|
edge = inventory.tls_mode(doc) != "none"
|
|
# Keycloak EKSTERNAL ber-URL `https://…` juga menuntut sslRequired=external (permintaan
|
|
# dari alamat non-lokal lewat TLS di depannya).
|
|
ext_https = _keycloak_external_scheme(doc) == "https"
|
|
return [
|
|
f"KEYCLOAK_SSL_REQUIRED={'external' if (edge or ext_https) else 'none'}",
|
|
# Di belakang Traefik, Keycloak WAJIB memercayai X-Forwarded-* — tanpa itu
|
|
# permintaan HTTPS dari proxy dianggap HTTP (redirect salah / tuntutan HTTPS).
|
|
# Sengaja KOSONG saat tanpa proxy: memasangnya membuat Keycloak memercayai
|
|
# header yang bisa dipalsukan klien.
|
|
f"KC_PROXY_HEADERS={'xforwarded' if edge else ''}",
|
|
# Trino: X-Forwarded-Proto supaya skema publik benar saat di belakang proxy.
|
|
f"TRINO_PROCESS_FORWARDED={'true' if edge else 'false'}",
|
|
]
|
|
|
|
|
|
# Rute untuk komponen publik yang TIDAK berjalan di host ingress. Traefik hanya melihat
|
|
# container di host-nya sendiri, jadi komponen di host lain dirutekan lewat berkas
|
|
# dinamis (provider file) ke PORT HOST-nya (12xxx), bukan port container.
|
|
# Nilai: (label rute, URL, butuh serverstransport-insecure?)
|
|
REMOTE_ROUTE_TARGETS = {
|
|
"keycloak": ("kc", "http://{addr}:12080", False),
|
|
"grafana": ("grafana", "http://{addr}:12300", False),
|
|
"openmetadata-server": ("om", "http://{addr}:12085", False),
|
|
"airflow-webserver": ("airflow", "http://{addr}:12082", False),
|
|
"trino-coordinator": ("trino", "https://{addr}:12443", True),
|
|
"minio": ("minio", "http://{addr}:12901", False),
|
|
}
|
|
|
|
|
|
def ingress_host(doc: dict, plan: dict) -> str:
|
|
"""Host yang menjalankan ingress Traefik (T-20).
|
|
|
|
Default: host yang menjalankan `keycloak` (tier identity — pada preset a3 itu
|
|
`control`), supaya tidak perlu setelan tambahan. Bisa dipatok eksplisit lewat
|
|
`tls.ingress_host`. Kosong bila tls.mode `none` (memang tanpa ingress).
|
|
"""
|
|
if inventory.tls_mode(doc) == "none":
|
|
return ""
|
|
explicit = ((doc.get("tls") or {}).get("ingress_host") or "").strip()
|
|
if explicit:
|
|
return explicit if explicit in plan else ""
|
|
owners = sorted(component_placements(doc).get("keycloak") or [])
|
|
if owners:
|
|
return owners[0]
|
|
return sorted(plan)[0] if plan else ""
|
|
|
|
|
|
def remote_route_doc(doc: dict, plan: dict) -> dict | None:
|
|
"""Isi berkas dinamis untuk rute komponen yang ada DI HOST LAIN.
|
|
|
|
Mengembalikan None bila tidak ada rute jarak jauh (semua komponen publik memang
|
|
berada di host ingress).
|
|
"""
|
|
ing = ingress_host(doc, plan)
|
|
if not ing:
|
|
return None
|
|
domain = inventory.cluster_domain(doc)
|
|
placement = component_placements(doc)
|
|
routers: dict = {}
|
|
services: dict = {}
|
|
for component, (label, tmpl, needs_insecure) in REMOTE_ROUTE_TARGETS.items():
|
|
owners = sorted(placement.get(component) or [])
|
|
if not owners or ing in owners:
|
|
continue # sudah ditangani label container di host ini
|
|
owner = owners[0]
|
|
router = {
|
|
"rule": f"Host(`{label}.{domain}`)",
|
|
"entryPoints": ["websecure"],
|
|
"service": label,
|
|
"tls": {
|
|
"certResolver": "le",
|
|
"domains": [{"main": domain, "sans": [f"*.{domain}"]}],
|
|
},
|
|
}
|
|
lb: dict = {"servers": [{"url": tmpl.format(addr=plan[owner]["address"])}]}
|
|
if needs_insecure:
|
|
lb["serversTransport"] = "trino-insecure"
|
|
routers[label] = router
|
|
services[label] = {"loadBalancer": lb}
|
|
if not routers:
|
|
return None
|
|
# serversTransport wajib ikut di berkas ini: berkas dinamis yang dibangkitkan
|
|
# MENGGANTIKAN salinan dasar, jadi definisinya harus ada di sini juga.
|
|
return {
|
|
"http": {
|
|
"serversTransports": {"trino-insecure": {"insecureSkipVerify": True}},
|
|
"routers": routers,
|
|
"services": services,
|
|
}
|
|
}
|
|
|
|
|
|
def node_targets_doc(plan: dict) -> list[dict]:
|
|
"""Berkas target file_sd untuk job `node`: satu entri `<alamat>:9100` per host.
|
|
|
|
node_exporter adalah PRA-SYARAT HOST (di luar installer), sehingga SEMUA host di
|
|
cluster.yml dianggap menjalankannya di port 9100. Label `host` disalin ke
|
|
`instance` oleh relabel_configs di prometheus.yml (panel menampilkan nama host).
|
|
"""
|
|
return [
|
|
{"targets": [f"{spec['address']}:{NODE_EXPORTER_PORT}"], "labels": {"host": name}}
|
|
for name, spec in sorted(plan.items())
|
|
]
|
|
|
|
|
|
def ingress_lines(doc: dict, host: str, plan: dict) -> list[str]:
|
|
"""Variabel ingress untuk .env host (T-20).
|
|
|
|
`CLUSTER_DOMAIN` dipakai label rute Traefik (Host(`<label>.<domain>`)), dan
|
|
`TRAEFIK_BIND_ADDR` membatasi port 80/443 ke alamat host ini — bukan semua antarmuka.
|
|
Hanya ditulis saat ada TLS/ingress; tanpa itu Traefik memang tidak dijalankan.
|
|
"""
|
|
if inventory.tls_mode(doc) == "none":
|
|
return []
|
|
if host != ingress_host(doc, plan):
|
|
return []
|
|
return [
|
|
f"CLUSTER_DOMAIN={inventory.cluster_domain(doc)}",
|
|
f"TRAEFIK_BIND_ADDR={plan[host]['address']}",
|
|
]
|
|
|
|
|
|
def _endpoint_port(var, env_path):
|
|
"""Port container untuk var URL (dari .env, fallback default URL_ENDPOINT_PORTS)."""
|
|
spec = URL_ENDPOINT_PORTS.get(var)
|
|
if not spec:
|
|
return ""
|
|
env_name, default = spec
|
|
val = _source_env_value(env_path, env_name) if env_path else None
|
|
return val if val else str(default)
|
|
|
|
|
|
def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[str]:
|
|
"""Baris `VAR=<url>` untuk konsumen bentuk URL.
|
|
|
|
SE-HOST → **nama service** (mis. `http://nessie:19120`, `http://opa:8181`);
|
|
LINTAS host → alamat IP pemilik (port internalnya diterbitkan generator).
|
|
Konsisten dengan endpoint_lines(): port container tak diterbitkan di host.
|
|
Port diambil dari .env (mis. NESSIE_INTERNAL_PORT) bila di-override.
|
|
"""
|
|
placement = component_placements(doc)
|
|
lines: list[str] = []
|
|
for var, (component, template) in URL_ENDPOINT_VARS.items():
|
|
ext = inventory.external_address(doc, component)
|
|
if ext:
|
|
lines.append(f"{var}={_ext_url(ext, template, _endpoint_port(var, env_path))}")
|
|
continue
|
|
owners = placement.get(component)
|
|
if not owners:
|
|
continue
|
|
addr = component if host in owners else plan[owners[0]]["address"]
|
|
lines.append(f"{var}={template.format(addr=addr, port=_endpoint_port(var, env_path))}")
|
|
return lines
|
|
|
|
|
|
def render_config_map(doc: dict, host: str, plan: dict) -> list[str]:
|
|
"""Peta hostname container → alamat, untuk render-config.sh (khusus host ini).
|
|
|
|
HANYA komponen yang dimiliki HOST LAIN yang dipetakan. Komponen SE-HOST dibiarkan
|
|
memakai NAMA container (satu jaringan Docker) — menulisnya ke alamat host justru
|
|
RUSAK: port container sering tak diterbitkan di host. Regresi nyata: `prometheus`
|
|
(se-host dengan grafana) ditulis `http://<ip-host>:9090` di datasources.yaml →
|
|
Grafana "connection refused"; `statsd-exporter` (se-host dengan prometheus) serupa.
|
|
"""
|
|
placement = component_placements(doc)
|
|
args: list[str] = []
|
|
for component in RENDER_HOSTS:
|
|
owners = placement.get(component)
|
|
if not owners:
|
|
continue
|
|
if host in owners:
|
|
continue # SE-HOST → biarkan nama container
|
|
args += ["--map", f"{component}={plan[owners[0]]['address']}"]
|
|
return args
|
|
|
|
|
|
def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None:
|
|
"""Ganti target scrape MinIO di prometheus.yml bila MinIO EKSTERNAL.
|
|
|
|
Prometheus TIDAK mendukung substitusi env (beda dgn config Trino), dan
|
|
render-config.sh hanya memetakan hostname komponen yang dimiliki HOST LAIN —
|
|
komponen EKSTERNAL (`components.minio.address`, skenario P3) TAK dipetakan →
|
|
target tetap `minio:9000` (nama container) → scrape GAGAL dari host prometheus.
|
|
Di sini diganti `<alamat-eksternal>:<MINIO_INTERNAL_PORT>`.
|
|
|
|
MinIO se-host (single-host / P1) → tak diubah (`minio:9000`, satu jaringan Docker).
|
|
|
|
Address boleh ber-URL penuh (`https://minio.corp:9000`): target ditulis `host:port`
|
|
(TANPA skema, tanpa port dobel) dan `scheme:` job minio disamakan (https) agar scrape
|
|
tak menembak http ke port TLS.
|
|
"""
|
|
yml = host_dir / "config" / "prometheus" / "prometheus.yml"
|
|
if not yml.is_file():
|
|
return
|
|
ext = inventory.external_address(doc, "minio")
|
|
if not ext:
|
|
return
|
|
scheme, host, p = inventory.parse_address(ext)
|
|
if scheme:
|
|
target = f"{host}:{p}" if p else host
|
|
eff_scheme = scheme
|
|
else:
|
|
# alamat polos (IP/hostname) → port internal, perilaku lama; skema ikut S3_ENDPOINT
|
|
port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000"
|
|
target = f"{host}:{port}"
|
|
eff_scheme = ((_source_env_value(env_path, "S3_ENDPOINT") or "").split("://", 1)[0]
|
|
or "http")
|
|
txt = yml.read_text(encoding="utf-8")
|
|
out: list[str] = []
|
|
in_minio = False
|
|
for line in txt.splitlines():
|
|
s = line.strip()
|
|
if s.startswith("- job_name:"):
|
|
in_minio = s.split(":", 1)[1].strip() in ("minio", "minio-bucket")
|
|
if "minio:9000" in line:
|
|
line = line.replace("minio:9000", target)
|
|
if in_minio and s == "scheme: http" and eff_scheme != "http":
|
|
line = line.replace("scheme: http", f"scheme: {eff_scheme}")
|
|
out.append(line)
|
|
new = "\n".join(out)
|
|
if txt.endswith("\n"):
|
|
new += "\n"
|
|
if new != txt:
|
|
yml.write_text(new, encoding="utf-8")
|
|
|
|
|
|
def cmd_render(argv: list[str]) -> int:
|
|
if len(argv) < 7:
|
|
sys.stderr.write(__doc__ or "")
|
|
return 2
|
|
inv_path, compose_path, env_path, out_base, renderer = argv[2:7]
|
|
preset_override = argv[7] if len(argv) > 7 else ""
|
|
doc = inventory.load(inv_path)
|
|
if preset_override:
|
|
doc["preset"] = preset_override
|
|
doc = inventory.apply_preset(doc)
|
|
errors = inventory.validate(doc)
|
|
if errors:
|
|
sys.stderr.write("cluster.yml tidak valid — jalankan 'inventory.py validate' untuk detail:\n")
|
|
for e in errors:
|
|
sys.stderr.write(f" ✗ {e}\n")
|
|
return 1
|
|
|
|
base = load_compose(compose_path)
|
|
extra_binds = extra_bind_dirs(base)
|
|
plan = inventory.plan(doc)
|
|
out_root = Path(out_base)
|
|
placement = component_placements(doc)
|
|
needed = required_cross_host(base, Path(compose_path).parent, plan, placement)
|
|
total_dropped: dict[str, list[str]] = {}
|
|
port_internal_all: set[str] = set()
|
|
|
|
# Trino cert/keystore: dibangkitkan SEKALI (SAN = ALAMAT HOST semua host), lalu
|
|
# disalin IDENTIK ke tiap bundle. WAJIB sekali — kalau tiap host beda keypair, cert
|
|
# klien (OM ingestion, dll.) tak cocok dgn cert server → verifikasi TLS gagal.
|
|
#
|
|
# DIPERSISTENKAN di deploy/.trino-keystore/ (gitignored): kalau cert dibuat BARU tiap
|
|
# `--generate`, server Trino yang belum di-recreate menyajikan cert LAMA → klien
|
|
# (GE/OM) "self-signed certificate". Stabil kecuali daftar SAN (alamat host) berubah.
|
|
_ks_cache = Path(__file__).resolve().parents[2] / ".trino-keystore"
|
|
_ks_cache.mkdir(parents=True, exist_ok=True)
|
|
_trino_ks_src = _ks_cache
|
|
_san = ",".join(f"IP:{plan[h]['address']}" for h in sorted(plan) if plan[h].get("address"))
|
|
_ks_script = Path(renderer).resolve().parent / "generate-trino-keystore.sh"
|
|
_marker = _ks_cache / "san.txt"
|
|
_need_regen = (not (_ks_cache / "keystore.p12").is_file()
|
|
or not _marker.is_file()
|
|
or _marker.read_text(encoding="utf-8").strip() != _san)
|
|
if _san and _ks_script.is_file() and _need_regen:
|
|
subprocess.run([str(_ks_script), "--out", str(_trino_ks_src), "--san-extra", _san],
|
|
capture_output=True, text=True)
|
|
try:
|
|
_marker.write_text(_san, encoding="utf-8")
|
|
except OSError:
|
|
pass
|
|
|
|
for host, spec in plan.items():
|
|
host_dir = out_root / f"host-{host}"
|
|
# 1) compose
|
|
# Ingress: setiap host menjalankan Traefik-nya sendiri saat ada TLS/proxy
|
|
# (satu sertifikat per host — acme.json tidak bisa dibagi antar host).
|
|
# Tanpa TLS, service traefik memang tidak ikut.
|
|
wanted = set(spec["services"])
|
|
# Ingress satu host: HANYA host ingress yang menjalankan Traefik. Ini juga
|
|
# menghilangkan perebutan record TXT DNS-01 (dulu tiap host menerbitkan
|
|
# wildcard yang sama) dan menekan pemakaian batas sertifikat Let's Encrypt.
|
|
ing = ingress_host(doc, plan)
|
|
if ing and host == ing:
|
|
wanted.add("traefik")
|
|
compose_out, dropped = build_compose(base, wanted)
|
|
# Fase 2: buka port internal ke alamat host ini agar komponen di host lain
|
|
# dapat menjangkau (pengganti ketiadaan satu bridge network lintas host).
|
|
added_ports = add_internal_ports(compose_out["services"], spec["address"], needed)
|
|
port_internal_all.update(added_ports)
|
|
write_compose(host_dir / "docker-compose.yml", compose_out)
|
|
total_dropped[host] = dropped
|
|
# 2) .env
|
|
render_env(
|
|
Path(env_path), host_dir / ".env", host, spec,
|
|
endpoint_lines(doc, host, plan)
|
|
+ url_endpoint_lines(doc, host, plan, env_path)
|
|
+ public_url_lines(doc, plan)
|
|
+ tls_policy_lines(doc)
|
|
+ ingress_lines(doc, host, plan),
|
|
)
|
|
# 3) config ter-render
|
|
# 3a) bind relatif non-config (mis. scripts/airflow) → bundle mandiri
|
|
copied = copy_extra_binds(Path(compose_path).parent, host_dir, extra_binds)
|
|
# 3b) config ter-render melalui render-config.sh
|
|
# Bersihkan target berkas runtime yang mungkin berwujud DIREKTORI (sisa `up`
|
|
# Docker saat sumber bind-mount belum ada). Tanpa ini render-config GAGAL
|
|
# ("tulis gagal") dan generator berhenti SEBELUM menulis berkas runtime.
|
|
for _rel in ("airflow/simple_auth_manager_passwords.json",
|
|
"trino/password.db"):
|
|
_t = host_dir / "config" / _rel
|
|
if _t.is_dir():
|
|
shutil.rmtree(_t)
|
|
rc = subprocess.run(
|
|
[renderer, "--out", str(host_dir / "config"), *render_config_map(doc, host, plan)],
|
|
capture_output=True, text=True,
|
|
)
|
|
# Berkas dinamis rute komponen di host lain (hanya di host ingress).
|
|
# Ditulis SETELAH render-config karena berkas ini menggantikan salinan dasar.
|
|
remote = remote_route_doc(doc, plan)
|
|
if remote is not None and host == ing:
|
|
(host_dir / "config" / "traefik").mkdir(parents=True, exist_ok=True)
|
|
with open(host_dir / "config" / "traefik" / "dynamic.yml", "w", encoding="utf-8") as fh:
|
|
fh.write("# DIBANGKITKAN OTOMATIS oleh install.sh --generate — jangan disunting tangan.\n")
|
|
fh.write("# Ingress satu host: rute untuk komponen yang TIDAK ada di host ini.\n")
|
|
yaml.safe_dump(remote, fh, sort_keys=False, allow_unicode=True)
|
|
# Target node_exporter (job `node`, file_sd) — hanya di host pemilik
|
|
# Prometheus. Ditulis SETELAH render-config karena menggantikan salinan dasar.
|
|
if host in (placement.get("prometheus") or []):
|
|
tdir = host_dir / "config" / "prometheus" / "targets"
|
|
tdir.mkdir(parents=True, exist_ok=True)
|
|
with open(tdir / "node.yml", "w", encoding="utf-8") as fh:
|
|
fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n")
|
|
fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n")
|
|
yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True)
|
|
# Prometheus tak mendukung substitusi env → target MinIO eksternal
|
|
# (`minio:9000`) di-rewrite di sini ke alamat:port publik.
|
|
_rewrite_prometheus_scrapes(host_dir, doc, env_path)
|
|
if rc.returncode != 0:
|
|
sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}")
|
|
return 1
|
|
# Salin cert/keystore Trino (dibangkitkan SEKALI dgn SAN IP semua host) ke bundle
|
|
# ini — keypair/cert IDENTIK lintas host agar verifikasi TLS server↔klien lolos.
|
|
if (_trino_ks_src / "keystore.p12").is_file():
|
|
(host_dir / "config" / "trino" / "certs").mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(_trino_ks_src / "keystore.p12",
|
|
host_dir / "config" / "trino" / "keystore.p12")
|
|
shutil.copy2(_trino_ks_src / "certs" / "trino.crt",
|
|
host_dir / "config" / "trino" / "certs" / "trino.crt")
|
|
# Berkas runtime gitignored (Airflow auth, Trino password.db) tidak ikut
|
|
# render-config → tulis eksplisit agar compose tak me-mount sumber hilang (DIRECTORY).
|
|
_write_bundle_runtime_files(host_dir, Path(env_path))
|
|
print(f"host-{host:9s} {spec['address']:15s} services={len(spec['services'])} "
|
|
f"depends_on_dibuang={len(dropped)} bind_disalin={len(copied)} "
|
|
f"port_internal={len(added_ports)}")
|
|
|
|
print(f"komponen butuh port lintas host: {', '.join(sorted(needed)) or '—'}")
|
|
if port_internal_all:
|
|
print(f"port internal dibuka: {len(port_internal_all)} ("
|
|
+ ", ".join(sorted(port_internal_all)) + ")")
|
|
|
|
# ringkasan untuk di-review
|
|
for host, dropped in total_dropped.items():
|
|
for item in dropped:
|
|
print(f" [host-{host}] depends_on lintas host dibuang: {item}")
|
|
print(json.dumps({h: len(s["services"]) for h, s in plan.items()}, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
def _source_env_value(env_path, key):
|
|
"""Ambil satu nilai dari .env sumber (kutip luar dibuang). None bila tak ada."""
|
|
try:
|
|
text = Path(env_path).read_text(encoding="utf-8")
|
|
except OSError:
|
|
return None
|
|
for line in text.splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
name, _, value = line.partition("=")
|
|
if name.strip() != key:
|
|
continue
|
|
value = value.strip()
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
value = value[1:-1]
|
|
return value
|
|
return None
|
|
|
|
|
|
def _write_bundle_runtime_files(host_dir, env_path):
|
|
"""Tulis berkas runtime yang TIDAK di-track git ke bundle host.
|
|
|
|
compose generated me-mount path ini sebagai FILE. Bila file tidak ada, Docker
|
|
membuat DIRECTORY di host → service gagal (Airflow SimpleAuthManager:
|
|
IsADirectoryError; Trino: password.db bukan file). ``render-config`` tidak
|
|
menyalinnya (gitignored), jadi harus ditulis eksplisit di sini.
|
|
"""
|
|
cfg = host_dir / "config"
|
|
# Dir bind-mount yang ABSEN di git (config/airflow/sling hanya memuat env.yaml yang
|
|
# gitignored) → Docker (daemon root) membuatnya sebagai ROOT saat `up` → operator
|
|
# non-root tak bisa menulis env.yaml (bootstrap gagal). Pastikan ada & user-owned.
|
|
(cfg / "airflow" / "sling").mkdir(parents=True, exist_ok=True)
|
|
# 1) Airflow SimpleAuthManager — {"admin": <password>} (mode a+ → wajib writable)
|
|
pw_json = cfg / "airflow" / "simple_auth_manager_passwords.json"
|
|
pw_json.parent.mkdir(parents=True, exist_ok=True)
|
|
if pw_json.is_dir():
|
|
shutil.rmtree(pw_json)
|
|
admin_pw = _source_env_value(env_path, "AIRFLOW_ADMIN_PASSWORD")
|
|
if admin_pw is not None:
|
|
pw_json.write_text(json.dumps({"admin": admin_pw}) + "\n", encoding="utf-8")
|
|
elif not pw_json.is_file():
|
|
pw_json.write_text("{}\n", encoding="utf-8")
|
|
pw_json.chmod(0o666)
|
|
# 2) Trino password.db — DIBANGKITKAN dari TRINO_SVC_*_PASSWORD di .env (service
|
|
# account, HTTP Basic/bcrypt). Dulu hanya MENYALIN config/trino/password.db
|
|
# sumber — di mode cluster sumber itu sering KOSONG → bundle kosong → Trino
|
|
# menolak dgn 401 "Access Denied: Invalid credentials" (mis. dbt `svc_dbt`).
|
|
# Kini bundle MANDIRI & konsisten setelah --sync. Fallback: salin sumber bila
|
|
# htpasswd tak tersedia; kalau tidak ada, file kosong (yang penting BUKAN direktori).
|
|
dst_db = cfg / "trino" / "password.db"
|
|
dst_db.parent.mkdir(parents=True, exist_ok=True)
|
|
if dst_db.is_dir():
|
|
shutil.rmtree(dst_db)
|
|
svc_accounts = (
|
|
("svc_seed", "TRINO_SVC_SEED_PASSWORD"),
|
|
("svc_dbt", "TRINO_SVC_DBT_PASSWORD"),
|
|
("svc_airflow", "TRINO_SVC_AIRFLOW_PASSWORD"),
|
|
("svc_om", "TRINO_SVC_OM_PASSWORD"),
|
|
)
|
|
pw_lines = []
|
|
htpasswd = shutil.which("htpasswd")
|
|
if htpasswd:
|
|
for user, key in svc_accounts:
|
|
secret = _source_env_value(env_path, key)
|
|
if not secret:
|
|
continue
|
|
proc = subprocess.run([htpasswd, "-nbB", "-C", "10", user, secret],
|
|
capture_output=True, text=True)
|
|
if proc.returncode == 0 and proc.stdout.strip():
|
|
pw_lines.append(proc.stdout.strip())
|
|
if pw_lines:
|
|
dst_db.write_text("\n".join(pw_lines) + "\n", encoding="utf-8")
|
|
dst_db.chmod(0o644)
|
|
else:
|
|
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
|
|
if src_db.is_file():
|
|
shutil.copy2(src_db, dst_db)
|
|
elif not dst_db.is_file():
|
|
dst_db.write_text("", encoding="utf-8")
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
if len(argv) < 2:
|
|
sys.stderr.write(__doc__ or "")
|
|
return 2
|
|
if argv[1] == "render":
|
|
return cmd_render(argv)
|
|
sys.stderr.write(f"perintah tak dikenal: {argv[1]}\n")
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main(sys.argv))
|