platform/sicabe-core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync dari sicabe-doc/deploy @ bc1a6e6

Browse Source
This commit is contained in:
subroto
2026-10-11 13:59:47 +07:00
parent 4ca7a9a58e
commit cbabe96061
29 changed files with 1210 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/lib/cluster/generate.py
+71 -7
View File
@@ -456,6 +456,10 @@ def endpoint_lines(doc: dict, host: str, plan: dict) -> list[str]:
placement = component_placements(doc)
lines: list[str] = []
for var, component in ENDPOINT_VARS.items():
ext = inventory.external_address(doc, component)
if ext:
lines.append(f"{var}={ext}") # EKSTERNAL (pihak lain) → alamat tetap
continue
owners = placement.get(component)
if not owners:
continue
@@ -492,6 +496,10 @@ def public_url_lines(doc: dict, plan: dict | None = None) -> list[str]:
placement = component_placements(doc)
resolved: dict[str, str] = {}
for var, (component, tmpl) in PUBLIC_URL_ENDPOINTS.items():
ext = inventory.external_address(doc, component)
if ext:
resolved[var] = tmpl.format(addr=ext) # eksternal → alamat tetap
continue
owners = placement.get(component)
if not owners:
continue
@@ -655,6 +663,10 @@ def url_endpoint_lines(doc: dict, host: str, plan: dict, env_path=None) -> list[
placement = component_placements(doc)
lines: list[str] = []
for var, (component, template) in URL_ENDPOINT_VARS.items():
ext = inventory.external_address(doc, component)
if ext:
lines.append(f"{var}={template.format(addr=ext, port=_endpoint_port(var, env_path))}")
continue
owners = placement.get(component)
if not owners:
continue
@@ -684,6 +696,30 @@ def render_config_map(doc: dict, host: str, plan: dict) -> list[str]:
return args
def _rewrite_prometheus_scrapes(host_dir, doc, env_path) -> None:
"""Ganti target scrape MinIO di prometheus.yml bila MinIO EKSTERNAL.
Prometheus TIDAK mendukung substitusi env (beda dgn config Trino), dan
render-config.sh hanya memetakan hostname komponen yang dimiliki HOST LAIN —
komponen EKSTERNAL (`components.minio.address`, skenario P3) TAK dipetakan →
target tetap `minio:9000` (nama container) → scrape GAGAL dari host prometheus.
Di sini diganti `<alamat-eksternal>:<MINIO_INTERNAL_PORT>`.
MinIO se-host (single-host / P1) → tak diubah (`minio:9000`, satu jaringan Docker).
"""
yml = host_dir / "config" / "prometheus" / "prometheus.yml"
if not yml.is_file():
return
ext = inventory.external_address(doc, "minio")
if not ext:
return
port = _source_env_value(env_path, "MINIO_INTERNAL_PORT") or "9000"
txt = yml.read_text(encoding="utf-8")
new = txt.replace("minio:9000", f"{ext}:{port}")
if new != txt:
yml.write_text(new, encoding="utf-8")
def cmd_render(argv: list[str]) -> int:
if len(argv) < 7:
sys.stderr.write(__doc__ or "")
@@ -797,6 +833,9 @@ def cmd_render(argv: list[str]) -> int:
fh.write("# DIBANGKITKAN OTOMATIS dari cluster.yml oleh install.sh --generate\n")
fh.write("# — jangan disunting tangan. Target node_exporter (job `node`) semua host.\n")
yaml.safe_dump(node_targets_doc(plan), fh, sort_keys=False, allow_unicode=True)
# Prometheus tak mendukung substitusi env → target MinIO eksternal
# (`minio:9000`) di-rewrite di sini ke alamat:port publik.
_rewrite_prometheus_scrapes(host_dir, doc, env_path)
if rc.returncode != 0:
sys.stderr.write(f"[host-{host}] render-config GAGAL:\n{rc.stdout}{rc.stderr}")
return 1
@@ -872,17 +911,42 @@ def _write_bundle_runtime_files(host_dir, env_path):
elif not pw_json.is_file():
pw_json.write_text("{}\n", encoding="utf-8")
pw_json.chmod(0o666)
# 2) Trino password.db — byte-identik dari sumber bila ada; kalau tidak, FILE kosong
# (yang penting BUKAN direktori; bootstrap mengisinya).
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
# 2) Trino password.db — DIBANGKITKAN dari TRINO_SVC_*_PASSWORD di .env (service
# account, HTTP Basic/bcrypt). Dulu hanya MENYALIN config/trino/password.db
# sumber — di mode cluster sumber itu sering KOSONG → bundle kosong → Trino
# menolak dgn 401 "Access Denied: Invalid credentials" (mis. dbt `svc_dbt`).
# Kini bundle MANDIRI & konsisten setelah --sync. Fallback: salin sumber bila
# htpasswd tak tersedia; kalau tidak ada, file kosong (yang penting BUKAN direktori).
dst_db = cfg / "trino" / "password.db"
dst_db.parent.mkdir(parents=True, exist_ok=True)
if dst_db.is_dir():
shutil.rmtree(dst_db)
if src_db.is_file():
shutil.copy2(src_db, dst_db)
elif not dst_db.is_file():
dst_db.write_text("", encoding="utf-8")
svc_accounts = (
("svc_seed", "TRINO_SVC_SEED_PASSWORD"),
("svc_dbt", "TRINO_SVC_DBT_PASSWORD"),
("svc_airflow", "TRINO_SVC_AIRFLOW_PASSWORD"),
("svc_om", "TRINO_SVC_OM_PASSWORD"),
)
pw_lines = []
htpasswd = shutil.which("htpasswd")
if htpasswd:
for user, key in svc_accounts:
secret = _source_env_value(env_path, key)
if not secret:
continue
proc = subprocess.run([htpasswd, "-nbB", "-C", "10", user, secret],
capture_output=True, text=True)
if proc.returncode == 0 and proc.stdout.strip():
pw_lines.append(proc.stdout.strip())
if pw_lines:
dst_db.write_text("\n".join(pw_lines) + "\n", encoding="utf-8")
dst_db.chmod(0o644)
else:
src_db = Path(env_path).resolve().parent / "config" / "trino" / "password.db"
if src_db.is_file():
shutil.copy2(src_db, dst_db)
elif not dst_db.is_file():
dst_db.write_text("", encoding="utf-8")
def main(argv: list[str]) -> int:
deploy/lib/cluster/inventory.py
+32 -3
View File
@@ -127,14 +127,36 @@ def apply_preset(doc: dict) -> dict:
return doc
def external_components(doc: dict) -> set[str]:
"""Komponen yang dikelola SISTEM LAIN: ditandai `components.<c>.address`.
Komponen ini TIDAK dijalankan cluster (tak masuk `placement`/`plan`); endpoint-nya
diarahkan ke alamat itu di SEMUA host (lihat generate.endpoint_lines). Berguna untuk
IAM/objek-storage milik pihak lain, termasuk di host/IP yang berbeda satu per komponen.
"""
return {n for n, s in (doc.get("components") or {}).items()
if (s.get("address") or "").strip()}
def external_address(doc: dict, service: str) -> str | None:
"""Alamat eksternal komponen (components.<svc>.address) bila ada, selain itu None."""
spec = (doc.get("components") or {}).get(service) or {}
a = (spec.get("address") or "").strip()
return a or None
def resolve(doc: dict) -> dict[str, list[str]]:
"""Komponen → daftar host (tier `engine` bisa >1 host)."""
"""Komponen → daftar host (tier `engine` bisa >1 host). Komponen EKSTERNAL
(ber-`address`) TIDAK masuk hasil — ia bukan milik host mana pun."""
hosts = doc.get("hosts") or {}
tiers_by_service = {svc: tier for tier, svcs in TIER_SERVICES.items() for svc in svcs}
explicit = {name: spec.get("host") for name, spec in (doc.get("components") or {}).items()}
external = external_components(doc)
placement: dict[str, list[str]] = {}
for service, tier in tiers_by_service.items():
if service in external:
continue # eksternal (pihak lain) → bukan milik host mana pun
if service in explicit and explicit[service]:
placement[service] = [explicit[service]]
continue
@@ -246,18 +268,23 @@ def validate(doc: dict) -> list[str]:
components = doc.get("components") or {}
for name, spec in components.items():
host = spec.get("host")
addr = (spec.get("address") or "").strip()
if host and host not in hosts:
errors.append(f"komponen '{name}': host '{host}' tidak ada di 'hosts'")
if host and addr:
errors.append(f"komponen '{name}': 'host' dan 'address' tak boleh bersamaan "
f"(host = dijalankan cluster; address = dikelola sistem lain)")
if tier_of(name) is None and name not in ("dremio",):
errors.append(f"komponen '{name}' tidak ada di taksonomi tier (TIER_SERVICES)")
# (c) tiap komponen harus bisa ditempatkan
placement = resolve(doc)
for service in (svc for svcs in TIER_SERVICES.values() for svc in svcs):
if service not in placement:
if service not in placement and not external_address(doc, service):
errors.append(
f"komponen '{service}' tidak bisa ditempatkan — tidak ada host bertier "
f"'{tier_of(service)}' dan tidak ada components.{service}.host"
f"'{tier_of(service)}', tidak ada components.{service}.host, "
f"dan bukan eksternal (components.{service}.address)"
)
# stateful: cegah pindah host tanpa migrasi (peringatan, bukan error)
@@ -331,6 +358,8 @@ def cmd_components(path: str) -> int:
doc = apply_preset(load(path))
for service, owners in sorted(resolve(doc).items()):
print(f"{service:24s} -> {','.join(owners)}")
for service in sorted(external_components(doc)):
print(f"{service:24s} -> (eksternal {external_address(doc, service)})")
return 0
deploy/lib/common.sh
+18
View File
@@ -145,6 +145,24 @@ sicabe_load_env() {
return 0
}
# ─── Helper: tujuan rsync default = CERMIN lokasi repo ini di bawah $HOME ────
# Dipakai `install.sh --sync` dan `scripts/check-trino-keystore.sh` supaya nama
# repo TIDAK dipatok (`~/sicabe-doc/deploy`). Repo distribusi bersih (mis.
# `sicabe-core` — isinya hanya folder `deploy/`) pun bekerja tanpa `--dest`:
# /home/u/sicabe-doc/deploy → ~/sicabe-doc/deploy
# /home/u/sicabe-core/deploy → ~/sicabe-core/deploy
# Bila repo di luar $HOME → ~/<nama-folder-induk>/deploy (fallback).
# Mengembalikan path ber-`~` (diekspansi shell REMOTE saat rsync/SSH).
sicabe_default_dest() { # <dir-absolut-dari-script> (default: $SCRIPT_DIR)
local _sdd_dir="${1:-${SCRIPT_DIR:-}}" _sdd_home="${HOME:-}"
[[ -n "${_sdd_dir}" ]] || { echo ""; return 1; }
if [[ -n "${_sdd_home}" && "${_sdd_dir}" == "${_sdd_home}/"* ]]; then
echo "~/${_sdd_dir#"${_sdd_home}/"}"
else
echo "~/$(basename "$(dirname "${_sdd_dir}")")/deploy"
fi
}
# ─── Helper: URL endpoint efektif (sadar EXTERNAL & multi-host) ─────────────
# Urutan: (1) <url_var> eksplisit NON-localhost (komponen EXTERNAL) → pakai apa adanya;
# (2) multi-host (bundle memuat SICABE_HOST_NAME) → scheme://<host_var>:<port>;