first commit
This commit is contained in:
Executable
+336
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# prefetch-images.sh — tarik SEMUA image Docker lebih dulu (agar install offline-ish)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Daftar image TIDAK di-hardcode: diambil dari `docker compose config --images`,
|
||||
# sehingga selalu sinkron dengan docker-compose.yml (termasuk image yang berada di
|
||||
# balik profil, mis. dremio).
|
||||
#
|
||||
# CATATAN PENTING — satu image dibangun LOKAL:
|
||||
# sicabe/airflow:3.3.2-om203-ol dibangun dari docker/airflow/Dockerfile.332
|
||||
# (base apache/airflow:slim-3.3.2-python3.11). Ia TIDAK ada di registry publik,
|
||||
# sehingga `pull` dari Docker Hub tidak mungkin.
|
||||
# ✅ SEJAK 2026-10-02 image ini TERSEDIA di REGISTRY INTERNAL:
|
||||
# registry.sicabe.com/airflow:3.3.2-om203-ol
|
||||
# Script ini OTOMATIS ikut menariknya bila `AIRFLOW_IMAGE` menunjuk REGISTRY
|
||||
# (host memuat titik, mis. `registry.sicabe.com/airflow:…`) — cukup set
|
||||
# AIRFLOW_IMAGE=registry.sicabe.com/airflow:3.3.2-om203-ol
|
||||
# di .env. Bila `AIRFLOW_IMAGE` dibiarkan nama LOKAL polos (`sicabe/airflow:…`),
|
||||
# image DIKECUALIKAN dari pull dan wajib dibangun/di-retag manual (alur di bawah).
|
||||
# Untuk host TANPA akses registry, tetap ada alur build-sekali-salin:
|
||||
# 1) di SATU host: ./scripts/prefetch-images.sh
|
||||
# docker build -t sicabe/airflow:3.3.2-om203-ol \
|
||||
# -f docker/airflow/Dockerfile.332 docker/airflow
|
||||
# ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar
|
||||
# 2) salin: scp /tmp/airflow.tar host2:/tmp/ ; scp … host3:/tmp/
|
||||
# 3) di host lain: ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar
|
||||
#
|
||||
# Pemakaian:
|
||||
# ./scripts/prefetch-images.sh [--all | --host <nama>] [--list] [--no-base]
|
||||
# ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar
|
||||
# ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
RED=$'\e[31m'; GRN=$'\e[32m'; YLW=$'\e[33m'; BLD=$'\e[1m'; NC=$'\e[0m'
|
||||
# PENTING: semua diagnostik ke STDERR. stdout hanya untuk DATA (daftar image),
|
||||
# karena collect_images() ditangkap dengan $( ) — pesan info yang ikut tertangkap
|
||||
# akan dianggap nama image (bug yang ditemukan saat menguji --list).
|
||||
ok() { echo -e " ${GRN}[OK]${NC} $*" >&2; }
|
||||
inf() { echo -e " ${YLW}[..]${NC} $*" >&2; }
|
||||
warn() { echo -e " ${YLW}[WARN]${NC} $*" >&2; }
|
||||
err() { echo -e " ${RED}[!!]${NC} $*" >&2; }
|
||||
|
||||
MODE="all" # all | host
|
||||
HOST_NAME=""
|
||||
LIST_ONLY=0
|
||||
# Dremio (opt-in di compose), image build lokal, dan base build Airflow DEFAULT
|
||||
# DIKELUARKAN dari daftar prefetch. Ketiganya diaktifkan lewat flag bila memang
|
||||
# diperlukan (mis. deployment yang menjalankan dremio / akan build Airflow).
|
||||
WITH_BASE=0
|
||||
WITH_DREMIO=0
|
||||
SAVE_AIRFLOW=""
|
||||
LOAD_AIRFLOW=""
|
||||
|
||||
usage() { sed -n '2,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0; }
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--all) MODE="all" ;;
|
||||
--host) MODE="host"; HOST_NAME="${2:-}"; shift ;;
|
||||
--list) LIST_ONLY=1 ;;
|
||||
--no-base) WITH_BASE=0 ;; # dipertahankan: kini defaultnya sudah tanpa base
|
||||
--with-base) WITH_BASE=1 ;;
|
||||
--with-dremio) WITH_DREMIO=1 ;;
|
||||
--save-airflow) SAVE_AIRFLOW="${2:-}"; shift ;;
|
||||
--load-airflow) LOAD_AIRFLOW="${2:-}"; shift ;;
|
||||
-h|--help) usage ;;
|
||||
*) err "Argumen tidak dikenal: $1"; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
}
|
||||
|
||||
require_docker() {
|
||||
if ! command -v docker &>/dev/null; then
|
||||
err "docker tidak ditemukan — jalankan ./scripts/prepare-host.sh lebih dulu."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Daftar image (tidak di-hardcode) ───────────────────────────────────────
|
||||
compose_images() { # <file> [--env-file <f>] [--profile …]…
|
||||
local file="$1"; shift
|
||||
local out err_file
|
||||
# stderr JANGAN ditelan: kalau kosong, tampilkan penyebabnya (pelajaran dari gate
|
||||
# Fase 5 yang sempat menyembunyikan error compose sehingga gagal sulit dilacak).
|
||||
#
|
||||
# Catatan bug: versi pertama memakai `local err` + `trap … RETURN`. Trap RETURN
|
||||
# berjalan SETELAH variabel local dibersihkan sehingga ${err} menjadi unbound, dan
|
||||
# nama `err` juga menutupi FUNGSI err(). Kini tanpa trap + bernama err_file.
|
||||
err_file="$(mktemp)"
|
||||
out="$(docker compose -f "${file}" "$@" config --images 2>"${err_file}" | sort -u)"
|
||||
if [[ -z "${out}" && -s "${err_file}" ]]; then
|
||||
warn "docker compose gagal mendaftar image:"
|
||||
sed 's/^/ /' "${err_file}" >&2
|
||||
fi
|
||||
rm -f "${err_file}"
|
||||
printf '%s\n' "${out}"
|
||||
}
|
||||
|
||||
# Isi ENV_ARGS (variabel global) dengan argumen env untuk compose.
|
||||
# Bila .env BELUM ADA (server masih bersih — .env dibuat install.sh saat instalasi),
|
||||
# pakai default dari config/environment.defaults.sh: ke-16 variabel yang diwajibkan
|
||||
# compose (:?err) semuanya ada di sana, dan nilainya TIDAK memengaruhi daftar image
|
||||
# (image di-pin di docker-compose.yml).
|
||||
#
|
||||
# PENTING: fungsi ini HARUS dipanggil di shell saat ini — bukan di dalam $( ) — supaya
|
||||
# `set -a; source defaults` benar-benar berlaku untuk proses compose kita. Versi pertama
|
||||
# saya memanggilnya dalam command substitution (subshell), jadi sourcing-nya sia-sia.
|
||||
ENV_ARGS=()
|
||||
resolve_env() { # <envfile>
|
||||
local envfile="$1"
|
||||
if [[ -f "${envfile}" ]]; then
|
||||
ENV_ARGS=(--env-file "${envfile}")
|
||||
return 0
|
||||
fi
|
||||
warn "berkas ${envfile} belum ada — memakai default config/environment.defaults.sh"
|
||||
echo " (.env dibuat oleh install.sh saat instalasi; tidak memengaruhi daftar image)" >&2
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "${DEPLOY_DIR}/config/environment.defaults.sh"
|
||||
set +a
|
||||
ENV_ARGS=()
|
||||
}
|
||||
|
||||
collect_images() {
|
||||
local file envfile
|
||||
if [[ "${MODE}" == "host" ]]; then
|
||||
[[ -z "${HOST_NAME}" ]] && { err "--host memerlukan nama host."; exit 2; }
|
||||
file="${DEPLOY_DIR}/generated/host-${HOST_NAME}/docker-compose.yml"
|
||||
if [[ ! -f "${file}" ]]; then
|
||||
err "Bundle host belum ada: ${file}"
|
||||
echo " Jalankan dulu: ./install.sh --generate (atau --generate --inventory cluster.yml)"
|
||||
exit 1
|
||||
fi
|
||||
envfile="${DEPLOY_DIR}/generated/host-${HOST_NAME}/.env"
|
||||
[[ -f "${envfile}" ]] || envfile="${DEPLOY_DIR}/.env"
|
||||
inf "sumber daftar image: bundle host '${HOST_NAME}'"
|
||||
resolve_env "${envfile}"
|
||||
compose_images "${file}" ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} --profile all
|
||||
else
|
||||
local extra=()
|
||||
if [[ ${WITH_DREMIO} -eq 1 ]]; then
|
||||
extra=(--profile dremio)
|
||||
inf "sumber daftar image: docker-compose.yml (profil all + dremio)"
|
||||
else
|
||||
inf "sumber daftar image: docker-compose.yml (profil all; dremio dikeluarkan)"
|
||||
fi
|
||||
resolve_env "${DEPLOY_DIR}/.env"
|
||||
compose_images "${DEPLOY_DIR}/docker-compose.yml" ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} \
|
||||
--profile all ${extra[@]+"${extra[@]}"}
|
||||
fi
|
||||
}
|
||||
|
||||
# Base image Airflow diambil dari Dockerfile agar tidak ter-hardcode di sini.
|
||||
airflow_base_image() {
|
||||
local df="${DEPLOY_DIR}/docker/airflow/Dockerfile.slim"
|
||||
[[ -f "${df}" ]] || return 1
|
||||
grep -m1 '^FROM ' "${df}" | awk '{print $2}'
|
||||
}
|
||||
|
||||
airflow_built_image() {
|
||||
# shellcheck disable=SC1091
|
||||
( set -a; [[ -f "${DEPLOY_DIR}/config/environment.defaults.sh" ]] && . "${DEPLOY_DIR}/config/environment.defaults.sh"
|
||||
[[ -f "${DEPLOY_DIR}/.env" ]] && . "${DEPLOY_DIR}/.env"
|
||||
echo "${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}" ) 2>/dev/null | tail -1
|
||||
}
|
||||
|
||||
# Host registry dari nama image — untuk petunjuk diagnosa saat pull gagal.
|
||||
# Image tanpa '/' atau dengan komponen pertama tanpa titik berasal dari Docker Hub
|
||||
# (endpoint API-nya registry-1.docker.io).
|
||||
registry_host() {
|
||||
# Dipecah DUA baris: `local x="$1" y="${x%%/*}"` mengevaluasi RHS `y` SEBELUM `x`
|
||||
# di-assign → "unbound variable" di bawah `set -u` (bug laten yang baru muncul saat
|
||||
# fungsi ini dipanggil di jalur normal).
|
||||
local img="$1"
|
||||
local first="${img%%/*}"
|
||||
# WAJIB ada '/' dulu: tanpa itu, "postgres:15" membuat ${img%%/*} = seluruh string
|
||||
# dan tanda ':' adalah TAG, bukan port registry (bug yang ditemukan saat menguji).
|
||||
if [[ "${img}" == */* && ( "${first}" == *.* || "${first}" == *:* || "${first}" == "localhost" ) ]]; then
|
||||
echo "${first}"
|
||||
else
|
||||
echo "registry-1.docker.io"
|
||||
fi
|
||||
}
|
||||
|
||||
image_size_mb() {
|
||||
docker image inspect -f '{{.Size}}' "$1" 2>/dev/null | awk '{printf "%d", $1/1024/1024}'
|
||||
}
|
||||
|
||||
# ─── Aksi: simpan / muat image Airflow yang dibangun lokal ──────────────────
|
||||
do_save_airflow() {
|
||||
local built; built="$(airflow_built_image)"
|
||||
if ! docker image inspect "${built}" &>/dev/null; then
|
||||
err "Image '${built}' belum ada — bangun dulu:"
|
||||
echo " docker build -t ${built} -f docker/airflow/Dockerfile.332 docker/airflow"
|
||||
exit 1
|
||||
fi
|
||||
inf "menyimpan '${built}' ke ${SAVE_AIRFLOW} (berisi seluruh layer, termasuk base)"
|
||||
docker save -o "${SAVE_AIRFLOW}" "${built}" && \
|
||||
ok "tersimpan ($(du -h "${SAVE_AIRFLOW}" | cut -f1)) — salin ke host lain lalu pakai --load-airflow"
|
||||
}
|
||||
|
||||
do_load_airflow() {
|
||||
[[ -f "${LOAD_AIRFLOW}" ]] || { err "berkas tidak ditemukan: ${LOAD_AIRFLOW}"; exit 1; }
|
||||
inf "memuat image dari ${LOAD_AIRFLOW}"
|
||||
docker load -i "${LOAD_AIRFLOW}" && ok "image dimuat"
|
||||
local built; built="$(airflow_built_image)"
|
||||
docker image inspect "${built}" &>/dev/null && ok "'${built}' tersedia" || warn "image '${built}' belum terlihat"
|
||||
}
|
||||
|
||||
# ─── Aksi: tarik image ──────────────────────────────────────────────────────
|
||||
do_pull() {
|
||||
local images base built built_local
|
||||
images="$(collect_images)"
|
||||
if [[ -z "${images}" ]]; then
|
||||
err "daftar image kosong — periksa docker-compose.yml / bundle host."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
base="$(airflow_base_image || true)"
|
||||
built="$(airflow_built_image)"
|
||||
|
||||
# Image Airflow: dikecualikan HANYA bila namanya menunjuk LOKAL polos
|
||||
# (`sicabe/airflow:…`, tidak ada host registry). Bila `AIRFLOW_IMAGE` menunjuk
|
||||
# REGISTRY (mis. `registry.sicabe.com/airflow:…`, host memuat titik), image
|
||||
# IKUT di-pull seperti image lain — sejak image dipublikasikan ke registry internal.
|
||||
# Deteksi memakai registry_host(): nama Docker Hub polos -> "registry-1.docker.io".
|
||||
built_local=0
|
||||
if [[ -n "${built}" && "$(registry_host "${built}")" == "registry-1.docker.io" ]]; then
|
||||
built_local=1
|
||||
# Keluarkan dari daftar (bukan sekadar dilewati) supaya daftar = yang benar-benar diunduh.
|
||||
images="$(printf '%s\n' "${images}" | grep -vxF "${built}" || true)"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${BLD}Daftar image (${MODE})${NC}"
|
||||
printf '%s\n' "${images}" | sed 's/^/ - /'
|
||||
[[ -n "${base}" && ${WITH_BASE} -eq 1 ]] && echo " - ${base} (base build Airflow)"
|
||||
if [[ -n "${built}" && ${built_local} -eq 1 ]]; then
|
||||
echo " (dikecualikan: ${built} — dibangun lokal, tidak ada di Docker Hub; tersedia di registry internal)"
|
||||
fi
|
||||
[[ ${WITH_DREMIO} -eq 0 ]] && echo " (dikecualikan: dremio/dremio-oss — opt-in, aktifkan dengan --with-dremio)"
|
||||
|
||||
if [[ ${LIST_ONLY} -eq 1 ]]; then
|
||||
echo
|
||||
inf "--list: tidak ada yang ditarik."
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${BLD}Menarik image${NC}"
|
||||
local failed=() okc=0
|
||||
while IFS= read -r img; do
|
||||
[[ -z "${img}" ]] && continue
|
||||
# Image yang dibangun lokal tidak ada di registry — jangan dicoba di-pull.
|
||||
if [[ ${built_local} -eq 1 && "${img}" == "${built}" ]]; then
|
||||
inf "${img} — dibangun lokal, dilewati (lihat alur build-sekali-salin di bawah)"
|
||||
continue
|
||||
fi
|
||||
if docker image inspect "${img}" &>/dev/null && [[ "${img}" != *":latest" ]]; then
|
||||
ok "${img} (sudah ada)"
|
||||
okc=$((okc + 1))
|
||||
continue
|
||||
fi
|
||||
inf "pull ${img} …"
|
||||
# Output pull ditangkap ke berkas, BUKAN dibuang: tanpa ini kegagalan hanya
|
||||
# berbunyi "GAGAL" tanpa sebab (dialami saat minio/quay.io gagal di server).
|
||||
local perr; perr="$(mktemp)"
|
||||
if docker pull "${img}" >"${perr}" 2>&1; then
|
||||
ok "${img}"
|
||||
okc=$((okc + 1))
|
||||
else
|
||||
warn "GAGAL: ${img}"
|
||||
tail -3 "${perr}" | sed 's/^/ /' >&2
|
||||
local reg; reg="$(registry_host "${img}")"
|
||||
echo " hint: periksa akses ke registry '${reg}' → curl -sI https://${reg}/v2/" >&2
|
||||
failed+=("${img}")
|
||||
fi
|
||||
rm -f "${perr}"
|
||||
done <<< "${images}"
|
||||
|
||||
if [[ -n "${base}" && ${WITH_BASE} -eq 1 ]]; then
|
||||
if docker image inspect "${base}" &>/dev/null; then
|
||||
ok "${base} (sudah ada)"
|
||||
elif docker pull -q "${base}" >/dev/null 2>&1; then
|
||||
ok "${base} (base build Airflow)"
|
||||
else
|
||||
warn "GAGAL: ${base} — build Airflow lokal akan mengunduh ulang"
|
||||
failed+=("${base}")
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${BLD}Ringkasan${NC}"
|
||||
echo " berhasil/sudah ada : ${okc}"
|
||||
if [[ ${#failed[@]} -gt 0 ]]; then
|
||||
echo " gagal : ${#failed[@]}"
|
||||
printf ' - %s\n' "${failed[@]}"
|
||||
fi
|
||||
|
||||
# Ukuran total image yang terdaftar (perkiraan kebutuhan disk)
|
||||
local total=0 size
|
||||
while IFS= read -r img; do
|
||||
size="$(image_size_mb "${img}")"
|
||||
[[ -n "${size}" ]] && total=$((total + size))
|
||||
done <<< "${images}"
|
||||
echo " ukuran total : ~${total} MB (image yang terdaftar)"
|
||||
|
||||
if [[ ${built_local} -eq 1 ]] && ! docker image inspect "${built}" &>/dev/null; then
|
||||
echo
|
||||
warn "'${built}' BELUM ada dan ia TIDAK dapat di-pull (dibangun lokal)."
|
||||
echo " Alur build-sekali-salin (opsi b):"
|
||||
echo " di SATU host : docker build -t ${built} -f docker/airflow/Dockerfile.332 docker/airflow"
|
||||
echo " ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar"
|
||||
echo " host lain : scp /tmp/airflow.tar <host>:/tmp/"
|
||||
echo " ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar"
|
||||
fi
|
||||
|
||||
[[ ${#failed[@]} -eq 0 ]]
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
require_docker
|
||||
if [[ -n "${SAVE_AIRFLOW}" ]]; then do_save_airflow; exit $?; fi
|
||||
if [[ -n "${LOAD_AIRFLOW}" ]]; then do_load_airflow; exit $?; fi
|
||||
echo -e "${BLD}Prefetch image SiCABE${NC}"
|
||||
do_pull
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user