337 lines
14 KiB
Bash
337 lines
14 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# =============================================================================
|
||
|
|
# prefetch-images.sh — tarik SEMUA image Docker lebih dulu (agar install offline-ish)
|
||
|
|
# -----------------------------------------------------------------------------
|
||
|
|
# Daftar image TIDAK di-hardcode: diambil dari `docker compose config --images`,
|
||
|
|
# sehingga selalu sinkron dengan docker-compose.yml (termasuk image yang berada di
|
||
|
|
# balik profil, mis. dremio).
|
||
|
|
#
|
||
|
|
# CATATAN PENTING — satu image dibangun LOKAL:
|
||
|
|
# sicabe/airflow:3.3.2-om203-ol dibangun dari docker/airflow/Dockerfile.332
|
||
|
|
# (base apache/airflow:slim-3.3.2-python3.11). Ia TIDAK ada di registry publik,
|
||
|
|
# sehingga `pull` dari Docker Hub tidak mungkin.
|
||
|
|
# ✅ SEJAK 2026-10-02 image ini TERSEDIA di REGISTRY INTERNAL:
|
||
|
|
# registry.sicabe.com/airflow:3.3.2-om203-ol
|
||
|
|
# Script ini OTOMATIS ikut menariknya bila `AIRFLOW_IMAGE` menunjuk REGISTRY
|
||
|
|
# (host memuat titik, mis. `registry.sicabe.com/airflow:…`) — cukup set
|
||
|
|
# AIRFLOW_IMAGE=registry.sicabe.com/airflow:3.3.2-om203-ol
|
||
|
|
# di .env. Bila `AIRFLOW_IMAGE` dibiarkan nama LOKAL polos (`sicabe/airflow:…`),
|
||
|
|
# image DIKECUALIKAN dari pull dan wajib dibangun/di-retag manual (alur di bawah).
|
||
|
|
# Untuk host TANPA akses registry, tetap ada alur build-sekali-salin:
|
||
|
|
# 1) di SATU host: ./scripts/prefetch-images.sh
|
||
|
|
# docker build -t sicabe/airflow:3.3.2-om203-ol \
|
||
|
|
# -f docker/airflow/Dockerfile.332 docker/airflow
|
||
|
|
# ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar
|
||
|
|
# 2) salin: scp /tmp/airflow.tar host2:/tmp/ ; scp … host3:/tmp/
|
||
|
|
# 3) di host lain: ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar
|
||
|
|
#
|
||
|
|
# Pemakaian:
|
||
|
|
# ./scripts/prefetch-images.sh [--all | --host <nama>] [--list] [--no-base]
|
||
|
|
# ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar
|
||
|
|
# ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar
|
||
|
|
# =============================================================================
|
||
|
|
set -uo pipefail
|
||
|
|
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
DEPLOY_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||
|
|
|
||
|
|
RED=$'\e[31m'; GRN=$'\e[32m'; YLW=$'\e[33m'; BLD=$'\e[1m'; NC=$'\e[0m'
|
||
|
|
# PENTING: semua diagnostik ke STDERR. stdout hanya untuk DATA (daftar image),
|
||
|
|
# karena collect_images() ditangkap dengan $( ) — pesan info yang ikut tertangkap
|
||
|
|
# akan dianggap nama image (bug yang ditemukan saat menguji --list).
|
||
|
|
ok() { echo -e " ${GRN}[OK]${NC} $*" >&2; }
|
||
|
|
inf() { echo -e " ${YLW}[..]${NC} $*" >&2; }
|
||
|
|
warn() { echo -e " ${YLW}[WARN]${NC} $*" >&2; }
|
||
|
|
err() { echo -e " ${RED}[!!]${NC} $*" >&2; }
|
||
|
|
|
||
|
|
MODE="all" # all | host
|
||
|
|
HOST_NAME=""
|
||
|
|
LIST_ONLY=0
|
||
|
|
# Dremio (opt-in di compose), image build lokal, dan base build Airflow DEFAULT
|
||
|
|
# DIKELUARKAN dari daftar prefetch. Ketiganya diaktifkan lewat flag bila memang
|
||
|
|
# diperlukan (mis. deployment yang menjalankan dremio / akan build Airflow).
|
||
|
|
WITH_BASE=0
|
||
|
|
WITH_DREMIO=0
|
||
|
|
SAVE_AIRFLOW=""
|
||
|
|
LOAD_AIRFLOW=""
|
||
|
|
|
||
|
|
usage() { sed -n '2,28p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0; }
|
||
|
|
|
||
|
|
parse_args() {
|
||
|
|
while [[ $# -gt 0 ]]; do
|
||
|
|
case "$1" in
|
||
|
|
--all) MODE="all" ;;
|
||
|
|
--host) MODE="host"; HOST_NAME="${2:-}"; shift ;;
|
||
|
|
--list) LIST_ONLY=1 ;;
|
||
|
|
--no-base) WITH_BASE=0 ;; # dipertahankan: kini defaultnya sudah tanpa base
|
||
|
|
--with-base) WITH_BASE=1 ;;
|
||
|
|
--with-dremio) WITH_DREMIO=1 ;;
|
||
|
|
--save-airflow) SAVE_AIRFLOW="${2:-}"; shift ;;
|
||
|
|
--load-airflow) LOAD_AIRFLOW="${2:-}"; shift ;;
|
||
|
|
-h|--help) usage ;;
|
||
|
|
*) err "Argumen tidak dikenal: $1"; exit 2 ;;
|
||
|
|
esac
|
||
|
|
shift
|
||
|
|
done
|
||
|
|
}
|
||
|
|
|
||
|
|
require_docker() {
|
||
|
|
if ! command -v docker &>/dev/null; then
|
||
|
|
err "docker tidak ditemukan — jalankan ./scripts/prepare-host.sh lebih dulu."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Daftar image (tidak di-hardcode) ───────────────────────────────────────
|
||
|
|
compose_images() { # <file> [--env-file <f>] [--profile …]…
|
||
|
|
local file="$1"; shift
|
||
|
|
local out err_file
|
||
|
|
# stderr JANGAN ditelan: kalau kosong, tampilkan penyebabnya (pelajaran dari gate
|
||
|
|
# Fase 5 yang sempat menyembunyikan error compose sehingga gagal sulit dilacak).
|
||
|
|
#
|
||
|
|
# Catatan bug: versi pertama memakai `local err` + `trap … RETURN`. Trap RETURN
|
||
|
|
# berjalan SETELAH variabel local dibersihkan sehingga ${err} menjadi unbound, dan
|
||
|
|
# nama `err` juga menutupi FUNGSI err(). Kini tanpa trap + bernama err_file.
|
||
|
|
err_file="$(mktemp)"
|
||
|
|
out="$(docker compose -f "${file}" "$@" config --images 2>"${err_file}" | sort -u)"
|
||
|
|
if [[ -z "${out}" && -s "${err_file}" ]]; then
|
||
|
|
warn "docker compose gagal mendaftar image:"
|
||
|
|
sed 's/^/ /' "${err_file}" >&2
|
||
|
|
fi
|
||
|
|
rm -f "${err_file}"
|
||
|
|
printf '%s\n' "${out}"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Isi ENV_ARGS (variabel global) dengan argumen env untuk compose.
|
||
|
|
# Bila .env BELUM ADA (server masih bersih — .env dibuat install.sh saat instalasi),
|
||
|
|
# pakai default dari config/environment.defaults.sh: ke-16 variabel yang diwajibkan
|
||
|
|
# compose (:?err) semuanya ada di sana, dan nilainya TIDAK memengaruhi daftar image
|
||
|
|
# (image di-pin di docker-compose.yml).
|
||
|
|
#
|
||
|
|
# PENTING: fungsi ini HARUS dipanggil di shell saat ini — bukan di dalam $( ) — supaya
|
||
|
|
# `set -a; source defaults` benar-benar berlaku untuk proses compose kita. Versi pertama
|
||
|
|
# saya memanggilnya dalam command substitution (subshell), jadi sourcing-nya sia-sia.
|
||
|
|
ENV_ARGS=()
|
||
|
|
resolve_env() { # <envfile>
|
||
|
|
local envfile="$1"
|
||
|
|
if [[ -f "${envfile}" ]]; then
|
||
|
|
ENV_ARGS=(--env-file "${envfile}")
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
warn "berkas ${envfile} belum ada — memakai default config/environment.defaults.sh"
|
||
|
|
echo " (.env dibuat oleh install.sh saat instalasi; tidak memengaruhi daftar image)" >&2
|
||
|
|
set -a
|
||
|
|
# shellcheck disable=SC1091
|
||
|
|
. "${DEPLOY_DIR}/config/environment.defaults.sh"
|
||
|
|
set +a
|
||
|
|
ENV_ARGS=()
|
||
|
|
}
|
||
|
|
|
||
|
|
collect_images() {
|
||
|
|
local file envfile
|
||
|
|
if [[ "${MODE}" == "host" ]]; then
|
||
|
|
[[ -z "${HOST_NAME}" ]] && { err "--host memerlukan nama host."; exit 2; }
|
||
|
|
file="${DEPLOY_DIR}/generated/host-${HOST_NAME}/docker-compose.yml"
|
||
|
|
if [[ ! -f "${file}" ]]; then
|
||
|
|
err "Bundle host belum ada: ${file}"
|
||
|
|
echo " Jalankan dulu: ./install.sh --generate (atau --generate --inventory cluster.yml)"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
envfile="${DEPLOY_DIR}/generated/host-${HOST_NAME}/.env"
|
||
|
|
[[ -f "${envfile}" ]] || envfile="${DEPLOY_DIR}/.env"
|
||
|
|
inf "sumber daftar image: bundle host '${HOST_NAME}'"
|
||
|
|
resolve_env "${envfile}"
|
||
|
|
compose_images "${file}" ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} --profile all
|
||
|
|
else
|
||
|
|
local extra=()
|
||
|
|
if [[ ${WITH_DREMIO} -eq 1 ]]; then
|
||
|
|
extra=(--profile dremio)
|
||
|
|
inf "sumber daftar image: docker-compose.yml (profil all + dremio)"
|
||
|
|
else
|
||
|
|
inf "sumber daftar image: docker-compose.yml (profil all; dremio dikeluarkan)"
|
||
|
|
fi
|
||
|
|
resolve_env "${DEPLOY_DIR}/.env"
|
||
|
|
compose_images "${DEPLOY_DIR}/docker-compose.yml" ${ENV_ARGS[@]+"${ENV_ARGS[@]}"} \
|
||
|
|
--profile all ${extra[@]+"${extra[@]}"}
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
# Base image Airflow diambil dari Dockerfile agar tidak ter-hardcode di sini.
|
||
|
|
airflow_base_image() {
|
||
|
|
local df="${DEPLOY_DIR}/docker/airflow/Dockerfile.slim"
|
||
|
|
[[ -f "${df}" ]] || return 1
|
||
|
|
grep -m1 '^FROM ' "${df}" | awk '{print $2}'
|
||
|
|
}
|
||
|
|
|
||
|
|
airflow_built_image() {
|
||
|
|
# shellcheck disable=SC1091
|
||
|
|
( set -a; [[ -f "${DEPLOY_DIR}/config/environment.defaults.sh" ]] && . "${DEPLOY_DIR}/config/environment.defaults.sh"
|
||
|
|
[[ -f "${DEPLOY_DIR}/.env" ]] && . "${DEPLOY_DIR}/.env"
|
||
|
|
echo "${AIRFLOW_IMAGE:-sicabe/airflow:3.3.2-om203-ol}" ) 2>/dev/null | tail -1
|
||
|
|
}
|
||
|
|
|
||
|
|
# Host registry dari nama image — untuk petunjuk diagnosa saat pull gagal.
|
||
|
|
# Image tanpa '/' atau dengan komponen pertama tanpa titik berasal dari Docker Hub
|
||
|
|
# (endpoint API-nya registry-1.docker.io).
|
||
|
|
registry_host() {
|
||
|
|
# Dipecah DUA baris: `local x="$1" y="${x%%/*}"` mengevaluasi RHS `y` SEBELUM `x`
|
||
|
|
# di-assign → "unbound variable" di bawah `set -u` (bug laten yang baru muncul saat
|
||
|
|
# fungsi ini dipanggil di jalur normal).
|
||
|
|
local img="$1"
|
||
|
|
local first="${img%%/*}"
|
||
|
|
# WAJIB ada '/' dulu: tanpa itu, "postgres:15" membuat ${img%%/*} = seluruh string
|
||
|
|
# dan tanda ':' adalah TAG, bukan port registry (bug yang ditemukan saat menguji).
|
||
|
|
if [[ "${img}" == */* && ( "${first}" == *.* || "${first}" == *:* || "${first}" == "localhost" ) ]]; then
|
||
|
|
echo "${first}"
|
||
|
|
else
|
||
|
|
echo "registry-1.docker.io"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
image_size_mb() {
|
||
|
|
docker image inspect -f '{{.Size}}' "$1" 2>/dev/null | awk '{printf "%d", $1/1024/1024}'
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Aksi: simpan / muat image Airflow yang dibangun lokal ──────────────────
|
||
|
|
do_save_airflow() {
|
||
|
|
local built; built="$(airflow_built_image)"
|
||
|
|
if ! docker image inspect "${built}" &>/dev/null; then
|
||
|
|
err "Image '${built}' belum ada — bangun dulu:"
|
||
|
|
echo " docker build -t ${built} -f docker/airflow/Dockerfile.332 docker/airflow"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
inf "menyimpan '${built}' ke ${SAVE_AIRFLOW} (berisi seluruh layer, termasuk base)"
|
||
|
|
docker save -o "${SAVE_AIRFLOW}" "${built}" && \
|
||
|
|
ok "tersimpan ($(du -h "${SAVE_AIRFLOW}" | cut -f1)) — salin ke host lain lalu pakai --load-airflow"
|
||
|
|
}
|
||
|
|
|
||
|
|
do_load_airflow() {
|
||
|
|
[[ -f "${LOAD_AIRFLOW}" ]] || { err "berkas tidak ditemukan: ${LOAD_AIRFLOW}"; exit 1; }
|
||
|
|
inf "memuat image dari ${LOAD_AIRFLOW}"
|
||
|
|
docker load -i "${LOAD_AIRFLOW}" && ok "image dimuat"
|
||
|
|
local built; built="$(airflow_built_image)"
|
||
|
|
docker image inspect "${built}" &>/dev/null && ok "'${built}' tersedia" || warn "image '${built}' belum terlihat"
|
||
|
|
}
|
||
|
|
|
||
|
|
# ─── Aksi: tarik image ──────────────────────────────────────────────────────
|
||
|
|
do_pull() {
|
||
|
|
local images base built built_local
|
||
|
|
images="$(collect_images)"
|
||
|
|
if [[ -z "${images}" ]]; then
|
||
|
|
err "daftar image kosong — periksa docker-compose.yml / bundle host."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
base="$(airflow_base_image || true)"
|
||
|
|
built="$(airflow_built_image)"
|
||
|
|
|
||
|
|
# Image Airflow: dikecualikan HANYA bila namanya menunjuk LOKAL polos
|
||
|
|
# (`sicabe/airflow:…`, tidak ada host registry). Bila `AIRFLOW_IMAGE` menunjuk
|
||
|
|
# REGISTRY (mis. `registry.sicabe.com/airflow:…`, host memuat titik), image
|
||
|
|
# IKUT di-pull seperti image lain — sejak image dipublikasikan ke registry internal.
|
||
|
|
# Deteksi memakai registry_host(): nama Docker Hub polos -> "registry-1.docker.io".
|
||
|
|
built_local=0
|
||
|
|
if [[ -n "${built}" && "$(registry_host "${built}")" == "registry-1.docker.io" ]]; then
|
||
|
|
built_local=1
|
||
|
|
# Keluarkan dari daftar (bukan sekadar dilewati) supaya daftar = yang benar-benar diunduh.
|
||
|
|
images="$(printf '%s\n' "${images}" | grep -vxF "${built}" || true)"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo -e "${BLD}Daftar image (${MODE})${NC}"
|
||
|
|
printf '%s\n' "${images}" | sed 's/^/ - /'
|
||
|
|
[[ -n "${base}" && ${WITH_BASE} -eq 1 ]] && echo " - ${base} (base build Airflow)"
|
||
|
|
if [[ -n "${built}" && ${built_local} -eq 1 ]]; then
|
||
|
|
echo " (dikecualikan: ${built} — dibangun lokal, tidak ada di Docker Hub; tersedia di registry internal)"
|
||
|
|
fi
|
||
|
|
[[ ${WITH_DREMIO} -eq 0 ]] && echo " (dikecualikan: dremio/dremio-oss — opt-in, aktifkan dengan --with-dremio)"
|
||
|
|
|
||
|
|
if [[ ${LIST_ONLY} -eq 1 ]]; then
|
||
|
|
echo
|
||
|
|
inf "--list: tidak ada yang ditarik."
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo -e "${BLD}Menarik image${NC}"
|
||
|
|
local failed=() okc=0
|
||
|
|
while IFS= read -r img; do
|
||
|
|
[[ -z "${img}" ]] && continue
|
||
|
|
# Image yang dibangun lokal tidak ada di registry — jangan dicoba di-pull.
|
||
|
|
if [[ ${built_local} -eq 1 && "${img}" == "${built}" ]]; then
|
||
|
|
inf "${img} — dibangun lokal, dilewati (lihat alur build-sekali-salin di bawah)"
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
if docker image inspect "${img}" &>/dev/null && [[ "${img}" != *":latest" ]]; then
|
||
|
|
ok "${img} (sudah ada)"
|
||
|
|
okc=$((okc + 1))
|
||
|
|
continue
|
||
|
|
fi
|
||
|
|
inf "pull ${img} …"
|
||
|
|
# Output pull ditangkap ke berkas, BUKAN dibuang: tanpa ini kegagalan hanya
|
||
|
|
# berbunyi "GAGAL" tanpa sebab (dialami saat minio/quay.io gagal di server).
|
||
|
|
local perr; perr="$(mktemp)"
|
||
|
|
if docker pull "${img}" >"${perr}" 2>&1; then
|
||
|
|
ok "${img}"
|
||
|
|
okc=$((okc + 1))
|
||
|
|
else
|
||
|
|
warn "GAGAL: ${img}"
|
||
|
|
tail -3 "${perr}" | sed 's/^/ /' >&2
|
||
|
|
local reg; reg="$(registry_host "${img}")"
|
||
|
|
echo " hint: periksa akses ke registry '${reg}' → curl -sI https://${reg}/v2/" >&2
|
||
|
|
failed+=("${img}")
|
||
|
|
fi
|
||
|
|
rm -f "${perr}"
|
||
|
|
done <<< "${images}"
|
||
|
|
|
||
|
|
if [[ -n "${base}" && ${WITH_BASE} -eq 1 ]]; then
|
||
|
|
if docker image inspect "${base}" &>/dev/null; then
|
||
|
|
ok "${base} (sudah ada)"
|
||
|
|
elif docker pull -q "${base}" >/dev/null 2>&1; then
|
||
|
|
ok "${base} (base build Airflow)"
|
||
|
|
else
|
||
|
|
warn "GAGAL: ${base} — build Airflow lokal akan mengunduh ulang"
|
||
|
|
failed+=("${base}")
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo
|
||
|
|
echo -e "${BLD}Ringkasan${NC}"
|
||
|
|
echo " berhasil/sudah ada : ${okc}"
|
||
|
|
if [[ ${#failed[@]} -gt 0 ]]; then
|
||
|
|
echo " gagal : ${#failed[@]}"
|
||
|
|
printf ' - %s\n' "${failed[@]}"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Ukuran total image yang terdaftar (perkiraan kebutuhan disk)
|
||
|
|
local total=0 size
|
||
|
|
while IFS= read -r img; do
|
||
|
|
size="$(image_size_mb "${img}")"
|
||
|
|
[[ -n "${size}" ]] && total=$((total + size))
|
||
|
|
done <<< "${images}"
|
||
|
|
echo " ukuran total : ~${total} MB (image yang terdaftar)"
|
||
|
|
|
||
|
|
if [[ ${built_local} -eq 1 ]] && ! docker image inspect "${built}" &>/dev/null; then
|
||
|
|
echo
|
||
|
|
warn "'${built}' BELUM ada dan ia TIDAK dapat di-pull (dibangun lokal)."
|
||
|
|
echo " Alur build-sekali-salin (opsi b):"
|
||
|
|
echo " di SATU host : docker build -t ${built} -f docker/airflow/Dockerfile.332 docker/airflow"
|
||
|
|
echo " ./scripts/prefetch-images.sh --save-airflow /tmp/airflow.tar"
|
||
|
|
echo " host lain : scp /tmp/airflow.tar <host>:/tmp/"
|
||
|
|
echo " ./scripts/prefetch-images.sh --load-airflow /tmp/airflow.tar"
|
||
|
|
fi
|
||
|
|
|
||
|
|
[[ ${#failed[@]} -eq 0 ]]
|
||
|
|
}
|
||
|
|
|
||
|
|
main() {
|
||
|
|
parse_args "$@"
|
||
|
|
require_docker
|
||
|
|
if [[ -n "${SAVE_AIRFLOW}" ]]; then do_save_airflow; exit $?; fi
|
||
|
|
if [[ -n "${LOAD_AIRFLOW}" ]]; then do_load_airflow; exit $?; fi
|
||
|
|
echo -e "${BLD}Prefetch image SiCABE${NC}"
|
||
|
|
do_pull
|
||
|
|
}
|
||
|
|
|
||
|
|
main "$@"
|