115 lines
6.5 KiB
Docker
115 lines
6.5 KiB
Docker
|
# syntax=docker/dockerfile:1
|
|||
|
|
# =============================================================================
|
|||
|
|
# eBdesk SiCABE — Custom Airflow Image (strategi S1: SATU environment)
|
|||
|
|
# =============================================================================
|
|||
|
|
# Base: apache/airflow:3.3.1-python3.11
|
|||
|
|
# - Python 3.11 → masuk matriks uji openmetadata-ingestion 1.13.x
|
|||
|
|
# - Airflow 3.3.1 (terbaru)
|
|||
|
|
#
|
|||
|
|
# Yang di-bake:
|
|||
|
|
# - apache-airflow 3.3.1 + providers (trino, amazon, http)
|
|||
|
|
# - dbt-core 1.11.x + dbt-trino, great_expectations
|
|||
|
|
# - openmetadata-ingestion==1.13.1.0 (match server OM 1.13.1; TANPA extra [airflow])
|
|||
|
|
# - Sling CLI (binary Go — BUKAN paket pip `sling`)
|
|||
|
|
#
|
|||
|
|
# Tiga aturan penting:
|
|||
|
|
# 1. ❌ JANGAN pakai extra `openmetadata-ingestion[airflow]` — extra itu mem-pin
|
|||
|
|
# `apache-airflow==3.2.1` dan langsung merusak resolver.
|
|||
|
|
# 2. ⚠️ Beberapa dependensi OM hanya tersedia sebagai **sdist** (tanpa wheel):
|
|||
|
|
# `antlr4-python3-runtime==4.9.2`, `dsnparse==0.1.15`, dst. Dengan build
|
|||
|
|
# isolation, pip MENGGANTUNG sangat lama; karena itu dipakai
|
|||
|
|
# `--no-build-isolation` (build hanya beberapa detik) + backend build disiapkan.
|
|||
|
|
# 3. ✅ `dbt-core<1.12` (lihat requirements.in): 1.12.x menarik
|
|||
|
|
# `dbt-core-experimental-parser` yang sdist-only.
|
|||
|
|
# Instalasi memakai `--constraint requirements.lock` (hasil `uv pip compile`)
|
|||
|
|
# agar tidak ada backtracking.
|
|||
|
|
#
|
|||
|
|
# Catatan `pip check` (NON-FATAL, sengaja): setelah langkah (3) tersisa 2 konflik
|
|||
|
|
# dengan komponen yang TIDAK dipakai DAG SiCABE:
|
|||
|
|
# - aiobotocore -> botocore>=1.43.3 vs botocore 1.41.6 (OM: boto3~=1.41.5)
|
|||
|
|
# - providers-mysql -> pymysql<1.2 vs pymysql 1.2.0
|
|||
|
|
# Bila kelak DAG memakai salah satunya, pindahkan toolchain-nya ke environment
|
|||
|
|
# terpisah (venv) — lihat strategi S2 di reviews/sicabe-10.
|
|||
|
|
#
|
|||
|
|
# Regenerasi lock bila dependensi berubah:
|
|||
|
|
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" apache/airflow:3.3.1-python3.11 \
|
|||
|
|
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.in --output-file /w/requirements.lock'
|
|||
|
|
#
|
|||
|
|
# Build (konteks = folder ini, karena COPY requirements.*):
|
|||
|
|
# docker build -t sicabe/airflow:3.3.1 -f deploy/docker/airflow/Dockerfile deploy/docker/airflow
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
FROM apache/airflow:3.3.1-python3.11
|
|||
|
|
|
|||
|
|
ARG AIRFLOW_VERSION=3.3.1
|
|||
|
|
ARG SLING_VERSION=1.5.23
|
|||
|
|
|
|||
|
|
USER airflow
|
|||
|
|
|
|||
|
|
COPY requirements.in /tmp/requirements.in
|
|||
|
|
COPY requirements.lock /tmp/requirements.lock
|
|||
|
|
|
|||
|
|
# ─── (1) Backend build untuk paket sdist ────────────────────────────────────
|
|||
|
|
# Dipakai oleh langkah (2) karena build isolation dimatikan.
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --user --no-compile \
|
|||
|
|
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
|||
|
|
|
|||
|
|
# ─── (2) Seluruh dependensi dari lock ───────────────────────────────────────
|
|||
|
|
# --no-build-isolation: kunci utama. Sebagian paket (antlr4-python3-runtime,
|
|||
|
|
# dsnparse, …) hanya sdist; dengan isolasi pip menggantung sangat lama.
|
|||
|
|
# --constraint: pin dari lock → tidak ada backtracking.
|
|||
|
|
# --no-compile: tidak menulis .pyc (menghemat puluhan MB; Python tetap jalan,
|
|||
|
|
# hanya sedikit lebih lambat saat import pertama).
|
|||
|
|
# Cache mount pip → rebuild tidak mengunduh ulang seluruh wheel.
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --user --no-build-isolation --no-compile \
|
|||
|
|
--constraint /tmp/requirements.lock \
|
|||
|
|
-r /tmp/requirements.in
|
|||
|
|
|
|||
|
|
# ─── (3) Buang providers-google + dependensi yatim-piatunya ─────────────────
|
|||
|
|
# SiCABE hanya memakai provider trino/amazon/http. `apache-airflow-providers-google`
|
|||
|
|
# (bawaan base image) menarik paket besar: ray (~186 MB), pyarrow, litellm,
|
|||
|
|
# scikit-learn, google-api-python-client, google-cloud-*, pandas-gbq, dsb.
|
|||
|
|
# `pip-autoremove` menghapus paket yang tidak lagi dibutuhkan paket lain.
|
|||
|
|
# Hasil uji: ~/.local 2.5G -> 1.8G (−0.7 GB); airflow/dbt/GE/OM ingestion tetap OK.
|
|||
|
|
#
|
|||
|
|
# ⚠️ Bila kelak butuh Google (mis. Workspace/Gmail/Drive), pasang provider-nya
|
|||
|
|
# kembali — provider Airflow tidak dipecah per produk, jadi akan menarik
|
|||
|
|
# seluruh dependensi google lagi (≈ +0.7 GB).
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --no-compile pip-autoremove \
|
|||
|
|
&& pip-autoremove apache-airflow-providers-google -y \
|
|||
|
|
&& pip uninstall -y pip-autoremove
|
|||
|
|
|
|||
|
|
# ─── Verifikasi inti (fatal bila gagal) ─────────────────────────────────────
|
|||
|
|
# `python -B` → jangan tulis .pyc saat verifikasi (menghemat ~50 MB layer).
|
|||
|
|
RUN python -B -c "import airflow; print('airflow', airflow.__version__)" \
|
|||
|
|
&& python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')"
|
|||
|
|
|
|||
|
|
# ─── Verifikasi OM (non-fatal: hanya melaporkan) ────────────────────────────
|
|||
|
|
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
|||
|
|
|| echo '[WARN] modul `metadata` gagal di-import — cek nama paket OM' ) \
|
|||
|
|
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
|||
|
|
|| echo '[WARN] CLI `metadata` tidak ditemukan' ) \
|
|||
|
|
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
|||
|
|
|
|||
|
|
# ─── Sling CLI (binary Go) ──────────────────────────────────────────────────
|
|||
|
|
# Tarball ±99 MB. Jaringan lambat → timeout tinggi + retry + resume (-C -),
|
|||
|
|
# dan hasil unduhan disimpan di cache mount agar percobaan ulang tidak mulai
|
|||
|
|
# dari nol. Nama file memuat versi supaya cache tidak salah pakai.
|
|||
|
|
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
|||
|
|
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
|||
|
|
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
|||
|
|
--max-time 2400 --connect-timeout 15 --continue-at - \
|
|||
|
|
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
|||
|
|
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
|||
|
|
&& mkdir -p "${HOME}/.local/bin" \
|
|||
|
|
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
|||
|
|
&& ( command -v strip >/dev/null && strip /tmp/sling 2>/dev/null || true ) \
|
|||
|
|
&& chmod +x /tmp/sling \
|
|||
|
|
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
|||
|
|
&& "${HOME}/.local/bin/sling" --version
|
|||
|
|
|
|||
|
|
USER airflow
|