134 lines
8.3 KiB
Docker
134 lines
8.3 KiB
Docker
# syntax=docker/dockerfile:1
|
|
# =============================================================================
|
|
# eBdesk SiCABE — Custom Airflow Image (base SLIM)
|
|
# =============================================================================
|
|
# Base: apache/airflow:slim-3.3.1-python3.11
|
|
# 1.15 GB · hanya 5 provider (standard, common-compat, common-io, common-sql, smtp)
|
|
# vs apache/airflow:3.3.1-python3.11 → 3.23 GB · 30 provider
|
|
#
|
|
# Yang di-bake:
|
|
# - provider tambahan: trino, amazon, http, postgres (standard/common-* dari base)
|
|
# - dbt-core 1.11.x + dbt-trino, great_expectations
|
|
# - openmetadata-ingestion[airflow,pii-processor]==2.0.2.0 + plugin
|
|
# openmetadata-managed-apis==2.0.2.0, disajikan Airflow di /pluginsv2/
|
|
# (inilah yang menghidupkan tombol Test Connection — §8.11)
|
|
# Extra `pii-processor` (presidio + spacy) WAJIB agar DAG yang dibangkitkan
|
|
# plugin untuk pipeline TestSuite bisa diimpor Airflow — tanpa itu:
|
|
# No module named 'presidio_analyzer' (§8.10)
|
|
# Konsekuensi: image +390 MB dan pandas diturunkan ke 2.3.3 (extra minta pandas<3).
|
|
# - Sling CLI (binary Go)
|
|
#
|
|
# Catatan penting:
|
|
# 1. ✅ Extra `openmetadata-ingestion[airflow]` DIPAKAI sejak OM 2.0.2. Di OM 2.0.2
|
|
# extra itu mem-pin apache-airflow==3.3.1 (sama dengan base image ini), sedang
|
|
# di OM 1.13.x mem-pin 3.2.1 sehingga menurunkan Airflow. Diuji di experiments/om20x.
|
|
# 2. ⚠️ Sebagian paket hanya sdist (antlr4-python3-runtime, dsnparse, …) → wajib
|
|
# `--no-build-isolation`. Keduanya **pure Python** sehingga TIDAK butuh gcc
|
|
# (base slim memang tidak punya gcc).
|
|
# 3. ✅ `dbt-core<1.12` (1.12.x menarik dbt-core-experimental-parser, sdist-only).
|
|
# 4. ✅ `--constraint requirements.lock` → tanpa backtracking.
|
|
# 5. ⚠️ Base slim TIDAK punya psycopg2, padahal Postgres adalah metadata DB
|
|
# Airflow (postgresql+psycopg2://) → disediakan oleh provider `postgres`.
|
|
# 6. `strip` tidak ada di slim → binary Sling dibiarkan apa adanya (255 MB).
|
|
#
|
|
# Provider MySQL — TIDAK dipasang (disengaja):
|
|
# `apache-airflow-providers-mysql` tidak dipakai SiCABE — koneksi Airflow hanya
|
|
# aws/trino/postgres/http, dan OpenMetadata memakai konektor + pymysql miliknya
|
|
# sendiri (`metadata/ingestion/source/database/mysql` + `pymysql==1.2.0`).
|
|
# Selain itu provider ini butuh `mysqlclient` yang TIDAK punya wheel Linux
|
|
# (sdist → butuh compiler, sedangkan base slim tanpa gcc), sehingga hook-nya
|
|
# (MySQLdb) tetap tidak berfungsi walau provider dipasang.
|
|
# Bila kelak benar-benar dibutuhkan, bangun mysqlclient di stage builder:
|
|
# FROM apache/airflow:slim-3.3.1-python3.11 AS mysqlwheel
|
|
# USER root
|
|
# RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
# gcc pkg-config default-libmysqlclient-dev python3-dev \
|
|
# && rm -rf /var/lib/apt/lists/*
|
|
# USER airflow
|
|
# RUN pip wheel --wheel-dir /wheelhouse "mysqlclient>=2.2.5"
|
|
# lalu di stage akhir: COPY --from=mysqlwheel /wheelhouse /wheelhouse +
|
|
# pip install --no-index --find-links /wheelhouse mysqlclient,
|
|
# serta apt-get install libmariadb3 (lib runtime).
|
|
#
|
|
# Regenerasi lock:
|
|
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" apache/airflow:slim-3.3.1-python3.11 \
|
|
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.in --output-file /w/requirements.lock'
|
|
#
|
|
# Build (konteks = folder ini, karena COPY requirements.*):
|
|
# docker build -t sicabe/airflow:3.3.1-om202 -f deploy/docker/airflow/Dockerfile.slim deploy/docker/airflow
|
|
# =============================================================================
|
|
|
|
FROM apache/airflow:slim-3.3.1-python3.11
|
|
|
|
ARG SLING_VERSION=1.5.23
|
|
|
|
USER airflow
|
|
|
|
COPY requirements.in /tmp/requirements.in
|
|
COPY requirements.lock /tmp/requirements.lock
|
|
|
|
# ─── (1) Backend build untuk paket sdist pure-Python ────────────────────────
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|
pip install --user --no-compile \
|
|
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
|
|
|
# ─── (2) Dependensi dari lock (termasuk provider postgres → psycopg2-binary) ─
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|
pip install --user --no-build-isolation --no-compile \
|
|
--constraint /tmp/requirements.lock \
|
|
-r /tmp/requirements.in
|
|
|
|
# ─── Verifikasi inti (fatal bila gagal) ─────────────────────────────────────
|
|
# `python -B` → jangan tulis .pyc (menghemat puluhan MB layer).
|
|
RUN python -B -c "import airflow; print('airflow', airflow.__version__)" \
|
|
&& python -B -c "import psycopg2; print('psycopg2', psycopg2.__version__)" \
|
|
&& python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')"
|
|
|
|
# ─── Plugin OM harus benar-benar DAPAT DIMUAT (FATAL) ───────────────────────
|
|
# Pelajaran dari experiments/airflow332 (reviews/sicabe-18, A1):
|
|
# `import openmetadata_managed_apis` BERHASIL, tetapi entry point-nya gagal
|
|
# `load()` karena rantai flask_appbuilder -> sqlalchemy_utils -> SQLAlchemy 2.1
|
|
# (AttributeError: module 'sqlalchemy.orm.attributes' has no attribute
|
|
# 'ScalarAttributeImpl'). Akibatnya plugin MATI saat Airflow start: tombol
|
|
# "Test Connection" / pipeline service client (§8.11) hilang, dan log hanya
|
|
# berbunyi "Failed to import plugin openmetadata_managed_apis" — tanpa
|
|
# menyebut SQLAlchemy sama sekali.
|
|
# ⚠️ Verifikasi lama (hanya mengimpor MODUL) LOLOS pada kondisi rusak itu.
|
|
# Karena itu di sini entry point benar-benar dimuat, dan kegagalannya FATAL:
|
|
# lebih baik build berhenti daripada menghasilkan image yang tombolnya mati.
|
|
# Catatan: ditulis SATU BARIS — kode Python multi-baris dalam `python -c "..."`
|
|
# tanpa `\` akan dibaca Docker sebagai instruksi baru ("unknown instruction").
|
|
RUN python -B -c "import flask_appbuilder; print('flask_appbuilder import OK')" \
|
|
&& python -B -c "import sqlalchemy as sa; print('sqlalchemy', sa.__version__)" \
|
|
&& python -B -c "import importlib.metadata as m; eps=[e for e in m.entry_points(group='airflow.plugins') if 'openmetadata_managed_apis' in e.name]; assert eps, 'entry point openmetadata_managed_apis tidak ada'; cls=eps[0].load(); print('plugin openmetadata_managed_apis DAPAT DIMUAT:', cls)"
|
|
|
|
# ─── Verifikasi OM & provider (non-fatal: hanya melaporkan) ─────────────────
|
|
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
|
|| echo '[WARN] modul `metadata` gagal di-import' ) \
|
|
&& ( python -B -c "import openmetadata_managed_apis; print('plugin openmetadata-managed-apis OK')" \
|
|
|| echo '[WARN] plugin openmetadata-managed-apis gagal di-import' ) \
|
|
&& python -B -c "import importlib.metadata as m; print('airflow.plugins entry points:', [e.name for e in m.entry_points(group='airflow.plugins')])" \
|
|
&& ( python -B -c "import importlib.metadata as m; assert 'openmetadata_managed_apis' in [e.name for e in m.entry_points(group='airflow.plugins')]" \
|
|
|| echo '[WARN] plugin TIDAK terdaftar pada entry point airflow.plugins' ) \
|
|
&& ( python -B -c "import airflow.providers.postgres; print('provider postgres OK')" \
|
|
|| echo '[WARN] provider postgres' ) \
|
|
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
|
|| echo '[WARN] CLI metadata tidak ada' ) \
|
|
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
|
|
|
# ─── Sling CLI (binary Go — BUKAN paket pip `sling`) ────────────────────────
|
|
# Tarball ±94 MB. Timeout tinggi + retry + resume; di-cache agar tidak mengulang.
|
|
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
|
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
|
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
|
--max-time 2400 --connect-timeout 15 --continue-at - \
|
|
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
|
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
|
&& mkdir -p "${HOME}/.local/bin" \
|
|
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
|
&& chmod +x /tmp/sling \
|
|
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
|
&& "${HOME}/.local/bin/sling" --version
|
|
|
|
USER airflow
|