240 lines
15 KiB
Docker
240 lines
15 KiB
Docker
|
# syntax=docker/dockerfile:1
|
|||
|
|
# =============================================================================
|
|||
|
|
# eBdesk SiCABE — Custom Airflow Image 3.3.2 / Python 3.11 (BERDAMPINGAN)
|
|||
|
|
# =============================================================================
|
|||
|
|
# VARIAN BARU (review R18 / BATCH B). Berdampingan dengan `Dockerfile.slim`
|
|||
|
|
# (3.3.1/py3.11) — yang lama TIDAK diubah, sehingga rollback hanya soal menunjuk
|
|||
|
|
# tag image. Rencana: reviews/sicabe-18-merge-airflow332-plan.md
|
|||
|
|
# Asal-usul & bukti validasi: experiments/airflow332/ (H1–H9 terbukti)
|
|||
|
|
#
|
|||
|
|
# ⚠️ REVISI PENTING (B3 → B2 revisi): percobaan pertama memakai base
|
|||
|
|
# **Python 3.13** dan DITOLAK pada uji go/no-go: `dbt debug`/`dbt compile`
|
|||
|
|
# GAGAL di 3.13 —
|
|||
|
|
# mashumaro → typing_extensions.Annotated.__class_getitem__
|
|||
|
|
# → AttributeError: __class_getitem__ (typing.py 3.13)
|
|||
|
|
# dan TIDAK ADA versi `mashumaro` yang sekaligus memenuhi batas dependensi kita
|
|||
|
|
# (`collate-data-diff <3.11`, `dbt-core <3.15`) dan berfungsi di 3.13.
|
|||
|
|
# Bukti lengkap: deploy/output/b3-hasil-uji.md
|
|||
|
|
# ➡️ Karena itu base dikembalikan ke **Python 3.11** — tumpukan yang sudah
|
|||
|
|
# terbukti di produksi. Tujuan BATCH B (Airflow 3.3.2 + OM 2.0.3 + provider
|
|||
|
|
# MySQL) tetap tercapai, karena pin OM memaksa 3.3.2 terlepas dari Python.
|
|||
|
|
#
|
|||
|
|
# Perbedaan dari Dockerfile.slim:
|
|||
|
|
# 1. base : slim-3.3.1-python3.11 -> slim-3.3.2-python3.11
|
|||
|
|
# 2. provider MySQL : tidak ada -> DIPASANG (keputusan K3), dengan mysqlclient
|
|||
|
|
# dibangun di STAGE BUILDER + libmariadb3 sebagai runtime
|
|||
|
|
# 3. openmetadata : 2.0.2.0 -> 2.0.3.0, extra `[airflow]` DILEPAS
|
|||
|
|
# 4. sqlalchemy<2.1 : PIN WAJIB (hasil A1) — lihat requirements.332.in
|
|||
|
|
# 5. paket tambahan (2026-10-06): statsd (metrik StatsD), duckdb,
|
|||
|
|
# pyiceberg[s3fs,pyarrow]
|
|||
|
|
#
|
|||
|
|
# Base: apache/airflow:slim-3.3.2-python3.11
|
|||
|
|
# Debian 12 (bookworm) · Python 3.11.16 · TANPA gcc (maka stage builder wajib)
|
|||
|
|
#
|
|||
|
|
# Fakta yang sudah diverifikasi pada base ini:
|
|||
|
|
# - Python.h ADA di /usr/python/include/python3.11/Python.h → mysqlclient BISA
|
|||
|
|
# dibangun. Karena itu `python3-dev` Debian TIDAK dipasang.
|
|||
|
|
# - libmariadb3 tersedia (1:10.11.19+maria~deb12); `libmariadb3t64` TIDAK ada.
|
|||
|
|
# - mysqlclient terbukti terbangun di base 3.3.2 (di 3.13:
|
|||
|
|
# mysqlclient-2.3.0-cp313-cp313-linux_x86_64.whl; di 3.11 ekuivalen cp311).
|
|||
|
|
# - `pip install --user` DITERIMA di base 3.11 (di base 3.13 ia DITOLAK karena
|
|||
|
|
# VIRTUAL_ENV=/home/airflow/.local). Berkas ini TIDAK memakai `--user`:
|
|||
|
|
# hasilnya sama, karena venv-nya memang di ~/.local.
|
|||
|
|
#
|
|||
|
|
# ⚠️ URUTAN PEMASANGAN YANG MENENTUKAN (jangan diubah tanpa alasan):
|
|||
|
|
# mysqlclient tidak punya wheel Linux → bila dibiarkan, pip akan mencoba
|
|||
|
|
# membangunnya dari sdist di stage akhir yang TIDAK punya compiler → gagal.
|
|||
|
|
# Karena itu: bangun wheel di stage builder, lalu pasang dari wheelhouse LEBIH
|
|||
|
|
# DULU (sebelum install utama).
|
|||
|
|
#
|
|||
|
|
# Regenerasi lock (CARA YANG DIPAKAI — menghasilkan diff minimal):
|
|||
|
|
# (1) ambil lock produksi 3.3.1 sebagai dasar constraint
|
|||
|
|
# (2) BUANG dari constraint paket yang SENGAJA dinaikkan:
|
|||
|
|
# grep -vE '^(apache-airflow|openmetadata-|collate-)' requirements.lock > c.lock
|
|||
|
|
# (tanpa ini, constraint justru memblokir kenaikan 3.3.2 / OM 2.0.3 yang kita inginkan)
|
|||
|
|
# (3) resolve:
|
|||
|
|
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" -v /path/c.lock:/c.lock:ro \
|
|||
|
|
# apache/airflow:slim-3.3.2-python3.11 \
|
|||
|
|
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.332.in --constraint /c.lock'
|
|||
|
|
# Hasil: hanya 44 baris paket berubah vs produksi (Airflow, provider, OM, collate,
|
|||
|
|
# + MySQL) — jauh lebih kecil daripada resolusi bebas.
|
|||
|
|
# ⚠️ Constraints RESMI Airflow 3.3.2 TIDAK dapat dipakai bersama OM 2.0.3.0
|
|||
|
|
# (bentrok tabulate==0.10.0 vs 0.9.0 dan kubernetes==36.0.3 vs <36).
|
|||
|
|
#
|
|||
|
|
# Build (konteks = folder ini, karena COPY requirements.332.*):
|
|||
|
|
# docker build -t sicabe/airflow:3.3.2-om203-ol \
|
|||
|
|
# -f deploy/docker/airflow/Dockerfile.332 deploy/docker/airflow
|
|||
|
|
# =============================================================================
|
|||
|
|
|
|||
|
|
# ─── STAGE 1 — bangun mysqlclient (compiler hanya ada di sini) ──────────────
|
|||
|
|
FROM apache/airflow:slim-3.3.2-python3.11 AS mysqlwheel
|
|||
|
|
|
|||
|
|
USER root
|
|||
|
|
# gcc + pkg-config + header MariaDB. SENGAJA TANPA `python3-dev` (lihat catatan atas).
|
|||
|
|
# ⚠️ `/wheelhouse` harus DIBUAT dan dimiliki `airflow`; tanpa itu build gagal:
|
|||
|
|
# PermissionError: [Errno 13] Permission denied: '/wheelhouse'
|
|||
|
|
RUN apt-get update \
|
|||
|
|
&& apt-get install -y --no-install-recommends \
|
|||
|
|
gcc pkg-config default-libmysqlclient-dev \
|
|||
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|||
|
|
&& mkdir -p /wheelhouse && chown airflow:0 /wheelhouse
|
|||
|
|
|
|||
|
|
USER airflow
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip wheel --no-cache-dir --wheel-dir /wheelhouse "mysqlclient>=2.2.5" \
|
|||
|
|
&& ls -la /wheelhouse
|
|||
|
|
|
|||
|
|
# ─── STAGE 2 — image akhir (tanpa compiler) ────────────────────────────────
|
|||
|
|
FROM apache/airflow:slim-3.3.2-python3.11
|
|||
|
|
|
|||
|
|
ARG SLING_VERSION=1.5.23
|
|||
|
|
|
|||
|
|
USER root
|
|||
|
|
# Library RUNTIME untuk ekstensi mysqlclient (libmariadb.so.3). Tanpa paket ini:
|
|||
|
|
# ImportError: libmariadb.so.3: cannot open shared object file
|
|||
|
|
RUN apt-get update \
|
|||
|
|
&& apt-get install -y --no-install-recommends libmariadb3 \
|
|||
|
|
&& rm -rf /var/lib/apt/lists/* \
|
|||
|
|
&& ldconfig
|
|||
|
|
USER airflow
|
|||
|
|
|
|||
|
|
COPY --from=mysqlwheel --chown=airflow:0 /wheelhouse /wheelhouse
|
|||
|
|
COPY requirements.332.in /tmp/requirements.in
|
|||
|
|
COPY requirements.332.lock /tmp/requirements.lock
|
|||
|
|
|
|||
|
|
# ─── (1) mysqlclient dari WHEELHOUSE, lebih dulu ────────────────────────────
|
|||
|
|
# `--no-index` + `--find-links` memaksa pip memakai wheel hasil stage builder,
|
|||
|
|
# sehingga tidak pernah mencoba membangun sdist di stage tanpa compiler ini.
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --no-compile --no-index --find-links /wheelhouse \
|
|||
|
|
--constraint /tmp/requirements.lock mysqlclient
|
|||
|
|
|
|||
|
|
# ─── (2) Backend build untuk paket sdist pure-Python ────────────────────────
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --no-compile \
|
|||
|
|
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
|||
|
|
|
|||
|
|
# ─── (3) Dependensi dari lock ───────────────────────────────────────────────
|
|||
|
|
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
|||
|
|
pip install --no-build-isolation --no-compile \
|
|||
|
|
--constraint /tmp/requirements.lock \
|
|||
|
|
-r /tmp/requirements.in
|
|||
|
|
|
|||
|
|
# ─── Verifikasi inti (FATAL bila gagal) ─────────────────────────────────────
|
|||
|
|
RUN python -B -c "import airflow; print('airflow', airflow.__version__)"
|
|||
|
|
|
|||
|
|
# Kelima klien koneksi harus dapat diimpor (inti eksperimen airflow332).
|
|||
|
|
RUN python -B -c "import MySQLdb; print('MySQLdb (mysqlclient) OK ->', MySQLdb.version_info)" \
|
|||
|
|
&& python -B -c "import mysql.connector; print('mysql-connector-python OK ->', mysql.connector.__version__)" \
|
|||
|
|
&& python -B -c "import pymysql; print('pymysql OK')" \
|
|||
|
|
&& python -B -c "import psycopg2; print('psycopg2 OK ->', psycopg2.__version__)" \
|
|||
|
|
&& python -B -c "import trino; print('trino client OK ->', trino.__version__)"
|
|||
|
|
|
|||
|
|
# Bukti mysqlclient benar-benar DIMUAT (bukan sekadar terpasang): impor modul C-nya
|
|||
|
|
# sudah memuat libmariadb.so.3 — kalau library runtime tidak ada, perintah gagal.
|
|||
|
|
RUN python -B -c "import MySQLdb, MySQLdb._mysql; print('modul C mysqlclient dimuat:', MySQLdb._mysql.__file__)" \
|
|||
|
|
&& ( ldconfig -p 2>/dev/null | grep -i mariadb || echo '[WARN] libmariadb tidak terlihat di ldconfig' )
|
|||
|
|
|
|||
|
|
RUN python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')" \
|
|||
|
|
&& python -B -c "import airflow.providers.mysql; print('provider mysql OK')"
|
|||
|
|
|
|||
|
|
# ─── Provider OpenLineage (FATAL bila gagal) ────────────────────────────────
|
|||
|
|
# Ditambahkan 2026-10-02 atas permintaan pemilik proyek. Verifikasi memastikan:
|
|||
|
|
# 1. paket openlineage-python terpasang
|
|||
|
|
# 2. provider Airflow-nya dapat diimpor
|
|||
|
|
# 3. entry point provider benar-benar terdaftar
|
|||
|
|
# ⚠️ Provider ini INERT tanpa backend: hanya aktif bila transport/endpoint
|
|||
|
|
# OpenLineage dikonfigurasi (OPENLINEAGE_URL + OPENLINEAGE_API_KEY, dst).
|
|||
|
|
# Belum ada backend OpenLineage di stack ini — jadi ini mengaktifkan KEMAMPUAN.
|
|||
|
|
# Catatan pemeriksaan: di Airflow 3, NAMA entry point provider selalu
|
|||
|
|
# `provider_info` (yang membedakan adalah VALUE/module path-nya). Percobaan
|
|||
|
|
# pertama saya memeriksa `e.name` -> positif palsu. Yang fatal di sini cukup
|
|||
|
|
# impor keduanya; info entry point hanya DICETAK (tidak di-assert) agar tidak rapuh.
|
|||
|
|
RUN python -B -c "import openlineage; print('openlineage-python OK')" \
|
|||
|
|
&& python -B -c "import airflow.providers.openlineage; print('provider openlineage import OK')" \
|
|||
|
|
&& python -B -c "import importlib.metadata as m; print('versi provider:', m.version('apache-airflow-providers-openlineage')); vals=[e.value for e in m.entry_points(group='apache_airflow_provider')]; print('entry point provider:', [v for v in vals if 'lineage' in v.lower()] or '(nama/value tidak memuat kata lineage — normal di Airflow 3)')"
|
|||
|
|
|
|||
|
|
# ─── Shim eventTime OpenLineage (FATAL bila gagal) ───────────────────────────
|
|||
|
|
# Ditemukan di experiments/lineage-airflow-om: OpenMetadata 2.0.3 menolak event
|
|||
|
|
# yang `eventTime`-nya berakhiran `+00:00` (bentuk yang dihasilkan
|
|||
|
|
# datetime.isoformat() untuk UTC) dengan
|
|||
|
|
# 400 {"code":400,"message":"Invalid request format"}
|
|||
|
|
# Shim ZuluHttpTransport menormalkan `+00:00` -> `Z` sebelum event dikirim.
|
|||
|
|
# Ditempatkan di direktori khusus + PYTHONPATH agar dapat diimpor sebagai
|
|||
|
|
# `ol_time_shim.ZuluHttpTransport` dari proses task maupun subprosesnya.
|
|||
|
|
COPY --chown=airflow:0 ol_time_shim.py /opt/airflow/ol_shim/ol_time_shim.py
|
|||
|
|
# Catatan: image dasar `apache/airflow:slim-3.3.2-python3.11` TIDAK mendefinisikan
|
|||
|
|
# PYTHONPATH (diverifikasi dengan `docker run --entrypoint env`), jadi tidak ada
|
|||
|
|
# nilai lama yang tertimpa. Jangan menulis "${PYTHONPATH}" di sini: BuildKit
|
|||
|
|
# memperingatkannya sebagai variabel tak terdefinisi.
|
|||
|
|
ENV PYTHONPATH="/opt/airflow/ol_shim"
|
|||
|
|
RUN python -B -c "from ol_time_shim import ZuluHttpTransport; print('shim eventTime OpenLineage DAPAT DIMUAT:', ZuluHttpTransport)"
|
|||
|
|
|
|||
|
|
# ─── Penjaga regresi dbt (FATAL) — pelajaran dari uji B3 ────────────────────
|
|||
|
|
# `import dbt` dan `dbt --version` TIDAK CUKUP: pada percobaan Python 3.13
|
|||
|
|
# keduanya LOLOS, tetapi `dbt debug`/`dbt compile` gagal dengan
|
|||
|
|
# mashumaro -> typing_extensions.Annotated.__class_getitem__
|
|||
|
|
# -> AttributeError: __class_getitem__ (typing.py)
|
|||
|
|
# Pemeriksaan di bawah memakai PEMAKAIAN YANG SAH (`Annotated[tipe, anotasi]`),
|
|||
|
|
# yang menempuh jalur kode yang sama dengan dbt saat mem-parse model — sehingga
|
|||
|
|
# kerusakan jenis itu tertangkap saat BUILD.
|
|||
|
|
# ⚠️ Percobaan pertama penjaga ini SALAH dan menghasilkan positif palsu:
|
|||
|
|
# memanggil `Annotated.__class_getitem__((int,))` dengan SATU argumen ->
|
|||
|
|
# TypeError "Annotated[...] should be used with at least two arguments".
|
|||
|
|
# (Catatan: galat itu sekaligus membuktikan fungsinya ADA di py3.11.)
|
|||
|
|
# Catatan: `dbt debug` penuh tidak bisa dijalankan di sini karena butuh koneksi
|
|||
|
|
# Trino; pemeriksaan ini adalah proksi yang tepat dan murah.
|
|||
|
|
RUN python -B -c "import typing_extensions as te; te.Annotated[int, 'meta']" \
|
|||
|
|
&& python -B -c "import importlib.metadata as m; print('jalur Annotated OK -> dbt dapat berjalan; mashumaro', m.version('mashumaro'))"
|
|||
|
|
|
|||
|
|
# ─── Plugin OM harus benar-benar DAPAT DIMUAT (FATAL) ───────────────────────
|
|||
|
|
# Pelajaran dari experiments/airflow332: `import openmetadata_managed_apis` BERHASIL,
|
|||
|
|
# tetapi entry point gagal `load()` karena rantai
|
|||
|
|
# flask_appbuilder -> sqlalchemy_utils -> SQLAlchemy 2.1
|
|||
|
|
# (AttributeError: 'ScalarAttributeImpl')
|
|||
|
|
# → plugin MATI saat Airflow start; tombol "Test Connection" hilang; log hanya
|
|||
|
|
# berbunyi "Failed to import plugin openmetadata_managed_apis".
|
|||
|
|
# ⚠️ Verifikasi lama (hanya mengimpor MODUL) LOLOS pada kondisi rusak itu — karena
|
|||
|
|
# itu di sini entry point benar-benar dimuat, dan kegagalannya FATAL.
|
|||
|
|
# Ditulis SATU BARIS: kode Python multi-baris dalam `python -c "..."` tanpa `\`
|
|||
|
|
# akan dibaca Docker sebagai instruksi baru ("unknown instruction: import").
|
|||
|
|
RUN python -B -c "import flask_appbuilder; print('flask_appbuilder import OK')" \
|
|||
|
|
&& python -B -c "import sqlalchemy as sa; print('sqlalchemy', sa.__version__)" \
|
|||
|
|
&& python -B -c "import importlib.metadata as m; eps=[e for e in m.entry_points(group='airflow.plugins') if 'openmetadata_managed_apis' in e.name]; assert eps, 'entry point openmetadata_managed_apis tidak ada'; cls=eps[0].load(); print('plugin openmetadata_managed_apis DAPAT DIMUAT:', cls)"
|
|||
|
|
|
|||
|
|
# ─── Paket tambahan 2026-10-06 (FATAL bila gagal) ───────────────────────────
|
|||
|
|
# 1) statsd — WAJIB agar Airflow memakai StatsClient (StatsD → statsd-exporter).
|
|||
|
|
# Tanpa ini Airflow jatuh ke NoStatsLogger dan TIDAK ada metrik terkirim.
|
|||
|
|
RUN python -B -c "import statsd; print('statsd OK (StatsClient aktif)')"
|
|||
|
|
# 2) duckdb — analitik lokal di task DAG.
|
|||
|
|
RUN python -B -c "import duckdb; print('duckdb OK ->', duckdb.__version__)"
|
|||
|
|
# 3) pyiceberg[s3fs,pyarrow] — S3FileIO via s3fs + tabel Arrow.
|
|||
|
|
# Impor ketiganya SEKALIGUS: pyiceberg tanpa s3fs/pyarrow kehilangan extra-nya.
|
|||
|
|
RUN python -B -c "import pyiceberg, pyarrow, s3fs; print('pyiceberg OK ->', pyiceberg.__version__); print('pyarrow OK ->', pyarrow.__version__); print('s3fs OK ->', s3fs.__version__)"
|
|||
|
|
|
|||
|
|
# ─── Verifikasi non-fatal (melaporkan saja) ─────────────────────────────────
|
|||
|
|
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
|||
|
|
|| echo '[WARN] modul `metadata` gagal di-import' ) \
|
|||
|
|
&& ( python -B -c "import openmetadata_managed_apis; print('plugin openmetadata-managed-apis OK')" \
|
|||
|
|
|| echo '[WARN] plugin openmetadata-managed-apis gagal di-import' ) \
|
|||
|
|
&& ( python -B -c "import airflow.providers.postgres; print('provider postgres OK')" \
|
|||
|
|
|| echo '[WARN] provider postgres' ) \
|
|||
|
|
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
|||
|
|
|| echo '[WARN] CLI metadata tidak ada' ) \
|
|||
|
|
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
|||
|
|
|
|||
|
|
# ─── Sling CLI (binary Go — BUKAN paket pip `sling`) ────────────────────────
|
|||
|
|
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
|||
|
|
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
|||
|
|
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
|||
|
|
--max-time 2400 --connect-timeout 15 --continue-at - \
|
|||
|
|
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
|||
|
|
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
|||
|
|
&& mkdir -p "${HOME}/.local/bin" \
|
|||
|
|
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
|||
|
|
&& chmod +x /tmp/sling \
|
|||
|
|
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
|||
|
|
&& "${HOME}/.local/bin/sling" --version
|
|||
|
|
|
|||
|
|
USER airflow
|