first commit
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Custom Airflow Image (strategi S1: SATU environment)
|
||||
# =============================================================================
|
||||
# Base: apache/airflow:3.3.1-python3.11
|
||||
# - Python 3.11 → masuk matriks uji openmetadata-ingestion 1.13.x
|
||||
# - Airflow 3.3.1 (terbaru)
|
||||
#
|
||||
# Yang di-bake:
|
||||
# - apache-airflow 3.3.1 + providers (trino, amazon, http)
|
||||
# - dbt-core 1.11.x + dbt-trino, great_expectations
|
||||
# - openmetadata-ingestion==1.13.1.0 (match server OM 1.13.1; TANPA extra [airflow])
|
||||
# - Sling CLI (binary Go — BUKAN paket pip `sling`)
|
||||
#
|
||||
# Tiga aturan penting:
|
||||
# 1. ❌ JANGAN pakai extra `openmetadata-ingestion[airflow]` — extra itu mem-pin
|
||||
# `apache-airflow==3.2.1` dan langsung merusak resolver.
|
||||
# 2. ⚠️ Beberapa dependensi OM hanya tersedia sebagai **sdist** (tanpa wheel):
|
||||
# `antlr4-python3-runtime==4.9.2`, `dsnparse==0.1.15`, dst. Dengan build
|
||||
# isolation, pip MENGGANTUNG sangat lama; karena itu dipakai
|
||||
# `--no-build-isolation` (build hanya beberapa detik) + backend build disiapkan.
|
||||
# 3. ✅ `dbt-core<1.12` (lihat requirements.in): 1.12.x menarik
|
||||
# `dbt-core-experimental-parser` yang sdist-only.
|
||||
# Instalasi memakai `--constraint requirements.lock` (hasil `uv pip compile`)
|
||||
# agar tidak ada backtracking.
|
||||
#
|
||||
# Catatan `pip check` (NON-FATAL, sengaja): setelah langkah (3) tersisa 2 konflik
|
||||
# dengan komponen yang TIDAK dipakai DAG SiCABE:
|
||||
# - aiobotocore -> botocore>=1.43.3 vs botocore 1.41.6 (OM: boto3~=1.41.5)
|
||||
# - providers-mysql -> pymysql<1.2 vs pymysql 1.2.0
|
||||
# Bila kelak DAG memakai salah satunya, pindahkan toolchain-nya ke environment
|
||||
# terpisah (venv) — lihat strategi S2 di reviews/sicabe-10.
|
||||
#
|
||||
# Regenerasi lock bila dependensi berubah:
|
||||
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" apache/airflow:3.3.1-python3.11 \
|
||||
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.in --output-file /w/requirements.lock'
|
||||
#
|
||||
# Build (konteks = folder ini, karena COPY requirements.*):
|
||||
# docker build -t sicabe/airflow:3.3.1 -f deploy/docker/airflow/Dockerfile deploy/docker/airflow
|
||||
# =============================================================================
|
||||
|
||||
FROM apache/airflow:3.3.1-python3.11
|
||||
|
||||
ARG AIRFLOW_VERSION=3.3.1
|
||||
ARG SLING_VERSION=1.5.23
|
||||
|
||||
USER airflow
|
||||
|
||||
COPY requirements.in /tmp/requirements.in
|
||||
COPY requirements.lock /tmp/requirements.lock
|
||||
|
||||
# ─── (1) Backend build untuk paket sdist ────────────────────────────────────
|
||||
# Dipakai oleh langkah (2) karena build isolation dimatikan.
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --user --no-compile \
|
||||
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
||||
|
||||
# ─── (2) Seluruh dependensi dari lock ───────────────────────────────────────
|
||||
# --no-build-isolation: kunci utama. Sebagian paket (antlr4-python3-runtime,
|
||||
# dsnparse, …) hanya sdist; dengan isolasi pip menggantung sangat lama.
|
||||
# --constraint: pin dari lock → tidak ada backtracking.
|
||||
# --no-compile: tidak menulis .pyc (menghemat puluhan MB; Python tetap jalan,
|
||||
# hanya sedikit lebih lambat saat import pertama).
|
||||
# Cache mount pip → rebuild tidak mengunduh ulang seluruh wheel.
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --user --no-build-isolation --no-compile \
|
||||
--constraint /tmp/requirements.lock \
|
||||
-r /tmp/requirements.in
|
||||
|
||||
# ─── (3) Buang providers-google + dependensi yatim-piatunya ─────────────────
|
||||
# SiCABE hanya memakai provider trino/amazon/http. `apache-airflow-providers-google`
|
||||
# (bawaan base image) menarik paket besar: ray (~186 MB), pyarrow, litellm,
|
||||
# scikit-learn, google-api-python-client, google-cloud-*, pandas-gbq, dsb.
|
||||
# `pip-autoremove` menghapus paket yang tidak lagi dibutuhkan paket lain.
|
||||
# Hasil uji: ~/.local 2.5G -> 1.8G (−0.7 GB); airflow/dbt/GE/OM ingestion tetap OK.
|
||||
#
|
||||
# ⚠️ Bila kelak butuh Google (mis. Workspace/Gmail/Drive), pasang provider-nya
|
||||
# kembali — provider Airflow tidak dipecah per produk, jadi akan menarik
|
||||
# seluruh dependensi google lagi (≈ +0.7 GB).
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --no-compile pip-autoremove \
|
||||
&& pip-autoremove apache-airflow-providers-google -y \
|
||||
&& pip uninstall -y pip-autoremove
|
||||
|
||||
# ─── Verifikasi inti (fatal bila gagal) ─────────────────────────────────────
|
||||
# `python -B` → jangan tulis .pyc saat verifikasi (menghemat ~50 MB layer).
|
||||
RUN python -B -c "import airflow; print('airflow', airflow.__version__)" \
|
||||
&& python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')"
|
||||
|
||||
# ─── Verifikasi OM (non-fatal: hanya melaporkan) ────────────────────────────
|
||||
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
||||
|| echo '[WARN] modul `metadata` gagal di-import — cek nama paket OM' ) \
|
||||
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
||||
|| echo '[WARN] CLI `metadata` tidak ditemukan' ) \
|
||||
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
||||
|
||||
# ─── Sling CLI (binary Go) ──────────────────────────────────────────────────
|
||||
# Tarball ±99 MB. Jaringan lambat → timeout tinggi + retry + resume (-C -),
|
||||
# dan hasil unduhan disimpan di cache mount agar percobaan ulang tidak mulai
|
||||
# dari nol. Nama file memuat versi supaya cache tidak salah pakai.
|
||||
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
||||
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
||||
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
--max-time 2400 --connect-timeout 15 --continue-at - \
|
||||
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
||||
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
||||
&& mkdir -p "${HOME}/.local/bin" \
|
||||
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
||||
&& ( command -v strip >/dev/null && strip /tmp/sling 2>/dev/null || true ) \
|
||||
&& chmod +x /tmp/sling \
|
||||
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
||||
&& "${HOME}/.local/bin/sling" --version
|
||||
|
||||
USER airflow
|
||||
@@ -0,0 +1,239 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Custom Airflow Image 3.3.2 / Python 3.11 (BERDAMPINGAN)
|
||||
# =============================================================================
|
||||
# VARIAN BARU (review R18 / BATCH B). Berdampingan dengan `Dockerfile.slim`
|
||||
# (3.3.1/py3.11) — yang lama TIDAK diubah, sehingga rollback hanya soal menunjuk
|
||||
# tag image. Rencana: reviews/sicabe-18-merge-airflow332-plan.md
|
||||
# Asal-usul & bukti validasi: experiments/airflow332/ (H1–H9 terbukti)
|
||||
#
|
||||
# ⚠️ REVISI PENTING (B3 → B2 revisi): percobaan pertama memakai base
|
||||
# **Python 3.13** dan DITOLAK pada uji go/no-go: `dbt debug`/`dbt compile`
|
||||
# GAGAL di 3.13 —
|
||||
# mashumaro → typing_extensions.Annotated.__class_getitem__
|
||||
# → AttributeError: __class_getitem__ (typing.py 3.13)
|
||||
# dan TIDAK ADA versi `mashumaro` yang sekaligus memenuhi batas dependensi kita
|
||||
# (`collate-data-diff <3.11`, `dbt-core <3.15`) dan berfungsi di 3.13.
|
||||
# Bukti lengkap: deploy/output/b3-hasil-uji.md
|
||||
# ➡️ Karena itu base dikembalikan ke **Python 3.11** — tumpukan yang sudah
|
||||
# terbukti di produksi. Tujuan BATCH B (Airflow 3.3.2 + OM 2.0.3 + provider
|
||||
# MySQL) tetap tercapai, karena pin OM memaksa 3.3.2 terlepas dari Python.
|
||||
#
|
||||
# Perbedaan dari Dockerfile.slim:
|
||||
# 1. base : slim-3.3.1-python3.11 -> slim-3.3.2-python3.11
|
||||
# 2. provider MySQL : tidak ada -> DIPASANG (keputusan K3), dengan mysqlclient
|
||||
# dibangun di STAGE BUILDER + libmariadb3 sebagai runtime
|
||||
# 3. openmetadata : 2.0.2.0 -> 2.0.3.0, extra `[airflow]` DILEPAS
|
||||
# 4. sqlalchemy<2.1 : PIN WAJIB (hasil A1) — lihat requirements.332.in
|
||||
# 5. paket tambahan (2026-10-06): statsd (metrik StatsD), duckdb,
|
||||
# pyiceberg[s3fs,pyarrow]
|
||||
#
|
||||
# Base: apache/airflow:slim-3.3.2-python3.11
|
||||
# Debian 12 (bookworm) · Python 3.11.16 · TANPA gcc (maka stage builder wajib)
|
||||
#
|
||||
# Fakta yang sudah diverifikasi pada base ini:
|
||||
# - Python.h ADA di /usr/python/include/python3.11/Python.h → mysqlclient BISA
|
||||
# dibangun. Karena itu `python3-dev` Debian TIDAK dipasang.
|
||||
# - libmariadb3 tersedia (1:10.11.19+maria~deb12); `libmariadb3t64` TIDAK ada.
|
||||
# - mysqlclient terbukti terbangun di base 3.3.2 (di 3.13:
|
||||
# mysqlclient-2.3.0-cp313-cp313-linux_x86_64.whl; di 3.11 ekuivalen cp311).
|
||||
# - `pip install --user` DITERIMA di base 3.11 (di base 3.13 ia DITOLAK karena
|
||||
# VIRTUAL_ENV=/home/airflow/.local). Berkas ini TIDAK memakai `--user`:
|
||||
# hasilnya sama, karena venv-nya memang di ~/.local.
|
||||
#
|
||||
# ⚠️ URUTAN PEMASANGAN YANG MENENTUKAN (jangan diubah tanpa alasan):
|
||||
# mysqlclient tidak punya wheel Linux → bila dibiarkan, pip akan mencoba
|
||||
# membangunnya dari sdist di stage akhir yang TIDAK punya compiler → gagal.
|
||||
# Karena itu: bangun wheel di stage builder, lalu pasang dari wheelhouse LEBIH
|
||||
# DULU (sebelum install utama).
|
||||
#
|
||||
# Regenerasi lock (CARA YANG DIPAKAI — menghasilkan diff minimal):
|
||||
# (1) ambil lock produksi 3.3.1 sebagai dasar constraint
|
||||
# (2) BUANG dari constraint paket yang SENGAJA dinaikkan:
|
||||
# grep -vE '^(apache-airflow|openmetadata-|collate-)' requirements.lock > c.lock
|
||||
# (tanpa ini, constraint justru memblokir kenaikan 3.3.2 / OM 2.0.3 yang kita inginkan)
|
||||
# (3) resolve:
|
||||
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" -v /path/c.lock:/c.lock:ro \
|
||||
# apache/airflow:slim-3.3.2-python3.11 \
|
||||
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.332.in --constraint /c.lock'
|
||||
# Hasil: hanya 44 baris paket berubah vs produksi (Airflow, provider, OM, collate,
|
||||
# + MySQL) — jauh lebih kecil daripada resolusi bebas.
|
||||
# ⚠️ Constraints RESMI Airflow 3.3.2 TIDAK dapat dipakai bersama OM 2.0.3.0
|
||||
# (bentrok tabulate==0.10.0 vs 0.9.0 dan kubernetes==36.0.3 vs <36).
|
||||
#
|
||||
# Build (konteks = folder ini, karena COPY requirements.332.*):
|
||||
# docker build -t sicabe/airflow:3.3.2-om203-ol \
|
||||
# -f deploy/docker/airflow/Dockerfile.332 deploy/docker/airflow
|
||||
# =============================================================================
|
||||
|
||||
# ─── STAGE 1 — bangun mysqlclient (compiler hanya ada di sini) ──────────────
|
||||
FROM apache/airflow:slim-3.3.2-python3.11 AS mysqlwheel
|
||||
|
||||
USER root
|
||||
# gcc + pkg-config + header MariaDB. SENGAJA TANPA `python3-dev` (lihat catatan atas).
|
||||
# ⚠️ `/wheelhouse` harus DIBUAT dan dimiliki `airflow`; tanpa itu build gagal:
|
||||
# PermissionError: [Errno 13] Permission denied: '/wheelhouse'
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
gcc pkg-config default-libmysqlclient-dev \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& mkdir -p /wheelhouse && chown airflow:0 /wheelhouse
|
||||
|
||||
USER airflow
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip wheel --no-cache-dir --wheel-dir /wheelhouse "mysqlclient>=2.2.5" \
|
||||
&& ls -la /wheelhouse
|
||||
|
||||
# ─── STAGE 2 — image akhir (tanpa compiler) ────────────────────────────────
|
||||
FROM apache/airflow:slim-3.3.2-python3.11
|
||||
|
||||
ARG SLING_VERSION=1.5.23
|
||||
|
||||
USER root
|
||||
# Library RUNTIME untuk ekstensi mysqlclient (libmariadb.so.3). Tanpa paket ini:
|
||||
# ImportError: libmariadb.so.3: cannot open shared object file
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libmariadb3 \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& ldconfig
|
||||
USER airflow
|
||||
|
||||
COPY --from=mysqlwheel --chown=airflow:0 /wheelhouse /wheelhouse
|
||||
COPY requirements.332.in /tmp/requirements.in
|
||||
COPY requirements.332.lock /tmp/requirements.lock
|
||||
|
||||
# ─── (1) mysqlclient dari WHEELHOUSE, lebih dulu ────────────────────────────
|
||||
# `--no-index` + `--find-links` memaksa pip memakai wheel hasil stage builder,
|
||||
# sehingga tidak pernah mencoba membangun sdist di stage tanpa compiler ini.
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --no-compile --no-index --find-links /wheelhouse \
|
||||
--constraint /tmp/requirements.lock mysqlclient
|
||||
|
||||
# ─── (2) Backend build untuk paket sdist pure-Python ────────────────────────
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --no-compile \
|
||||
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
||||
|
||||
# ─── (3) Dependensi dari lock ───────────────────────────────────────────────
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --no-build-isolation --no-compile \
|
||||
--constraint /tmp/requirements.lock \
|
||||
-r /tmp/requirements.in
|
||||
|
||||
# ─── Verifikasi inti (FATAL bila gagal) ─────────────────────────────────────
|
||||
RUN python -B -c "import airflow; print('airflow', airflow.__version__)"
|
||||
|
||||
# Kelima klien koneksi harus dapat diimpor (inti eksperimen airflow332).
|
||||
RUN python -B -c "import MySQLdb; print('MySQLdb (mysqlclient) OK ->', MySQLdb.version_info)" \
|
||||
&& python -B -c "import mysql.connector; print('mysql-connector-python OK ->', mysql.connector.__version__)" \
|
||||
&& python -B -c "import pymysql; print('pymysql OK')" \
|
||||
&& python -B -c "import psycopg2; print('psycopg2 OK ->', psycopg2.__version__)" \
|
||||
&& python -B -c "import trino; print('trino client OK ->', trino.__version__)"
|
||||
|
||||
# Bukti mysqlclient benar-benar DIMUAT (bukan sekadar terpasang): impor modul C-nya
|
||||
# sudah memuat libmariadb.so.3 — kalau library runtime tidak ada, perintah gagal.
|
||||
RUN python -B -c "import MySQLdb, MySQLdb._mysql; print('modul C mysqlclient dimuat:', MySQLdb._mysql.__file__)" \
|
||||
&& ( ldconfig -p 2>/dev/null | grep -i mariadb || echo '[WARN] libmariadb tidak terlihat di ldconfig' )
|
||||
|
||||
RUN python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')" \
|
||||
&& python -B -c "import airflow.providers.mysql; print('provider mysql OK')"
|
||||
|
||||
# ─── Provider OpenLineage (FATAL bila gagal) ────────────────────────────────
|
||||
# Ditambahkan 2026-10-02 atas permintaan pemilik proyek. Verifikasi memastikan:
|
||||
# 1. paket openlineage-python terpasang
|
||||
# 2. provider Airflow-nya dapat diimpor
|
||||
# 3. entry point provider benar-benar terdaftar
|
||||
# ⚠️ Provider ini INERT tanpa backend: hanya aktif bila transport/endpoint
|
||||
# OpenLineage dikonfigurasi (OPENLINEAGE_URL + OPENLINEAGE_API_KEY, dst).
|
||||
# Belum ada backend OpenLineage di stack ini — jadi ini mengaktifkan KEMAMPUAN.
|
||||
# Catatan pemeriksaan: di Airflow 3, NAMA entry point provider selalu
|
||||
# `provider_info` (yang membedakan adalah VALUE/module path-nya). Percobaan
|
||||
# pertama saya memeriksa `e.name` -> positif palsu. Yang fatal di sini cukup
|
||||
# impor keduanya; info entry point hanya DICETAK (tidak di-assert) agar tidak rapuh.
|
||||
RUN python -B -c "import openlineage; print('openlineage-python OK')" \
|
||||
&& python -B -c "import airflow.providers.openlineage; print('provider openlineage import OK')" \
|
||||
&& python -B -c "import importlib.metadata as m; print('versi provider:', m.version('apache-airflow-providers-openlineage')); vals=[e.value for e in m.entry_points(group='apache_airflow_provider')]; print('entry point provider:', [v for v in vals if 'lineage' in v.lower()] or '(nama/value tidak memuat kata lineage — normal di Airflow 3)')"
|
||||
|
||||
# ─── Shim eventTime OpenLineage (FATAL bila gagal) ───────────────────────────
|
||||
# Ditemukan di experiments/lineage-airflow-om: OpenMetadata 2.0.3 menolak event
|
||||
# yang `eventTime`-nya berakhiran `+00:00` (bentuk yang dihasilkan
|
||||
# datetime.isoformat() untuk UTC) dengan
|
||||
# 400 {"code":400,"message":"Invalid request format"}
|
||||
# Shim ZuluHttpTransport menormalkan `+00:00` -> `Z` sebelum event dikirim.
|
||||
# Ditempatkan di direktori khusus + PYTHONPATH agar dapat diimpor sebagai
|
||||
# `ol_time_shim.ZuluHttpTransport` dari proses task maupun subprosesnya.
|
||||
COPY --chown=airflow:0 ol_time_shim.py /opt/airflow/ol_shim/ol_time_shim.py
|
||||
# Catatan: image dasar `apache/airflow:slim-3.3.2-python3.11` TIDAK mendefinisikan
|
||||
# PYTHONPATH (diverifikasi dengan `docker run --entrypoint env`), jadi tidak ada
|
||||
# nilai lama yang tertimpa. Jangan menulis "${PYTHONPATH}" di sini: BuildKit
|
||||
# memperingatkannya sebagai variabel tak terdefinisi.
|
||||
ENV PYTHONPATH="/opt/airflow/ol_shim"
|
||||
RUN python -B -c "from ol_time_shim import ZuluHttpTransport; print('shim eventTime OpenLineage DAPAT DIMUAT:', ZuluHttpTransport)"
|
||||
|
||||
# ─── Penjaga regresi dbt (FATAL) — pelajaran dari uji B3 ────────────────────
|
||||
# `import dbt` dan `dbt --version` TIDAK CUKUP: pada percobaan Python 3.13
|
||||
# keduanya LOLOS, tetapi `dbt debug`/`dbt compile` gagal dengan
|
||||
# mashumaro -> typing_extensions.Annotated.__class_getitem__
|
||||
# -> AttributeError: __class_getitem__ (typing.py)
|
||||
# Pemeriksaan di bawah memakai PEMAKAIAN YANG SAH (`Annotated[tipe, anotasi]`),
|
||||
# yang menempuh jalur kode yang sama dengan dbt saat mem-parse model — sehingga
|
||||
# kerusakan jenis itu tertangkap saat BUILD.
|
||||
# ⚠️ Percobaan pertama penjaga ini SALAH dan menghasilkan positif palsu:
|
||||
# memanggil `Annotated.__class_getitem__((int,))` dengan SATU argumen ->
|
||||
# TypeError "Annotated[...] should be used with at least two arguments".
|
||||
# (Catatan: galat itu sekaligus membuktikan fungsinya ADA di py3.11.)
|
||||
# Catatan: `dbt debug` penuh tidak bisa dijalankan di sini karena butuh koneksi
|
||||
# Trino; pemeriksaan ini adalah proksi yang tepat dan murah.
|
||||
RUN python -B -c "import typing_extensions as te; te.Annotated[int, 'meta']" \
|
||||
&& python -B -c "import importlib.metadata as m; print('jalur Annotated OK -> dbt dapat berjalan; mashumaro', m.version('mashumaro'))"
|
||||
|
||||
# ─── Plugin OM harus benar-benar DAPAT DIMUAT (FATAL) ───────────────────────
|
||||
# Pelajaran dari experiments/airflow332: `import openmetadata_managed_apis` BERHASIL,
|
||||
# tetapi entry point gagal `load()` karena rantai
|
||||
# flask_appbuilder -> sqlalchemy_utils -> SQLAlchemy 2.1
|
||||
# (AttributeError: 'ScalarAttributeImpl')
|
||||
# → plugin MATI saat Airflow start; tombol "Test Connection" hilang; log hanya
|
||||
# berbunyi "Failed to import plugin openmetadata_managed_apis".
|
||||
# ⚠️ Verifikasi lama (hanya mengimpor MODUL) LOLOS pada kondisi rusak itu — karena
|
||||
# itu di sini entry point benar-benar dimuat, dan kegagalannya FATAL.
|
||||
# Ditulis SATU BARIS: kode Python multi-baris dalam `python -c "..."` tanpa `\`
|
||||
# akan dibaca Docker sebagai instruksi baru ("unknown instruction: import").
|
||||
RUN python -B -c "import flask_appbuilder; print('flask_appbuilder import OK')" \
|
||||
&& python -B -c "import sqlalchemy as sa; print('sqlalchemy', sa.__version__)" \
|
||||
&& python -B -c "import importlib.metadata as m; eps=[e for e in m.entry_points(group='airflow.plugins') if 'openmetadata_managed_apis' in e.name]; assert eps, 'entry point openmetadata_managed_apis tidak ada'; cls=eps[0].load(); print('plugin openmetadata_managed_apis DAPAT DIMUAT:', cls)"
|
||||
|
||||
# ─── Paket tambahan 2026-10-06 (FATAL bila gagal) ───────────────────────────
|
||||
# 1) statsd — WAJIB agar Airflow memakai StatsClient (StatsD → statsd-exporter).
|
||||
# Tanpa ini Airflow jatuh ke NoStatsLogger dan TIDAK ada metrik terkirim.
|
||||
RUN python -B -c "import statsd; print('statsd OK (StatsClient aktif)')"
|
||||
# 2) duckdb — analitik lokal di task DAG.
|
||||
RUN python -B -c "import duckdb; print('duckdb OK ->', duckdb.__version__)"
|
||||
# 3) pyiceberg[s3fs,pyarrow] — S3FileIO via s3fs + tabel Arrow.
|
||||
# Impor ketiganya SEKALIGUS: pyiceberg tanpa s3fs/pyarrow kehilangan extra-nya.
|
||||
RUN python -B -c "import pyiceberg, pyarrow, s3fs; print('pyiceberg OK ->', pyiceberg.__version__); print('pyarrow OK ->', pyarrow.__version__); print('s3fs OK ->', s3fs.__version__)"
|
||||
|
||||
# ─── Verifikasi non-fatal (melaporkan saja) ─────────────────────────────────
|
||||
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
||||
|| echo '[WARN] modul `metadata` gagal di-import' ) \
|
||||
&& ( python -B -c "import openmetadata_managed_apis; print('plugin openmetadata-managed-apis OK')" \
|
||||
|| echo '[WARN] plugin openmetadata-managed-apis gagal di-import' ) \
|
||||
&& ( python -B -c "import airflow.providers.postgres; print('provider postgres OK')" \
|
||||
|| echo '[WARN] provider postgres' ) \
|
||||
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
||||
|| echo '[WARN] CLI metadata tidak ada' ) \
|
||||
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
||||
|
||||
# ─── Sling CLI (binary Go — BUKAN paket pip `sling`) ────────────────────────
|
||||
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
||||
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
||||
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
--max-time 2400 --connect-timeout 15 --continue-at - \
|
||||
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
||||
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
||||
&& mkdir -p "${HOME}/.local/bin" \
|
||||
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
||||
&& chmod +x /tmp/sling \
|
||||
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
||||
&& "${HOME}/.local/bin/sling" --version
|
||||
|
||||
USER airflow
|
||||
@@ -0,0 +1,133 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — Custom Airflow Image (base SLIM)
|
||||
# =============================================================================
|
||||
# Base: apache/airflow:slim-3.3.1-python3.11
|
||||
# 1.15 GB · hanya 5 provider (standard, common-compat, common-io, common-sql, smtp)
|
||||
# vs apache/airflow:3.3.1-python3.11 → 3.23 GB · 30 provider
|
||||
#
|
||||
# Yang di-bake:
|
||||
# - provider tambahan: trino, amazon, http, postgres (standard/common-* dari base)
|
||||
# - dbt-core 1.11.x + dbt-trino, great_expectations
|
||||
# - openmetadata-ingestion[airflow,pii-processor]==2.0.2.0 + plugin
|
||||
# openmetadata-managed-apis==2.0.2.0, disajikan Airflow di /pluginsv2/
|
||||
# (inilah yang menghidupkan tombol Test Connection — §8.11)
|
||||
# Extra `pii-processor` (presidio + spacy) WAJIB agar DAG yang dibangkitkan
|
||||
# plugin untuk pipeline TestSuite bisa diimpor Airflow — tanpa itu:
|
||||
# No module named 'presidio_analyzer' (§8.10)
|
||||
# Konsekuensi: image +390 MB dan pandas diturunkan ke 2.3.3 (extra minta pandas<3).
|
||||
# - Sling CLI (binary Go)
|
||||
#
|
||||
# Catatan penting:
|
||||
# 1. ✅ Extra `openmetadata-ingestion[airflow]` DIPAKAI sejak OM 2.0.2. Di OM 2.0.2
|
||||
# extra itu mem-pin apache-airflow==3.3.1 (sama dengan base image ini), sedang
|
||||
# di OM 1.13.x mem-pin 3.2.1 sehingga menurunkan Airflow. Diuji di experiments/om20x.
|
||||
# 2. ⚠️ Sebagian paket hanya sdist (antlr4-python3-runtime, dsnparse, …) → wajib
|
||||
# `--no-build-isolation`. Keduanya **pure Python** sehingga TIDAK butuh gcc
|
||||
# (base slim memang tidak punya gcc).
|
||||
# 3. ✅ `dbt-core<1.12` (1.12.x menarik dbt-core-experimental-parser, sdist-only).
|
||||
# 4. ✅ `--constraint requirements.lock` → tanpa backtracking.
|
||||
# 5. ⚠️ Base slim TIDAK punya psycopg2, padahal Postgres adalah metadata DB
|
||||
# Airflow (postgresql+psycopg2://) → disediakan oleh provider `postgres`.
|
||||
# 6. `strip` tidak ada di slim → binary Sling dibiarkan apa adanya (255 MB).
|
||||
#
|
||||
# Provider MySQL — TIDAK dipasang (disengaja):
|
||||
# `apache-airflow-providers-mysql` tidak dipakai SiCABE — koneksi Airflow hanya
|
||||
# aws/trino/postgres/http, dan OpenMetadata memakai konektor + pymysql miliknya
|
||||
# sendiri (`metadata/ingestion/source/database/mysql` + `pymysql==1.2.0`).
|
||||
# Selain itu provider ini butuh `mysqlclient` yang TIDAK punya wheel Linux
|
||||
# (sdist → butuh compiler, sedangkan base slim tanpa gcc), sehingga hook-nya
|
||||
# (MySQLdb) tetap tidak berfungsi walau provider dipasang.
|
||||
# Bila kelak benar-benar dibutuhkan, bangun mysqlclient di stage builder:
|
||||
# FROM apache/airflow:slim-3.3.1-python3.11 AS mysqlwheel
|
||||
# USER root
|
||||
# RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
# gcc pkg-config default-libmysqlclient-dev python3-dev \
|
||||
# && rm -rf /var/lib/apt/lists/*
|
||||
# USER airflow
|
||||
# RUN pip wheel --wheel-dir /wheelhouse "mysqlclient>=2.2.5"
|
||||
# lalu di stage akhir: COPY --from=mysqlwheel /wheelhouse /wheelhouse +
|
||||
# pip install --no-index --find-links /wheelhouse mysqlclient,
|
||||
# serta apt-get install libmariadb3 (lib runtime).
|
||||
#
|
||||
# Regenerasi lock:
|
||||
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" apache/airflow:slim-3.3.1-python3.11 \
|
||||
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.in --output-file /w/requirements.lock'
|
||||
#
|
||||
# Build (konteks = folder ini, karena COPY requirements.*):
|
||||
# docker build -t sicabe/airflow:3.3.1-om202 -f deploy/docker/airflow/Dockerfile.slim deploy/docker/airflow
|
||||
# =============================================================================
|
||||
|
||||
FROM apache/airflow:slim-3.3.1-python3.11
|
||||
|
||||
ARG SLING_VERSION=1.5.23
|
||||
|
||||
USER airflow
|
||||
|
||||
COPY requirements.in /tmp/requirements.in
|
||||
COPY requirements.lock /tmp/requirements.lock
|
||||
|
||||
# ─── (1) Backend build untuk paket sdist pure-Python ────────────────────────
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --user --no-compile \
|
||||
"setuptools" "wheel" "hatchling" "hatch-vcs" "poetry-core" "flit-core" "setuptools-scm"
|
||||
|
||||
# ─── (2) Dependensi dari lock (termasuk provider postgres → psycopg2-binary) ─
|
||||
RUN --mount=type=cache,target=/home/airflow/.cache/pip,uid=50000 \
|
||||
pip install --user --no-build-isolation --no-compile \
|
||||
--constraint /tmp/requirements.lock \
|
||||
-r /tmp/requirements.in
|
||||
|
||||
# ─── Verifikasi inti (fatal bila gagal) ─────────────────────────────────────
|
||||
# `python -B` → jangan tulis .pyc (menghemat puluhan MB layer).
|
||||
RUN python -B -c "import airflow; print('airflow', airflow.__version__)" \
|
||||
&& python -B -c "import psycopg2; print('psycopg2', psycopg2.__version__)" \
|
||||
&& python -B -c "import dbt, great_expectations; print('dbt + great_expectations OK')"
|
||||
|
||||
# ─── Plugin OM harus benar-benar DAPAT DIMUAT (FATAL) ───────────────────────
|
||||
# Pelajaran dari experiments/airflow332 (reviews/sicabe-18, A1):
|
||||
# `import openmetadata_managed_apis` BERHASIL, tetapi entry point-nya gagal
|
||||
# `load()` karena rantai flask_appbuilder -> sqlalchemy_utils -> SQLAlchemy 2.1
|
||||
# (AttributeError: module 'sqlalchemy.orm.attributes' has no attribute
|
||||
# 'ScalarAttributeImpl'). Akibatnya plugin MATI saat Airflow start: tombol
|
||||
# "Test Connection" / pipeline service client (§8.11) hilang, dan log hanya
|
||||
# berbunyi "Failed to import plugin openmetadata_managed_apis" — tanpa
|
||||
# menyebut SQLAlchemy sama sekali.
|
||||
# ⚠️ Verifikasi lama (hanya mengimpor MODUL) LOLOS pada kondisi rusak itu.
|
||||
# Karena itu di sini entry point benar-benar dimuat, dan kegagalannya FATAL:
|
||||
# lebih baik build berhenti daripada menghasilkan image yang tombolnya mati.
|
||||
# Catatan: ditulis SATU BARIS — kode Python multi-baris dalam `python -c "..."`
|
||||
# tanpa `\` akan dibaca Docker sebagai instruksi baru ("unknown instruction").
|
||||
RUN python -B -c "import flask_appbuilder; print('flask_appbuilder import OK')" \
|
||||
&& python -B -c "import sqlalchemy as sa; print('sqlalchemy', sa.__version__)" \
|
||||
&& python -B -c "import importlib.metadata as m; eps=[e for e in m.entry_points(group='airflow.plugins') if 'openmetadata_managed_apis' in e.name]; assert eps, 'entry point openmetadata_managed_apis tidak ada'; cls=eps[0].load(); print('plugin openmetadata_managed_apis DAPAT DIMUAT:', cls)"
|
||||
|
||||
# ─── Verifikasi OM & provider (non-fatal: hanya melaporkan) ─────────────────
|
||||
RUN ( python -B -c "import metadata; print('openmetadata-ingestion OK')" \
|
||||
|| echo '[WARN] modul `metadata` gagal di-import' ) \
|
||||
&& ( python -B -c "import openmetadata_managed_apis; print('plugin openmetadata-managed-apis OK')" \
|
||||
|| echo '[WARN] plugin openmetadata-managed-apis gagal di-import' ) \
|
||||
&& python -B -c "import importlib.metadata as m; print('airflow.plugins entry points:', [e.name for e in m.entry_points(group='airflow.plugins')])" \
|
||||
&& ( python -B -c "import importlib.metadata as m; assert 'openmetadata_managed_apis' in [e.name for e in m.entry_points(group='airflow.plugins')]" \
|
||||
|| echo '[WARN] plugin TIDAK terdaftar pada entry point airflow.plugins' ) \
|
||||
&& ( python -B -c "import airflow.providers.postgres; print('provider postgres OK')" \
|
||||
|| echo '[WARN] provider postgres' ) \
|
||||
&& ( command -v metadata >/dev/null && echo 'CLI metadata OK' \
|
||||
|| echo '[WARN] CLI metadata tidak ada' ) \
|
||||
&& ( pip check || echo '[WARN] pip check melaporkan ketidaksesuaian (lihat log)' )
|
||||
|
||||
# ─── Sling CLI (binary Go — BUKAN paket pip `sling`) ────────────────────────
|
||||
# Tarball ±94 MB. Timeout tinggi + retry + resume; di-cache agar tidak mengulang.
|
||||
RUN --mount=type=cache,target=/tmp/sling-dl,uid=50000 \
|
||||
( tar -tzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" >/dev/null 2>&1 \
|
||||
|| curl -fL --retry 5 --retry-delay 5 --retry-all-errors \
|
||||
--max-time 2400 --connect-timeout 15 --continue-at - \
|
||||
"https://github.com/slingdata-io/sling-cli/releases/download/v${SLING_VERSION}/sling_linux_amd64.tar.gz" \
|
||||
-o "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" ) \
|
||||
&& mkdir -p "${HOME}/.local/bin" \
|
||||
&& tar -xzf "/tmp/sling-dl/sling-${SLING_VERSION}.tar.gz" -C /tmp/ \
|
||||
&& chmod +x /tmp/sling \
|
||||
&& mv /tmp/sling "${HOME}/.local/bin/sling" \
|
||||
&& "${HOME}/.local/bin/sling" --version
|
||||
|
||||
USER airflow
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Shim transport OpenLineage — perbaiki `eventTime` `+00:00` -> `Z`.
|
||||
|
||||
Kenapa ada:
|
||||
OpenMetadata 2.0.3 hanya menerima `eventTime` berakhiran `Z`. Provider
|
||||
OpenLineage Airflow membangun `eventTime` dengan `datetime.isoformat()`
|
||||
(lihat airflow/providers/openlineage/plugins/adapter.py) yang untuk datetime
|
||||
UTC menghasilkan akhiran `+00:00`. Akibatnya OM menjawab
|
||||
|
||||
400 {"code":400,"message":"Invalid request format"}
|
||||
|
||||
untuk SETIAP event (pesan berasal dari JsonMappingExceptionMapper —
|
||||
kegagalan Jackson memetakan JSON ke OpenLineageRunEvent).
|
||||
|
||||
Cara pakai:
|
||||
Transport kustom didukung resmi oleh provider OpenLineage lewat `type`
|
||||
berupa path kelas Python (openlineage/client/transport/factory.py ->
|
||||
import_from_string). Konfigurasikan transport seperti biasa, tetapi
|
||||
`type` diarahkan ke kelas ini, mis.:
|
||||
|
||||
{"type": "ol_time_shim.ZuluHttpTransport",
|
||||
"url": "http://openmetadata-server:8585/api/v1/openlineage/",
|
||||
"endpoint": "lineage",
|
||||
"auth": {"type": "airflow_connection_api_key",
|
||||
"conn_id": "openmetadata_lineage_api"}}
|
||||
|
||||
Referensi: experiments/lineage-airflow-om/{FASE3-HASIL.md,shim/ol_time_shim.py}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from openlineage.client.transport.http import HttpConfig, HttpTransport
|
||||
|
||||
|
||||
class ZuluHttpTransport(HttpTransport):
|
||||
"""HttpTransport yang menormalkan `eventTime` ke bentuk `Z` (UTC)."""
|
||||
|
||||
kind = "zuluhttp"
|
||||
config_class = HttpConfig
|
||||
|
||||
def emit(self, event): # type: ignore[no-untyped-def]
|
||||
et = getattr(event, "eventTime", None)
|
||||
if isinstance(et, str) and et.endswith("+00:00"):
|
||||
event.eventTime = et[:-6] + "Z"
|
||||
return super().emit(event)
|
||||
@@ -0,0 +1,99 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dependensi image Airflow 3.3.2 / Python 3.11 (BERDAMPINGAN)
|
||||
# =============================================================================
|
||||
# SUMBER (input) untuk `uv pip compile`. Jangan tambahkan pin hasil ke sini —
|
||||
# hasilnya ada di requirements.332.lock.
|
||||
# Berdampingan dengan `requirements.in` (3.3.1) yang TIDAK diubah.
|
||||
# Rencana & bukti: reviews/sicabe-18-merge-airflow332-plan.md · experiments/airflow332/
|
||||
#
|
||||
# Perubahan dari requirements.in (3.3.1):
|
||||
# 1. apache-airflow 3.3.1 -> 3.3.2
|
||||
# 2. + apache-airflow-providers-mysql (keputusan K3 — sebelumnya sengaja tidak ada)
|
||||
# 3. openmetadata-* 2.0.2.0 -> 2.0.3.0 ; extra `[airflow]` DILEPAS
|
||||
# 4. + sqlalchemy<2.1 (PIN WAJIB, hasil langkah A1)
|
||||
#
|
||||
# ⚠️ CONSTRAINT: resolusi ini di-resolve BEBAS (tanpa constraints resmi Airflow),
|
||||
# karena constraints resmi 3.3.2 TIDAK dapat dipakai bersama OpenMetadata 2.0.3.0:
|
||||
# constraints tabulate==0.10.0 vs OM tabulate==0.9.0 (keras)
|
||||
# constraints kubernetes==36.0.3 vs OM kubernetes<36
|
||||
# Jalur `--override` sudah dicoba dan TIDAK menang atas constraints.
|
||||
# Preseden: lock produksi 3.3.1 pun di-resolve bebas (ia memakai tabulate 0.9.0).
|
||||
# =============================================================================
|
||||
|
||||
apache-airflow==3.3.2
|
||||
|
||||
# Provider Airflow yang dipakai DAG / konektor
|
||||
apache-airflow-providers-trino
|
||||
apache-airflow-providers-amazon
|
||||
apache-airflow-providers-http
|
||||
# WAJIB: Postgres adalah metadata DB Airflow (postgresql+psycopg2://).
|
||||
apache-airflow-providers-postgres
|
||||
# ⚠️ BARU (keputusan K3): untuk konek ke server MySQL eksternal. Provider ini
|
||||
# menarik EMPAT klien (mysqlclient, mysql-connector-python, aiomysql, pymysql)
|
||||
# sehingga ketiga rute Extra `client` tersedia. `mysqlclient` tidak punya wheel
|
||||
# Linux → dibangun di stage builder Dockerfile.332 + libmariadb3 sebagai runtime.
|
||||
apache-airflow-providers-mysql
|
||||
|
||||
# OpenLineage — memancarkan event lineage dari task Airflow.
|
||||
# ⚠️ INERT tanpa backend: provider ini hanya aktif bila transport/endpoint
|
||||
# OpenLineage dikonfigurasi (mis. OPENLINEAGE_URL + OPENLINEAGE_API_KEY, atau
|
||||
# transport lain). Di stack ini BELUM ada backend OpenLineage, jadi penambahan
|
||||
# ini mengaktifkan KEMAMPUAN — belum menghasilkan lineage sampai dikonfigurasi.
|
||||
# Dependensinya ringan dan hanya batas bawah: attrs>=22.2 (kita punya 26.1.0),
|
||||
# providers-common-sql/common-compat (sudah ada), plus dua paket baru:
|
||||
# openlineage-python + openlineage-integration-common.
|
||||
apache-airflow-providers-openlineage
|
||||
|
||||
# ─── PIN WAJIB: melindungi plugin OpenMetadata dari kerusakan SENYAP ─────────
|
||||
# SQLAlchemy 2.1 mematahkan rantai:
|
||||
# flask_appbuilder -> sqlalchemy_utils -> sqlalchemy.orm.attributes
|
||||
# AttributeError: module 'sqlalchemy.orm.attributes' has no attribute
|
||||
# 'ScalarAttributeImpl'
|
||||
# Akibatnya plugin `openmetadata-managed-apis` GAGAL DIMUAT saat Airflow start
|
||||
# (log hanya "Failed to import plugin openmetadata_managed_apis"), sehingga tombol
|
||||
# "Test Connection" dan jalur pipeline service client (install-phases §8.11) mati.
|
||||
# ⚠️ Berkas constraints RESMI Airflow pun TIDAK mem-pin sqlalchemy — jadi hanya
|
||||
# pin eksplisit inilah yang melindungi (ditemukan di experiments/airflow332).
|
||||
sqlalchemy<2.1
|
||||
|
||||
# Transformasi & data quality
|
||||
# dbt-core<1.12: 1.12.x menarik `dbt-core-experimental-parser` yang sdist-only.
|
||||
dbt-core<1.12
|
||||
dbt-trino>=1.8,<2.0
|
||||
great_expectations>=1.0,<2.0
|
||||
|
||||
# Metadata ingestion
|
||||
# ⚠️ Extra `[airflow]` DILEPAS (keputusan pemilik proyek). Dampaknya TERUKUR di H7:
|
||||
# produksi (dengan extra) 18/18 modul · eksperimen (tanpa extra) 13/18
|
||||
# yang hilang: tornado, elasticsearch, opensearchpy,
|
||||
# providers-elasticsearch, providers-opensearch
|
||||
# TETAP ADA (yang menentukan): workflow_factory, api.routes.deploy, pygtrie,
|
||||
# attrs, werkzeug, starlette, python-multipart, otlp, presidio, spacy
|
||||
# → jalur DAG ingestion metadata TETAP UTUH; OM 2.x hanya punya 2 sink
|
||||
# (`file`, `metadata_rest`) dan keduanya impor bersih.
|
||||
# Bila kelak memakai pipeline yang meng-ingest DARI Elasticsearch/OpenSearch,
|
||||
# tambahkan paket spesifik di atas — bukan mengembalikan seluruh extra.
|
||||
# Extra `[pii-processor]` TETAP dipasang (presidio + spacy) agar DAG pipeline DQ
|
||||
# yang dibangkitkan plugin dapat diimpor Airflow. Versi WAJIB sama dengan server.
|
||||
openmetadata-ingestion[pii-processor]==2.0.3.0
|
||||
|
||||
# Plugin REST API OpenMetadata yang disajikan Airflow di prefix /pluginsv2/
|
||||
# (yang menghidupkan tombol "Test Connection"). Paket ini TIDAK mem-pin Airflow
|
||||
# (`apache-airflow>=2.2.2`) sehingga netral terhadap pilihan versi kita.
|
||||
openmetadata-managed-apis==2.0.3.0
|
||||
|
||||
# Lineage dbt (bronze→silver→gold) di OpenMetadata — tanpa ini workflow dbt gagal
|
||||
# dengan "No module named 'collate_dbt_artifacts_parser'".
|
||||
collate-dbt-artifacts-parser==0.1.4
|
||||
|
||||
# ─── Ditambahkan 2026-10-06 atas permintaan pemilik proyek ──────────────────
|
||||
# 1) statsd — WAJIB agar Airflow memakai StatsClient (StatsD → statsd-exporter).
|
||||
# Tanpa paket ini Airflow jatuh ke NoStatsLogger:
|
||||
# "Could not configure StatsClient: No module named 'statsd'"
|
||||
# sehingga TIDAK ada metrik yang terkirim (menutup caveat non-permanen M2).
|
||||
statsd
|
||||
# 2) duckdb — analitik lokal di dalam task DAG.
|
||||
duckdb
|
||||
# 3) pyiceberg[s3fs,pyarrow] — akses tabel Iceberg langsung (S3FileIO via s3fs)
|
||||
# + konversi tabel Arrow. Menarik pyarrow, s3fs, dan aiobotocore (lihat lock).
|
||||
pyiceberg[s3fs,pyarrow]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,71 @@
|
||||
# =============================================================================
|
||||
# eBdesk SiCABE — dependensi top-level image Airflow (strategi S1: satu env)
|
||||
# =============================================================================
|
||||
# File ini adalah SUMBER (input) untuk `uv pip compile`. Jangan tambahkan pin
|
||||
# hasil ke sini — hasilnya ada di requirements.lock.
|
||||
#
|
||||
# Regenerasi lock (lihat juga header deploy/docker/airflow/Dockerfile):
|
||||
# docker run --rm -v "$PWD/deploy/docker/airflow:/w" apache/airflow:3.3.1-python3.11 \
|
||||
# bash -lc 'pip -q install uv && uv pip compile /w/requirements.in --output-file /w/requirements.lock'
|
||||
#
|
||||
# ✅ Extra `[airflow]` WAJIB dipakai sejak OpenMetadata 2.0.2.
|
||||
# Di OM 1.13.x extra itu mem-pin `apache-airflow==3.2.1` sehingga menurunkan
|
||||
# Airflow; di OM 2.0.2 extra yang sama mem-pin `apache-airflow==3.3.1` — sama
|
||||
# persis dengan versi terbaru dan base image kita. Diuji di experiments/om20x.
|
||||
# =============================================================================
|
||||
|
||||
apache-airflow==3.3.1
|
||||
|
||||
# Provider Airflow yang dipakai DAG / konektor
|
||||
apache-airflow-providers-trino
|
||||
apache-airflow-providers-amazon
|
||||
apache-airflow-providers-http
|
||||
# WAJIB: Postgres adalah metadata DB Airflow (postgresql+psycopg2://) — tanpa
|
||||
# driver-nya container gagal start. Base `slim` TIDAK membawanya.
|
||||
apache-airflow-providers-postgres
|
||||
|
||||
# ─── PIN WAJIB: melindungi plugin OpenMetadata dari kerusakan SENYAP ─────────
|
||||
# SQLAlchemy 2.1 mematahkan rantai:
|
||||
# flask_appbuilder -> sqlalchemy_utils -> sqlalchemy.orm.attributes
|
||||
# AttributeError: module 'sqlalchemy.orm.attributes' has no attribute
|
||||
# 'ScalarAttributeImpl'
|
||||
# Akibatnya plugin `openmetadata-managed-apis` GAGAL DIMUAT saat Airflow start
|
||||
# (log hanya berbunyi "Failed to import plugin openmetadata_managed_apis"),
|
||||
# sehingga tombol "Test Connection" dan jalur pipeline service client
|
||||
# (docs/sicabe-install-phases-detail.md §8.11) mati — dengan pesan yang TIDAK
|
||||
# menyebut SQLAlchemy sama sekali.
|
||||
# Lock saat ini memakai 2.0.54 HANYA karena di-resolve ketika 2.1 belum ada;
|
||||
# tanpa pin ini, regenerasi lock berikutnya akan menarik 2.1.x dan mematahkannya.
|
||||
# ⚠️ Berkas constraints RESMI Airflow pun TIDAK mem-pin sqlalchemy — jadi hanya
|
||||
# pin eksplisit inilah yang melindungi.
|
||||
# Ditemukan & dibuktikan di experiments/airflow332 — lihat reviews/sicabe-18.
|
||||
sqlalchemy<2.1
|
||||
|
||||
# Transformasi & data quality
|
||||
# dbt-core<1.12: 1.12.x menarik `dbt-core-experimental-parser` yang sdist-only
|
||||
# (tanpa wheel) sehingga memaksa build dari source. 1.11.x tidak.
|
||||
dbt-core<1.12
|
||||
dbt-trino>=1.8,<2.0
|
||||
great_expectations>=1.0,<2.0
|
||||
|
||||
# Metadata ingestion
|
||||
# WAJIB sama dengan versi server OpenMetadata (deploy/ = 2.0.2).
|
||||
# Extra `[airflow]` membawa plugin-side deps + provider elasticsearch/opensearch.
|
||||
# Extra `[pii-processor]` (presidio + spacy) dibutuhkan agar DAG yang DIBANGKITKAN
|
||||
# plugin OpenMetadata untuk pipeline TestSuite bisa diimpor Airflow: DAG tsb
|
||||
# mengimpor workflow ingestion yang menyentuh presidio_analyzer. Tanpa extra ini
|
||||
# deploy pipeline gagal dengan:
|
||||
# Failed to import dag_file [.../sicabe_dq_gold_native.py]: No module named 'presidio_analyzer'
|
||||
openmetadata-ingestion[airflow,pii-processor]==2.0.2.0
|
||||
|
||||
# Plugin REST API OpenMetadata yang disajikan Airflow di prefix /pluginsv2/.
|
||||
# Inilah yang di-probe server OM (GET /pluginsv2/api/v2/openmetadata/health-auth)
|
||||
# dan yang menghidupkan tombol "Test Connection" di UI — lihat
|
||||
# docs/sicabe-install-phases-detail.md §8.11.
|
||||
openmetadata-managed-apis==2.0.2.0
|
||||
|
||||
# Lineage dbt (bronze→silver→gold) di OpenMetadata.
|
||||
# Extra `[dbt]` dari openmetadata-ingestion memerlukan paket ini untuk membaca
|
||||
# artefak dbt (manifest.json/catalog.json). Tanpa ini, workflow dbt gagal dengan
|
||||
# "No module named 'collate_dbt_artifacts_parser'" dan lineage tidak terbentuk.
|
||||
collate-dbt-artifacts-parser==0.1.4
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user