29 lines
1.2 KiB
Plaintext
29 lines
1.2 KiB
Plaintext
# ==============================================================================
|
|
# MERICA Data Lakehouse - Airflow Multi-Engine Pinned Dependencies
|
|
# Target Runtime: Apache Airflow 3.3.2 / Python 3.11 (registry.sicabe.com/airflow:3.3.2-om203-ol)
|
|
# ==============================================================================
|
|
|
|
# Fast OLAP Ingestion & Transformation Engine (Embedded / Zero-Bloat)
|
|
duckdb==1.1.3
|
|
|
|
# Apache Iceberg Lakehouse Table Management & Metadata Tracking
|
|
pyiceberg[s3fs,pyarrow]==0.8.0
|
|
|
|
# High-Performance Columnar Memory & Storage Format
|
|
pyarrow==17.0.0
|
|
|
|
# S3 File System & Filesystem Spec for MinIO/Lakehouse Object Storage
|
|
s3fs==2024.9.0
|
|
fsspec==2024.9.0
|
|
|
|
# Granular dbt Orchestration & Shift-Left Testing (Astronomer Cosmos)
|
|
# WAJIB DIBUILD KE DALAM REGISTRY IMAGE: registry.sicabe.com/airflow:3.3.2-om203-ol
|
|
astronomer-cosmos==1.15.1
|
|
dbt-postgres==1.11.0
|
|
|
|
# NOTE ON PYSPARK:
|
|
# PySpark and Java JDK are deliberately EXCLUDED from this image to prevent bloat (>4GB)
|
|
# and avoid OOM risks on Airflow Scheduler / DAG Processor.
|
|
# Distributed PySpark jobs are dispatched externally via Livy REST API (http://spark-master:8998)
|
|
# or SparkSubmitOperator pointing to an external cluster.
|