# ==============================================================================
# MERICA Data Lakehouse - Airflow Multi-Engine Pinned Dependencies
# Target Runtime: Apache Airflow 3.3.2 / Python 3.11 (registry.sicabe.com/airflow:3.3.2-om203-ol)
# ==============================================================================

# Fast OLAP Ingestion & Transformation Engine (Embedded / Zero-Bloat)
duckdb==1.1.3

# Apache Iceberg Lakehouse Table Management & Metadata Tracking
pyiceberg[s3fs,pyarrow]==0.8.0

# High-Performance Columnar Memory & Storage Format
pyarrow==17.0.0

# S3 File System & Filesystem Spec for MinIO/Lakehouse Object Storage
s3fs==2024.9.0
fsspec==2024.9.0

# Granular dbt Orchestration & Shift-Left Testing (Astronomer Cosmos)
# WAJIB DIBUILD KE DALAM REGISTRY IMAGE: registry.sicabe.com/airflow:3.3.2-om203-ol
astronomer-cosmos==1.15.1
dbt-postgres==1.11.0

# NOTE ON PYSPARK:
# PySpark and Java JDK are deliberately EXCLUDED from this image to prevent bloat (>4GB)
# and avoid OOM risks on Airflow Scheduler / DAG Processor.
# Distributed PySpark jobs are dispatched externally via Livy REST API (http://spark-master:8998)
# or SparkSubmitOperator pointing to an external cluster.