153 lines
7.0 KiB
YAML
153 lines
7.0 KiB
YAML
|
# ==============================================================================
|
||
|
|
# MERICA PRODUCTION APPLIANCE — DOCKER COMPOSE CONFIGURATION
|
||
|
|
# ==============================================================================
|
||
|
|
# Ultra-lean decoupled architecture containing strictly:
|
||
|
|
# 1. MERICA (Unified Nginx + FastAPI + Vite Frontend Appliance on Port 14000)
|
||
|
|
# 2. Apache Airflow 3 (API Server, Scheduler, DAG Processor with Git DAG Bundles)
|
||
|
|
#
|
||
|
|
# Clean production deployment:
|
||
|
|
# - Zero test suites or developer tooling
|
||
|
|
# - Zero internal database bloat (PostgreSQL MetaDB is configured externally via .env)
|
||
|
|
# - All host ports mapped cleanly to 14000 series
|
||
|
|
# ==============================================================================
|
||
|
|
|
||
|
|
x-airflow-common:
|
||
|
|
&airflow-common
|
||
|
|
image: ${AIRFLOW_IMAGE:-registry.sicabe.com/airflow:3.3.2-om203-ol}
|
||
|
|
extra_hosts:
|
||
|
|
- "host.docker.internal:host-gateway"
|
||
|
|
environment:
|
||
|
|
&airflow-common-env
|
||
|
|
# [NOTE REGISTRY IMAGE BUILD]: Paket berikut disuntikkan dinamis via _PIP_ADDITIONAL_REQUIREMENTS.
|
||
|
|
# Harap sertakan RUN pip install astronomer-cosmos==1.15.1 saat update image registry.sicabe.com/airflow.
|
||
|
|
_PIP_ADDITIONAL_REQUIREMENTS: '${AIRFLOW_ADDITIONAL_REQUIREMENTS:-apache-airflow-providers-git==1.0.0 dbt-postgres==1.11.0 astronomer-cosmos==1.15.1}'
|
||
|
|
DBT_PROFILE_NAME: '${DBT_PROFILE_NAME:-dlh_postgres}'
|
||
|
|
AIRFLOW__CORE__EXECUTOR: '${AIRFLOW_EXECUTOR:-LocalExecutor}'
|
||
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: 'postgresql+psycopg2://${POSTGRES_USER:-dlh_user}:${POSTGRES_PASSWORD:-dlh_password}@${POSTGRES_HOST:-host.docker.internal}:${POSTGRES_PORT:-14432}/${POSTGRES_DB:-dlh_lakehouse}'
|
||
|
|
AIRFLOW__CORE__FERNET_KEY: '${AIRFLOW__CORE__FERNET_KEY:-FnhjRG8hCAWFi7xAcd1OPCf-t1JfZQmaiMT9Cjp0h6U=}'
|
||
|
|
AIRFLOW__CORE__SECRET_KEY: '${AIRFLOW_SECRET_KEY:-secret-key-dlh-12345678901234567890123456789012}'
|
||
|
|
AIRFLOW__API_SERVER__JWT_SECRET: '${AIRFLOW_SECRET_KEY:-secret-key-dlh-12345678901234567890123456789012}'
|
||
|
|
AIRFLOW__API_AUTH__JWT_SECRET: '${AIRFLOW_SECRET_KEY:-secret-key-dlh-12345678901234567890123456789012}'
|
||
|
|
AIRFLOW__API_AUTH__SECRET_KEY: '${AIRFLOW_SECRET_KEY:-secret-key-dlh-12345678901234567890123456789012}'
|
||
|
|
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: '${AIRFLOW_DAGS_ARE_PAUSED_AT_CREATION:-false}'
|
||
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: '${AIRFLOW_LOAD_EXAMPLES:-false}'
|
||
|
|
AIRFLOW__CORE__AUTH_MANAGER: 'airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager'
|
||
|
|
|
||
|
|
# OpenLineage & Metadata Governance
|
||
|
|
AIRFLOW__OPENLINEAGE__DISABLED: '${OPENLINEAGE_DISABLED:-False}'
|
||
|
|
AIRFLOW__OPENLINEAGE__URL: '${OPENLINEAGE_URL:-http://host.docker.internal:14585/api/v1/openlineage/lineage}'
|
||
|
|
AIRFLOW__OPENLINEAGE__NAMESPACE: '${OPENLINEAGE_NAMESPACE:-dlh-airflow-prod}'
|
||
|
|
OPENMETADATA_HOST: '${OPENMETADATA_HOST:-http://host.docker.internal:14585}'
|
||
|
|
ENABLE_OPENMETADATA_SYNC: '${ENABLE_OPENMETADATA_SYNC:-false}'
|
||
|
|
|
||
|
|
# Git DAG Bundle Configuration (AIP-66)
|
||
|
|
AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH: '${AIRFLOW_DAG_BUNDLE_STORAGE_PATH:-/opt/airflow/dag_bundles}'
|
||
|
|
AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST: >-
|
||
|
|
[
|
||
|
|
{
|
||
|
|
"name": "${GIT_PIPELINE_BUNDLE_NAME:-merica-pipeline}",
|
||
|
|
"classpath": "airflow.providers.git.bundles.git.GitDagBundle",
|
||
|
|
"kwargs": {
|
||
|
|
"repo_url": "${GIT_PIPELINE_REPO_URL:-https://repo.sicabe.com/merica-dlh/merica-pipelines.git}",
|
||
|
|
"tracking_ref": "${GIT_PIPELINE_REPO_BRANCH:-main}",
|
||
|
|
"subdir": "${GIT_PIPELINE_REPO_SUBDIR:-dags}",
|
||
|
|
"refresh_interval": ${AIRFLOW_DAG_BUNDLE_REFRESH_INTERVAL:-30}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
|
||
|
|
POSTGRES_HOST: '${POSTGRES_HOST:-host.docker.internal}'
|
||
|
|
POSTGRES_PORT: '${POSTGRES_PORT:-14432}'
|
||
|
|
POSTGRES_DB: '${POSTGRES_DB:-dlh_lakehouse}'
|
||
|
|
POSTGRES_USER: '${POSTGRES_USER:-dlh_user}'
|
||
|
|
POSTGRES_PASSWORD: '${POSTGRES_PASSWORD:-dlh_password}'
|
||
|
|
volumes:
|
||
|
|
- airflow_dag_bundles:/opt/airflow/dag_bundles
|
||
|
|
networks:
|
||
|
|
- merica-appliance-net
|
||
|
|
|
||
|
|
services:
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
# 1. MERICA Unified Appliance (Frontend UI + Backend API Gateway)
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
merica:
|
||
|
|
build: .
|
||
|
|
image: ${MERICA_IMAGE:-registry.sicabe.com/merica:1.0.0}
|
||
|
|
container_name: merica
|
||
|
|
restart: always
|
||
|
|
ports:
|
||
|
|
- "${MERICA_PORT:-14000}:80"
|
||
|
|
extra_hosts:
|
||
|
|
- "host.docker.internal:host-gateway"
|
||
|
|
environment:
|
||
|
|
- AIRFLOW_API_URL=http://airflow-webserver:8080
|
||
|
|
- AIRFLOW_WEB_URL=http://localhost:${AIRFLOW_WEBSERVER_HOST_PORT:-14080}
|
||
|
|
- OPENMETADATA_HOST=${OPENMETADATA_HOST:-http://host.docker.internal:14585}
|
||
|
|
- OPENMETADATA_JWT_TOKEN=${OPENMETADATA_JWT_TOKEN:-}
|
||
|
|
- ENABLE_OPENMETADATA_SYNC=${ENABLE_OPENMETADATA_SYNC:-false}
|
||
|
|
- GOVERNANCE_DOMAIN=${GOVERNANCE_DOMAIN:-Lingkungan Hidup & Kehutanan}
|
||
|
|
- GOVERNANCE_OWNER_TEAM=${GOVERNANCE_OWNER_TEAM:-de_team_dlh}
|
||
|
|
- GOVERNANCE_OWNER_NAME=${GOVERNANCE_OWNER_NAME:-Data Engineering DLH}
|
||
|
|
- GOVERNANCE_TIER=${GOVERNANCE_TIER:-Tier.Tier1}
|
||
|
|
- GOVERNANCE_CERTIFICATION=${GOVERNANCE_CERTIFICATION:-Certification.Gold}
|
||
|
|
- POSTGRES_HOST=${POSTGRES_HOST:-host.docker.internal}
|
||
|
|
- POSTGRES_PORT=${POSTGRES_PORT:-14432}
|
||
|
|
- POSTGRES_DB=${POSTGRES_DB:-dlh_lakehouse}
|
||
|
|
- POSTGRES_USER=${POSTGRES_USER:-dlh_user}
|
||
|
|
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-dlh_password}
|
||
|
|
volumes:
|
||
|
|
- airflow_dag_bundles:/opt/airflow/dag_bundles:ro
|
||
|
|
- studio_storage:/app/storage
|
||
|
|
- studio_data:/app/data
|
||
|
|
depends_on:
|
||
|
|
- airflow-webserver
|
||
|
|
networks:
|
||
|
|
- merica-appliance-net
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
# 2. Apache Airflow 3 Webserver / API Server
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
airflow-webserver:
|
||
|
|
<<: *airflow-common
|
||
|
|
container_name: merica-airflow-webserver
|
||
|
|
command: api-server
|
||
|
|
restart: always
|
||
|
|
ports:
|
||
|
|
- "${AIRFLOW_WEBSERVER_HOST_PORT:-14080}:8080"
|
||
|
|
healthcheck:
|
||
|
|
test: ["CMD-SHELL", "curl -f http://localhost:8080/health || exit 1"]
|
||
|
|
interval: 30s
|
||
|
|
timeout: 10s
|
||
|
|
retries: 5
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
# 3. Apache Airflow 3 Scheduler
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
airflow-scheduler:
|
||
|
|
<<: *airflow-common
|
||
|
|
container_name: merica-airflow-scheduler
|
||
|
|
command: scheduler
|
||
|
|
restart: always
|
||
|
|
depends_on:
|
||
|
|
- airflow-webserver
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
# 4. Apache Airflow 3 DAG Processor (AIP-66 Git Sync Engine)
|
||
|
|
# ----------------------------------------------------------------------------
|
||
|
|
airflow-dag-processor:
|
||
|
|
<<: *airflow-common
|
||
|
|
container_name: merica-airflow-dag-processor
|
||
|
|
command: dag-processor
|
||
|
|
restart: always
|
||
|
|
depends_on:
|
||
|
|
- airflow-webserver
|
||
|
|
|
||
|
|
volumes:
|
||
|
|
airflow_dag_bundles:
|
||
|
|
studio_storage:
|
||
|
|
studio_data:
|
||
|
|
|
||
|
|
networks:
|
||
|
|
merica-appliance-net:
|
||
|
|
driver: bridge
|