feat(appliance): initial release of MERICA standalone appliance v1.0.0
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Core package initialization."""
|
||||
@@ -0,0 +1,51 @@
|
||||
from app.core.auth.models import (
|
||||
UserSession,
|
||||
CapabilityTier,
|
||||
LoginRequest,
|
||||
SSOCallbackRequest,
|
||||
SSOTokenVerifyRequest,
|
||||
TokenResponse,
|
||||
RBACSettingsSchema
|
||||
)
|
||||
from app.core.auth.security import (
|
||||
verify_password,
|
||||
hash_password,
|
||||
create_session_token,
|
||||
decode_session_token
|
||||
)
|
||||
from app.core.auth.role_resolver import (
|
||||
resolve_user_capabilities,
|
||||
load_rbac_settings,
|
||||
save_rbac_settings
|
||||
)
|
||||
from app.core.auth.keycloak import keycloak_client
|
||||
from app.core.auth.dependencies import (
|
||||
get_current_user,
|
||||
get_optional_current_user,
|
||||
require_capabilities,
|
||||
require_admin,
|
||||
require_editor
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"UserSession",
|
||||
"CapabilityTier",
|
||||
"LoginRequest",
|
||||
"SSOCallbackRequest",
|
||||
"SSOTokenVerifyRequest",
|
||||
"TokenResponse",
|
||||
"RBACSettingsSchema",
|
||||
"verify_password",
|
||||
"hash_password",
|
||||
"create_session_token",
|
||||
"decode_session_token",
|
||||
"resolve_user_capabilities",
|
||||
"load_rbac_settings",
|
||||
"save_rbac_settings",
|
||||
"keycloak_client",
|
||||
"get_current_user",
|
||||
"get_optional_current_user",
|
||||
"require_capabilities",
|
||||
"require_admin",
|
||||
"require_editor",
|
||||
]
|
||||
@@ -0,0 +1,82 @@
|
||||
from typing import List, Optional
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
|
||||
from app.core.auth.models import UserSession
|
||||
from app.core.auth.security import decode_session_token
|
||||
|
||||
security_scheme = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
async def get_current_user(credentials: Optional[HTTPAuthorizationCredentials] = Depends(security_scheme)) -> UserSession:
|
||||
"""Extract and validate current authenticated user from session token."""
|
||||
if not credentials or not credentials.credentials:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Otentikasi diperlukan. Silakan login terlebih dahulu.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
payload = decode_session_token(credentials.credentials)
|
||||
if not payload:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Token sesi tidak valid atau telah kedaluwarsa. Silakan login kembali.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
user_data = payload.get("user")
|
||||
if not user_data:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Struktur data sesi tidak valid.",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
return UserSession(**user_data)
|
||||
|
||||
|
||||
async def get_optional_current_user(credentials: Optional[HTTPAuthorizationCredentials] = Depends(security_scheme)) -> Optional[UserSession]:
|
||||
"""Extract current user if token is present, otherwise return None."""
|
||||
if not credentials or not credentials.credentials:
|
||||
return None
|
||||
payload = decode_session_token(credentials.credentials)
|
||||
if not payload or not payload.get("user"):
|
||||
return None
|
||||
try:
|
||||
return UserSession(**payload["user"])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def require_capabilities(allowed_caps: List[str]):
|
||||
"""Enforce that the current user possesses one of the allowed capabilities."""
|
||||
async def capability_checker(current_user: UserSession = Depends(get_current_user)) -> UserSession:
|
||||
if current_user.capability not in allowed_caps and not current_user.is_admin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"Akses ditolak: Akun Anda memiliki peran '{current_user.capability}'. Tindakan ini memerlukan hak akses '{'/'.join(allowed_caps)}'."
|
||||
)
|
||||
return current_user
|
||||
return capability_checker
|
||||
|
||||
|
||||
# Shortcuts
|
||||
async def require_admin(current_user: UserSession = Depends(get_current_user)) -> UserSession:
|
||||
"""Enforce Administrator role (Full access + Settings control)."""
|
||||
if not current_user.is_admin and current_user.capability != "admin":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Akses ditolak: Menu Pengaturan dan kontrol IAM hanya dapat diakses oleh Administrator / Superadmin."
|
||||
)
|
||||
return current_user
|
||||
|
||||
|
||||
async def require_editor(current_user: UserSession = Depends(get_current_user)) -> UserSession:
|
||||
"""Enforce Editor capability (Data Engineer / Full Canvas Access)."""
|
||||
if not current_user.can_edit and not current_user.is_admin:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Akses ditolak: Akun Anda memiliki peran Read-Only (Pimpinan/Auditor). Perubahan atau deployment DAG hanya dapat dilakukan oleh Data Engineer."
|
||||
)
|
||||
return current_user
|
||||
@@ -0,0 +1,112 @@
|
||||
import json
|
||||
import logging
|
||||
import base64
|
||||
from typing import Dict, Any, Optional, Tuple, List
|
||||
import requests
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.auth.models import UserSession
|
||||
from app.core.auth.role_resolver import resolve_user_capabilities
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KeycloakClient:
|
||||
def __init__(self):
|
||||
self.base_url = settings.KEYCLOAK_INTERNAL_URL.rstrip("/")
|
||||
self.realm = settings.KEYCLOAK_REALM
|
||||
self.client_id = settings.KEYCLOAK_CLIENT_ID
|
||||
|
||||
@property
|
||||
def realm_url(self) -> str:
|
||||
return f"{self.base_url}/realms/{self.realm}"
|
||||
|
||||
@property
|
||||
def token_endpoint(self) -> str:
|
||||
return f"{self.realm_url}/protocol/openid-connect/token"
|
||||
|
||||
@property
|
||||
def userinfo_endpoint(self) -> str:
|
||||
return f"{self.realm_url}/protocol/openid-connect/userinfo"
|
||||
|
||||
@property
|
||||
def jwks_endpoint(self) -> str:
|
||||
return f"{self.realm_url}/protocol/openid-connect/certs"
|
||||
|
||||
def exchange_code(self, code: str, redirect_uri: str, code_verifier: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Exchange authorization code for tokens from Keycloak."""
|
||||
data = {
|
||||
"grant_type": "authorization_code",
|
||||
"client_id": self.client_id,
|
||||
"code": code,
|
||||
"redirect_uri": redirect_uri,
|
||||
}
|
||||
if code_verifier:
|
||||
data["code_verifier"] = code_verifier
|
||||
|
||||
resp = requests.post(self.token_endpoint, data=data, timeout=10)
|
||||
if not resp.ok:
|
||||
logger.error(f"Keycloak code exchange failed: {resp.status_code} - {resp.text}")
|
||||
raise ValueError(f"Keycloak token exchange error: {resp.text}")
|
||||
return resp.json()
|
||||
|
||||
def parse_token_claims(self, access_token: str) -> Dict[str, Any]:
|
||||
"""Parse unverified payload claims from Keycloak JWT token."""
|
||||
try:
|
||||
parts = access_token.split(".")
|
||||
if len(parts) != 3:
|
||||
raise ValueError("Invalid JWT format")
|
||||
payload_b64 = parts[1]
|
||||
# Add padding
|
||||
rem = len(payload_b64) % 4
|
||||
if rem > 0:
|
||||
payload_b64 += "=" * (4 - rem)
|
||||
payload_json = base64.urlsafe_b64decode(payload_b64).decode("utf-8")
|
||||
return json.loads(payload_json)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse Keycloak access token: {e}")
|
||||
raise ValueError(f"Invalid access token: {e}")
|
||||
|
||||
def build_user_session_from_token(self, token_data: Dict[str, Any]) -> UserSession:
|
||||
"""Construct unified UserSession from Keycloak access_token and id_token."""
|
||||
access_token = token_data.get("access_token", "")
|
||||
claims = self.parse_token_claims(access_token)
|
||||
|
||||
sub = claims.get("sub", "")
|
||||
username = claims.get("preferred_username") or claims.get("username") or sub
|
||||
name = claims.get("name") or username
|
||||
email = claims.get("email")
|
||||
|
||||
# Extract roles from realm_access or direct roles claim
|
||||
raw_roles: List[str] = []
|
||||
if isinstance(claims.get("roles"), list):
|
||||
raw_roles.extend(claims["roles"])
|
||||
realm_access = claims.get("realm_access", {})
|
||||
if isinstance(realm_access.get("roles"), list):
|
||||
raw_roles.extend(realm_access["roles"])
|
||||
|
||||
# Extract groups
|
||||
raw_groups: List[str] = claims.get("groups", [])
|
||||
if not isinstance(raw_groups, list):
|
||||
raw_groups = []
|
||||
|
||||
# Resolve internal capabilities dynamically
|
||||
cap, can_edit, can_deploy, is_admin, is_readonly = resolve_user_capabilities(raw_roles, raw_groups)
|
||||
|
||||
return UserSession(
|
||||
id=sub,
|
||||
username=username,
|
||||
name=name,
|
||||
email=email,
|
||||
auth_provider="keycloak",
|
||||
raw_roles=raw_roles,
|
||||
raw_groups=raw_groups,
|
||||
capability=cap,
|
||||
can_edit=can_edit,
|
||||
can_deploy=can_deploy,
|
||||
is_admin=is_admin,
|
||||
is_readonly=is_readonly
|
||||
)
|
||||
|
||||
|
||||
keycloak_client = KeycloakClient()
|
||||
@@ -0,0 +1,57 @@
|
||||
from typing import List, Optional, Dict, Any
|
||||
from pydantic import BaseModel, Field
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class CapabilityTier(str, Enum):
|
||||
ADMIN = "admin"
|
||||
EDITOR = "editor"
|
||||
VIEWER = "viewer"
|
||||
|
||||
|
||||
class UserSession(BaseModel):
|
||||
id: str
|
||||
username: str
|
||||
name: str
|
||||
email: Optional[str] = None
|
||||
auth_provider: str = "local" # "local" or "keycloak"
|
||||
raw_roles: List[str] = Field(default_factory=list)
|
||||
raw_groups: List[str] = Field(default_factory=list)
|
||||
capability: str = "viewer" # "admin", "editor", "viewer"
|
||||
can_edit: bool = False
|
||||
can_deploy: bool = False
|
||||
is_admin: bool = False
|
||||
is_readonly: bool = True
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class SSOCallbackRequest(BaseModel):
|
||||
code: str
|
||||
redirect_uri: str
|
||||
code_verifier: Optional[str] = None
|
||||
|
||||
|
||||
class SSOTokenVerifyRequest(BaseModel):
|
||||
access_token: str
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_in: int = 86400
|
||||
user: UserSession
|
||||
|
||||
|
||||
class RBACSettingsSchema(BaseModel):
|
||||
editor_roles: List[str] = Field(default_factory=list)
|
||||
editor_groups: List[str] = Field(default_factory=list)
|
||||
viewer_roles: List[str] = Field(default_factory=list)
|
||||
viewer_groups: List[str] = Field(default_factory=list)
|
||||
admin_roles: List[str] = Field(default_factory=list)
|
||||
admin_groups: List[str] = Field(default_factory=list)
|
||||
updated_at: Optional[str] = None
|
||||
updated_by: Optional[str] = None
|
||||
@@ -0,0 +1,88 @@
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from app.core.config import settings
|
||||
from app.core.auth.models import RBACSettingsSchema, CapabilityTier
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Cache for loaded settings
|
||||
_SETTINGS_CACHE: Dict[str, Any] = {}
|
||||
|
||||
|
||||
def get_rbac_storage_path() -> str:
|
||||
"""Resolve storage path for dynamic RBAC settings file."""
|
||||
base_dir = settings.METADATA_DIR
|
||||
if not os.path.exists(base_dir):
|
||||
base_dir = settings.CONFIGS_DIR
|
||||
return os.path.join(base_dir, "rbac_mappings.json")
|
||||
|
||||
|
||||
def load_rbac_settings() -> RBACSettingsSchema:
|
||||
"""Load dynamic RBAC settings from persistent storage, fallback to config.py defaults."""
|
||||
filepath = get_rbac_storage_path()
|
||||
|
||||
if os.path.exists(filepath):
|
||||
try:
|
||||
with open(filepath, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return RBACSettingsSchema(**data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to read custom RBAC settings from {filepath}: {e}. Using defaults.")
|
||||
|
||||
# Fallback to defaults defined in config.py
|
||||
return RBACSettingsSchema(
|
||||
editor_roles=list(settings.AUTH_EDITOR_ROLES),
|
||||
editor_groups=list(settings.AUTH_EDITOR_GROUPS),
|
||||
viewer_roles=list(settings.AUTH_VIEWER_ROLES),
|
||||
viewer_groups=list(settings.AUTH_VIEWER_GROUPS),
|
||||
admin_roles=list(settings.AUTH_ADMIN_ROLES),
|
||||
admin_groups=list(settings.AUTH_ADMIN_GROUPS),
|
||||
updated_at=datetime.now(timezone.utc).isoformat(),
|
||||
updated_by="system_default"
|
||||
)
|
||||
|
||||
|
||||
def save_rbac_settings(schema: RBACSettingsSchema, author: str = "admin") -> RBACSettingsSchema:
|
||||
"""Save updated dynamic RBAC settings to persistent storage."""
|
||||
filepath = get_rbac_storage_path()
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
|
||||
schema.updated_at = datetime.now(timezone.utc).isoformat()
|
||||
schema.updated_by = author
|
||||
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
json.dump(schema.model_dump(), f, indent=2)
|
||||
|
||||
logger.info(f"Dynamic RBAC settings successfully updated by {author} at {filepath}")
|
||||
return schema
|
||||
|
||||
|
||||
def resolve_user_capabilities(user_roles: List[str], user_groups: List[str]) -> Tuple[str, bool, bool, bool, bool]:
|
||||
"""
|
||||
Resolve raw user roles & groups into internal capabilities:
|
||||
Returns (capability, can_edit, can_deploy, is_admin, is_readonly)
|
||||
"""
|
||||
rbac = load_rbac_settings()
|
||||
|
||||
roles_set = {str(r).strip().lower() for r in user_roles if r}
|
||||
groups_set = {str(g).strip().lower() for g in user_groups if g}
|
||||
|
||||
admin_roles_set = {str(r).strip().lower() for r in rbac.admin_roles}
|
||||
admin_groups_set = {str(g).strip().lower() for g in rbac.admin_groups}
|
||||
|
||||
editor_roles_set = {str(r).strip().lower() for r in rbac.editor_roles}
|
||||
editor_groups_set = {str(g).strip().lower() for g in rbac.editor_groups}
|
||||
|
||||
# 1. Check Admin Capability (Highest Priority)
|
||||
if roles_set.intersection(admin_roles_set) or groups_set.intersection(admin_groups_set):
|
||||
return CapabilityTier.ADMIN.value, True, True, True, False
|
||||
|
||||
# 2. Check Editor Capability (Data Engineer - Full Canvas Access)
|
||||
if roles_set.intersection(editor_roles_set) or groups_set.intersection(editor_groups_set):
|
||||
return CapabilityTier.EDITOR.value, True, True, False, False
|
||||
|
||||
# 3. Default Safe Fallback: Viewer (Pimpinan / Auditor / Read-Only)
|
||||
return CapabilityTier.VIEWER.value, False, False, False, True
|
||||
@@ -0,0 +1,52 @@
|
||||
import hmac
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional, Dict, Any
|
||||
import jwt
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
|
||||
def verify_password(plain_password: str, stored_password: str) -> bool:
|
||||
"""Constant-time password comparison to prevent timing attacks."""
|
||||
if not plain_password or not stored_password:
|
||||
return False
|
||||
# Supports direct string comparison via hmac or sha256 hash
|
||||
if stored_password.startswith("sha256$"):
|
||||
_, salt, expected_hash = stored_password.split("$", 2)
|
||||
calc_hash = hashlib.sha256((salt + plain_password).encode("utf-8")).hexdigest()
|
||||
return hmac.compare_digest(calc_hash, expected_hash)
|
||||
|
||||
return hmac.compare_digest(plain_password, stored_password)
|
||||
|
||||
|
||||
def hash_password(plain_password: str, salt: str = "merica_dlh_salt") -> str:
|
||||
"""Generate salted sha256 password hash."""
|
||||
calc_hash = hashlib.sha256((salt + plain_password).encode("utf-8")).hexdigest()
|
||||
return f"sha256${salt}${calc_hash}"
|
||||
|
||||
|
||||
def create_session_token(data: Dict[str, Any], expires_delta: Optional[timedelta] = None) -> str:
|
||||
"""Create a unified session JWT token signed with SECRET_KEY."""
|
||||
to_encode = data.copy()
|
||||
now = datetime.now(timezone.utc)
|
||||
if expires_delta:
|
||||
expire = now + expires_delta
|
||||
else:
|
||||
expire = now + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
|
||||
to_encode.update({
|
||||
"exp": int(expire.timestamp()),
|
||||
"iat": int(now.timestamp())
|
||||
})
|
||||
encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
|
||||
return encoded_jwt
|
||||
|
||||
|
||||
def decode_session_token(token: str) -> Optional[Dict[str, Any]]:
|
||||
"""Decode and validate a session JWT token."""
|
||||
try:
|
||||
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM])
|
||||
return payload
|
||||
except Exception:
|
||||
return None
|
||||
@@ -0,0 +1,212 @@
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
# Directory Anchors
|
||||
CORE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
APP_DIR = os.path.dirname(CORE_DIR)
|
||||
BACKEND_DIR = os.path.dirname(APP_DIR)
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(BACKEND_DIR, ".."))
|
||||
|
||||
# Load environment variables from .env if present
|
||||
ENV_FILE_PATH = os.path.join(PROJECT_ROOT, ".env")
|
||||
if not os.path.exists(ENV_FILE_PATH):
|
||||
ENV_FILE_PATH = os.path.join(BACKEND_DIR, ".env")
|
||||
|
||||
if os.path.exists(ENV_FILE_PATH):
|
||||
with open(ENV_FILE_PATH, "r", encoding="utf-8") as ef:
|
||||
for line in ef:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
os.environ.setdefault(k.strip(), v.strip())
|
||||
|
||||
|
||||
def resolve_dir_path(env_var_name: str, relative_subpath: str) -> str:
|
||||
"""Resolve directory path with fallback across project root and backend dir."""
|
||||
env_val = os.getenv(env_var_name)
|
||||
if env_val:
|
||||
if os.path.isabs(env_val) and os.path.exists(env_val):
|
||||
return os.path.abspath(env_val)
|
||||
c1 = os.path.abspath(os.path.join(BACKEND_DIR, env_val.lstrip("./")))
|
||||
if os.path.exists(c1):
|
||||
return c1
|
||||
c2 = os.path.abspath(os.path.join(PROJECT_ROOT, env_val.lstrip("./")))
|
||||
if os.path.exists(c2):
|
||||
return c2
|
||||
|
||||
c1 = os.path.abspath(os.path.join(BACKEND_DIR, relative_subpath))
|
||||
if os.path.exists(c1):
|
||||
return c1
|
||||
c2 = os.path.abspath(os.path.join(PROJECT_ROOT, relative_subpath))
|
||||
if os.path.exists(c2):
|
||||
return c2
|
||||
return os.path.abspath(os.path.join(PROJECT_ROOT, relative_subpath))
|
||||
|
||||
|
||||
import glob
|
||||
|
||||
def resolve_configs_dir() -> str:
|
||||
"""Resolve pipeline configs directory with SSOT priority:
|
||||
1. Explicit custom AIRFLOW_CONFIGS_DIR (if valid and exists)
|
||||
2. Container mount /app/merica-pipelines/configs
|
||||
3. Host repo ../release/merica-pipelines/configs
|
||||
4. Sibling repo ../merica-pipelines/configs or ../merica-pipeline-bundles/configs
|
||||
5. Airflow 3 GitDAGBundle directories under /opt/airflow/dag_bundles/*/tracking_repo/configs
|
||||
6. Local fallback directory airflow/configs
|
||||
"""
|
||||
env_val = os.getenv("AIRFLOW_CONFIGS_DIR")
|
||||
if env_val:
|
||||
resolved = resolve_dir_path("AIRFLOW_CONFIGS_DIR", env_val)
|
||||
if os.path.exists(resolved) and (os.path.isdir(resolved) and any(f.endswith(('.yaml', '.yml')) for f in os.listdir(resolved)) or env_val.strip("./") not in ("airflow/configs", "configs")):
|
||||
return resolved
|
||||
|
||||
# Check container mount
|
||||
container_cfg = "/app/merica-pipelines/configs"
|
||||
if os.path.exists(container_cfg) and os.path.isdir(container_cfg):
|
||||
return container_cfg
|
||||
|
||||
# Check release/merica-pipelines and sibling repos
|
||||
for rel_path in [
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "release", "merica-pipelines", "configs")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "release", "merica-pipelines", "configs")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipelines", "configs")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipeline-bundles", "configs")),
|
||||
]:
|
||||
if os.path.exists(rel_path) and os.path.isdir(rel_path):
|
||||
return rel_path
|
||||
|
||||
# Auto-detect AIP-66 Git DAG Bundles in standard Airflow 3 bundle storage
|
||||
for bundle_cfg in sorted(glob.glob("/opt/airflow/dag_bundles/*/tracking_repo/configs")):
|
||||
if os.path.exists(bundle_cfg):
|
||||
return bundle_cfg
|
||||
|
||||
bundle_path = os.getenv("AIRFLOW_DAG_BUNDLE_PATH", "/opt/airflow/dag_bundles/merica-pipeline/tracking_repo/configs")
|
||||
if os.path.exists(bundle_path):
|
||||
return bundle_path
|
||||
|
||||
return resolve_dir_path("AIRFLOW_CONFIGS_DIR", os.path.join("airflow", "configs"))
|
||||
|
||||
|
||||
def resolve_metadata_dir() -> str:
|
||||
"""Resolve pipeline metadata directory with SSOT priority:
|
||||
1. Explicit custom AIRFLOW_METADATA_DIR
|
||||
2. Container mount /app/merica-pipelines/metadata
|
||||
3. Host repo ../release/merica-pipelines/metadata
|
||||
4. Sibling repo ../merica-pipelines/metadata or ../merica-pipeline-bundles/metadata
|
||||
5. Airflow 3 GitDAGBundle metadata directories (/opt/airflow/dag_bundles/*/tracking_repo/metadata)
|
||||
6. Fallback to CONFIGS_DIR
|
||||
"""
|
||||
env_val = os.getenv("AIRFLOW_METADATA_DIR")
|
||||
if env_val:
|
||||
resolved = resolve_dir_path("AIRFLOW_METADATA_DIR", env_val)
|
||||
if os.path.exists(resolved):
|
||||
return resolved
|
||||
|
||||
container_meta = "/app/merica-pipelines/metadata"
|
||||
if os.path.exists(container_meta) and os.path.isdir(container_meta):
|
||||
return container_meta
|
||||
|
||||
for rel_path in [
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "release", "merica-pipelines", "metadata")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "release", "merica-pipelines", "metadata")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipelines", "metadata")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipeline-bundles", "metadata")),
|
||||
]:
|
||||
if os.path.exists(rel_path) and os.path.isdir(rel_path):
|
||||
return rel_path
|
||||
|
||||
for bundle_meta in sorted(glob.glob("/opt/airflow/dag_bundles/*/tracking_repo/metadata")):
|
||||
if os.path.exists(bundle_meta):
|
||||
return bundle_meta
|
||||
|
||||
bundle_meta = os.path.abspath(os.path.join(os.getenv("AIRFLOW_DAG_BUNDLE_PATH", "/opt/airflow/dag_bundles/merica-pipeline/tracking_repo"), "metadata"))
|
||||
if os.path.exists(bundle_meta):
|
||||
return bundle_meta
|
||||
|
||||
def resolve_dbt_dir() -> str:
|
||||
"""Resolve dbt transformation project directory with SSOT priority:
|
||||
1. Explicit custom AIRFLOW_DBT_DIR (if valid and exists)
|
||||
2. Container mount /app/merica-pipelines/dbt
|
||||
3. Host repo ../release/merica-pipelines/dbt
|
||||
4. Sibling repo ../merica-pipelines/dbt or ../merica-pipeline-bundles/dbt
|
||||
5. Local fallback directory airflow/dbt/dlh_transformation
|
||||
"""
|
||||
env_val = os.getenv("AIRFLOW_DBT_DIR")
|
||||
if env_val:
|
||||
resolved = resolve_dir_path("AIRFLOW_DBT_DIR", env_val)
|
||||
if os.path.exists(resolved) and os.path.isdir(resolved):
|
||||
return resolved
|
||||
|
||||
container_dbt = "/app/merica-pipelines/dbt"
|
||||
if os.path.exists(container_dbt) and os.path.isdir(container_dbt):
|
||||
return container_dbt
|
||||
|
||||
for rel_path in [
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "release", "merica-pipelines", "dbt")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "release", "merica-pipelines", "dbt")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipelines", "dbt")),
|
||||
os.path.abspath(os.path.join(PROJECT_ROOT, "..", "merica-pipeline-bundles", "dbt")),
|
||||
]:
|
||||
if os.path.exists(rel_path) and os.path.isdir(rel_path):
|
||||
return rel_path
|
||||
|
||||
return resolve_dir_path("AIRFLOW_DBT_DIR", os.path.join("airflow", "dbt", "dlh_transformation"))
|
||||
|
||||
|
||||
|
||||
class Settings:
|
||||
PROJECT_NAME: str = "DLH Data Lakehouse - Dynamic Pipeline & Storage API"
|
||||
VERSION: str = "2.0.0"
|
||||
API_V1_PREFIX: str = "/api/v1"
|
||||
|
||||
# Path settings (SSOT decoupled with merica-pipeline-bundles)
|
||||
CONFIGS_DIR: str = resolve_configs_dir()
|
||||
METADATA_DIR: str = resolve_metadata_dir()
|
||||
UPLOADS_DIR: str = os.path.abspath(os.path.join(PROJECT_ROOT, "airflow", "configs", "uploads"))
|
||||
DOCS_DIR: str = resolve_dir_path("FRONTEND_DOCS_DIR", os.path.join("frontend", "src", "docs"))
|
||||
|
||||
DEFAULT_DBT_DIR: str = os.path.abspath(os.path.join(PROJECT_ROOT, "airflow", "dbt", "dlh_transformation"))
|
||||
DBT_DIR: str = resolve_dbt_dir()
|
||||
|
||||
# Database
|
||||
AIRFLOW_DATABASE_URL: str = os.getenv(
|
||||
"AIRFLOW__DATABASE__SQL_ALCHEMY_CONN",
|
||||
"postgresql+psycopg2://dlh_user:dlh_password@postgres:5432/dlh_lakehouse"
|
||||
)
|
||||
|
||||
# CORS
|
||||
CORS_ORIGINS: List[str] = ["*"]
|
||||
|
||||
# Security & JWT Auth
|
||||
SECRET_KEY: str = os.getenv("SECRET_KEY", "merica-secret-key-dlh-12345678901234567890123456789012")
|
||||
ALGORITHM: str = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = int(os.getenv("ACCESS_TOKEN_EXPIRE_MINUTES", "1440")) # 24 hours
|
||||
|
||||
# Local Admin Fallback Credentials (Zero-SSO emergency access)
|
||||
ADMIN_USERNAME: str = os.getenv("MERICA_ADMIN_USERNAME", "admin")
|
||||
ADMIN_PASSWORD: str = os.getenv("MERICA_ADMIN_PASSWORD", "admin")
|
||||
|
||||
# Keycloak OIDC Settings
|
||||
KEYCLOAK_URL: str = os.getenv("KEYCLOAK_URL", "http://localhost:14180")
|
||||
KEYCLOAK_INTERNAL_URL: str = os.getenv("KEYCLOAK_INTERNAL_URL", os.getenv("KEYCLOAK_URL", "http://localhost:14180"))
|
||||
KEYCLOAK_REALM: str = os.getenv("KEYCLOAK_REALM", "merica")
|
||||
KEYCLOAK_CLIENT_ID: str = os.getenv("KEYCLOAK_CLIENT_ID", "merica-app")
|
||||
|
||||
# Default Dynamic Role & Group Mappings (Fallbacks if not in DB/JSON)
|
||||
AUTH_ADMIN_ROLES: List[str] = [r.strip().lower() for r in os.getenv("AUTH_ADMIN_ROLES", "merica_admin,admin,administrator,dlh_admin,role_admin,superuser").split(",") if r.strip()]
|
||||
AUTH_ADMIN_GROUPS: List[str] = [g.strip().lower() for g in os.getenv("AUTH_ADMIN_GROUPS", "/platform administrators,/admins,/dlh/admins").split(",") if g.strip()]
|
||||
|
||||
AUTH_EDITOR_ROLES: List[str] = [r.strip().lower() for r in os.getenv("AUTH_EDITOR_ROLES", "data_engineer,engineer,editor,dlh_engineer,role_data_engineer,pranata_komputer,pipeline_editor,de_bpkp").split(",") if r.strip()]
|
||||
AUTH_EDITOR_GROUPS: List[str] = [g.strip().lower() for g in os.getenv("AUTH_EDITOR_GROUPS", "/data engineers,/ingestion engineers,/tim pengembang,/dlh/engineers").split(",") if g.strip()]
|
||||
|
||||
AUTH_VIEWER_ROLES: List[str] = [r.strip().lower() for r in os.getenv("AUTH_VIEWER_ROLES", "merica_viewer,viewer,auditor,pimpinan,analyst,dlh_auditor,role_viewer,pipeline_viewer,pengawas").split(",") if r.strip()]
|
||||
AUTH_VIEWER_GROUPS: List[str] = [g.strip().lower() for g in os.getenv("AUTH_VIEWER_GROUPS", "/pimpinan unit kerja,/auditor internal bpkp,/data analysts,/auditors,/dlh/viewers").split(",") if g.strip()]
|
||||
|
||||
|
||||
settings = Settings()
|
||||
for _dir_path in (settings.CONFIGS_DIR, settings.METADATA_DIR, settings.UPLOADS_DIR):
|
||||
if not _dir_path.startswith("/opt/airflow/dag_bundles"):
|
||||
try:
|
||||
os.makedirs(_dir_path, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
Reference in New Issue
Block a user