rewrite logger to support custom config files (#3104)

mealie/core/logger/config.py

@@ -0,0 +1,67 @@
+import json
+import logging
+import pathlib
+import typing
+from logging import config as logging_config
+
+__dir = pathlib.Path(__file__).parent
+__conf: dict[str, str] | None = None
+
+
+def _load_config(path: pathlib.Path, substitutions: dict[str, str] | None = None) -> dict[str, typing.Any]:
+    with open(path) as file:
+        if substitutions:
+            contents = file.read()
+            for key, value in substitutions.items():
+                # Replaces the key matches
+                #
+                # Example:
+                #   {"key": "value"}
+                #   "/path/to/${key}/file" -> "/path/to/value/file"
+                contents = contents.replace(f"${{{key}}}", value)
+
+            json_data = json.loads(contents)
+
+        else:
+            json_data = json.load(file)
+
+    return json_data
+
+
+def log_config() -> dict[str, str]:
+    if __conf is None:
+        raise ValueError("logger not configured, must call configured_logger first")
+
+    return __conf
+
+
+def configured_logger(
+    *,
+    mode: str,
+    config_override: pathlib.Path | None = None,
+    substitutions: dict[str, str] | None = None,
+) -> logging.Logger:
+    """
+    Configure the logger based on the mode and return the root logger
+
+    Args:
+        mode (str): The mode to configure the logger for (production, development, testing)
+        config_override (pathlib.Path, optional): A path to a custom logging config. Defaults to None.
+        substitutions (dict[str, str], optional): A dictionary of substitutions to apply to the logging config.
+    """
+    global __conf
+
+    if config_override:
+        __conf = _load_config(config_override, substitutions)
+    else:
+        if mode == "production":
+            __conf = _load_config(__dir / "logconf.prod.json", substitutions)
+        elif mode == "development":
+            __conf = _load_config(__dir / "logconf.dev.json", substitutions)
+        elif mode == "testing":
+            __conf = _load_config(__dir / "logconf.test.json", substitutions)
+        else:
+            raise ValueError(f"Invalid mode: {mode}")
+
+    logging_config.dictConfig(config=__conf)
+    return logging.getLogger()

mealie/core/logger/logconf.dev.json

@@ -0,0 +1,17 @@
+{
+  "version": 1,
+  "disable_existing_loggers": false,
+  "handlers": {
+    "rich": {
+      "class": "rich.logging.RichHandler"
+    }
+  },
+  "loggers": {
+    "root": {
+      "level": "DEBUG",
+      "handlers": [
+        "rich"
+      ]
+    }
+  }
+}

mealie/core/logger/logconf.prod.json

@@ -0,0 +1,74 @@
+{
+  "version": 1,
+  "disable_existing_loggers": false,
+  "formatters": {
+    "simple": {
+      "format": "%(levelname)-8s %(asctime)s - %(message)s",
+      "datefmt": "%Y-%m-%dT%H:%M:%S"
+    },
+    "detailed": {
+      "format": "[%(levelname)s|%(module)s|L%(lineno)d] %(asctime)s: %(message)s",
+      "datefmt": "%Y-%m-%dT%H:%M:%S"
+    },
+    "access": {
+      "()": "uvicorn.logging.AccessFormatter",
+      "fmt": "%(levelname)-8s %(asctime)s - [%(client_addr)s] %(status_code)s \"%(request_line)s\"",
+      "datefmt": "%Y-%m-%dT%H:%M:%S"
+    }
+  },
+  "handlers": {
+    "stderr": {
+      "class": "logging.StreamHandler",
+      "level": "WARNING",
+      "formatter": "simple",
+      "stream": "ext://sys.stderr"
+    },
+    "stdout": {
+      "class": "logging.StreamHandler",
+      "level": "${LOG_LEVEL}",
+      "formatter": "simple",
+      "stream": "ext://sys.stdout"
+    },
+    "access": {
+      "class": "logging.StreamHandler",
+      "level": "${LOG_LEVEL}",
+      "formatter": "access",
+      "stream": "ext://sys.stdout"
+    },
+    "file": {
+      "class": "logging.handlers.RotatingFileHandler",
+      "level": "DEBUG",
+      "formatter": "detailed",
+      "filename": "${DATA_DIR}/mealie.log",
+      "maxBytes": 10000,
+      "backupCount": 3
+    }
+  },
+  "loggers": {
+    "root": {
+      "level": "${LOG_LEVEL}",
+      "handlers": [
+        "stderr",
+        "file",
+        "stdout"
+      ]
+    },
+    "uvicorn.error": {
+      "handlers": [
+        "stderr",
+        "file",
+        "stdout"
+      ],
+      "level": "${LOG_LEVEL}",
+      "propagate": false
+    },
+    "uvicorn.access": {
+      "handlers": [
+        "access",
+        "file"
+      ],
+      "level": "${LOG_LEVEL}",
+      "propagate": false
+    }
+  }
+}

mealie/core/logger/logconf.test.json

@@ -0,0 +1,26 @@
+{
+  "version": 1,
+  "disable_existing_loggers": false,
+  "formatters": {
+    "detailed": {
+      "format": "[%(levelname)s|%(module)s|L%(lineno)d] %(asctime)s: %(message)s",
+      "datefmt": "%Y-%m-%dT%H:%M:%S"
+    }
+  },
+  "handlers": {
+    "stdout": {
+      "class": "logging.StreamHandler",
+      "level": "DEBUG",
+      "formatter": "detailed",
+      "stream": "ext://sys.stdout"
+    }
+  },
+  "loggers": {
+    "root": {
+      "level": "${LOG_LEVEL}",
+      "handlers": [
+        "stdout"
+      ]
+    }
+  }
+}

mealie/core/root_logger.py

@@ -1,85 +1,46 @@
 import logging
-import sys
-from dataclasses import dataclass
-from functools import lru_cache
 
-from mealie.core.config import determine_data_dir
+from .config import get_app_dirs, get_app_settings
+from .logger.config import configured_logger
 
-DATA_DIR = determine_data_dir()
-
-from .config import get_app_settings  # noqa E402
-
-LOGGER_FILE = DATA_DIR.joinpath("mealie.log")
-DATE_FORMAT = "%d-%b-%y %H:%M:%S"
-LOGGER_FORMAT = "%(levelname)s: %(asctime)s \t%(message)s"
-
-
-@dataclass
-class LoggerConfig:
-    handlers: list
-    format: str
-    date_format: str
-    logger_file: str
-    level: int = logging.INFO
-
-
-@lru_cache
-def get_logger_config():
-    settings = get_app_settings()
-
-    log_level = logging._nameToLevel[settings.LOG_LEVEL]
-
-    if not settings.PRODUCTION:
-        from rich.logging import RichHandler
-
-        return LoggerConfig(
-            handlers=[RichHandler(rich_tracebacks=True, tracebacks_show_locals=True)],
-            format=None,
-            date_format=None,
-            logger_file=None,
-            level=log_level,
-        )
-
-    output_file_handler = logging.FileHandler(LOGGER_FILE)
-    handler_format = logging.Formatter(LOGGER_FORMAT, datefmt=DATE_FORMAT)
-    output_file_handler.setFormatter(handler_format)
-
-    # Stdout
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setFormatter(handler_format)
-
-    return LoggerConfig(
-        handlers=[output_file_handler, stdout_handler],
-        format="%(levelname)s: %(asctime)s \t%(message)s",
-        date_format="%d-%b-%y %H:%M:%S",
-        logger_file=LOGGER_FILE,
-        level=log_level,
-    )
-
-
-logger_config = get_logger_config()
-
-logging.basicConfig(
-    level=logger_config.level,
-    format=logger_config.format,
-    datefmt=logger_config.date_format,
-    handlers=logger_config.handlers,
-)
-
-
-def logger_init() -> logging.Logger:
-    """Returns the Root Logging Object for Mealie"""
-    return logging.getLogger("mealie")
-
-
-root_logger = logger_init()
+__root_logger: None | logging.Logger = None
 
 
 def get_logger(module=None) -> logging.Logger:
-    """Returns a child logger for mealie"""
-    global root_logger
+    """
+    Get a logger instance for a module, in most cases module should not be
+    provided. Simply using the root logger is sufficient.
+
+    Cases where you would want to use a module specific logger might be a background
+    task or a long running process where you want to easily identify the source of
+    those messages
+    """
+    global __root_logger
+
+    if __root_logger is None:
+        app_settings = get_app_settings()
+
+        mode = "development"
+
+        if app_settings.TESTING:
+            mode = "testing"
+        elif app_settings.PRODUCTION:
+            mode = "production"
+
+        dirs = get_app_dirs()
+
+        substitutions = {
+            "DATA_DIR": dirs.DATA_DIR.as_posix(),
+            "LOG_LEVEL": app_settings.LOG_LEVEL.upper(),
+        }
+
+        __root_logger = configured_logger(
+            mode=mode,
+            config_override=app_settings.LOG_CONFIG_OVERRIDE,
+            substitutions=substitutions,
+        )
 
     if module is None:
-        return root_logger
+        return __root_logger
 
-    return root_logger.getChild(module)
+    return __root_logger.getChild(module)

mealie/core/settings/settings.py

@@ -36,13 +36,21 @@ class AppSettings(BaseSettings):
     """path to static files directory (ex. `mealie/dist`)"""
 
     IS_DEMO: bool = False
+
+    HOST_IP: str = "*"
+
+    API_HOST: str = "0.0.0.0"
     API_PORT: int = 9000
     API_DOCS: bool = True
     TOKEN_TIME: int = 48
     """time in hours"""
 
     SECRET: str
-    LOG_LEVEL: str = "INFO"
+
+    LOG_CONFIG_OVERRIDE: Path | None = None
+    """path to custom logging configuration file"""
+
+    LOG_LEVEL: str = "info"
     """corresponds to standard Python log levels"""
 
     GIT_COMMIT_HASH: str = "unknown"

mealie/main.py

@@ -0,0 +1,20 @@
+import uvicorn
+
+from mealie.app import settings
+from mealie.core.logger.config import log_config
+
+
+def main():
+    uvicorn.run(
+        "app:app",
+        host=settings.API_HOST,
+        port=settings.API_PORT,
+        log_level=settings.LOG_LEVEL.lower(),
+        log_config=log_config(),
+        workers=1,
+        forwarded_allow_ips=settings.HOST_IP,
+    )
+
+
+if __name__ == "__main__":
+    main()

mealie/routes/admin/__init__.py

@@ -5,7 +5,6 @@ from . import (
     admin_analytics,
     admin_backups,
     admin_email,
-    admin_log,
     admin_maintenance,
     admin_management_groups,
     admin_management_users,
@@ -15,7 +14,6 @@ from . import (
 router = AdminAPIRouter(prefix="/admin")
 
 router.include_router(admin_about.router, tags=["Admin: About"])
-router.include_router(admin_log.router, tags=["Admin: Log"])
 router.include_router(admin_management_users.router, tags=["Admin: Manage Users"])
 router.include_router(admin_management_groups.router, tags=["Admin: Manage Groups"])
 router.include_router(admin_email.router, tags=["Admin: Email"])

mealie/routes/admin/admin_log.py

@@ -1,44 +0,0 @@
-from fastapi import APIRouter
-
-from mealie.core.root_logger import LOGGER_FILE
-from mealie.core.security import create_file_token
-
-router = APIRouter(prefix="/logs")
-
-
-@router.get("/{num}")
-async def get_log(num: int):
-    """Doc Str"""
-    with open(LOGGER_FILE, "rb") as f:
-        log_text = tail(f, num)
-    return log_text
-
-
-@router.get("")
-async def get_log_file():
-    """Returns a token to download a file"""
-    return {"fileToken": create_file_token(LOGGER_FILE)}
-
-
-def tail(f, lines=20):
-    total_lines_wanted = lines
-
-    BLOCK_SIZE = 1024
-    f.seek(0, 2)
-    block_end_byte = f.tell()
-    lines_to_go = total_lines_wanted
-    block_number = -1
-    blocks = []
-    while lines_to_go > 0 and block_end_byte > 0:
-        if block_end_byte - BLOCK_SIZE > 0:
-            f.seek(block_number * BLOCK_SIZE, 2)
-            blocks.append(f.read(BLOCK_SIZE))
-        else:
-            f.seek(0, 0)
-            blocks.append(f.read(block_end_byte))
-        lines_found = blocks[-1].count(b"\n")
-        lines_to_go -= lines_found
-        block_end_byte -= BLOCK_SIZE
-        block_number -= 1
-    all_read_text = b"".join(reversed(blocks))
-    return b"/n".join(all_read_text.splitlines()[-total_lines_wanted:])

mealie/routes/admin/admin_maintenance.py

@@ -1,16 +1,13 @@
-import contextlib
-import os
 import shutil
 import uuid
 from pathlib import Path
 
 from fastapi import APIRouter, HTTPException
 
-from mealie.core.root_logger import LOGGER_FILE
 from mealie.pkgs.stats import fs_stats
 from mealie.routes._base import BaseAdminController, controller
 from mealie.schema.admin import MaintenanceSummary
-from mealie.schema.admin.maintenance import MaintenanceLogs, MaintenanceStorageDetails
+from mealie.schema.admin.maintenance import MaintenanceStorageDetails
 from mealie.schema.response import ErrorResponse, SuccessResponse
 
 router = APIRouter(prefix="/maintenance")
@@ -72,21 +69,13 @@ class AdminMaintenanceController(BaseAdminController):
         """
         Get the maintenance summary
         """
-        log_file_size = 0
-        with contextlib.suppress(FileNotFoundError):
-            log_file_size = os.path.getsize(LOGGER_FILE)
 
         return MaintenanceSummary(
             data_dir_size=fs_stats.pretty_size(fs_stats.get_dir_size(self.folders.DATA_DIR)),
-            log_file_size=fs_stats.pretty_size(log_file_size),
             cleanable_images=clean_images(self.folders.RECIPE_DATA_DIR, dry_run=True),
             cleanable_dirs=clean_recipe_folders(self.folders.RECIPE_DATA_DIR, dry_run=True),
         )
 
-    @router.get("/logs", response_model=MaintenanceLogs)
-    def get_logs(self, lines: int = 200):
-        return MaintenanceLogs(logs=tail_log(LOGGER_FILE, lines))
-
     @router.get("/storage", response_model=MaintenanceStorageDetails)
     def get_storage_details(self):
         return MaintenanceStorageDetails(
@@ -130,16 +119,3 @@ class AdminMaintenanceController(BaseAdminController):
             return SuccessResponse.respond(f"{cleaned_dirs} Recipe folders removed")
         except Exception as e:
             raise HTTPException(status_code=500, detail=ErrorResponse.respond("Failed to clean directories")) from e
-
-    @router.post("/clean/logs", response_model=SuccessResponse)
-    def clean_logs(self):
-        """
-        Purges the logs
-        """
-        try:
-            with contextlib.suppress(FileNotFoundError):
-                os.remove(LOGGER_FILE)
-                LOGGER_FILE.touch()
-            return SuccessResponse.respond("Logs cleaned")
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=ErrorResponse.respond("Failed to clean logs")) from e

mealie/schema/admin/maintenance.py

@@ -3,7 +3,6 @@ from mealie.schema._mealie import MealieModel
 
 class MaintenanceSummary(MealieModel):
     data_dir_size: str
-    log_file_size: str
     cleanable_images: int
     cleanable_dirs: int