feature/additional-db (#371)

* add support for setting db_url

* fix tests

* add db_username/password env variables

* init db if super user doesn't exist

* fix tests

* fix tests

* set SQLite default DB_URL

* don't run tests on draft PRs

* add lint/black tests

* add test-all

* spell check settings

* black/flake8

* check format fail

* new badges

* rename workflow

* fix formatting

* remove white-space

* test connection arguments for pg

* format

* add new values to template

* format

* remove old script

* monkeypatch test db

* working docker-compose for postgres

* update docs

* test pg workflow

* format

* add driver

* install w/ poetry

* setup container

* change image

* set database to localhost

* update tests

* set url

* fix url path

* disable cache

* database init

* bust cache

* get by name

Co-authored-by: hay-kot <hay-kot@pm.me>

mealie/core/config.py

@@ -1,10 +1,10 @@
 import os
 import secrets
 from pathlib import Path
-from typing import Optional, Union
+from typing import Any, Optional, Union
 
 import dotenv
-from pydantic import BaseSettings, Field, validator
+from pydantic import BaseSettings, Field, PostgresDsn, validator
 
 APP_VERSION = "v0.5.0beta"
 DB_VERSION = "v0.5.0"
@@ -57,7 +57,6 @@ class AppDirectories:
         self.CHOWDOWN_DIR: Path = self.MIGRATION_DIR.joinpath("chowdown")
         self.TEMPLATE_DIR: Path = data_dir.joinpath("templates")
         self.USER_DIR: Path = data_dir.joinpath("users")
-        self.SQLITE_DIR: Path = data_dir.joinpath("db")
         self.RECIPE_DATA_DIR: Path = data_dir.joinpath("recipes")
         self.TEMP_DIR: Path = data_dir.joinpath(".temp")
 
@@ -70,7 +69,6 @@ class AppDirectories:
             self.DEBUG_DIR,
             self.MIGRATION_DIR,
             self.TEMPLATE_DIR,
-            self.SQLITE_DIR,
             self.NEXTCLOUD_DIR,
             self.CHOWDOWN_DIR,
             self.RECIPE_DATA_DIR,
@@ -84,6 +82,16 @@ class AppDirectories:
 app_dirs = AppDirectories(CWD, DATA_DIR)
 
 
+def determine_sqlite_path(path=False, suffix=DB_VERSION) -> str:
+    global app_dirs
+    db_path = app_dirs.DATA_DIR.joinpath(f"mealie_{suffix}.db")  # ! Temporary Until Alembic
+
+    if path:
+        return db_path
+
+    return "sqlite:///" + str(db_path.absolute())
+
+
 class AppSettings(BaseSettings):
     global DATA_DIR
     PRODUCTION: bool = Field(True, env="PRODUCTION")
@@ -100,21 +108,29 @@ class AppSettings(BaseSettings):
         return "/redoc" if self.API_DOCS else None
 
     SECRET: str = determine_secrets(DATA_DIR, PRODUCTION)
-    DATABASE_TYPE: str = Field("sqlite", env="DB_TYPE")
 
-    @validator("DATABASE_TYPE", pre=True)
-    def validate_db_type(cls, v: str) -> Optional[str]:
-        if v != "sqlite":
-            raise ValueError("Unable to determine database type. Acceptible options are 'sqlite'")
-        else:
-            return v
+    DB_ENGINE: Optional[str] = None  # Optional: 'sqlite', 'postgres'
+    POSTGRES_USER: str = "mealie"
+    POSTGRES_PASSWORD: str = "mealie"
+    POSTGRES_SERVER: str = "postgres"
+    POSTGRES_PORT: str = 5432
+    POSTGRES_DB: str = "mealie"
 
-    # Used to Set SQLite File Version
-    SQLITE_FILE: Optional[Union[str, Path]]
+    DB_URL: Union[str, PostgresDsn] = None  # Actual DB_URL is calculated with `assemble_db_connection`
 
-    @validator("SQLITE_FILE", pre=True)
-    def identify_sqlite_file(cls, v: str) -> Optional[str]:
-        return app_dirs.SQLITE_DIR.joinpath(f"mealie_{DB_VERSION}.sqlite")
+    @validator("DB_URL", pre=True)
+    def assemble_db_connection(cls, v: Optional[str], values: dict[str, Any]) -> Any:
+        engine = values.get("DB_ENGINE", "sqlite")
+        if engine == "postgres":
+            host = f"{values.get('POSTGRES_SERVER')}:{values.get('POSTGRES_PORT')}"
+            return PostgresDsn.build(
+                scheme="postgresql",
+                user=values.get("POSTGRES_USER"),
+                password=values.get("POSTGRES_PASSWORD"),
+                host=host,
+                path=f"/{values.get('POSTGRES_DB') or ''}",
+            )
+        return determine_sqlite_path()
 
     DEFAULT_GROUP: str = "Home"
     DEFAULT_EMAIL: str = "changeme@email.com"

mealie/db/database.py

@@ -18,6 +18,7 @@ from mealie.schema.theme import SiteTheme
 from mealie.schema.user import GroupInDB, UserInDB
 from sqlalchemy.orm.session import Session
 
+
 logger = getLogger()
 
 

mealie/db/db_setup.py

@@ -1,12 +1,8 @@
 from mealie.core.config import settings
+from mealie.db.models.db_session import sql_global_init
 from sqlalchemy.orm.session import Session
 
-from mealie.db.models.db_session import sql_global_init
-
-sql_exists = True
-
-sql_exists = settings.SQLITE_FILE.is_file()
-SessionLocal = sql_global_init(settings.SQLITE_FILE)
+SessionLocal = sql_global_init(settings.DB_URL)
 
 
 def create_session() -> Session:

mealie/db/init_db.py

@@ -2,7 +2,7 @@ from mealie.core import root_logger
 from mealie.core.config import settings
 from mealie.core.security import get_password_hash
 from mealie.db.database import db
-from mealie.db.db_setup import create_session, sql_exists
+from mealie.db.db_setup import create_session
 from mealie.schema.settings import SiteSettings
 from mealie.schema.theme import SiteTheme
 from sqlalchemy.orm import Session
@@ -51,7 +51,9 @@ def default_user_init(session: Session):
 
 
 def main():
-    if sql_exists:
+    session = create_session()
+    init_user = db.users.get(session, "1", "id")
+    if init_user:
         print("Database Exists")
     else:
         print("Database Doesn't Exists, Initializing...")

mealie/db/models/db_session.py

@@ -1,20 +1,14 @@
-from pathlib import Path
-
 import sqlalchemy as sa
 from mealie.db.models.model_base import SqlAlchemyBase
 from sqlalchemy.orm import sessionmaker
 
 
-def sql_global_init(db_file: Path, check_thread=False):
+def sql_global_init(db_url: str):
+    connect_args = {}
+    if "sqlite" in db_url:
+        connect_args["check_same_thread"] = False
 
-    SQLALCHEMY_DATABASE_URL = "sqlite:///" + str(db_file.absolute())
-    # SQLALCHEMY_DATABASE_URL = "postgresql://user:password@postgresserver/db"
-
-    engine = sa.create_engine(
-        SQLALCHEMY_DATABASE_URL,
-        echo=False,
-        connect_args={"check_same_thread": check_thread},
-    )
+    engine = sa.create_engine(db_url, echo=False, connect_args=connect_args)
 
     SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 

mealie/db/models/group.py

@@ -3,6 +3,7 @@ import sqlalchemy.orm as orm
 from mealie.db.models.model_base import BaseMixins, SqlAlchemyBase
 from mealie.db.models.recipe.category import Category, group2categories
 from sqlalchemy.orm.session import Session
+from mealie.core.config import settings
 
 
 class WebhookURLModel(SqlAlchemyBase):
@@ -57,5 +58,5 @@ class Group(SqlAlchemyBase, BaseMixins):
     def get_ref(session: Session, name: str):
         item = session.query(Group).filter(Group.name == name).one_or_none()
         if item is None:
-            item = session.query(Group).filter(Group.id == 1).one()
+            item = session.query(Group).filter(Group.name == settings.DEFAULT_GROUP).one()
         return item

mealie/db/models/mealplan.py

@@ -30,7 +30,7 @@ class MealPlanModel(SqlAlchemyBase, BaseMixins):
     startDate = sa.Column(sa.Date)
     endDate = sa.Column(sa.Date)
     meals: List[Meal] = orm.relationship(Meal, cascade="all, delete, delete-orphan")
-    group_id = sa.Column(sa.String, sa.ForeignKey("groups.id"))
+    group_id = sa.Column(sa.Integer, sa.ForeignKey("groups.id"))
     group = orm.relationship("Group", back_populates="mealplans")
 
     def __init__(self, startDate, endDate, meals, group: str, uid=None, session=None) -> None:

mealie/db/models/recipe/category.py

@@ -8,31 +8,31 @@ from sqlalchemy.orm import validates
 logger = root_logger.get_logger()
 
 site_settings2categories = sa.Table(
-    "site_settings2categoories",
+    "site_settings2categories",
     SqlAlchemyBase.metadata,
-    sa.Column("sidebar_id", sa.Integer, sa.ForeignKey("site_settings.id")),
-    sa.Column("category_id", sa.String, sa.ForeignKey("categories.id")),
+    sa.Column("site_settings.id", sa.Integer, sa.ForeignKey("site_settings.id")),
+    sa.Column("category_id", sa.Integer, sa.ForeignKey("categories.id")),
 )
 
 group2categories = sa.Table(
     "group2categories",
     SqlAlchemyBase.metadata,
     sa.Column("group_id", sa.Integer, sa.ForeignKey("groups.id")),
-    sa.Column("category_id", sa.String, sa.ForeignKey("categories.id")),
+    sa.Column("category_id", sa.Integer, sa.ForeignKey("categories.id")),
 )
 
 recipes2categories = sa.Table(
     "recipes2categories",
     SqlAlchemyBase.metadata,
     sa.Column("recipe_id", sa.Integer, sa.ForeignKey("recipes.id")),
-    sa.Column("category_id", sa.String, sa.ForeignKey("categories.id")),
+    sa.Column("category_id", sa.Integer, sa.ForeignKey("categories.id")),
 )
 
 custom_pages2categories = sa.Table(
     "custom_pages2categories",
     SqlAlchemyBase.metadata,
     sa.Column("custom_page_id", sa.Integer, sa.ForeignKey("custom_pages.id")),
-    sa.Column("category_id", sa.String, sa.ForeignKey("categories.id")),
+    sa.Column("category_id", sa.Integer, sa.ForeignKey("categories.id")),
 )
 
 

mealie/db/models/settings.py

@@ -10,11 +10,7 @@ class SiteSettings(SqlAlchemyBase, BaseMixins):
     id = sa.Column(sa.Integer, primary_key=True)
     language = sa.Column(sa.String)
     first_day_of_week = sa.Column(sa.Integer)
-    categories = orm.relationship(
-        "Category",
-        secondary=site_settings2categories,
-        single_parent=True,
-    )
+    categories = orm.relationship("Category", secondary=site_settings2categories, single_parent=True)
     show_recent = sa.Column(sa.Boolean, default=True)
     cards_per_section = sa.Column(sa.Integer)
 
@@ -44,11 +40,7 @@ class CustomPage(SqlAlchemyBase, BaseMixins):
     position = sa.Column(sa.Integer, nullable=False)
     name = sa.Column(sa.String, nullable=False)
     slug = sa.Column(sa.String, nullable=False)
-    categories = orm.relationship(
-        "Category",
-        secondary=custom_pages2categories,
-        single_parent=True,
-    )
+    categories = orm.relationship("Category", secondary=custom_pages2categories, single_parent=True)
 
     def __init__(self, session=None, name=None, slug=None, position=0, categories=[], *args, **kwargs) -> None:
         self.name = name

mealie/db/models/users.py

@@ -16,7 +16,7 @@ class User(SqlAlchemyBase, BaseMixins):
     full_name = Column(String, index=True)
     email = Column(String, unique=True, index=True)
     password = Column(String)
-    group_id = Column(String, ForeignKey("groups.id"))
+    group_id = Column(Integer, ForeignKey("groups.id"))
     group = orm.relationship("Group", back_populates="users")
     admin = Column(Boolean, default=False)
 

mealie/routes/debug_routes.py

@@ -18,8 +18,7 @@ async def get_debug_info(current_user=Depends(get_current_user)):
         demo_status=settings.IS_DEMO,
         api_port=settings.API_PORT,
         api_docs=settings.API_DOCS,
-        db_type=settings.DATABASE_TYPE,
-        sqlite_file=settings.SQLITE_FILE,
+        db_url=settings.DB_URL,
         default_group=settings.DEFAULT_GROUP,
     )
 

mealie/schema/debug.py

@@ -11,6 +11,5 @@ class AppInfo(CamelModel):
 class DebugInfo(AppInfo):
     api_port: int
     api_docs: bool
-    db_type: str
-    sqlite_file: Path
+    db_url: Path
     default_group: str