fix: Foreign Key Violations During Backup Restore (#2986)

* added more test data

* added missing pytest id

* add fk validation to backup restore

* removed bad type imports

* actually apply the invalid fk filter and clean up types

* fix key name

* added log when removing bad rows

* removed unused import

* bumped info to warning

mealie/services/backups_v2/alchemy_exporter.py

@@ -2,10 +2,11 @@ import datetime
 import uuid
 from os import path
 from pathlib import Path
+from typing import Any
 
 from fastapi.encoders import jsonable_encoder
 from pydantic import BaseModel
-from sqlalchemy import ForeignKeyConstraint, MetaData, create_engine, insert, text
+from sqlalchemy import ForeignKey, ForeignKeyConstraint, MetaData, Table, create_engine, insert, text
 from sqlalchemy.engine import base
 from sqlalchemy.orm import sessionmaker
 
@@ -41,13 +42,27 @@ class AlchemyExporter(BaseService):
         self.session_maker = sessionmaker(bind=self.engine)
 
     @staticmethod
-    def is_uuid(value: str) -> bool:
+    def is_uuid(value: Any) -> bool:
         try:
             uuid.UUID(value)
             return True
         except ValueError:
             return False
 
+    @staticmethod
+    def is_valid_foreign_key(db_dump: dict[str, list[dict]], fk: ForeignKey, fk_value: Any) -> bool:
+        if not fk_value:
+            return True
+
+        foreign_table_name = fk.column.table.name
+        foreign_field_name = fk.column.name
+
+        for row in db_dump.get(foreign_table_name, []):
+            if row[foreign_field_name] == fk_value:
+                return True
+
+        return False
+
     def convert_types(self, data: dict) -> dict:
         """
         walks the dictionary to restore all things that look like string representations of their complex types
@@ -70,6 +85,33 @@ class AlchemyExporter(BaseService):
                     data[key] = self.DateTimeParser(time=value).time
         return data
 
+    def clean_rows(self, db_dump: dict[str, list[dict]], table: Table, rows: list[dict]) -> list[dict]:
+        """
+        Checks rows against foreign key restraints and removes any rows that would violate them
+        """
+
+        fks = table.foreign_keys
+
+        valid_rows = []
+        for row in rows:
+            is_valid_row = True
+            for fk in fks:
+                fk_value = row.get(fk.parent.name)
+                if self.is_valid_foreign_key(db_dump, fk, row.get(fk.parent.name)):
+                    continue
+
+                is_valid_row = False
+                self.logger.warning(
+                    f"Removing row from table {table.name} because of invalid foreign key {fk.parent.name}: {fk_value}"
+                )
+                self.logger.warning(f"Row: {row}")
+                break
+
+            if is_valid_row:
+                valid_rows.append(row)
+
+        return valid_rows
+
     def dump_schema(self) -> dict:
         """
         Returns the schema of the SQLAlchemy database as a python dictionary. This dictionary is wrapped by
@@ -125,6 +167,7 @@ class AlchemyExporter(BaseService):
                 if not rows:
                     continue
                 table = self.meta.tables[table_name]
+                rows = self.clean_rows(db_dump, table, rows)
 
                 connection.execute(table.delete())
                 connection.execute(insert(table), rows)

mealie/services/backups_v2/backup_v2.py

@@ -69,7 +69,7 @@ class BackupV2(BaseService):
             shutil.copytree(f, self.directories.DATA_DIR / f.name)
 
     def restore(self, backup_path: Path) -> None:
-        self.logger.info("initially backup restore")
+        self.logger.info("initializing backup restore")
 
         backup = BackupFile(backup_path)
 

tests/data/__init__.py

@@ -10,6 +10,9 @@ backup_version_44e8d670719d_1 = CWD / "backups/backup_version_44e8d670719d_1.zip
 backup_version_44e8d670719d_2 = CWD / "backups/backup_version_44e8d670719d_2.zip"
 """44e8d670719d: add extras to shopping lists, list items, and ingredient foods"""
 
+backup_version_44e8d670719d_3 = CWD / "backups/backup_version_44e8d670719d_3.zip"
+"""44e8d670719d: add extras to shopping lists, list items, and ingredient foods"""
+
 backup_version_ba1e4a6cfe99_1 = CWD / "backups/backup_version_ba1e4a6cfe99_1.zip"
 """ba1e4a6cfe99: added plural names and alias tables for foods and units"""
 

tests/unit_tests/services_tests/backup_v2_tests/test_backup_v2.py

@@ -73,12 +73,14 @@ def test_database_restore():
     [
         test_data.backup_version_44e8d670719d_1,
         test_data.backup_version_44e8d670719d_2,
+        test_data.backup_version_44e8d670719d_3,
         test_data.backup_version_ba1e4a6cfe99_1,
         test_data.backup_version_bcfdad6b7355_1,
     ],
     ids=[
         "44e8d670719d_1: add extras to shopping lists, list items, and ingredient foods",
         "44e8d670719d_2: add extras to shopping lists, list items, and ingredient foods",
+        "44e8d670719d_3: add extras to shopping lists, list items, and ingredient foods",
         "ba1e4a6cfe99_1: added plural names and alias tables for foods and units",
         "bcfdad6b7355_1: remove tool name and slug unique contraints",
     ],