fix: More Backup Restore Fixes (#2859)

* refactor normalized search migration to use dummy default

* changed group slug migration to use raw SQL

* updated comment

* added tests with anonymized backups (currently failing)

* typo

* fixed LDAP enum in test data

* fix for adding label settings across groups

* add migration data fixes

* fix shopping list label settings test

* re-run db init instead of just running alembic migration, to include fixes

* intentionally broke SQLAlchemy GUID handling

* safely convert between GUID types in different databases

* restore original test data after testing backup restores

* added missing group name update to migration

mealie/services/backups_v2/alchemy_exporter.py

@@ -1,4 +1,5 @@
 import datetime
+import uuid
 from os import path
 from pathlib import Path
 
@@ -10,6 +11,8 @@ from sqlalchemy.orm import sessionmaker
 
 from alembic import command
 from alembic.config import Config
+from mealie.db import init_db
+from mealie.db.models._model_utils import GUID
 from mealie.services._base_service import BaseService
 
 PROJECT_DIR = Path(__file__).parent.parent.parent.parent
@@ -38,23 +41,33 @@ class AlchemyExporter(BaseService):
         self.session_maker = sessionmaker(bind=self.engine)
 
     @staticmethod
-    def convert_to_datetime(data: dict) -> dict:
+    def is_uuid(value: str) -> bool:
+        try:
+            uuid.UUID(value)
+            return True
+        except ValueError:
+            return False
+
+    def convert_types(self, data: dict) -> dict:
         """
-        walks the dictionary to convert all things that look like timestamps to datetime objects
+        walks the dictionary to restore all things that look like string representations of their complex types
         used in the context of reading a json file into a database via SQLAlchemy.
         """
         for key, value in data.items():
             if isinstance(value, dict):
-                data = AlchemyExporter.convert_to_datetime(value)
+                data = self.convert_types(value)
             elif isinstance(value, list):  # assume that this is a list of dictionaries
-                data[key] = [AlchemyExporter.convert_to_datetime(item) for item in value]
+                data[key] = [self.convert_types(item) for item in value]
             elif isinstance(value, str):
-                if key in AlchemyExporter.look_for_datetime:
-                    data[key] = AlchemyExporter.DateTimeParser(dt=value).dt
-                if key in AlchemyExporter.look_for_date:
-                    data[key] = AlchemyExporter.DateTimeParser(date=value).date
-                if key in AlchemyExporter.look_for_time:
-                    data[key] = AlchemyExporter.DateTimeParser(time=value).time
+                if self.is_uuid(value):
+                    # convert the data to the current database's native GUID type
+                    data[key] = GUID.convert_value_to_guid(value, self.engine.dialect)
+                if key in self.look_for_datetime:
+                    data[key] = self.DateTimeParser(dt=value).dt
+                if key in self.look_for_date:
+                    data[key] = self.DateTimeParser(date=value).date
+                if key in self.look_for_time:
+                    data[key] = self.DateTimeParser(time=value).time
         return data
 
     def dump_schema(self) -> dict:
@@ -105,7 +118,7 @@ class AlchemyExporter(BaseService):
         del db_dump["alembic_version"]
         """Restores all data from dictionary into the database"""
         with self.engine.begin() as connection:
-            data = AlchemyExporter.convert_to_datetime(db_dump)
+            data = self.convert_types(db_dump)
 
             self.meta.reflect(bind=self.engine)
             for table_name, rows in data.items():
@@ -139,8 +152,8 @@ SELECT SETVAL('shopping_list_item_extras_id_seq', (SELECT MAX(id) FROM shopping_
                     )
                 )
 
-        # Run all migrations up to current version
-        command.upgrade(alembic_cfg, "head")
+        # Re-init database to finish migrations
+        init_db.main()
 
     def drop_all(self) -> None:
         """Drops all data from the database"""

mealie/services/backups_v2/backup_v2.py

@@ -83,7 +83,7 @@ class BackupV2(BaseService):
             # Validation
             if not contents.validate():
                 self.logger.error(
-                    "Invalid backup file. file does not contain required elements (data directory and database.json"
+                    "Invalid backup file. file does not contain required elements (data directory and database.json)"
                 )
                 raise ValueError("Invalid backup file")