feat(backend): ✨ Minor linting, bulk URL import, and improve BG tasks (#760)

* Fixes #751

* Fixes not showing original URL

* start slice at 0 instead of 1

* remove print statements

* add linter for print statements and remove print

* hide all buttons when edit disabled

* add bulk import API

* update attribute bindings

* unify button styles

* bulk add recipe feature

* thanks linter!

* uncomment code

Co-authored-by: Hayden <hay-kot@pm.me>

mealie/db/data_initialization/init_units_foods.py

@@ -1,9 +1,11 @@
 import json
 from pathlib import Path
 
+from mealie.core.root_logger import get_logger
 from mealie.db.data_access_layer.access_model_factory import Database
 
 CWD = Path(__file__).parent
+logger = get_logger(__name__)
 
 
 def get_default_foods():
@@ -23,10 +25,10 @@ def default_recipe_unit_init(db: Database) -> None:
         try:
             db.ingredient_units.create(unit)
         except Exception as e:
-            print(e)
+            logger.error(e)
 
     for food in get_default_foods():
         try:
             db.ingredient_foods.create(food)
         except Exception as e:
-            print(e)
+            logger.error(e)

mealie/routes/admin/admin_email.py

@@ -33,7 +33,6 @@ async def check_email_config():
 
 @router.post("", response_model=EmailSuccess)
 async def send_test_email(data: EmailTest):
-    print(data)
     service = EmailService()
     status = False
     error = None

mealie/routes/recipe/image_and_assets.py

@@ -8,17 +8,14 @@ from sqlalchemy.orm.session import Session
 from mealie.db.database import get_database
 from mealie.db.db_setup import generate_session
 from mealie.routes.routers import UserAPIRouter
-from mealie.schema.recipe import CreateRecipeByURL, Recipe, RecipeAsset
+from mealie.schema.recipe import CreateRecipeByUrl, Recipe, RecipeAsset
 from mealie.services.image.image import scrape_image, write_image
 
 user_router = UserAPIRouter()
 
 
 @user_router.post("/{slug}/image")
-def scrape_image_url(
-    slug: str,
-    url: CreateRecipeByURL,
-):
+def scrape_image_url(slug: str, url: CreateRecipeByUrl):
     """ Removes an existing image and replaces it with the incoming file. """
 
     scrape_image(url.url, slug)

mealie/routes/recipe/recipe_crud_routes.py

@@ -12,10 +12,12 @@ from mealie.core.root_logger import get_logger
 from mealie.db.database import get_database
 from mealie.db.db_setup import generate_session
 from mealie.routes.routers import UserAPIRouter
-from mealie.schema.recipe import CreateRecipeByURL, Recipe, RecipeImageTypes
-from mealie.schema.recipe.recipe import CreateRecipe, RecipeSummary
+from mealie.schema.recipe import CreateRecipeByUrl, Recipe, RecipeImageTypes
+from mealie.schema.recipe.recipe import CreateRecipe, CreateRecipeByUrlBulk, RecipeSummary
+from mealie.schema.server.tasks import ServerTaskNames
 from mealie.services.recipe.recipe_service import RecipeService
 from mealie.services.scraper.scraper import create_from_url, scrape_from_url
+from mealie.services.server_tasks.background_executory import BackgroundExecutor
 
 user_router = UserAPIRouter()
 logger = get_logger()
@@ -34,15 +36,55 @@ def create_from_name(data: CreateRecipe, recipe_service: RecipeService = Depends
 
 
 @user_router.post("/create-url", status_code=201, response_model=str)
-def parse_recipe_url(url: CreateRecipeByURL, recipe_service: RecipeService = Depends(RecipeService.private)):
+def parse_recipe_url(url: CreateRecipeByUrl, recipe_service: RecipeService = Depends(RecipeService.private)):
     """ Takes in a URL and attempts to scrape data and load it into the database """
-
     recipe = create_from_url(url.url)
     return recipe_service.create_one(recipe).slug
 
 
+@user_router.post("/create-url/bulk", status_code=202)
+def parse_recipe_url_bulk(
+    bulk: CreateRecipeByUrlBulk,
+    recipe_service: RecipeService = Depends(RecipeService.private),
+    bg_service: BackgroundExecutor = Depends(BackgroundExecutor.private),
+):
+    """ Takes in a URL and attempts to scrape data and load it into the database """
+
+    def bulk_import_func(task_id: int, session: Session) -> None:
+        database = get_database(session)
+        task = database.server_tasks.get_one(task_id)
+
+        task.append_log("test task has started")
+
+        for b in bulk.imports:
+            try:
+                recipe = create_from_url(b.url)
+
+                if b.tags:
+                    recipe.tags = b.tags
+
+                if b.categories:
+                    recipe.recipe_category = b.categories
+
+                recipe_service.create_one(recipe)
+                task.append_log(f"INFO: Created recipe from url: {b.url}")
+            except Exception as e:
+                task.append_log(f"Error: Failed to create recipe from url: {b.url}")
+                task.append_log(f"Error: {e}")
+                logger.error(f"Failed to create recipe from url: {b.url}")
+                logger.error(e)
+            database.server_tasks.update(task.id, task)
+
+        task.set_finished()
+        database.server_tasks.update(task.id, task)
+
+    bg_service.dispatch(ServerTaskNames.bulk_recipe_import, bulk_import_func)
+
+    return {"details": "task has been started"}
+
+
 @user_router.post("/test-scrape-url")
-def test_parse_recipe_url(url: CreateRecipeByURL):
+def test_parse_recipe_url(url: CreateRecipeByUrl):
     # Debugger should produce the same result as the scraper sees before cleaning
     scraped_data = scrape_from_url(url.url)
     if scraped_data:
@@ -73,11 +115,8 @@ async def get_recipe_as_zip(
 ):
     """ Get a Recipe and It's Original Image as a Zip File """
     db = get_database(session)
-
     recipe: Recipe = db.recipes.get(slug)
-
     image_asset = recipe.image_dir.joinpath(RecipeImageTypes.original.value)
-
     with ZipFile(temp_path, "w") as myzip:
         myzip.writestr(f"{slug}.json", recipe.json())
 

mealie/schema/meal_plan/new_meal.py

@@ -25,7 +25,6 @@ class CreatePlanEntry(CamelModel):
     @validator("recipe_id", always=True)
     @classmethod
     def id_or_title(cls, value, values):
-        print(value, values)
         if bool(value) is False and bool(values["title"]) is False:
             raise ValueError(f"`recipe_id={value}` or `title={values['title']}` must be provided")
 

mealie/schema/recipe/recipe.py

@@ -21,17 +21,6 @@ from .recipe_step import RecipeStep
 app_dirs = get_app_dirs()
 
 
-class CreateRecipeByURL(BaseModel):
-    url: str
-
-    class Config:
-        schema_extra = {"example": {"url": "https://myfavoriterecipes.com/recipes"}}
-
-
-class CreateRecipe(CamelModel):
-    name: str
-
-
 class RecipeTag(CamelModel):
     name: str
     slug: str
@@ -44,6 +33,27 @@ class RecipeCategory(RecipeTag):
     pass
 
 
+class CreateRecipeByUrl(BaseModel):
+    url: str
+
+    class Config:
+        schema_extra = {"example": {"url": "https://myfavoriterecipes.com/recipes"}}
+
+
+class CreateRecipeBulk(BaseModel):
+    url: str
+    categories: list[RecipeCategory] = None
+    tags: list[RecipeTag] = None
+
+
+class CreateRecipeByUrlBulk(BaseModel):
+    imports: list[CreateRecipeBulk]
+
+
+class CreateRecipe(CamelModel):
+    name: str
+
+
 class RecipeSummary(CamelModel):
     id: Optional[int]
 

mealie/schema/server/tasks.py

@@ -8,6 +8,7 @@ from pydantic import Field
 class ServerTaskNames(str, enum.Enum):
     default = "Background Task"
     backup_task = "Database Backup"
+    bulk_recipe_import = "Bulk Recipe Import"
 
 
 class ServerTaskStatus(str, enum.Enum):

mealie/services/_base_http_service/crud_http_mixins.py

@@ -37,6 +37,7 @@ class CrudHttpMixins(Generic[C, R, U], ABC):
             self.item = self.dal.create(data)
         except Exception as ex:
             logger.exception(ex)
+            self.session.rollback()
 
             msg = default_msg
             if exception_msgs:

mealie/services/email/email_service.py

@@ -73,14 +73,3 @@ class EmailService(BaseService):
             button_text="Test Email",
         )
         return self.send_email(address, test_email)
-
-
-def main():
-    print("Starting...")
-    service = EmailService()
-    service.send_test_email("hay-kot@pm.me")
-    print("Finished...")
-
-
-if __name__ == "__main__":
-    main()

mealie/services/parser_services/crfpp/pre_processor.py

@@ -19,7 +19,7 @@ replace_abbreviations = {
 def replace_common_abbreviations(string: str) -> str:
 
     for k, v in replace_abbreviations.items():
-        regex = rf"(?<=\d)\s?({k}s?)"
+        regex = rf"(?<=\d)\s?({k}\bs?)"
         string = re.sub(regex, v, string)
 
     return string

mealie/services/scraper/cleaner.py

@@ -43,13 +43,9 @@ def clean_string(text: str) -> str:
     if isinstance(text, list):
         text = text[0]
 
-    print(type(text))
-
     if text == "" or text is None:
         return ""
 
-    print(text)
-
     cleaned_text = html.unescape(text)
     cleaned_text = re.sub("<[^<]+?>", "", cleaned_text)
     cleaned_text = re.sub(" +", " ", cleaned_text)
@@ -201,9 +197,10 @@ def clean_time(time_entry):
     if time_entry is None:
         return None
     elif isinstance(time_entry, timedelta):
-        pretty_print_timedelta(time_entry)
+        return pretty_print_timedelta(time_entry)
     elif isinstance(time_entry, datetime):
-        print(time_entry)
+        pass
+        # print(time_entry)
     elif isinstance(time_entry, str):
         try:
             time_delta_object = parse_duration(time_entry)