test(backend): ✅ refactor testing to reduce network reliance and speed up suite

tests/integration_tests/user_recipe_tests/test_recipe_crud.py

@@ -1,9 +1,17 @@
 import json
+from pathlib import Path
+from typing import Optional, Tuple, Union
 
 import pytest
+from bs4 import BeautifulSoup
 from fastapi.testclient import TestClient
+from pytest import MonkeyPatch
+from recipe_scrapers._abstract import AbstractScraper
+from recipe_scrapers._schemaorg import SchemaOrg
 from slugify import slugify
 
+from mealie.services.scraper import scraper
+from mealie.services.scraper.scraper_strategies import RecipeScraperOpenGraph
 from tests.utils.app_routes import AppRoutes
 from tests.utils.fixture_schemas import TestUser
 from tests.utils.recipe_data import RecipeSiteTestCase, get_recipe_test_cases
@@ -11,10 +19,65 @@ from tests.utils.recipe_data import RecipeSiteTestCase, get_recipe_test_cases
 recipe_test_data = get_recipe_test_cases()
 
 
+def get_init(html_path: Path):
+    """
+    Override the init method of the abstract scraper to return a bootstrapped init function that
+    serves the html from the given path instead of calling the url.
+    """
+
+    def init_override(
+        self,
+        url,
+        proxies: Optional[str] = None,
+        timeout: Optional[Union[float, Tuple, None]] = None,
+        wild_mode: Optional[bool] = False,
+        **_,
+    ):
+        page_data = html_path.read_bytes()
+        url = "https://test.example.com/"
+
+        self.wild_mode = wild_mode
+        self.soup = BeautifulSoup(page_data, "html.parser")
+        self.url = url
+        self.schema = SchemaOrg(page_data)
+
+    return init_override
+
+
+def open_graph_override(html: str):
+    def get_html(self) -> str:
+        return html
+
+    return get_html
+
+
 @pytest.mark.parametrize("recipe_data", recipe_test_data)
 def test_create_by_url(
-    api_client: TestClient, api_routes: AppRoutes, recipe_data: RecipeSiteTestCase, unique_user: TestUser
+    api_client: TestClient,
+    api_routes: AppRoutes,
+    recipe_data: RecipeSiteTestCase,
+    unique_user: TestUser,
+    monkeypatch: MonkeyPatch,
 ):
+    # Override init function for AbstractScraper to use the test html instead of calling the url
+    monkeypatch.setattr(
+        AbstractScraper,
+        "__init__",
+        get_init(recipe_data.html_file),
+    )
+    # Override the get_html method of the RecipeScraperOpenGraph to return the test html
+    monkeypatch.setattr(
+        RecipeScraperOpenGraph,
+        "get_html",
+        open_graph_override(recipe_data.html_file.read_text()),
+    )
+    # Skip image downloader
+    monkeypatch.setattr(
+        scraper,
+        "download_image_for_recipe",
+        lambda *_: "TEST_IMAGE",
+    )
+
     api_client.delete(api_routes.recipes_recipe_slug(recipe_data.expected_slug), headers=unique_user.token)
 
     response = api_client.post(api_routes.recipes_create_url, json={"url": recipe_data.url}, headers=unique_user.token)

tests/pre_test.py

@@ -1,6 +1,10 @@
+import os
+
 from mealie.core.config import get_app_dirs, get_app_settings
 from mealie.core.settings.db_providers import SQLiteProvider
 
+os.environ["PRODUCTION"] = "True"
+
 settings = get_app_settings()
 app_dirs = get_app_dirs()
 settings.DB_PROVIDER = SQLiteProvider(data_dir=app_dirs.DATA_DIR, prefix="test_")

tests/test_config.py

@@ -14,7 +14,4 @@ TEST_RAW_RECIPES = TEST_DATA.joinpath("scraper", "recipes-raw")
 TEST_CHOWDOWN_DIR = TEST_DATA.joinpath("migrations", "chowdown")
 TEST_NEXTCLOUD_DIR = TEST_DATA.joinpath("migrations", "nextcloud")
 
-# Routes
+TEST_HTML_DIR = TEST_DATA.joinpath("html")
-
-if __name__ == "__main__":
-    pass

tests/unit_tests/test_cleaner.py

@@ -5,7 +5,7 @@ from datetime import timedelta
 import pytest
 
 from mealie.services.scraper import cleaner
-from mealie.services.scraper.scraper import open_graph
+from mealie.services.scraper.scraper_strategies import RecipeScraperOpenGraph
 from tests.test_config import TEST_RAW_HTML, TEST_RAW_RECIPES
 
 # https://github.com/django/django/blob/stable/1.3.x/django/core/validators.py#L45
@@ -100,7 +100,10 @@ def test_cleaner_instructions(instructions):
 def test_html_with_recipe_data():
     path = TEST_RAW_HTML.joinpath("healthy_pasta_bake_60759.html")
     url = "https://www.bbc.co.uk/food/recipes/healthy_pasta_bake_60759"
-    recipe_data = open_graph.basic_recipe_from_opengraph(path.read_text(), url)
+
+    open_graph_strategy = RecipeScraperOpenGraph(url)
+
+    recipe_data = open_graph_strategy.get_recipe_fields(path.read_text())
 
     assert len(recipe_data["name"]) > 10
     assert len(recipe_data["slug"]) > 10

tests/utils/recipe_data.py

@@ -1,48 +1,62 @@
 from dataclasses import dataclass
+from pathlib import Path
+
+from tests.test_config import TEST_HTML_DIR
 
 
 @dataclass
 class RecipeSiteTestCase:
     url: str
+    html: str
     expected_slug: str
     num_ingredients: int
     num_steps: int
 
+    @property
+    def html_file(self) -> Path:
+        return TEST_HTML_DIR / self.html
+
 
 def get_recipe_test_cases():
     return [
         RecipeSiteTestCase(
             url="https://www.seriouseats.com/taiwanese-three-cup-chicken-san-bei-gi-recipe",
+            html="taiwanese-three-cup-chicken-san-bei-gi-recipe.html",
             expected_slug="taiwanese-three-cup-chicken-san-bei-ji-recipe",
             num_ingredients=10,
             num_steps=3,
         ),
         RecipeSiteTestCase(
             url="https://www.rezeptwelt.de/backen-herzhaft-rezepte/schinken-kaese-waffeln-ohne-viel-schnickschnack/4j0bkiig-94d4d-106529-cfcd2-is97x2ml",
+            html="schinken-kase-waffeln-ohne-viel-schnickschnack.html",
             expected_slug="schinken-kase-waffeln-ohne-viel-schnickschnack",
             num_ingredients=7,
             num_steps=1,  # Malformed JSON Data, can't parse steps just get one string
         ),
         RecipeSiteTestCase(
             url="https://cookpad.com/us/recipes/5544853-sous-vide-smoked-beef-ribs",
+            html="sous-vide-smoked-beef-ribs.html",
             expected_slug="sous-vide-smoked-beef-ribs",
             num_ingredients=7,
             num_steps=12,
         ),
         RecipeSiteTestCase(
             url="https://www.greatbritishchefs.com/recipes/jam-roly-poly-recipe",
+            html="jam-roly-poly-with-custard.html",
             expected_slug="jam-roly-poly-with-custard",
             num_ingredients=13,
             num_steps=9,
         ),
         RecipeSiteTestCase(
             url="https://recipes.anovaculinary.com/recipe/sous-vide-shrimp",
+            html="sous-vide-shrimp.html",
             expected_slug="sous-vide-shrimp",
             num_ingredients=5,
             num_steps=0,
         ),
         RecipeSiteTestCase(
             url="https://www.bonappetit.com/recipe/detroit-style-pepperoni-pizza",
+            html="detroit-style-pepperoni-pizza.html",
             expected_slug="detroit-style-pepperoni-pizza",
             num_ingredients=8,
             num_steps=5,