improved scraper with debug

.vscode/settings.json

@@ -9,5 +9,8 @@
   "python.testing.unittestEnabled": false,
   "python.testing.nosetestsEnabled": false,
   "python.testing.pytestEnabled": false,
-  "python.testing.promptToConfigure": false
+  "python.testing.promptToConfigure": false,
+  "cSpell.enableFiletypes": [
+    "!python"
+  ]
 }

dev/todos.md

@@ -0,0 +1,7 @@
+Frontend
+- [ ] Catch 400 / bad response on create from URL
+
+Backend
+- [x] Add Debug folder for writing the last pulled recipe data to. 
+- [ ] Normalize Recipe data on scrape
+- [ ] Support how to Sections

mealie/data/debug/last_recipe.json

@@ -0,0 +1,74 @@
+{
+    "@context": "http://schema.org/",
+    "@type": "Recipe",
+    "name": "Moscow Mule Recipe",
+    "author": {
+        "@type": "Person",
+        "name": "John Kanell"
+    },
+    "description": "How to make a classic Moscow Mule using 3 simple ingredients. Don't forget to check out my step by step photos, tips and other cocktail varations above!",
+    "datePublished": "2020-12-28 03:59:30+00:00",
+    "image": "moscow-mule-recipe.jpg",
+    "recipeYield": [
+        "1",
+        "1 serving"
+    ],
+    "prepTime": "0:05:00",
+    "totalTime": "0:05:00",
+    "recipeIngredient": [
+        "2 oz vodka",
+        "1 oz fresh lime juice",
+        "4 oz ginger beer",
+        "Garnish with lime wedges"
+    ],
+    "recipeInstructions": [
+        {
+            "@type": "HowToStep",
+            "text": "Juice the lime and cut extra for garnish.",
+            "name": "Juice the lime and cut extra for garnish.",
+            "url": "https://preppykitchen.com/moscow-mule/#wprm-recipe-37963-step-0-0"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Fill a Moscow mule mug or glass with ice then add the lime juice, vodka and ginger beer.",
+            "name": "Fill a Moscow mule mug or glass with ice then add the lime juice, vodka and ginger beer.",
+            "url": "https://preppykitchen.com/moscow-mule/#wprm-recipe-37963-step-0-1"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Stir and top with lime wedges.",
+            "name": "Stir and top with lime wedges.",
+            "url": "https://preppykitchen.com/moscow-mule/#wprm-recipe-37963-step-0-2"
+        }
+    ],
+    "recipeCategory": [
+        "Drinks"
+    ],
+    "recipeCuisine": [
+        "American"
+    ],
+    "keywords": "Classic Moscow Mule, Moscow mule recipe, Moscow Mule variations",
+    "nutrition": {
+        "@type": "NutritionInformation",
+        "calories": "177 kcal",
+        "carbohydrateContent": "12 g",
+        "proteinContent": "1 g",
+        "fatContent": "1 g",
+        "sodiumContent": "9 mg",
+        "sugarContent": "10 g",
+        "servingSize": "1 serving"
+    },
+    "@id": "https://preppykitchen.com/moscow-mule/#recipe",
+    "isPartOf": {
+        "@id": "https://preppykitchen.com/moscow-mule/#article"
+    },
+    "mainEntityOfPage": "https://preppykitchen.com/moscow-mule/#webpage",
+    "url": "https://preppykitchen.com/moscow-mule/",
+    "slug": "moscow-mule-recipe",
+    "orgURL": "https://preppykitchen.com/moscow-mule/",
+    "categories": [],
+    "tags": [],
+    "dateAdded": null,
+    "notes": [],
+    "extras": []
+}

mealie/data/debug/scratch.json

@@ -0,0 +1,107 @@
+{
+    "@context": "http://schema.org/",
+    "@type": "Recipe",
+    "name": "Dairy-Free Impossible Pumpkin Pie",
+    "author": {
+        "@type": "Person",
+        "name": "Kare for Kitchen Treaty"
+    },
+    "description": "This crustless pumpkin pie might just be the\u00a0easiest\u00a0you'll ever make. Simply blend the ingredients together, pour into your pie pan, and bake!",
+    "datePublished": "2017-11-10 16:12:06+00:00",
+    "image": "dairy-free-impossible-pumpkin-pie.jpg",
+    "recipeYield": [
+        "8"
+    ],
+    "prepTime": "0:10:00",
+    "cookTime": "0:45:00",
+    "totalTime": "0:55:00",
+    "recipeIngredient": [
+        "1  (15-ounce) can coconut milk (I recommend full-fat for a richer pie, but lite also works)",
+        "1 cup pumpkin puree",
+        "4  large eggs",
+        "1/2 cup granulated sugar",
+        "1 tablespoon pure vanilla extract",
+        "1/2 cup all-purpose flour (or your favorite cup-for-cup gluten-free flour blend*)",
+        "1 teaspoon baking powder",
+        "1 tablespoon pumpkin pie spice",
+        "1/2 teaspoon fine-grain sea salt or table salt",
+        "Coconut whipped cream (for serving**)"
+    ],
+    "recipeInstructions": [
+        {
+            "@type": "HowToStep",
+            "text": "Preheat oven to 375 degrees Fahrenheit and position rack in the middle of the oven. Spray a 9- or 10-inch pie pan with baking spray or oil the pan with coconut oil or vegan butter.",
+            "name": "Preheat oven to 375 degrees Fahrenheit and position rack in the middle of the oven. Spray a 9- or 10-inch pie pan with baking spray or oil the pan with coconut oil or vegan butter.",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-0"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Add the coconut milk, pumpkin, eggs, sugar, and vanilla to the pitcher of a blender. Blend until combined, about 20 seconds. Add the flour, baking powder, pumpkin pie spice, and salt. Blend again until well-combined, another 20\u00a0seconds.",
+            "name": "Add the coconut milk, pumpkin, eggs, sugar, and vanilla to the pitcher of a blender. Blend until combined, about 20 seconds. Add the flour, baking powder, pumpkin pie spice, and salt. Blend again until well-combined, another 20\u00a0seconds.",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-1"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Pour filling into the pie plate. The mixture will be fairly runny. Carefully transfer to the preheated oven.",
+            "name": "Pour filling into the pie plate. The mixture will be fairly runny. Carefully transfer to the preheated oven.",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-2"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Bake\u00a0until the middle just barely jiggles, 40-50 minutes. I like to check the middle by giving the pie pan a little nudge, and if it seems like it's no longer liquid, I'll pull the pie out and insert a butter knife about halfway between the center and the edge. If the knife comes out relatively clean - no runny pie filling - it's\u00a0done!",
+            "name": "Bake\u00a0until the middle just barely jiggles, 40-50 minutes. I like to check the middle by giving the pie pan a little nudge, and if it seems like it's no longer liquid, I'll pull the pie out and insert a butter knife about halfway between the center and the edge. If the knife comes out relatively clean - no runny pie filling - it's\u00a0done!",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-3"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Place on a cooling rack and let cool, about 1 hour. Transfer to refrigerator to completely\u00a0cool, at least one more hour (or up to 3 days in advance).",
+            "name": "Place on a cooling rack and let cool, about 1 hour. Transfer to refrigerator to completely\u00a0cool, at least one more hour (or up to 3 days in advance).",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-4"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "If desired, top pie with dollops of coconut whipped cream. Or simply cut slices, transfer to a plate, and top\u00a0individual servings with the whipped cream.",
+            "name": "If desired, top pie with dollops of coconut whipped cream. Or simply cut slices, transfer to a plate, and top\u00a0individual servings with the whipped cream.",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-5"
+        },
+        {
+            "@type": "HowToStep",
+            "text": "Keeps in the refrigerator for 3-4 days. I suggest covering the completely cooled pie with plastic wrap if not serving right away.",
+            "name": "Keeps in the refrigerator for 3-4 days. I suggest covering the completely cooled pie with plastic wrap if not serving right away.",
+            "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/#wprm-recipe-32856-step-0-6"
+        }
+    ],
+    "aggregateRating": {
+        "@type": "AggregateRating",
+        "ratingValue": "5",
+        "ratingCount": "4"
+    },
+    "recipeCategory": [
+        "Dessert"
+    ],
+    "recipeCuisine": [
+        "American"
+    ],
+    "keywords": "pie",
+    "nutrition": {
+        "@type": "NutritionInformation",
+        "calories": "231 kcal",
+        "sugarContent": "14 g",
+        "sodiumContent": "239 mg",
+        "fatContent": "14 g",
+        "saturatedFatContent": "11 g",
+        "carbohydrateContent": "23 g",
+        "fiberContent": "1 g",
+        "proteinContent": "5 g",
+        "cholesterolContent": "82 mg",
+        "servingSize": "1 serving"
+    },
+    "url": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/",
+    "slug": "dairy-free-impossible-pumpkin-pie",
+    "orgURL": "https://www.kitchentreaty.com/dairy-free-impossible-pumpkin-pie/",
+    "categories": [],
+    "tags": [],
+    "dateAdded": null,
+    "notes": [],
+    "extras": []
+}

mealie/routes/recipe_routes.py

@@ -42,9 +42,11 @@ async def get_recipe_url(url: dict):
     """ Takes in a URL and Attempts to scrape data and load it into the database """
 
     url = url.get("url")
+    slug = create_from_url(url)
 
     try:
-        slug = create_from_url(url)
+        pass
+        # slug = create_from_url(url)
     except:
         raise HTTPException(
             status_code=400, detail=SnackResponse.error("Unable to Parse URL")

mealie/services/scrape_services.py

@@ -1,3 +1,6 @@
+import json
+from pathlib import Path
+
 from scrape_schema_recipe import scrape_url
 from slugify import slugify
 from utils.logger import logger
@@ -5,9 +8,24 @@ from utils.logger import logger
 from services.image_services import scrape_image
 from services.recipe_services import Recipe
 
+CWD = Path(__file__).parent
+TEMP_FILE = CWD.parent.joinpath("data", "debug", "last_recipe.json")
+
+
+def normalize_data(recipe_data: dict) -> dict:
+    if type(recipe_data["recipeYield"]) == list:
+        recipe_data["recipeYield"] = recipe_data["recipeYield"][0]
+
+    return recipe_data
+
 
 def create_from_url(url: str) -> dict:
     recipe_data = process_recipe_url(url)
+
+    with open(TEMP_FILE, "w") as f:
+        f.write(json.dumps(recipe_data, indent=4, default=str))
+
+    recipe_data = normalize_data(recipe_data)
     recipe = Recipe(**recipe_data)
 
     return recipe.save_to_db()