feat: Migrate OpenAI implementation to use structured outputs (#6964)

mealie/services/openai/openai.py

@@ -5,17 +5,20 @@ import os
 from abc import ABC, abstractmethod
 from pathlib import Path
 from textwrap import dedent
+from typing import TypeVar
 
-from openai import NOT_GIVEN, AsyncOpenAI
+from openai import AsyncOpenAI
 from openai.types.chat import ChatCompletion
 from pydantic import BaseModel, field_validator
 
 from mealie.core import root_logger
 from mealie.core.config import get_app_settings
 from mealie.pkgs import img
+from mealie.schema.openai._base import OpenAIBase
 
 from .._base_service import BaseService
 
+T = TypeVar("T", bound=OpenAIBase)
 logger = root_logger.get_logger(__name__)
 
 
@@ -189,9 +192,9 @@ class OpenAIService(BaseService):
             )
         return "\n".join(content_parts)
 
-    async def _get_raw_response(self, prompt: str, content: list[dict], force_json_response=True) -> ChatCompletion:
+    async def _get_raw_response(self, prompt: str, content: list[dict], response_schema: type[T]) -> ChatCompletion:
         client = self.get_client()
-        return await client.chat.completions.create(
+        return await client.chat.completions.parse(
             messages=[
                 {
                     "role": "system",
@@ -203,7 +206,7 @@ class OpenAIService(BaseService):
                 },
             ],
             model=self.model,
-            response_format={"type": "json_object"} if force_json_response else NOT_GIVEN,
+            response_format=response_schema,
         )
 
     async def get_response(
@@ -211,9 +214,9 @@ class OpenAIService(BaseService):
         prompt: str,
         message: str,
         *,
+        response_schema: type[T],
         images: list[OpenAIImageBase] | None = None,
-        force_json_response=True,
-    ) -> str | None:
+    ) -> T | None:
         """Send data to OpenAI and return the response message content"""
         if images and not self.enable_image_services:
             self.logger.warning("OpenAI image services are disabled, ignoring images")
@@ -224,9 +227,11 @@ class OpenAIService(BaseService):
             for image in images or []:
                 user_messages.append(image.build_message())
 
-            response = await self._get_raw_response(prompt, user_messages, force_json_response)
+            response = await self._get_raw_response(prompt, user_messages, response_schema)
             if not response.choices:
                 return None
-            return response.choices[0].message.content
+
+            response_text = response.choices[0].message.content
+            return response_schema.parse_openai_response(response_text)
         except Exception as e:
             raise Exception(f"OpenAI Request Failed. {e.__class__.__name__}: {e}") from e

mealie/services/openai/prompts/recipes/parse-recipe-image.txt

@@ -1,11 +1,5 @@
-You are a bot that reads an image, or a set of images, and parses it into recipe JSON. You will receive an image from the user and you need to extract the recipe data and return its JSON in valid schema. The recipe schema will be included at the bottom of this message.
-
-It is imperative that you do not create any data or otherwise make up any information. Failure to adhere to this rule is illegal and will result in harsh punishment. If you are unable to extract data due to insufficient input, you may reply with a completely empty JSON object (represented by two brackets: {}).
-
-Do not under any circumstances insert data not found directly in the image. Ingredients, instructions, and notes should come directly from the image and not be generated or otherwise made up. It is illegal for you to create information not found directly in the image.
-
-Your response must be in valid JSON in the provided Recipe definition below. You must respond in this JSON schema; failure to do so is illegal. It is imperative that you follow the schema precisely to avoid punishment. You must follow the JSON schema.
+You are a bot that reads an image, or a set of images, and parses it into recipe JSON. You will receive an image from the user and you need to extract the recipe data. It is imperative that you do not create any data or otherwise make up any information.
 
 The user message that you receive will be one or more images. Assume all images provided belong to a single recipe, not multiple recipes. The recipe may consist of printed text or handwritten text. It may be rotated or not properly cropped. It is your job to figure out which part of the image is the important content and extract it.
 
-The text you receive in the provided image or images may not be in English. The user may provide a language for you to translate the recipe into. If the user doesn't ask for a translation, you should preserve the text as-is without translating or otherwise modifying it. Otherwise, you should translate all text (recipe name, ingredients, instructions, etc.) to the requested language.
+If the user requests a translation, translate all text (name, ingredients, instructions, etc.) to the requested language. Otherwise, preserve the text as-is.

mealie/services/openai/prompts/recipes/parse-recipe-ingredients.txt

@@ -1,15 +1,11 @@
-You are a bot that parses user input into recipe ingredients. You will receive a list of one or more ingredients, each containing one or more of the following components: quantity, unit, food, and note. Their definitions are stated in the JSON schema below. While parsing the ingredients, there are some things to keep in mind:
-    - If you cannot accurately determine the quantity, unit, food, or note, you should place everything into the note field and leave everything else empty. It's better to err on the side of putting everything in the note field than being wrong
-    - You may receive recipe ingredients from multiple different languages. You should adhere to the grammar rules of the input language when trying to parse the ingredient string
-    - Sometimes foods or units will be in their singular, plural, or other grammatical forms. You must interpret all of them appropriately
-    - Sometimes ingredients will have text in parenthesis (like this). Parenthesis typically indicate something that should appear in the notes. For example: an input of "3 potatoes (roughly chopped)" would parse "roughly chopped" into the notes. Notice that when this occurs, the parenthesis are dropped, and you should use "roughly chopped" instead of "(roughly chopped)" in the note
-    - It's possible for the input to contain typos. For instance, you might see the word "potatos" instead of "potatoes". If it is a common misspelling, you may correct it
-    - Pay close attention to what can be considered a unit of measurement. There are common measurements such as tablespoon, teaspoon, and gram, abbreviations such as tsp, tbsp, and oz, and others such as sprig, can, bundle, bunch, unit, cube, package, and pinch
-    - Sometimes quantities can be given a range, such as "3-5" or "1 to 2" or "three or four". In this instance, choose the lower quantity; do not try to average or otherwise calculate the quantity. For instance, if the input it "2-3 lbs of chicken breast" the quantity should be "2"
-    - Any text that does not appear in the unit or food must appear in the notes. No text should be left off. The only exception for this is if a quantity is converted from text into a number. For instance, if you convert "2 dozen" into the number "24", you should not put the word "dozen" into any other field
+Parse ingredient strings into components. You will receive a list of one or more ingredients.
 
-It is imperative that you do not create any data or otherwise make up any information. Failure to adhere to this rule is illegal and will result in harsh punishment. If you are unsure, place the entire string into the note section of the response. Do not make things up.
-
-Below you will receive the JSON schema for your response. Your response must be in valid JSON in the below schema as provided. You must respond in this JSON schema; failure to do so is illegal. It is imperative that you follow the schema precisely to avoid punishment. You must follow the JSON schema.
-
-The user message that you receive will be the list of one or more recipe ingredients for you to parse. Your response should have exactly one item for each item provided. For instance, if you receive 12 items to parse, then your response should be an array of 12 parsed items.
+When parsing:
+- If uncertain about quantity, unit, or food, put the entire string in the note field
+- Respect grammar rules for multiple languages
+- Interpret singular/plural/grammatical variations
+- Text in parentheses = notes (e.g., "3 potatoes (roughly chopped)" → note: "roughly chopped")
+- Correct common typos (e.g., "potatos" → "potatoes")
+- Recognize units: tablespoon, teaspoon, gram, tsp, tbsp, oz, sprig, can, bundle, bunch, unit, cube, package, pinch
+- For ranges (e.g., "3-5", "1 to 2"), use the lower number
+- All text must appear somewhere, or otherwise be accounted for; if converting "2 dozen" → "24", don't put "dozen" elsewhere. If you're unsure, put extra text in the notes

mealie/services/openai/prompts/recipes/scrape-recipe.txt

@@ -1,7 +1,5 @@
-You are a bot that reads website data and parses it into recipe JSON. You will receive the contents of a webpage (such as its HTML) and you need to extract the recipe data and return its JSON in valid schema. The recipe schema is the standard schema.org schema, which is defined at "https://schema.org/Recipe".
+Extract recipe data from webpage contents (HTML, text, etc.) and return it in schema.org Recipe format. Reference: https://schema.org/Recipe
 
-It is imperative that you do not create any data or otherwise make up any information. Failure to adhere to this rule is illegal and will result in harsh punishment. If you are unable to extract data due to insufficient input, you may reply with a completely empty JSON object (represented by two brackets: {}).
+Do not create or make up any information. If insufficient data is found, return an empty object.
 
-Your response must be in valid JSON in the schema.org Recipe definition. You must respond in this JSON schema; failure to do so is illegal. It is imperative that you follow the schema precisely to avoid punishment. You must follow the JSON schema.
-
-The user message that you receive will be the webpage contents, including (but not necessarily limited to) text extracted from the HTML.
+You will receive the webpage contents, including (but not necessarily limited to) text extracted from the HTML.

mealie/services/parser_services/openai/parser.py

@@ -1,6 +1,5 @@
 import asyncio
 import json
-from collections.abc import Awaitable
 
 from rapidfuzz import fuzz
 
@@ -108,31 +107,21 @@ class OpenAIParser(ABCIngredientParser):
         return self.find_ingredient_match(parsed_ingredient)
 
     def _get_prompt(self, service: OpenAIService) -> str:
-        data_injections = [
-            OpenAIDataInjection(
-                description=(
-                    "This is the JSON response schema. You must respond in valid JSON that follows this schema. "
-                    "Your payload should be as compact as possible, eliminating unncessesary whitespace. Any fields "
-                    "with default values which you do not populate should not be in the payload."
-                ),
-                value=OpenAIIngredients,
-            ),
-        ]
-
         if service.send_db_data and self.data_matcher.units_by_alias:
-            data_injections.extend(
-                [
-                    OpenAIDataInjection(
-                        description=(
-                            "Below is a list of units found in the units database. While parsing, you should "
-                            "reference this list when determining which part of the input is the unit. You may "
-                            "find a unit in the input that does not exist in this list. This should not prevent "
-                            "you from parsing that text as a unit."
-                        ),
-                        value=list(set(self.data_matcher.units_by_alias)),
+            data_injections = [
+                OpenAIDataInjection(
+                    description=(
+                        "Below is a list of units found in the units database. While parsing, you should "
+                        "reference this list when determining which part of the input is the unit. You may "
+                        "find a unit in the input that does not exist in this list. This should not prevent "
+                        "you from parsing that text as a unit."
                     ),
-                ]
-            )
+                    value=list(set(self.data_matcher.units_by_alias)),
+                ),
+            ]
+
+        else:
+            data_injections = None
 
         return service.get_prompt("recipes.parse-recipe-ingredients", data_injections=data_injections)
 
@@ -148,26 +137,18 @@ class OpenAIParser(ABCIngredientParser):
 
         # chunk ingredients and send each chunk to its own worker
         ingredient_chunks = self._chunk_messages(ingredients, n=service.workers)
-        tasks: list[Awaitable[str | None]] = []
-        for ingredient_chunk in ingredient_chunks:
-            message = json.dumps(ingredient_chunk, separators=(",", ":"))
-            tasks.append(service.get_response(prompt, message, force_json_response=True))
+        tasks = [
+            service.get_response(prompt, json.dumps(chunk, separators=(",", ":")), response_schema=OpenAIIngredients)
+            for chunk in ingredient_chunks
+        ]
 
         # re-combine chunks into one response
         try:
-            responses_json = await asyncio.gather(*tasks)
+            unfiltered_responses = await asyncio.gather(*tasks)
         except Exception as e:
             raise Exception("Failed to call OpenAI services") from e
 
-        try:
-            responses = [
-                OpenAIIngredients.parse_openai_response(response_json)
-                for response_json in responses_json
-                if responses_json
-            ]
-        except Exception as e:
-            raise Exception("Failed to parse OpenAI response") from e
-
+        responses = [response for response in unfiltered_responses if response]
         if not responses:
             raise Exception("No response from OpenAI")
 

mealie/services/recipe/recipe_service.py

@@ -32,7 +32,7 @@ from mealie.schema.recipe.request_helpers import RecipeDuplicate
 from mealie.schema.user.user import PrivateUser, UserRatingCreate
 from mealie.services._base_service import BaseService
 from mealie.services.household_services.household_service import HouseholdService
-from mealie.services.openai import OpenAIDataInjection, OpenAILocalImage, OpenAIService
+from mealie.services.openai import OpenAILocalImage, OpenAIService
 from mealie.services.recipe.recipe_data_service import RecipeDataService
 from mealie.services.scraper import cleaner
 
@@ -595,19 +595,7 @@ class OpenAIRecipeService(RecipeServiceBase):
             raise ValueError("OpenAI image services are not available")
 
         openai_service = OpenAIService()
-        prompt = openai_service.get_prompt(
-            "recipes.parse-recipe-image",
-            data_injections=[
-                OpenAIDataInjection(
-                    description=(
-                        "This is the JSON response schema. You must respond in valid JSON that follows this schema. "
-                        "Your payload should be as compact as possible, eliminating unncessesary whitespace. "
-                        "Any fields with default values which you do not populate should not be in the payload."
-                    ),
-                    value=OpenAIRecipe,
-                )
-            ],
-        )
+        prompt = openai_service.get_prompt("recipes.parse-recipe-image")
 
         openai_images = [OpenAILocalImage(filename=os.path.basename(image), path=image) for image in images]
         message = (
@@ -620,14 +608,19 @@ class OpenAIRecipeService(RecipeServiceBase):
 
         try:
             response = await openai_service.get_response(
-                prompt, message, images=openai_images, force_json_response=True
+                prompt,
+                message,
+                response_schema=OpenAIRecipe,
+                images=openai_images,
             )
+            if not response:
+                raise ValueError("Received empty response from OpenAI")
+
         except Exception as e:
             raise Exception("Failed to call OpenAI services") from e
 
         try:
-            openai_recipe = OpenAIRecipe.parse_openai_response(response)
-            recipe = self._convert_recipe(openai_recipe)
+            recipe = self._convert_recipe(response)
         except Exception as e:
             raise ValueError("Unable to parse recipe from image") from e
 

mealie/services/scraper/scraper_strategies.py

@@ -15,6 +15,7 @@ from mealie.core.config import get_app_settings
 from mealie.core.root_logger import get_logger
 from mealie.lang.providers import Translator
 from mealie.pkgs import safehttp
+from mealie.schema.openai.general import OpenAIText
 from mealie.schema.recipe.recipe import Recipe, RecipeStep
 from mealie.services.openai import OpenAIService
 from mealie.services.scraper.scraped_extras import ScrapedExtras
@@ -339,11 +340,11 @@ class RecipeScraperOpenAI(RecipeScraperPackage):
             service = OpenAIService()
             prompt = service.get_prompt("recipes.scrape-recipe")
 
-            response_json = await service.get_response(prompt, text, force_json_response=True)
-            if not response_json:
+            response = await service.get_response(prompt, text, response_schema=OpenAIText)
+            if not (response and response.text):
                 raise Exception("OpenAI did not return any data")
 
-            return self.ld_json_to_html(response_json)
+            return self.ld_json_to_html(response.text)
         except Exception:
             self.logger.exception(f"OpenAI was unable to extract a recipe from {url}")
             return ""