mirror of
				https://github.com/mealie-recipes/mealie.git
				synced 2025-10-31 02:03:35 -04:00 
			
		
		
		
	feat: ✨ Add brute strategy to ingredient processor (#744)
* fix UI column width * words * update parser to support diff strats * add new model url * make button more visible * fix nutrition error * feat(backend): ✨ add 'brute' strategy for parsing ingredients * satisfy linter * update UI for creation page * feat(backend): ✨ log 422 errors in detail when not in PRODUCTION * add strategy selector Co-authored-by: Hayden <hay-kot@pm.me>
This commit is contained in:
		| @@ -106,7 +106,7 @@ COPY --from=builder-base $POETRY_HOME $POETRY_HOME | ||||
| COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH | ||||
|  | ||||
| # copy CRF++ Binary from crfpp | ||||
| ENV CRF_MODEL_URL=https://github.com/hay-kot/mealie-nlp-model/releases/download/v1.0.0/model.crfmodel | ||||
| ENV CRF_MODEL_URL=https://github.com/mealie-recipes/nlp-model/releases/download/v1.0.0/model.crfmodel | ||||
|  | ||||
| ENV LD_LIBRARY_PATH=/usr/local/lib | ||||
| COPY --from=crfpp /usr/local/lib/ /usr/local/lib | ||||
|   | ||||
| @@ -22,6 +22,43 @@ const routes = { | ||||
|   recipesSlugCommentsId: (slug: string, id: number) => `${prefix}/recipes/${slug}/comments/${id}`, | ||||
| }; | ||||
|  | ||||
| export type Parser = "nlp" | "brute"; | ||||
|  | ||||
| export interface Confidence { | ||||
|   average?: number; | ||||
|   comment?: number; | ||||
|   name?: number; | ||||
|   unit?: number; | ||||
|   quantity?: number; | ||||
|   food?: number; | ||||
| } | ||||
|  | ||||
| export interface Unit { | ||||
|   name: string; | ||||
|   description: string; | ||||
|   fraction: boolean; | ||||
|   abbreviation: string; | ||||
| } | ||||
|  | ||||
| export interface Food { | ||||
|   name: string; | ||||
|   description: string; | ||||
| } | ||||
|  | ||||
| export interface Ingredient { | ||||
|   title: string; | ||||
|   note: string; | ||||
|   unit: Unit; | ||||
|   food: Food; | ||||
|   disableAmount: boolean; | ||||
|   quantity: number; | ||||
| } | ||||
|  | ||||
| export interface ParsedIngredient { | ||||
|   confidence: Confidence; | ||||
|   ingredient: Ingredient; | ||||
| } | ||||
|  | ||||
| export class RecipeAPI extends BaseCRUDAPI<Recipe, CreateRecipe> { | ||||
|   baseRoute: string = routes.recipesBase; | ||||
|   itemRoute = routes.recipesRecipeSlug; | ||||
| @@ -84,11 +121,13 @@ export class RecipeAPI extends BaseCRUDAPI<Recipe, CreateRecipe> { | ||||
|     return await this.requests.delete(routes.recipesSlugCommentsId(slug, id)); | ||||
|   } | ||||
|  | ||||
|   async parseIngredients(ingredients: Array<string>) { | ||||
|     return await this.requests.post(routes.recipesParseIngredients, { ingredients }); | ||||
|   async parseIngredients(parser: Parser, ingredients: Array<string>) { | ||||
|     parser = parser || "nlp"; | ||||
|     return await this.requests.post<ParsedIngredient[]>(routes.recipesParseIngredients, { parser, ingredients }); | ||||
|   } | ||||
|  | ||||
|   async parseIngredient(ingredient: string) { | ||||
|     return await this.requests.post(routes.recipesParseIngredient, { ingredient }); | ||||
|   async parseIngredient(parser: Parser, ingredient: string) { | ||||
|     parser = parser || "nlp"; | ||||
|     return await this.requests.post<ParsedIngredient>(routes.recipesParseIngredient, { parser, ingredient }); | ||||
|   } | ||||
| } | ||||
|   | ||||
							
								
								
									
										82
									
								
								frontend/components/global/BaseOverflowButton.vue
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										82
									
								
								frontend/components/global/BaseOverflowButton.vue
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,82 @@ | ||||
|   <template> | ||||
|   <v-menu offset-y> | ||||
|     <template #activator="{ on, attrs }"> | ||||
|       <v-btn color="primary" v-bind="attrs" :class="btnClass" v-on="on"> | ||||
|         <v-icon v-if="activeObj.icon" left> | ||||
|           {{ activeObj.icon }} | ||||
|         </v-icon> | ||||
|         {{ activeObj.text }} | ||||
|         <v-icon right> | ||||
|           {{ $globals.icons.chevronDown }} | ||||
|         </v-icon> | ||||
|       </v-btn> | ||||
|     </template> | ||||
|     <v-list> | ||||
|       <v-list-item-group v-model="itemGroup"> | ||||
|         <v-list-item v-for="(item, index) in items" :key="index" @click="setValue(item)"> | ||||
|           <v-list-item-icon v-if="item.icon"> | ||||
|             <v-icon>{{ item.icon }}</v-icon> | ||||
|           </v-list-item-icon> | ||||
|           <v-list-item-title>{{ item.text }}</v-list-item-title> | ||||
|         </v-list-item> | ||||
|       </v-list-item-group> | ||||
|     </v-list> | ||||
|   </v-menu> | ||||
| </template> | ||||
|  | ||||
| <script lang="ts"> | ||||
| import { defineComponent, ref } from "@nuxtjs/composition-api"; | ||||
|  | ||||
| const INPUT_EVENT = "input"; | ||||
|  | ||||
| export default defineComponent({ | ||||
|   props: { | ||||
|     items: { | ||||
|       type: Array, | ||||
|       required: true, | ||||
|     }, | ||||
|     value: { | ||||
|       type: String, | ||||
|       required: false, | ||||
|       default: "", | ||||
|     }, | ||||
|     btnClass: { | ||||
|       type: String, | ||||
|       required: false, | ||||
|       default: "", | ||||
|     }, | ||||
|   }, | ||||
|   setup(props, context) { | ||||
|     const activeObj = ref({ | ||||
|       text: "DEFAULT", | ||||
|       value: "", | ||||
|     }); | ||||
|  | ||||
|     let startIndex = 0; | ||||
|     props.items.forEach((item, index) => { | ||||
|       // @ts-ignore | ||||
|       if (item.value === props.value) { | ||||
|         startIndex = index; | ||||
|  | ||||
|         // @ts-ignore | ||||
|         activeObj.value = item; | ||||
|       } | ||||
|     }); | ||||
|     const itemGroup = ref(startIndex); | ||||
|  | ||||
|     function setValue(v: any) { | ||||
|       context.emit(INPUT_EVENT, v.value); | ||||
|       activeObj.value = v; | ||||
|     } | ||||
|  | ||||
|     return { | ||||
|       activeObj, | ||||
|       itemGroup, | ||||
|       setValue, | ||||
|     }; | ||||
|   }, | ||||
| }); | ||||
| </script> | ||||
|  | ||||
|  | ||||
|      | ||||
| @@ -1,18 +1,28 @@ | ||||
| <template> | ||||
|   <v-container> | ||||
|   <v-container class="pa-0"> | ||||
|     <v-container> | ||||
|       <BaseCardSectionTitle title="Ingredients Natural Language Processor"> | ||||
|         Mealie uses conditional random Conditional Random Fields (CRFs) for parsing and processing ingredients. The | ||||
|         model used for ingredients is based off a data set of over 100,000 ingredients from a dataset compiled by the | ||||
|         New York Times. Note that as the model is trained in English only, you may have varied results when using the | ||||
|         model in other languages. This page is a playground for testing the model. | ||||
|         Mealie uses Conditional Random Fields (CRFs) for parsing and processing ingredients. The model used for | ||||
|         ingredients is based off a data set of over 100,000 ingredients from a dataset compiled by the New York Times. | ||||
|         Note that as the model is trained in English only, you may have varied results when using the model in other | ||||
|         languages. This page is a playground for testing the model. | ||||
|  | ||||
|         <p class="pt-3"> | ||||
|           It's not perfect, but it yields great results in general and is a good starting point for manually parsing | ||||
|           ingredients into individual fields. | ||||
|           ingredients into individual fields. Alternatively, you can also use the "Brute" processor that uses a pattern | ||||
|           matching technique to identify ingredients. | ||||
|         </p> | ||||
|       </BaseCardSectionTitle> | ||||
|  | ||||
|       <div class="d-flex align-center justify-center justify-md-start flex-wrap"> | ||||
|         <v-btn-toggle v-model="parser" dense mandatory @change="processIngredient"> | ||||
|           <v-btn value="nlp"> NLP </v-btn> | ||||
|           <v-btn value="brute"> Brute </v-btn> | ||||
|         </v-btn-toggle> | ||||
|  | ||||
|         <v-checkbox v-model="showConfidence" class="ml-5" label="Show individual confidence"></v-checkbox> | ||||
|       </div> | ||||
|  | ||||
|       <v-card flat> | ||||
|         <v-card-text> | ||||
|           <v-text-field v-model="ingredient" label="Ingredient Text"> </v-text-field> | ||||
| @@ -26,22 +36,29 @@ | ||||
|       </v-card> | ||||
|     </v-container> | ||||
|     <v-container v-if="results"> | ||||
|       <v-row class="d-flex"> | ||||
|       <div v-if="parser !== 'brute' && getConfidence('average')" class="d-flex"> | ||||
|         <v-chip dark :color="getColor('average')" class="mx-auto mb-2"> | ||||
|           {{ getConfidence("average") }} Confident | ||||
|         </v-chip> | ||||
|       </div> | ||||
|       <div class="d-flex justify-center flex-wrap" style="gap: 1.5rem"> | ||||
|         <template v-for="(prop, index) in properties"> | ||||
|           <v-col v-if="prop.value" :key="index" xs="12" sm="6" lg="3"> | ||||
|             <v-card> | ||||
|           <div v-if="prop.value" :key="index" class="flex-grow-1"> | ||||
|             <v-card min-width="200px"> | ||||
|               <v-card-title> {{ prop.value }} </v-card-title> | ||||
|               <v-card-text> | ||||
|                 {{ prop.subtitle }} | ||||
|               </v-card-text> | ||||
|             </v-card> | ||||
|           </v-col> | ||||
|             <v-chip v-if="prop.confidence && showConfidence" dark :color="prop.color" class="mt-2"> | ||||
|               {{ prop.confidence }} Confident | ||||
|             </v-chip> | ||||
|           </div> | ||||
|         </template> | ||||
|       </v-row> | ||||
|       </div> | ||||
|     </v-container> | ||||
|     <v-container class="narrow-container"> | ||||
|       <v-card-title> Try an example </v-card-title> | ||||
|  | ||||
|       <v-card v-for="(text, idx) in tryText" :key="idx" class="my-2" hover @click="processTryText(text)"> | ||||
|         <v-card-text> {{ text }} </v-card-text> | ||||
|       </v-card> | ||||
| @@ -50,7 +67,8 @@ | ||||
| </template> | ||||
|      | ||||
| <script lang="ts"> | ||||
| import { defineComponent, reactive, toRefs } from "@nuxtjs/composition-api"; | ||||
| import { defineComponent, reactive, ref, toRefs } from "@nuxtjs/composition-api"; | ||||
| import { Confidence, Parser } from "~/api/class-interfaces/recipes"; | ||||
| import { useApiSingleton } from "~/composables/use-api"; | ||||
|  | ||||
| export default defineComponent({ | ||||
| @@ -62,8 +80,41 @@ export default defineComponent({ | ||||
|       loading: false, | ||||
|       ingredient: "", | ||||
|       results: false, | ||||
|       parser: "nlp" as Parser, | ||||
|     }); | ||||
|  | ||||
|     const confidence = ref<Confidence>({}); | ||||
|  | ||||
|     function getColor(attribute: string) { | ||||
|       const percentage = getConfidence(attribute); | ||||
|  | ||||
|       // @ts-ignore | ||||
|       const p_as_num = parseFloat(percentage?.replace("%", "")); | ||||
|  | ||||
|       // Set color based off range | ||||
|       if (p_as_num > 75) { | ||||
|         return "success"; | ||||
|       } else if (p_as_num > 60) { | ||||
|         return "warning"; | ||||
|       } else { | ||||
|         return "error"; | ||||
|       } | ||||
|     } | ||||
|  | ||||
|     function getConfidence(attribute: string) { | ||||
|       attribute = attribute.toLowerCase(); | ||||
|       if (!confidence.value) { | ||||
|         return; | ||||
|       } | ||||
|  | ||||
|       // @ts-ignore | ||||
|       const property: number = confidence.value[attribute]; | ||||
|       if (property) { | ||||
|         return `${(property * 100).toFixed(0)}%`; | ||||
|       } | ||||
|       return null; | ||||
|     } | ||||
|  | ||||
|     const tryText = [ | ||||
|       "2 tbsp minced cilantro, leaves and stems", | ||||
|       "1 large yellow onion, coarsely chopped", | ||||
| @@ -78,23 +129,39 @@ export default defineComponent({ | ||||
|     } | ||||
|  | ||||
|     async function processIngredient() { | ||||
|       if (state.ingredient === "") { | ||||
|         return; | ||||
|       } | ||||
|  | ||||
|       state.loading = true; | ||||
|       const { data } = await api.recipes.parseIngredient(state.ingredient); | ||||
|  | ||||
|       const { data } = await api.recipes.parseIngredient(state.parser, state.ingredient); | ||||
|  | ||||
|       if (data) { | ||||
|         state.results = true; | ||||
|  | ||||
|         confidence.value = data.confidence; | ||||
|  | ||||
|         // TODO: Remove ts-ignore | ||||
|         // ts-ignore because data will likely change significantly once I figure out how to return results | ||||
|         // for the parser. For now we'll leave it like this | ||||
|         // @ts-ignore | ||||
|         properties.comments.value = data.ingredient.note || null; | ||||
|         // @ts-ignore | ||||
|         properties.quantity.value = data.ingredient.quantity || null; | ||||
|         // @ts-ignore | ||||
|         properties.unit.value = data.ingredient.unit.name || null; | ||||
|         // @ts-ignore | ||||
|         properties.food.value = data.ingredient.food.name || null; | ||||
|         properties.comment.value = data.ingredient.note || ""; | ||||
|         properties.quantity.value = data.ingredient.quantity || ""; | ||||
|         properties.unit.value = data.ingredient.unit.name || ""; | ||||
|         properties.food.value = data.ingredient.food.name || ""; | ||||
|  | ||||
|         for (const property in properties) { | ||||
|           const color = getColor(property); | ||||
|           const confidence = getConfidence(property); | ||||
|           if (color) { | ||||
|             // @ts-ignore | ||||
|             properties[property].color = color; | ||||
|           } | ||||
|           if (confidence) { | ||||
|             // @ts-ignore | ||||
|             properties[property].confidence = confidence; | ||||
|           } | ||||
|         } | ||||
|       } | ||||
|       state.loading = false; | ||||
|     } | ||||
| @@ -102,23 +169,37 @@ export default defineComponent({ | ||||
|     const properties = reactive({ | ||||
|       quantity: { | ||||
|         subtitle: "Quantity", | ||||
|         value: "Value", | ||||
|         value: "" as any, | ||||
|         color: null, | ||||
|         confidence: null, | ||||
|       }, | ||||
|       unit: { | ||||
|         subtitle: "Unit", | ||||
|         value: "Value", | ||||
|         value: "", | ||||
|         color: null, | ||||
|         confidence: null, | ||||
|       }, | ||||
|       food: { | ||||
|         subtitle: "Food", | ||||
|         value: "Value", | ||||
|         value: "", | ||||
|         color: null, | ||||
|         confidence: null, | ||||
|       }, | ||||
|       comments: { | ||||
|         subtitle: "Comments", | ||||
|         value: "Value", | ||||
|       comment: { | ||||
|         subtitle: "Comment", | ||||
|         value: "", | ||||
|         color: null, | ||||
|         confidence: null, | ||||
|       }, | ||||
|     }); | ||||
|  | ||||
|     const showConfidence = ref(false); | ||||
|  | ||||
|     return { | ||||
|       showConfidence, | ||||
|       getColor, | ||||
|       confidence, | ||||
|       getConfidence, | ||||
|       ...toRefs(state), | ||||
|       tryText, | ||||
|       properties, | ||||
|   | ||||
| @@ -1,23 +1,69 @@ | ||||
| <template> | ||||
|   <v-container v-if="recipe"> | ||||
|     <v-container> | ||||
|       <BaseCardSectionTitle title="Ingredients Processor"> </BaseCardSectionTitle> | ||||
|       <BaseCardSectionTitle title="Ingredients Processor"> | ||||
|         To use the ingredient parser, click the "Parse All" button and the process will start. When the processed | ||||
|         ingredients are available, you can look through the items and verify that they were parsed correctly. The models | ||||
|         confidence score is displayed on the right of the title item. This is an average of all scores and may not be | ||||
|         wholey accurate. | ||||
|  | ||||
|         <div class="mt-6"> | ||||
|           Alerts will be displayed if a matching foods or unit is found but does not exists in the database. | ||||
|         </div> | ||||
|         <v-divider class="my-4"> </v-divider> | ||||
|         <div class="mb-n4"> | ||||
|           Select Parser | ||||
|           <BaseOverflowButton | ||||
|             v-model="parser" | ||||
|             btn-class="mx-2" | ||||
|             :items="[ | ||||
|               { | ||||
|                 text: 'Natural Language Processor ', | ||||
|                 value: 'nlp', | ||||
|               }, | ||||
|               { | ||||
|                 text: 'Brute Parser', | ||||
|                 value: 'brute', | ||||
|               }, | ||||
|             ]" | ||||
|           /> | ||||
|         </div> | ||||
|       </BaseCardSectionTitle> | ||||
|  | ||||
|       <v-card-actions class="justify-end"> | ||||
|         <BaseButton color="info"> | ||||
|         <BaseButton color="info" @click="fetchParsed"> | ||||
|           <template #icon> {{ $globals.icons.foods }}</template> | ||||
|           Parse All | ||||
|         </BaseButton> | ||||
|         <BaseButton save> Save All </BaseButton> | ||||
|         <BaseButton save @click="saveAll"> Save All </BaseButton> | ||||
|       </v-card-actions> | ||||
|  | ||||
|       </v-card> | ||||
|       <v-expansion-panels v-model="panels" multiple> | ||||
|         <v-expansion-panel v-for="(ing, index) in ingredients" :key="index"> | ||||
|           <v-expansion-panel-header class="my-0 py-0"> | ||||
|             {{ recipe.recipeIngredient[index].note }} | ||||
|         <v-expansion-panel v-for="(ing, index) in parsedIng" :key="index"> | ||||
|           <v-expansion-panel-header class="my-0 py-0" disable-icon-rotate> | ||||
|             {{ ing.input }} | ||||
|             <template #actions> | ||||
|               <v-icon left :color="isError(ing) ? 'error' : 'success'"> | ||||
|                 {{ isError(ing) ? $globals.icons.alert : $globals.icons.check }} | ||||
|               </v-icon> | ||||
|               <div class="my-auto" :color="isError(ing) ? 'error-text' : 'success-text'"> | ||||
|                 {{ asPercentage(ing.confidence.average) }} | ||||
|               </div> | ||||
|             </template> | ||||
|           </v-expansion-panel-header> | ||||
|           <v-expansion-panel-content class="pb-0 mb-0"> | ||||
|             <RecipeIngredientEditor v-model="ingredients[index]" /> | ||||
|             <RecipeIngredientEditor v-model="parsedIng[index].ingredient" /> | ||||
|             <v-card-actions> | ||||
|               <v-spacer></v-spacer> | ||||
|               <BaseButton | ||||
|                 v-if="errors[index].foodError && errors[index].foodErrorMessage !== ''" | ||||
|                 color="warning" | ||||
|                 small | ||||
|                 @click="createFood(ing.ingredient.food, index)" | ||||
|               > | ||||
|                 {{ errors[index].foodErrorMessage }} | ||||
|               </BaseButton> | ||||
|             </v-card-actions> | ||||
|           </v-expansion-panel-content> | ||||
|         </v-expansion-panel> | ||||
|       </v-expansion-panels> | ||||
| @@ -26,19 +72,32 @@ | ||||
| </template> | ||||
|    | ||||
| <script lang="ts"> | ||||
| import { defineComponent, reactive, ref, toRefs, useRoute, watch } from "@nuxtjs/composition-api"; | ||||
| import { defineComponent, ref, useRoute, useRouter } from "@nuxtjs/composition-api"; | ||||
| import { Food, ParsedIngredient, Parser } from "~/api/class-interfaces/recipes"; | ||||
| import RecipeIngredientEditor from "~/components/Domain/Recipe/RecipeIngredientEditor.vue"; | ||||
| import { useApiSingleton } from "~/composables/use-api"; | ||||
| import { useRecipeContext } from "~/composables/use-recipe-context"; | ||||
| import { useFoods } from "~/composables/use-recipe-foods"; | ||||
| import { useUnits } from "~/composables/use-recipe-units"; | ||||
| import { RecipeIngredientUnit } from "~/types/api-types/recipe"; | ||||
|  | ||||
| interface Error { | ||||
|   ingredientIndex: number; | ||||
|   unitError: Boolean; | ||||
|   unitErrorMessage: string; | ||||
|   foodError: Boolean; | ||||
|   foodErrorMessage: string; | ||||
| } | ||||
|  | ||||
| export default defineComponent({ | ||||
|   components: { | ||||
|     RecipeIngredientEditor, | ||||
|   }, | ||||
|   setup() { | ||||
|     const state = reactive({ | ||||
|       panels: null, | ||||
|     }); | ||||
|     const panels = ref<number[]>([]); | ||||
|  | ||||
|     const route = useRoute(); | ||||
|     const router = useRouter(); | ||||
|     const slug = route.value.params.slug; | ||||
|     const api = useApiSingleton(); | ||||
|  | ||||
| @@ -48,14 +107,150 @@ export default defineComponent({ | ||||
|  | ||||
|     const ingredients = ref<any[]>([]); | ||||
|  | ||||
|     watch(recipe, () => { | ||||
|       const copy = recipe?.value?.recipeIngredient || []; | ||||
|       ingredients.value = [...copy]; | ||||
|     }); | ||||
|     // ========================================================= | ||||
|     // Parser Logic | ||||
|  | ||||
|     const parser = ref<Parser>("nlp"); | ||||
|  | ||||
|     const parsedIng = ref<any[]>([]); | ||||
|  | ||||
|     async function fetchParsed() { | ||||
|       if (!recipe.value) { | ||||
|         return; | ||||
|       } | ||||
|       const raw = recipe.value.recipeIngredient.map((ing) => ing.note); | ||||
|       const { response, data } = await api.recipes.parseIngredients(parser.value, raw); | ||||
|       console.log({ response }); | ||||
|  | ||||
|       if (data) { | ||||
|         parsedIng.value = data; | ||||
|  | ||||
|         console.log(data); | ||||
|  | ||||
|         // @ts-ignore | ||||
|         errors.value = data.map((ing, index: number) => { | ||||
|           const unitError = !checkForUnit(ing.ingredient.unit); | ||||
|           const foodError = !checkForFood(ing.ingredient.food); | ||||
|  | ||||
|           let unitErrorMessage = ""; | ||||
|           let foodErrorMessage = ""; | ||||
|  | ||||
|           if (unitError || foodError) { | ||||
|             if (unitError) { | ||||
|               if (ing?.ingredient?.unit?.name) { | ||||
|                 unitErrorMessage = `Create missing unit '${ing?.ingredient?.unit?.name || "No unit"}'`; | ||||
|               } | ||||
|             } | ||||
|  | ||||
|             if (foodError) { | ||||
|               if (ing?.ingredient?.food?.name) { | ||||
|                 foodErrorMessage = `Create missing food '${ing.ingredient.food.name || "No food"}'?`; | ||||
|               } | ||||
|               panels.value.push(index); | ||||
|             } | ||||
|           } | ||||
|  | ||||
|           return { | ||||
|             ingredientIndex: index, | ||||
|             unitError, | ||||
|             unitErrorMessage, | ||||
|             foodError, | ||||
|             foodErrorMessage, | ||||
|           }; | ||||
|         }); | ||||
|       } | ||||
|     } | ||||
|  | ||||
|     function isError(ing: ParsedIngredient) { | ||||
|       if (!ing?.confidence?.average) { | ||||
|         return true; | ||||
|       } | ||||
|       return !(ing.confidence.average >= 0.75); | ||||
|     } | ||||
|  | ||||
|     function asPercentage(num: number) { | ||||
|       return Math.round(num * 100).toFixed(2) + "%"; | ||||
|     } | ||||
|  | ||||
|     // ========================================================= | ||||
|     // Food and Ingredient Logic | ||||
|  | ||||
|     const { foods, workingFoodData, actions } = useFoods(); | ||||
|     const { units } = useUnits(); | ||||
|  | ||||
|     const errors = ref<Error[]>([]); | ||||
|  | ||||
|     function checkForUnit(unit: RecipeIngredientUnit) { | ||||
|       if (units.value && unit?.name) { | ||||
|         return units.value.some((u) => u.name === unit.name); | ||||
|       } | ||||
|       return false; | ||||
|     } | ||||
|  | ||||
|     function checkForFood(food: Food) { | ||||
|       if (foods.value && food?.name) { | ||||
|         return foods.value.some((f) => f.name === food.name); | ||||
|       } | ||||
|       return false; | ||||
|     } | ||||
|  | ||||
|     async function createFood(food: Food, index: number) { | ||||
|       workingFoodData.name = food.name; | ||||
|       await actions.createOne(); | ||||
|       errors.value[index].foodError = false; | ||||
|     } | ||||
|  | ||||
|     // ========================================================= | ||||
|     // Save All Loginc | ||||
|     async function saveAll() { | ||||
|       let ingredients = parsedIng.value.map((ing) => { | ||||
|         return { | ||||
|           ...ing.ingredient, | ||||
|         }; | ||||
|       }); | ||||
|  | ||||
|       console.log(ingredients); | ||||
|  | ||||
|       ingredients = ingredients.map((ing) => { | ||||
|         if (!foods.value || !units.value) { | ||||
|           return ing; | ||||
|         } | ||||
|         // Get food from foods | ||||
|         const food = foods.value.find((f) => f.name === ing.food.name); | ||||
|         ing.food = food || null; | ||||
|  | ||||
|         // Get unit from units | ||||
|         const unit = units.value.find((u) => u.name === ing.unit.name); | ||||
|         ing.unit = unit || null; | ||||
|         console.log(ing); | ||||
|  | ||||
|         return ing; | ||||
|       }); | ||||
|  | ||||
|       if (!recipe.value) { | ||||
|         return; | ||||
|       } | ||||
|  | ||||
|       recipe.value.recipeIngredient = ingredients; | ||||
|       const { response } = await api.recipes.updateOne(recipe.value.slug, recipe.value); | ||||
|  | ||||
|       if (response?.status === 200) { | ||||
|         router.push("/recipe/" + recipe.value.slug); | ||||
|       } | ||||
|     } | ||||
|  | ||||
|     return { | ||||
|       ...toRefs(state), | ||||
|       api, | ||||
|       parser, | ||||
|       saveAll, | ||||
|       createFood, | ||||
|       errors, | ||||
|       actions, | ||||
|       workingFoodData, | ||||
|       isError, | ||||
|       panels, | ||||
|       asPercentage, | ||||
|       fetchParsed, | ||||
|       parsedIng, | ||||
|       recipe, | ||||
|       loading, | ||||
|       ingredients, | ||||
| @@ -69,5 +264,3 @@ export default defineComponent({ | ||||
| }); | ||||
| </script> | ||||
|    | ||||
| <style scoped> | ||||
| </style> | ||||
| @@ -7,11 +7,8 @@ | ||||
|       <template #title> Recipe Creation </template> | ||||
|       Select one of the various ways to create a recipe | ||||
|     </BasePageTitle> | ||||
|     <v-tabs v-model="tab"> | ||||
|       <v-tab href="#url">From URL</v-tab> | ||||
|       <v-tab href="#new">Create</v-tab> | ||||
|       <v-tab href="#zip">Import Zip</v-tab> | ||||
|     </v-tabs> | ||||
|     <BaseOverflowButton v-model="tab" rounded class="mx-2" outlined :items="tabs"> </BaseOverflowButton> | ||||
|    | ||||
|     <section> | ||||
|       <v-tabs-items v-model="tab" class="mt-10"> | ||||
|         <v-tab-item value="url" eager> | ||||
| @@ -127,7 +124,7 @@ | ||||
| </template> | ||||
|  | ||||
| <script lang="ts"> | ||||
| import { defineComponent, reactive, toRefs, ref, useRouter } from "@nuxtjs/composition-api"; | ||||
| import { defineComponent, reactive, toRefs, ref, useRouter, useContext } from "@nuxtjs/composition-api"; | ||||
| import { useApiSingleton } from "~/composables/use-api"; | ||||
| import { validators } from "~/composables/use-validators"; | ||||
| export default defineComponent({ | ||||
| @@ -137,6 +134,27 @@ export default defineComponent({ | ||||
|       loading: false, | ||||
|     }); | ||||
|  | ||||
|     // @ts-ignore - $globals not found in type definition | ||||
|     const { $globals } = useContext(); | ||||
|  | ||||
|     const tabs = [ | ||||
|       { | ||||
|         icon: $globals.icons.edit, | ||||
|         text: "Create Recipe", | ||||
|         value: "new", | ||||
|       }, | ||||
|       { | ||||
|         icon: $globals.icons.link, | ||||
|         text: "Import with URL", | ||||
|         value: "url", | ||||
|       }, | ||||
|       { | ||||
|         icon: $globals.icons.zip, | ||||
|         text: "Import with .zip", | ||||
|         value: "zip", | ||||
|       }, | ||||
|     ]; | ||||
|  | ||||
|     const api = useApiSingleton(); | ||||
|     const router = useRouter(); | ||||
|  | ||||
| @@ -203,6 +221,7 @@ export default defineComponent({ | ||||
|     } | ||||
|  | ||||
|     return { | ||||
|       tabs, | ||||
|       domCreateByName, | ||||
|       domUrlForm, | ||||
|       newRecipeName, | ||||
|   | ||||
| @@ -10,11 +10,11 @@ | ||||
|     <section> | ||||
|       <ToggleState tag="article"> | ||||
|         <template #activator="{ toggle, state }"> | ||||
|           <v-btn v-if="!state" text color="info" class="mt-2 mb-n3" @click="toggle"> | ||||
|           <v-btn v-if="!state" color="info" class="mt-2 mb-n3" @click="toggle"> | ||||
|             <v-icon left>{{ $globals.icons.lock }}</v-icon> | ||||
|             {{ $t("settings.change-password") }} | ||||
|           </v-btn> | ||||
|           <v-btn v-else text color="info" class="mt-2 mb-n3" @click="toggle"> | ||||
|           <v-btn v-else color="info" class="mt-2 mb-n3" @click="toggle"> | ||||
|             <v-icon left>{{ $globals.icons.user }}</v-icon> | ||||
|             {{ $t("settings.profile") }} | ||||
|           </v-btn> | ||||
|   | ||||
| @@ -30,6 +30,7 @@ import { | ||||
|   mdiDotsVertical, | ||||
|   mdiPrinter, | ||||
|   mdiShareVariant, | ||||
|   mdiChevronDown, | ||||
|   mdiHeart, | ||||
|   mdiHeartOutline, | ||||
|   mdiDotsHorizontal, | ||||
| @@ -210,4 +211,5 @@ export const icons = { | ||||
|   forward: mdiArrowRightBoldOutline, | ||||
|   back: mdiArrowLeftBoldOutline, | ||||
|   slotMachine: mdiSlotMachine, | ||||
|   chevronDown: mdiChevronDown, | ||||
| }; | ||||
|   | ||||
| @@ -7,6 +7,7 @@ from mealie.core.root_logger import get_logger | ||||
| from mealie.core.settings.static import APP_VERSION | ||||
| from mealie.routes import backup_routes, migration_routes, router, utility_routes | ||||
| from mealie.routes.about import about_router | ||||
| from mealie.routes.handlers import register_debug_handler | ||||
| from mealie.routes.media import media_router | ||||
| from mealie.routes.site_settings import settings_router | ||||
| from mealie.services.events import create_general_event | ||||
| @@ -25,6 +26,8 @@ app = FastAPI( | ||||
|  | ||||
| app.add_middleware(GZipMiddleware, minimum_size=1000) | ||||
|  | ||||
| register_debug_handler(app) | ||||
|  | ||||
|  | ||||
| def start_scheduler(): | ||||
|     SchedulerService.start() | ||||
|   | ||||
| @@ -117,7 +117,7 @@ class RecipeModel(SqlAlchemyBase, BaseMixins): | ||||
|         tools: list[str] = None, | ||||
|         **_ | ||||
|     ) -> None: | ||||
|         self.nutrition = Nutrition(**nutrition) if self.nutrition else Nutrition() | ||||
|         self.nutrition = Nutrition(**nutrition) if nutrition else Nutrition() | ||||
|         self.tools = [Tool(tool=x) for x in tools] if tools else [] | ||||
|         self.recipe_ingredient = [RecipeIngredient(**ingr, session=session) for ingr in recipe_ingredient] | ||||
|         self.assets = [RecipeAsset(**a) for a in assets] | ||||
|   | ||||
							
								
								
									
										33
									
								
								mealie/routes/handlers.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										33
									
								
								mealie/routes/handlers.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,33 @@ | ||||
| from fastapi import FastAPI, Request, status | ||||
| from fastapi.exceptions import RequestValidationError | ||||
| from fastapi.responses import JSONResponse | ||||
|  | ||||
| from mealie.core.config import get_app_settings | ||||
| from mealie.core.root_logger import get_logger | ||||
|  | ||||
| logger = get_logger(__name__) | ||||
|  | ||||
|  | ||||
| def log_wrapper(request: Request, e): | ||||
|  | ||||
|     logger.error("Start 422 Error".center(60, "-")) | ||||
|     logger.error(f"{request.method} {request.url}") | ||||
|     logger.error(f"error is {e}") | ||||
|     logger.error("End 422 Error".center(60, "-")) | ||||
|  | ||||
|  | ||||
| def register_debug_handler(app: FastAPI): | ||||
|     settings = get_app_settings() | ||||
|  | ||||
|     if settings.PRODUCTION: | ||||
|         return | ||||
|  | ||||
|     @app.exception_handler(RequestValidationError) | ||||
|     async def validation_exception_handler(request: Request, exc: RequestValidationError): | ||||
|  | ||||
|         exc_str = f"{exc}".replace("\n", " ").replace("   ", " ") | ||||
|         log_wrapper(request, exc) | ||||
|         content = {"status_code": status.HTTP_422_UNPROCESSABLE_ENTITY, "message": exc_str, "data": None} | ||||
|         return JSONResponse(content=content, status_code=status.HTTP_422_UNPROCESSABLE_ENTITY) | ||||
|  | ||||
|     return validation_exception_handler | ||||
| @@ -1,31 +1,25 @@ | ||||
| from fastapi import APIRouter, Depends | ||||
| from pydantic import BaseModel | ||||
|  | ||||
| from mealie.schema.recipe import RecipeIngredient | ||||
| from mealie.schema.recipe import ParsedIngredient | ||||
| from mealie.schema.recipe.recipe_ingredient import IngredientRequest, IngredientsRequest | ||||
| from mealie.services.parser_services import IngredientParserService | ||||
|  | ||||
| public_router = APIRouter(prefix="/parser") | ||||
|  | ||||
|  | ||||
| class IngredientsRequest(BaseModel): | ||||
|     ingredients: list[str] | ||||
|  | ||||
|  | ||||
| class IngredientRequest(BaseModel): | ||||
|     ingredient: str | ||||
|  | ||||
|  | ||||
| @public_router.post("/ingredients", response_model=list[RecipeIngredient]) | ||||
| @public_router.post("/ingredients", response_model=list[ParsedIngredient]) | ||||
| def parse_ingredients( | ||||
|     ingredients: IngredientsRequest, | ||||
|     p_service: IngredientParserService = Depends(IngredientParserService.private), | ||||
| ): | ||||
|     return {"ingredients": p_service.parse_ingredients(ingredients.ingredients)} | ||||
|     p_service.set_parser(parser=ingredients.parser) | ||||
|     return p_service.parse_ingredients(ingredients.ingredients) | ||||
|  | ||||
|  | ||||
| @public_router.post("/ingredient") | ||||
| @public_router.post("/ingredient", response_model=ParsedIngredient) | ||||
| def parse_ingredient( | ||||
|     ingredient: IngredientRequest, | ||||
|     p_service: IngredientParserService = Depends(IngredientParserService.private), | ||||
| ): | ||||
|     return {"ingredient": p_service.parse_ingredient(ingredient.ingredient)} | ||||
|     p_service.set_parser(parser=ingredient.parser) | ||||
|     return p_service.parse_ingredient(ingredient.ingredient) | ||||
|   | ||||
| @@ -1,3 +1,4 @@ | ||||
| import enum | ||||
| from typing import Optional, Union | ||||
|  | ||||
| from fastapi_camelcase import CamelModel | ||||
| @@ -30,10 +31,40 @@ class IngredientUnit(CreateIngredientUnit): | ||||
| class RecipeIngredient(CamelModel): | ||||
|     title: Optional[str] | ||||
|     note: Optional[str] | ||||
|     unit: Optional[Union[CreateIngredientUnit, IngredientUnit]] | ||||
|     food: Optional[Union[CreateIngredientFood, IngredientFood]] | ||||
|     unit: Optional[Union[IngredientUnit, CreateIngredientUnit]] | ||||
|     food: Optional[Union[IngredientFood, CreateIngredientFood]] | ||||
|     disable_amount: bool = True | ||||
|     quantity: float = 1 | ||||
|  | ||||
|     class Config: | ||||
|         orm_mode = True | ||||
|  | ||||
|  | ||||
| class IngredientConfidence(CamelModel): | ||||
|     average: float = None | ||||
|     comment: float = None | ||||
|     name: float = None | ||||
|     unit: float = None | ||||
|     quantity: float = None | ||||
|     food: float = None | ||||
|  | ||||
|  | ||||
| class ParsedIngredient(CamelModel): | ||||
|     input: Optional[str] | ||||
|     confidence: IngredientConfidence = IngredientConfidence() | ||||
|     ingredient: RecipeIngredient | ||||
|  | ||||
|  | ||||
| class RegisteredParser(str, enum.Enum): | ||||
|     nlp = "nlp" | ||||
|     brute = "brute" | ||||
|  | ||||
|  | ||||
| class IngredientsRequest(CamelModel): | ||||
|     parser: RegisteredParser = RegisteredParser.nlp | ||||
|     ingredients: list[str] | ||||
|  | ||||
|  | ||||
| class IngredientRequest(CamelModel): | ||||
|     parser: RegisteredParser = RegisteredParser.nlp | ||||
|     ingredient: str | ||||
|   | ||||
| @@ -1 +1,2 @@ | ||||
| from .ingredient_parser import * | ||||
| from .ingredient_parser_service import * | ||||
|   | ||||
							
								
								
									
										1
									
								
								mealie/services/parser_services/_helpers/__init__.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1
									
								
								mealie/services/parser_services/_helpers/__init__.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1 @@ | ||||
| from .string_utils import * | ||||
							
								
								
									
										23
									
								
								mealie/services/parser_services/_helpers/string_utils.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										23
									
								
								mealie/services/parser_services/_helpers/string_utils.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,23 @@ | ||||
| import re | ||||
|  | ||||
| compiled_match = re.compile(r"(.){1,6}\s\((.[^\(\)])+\)\s") | ||||
| compiled_search = re.compile(r"\((.[^\(])+\)") | ||||
|  | ||||
|  | ||||
| def move_parens_to_end(ing_str) -> str: | ||||
|     """ | ||||
|     Moves all parentheses in the string to the end of the string using Regex. | ||||
|     If no parentheses are found, the string is returned unchanged. | ||||
|     """ | ||||
|     if re.match(compiled_match, ing_str): | ||||
|         match = re.search(compiled_search, ing_str) | ||||
|         start = match.start() | ||||
|         end = match.end() | ||||
|         ing_str = ing_str[:start] + ing_str[end:] + " " + ing_str[start:end] | ||||
|  | ||||
|     return ing_str | ||||
|  | ||||
|  | ||||
| def check_char(char, *eql) -> bool: | ||||
|     """Helper method to check if a charaters matches any of the additional provided arguments""" | ||||
|     return any(char == eql_char for eql_char in eql) | ||||
							
								
								
									
										1
									
								
								mealie/services/parser_services/brute/__init__.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										1
									
								
								mealie/services/parser_services/brute/__init__.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1 @@ | ||||
| from .process import parse | ||||
							
								
								
									
										204
									
								
								mealie/services/parser_services/brute/process.py
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										204
									
								
								mealie/services/parser_services/brute/process.py
									
									
									
									
									
										Normal file
									
								
							| @@ -0,0 +1,204 @@ | ||||
| import string | ||||
| import unicodedata | ||||
| from typing import Tuple | ||||
|  | ||||
| from pydantic import BaseModel | ||||
|  | ||||
| from .._helpers import check_char, move_parens_to_end | ||||
|  | ||||
|  | ||||
| class BruteParsedIngredient(BaseModel): | ||||
|     food: str = "" | ||||
|     note: str = "" | ||||
|     amount: float = "" | ||||
|     unit: str = "" | ||||
|  | ||||
|     class Config: | ||||
|         anystr_strip_whitespace = True | ||||
|  | ||||
|  | ||||
| def parse_fraction(x): | ||||
|     if len(x) == 1 and "fraction" in unicodedata.decomposition(x): | ||||
|         frac_split = unicodedata.decomposition(x[-1:]).split() | ||||
|         return float((frac_split[1]).replace("003", "")) / float((frac_split[3]).replace("003", "")) | ||||
|     else: | ||||
|         frac_split = x.split("/") | ||||
|         if len(frac_split) != 2: | ||||
|             raise ValueError | ||||
|         try: | ||||
|             return int(frac_split[0]) / int(frac_split[1]) | ||||
|         except ZeroDivisionError: | ||||
|             raise ValueError | ||||
|  | ||||
|  | ||||
| def parse_amount(ing_str) -> Tuple[float, str, str]: | ||||
|     def keep_looping(ing_str, end) -> bool: | ||||
|         """ | ||||
|         Checks if: | ||||
|         1. the end of the string is reached | ||||
|         2. or if the next character is a digit | ||||
|         3. or if the next character looks like an number (e.g. 1/2, 1.3, 1,500) | ||||
|         """ | ||||
|         if end >= len(ing_str): | ||||
|             return False | ||||
|  | ||||
|         if ing_str[end] in string.digits: | ||||
|             return True | ||||
|  | ||||
|         if check_char(ing_str[end], ".", ",", "/") and end + 1 < len(ing_str) and ing_str[end + 1] in string.digits: | ||||
|             return True | ||||
|  | ||||
|     amount = 0 | ||||
|     unit = "" | ||||
|     note = "" | ||||
|  | ||||
|     did_check_frac = False | ||||
|     end = 0 | ||||
|  | ||||
|     while keep_looping(ing_str, end): | ||||
|         end += 1 | ||||
|  | ||||
|     if end > 0: | ||||
|         if "/" in ing_str[:end]: | ||||
|             amount = parse_fraction(ing_str[:end]) | ||||
|         else: | ||||
|             amount = float(ing_str[:end].replace(",", ".")) | ||||
|     else: | ||||
|         amount = parse_fraction(ing_str[0]) | ||||
|         end += 1 | ||||
|         did_check_frac = True | ||||
|     if end < len(ing_str): | ||||
|         if did_check_frac: | ||||
|             unit = ing_str[end:] | ||||
|         else: | ||||
|             try: | ||||
|                 amount += parse_fraction(ing_str[end]) | ||||
|  | ||||
|                 unit_end = end + 1 | ||||
|                 unit = ing_str[unit_end:] | ||||
|             except ValueError: | ||||
|                 unit = ing_str[end:] | ||||
|  | ||||
|     # i dont know any unit that starts with ( or - so its likely an alternative like 1L (500ml) Water or 2-3 | ||||
|     if unit.startswith("(") or unit.startswith("-"): | ||||
|         unit = "" | ||||
|         note = ing_str | ||||
|  | ||||
|     return amount, unit, note | ||||
|  | ||||
|  | ||||
| def parse_ingredient_with_comma(tokens) -> Tuple[str, str]: | ||||
|     ingredient = "" | ||||
|     note = "" | ||||
|     start = 0 | ||||
|     # search for first occurrence of an argument ending in a comma | ||||
|     while start < len(tokens) and not tokens[start].endswith(","): | ||||
|         start += 1 | ||||
|     if start == len(tokens): | ||||
|         # no token ending in a comma found -> use everything as ingredient | ||||
|         ingredient = " ".join(tokens) | ||||
|     else: | ||||
|         ingredient = " ".join(tokens[: start + 1])[:-1] | ||||
|  | ||||
|         note_end = start + 1 | ||||
|         note = " ".join(tokens[note_end:]) | ||||
|     return ingredient, note | ||||
|  | ||||
|  | ||||
| def parse_ingredient(tokens) -> Tuple[str, str]: | ||||
|     ingredient = "" | ||||
|     note = "" | ||||
|     if tokens[-1].endswith(")"): | ||||
|         # Check if the matching opening bracket is in the same token | ||||
|         if (not tokens[-1].startswith("(")) and ("(" in tokens[-1]): | ||||
|             return parse_ingredient_with_comma(tokens) | ||||
|         # last argument ends with closing bracket -> look for opening bracket | ||||
|         start = len(tokens) - 1 | ||||
|         while not tokens[start].startswith("(") and start != 0: | ||||
|             start -= 1 | ||||
|         if start == 0: | ||||
|             # the whole list is wrapped in brackets -> assume it is an error (e.g. assumed first argument was the unit)  # noqa: E501 | ||||
|             raise ValueError | ||||
|         elif start < 0: | ||||
|             # no opening bracket anywhere -> just ignore the last bracket | ||||
|             ingredient, note = parse_ingredient_with_comma(tokens) | ||||
|         else: | ||||
|             # opening bracket found -> split in ingredient and note, remove brackets from note  # noqa: E501 | ||||
|             note = " ".join(tokens[start:])[1:-1] | ||||
|             ingredient = " ".join(tokens[:start]) | ||||
|     else: | ||||
|         ingredient, note = parse_ingredient_with_comma(tokens) | ||||
|     return ingredient, note | ||||
|  | ||||
|  | ||||
| def parse(ing_str) -> BruteParsedIngredient: | ||||
|     amount = 0 | ||||
|     unit = "" | ||||
|     ingredient = "" | ||||
|     note = "" | ||||
|     unit_note = "" | ||||
|  | ||||
|     ing_str = move_parens_to_end(ing_str) | ||||
|  | ||||
|     tokens = ing_str.split() | ||||
|  | ||||
|     # Early return if the ingrdient is a single token and therefore has no other properties | ||||
|     if len(tokens) == 1: | ||||
|         ingredient = tokens[0] | ||||
|         # TODO Refactor to expect BFP to be returned instead of Tuple | ||||
|         return BruteParsedIngredient(food=ingredient, note=note, amount=amount, unit=unit) | ||||
|  | ||||
|     try: | ||||
|         # try to parse first argument as amount | ||||
|         amount, unit, unit_note = parse_amount(tokens[0]) | ||||
|         # only try to parse second argument as amount if there are at least | ||||
|         # three arguments if it already has a unit there can't be | ||||
|         # a fraction for the amount | ||||
|         if len(tokens) > 2: | ||||
|             try: | ||||
|                 if unit != "": | ||||
|                     # a unit is already found, no need to try the second argument for a fraction | ||||
|                     # probably not the best method to do it, but I didn't want to make an if check and paste the exact same thing in the else as already is in the except  # noqa: E501 | ||||
|                     raise ValueError | ||||
|                 # try to parse second argument as amount and add that, in case of '2 1/2' or '2 ½' | ||||
|                 amount += parse_fraction(tokens[1]) | ||||
|                 # assume that units can't end with a comma | ||||
|                 if len(tokens) > 3 and not tokens[2].endswith(","): | ||||
|                     # try to use third argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501 | ||||
|                     try: | ||||
|                         ingredient, note = parse_ingredient(tokens[3:]) | ||||
|                         unit = tokens[2] | ||||
|                     except ValueError: | ||||
|                         ingredient, note = parse_ingredient(tokens[2:]) | ||||
|                 else: | ||||
|                     ingredient, note = parse_ingredient(tokens[2:]) | ||||
|             except ValueError: | ||||
|                 # assume that units can't end with a comma | ||||
|                 if not tokens[1].endswith(","): | ||||
|                     # try to use second argument as unit and everything else as ingredient, use everything as ingredient if it fails  # noqa: E501 | ||||
|                     try: | ||||
|                         ingredient, note = parse_ingredient(tokens[2:]) | ||||
|                         if unit == "": | ||||
|                             unit = tokens[1] | ||||
|                         else: | ||||
|                             note = tokens[1] | ||||
|                     except ValueError: | ||||
|                         ingredient, note = parse_ingredient(tokens[1:]) | ||||
|                 else: | ||||
|                     ingredient, note = parse_ingredient(tokens[1:]) | ||||
|         else: | ||||
|             # only two arguments, first one is the amount | ||||
|             # which means this is the ingredient | ||||
|             ingredient = tokens[1] | ||||
|     except ValueError: | ||||
|         try: | ||||
|             # can't parse first argument as amount | ||||
|             # -> no unit -> parse everything as ingredient | ||||
|             ingredient, note = parse_ingredient(tokens) | ||||
|         except ValueError: | ||||
|             ingredient = " ".join(tokens[1:]) | ||||
|  | ||||
|     if unit_note not in note: | ||||
|         note += " " + unit_note | ||||
|  | ||||
|     return BruteParsedIngredient(food=ingredient, note=note, amount=amount, unit=unit) | ||||
| @@ -0,0 +1 @@ | ||||
| from .processor import * | ||||
|   | ||||
| @@ -2,23 +2,25 @@ import re | ||||
| import unicodedata | ||||
|  | ||||
| replace_abbreviations = { | ||||
|     "cup ": "cup ", | ||||
|     " g ": "gram ", | ||||
|     "kg ": "kilogram ", | ||||
|     "lb ": "pound ", | ||||
|     "ml ": "milliliter ", | ||||
|     "oz ": "ounce ", | ||||
|     "pint ": "pint ", | ||||
|     "qt ": "quart ", | ||||
|     "tbs ": "tablespoon ", | ||||
|     "tbsp ": "tablespoon ", | ||||
|     "tsp ": "teaspoon ", | ||||
|     "cup": " cup ", | ||||
|     "g": " gram ", | ||||
|     "kg": " kilogram ", | ||||
|     "lb": " pound ", | ||||
|     "ml": " milliliter ", | ||||
|     "oz": " ounce ", | ||||
|     "pint": " pint ", | ||||
|     "qt": " quart ", | ||||
|     "tbsp": " tablespoon ", | ||||
|     "tbs": " tablespoon ",  # Order Matters!, 'tsb' must come after 'tbsp' incase of duplicate matches | ||||
|     "tsp": " teaspoon ", | ||||
| } | ||||
|  | ||||
|  | ||||
| def replace_common_abbreviations(string: str) -> str: | ||||
|  | ||||
|     for k, v in replace_abbreviations.items(): | ||||
|         string = string.replace(k, v) | ||||
|         regex = rf"(?<=\d)\s?({k}s?)" | ||||
|         string = re.sub(regex, v, string) | ||||
|  | ||||
|     return string | ||||
|  | ||||
| @@ -81,17 +83,3 @@ def pre_process_string(string: str) -> str: | ||||
|         string = wrap_or_clause(string) | ||||
|  | ||||
|     return string | ||||
|  | ||||
|  | ||||
| def main(): | ||||
|     # TODO: Migrate to unittests | ||||
|     print("Starting...") | ||||
|     print(pre_process_string("1 tsp. Diamond Crystal or ½ tsp. Morton kosher salt, plus more")) | ||||
|     print(pre_process_string("1 tsp. Diamond Crystal or ½ tsp. Morton kosher salt")) | ||||
|     print(pre_process_string("¼ cup michiu tou or other rice wine")) | ||||
|     print(pre_process_string("1 tbs. wine, expensive or other white wine, plus more")) | ||||
|     print("Finished...") | ||||
|  | ||||
|  | ||||
| if __name__ == "__main__": | ||||
|     main() | ||||
|   | ||||
| @@ -12,6 +12,14 @@ CWD = Path(__file__).parent | ||||
| MODEL_PATH = CWD / "model.crfmodel" | ||||
|  | ||||
|  | ||||
| class CRFConfidence(BaseModel): | ||||
|     average: float = 0.0 | ||||
|     comment: float = None | ||||
|     name: float = None | ||||
|     unit: float = None | ||||
|     qty: float = None | ||||
|  | ||||
|  | ||||
| class CRFIngredient(BaseModel): | ||||
|     input: str = "" | ||||
|     name: str = "" | ||||
| @@ -19,15 +27,19 @@ class CRFIngredient(BaseModel): | ||||
|     qty: str = "" | ||||
|     comment: str = "" | ||||
|     unit: str = "" | ||||
|     confidence: CRFConfidence | ||||
|  | ||||
|     @validator("qty", always=True, pre=True) | ||||
|     def validate_qty(qty, values):  # sourcery skip: merge-nested-ifs | ||||
|         if qty is None or qty == "": | ||||
|             # Check if other contains a fraction | ||||
|             if values["other"] is not None and values["other"].find("/") != -1: | ||||
|                 return float(Fraction(values["other"])).__round__(1) | ||||
|             else: | ||||
|                 return 1 | ||||
|             try: | ||||
|                 if values["other"] is not None and values["other"].find("/") != -1: | ||||
|                     return float(Fraction(values["other"])).__round__(1) | ||||
|                 else: | ||||
|                     return 1 | ||||
|             except Exception: | ||||
|                 pass | ||||
|  | ||||
|         return qty | ||||
|  | ||||
|   | ||||
| @@ -1,4 +1,5 @@ | ||||
| import re | ||||
| from statistics import mean | ||||
|  | ||||
| from . import tokenizer | ||||
|  | ||||
| @@ -179,6 +180,9 @@ def import_data(lines): | ||||
|     data = [{}] | ||||
|     display = [[]] | ||||
|     prevTag = None | ||||
|  | ||||
|     confidence_all = [{}] | ||||
|  | ||||
|     # | ||||
|     # iterate lines in the data file, which looks like: | ||||
|     # | ||||
| @@ -208,6 +212,8 @@ def import_data(lines): | ||||
|             display.append([]) | ||||
|             prevTag = None | ||||
|  | ||||
|             confidence_all.append({}) | ||||
|  | ||||
|         # ignore comments | ||||
|         elif line[0] == "#": | ||||
|             pass | ||||
| @@ -226,6 +232,18 @@ def import_data(lines): | ||||
|             tag, confidence = re.split(r"/", columns[-1], 1) | ||||
|             tag = re.sub("^[BI]\-", "", tag).lower()  # noqa: W605 - invalid dscape sequence | ||||
|  | ||||
|             # ==================== | ||||
|             # Confidence Getter | ||||
|             if prevTag != tag: | ||||
|                 if confidence_all[-1].get(tag): | ||||
|                     confidence_all[-1][tag].append(confidence) | ||||
|                 else: | ||||
|                     confidence_all[-1][tag] = [confidence] | ||||
|             else: | ||||
|                 if confidence_all[-1].get(tag): | ||||
|                     confidence_all[-1][tag].append(confidence) | ||||
|                 else: | ||||
|                     confidence_all[-1][tag] = [confidence] | ||||
|             # ---- DISPLAY ---- | ||||
|             # build a structure which groups each token by its tag, so we can | ||||
|             # rebuild the original display name later. | ||||
| @@ -257,13 +275,23 @@ def import_data(lines): | ||||
|     output = [ | ||||
|         dict([(k, smartJoin(tokens)) for k, tokens in ingredient.items()]) for ingredient in data if len(ingredient) | ||||
|     ] | ||||
|     # Add the marked-up display data | ||||
|     for i, v in enumerate(output): | ||||
|         output[i]["display"] = displayIngredient(display[i]) | ||||
|  | ||||
|     # Preclean Confidence | ||||
|     for i, c in enumerate(confidence_all): | ||||
|         avg_of_all = [] | ||||
|         for k, v in c.items(): | ||||
|             v = [float(x) for x in v] | ||||
|             avg = round(mean(v), 2) | ||||
|             avg_of_all.append(avg) | ||||
|             confidence_all[i][k] = avg | ||||
|  | ||||
|         if avg_of_all: | ||||
|             confidence_all[i]["average"] = round(mean(avg_of_all), 2) | ||||
|  | ||||
|     # Add the raw ingredient phrase | ||||
|     for i, v in enumerate(output): | ||||
|         output[i]["input"] = smartJoin([" ".join(tokens) for k, tokens in display[i]]) | ||||
|     for i, _ in enumerate(output): | ||||
|         output[i]["input"] = smartJoin([" ".join(tokens) for _, tokens in display[i]]) | ||||
|         output[i]["confidence"] = confidence_all[i] | ||||
|  | ||||
|     return output | ||||
|  | ||||
|   | ||||
| @@ -3,9 +3,15 @@ from fractions import Fraction | ||||
|  | ||||
| from mealie.core.root_logger import get_logger | ||||
| from mealie.schema.recipe import RecipeIngredient | ||||
| from mealie.schema.recipe.recipe_ingredient import CreateIngredientFood, CreateIngredientUnit | ||||
| from mealie.schema.recipe.recipe_ingredient import ( | ||||
|     CreateIngredientFood, | ||||
|     CreateIngredientUnit, | ||||
|     IngredientConfidence, | ||||
|     ParsedIngredient, | ||||
|     RegisteredParser, | ||||
| ) | ||||
|  | ||||
| from .crfpp.processor import CRFIngredient, convert_list_to_crf_model | ||||
| from . import brute, crfpp | ||||
|  | ||||
| logger = get_logger(__name__) | ||||
|  | ||||
| @@ -15,12 +21,41 @@ class ABCIngredientParser(ABC): | ||||
|     Abstract class for ingredient parsers. | ||||
|     """ | ||||
|  | ||||
|     def parse_one(self, ingredient_string: str) -> ParsedIngredient: | ||||
|         pass | ||||
|  | ||||
|     @abstractmethod | ||||
|     def parse(self, ingredients: list[str]) -> list[RecipeIngredient]: | ||||
|     def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: | ||||
|         ... | ||||
|  | ||||
|  | ||||
| class CRFPPIngredientParser(ABCIngredientParser): | ||||
| class BruteForceParser(ABCIngredientParser): | ||||
|     """ | ||||
|     Brute force ingredient parser. | ||||
|     """ | ||||
|  | ||||
|     def __init__(self) -> None: | ||||
|         pass | ||||
|  | ||||
|     def parse_one(self, ingredient: str) -> ParsedIngredient: | ||||
|         bfi = brute.parse(ingredient) | ||||
|  | ||||
|         return ParsedIngredient( | ||||
|             input=ingredient, | ||||
|             ingredient=RecipeIngredient( | ||||
|                 unit=CreateIngredientUnit(name=bfi.unit), | ||||
|                 food=CreateIngredientFood(name=bfi.food), | ||||
|                 disable_amount=False, | ||||
|                 quantity=bfi.amount, | ||||
|                 note=bfi.note, | ||||
|             ), | ||||
|         ) | ||||
|  | ||||
|     def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: | ||||
|         return [self.parse_one(ingredient) for ingredient in ingredients] | ||||
|  | ||||
|  | ||||
| class NLPParser(ABCIngredientParser): | ||||
|     """ | ||||
|     Class for CRFPP ingredient parsers. | ||||
|     """ | ||||
| @@ -28,7 +63,7 @@ class CRFPPIngredientParser(ABCIngredientParser): | ||||
|     def __init__(self) -> None: | ||||
|         pass | ||||
|  | ||||
|     def _crf_to_ingredient(self, crf_model: CRFIngredient) -> RecipeIngredient: | ||||
|     def _crf_to_ingredient(self, crf_model: crfpp.CRFIngredient) -> ParsedIngredient: | ||||
|         ingredient = None | ||||
|  | ||||
|         try: | ||||
| @@ -41,15 +76,37 @@ class CRFPPIngredientParser(ABCIngredientParser): | ||||
|                 quantity=float(sum(Fraction(s) for s in crf_model.qty.split())), | ||||
|             ) | ||||
|         except Exception as e: | ||||
|             logger.error(f"Failed to parse ingredient: {crf_model}: {e}") | ||||
|             # TODO: Capture some sort of state for the user to see that an exception occured | ||||
|             logger.exception(e) | ||||
|             ingredient = RecipeIngredient( | ||||
|                 title="", | ||||
|                 note=crf_model.input, | ||||
|             ) | ||||
|  | ||||
|         return ingredient | ||||
|         return ParsedIngredient( | ||||
|             input=crf_model.input, | ||||
|             ingredient=ingredient, | ||||
|             confidence=IngredientConfidence( | ||||
|                 quantity=crf_model.confidence.qty, | ||||
|                 food=crf_model.confidence.name, | ||||
|                 **crf_model.confidence.dict(), | ||||
|             ), | ||||
|         ) | ||||
|  | ||||
|     def parse(self, ingredients: list[str]) -> list[RecipeIngredient]: | ||||
|         crf_models = convert_list_to_crf_model(ingredients) | ||||
|     def parse(self, ingredients: list[str]) -> list[ParsedIngredient]: | ||||
|         crf_models = crfpp.convert_list_to_crf_model(ingredients) | ||||
|         return [self._crf_to_ingredient(crf_model) for crf_model in crf_models] | ||||
|  | ||||
|  | ||||
| __registrar = { | ||||
|     RegisteredParser.nlp: NLPParser, | ||||
|     RegisteredParser.brute: BruteForceParser, | ||||
| } | ||||
|  | ||||
|  | ||||
| def get_parser(parser: RegisteredParser) -> ABCIngredientParser: | ||||
|     """ | ||||
|     get_parser returns an ingrdeint parser based on the string enum value | ||||
|     passed in. | ||||
|     """ | ||||
|     return __registrar.get(parser, NLPParser)() | ||||
|   | ||||
| @@ -1,14 +1,19 @@ | ||||
| from mealie.schema.recipe import RecipeIngredient | ||||
| from mealie.services._base_http_service.http_services import UserHttpService | ||||
|  | ||||
| from .ingredient_parser import ABCIngredientParser, CRFPPIngredientParser | ||||
| from .ingredient_parser import ABCIngredientParser, RegisteredParser, get_parser | ||||
|  | ||||
|  | ||||
| class IngredientParserService(UserHttpService): | ||||
|     def __init__(self, parser: ABCIngredientParser = None, *args, **kwargs) -> None: | ||||
|         self.parser: ABCIngredientParser = parser() if parser else CRFPPIngredientParser() | ||||
|     parser: ABCIngredientParser | ||||
|  | ||||
|     def __init__(self, parser: RegisteredParser = RegisteredParser.nlp, *args, **kwargs) -> None: | ||||
|         self.set_parser(parser) | ||||
|         super().__init__(*args, **kwargs) | ||||
|  | ||||
|     def set_parser(self, parser: RegisteredParser) -> None: | ||||
|         self.parser = get_parser(parser) | ||||
|  | ||||
|     def populate_item(self) -> None: | ||||
|         """Satisfy abstract method""" | ||||
|         pass | ||||
|   | ||||
| @@ -3,6 +3,7 @@ from fractions import Fraction | ||||
| 
 | ||||
| import pytest | ||||
| 
 | ||||
| from mealie.services.parser_services import RegisteredParser, get_parser | ||||
| from mealie.services.parser_services.crfpp.processor import CRFIngredient, convert_list_to_crf_model | ||||
| 
 | ||||
| 
 | ||||
| @@ -15,6 +16,12 @@ class TestIngredient: | ||||
|     comments: str | ||||
| 
 | ||||
| 
 | ||||
| def crf_exists() -> bool: | ||||
|     import shutil | ||||
| 
 | ||||
|     return shutil.which("crf_test") is not None | ||||
| 
 | ||||
| 
 | ||||
| # TODO - add more robust test cases | ||||
| test_ingredients = [ | ||||
|     TestIngredient("½ cup all-purpose flour", 0.5, "cup", "all-purpose flour", ""), | ||||
| @@ -24,12 +31,6 @@ test_ingredients = [ | ||||
| ] | ||||
| 
 | ||||
| 
 | ||||
| def crf_exists() -> bool: | ||||
|     import shutil | ||||
| 
 | ||||
|     return shutil.which("crf_test") is not None | ||||
| 
 | ||||
| 
 | ||||
| @pytest.mark.skipif(not crf_exists(), reason="CRF++ not installed") | ||||
| def test_nlp_parser(): | ||||
|     models: list[CRFIngredient] = convert_list_to_crf_model([x.input for x in test_ingredients]) | ||||
| @@ -41,3 +42,34 @@ def test_nlp_parser(): | ||||
|         assert model.comment == test_ingredient.comments | ||||
|         assert model.name == test_ingredient.food | ||||
|         assert model.unit == test_ingredient.unit | ||||
| 
 | ||||
| 
 | ||||
| def test_brute_parser(): | ||||
|     # input: (quantity, unit, food, comments) | ||||
|     expectations = { | ||||
|         # Dutch | ||||
|         "1 theelepel koffie": (1, "theelepel", "koffie", ""), | ||||
|         "3 theelepels koffie": (3, "theelepels", "koffie", ""), | ||||
|         "1 eetlepel tarwe": (1, "eetlepel", "tarwe", ""), | ||||
|         "20 eetlepels bloem": (20, "eetlepels", "bloem", ""), | ||||
|         "1 mespunt kaneel": (1, "mespunt", "kaneel", ""), | ||||
|         "1 snuf(je) zout": (1, "snuf(je)", "zout", ""), | ||||
|         "2 tbsp minced cilantro, leaves and stems": (2, "tbsp", "minced cilantro", "leaves and stems"), | ||||
|         "1 large yellow onion, coarsely chopped": (1, "large", "yellow onion", "coarsely chopped"), | ||||
|         "1 1/2 tsp garam masala": (1.5, "tsp", "garam masala", ""), | ||||
|         "2 cups mango chunks, (2 large mangoes) (fresh or frozen)": ( | ||||
|             2, | ||||
|             "cups", | ||||
|             "mango chunks, (2 large mangoes)", | ||||
|             "fresh or frozen", | ||||
|         ), | ||||
|     } | ||||
|     parser = get_parser(RegisteredParser.brute) | ||||
| 
 | ||||
|     for key, val in expectations.items(): | ||||
|         parsed = parser.parse_one(key) | ||||
| 
 | ||||
|         assert parsed.ingredient.quantity == val[0] | ||||
|         assert parsed.ingredient.unit.name == val[1] | ||||
|         assert parsed.ingredient.food.name == val[2] | ||||
|         assert parsed.ingredient.note in {val[3], None} | ||||
		Reference in New Issue
	
	Block a user