feat: Update food seeding logic to use new format, now with removed CrowdIn limits? (#5514)

Co-authored-by: Michael Genson <71845777+michael-genson@users.noreply.github.com>
Co-authored-by: Hayden <64056131+hay-kot@users.noreply.github.com>

mealie/repos/seed/resources/foods/locales/ar-SA.json

@@ -1,692 +1,62 @@
 {
-	"acorn-squash": {
-		"name": "acorn squash"
-	},
-	"alfalfa-sprouts": {
-		"name": "alfalfa sprouts"
-	},
-	"anchovies": {
-		"name": "anchovies"
-	},
-	"apples": {
-		"name": "apple",
-		"plural_name": "apples"
-	},
-	"artichoke": {
-		"name": "artichoke"
-	},
-	"arugula": {
-		"name": "arugula"
-	},
-	"asparagus": {
-		"name": "asparagus"
-	},
-	"avocado": {
-		"name": "avocado",
-		"plural_name": "avocado"
-	},
-	"bacon": {
-		"name": "bacon"
-	},
-	"baking-powder": {
-		"name": "baking powder"
-	},
-	"baking-soda": {
-		"name": "baking soda"
-	},
-	"baking-sugar": {
-		"name": "baking sugar"
-	},
-	"bar-sugar": {
-		"name": "bar sugar"
-	},
-	"basil": {
-		"name": "basil"
-	},
-	"beans": {
-		"name": "beans"
-	},
-	"bell-peppers": {
-		"name": "bell peppers",
-		"plural_name": "bell peppers"
-	},
-	"blackberries": {
-		"name": "blackberries"
-	},
-	"bok-choy": {
-		"name": "bok choy"
-	},
-	"brassicas": {
-		"name": "brassicas"
-	},
-	"bread": {
-		"name": "bread"
-	},
-	"breadfruit": {
-		"name": "breadfruit"
-	},
-	"broccoflower": {
-		"name": "broccoflower"
-	},
-	"broccoli": {
-		"name": "broccoli"
-	},
-	"broccoli-rabe": {
-		"name": "broccoli rabe"
-	},
-	"broccolini": {
-		"name": "broccolini"
-	},
-	"brown-sugar": {
-		"name": "brown sugar"
-	},
-	"brussels-sprouts": {
-		"name": "brussels sprouts"
-	},
-	"butter": {
-		"name": "butter"
-	},
-	"butternut-pumpkin": {
-		"name": "butternut pumpkin"
-	},
-	"butternut-squash": {
-		"name": "butternut squash"
-	},
-	"cabbage": {
-		"name": "cabbage",
-		"plural_name": "cabbages"
-	},
-	"cactus-edible": {
-		"name": "cactus, edible"
-	},
-	"calabrese": {
-		"name": "calabrese"
-	},
-	"cane-sugar": {
-		"name": "cane sugar"
-	},
-	"cannabis": {
-		"name": "cannabis"
-	},
-	"capsicum": {
-		"name": "capsicum"
-	},
-	"caraway": {
-		"name": "caraway"
-	},
-	"carrot": {
-		"name": "carrot",
-		"plural_name": "carrots"
-	},
-	"caster-sugar": {
-		"name": "caster sugar"
-	},
-	"castor-sugar": {
-		"name": "castor sugar"
-	},
-	"catfish": {
-		"name": "catfish"
-	},
-	"cauliflower": {
-		"name": "cauliflower",
-		"plural_name": "cauliflowers"
-	},
-	"cayenne-pepper": {
-		"name": "cayenne pepper"
-	},
-	"celeriac": {
-		"name": "celery root"
-	},
-	"celery": {
-		"name": "celery"
-	},
-	"cereal-grains": {
-		"name": "cereal grains"
-	},
-	"chard": {
-		"name": "chard"
-	},
-	"cheese": {
-		"name": "cheese"
-	},
-	"chicory": {
-		"name": "chicory"
-	},
-	"chilli-peppers": {
-		"name": "chilli pepper",
-		"plural_name": "chilli peppers"
-	},
-	"chinese-leaves": {
-		"name": "chinese leaves"
-	},
-	"chives": {
-		"name": "chives"
-	},
-	"chocolate": {
-		"name": "chocolate"
-	},
-	"cilantro": {
-		"name": "cilantro"
-	},
-	"cinnamon": {
-		"name": "cinnamon"
-	},
-	"clarified-butter": {
-		"name": "clarified butter"
-	},
-	"coconut": {
-		"name": "coconut",
-		"plural_name": "coconuts"
-	},
-	"coconut-milk": {
-		"name": "coconut milk"
-	},
-	"cod": {
-		"name": "cod"
-	},
-	"coffee": {
-		"name": "coffee"
-	},
-	"collard-greens": {
-		"name": "collard greens"
-	},
-	"confectioners-sugar": {
-		"name": "confectioners' sugar"
-	},
-	"coriander": {
-		"name": "coriander"
-	},
-	"corn": {
-		"name": "corn",
-		"plural_name": "corns"
-	},
-	"corn-syrup": {
-		"name": "corn syrup"
-	},
-	"cottonseed-oil": {
-		"name": "cottonseed oil"
-	},
-	"courgette": {
-		"name": "courgette"
-	},
-	"cream-of-tartar": {
-		"name": "cream of tartar"
-	},
-	"cucumber": {
-		"name": "cucumber",
-		"plural_name": "cucumbers"
-	},
-	"cumin": {
-		"name": "cumin"
-	},
-	"daikon": {
-		"name": "daikon",
-		"plural_name": "daikons"
-	},
-	"dairy-products-and-dairy-substitutes": {
-		"name": "dairy products and dairy substitutes"
-	},
-	"dandelion": {
-		"name": "dandelion"
-	},
-	"demerara-sugar": {
-		"name": "demerara sugar"
-	},
-	"dough": {
-		"name": "dough"
-	},
-	"edible-cactus": {
-		"name": "edible cactus"
-	},
-	"eggplant": {
-		"name": "eggplant",
-		"plural_name": "eggplants"
-	},
-	"eggs": {
-		"name": "egg",
-		"plural_name": "eggs"
-	},
-	"endive": {
-		"name": "endive",
-		"plural_name": "endives"
-	},
-	"fats": {
-		"name": "fats"
-	},
-	"fava-beans": {
-		"name": "fava beans"
-	},
-	"fiddlehead": {
-		"name": "fiddlehead"
-	},
-	"fiddlehead-fern": {
-		"name": "fiddlehead fern",
-		"plural_name": "fiddlehead ferns"
-	},
-	"fish": {
-		"name": "fish"
-	},
-	"five-spice-powder": {
-		"name": "five spice powder"
-	},
-	"flour": {
-		"name": "flour"
-	},
-	"frisee": {
-		"name": "frisee"
-	},
-	"fructose": {
-		"name": "fructose"
-	},
-	"fruit": {
-		"name": "fruit"
-	},
-	"fruit-sugar": {
-		"name": "fruit sugar"
-	},
-	"ful": {
-		"name": "ful"
-	},
-	"garam-masala": {
-		"name": "garam masala"
-	},
-	"garlic": {
-		"name": "garlic",
-		"plural_name": "garlics"
-	},
-	"gem-squash": {
-		"name": "gem squash"
-	},
-	"ghee": {
-		"name": "ghee"
-	},
-	"giblets": {
-		"name": "giblets"
-	},
-	"ginger": {
-		"name": "ginger"
-	},
-	"grains": {
-		"name": "grains"
-	},
-	"granulated-sugar": {
-		"name": "granulated sugar"
-	},
-	"grape-seed-oil": {
-		"name": "grape seed oil"
-	},
-	"green-onion": {
-		"name": "green onion",
-		"plural_name": "green onions"
-	},
-	"heart-of-palm": {
-		"name": "heart of palm",
-		"plural_name": "heart of palms"
-	},
-	"hemp": {
-		"name": "قنب والعياذ بالله"
-	},
-	"herbs": {
-		"name": "herbs"
-	},
-	"honey": {
-		"name": "honey"
-	},
-	"isomalt": {
-		"name": "isomalt"
-	},
-	"jackfruit": {
-		"name": "jackfruit",
-		"plural_name": "jackfruits"
-	},
-	"jaggery": {
-		"name": "jaggery"
-	},
-	"jams": {
-		"name": "jams"
-	},
-	"jellies": {
-		"name": "jellies"
-	},
-	"jerusalem-artichoke": {
-		"name": "jerusalem artichoke"
-	},
-	"jicama": {
-		"name": "jicama"
-	},
-	"kale": {
-		"name": "kale"
-	},
-	"kohlrabi": {
-		"name": "kohlrabi"
-	},
-	"kumara": {
-		"name": "kumara"
-	},
-	"leavening-agents": {
-		"name": "leavening agents"
-	},
-	"leek": {
-		"name": "leek",
-		"plural_name": "leeks"
-	},
-	"legumes": {
-		"name": "legumes"
-	},
-	"lemongrass": {
-		"name": "lemongrass"
-	},
-	"lentils": {
-		"name": "lentils"
-	},
-	"lettuce": {
-		"name": "lettuce"
-	},
-	"liver": {
-		"name": "liver",
-		"plural_name": "livers"
-	},
-	"maize": {
-		"name": "maize"
-	},
-	"maple-syrup": {
-		"name": "maple syrup"
-	},
-	"meat": {
-		"name": "meat"
-	},
-	"milk": {
-		"name": "milk"
-	},
-	"mortadella": {
-		"name": "mortadella"
-	},
-	"mushroom": {
-		"name": "mushroom",
-		"plural_name": "mushrooms"
-	},
-	"mussels": {
-		"name": "mussels"
-	},
-	"nanaimo-bar-mix": {
-		"name": "nanaimo bar mix"
-	},
-	"nori": {
-		"name": "nori"
-	},
-	"nutmeg": {
-		"name": "nutmeg"
-	},
-	"nutritional-yeast-flakes": {
-		"name": "nutritional yeast flakes"
-	},
-	"nuts": {
-		"name": "nuts"
-	},
-	"octopuses": {
-		"name": "octopus",
-		"plural_name": "octopuses"
-	},
-	"oils": {
-		"name": "oils"
-	},
-	"okra": {
-		"name": "okra"
-	},
-	"olive": {
-		"name": "olive"
-	},
-	"olive-oil": {
-		"name": "olive oil"
-	},
-	"onion": {
-		"name": "onion"
-	},
-	"onion-family": {
-		"name": "onion family"
-	},
-	"orange-blossom-water": {
-		"name": "orange blossom water"
-	},
-	"oranges": {
-		"name": "orange",
-		"plural_name": "oranges"
-	},
-	"oregano": {
-		"name": "oregano"
-	},
-	"oysters": {
-		"name": "oysters"
-	},
-	"panch-puran": {
-		"name": "panch puran"
-	},
-	"paprika": {
-		"name": "paprika"
-	},
-	"parsley": {
-		"name": "parsley"
-	},
-	"parsnip": {
-		"name": "parsnip",
-		"plural_name": "parsnips"
-	},
-	"pear": {
-		"name": "pear",
-		"plural_name": "pears"
-	},
-	"peas": {
-		"name": "peas"
-	},
-	"pepper": {
-		"name": "pepper",
-		"plural_name": "peppers"
-	},
-	"pineapple": {
-		"name": "pineapple",
-		"plural_name": "pineapples"
-	},
-	"plantain": {
-		"name": "plantain",
-		"plural_name": "plantains"
-	},
-	"poppy-seeds": {
-		"name": "poppy seeds"
-	},
-	"potato": {
-		"name": "potato",
-		"plural_name": "potatoes"
-	},
-	"poultry": {
-		"name": "poultry"
-	},
-	"powdered-sugar": {
-		"name": "powdered sugar"
-	},
-	"pumpkin": {
-		"name": "pumpkin",
-		"plural_name": "pumpkins"
-	},
-	"pumpkin-seeds": {
-		"name": "pumpkin seeds"
-	},
-	"radish": {
-		"name": "radish",
-		"plural_name": "radishes"
-	},
-	"raw-sugar": {
-		"name": "raw sugar"
-	},
-	"refined-sugar": {
-		"name": "refined sugar"
-	},
-	"rice": {
-		"name": "rice"
-	},
-	"rice-flour": {
-		"name": "rice flour"
-	},
-	"rock-sugar": {
-		"name": "rock sugar"
-	},
-	"rum": {
-		"name": "rum"
-	},
-	"salmon": {
-		"name": "salmon"
-	},
-	"salt": {
-		"name": "salt"
-	},
-	"salt-cod": {
-		"name": "salt cod"
-	},
-	"scallion": {
-		"name": "scallion",
-		"plural_name": "scallions"
-	},
-	"seafood": {
-		"name": "seafood"
-	},
-	"seeds": {
-		"name": "seeds"
-	},
-	"sesame-seeds": {
-		"name": "sesame seeds"
-	},
-	"shallot": {
-		"name": "shallot",
-		"plural_name": "shallots"
-	},
-	"skate": {
-		"name": "skate"
-	},
-	"soda": {
-		"name": "soda"
-	},
-	"soda-baking": {
-		"name": "soda, baking"
-	},
-	"soybean": {
-		"name": "soybean"
-	},
-	"spaghetti-squash": {
-		"name": "spaghetti squash",
-		"plural_name": "spaghetti squashes"
-	},
-	"speck": {
-		"name": "speck"
-	},
-	"spices": {
-		"name": "spices"
-	},
-	"spinach": {
-		"name": "spinach"
-	},
-	"spring-onion": {
-		"name": "spring onion",
-		"plural_name": "spring onions"
-	},
-	"squash": {
-		"name": "squash",
-		"plural_name": "squashes"
-	},
-	"squash-family": {
-		"name": "squash family"
-	},
-	"stockfish": {
-		"name": "stockfish"
-	},
-	"sugar": {
-		"name": "sugar"
-	},
-	"sunchoke": {
-		"name": "sunchoke",
-		"plural_name": "sunchokes"
-	},
-	"sunflower-seeds": {
-		"name": "sunflower seeds"
-	},
-	"superfine-sugar": {
-		"name": "superfine sugar"
-	},
-	"sweet-potato": {
-		"name": "sweet potato",
-		"plural_name": "sweet potatoes"
-	},
-	"sweetcorn": {
-		"name": "sweetcorn",
-		"plural_name": "sweetcorns"
-	},
-	"sweeteners": {
-		"name": "sweeteners"
-	},
-	"tahini": {
-		"name": "tahini"
-	},
-	"taro": {
-		"name": "taro",
-		"plural_name": "taroes"
-	},
-	"teff": {
-		"name": "teff"
-	},
-	"tomato": {
-		"name": "tomato",
-		"plural_name": "tomatoes"
-	},
-	"trout": {
-		"name": "trout"
-	},
-	"tubers": {
-		"name": "tuber",
-		"plural_name": "tubers"
-	},
-	"tuna": {
-		"name": "tuna"
-	},
-	"turbanado-sugar": {
-		"name": "turbanado sugar"
-	},
-	"turnip": {
-		"name": "turnip",
-		"plural_name": "turnips"
-	},
-	"unrefined-sugar": {
-		"name": "unrefined sugar"
-	},
-	"vanilla": {
-		"name": "vanilla"
-	},
-	"vegetables": {
-		"name": "vegetables"
-	},
-	"watercress": {
-		"name": "watercress"
-	},
-	"watermelon": {
-		"name": "watermelon",
-		"plural_name": "watermelons"
-	},
-	"white-mushroom": {
-		"name": "white mushroom",
-		"plural_name": "white mushrooms"
-	},
-	"white-sugar": {
-		"name": "white sugar"
-	},
-	"xanthan-gum": {
-		"name": "xanthan gum"
-	},
-	"yam": {
-		"name": "yam",
-		"plural_name": "yams"
-	},
-	"yeast": {
-		"name": "yeast"
-	},
-	"zucchini": {
-		"name": "zucchini",
-		"plural_name": "zucchinis"
-	}
-}
+    "": {
+        "foods": {}
+    },
+    "خَضْراوات وفواكه": {
+        "foods": {}
+    },
+    "الحبوب": {
+        "foods": {}
+    },
+    "آلفواكه": {
+        "foods": {}
+    },
+    "الخضراوات": {
+        "foods": {}
+    },
+    "اللحوم": {
+        "foods": {}
+    },
+    "المأكولات البحرية": {
+        "foods": {}
+    },
+    "المشروبات": {
+        "foods": {}
+    },
+    "المخبوزات": {
+        "foods": {}
+    },
+    "المعلبات": {
+        "foods": {}
+    },
+    "الباهرات": {
+        "foods": {}
+    },
+    "الحَلْوَيَات": {
+        "foods": {}
+    },
+    "منتجات الألبان": {
+        "foods": {}
+    },
+    "الأطعمة المجمدة": {
+        "foods": {}
+    },
+    "الأغذية الصحية": {
+        "foods": {}
+    },
+    "المنزل": {
+        "foods": {}
+    },
+    "منتجات اللحوم": {
+        "foods": {}
+    },
+    "الوجبات الخفيفة": {
+        "foods": {}
+    },
+    "التوابل": {
+        "foods": {}
+    },
+    "أخرى": {
+        "foods": {}
+    }
+}

mealie/repos/seed/seeders.py

@@ -3,12 +3,17 @@ import pathlib
 from collections.abc import Generator
 from functools import cached_property
 
-from mealie.schema.labels import MultiPurposeLabelSave
-from mealie.schema.recipe.recipe_ingredient import SaveIngredientFood, SaveIngredientUnit
+from mealie.schema.labels import MultiPurposeLabelOut, MultiPurposeLabelSave
+from mealie.schema.recipe.recipe_ingredient import (
+    IngredientFood,
+    IngredientUnit,
+    SaveIngredientFood,
+    SaveIngredientUnit,
+)
 from mealie.services.group_services.labels_service import MultiPurposeLabelService
 
 from ._abstract_seeder import AbstractSeeder
-from .resources import foods, labels, units
+from .resources import foods, units
 
 
 class MultiPurposeLabelSeeder(AbstractSeeder):
@@ -17,20 +22,24 @@ class MultiPurposeLabelSeeder(AbstractSeeder):
         return MultiPurposeLabelService(self.repos)
 
     def get_file(self, locale: str | None = None) -> pathlib.Path:
-        locale_path = self.resources / "labels" / "locales" / f"{locale}.json"
-        return locale_path if locale_path.exists() else labels.en_US
+        # Get the labels from the foods seed file now
+        locale_path = self.resources / "foods" / "locales" / f"{locale}.json"
+        return locale_path if locale_path.exists() else foods.en_US
+
+    def get_all_labels(self) -> list[MultiPurposeLabelOut]:
+        return self.repos.group_multi_purpose_labels.get_all()
 
     def load_data(self, locale: str | None = None) -> Generator[MultiPurposeLabelSave, None, None]:
         file = self.get_file(locale)
 
-        seen_label_names = set()
-        for label in json.loads(file.read_text(encoding="utf-8")):
-            if label["name"] in seen_label_names:
-                continue
-
-            seen_label_names.add(label["name"])
+        current_label_names = {label.name for label in self.get_all_labels()}
+        # load from the foods locale file and remove any empty strings
+        seed_label_names = set(filter(None, json.loads(file.read_text(encoding="utf-8")).keys()))  # type: set[str]
+        # only seed new labels
+        to_seed_labels = seed_label_names - current_label_names
+        for label in to_seed_labels:
             yield MultiPurposeLabelSave(
-                name=label["name"],
+                name=label,
                 group_id=self.repos.group_id,
             )
 
@@ -48,10 +57,13 @@ class IngredientUnitsSeeder(AbstractSeeder):
         locale_path = self.resources / "units" / "locales" / f"{locale}.json"
         return locale_path if locale_path.exists() else units.en_US
 
+    def get_all_units(self) -> list[IngredientUnit]:
+        return self.repos.ingredient_units.get_all()
+
     def load_data(self, locale: str | None = None) -> Generator[SaveIngredientUnit, None, None]:
         file = self.get_file(locale)
 
-        seen_unit_names = set()
+        seen_unit_names = {unit.name for unit in self.get_all_units()}
         for unit in json.loads(file.read_text(encoding="utf-8")).values():
             if unit["name"] in seen_unit_names:
                 continue
@@ -80,21 +92,32 @@ class IngredientFoodsSeeder(AbstractSeeder):
         locale_path = self.resources / "foods" / "locales" / f"{locale}.json"
         return locale_path if locale_path.exists() else foods.en_US
 
+    def get_label(self, value: str) -> MultiPurposeLabelOut | None:
+        return self.repos.group_multi_purpose_labels.get_one(value, "name")
+
+    def get_all_foods(self) -> list[IngredientFood]:
+        return self.repos.ingredient_foods.get_all()
+
     def load_data(self, locale: str | None = None) -> Generator[SaveIngredientFood, None, None]:
         file = self.get_file(locale)
 
-        seed_foods_names = set()
-        for food in json.loads(file.read_text(encoding="utf-8")).values():
-            if food["name"] in seed_foods_names:
-                continue
+        # get all current unique foods
+        seen_foods_names = {food.name for food in self.get_all_foods()}
+        for label, values in json.loads(file.read_text(encoding="utf-8")).items():
+            label_out = self.get_label(label)
 
-            seed_foods_names.add(food["name"])
-            yield SaveIngredientFood(
-                group_id=self.repos.group_id,
-                name=food["name"],
-                plural_name=food.get("plural_name"),
-                description="",
-            )
+            for food_name, attributes in values["foods"].items():
+                if food_name in seen_foods_names:
+                    continue
+
+                seen_foods_names.add(food_name)
+                yield SaveIngredientFood(
+                    group_id=self.repos.group_id,
+                    name=attributes["name"],
+                    plural_name=attributes.get("plural_name"),
+                    description="",  # description expected to be empty string by UnitFoodBase class
+                    label_id=label_out.id if label_out and label_out.id else None,
+                )
 
     def seed(self, locale: str | None = None) -> None:
         self.logger.info("Seeding Ingredient Foods")