mirror of
https://github.com/mealie-recipes/mealie.git
synced 2025-10-31 10:13:32 -04:00
feat (WIP): bring png OCR scanning support (#1670)
* Add pytesseract * Add simple ocr endpoint replace extension argument * feat/ocr-editor gui * fix frontend linting issues * Add service unit tests * Add split text modes & single ingredient/instruction editing * make split mode really reactive * Remove default step and ingredient * make the linter haappy * Accept only image uploads * Add automatic recipe title suggestion * Correct regex * fix incorrect array.map method usage * make the linter happy again * Swap route to use asset name * Rearange buttons * fix test data * feat: Allow making image the recipe image * Add translation * Make the linter happy * Restrict function setPropertyValueByPath generic * Restrict template literal type * Add a more friendly icon to creation page * update poetry lock file * Correct sloppy ocr classes * Make MyPy happy * Rewrite safer tests * Add tesseract to backend test CI container dependencies * Make canvas element a component global * Remove unwanted spaces in selected text * Add way to know if recipe was created with ocr * Access to ocr-editor for ocr recipes * Update Alembic revision * Make the frontend build * Fix scrolling offset bug * Allow creation of recipes with custom settings * Fix rebasing mistakes * Add format_tsv_output test * Exclude the tests data directory only * Enforce camelCase for frontend functions * Remove import of unused component * Fix type and class initialization * Add multi-language support * Highlight words in mount * Fix image ratio bug * Better ocr creation page * Revert awkward feature to scroll in Selection mode * Rebasing alembic migrations sux * Remove obsolete getShared function * Add function docstring * Move down ocr creation option * Make toolbar icons more generic * Show help at the bottom of the page * move ocr types to own file * Use template ref for the canvas * Use i18n.tc to get strings directly * Correct naming mistake * Move Ocr editor to own directory * Create Ocr Editor parts * Safeguard recipe properties access * Add loading frontend animation due to longer request time * minor cleanup chores Co-authored-by: Miroito <alban.vachette@gmail.com>
This commit is contained in:
Hayden
GitHub
2
.github/workflows/partial-backend.yml
vendored
2
.github/workflows/partial-backend.yml
vendored
@@ -62,7 +62,7 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install libsasl2-dev libldap2-dev libssl-dev
|
||||
sudo apt-get install libsasl2-dev libldap2-dev libssl-dev tesseract-ocr-all
|
||||
poetry install
|
||||
poetry add "psycopg2-binary==2.8.6"
|
||||
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
|
||||
|
||||
@@ -9,6 +9,7 @@ repos:
|
||||
- id: check-toml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
exclude: ^tests/data/
|
||||
- repo: https://github.com/sondrelg/pep585-upgrade
|
||||
rev: "v1.0.1" # Use the sha / tag you want to point at
|
||||
hooks:
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Add is_ocr_recipe column to recipes
|
||||
|
||||
Revision ID: 089bfa50d0ed
|
||||
Revises: f30cf048c228
|
||||
Create Date: 2022-08-05 17:07:07.389271
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "089bfa50d0ed"
|
||||
down_revision = "188374910655"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column("recipes", sa.Column("is_ocr_recipe", sa.Boolean(), default=False, nullable=True))
|
||||
op.execute("UPDATE recipes SET is_ocr_recipe = FALSE")
|
||||
# SQLITE does not support ALTER COLUMN, so the column will stay nullable to prevent making this migration a mess
|
||||
# The Recipe pydantic model and the SQL server use False as default value anyway for this column so Null should be a very rare sight
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("recipes", "is_ocr_recipe")
|
||||
18
frontend/api/class-interfaces/ocr.ts
Normal file
18
frontend/api/class-interfaces/ocr.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { BaseAPI } from "~/api/_base";
|
||||
|
||||
const prefix = "/api";
|
||||
|
||||
export class OcrAPI extends BaseAPI {
|
||||
|
||||
// Currently unused in favor for the endpoint using asset names
|
||||
async fileToTsv(file: File) {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
return await this.requests.post(`${prefix}/ocr/file-to-tsv`, formData);
|
||||
}
|
||||
|
||||
async assetToTsv(recipeSlug: string, assetName: string) {
|
||||
return await this.requests.post(`${prefix}/ocr/asset-to-tsv`, { recipeSlug, assetName });
|
||||
}
|
||||
|
||||
}
|
||||
@@ -34,6 +34,7 @@ const routes = {
|
||||
recipesCategory: `${prefix}/recipes/category`,
|
||||
recipesParseIngredient: `${prefix}/parser/ingredient`,
|
||||
recipesParseIngredients: `${prefix}/parser/ingredients`,
|
||||
recipesCreateFromOcr: `${prefix}/recipes/create-ocr`,
|
||||
|
||||
recipesRecipeSlug: (recipe_slug: string) => `${prefix}/recipes/${recipe_slug}`,
|
||||
recipesRecipeSlugExport: (recipe_slug: string) => `${prefix}/recipes/${recipe_slug}/exports`,
|
||||
@@ -116,4 +117,13 @@ export class RecipeAPI extends BaseCRUDAPI<CreateRecipe, Recipe, Recipe> {
|
||||
getZipRedirectUrl(recipeSlug: string, token: string) {
|
||||
return `${routes.recipesRecipeSlugExportZip(recipeSlug)}?token=${token}`;
|
||||
}
|
||||
|
||||
async createFromOcr(file: File, makeFileRecipeImage: boolean) {
|
||||
const formData = new FormData();
|
||||
formData.append("file", file);
|
||||
formData.append("extension", file.name.split(".").pop() ?? "");
|
||||
formData.append("makefilerecipeimage", String(makeFileRecipeImage));
|
||||
|
||||
return await this.requests.post(routes.recipesCreateFromOcr, formData);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import { MultiPurposeLabelsApi } from "./class-interfaces/group-multiple-purpose
|
||||
import { GroupEventNotifierApi } from "./class-interfaces/group-event-notifier";
|
||||
import { MealPlanRulesApi } from "./class-interfaces/group-mealplan-rules";
|
||||
import { GroupDataSeederApi } from "./class-interfaces/group-seeder";
|
||||
import {OcrAPI} from "./class-interfaces/ocr";
|
||||
import { ApiRequestInstance } from "~/types/api";
|
||||
|
||||
class Api {
|
||||
@@ -52,6 +53,7 @@ class Api {
|
||||
public groupEventNotifier: GroupEventNotifierApi;
|
||||
public upload: UploadFile;
|
||||
public seeders: GroupDataSeederApi;
|
||||
public ocr: OcrAPI;
|
||||
|
||||
constructor(requests: ApiRequestInstance) {
|
||||
// Recipes
|
||||
@@ -90,6 +92,9 @@ class Api {
|
||||
this.bulk = new BulkActionsAPI(requests);
|
||||
this.groupEventNotifier = new GroupEventNotifierApi(requests);
|
||||
|
||||
// ocr
|
||||
this.ocr = new OcrAPI(requests);
|
||||
|
||||
Object.freeze(this);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,7 @@ const SAVE_EVENT = "save";
|
||||
const DELETE_EVENT = "delete";
|
||||
const CLOSE_EVENT = "close";
|
||||
const JSON_EVENT = "json";
|
||||
const OCR_EVENT = "ocr";
|
||||
|
||||
export default defineComponent({
|
||||
components: { RecipeContextMenu, RecipeFavoriteBadge },
|
||||
@@ -122,8 +123,12 @@ export default defineComponent({
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
showOcrButton: {
|
||||
type: Boolean,
|
||||
default: false,
|
||||
},
|
||||
setup(_, context) {
|
||||
},
|
||||
setup(props, context) {
|
||||
const deleteDialog = ref(false);
|
||||
|
||||
const { i18n, $globals } = useContext();
|
||||
@@ -154,22 +159,26 @@ export default defineComponent({
|
||||
},
|
||||
];
|
||||
|
||||
if (props.showOcrButton) {
|
||||
editorButtons.splice(2, 0, {
|
||||
text: i18n.t("ocr-editor.ocr-editor"),
|
||||
icon: $globals.icons.eye,
|
||||
event: OCR_EVENT,
|
||||
color: "accent",
|
||||
});
|
||||
}
|
||||
|
||||
function emitHandler(event: string) {
|
||||
switch (event) {
|
||||
case CLOSE_EVENT:
|
||||
context.emit(CLOSE_EVENT);
|
||||
context.emit("input", false);
|
||||
break;
|
||||
case SAVE_EVENT:
|
||||
context.emit(SAVE_EVENT);
|
||||
break;
|
||||
case JSON_EVENT:
|
||||
context.emit(JSON_EVENT);
|
||||
break;
|
||||
case DELETE_EVENT:
|
||||
deleteDialog.value = true;
|
||||
break;
|
||||
default:
|
||||
context.emit(event);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
<div class="text-center">
|
||||
<v-dialog v-model="dialog" width="800">
|
||||
<template #activator="{ on, attrs }">
|
||||
<BaseButton v-bind="attrs" v-on="on" @click="inputText = ''">
|
||||
<BaseButton v-bind="attrs" v-on="on" @click="inputText = inputTextProp">
|
||||
{{ $t("new-recipe.bulk-add") }}
|
||||
</BaseButton>
|
||||
</template>
|
||||
@@ -58,10 +58,17 @@
|
||||
<script lang="ts">
|
||||
import { reactive, toRefs, defineComponent, useContext } from "@nuxtjs/composition-api";
|
||||
export default defineComponent({
|
||||
setup(_, context) {
|
||||
props: {
|
||||
inputTextProp: {
|
||||
type: String,
|
||||
required: false,
|
||||
default: "",
|
||||
},
|
||||
},
|
||||
setup(props, context) {
|
||||
const state = reactive({
|
||||
dialog: false,
|
||||
inputText: "",
|
||||
inputText: props.inputTextProp,
|
||||
});
|
||||
|
||||
function splitText() {
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
class="mx-1 mt-3 mb-4"
|
||||
:placeholder="$t('recipe.section-title')"
|
||||
style="max-width: 500px"
|
||||
@click="$emit('clickIngredientField', 'title')"
|
||||
>
|
||||
</v-text-field>
|
||||
<v-row :no-gutters="$vuetify.breakpoint.mdAndUp" dense class="d-flex flex-wrap my-1">
|
||||
@@ -81,7 +82,15 @@
|
||||
</v-col>
|
||||
<v-col sm="12" md="" cols="12">
|
||||
<div class="d-flex">
|
||||
<v-text-field v-model="value.note" hide-details dense solo class="mx-1" :placeholder="$t('recipe.notes')">
|
||||
<v-text-field
|
||||
v-model="value.note"
|
||||
hide-details
|
||||
dense
|
||||
solo
|
||||
class="mx-1"
|
||||
:placeholder="$t('recipe.notes')"
|
||||
@click="$emit('clickIngredientField', 'note')"
|
||||
>
|
||||
<v-icon v-if="disableAmount && $listeners && $listeners.delete" slot="prepend" class="mr-n1 handle">
|
||||
{{ $globals.icons.arrowUpDown }}
|
||||
</v-icon>
|
||||
@@ -93,12 +102,12 @@
|
||||
:buttons="[
|
||||
{
|
||||
icon: $globals.icons.delete,
|
||||
text: $t('general.delete'),
|
||||
text: $tc('general.delete'),
|
||||
event: 'delete',
|
||||
},
|
||||
{
|
||||
icon: $globals.icons.dotsVertical,
|
||||
text: $t('general.menu'),
|
||||
text: $tc('general.menu'),
|
||||
event: 'open',
|
||||
children: contextMenuOptions,
|
||||
},
|
||||
|
||||
@@ -176,6 +176,7 @@
|
||||
blur: imageUploadMode,
|
||||
}"
|
||||
@drop.stop.prevent="handleImageDrop(index, $event)"
|
||||
@click="$emit('clickInstructionField', `${index}.text`)"
|
||||
>
|
||||
<MarkdownEditor
|
||||
v-model="value[index]['text']"
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
<template>
|
||||
<v-container
|
||||
v-if="recipe && recipe.slug && recipe.settings && recipe.recipeIngredient"
|
||||
:class="{
|
||||
'pa-0': $vuetify.breakpoint.smAndDown,
|
||||
}"
|
||||
>
|
||||
<BannerExperimental />
|
||||
|
||||
<div v-if="loading">
|
||||
<v-spacer />
|
||||
<v-progress-circular indeterminate class="" color="primary"> </v-progress-circular>
|
||||
{{ loadingText }}
|
||||
<v-spacer />
|
||||
</div>
|
||||
<v-row v-if="!loading">
|
||||
<v-col cols="12" sm="7" md="7" lg="7">
|
||||
<RecipeOcrEditorPageCanvas
|
||||
:image="canvasImage"
|
||||
:tsv="tsv"
|
||||
@setText="canvasSetText"
|
||||
@update-recipe="updateRecipe"
|
||||
@close-editor="closeEditor"
|
||||
@text-selected="updateSelectedText"
|
||||
>
|
||||
</RecipeOcrEditorPageCanvas>
|
||||
|
||||
<RecipeOcrEditorPageHelp />
|
||||
</v-col>
|
||||
<v-col cols="12" sm="5" md="5" lg="5">
|
||||
<v-tabs v-model="tab" fixed-tabs>
|
||||
<v-tab key="header">
|
||||
{{ $t("general.recipe") }}
|
||||
</v-tab>
|
||||
<v-tab key="ingredients">
|
||||
{{ $t("recipe.ingredients") }}
|
||||
</v-tab>
|
||||
<v-tab key="instructions">
|
||||
{{ $t("recipe.instructions") }}
|
||||
</v-tab>
|
||||
</v-tabs>
|
||||
<v-tabs-items v-model="tab">
|
||||
<v-tab-item key="header">
|
||||
<v-text-field
|
||||
v-model="recipe.name"
|
||||
class="my-3"
|
||||
:label="$t('recipe.recipe-name')"
|
||||
:rules="[validators.required]"
|
||||
@focus="selectedRecipeField = 'name'"
|
||||
>
|
||||
</v-text-field>
|
||||
|
||||
<div class="d-flex flex-wrap">
|
||||
<v-text-field
|
||||
v-model="recipe.totalTime"
|
||||
class="mx-2"
|
||||
:label="$t('recipe.total-time')"
|
||||
@click="selectedRecipeField = 'totalTime'"
|
||||
></v-text-field>
|
||||
<v-text-field
|
||||
v-model="recipe.prepTime"
|
||||
class="mx-2"
|
||||
:label="$t('recipe.prep-time')"
|
||||
@click="selectedRecipeField = 'prepTime'"
|
||||
></v-text-field>
|
||||
<v-text-field
|
||||
v-model="recipe.performTime"
|
||||
class="mx-2"
|
||||
:label="$t('recipe.perform-time')"
|
||||
@click="selectedRecipeField = 'performTime'"
|
||||
></v-text-field>
|
||||
</div>
|
||||
|
||||
<v-textarea
|
||||
v-model="recipe.description"
|
||||
auto-grow
|
||||
min-height="100"
|
||||
:label="$t('recipe.description')"
|
||||
@click="selectedRecipeField = 'description'"
|
||||
>
|
||||
</v-textarea>
|
||||
<v-text-field
|
||||
v-model="recipe.recipeYield"
|
||||
dense
|
||||
:label="$t('recipe.servings')"
|
||||
@click="selectedRecipeField = 'recipeYield'"
|
||||
>
|
||||
</v-text-field>
|
||||
</v-tab-item>
|
||||
<v-tab-item key="ingredients">
|
||||
<div class="d-flex justify-end mt-2">
|
||||
<RecipeDialogBulkAdd class="ml-1 mr-1" :input-text-prop="canvasSelectedText" @bulk-data="addIngredient" />
|
||||
<BaseButton @click="addIngredient"> {{ $t("general.new") }} </BaseButton>
|
||||
</div>
|
||||
<draggable
|
||||
v-if="recipe.recipeIngredient.length > 0"
|
||||
v-model="recipe.recipeIngredient"
|
||||
handle=".handle"
|
||||
v-bind="{
|
||||
animation: 200,
|
||||
group: 'description',
|
||||
disabled: false,
|
||||
ghostClass: 'ghost',
|
||||
}"
|
||||
@start="drag = true"
|
||||
@end="drag = false"
|
||||
>
|
||||
<TransitionGroup type="transition" :name="!drag ? 'flip-list' : ''">
|
||||
<RecipeIngredientEditor
|
||||
v-for="(ingredient, index) in recipe.recipeIngredient"
|
||||
:key="ingredient.referenceId"
|
||||
v-model="recipe.recipeIngredient[index]"
|
||||
class="list-group-item"
|
||||
:disable-amount="recipe.settings.disableAmount"
|
||||
@delete="recipe.recipeIngredient.splice(index, 1)"
|
||||
@clickIngredientField="setSingleIngredient($event, index)"
|
||||
/>
|
||||
</TransitionGroup>
|
||||
</draggable>
|
||||
</v-tab-item>
|
||||
<v-tab-item key="instructions">
|
||||
<div class="d-flex justify-end mt-2">
|
||||
<RecipeDialogBulkAdd class="ml-1 mr-1" :input-text-prop="canvasSelectedText" @bulk-data="addStep" />
|
||||
<BaseButton @click="addStep()"> {{ $t("general.new") }}</BaseButton>
|
||||
</div>
|
||||
<RecipeInstructions
|
||||
v-model="recipe.recipeInstructions"
|
||||
:ingredients="recipe.recipeIngredient"
|
||||
:disable-amount="recipe.settings.disableAmount"
|
||||
:edit="true"
|
||||
:recipe-id="recipe.id"
|
||||
:recipe-slug="recipe.slug"
|
||||
:assets.sync="recipe.assets"
|
||||
@clickInstructionField="setSingleStep"
|
||||
/>
|
||||
</v-tab-item>
|
||||
</v-tabs-items>
|
||||
</v-col>
|
||||
</v-row>
|
||||
</v-container>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent, ref, onMounted, reactive, toRefs, useRouter } from "@nuxtjs/composition-api";
|
||||
import { until } from "@vueuse/core";
|
||||
import { invoke } from "@vueuse/shared";
|
||||
import draggable from "vuedraggable";
|
||||
import { useUserApi, useStaticRoutes } from "~/composables/api";
|
||||
import { OcrTsvResponse } from "~/types/api-types/ocr";
|
||||
import { validators } from "~/composables/use-validators";
|
||||
import { Recipe, RecipeIngredient, RecipeStep } from "~/types/api-types/recipe";
|
||||
import { Paths, Leaves, SelectedRecipeLeaves } from "~/types/ocr-types";
|
||||
import BannerExperimental from "~/components/global/BannerExperimental.vue";
|
||||
import RecipeDialogBulkAdd from "~/components/Domain/Recipe/RecipeDialogBulkAdd.vue";
|
||||
import RecipeInstructions from "~/components/Domain/Recipe/RecipeInstructions.vue";
|
||||
import RecipeIngredientEditor from "~/components/Domain/Recipe/RecipeIngredientEditor.vue";
|
||||
import RecipeOcrEditorPageCanvas from "~/components/Domain/Recipe/RecipeOcrEditorPage/RecipeOcrEditorPageParts/RecipeOcrEditorPageCanvas.vue";
|
||||
import RecipeOcrEditorPageHelp from "~/components/Domain/Recipe/RecipeOcrEditorPage/RecipeOcrEditorPageParts/RecipeOcrEditorPageHelp.vue";
|
||||
import { uuid4 } from "~/composables/use-utils";
|
||||
import { NoUndefinedField } from "~/types/api";
|
||||
|
||||
export default defineComponent({
|
||||
components: {
|
||||
RecipeIngredientEditor,
|
||||
draggable,
|
||||
BannerExperimental,
|
||||
RecipeDialogBulkAdd,
|
||||
RecipeInstructions,
|
||||
RecipeOcrEditorPageCanvas,
|
||||
RecipeOcrEditorPageHelp,
|
||||
},
|
||||
props: {
|
||||
recipe: {
|
||||
type: Object as () => NoUndefinedField<Recipe>,
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
setup(props) {
|
||||
const router = useRouter();
|
||||
const api = useUserApi();
|
||||
|
||||
const tsv = ref<OcrTsvResponse[]>([]);
|
||||
|
||||
const drag = ref(false);
|
||||
|
||||
const { recipeAssetPath } = useStaticRoutes();
|
||||
|
||||
function assetURL(assetName: string) {
|
||||
return recipeAssetPath(props.recipe.id, assetName);
|
||||
}
|
||||
|
||||
const state = reactive({
|
||||
loading: true,
|
||||
loadingText: "Loading recipe...",
|
||||
tab: null,
|
||||
selectedRecipeField: "" as SelectedRecipeLeaves | "",
|
||||
canvasSelectedText: "",
|
||||
canvasImage: new Image(),
|
||||
});
|
||||
|
||||
const setPropertyValueByPath = function <T extends Recipe>(object: T, path: Paths<T>, value: any) {
|
||||
const a = path.split(".");
|
||||
let nextProperty: any = object;
|
||||
for (let i = 0, n = a.length - 1; i < n; ++i) {
|
||||
const k = a[i];
|
||||
if (k in nextProperty) {
|
||||
nextProperty = nextProperty[k];
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
}
|
||||
nextProperty[a[a.length - 1]] = value;
|
||||
};
|
||||
|
||||
/**
|
||||
* This function will find the title of a recipe with the assumption that the title
|
||||
* has the biggest ratio of surface area / number of words on the image.
|
||||
* @return Returns the text parts of the block with the highest score.
|
||||
*/
|
||||
function findRecipeTitle() {
|
||||
const filtered = tsv.value.filter((element) => element.level === 2 || element.level === 5);
|
||||
const blocks = [[]] as OcrTsvResponse[][];
|
||||
let blockNum = 1;
|
||||
filtered.forEach((element, index, array) => {
|
||||
if (index !== 0 && array[index - 1].blockNum !== element.blockNum) {
|
||||
blocks.push([]);
|
||||
blockNum = element.blockNum;
|
||||
}
|
||||
blocks[blockNum - 1].push(element);
|
||||
});
|
||||
|
||||
let bestScore = 0;
|
||||
let bestBlock = blocks[0];
|
||||
blocks.forEach((element) => {
|
||||
// element[0] is the block declaration line containing the blocks total dimensions
|
||||
// element.length is the number of words (+ 2) contained in that block
|
||||
const elementScore = (element[0].height * element[0].width) / element.length; // Prettier is adding useless parenthesis for a mysterious reason
|
||||
const elementText = element.map((element) => element.text).join(""); // Identify empty blocks and don't count them
|
||||
if (elementScore > bestScore && elementText !== "") {
|
||||
bestBlock = element;
|
||||
bestScore = elementScore;
|
||||
}
|
||||
});
|
||||
|
||||
return bestBlock
|
||||
.filter((element) => element.level === 5 && element.conf >= 40)
|
||||
.map((element) => {
|
||||
return element.text.trim();
|
||||
})
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
invoke(async () => {
|
||||
await until(props.recipe).not.toBeNull();
|
||||
state.loadingText = "Loading OCR data...";
|
||||
|
||||
const assetName = props.recipe.assets[0].fileName;
|
||||
const imagesrc = assetURL(assetName);
|
||||
state.canvasImage.src = imagesrc;
|
||||
|
||||
const res = await api.ocr.assetToTsv(props.recipe.slug, assetName);
|
||||
tsv.value = res.data as OcrTsvResponse[];
|
||||
state.loading = false;
|
||||
|
||||
if (props.recipe.name.match(/New\sOCR\sRecipe(\s\([0-9]+\))?/g)) {
|
||||
props.recipe.name = findRecipeTitle();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
function addIngredient(ingredients: Array<string> | null = null) {
|
||||
if (ingredients?.length) {
|
||||
const newIngredients = ingredients.map((x) => {
|
||||
return {
|
||||
referenceId: uuid4(),
|
||||
title: "",
|
||||
note: x,
|
||||
unit: undefined,
|
||||
food: undefined,
|
||||
disableAmount: true,
|
||||
quantity: 1,
|
||||
originalText: "",
|
||||
};
|
||||
});
|
||||
|
||||
if (newIngredients) {
|
||||
// @ts-expect-error - prop can be null-type by NoUndefinedField type forces it to be set
|
||||
props.recipe.recipeIngredient.push(...newIngredients);
|
||||
}
|
||||
} else {
|
||||
props.recipe.recipeIngredient.push({
|
||||
referenceId: uuid4(),
|
||||
title: "",
|
||||
note: "",
|
||||
// @ts-expect-error - prop can be null-type by NoUndefinedField type forces it to be set
|
||||
unit: undefined,
|
||||
// @ts-expect-error - prop can be null-type by NoUndefinedField type forces it to be set
|
||||
food: undefined,
|
||||
disableAmount: true,
|
||||
quantity: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function addStep(steps: Array<string> | null = null) {
|
||||
if (!props.recipe.recipeInstructions) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (steps) {
|
||||
const cleanedSteps = steps.map((step) => {
|
||||
return { id: uuid4(), text: step, title: "", ingredientReferences: [] };
|
||||
});
|
||||
|
||||
props.recipe.recipeInstructions.push(...cleanedSteps);
|
||||
} else {
|
||||
props.recipe.recipeInstructions.push({ id: uuid4(), text: "", title: "", ingredientReferences: [] });
|
||||
}
|
||||
}
|
||||
|
||||
// EVENT HANDLERS
|
||||
|
||||
// Canvas component event handlers
|
||||
async function updateRecipe() {
|
||||
const { data } = await api.recipes.updateOne(props.recipe.slug, props.recipe);
|
||||
if (data?.slug) {
|
||||
router.push("/recipe/" + data.slug);
|
||||
}
|
||||
}
|
||||
|
||||
function closeEditor() {
|
||||
router.push("/recipe/" + props.recipe.slug);
|
||||
}
|
||||
|
||||
const canvasSetText = function () {
|
||||
if (state.selectedRecipeField !== "") {
|
||||
setPropertyValueByPath<Recipe>(props.recipe, state.selectedRecipeField, state.canvasSelectedText);
|
||||
}
|
||||
};
|
||||
|
||||
function updateSelectedText(value: string) {
|
||||
state.canvasSelectedText = value;
|
||||
}
|
||||
|
||||
// Recipe field selection event handlers
|
||||
function setSingleIngredient(f: keyof RecipeIngredient, index: number) {
|
||||
state.selectedRecipeField = `recipeIngredient.${index}.${f}` as SelectedRecipeLeaves;
|
||||
}
|
||||
|
||||
// Leaves<RecipeStep[]> will return some function types making eslint very unhappy
|
||||
type RecipeStepsLeaves = `${number}.${Leaves<RecipeStep>}`;
|
||||
|
||||
function setSingleStep(path: RecipeStepsLeaves) {
|
||||
state.selectedRecipeField = `recipeInstructions.${path}` as SelectedRecipeLeaves;
|
||||
}
|
||||
|
||||
return {
|
||||
...toRefs(state),
|
||||
addIngredient,
|
||||
addStep,
|
||||
drag,
|
||||
assetURL,
|
||||
updateRecipe,
|
||||
closeEditor,
|
||||
updateSelectedText,
|
||||
tsv,
|
||||
validators,
|
||||
setSingleIngredient,
|
||||
setSingleStep,
|
||||
canvasSetText,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="css">
|
||||
.ghost {
|
||||
opacity: 0.5;
|
||||
}
|
||||
</style>
|
||||
@@ -0,0 +1,484 @@
|
||||
<template>
|
||||
<v-card flat tile>
|
||||
<v-toolbar v-for="(section, idx) in toolbarIcons" :key="section.sectionTitle" dense style="float: left">
|
||||
<v-toolbar-title bottom>
|
||||
{{ section.sectionTitle }}
|
||||
</v-toolbar-title>
|
||||
<v-tooltip v-for="icon in section.icons" :key="icon.name" bottom>
|
||||
<template #activator="{ on, attrs }">
|
||||
<v-btn icon @click="section.eventHandler(icon.name)">
|
||||
<v-icon :color="section.highlight === icon.name ? 'primary' : 'default'" v-bind="attrs" v-on="on">
|
||||
{{ icon.icon }}
|
||||
</v-icon>
|
||||
</v-btn>
|
||||
</template>
|
||||
<span>{{ icon.tooltip }}</span>
|
||||
</v-tooltip>
|
||||
<v-divider v-if="idx != toolbarIcons.length - 1" vertical class="mx-2" />
|
||||
</v-toolbar>
|
||||
<v-toolbar dense style="float: right">
|
||||
<BaseButton class="ml-1 mr-1" save @click="updateRecipe()">
|
||||
{{ $t("general.save") }}
|
||||
</BaseButton>
|
||||
<BaseButton cancel @click="closeEditor()">
|
||||
{{ $t("general.close") }}
|
||||
</BaseButton>
|
||||
</v-toolbar>
|
||||
<canvas
|
||||
ref="canvas"
|
||||
@mousedown="handleMouseDown"
|
||||
@mouseup="handleMouseUp"
|
||||
@mousemove="handleMouseMove"
|
||||
@wheel="handleMouseScroll"
|
||||
>
|
||||
</canvas>
|
||||
<span style="white-space: pre-wrap">
|
||||
{{ selectedText }}
|
||||
</span>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent, reactive, useContext, ref, toRefs, watch } from "@nuxtjs/composition-api";
|
||||
import { onMounted } from "vue-demi";
|
||||
import { OcrTsvResponse } from "~/types/api-types/ocr";
|
||||
import { CanvasModes, SelectedTextSplitModes, ImagePosition, Mouse, CanvasRect, ToolbarIcons } from "~/types/ocr-types";
|
||||
|
||||
export default defineComponent({
|
||||
props: {
|
||||
image: {
|
||||
type: HTMLImageElement,
|
||||
required: true,
|
||||
},
|
||||
tsv: {
|
||||
type: Array as () => OcrTsvResponse[],
|
||||
required: true,
|
||||
},
|
||||
},
|
||||
setup(props, context) {
|
||||
const state = reactive({
|
||||
canvas: null as HTMLCanvasElement | null,
|
||||
ctx: null as CanvasRenderingContext2D | null,
|
||||
canvasRect: null as DOMRect | null,
|
||||
rect: {
|
||||
startX: 0,
|
||||
startY: 0,
|
||||
w: 0,
|
||||
h: 0,
|
||||
},
|
||||
mouse: {
|
||||
current: {
|
||||
x: 0,
|
||||
y: 0,
|
||||
},
|
||||
down: false,
|
||||
},
|
||||
selectedText: "",
|
||||
canvasMode: "selection" as CanvasModes,
|
||||
imagePosition: {
|
||||
sx: 0,
|
||||
sy: 0,
|
||||
sWidth: 0,
|
||||
sHeight: 0,
|
||||
dx: 0,
|
||||
dy: 0,
|
||||
dWidth: 0,
|
||||
dHeight: 0,
|
||||
scale: 1,
|
||||
panStartPoint: {
|
||||
x: 0,
|
||||
y: 0,
|
||||
},
|
||||
} as ImagePosition,
|
||||
isImageSmallerThanCanvas: false,
|
||||
selectedTextSplitMode: "lineNum" as SelectedTextSplitModes,
|
||||
});
|
||||
|
||||
watch(
|
||||
() => state.selectedText,
|
||||
(value) => {
|
||||
context.emit("text-selected", value);
|
||||
}
|
||||
);
|
||||
|
||||
onMounted(() => {
|
||||
if (state.canvas === null) return; // never happens because the ref "canvas" is in the template
|
||||
state.ctx = state.canvas.getContext("2d") as CanvasRenderingContext2D;
|
||||
state.ctx.imageSmoothingEnabled = false;
|
||||
state.canvasRect = state.canvas.getBoundingClientRect();
|
||||
|
||||
state.canvas.width = state.canvasRect.width;
|
||||
if (props.image.width < state.canvas.width) {
|
||||
state.isImageSmallerThanCanvas = true;
|
||||
}
|
||||
state.imagePosition.dWidth = state.canvas.width;
|
||||
|
||||
updateImageScale();
|
||||
state.canvas.height = Math.min(props.image.height * state.imagePosition.scale, 700); // Max height of 700px
|
||||
|
||||
state.imagePosition.sWidth = props.image.width;
|
||||
state.imagePosition.sHeight = props.image.height;
|
||||
state.imagePosition.dWidth = state.canvas.width;
|
||||
drawImage(state.ctx);
|
||||
drawWordBoxesOnCanvas(props.tsv);
|
||||
});
|
||||
|
||||
function handleMouseDown(event: MouseEvent) {
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
state.mouse.down = true;
|
||||
|
||||
updateMousePos(event);
|
||||
|
||||
if (state.canvasMode === "selection") {
|
||||
if (isMouseInRect(state.mouse, state.rect)) {
|
||||
context.emit("setText", state.selectedText);
|
||||
} else {
|
||||
state.ctx.fillStyle = "rgb(255, 255, 255)";
|
||||
state.ctx.fillRect(0, 0, state.canvas.width, state.canvas.height);
|
||||
drawImage(state.ctx);
|
||||
state.rect.startX = state.mouse.current.x;
|
||||
state.rect.startY = state.mouse.current.y;
|
||||
resetSelection();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.canvasMode === "panAndZoom") {
|
||||
state.imagePosition.panStartPoint.x = state.mouse.current.x - state.imagePosition.dx;
|
||||
state.imagePosition.panStartPoint.y = state.mouse.current.y - state.imagePosition.dy;
|
||||
resetSelection();
|
||||
}
|
||||
}
|
||||
|
||||
function handleMouseUp(_event: MouseEvent) {
|
||||
if (state.canvasRect === null) return;
|
||||
state.mouse.down = false;
|
||||
state.selectedText = getWordsInSelection(props.tsv, state.rect);
|
||||
}
|
||||
|
||||
function handleMouseMove(event: MouseEvent) {
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
|
||||
updateMousePos(event);
|
||||
|
||||
if (state.mouse.down) {
|
||||
if (state.canvasMode === "selection") {
|
||||
state.rect.w = state.mouse.current.x - state.rect.startX;
|
||||
state.rect.h = state.mouse.current.y - state.rect.startY;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.canvasMode === "panAndZoom") {
|
||||
state.canvas.style.cursor = "move";
|
||||
state.imagePosition.dx = state.mouse.current.x - state.imagePosition.panStartPoint.x;
|
||||
state.imagePosition.dy = state.mouse.current.y - state.imagePosition.panStartPoint.y;
|
||||
keepImageInCanvas();
|
||||
state.ctx.fillStyle = "rgb(255, 255, 255)";
|
||||
state.ctx.fillRect(0, 0, state.canvas.width, state.canvas.height);
|
||||
drawImage(state.ctx);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (isMouseInRect(state.mouse, state.rect) && state.canvasMode === "selection") {
|
||||
state.canvas.style.cursor = "pointer";
|
||||
} else {
|
||||
state.canvas.style.cursor = "default";
|
||||
}
|
||||
}
|
||||
|
||||
const scrollSensitivity = 0.05;
|
||||
|
||||
function handleMouseScroll(event: WheelEvent) {
|
||||
if (state.isImageSmallerThanCanvas) return;
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
|
||||
if (state.canvasMode === "panAndZoom") {
|
||||
event.preventDefault();
|
||||
|
||||
updateMousePos(event);
|
||||
|
||||
const m = Math.sign(event.deltaY);
|
||||
|
||||
const ndx = state.imagePosition.dx + m * state.imagePosition.dWidth * scrollSensitivity;
|
||||
const ndy = state.imagePosition.dy + m * state.imagePosition.dHeight * scrollSensitivity;
|
||||
const ndw = state.imagePosition.dWidth + -m * state.imagePosition.dWidth * scrollSensitivity * 2;
|
||||
const ndh = state.imagePosition.dHeight + -m * state.imagePosition.dHeight * scrollSensitivity * 2;
|
||||
|
||||
if (ndw < props.image.width) {
|
||||
state.imagePosition.dx = ndx;
|
||||
state.imagePosition.dy = ndy;
|
||||
state.imagePosition.dWidth = ndw;
|
||||
state.imagePosition.dHeight = ndh;
|
||||
}
|
||||
|
||||
keepImageInCanvas();
|
||||
updateImageScale();
|
||||
|
||||
state.ctx.fillStyle = "rgb(255, 255, 255)";
|
||||
state.ctx.fillRect(0, 0, state.canvas.width, state.canvas.height);
|
||||
drawImage(state.ctx);
|
||||
}
|
||||
}
|
||||
|
||||
function draw() {
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
if (state.mouse.down) {
|
||||
state.ctx.imageSmoothingEnabled = false;
|
||||
state.ctx.fillStyle = "rgb(255, 255, 255)";
|
||||
state.ctx.fillRect(0, 0, state.canvas.width, state.canvas.height);
|
||||
drawImage(state.ctx);
|
||||
state.ctx.fillStyle = "rgba(255, 255, 255, 0.1)";
|
||||
state.ctx.setLineDash([6]);
|
||||
state.ctx.fillRect(state.rect.startX, state.rect.startY, state.rect.w, state.rect.h);
|
||||
state.ctx.strokeRect(state.rect.startX, state.rect.startY, state.rect.w, state.rect.h);
|
||||
}
|
||||
}
|
||||
|
||||
function drawImage(ctx: CanvasRenderingContext2D) {
|
||||
ctx.drawImage(
|
||||
props.image,
|
||||
state.imagePosition.sx,
|
||||
state.imagePosition.sy,
|
||||
state.imagePosition.sWidth,
|
||||
state.imagePosition.sHeight,
|
||||
state.imagePosition.dx,
|
||||
state.imagePosition.dy,
|
||||
state.imagePosition.dWidth,
|
||||
state.imagePosition.dHeight
|
||||
);
|
||||
}
|
||||
|
||||
function keepImageInCanvas() {
|
||||
if (state.canvasRect === null || state.canvas === null) return;
|
||||
|
||||
// Prevent image from being smaller than the canvas width
|
||||
if (state.imagePosition.dWidth - state.canvas.width < 0) {
|
||||
state.imagePosition.dWidth = state.canvas.width;
|
||||
}
|
||||
|
||||
// Prevent image from being smaller than the canvas height
|
||||
if (state.imagePosition.dHeight - state.canvas.height < 0) {
|
||||
state.imagePosition.dHeight = props.image.height * state.imagePosition.scale;
|
||||
}
|
||||
|
||||
// Prevent to move the image too much to the left
|
||||
if (state.canvas.width - state.imagePosition.dx - state.imagePosition.dWidth > 0) {
|
||||
state.imagePosition.dx = state.canvas.width - state.imagePosition.dWidth;
|
||||
}
|
||||
|
||||
// Prevent to move the image too much to the top
|
||||
if (state.canvas.height - state.imagePosition.dy - state.imagePosition.dHeight > 0) {
|
||||
state.imagePosition.dy = state.canvas.height - state.imagePosition.dHeight;
|
||||
}
|
||||
|
||||
// Prevent to move the image too much to the right
|
||||
if (state.imagePosition.dx > 0) {
|
||||
state.imagePosition.dx = 0;
|
||||
}
|
||||
|
||||
// Prevent to move the image too much to the bottom
|
||||
if (state.imagePosition.dy > 0) {
|
||||
state.imagePosition.dy = 0;
|
||||
}
|
||||
}
|
||||
|
||||
function updateImageScale() {
|
||||
state.imagePosition.scale = state.imagePosition.dWidth / props.image.width;
|
||||
|
||||
// force the original ratio to be respected
|
||||
state.imagePosition.dHeight = props.image.height * state.imagePosition.scale;
|
||||
|
||||
// Don't let images bigger than the canvas be zoomed in more than 1:1 scale
|
||||
// Meaning only let images smaller than the canvas to have a scale > 1
|
||||
if (!state.isImageSmallerThanCanvas && state.imagePosition.scale > 1) {
|
||||
state.imagePosition.scale = 1;
|
||||
}
|
||||
}
|
||||
|
||||
function resetSelection() {
|
||||
if (state.canvasRect === null) return;
|
||||
state.rect.w = 0;
|
||||
state.rect.h = 0;
|
||||
state.selectedText = "";
|
||||
}
|
||||
|
||||
function updateMousePos<T extends MouseEvent>(event: T) {
|
||||
if (state.canvas === null) return;
|
||||
state.canvasRect = state.canvas.getBoundingClientRect();
|
||||
state.mouse.current = {
|
||||
x: event.clientX - state.canvasRect.left,
|
||||
y: event.clientY - state.canvasRect.top,
|
||||
};
|
||||
}
|
||||
|
||||
function isMouseInRect(mouse: Mouse, rect: CanvasRect) {
|
||||
if (state.canvasRect === null) return;
|
||||
const correctRect = correctRectCoordinates(rect);
|
||||
|
||||
return (
|
||||
mouse.current.x > correctRect.startX &&
|
||||
mouse.current.x < correctRect.startX + correctRect.w &&
|
||||
mouse.current.y > correctRect.startY &&
|
||||
mouse.current.y < correctRect.startY + correctRect.h
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns rectangle coordinates with positive dimensions
|
||||
* @param rect A rectangle
|
||||
* @returns An equivalent rectangle with width and height > 0
|
||||
*/
|
||||
function correctRectCoordinates(rect: CanvasRect) {
|
||||
if (rect.w < 0) {
|
||||
rect.startX = rect.startX + rect.w;
|
||||
rect.w = -rect.w;
|
||||
}
|
||||
|
||||
if (rect.h < 0) {
|
||||
rect.startY = rect.startY + rect.h;
|
||||
rect.h = -rect.h;
|
||||
}
|
||||
return rect;
|
||||
}
|
||||
|
||||
function drawWordBoxesOnCanvas(tsv: OcrTsvResponse[]) {
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
|
||||
state.ctx.fillStyle = "rgb(255, 255, 255, 0.3)";
|
||||
tsv
|
||||
.filter((element) => element.level === 5)
|
||||
.forEach((element) => {
|
||||
if (state.canvasRect === null || state.canvas === null || state.ctx === null) return;
|
||||
state.ctx.fillRect(
|
||||
element.left * state.imagePosition.scale,
|
||||
element.top * state.imagePosition.scale,
|
||||
element.width * state.imagePosition.scale,
|
||||
element.height * state.imagePosition.scale
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Event emitters
|
||||
const updateRecipe = function () {
|
||||
context.emit("update-recipe");
|
||||
};
|
||||
|
||||
const closeEditor = function () {
|
||||
context.emit("close-editor");
|
||||
};
|
||||
|
||||
// TOOLBAR STUFF
|
||||
|
||||
const { $globals, i18n } = useContext();
|
||||
|
||||
const toolbarIcons = ref<ToolbarIcons<CanvasModes | SelectedTextSplitModes>>([
|
||||
{
|
||||
sectionTitle: "Toolbar",
|
||||
eventHandler: switchCanvasMode,
|
||||
highlight: state.canvasMode,
|
||||
icons: [
|
||||
{
|
||||
name: "selection",
|
||||
icon: $globals.icons.selectMode,
|
||||
tooltip: i18n.tc("ocr-editor.selection-mode"),
|
||||
},
|
||||
{
|
||||
name: "panAndZoom",
|
||||
icon: $globals.icons.panAndZoom,
|
||||
tooltip: i18n.tc("ocr-editor.pan-and-zoom-picture"),
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
sectionTitle: i18n.tc("ocr-editor.split-text"),
|
||||
eventHandler: switchSplitTextMode,
|
||||
highlight: state.selectedTextSplitMode,
|
||||
icons: [
|
||||
{
|
||||
name: "lineNum",
|
||||
icon: $globals.icons.preserveLines,
|
||||
tooltip: i18n.tc("ocr-editor.preserve-line-breaks"),
|
||||
},
|
||||
{
|
||||
name: "blockNum",
|
||||
icon: $globals.icons.preserveBlocks,
|
||||
tooltip: i18n.tc("ocr-editor.split-by-block"),
|
||||
},
|
||||
{
|
||||
name: "flatten",
|
||||
icon: $globals.icons.flatten,
|
||||
tooltip: i18n.tc("ocr-editor.flatten"),
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
function switchCanvasMode(mode: CanvasModes) {
|
||||
if (state.canvasRect === null || state.canvas === null) return;
|
||||
state.canvasMode = mode;
|
||||
toolbarIcons.value[0].highlight = mode;
|
||||
if (mode === "panAndZoom") {
|
||||
state.canvas.style.cursor = "pointer";
|
||||
} else {
|
||||
state.canvas.style.cursor = "default";
|
||||
}
|
||||
}
|
||||
|
||||
function switchSplitTextMode(mode: SelectedTextSplitModes) {
|
||||
if (state.canvasRect === null) return;
|
||||
state.selectedTextSplitMode = mode;
|
||||
toolbarIcons.value[1].highlight = mode;
|
||||
state.selectedText = getWordsInSelection(props.tsv, state.rect);
|
||||
}
|
||||
|
||||
/**
|
||||
* Using rectangle coordinates, filters the tsv to get text elements contained
|
||||
* inside the rectangle
|
||||
* Additionaly adds newlines depending on the current "text split" mode
|
||||
* @param tsv An Object containing tesseracts tsv fields
|
||||
* @param rect Coordinates of a rectangle
|
||||
* @returns Text from tsv contained in the rectangle
|
||||
*/
|
||||
function getWordsInSelection(tsv: OcrTsvResponse[], rect: CanvasRect) {
|
||||
const correctedRect = correctRectCoordinates(rect);
|
||||
|
||||
return tsv
|
||||
.filter(
|
||||
(element) =>
|
||||
element.level === 5 &&
|
||||
correctedRect.startY - state.imagePosition.dy < element.top * state.imagePosition.scale &&
|
||||
correctedRect.startX - state.imagePosition.dx < element.left * state.imagePosition.scale &&
|
||||
correctedRect.startX + correctedRect.w >
|
||||
(element.left + element.width) * state.imagePosition.scale + state.imagePosition.dx &&
|
||||
correctedRect.startY + correctedRect.h >
|
||||
(element.top + element.height) * state.imagePosition.scale + state.imagePosition.dy
|
||||
)
|
||||
.map((element, index, array) => {
|
||||
let separator = " ";
|
||||
if (
|
||||
state.selectedTextSplitMode !== "flatten" &&
|
||||
index !== array.length - 1 &&
|
||||
element[state.selectedTextSplitMode] !== array[index + 1][state.selectedTextSplitMode]
|
||||
) {
|
||||
separator = "\n";
|
||||
}
|
||||
return element.text + separator;
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
return {
|
||||
...toRefs(state),
|
||||
handleMouseDown,
|
||||
handleMouseUp,
|
||||
handleMouseMove,
|
||||
handleMouseScroll,
|
||||
toolbarIcons,
|
||||
updateRecipe,
|
||||
closeEditor,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
@@ -0,0 +1,54 @@
|
||||
<template>
|
||||
<v-card>
|
||||
<v-app-bar dense dark color="primary" class="mb-2">
|
||||
<v-icon large left>
|
||||
{{ $globals.icons.help }}
|
||||
</v-icon>
|
||||
<v-toolbar-title class="headline"> Help </v-toolbar-title>
|
||||
<v-spacer></v-spacer>
|
||||
</v-app-bar>
|
||||
<v-card-text>
|
||||
<h1>Mouse modes</h1>
|
||||
<v-divider class="mb-2 mt-1" />
|
||||
<h2 class="my-2">
|
||||
<v-icon> {{ $globals.icons.selectMode }} </v-icon>{{ $t("ocr-editor.help.selection-mode") }}
|
||||
</h2>
|
||||
<p class="my-1">{{ $t("ocr-editor.help.selection-mode") }}</p>
|
||||
<ol>
|
||||
<li>{{ $t("ocr-editor.help.selection-mode-steps.draw") }}</li>
|
||||
<li>{{ $t("ocr-editor.help.selection-mode-steps.click") }}</li>
|
||||
<li>{{ $t("ocr-editor.help.selection-mode-steps.result") }}</li>
|
||||
</ol>
|
||||
<h2 class="my-2">
|
||||
<v-icon> {{ $globals.icons.panAndZoom }} </v-icon>{{ $t("ocr-editor.help.pan-and-zoom-mode") }}
|
||||
</h2>
|
||||
{{ $t("ocr-editor.help.pan-and-zoom-desc") }}
|
||||
<h1 class="mt-5">{{ $t("ocr-editor.help.split-text-mode") }}</h1>
|
||||
<v-divider class="mb-2 mt-1" />
|
||||
<h2 class="my-2">
|
||||
<v-icon> {{ $globals.icons.preserveLines }} </v-icon>
|
||||
{{ $t("ocr-editor.help.split-modes.line-mode") }}
|
||||
</h2>
|
||||
<p>
|
||||
{{ $t("ocr-editor.help.split-modes.line-mode-desc") }}
|
||||
</p>
|
||||
<h2 class="my-2">
|
||||
<v-icon> {{ $globals.icons.preserveBlocks }} </v-icon>
|
||||
{{ $t("ocr-editor.help.split-modes.block-mode") }}
|
||||
</h2>
|
||||
<p>
|
||||
{{ $t("ocr-editor.help.split-modes.block-mode-desc") }}
|
||||
</p>
|
||||
<h2 class="my-2">
|
||||
<v-icon> {{ $globals.icons.flatten }} </v-icon> {{ $t("ocr-editor.help.split-modes.flat-mode") }}
|
||||
</h2>
|
||||
<p>{{ $t("ocr-editor.help.split-modes.flat-mode-desc") }}</p>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent } from "@nuxtjs/composition-api";
|
||||
|
||||
export default defineComponent({});
|
||||
</script>
|
||||
@@ -0,0 +1,3 @@
|
||||
import RecipeOcrEditorPage from "./RecipeOcrEditorPage.vue";
|
||||
|
||||
export default RecipeOcrEditorPage;
|
||||
@@ -42,6 +42,7 @@
|
||||
:logged-in="$auth.loggedIn"
|
||||
:open="isEditMode"
|
||||
:recipe-id="recipe.id"
|
||||
:show-ocr-button="recipe.isOcrRecipe"
|
||||
class="ml-auto mt-n8 pb-4"
|
||||
@close="setMode(PageMode.VIEW)"
|
||||
@json="toggleEditMode()"
|
||||
@@ -49,12 +50,13 @@
|
||||
@save="$emit('save')"
|
||||
@delete="$emit('delete')"
|
||||
@print="printRecipe"
|
||||
@ocr="goToOcrEditor"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent, useContext, computed, ref, watch } from "@nuxtjs/composition-api";
|
||||
import { defineComponent, useContext, computed, ref, watch, useRouter } from "@nuxtjs/composition-api";
|
||||
import RecipeRating from "~/components/Domain/Recipe/RecipeRating.vue";
|
||||
import RecipeActionMenu from "~/components/Domain/Recipe/RecipeActionMenu.vue";
|
||||
import RecipeTimeCard from "~/components/Domain/Recipe/RecipeTimeCard.vue";
|
||||
@@ -82,6 +84,7 @@ export default defineComponent({
|
||||
const { recipeImage } = useStaticRoutes();
|
||||
const { imageKey, pageMode, editMode, setMode, toggleEditMode, isEditMode } = usePageState(props.recipe.slug);
|
||||
const { user } = usePageUser();
|
||||
const router = useRouter();
|
||||
|
||||
function printRecipe() {
|
||||
window.print();
|
||||
@@ -98,6 +101,10 @@ export default defineComponent({
|
||||
return recipeImage(props.recipe.id, props.recipe.image, imageKey.value);
|
||||
});
|
||||
|
||||
function goToOcrEditor() {
|
||||
router.push("/recipe/" + props.recipe.slug + "/ocr-editor");
|
||||
}
|
||||
|
||||
watch(
|
||||
() => recipeImageUrl.value,
|
||||
() => {
|
||||
@@ -120,6 +127,7 @@ export default defineComponent({
|
||||
hideImage,
|
||||
isEditMode,
|
||||
recipeImageUrl,
|
||||
goToOcrEditor,
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
@@ -248,7 +248,8 @@
|
||||
"trim-prefix-description": "Trim first character from each line",
|
||||
"split-by-numbered-line-description": "Attempts to split a paragraph by matching '1)' or '1.' patterns",
|
||||
"import-by-url": "Import a recipe by URL",
|
||||
"create-manually": "Create a recipe manually"
|
||||
"create-manually": "Create a recipe manually",
|
||||
"make-recipe-image": "Make this the recipe image"
|
||||
},
|
||||
"page": {
|
||||
"404-page-not-found": "404 Page not found",
|
||||
@@ -660,5 +661,34 @@
|
||||
"info_message_with_version": "This is a Demo for version: {version}",
|
||||
"demo_username": "Username: {username}",
|
||||
"demo_password": "Password: {password}"
|
||||
},
|
||||
"ocr-editor": {
|
||||
"ocr-editor": "Ocr editor",
|
||||
"selection-mode": "Selection mode",
|
||||
"pan-and-zoom-picture": "Pan and zoom picture",
|
||||
"split-text": "Split text",
|
||||
"preserve-line-breaks": "Preserve original line breaks",
|
||||
"split-by-block": "Split by text block",
|
||||
"flatten": "Flatten regardless of original formating",
|
||||
"help": {
|
||||
"selection-mode": "Selection Mode (default)",
|
||||
"selection-mode-desc": "The selection mode is the main mode that can be used to enter data:",
|
||||
"selection-mode-steps": {
|
||||
"draw": "Draw a rectangle on the text you want to select.",
|
||||
"click": "Click on any field on the right and then click back on the rectangle above the image.",
|
||||
"result": "The selected text will appear inside the previously selected field."
|
||||
},
|
||||
"pan-and-zoom-mode": "Pan and Zoom Mode",
|
||||
"pan-and-zoom-desc": "Select pan and zoom by clicking the icon. This mode allows to zoom inside the image and move around to make using big images easier.",
|
||||
"split-text-mode": "Split Text modes",
|
||||
"split-modes": {
|
||||
"line-mode": "Line mode (default)",
|
||||
"line-mode-desc": "In line mode, the text will be propagated by keeping the original line breaks. This mode is useful when using bulk add on a list of ingredients where one ingredient is one line.",
|
||||
"block-mode": "Block mode",
|
||||
"block-mode-desc": "In block mode, the text will be split in blocks. This mode is useful when bulk adding instructions that are usually written in paragraphs.",
|
||||
"flat-mode": "Flat mode",
|
||||
"flat-mode-desc": "In flat mode, the text will be added to the selected recipe field with no line breaks."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,7 +60,7 @@
|
||||
{{ isError(ing) ? $globals.icons.alert : $globals.icons.check }}
|
||||
</v-icon>
|
||||
<div class="my-auto" :color="isError(ing) ? 'error-text' : 'success-text'">
|
||||
{{ asPercentage(ing.confidence.average) }}
|
||||
{{ ing.confidence ? asPercentage(ing.confidence.average) : "" }}
|
||||
</div>
|
||||
</template>
|
||||
</v-expansion-panel-header>
|
||||
@@ -197,7 +197,11 @@ export default defineComponent({
|
||||
return !(ing.confidence.average >= 0.75);
|
||||
}
|
||||
|
||||
function asPercentage(num: number) {
|
||||
function asPercentage(num: number | undefined): string {
|
||||
if (!num) {
|
||||
return "0%";
|
||||
}
|
||||
|
||||
return Math.round(num * 100).toFixed(2) + "%";
|
||||
}
|
||||
|
||||
@@ -230,7 +234,11 @@ export default defineComponent({
|
||||
return false;
|
||||
}
|
||||
|
||||
async function createFood(food: CreateIngredientFood, index: number) {
|
||||
async function createFood(food: CreateIngredientFood | undefined, index: number) {
|
||||
if (!food) {
|
||||
return;
|
||||
}
|
||||
|
||||
foodData.data.name = food.name;
|
||||
await foodStore.actions.createOne(foodData.data);
|
||||
errors.value[index].foodError = false;
|
||||
|
||||
51
frontend/pages/recipe/_slug/ocr-editor.vue
Normal file
51
frontend/pages/recipe/_slug/ocr-editor.vue
Normal file
@@ -0,0 +1,51 @@
|
||||
<template>
|
||||
<div>
|
||||
<RecipeOcrEditorPage v-if="recipe" :recipe="recipe" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script lang="ts">
|
||||
import { defineComponent, useRoute } from "@nuxtjs/composition-api";
|
||||
import RecipeOcrEditorPage from "~/components/Domain/Recipe/RecipeOcrEditorPage/RecipeOcrEditorPage.vue";
|
||||
import { useRecipe } from "~/composables/recipes";
|
||||
|
||||
export default defineComponent({
|
||||
components: { RecipeOcrEditorPage },
|
||||
setup() {
|
||||
const route = useRoute();
|
||||
const slug = route.value.params.slug;
|
||||
|
||||
const { recipe, loading } = useRecipe(slug);
|
||||
|
||||
return {
|
||||
recipe,
|
||||
loading,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
|
||||
<style lang="css">
|
||||
.ghost {
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
body {
|
||||
background: #eee;
|
||||
}
|
||||
|
||||
canvas {
|
||||
background: white;
|
||||
box-shadow: 0px 2px 3px rgba(0, 0, 0, 0.2);
|
||||
width: 100%;
|
||||
image-rendering: optimizeQuality;
|
||||
}
|
||||
|
||||
.box {
|
||||
position: absolute;
|
||||
border: 2px #90ee90 solid;
|
||||
background-color: #90ee90;
|
||||
|
||||
z-index: 3;
|
||||
}
|
||||
</style>
|
||||
@@ -52,6 +52,11 @@ export default defineComponent({
|
||||
text: "Import with .zip",
|
||||
value: "zip",
|
||||
},
|
||||
{
|
||||
icon: $globals.icons.fileImage,
|
||||
text: "Create recipe from an image",
|
||||
value: "ocr",
|
||||
},
|
||||
{
|
||||
icon: $globals.icons.link,
|
||||
text: "Bulk URL Import",
|
||||
|
||||
81
frontend/pages/recipe/create/ocr.vue
Normal file
81
frontend/pages/recipe/create/ocr.vue
Normal file
@@ -0,0 +1,81 @@
|
||||
<template>
|
||||
<div>
|
||||
<v-card-title class="headline"> Create Recipe from an Image </v-card-title>
|
||||
<v-card-text>
|
||||
Create a recipe by uploading a scan.
|
||||
<v-form ref="domCreateByOcr"> </v-form>
|
||||
</v-card-text>
|
||||
<v-card-actions class="justify-center">
|
||||
<v-file-input
|
||||
v-model="imageUpload"
|
||||
accept=".png"
|
||||
label="recipe.png"
|
||||
filled
|
||||
clearable
|
||||
class="rounded-lg mt-2"
|
||||
rounded
|
||||
truncate-length="100"
|
||||
hint="Upload a png image from a recipe book"
|
||||
persistent-hint
|
||||
prepend-icon=""
|
||||
:prepend-inner-icon="$globals.icons.fileImage"
|
||||
/>
|
||||
</v-card-actions>
|
||||
<v-card-actions class="justify-center">
|
||||
<v-checkbox v-model="makeFileRecipeImage" :label="$t('new-recipe.make-recipe-image')" />
|
||||
</v-card-actions>
|
||||
<v-card-actions class="justify-center">
|
||||
<div style="width: 250px">
|
||||
<BaseButton :disabled="imageUpload === null" large rounded block :loading="loading" @click="createByOcr" />
|
||||
</div>
|
||||
</v-card-actions>
|
||||
</div>
|
||||
</template>
|
||||
<script lang="ts">
|
||||
import { defineComponent, reactive, toRefs, ref, useRouter } from "@nuxtjs/composition-api";
|
||||
import { AxiosResponse } from "axios";
|
||||
import { useUserApi } from "~/composables/api";
|
||||
import { validators } from "~/composables/use-validators";
|
||||
import { VForm } from "~/types/vuetify";
|
||||
|
||||
export default defineComponent({
|
||||
setup() {
|
||||
const state = reactive({
|
||||
error: false,
|
||||
loading: false,
|
||||
makeFileRecipeImage: false,
|
||||
});
|
||||
const api = useUserApi();
|
||||
const router = useRouter();
|
||||
|
||||
const imageUpload = ref<File | null>(null);
|
||||
|
||||
function handleResponse(response: AxiosResponse<string> | null) {
|
||||
if (response?.status !== 201) {
|
||||
state.error = true;
|
||||
state.loading = false;
|
||||
return;
|
||||
}
|
||||
router.push(`/recipe/${response.data}/ocr-editor`);
|
||||
}
|
||||
|
||||
const domCreateByOcr = ref<VForm | null>(null);
|
||||
|
||||
async function createByOcr() {
|
||||
if (imageUpload.value === null) return; // Should never be true due to circumstances
|
||||
state.loading = true;
|
||||
const { response } = await api.recipes.createFromOcr(imageUpload.value, state.makeFileRecipeImage);
|
||||
// @ts-ignore returns a string and not a full Recipe
|
||||
handleResponse(response);
|
||||
}
|
||||
|
||||
return {
|
||||
domCreateByOcr,
|
||||
createByOcr,
|
||||
...toRefs(state),
|
||||
validators,
|
||||
imageUpload,
|
||||
};
|
||||
},
|
||||
});
|
||||
</script>
|
||||
21
frontend/types/api-types/ocr.ts
Normal file
21
frontend/types/api-types/ocr.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
/* tslint:disable */
|
||||
/* eslint-disable */
|
||||
/**
|
||||
/* This file was automatically generated from pydantic models by running pydantic2ts.
|
||||
/* Do not modify it by hand - just update the pydantic models and then re-run the script
|
||||
*/
|
||||
|
||||
export interface OcrTsvResponse {
|
||||
level: number;
|
||||
pageNum: number;
|
||||
blockNum: number;
|
||||
parNum: number;
|
||||
lineNum: number;
|
||||
wordNum: number;
|
||||
left: number;
|
||||
top: number;
|
||||
width: number;
|
||||
height: number;
|
||||
conf: number;
|
||||
text: string;
|
||||
}
|
||||
@@ -214,6 +214,7 @@ export interface Recipe {
|
||||
[k: string]: unknown;
|
||||
};
|
||||
comments?: RecipeCommentOut[];
|
||||
isOcrRecipe?: boolean;
|
||||
}
|
||||
export interface RecipeTool {
|
||||
id: string;
|
||||
|
||||
4
frontend/types/components.d.ts
vendored
4
frontend/types/components.d.ts
vendored
@@ -30,10 +30,6 @@ import AdvancedOnly from "@/components/global/AdvancedOnly.vue";
|
||||
import BasePageTitle from "@/components/global/BasePageTitle.vue";
|
||||
import ButtonLink from "@/components/global/ButtonLink.vue";
|
||||
|
||||
import TheSnackbar from "@/components/layout/TheSnackbar.vue";
|
||||
import AppHeader from "@/components/layout/AppHeader.vue";
|
||||
import AppSidebar from "@/components/layout/AppSidebar.vue";
|
||||
import AppFooter from "@/components/layout/AppFooter.vue";
|
||||
|
||||
declare module "vue" {
|
||||
export interface GlobalComponents {
|
||||
|
||||
73
frontend/types/ocr-types.ts
Normal file
73
frontend/types/ocr-types.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import { OcrTsvResponse } from "~/types/api-types/ocr";
|
||||
import { Recipe } from "~/types/api-types/recipe";
|
||||
|
||||
export type CanvasRect = {
|
||||
startX: number;
|
||||
startY: number;
|
||||
w: number;
|
||||
h: number;
|
||||
};
|
||||
|
||||
export type ImagePosition = {
|
||||
sx: number;
|
||||
sy: number;
|
||||
sWidth: number;
|
||||
sHeight: number;
|
||||
dx: number;
|
||||
dy: number;
|
||||
dWidth: number;
|
||||
dHeight: number;
|
||||
scale: number;
|
||||
panStartPoint: {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type Mouse = {
|
||||
current: {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
down: boolean;
|
||||
};
|
||||
|
||||
// https://stackoverflow.com/questions/58434389/export typescript-deep-keyof-of-a-nested-object/58436959#58436959
|
||||
type Prev = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...0[]];
|
||||
|
||||
type Join<K, P> = K extends string | number
|
||||
? P extends string | number
|
||||
? `${K}${"" extends P ? "" : "."}${P}`
|
||||
: never
|
||||
: never;
|
||||
|
||||
export type Leaves<T, D extends number = 10> = [D] extends [never]
|
||||
? never
|
||||
: T extends object
|
||||
? { [K in keyof T]-?: Join<K, Leaves<T[K], Prev[D]>> }[keyof T]
|
||||
: "";
|
||||
|
||||
export type Paths<T, D extends number = 10> = [D] extends [never]
|
||||
? never
|
||||
: T extends object
|
||||
? {
|
||||
[K in keyof T]-?: K extends string | number ? `${K}` | Join<K, Paths<T[K], Prev[D]>> : never;
|
||||
}[keyof T]
|
||||
: "";
|
||||
|
||||
export type SelectedRecipeLeaves = Leaves<Recipe>;
|
||||
|
||||
export type CanvasModes = "selection" | "panAndZoom";
|
||||
|
||||
export type SelectedTextSplitModes = keyof OcrTsvResponse | "flatten";
|
||||
|
||||
export type ToolbarIcons<T extends string> = {
|
||||
sectionTitle: string;
|
||||
eventHandler(mode: T): void;
|
||||
highlight: T;
|
||||
icons: {
|
||||
name: T;
|
||||
icon: string;
|
||||
tooltip: string;
|
||||
}[];
|
||||
}[];
|
||||
@@ -125,4 +125,11 @@ export interface Icon {
|
||||
back: string;
|
||||
slotMachine: string;
|
||||
chevronDown: string;
|
||||
|
||||
// Ocr toolbar
|
||||
selectMode: string;
|
||||
panAndZoom: string;
|
||||
preserveLines: string;
|
||||
preserveBlocks: string;
|
||||
flatten: string;
|
||||
}
|
||||
|
||||
@@ -118,6 +118,10 @@ import {
|
||||
mdiHelpCircleOutline,
|
||||
mdiDocker,
|
||||
mdiUndo,
|
||||
mdiSelectionDrag,
|
||||
mdiCursorMove,
|
||||
mdiText,
|
||||
mdiTextBoxOutline,
|
||||
} from "@mdi/js";
|
||||
|
||||
export const icons = {
|
||||
@@ -253,4 +257,12 @@ export const icons = {
|
||||
slotMachine: mdiSlotMachine,
|
||||
chevronDown: mdiChevronDown,
|
||||
chevronRight: mdiChevronRight,
|
||||
|
||||
// Ocr toolbar
|
||||
selectMode: mdiSelectionDrag,
|
||||
panAndZoom: mdiCursorMove,
|
||||
preserveLines: mdiText,
|
||||
preserveBlocks: mdiTextBoxOutline,
|
||||
flatten: mdiMinus,
|
||||
|
||||
};
|
||||
|
||||
@@ -104,6 +104,7 @@ class RecipeModel(SqlAlchemyBase, BaseMixins):
|
||||
rating = sa.Column(sa.Integer)
|
||||
org_url = sa.Column(sa.String)
|
||||
extras: list[ApiExtras] = orm.relationship("ApiExtras", cascade="all, delete-orphan")
|
||||
is_ocr_recipe = sa.Column(sa.Boolean, default=False)
|
||||
|
||||
# Time Stamp Properties
|
||||
date_added = sa.Column(sa.Date, default=datetime.date.today)
|
||||
|
||||
@@ -7,6 +7,7 @@ from . import (
|
||||
comments,
|
||||
explore,
|
||||
groups,
|
||||
ocr,
|
||||
organizers,
|
||||
parser,
|
||||
recipe,
|
||||
@@ -31,3 +32,4 @@ router.include_router(unit_and_foods.router)
|
||||
router.include_router(admin.router)
|
||||
router.include_router(validators.router)
|
||||
router.include_router(explore.router)
|
||||
router.include_router(ocr.router)
|
||||
|
||||
7
mealie/routes/ocr/__init__.py
Normal file
7
mealie/routes/ocr/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from . import pytesseract
|
||||
|
||||
router = APIRouter(prefix="/ocr")
|
||||
|
||||
router.include_router(pytesseract.router)
|
||||
37
mealie/routes/ocr/pytesseract.py
Normal file
37
mealie/routes/ocr/pytesseract.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from fastapi import APIRouter, File
|
||||
|
||||
from mealie.routes._base import BaseUserController, controller
|
||||
from mealie.schema.ocr.ocr import OcrAssetReq, OcrTsvResponse
|
||||
from mealie.services.ocr.pytesseract import OcrService
|
||||
from mealie.services.recipe.recipe_data_service import RecipeDataService
|
||||
from mealie.services.recipe.recipe_service import RecipeService
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@controller(router)
|
||||
class OCRController(BaseUserController):
|
||||
def __init__(self):
|
||||
self.ocr_service = OcrService()
|
||||
|
||||
@router.post("/", response_model=str)
|
||||
def image_to_string(self, file: bytes = File(...)):
|
||||
return self.ocr_service.image_to_string(file)
|
||||
|
||||
@router.post("/file-to-tsv", response_model=list[OcrTsvResponse])
|
||||
def file_to_tsv(self, file: bytes = File(...)):
|
||||
tsv = self.ocr_service.image_to_tsv(file)
|
||||
return self.ocr_service.format_tsv_output(tsv)
|
||||
|
||||
@router.post("/asset-to-tsv", response_model=list[OcrTsvResponse])
|
||||
def asset_to_tsv(self, req: OcrAssetReq):
|
||||
recipe_service = RecipeService(self.repos, self.user, self.group)
|
||||
recipe = recipe_service._get_recipe(req.recipe_slug)
|
||||
if recipe.id is None:
|
||||
return []
|
||||
data_service = RecipeDataService(recipe.id, recipe.group_id)
|
||||
asset_path = data_service.dir_assets.joinpath(req.asset_name)
|
||||
file = open(asset_path, "rb")
|
||||
tsv = self.ocr_service.image_to_tsv(file.read())
|
||||
|
||||
return self.ocr_service.format_tsv_output(tsv)
|
||||
@@ -33,7 +33,10 @@ from mealie.schema.recipe.recipe import (
|
||||
RecipeSummary,
|
||||
)
|
||||
from mealie.schema.recipe.recipe_asset import RecipeAsset
|
||||
from mealie.schema.recipe.recipe_ingredient import RecipeIngredient
|
||||
from mealie.schema.recipe.recipe_scraper import ScrapeRecipeTest
|
||||
from mealie.schema.recipe.recipe_settings import RecipeSettings
|
||||
from mealie.schema.recipe.recipe_step import RecipeStep
|
||||
from mealie.schema.recipe.request_helpers import RecipeZipTokenResponse, UpdateImageResponse
|
||||
from mealie.schema.response.responses import ErrorResponse
|
||||
from mealie.services import urls
|
||||
@@ -435,3 +438,37 @@ class RecipeController(BaseRecipeController):
|
||||
self.mixins.update_one(recipe, slug)
|
||||
|
||||
return asset_in
|
||||
|
||||
# ==================================================================================================================
|
||||
# OCR
|
||||
@router.post("/create-ocr", status_code=201, response_model=str)
|
||||
def create_recipe_ocr(
|
||||
self, extension: str = Form(...), file: UploadFile = File(...), makefilerecipeimage: bool = Form(...)
|
||||
):
|
||||
"""Takes an image and creates a recipe based on the image"""
|
||||
slug = self.service.create_one(
|
||||
Recipe(
|
||||
name="New OCR Recipe",
|
||||
recipe_ingredient=[RecipeIngredient(note="", title=None, unit=None, food=None, original_text=None)],
|
||||
recipe_instructions=[RecipeStep(text="")],
|
||||
is_ocr_recipe=True,
|
||||
settings=RecipeSettings(show_assets=True),
|
||||
id=None,
|
||||
image=None,
|
||||
recipe_yield=None,
|
||||
rating=None,
|
||||
orgURL=None,
|
||||
date_added=None,
|
||||
date_updated=None,
|
||||
created_at=None,
|
||||
update_at=None,
|
||||
nutrition=None,
|
||||
)
|
||||
).slug
|
||||
RecipeController.upload_recipe_asset(self, slug, "Original recipe image", "", extension, file)
|
||||
if makefilerecipeimage:
|
||||
# Get the pointer to the beginning of the file to read it once more
|
||||
file.file.seek(0)
|
||||
self.update_recipe_image(slug, file.file.read(), extension)
|
||||
|
||||
return slug
|
||||
|
||||
0
mealie/schema/ocr/__init__.py
Normal file
0
mealie/schema/ocr/__init__.py
Normal file
21
mealie/schema/ocr/ocr.py
Normal file
21
mealie/schema/ocr/ocr.py
Normal file
@@ -0,0 +1,21 @@
|
||||
from mealie.schema._mealie import MealieModel
|
||||
|
||||
|
||||
class OcrTsvResponse(MealieModel):
|
||||
level: int = 0
|
||||
page_num: int = 0
|
||||
block_num: int = 0
|
||||
par_num: int = 0
|
||||
line_num: int = 0
|
||||
word_num: int = 0
|
||||
left: int = 0
|
||||
top: int = 0
|
||||
width: int = 0
|
||||
height: int = 0
|
||||
conf: float = 0.0
|
||||
text: str = ""
|
||||
|
||||
|
||||
class OcrAssetReq(MealieModel):
|
||||
recipe_slug: str
|
||||
asset_name: str
|
||||
@@ -141,10 +141,11 @@ class Recipe(RecipeSummary):
|
||||
nutrition: Optional[Nutrition]
|
||||
|
||||
# Mealie Specific
|
||||
settings: Optional[RecipeSettings] = RecipeSettings()
|
||||
settings: Optional[RecipeSettings] = None
|
||||
assets: Optional[list[RecipeAsset]] = []
|
||||
notes: Optional[list[RecipeNote]] = []
|
||||
extras: Optional[dict] = {}
|
||||
is_ocr_recipe: Optional[bool] = False
|
||||
|
||||
comments: Optional[list[RecipeCommentOut]] = []
|
||||
|
||||
|
||||
0
mealie/services/ocr/__init__.py
Normal file
0
mealie/services/ocr/__init__.py
Normal file
56
mealie/services/ocr/pytesseract.py
Normal file
56
mealie/services/ocr/pytesseract.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from io import BytesIO
|
||||
|
||||
import pytesseract
|
||||
from PIL import Image
|
||||
|
||||
from mealie.schema.ocr.ocr import OcrTsvResponse
|
||||
from mealie.services._base_service import BaseService
|
||||
|
||||
|
||||
class OcrService(BaseService):
|
||||
"""
|
||||
Class for ocr engines.
|
||||
"""
|
||||
|
||||
def image_to_string(self, image_data):
|
||||
"""
|
||||
Returns a plain text translation of an image
|
||||
"""
|
||||
return pytesseract.image_to_string(Image.open(image_data))
|
||||
|
||||
def image_to_tsv(self, image_data, lang=None):
|
||||
"""
|
||||
Returns the pytesseract default tsv output
|
||||
"""
|
||||
if lang is not None:
|
||||
return pytesseract.image_to_data(Image.open(BytesIO(image_data)), lang=lang)
|
||||
|
||||
return pytesseract.image_to_data(Image.open(BytesIO(image_data)))
|
||||
|
||||
def format_tsv_output(self, tsv: str) -> list[OcrTsvResponse]:
|
||||
"""
|
||||
Returns a OcrTsvResponse from a default pytesseract tsv output
|
||||
"""
|
||||
lines = tsv.split("\n")
|
||||
titles = [t.strip() for t in lines[0].split("\t")]
|
||||
response: list[OcrTsvResponse] = []
|
||||
|
||||
for i in range(1, len(lines)):
|
||||
if lines[i] == "":
|
||||
continue
|
||||
|
||||
line = OcrTsvResponse()
|
||||
for key, value in zip(titles, lines[i].split("\t")):
|
||||
if key == "text":
|
||||
setattr(line, key, value.strip())
|
||||
elif key == "conf":
|
||||
setattr(line, key, float(value.strip()))
|
||||
elif key in OcrTsvResponse.__fields__:
|
||||
setattr(line, key, int(value.strip()))
|
||||
else:
|
||||
continue
|
||||
|
||||
if isinstance(line, OcrTsvResponse):
|
||||
response.append(line)
|
||||
|
||||
return response
|
||||
@@ -111,6 +111,8 @@ class RecipeService(BaseService):
|
||||
additional_attrs=create_data.dict(),
|
||||
)
|
||||
|
||||
if isinstance(create_data, CreateRecipe) or create_data.settings is None:
|
||||
if self.group.preferences is not None:
|
||||
data.settings = RecipeSettings(
|
||||
public=self.group.preferences.recipe_public,
|
||||
show_nutrition=self.group.preferences.recipe_show_nutrition,
|
||||
@@ -119,6 +121,8 @@ class RecipeService(BaseService):
|
||||
disable_comments=self.group.preferences.recipe_disable_comments,
|
||||
disable_amount=self.group.preferences.recipe_disable_amount,
|
||||
)
|
||||
else:
|
||||
data.settings = RecipeSettings()
|
||||
|
||||
return self.repos.recipes.create(data)
|
||||
|
||||
|
||||
18
poetry.lock
generated
18
poetry.lock
generated
@@ -821,7 +821,7 @@ requests = ["requests"]
|
||||
name = "packaging"
|
||||
version = "21.3"
|
||||
description = "Core utilities for Python packages"
|
||||
category = "dev"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
@@ -1083,6 +1083,18 @@ category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[[package]]
|
||||
name = "pytesseract"
|
||||
version = "0.3.9"
|
||||
description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
packaging = ">=21.3"
|
||||
Pillow = ">=8.0.0"
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "6.2.5"
|
||||
@@ -2309,6 +2321,10 @@ pyrsistent = [
|
||||
{file = "pyrsistent-0.18.1-cp39-cp39-win_amd64.whl", hash = "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07"},
|
||||
{file = "pyrsistent-0.18.1.tar.gz", hash = "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96"},
|
||||
]
|
||||
pytesseract = [
|
||||
{file = "pytesseract-0.3.9-py2.py3-none-any.whl", hash = "sha256:fecda37d1e4eaf744c657cd03a5daab4eb97c61506ac5550274322c8ae32eca2"},
|
||||
{file = "pytesseract-0.3.9.tar.gz", hash = "sha256:7e2bafc7f48d1bb71443ce4633a56f5e21925a98f220a36c336297edcd1956d0"},
|
||||
]
|
||||
pytest = [
|
||||
{file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
|
||||
{file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
|
||||
|
||||
@@ -38,6 +38,7 @@ python-ldap = "^3.3.1"
|
||||
pydantic = "^1.9.1"
|
||||
tzdata = "^2021.5"
|
||||
pyhumps = "^3.5.3"
|
||||
pytesseract = "^0.3.9"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
pylint = "^2.6.0"
|
||||
|
||||
BIN
tests/data/images/test-ocr.png
Normal file
BIN
tests/data/images/test-ocr.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
73
tests/data/text/test-ocr.tsv
Normal file
73
tests/data/text/test-ocr.tsv
Normal file
@@ -0,0 +1,73 @@
|
||||
level page_num block_num par_num line_num word_num left top width height conf text
|
||||
1 1 0 0 0 0 0 0 640 480 -1
|
||||
2 1 1 0 0 0 36 92 582 269 -1
|
||||
3 1 1 1 0 0 36 92 582 92 -1
|
||||
4 1 1 1 1 0 36 92 544 30 -1
|
||||
5 1 1 1 1 1 36 92 60 24 87.137558 This
|
||||
5 1 1 1 1 2 109 92 20 24 87.137558 is
|
||||
5 1 1 1 1 3 141 98 15 18 87.823906 a
|
||||
5 1 1 1 1 4 169 92 32 24 87.823906 lot
|
||||
5 1 1 1 1 5 212 92 28 24 92.965874 of
|
||||
5 1 1 1 1 6 251 92 31 24 93.247513 12
|
||||
5 1 1 1 1 7 296 92 68 30 92.734741 point
|
||||
5 1 1 1 1 8 374 93 53 23 92.996040 text
|
||||
5 1 1 1 1 9 437 93 26 23 93.160057 to
|
||||
5 1 1 1 1 10 474 93 52 23 92.312637 test
|
||||
5 1 1 1 1 11 536 92 44 24 92.312637 the
|
||||
4 1 1 1 2 0 36 126 582 31 -1
|
||||
5 1 1 1 2 1 36 132 45 18 90.505524 ocr
|
||||
5 1 1 1 2 2 91 126 69 24 90.505524 code
|
||||
5 1 1 1 2 3 172 126 51 24 91.169167 and
|
||||
5 1 1 1 2 4 236 132 50 18 89.765854 see
|
||||
5 1 1 1 2 5 299 126 15 24 85.827324 if
|
||||
5 1 1 1 2 6 325 126 14 24 93.116241 it
|
||||
5 1 1 1 2 7 348 126 85 24 92.394562 works
|
||||
5 1 1 1 2 8 445 132 33 18 30.119690 on
|
||||
5 1 1 1 2 9 500 126 29 24 30.119690 all
|
||||
5 1 1 1 2 10 541 127 77 30 92.090988 types
|
||||
4 1 1 1 3 0 36 160 187 24 -1
|
||||
5 1 1 1 3 1 36 160 28 24 92.476135 of
|
||||
5 1 1 1 3 2 72 160 41 24 90.919365 file
|
||||
5 1 1 1 3 3 123 160 100 24 91.360367 format.
|
||||
3 1 1 2 0 0 36 194 561 167 -1
|
||||
4 1 1 2 1 0 36 194 549 31 -1
|
||||
5 1 1 2 1 1 36 194 55 24 89.098892 The
|
||||
5 1 1 2 1 2 102 194 75 30 89.098892 quick
|
||||
5 1 1 2 1 3 189 194 85 24 91.415680 brown
|
||||
5 1 1 2 1 4 287 194 52 31 91.943085 dog
|
||||
5 1 1 2 1 5 348 194 108 31 92.167969 jumped
|
||||
5 1 1 2 1 6 468 200 63 18 91.970985 over
|
||||
5 1 1 2 1 7 540 194 45 24 92.843704 the
|
||||
4 1 1 2 2 0 37 228 548 31 -1
|
||||
5 1 1 2 2 1 37 228 55 31 92.262550 lazy
|
||||
5 1 1 2 2 2 103 228 50 24 92.693161 fox.
|
||||
5 1 1 2 2 3 165 228 55 24 92.947639 The
|
||||
5 1 1 2 2 4 232 228 75 30 90.589806 quick
|
||||
5 1 1 2 2 5 319 228 85 24 91.051247 brown
|
||||
5 1 1 2 2 6 417 228 51 31 91.925011 dog
|
||||
5 1 1 2 2 7 478 228 107 31 91.471077 jumped
|
||||
4 1 1 2 3 0 36 262 561 31 -1
|
||||
5 1 1 2 3 1 36 268 63 18 90.210129 over
|
||||
5 1 1 2 3 2 109 262 44 24 90.210129 the
|
||||
5 1 1 2 3 3 165 262 56 31 91.178192 lazy
|
||||
5 1 1 2 3 4 231 262 50 24 92.794647 fox.
|
||||
5 1 1 2 3 5 294 262 55 24 91.388016 The
|
||||
5 1 1 2 3 6 360 262 75 30 92.525742 quick
|
||||
5 1 1 2 3 7 447 262 85 24 90.425552 brown
|
||||
5 1 1 2 3 8 545 262 52 31 90.425552 dog
|
||||
4 1 1 2 4 0 43 296 518 31 -1
|
||||
5 1 1 2 4 1 43 296 107 31 91.759590 jumped
|
||||
5 1 1 2 4 2 162 302 64 18 92.923576 over
|
||||
5 1 1 2 4 3 235 296 44 24 92.017929 the
|
||||
5 1 1 2 4 4 292 296 55 31 91.558884 lazy
|
||||
5 1 1 2 4 5 357 296 50 24 92.687485 fox.
|
||||
5 1 1 2 4 6 420 296 55 24 91.922661 The
|
||||
5 1 1 2 4 7 486 296 75 30 91.870224 quick
|
||||
4 1 1 2 5 0 37 330 524 31 -1
|
||||
5 1 1 2 5 1 37 330 85 24 92.923935 brown
|
||||
5 1 1 2 5 2 135 330 52 31 91.468765 dog
|
||||
5 1 1 2 5 3 196 330 108 31 91.425491 jumped
|
||||
5 1 1 2 5 4 316 336 63 18 91.489830 over
|
||||
5 1 1 2 5 5 388 330 45 24 91.740379 the
|
||||
5 1 1 2 5 6 445 330 55 31 92.110054 lazy
|
||||
5 1 1 2 5 7 511 330 50 24 93.180054 fox.
|
||||
|
9
tests/data/text/test-ocr.txt
Normal file
9
tests/data/text/test-ocr.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
This is a lot of 12 point text to test the
|
||||
ocr code and see if it works on all types
|
||||
of file format.
|
||||
|
||||
The quick brown dog jumped over the
|
||||
lazy fox. The quick brown dog jumped
|
||||
over the lazy fox. The quick brown dog
|
||||
jumped over the lazy fox. The quick
|
||||
brown dog jumped over the lazy fox.
|
||||
@@ -4,7 +4,7 @@ from mealie.core.config import get_app_settings
|
||||
from mealie.services.backups_v2.alchemy_exporter import AlchemyExporter
|
||||
|
||||
ALEMBIC_VERSIONS = [
|
||||
{"version_num": "188374910655"},
|
||||
{"version_num": "089bfa50d0ed"},
|
||||
]
|
||||
|
||||
|
||||
|
||||
58
tests/unit_tests/services_tests/test_ocr_service.py
Normal file
58
tests/unit_tests/services_tests/test_ocr_service.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from mealie.services.ocr.pytesseract import OcrService
|
||||
|
||||
ocr_service = OcrService()
|
||||
|
||||
|
||||
@pytest.mark.skip("Tesseract is not reliable between environments")
|
||||
def test_image_to_string():
|
||||
with open(Path("tests/data/images/test-ocr.png"), "rb") as image:
|
||||
result = ocr_service.image_to_string(image)
|
||||
with open(Path("tests/data/text/test-ocr.txt"), "r", encoding="utf-8") as expected_result:
|
||||
assert result == expected_result.read()
|
||||
|
||||
|
||||
@pytest.mark.skip("Tesseract is not reliable between environments")
|
||||
def test_image_to_tsv():
|
||||
with open(Path("tests/data/images/test-ocr.png"), "rb") as image:
|
||||
result = ocr_service.image_to_tsv(image.read())
|
||||
with open(Path("tests/data/text/test-ocr.tsv"), "r", encoding="utf-8") as expected_result:
|
||||
assert result == expected_result.read()
|
||||
|
||||
|
||||
def test_format_tsv_output():
|
||||
tsv = " level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext \n1\t1\t0\t0\t0\t0\t0\t0\t640\t480\t-1\t\n5\t1\t1\t1\t1\t1\t36\t92\t60\t24\t87.137558\tThis"
|
||||
expected_result = [
|
||||
{
|
||||
"level": 1,
|
||||
"page_num": 1,
|
||||
"block_num": 0,
|
||||
"par_num": 0,
|
||||
"line_num": 0,
|
||||
"word_num": 0,
|
||||
"left": 0,
|
||||
"top": 0,
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"conf": -1.0,
|
||||
"text": "",
|
||||
},
|
||||
{
|
||||
"level": 5,
|
||||
"page_num": 1,
|
||||
"block_num": 1,
|
||||
"par_num": 1,
|
||||
"line_num": 1,
|
||||
"word_num": 1,
|
||||
"left": 36,
|
||||
"top": 92,
|
||||
"width": 60,
|
||||
"height": 24,
|
||||
"conf": 87.137558,
|
||||
"text": "This",
|
||||
},
|
||||
]
|
||||
assert ocr_service.format_tsv_output(tsv) == expected_result
|
||||
Reference in New Issue
Block a user