Mirrors/mealie
1
0
Fork 0
mirror of https://github.com/mealie-recipes/mealie.git synced 2026-02-10 09:53:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: Bulk URL Import Fixes (#2796)

Browse Source 
* allow expections when fetching content

* removed extra bracket on import text

* added more fault tolerance and limited concurrency

* fix entries not being saved to report

* disable clicking into in-proress import

* conditionally render expansion
This commit is contained in:
Michael Genson
2023-12-07 11:08:47 -06:00
committed by GitHub
parent 449bb6f0ce
commit f8ad72ec31
6 changed files with 37 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
mealie/services/recipe/recipe_data_service.py
View File

@@ -12,14 +12,17 @@ from mealie.services._base_service import BaseService
_FIREFOX_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"
async def gather_with_concurrency(n, *coros):
async def gather_with_concurrency(n, *coros, ignore_exceptions=False):
semaphore = asyncio.Semaphore(n)
async def sem_coro(coro):
async with semaphore:
return await coro
return await asyncio.gather(*(sem_coro(c) for c in coros))
results = await asyncio.gather(*(sem_coro(c) for c in coros), return_exceptions=ignore_exceptions)
if ignore_exceptions:
results = [r for r in results if not isinstance(r, Exception)]
return results
async def largest_content_len(urls: list[str]) -> tuple[str, int]:
@@ -31,7 +34,7 @@ async def largest_content_len(urls: list[str]) -> tuple[str, int]:
async with AsyncClient() as client:
tasks = [do(client, url) for url in urls]
responses: list[Response] = await gather_with_concurrency(10, *tasks)
responses: list[Response] = await gather_with_concurrency(10, *tasks, ignore_exceptions=True)
for response in responses:
len_int = int(response.headers.get("Content-Length", 0))
if len_int > largest_len: