Mirrors/mealie
1
0
Fork 0
mirror of https://github.com/mealie-recipes/mealie.git synced 2026-04-14 08:55:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: Don't continue parsing recipes with errored HTTP status codes (#7230)

Browse Source
This commit is contained in:
Dean Malan
2026-03-12 15:58:40 +02:00
committed by GitHub
parent 8189416495
commit 6a3b38a31e
4 changed files with 14 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
mealie/services/scraper/recipe_scraper.py
View File

@@ -44,6 +44,10 @@ class RecipeScraper:
"""
raw_html = html or await safe_scrape_html(url)
if not raw_html:
return None, None
for ScraperClass in self.scrapers:
scraper = ScraperClass(url, self.translator, raw_html=raw_html)
if not scraper.can_scrape():

4
mealie/services/scraper/scraper_strategies.py
View File

@@ -70,8 +70,8 @@ async def safe_scrape_html(url: str) -> str:
headers=user_agents_manager.get_scrape_headers(user_agent),
follow_redirects=True,
) as resp:
if resp.status_code == status.HTTP_403_FORBIDDEN:
logger.debug(f'403 Forbidden with User-Agent: "{user_agent}"')
if resp.status_code >= status.HTTP_400_BAD_REQUEST:
logger.debug(f'Error status code {resp.status_code} with User-Agent: "{user_agent}"')
continue
start_time = time.time()