Mirrors/mealie
1
0
Fork 0
mirror of https://github.com/mealie-recipes/mealie.git synced 2025-12-09 20:05:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: missing user agent for scraper (#1586)

Browse Source 
* set user agent for requests.get

* bump scraper version
This commit is contained in:
Hayden
2022-08-20 17:54:06 -08:00
committed by GitHub
parent 007b861ad6
commit 2865bcbb04
3 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mealie/services/scraper/scraper_strategies.py
View File

@@ -15,6 +15,7 @@ from mealie.services.scraper.scraped_extras import ScrapedExtras
from . import cleaner
_FIREFOX_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:86.0) Gecko/20100101 Firefox/86.0"
SCRAPER_TIMEOUT = 15
@@ -28,7 +29,7 @@ def safe_scrape_html(url: str) -> str:
if the request takes longer than 15 seconds. This is used to mitigate
DDOS attacks from users providing a url with arbitrary large content.
"""
resp = requests.get(url, timeout=SCRAPER_TIMEOUT, stream=True)
resp = requests.get(url, timeout=SCRAPER_TIMEOUT, stream=True, headers={"User-Agent": _FIREFOX_UA})
html_bytes = b""