From c93066980a64ca8b417a5b49b5fdc5effc189a95 Mon Sep 17 00:00:00 2001 From: yym68686 Date: Wed, 27 Sep 2023 23:30:20 +0800 Subject: [PATCH] change the webpage size limit to 5,000,000 --- README.md | 2 +- agent.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e6fbf31a..a3c97992 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Join the [Telegram Group](https://t.me/+_01cz9tAkUc1YzZl) chat to share your user experience or report Bugs. -[English](./README.md)|[Simplified Chinese](./README.zh-CN.md)|[Traditional Chinese](./README.zh-TW.md) +[English](./README.md) | [Simplified Chinese](./README.zh-CN.md) | [Traditional Chinese](./README.zh-TW.md) ## ✨ Features diff --git a/agent.py b/agent.py index 387da35c..9864adf0 100644 --- a/agent.py +++ b/agent.py @@ -218,7 +218,7 @@ def Web_crawler(url: str) -> str: requests.packages.urllib3.disable_warnings() response = requests.get(url, headers=headers, verify=False, timeout=5, stream=True) content_length = int(response.headers.get('Content-Length', 0)) - if content_length > 500000: + if content_length > 5000000: print("Skipping large file:", url) return result soup = BeautifulSoup(response.text.encode(response.encoding), 'lxml', from_encoding='utf-8')