Skip to content

Commit 290a41c

Browse files
kyteinskykyteinsky
committed
fix: web_fetch redirect to internal file fetch for regex-matched links
also more hardenings of the fetches: - don't fetch folders - limit file/content size to 100kB - limit mimetype to text-like Signed-off-by: kyteinsky <kyteinsky@gmail.com> Assisted-by: Github Copilot: claude-opus-4-6
1 parent c29d561 commit 290a41c

3 files changed

Lines changed: 122 additions & 59 deletions

File tree

‎ex_app/lib/all_tools/files.py‎

Lines changed: 3 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@
33
import xml.etree.ElementTree as ET
44
from urllib.parse import unquote
55

6-
import niquests
76
from langchain_core.tools import tool
87
from nc_py_api import AsyncNextcloudApp
98
from nc_py_api.files.files_async import AsyncFilesAPI, FsNode
109

1110
from ex_app.lib.all_tools.lib.decorator import dangerous_tool, safe_tool
12-
from ex_app.lib.all_tools.lib.files import get_file_id_from_file_url
11+
from ex_app.lib.all_tools.lib.files import format_fs_node, get_file_content_from_int_link, get_file_id_from_file_url
1312

1413

1514
def _validate_path(path: str) -> str:
@@ -53,53 +52,7 @@ async def get_file_content_by_file_link(file_url: str):
5352
:return: text content of the file
5453
"""
5554

56-
file_id = get_file_id_from_file_url(file_url)
57-
# Generate a direct download link using the fileId
58-
info = await nc.ocs('POST', '/ocs/v2.php/apps/dav/api/v1/direct', json={'fileId': file_id}, response_type='json')
59-
download_url = info.get('ocs', {}).get('data', {}).get('url', None)
60-
61-
if not download_url:
62-
raise Exception('Could not generate download URL from file id')
63-
64-
# Download the file from the direct download URL
65-
response = await niquests.async_api.get(download_url)
66-
67-
return response.text
68-
69-
def __format_fs_node(fsnode: FsNode) -> dict:
70-
# todo: permissions info
71-
return {
72-
'path': fsnode.user_path,
73-
'file_id': fsnode.info.fileid,
74-
'etag': fsnode.etag.replace('"', '').replace("'", ''),
75-
'bytes': fsnode.info.size,
76-
'creation_date': fsnode.info.creation_date.isoformat(),
77-
'last_modified': fsnode.info.last_modified.isoformat(),
78-
'mimetype': fsnode.info.mimetype,
79-
'is_shared': fsnode.is_shared,
80-
'is_favourite': fsnode.info.favorite,
81-
'is_version': fsnode.info.is_version,
82-
'trash_info': {
83-
'in_trash': fsnode.info.in_trash,
84-
**({
85-
'trashbin_filename': fsnode.info.trashbin_filename,
86-
'original_location': fsnode.info.trashbin_original_location,
87-
'deletion_time': fsnode.info.trashbin_deletion_time,
88-
} if fsnode.info.in_trash else {}),
89-
},
90-
'lock_info': {
91-
'is_locked': fsnode.lock_info.is_locked,
92-
**({
93-
'owner': fsnode.lock_info.owner,
94-
'owner_display_name': fsnode.lock_info.owner_display_name,
95-
'type': fsnode.lock_info.type.name,
96-
'creation_time': fsnode.lock_info.lock_creation_time,
97-
'ttl': fsnode.lock_info.lock_ttl,
98-
'locked_by_app': fsnode.lock_info.owner_editor,
99-
} if fsnode.lock_info.is_locked else {}),
100-
},
101-
}
102-
55+
return await get_file_content_from_int_link(nc, file_url)
10356

10457
@tool
10558
@safe_tool
@@ -115,7 +68,7 @@ async def get_file_tree(path: str = '/', include_metadata = False, depth: int =
11568
files_handle = AsyncFilesAPI(nc._session)
11669
fsnode_list = await files_handle.listdir(path, min(5, depth))
11770
if include_metadata:
118-
return [__format_fs_node(fsnode) for fsnode in fsnode_list]
71+
return [format_fs_node(fsnode) for fsnode in fsnode_list]
11972

12073
return [fsnode.user_path for fsnode in fsnode_list]
12174

‎ex_app/lib/all_tools/lib/files.py‎

Lines changed: 94 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,100 @@
22
# SPDX-License-Identifier: AGPL-3.0-or-later
33
import re
44

5-
def get_file_id_from_file_url(file_url) -> int:
6-
# Define the regex pattern to capture only the digits
7-
pattern = r"https?://[a-zA-Z-_.:0-9]+/(index\.php/)?f/(\d+)"
8-
match = re.search(pattern, file_url)
5+
from nc_py_api import AsyncNextcloudApp
6+
from nc_py_api.files.files_async import AsyncFilesAPI, FsNode
97

8+
TEXT_LIKE_MIMETYPE_PARTS = ('text/', 'application/json', 'application/xml', 'application/xhtml', '+xml', '+json')
9+
MAX_FILE_SIZE = 100_000 # 100kB, approx. 25k tokens
10+
11+
_FILE_ID_SUFFIX_RE = re.compile(r'/(index\.php/)?f/(\d+)/?$')
12+
13+
14+
def _strip_scheme(url: str) -> str:
15+
"""Remove the http(s):// scheme from a URL for scheme-agnostic comparison."""
16+
return re.sub(r'^https?://', '', url)
17+
18+
19+
def is_an_internal_file_link(nc: AsyncNextcloudApp, url: str) -> bool:
20+
"""Check whether a URL is an internal file link for this Nextcloud instance (scheme-agnostic)."""
21+
nc_host = _strip_scheme(nc.app_cfg.endpoint.rstrip('/'))
22+
url_without_scheme = _strip_scheme(url)
23+
return url_without_scheme.startswith(nc_host) and bool(_FILE_ID_SUFFIX_RE.search(url_without_scheme[len(nc_host):]))
24+
25+
26+
def get_file_id_from_file_url(file_url: str) -> int:
27+
"""Extract the numeric file ID from a Nextcloud-internal file URL."""
28+
match = _FILE_ID_SUFFIX_RE.search(file_url)
1029
if match:
1130
return int(match.group(2))
12-
else:
13-
raise Exception("Not a valid nextcloud file URL")
31+
raise Exception("Not a valid nextcloud file URL")
32+
33+
34+
def format_fs_node(fsnode: FsNode) -> dict:
35+
# todo: permissions info
36+
return {
37+
'path': fsnode.user_path,
38+
'file_id': fsnode.info.fileid,
39+
'etag': fsnode.etag.replace('"', '').replace("'", ''),
40+
'bytes': fsnode.info.size,
41+
'creation_date': fsnode.info.creation_date.isoformat(),
42+
'last_modified': fsnode.info.last_modified.isoformat(),
43+
'mimetype': fsnode.info.mimetype,
44+
'is_shared': fsnode.is_shared,
45+
'is_favourite': fsnode.info.favorite,
46+
'is_version': fsnode.info.is_version,
47+
'trash_info': {
48+
'in_trash': fsnode.info.in_trash,
49+
**({
50+
'trashbin_filename': fsnode.info.trashbin_filename,
51+
'original_location': fsnode.info.trashbin_original_location,
52+
'deletion_time': fsnode.info.trashbin_deletion_time,
53+
} if fsnode.info.in_trash else {}),
54+
},
55+
'lock_info': {
56+
'is_locked': fsnode.lock_info.is_locked,
57+
**({
58+
'owner': fsnode.lock_info.owner,
59+
'owner_display_name': fsnode.lock_info.owner_display_name,
60+
'type': fsnode.lock_info.type.name,
61+
'creation_time': fsnode.lock_info.lock_creation_time,
62+
'ttl': fsnode.lock_info.lock_ttl,
63+
'locked_by_app': fsnode.lock_info.owner_editor,
64+
} if fsnode.lock_info.is_locked else {}),
65+
},
66+
}
67+
68+
69+
async def get_file_node(nc: AsyncNextcloudApp, file_id: int) -> FsNode:
70+
files_handle = AsyncFilesAPI(nc._session)
71+
node = await files_handle.by_id(file_id)
72+
if not node:
73+
raise RuntimeError(f'No file/folder found with id: {file_id}')
74+
return node
75+
76+
77+
async def get_file_contents(nc: AsyncNextcloudApp, fsnode: FsNode) -> str:
78+
"""
79+
RuntimeError: just return the metadata to the model since the node is one of the following:
80+
- a folder
81+
- very large in size
82+
- non-text mimetype
83+
"""
84+
if fsnode.is_dir:
85+
raise RuntimeError('Folder found at the given file id, skipping download')
86+
if fsnode.info.content_length > MAX_FILE_SIZE:
87+
raise RuntimeError(f'File id {fsnode.info.fileid} is too large to download at {fsnode.info.content_length} bytes')
88+
if not any(t in fsnode.info.mimetype for t in TEXT_LIKE_MIMETYPE_PARTS):
89+
raise RuntimeError(f'File id {fsnode.info.fileid} is of content type {fsnode.info.mimetype} so cannot be displayed as text')
90+
files_handle = AsyncFilesAPI(nc._session)
91+
return (await files_handle.download(fsnode)).decode(encoding='utf-8', errors='ignore')
92+
93+
94+
async def get_file_content_from_int_link(nc: AsyncNextcloudApp, url: str) -> str:
95+
file_id = get_file_id_from_file_url(url)
96+
fsnode = await get_file_node(nc, file_id)
97+
98+
try:
99+
return await get_file_contents(nc, fsnode)
100+
except RuntimeError as e:
101+
return f'Failed to download the file/folder: {e}.\nMore info about the node:{format_fs_node(fsnode)}'

‎ex_app/lib/all_tools/web.py‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
from nc_py_api import AsyncNextcloudApp
66

77
from ex_app.lib.all_tools.lib.decorator import safe_tool
8+
from ex_app.lib.all_tools.lib.files import (
9+
MAX_FILE_SIZE,
10+
TEXT_LIKE_MIMETYPE_PARTS,
11+
get_file_content_from_int_link,
12+
is_an_internal_file_link,
13+
)
814

915

1016
async def get_tools(nc: AsyncNextcloudApp):
@@ -14,13 +20,29 @@ async def get_tools(nc: AsyncNextcloudApp):
1420
async def web_fetch(url: str) -> str:
1521
"""
1622
Fetch the contents of an external web page via HTTP.
17-
This is NOT for Nextcloud-internal file links (like https://<host>/f/12345), use get_file_content_by_file_link for those.
1823
Use this for any URL on the public internet or intranet (e.g., https://www.eff.org/).
1924
:param url: the HTTP(S) URL of the web page to fetch
2025
:return: the raw web page content (HTML, JSON, etc.)
2126
"""
22-
res = await niquests.async_api.get(url)
23-
return res.text or "(empty content)"
27+
# Detect Nextcloud-internal file links and fetch via the internal API, for the models that would still call this tool.
28+
if is_an_internal_file_link(nc, url):
29+
return await get_file_content_from_int_link(nc, url)
30+
31+
# Pre-flight HEAD request to check content type and size before downloading
32+
head_res = await niquests.async_api.head(url, allow_redirects=True)
33+
34+
content_type = head_res.headers.get('Content-Type', '')
35+
if not any(t in content_type for t in TEXT_LIKE_MIMETYPE_PARTS):
36+
return f"(binary or unknown content detected: {content_type or 'no Content-Type header'}. Cannot display as text.)"
37+
38+
# download first 100kB of the file
39+
res = await niquests.async_api.get(url, headers={'Range': f'bytes=0-{MAX_FILE_SIZE - 1}'})
40+
text = res.text or "(empty content)"
41+
42+
if res.status_code == 206:
43+
text += "\n\n(truncated: only the first 100 kB of content was fetched.)"
44+
45+
return text
2446

2547
return [
2648
web_fetch,

0 commit comments

Comments
 (0)