Skip to content

Commit c36c0d2

Browse files
kyteinskykyteinsky
authored
fix: improve tool definitions for web_fetch and get_file_content_by_file_link (#228)
* fix: disambiguate between nextcloud internal file links tool and web_fetch Signed-off-by: kyteinsky <kyteinsky@gmail.com> Assisted-by: Github Copilot: claude-opus-4-6 * fix: web_fetch redirect to internal file fetch for regex-matched links also more hardenings of the fetches: - don't fetch folders - limit file/content size to 100kB - limit mimetype to text-like Signed-off-by: kyteinsky <kyteinsky@gmail.com> Assisted-by: Github Copilot: claude-opus-4-6 --------- Signed-off-by: kyteinsky <kyteinsky@gmail.com>
2 parents d220114 + 290a41c commit c36c0d2

3 files changed

Lines changed: 131 additions & 63 deletions

File tree

‎ex_app/lib/all_tools/files.py‎

Lines changed: 8 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22
# SPDX-License-Identifier: AGPL-3.0-or-later
33
import xml.etree.ElementTree as ET
44
from urllib.parse import unquote
5-
import niquests
5+
66
from langchain_core.tools import tool
77
from nc_py_api import AsyncNextcloudApp
88
from nc_py_api.files.files_async import AsyncFilesAPI, FsNode
99

1010
from ex_app.lib.all_tools.lib.decorator import dangerous_tool, safe_tool
11-
from ex_app.lib.all_tools.lib.files import get_file_id_from_file_url
11+
from ex_app.lib.all_tools.lib.files import format_fs_node, get_file_content_from_int_link, get_file_id_from_file_url
1212

1313

1414
def _validate_path(path: str) -> str:
@@ -45,58 +45,14 @@ async def get_file_content(file_path: str):
4545
@safe_tool
4646
async def get_file_content_by_file_link(file_url: str):
4747
"""
48-
Get the content of a file given an internal Nextcloud link (e.g., https://host/index.php/f/12345)
49-
:param file_url: the nextcloud-internal file URL
48+
Get the content of a Nextcloud-internal file using its internal file link.
49+
This is NOT for fetching arbitrary web URLs, use web_fetch for those.
50+
Only use this tool when the URL points to a file stored in Nextcloud (e.g., https://cloud.example.com/index.php/f/12345 or https://cloud.example.com/f/12345).
51+
:param file_url: a Nextcloud-internal file URL (must match the pattern https://<host>/f/<fileId> or https://<host>/index.php/f/<fileId>)
5052
:return: text content of the file
5153
"""
5254

53-
file_id = get_file_id_from_file_url(file_url)
54-
# Generate a direct download link using the fileId
55-
info = await nc.ocs('POST', '/ocs/v2.php/apps/dav/api/v1/direct', json={'fileId': file_id}, response_type='json')
56-
download_url = info.get('ocs', {}).get('data', {}).get('url', None)
57-
58-
if not download_url:
59-
raise Exception('Could not generate download URL from file id')
60-
61-
# Download the file from the direct download URL
62-
response = await niquests.async_api.get(download_url)
63-
64-
return response.text
65-
66-
def __format_fs_node(fsnode: FsNode) -> dict:
67-
# todo: permissions info
68-
return {
69-
'path': fsnode.user_path,
70-
'file_id': fsnode.info.fileid,
71-
'etag': fsnode.etag.replace('"', '').replace("'", ''),
72-
'bytes': fsnode.info.size,
73-
'creation_date': fsnode.info.creation_date.isoformat(),
74-
'last_modified': fsnode.info.last_modified.isoformat(),
75-
'mimetype': fsnode.info.mimetype,
76-
'is_shared': fsnode.is_shared,
77-
'is_favourite': fsnode.info.favorite,
78-
'is_version': fsnode.info.is_version,
79-
'trash_info': {
80-
'in_trash': fsnode.info.in_trash,
81-
**({
82-
'trashbin_filename': fsnode.info.trashbin_filename,
83-
'original_location': fsnode.info.trashbin_original_location,
84-
'deletion_time': fsnode.info.trashbin_deletion_time,
85-
} if fsnode.info.in_trash else {}),
86-
},
87-
'lock_info': {
88-
'is_locked': fsnode.lock_info.is_locked,
89-
**({
90-
'owner': fsnode.lock_info.owner,
91-
'owner_display_name': fsnode.lock_info.owner_display_name,
92-
'type': fsnode.lock_info.type.name,
93-
'creation_time': fsnode.lock_info.lock_creation_time,
94-
'ttl': fsnode.lock_info.lock_ttl,
95-
'locked_by_app': fsnode.lock_info.owner_editor,
96-
} if fsnode.lock_info.is_locked else {}),
97-
},
98-
}
99-
55+
return await get_file_content_from_int_link(nc, file_url)
10056

10157
@tool
10258
@safe_tool
@@ -112,7 +68,7 @@ async def get_file_tree(path: str = '/', include_metadata = False, depth: int =
11268
files_handle = AsyncFilesAPI(nc._session)
11369
fsnode_list = await files_handle.listdir(path, min(5, depth))
11470
if include_metadata:
115-
return [__format_fs_node(fsnode) for fsnode in fsnode_list]
71+
return [format_fs_node(fsnode) for fsnode in fsnode_list]
11672

11773
return [fsnode.user_path for fsnode in fsnode_list]
11874

‎ex_app/lib/all_tools/lib/files.py‎

Lines changed: 94 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,100 @@
22
# SPDX-License-Identifier: AGPL-3.0-or-later
33
import re
44

5-
def get_file_id_from_file_url(file_url) -> int:
6-
# Define the regex pattern to capture only the digits
7-
pattern = r"https?://[a-zA-Z-_.:0-9]+/(index\.php/)?f/(\d+)"
8-
match = re.search(pattern, file_url)
5+
from nc_py_api import AsyncNextcloudApp
6+
from nc_py_api.files.files_async import AsyncFilesAPI, FsNode
97

8+
TEXT_LIKE_MIMETYPE_PARTS = ('text/', 'application/json', 'application/xml', 'application/xhtml', '+xml', '+json')
9+
MAX_FILE_SIZE = 100_000 # 100kB, approx. 25k tokens
10+
11+
_FILE_ID_SUFFIX_RE = re.compile(r'/(index\.php/)?f/(\d+)/?$')
12+
13+
14+
def _strip_scheme(url: str) -> str:
15+
"""Remove the http(s):// scheme from a URL for scheme-agnostic comparison."""
16+
return re.sub(r'^https?://', '', url)
17+
18+
19+
def is_an_internal_file_link(nc: AsyncNextcloudApp, url: str) -> bool:
20+
"""Check whether a URL is an internal file link for this Nextcloud instance (scheme-agnostic)."""
21+
nc_host = _strip_scheme(nc.app_cfg.endpoint.rstrip('/'))
22+
url_without_scheme = _strip_scheme(url)
23+
return url_without_scheme.startswith(nc_host) and bool(_FILE_ID_SUFFIX_RE.search(url_without_scheme[len(nc_host):]))
24+
25+
26+
def get_file_id_from_file_url(file_url: str) -> int:
27+
"""Extract the numeric file ID from a Nextcloud-internal file URL."""
28+
match = _FILE_ID_SUFFIX_RE.search(file_url)
1029
if match:
1130
return int(match.group(2))
12-
else:
13-
raise Exception("Not a valid nextcloud file URL")
31+
raise Exception("Not a valid nextcloud file URL")
32+
33+
34+
def format_fs_node(fsnode: FsNode) -> dict:
35+
# todo: permissions info
36+
return {
37+
'path': fsnode.user_path,
38+
'file_id': fsnode.info.fileid,
39+
'etag': fsnode.etag.replace('"', '').replace("'", ''),
40+
'bytes': fsnode.info.size,
41+
'creation_date': fsnode.info.creation_date.isoformat(),
42+
'last_modified': fsnode.info.last_modified.isoformat(),
43+
'mimetype': fsnode.info.mimetype,
44+
'is_shared': fsnode.is_shared,
45+
'is_favourite': fsnode.info.favorite,
46+
'is_version': fsnode.info.is_version,
47+
'trash_info': {
48+
'in_trash': fsnode.info.in_trash,
49+
**({
50+
'trashbin_filename': fsnode.info.trashbin_filename,
51+
'original_location': fsnode.info.trashbin_original_location,
52+
'deletion_time': fsnode.info.trashbin_deletion_time,
53+
} if fsnode.info.in_trash else {}),
54+
},
55+
'lock_info': {
56+
'is_locked': fsnode.lock_info.is_locked,
57+
**({
58+
'owner': fsnode.lock_info.owner,
59+
'owner_display_name': fsnode.lock_info.owner_display_name,
60+
'type': fsnode.lock_info.type.name,
61+
'creation_time': fsnode.lock_info.lock_creation_time,
62+
'ttl': fsnode.lock_info.lock_ttl,
63+
'locked_by_app': fsnode.lock_info.owner_editor,
64+
} if fsnode.lock_info.is_locked else {}),
65+
},
66+
}
67+
68+
69+
async def get_file_node(nc: AsyncNextcloudApp, file_id: int) -> FsNode:
70+
files_handle = AsyncFilesAPI(nc._session)
71+
node = await files_handle.by_id(file_id)
72+
if not node:
73+
raise RuntimeError(f'No file/folder found with id: {file_id}')
74+
return node
75+
76+
77+
async def get_file_contents(nc: AsyncNextcloudApp, fsnode: FsNode) -> str:
78+
"""
79+
RuntimeError: just return the metadata to the model since the node is one of the following:
80+
- a folder
81+
- very large in size
82+
- non-text mimetype
83+
"""
84+
if fsnode.is_dir:
85+
raise RuntimeError('Folder found at the given file id, skipping download')
86+
if fsnode.info.content_length > MAX_FILE_SIZE:
87+
raise RuntimeError(f'File id {fsnode.info.fileid} is too large to download at {fsnode.info.content_length} bytes')
88+
if not any(t in fsnode.info.mimetype for t in TEXT_LIKE_MIMETYPE_PARTS):
89+
raise RuntimeError(f'File id {fsnode.info.fileid} is of content type {fsnode.info.mimetype} so cannot be displayed as text')
90+
files_handle = AsyncFilesAPI(nc._session)
91+
return (await files_handle.download(fsnode)).decode(encoding='utf-8', errors='ignore')
92+
93+
94+
async def get_file_content_from_int_link(nc: AsyncNextcloudApp, url: str) -> str:
95+
file_id = get_file_id_from_file_url(url)
96+
fsnode = await get_file_node(nc, file_id)
97+
98+
try:
99+
return await get_file_contents(nc, fsnode)
100+
except RuntimeError as e:
101+
return f'Failed to download the file/folder: {e}.\nMore info about the node:{format_fs_node(fsnode)}'

‎ex_app/lib/all_tools/web.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
from nc_py_api import AsyncNextcloudApp
66

77
from ex_app.lib.all_tools.lib.decorator import safe_tool
8+
from ex_app.lib.all_tools.lib.files import (
9+
MAX_FILE_SIZE,
10+
TEXT_LIKE_MIMETYPE_PARTS,
11+
get_file_content_from_int_link,
12+
is_an_internal_file_link,
13+
)
814

915

1016
async def get_tools(nc: AsyncNextcloudApp):
@@ -13,12 +19,30 @@ async def get_tools(nc: AsyncNextcloudApp):
1319
@safe_tool
1420
async def web_fetch(url: str) -> str:
1521
"""
16-
Get the contents of a web page via HTTP
17-
:param url: The HTTP URL to the web page (e.g. https://nextcloud.com/team/ )
18-
:return: the web page content
22+
Fetch the contents of an external web page via HTTP.
23+
Use this for any URL on the public internet or intranet (e.g., https://www.eff.org/).
24+
:param url: the HTTP(S) URL of the web page to fetch
25+
:return: the raw web page content (HTML, JSON, etc.)
1926
"""
20-
res = await niquests.get(url)
21-
return res.text()
27+
# Detect Nextcloud-internal file links and fetch via the internal API, for the models that would still call this tool.
28+
if is_an_internal_file_link(nc, url):
29+
return await get_file_content_from_int_link(nc, url)
30+
31+
# Pre-flight HEAD request to check content type and size before downloading
32+
head_res = await niquests.async_api.head(url, allow_redirects=True)
33+
34+
content_type = head_res.headers.get('Content-Type', '')
35+
if not any(t in content_type for t in TEXT_LIKE_MIMETYPE_PARTS):
36+
return f"(binary or unknown content detected: {content_type or 'no Content-Type header'}. Cannot display as text.)"
37+
38+
# download first 100kB of the file
39+
res = await niquests.async_api.get(url, headers={'Range': f'bytes=0-{MAX_FILE_SIZE - 1}'})
40+
text = res.text or "(empty content)"
41+
42+
if res.status_code == 206:
43+
text += "\n\n(truncated: only the first 100 kB of content was fetched.)"
44+
45+
return text
2246

2347
return [
2448
web_fetch,

0 commit comments

Comments
 (0)