Skip to content

Commit 5886641

Browse files
voidstackloopvoidstackloop
committed
Extract network tool group into agent-tools/network.ts
Moves web_search/fetch_url/http_request, the shared htmlToText helper, and the NETWORK_TOOLS gate set out of agent-tools.ts. The gate check itself stays in executeTool. Pure move, dispatch is unchanged.
1 parent 5e19567 commit 5886641

2 files changed

Lines changed: 137 additions & 135 deletions

File tree

‎app/src/agent-tools.ts‎

Lines changed: 4 additions & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ import {
3535
killBackgroundCommandsForWorkspace,
3636
} from "./agent-tools/execution";
3737
import { gitStatus, gitDiff, gitLog, gitCommit } from "./agent-tools/git";
38+
import { NETWORK_TOOLS, fetchUrl, httpRequest, webSearch } from "./agent-tools/network";
39+
import type { WebSearchResult } from "./agent-tools/network";
3840

3941
export { AGENT_TOOLS };
4042
export {
@@ -68,126 +70,8 @@ export {
6870
killBackgroundCommandsForWorkspace,
6971
};
7072
export { gitStatus, gitDiff, gitLog, gitCommit };
71-
72-
const WEB_FETCH_TIMEOUT_MS = 15_000;
73-
const MAX_FETCH_CHARS = 30_000;
74-
const MAX_SEARCH_RESULTS_WEB = 5;
75-
76-
// Crude HTML-to-text: drop non-content tags outright, then strip remaining
77-
// markup and collapse whitespace. Not a real HTML parser — good enough for
78-
// giving a model readable page text without pulling in a DOM library in the
79-
// main process.
80-
function htmlToText(html: string): string {
81-
return html
82-
.replace(/<script[\s\S]*?<\/script>/gi, " ")
83-
.replace(/<style[\s\S]*?<\/style>/gi, " ")
84-
.replace(/<!--[\s\S]*?-->/g, " ")
85-
.replace(/<[^>]+>/g, " ")
86-
.replace(/&nbsp;/g, " ")
87-
.replace(/&amp;/g, "&")
88-
.replace(/&lt;/g, "<")
89-
.replace(/&gt;/g, ">")
90-
.replace(/&quot;/g, '"')
91-
.replace(/&#39;/g, "'")
92-
.replace(/[ \t]+/g, " ")
93-
.replace(/\n\s*\n+/g, "\n\n")
94-
.trim();
95-
}
96-
97-
export async function fetchUrl(url: string): Promise<string> {
98-
let parsed: URL;
99-
try {
100-
parsed = new URL(url);
101-
} catch {
102-
throw new Error(`"${url}" is not a valid URL.`);
103-
}
104-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
105-
throw new Error("Only http:// and https:// URLs can be fetched.");
106-
}
107-
108-
const res = await fetch(parsed, {
109-
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
110-
headers: { "User-Agent": "Mozilla/5.0 (compatible; Modelforge/1.0)" },
111-
});
112-
if (!res.ok) throw new Error(`Failed to fetch ${url}: HTTP ${res.status}`);
113-
const contentType = res.headers.get("content-type") ?? "";
114-
const raw = await res.text();
115-
const text = contentType.includes("html") ? htmlToText(raw) : raw;
116-
return text.length > MAX_FETCH_CHARS
117-
? `${text.slice(0, MAX_FETCH_CHARS)}\n\n[truncated — page is ${text.length} characters]`
118-
: text;
119-
}
120-
121-
const MAX_HTTP_BODY_CHARS = 20_000;
122-
123-
export async function httpRequest(
124-
url: string,
125-
method = "GET",
126-
headers?: Record<string, string>,
127-
body?: string
128-
): Promise<string> {
129-
let parsed: URL;
130-
try {
131-
parsed = new URL(url);
132-
} catch {
133-
throw new Error(`"${url}" is not a valid URL.`);
134-
}
135-
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
136-
throw new Error("Only http:// and https:// URLs can be requested.");
137-
}
138-
139-
const res = await fetch(parsed, {
140-
method,
141-
headers,
142-
body: method !== "GET" && method !== "DELETE" ? body : undefined,
143-
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
144-
});
145-
const text = await res.text();
146-
const truncated =
147-
text.length > MAX_HTTP_BODY_CHARS
148-
? `${text.slice(0, MAX_HTTP_BODY_CHARS)}\n[truncated — body is ${text.length} characters]`
149-
: text;
150-
return `HTTP ${res.status} ${res.statusText}\n--- body ---\n${truncated || "(empty body)"}`;
151-
}
152-
153-
export interface WebSearchResult {
154-
title: string;
155-
url: string;
156-
snippet: string;
157-
}
158-
159-
// Uses DuckDuckGo's keyless HTML endpoint (no API key/signup needed, unlike
160-
// most search APIs) and regex-scrapes the result markup — brittle if DDG
161-
// changes its HTML, but keeps this tool usable out of the box with zero
162-
// configuration, consistent with the rest of Agent mode's tools.
163-
export async function webSearch(query: string): Promise<WebSearchResult[]> {
164-
const res = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, {
165-
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
166-
headers: { "User-Agent": "Mozilla/5.0 (compatible; Modelforge/1.0)" },
167-
});
168-
if (!res.ok) throw new Error(`Web search failed: HTTP ${res.status}`);
169-
const html = await res.text();
170-
171-
const results: WebSearchResult[] = [];
172-
const linkPattern = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
173-
const snippetPattern = /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
174-
const links = [...html.matchAll(linkPattern)];
175-
const snippets = [...html.matchAll(snippetPattern)];
176-
177-
for (let i = 0; i < links.length && results.length < MAX_SEARCH_RESULTS_WEB; i++) {
178-
const href = links[i][1];
179-
// DuckDuckGo's HTML endpoint wraps result URLs in a redirect
180-
// (/l/?uddg=<encoded target>) rather than linking straight to them.
181-
const uddgMatch = href.match(/[?&]uddg=([^&]+)/);
182-
const url = uddgMatch ? decodeURIComponent(uddgMatch[1]) : href;
183-
results.push({
184-
title: htmlToText(links[i][2]),
185-
url,
186-
snippet: htmlToText(snippets[i]?.[1] ?? ""),
187-
});
188-
}
189-
return results;
190-
}
73+
export { fetchUrl, httpRequest, webSearch };
74+
export type { WebSearchResult };
19175

19276
function requireGitHubToken(): string {
19377
const token = getAccountToken("github");
@@ -267,21 +151,6 @@ export async function githubReadFile(repository: string, filePath: string, ref?:
267151
return content.length > MAX_READ_CHARS ? `${content.slice(0, MAX_READ_CHARS)}\n\n[truncated]` : content;
268152
}
269153

270-
// Tools that reach the network — gated by settings.networkToolsEnabled as a
271-
// baseline that's 100% enforceable on every platform (refusing to run the
272-
// tool at all, rather than trying to block network access after the fact,
273-
// which is what command-sandbox.ts's per-call `network` argument does for
274-
// run_command/run_code/start_background_command instead).
275-
const NETWORK_TOOLS = new Set([
276-
"web_search",
277-
"fetch_url",
278-
"http_request",
279-
"capture_page_screenshot",
280-
"github_list_repositories",
281-
"github_repository_tree",
282-
"github_read_file",
283-
]);
284-
285154
export async function executeTool(workspaceRoot: string, name: string, args: Record<string, unknown>): Promise<unknown> {
286155
if (NETWORK_TOOLS.has(name) && settingsStore.getSettings().networkToolsEnabled === false) {
287156
throw new Error(`Network access for agent tools is turned off in Settings — "${name}" can't run.`);

‎app/src/agent-tools/network.ts‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
const WEB_FETCH_TIMEOUT_MS = 15_000;
2+
const MAX_FETCH_CHARS = 30_000;
3+
const MAX_SEARCH_RESULTS_WEB = 5;
4+
const MAX_HTTP_BODY_CHARS = 20_000;
5+
6+
// Tools that reach the network — gated by settings.networkToolsEnabled as a
7+
// baseline that's 100% enforceable on every platform (refusing to run the
8+
// tool at all, rather than trying to block network access after the fact,
9+
// which is what command-sandbox.ts's per-call `network` argument does for
10+
// run_command/run_code/start_background_command instead).
11+
export const NETWORK_TOOLS = new Set([
12+
"web_search",
13+
"fetch_url",
14+
"http_request",
15+
"capture_page_screenshot",
16+
"github_list_repositories",
17+
"github_repository_tree",
18+
"github_read_file",
19+
]);
20+
21+
// Crude HTML-to-text: drop non-content tags outright, then strip remaining
22+
// markup and collapse whitespace. Not a real HTML parser — good enough for
23+
// giving a model readable page text without pulling in a DOM library in the
24+
// main process.
25+
export function htmlToText(html: string): string {
26+
return html
27+
.replace(/<script[\s\S]*?<\/script>/gi, " ")
28+
.replace(/<style[\s\S]*?<\/style>/gi, " ")
29+
.replace(/<!--[\s\S]*?-->/g, " ")
30+
.replace(/<[^>]+>/g, " ")
31+
.replace(/&nbsp;/g, " ")
32+
.replace(/&amp;/g, "&")
33+
.replace(/&lt;/g, "<")
34+
.replace(/&gt;/g, ">")
35+
.replace(/&quot;/g, '"')
36+
.replace(/&#39;/g, "'")
37+
.replace(/[ \t]+/g, " ")
38+
.replace(/\n\s*\n+/g, "\n\n")
39+
.trim();
40+
}
41+
42+
export async function fetchUrl(url: string): Promise<string> {
43+
let parsed: URL;
44+
try {
45+
parsed = new URL(url);
46+
} catch {
47+
throw new Error(`"${url}" is not a valid URL.`);
48+
}
49+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
50+
throw new Error("Only http:// and https:// URLs can be fetched.");
51+
}
52+
53+
const res = await fetch(parsed, {
54+
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
55+
headers: { "User-Agent": "Mozilla/5.0 (compatible; Modelforge/1.0)" },
56+
});
57+
if (!res.ok) throw new Error(`Failed to fetch ${url}: HTTP ${res.status}`);
58+
const contentType = res.headers.get("content-type") ?? "";
59+
const raw = await res.text();
60+
const text = contentType.includes("html") ? htmlToText(raw) : raw;
61+
return text.length > MAX_FETCH_CHARS
62+
? `${text.slice(0, MAX_FETCH_CHARS)}\n\n[truncated — page is ${text.length} characters]`
63+
: text;
64+
}
65+
66+
export async function httpRequest(
67+
url: string,
68+
method = "GET",
69+
headers?: Record<string, string>,
70+
body?: string
71+
): Promise<string> {
72+
let parsed: URL;
73+
try {
74+
parsed = new URL(url);
75+
} catch {
76+
throw new Error(`"${url}" is not a valid URL.`);
77+
}
78+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
79+
throw new Error("Only http:// and https:// URLs can be requested.");
80+
}
81+
82+
const res = await fetch(parsed, {
83+
method,
84+
headers,
85+
body: method !== "GET" && method !== "DELETE" ? body : undefined,
86+
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
87+
});
88+
const text = await res.text();
89+
const truncated =
90+
text.length > MAX_HTTP_BODY_CHARS
91+
? `${text.slice(0, MAX_HTTP_BODY_CHARS)}\n[truncated — body is ${text.length} characters]`
92+
: text;
93+
return `HTTP ${res.status} ${res.statusText}\n--- body ---\n${truncated || "(empty body)"}`;
94+
}
95+
96+
export interface WebSearchResult {
97+
title: string;
98+
url: string;
99+
snippet: string;
100+
}
101+
102+
// Uses DuckDuckGo's keyless HTML endpoint (no API key/signup needed, unlike
103+
// most search APIs) and regex-scrapes the result markup — brittle if DDG
104+
// changes its HTML, but keeps this tool usable out of the box with zero
105+
// configuration, consistent with the rest of Agent mode's tools.
106+
export async function webSearch(query: string): Promise<WebSearchResult[]> {
107+
const res = await fetch(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, {
108+
signal: AbortSignal.timeout(WEB_FETCH_TIMEOUT_MS),
109+
headers: { "User-Agent": "Mozilla/5.0 (compatible; Modelforge/1.0)" },
110+
});
111+
if (!res.ok) throw new Error(`Web search failed: HTTP ${res.status}`);
112+
const html = await res.text();
113+
114+
const results: WebSearchResult[] = [];
115+
const linkPattern = /<a[^>]*class="result__a"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
116+
const snippetPattern = /<a[^>]*class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
117+
const links = [...html.matchAll(linkPattern)];
118+
const snippets = [...html.matchAll(snippetPattern)];
119+
120+
for (let i = 0; i < links.length && results.length < MAX_SEARCH_RESULTS_WEB; i++) {
121+
const href = links[i][1];
122+
// DuckDuckGo's HTML endpoint wraps result URLs in a redirect
123+
// (/l/?uddg=<encoded target>) rather than linking straight to them.
124+
const uddgMatch = href.match(/[?&]uddg=([^&]+)/);
125+
const url = uddgMatch ? decodeURIComponent(uddgMatch[1]) : href;
126+
results.push({
127+
title: htmlToText(links[i][2]),
128+
url,
129+
snippet: htmlToText(snippets[i]?.[1] ?? ""),
130+
});
131+
}
132+
return results;
133+
}

0 commit comments

Comments
 (0)