From 237ede77200deb2d715df4b89f4ff9e6fc904eea Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Mon, 30 Mar 2026 08:57:28 +0400 Subject: [PATCH 1/3] fix(seeder): use TLS for proxy CONNECT tunnel to fix FRED fetch failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decodo gate.decodo.com:10001 requires TLS. Previous code used http.request (plain TCP) which received SOCKS5 rejection bytes instead of HTTP 200. Two issues fixed: 1. Replace http.request CONNECT with tls.connect + manual CONNECT handshake. Node.js http.request also auto-sets Host to the proxy hostname; Decodo rejects this and responds with SOCKS5 bytes (0x05 0xff). Manual CONNECT over a raw TLS socket avoids both issues. 2. Handle https:// and plain "user:pass@host:port" proxy URL formats — always uses TLS regardless of PROXY_URL prefix. _proxy-utils.cjs: resolveProxyStringConnect now preserves https:// prefix from PROXY_URL so callers can detect TLS proxies explicitly. All 24 FRED series (BAMLH0A0HYM2, FEDFUNDS, DGS10, etc.) confirmed working locally via gate.decodo.com:10001. --- scripts/_proxy-utils.cjs | 5 +++- scripts/_seed-utils.mjs | 63 +++++++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/_proxy-utils.cjs b/scripts/_proxy-utils.cjs index 94b4eb6003..4323770295 100644 --- a/scripts/_proxy-utils.cjs +++ b/scripts/_proxy-utils.cjs @@ -82,12 +82,15 @@ function resolveProxyString() { /** * Returns proxy as "user:pass@host:port" string for use with HTTP CONNECT tunneling. * Does NOT replace gate.decodo.com → us.decodo.com; CONNECT endpoint is gate.decodo.com. + * When PROXY_URL uses https:// (TLS proxy), returns "https://user:pass@host:port" so + * httpsProxyFetchJson can detect and use https.request instead of http.request. * Returns empty string if no proxy configured. */ function resolveProxyStringConnect() { const cfg = resolveProxyConfigWithFallback(); if (!cfg) return ''; - return cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`; + const base = cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`; + return cfg.tls ? `https://${base}` : base; } module.exports = { parseProxyConfig, resolveProxyConfig, resolveProxyConfigWithFallback, resolveProxyString, resolveProxyStringConnect }; diff --git a/scripts/_seed-utils.mjs b/scripts/_seed-utils.mjs index d389a280ea..b3e50fd26f 100644 --- a/scripts/_seed-utils.mjs +++ b/scripts/_seed-utils.mjs @@ -342,43 +342,60 @@ export function curlFetch(url, proxyAuth, headers = {}) { return raw.slice(0, nl); } -// Pure Node.js HTTPS-through-HTTP-proxy (CONNECT tunnel). -// Replaces curlFetch for seeder scripts running in containers without curl. -// proxyAuth format: "user:pass@host:port" +// Pure Node.js HTTPS-through-TLS-proxy (CONNECT tunnel). +// Always connects to the proxy over TLS (tls.connect), then manually sends the HTTP +// CONNECT request over the TLS socket. This works for both plain PROXY_URL values +// ("user:pass@host:port") and https:// prefixed values — always uses TLS to proxy. +// proxyAuth format: "user:pass@host:port" OR "https://user:pass@host:port" async function httpsProxyFetchJson(url, proxyAuth) { const targetUrl = new URL(url); - const atIdx = proxyAuth.lastIndexOf('@'); - const credentials = atIdx >= 0 ? proxyAuth.slice(0, atIdx) : ''; - const hostPort = atIdx >= 0 ? proxyAuth.slice(atIdx + 1) : proxyAuth; + + // Normalise proxyAuth: strip https:// prefix if present, parse user:pass@host:port. + let proxyAuthStr = proxyAuth; + if (proxyAuth.startsWith('https://') || proxyAuth.startsWith('http://')) { + const u = new URL(proxyAuth); + proxyAuthStr = (u.username ? `${decodeURIComponent(u.username)}:${decodeURIComponent(u.password)}@` : '') + `${u.hostname}:${u.port}`; + } + + const atIdx = proxyAuthStr.lastIndexOf('@'); + const credentials = atIdx >= 0 ? proxyAuthStr.slice(0, atIdx) : ''; + const hostPort = atIdx >= 0 ? proxyAuthStr.slice(atIdx + 1) : proxyAuthStr; const colonIdx = hostPort.lastIndexOf(':'); const proxyHost = hostPort.slice(0, colonIdx); const proxyPort = parseInt(hostPort.slice(colonIdx + 1), 10); - const connectHeaders = {}; - if (credentials) { - connectHeaders['Proxy-Authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`; - } + // Step 1: TLS connect to proxy (always TLS — Decodo gate.decodo.com requires it). + const proxySock = await new Promise((resolve, reject) => { + const s = tls.connect({ host: proxyHost, port: proxyPort, servername: proxyHost, ALPNProtocols: ['http/1.1'] }, () => resolve(s)); + s.on('error', reject); + }); + + // Step 2: Send HTTP CONNECT over the TLS socket manually (avoids Node.js http.request + // auto-setting Host to the proxy hostname, which Decodo rejects with SOCKS5 bytes). + const authHeader = credentials ? `\r\nProxy-Authorization: Basic ${Buffer.from(credentials).toString('base64')}` : ''; + proxySock.write(`CONNECT ${targetUrl.hostname}:443 HTTP/1.1\r\nHost: ${targetUrl.hostname}:443${authHeader}\r\n\r\n`); - const { socket } = await new Promise((resolve, reject) => { - http.request({ - host: proxyHost, port: proxyPort, - method: 'CONNECT', - path: `${targetUrl.hostname}:443`, - headers: connectHeaders, - }).on('connect', (res, socket) => { - if (res.statusCode !== 200) { - socket.destroy(); - return reject(Object.assign(new Error(`Proxy CONNECT: ${res.statusCode}`), { status: res.statusCode })); + // Step 3: Read CONNECT response (first data chunk contains the status line). + await new Promise((resolve, reject) => { + proxySock.once('data', (chunk) => { + const resp = chunk.toString('ascii'); + if (!resp.startsWith('HTTP/1.1 200') && !resp.startsWith('HTTP/1.0 200')) { + proxySock.destroy(); + return reject(Object.assign(new Error(`Proxy CONNECT: ${resp.split('\r\n')[0]}`), { status: parseInt(resp.split(' ')[1]) || 0 })); } - resolve({ socket }); - }).on('error', reject).end(); + proxySock.pause(); + resolve(); + }); + proxySock.on('error', reject); }); - const tlsSock = tls.connect({ socket, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] }); + // Step 4: TLS over the proxy tunnel (TLS-in-TLS) to reach the target server. + const tlsSock = tls.connect({ socket: proxySock, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] }); await new Promise((resolve, reject) => { tlsSock.on('secureConnect', resolve); tlsSock.on('error', reject); }); + proxySock.resume(); return new Promise((resolve, reject) => { const timer = setTimeout(() => { tlsSock.destroy(); reject(new Error('FRED proxy fetch timeout')); }, 20000); From 5fceb97e1cf469f703167638bd90dccc15ebe385 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Mon, 30 Mar 2026 09:18:17 +0400 Subject: [PATCH 2/3] fix(seeder): respect http:// proxy scheme + buffer full CONNECT response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two protocol-correctness fixes: 1. http:// proxies used plain TCP before; always-TLS regressed them. Now: bare/undeclared format → TLS (Decodo requires it), explicit http:// → plain net.connect, explicit https:// → TLS. 2. CONNECT response buffered until \r\n\r\n instead of acting on the first data chunk. Fragmented proxy responses (headers split across packets) could corrupt the TLS handshake by leaving header bytes on the wire when tls.connect() was called too early. Verified locally: BAMLH0A0HYM2 → { date: 2026-03-26, value: 3.21 } --- scripts/_seed-utils.mjs | 53 ++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/scripts/_seed-utils.mjs b/scripts/_seed-utils.mjs index b3e50fd26f..086d9c3efb 100644 --- a/scripts/_seed-utils.mjs +++ b/scripts/_seed-utils.mjs @@ -5,6 +5,7 @@ import { execFileSync } from 'node:child_process'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; +import * as net from 'node:net'; import * as http from 'node:http'; import * as tls from 'node:tls'; import * as https from 'node:https'; @@ -342,15 +343,21 @@ export function curlFetch(url, proxyAuth, headers = {}) { return raw.slice(0, nl); } -// Pure Node.js HTTPS-through-TLS-proxy (CONNECT tunnel). -// Always connects to the proxy over TLS (tls.connect), then manually sends the HTTP -// CONNECT request over the TLS socket. This works for both plain PROXY_URL values -// ("user:pass@host:port") and https:// prefixed values — always uses TLS to proxy. -// proxyAuth format: "user:pass@host:port" OR "https://user:pass@host:port" +// Pure Node.js HTTPS-through-proxy (CONNECT tunnel). +// proxyAuth format: "user:pass@host:port" (bare/Decodo → TLS) OR +// "https://user:pass@host:port" (explicit TLS) OR +// "http://user:pass@host:port" (explicit plain TCP) +// Bare/undeclared-scheme proxies always use TLS (Decodo gate.decodo.com requires it). +// Explicit http:// proxies use plain TCP to avoid breaking non-TLS setups. async function httpsProxyFetchJson(url, proxyAuth) { const targetUrl = new URL(url); - // Normalise proxyAuth: strip https:// prefix if present, parse user:pass@host:port. + // Detect whether the proxy URL specifies http:// explicitly (plain TCP) or not + // (bare format or https:// → TLS). User instruction: bare → always TLS. + const explicitHttp = proxyAuth.startsWith('http://') && !proxyAuth.startsWith('https://'); + const useTls = !explicitHttp; + + // Strip scheme prefix, parse user:pass@host:port. let proxyAuthStr = proxyAuth; if (proxyAuth.startsWith('https://') || proxyAuth.startsWith('http://')) { const u = new URL(proxyAuth); @@ -364,28 +371,40 @@ async function httpsProxyFetchJson(url, proxyAuth) { const proxyHost = hostPort.slice(0, colonIdx); const proxyPort = parseInt(hostPort.slice(colonIdx + 1), 10); - // Step 1: TLS connect to proxy (always TLS — Decodo gate.decodo.com requires it). + // Step 1: Open socket to proxy (TLS for https:// or bare, plain TCP for http://). const proxySock = await new Promise((resolve, reject) => { - const s = tls.connect({ host: proxyHost, port: proxyPort, servername: proxyHost, ALPNProtocols: ['http/1.1'] }, () => resolve(s)); - s.on('error', reject); + if (useTls) { + const s = tls.connect({ host: proxyHost, port: proxyPort, servername: proxyHost, ALPNProtocols: ['http/1.1'] }, () => resolve(s)); + s.on('error', reject); + } else { + const s = net.connect({ host: proxyHost, port: proxyPort }, () => resolve(s)); + s.on('error', reject); + } }); - // Step 2: Send HTTP CONNECT over the TLS socket manually (avoids Node.js http.request - // auto-setting Host to the proxy hostname, which Decodo rejects with SOCKS5 bytes). + // Step 2: Send HTTP CONNECT manually (avoids Node.js http.request auto-setting + // Host to the proxy hostname, which Decodo rejects with SOCKS5 bytes). const authHeader = credentials ? `\r\nProxy-Authorization: Basic ${Buffer.from(credentials).toString('base64')}` : ''; proxySock.write(`CONNECT ${targetUrl.hostname}:443 HTTP/1.1\r\nHost: ${targetUrl.hostname}:443${authHeader}\r\n\r\n`); - // Step 3: Read CONNECT response (first data chunk contains the status line). + // Step 3: Buffer until the full CONNECT response headers arrive (\r\n\r\n). + // Using a single 'data' event is not safe — headers may arrive fragmented across + // multiple packets, leaving unread bytes that corrupt the subsequent TLS handshake. await new Promise((resolve, reject) => { - proxySock.once('data', (chunk) => { - const resp = chunk.toString('ascii'); - if (!resp.startsWith('HTTP/1.1 200') && !resp.startsWith('HTTP/1.0 200')) { + let buf = ''; + const onData = (chunk) => { + buf += chunk.toString('ascii'); + if (!buf.includes('\r\n\r\n')) return; + proxySock.removeListener('data', onData); + const statusLine = buf.split('\r\n')[0]; + if (!statusLine.startsWith('HTTP/1.1 200') && !statusLine.startsWith('HTTP/1.0 200')) { proxySock.destroy(); - return reject(Object.assign(new Error(`Proxy CONNECT: ${resp.split('\r\n')[0]}`), { status: parseInt(resp.split(' ')[1]) || 0 })); + return reject(Object.assign(new Error(`Proxy CONNECT: ${statusLine}`), { status: parseInt(statusLine.split(' ')[1]) || 0 })); } proxySock.pause(); resolve(); - }); + }; + proxySock.on('data', onData); proxySock.on('error', reject); }); From f7c9604081ae3b648930e1c7a6fb212c0ee76b08 Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Mon, 30 Mar 2026 11:03:43 +0400 Subject: [PATCH 3/3] chore(seeder): remove unused http import, fix stale JSDoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop `import * as http from 'node:http'` — no longer used after replacing http.request CONNECT with tls.connect + manual handshake - Update resolveProxyStringConnect() JSDoc: https.request → tls.connect --- scripts/_proxy-utils.cjs | 2 +- scripts/_seed-utils.mjs | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/_proxy-utils.cjs b/scripts/_proxy-utils.cjs index 4323770295..f79bfa372c 100644 --- a/scripts/_proxy-utils.cjs +++ b/scripts/_proxy-utils.cjs @@ -83,7 +83,7 @@ function resolveProxyString() { * Returns proxy as "user:pass@host:port" string for use with HTTP CONNECT tunneling. * Does NOT replace gate.decodo.com → us.decodo.com; CONNECT endpoint is gate.decodo.com. * When PROXY_URL uses https:// (TLS proxy), returns "https://user:pass@host:port" so - * httpsProxyFetchJson can detect and use https.request instead of http.request. + * httpsProxyFetchJson uses tls.connect to the proxy instead of plain net.connect. * Returns empty string if no proxy configured. */ function resolveProxyStringConnect() { diff --git a/scripts/_seed-utils.mjs b/scripts/_seed-utils.mjs index 086d9c3efb..f58c117804 100644 --- a/scripts/_seed-utils.mjs +++ b/scripts/_seed-utils.mjs @@ -6,7 +6,6 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; import * as net from 'node:net'; -import * as http from 'node:http'; import * as tls from 'node:tls'; import * as https from 'node:https'; import { promisify } from 'node:util';