fix(seeder): TLS proxy CONNECT — fixes FRED fetch failures (FSI, yield curve, macro) (#2538)

* fix(seeder): use TLS for proxy CONNECT tunnel to fix FRED fetch failures

Decodo gate.decodo.com:10001 requires TLS. Previous code used http.request
(plain TCP) which received SOCKS5 rejection bytes instead of HTTP 200.

Two issues fixed:
1. Replace http.request CONNECT with tls.connect + manual CONNECT handshake.
   Node.js http.request also auto-sets Host to the proxy hostname; Decodo
   rejects this and responds with SOCKS5 bytes (0x05 0xff). Manual CONNECT
   over a raw TLS socket avoids both issues.
2. Handle https:// and plain "user:pass@host:port" proxy URL formats — always
   uses TLS regardless of PROXY_URL prefix.

_proxy-utils.cjs: resolveProxyStringConnect now preserves https:// prefix
from PROXY_URL so callers can detect TLS proxies explicitly.

All 24 FRED series (BAMLH0A0HYM2, FEDFUNDS, DGS10, etc.) confirmed working
locally via gate.decodo.com:10001.

* fix(seeder): respect http:// proxy scheme + buffer full CONNECT response

Two protocol-correctness fixes:

1. http:// proxies used plain TCP before; always-TLS regressed them.
   Now: bare/undeclared format → TLS (Decodo requires it), explicit
   http:// → plain net.connect, explicit https:// → TLS.

2. CONNECT response buffered until \r\n\r\n instead of acting on the
   first data chunk. Fragmented proxy responses (headers split across
   packets) could corrupt the TLS handshake by leaving header bytes
   on the wire when tls.connect() was called too early.

Verified locally: BAMLH0A0HYM2 → { date: 2026-03-26, value: 3.21 }

* chore(seeder): remove unused http import, fix stale JSDoc

- Drop `import * as http from 'node:http'` — no longer used after
  replacing http.request CONNECT with tls.connect + manual handshake
- Update resolveProxyStringConnect() JSDoc: https.request → tls.connect

Author: koala73

scripts/_proxy-utils.cjs

@@ -82,12 +82,15 @@ function resolveProxyString() {
 /**
  * Returns proxy as "user:pass@host:port" string for use with HTTP CONNECT tunneling.
  * Does NOT replace gate.decodo.com → us.decodo.com; CONNECT endpoint is gate.decodo.com.
+ * When PROXY_URL uses https:// (TLS proxy), returns "https://user:pass@host:port" so
+ * httpsProxyFetchJson uses tls.connect to the proxy instead of plain net.connect.
  * Returns empty string if no proxy configured.
  */
 function resolveProxyStringConnect() {
   const cfg = resolveProxyConfigWithFallback();
   if (!cfg) return '';
-  return cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`;
+  const base = cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`;
+  return cfg.tls ? `https://${base}` : base;
 }
 
 module.exports = { parseProxyConfig, resolveProxyConfig, resolveProxyConfigWithFallback, resolveProxyString, resolveProxyStringConnect };

scripts/_seed-utils.mjs

@@ -5,7 +5,7 @@ import { execFileSync } from 'node:child_process';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { createRequire } from 'node:module';
-import * as http from 'node:http';
+import * as net from 'node:net';
 import * as tls from 'node:tls';
 import * as https from 'node:https';
 import { promisify } from 'node:util';
@@ -342,43 +342,78 @@ export function curlFetch(url, proxyAuth, headers = {}) {
   return raw.slice(0, nl);
 }
 
-// Pure Node.js HTTPS-through-HTTP-proxy (CONNECT tunnel).
-// Replaces curlFetch for seeder scripts running in containers without curl.
-// proxyAuth format: "user:pass@host:port"
+// Pure Node.js HTTPS-through-proxy (CONNECT tunnel).
+// proxyAuth format: "user:pass@host:port" (bare/Decodo → TLS) OR
+//                  "https://user:pass@host:port" (explicit TLS) OR
+//                  "http://user:pass@host:port"  (explicit plain TCP)
+// Bare/undeclared-scheme proxies always use TLS (Decodo gate.decodo.com requires it).
+// Explicit http:// proxies use plain TCP to avoid breaking non-TLS setups.
 async function httpsProxyFetchJson(url, proxyAuth) {
   const targetUrl = new URL(url);
-  const atIdx = proxyAuth.lastIndexOf('@');
-  const credentials = atIdx >= 0 ? proxyAuth.slice(0, atIdx) : '';
-  const hostPort = atIdx >= 0 ? proxyAuth.slice(atIdx + 1) : proxyAuth;
+
+  // Detect whether the proxy URL specifies http:// explicitly (plain TCP) or not
+  // (bare format or https:// → TLS). User instruction: bare → always TLS.
+  const explicitHttp = proxyAuth.startsWith('http://') && !proxyAuth.startsWith('https://');
+  const useTls = !explicitHttp;
+
+  // Strip scheme prefix, parse user:pass@host:port.
+  let proxyAuthStr = proxyAuth;
+  if (proxyAuth.startsWith('https://') || proxyAuth.startsWith('http://')) {
+    const u = new URL(proxyAuth);
+    proxyAuthStr = (u.username ? `${decodeURIComponent(u.username)}:${decodeURIComponent(u.password)}@` : '') + `${u.hostname}:${u.port}`;
+  }
+
+  const atIdx = proxyAuthStr.lastIndexOf('@');
+  const credentials = atIdx >= 0 ? proxyAuthStr.slice(0, atIdx) : '';
+  const hostPort = atIdx >= 0 ? proxyAuthStr.slice(atIdx + 1) : proxyAuthStr;
   const colonIdx = hostPort.lastIndexOf(':');
   const proxyHost = hostPort.slice(0, colonIdx);
   const proxyPort = parseInt(hostPort.slice(colonIdx + 1), 10);
 
-  const connectHeaders = {};
-  if (credentials) {
-    connectHeaders['Proxy-Authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`;
-  }
+  // Step 1: Open socket to proxy (TLS for https:// or bare, plain TCP for http://).
+  const proxySock = await new Promise((resolve, reject) => {
+    if (useTls) {
+      const s = tls.connect({ host: proxyHost, port: proxyPort, servername: proxyHost, ALPNProtocols: ['http/1.1'] }, () => resolve(s));
+      s.on('error', reject);
+    } else {
+      const s = net.connect({ host: proxyHost, port: proxyPort }, () => resolve(s));
+      s.on('error', reject);
+    }
+  });
 
-  const { socket } = await new Promise((resolve, reject) => {
-    http.request({
-      host: proxyHost, port: proxyPort,
-      method: 'CONNECT',
-      path: `${targetUrl.hostname}:443`,
-      headers: connectHeaders,
-    }).on('connect', (res, socket) => {
-      if (res.statusCode !== 200) {
-        socket.destroy();
-        return reject(Object.assign(new Error(`Proxy CONNECT: ${res.statusCode}`), { status: res.statusCode }));
+  // Step 2: Send HTTP CONNECT manually (avoids Node.js http.request auto-setting
+  // Host to the proxy hostname, which Decodo rejects with SOCKS5 bytes).
+  const authHeader = credentials ? `\r\nProxy-Authorization: Basic ${Buffer.from(credentials).toString('base64')}` : '';
+  proxySock.write(`CONNECT ${targetUrl.hostname}:443 HTTP/1.1\r\nHost: ${targetUrl.hostname}:443${authHeader}\r\n\r\n`);
+
+  // Step 3: Buffer until the full CONNECT response headers arrive (\r\n\r\n).
+  // Using a single 'data' event is not safe — headers may arrive fragmented across
+  // multiple packets, leaving unread bytes that corrupt the subsequent TLS handshake.
+  await new Promise((resolve, reject) => {
+    let buf = '';
+    const onData = (chunk) => {
+      buf += chunk.toString('ascii');
+      if (!buf.includes('\r\n\r\n')) return;
+      proxySock.removeListener('data', onData);
+      const statusLine = buf.split('\r\n')[0];
+      if (!statusLine.startsWith('HTTP/1.1 200') && !statusLine.startsWith('HTTP/1.0 200')) {
+        proxySock.destroy();
+        return reject(Object.assign(new Error(`Proxy CONNECT: ${statusLine}`), { status: parseInt(statusLine.split(' ')[1]) || 0 }));
       }
-      resolve({ socket });
-    }).on('error', reject).end();
+      proxySock.pause();
+      resolve();
+    };
+    proxySock.on('data', onData);
+    proxySock.on('error', reject);
   });
 
-  const tlsSock = tls.connect({ socket, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] });
+  // Step 4: TLS over the proxy tunnel (TLS-in-TLS) to reach the target server.
+  const tlsSock = tls.connect({ socket: proxySock, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] });
   await new Promise((resolve, reject) => {
     tlsSock.on('secureConnect', resolve);
     tlsSock.on('error', reject);
   });
+  proxySock.resume();
 
   return new Promise((resolve, reject) => {
     const timer = setTimeout(() => { tlsSock.destroy(); reject(new Error('FRED proxy fetch timeout')); }, 20000);