fix(seeder): TLS proxy CONNECT — fixes FRED fetch failures (FSI, yield curve, macro)

scripts/_proxy-utils.cjs

@@ -82,12 +82,15 @@ function resolveProxyString() {
 /**
  * Returns proxy as "user:pass@host:port" string for use with HTTP CONNECT tunneling.
  * Does NOT replace gate.decodo.com → us.decodo.com; CONNECT endpoint is gate.decodo.com.
+ * When PROXY_URL uses https:// (TLS proxy), returns "https://user:pass@host:port" so
+ * httpsProxyFetchJson uses tls.connect to the proxy instead of plain net.connect.
  * Returns empty string if no proxy configured.
  */
 function resolveProxyStringConnect() {
   const cfg = resolveProxyConfigWithFallback();
   if (!cfg) return '';
-  return cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`;
+  const base = cfg.auth ? `${cfg.auth}@${cfg.host}:${cfg.port}` : `${cfg.host}:${cfg.port}`;
+  return cfg.tls ? `https://${base}` : base;
 }
 
 module.exports = { parseProxyConfig, resolveProxyConfig, resolveProxyConfigWithFallback, resolveProxyString, resolveProxyStringConnect };

scripts/_seed-utils.mjs

@@ -5,7 +5,7 @@ import { execFileSync } from 'node:child_process';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { createRequire } from 'node:module';
-import * as http from 'node:http';
+import * as net from 'node:net';
 import * as tls from 'node:tls';
 import * as https from 'node:https';
 import { promisify } from 'node:util';
@@ -342,43 +342,78 @@ export function curlFetch(url, proxyAuth, headers = {}) {
   return raw.slice(0, nl);
 }
 
-// Pure Node.js HTTPS-through-HTTP-proxy (CONNECT tunnel).
-// Replaces curlFetch for seeder scripts running in containers without curl.
-// proxyAuth format: "user:pass@host:port"
+// Pure Node.js HTTPS-through-proxy (CONNECT tunnel).
+// proxyAuth format: "user:pass@host:port" (bare/Decodo → TLS) OR
+//                  "https://user:pass@host:port" (explicit TLS) OR
+//                  "http://user:pass@host:port"  (explicit plain TCP)
+// Bare/undeclared-scheme proxies always use TLS (Decodo gate.decodo.com requires it).
+// Explicit http:// proxies use plain TCP to avoid breaking non-TLS setups.
 async function httpsProxyFetchJson(url, proxyAuth) {
   const targetUrl = new URL(url);
-  const atIdx = proxyAuth.lastIndexOf('@');
-  const credentials = atIdx >= 0 ? proxyAuth.slice(0, atIdx) : '';
-  const hostPort = atIdx >= 0 ? proxyAuth.slice(atIdx + 1) : proxyAuth;
+
+  // Detect whether the proxy URL specifies http:// explicitly (plain TCP) or not
+  // (bare format or https:// → TLS). User instruction: bare → always TLS.
+  const explicitHttp = proxyAuth.startsWith('http://') && !proxyAuth.startsWith('https://');
+  const useTls = !explicitHttp;
+
+  // Strip scheme prefix, parse user:pass@host:port.
+  let proxyAuthStr = proxyAuth;
+  if (proxyAuth.startsWith('https://') || proxyAuth.startsWith('http://')) {
+    const u = new URL(proxyAuth);
+    proxyAuthStr = (u.username ? `${decodeURIComponent(u.username)}:${decodeURIComponent(u.password)}@` : '') + `${u.hostname}:${u.port}`;
+  }
+
+  const atIdx = proxyAuthStr.lastIndexOf('@');
+  const credentials = atIdx >= 0 ? proxyAuthStr.slice(0, atIdx) : '';
+  const hostPort = atIdx >= 0 ? proxyAuthStr.slice(atIdx + 1) : proxyAuthStr;
   const colonIdx = hostPort.lastIndexOf(':');
   const proxyHost = hostPort.slice(0, colonIdx);
   const proxyPort = parseInt(hostPort.slice(colonIdx + 1), 10);
 
-  const connectHeaders = {};
-  if (credentials) {
-    connectHeaders['Proxy-Authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`;
-  }
+  // Step 1: Open socket to proxy (TLS for https:// or bare, plain TCP for http://).
+  const proxySock = await new Promise((resolve, reject) => {
+    if (useTls) {
+      const s = tls.connect({ host: proxyHost, port: proxyPort, servername: proxyHost, ALPNProtocols: ['http/1.1'] }, () => resolve(s));
+      s.on('error', reject);
+    } else {
+      const s = net.connect({ host: proxyHost, port: proxyPort }, () => resolve(s));
+      s.on('error', reject);
+    }
+  });
 
-  const { socket } = await new Promise((resolve, reject) => {
-    http.request({
-      host: proxyHost, port: proxyPort,
-      method: 'CONNECT',
-      path: `${targetUrl.hostname}:443`,
-      headers: connectHeaders,
-    }).on('connect', (res, socket) => {
-      if (res.statusCode !== 200) {
-        socket.destroy();
-        return reject(Object.assign(new Error(`Proxy CONNECT: ${res.statusCode}`), { status: res.statusCode }));
+  // Step 2: Send HTTP CONNECT manually (avoids Node.js http.request auto-setting
+  // Host to the proxy hostname, which Decodo rejects with SOCKS5 bytes).
+  const authHeader = credentials ? `\r\nProxy-Authorization: Basic ${Buffer.from(credentials).toString('base64')}` : '';
+  proxySock.write(`CONNECT ${targetUrl.hostname}:443 HTTP/1.1\r\nHost: ${targetUrl.hostname}:443${authHeader}\r\n\r\n`);
+
+  // Step 3: Buffer until the full CONNECT response headers arrive (\r\n\r\n).
+  // Using a single 'data' event is not safe — headers may arrive fragmented across
+  // multiple packets, leaving unread bytes that corrupt the subsequent TLS handshake.
+  await new Promise((resolve, reject) => {
+    let buf = '';
+    const onData = (chunk) => {
+      buf += chunk.toString('ascii');
+      if (!buf.includes('\r\n\r\n')) return;
+      proxySock.removeListener('data', onData);
+      const statusLine = buf.split('\r\n')[0];
+      if (!statusLine.startsWith('HTTP/1.1 200') && !statusLine.startsWith('HTTP/1.0 200')) {
+        proxySock.destroy();
+        return reject(Object.assign(new Error(`Proxy CONNECT: ${statusLine}`), { status: parseInt(statusLine.split(' ')[1]) || 0 }));
       }
-      resolve({ socket });
-    }).on('error', reject).end();
+      proxySock.pause();
+      resolve();
+    };
+    proxySock.on('data', onData);
+    proxySock.on('error', reject);
   });
 
-  const tlsSock = tls.connect({ socket, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] });
+  // Step 4: TLS over the proxy tunnel (TLS-in-TLS) to reach the target server.
+  const tlsSock = tls.connect({ socket: proxySock, servername: targetUrl.hostname, ALPNProtocols: ['http/1.1'] });
   await new Promise((resolve, reject) => {
     tlsSock.on('secureConnect', resolve);
     tlsSock.on('error', reject);
   });
+  proxySock.resume();
 
   return new Promise((resolve, reject) => {
     const timer = setTimeout(() => { tlsSock.destroy(); reject(new Error('FRED proxy fetch timeout')); }, 20000);