Skip to content

fix(cli): recover transient publish failures #3087

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
miguel-heygen merged 2 commits into from
Aug 7, 2026
Merged

fix(cli): recover transient publish failures #3087

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2d6ce3
fix(cli): recover transient publish failures
miguel-heygen Aug 7, 2026
dd62969
fix(cli): harden publish retry behavior
miguel-heygen Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 125 additions & 0 deletions packages/cli/src/utils/publishProject.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ function directFetch(completeData?: Record<string, unknown>) {
.mockResolvedValueOnce(publishedResponse(completeData));
}

function networkFailure(code: string, message: string): TypeError {
return new TypeError("fetch failed", {
cause: Object.assign(new Error(message), { code }),
});
}

/** Asserts the Nth fetch call, always requiring an AbortSignal alongside the given init. */
function expectFetchCall(
fetchMock: ReturnType<typeof vi.fn>,
Expand Down Expand Up @@ -729,6 +735,125 @@ describe("publishProjectArchive", () => {
rmSync(dir, { recursive: true, force: true });
}
});

it("retries a transient presigned upload network failure once", async () => {
const dir = makeProjectDir();
const fetchMock = vi
.fn()
.mockResolvedValueOnce(uploadResponse())
.mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"))
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockResolvedValueOnce(publishedResponse());
vi.stubGlobal("fetch", fetchMock);

try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");

const result = await publishProjectArchive(dir);

expect(result.projectId).toBe("hfp_123");
expect(fetchMock).toHaveBeenCalledTimes(4);
expect(fetchMock.mock.calls[1]![0]).toBe("https://s3.example.com/upload");
expect(fetchMock.mock.calls[2]![0]).toBe("https://s3.example.com/upload");
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("reports the upload stage and transport cause after the retry is exhausted", async () => {
const dir = makeProjectDir();
const signedUrl =
"https://s3.example.com/upload?X-Amz-Credential=secret&X-Amz-Signature=do-not-print";
const fetchMock = vi
.fn()
.mockResolvedValueOnce(uploadResponse({ upload_url: signedUrl }))
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"))
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "getaddrinfo EAI_AGAIN s3.example.com"));
vi.stubGlobal("fetch", fetchMock);

try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");

const promise = publishProjectArchive(dir);
await expect(promise).rejects.toThrow(
"Failed to upload project archive after 2 attempts: fetch failed (EAI_AGAIN: getaddrinfo EAI_AGAIN s3.example.com)",
);
await expect(promise).rejects.not.toThrow("do-not-print");
expect(fetchMock).toHaveBeenCalledTimes(3);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("reports the prepare stage and explains disabled Node proxy support", async () => {
const dir = makeProjectDir();
vi.stubEnv("HTTPS_PROXY", "http://proxy.example.com:8080");
vi.stubEnv("NODE_USE_ENV_PROXY", "");
const fetchMock = vi
.fn()
.mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"))
.mockRejectedValueOnce(networkFailure("ENETUNREACH", "network is unreachable"));
vi.stubGlobal("fetch", fetchMock);

try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");

await expect(publishProjectArchive(dir)).rejects.toThrow(
"Failed to prepare project upload after 2 attempts: fetch failed (ENETUNREACH: network is unreachable). Proxy variables are set, but Node fetch proxy support is disabled; retry with NODE_USE_ENV_PROXY=1",
);
expect(fetchMock).toHaveBeenCalledTimes(2);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("retries a transient prepare-upload network failure once", async () => {
const dir = makeProjectDir();
const fetchMock = vi
.fn()
.mockRejectedValueOnce(networkFailure("EAI_AGAIN", "temporary DNS failure"))
.mockResolvedValueOnce(uploadResponse())
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockResolvedValueOnce(publishedResponse());
vi.stubGlobal("fetch", fetchMock);

try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");

const result = await publishProjectArchive(dir);

expect(result.projectId).toBe("hfp_123");
expect(fetchMock).toHaveBeenCalledTimes(4);
expect(fetchMock.mock.calls[0]![0]).toBe(
"https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
);
expect(fetchMock.mock.calls[1]![0]).toBe(
"https://api2.heygen.com/v1/hyperframes/projects/publish/upload",
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});

it("reports the finalize stage and transport cause", async () => {
const dir = makeProjectDir();
const fetchMock = vi
.fn()
.mockResolvedValueOnce(uploadResponse())
.mockResolvedValueOnce(new Response(null, { status: 200 }))
.mockRejectedValueOnce(networkFailure("ECONNRESET", "socket disconnected"));
vi.stubGlobal("fetch", fetchMock);

try {
writeFileSync(join(dir, "index.html"), "<html></html>", "utf-8");

await expect(publishProjectArchive(dir)).rejects.toThrow(
"Failed to finalize project publish: fetch failed (ECONNRESET: socket disconnected)",
);
} finally {
rmSync(dir, { recursive: true, force: true });
}
});
});

/** Staged flow returning an already-owned, already-claimed stable project. */
Expand Down
159 changes: 117 additions & 42 deletions packages/cli/src/utils/publishProject.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ const DEFAULT_PROJECT_IGNORE = ["/renders/", "/snapshots/"];
const PUBLISH_CONTENT_TYPE = "application/zip";
const PUBLISH_METADATA_TIMEOUT_MS = 30_000;
const PUBLISH_UPLOAD_MIN_TIMEOUT_MS = 120_000;
const PUBLISH_TRANSPORT_ATTEMPTS = 2;
// Conservative floor — most connections are faster, but this prevents
// premature aborts on slow/unstable networks (hotel wifi, tethering).
const PUBLISH_UPLOAD_BYTES_PER_SECOND = 500_000;
Expand Down Expand Up @@ -166,6 +167,62 @@ async function readErrorMessage(response: Response, fallback: string): Promise<s
return text.trim() ? `${fallback}: ${text.trim().slice(0, 180)}` : fallback;
}

function errorCode(value: unknown): string | null {
if (!isRecord(value)) return null;
return typeof value["code"] === "string" ? value["code"] : null;
}

function redactUrlQuery(message: string): string {
return message.replace(/(https?:\/\/[^\s?]+)\?[^\s]+/gu, "$1?[redacted]");
}

function proxySupportHint(): string {
const proxyConfigured = ["HTTPS_PROXY", "HTTP_PROXY", "https_proxy", "http_proxy"].some((key) =>
Boolean(process.env[key]?.trim()),
);
const proxyEnabled =
process.env["NODE_USE_ENV_PROXY"] === "1" ||
process.execArgv.includes("--use-env-proxy") ||
process.env["NODE_OPTIONS"]?.split(/\s+/u).includes("--use-env-proxy") === true;
if (!proxyConfigured || proxyEnabled) return "";
return (
". Proxy variables are set, but Node fetch proxy support is disabled; retry with " +
"NODE_USE_ENV_PROXY=1 (Node 22.21+)"
);
}

function describeFetchFailure(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
const cause = error instanceof Error ? error.cause : undefined;
const causeMessage = cause instanceof Error ? cause.message : "";
const code = errorCode(cause) ?? errorCode(error);
const detail = [code, causeMessage && causeMessage !== message ? causeMessage : ""]
.filter(Boolean)
.join(": ");
return `${redactUrlQuery(message)}${detail ? ` (${redactUrlQuery(detail)})` : ""}${proxySupportHint()}`;
}

async function fetchForPublish(
input: string,
createInit: () => RequestInit,
failureStage: string,
attempts = 1,
): Promise<Response> {
for (let attempt = 1; attempt <= attempts; attempt += 1) {
try {
return await fetch(input, createInit());
} catch (error) {
if (attempt === attempts) {
const attemptDetail = attempts > 1 ? ` after ${attempts} attempts` : "";
throw new Error(`${failureStage}${attemptDetail}: ${describeFetchFailure(error)}`, {
cause: error instanceof Error ? error : undefined,
});
}
}
}
throw new Error(failureStage);
}

export function uploadTimeoutMs(byteLength: number): number {
return Math.max(
PUBLISH_UPLOAD_MIN_TIMEOUT_MS,
Expand Down Expand Up @@ -483,12 +540,16 @@ async function publishProjectArchiveDirect(
heygen_route: "canary",
};

const response = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish`, {
method: "POST",
body,
headers,
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
});
const response = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish`,
() => ({
method: "POST",
body,
headers,
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
}),
"Failed to publish project",
);

const payload = await readJson(response);
const publishedProject = parsePublishedProjectResponse(payload);
Expand All @@ -504,14 +565,19 @@ async function uploadArchiveToPresignedUrl(
archive: PublishArchiveResult,
): Promise<void> {
const presignedUrlTtlMs = stagedUpload.expiresInSeconds * 1000 - PUBLISH_METADATA_TIMEOUT_MS;
const s3Response = await fetch(stagedUpload.uploadUrl, {
method: "PUT",
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
headers: stagedUpload.uploadHeaders,
signal: AbortSignal.timeout(
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
),
});
const s3Response = await fetchForPublish(
stagedUpload.uploadUrl,
() => ({
method: "PUT",
body: new Blob([archiveArrayBuffer(archive)], { type: stagedUpload.contentType }),
headers: stagedUpload.uploadHeaders,
signal: AbortSignal.timeout(
Math.min(uploadTimeoutMs(archive.buffer.byteLength), presignedUrlTtlMs),
),
}),
"Failed to upload project archive",
PUBLISH_TRANSPORT_ATTEMPTS,
);
if (!s3Response.ok) {
throw new Error(await readErrorMessage(s3Response, "Failed to upload project archive"));
}
Expand All @@ -526,20 +592,25 @@ async function publishProjectArchiveStaged(
projectId: string | undefined,
): Promise<PublishedProjectResponse | null> {
const fileName = `${title}.zip`;
const uploadResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`, {
method: "POST",
body: JSON.stringify({
file_name: fileName,
content_type: PUBLISH_CONTENT_TYPE,
content_length: archive.buffer.byteLength,
const uploadResponse = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish/upload`,
() => ({
method: "POST",
body: JSON.stringify({
file_name: fileName,
content_type: PUBLISH_CONTENT_TYPE,
content_length: archive.buffer.byteLength,
}),
headers: {
...authHeaders,
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
}),
headers: {
...authHeaders,
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(PUBLISH_METADATA_TIMEOUT_MS),
});
"Failed to prepare project upload",
PUBLISH_TRANSPORT_ATTEMPTS,
);

if (uploadResponse.status === 404 || uploadResponse.status === 405) {
return null;
Expand All @@ -553,22 +624,26 @@ async function publishProjectArchiveStaged(

await uploadArchiveToPresignedUrl(stagedUpload, archive);

const completeResponse = await fetch(`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`, {
method: "POST",
body: JSON.stringify({
upload_key: stagedUpload.uploadKey,
file_name: fileName,
title,
...(isPublic ? { is_public: true } : {}),
...(projectId ? { project_id: projectId } : {}),
const completeResponse = await fetchForPublish(
`${apiBaseUrl}/v1/hyperframes/projects/publish/complete`,
() => ({
method: "POST",
body: JSON.stringify({
upload_key: stagedUpload.uploadKey,
file_name: fileName,
title,
...(isPublic ? { is_public: true } : {}),
...(projectId ? { project_id: projectId } : {}),
}),
headers: {
...authHeaders,
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
}),
headers: {
...authHeaders,
"content-type": "application/json",
heygen_route: "canary",
},
signal: AbortSignal.timeout(uploadTimeoutMs(archive.buffer.byteLength)),
});
"Failed to finalize project publish",
);

const completePayload = await readJson(completeResponse);
const publishedProject = parsePublishedProjectResponse(completePayload);
Expand Down
Loading