fix: xhr queue plugin

packages/analytics-js-common/src/utilities/http.ts

@@ -1,7 +1,7 @@
 import type { IResponseDetails } from '../types/HttpClient';
 
 const isErrRetryable = (details: IResponseDetails) => {
-  const status = details.error?.status || 0;
+  const status = details.error?.status ?? 0;
   return status === 429 || (status >= 500 && status < 600);
 };
 

packages/analytics-js-common/src/utilities/retryQueue/RetryQueue.ts

@@ -37,6 +37,7 @@ import {
   DEFAULT_BATCH_FLUSH_INTERVAL_MS,
   MIN_TIMER_SCALE_FACTOR,
   MAX_TIMER_SCALE_FACTOR,
+  MAX_PAGE_UNLOAD_BATCH_SIZE_BYTES,
 } from './constants';
 
 const sortByTime = (a: QueueItem, b: QueueItem) => a.time - b.time;
@@ -244,14 +245,33 @@ class RetryQueue implements IQueue<QueueItemData> {
     this.isPageAccessible = isAccessible;
     if (!this.batchingInProgress) {
       this.batchingInProgress = true;
-      let batchQueue =
+      const batchQueue =
         (this.getStorageEntry(QueueStatuses.BATCH_QUEUE) as Nullable<QueueItem[]>) ?? [];
       if (batchQueue.length > 0) {
-        batchQueue = batchQueue.slice(-batchQueue.length);
+        let batchItems: QueueItem[] = [];
+        let remainingBatchItems: QueueItem[] = [];
+        if (!this.isPageAccessible) {
+          // eslint-disable-next-line no-restricted-syntax
+          for (const queueItem of batchQueue) {
+            if (
+              (this.batchSizeCalcCb as QueueBatchItemsSizeCalculatorCallback<QueueItemData>)(
+                [...batchItems, queueItem].map(queueItem => queueItem.item),
+              ) > MAX_PAGE_UNLOAD_BATCH_SIZE_BYTES
+            ) {
+              break;
+            }
+
+            batchItems.push(queueItem);
+          }
+
+          remainingBatchItems = batchQueue.slice(batchItems.length);
+        } else {
+          batchItems = batchQueue.slice(-batchQueue.length);
+        }
 
-        const batchEntry = this.genQueueItem(batchQueue.map(queueItem => queueItem.item));
+        const batchEntry = this.genQueueItem(batchItems.map(queueItem => queueItem.item));
 
-        this.setStorageEntry(QueueStatuses.BATCH_QUEUE, []);
+        this.setStorageEntry(QueueStatuses.BATCH_QUEUE, remainingBatchItems);
 
         this.pushToMainQueue(batchEntry);
       }

packages/analytics-js-common/src/utilities/retryQueue/constants.ts

@@ -17,6 +17,8 @@ const DEFAULT_MAX_BATCH_SIZE_BYTES = 512 * 1024; // 512 KB; this is also the max
 const DEFAULT_MAX_BATCH_ITEMS = 100;
 const DEFAULT_BATCH_FLUSH_INTERVAL_MS = 60 * 1000; // 1 minutes
 
+const MAX_PAGE_UNLOAD_BATCH_SIZE_BYTES = 64 * 1024; // 64 KB
+
 export {
   DEFAULT_MIN_RETRY_DELAY_MS,
   DEFAULT_MAX_RETRY_DELAY_MS,
@@ -33,4 +35,5 @@ export {
   DEFAULT_BATCH_FLUSH_INTERVAL_MS,
   MIN_TIMER_SCALE_FACTOR,
   MAX_TIMER_SCALE_FACTOR,
+  MAX_PAGE_UNLOAD_BATCH_SIZE_BYTES,
 };

packages/analytics-js-common/src/utilities/retryQueue/types.ts

@@ -67,17 +67,7 @@ export type QueueBatchItemsSizeCalculatorCallback<T = any> = (item: T) => number
 export interface IQueue<T = any> {
   name: string;
   id: string;
-  processQueueCb: QueueProcessCallback;
-  store: IStore;
-  storeManager: IStoreManager;
-  maxItems: number;
-  timeouts: Record<string, number>;
   scheduleTimeoutActive: boolean;
-  getStorageEntry(name?: string): Nullable<QueueItem<T>[] | Record<string, any> | number>;
-  setStorageEntry(
-    name?: string,
-    value?: Nullable<QueueItem<T>[] | Record<string, any> | number>,
-  ): void;
   start(): void;
   stop(): void;
   enqueue(item: QueueItem<T>): void;

packages/analytics-js-plugins/__tests__/xhrQueue/utilities.test.ts

@@ -118,103 +118,70 @@ describe('xhrQueue Plugin Utilities', () => {
 
   describe('logErrorOnFailure', () => {
     it('should not log error if there is no error', () => {
-      const details = {
-        response: {},
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', false, 1, 10, mockLogger);
+      logErrorOnFailure(false, 'https://test.com/v1/page', undefined, false, 1, 10, mockLogger);
 
       expect(mockLogger.error).not.toHaveBeenCalled();
     });
 
     it('should log an error for delivery failure', () => {
-      const details = {
-        error: {},
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', false, 1, 10, mockLogger);
+      logErrorOnFailure(
+        false,
+        'https://test.com/v1/page',
+        'Something bad happened',
+        false,
+        1,
+        10,
+        mockLogger,
+      );
 
       expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. The event(s) will be dropped.',
+        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. Original error: Something bad happened. The event(s) will be dropped.',
       );
     });
 
     it('should log an error for retryable network failure', () => {
-      let details = {
-        error: {
-          status: 429,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', true, 1, 10, mockLogger);
-
-      expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. It/they will be retried. Retry attempt 1 of 10.',
-      );
-
-      // Retryable error but it's the first attempt
-      details = {
-        error: {
-          status: 429,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', true, 0, 10, mockLogger);
-
-      expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. It/they will be retried.',
+      logErrorOnFailure(
+        true,
+        'https://test.com/v1/page',
+        'Something bad happened',
+        true,
+        1,
+        10,
+        mockLogger,
       );
 
-      // 500 error
-      details = {
-        error: {
-          status: 500,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', true, 1, 10, mockLogger);
-
       expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. It/they will be retried. Retry attempt 1 of 10.',
+        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. Original error: Something bad happened. It/they will be retried. Retry attempt 1 of 10.',
       );
 
-      // 5xx error
-      details = {
-        error: {
-          status: 501,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', true, 1, 10, mockLogger);
-
-      expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. It/they will be retried. Retry attempt 1 of 10.',
+      // Retryable error but it's the first attempt
+      logErrorOnFailure(
+        true,
+        'https://test.com/v1/page',
+        'Something bad happened',
+        true,
+        0,
+        10,
+        mockLogger,
       );
 
-      // 600 error
-      details = {
-        error: {
-          status: 600,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', true, 1, 10, mockLogger);
-
       expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. The event(s) will be dropped.',
+        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. Original error: Something bad happened. It/they will be retried.',
       );
 
       // Retryable error but exhausted all tries
-      details = {
-        error: {
-          status: 520,
-        },
-      } as IResponseDetails;
-
-      logErrorOnFailure(details, 'https://test.com/v1/page', false, 10, 10, mockLogger);
+      logErrorOnFailure(
+        true,
+        'https://test.com/v1/page',
+        'Something bad happened',
+        false,
+        10,
+        10,
+        mockLogger,
+      );
 
       expect(mockLogger.error).toHaveBeenCalledWith(
-        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. Retries exhausted (10). The event(s) will be dropped.',
+        'XhrQueuePlugin:: Failed to deliver event(s) to https://test.com/v1/page. Original error: Something bad happened. Retries exhausted (10). The event(s) will be dropped.',
       );
     });
   });

packages/analytics-js-plugins/src/xhrQueue/index.ts

@@ -2,7 +2,10 @@
 /* eslint-disable no-param-reassign */
 import type { ExtensionPlugin } from '@rudderstack/analytics-js-common/types/PluginEngine';
 import type { ApplicationState } from '@rudderstack/analytics-js-common/types/ApplicationState';
-import type { IHttpClient } from '@rudderstack/analytics-js-common/types/HttpClient';
+import type {
+  IHttpClient,
+  IResponseDetails,
+} from '@rudderstack/analytics-js-common/types/HttpClient';
 import type { IErrorHandler } from '@rudderstack/analytics-js-common/types/ErrorHandler';
 import type { ILogger } from '@rudderstack/analytics-js-common/types/Logger';
 import type { IStoreManager } from '@rudderstack/analytics-js-common/types/Store';
@@ -15,6 +18,8 @@ import type {
   QueueItemData,
   DoneCallback,
 } from '@rudderstack/analytics-js-common/utilities/retryQueue/types';
+import { toBase64 } from '@rudderstack/analytics-js-common/utilities/string';
+import { FAILED_REQUEST_ERR_MSG_PREFIX } from '@rudderstack/analytics-js-common/constants/errors';
 import { storages, http, timestamp, string, eventsDelivery } from '../shared-chunks/common';
 import {
   getNormalizedQueueOptions,
@@ -26,6 +31,7 @@ import {
 import { QUEUE_NAME, REQUEST_TIMEOUT_MS } from './constants';
 import type { XHRRetryQueueItemData, XHRQueueItemData } from './types';
 import { RetryQueue } from '../shared-chunks/retryQueue';
+import { DELIVERY_ERROR, REQUEST_ERROR } from './logMessages';
 
 const pluginName: PluginName = 'XhrQueue';
 
@@ -76,33 +82,87 @@ const XhrQueue = (): ExtensionPlugin => ({
             logger,
           );
 
-          httpClient.request({
-            url,
-            options: {
-              method: 'POST',
-              headers,
-              body: data as string,
-              sendRawData: true,
-              useAuth: true,
-            },
-            isRawResponse: true,
-            timeout: REQUEST_TIMEOUT_MS,
-            callback: (result, details) => {
-              // null means item will not be requeued
-              const queueErrResp = http.isErrRetryable(details) ? details : null;
-
-              logErrorOnFailure(
-                details,
+          const handleResponse = (
+            err?: any,
+            status?: number,
+            statusText?: string,
+            ev?: ProgressEvent,
+          ) => {
+            let errMsg;
+            if (err) {
+              errMsg = REQUEST_ERROR(
+                FAILED_REQUEST_ERR_MSG_PREFIX,
                 url,
-                willBeRetried,
-                attemptNumber,
-                maxRetryAttempts,
-                logger,
+                REQUEST_TIMEOUT_MS,
+                err,
+                ev,
               );
+            } else if (status && (status < 200 || status > 300)) {
+              errMsg = DELIVERY_ERROR(FAILED_REQUEST_ERR_MSG_PREFIX, status, url, statusText, ev);
+            }
+
+            const details = {
+              error: {
+                status,
+              },
+            } as IResponseDetails;
+
+            const isRetryableFailure = http.isErrRetryable(details);
+
+            // null means item will not be requeued
+            const queueErrResp = isRetryableFailure ? details : null;
+
+            logErrorOnFailure(
+              isRetryableFailure,
+              url,
+              errMsg,
+              willBeRetried,
+              attemptNumber,
+              maxRetryAttempts,
+              logger,
+            );
+
+            done(queueErrResp);
+          };
+
+          const xhr = new XMLHttpRequest();
+          try {
+            xhr.open('POST', url, true);
+          } catch (err: any) {
+            handleResponse(err);
+            return;
+          }
+
+          // The timeout property may be set only in the time interval between a call to the open method
+          // and the first call to the send method in legacy browsers
+          xhr.timeout = REQUEST_TIMEOUT_MS;
+
+          if (headers) {
+            Object.entries(headers).forEach(([headerName, headerValue]) => {
+              xhr.setRequestHeader(headerName, headerValue);
+            });
+          }
+
+          xhr.onload = () => {
+            // This is same as fetch API
+            if (xhr.status >= 200 && xhr.status < 300) {
+              handleResponse(null, xhr.status, xhr.statusText);
+            }
+          };
+
+          const xhrCallback = (ev: ProgressEvent) => {
+            handleResponse(undefined, xhr.status, xhr.statusText, ev);
+          };
+
+          xhr.ontimeout = xhrCallback;
+          xhr.onerror = xhrCallback;
+          xhr.onabort = xhrCallback;
 
-              done(queueErrResp, result);
-            },
-          });
+          try {
+            xhr.send(data);
+          } catch (err: any) {
+            handleResponse(err, xhr.status, xhr.statusText);
+          }
         },
         storeManager,
         storages.LOCAL_STORAGE,
@@ -146,6 +206,7 @@ const XhrQueue = (): ExtensionPlugin => ({
       const headers = {
         Accept: 'application/json',
         'Content-Type': 'application/json;charset=UTF-8',
+        Authorization: `Basic ${toBase64(`${state.lifecycle.writeKey.value as string}:`)}`,
         // To maintain event ordering while using the HTTP API as per is documentation,
         // make sure to include anonymousId as a header
         AnonymousId: string.toBase64(event.anonymousId),