feat(e2e): Migrated analytics tests to playwright (#16872)

* feat(e2e): Migrated analytics tests to playwright

* fix(e2e): Fixed desktop bridge connection

* fix(e2e): Unified waiting for requests

.github/workflows/test-suite-desktop-e2e.yml

@@ -44,8 +44,8 @@ jobs:
           #   CONTAINERS: "trezor-user-env-unix"
           # - TEST_GROUP: "@group=passphrase"
           #   CONTAINERS: "trezor-user-env-unix"
-          # - TEST_GROUP: "@group=other"
-          #   CONTAINERS: "trezor-user-env-unix"
+          - TEST_GROUP: "@group=other"
+            CONTAINERS: "trezor-user-env-unix"
           - TEST_GROUP: "@group=wallet"
             CONTAINERS: "trezor-user-env-unix bitcoin-regtest"
 

.github/workflows/test-suite-web-e2e.yml

@@ -106,9 +106,6 @@ jobs:
           - TEST_GROUP: "@group_passphrase"
             CONTAINERS: "trezor-user-env-unix"
             CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
-          - TEST_GROUP: "@group_other"
-            CONTAINERS: "trezor-user-env-unix"
-            CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"
           - TEST_GROUP: "@group_wallet"
             CONTAINERS: "trezor-user-env-unix bitcoin-regtest"
             CYPRESS_USE_TREZOR_USER_ENV_BRIDGE: "1"

packages/suite-desktop-core/e2e/support/analytics.ts

@@ -25,6 +25,10 @@ export class AnalyticsFixture {
         return event;
     }
 
+    extractRequestTypes() {
+        return this.requests.map(request => request['c_type']);
+    }
+
     //TODO: #15811 To be refactored
     @step()
     async interceptAnalytics() {

packages/suite-desktop-core/e2e/support/pageActions/settings/settingsActions.ts

@@ -65,6 +65,7 @@ export class SettingsActions {
         this.page.getByTestId(`@settings/language-select/option/${language}`);
     readonly checkSeedButton: Locator;
     readonly metadataSwitch: Locator;
+    readonly analyticsSwitch: Locator;
 
     constructor(
         private readonly page: Page,
@@ -98,6 +99,7 @@ export class SettingsActions {
         this.languageInput = this.page.getByTestId('@settings/language-select/input');
         this.checkSeedButton = this.page.getByTestId('@settings/device/check-seed-button');
         this.metadataSwitch = this.page.getByTestId('@settings/metadata-switch');
+        this.analyticsSwitch = this.page.getByTestId('@analytics/toggle-switch');
     }
 
     @step()

packages/suite-desktop-core/e2e/tests/analytics/events.test.ts

@@ -0,0 +1,179 @@
+import { EventType } from '@trezor/suite-analytics';
+import { ExtractByEventType } from '@trezor/suite-web/e2e/support/types';
+
+import { expect, test } from '../../support/fixtures';
+
+test.describe('Analytics Events', { tag: ['@group=suite', '@webOnly'] }, () => {
+    test.use({
+        startEmulator: false,
+    });
+    test.beforeEach(async ({ trezorUserEnvLink, onboardingPage }) => {
+        await trezorUserEnvLink.stopBridge();
+        await trezorUserEnvLink.stopEmu();
+        await onboardingPage.disableFirmwareHashCheck();
+    });
+
+    test('reports transport-type, suite-ready and device-connect/device-disconnect events when analytics is initialized and enabled', async ({
+        page,
+        analytics,
+        onboardingPage,
+        settingsPage,
+        trezorUserEnvLink,
+    }) => {
+        // go to settings and enable analytics (makes analytics enabled and initialized)
+        await settingsPage.navigateTo('application');
+        await settingsPage.analyticsSwitch.click();
+        await settingsPage.closeSettings();
+
+        await trezorUserEnvLink.startEmu({ wipe: true, model: 'T3T1', version: '2.8.1' });
+        await trezorUserEnvLink.setupEmu({
+            passphrase_protection: true,
+        });
+
+        await trezorUserEnvLink.startBridge();
+
+        // reload to activate bridge and start testing app with enabled analytics
+        await page.reload();
+        await analytics.interceptAnalytics();
+        await onboardingPage.optionallyDismissFwHashCheckError();
+        await page.getByTestId('@onboarding/exit-app-button').click();
+
+        // suite-ready is logged 1st, just check that it is reported when app is initialized and enabled
+        // device-connect is logged 2nd
+        // transport-type is logged 3rd
+        expect(analytics.requests[0]).toHaveProperty('c_type', EventType.SuiteReady);
+        expect(analytics.requests[1]).toHaveProperty('c_type', EventType.DeviceConnect);
+        expect(analytics.requests[2]).toHaveProperty('c_type', EventType.TransportType);
+        expect(analytics.requests).toHaveLength(3);
+
+        const deviceConnectEvent = analytics.findAnalyticsEventByType<
+            ExtractByEventType<EventType.DeviceConnect>
+        >(EventType.DeviceConnect);
+        expect(deviceConnectEvent.mode).toBe('normal'); // not in BL
+        expect(deviceConnectEvent.firmware).toBe('2.8.1'); // 2.6.0 is hardcoded in startEmu to always match
+        expect(deviceConnectEvent.firmwareRevision).toBe(
+            // good to check because of phishing
+            '632b9561559b7ab6824bb7eeac072874e07b7b82', // https://github.com/trezor/trezor-firmware/releases/tag/core%2Fv2.6.0
+        );
+        expect(deviceConnectEvent.bootloaderHash).toBe('');
+        expect(deviceConnectEvent.backup_type).toBe('Bip39');
+        expect(deviceConnectEvent.pin_protection).toBe('false');
+        expect(deviceConnectEvent.passphrase_protection).toBe('true'); // set in startEmu
+        expect(deviceConnectEvent.totalInstances).toBe('1');
+        expect(deviceConnectEvent.isBitcoinOnly).toBe('false');
+        expect(deviceConnectEvent.totalDevices).toBe('1');
+        expect(deviceConnectEvent.language).toBe('en-US');
+        expect(deviceConnectEvent.model).toBe('T3T1');
+
+        const transportTypeEvent = analytics.findAnalyticsEventByType<
+            ExtractByEventType<EventType.TransportType>
+        >(EventType.TransportType);
+        expect(transportTypeEvent.type).toBe('BridgeTransport');
+        expect(parseInt(transportTypeEvent.version, 10)).not.toBeNaN();
+
+        // device-disconnect is logged 4th
+        await trezorUserEnvLink.stopEmu();
+
+        await expect.poll(() => analytics.requests).toHaveLength(4); // Poll to prevent race condition
+        expect(analytics.requests[3]).toHaveProperty('c_type', EventType.DeviceDisconnect);
+    });
+
+    test('reports suite-ready after enabling analytics on app initial run', async ({
+        analytics,
+        page,
+        analyticsPage,
+        settingsPage,
+        onboardingPage,
+        trezorUserEnvLink,
+    }) => {
+        await trezorUserEnvLink.startEmu({ wipe: true, model: 'T3T1' });
+        await trezorUserEnvLink.setupEmu({
+            passphrase_protection: true,
+        });
+
+        await trezorUserEnvLink.startBridge();
+
+        await analytics.interceptAnalytics();
+
+        await settingsPage.navigateTo('application');
+
+        // change language
+        await page.getByTestId('@settings/language-select/input').scrollIntoViewIfNeeded();
+        await page.getByTestId('@settings/language-select/input').click();
+        await page.getByTestId('@settings/language-select/option/cs').click();
+
+        // change fiat
+        await page.getByTestId('@settings/fiat-select/input').scrollIntoViewIfNeeded();
+        await page.getByTestId('@settings/fiat-select/input').click();
+        await page.getByTestId('@settings/fiat-select/option/czk').click();
+
+        // change BTC units
+        await page.getByTestId('@settings/btc-units-select/input').scrollIntoViewIfNeeded();
+        await page.getByTestId('@settings/btc-units-select/input').click();
+        await page.getByTestId('@settings/btc-units-select/option/Satoshis').click();
+
+        // change dark mode
+        await page.getByTestId('@theme/color-scheme-select/input').scrollIntoViewIfNeeded();
+        await page.getByTestId('@theme/color-scheme-select/input').click();
+        await page.getByTestId('@theme/color-scheme-select/option/dark').click();
+
+        // disable btc, enable ethereum and holesky
+        await page.getByTestId('@settings/menu/wallet').click();
+        await page.getByTestId('@settings/wallet/network/btc').click();
+        await page.getByTestId('@settings/wallet/network/eth').click();
+        await page.getByTestId('@settings/wallet/network/thol').click();
+
+        // custom eth backend
+        await page.getByTestId('@settings/wallet/network/eth/advance').click();
+        await page.getByTestId('@settings/advance/select-type/input').click();
+        await page.getByTestId('@settings/advance/select-type/option/blockbook').click();
+        await page.getByTestId('@settings/advance/url').fill('https://eth.marek.pl/');
+        await page.getByTestId('@settings/advance/button/save').click();
+
+        await settingsPage.closeSettings();
+        await onboardingPage.optionallyDismissFwHashCheckError();
+
+        expect(analytics.requests).toHaveLength(0);
+        await analyticsPage.continueButton.click();
+        await page.getByTestId('@onboarding/exit-app-button').click();
+
+        expect(analytics.requests.length).toBeGreaterThanOrEqual(2);
+        expect(analytics.requests[0]).toHaveProperty('c_type', EventType.SettingsAnalytics);
+        expect(analytics.requests[1]).toHaveProperty('c_type', EventType.SuiteReady);
+
+        // settings/analytics
+        const settingsAnalyticsEvent = analytics.findAnalyticsEventByType<
+            ExtractByEventType<EventType.SettingsAnalytics>
+        >(EventType.SettingsAnalytics);
+        expect(settingsAnalyticsEvent.value).toBe('true');
+
+        // suite-ready reflects state when app was launched, does not include changes
+        const suiteReadyEvent = analytics.findAnalyticsEventByType<
+            ExtractByEventType<EventType.SuiteReady>
+        >(EventType.SuiteReady);
+        expect(suiteReadyEvent.language).toBe('en');
+        expect(suiteReadyEvent.enabledNetworks).toBe('btc');
+        expect(suiteReadyEvent.customBackends).toBe('');
+        expect(suiteReadyEvent.localCurrency).toBe('usd');
+        expect(suiteReadyEvent.bitcoinUnit).toBe('BTC');
+        expect(suiteReadyEvent.discreetMode).toBe('false');
+        expect(suiteReadyEvent.screenWidth).toBeDefined();
+        expect(suiteReadyEvent.screenHeight).toBeDefined();
+        expect(suiteReadyEvent.platformLanguages).toBeDefined();
+        expect(suiteReadyEvent.tor).toBe('false');
+        expect(suiteReadyEvent.labeling).toBeDefined();
+        expect(suiteReadyEvent.rememberedStandardWallets).toBe('0');
+        expect(suiteReadyEvent.rememberedHiddenWallets).toBe('0');
+        expect(suiteReadyEvent.theme).toBe('light');
+        expect(parseInt(suiteReadyEvent.suiteVersion, 10)).not.toBeNaN();
+        expect(suiteReadyEvent.earlyAccessProgram).toBe('false');
+        expect(parseInt(suiteReadyEvent.browserVersion, 10)).not.toBeNaN();
+        expect(suiteReadyEvent.osName).toBeDefined();
+        expect(parseInt(suiteReadyEvent.osVersion, 10)).not.toBeNaN();
+        const viewport = page.viewportSize();
+        expect(suiteReadyEvent.windowWidth).toBe(viewport?.width.toString());
+        expect(suiteReadyEvent.windowHeight).toBe(viewport?.height.toString());
+        expect(suiteReadyEvent.autodetectLanguage).toBe('true');
+        expect(suiteReadyEvent.autodetectTheme).toBe('true');
+    });
+});

packages/suite-desktop-core/e2e/tests/analytics/toggle.test.ts

@@ -0,0 +1,143 @@
+import { EventType } from '@trezor/suite-analytics';
+
+import { expect, test } from '../../support/fixtures';
+
+test.describe('Analytics Toggle - Enabling and Disabling', { tag: ['@group=other'] }, () => {
+    test.beforeEach(async ({ analytics, onboardingPage }) => {
+        await analytics.interceptAnalytics();
+        await onboardingPage.disableFirmwareHashCheck();
+    });
+
+    test('should respect disabled analytics in onboarding with following enabling in settings', async ({
+        analytics,
+        page,
+        analyticsPage,
+        onboardingPage,
+        dashboardPage,
+        settingsPage,
+    }) => {
+        // pass through onboarding with disabled analytics
+        await expect(settingsPage.analyticsSwitch.locator('input')).toBeChecked();
+        await settingsPage.analyticsSwitch.click();
+        await expect(settingsPage.analyticsSwitch.locator('input')).not.toBeChecked();
+
+        await analyticsPage.continueButton.click(); // Click the button and trigger the request
+        await expect.poll(() => analytics.requests).toHaveLength(1);
+
+        // assert that only "analytics/dispose" event was fired
+        const disposeRequest = analytics.requests[0];
+        expect(disposeRequest).toHaveProperty('c_type', EventType.SettingsAnalytics);
+        expect(disposeRequest).toHaveProperty('value', 'false');
+        expect(disposeRequest).toHaveProperty('c_session_id');
+        expect(disposeRequest).toHaveProperty('c_instance_id');
+        expect(disposeRequest).toHaveProperty('c_timestamp');
+        expect(disposeRequest.c_timestamp).toMatch(/^\d+$/);
+
+        await page.getByTestId('@onboarding/exit-app-button').click();
+
+        if (onboardingPage.isModelWithSecureElement()) {
+            await onboardingPage.passThroughAuthenticityCheck();
+        }
+
+        await onboardingPage.onboardingViewOnlyEnableButton.click();
+        await onboardingPage.viewOnlyTooltipGotItButton.click();
+
+        // reload app (important, app needs time to save initialRun flag into storage) to change session id
+        await page.getByTestId('@suite/loading').waitFor({ state: 'hidden' });
+        await dashboardPage.discoveryShouldFinish();
+        await page.reload();
+
+        // go to settings, analytics should not enabled and no additional analytics requests should be fired
+        await settingsPage.navigateTo('application');
+        await expect(settingsPage.analyticsSwitch.locator('input')).not.toBeChecked();
+        expect(analytics.requests).toHaveLength(1);
+
+        // enable analytics and check "analytics/enable" event was fired
+        await settingsPage.analyticsSwitch.click();
+        await expect(settingsPage.analyticsSwitch.locator('input')).toBeChecked();
+        await expect.poll(() => analytics.requests).toHaveLength(2);
+
+        const enableRequest = analytics.requests[1];
+        expect(enableRequest).toHaveProperty('c_type', EventType.SettingsAnalytics);
+        expect(enableRequest).toHaveProperty('c_session_id');
+        expect(enableRequest).toHaveProperty('c_instance_id');
+        expect(enableRequest).toHaveProperty('c_timestamp');
+        expect(enableRequest.c_timestamp).toMatch(/^\d+$/);
+        expect(analytics.requests).toHaveLength(2);
+
+        // check that timestamps are different
+        expect(disposeRequest.c_timestamp).not.toEqual(enableRequest.c_timestamp);
+
+        // check that session ids changed after reload
+        expect(disposeRequest.c_session_id).not.toEqual(enableRequest.c_session_id);
+
+        // check that instance ids are the same after reload
+        expect(disposeRequest.c_instance_id).toEqual(enableRequest.c_instance_id);
+
+        // change fiat and check that it was logged
+        await page.getByTestId('@settings/fiat-select/input').scrollIntoViewIfNeeded(); // Shouldn't be necessary, but without it the dropdown doesn't open
+        await page.getByTestId('@settings/fiat-select/input').click();
+        await page.getByTestId('@settings/fiat-select/option/huf').click();
+        await expect.poll(() => analytics.requests).toHaveLength(3);
+        expect(analytics.requests[2]).toHaveProperty('c_type', EventType.SettingsGeneralChangeFiat);
+        expect(analytics.requests[2]).toHaveProperty('fiat', 'huf');
+        expect(analytics.requests[2]).toHaveProperty(
+            'c_instance_id',
+            analytics.requests[1].c_instance_id,
+        );
+        expect(analytics.requests).toHaveLength(3);
+
+        // open device modal and check that it was logged
+        await dashboardPage.openDeviceSwitcher();
+        await expect.poll(() => analytics.requests).toHaveLength(4);
+
+        const deviceModalRequest = analytics.requests[3];
+        expect(deviceModalRequest).toHaveProperty('c_type', EventType.RouterLocationChange);
+        expect(analytics.requests).toHaveLength(4);
+    });
+
+    test('should respect enabled analytics in onboarding with following disabling in settings', async ({
+        analytics,
+        page,
+        analyticsPage,
+        onboardingPage,
+        settingsPage,
+    }) => {
+        // pass through onboarding with enabled analytics
+        await expect(settingsPage.analyticsSwitch.locator('input')).toBeChecked();
+
+        await analyticsPage.continueButton.click(); // Click the button and trigger the request
+        await expect.poll(() => analytics.requests.length).toBeGreaterThan(2);
+
+        // assert that more than 1 event was fired and it was "suite/ready" and "analytics/enable" for sure
+        expect(analytics.requests.length).toBeGreaterThan(1);
+        expect(analytics.extractRequestTypes()).toContain(EventType.SuiteReady);
+        expect(analytics.extractRequestTypes()).toContain(EventType.SettingsAnalytics);
+
+        // finish onboarding
+        await page.getByTestId('@onboarding/exit-app-button').click();
+        if (onboardingPage.isModelWithSecureElement()) {
+            await onboardingPage.passThroughAuthenticityCheck();
+        }
+
+        // go to settings, analytics should be enabled
+        await settingsPage.navigateTo('application');
+        await expect(settingsPage.analyticsSwitch.locator('input')).toBeChecked();
+
+        // disable analytics
+        await settingsPage.analyticsSwitch.click();
+        await expect(settingsPage.analyticsSwitch.locator('input')).not.toBeChecked();
+
+        // change fiat and check that it was not logged
+        await page.getByTestId('@settings/fiat-select/input').scrollIntoViewIfNeeded(); // Shouldn't be necessary, but without it the dropdown doesn't open
+        await page.getByTestId('@settings/fiat-select/input').click();
+        await page.getByTestId('@settings/fiat-select/option/huf').click();
+
+        // check that analytics disable event was fired
+        await expect.poll(() => analytics.requests.length).toBeGreaterThan(3);
+        expect(analytics.extractRequestTypes()).toContain(EventType.SettingsAnalytics);
+
+        // check that "settings/general/change-fiat" event was not fired
+        expect(analytics.extractRequestTypes()).not.toContain(EventType.SettingsGeneralChangeFiat);
+    });
+});