From 71e28b7890bfdd77263c0495e0b59f787929089f Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 10 Nov 2025 03:52:33 +0530 Subject: [PATCH 01/46] TRA changes --- nightwatch/globals.js | 54 ++-- src/accessibilityAutomation.js | 177 +---------- src/scripts/accessibilityScripts.js | 97 ++++++ src/testObservability.js | 332 +++++++++++++++----- src/testorchestration/orchestrationUtils.js | 6 +- src/testorchestration/requestUtils.js | 4 +- src/utils/crashReporter.js | 2 +- src/utils/helper.js | 22 +- src/utils/requestQueueHandler.js | 19 +- 9 files changed, 411 insertions(+), 302 deletions(-) create mode 100644 src/scripts/accessibilityScripts.js diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 38ade3e..98a2245 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -257,10 +257,12 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { + await testObservability.sendTestRunEvent("TestRunStarted",test); await accessibilityAutomation.beforeEachExecution(test); }); eventBroadcaster.on('TestRunFinished', async (test) => { + await testObservability.sendTestRunEvent("TestRunFinished",test); await accessibilityAutomation.afterEachExecution(test); }); }, @@ -272,10 +274,11 @@ module.exports = { }, async before(settings, testEnvSettings) { + const pluginSettings = settings['@nightwatch/browserstack']; if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; @@ -312,7 +315,7 @@ module.exports = { settings.test_runner.options['require'] = path.resolve(__dirname, 'observabilityLogPatcherHook.js'); } settings.globals['customReporterCallbackTimeout'] = CUSTOM_REPORTER_CALLBACK_TIMEOUT; - if (testObservability._user && testObservability._key) { + if (helper.isTestObservabilitySession() || pluginSettings?.accessibility === true) { await testObservability.launchTestSession(); } if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS && process.env.BROWSERSTACK_RERUN_TESTS!=='null') { @@ -321,8 +324,17 @@ module.exports = { } } } catch (error) { - Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); + Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); + } + + try { + if (helper.isAccessibilitySession() && !settings.parallel_mode) { + accessibilityAutomation.setAccessibilityCapabilities(settings); + } + } catch (err){ + Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); } + // Initialize and configure test orchestration try { @@ -408,22 +420,6 @@ module.exports = { Logger.error(`Could not configure test orchestration - ${error}`); } - try { - accessibilityAutomation.configure(settings); - if (helper.isAccessibilitySession()) { - if (accessibilityAutomation._user && accessibilityAutomation._key) { - const [jwtToken, testRunId] = await accessibilityAutomation.createAccessibilityTestRun(); - process.env.BS_A11Y_JWT = jwtToken; - process.env.BS_A11Y_TEST_RUN_ID = testRunId; - if (helper.isAccessibilitySession()) { - accessibilityAutomation.setAccessibilityCapabilities(settings); - } - } - } - } catch (error) { - Logger.error(`Could not configure or launch accessibility automation - ${error}`); - } - addProductMapAndbuildUuidCapability(settings); }, @@ -443,8 +439,8 @@ module.exports = { } catch (error) { Logger.error(`Error collecting build data for test orchestration: ${error}`); } - - if (helper.isTestObservabilitySession()) { + + if (helper.isTestObservabilitySession() || helper.isAccessibilitySession()) { process.env.NIGHTWATCH_RERUN_FAILED = nightwatchRerun; process.env.NIGHTWATCH_RERUN_REPORT_FILE = nightwatchRerunFile; if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS) { @@ -452,22 +448,14 @@ module.exports = { } try { await testObservability.stopBuildUpstream(); - if (process.env.BS_TESTOPS_BUILD_HASHED_ID) { - Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID} to view build report, insights, and many more debugging information all at one place!\n`); + if (process.env.BROWSERSTACK_TESTHUB_UUID) { + Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); } } catch (error) { Logger.error(`Something went wrong in stopping build session for test reporting and analytics - ${error}`); } process.exit(); } - if (helper.isAccessibilitySession()){ - try { - await accessibilityAutomation.stopAccessibilityTestRun(); - } catch (error) { - Logger.error(`Exception in stop accessibility test run: ${error}`); - } - - } }, async beforeEach(settings) { @@ -569,10 +557,10 @@ const addProductMapAndbuildUuidCapability = (settings) => { if (settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options']['buildProductMap'] = buildProductMap; - settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['bstack:options']['testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } else { settings.desiredCapabilities['browserstack.buildProductMap'] = buildProductMap; - settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BS_TESTOPS_BUILD_HASHED_ID ? process.env.BS_TESTOPS_BUILD_HASHED_ID : '' ; + settings.desiredCapabilities['browserstack.testhubBuildUuid'] = process.env.BROWSERSTACK_TESTHUB_UUID ? process.env.BROWSERSTACK_TESTHUB_UUID : '' ; } } catch (error) { Logger.debug(`Error while sending productmap and build capabilities ${error}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index af97fa0..45c11fd 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -7,22 +7,10 @@ const util = require('util'); class AccessibilityAutomation { configure(settings = {}) { - this._settings = settings['@nightwatch/browserstack'] || {}; - - if (this._settings.accessibility) { - process.env.BROWSERSTACK_ACCESSIBILITY = - String(this._settings.accessibility).toLowerCase() === 'true'; - } if (process.argv.includes('--disable-accessibility')) { process.env.BROWSERSTACK_ACCESSIBILITY = false; - return; } - process.env.BROWSERSTACK_INFRA = true; - if (settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1){ - process.env.BROWSERSTACK_INFRA = false; - } - this._testRunner = settings.test_runner; this._bstackOptions = {}; if ( @@ -39,165 +27,6 @@ class AccessibilityAutomation { } } - async createAccessibilityTestRun() { - const userName = this._user; - const accessKey = this._key; - - if (helper.isUndefined(userName) || helper.isUndefined(accessKey)) { - Logger.error( - 'Exception while creating test run for BrowserStack Accessibility Automation: Missing authentication token' - ); - - return [null, null]; - } - - try { - let accessibilityOptions; - if (helper.isUndefined(this._settings.accessibilityOptions)) { - accessibilityOptions = {}; - } else { - accessibilityOptions = this._settings.accessibilityOptions; - } - - const fromProduct = { - accessibility: true - }; - - const data = { - projectName: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), - buildName: - helper.getBuildName(this._settings, this._bstackOptions, fromProduct) || - path.basename(path.resolve(process.cwd())), - startTime: new Date().toISOString(), - description: accessibilityOptions.buildDescription || '', - source: { - frameworkName: helper.getFrameworkName(this._testRunner), - frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() - }, - settings: accessibilityOptions, - versionControl: await helper.getGitMetaData(), - ciInfo: helper.getCiInfo(), - hostInfo: helper.getHostInfo(), - browserstackAutomation: helper.isBrowserstackInfra() - }; - const config = { - auth: { - user: userName, - pass: accessKey - }, - headers: { - 'Content-Type': 'application/json' - } - }; - - const response = await makeRequest('POST', 'test_runs', data, config, ACCESSIBILITY_URL); - const responseData = response.data.data || {}; - - accessibilityOptions.scannerVersion = responseData.scannerVersion; - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); - - return [responseData.accessibilityToken, responseData.id]; - } catch (error) { - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - if (error.response) { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - if (error.message === 'Invalid configuration passed.') { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - for (const errorkey of error.errors) { - Logger.error(errorkey.message); - } - process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - } else { - Logger.error( - `Exception while creating test run for BrowserStack Accessibility Automation: ${ - error.message || error.stack - }` - ); - } - } - - return [null, null]; - } - } - - async stopAccessibilityTestRun() { - if ( - helper.isUndefined(process.env.BS_A11Y_JWT) || - typeof process.env.BS_A11Y_JWT !== 'string' - ) { - return { - status: 'error', - message: 'Build creation had failed.' - }; - } - - const data = {endTime: new Date().toISOString()}; - const config = { - headers: { - Authorization: `Bearer ${process.env.BS_A11Y_JWT}`, - 'Content-Type': 'application/json' - } - }; - const options = { - ...config, - ...{ - body: data, - auth: null, - json: true - } - }; - - try { - const response = await makeRequest( - 'PUT', - 'test_runs/stop', - options, - config, - ACCESSIBILITY_URL - ); - if (response.data.error) { - throw new Error('Invalid request: ' + response.data.error); - } else { - Logger.info( - `BrowserStack Accessibility Automation Test Run marked as completed at ${new Date().toISOString()}` - ); - - return {status: 'success', message: ''}; - } - } catch (error) { - if (error.response) { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.response.status - } ${error.response.statusText} ${JSON.stringify(error.response.data)}` - ); - } else { - Logger.error( - `Exception while marking completion of BrowserStack Accessibility Automation Test Run: ${ - error.message || util.format(error) - }` - ); - } - - return { - status: 'error', - message: - error.message || - (error.response ? `${error.response.status}:${error.response.statusText}` : error) - }; - } - } - setAccessibilityCapabilities(settings) { try { settings.desiredCapabilities = settings.desiredCapabilities || {}; @@ -313,7 +142,7 @@ class AccessibilityAutomation { return response; } - setExtension(driver) { + validateA11yCaps(driver) { try { const capabilityConfig = driver.desiredCapabilities || {}; const deviceName = driver.capabilities.deviceName || (capabilityConfig['bstack:options'] ? capabilityConfig['bstack:options'].deviceName : capabilityConfig.device) || ''; @@ -345,7 +174,7 @@ class AccessibilityAutomation { return true; } catch (error) { - Logger.debug(`Exception in setExtension Error: ${error}`); + Logger.debug(`Exception in validateA11yCaps Error: ${error}`); } return false; @@ -358,7 +187,7 @@ class AccessibilityAutomation { testMetaData ); this.currentTest.accessibilityScanStarted = true; - this._isAccessibilitySession = this.setExtension(browser); + this._isAccessibilitySession = this.validateA11yCaps(browser); if (this.isAccessibilityAutomationSession() && browser && helper.isAccessibilitySession() && this._isAccessibilitySession) { try { diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js new file mode 100644 index 0000000..cd4bae8 --- /dev/null +++ b/src/scripts/accessibilityScripts.js @@ -0,0 +1,97 @@ +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +class AccessibilityScripts { + static instance = null; + + performScan = null; + getResults = null; + getResultsSummary = null; + saveTestResults = null; + commandsToWrap = null; + ChromeExtension = {}; + + browserstackFolderPath = ''; + commandsPath = ''; + + // don't allow to create instances from it other than through `checkAndGetInstance` + constructor() { + this.browserstackFolderPath = this.getWritableDir(); + this.commandsPath = path.join(this.browserstackFolderPath, 'commands.json'); + } + + static checkAndGetInstance() { + if (!AccessibilityScripts.instance) { + AccessibilityScripts.instance = new AccessibilityScripts(); + AccessibilityScripts.instance.readFromExistingFile(); + } + return AccessibilityScripts.instance; + } + + getWritableDir() { + const orderedPaths = [ + path.join(os.homedir(), '.browserstack'), + process.cwd(), + os.tmpdir() + ]; + for (const orderedPath of orderedPaths) { + try { + if (fs.existsSync(orderedPath)) { + fs.accessSync(orderedPath); + return orderedPath; + } + + fs.mkdirSync(orderedPath, { recursive: true }); + return orderedPath; + + } catch (e) { + /* no-empty */ + } + } + return ''; + } + + readFromExistingFile() { + try { + if (fs.existsSync(this.commandsPath)) { + const data = fs.readFileSync(this.commandsPath, 'utf8'); + if (data) { + this.update(JSON.parse(data)); + } + } + } catch (error) { + /* Do nothing */ + } + } + + update(data) { + if (data.scripts) { + this.performScan = data.scripts.scan; + this.getResults = data.scripts.getResults; + this.getResultsSummary = data.scripts.getResultsSummary; + this.saveTestResults = data.scripts.saveResults; + } + if (data.commands && data.commands.length) { + this.commandsToWrap = data.commands; + } + } + + store() { + if (!fs.existsSync(this.browserstackFolderPath)){ + fs.mkdirSync(this.browserstackFolderPath); + } + + fs.writeFileSync(this.commandsPath, JSON.stringify({ + commands: this.commandsToWrap, + scripts: { + scan: this.performScan, + getResults: this.getResults, + getResultsSummary: this.getResultsSummary, + saveResults: this.saveTestResults, + }, + })); + } +} + +module.exports = AccessibilityScripts.checkAndGetInstance(); diff --git a/src/testObservability.js b/src/testObservability.js index eb04010..f6b25ea 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -9,6 +9,7 @@ const CrashReporter = require('./utils/crashReporter'); const Logger = require('./utils/logger'); const {API_URL, TAKE_SCREENSHOT_REGEX} = require('./utils/constants'); const OrchestrationUtils = require('./testorchestration/orchestrationUtils'); +const accessibilityScripts = require('./scripts/accessibilityScripts'); const hooksMap = {}; class TestObservability { @@ -83,36 +84,49 @@ class TestObservability { async launchTestSession() { // Support both old and new configuration options at different levels - const options = this._settings.test_observability || + const testReportingOptions = this._settings.test_observability || this._settings.test_reporting || this._settings.testReportingOptions || this._settings.testObservabilityOptions || this._parentSettings?.testReportingOptions || this._parentSettings?.testObservabilityOptions || {}; + const accessibility = this._settings.accessibility || false; + const accessibilityOptions = accessibility ? this._settings.accessibilityOptions || {} : {}; this._gitMetadata = await helper.getGitMetaData(); const fromProduct = { - test_observability: true, - test_reporting: true + test_observability: this._settings.test_observability.enabled || this._settings.test_reporting.enabled || false, + accessibility: accessibility }; const data = { format: 'json', project_name: helper.getProjectName(this._settings, this._bstackOptions, fromProduct), name: helper.getBuildName(this._settings, this._bstackOptions, fromProduct), - build_identifier: options.buildIdentifier, - description: options.buildDescription || '', - start_time: new Date().toISOString(), + build_identifier: testReportingOptions.buildIdentifier, + description: testReportingOptions.buildDescription || '', + started_at: new Date().toISOString(), tags: helper.getObservabilityBuildTags(this._settings, this._bstackOptions), host_info: helper.getHostInfo(), ci_info: helper.getCiInfo(), build_run_identifier: process.env.BROWSERSTACK_BUILD_RUN_IDENTIFIER, failed_tests_rerun: process.env.BROWSERSTACK_RERUN || false, version_control: this._gitMetadata, - observability_version: { - frameworkName: helper.getFrameworkName(this._testRunner), - frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion() + accessibility: { + settings: accessibilityOptions }, + framework_details: { + frameworkName: helper.getFrameworkName(this._testRunner), + frameworkVersion: helper.getPackageVersion('nightwatch'), + sdkVersion: helper.getAgentVersion(), + language: 'javascript', + testFramework: { + name: 'nightwatch', + version: helper.getPackageVersion('nightwatch') + } + }, + product_map: this.getProductMapForBuildStartCall(this._parentSettings), + browserstackAutomation: helper.isBrowserstackInfra(this._settings), + config: {}, test_orchestration: this.getTestOrchestrationBuildStartData(this._parentSettings) }; @@ -128,20 +142,19 @@ class TestObservability { }; try { - const response = await makeRequest('POST', 'api/v1/builds', data, config, API_URL); - Logger.info('Build creation successful!'); + const response = await makeRequest('POST', 'api/v2/builds', data, config, API_URL); + Logger.debug(`[Start_Build] Success response: ${JSON.stringify(response)}`) + console.log(`[Start_Build] Success response: ${JSON.stringify(response)}`) process.env.BS_TESTOPS_BUILD_COMPLETED = true; const responseData = response.data || {}; if (responseData.jwt) { - process.env.BS_TESTOPS_JWT = responseData.jwt; + process.env.BROWSERSTACK_TESTHUB_JWT = responseData.jwt; } if (responseData.build_hashed_id) { - process.env.BS_TESTOPS_BUILD_HASHED_ID = responseData.build_hashed_id; - } - if (responseData.allow_screenshots) { - process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.allow_screenshots.toString(); + process.env.BROWSERSTACK_TESTHUB_UUID = responseData.build_hashed_id; } + this.processLaunchBuildResponse(responseData, this._settings); } catch (error) { if (error.response) { Logger.error(`EXCEPTION IN BUILD START EVENT : ${error.response.status} ${error.response.statusText} ${JSON.stringify(error.response.data)}`); @@ -159,11 +172,82 @@ class TestObservability { return orchestrationUtils.getBuildStartData(); } + processLaunchBuildResponse(responseData, settings) { + if (helper.isTestObservabilitySession()) { + this.processTestObservabilityResponse(responseData); + } + this.processAccessibilityResponse(responseData, settings); + } + + processTestObservabilityResponse(responseData) { + if (!responseData.observability) { + this.handleErrorForObservability(null) + return + } + if (!responseData.observability.success) { + this.handleErrorForObservability(responseData.observability) + return + } + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'true'; + process.env.BROWSERSTACK_TEST_REPORTING = 'true'; + if (responseData.observability.options.allow_screenshots) { + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.observability.options.allow_screenshots.toString(); + } + } + + processAccessibilityResponse(responseData, settings) { + if (!responseData.accessibility) { + if (settings.accessibility === true) { + this.handleErrorForAccessibility(null) + } + return + } + if (!responseData.accessibility.success) { + this.handleErrorForAccessibility(responseData.accessibility) + return + } + + if (responseData.accessibility.options) { + const { accessibilityToken, pollingTimeout, scannerVersion } = helper.jsonifyAccessibilityArray(responseData.accessibility.options.capabilities, 'name', 'value') + const scriptsJson = { + 'scripts': helper.jsonifyAccessibilityArray(responseData.accessibility.options.scripts, 'name', 'command'), + 'commands': responseData.accessibility.options.commandsToWrap?.commands ?? [], + } + if (scannerVersion) { + process.env.BSTACK_A11Y_SCANNER_VERSION = scannerVersion + Logger.debug(`Accessibility scannerVersion ${scannerVersion}`) + } + if (accessibilityToken) { + process.env.BS_A11Y_JWT = accessibilityToken + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + } + if (pollingTimeout) { + process.env.BSTACK_A11Y_POLLING_TIMEOUT = pollingTimeout + } + if (scriptsJson) { + accessibilityScripts.update(scriptsJson); + accessibilityScripts.store(); + } + } + + } + + handleErrorForObservability(observabilityData) { + process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'false'; + process.env.BROWSERSTACK_TEST_REPORTING = 'false'; + Logger.error(`Observability could not be started`); //make logging better later + } + + handleErrorForAccessibility(accessibilityData) { + process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; + Logger.error(`Accessibility could not be started`); //make logging better later + } + async stopBuildUpstream () { if (!process.env.BS_TESTOPS_BUILD_COMPLETED) { return; } - if (!process.env.BS_TESTOPS_JWT) { + if (!process.env.BROWSERSTACK_TESTHUB_JWT) { Logger.info('[STOP_BUILD] Missing Authentication Token/ Build ID'); return { @@ -172,11 +256,11 @@ class TestObservability { }; } const data = { - 'stop_time': new Date().toISOString() + 'finished_at': new Date().toISOString() }; const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -184,7 +268,7 @@ class TestObservability { await helper.uploadPending(); await helper.shutDownRequestHandler(); try { - const response = await makeRequest('PUT', `api/v1/builds/${process.env.BS_TESTOPS_BUILD_HASHED_ID}/stop`, data, config, API_URL, false); + const response = await makeRequest('PUT', `api/v1/builds/${process.env.BROWSERSTACK_TESTHUB_UUID}/stop`, data, config, API_URL, false); if (response.data?.error) { throw {message: response.data.error}; } else { @@ -207,8 +291,8 @@ class TestObservability { } } - async sendEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { - await this.sendTestRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); + async sendHookEvents(eventData, testFileReport, startEventType, finishedEventType, hookId, hookType, sectionName) { + await this.sendHookRunEvent(eventData, testFileReport, startEventType, hookId, hookType, sectionName); if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { @@ -216,12 +300,12 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); + await this.sendHookRunEvent(eventData, testFileReport, finishedEventType, hookId, hookType, sectionName); } async processTestReportFile(testFileReport) { const completedSections = testFileReport['completedSections']; - const skippedTests = testFileReport['skippedAtRuntime'].concat(testFileReport['skippedByUser']); + // const skippedTests = testFileReport['skippedAtRuntime'].concat(testFileReport['skippedByUser']); if (completedSections) { const globalBeforeEachHookId = uuidv4(); const beforeHookId = uuidv4(); @@ -232,49 +316,49 @@ class TestObservability { const eventData = completedSections[sectionName]; switch (sectionName) { case '__global_beforeEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalBeforeEachHookId, 'GLOBAL_BEFORE_EACH', sectionName); break; } case '__before_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', beforeHookId, 'BEFORE_ALL', sectionName); break; } case '__after_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', afterHookId, 'AFTER_ALL', sectionName); break; } case '__global_afterEach_hook': { - await this.sendEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); - break; - } - default: { - if (eventData.retryTestData?.length>0) { - for (const retryTest of eventData.retryTestData) { - await this.processTestRunData(retryTest, sectionName, testFileReport, hookIds); - } - } - await this.processTestRunData(eventData, sectionName, testFileReport, hookIds); + await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } + // default: { + // if (eventData.retryTestData?.length>0) { + // for (const retryTest of eventData.retryTestData) { + // await this.processTestRunData(retryTest, sectionName, testFileReport, hookIds); + // } + // } + // await this.processTestRunData(eventData, sectionName, testFileReport, hookIds); + // break; + // } } } - if (skippedTests?.length > 0) { - for (const skippedTest of skippedTests) { - await this.sendSkippedTestEvent(skippedTest, testFileReport); - } - } + // if (skippedTests?.length > 0) { + // for (const skippedTest of skippedTests) { + // await this.sendSkippedTestEvent(skippedTest, testFileReport); + // } + // } } } - async processTestRunData (eventData, sectionName, testFileReport, hookIds) { - const testUuid = uuidv4(); - const errorData = eventData.commands.find(command => command.result?.stack); - eventData.lastError = errorData ? errorData.result : null; - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunStarted', testUuid, null, sectionName, hookIds); + async processTestRunData (eventData, uuid) { + // const testUuid = uuidv4(); + // const errorData = eventData.commands.find(command => command.result?.stack); + // eventData.lastError = errorData ? errorData.result : null; + // await this.sendTestRunEvent(eventData, testFileReport, 'TestRunStarted', testUuid, null, sectionName, hookIds); if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { - await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], testUuid); + await this.createHttpLogEvent(output, eventData.httpOutput[index + 1], uuid); } } } @@ -291,7 +375,7 @@ class TestObservability { try { if (fs.existsSync(screenshotPath)) { const screenshot = fs.readFileSync(screenshotPath, 'base64'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } catch (err) { Logger.debug(`Failed to upload screenshot from saveScreenshot: ${err.message}`); @@ -299,11 +383,11 @@ class TestObservability { } else if (TAKE_SCREENSHOT_REGEX.test(command.name) && command.result) { try { if (command.result.value) { - await this.createScreenshotLogEvent(testUuid, command.result.value, command.startTime); + await this.createScreenshotLogEvent(uuid, command.result.value, command.startTime); } else if (command.result.valuePath) { if (fs.existsSync(command.result.valuePath)) { const screenshot = fs.readFileSync(command.result.valuePath, 'utf8'); - await this.createScreenshotLogEvent(testUuid, screenshot, command.startTime); + await this.createScreenshotLogEvent(uuid, screenshot, command.startTime); } } } catch (err) { @@ -312,7 +396,7 @@ class TestObservability { } } } - await this.sendTestRunEvent(eventData, testFileReport, 'TestRunFinished', testUuid, null, sectionName, hookIds); + // await this.sendTestRunEvent(eventData, testFileReport, 'TestRunFinished', testUuid, null, sectionName, hookIds); } async sendSkippedTestEvent(skippedTest, testFileReport) { @@ -329,7 +413,7 @@ class TestObservability { file_name: path.relative(process.cwd(), testFileReport.modulePath), location: path.relative(process.cwd(), testFileReport.modulePath), vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testFileReport.modulePath) : null, - started_at: new Date(testFileReport.endTimestamp).toISOString(), + started_at: new Date(testFileReport.startTimestamp).toISOString(), finished_at: new Date(testFileReport.endTimestamp).toISOString(), duration_in_ms: 0, result: 'skipped', @@ -384,7 +468,91 @@ class TestObservability { } } - async sendTestRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { + async sendTestRunEvent(eventType, test) { + Logger.debug(`[sendTestRunEvent] Reached sendTestRunEvent with eventType: ${eventType}`); + const testMetaData = test.metadata; + const uuid = test.uuid; + const testName = test.testcase; + const settings = test.settings || {}; + const startTimestamp = test.envelope[testName].startTimestamp; + let testResults = {}; + const testBody = test.testBody; + Logger.debug(`[sendTestRunEvent] Values - testMetaData: ${JSON.stringify(testMetaData)}, uuid: ${uuid}, testName: ${testName}, settings: ${JSON.stringify(settings)},testResults: ${JSON.stringify(testResults)}, testBody: ${testBody}`); + const provider = helper.getCloudProvider(testMetaData.host); + const testData = { + uuid: uuid, + type: 'test', + name: testName, + body: { + lang: 'javascript', + code: testBody ? testBody.toString() : null + }, + scope: `${testMetaData.name} - ${testName}`, + scopes: [ + testMetaData.name + ], + tags: testMetaData.tags, + identifier: `${testMetaData.name} - ${testName}`, + file_name: path.relative(process.cwd(), testMetaData.modulePath), + location: path.relative(process.cwd(), testMetaData.modulePath), + vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testMetaData.modulePath) : null, + started_at: new Date(startTimestamp).toISOString(), + result: 'pending', + framework: 'nightwatch', + integrations: { + [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings.desiredCapabilities['bstack:options']?.osVersion) + }, + // customRerunParam: { + // rerun_name: '' // Rerun name of test that can be sent in Rerun request to uniquely identify the test + // }, + // retry_of: '', + product_map: { + accessibility: helper.isAccessibilitySession(), + } + }; + + if (eventType === 'TestRunFinished') { + const eventData = test.envelope[testName].testcase; + testResults = test.testResults; + // const skippedTests =testResults['skippedAtRuntime'].concat(testResults['skippedByUser']) + // if (skippedTests?.length > 0){ + // for (const skippedTest of skippedTests) { + // await this.sendSkippedTestEvent(skippedTest, testMetaData); + // } + // } + testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); + testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; + if (testData.result === 'failed' && testResults.__lastError) { + testData.failure = [ + { + 'backtrace': [stripAnsi(testResults.__lastError.message), testResults.__lastError.stack] + } + ]; + testData.failure_reason = testResults.__lastError ? stripAnsi(testResults.__lastError.message) : null; + if (testResults.__lastError && testResults.__lastError.name) { + testData.failure_type = testResults.__lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } + } + + this.processTestRunData (eventData,uuid) + // else if (eventData.status === 'fail' && (testFileReport?.completed[sectionName]?.lastError || testFileReport?.completed[sectionName]?.stackTrace)) { + // const testCompletionData = testFileReport.completed[sectionName]; + // testData.failure = [ + // {'backtrace': [testCompletionData?.stackTrace]} + // ]; + // testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; + // testData.failure_type = testCompletionData?.stackTrace.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + // } + } + + const uploadData = { + event_type: eventType, + test_run: testData + }; + await helper.uploadEventData(uploadData); + } + + async sendHookRunEvent(eventData, testFileReport, eventType, uuid, hookType, sectionName, hooks) { const testData = { uuid: uuid, type: 'hook', @@ -404,12 +572,12 @@ class TestObservability { hook_type: hookType }; - if (eventType === 'HookRunFinished' || eventType === 'TestRunFinished') { + if (eventType === 'HookRunFinished') { testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(eventData.startTimestamp).toISOString(); testData.result = eventData.status === 'pass' ? 'passed' : 'failed'; testData.duration_in_ms = 'timeMs' in eventData ? eventData.timeMs : eventData.time; if (eventData.status === 'fail' && eventData.lastError) { - testData.failure = [ + testData.failure_data = [ { 'backtrace': [stripAnsi(eventData.lastError.message), eventData.lastError.stack] } @@ -420,7 +588,7 @@ class TestObservability { } } else if (eventData.status === 'fail' && (testFileReport?.completed[sectionName]?.lastError || testFileReport?.completed[sectionName]?.stackTrace)) { const testCompletionData = testFileReport.completed[sectionName]; - testData.failure = [ + testData.failure_data = [ {'backtrace': [testCompletionData?.stackTrace]} ]; testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; @@ -434,26 +602,27 @@ class TestObservability { testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); } - if (eventType === 'TestRunStarted') { - testData.type = 'test'; - testData.integrations = {}; - const provider = helper.getCloudProvider(testFileReport.host); - testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); - } + // if (eventType === 'TestRunStarted') { + // testData.type = 'test'; + // testData.integrations = {}; + // const provider = helper.getCloudProvider(testFileReport.host); + // testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); + // } - if (eventType === 'TestRunFinished') { - testData.type = 'test'; - testData.hooks = hooks; - } + // if (eventType === 'TestRunFinished') { + // testData.type = 'test'; + // testData.hooks = hooks; + // } const uploadData = { - event_type: eventType + event_type: eventType, + hook_run: testData }; - if (eventType.match(/HookRun/)) { - uploadData['hook_run'] = testData; - } else { - uploadData['test_run'] = testData; - } + // if (eventType.match(/HookRun/)) { + // uploadData['hook_run'] = testData; + // } else { + // uploadData['test_run'] = testData; + // } await helper.uploadEventData(uploadData); } @@ -709,6 +878,21 @@ class TestObservability { Logger.error(`Exception in uploading log data to Test Reporting and Analytics with error : ${error}`); } } + + getProductMapForBuildStartCall(settings) { + const product = helper.getObservabilityLinkedProductName(settings.desiredCapabilities, settings?.selenium?.host); + + const buildProductMap = { + automate: product === 'automate', + app_automate: product === 'app-automate', + observability: helper.isTestObservabilitySession(), + accessibility: settings['@nightwatch/browserstack']?.accessibility === true, + turboscale: product === 'turboscale', + percy: false + }; + + return buildProductMap; + } } module.exports = TestObservability; diff --git a/src/testorchestration/orchestrationUtils.js b/src/testorchestration/orchestrationUtils.js index c9965f6..63787f3 100644 --- a/src/testorchestration/orchestrationUtils.js +++ b/src/testorchestration/orchestrationUtils.js @@ -145,7 +145,7 @@ class OrchestrationUtils { * Check if the abort build file exists */ static checkAbortBuildFileExists() { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const filePath = path.join(tmpdir(), `abort_build_${buildUuid}`); return fs.existsSync(filePath); @@ -155,7 +155,7 @@ class OrchestrationUtils { * Write failure to file */ static writeFailureToFile(testName) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; const failedTestsFile = path.join(tmpdir(), `failed_tests_${buildUuid}.txt`); fs.appendFileSync(failedTestsFile, `${testName}\n`); @@ -433,7 +433,7 @@ class OrchestrationUtils { * Collects build data by making a call to the collect-build-data endpoint */ async collectBuildData(config) { - const buildUuid = process.env.BS_TESTOPS_BUILD_HASHED_ID; + const buildUuid = process.env.BROWSERSTACK_TESTHUB_UUID; this.logger.debug(`[collectBuildData] Collecting build data for build UUID: ${buildUuid}`); try { diff --git a/src/testorchestration/requestUtils.js b/src/testorchestration/requestUtils.js index 254dc9e..ef7e2d0 100644 --- a/src/testorchestration/requestUtils.js +++ b/src/testorchestration/requestUtils.js @@ -36,11 +36,11 @@ class RequestUtils { * Makes an orchestration request with the given method and data */ static async makeOrchestrationRequest(method, reqEndpoint, options) { - const jwtToken = process.env.BS_TESTOPS_JWT || ''; + const jwtToken = process.env.BROWSERSTACK_TESTHUB_JWT || ''; // Validate JWT token if (!jwtToken) { - Logger.error('BS_TESTOPS_JWT environment variable is not set. This is required for test orchestration.'); + Logger.error('BROWSERSTACK_TESTHUB_JWT environment variable is not set. This is required for test orchestration.'); return null; } diff --git a/src/utils/crashReporter.js b/src/utils/crashReporter.js index 8caa23f..1255def 100644 --- a/src/utils/crashReporter.js +++ b/src/utils/crashReporter.js @@ -52,7 +52,7 @@ class CrashReporter { try { const data = { - hashed_id: process.env.BS_TESTOPS_BUILD_HASHED_ID, + hashed_id: process.env.BROWSERSTACK_TESTHUB_UUID, observability_version: { frameworkName: 'nightwatch-default', frameworkVersion: helper.getPackageVersion('nightwatch'), diff --git a/src/utils/helper.js b/src/utils/helper.js index 1054240..82ccdcd 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -376,8 +376,9 @@ exports.getHostInfo = () => { }; }; -exports.isBrowserstackInfra = () => { - return process.env.BROWSERSTACK_INFRA === 'true'; +exports.isBrowserstackInfra = (settings) => { + const isBrowserstackInfra = settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1 ? false : true; + return isBrowserstackInfra; }; const findGitConfig = async (filePath) => { @@ -505,12 +506,12 @@ exports.uploadEventData = async (eventData) => { ['HookRunFinished']: 'Hook_End_Upload' }[eventData.event_type]; - if (process.env.BS_TESTOPS_JWT && process.env.BS_TESTOPS_JWT !== 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT && process.env.BROWSERSTACK_TESTHUB_JWT !== 'null') { requestQueueHandler.pending_test_uploads += 1; } if (process.env.BS_TESTOPS_BUILD_COMPLETED === 'true') { - if (process.env.BS_TESTOPS_JWT === 'null') { + if (process.env.BROWSERSTACK_TESTHUB_JWT === 'null') { Logger.info(`EXCEPTION IN ${log_tag} REQUEST TO TEST REPORTING AND ANALYTICS : missing authentication token`); requestQueueHandler.pending_test_uploads = Math.max(0, requestQueueHandler.pending_test_uploads-1); @@ -537,7 +538,7 @@ exports.uploadEventData = async (eventData) => { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } @@ -631,13 +632,14 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { return product; }; -exports.getIntegrationsObject = (capabilities, sessionId, hostname) => { +exports.getIntegrationsObject = (capabilities, sessionId, hostname, platform_version) => { return { capabilities: capabilities, session_id: sessionId, browser: capabilities.browserName, browser_version: capabilities.browserVersion, platform: capabilities.platformName, + platform_version: platform_version, product: this.getObservabilityLinkedProductName(capabilities, hostname) }; }; @@ -1301,4 +1303,12 @@ exports.getGitMetadataForAiSelection = (folders = []) => { })); return formattedResults; +}; + +exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { + const result = {}; + dataArray.forEach((element) => { + result[element[keyName]] = element[valueName]; + }); + return result; }; \ No newline at end of file diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index e4363ec..18543a3 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -11,6 +11,7 @@ class RequestQueueHandler { this.BATCH_EVENT_TYPES = ['LogCreated', 'TestRunFinished', 'TestRunSkipped', 'HookRunFinished', 'TestRunStarted', 'HookRunStarted']; this.pollEventBatchInterval = null; this.pending_test_uploads = 0; + this.data = null; } start() { @@ -31,17 +32,17 @@ class RequestQueueHandler { } this.queue.push(event); - let data = null; + // let data = null; const shouldProceed = this.shouldProceed(); if (shouldProceed) { - data = this.queue.slice(0, BATCH_SIZE); + this.data = this.queue.slice(0, BATCH_SIZE); this.queue.splice(0, BATCH_SIZE); this.resetEventBatchPolling(); } return { shouldProceed: shouldProceed, - proceedWithData: data, + proceedWithData: this.data, proceedWithUrl: this.eventUrl }; } @@ -63,10 +64,10 @@ class RequestQueueHandler { startEventBatchPolling () { this.pollEventBatchInterval = setInterval(async () => { - if (this.queue.length > 0) { - const data = this.queue.slice(0, BATCH_SIZE); - this.queue.splice(0, BATCH_SIZE); - await this.batchAndPostEvents(this.eventUrl, 'Interval-Queue', data); + if (this.data.length > 0) { + // const data = this.queue.slice(0, BATCH_SIZE); + // this.queue.splice(0, BATCH_SIZE); + await this.batchAndPostEvents(this.eventUrl, 'Interval-Queue', this.data); } }, BATCH_INTERVAL); } @@ -87,13 +88,13 @@ class RequestQueueHandler { } shouldProceed () { - return this.queue.length >= BATCH_SIZE; + return this.queue.length <= BATCH_SIZE; } async batchAndPostEvents (eventUrl, kind, data) { const config = { headers: { - 'Authorization': `Bearer ${process.env.BS_TESTOPS_JWT}`, + 'Authorization': `Bearer ${process.env.BROWSERSTACK_TESTHUB_JWT}`, 'Content-Type': 'application/json', 'X-BSTACK-TESTOPS': 'true' } From 94fbd890bb0d49eb62778d4c9b9ae49112ecb511 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 11 Nov 2025 15:47:48 +0530 Subject: [PATCH 02/46] changes for testMap implementation --- nightwatch/globals.js | 9 ++++-- src/testObservability.js | 55 +++++-------------------------------- src/utils/testMap.js | 59 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 50 deletions(-) create mode 100644 src/utils/testMap.js diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 98a2245..8bab354 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -9,9 +9,11 @@ const path = require('path'); const AccessibilityAutomation = require('../src/accessibilityAutomation'); const eventHelper = require('../src/utils/eventHelper'); const OrchestrationUtils = require('../src/testorchestration/orchestrationUtils'); +const TestMap = require('../src/utils/testMap'); const localTunnel = new LocalTunnel(); const testObservability = new TestObservability(); const accessibilityAutomation = new AccessibilityAutomation(); +const testMapInstance = new TestMap(); const nightwatchRerun = process.env.NIGHTWATCH_RERUN_FAILED; const nightwatchRerunFile = process.env.NIGHTWATCH_RERUN_REPORT_FILE; @@ -257,12 +259,15 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - await testObservability.sendTestRunEvent("TestRunStarted",test); + testMapInstance.storeTestDetails(test); + const uuid = testMapInstance.getUUID(test); + await testObservability.sendTestRunEvent("TestRunStarted", test, uuid); await accessibilityAutomation.beforeEachExecution(test); }); eventBroadcaster.on('TestRunFinished', async (test) => { - await testObservability.sendTestRunEvent("TestRunFinished",test); + const uuid = testMapInstance.getUUID(test); + await testObservability.sendTestRunEvent("TestRunFinished",test, uuid); await accessibilityAutomation.afterEachExecution(test); }); }, diff --git a/src/testObservability.js b/src/testObservability.js index f6b25ea..7849a2f 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -305,7 +305,7 @@ class TestObservability { async processTestReportFile(testFileReport) { const completedSections = testFileReport['completedSections']; - // const skippedTests = testFileReport['skippedAtRuntime'].concat(testFileReport['skippedByUser']); + const skippedTests = testFileReport['skippedAtRuntime'].concat(testFileReport['skippedByUser']); if (completedSections) { const globalBeforeEachHookId = uuidv4(); const beforeHookId = uuidv4(); @@ -342,19 +342,15 @@ class TestObservability { // } } } - // if (skippedTests?.length > 0) { - // for (const skippedTest of skippedTests) { - // await this.sendSkippedTestEvent(skippedTest, testFileReport); - // } - // } + if (skippedTests?.length > 0) { + for (const skippedTest of skippedTests) { + await this.sendSkippedTestEvent(skippedTest, testFileReport); + } + } } } async processTestRunData (eventData, uuid) { - // const testUuid = uuidv4(); - // const errorData = eventData.commands.find(command => command.result?.stack); - // eventData.lastError = errorData ? errorData.result : null; - // await this.sendTestRunEvent(eventData, testFileReport, 'TestRunStarted', testUuid, null, sectionName, hookIds); if (eventData.httpOutput && eventData.httpOutput.length > 0) { for (const [index, output] of eventData.httpOutput.entries()) { if (index % 2 === 0) { @@ -396,7 +392,6 @@ class TestObservability { } } } - // await this.sendTestRunEvent(eventData, testFileReport, 'TestRunFinished', testUuid, null, sectionName, hookIds); } async sendSkippedTestEvent(skippedTest, testFileReport) { @@ -468,10 +463,9 @@ class TestObservability { } } - async sendTestRunEvent(eventType, test) { + async sendTestRunEvent(eventType, test, uuid) { Logger.debug(`[sendTestRunEvent] Reached sendTestRunEvent with eventType: ${eventType}`); const testMetaData = test.metadata; - const uuid = test.uuid; const testName = test.testcase; const settings = test.settings || {}; const startTimestamp = test.envelope[testName].startTimestamp; @@ -502,10 +496,6 @@ class TestObservability { integrations: { [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings.desiredCapabilities['bstack:options']?.osVersion) }, - // customRerunParam: { - // rerun_name: '' // Rerun name of test that can be sent in Rerun request to uniquely identify the test - // }, - // retry_of: '', product_map: { accessibility: helper.isAccessibilitySession(), } @@ -514,12 +504,6 @@ class TestObservability { if (eventType === 'TestRunFinished') { const eventData = test.envelope[testName].testcase; testResults = test.testResults; - // const skippedTests =testResults['skippedAtRuntime'].concat(testResults['skippedByUser']) - // if (skippedTests?.length > 0){ - // for (const skippedTest of skippedTests) { - // await this.sendSkippedTestEvent(skippedTest, testMetaData); - // } - // } testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; if (testData.result === 'failed' && testResults.__lastError) { @@ -535,14 +519,6 @@ class TestObservability { } this.processTestRunData (eventData,uuid) - // else if (eventData.status === 'fail' && (testFileReport?.completed[sectionName]?.lastError || testFileReport?.completed[sectionName]?.stackTrace)) { - // const testCompletionData = testFileReport.completed[sectionName]; - // testData.failure = [ - // {'backtrace': [testCompletionData?.stackTrace]} - // ]; - // testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; - // testData.failure_type = testCompletionData?.stackTrace.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; - // } } const uploadData = { @@ -602,27 +578,10 @@ class TestObservability { testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); } - // if (eventType === 'TestRunStarted') { - // testData.type = 'test'; - // testData.integrations = {}; - // const provider = helper.getCloudProvider(testFileReport.host); - // testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); - // } - - // if (eventType === 'TestRunFinished') { - // testData.type = 'test'; - // testData.hooks = hooks; - // } - const uploadData = { event_type: eventType, hook_run: testData }; - // if (eventType.match(/HookRun/)) { - // uploadData['hook_run'] = testData; - // } else { - // uploadData['test_run'] = testData; - // } await helper.uploadEventData(uploadData); } diff --git a/src/utils/testMap.js b/src/utils/testMap.js new file mode 100644 index 0000000..0e80e5f --- /dev/null +++ b/src/utils/testMap.js @@ -0,0 +1,59 @@ +const { v4: uuidv4 } = require('uuid'); + +class TestMap { + constructor() { + this.testMap = new Map(); + this.currentTest = null; + } + + storeTestDetails(test) { + const testIdentifier = this.generateTestIdentifier(test); + + if (!this.testMap.has(testIdentifier)) { + const uuid = this.generateUUID(); + this.testMap.set(testIdentifier, { + uuid, + test, + createdAt: new Date().toISOString() + }); + } + this.currentTest = testIdentifier; + } + + getUUID(test = null) { + if (test) { + const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); + const testData = this.testMap.get(testIdentifier); + return testData ? testData.uuid : null; + } + } + + getTestDetails(identifier) { + if (this.testMap.has(identifier)) { + return this.testMap.get(identifier); + } + return null; + } + + generateTestIdentifier(test) { + if (!test) { + throw new Error('Test object is required to generate identifier'); + } + const testName = test.testcase; + const moduleName = test.metadata.name; + + return `${moduleName}::${testName}`; + } + + generateUUID() { + return uuidv4(); + } + + getAllTests() { + return new Map(this.testMap); + } + + +} + +module.exports = TestMap; \ No newline at end of file From c9e68626960b6af6c54500ca9ae1e0c80d24fec1 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 11 Nov 2025 15:50:20 +0530 Subject: [PATCH 03/46] added EOF lines --- src/utils/helper.js | 3 ++- src/utils/testMap.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/utils/helper.js b/src/utils/helper.js index 82ccdcd..7d53654 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1311,4 +1311,5 @@ exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { result[element[keyName]] = element[valueName]; }); return result; -}; \ No newline at end of file +}; + diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 0e80e5f..69b3004 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -56,4 +56,5 @@ class TestMap { } -module.exports = TestMap; \ No newline at end of file +module.exports = TestMap; + From 8ab63b8febea71cc5b586b8775f37941eb7ec905 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Fri, 14 Nov 2025 21:42:17 +0530 Subject: [PATCH 04/46] TRA changes --- src/testObservability.js | 7 ++----- src/utils/helper.js | 7 +++++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/testObservability.js b/src/testObservability.js index 7849a2f..3fc490b 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -95,7 +95,7 @@ class TestObservability { const accessibilityOptions = accessibility ? this._settings.accessibilityOptions || {} : {}; this._gitMetadata = await helper.getGitMetaData(); const fromProduct = { - test_observability: this._settings.test_observability.enabled || this._settings.test_reporting.enabled || false, + test_observability: this._settings.test_observability?.enabled || this._settings.test_reporting?.enabled || false, accessibility: accessibility }; const data = { @@ -143,8 +143,6 @@ class TestObservability { try { const response = await makeRequest('POST', 'api/v2/builds', data, config, API_URL); - Logger.debug(`[Start_Build] Success response: ${JSON.stringify(response)}`) - console.log(`[Start_Build] Success response: ${JSON.stringify(response)}`) process.env.BS_TESTOPS_BUILD_COMPLETED = true; const responseData = response.data || {}; @@ -471,7 +469,6 @@ class TestObservability { const startTimestamp = test.envelope[testName].startTimestamp; let testResults = {}; const testBody = test.testBody; - Logger.debug(`[sendTestRunEvent] Values - testMetaData: ${JSON.stringify(testMetaData)}, uuid: ${uuid}, testName: ${testName}, settings: ${JSON.stringify(settings)},testResults: ${JSON.stringify(testResults)}, testBody: ${testBody}`); const provider = helper.getCloudProvider(testMetaData.host); const testData = { uuid: uuid, @@ -845,7 +842,7 @@ class TestObservability { automate: product === 'automate', app_automate: product === 'app-automate', observability: helper.isTestObservabilitySession(), - accessibility: settings['@nightwatch/browserstack']?.accessibility === true, + accessibility: helper.isAccessibilityEnabled(settings), turboscale: product === 'turboscale', percy: false }; diff --git a/src/utils/helper.js b/src/utils/helper.js index 7d53654..bddbd1d 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -76,6 +76,13 @@ exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; +exports.isAccessibilityEnabled = (settings) => { + if (process.argv.includes('--disable-accessibility')) + return false; + else + return settings['@nightwatch/browserstack']?.accessibility === true; +}; + exports.getProjectName = (options, bstackOptions={}, fromProduct={}) => { if ((fromProduct.test_observability || fromProduct.test_reporting) && ((options.test_observability && options.test_observability.projectName) || From 506dd0bc09c0a315014faa0f62bcfd9fd268a75b Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 18 Nov 2025 02:14:15 +0530 Subject: [PATCH 05/46] TRA changes pt.3 --- nightwatch/globals.js | 20 ++++++++++++++------ src/testObservability.js | 27 +++++++++++---------------- src/utils/helper.js | 29 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 8bab354..1ef037c 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -21,6 +21,7 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; +let allPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -261,14 +262,15 @@ module.exports = { eventBroadcaster.on('TestRunStarted', async (test) => { testMapInstance.storeTestDetails(test); const uuid = testMapInstance.getUUID(test); - await testObservability.sendTestRunEvent("TestRunStarted", test, uuid); - await accessibilityAutomation.beforeEachExecution(test); + allPromises.push(accessibilityAutomation.beforeEachExecution(test)); + allPromises.push(testObservability.sendTestRunEvent("TestRunStarted", test, uuid)); + }); eventBroadcaster.on('TestRunFinished', async (test) => { const uuid = testMapInstance.getUUID(test); - await testObservability.sendTestRunEvent("TestRunFinished",test, uuid); - await accessibilityAutomation.afterEachExecution(test); + allPromises.push(accessibilityAutomation.afterEachExecution(test, uuid)); + allPromises.push(testObservability.sendTestRunEvent("TestRunFinished", test, uuid)); }); }, @@ -303,6 +305,7 @@ module.exports = { try { testObservability.configure(settings); + accessibilityAutomation.configure(settings); if (helper.isTestObservabilitySession()) { if (settings.reporter_options) { if (settings.reporter_options['save_command_result_value'] !== true){ @@ -335,6 +338,7 @@ module.exports = { try { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); @@ -471,11 +475,14 @@ module.exports = { // This will be run after each test suite is finished async afterEach(settings) { - // await accessibilityAutomation.afterEachExecution(browser); + if (allPromises.length > 0) { + await Promise.all(allPromises); + allPromises = []; + } }, beforeChildProcess(settings) { - + if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } @@ -507,6 +514,7 @@ module.exports = { try { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); + accessibilityAutomation.commandWrapper(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); diff --git a/src/testObservability.js b/src/testObservability.js index 3fc490b..5f39bf0 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -230,15 +230,15 @@ class TestObservability { } - handleErrorForObservability(observabilityData) { + handleErrorForObservability(error) { process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'false'; process.env.BROWSERSTACK_TEST_REPORTING = 'false'; - Logger.error(`Observability could not be started`); //make logging better later + helper.logBuildError(error, 'Test Reporting and Analytics'); } - handleErrorForAccessibility(accessibilityData) { + handleErrorForAccessibility(error) { process.env.BROWSERSTACK_ACCESSIBILITY = 'false'; - Logger.error(`Accessibility could not be started`); //make logging better later + helper.logBuildError(error, 'Accessibility'); } async stopBuildUpstream () { @@ -329,15 +329,6 @@ class TestObservability { await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } - // default: { - // if (eventData.retryTestData?.length>0) { - // for (const retryTest of eventData.retryTestData) { - // await this.processTestRunData(retryTest, sectionName, testFileReport, hookIds); - // } - // } - // await this.processTestRunData(eventData, sectionName, testFileReport, hookIds); - // break; - // } } } if (skippedTests?.length > 0) { @@ -462,13 +453,13 @@ class TestObservability { } async sendTestRunEvent(eventType, test, uuid) { - Logger.debug(`[sendTestRunEvent] Reached sendTestRunEvent with eventType: ${eventType}`); + Logger.debug(`Sending test run event with eventType: ${eventType}`); const testMetaData = test.metadata; const testName = test.testcase; const settings = test.settings || {}; const startTimestamp = test.envelope[testName].startTimestamp; let testResults = {}; - const testBody = test.testBody; + const testBody = this.getTestBody(test.testCaseData); const provider = helper.getCloudProvider(testMetaData.host); const testData = { uuid: uuid, @@ -849,6 +840,10 @@ class TestObservability { return buildProductMap; } -} + + getTestBody(testCaseData) { + return testCaseData.context.__module[testCaseData.testName] || null; + } +} module.exports = TestObservability; diff --git a/src/utils/helper.js b/src/utils/helper.js index bddbd1d..17d610e 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1320,3 +1320,32 @@ exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { return result; }; +exports.logBuildError = (error, product = '') => { + if (!error || !error.errors) { + Logger.error(`${product.toUpperCase()} Build creation failed ${error}`); + return; + } + + for (const errorJson of error.errors) { + const errorType = errorJson.key; + const errorMessage = errorJson.message; + if (errorMessage) { + switch (errorType) { + case 'ERROR_INVALID_CREDENTIALS': + Logger.error(errorMessage); + break; + case 'ERROR_ACCESS_DENIED': + Logger.info(errorMessage); + break; + case 'ERROR_SDK_DEPRECATED': + Logger.error(errorMessage); + break; + default: + Logger.error(errorMessage); + } + } + } +}; + + + From c876a423641fcec1698c43e911f92a7792093bb8 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 18 Nov 2025 02:17:19 +0530 Subject: [PATCH 06/46] fix: lint error --- src/utils/helper.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/helper.js b/src/utils/helper.js index 17d610e..3da3bd0 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1347,5 +1347,3 @@ exports.logBuildError = (error, product = '') => { } }; - - From c107fa112338fcd6e1ea1a64d8900208a97a4129 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 18 Nov 2025 02:27:25 +0530 Subject: [PATCH 07/46] accessibility changes --- src/accessibilityAutomation.js | 249 +++++++++++++++------------------ 1 file changed, 112 insertions(+), 137 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 45c11fd..f58de41 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,16 +1,12 @@ const path = require('path'); const helper = require('./utils/helper'); -const {makeRequest} = require('./utils/requestHelper'); const Logger = require('./utils/logger'); -const {ACCESSIBILITY_URL} = require('./utils/constants'); const util = require('util'); +const AccessibilityScripts = require('./scripts/accessibilityScripts'); class AccessibilityAutomation { configure(settings = {}) { - if (process.argv.includes('--disable-accessibility')) { - process.env.BROWSERSTACK_ACCESSIBILITY = false; - return; - } + this._settings = settings['@nightwatch/browserstack'] || {}; this._testRunner = settings.test_runner; this._bstackOptions = {}; if ( @@ -25,6 +21,14 @@ class AccessibilityAutomation { this._user = helper.getUserName(settings, this._settings); this._key = helper.getAccessKey(settings, this._settings); } + + let accessibilityOptions; + if (helper.isUndefined(this._settings.accessibilityOptions)) { + accessibilityOptions = {}; + } else { + accessibilityOptions = this._settings.accessibilityOptions; + } + process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); } setAccessibilityCapabilities(settings) { @@ -56,9 +60,7 @@ class AccessibilityAutomation { } else { this._bstackOptions.accessibilityOptions = {authToken: process.env.BS_A11Y_JWT}; } - this._bstackOptions.accessibilityOptions.scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + this._bstackOptions.accessibilityOptions.scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } else if (settings.desiredCapabilities['browserstack.accessibility']) { if (settings.desiredCapabilities['browserstack.accessibilityOptions']) { settings.desiredCapabilities['browserstack.accessibilityOptions'].authToken = @@ -68,9 +70,7 @@ class AccessibilityAutomation { authToken: process.env.BS_A11Y_JWT }; } - settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = JSON.parse( - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS - ).scannerVersion; + settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } } } catch (e) { @@ -126,22 +126,6 @@ class AccessibilityAutomation { return false; } - fetchPlatformDetails(driver) { - let response = {}; - try { - response = { - os_name: driver.capabilities.platformName, - os_version: helper.getPlatformVersion(driver), - browser_name: driver.capabilities.browserName, - browser_version: driver.capabilities.browserVersion - }; - } catch (error) { - Logger.debug(`Exception in fetching platform details with error : ${error}`); - } - - return response; - } - validateA11yCaps(driver) { try { const capabilityConfig = driver.desiredCapabilities || {}; @@ -189,54 +173,13 @@ class AccessibilityAutomation { this.currentTest.accessibilityScanStarted = true; this._isAccessibilitySession = this.validateA11yCaps(browser); - if (this.isAccessibilityAutomationSession() && browser && helper.isAccessibilitySession() && this._isAccessibilitySession) { - try { - const session = await browser.session(); - if (session) { - let pageOpen = true; - const currentURL = await browser.driver.getCurrentUrl(); - - let url = {}; - try { - url = new URL(currentURL); - pageOpen = true; - } catch (e) { - pageOpen = false; - } - pageOpen = url.protocol === 'http:' || url.protocol === 'https:'; - - if (pageOpen) { - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info( - 'Setup for Accessibility testing has started. Automate test case execution will begin momentarily.' - ); - - await browser.executeAsyncScript(` - const callback = arguments[arguments.length - 1]; - const fn = () => { - window.addEventListener('A11Y_TAP_STARTED', fn2); - const e = new CustomEvent('A11Y_FORCE_START'); - window.dispatchEvent(e); - }; - const fn2 = () => { - window.removeEventListener('A11Y_TAP_STARTED', fn); - callback(); - } - fn(); - `); - } else { - await browser.executeAsyncScript(` - const e = new CustomEvent('A11Y_FORCE_STOP'); - window.dispatchEvent(e); - `); - } - } + if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { + try { this.currentTest.accessibilityScanStarted = this.currentTest.shouldScanTestForAccessibility; if (this.currentTest.shouldScanTestForAccessibility) { Logger.info('Automate test case execution has started.'); } - } } catch (e) { Logger.error('Exception in starting accessibility automation scan for this test case', e); } @@ -246,46 +189,21 @@ class AccessibilityAutomation { } } - async afterEachExecution(testMetaData) { + async afterEachExecution(testMetaData, uuid) { try { if (this.currentTest.accessibilityScanStarted && this.isAccessibilityAutomationSession() && this._isAccessibilitySession) { if (this.currentTest.shouldScanTestForAccessibility) { Logger.info( 'Automate test case execution has ended. Processing for accessibility testing is underway. ' ); - } - const dataForExtension = { - saveResults: this.currentTest.shouldScanTestForAccessibility, - testDetails: { - name: testMetaData.testcase, - testRunId: process.env.BS_A11Y_TEST_RUN_ID, - filePath: testMetaData.metadata.modulePath, - scopeList: [testMetaData.metadata.name, testMetaData.testcase] - }, - platform: await this.fetchPlatformDetails(browser) - }; - const final_res = await browser.executeAsyncScript( - ` - const callback = arguments[arguments.length - 1]; - - this.res = null; - if (arguments[0].saveResults) { - window.addEventListener('A11Y_TAP_TRANSPORTER', (event) => { - window.tapTransporterData = event.detail; - this.res = window.tapTransporterData; - callback(this.res); - }); - } - const e = new CustomEvent('A11Y_TEST_END', {detail: arguments[0]}); - window.dispatchEvent(e); - if (arguments[0].saveResults !== true ) { - callback(); - } - `, - dataForExtension - ); - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Accessibility testing for this test case has ended.'); + + const dataForExtension = { + 'thTestRunUuid': uuid, + 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, + 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT + } + await this.sendTestStopEvent(browser, dataForExtension) + Logger.info('Accessibility testing for this test case has ended.'); } } } catch (er) { @@ -303,22 +221,9 @@ class AccessibilityAutomation { return {}; } try { - const results = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try { - const event = new CustomEvent('A11Y_TAP_GET_RESULTS'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_RESPONSE', fn); - resolve(event.detail.data); - }; - window.addEventListener('A11Y_RESULTS_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); - + Logger.debug('Performing scan before getting results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.getResults); return results; } catch { Logger.error('No accessibility results were found.'); @@ -336,22 +241,9 @@ class AccessibilityAutomation { return {}; } try { - const summaryResults = await browser.executeScript(` - return new Promise(function (resolve, reject) { - try{ - const event = new CustomEvent('A11Y_TAP_GET_RESULTS_SUMMARY'); - const fn = function (event) { - window.removeEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - resolve(event.detail.summary); - }; - window.addEventListener('A11Y_RESULTS_SUMMARY_RESPONSE', fn); - window.dispatchEvent(event); - } catch { - reject(); - } - }); - `); - + Logger.debug('Performing scan before getting results summary'); + await this.performScan(browser); + const summaryResults = await browser.executeAsyncScript(AccessibilityScripts.getResultsSummary); return summaryResults; } catch { Logger.error('No accessibility summary was found.'); @@ -364,6 +256,89 @@ class AccessibilityAutomation { return Object.fromEntries(Object.entries(accessibilityOptions).filter(([k, v]) => !(k.toLowerCase() === 'excludetagsintestingscope' || k.toLowerCase() === 'includetagsintestingscope'))); } + async performScan( browserInstance = null, commandName = '') { + + if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { + Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); + return; + } + + if (this.currentTest.shouldScanTestForAccessibility === false) { + Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); + return; + } + try { + const browser = browserInstance; + + if (!browser) { + Logger.error('No browser instance available for accessibility scan'); + return; + } + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { + method: commandName || '' + }); + Logger.debug(util.inspect(results)); + return results; + + } catch (err) { + Logger.error('Accessibility Scan could not be performed: ' + err.message); + Logger.debug('Stack trace:', err.stack); + return; + } + } + +async sendTestStopEvent(browser, dataForExtension = {}) { + Logger.debug('Performing scan before saving results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); + Logger.debug(util.inspect(results)); +} + +async commandWrapper() { + try { + const nightwatchMain = require.resolve('nightwatch'); + const nightwatchDir = path.dirname(nightwatchMain); + + const commandJson = AccessibilityScripts.commandsToWrap; + const accessibilityInstance = this; + for (const commandKey in commandJson) { + if (commandJson[commandKey].method === 'protocolAction'){ + try { + commandJson[commandKey].name.forEach(commandName => { + const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const OriginalClass = require(commandPath); + const originalProtocolAction = OriginalClass.prototype.protocolAction; + + OriginalClass.prototype.protocolAction = async function() { + await accessibilityInstance.performScan(browser, commandName); + return originalProtocolAction.apply(this); + }; + }); + } catch (error) { + Logger.debug(`Failed to patch protocolAction for command: ${error.message}`); + } + } + else { + try { + commandJson[commandKey].name.forEach(commandName => { + const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const originalCommand = require(webElementCommandPath); + const originalCommandFn = originalCommand.command; + + originalCommand.command = async function(...args) { + await accessibilityInstance.performScan(browser, commandName); + return originalCommandFn.apply(this, args); + }; + }); + } catch (error) { + Logger.debug(`Failed to patch command ${commandName}: ${error.message}`); + } + } + } + } catch (error) { + Logger.debug(`Command patching failed: ${error.message}`); + } +} } module.exports = AccessibilityAutomation; From 9ada09f4b8dae8c9477100e8743da85f1bf18f70 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 18 Nov 2025 02:30:11 +0530 Subject: [PATCH 08/46] fix: lint error --- src/utils/testMap.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 69b3004..8bb2c1a 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -52,8 +52,6 @@ class TestMap { getAllTests() { return new Map(this.testMap); } - - } module.exports = TestMap; From ff58a6af07352091e31a3168fe4b08dc5182922a Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 18 Nov 2025 17:01:37 +0530 Subject: [PATCH 09/46] minor change --- src/testObservability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/testObservability.js b/src/testObservability.js index 5f39bf0..ff96d1d 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -397,7 +397,7 @@ class TestObservability { file_name: path.relative(process.cwd(), testFileReport.modulePath), location: path.relative(process.cwd(), testFileReport.modulePath), vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testFileReport.modulePath) : null, - started_at: new Date(testFileReport.startTimestamp).toISOString(), + started_at: new Date(testFileReport.endTimestamp).toISOString(), finished_at: new Date(testFileReport.endTimestamp).toISOString(), duration_in_ms: 0, result: 'skipped', From 7344585bd5d1ce2f900e10f81abe389bd029cebf Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 19 Nov 2025 13:51:36 +0530 Subject: [PATCH 10/46] minor change --- src/utils/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/helper.js b/src/utils/helper.js index 3da3bd0..f4432e5 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -646,7 +646,7 @@ exports.getIntegrationsObject = (capabilities, sessionId, hostname, platform_ver browser: capabilities.browserName, browser_version: capabilities.browserVersion, platform: capabilities.platformName, - platform_version: platform_version, + platform_version: capabilities.platformVersion || platform_version, product: this.getObservabilityLinkedProductName(capabilities, hostname) }; }; From 3a3910a8b10222a7b11b13388d059714206447a0 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 19 Nov 2025 14:02:34 +0530 Subject: [PATCH 11/46] fix:lint errors --- nightwatch/globals.js | 6 +- src/accessibilityAutomation.js | 122 +++++++++++---------- src/scripts/accessibilityScripts.js | 158 ++++++++++++++-------------- src/testObservability.js | 106 ++++++++++--------- src/utils/helper.js | 32 +++--- src/utils/requestQueueHandler.js | 2 +- src/utils/testMap.js | 4 +- 7 files changed, 225 insertions(+), 205 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 1ef037c..fbada87 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -263,14 +263,14 @@ module.exports = { testMapInstance.storeTestDetails(test); const uuid = testMapInstance.getUUID(test); allPromises.push(accessibilityAutomation.beforeEachExecution(test)); - allPromises.push(testObservability.sendTestRunEvent("TestRunStarted", test, uuid)); + allPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); }); eventBroadcaster.on('TestRunFinished', async (test) => { const uuid = testMapInstance.getUUID(test); allPromises.push(accessibilityAutomation.afterEachExecution(test, uuid)); - allPromises.push(testObservability.sendTestRunEvent("TestRunFinished", test, uuid)); + allPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); }); }, @@ -332,7 +332,7 @@ module.exports = { } } } catch (error) { - Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); + Logger.error(`Could not configure or launch test reporting and analytics - ${error}`); } try { diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index f58de41..6c93c67 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -22,13 +22,13 @@ class AccessibilityAutomation { this._key = helper.getAccessKey(settings, this._settings); } - let accessibilityOptions; - if (helper.isUndefined(this._settings.accessibilityOptions)) { - accessibilityOptions = {}; - } else { - accessibilityOptions = this._settings.accessibilityOptions; - } - process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); + let accessibilityOptions; + if (helper.isUndefined(this._settings.accessibilityOptions)) { + accessibilityOptions = {}; + } else { + accessibilityOptions = this._settings.accessibilityOptions; + } + process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); } setAccessibilityCapabilities(settings) { @@ -175,11 +175,11 @@ class AccessibilityAutomation { if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { try { - this.currentTest.accessibilityScanStarted = + this.currentTest.accessibilityScanStarted = this.currentTest.shouldScanTestForAccessibility; - if (this.currentTest.shouldScanTestForAccessibility) { - Logger.info('Automate test case execution has started.'); - } + if (this.currentTest.shouldScanTestForAccessibility) { + Logger.info('Automate test case execution has started.'); + } } catch (e) { Logger.error('Exception in starting accessibility automation scan for this test case', e); } @@ -201,8 +201,8 @@ class AccessibilityAutomation { 'thTestRunUuid': uuid, 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT - } - await this.sendTestStopEvent(browser, dataForExtension) + }; + await this.sendTestStopEvent(browser, dataForExtension); Logger.info('Accessibility testing for this test case has ended.'); } } @@ -224,6 +224,7 @@ class AccessibilityAutomation { Logger.debug('Performing scan before getting results'); await this.performScan(browser); const results = await browser.executeAsyncScript(AccessibilityScripts.getResults); + return results; } catch { Logger.error('No accessibility results were found.'); @@ -244,6 +245,7 @@ class AccessibilityAutomation { Logger.debug('Performing scan before getting results summary'); await this.performScan(browser); const summaryResults = await browser.executeAsyncScript(AccessibilityScripts.getResultsSummary); + return summaryResults; } catch { Logger.error('No accessibility summary was found.'); @@ -256,46 +258,51 @@ class AccessibilityAutomation { return Object.fromEntries(Object.entries(accessibilityOptions).filter(([k, v]) => !(k.toLowerCase() === 'excludetagsintestingscope' || k.toLowerCase() === 'includetagsintestingscope'))); } - async performScan( browserInstance = null, commandName = '') { + async performScan(browserInstance = null, commandName = '') { - if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { - Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); - return; - } + if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { + Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); - if (this.currentTest.shouldScanTestForAccessibility === false) { - Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); - return; - } - try { - const browser = browserInstance; + return; + } + + if (this.currentTest.shouldScanTestForAccessibility === false) { + Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); + + return; + } + try { + const browser = browserInstance; - if (!browser) { - Logger.error('No browser instance available for accessibility scan'); - return; - } - const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { - method: commandName || '' - }); - Logger.debug(util.inspect(results)); - return results; - - } catch (err) { - Logger.error('Accessibility Scan could not be performed: ' + err.message); - Logger.debug('Stack trace:', err.stack); + if (!browser) { + Logger.error('No browser instance available for accessibility scan'); + return; } + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { + method: commandName || '' + }); + Logger.debug(util.inspect(results)); + + return results; + + } catch (err) { + Logger.error('Accessibility Scan could not be performed: ' + err.message); + Logger.debug('Stack trace:', err.stack); + + return; + } } -async sendTestStopEvent(browser, dataForExtension = {}) { - Logger.debug('Performing scan before saving results'); - await this.performScan(browser); - const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); - Logger.debug(util.inspect(results)); -} + async sendTestStopEvent(browser, dataForExtension = {}) { + Logger.debug('Performing scan before saving results'); + await this.performScan(browser); + const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); + Logger.debug(util.inspect(results)); + } -async commandWrapper() { - try { + async commandWrapper() { + try { const nightwatchMain = require.resolve('nightwatch'); const nightwatchDir = path.dirname(nightwatchMain); @@ -303,22 +310,22 @@ async commandWrapper() { const accessibilityInstance = this; for (const commandKey in commandJson) { if (commandJson[commandKey].method === 'protocolAction'){ - try { + try { commandJson[commandKey].name.forEach(commandName => { const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); const OriginalClass = require(commandPath); const originalProtocolAction = OriginalClass.prototype.protocolAction; OriginalClass.prototype.protocolAction = async function() { - await accessibilityInstance.performScan(browser, commandName); - return originalProtocolAction.apply(this); + await accessibilityInstance.performScan(browser, commandName); + + return originalProtocolAction.apply(this); }; }); - } catch (error) { + } catch (error) { Logger.debug(`Failed to patch protocolAction for command: ${error.message}`); - } - } - else { + } + } else { try { commandJson[commandKey].name.forEach(commandName => { const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); @@ -326,8 +333,9 @@ async commandWrapper() { const originalCommandFn = originalCommand.command; originalCommand.command = async function(...args) { - await accessibilityInstance.performScan(browser, commandName); - return originalCommandFn.apply(this, args); + await accessibilityInstance.performScan(browser, commandName); + + return originalCommandFn.apply(this, args); }; }); } catch (error) { @@ -335,10 +343,10 @@ async commandWrapper() { } } } - } catch (error) { - Logger.debug(`Command patching failed: ${error.message}`); + } catch (error) { + Logger.debug(`Command patching failed: ${error.message}`); + } } } -} module.exports = AccessibilityAutomation; diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js index cd4bae8..90844d3 100644 --- a/src/scripts/accessibilityScripts.js +++ b/src/scripts/accessibilityScripts.js @@ -3,95 +3,99 @@ const fs = require('fs'); const os = require('os'); class AccessibilityScripts { - static instance = null; - - performScan = null; - getResults = null; - getResultsSummary = null; - saveTestResults = null; - commandsToWrap = null; - ChromeExtension = {}; - - browserstackFolderPath = ''; - commandsPath = ''; - - // don't allow to create instances from it other than through `checkAndGetInstance` - constructor() { - this.browserstackFolderPath = this.getWritableDir(); - this.commandsPath = path.join(this.browserstackFolderPath, 'commands.json'); - } + static instance = null; - static checkAndGetInstance() { - if (!AccessibilityScripts.instance) { - AccessibilityScripts.instance = new AccessibilityScripts(); - AccessibilityScripts.instance.readFromExistingFile(); - } - return AccessibilityScripts.instance; - } + performScan = null; + getResults = null; + getResultsSummary = null; + saveTestResults = null; + commandsToWrap = null; + ChromeExtension = {}; - getWritableDir() { - const orderedPaths = [ - path.join(os.homedir(), '.browserstack'), - process.cwd(), - os.tmpdir() - ]; - for (const orderedPath of orderedPaths) { - try { - if (fs.existsSync(orderedPath)) { - fs.accessSync(orderedPath); - return orderedPath; - } - - fs.mkdirSync(orderedPath, { recursive: true }); - return orderedPath; - - } catch (e) { - /* no-empty */ - } - } - return ''; + browserstackFolderPath = ''; + commandsPath = ''; + + // don't allow to create instances from it other than through `checkAndGetInstance` + constructor() { + this.browserstackFolderPath = this.getWritableDir(); + this.commandsPath = path.join(this.browserstackFolderPath, 'commands.json'); + } + + static checkAndGetInstance() { + if (!AccessibilityScripts.instance) { + AccessibilityScripts.instance = new AccessibilityScripts(); + AccessibilityScripts.instance.readFromExistingFile(); } - readFromExistingFile() { - try { - if (fs.existsSync(this.commandsPath)) { - const data = fs.readFileSync(this.commandsPath, 'utf8'); - if (data) { - this.update(JSON.parse(data)); - } - } - } catch (error) { - /* Do nothing */ + return AccessibilityScripts.instance; + } + + getWritableDir() { + const orderedPaths = [ + path.join(os.homedir(), '.browserstack'), + process.cwd(), + os.tmpdir() + ]; + for (const orderedPath of orderedPaths) { + try { + if (fs.existsSync(orderedPath)) { + fs.accessSync(orderedPath); + + return orderedPath; } + + fs.mkdirSync(orderedPath, {recursive: true}); + + return orderedPath; + + } catch (e) { + /* no-empty */ + } } - update(data) { - if (data.scripts) { - this.performScan = data.scripts.scan; - this.getResults = data.scripts.getResults; - this.getResultsSummary = data.scripts.getResultsSummary; - this.saveTestResults = data.scripts.saveResults; - } - if (data.commands && data.commands.length) { - this.commandsToWrap = data.commands; + return ''; + } + + readFromExistingFile() { + try { + if (fs.existsSync(this.commandsPath)) { + const data = fs.readFileSync(this.commandsPath, 'utf8'); + if (data) { + this.update(JSON.parse(data)); } + } + } catch (error) { + /* Do nothing */ } + } - store() { - if (!fs.existsSync(this.browserstackFolderPath)){ - fs.mkdirSync(this.browserstackFolderPath); - } + update(data) { + if (data.scripts) { + this.performScan = data.scripts.scan; + this.getResults = data.scripts.getResults; + this.getResultsSummary = data.scripts.getResultsSummary; + this.saveTestResults = data.scripts.saveResults; + } + if (data.commands && data.commands.length) { + this.commandsToWrap = data.commands; + } + } - fs.writeFileSync(this.commandsPath, JSON.stringify({ - commands: this.commandsToWrap, - scripts: { - scan: this.performScan, - getResults: this.getResults, - getResultsSummary: this.getResultsSummary, - saveResults: this.saveTestResults, - }, - })); + store() { + if (!fs.existsSync(this.browserstackFolderPath)){ + fs.mkdirSync(this.browserstackFolderPath); } + + fs.writeFileSync(this.commandsPath, JSON.stringify({ + commands: this.commandsToWrap, + scripts: { + scan: this.performScan, + getResults: this.getResults, + getResultsSummary: this.getResultsSummary, + saveResults: this.saveTestResults + } + })); + } } module.exports = AccessibilityScripts.checkAndGetInstance(); diff --git a/src/testObservability.js b/src/testObservability.js index ff96d1d..d2f45ce 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -115,14 +115,14 @@ class TestObservability { settings: accessibilityOptions }, framework_details: { - frameworkName: helper.getFrameworkName(this._testRunner), - frameworkVersion: helper.getPackageVersion('nightwatch'), - sdkVersion: helper.getAgentVersion(), - language: 'javascript', - testFramework: { - name: 'nightwatch', - version: helper.getPackageVersion('nightwatch') - } + frameworkName: helper.getFrameworkName(this._testRunner), + frameworkVersion: helper.getPackageVersion('nightwatch'), + sdkVersion: helper.getAgentVersion(), + language: 'javascript', + testFramework: { + name: 'nightwatch', + version: helper.getPackageVersion('nightwatch') + } }, product_map: this.getProductMapForBuildStartCall(this._parentSettings), browserstackAutomation: helper.isBrowserstackInfra(this._settings), @@ -178,54 +178,58 @@ class TestObservability { } processTestObservabilityResponse(responseData) { - if (!responseData.observability) { - this.handleErrorForObservability(null) - return + if (!responseData.observability) { + this.handleErrorForObservability(null); + + return; } if (!responseData.observability.success) { - this.handleErrorForObservability(responseData.observability) - return + this.handleErrorForObservability(responseData.observability); + + return; } process.env.BROWSERSTACK_TEST_OBSERVABILITY = 'true'; process.env.BROWSERSTACK_TEST_REPORTING = 'true'; if (responseData.observability.options.allow_screenshots) { - process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.observability.options.allow_screenshots.toString(); + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = responseData.observability.options.allow_screenshots.toString(); } } processAccessibilityResponse(responseData, settings) { if (!responseData.accessibility) { - if (settings.accessibility === true) { - this.handleErrorForAccessibility(null) - } - return + if (settings.accessibility === true) { + this.handleErrorForAccessibility(null); + } + + return; } if (!responseData.accessibility.success) { - this.handleErrorForAccessibility(responseData.accessibility) - return + this.handleErrorForAccessibility(responseData.accessibility); + + return; } if (responseData.accessibility.options) { - const { accessibilityToken, pollingTimeout, scannerVersion } = helper.jsonifyAccessibilityArray(responseData.accessibility.options.capabilities, 'name', 'value') - const scriptsJson = { - 'scripts': helper.jsonifyAccessibilityArray(responseData.accessibility.options.scripts, 'name', 'command'), - 'commands': responseData.accessibility.options.commandsToWrap?.commands ?? [], - } - if (scannerVersion) { - process.env.BSTACK_A11Y_SCANNER_VERSION = scannerVersion - Logger.debug(`Accessibility scannerVersion ${scannerVersion}`) - } - if (accessibilityToken) { - process.env.BS_A11Y_JWT = accessibilityToken - process.env.BROWSERSTACK_ACCESSIBILITY = 'true' - } - if (pollingTimeout) { - process.env.BSTACK_A11Y_POLLING_TIMEOUT = pollingTimeout - } - if (scriptsJson) { - accessibilityScripts.update(scriptsJson); - accessibilityScripts.store(); - } + const {accessibilityToken, pollingTimeout, scannerVersion} = helper.jsonifyAccessibilityArray(responseData.accessibility.options.capabilities, 'name', 'value'); + const scriptsJson = { + 'scripts': helper.jsonifyAccessibilityArray(responseData.accessibility.options.scripts, 'name', 'command'), + 'commands': responseData.accessibility.options.commandsToWrap?.commands ?? [] + }; + if (scannerVersion) { + process.env.BSTACK_A11Y_SCANNER_VERSION = scannerVersion; + Logger.debug(`Accessibility scannerVersion ${scannerVersion}`); + } + if (accessibilityToken) { + process.env.BS_A11Y_JWT = accessibilityToken; + process.env.BROWSERSTACK_ACCESSIBILITY = 'true'; + } + if (pollingTimeout) { + process.env.BSTACK_A11Y_POLLING_TIMEOUT = pollingTimeout; + } + if (scriptsJson) { + accessibilityScripts.update(scriptsJson); + accessibilityScripts.store(); + } } } @@ -485,7 +489,7 @@ class TestObservability { [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings.desiredCapabilities['bstack:options']?.osVersion) }, product_map: { - accessibility: helper.isAccessibilitySession(), + accessibility: helper.isAccessibilitySession() } }; @@ -506,7 +510,7 @@ class TestObservability { } } - this.processTestRunData (eventData,uuid) + this.processTestRunData (eventData, uuid); } const uploadData = { @@ -829,16 +833,16 @@ class TestObservability { getProductMapForBuildStartCall(settings) { const product = helper.getObservabilityLinkedProductName(settings.desiredCapabilities, settings?.selenium?.host); - const buildProductMap = { - automate: product === 'automate', - app_automate: product === 'app-automate', - observability: helper.isTestObservabilitySession(), - accessibility: helper.isAccessibilityEnabled(settings), - turboscale: product === 'turboscale', - percy: false - }; + const buildProductMap = { + automate: product === 'automate', + app_automate: product === 'app-automate', + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilityEnabled(settings), + turboscale: product === 'turboscale', + percy: false + }; - return buildProductMap; + return buildProductMap; } getTestBody(testCaseData) { diff --git a/src/utils/helper.js b/src/utils/helper.js index f4432e5..96abd73 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -77,10 +77,9 @@ exports.isAccessibilitySession = () => { }; exports.isAccessibilityEnabled = (settings) => { - if (process.argv.includes('--disable-accessibility')) - return false; - else - return settings['@nightwatch/browserstack']?.accessibility === true; + if (process.argv.includes('--disable-accessibility')) {return false} + + return settings['@nightwatch/browserstack']?.accessibility === true; }; exports.getProjectName = (options, bstackOptions={}, fromProduct={}) => { @@ -385,6 +384,7 @@ exports.getHostInfo = () => { exports.isBrowserstackInfra = (settings) => { const isBrowserstackInfra = settings && settings.webdriver && settings.webdriver.host && settings.webdriver.host.indexOf('browserstack') === -1 ? false : true; + return isBrowserstackInfra; }; @@ -1317,12 +1317,14 @@ exports.jsonifyAccessibilityArray = (dataArray, keyName, valueName) => { dataArray.forEach((element) => { result[element[keyName]] = element[valueName]; }); + return result; }; exports.logBuildError = (error, product = '') => { if (!error || !error.errors) { Logger.error(`${product.toUpperCase()} Build creation failed ${error}`); + return; } @@ -1331,17 +1333,17 @@ exports.logBuildError = (error, product = '') => { const errorMessage = errorJson.message; if (errorMessage) { switch (errorType) { - case 'ERROR_INVALID_CREDENTIALS': - Logger.error(errorMessage); - break; - case 'ERROR_ACCESS_DENIED': - Logger.info(errorMessage); - break; - case 'ERROR_SDK_DEPRECATED': - Logger.error(errorMessage); - break; - default: - Logger.error(errorMessage); + case 'ERROR_INVALID_CREDENTIALS': + Logger.error(errorMessage); + break; + case 'ERROR_ACCESS_DENIED': + Logger.info(errorMessage); + break; + case 'ERROR_SDK_DEPRECATED': + Logger.error(errorMessage); + break; + default: + Logger.error(errorMessage); } } } diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index 18543a3..73a6e7c 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -64,7 +64,7 @@ class RequestQueueHandler { startEventBatchPolling () { this.pollEventBatchInterval = setInterval(async () => { - if (this.data.length > 0) { + if (this.data.length > 0) { // const data = this.queue.slice(0, BATCH_SIZE); // this.queue.splice(0, BATCH_SIZE); await this.batchAndPostEvents(this.eventUrl, 'Interval-Queue', this.data); diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 8bb2c1a..e19aa84 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -1,4 +1,4 @@ -const { v4: uuidv4 } = require('uuid'); +const {v4: uuidv4} = require('uuid'); class TestMap { constructor() { @@ -24,6 +24,7 @@ class TestMap { if (test) { const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); const testData = this.testMap.get(testIdentifier); + return testData ? testData.uuid : null; } } @@ -32,6 +33,7 @@ class TestMap { if (this.testMap.has(identifier)) { return this.testMap.get(identifier); } + return null; } From 04a4ab0bd70f53f01f3b719ebe28819d263db94b Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 19 Nov 2025 15:36:06 +0530 Subject: [PATCH 12/46] added null checks --- src/testObservability.js | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/testObservability.js b/src/testObservability.js index d2f45ce..1472e04 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -486,7 +486,7 @@ class TestObservability { result: 'pending', framework: 'nightwatch', integrations: { - [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings.desiredCapabilities['bstack:options']?.osVersion) + [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) }, product_map: { accessibility: helper.isAccessibilitySession() @@ -496,19 +496,23 @@ class TestObservability { if (eventType === 'TestRunFinished') { const eventData = test.envelope[testName].testcase; testResults = test.testResults; - testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); - testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; - if (testData.result === 'failed' && testResults.__lastError) { - testData.failure = [ - { - 'backtrace': [stripAnsi(testResults.__lastError.message), testResults.__lastError.stack] + if (!testResults) { + Logger.debug(`Test results could not be retrieved for test: ${testName}. Skipping result processing.`); + } else { + testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); + testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; + if (testData.result === 'failed' && testResults.__lastError) { + testData.failure = [ + { + 'backtrace': [stripAnsi(testResults.__lastError.message), testResults.__lastError.stack] + } + ]; + testData.failure_reason = testResults.__lastError ? stripAnsi(testResults.__lastError.message) : null; + if (testResults.__lastError && testResults.__lastError.name) { + testData.failure_type = testResults.__lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; } - ]; - testData.failure_reason = testResults.__lastError ? stripAnsi(testResults.__lastError.message) : null; - if (testResults.__lastError && testResults.__lastError.name) { - testData.failure_type = testResults.__lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; } - } + } this.processTestRunData (eventData, uuid); } @@ -846,7 +850,7 @@ class TestObservability { } getTestBody(testCaseData) { - return testCaseData.context.__module[testCaseData.testName] || null; + return testCaseData?.context.__module[testCaseData.testName] || null; } } From 04f5edbd51392444d201d9a2d4d8573a2a93dde6 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 00:45:15 +0530 Subject: [PATCH 13/46] review changes pt.1 --- nightwatch/globals.js | 15 ++++----------- src/accessibilityAutomation.js | 2 +- src/scripts/accessibilityScripts.js | 6 ++++-- src/testObservability.js | 2 +- src/utils/requestQueueHandler.js | 21 ++++++++++----------- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index fbada87..715298f 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -21,7 +21,6 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; -let allPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -262,15 +261,14 @@ module.exports = { eventBroadcaster.on('TestRunStarted', async (test) => { testMapInstance.storeTestDetails(test); const uuid = testMapInstance.getUUID(test); - allPromises.push(accessibilityAutomation.beforeEachExecution(test)); - allPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); - + await accessibilityAutomation.beforeEachExecution(test); + await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); }); eventBroadcaster.on('TestRunFinished', async (test) => { const uuid = testMapInstance.getUUID(test); - allPromises.push(accessibilityAutomation.afterEachExecution(test, uuid)); - allPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); + await accessibilityAutomation.afterEachExecution(test, uuid); + await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); }); }, @@ -470,15 +468,10 @@ module.exports = { async beforeEach(settings) { browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; - // await accessibilityAutomation.beforeEachExecution(browser); }, // This will be run after each test suite is finished async afterEach(settings) { - if (allPromises.length > 0) { - await Promise.all(allPromises); - allPromises = []; - } }, beforeChildProcess(settings) { diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 6c93c67..534a7be 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -339,7 +339,7 @@ class AccessibilityAutomation { }; }); } catch (error) { - Logger.debug(`Failed to patch command ${commandName}: ${error.message}`); + Logger.debug(`Failed to patch command: ${error.message}`); } } } diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js index 90844d3..74d766d 100644 --- a/src/scripts/accessibilityScripts.js +++ b/src/scripts/accessibilityScripts.js @@ -1,6 +1,8 @@ const path = require('path'); const fs = require('fs'); const os = require('os'); +const Logger = require('../utils/logger.js'); + class AccessibilityScripts { static instance = null; @@ -49,7 +51,7 @@ class AccessibilityScripts { return orderedPath; } catch (e) { - /* no-empty */ + Logger.debug(`Failed to access or create directory ${orderedPath}: ${e.message}`); } } @@ -65,7 +67,7 @@ class AccessibilityScripts { } } } catch (error) { - /* Do nothing */ + Logger.debug(`Failed to read accessibility commands file: ${error.message}`); } } diff --git a/src/testObservability.js b/src/testObservability.js index 1472e04..6c9bde9 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -514,7 +514,7 @@ class TestObservability { } } - this.processTestRunData (eventData, uuid); + await this.processTestRunData (eventData, uuid); } const uploadData = { diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index 73a6e7c..a51f4f0 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -11,7 +11,6 @@ class RequestQueueHandler { this.BATCH_EVENT_TYPES = ['LogCreated', 'TestRunFinished', 'TestRunSkipped', 'HookRunFinished', 'TestRunStarted', 'HookRunStarted']; this.pollEventBatchInterval = null; this.pending_test_uploads = 0; - this.data = null; } start() { @@ -32,17 +31,17 @@ class RequestQueueHandler { } this.queue.push(event); - // let data = null; const shouldProceed = this.shouldProceed(); + let data = null; if (shouldProceed) { - this.data = this.queue.slice(0, BATCH_SIZE); + data = this.queue.slice(0, BATCH_SIZE); this.queue.splice(0, BATCH_SIZE); this.resetEventBatchPolling(); } return { shouldProceed: shouldProceed, - proceedWithData: this.data, + proceedWithData: data, proceedWithUrl: this.eventUrl }; } @@ -64,10 +63,10 @@ class RequestQueueHandler { startEventBatchPolling () { this.pollEventBatchInterval = setInterval(async () => { - if (this.data.length > 0) { - // const data = this.queue.slice(0, BATCH_SIZE); - // this.queue.splice(0, BATCH_SIZE); - await this.batchAndPostEvents(this.eventUrl, 'Interval-Queue', this.data); + if (this.queue.length > 0) { + const data = this.queue.slice(0, BATCH_SIZE); + this.queue.splice(0, BATCH_SIZE); + await this.batchAndPostEvents(this.eventUrl, 'Interval-Queue', data); } }, BATCH_INTERVAL); } @@ -88,7 +87,7 @@ class RequestQueueHandler { } shouldProceed () { - return this.queue.length <= BATCH_SIZE; + return this.queue.length >= BATCH_SIZE; } async batchAndPostEvents (eventUrl, kind, data) { @@ -102,7 +101,7 @@ class RequestQueueHandler { try { const response = await makeRequest('POST', eventUrl, data, config); - if (response.data.error) { + if (response.data && response.data.error) { throw ({message: response.data.error}); } else { this.pending_test_uploads = Math.max(0, this.pending_test_uploads - data.length); @@ -115,7 +114,7 @@ class RequestQueueHandler { } this.pending_test_uploads = Math.max(0, this.pending_test_uploads - data.length); } - }; + } } module.exports = new RequestQueueHandler(); From dc47ae8cee787ca98d038e2561a72071fe8a6548 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 00:47:01 +0530 Subject: [PATCH 14/46] minor change --- src/utils/requestQueueHandler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/requestQueueHandler.js b/src/utils/requestQueueHandler.js index a51f4f0..f9ad180 100644 --- a/src/utils/requestQueueHandler.js +++ b/src/utils/requestQueueHandler.js @@ -31,8 +31,8 @@ class RequestQueueHandler { } this.queue.push(event); - const shouldProceed = this.shouldProceed(); let data = null; + const shouldProceed = this.shouldProceed(); if (shouldProceed) { data = this.queue.slice(0, BATCH_SIZE); this.queue.splice(0, BATCH_SIZE); @@ -114,7 +114,7 @@ class RequestQueueHandler { } this.pending_test_uploads = Math.max(0, this.pending_test_uploads - data.length); } - } + }; } module.exports = new RequestQueueHandler(); From 1d9431f8a04791e11c3c4d968464369d5fbad0de Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 15:09:28 +0530 Subject: [PATCH 15/46] fix: static testMap added --- nightwatch/globals.js | 15 +++++++-------- src/utils/testMap.js | 37 ++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 715298f..690a3ce 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -13,7 +13,6 @@ const TestMap = require('../src/utils/testMap'); const localTunnel = new LocalTunnel(); const testObservability = new TestObservability(); const accessibilityAutomation = new AccessibilityAutomation(); -const testMapInstance = new TestMap(); const nightwatchRerun = process.env.NIGHTWATCH_RERUN_FAILED; const nightwatchRerunFile = process.env.NIGHTWATCH_RERUN_REPORT_FILE; @@ -67,8 +66,8 @@ module.exports = { promises.push(testObservability.processTestReportFile(JSON.parse(JSON.stringify(modulesWithEnv[testSetting][testFile])))); } } - await Promise.all(promises); + done(); } catch (error) { CrashReporter.uploadCrashReport(error.message, error.stack); @@ -259,16 +258,16 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - testMapInstance.storeTestDetails(test); - const uuid = testMapInstance.getUUID(test); - await accessibilityAutomation.beforeEachExecution(test); - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); + TestMap.storeTestDetails(test); + const uuid = TestMap.getUUID(test); + await accessibilityAutomation.beforeEachExecution(test) + await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) }); eventBroadcaster.on('TestRunFinished', async (test) => { - const uuid = testMapInstance.getUUID(test); + const uuid = TestMap.getUUID(test); await accessibilityAutomation.afterEachExecution(test, uuid); - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); + await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) }); }, diff --git a/src/utils/testMap.js b/src/utils/testMap.js index e19aa84..5afe57b 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -1,43 +1,42 @@ const {v4: uuidv4} = require('uuid'); -class TestMap { - constructor() { - this.testMap = new Map(); - this.currentTest = null; - } +const sharedTestMap = new Map(); +let sharedCurrentTest = null; - storeTestDetails(test) { +class TestMap { + + static storeTestDetails(test) { const testIdentifier = this.generateTestIdentifier(test); - if (!this.testMap.has(testIdentifier)) { + if (!sharedTestMap.has(testIdentifier)) { const uuid = this.generateUUID(); - this.testMap.set(testIdentifier, { + sharedTestMap.set(testIdentifier, { uuid, test, createdAt: new Date().toISOString() }); } - this.currentTest = testIdentifier; + sharedCurrentTest = testIdentifier; } - getUUID(test = null) { + static getUUID(test = null) { if (test) { - const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); - const testData = this.testMap.get(testIdentifier); + const testIdentifier = typeof test === 'string' ? test : generateTestIdentifier(test); + const testData = sharedTestMap.get(testIdentifier); return testData ? testData.uuid : null; } } - getTestDetails(identifier) { - if (this.testMap.has(identifier)) { - return this.testMap.get(identifier); + static getTestDetails(identifier) { + if (sharedTestMap.has(identifier)) { + return sharedTestMap.get(identifier); } return null; } - generateTestIdentifier(test) { + static generateTestIdentifier(test) { if (!test) { throw new Error('Test object is required to generate identifier'); } @@ -47,12 +46,12 @@ class TestMap { return `${moduleName}::${testName}`; } - generateUUID() { + static generateUUID() { return uuidv4(); } - getAllTests() { - return new Map(this.testMap); + static getAllTests() { + return new Map(sharedTestMap); } } From 2f0befb8a706c67d70840a0f0135a619c991e97e Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 15:17:50 +0530 Subject: [PATCH 16/46] minor change --- src/utils/testMap.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 5afe57b..2828e87 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -21,7 +21,7 @@ class TestMap { static getUUID(test = null) { if (test) { - const testIdentifier = typeof test === 'string' ? test : generateTestIdentifier(test); + const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); const testData = sharedTestMap.get(testIdentifier); return testData ? testData.uuid : null; From b31cdc9019a9261d6338ac6d2329219e797b6269 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 19:14:05 +0530 Subject: [PATCH 17/46] minor change in try-catch --- src/accessibilityAutomation.js | 64 ++++++++++++++++------------------ 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 534a7be..1f08fce 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -302,49 +302,45 @@ class AccessibilityAutomation { } async commandWrapper() { - try { - const nightwatchMain = require.resolve('nightwatch'); - const nightwatchDir = path.dirname(nightwatchMain); - - const commandJson = AccessibilityScripts.commandsToWrap; - const accessibilityInstance = this; - for (const commandKey in commandJson) { - if (commandJson[commandKey].method === 'protocolAction'){ + const nightwatchMain = require.resolve('nightwatch'); + const nightwatchDir = path.dirname(nightwatchMain); + + const commandJson = AccessibilityScripts.commandsToWrap; + const accessibilityInstance = this; + for (const commandKey in commandJson) { + if (commandJson[commandKey].method === 'protocolAction'){ + commandJson[commandKey].name.forEach(commandName => { try { - commandJson[commandKey].name.forEach(commandName => { - const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); - const OriginalClass = require(commandPath); - const originalProtocolAction = OriginalClass.prototype.protocolAction; - - OriginalClass.prototype.protocolAction = async function() { - await accessibilityInstance.performScan(browser, commandName); - - return originalProtocolAction.apply(this); - }; - }); + const commandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const OriginalClass = require(commandPath); + const originalProtocolAction = OriginalClass.prototype.protocolAction; + + OriginalClass.prototype.protocolAction = async function() { + await accessibilityInstance.performScan(browser, commandName); + + return originalProtocolAction.apply(this); + }; } catch (error) { - Logger.debug(`Failed to patch protocolAction for command: ${error.message}`); + Logger.debug(`Failed to patch protocolAction for command ${commandName}`); } - } else { + }); + } else { + commandJson[commandKey].name.forEach(commandName => { try { - commandJson[commandKey].name.forEach(commandName => { - const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); - const originalCommand = require(webElementCommandPath); - const originalCommandFn = originalCommand.command; + const webElementCommandPath = path.join(nightwatchDir, `${commandJson[commandKey].path}`, `${commandName}.js`); + const originalCommand = require(webElementCommandPath); + const originalCommandFn = originalCommand.command; - originalCommand.command = async function(...args) { - await accessibilityInstance.performScan(browser, commandName); + originalCommand.command = async function(...args) { + await accessibilityInstance.performScan(browser, commandName); - return originalCommandFn.apply(this, args); - }; - }); + return originalCommandFn.apply(this, args); + }; } catch (error) { - Logger.debug(`Failed to patch command: ${error.message}`); + Logger.debug(`Failed to patch command ${commandName}`); } - } + }); } - } catch (error) { - Logger.debug(`Command patching failed: ${error.message}`); } } } From bc916ed14fbcb43c0c1e345d79760848afc85a61 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 19:15:08 +0530 Subject: [PATCH 18/46] temp: fallback for old core version --- src/testObservability.js | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/testObservability.js b/src/testObservability.js index 6c9bde9..b65a4a7 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -10,6 +10,7 @@ const Logger = require('./utils/logger'); const {API_URL, TAKE_SCREENSHOT_REGEX} = require('./utils/constants'); const OrchestrationUtils = require('./testorchestration/orchestrationUtils'); const accessibilityScripts = require('./scripts/accessibilityScripts'); +const TestMap = require('./utils/testMap'); const hooksMap = {}; class TestObservability { @@ -333,6 +334,13 @@ class TestObservability { await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } + default: { + if(process.env.TEST_RESULT_PENDING) { + const uuid = TestMap.getUUID(`${testFileReport.name}::${sectionName}`); + await this.sendPendingTestEvent(eventData, sectionName, testFileReport, uuid); + await this.processTestRunData(eventData, uuid); + } + } } } if (skippedTests?.length > 0) { @@ -386,6 +394,54 @@ class TestObservability { } } } + + // fallback for older core versions + async sendPendingTestEvent(eventData, testName, testFileReport, uuid) { + const testData = { + uuid: uuid, + type: 'test', + name: testName, + scope: `${testFileReport.name} - ${testName}`, + scopes: [ + testFileReport.name + ], + tags: testFileReport.tags, + identifier: `${testFileReport.name} - ${testName}`, + file_name: path.relative(process.cwd(), testFileReport.modulePath), + location: path.relative(process.cwd(), testFileReport.modulePath), + vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testFileReport.modulePath) : null, + started_at: new Date(eventData.startTimestamp).toISOString(), + framework: 'nightwatch' + }; + testData.integrations = {}; + const provider = helper.getCloudProvider(testFileReport.host); + testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); + testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(eventData.startTimestamp).toISOString(); + testData.result = eventData.status === 'pass' ? 'passed' : 'failed'; + if (eventData.status === 'fail' && eventData.lastError) { + testData.failure = [ + { + 'backtrace': [stripAnsi(eventData.lastError.message), eventData.lastError.stack] + } + ]; + testData.failure_reason = eventData.lastError ? stripAnsi(eventData.lastError.message) : null; + if (eventData.lastError && eventData.lastError.name) { + testData.failure_type = eventData.lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } + } else if (eventData.status === 'fail' && (testFileReport?.completed[testName]?.lastError || testFileReport?.completed[testName]?.stackTrace)) { + const testCompletionData = testFileReport.completed[testName]; + testData.failure = [ + {'backtrace': [testCompletionData?.stackTrace]} + ]; + testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; + testData.failure_type = testCompletionData?.stackTrace.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } + const uploadData = { + event_type: 'TestRunFinished' + }; + uploadData['test_run'] = testData; + await helper.uploadEventData(uploadData); + } async sendSkippedTestEvent(skippedTest, testFileReport) { const testData = { @@ -497,8 +553,11 @@ class TestObservability { const eventData = test.envelope[testName].testcase; testResults = test.testResults; if (!testResults) { + process.env.TEST_RESULT_PENDING = true; Logger.debug(`Test results could not be retrieved for test: ${testName}. Skipping result processing.`); + return; } else { + process.env.TEST_RESULT_PENDING = false; testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; if (testData.result === 'failed' && testResults.__lastError) { From 4893c7a794586e84e0c6e4ce3a3040ad293de863 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 19:19:03 +0530 Subject: [PATCH 19/46] eslintrc change --- .eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc b/.eslintrc index fc74d48..09780ca 100644 --- a/.eslintrc +++ b/.eslintrc @@ -102,6 +102,7 @@ "Promise": true, "Proxy": true, "Set": true, + "Map": true, "Reflect": true, "element": "readonly", "by": "readonly", From 1f5138fab36c1e843d07f51c72b6f3e6c2bd6fe9 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 23:21:17 +0530 Subject: [PATCH 20/46] fix for double test events in cucumber runner --- nightwatch/globals.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 690a3ce..87f66be 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -20,6 +20,7 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; +let testRunner = ""; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -261,13 +262,19 @@ module.exports = { TestMap.storeTestDetails(test); const uuid = TestMap.getUUID(test); await accessibilityAutomation.beforeEachExecution(test) - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) + if (testRunner != "cucumber"){ + await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) + } + }); eventBroadcaster.on('TestRunFinished', async (test) => { const uuid = TestMap.getUUID(test); await accessibilityAutomation.afterEachExecution(test, uuid); - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) + if (testRunner != "cucumber"){ + await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) + } + }); }, @@ -278,6 +285,7 @@ module.exports = { }, async before(settings, testEnvSettings) { + testRunner = settings.test_runner.type; const pluginSettings = settings['@nightwatch/browserstack']; if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; From e932fab92c86c18d2ef089e02064d83bb33b9b22 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 20 Nov 2025 23:22:57 +0530 Subject: [PATCH 21/46] env var name changed --- src/accessibilityAutomation.js | 14 +++++++------- src/testObservability.js | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 1f08fce..0cf908d 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -56,18 +56,18 @@ class AccessibilityAutomation { if (this._bstackOptions) { this._bstackOptions.accessibility = this._settings.accessibility; if (this._bstackOptions.accessibilityOptions) { - this._bstackOptions.accessibilityOptions.authToken = process.env.BS_A11Y_JWT; + this._bstackOptions.accessibilityOptions.authToken = process.env.BSTACK_A11Y_JWT; } else { - this._bstackOptions.accessibilityOptions = {authToken: process.env.BS_A11Y_JWT}; + this._bstackOptions.accessibilityOptions = {authToken: process.env.BSTACK_A11Y_JWT}; } this._bstackOptions.accessibilityOptions.scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; } else if (settings.desiredCapabilities['browserstack.accessibility']) { if (settings.desiredCapabilities['browserstack.accessibilityOptions']) { settings.desiredCapabilities['browserstack.accessibilityOptions'].authToken = - process.env.BS_A11Y_JWT; + process.env.BSTACK_A11Y_JWT; } else { settings.desiredCapabilities['browserstack.accessibilityOptions'] = { - authToken: process.env.BS_A11Y_JWT + authToken: process.env.BSTACK_A11Y_JWT }; } settings.desiredCapabilities['browserstack.accessibilityOptions'].scannerVersion = process.env.BSTACK_A11Y_SCANNER_VERSION; @@ -85,9 +85,9 @@ class AccessibilityAutomation { } const isBrowserstackAccessibilityEnabled = process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; const hasA11yJwtToken = - typeof process.env.BS_A11Y_JWT === 'string' && - process.env.BS_A11Y_JWT.length > 0 && - process.env.BS_A11Y_JWT !== 'null'; + typeof process.env.BSTACK_A11Y_JWT === 'string' && + process.env.BSTACK_A11Y_JWT.length > 0 && + process.env.BSTACK_A11Y_JWT !== 'null'; return isBrowserstackAccessibilityEnabled && hasA11yJwtToken; } catch (error) { diff --git a/src/testObservability.js b/src/testObservability.js index b65a4a7..25bbf0d 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -221,7 +221,7 @@ class TestObservability { Logger.debug(`Accessibility scannerVersion ${scannerVersion}`); } if (accessibilityToken) { - process.env.BS_A11Y_JWT = accessibilityToken; + process.env.BSTACK_A11Y_JWT = accessibilityToken; process.env.BROWSERSTACK_ACCESSIBILITY = 'true'; } if (pollingTimeout) { From 6e0a90ab7b804c569a074ac9ba8c54d066208766 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 24 Nov 2025 15:59:29 +0530 Subject: [PATCH 22/46] minor changes --- nightwatch/globals.js | 10 +++++----- src/utils/logPatcher.js | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 87f66be..e06d3b4 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -94,7 +94,7 @@ module.exports = { const gherkinDocument = reportData?.gherkinDocument.find((document) => document.uri === pickleData.uri); const featureData = gherkinDocument.feature; const uniqueId = uuidv4(); - process.env.TEST_OPS_TEST_UUID = uniqueId; + process.env.TEST_RUN_UUID = uniqueId; Object.values(workerList).forEach((worker) => { worker.process.on('message', async (data) => { @@ -255,26 +255,26 @@ module.exports = { eventBroadcaster.on('ScreenshotCreated', async (args) => { if (!helper.isTestObservabilitySession()) {return} - handleScreenshotUpload({args: args, uuid: process.env.TEST_OPS_TEST_UUID}); + handleScreenshotUpload({args: args, uuid: process.env.TEST_RUN_UUID}); }); eventBroadcaster.on('TestRunStarted', async (test) => { TestMap.storeTestDetails(test); - const uuid = TestMap.getUUID(test); await accessibilityAutomation.beforeEachExecution(test) if (testRunner != "cucumber"){ + const uuid = TestMap.getUUID(test); + process.env.TEST_RUN_UUID = uuid; await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) } }); eventBroadcaster.on('TestRunFinished', async (test) => { - const uuid = TestMap.getUUID(test); + const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner != "cucumber"){ await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) } - }); }, diff --git a/src/utils/logPatcher.js b/src/utils/logPatcher.js index 9cd08a6..e0ecd1e 100644 --- a/src/utils/logPatcher.js +++ b/src/utils/logPatcher.js @@ -68,7 +68,7 @@ class LogPatcher extends Transport { process.on('message', (data) => { if (data.uuid !== undefined){ _uuid = data.uuid; - process.env.TEST_OPS_TEST_UUID = _uuid; + process.env.TEST_RUN_UUID = _uuid; } }); process.on('disconnect', async () => { From 724033d1e3c49961742d2b4468e2e7f662b14055 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 24 Nov 2025 22:12:32 +0530 Subject: [PATCH 23/46] removed the fallback and added alternative --- src/testObservability.js | 92 ++++++++-------------------------------- 1 file changed, 17 insertions(+), 75 deletions(-) diff --git a/src/testObservability.js b/src/testObservability.js index 25bbf0d..c79af3e 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -334,13 +334,6 @@ class TestObservability { await this.sendHookEvents(eventData, testFileReport, 'HookRunStarted', 'HookRunFinished', globalAfterEachHookId, 'GLOBAL_AFTER_EACH', sectionName); break; } - default: { - if(process.env.TEST_RESULT_PENDING) { - const uuid = TestMap.getUUID(`${testFileReport.name}::${sectionName}`); - await this.sendPendingTestEvent(eventData, sectionName, testFileReport, uuid); - await this.processTestRunData(eventData, uuid); - } - } } } if (skippedTests?.length > 0) { @@ -395,54 +388,6 @@ class TestObservability { } } - // fallback for older core versions - async sendPendingTestEvent(eventData, testName, testFileReport, uuid) { - const testData = { - uuid: uuid, - type: 'test', - name: testName, - scope: `${testFileReport.name} - ${testName}`, - scopes: [ - testFileReport.name - ], - tags: testFileReport.tags, - identifier: `${testFileReport.name} - ${testName}`, - file_name: path.relative(process.cwd(), testFileReport.modulePath), - location: path.relative(process.cwd(), testFileReport.modulePath), - vc_filepath: (this._gitMetadata && this._gitMetadata.root) ? path.relative(this._gitMetadata.root, testFileReport.modulePath) : null, - started_at: new Date(eventData.startTimestamp).toISOString(), - framework: 'nightwatch' - }; - testData.integrations = {}; - const provider = helper.getCloudProvider(testFileReport.host); - testData.integrations[provider] = helper.getIntegrationsObject(testFileReport.sessionCapabilities, testFileReport.sessionId, testFileReport.host); - testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(eventData.startTimestamp).toISOString(); - testData.result = eventData.status === 'pass' ? 'passed' : 'failed'; - if (eventData.status === 'fail' && eventData.lastError) { - testData.failure = [ - { - 'backtrace': [stripAnsi(eventData.lastError.message), eventData.lastError.stack] - } - ]; - testData.failure_reason = eventData.lastError ? stripAnsi(eventData.lastError.message) : null; - if (eventData.lastError && eventData.lastError.name) { - testData.failure_type = eventData.lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; - } - } else if (eventData.status === 'fail' && (testFileReport?.completed[testName]?.lastError || testFileReport?.completed[testName]?.stackTrace)) { - const testCompletionData = testFileReport.completed[testName]; - testData.failure = [ - {'backtrace': [testCompletionData?.stackTrace]} - ]; - testData.failure_reason = testCompletionData?.assertions.find(val => val.stackTrace === testCompletionData.stackTrace)?.failure; - testData.failure_type = testCompletionData?.stackTrace.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; - } - const uploadData = { - event_type: 'TestRunFinished' - }; - uploadData['test_run'] = testData; - await helper.uploadEventData(uploadData); - } - async sendSkippedTestEvent(skippedTest, testFileReport) { const testData = { uuid: uuidv4(), @@ -551,28 +496,25 @@ class TestObservability { if (eventType === 'TestRunFinished') { const eventData = test.envelope[testName].testcase; - testResults = test.testResults; - if (!testResults) { - process.env.TEST_RESULT_PENDING = true; - Logger.debug(`Test results could not be retrieved for test: ${testName}. Skipping result processing.`); - return; - } else { - process.env.TEST_RESULT_PENDING = false; - testData.finished_at = testResults.endTimestamp ? new Date(testResults.endTimestamp).toISOString() : new Date(testResults.startTimestamp).toISOString(); - testData.result = testResults.__failedCount > 0 ? 'failed' : 'passed'; - if (testData.result === 'failed' && testResults.__lastError) { - testData.failure = [ - { - 'backtrace': [stripAnsi(testResults.__lastError.message), testResults.__lastError.stack] + testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString(); + testData.result = 'passed'; + if (eventData && eventData.commands && Array.isArray(eventData.commands)) { + const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); + if (failedCommand) { + testData.result = 'failed'; + if (failedCommand.result) { + testData.failure = [ + { + 'backtrace': [stripAnsi(failedCommand.result.message || ''), failedCommand.result.stack || ''] + } + ]; + testData.failure_reason = failedCommand.result.message ? stripAnsi(failedCommand.result.message) : null; + if (failedCommand.result.name) { + testData.failure_type = failedCommand.result.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + } } - ]; - testData.failure_reason = testResults.__lastError ? stripAnsi(testResults.__lastError.message) : null; - if (testResults.__lastError && testResults.__lastError.name) { - testData.failure_type = testResults.__lastError.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; } - } - } - + } await this.processTestRunData (eventData, uuid); } From a25e71ef4bc3e1357fa91d7e6f3c9afd12007b00 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 25 Nov 2025 19:46:51 +0530 Subject: [PATCH 24/46] review changes pt.2 --- nightwatch/globals.js | 4 ++-- src/accessibilityAutomation.js | 13 +++++-------- src/utils/helper.js | 11 +++++++++++ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index e06d3b4..2fcf635 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -328,7 +328,7 @@ module.exports = { settings.test_runner.options['require'] = path.resolve(__dirname, 'observabilityLogPatcherHook.js'); } settings.globals['customReporterCallbackTimeout'] = CUSTOM_REPORTER_CALLBACK_TIMEOUT; - if (helper.isTestObservabilitySession() || pluginSettings?.accessibility === true) { + if (helper.isTestHubBuild(pluginSettings,true)) { await testObservability.launchTestSession(); } if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS && process.env.BROWSERSTACK_RERUN_TESTS!=='null') { @@ -454,7 +454,7 @@ module.exports = { Logger.error(`Error collecting build data for test orchestration: ${error}`); } - if (helper.isTestObservabilitySession() || helper.isAccessibilitySession()) { + if (helper.isTestHubBuild()) { process.env.NIGHTWATCH_RERUN_FAILED = nightwatchRerun; process.env.NIGHTWATCH_RERUN_REPORT_FILE = nightwatchRerunFile; if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS) { diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 0cf908d..254125c 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -22,12 +22,9 @@ class AccessibilityAutomation { this._key = helper.getAccessKey(settings, this._settings); } - let accessibilityOptions; - if (helper.isUndefined(this._settings.accessibilityOptions)) { - accessibilityOptions = {}; - } else { - accessibilityOptions = this._settings.accessibilityOptions; - } + const accessibilityOptions = helper.isUndefined(this._settings.accessibilityOptions) + ? {} + : this._settings.accessibilityOptions; process.env.BROWSERSTACK_ACCESSIBILITY_OPTIONS = JSON.stringify(accessibilityOptions); } @@ -202,7 +199,7 @@ class AccessibilityAutomation { 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT }; - await this.sendTestStopEvent(browser, dataForExtension); + await this.saveAccessibilityResults(browser, dataForExtension); Logger.info('Accessibility testing for this test case has ended.'); } } @@ -294,7 +291,7 @@ class AccessibilityAutomation { } } - async sendTestStopEvent(browser, dataForExtension = {}) { + async saveAccessibilityResults(browser, dataForExtension = {}) { Logger.debug('Performing scan before saving results'); await this.performScan(browser); const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); diff --git a/src/utils/helper.js b/src/utils/helper.js index 96abd73..4ed4667 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -76,8 +76,19 @@ exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; +exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { + if(isBuildStart) { + return pluginSettings?.test_reporting?.enabled === true || pluginSettings?.test_observability?.enabled === true || pluginSettings?.accessibility === true; + } + else { + return this.isTestObservabilitySession() || this.isAccessibilitySession(); + } +}; + exports.isAccessibilityEnabled = (settings) => { if (process.argv.includes('--disable-accessibility')) {return false} + + if(process.env.BROWSERSTACK_ACCESSIBILITY === 'false') {return false} return settings['@nightwatch/browserstack']?.accessibility === true; }; From e4aac46ae547e1f40ecf64504529484120fd3805 Mon Sep 17 00:00:00 2001 From: bhargavi vaidya Date: Wed, 26 Nov 2025 16:38:51 +0530 Subject: [PATCH 25/46] Update src/utils/testMap.js Co-authored-by: Amaan Hakim <89768375+amaanbs@users.noreply.github.com> --- src/utils/testMap.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 2828e87..7ca6cd4 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -29,6 +29,7 @@ class TestMap { } static getTestDetails(identifier) { + return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; if (sharedTestMap.has(identifier)) { return sharedTestMap.get(identifier); } From bb8612972cc347912d186448448c3738fb0d754f Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 26 Nov 2025 16:41:42 +0530 Subject: [PATCH 26/46] review changes pt.3 --- src/utils/testMap.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 7ca6cd4..95a8c54 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -30,11 +30,6 @@ class TestMap { static getTestDetails(identifier) { return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; - if (sharedTestMap.has(identifier)) { - return sharedTestMap.get(identifier); - } - - return null; } static generateTestIdentifier(test) { @@ -50,10 +45,6 @@ class TestMap { static generateUUID() { return uuidv4(); } - - static getAllTests() { - return new Map(sharedTestMap); - } } module.exports = TestMap; From 8539ae87f04ac5c938e722cde9b1268d694410b4 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 26 Nov 2025 22:33:17 +0530 Subject: [PATCH 27/46] review changes pt.4 --- nightwatch/globals.js | 23 +++++++++---- src/scripts/accessibilityScripts.js | 28 +++++++++------- src/testObservability.js | 7 ++-- src/utils/testMap.js | 51 ++++++++++++++++++++++++++--- 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 2fcf635..067a724 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -259,21 +259,31 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - TestMap.storeTestDetails(test); + process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser) await accessibilityAutomation.beforeEachExecution(test) if (testRunner != "cucumber"){ - const uuid = TestMap.getUUID(test); + const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) } - }); eventBroadcaster.on('TestRunFinished', async (test) => { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); - await accessibilityAutomation.afterEachExecution(test, uuid); - if (testRunner != "cucumber"){ - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) + if (TestMap.hasTestFinished(uuid)) { + Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + return; + } + try { + await accessibilityAutomation.afterEachExecution(test, uuid) + if (testRunner != "cucumber"){ + await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) + TestMap.markTestFinished(uuid); + } + + } catch (error) { + Logger.error(`Error in TestRunFinished event: ${error.message}`); + TestMap.markTestFinished(uuid); } }); }, @@ -341,6 +351,7 @@ module.exports = { } try { + // In parallel mode, env-specific settings are passed to beforeChildProcess hook instead of before hook, if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); diff --git a/src/scripts/accessibilityScripts.js b/src/scripts/accessibilityScripts.js index 74d766d..a399aa8 100644 --- a/src/scripts/accessibilityScripts.js +++ b/src/scripts/accessibilityScripts.js @@ -84,19 +84,23 @@ class AccessibilityScripts { } store() { - if (!fs.existsSync(this.browserstackFolderPath)){ - fs.mkdirSync(this.browserstackFolderPath); - } - - fs.writeFileSync(this.commandsPath, JSON.stringify({ - commands: this.commandsToWrap, - scripts: { - scan: this.performScan, - getResults: this.getResults, - getResultsSummary: this.getResultsSummary, - saveResults: this.saveTestResults + try { + if (!fs.existsSync(this.browserstackFolderPath)){ + fs.mkdirSync(this.browserstackFolderPath); } - })); + + fs.writeFileSync(this.commandsPath, JSON.stringify({ + commands: this.commandsToWrap, + scripts: { + scan: this.performScan, + getResults: this.getResults, + getResultsSummary: this.getResultsSummary, + saveResults: this.saveTestResults + } + })); + } catch (error) { + Logger.debug(`Failed to store accessibility commands file: ${error.message}`); + } } } diff --git a/src/testObservability.js b/src/testObservability.js index c79af3e..009501e 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -9,9 +9,11 @@ const CrashReporter = require('./utils/crashReporter'); const Logger = require('./utils/logger'); const {API_URL, TAKE_SCREENSHOT_REGEX} = require('./utils/constants'); const OrchestrationUtils = require('./testorchestration/orchestrationUtils'); +const AccessibilityAutomation = require('./accessibilityAutomation'); const accessibilityScripts = require('./scripts/accessibilityScripts'); const TestMap = require('./utils/testMap'); const hooksMap = {}; +const accessibilityAutomation = new AccessibilityAutomation(); class TestObservability { configure(settings = {}) { @@ -92,7 +94,7 @@ class TestObservability { this._parentSettings?.testReportingOptions || this._parentSettings?.testObservabilityOptions || {}; - const accessibility = this._settings.accessibility || false; + const accessibility = helper.isAccessibilityEnabled(this._parentSettings); const accessibilityOptions = accessibility ? this._settings.accessibilityOptions || {} : {}; this._gitMetadata = await helper.getGitMetaData(); const fromProduct = { @@ -490,7 +492,8 @@ class TestObservability { [provider]: helper.getIntegrationsObject(testMetaData.sessionCapabilities, testMetaData.sessionId, testMetaData.host, settings?.desiredCapabilities?.['bstack:options']?.osVersion) }, product_map: { - accessibility: helper.isAccessibilitySession() + observability: helper.isTestObservabilitySession(), + accessibility: helper.isAccessibilitySession() && accessibilityAutomation.shouldScanTestForAccessibility() && process.env.VALID_ALLY_PLATFORM } }; diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 95a8c54..9e168cd 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -2,21 +2,43 @@ const {v4: uuidv4} = require('uuid'); const sharedTestMap = new Map(); let sharedCurrentTest = null; +let activeTestRuns = new Map(); class TestMap { static storeTestDetails(test) { const testIdentifier = this.generateTestIdentifier(test); + const uuid = this.generateUUID(); if (!sharedTestMap.has(testIdentifier)) { - const uuid = this.generateUUID(); sharedTestMap.set(testIdentifier, { - uuid, + baseUuid: uuid, // Store the first UUID as base + retries: [], + currentUuid: uuid, test, createdAt: new Date().toISOString() }); + } else { + // This is a retry - add new UUID to retries array + const testData = sharedTestMap.get(testIdentifier); + testData.retries.push({ + uuid, + startedAt: new Date().toISOString() + }); + testData.currentUuid = uuid; // Update to latest UUID + sharedTestMap.set(testIdentifier, testData); } + + // Track this as an active test run + activeTestRuns.set(uuid, { + identifier: testIdentifier, + startedAt: new Date().toISOString(), + hasFinished: false + }); + sharedCurrentTest = testIdentifier; + + return uuid; } static getUUID(test = null) { @@ -24,12 +46,33 @@ class TestMap { const testIdentifier = typeof test === 'string' ? test : this.generateTestIdentifier(test); const testData = sharedTestMap.get(testIdentifier); - return testData ? testData.uuid : null; + return testData ? testData.currentUuid : null; } + + return null; } + static markTestFinished(uuid) { + if (activeTestRuns.has(uuid)) { + const testRun = activeTestRuns.get(uuid); + testRun.hasFinished = true; + testRun.finishedAt = new Date().toISOString(); + activeTestRuns.set(uuid, testRun); + + return true; + } + + return false; + } + + static hasTestFinished(uuid) { + const testRun = activeTestRuns.get(uuid); + return testRun ? testRun.hasFinished : false; + } + + static getTestDetails(identifier) { - return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; + return sharedTestMap.has(identifier) ? sharedTestMap.get(identifier) : null; } static generateTestIdentifier(test) { From 75a5ec9eabca77a570027ebce3bf020f1bb5293f Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 03:27:08 +0530 Subject: [PATCH 28/46] minor change --- src/testObservability.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/testObservability.js b/src/testObservability.js index 009501e..24b8317 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -493,7 +493,7 @@ class TestObservability { }, product_map: { observability: helper.isTestObservabilitySession(), - accessibility: helper.isAccessibilitySession() && accessibilityAutomation.shouldScanTestForAccessibility() && process.env.VALID_ALLY_PLATFORM + accessibility: helper.isAccessibilitySession() && accessibilityAutomation.shouldScanTestForAccessibility(test) && process.env.VALID_ALLY_PLATFORM } }; From 6934bf72d551bb57d940491eef9ab0c0c5900a2e Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 03:31:00 +0530 Subject: [PATCH 29/46] app accessibility changes --- nightwatch/globals.js | 13 +++- src/accessibilityAutomation.js | 135 ++++++++++++++++++++++++++++++++- src/testObservability.js | 1 + src/utils/constants.js | 4 +- src/utils/helper.js | 112 ++++++++++++++++++++++++++- 5 files changed, 256 insertions(+), 9 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 067a724..bc62de7 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -300,7 +300,7 @@ module.exports = { if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + process.env.BROWSERSTACK_APP_AUTOMATE = helper.checkTestEnvironmentForAppAutomate(testEnvSettings); // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; @@ -484,8 +484,15 @@ module.exports = { }, async beforeEach(settings) { - browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; - browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; + if (helper.isAppAccessibilitySession()){ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js + } + else{ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; + } + // await accessibilityAutomation.beforeEachExecution(browser); }, // This will be run after each test suite is finished diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 254125c..801bbde 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,6 +1,7 @@ const path = require('path'); const helper = require('./utils/helper'); const Logger = require('./utils/logger'); +const {APP_ALLY_ENDPOINT,APP_ALLY_ISSUES_SUMMARY_ENDPOINT,APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); const util = require('util'); const AccessibilityScripts = require('./scripts/accessibilityScripts'); @@ -161,6 +162,23 @@ class AccessibilityAutomation { return false; } + validateAppA11yCaps(capabilities = {}) { + /* Check if the current driver platform is eligible for AppAccessibility scan */ + Logger.debug(`capabilities ${JSON.stringify(capabilities)}`); + if ( + capabilities?.platformName && + String(capabilities?.platformName).toLowerCase() === 'android' && + capabilities?.platformVersion && + parseInt(capabilities?.platformVersion?.toString()) < 11 + ) { + Logger.warn( + 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' + ); + return false; + } + return true; + } + async beforeEachExecution(testMetaData) { try { this.currentTest = browser.currentTest; @@ -168,7 +186,13 @@ class AccessibilityAutomation { testMetaData ); this.currentTest.accessibilityScanStarted = true; - this._isAccessibilitySession = this.validateA11yCaps(browser); + + this._isAppAccessibility = helper.isAppAccessibilitySession(); + if (this._isAppAccessibility) { + this._isAccessibilitySession = this.validateAppA11yCaps(testMetaData.metadata.sessionCapabilities); + } else { + this._isAccessibilitySession = this.validateA11yCaps(browser); + } if (this.isAccessibilityAutomationSession() && browser && this._isAccessibilitySession) { try { @@ -264,10 +288,14 @@ class AccessibilityAutomation { } if (this.currentTest.shouldScanTestForAccessibility === false) { - Logger.info('Skipping Accessibility scan for this test as it\'s disabled.'); - + if(commandName && commandName !== '') { + Logger.debug(`Skipping Accessibility scan for command ${commandName} as the test is excluded from accessibility scanning.`); + } else { + Logger.debug('Skipping Accessibility scan as the test is excluded from accessibility scanning.'); + } return; } + try { const browser = browserInstance; @@ -276,6 +304,16 @@ class AccessibilityAutomation { return; } + + if (helper.isAppAccessibilitySession()){ + const results = await browser.executeScript( + helper.formatString(AccessibilityScripts.performScan, JSON.stringify(this.getParamsForAppAccessibility(commandName))), + {} + ); + Logger.debug(util.inspect(results)); + return results; + } + const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { method: commandName || '' }); @@ -291,9 +329,71 @@ class AccessibilityAutomation { } } + async getAppAccessibilityResults(browser) { + if (!helper.isBrowserstackInfra()) { + return []; + } + + if (!helper.isAppAccessibilitySession()) { + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') + return []; + } + + try { + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.issues; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + return result; + } catch (error) { + Logger.error('No accessibility summary was found.'); + Logger.debug(`getAppA11yResults Failed. Error: ${error}`); + return []; + } + + } + + async getAppAccessibilityResultsSummary(browser) { + if (!helper.isBrowserstackInfra()) { + return {}; + } + + if (!helper.isAppAccessibilitySession()) { + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') + return {} + } + try { + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.summary; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + return result; + } catch { + Logger.error('No accessibility summary was found.'); + return {}; + } + } + + async getAppA11yResultResponse(apiUrl, browser, sessionId){ + Logger.debug('Performing scan before getting results/results summary'); + await this.performScan(browser); + + const upperTimeLimit = process.env.BSTACK_A11Y_POLLING_TIMEOUT ? Date.now() + parseInt(process.env.BSTACK_A11Y_POLLING_TIMEOUT) * 1000 : Date.now() + 30000; + const params = { test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now() }; // Query params to pass + const header = { Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}` }; + const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); + Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + return apiRespone; + + } + + async saveAccessibilityResults(browser, dataForExtension = {}) { Logger.debug('Performing scan before saving results'); await this.performScan(browser); + if (helper.isAppAccessibilitySession()){ + return; + } const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); Logger.debug(util.inspect(results)); } @@ -329,7 +429,12 @@ class AccessibilityAutomation { const originalCommandFn = originalCommand.command; originalCommand.command = async function(...args) { - await accessibilityInstance.performScan(browser, commandName); + if ( + !commandName.includes('execute') || + !accessibilityInstance.shouldPatchExecuteScript(args.length ? args[0] : null) + ) { + await accessibilityInstance.performScan(browser, commandName); + } return originalCommandFn.apply(this, args); }; @@ -340,6 +445,28 @@ class AccessibilityAutomation { } } } + + shouldPatchExecuteScript(script) { + if (!script || typeof script !== 'string') { + return true; + } + + return ( + script.toLowerCase().indexOf('browserstack_executor') !== -1 || + script.toLowerCase().indexOf('browserstack_accessibility_automation_script') !== -1 + ); + } + + getParamsForAppAccessibility(commandName) { + return { + 'thTestRunUuid': process.env.TEST_RUN_UUID, + 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, + 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT, + 'authHeader': process.env.BSTACK_A11Y_JWT, + 'scanTimestamp': Date.now(), + 'method': commandName + }; + } } module.exports = AccessibilityAutomation; diff --git a/src/testObservability.js b/src/testObservability.js index 24b8317..964117a 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -234,6 +234,7 @@ class TestObservability { accessibilityScripts.store(); } } + process.env.IS_APP_ACCESSIBILITY = accessibilityAutomation.isAccessibilityAutomationSession() && helper.isAppAutomate(); } diff --git a/src/utils/constants.js b/src/utils/constants.js index 4460c4e..953e360 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -18,7 +18,9 @@ exports.EVENTS = { SCREENSHOT: 'testObservability:screenshot' }; exports.ACCESSIBILITY_URL= 'https://accessibility.browserstack.com/api'; - +exports.APP_ALLY_ENDPOINT = 'https://app-accessibility.browserstack.com/automate'; +exports.APP_ALLY_ISSUES_SUMMARY_ENDPOINT ='api/v1/issues-summary'; +exports.APP_ALLY_ISSUES_ENDPOINT = 'api/v1/issues'; // Maximum size of VCS info which is allowed exports.MAX_GIT_META_DATA_SIZE_IN_BYTES = 64 * 1024; diff --git a/src/utils/helper.js b/src/utils/helper.js index 4ed4667..94a585b 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -16,6 +16,7 @@ const LogPatcher = require('./logPatcher'); const BSTestOpsPatcher = new LogPatcher({}); const sessions = {}; const {execSync} = require('child_process'); +const request = require('@cypress/request'); console = {}; Object.keys(consoleHolder).forEach(method => { @@ -72,6 +73,21 @@ exports.getObservabilityKey = (config, bstackOptions={}) => { return process.env.BROWSERSTACK_ACCESS_KEY || config?.key || bstackOptions?.accessKey; }; +exports.isAppAutomate = () => { + return process.env.BROWSERSTACK_APP_AUTOMATE === 'true'; +}; + +exports.checkTestEnvironmentForAppAutomate = (testEnvSettings) => { + + const firstEnvKey = Object.keys(testEnvSettings)[0]; + const firstEnv = testEnvSettings[firstEnvKey]; + if (firstEnv?.desiredCapabilities?.['appium:options']?.app) { + return true; + } + + return false; +}; + exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; @@ -85,6 +101,10 @@ exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { } }; +exports.isAppAccessibilitySession = () => { + return process.env.IS_APP_ACCESSIBILITY === 'true'; +}; + exports.isAccessibilityEnabled = (settings) => { if (process.argv.includes('--disable-accessibility')) {return false} @@ -637,7 +657,7 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { if (hostname) { if (hostname.includes('browserstack.com') && !hostname.includes('hub-ft')) { - if (this.isUndefined(caps.browserName)) { + if (this.isAppAutomate()) { product = 'app-automate'; } else { product = 'automate'; @@ -1360,3 +1380,93 @@ exports.logBuildError = (error, product = '') => { } }; +exports.formatString = (template, ...values) => { + let i = 0; + if (template === null) { + return ''; + } + return template.replace(/%s/g, () => { + const value = values[i++]; + return value !== null && value !== undefined ? value : ''; + }); +}; + +exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now()) => { + params.timestamp = Math.round(Date.now() / 1000); + Logger.debug(`current timestamp ${params.timestamp}`); + + try { + const queryString = new URLSearchParams(params).toString(); + const fullUrl = `${url}?${queryString}`; + + const response = await new Promise((resolve, reject) => { + request({ + method: 'GET', + url: fullUrl, + headers: headers, + json: false + }, (error, response, body) => { + if (error) { + reject(error); + } else { + resolve(response); + } + }); + }); + + const responseData = JSON.parse(response.body); + return { + data: responseData, + headers: response.headers, + message: 'Polling succeeded.' + }; + } catch (error) { + if (error.response && error.response.statusCode === 404) { + const nextPollTime = parseInt(error.response.headers.next_poll_time, 10) * 1000; + Logger.debug(`timeInMillis ${nextPollTime}`); + + if (isNaN(nextPollTime)) { + Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); + return { + data: {}, + headers: error.response.headers, + message: 'Invalid nextPollTime header value. Polling stopped.' + }; + } + + const elapsedTime = nextPollTime - Date.now(); + Logger.debug( + `elapsedTime ${elapsedTime} timeInMillis ${nextPollTime} upperLimit ${upperLimit}` + ); + + // Stop polling if the upper time limit is reached + if (nextPollTime > upperLimit) { + Logger.warn('Polling stopped due to upper time limit.'); + return { + data: {}, + headers: error.response.headers, + message: 'Polling stopped due to upper time limit.' + }; + } + + Logger.debug(`Polling again in ${elapsedTime}ms with params:`, params); + + // Wait for the specified time and poll again + await new Promise((resolve) => setTimeout(resolve, elapsedTime)); + return exports.pollApi(url, params, headers, upperLimit, startTime); + } else if (error.response) { + throw { + data: {}, + headers: {}, + message: error.response.body ? JSON.parse(error.response.body).message : 'Unknown error' + }; + } else { + Logger.error(`Unexpected error occurred: ${error}`); + return { data: {}, headers: {}, message: 'Unexpected error occurred.' }; + } + } +}; + + + + From 9ca5ff57dc51dd294e490267d0804a9c68906589 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 12:33:56 +0530 Subject: [PATCH 30/46] fix: lint issues --- nightwatch/globals.js | 19 ++++++++++--------- src/testObservability.js | 30 +++++++++++++++--------------- src/utils/helper.js | 10 +++++----- src/utils/testMap.js | 3 ++- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 067a724..5c1a161 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -20,7 +20,7 @@ const _tests = {}; const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; -let testRunner = ""; +let testRunner = ''; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -259,12 +259,12 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser) - await accessibilityAutomation.beforeEachExecution(test) - if (testRunner != "cucumber"){ + process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser); + await accessibilityAutomation.beforeEachExecution(test); + if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid) + await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); } }); @@ -272,12 +272,13 @@ module.exports = { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); if (TestMap.hasTestFinished(uuid)) { Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + return; } try { - await accessibilityAutomation.afterEachExecution(test, uuid) - if (testRunner != "cucumber"){ - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid) + await accessibilityAutomation.afterEachExecution(test, uuid); + if (testRunner !== 'cucumber'){ + await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); TestMap.markTestFinished(uuid); } @@ -338,7 +339,7 @@ module.exports = { settings.test_runner.options['require'] = path.resolve(__dirname, 'observabilityLogPatcherHook.js'); } settings.globals['customReporterCallbackTimeout'] = CUSTOM_REPORTER_CALLBACK_TIMEOUT; - if (helper.isTestHubBuild(pluginSettings,true)) { + if (helper.isTestHubBuild(pluginSettings, true)) { await testObservability.launchTestSession(); } if (process.env.BROWSERSTACK_RERUN === 'true' && process.env.BROWSERSTACK_RERUN_TESTS && process.env.BROWSERSTACK_RERUN_TESTS!=='null') { diff --git a/src/testObservability.js b/src/testObservability.js index 24b8317..f5caa1b 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -465,7 +465,7 @@ class TestObservability { const testName = test.testcase; const settings = test.settings || {}; const startTimestamp = test.envelope[testName].startTimestamp; - let testResults = {}; + const testResults = {}; const testBody = this.getTestBody(test.testCaseData); const provider = helper.getCloudProvider(testMetaData.host); const testData = { @@ -501,23 +501,23 @@ class TestObservability { const eventData = test.envelope[testName].testcase; testData.finished_at = eventData.endTimestamp ? new Date(eventData.endTimestamp).toISOString() : new Date(startTimestamp).toISOString(); testData.result = 'passed'; - if (eventData && eventData.commands && Array.isArray(eventData.commands)) { - const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); - if (failedCommand) { - testData.result = 'failed'; - if (failedCommand.result) { - testData.failure = [ - { - 'backtrace': [stripAnsi(failedCommand.result.message || ''), failedCommand.result.stack || ''] - } - ]; - testData.failure_reason = failedCommand.result.message ? stripAnsi(failedCommand.result.message) : null; - if (failedCommand.result.name) { - testData.failure_type = failedCommand.result.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; + if (eventData && eventData.commands && Array.isArray(eventData.commands)) { + const failedCommand = eventData.commands.find(cmd => cmd.status === 'fail'); + if (failedCommand) { + testData.result = 'failed'; + if (failedCommand.result) { + testData.failure = [ + { + 'backtrace': [stripAnsi(failedCommand.result.message || ''), failedCommand.result.stack || ''] } + ]; + testData.failure_reason = failedCommand.result.message ? stripAnsi(failedCommand.result.message) : null; + if (failedCommand.result.name) { + testData.failure_type = failedCommand.result.name.match(/Assert/) ? 'AssertionError' : 'UnhandledError'; } } - } + } + } await this.processTestRunData (eventData, uuid); } diff --git a/src/utils/helper.js b/src/utils/helper.js index 4ed4667..34500ad 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -77,18 +77,18 @@ exports.isAccessibilitySession = () => { }; exports.isTestHubBuild = (pluginSettings = {}, isBuildStart = false) => { - if(isBuildStart) { + if (isBuildStart) { return pluginSettings?.test_reporting?.enabled === true || pluginSettings?.test_observability?.enabled === true || pluginSettings?.accessibility === true; } - else { - return this.isTestObservabilitySession() || this.isAccessibilitySession(); - } + + return this.isTestObservabilitySession() || this.isAccessibilitySession(); + }; exports.isAccessibilityEnabled = (settings) => { if (process.argv.includes('--disable-accessibility')) {return false} - if(process.env.BROWSERSTACK_ACCESSIBILITY === 'false') {return false} + if (process.env.BROWSERSTACK_ACCESSIBILITY === 'false') {return false} return settings['@nightwatch/browserstack']?.accessibility === true; }; diff --git a/src/utils/testMap.js b/src/utils/testMap.js index 9e168cd..39b774c 100644 --- a/src/utils/testMap.js +++ b/src/utils/testMap.js @@ -2,7 +2,7 @@ const {v4: uuidv4} = require('uuid'); const sharedTestMap = new Map(); let sharedCurrentTest = null; -let activeTestRuns = new Map(); +const activeTestRuns = new Map(); class TestMap { @@ -67,6 +67,7 @@ class TestMap { static hasTestFinished(uuid) { const testRun = activeTestRuns.get(uuid); + return testRun ? testRun.hasFinished : false; } From 73d96baca9297d961cf048327c4394a0ca9e1c51 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 12:53:45 +0530 Subject: [PATCH 31/46] fixed the UTs --- .../test-observability/processTestRunData.js | 21 ++++--------- test/src/utils/helper.js | 31 ++++++++++++------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/test/src/test-observability/processTestRunData.js b/test/src/test-observability/processTestRunData.js index 4a4c6aa..c5c46aa 100644 --- a/test/src/test-observability/processTestRunData.js +++ b/test/src/test-observability/processTestRunData.js @@ -8,37 +8,28 @@ describe('TestObservability - processTestRunData', function () { this.sandbox = sinon.createSandbox(); this.testObservability = new TestObservability(); - this.eventData = {commands: []}; - this.sectionName = 'testSection'; - this.testFileReport = {}; - this.hookIds = []; - this.sendTestRunEventStub = this.sandbox.stub(this.testObservability, 'sendTestRunEvent').resolves(); + this.eventData = {commands: [], httpOutput: []}; + this.uuid = 'test-uuid-123'; }); afterEach(() => { this.sandbox.restore(); }); - it('should send test run events', async () => { - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); - sinon.assert.calledTwice(this.sendTestRunEventStub); - sinon.assert.calledWith(this.sendTestRunEventStub.firstCall, this.eventData, this.testFileReport, 'TestRunStarted', sinon.match.string, null, this.sectionName, this.hookIds); - sinon.assert.calledWith(this.sendTestRunEventStub.secondCall, this.eventData, this.testFileReport, 'TestRunFinished', sinon.match.string, null, this.sectionName, this.hookIds); - }); - it('should create screenshot log events', async () => { this.eventData = { commands: [ {name: 'saveScreenshot', args: ['path/to/screenshot.png'], startTime: 'start_time'} - ] + ], + httpOutput: [] }; this.sandbox.stub(fs, 'existsSync').callsFake(() => true); this.sandbox.stub(fs, 'readFileSync').callsFake(() => 'screenshot-base-64'); const createScreenshotLogEventStub = this.sandbox.stub(this.testObservability, 'createScreenshotLogEvent'); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'true'; - await this.testObservability.processTestRunData(this.eventData, this.sectionName, this.testFileReport, this.hookIds); + await this.testObservability.processTestRunData(this.eventData, this.uuid); process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'false'; - sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, sinon.match.string, 'screenshot-base-64', 'start_time'); + sinon.assert.calledOnceWithExactly(createScreenshotLogEventStub, this.uuid, 'screenshot-base-64', 'start_time'); }); }); diff --git a/test/src/utils/helper.js b/test/src/utils/helper.js index c82372f..2d54cb7 100644 --- a/test/src/utils/helper.js +++ b/test/src/utils/helper.js @@ -326,21 +326,30 @@ describe('isBrowserstackInfra', () => { isBrowserstackInfra = require('../../../src/utils/helper').isBrowserstackInfra; }); - it('returns false for undefined', async () => { - delete process.env.BROWSERSTACK_INFRA; - expect(isBrowserstackInfra()).to.be.false; + it('returns true for undefined settings', async () => { + expect(isBrowserstackInfra()).to.be.true; }); - it('returns true if env variable is set to true', async () => { - process.env.BROWSERSTACK_INFRA = true; - expect(isBrowserstackInfra()).to.be.true; - delete process.env.BROWSERSTACK_INFRA; + it('returns true for empty settings', async () => { + expect(isBrowserstackInfra({})).to.be.true; }); - it('returns false if env variable is set to false', async () => { - process.env.BROWSERSTACK_INFRA = false; - expect(isBrowserstackInfra()).to.be.false; - delete process.env.BROWSERSTACK_INFRA; + it('returns true if webdriver.host contains browserstack', async () => { + const settings = { + webdriver: { + host: 'hub-cloud.browserstack.com' + } + }; + expect(isBrowserstackInfra(settings)).to.be.true; + }); + + it('returns false if webdriver.host does not contain browserstack', async () => { + const settings = { + webdriver: { + host: 'localhost' + } + }; + expect(isBrowserstackInfra(settings)).to.be.false; }); }); From b42efb4918a6d4914e49e679215ed90f6b6f6aa7 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 13:16:49 +0530 Subject: [PATCH 32/46] minor change --- src/testObservability.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/testObservability.js b/src/testObservability.js index f5caa1b..11f78ee 100644 --- a/src/testObservability.js +++ b/src/testObservability.js @@ -146,6 +146,7 @@ class TestObservability { try { const response = await makeRequest('POST', 'api/v2/builds', data, config, API_URL); + Logger.info('Build creation successful!'); process.env.BS_TESTOPS_BUILD_COMPLETED = true; const responseData = response.data || {}; From 5627070f7229b3f44c311ad9d0be431cfc37b3d4 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 19:55:05 +0530 Subject: [PATCH 33/46] review changes pt.5 --- nightwatch/globals.js | 4 ++-- src/accessibilityAutomation.js | 6 ++++++ src/utils/helper.js | 25 +++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 5c1a161..2abb361 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -356,12 +356,11 @@ module.exports = { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); } - - // Initialize and configure test orchestration try { if (helper.isTestObservabilitySession()) { @@ -527,6 +526,7 @@ module.exports = { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); + helper.patchBrowserTerminateCommand(); } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 254125c..04a19d2 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -5,6 +5,7 @@ const util = require('util'); const AccessibilityScripts = require('./scripts/accessibilityScripts'); class AccessibilityAutomation { + static pendingAllyReq = 0; configure(settings = {}) { this._settings = settings['@nightwatch/browserstack'] || {}; this._testRunner = settings.test_runner; @@ -276,14 +277,17 @@ class AccessibilityAutomation { return; } + AccessibilityAutomation.pendingAllyReq++; const results = await browser.executeAsyncScript(AccessibilityScripts.performScan, { method: commandName || '' }); + AccessibilityAutomation.pendingAllyReq--; Logger.debug(util.inspect(results)); return results; } catch (err) { + AccessibilityAutomation.pendingAllyReq--; Logger.error('Accessibility Scan could not be performed: ' + err.message); Logger.debug('Stack trace:', err.stack); @@ -294,7 +298,9 @@ class AccessibilityAutomation { async saveAccessibilityResults(browser, dataForExtension = {}) { Logger.debug('Performing scan before saving results'); await this.performScan(browser); + AccessibilityAutomation.pendingAllyReq++; const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); + AccessibilityAutomation.pendingAllyReq--; Logger.debug(util.inspect(results)); } diff --git a/src/utils/helper.js b/src/utils/helper.js index 34500ad..8496a04 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1360,3 +1360,28 @@ exports.logBuildError = (error, product = '') => { } }; +exports.patchBrowserTerminateCommand = () =>{ + + const nightwatchDir = path.dirname(require.resolve('nightwatch')); + const CommandPath = path.join(nightwatchDir, `testsuite/index.js`); + const TestSuite = require(CommandPath); + const originalFn = TestSuite.prototype.terminate; + TestSuite.prototype.terminate = async function(...args) { + const maxWaitTime = 30000; + const pollInterval = 500; + const startTime = Date.now(); + const AccessibilityAutomation = require('../accessibilityAutomation'); + while (Date.now() - startTime < maxWaitTime) { + const pendingAllyReq = AccessibilityAutomation.pendingAllyReq || 0; + + if (pendingAllyReq === 0) { + break; + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); + } + Logger.debug(`Pending A11y Scans at session end: ${AccessibilityAutomation.pendingAllyReq }`); + + return originalFn.apply(this, args); + }; +}; + From 5220e0a5e840d26e7e387ab42f55a2a08370ee91 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 27 Nov 2025 20:24:21 +0530 Subject: [PATCH 34/46] fixed lint issues --- src/utils/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/helper.js b/src/utils/helper.js index 8496a04..4ce9d0c 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1363,7 +1363,7 @@ exports.logBuildError = (error, product = '') => { exports.patchBrowserTerminateCommand = () =>{ const nightwatchDir = path.dirname(require.resolve('nightwatch')); - const CommandPath = path.join(nightwatchDir, `testsuite/index.js`); + const CommandPath = path.join(nightwatchDir, 'testsuite/index.js'); const TestSuite = require(CommandPath); const originalFn = TestSuite.prototype.terminate; TestSuite.prototype.terminate = async function(...args) { From adda0beb7fe99e50de1d856fd63c5610a3ecc439 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Fri, 28 Nov 2025 11:56:30 +0530 Subject: [PATCH 35/46] minor log change --- src/utils/helper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/helper.js b/src/utils/helper.js index 4ce9d0c..23bd35b 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1379,7 +1379,7 @@ exports.patchBrowserTerminateCommand = () =>{ } await new Promise(resolve => setTimeout(resolve, pollInterval)); } - Logger.debug(`Pending A11y Scans at session end: ${AccessibilityAutomation.pendingAllyReq }`); + Logger.debug(`Pending Accessibility requests at session end: ${AccessibilityAutomation.pendingAllyReq }`); return originalFn.apply(this, args); }; From 990102cc9379b1addb06f1749f8d2a2e55bf39f6 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Fri, 28 Nov 2025 19:46:48 +0530 Subject: [PATCH 36/46] minor change --- src/accessibilityAutomation.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 04a19d2..c225644 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -200,7 +200,9 @@ class AccessibilityAutomation { 'thBuildUuid': process.env.BROWSERSTACK_TESTHUB_UUID, 'thJwtToken': process.env.BROWSERSTACK_TESTHUB_JWT }; + AccessibilityAutomation.pendingAllyReq++; await this.saveAccessibilityResults(browser, dataForExtension); + AccessibilityAutomation.pendingAllyReq--; Logger.info('Accessibility testing for this test case has ended.'); } } @@ -298,9 +300,7 @@ class AccessibilityAutomation { async saveAccessibilityResults(browser, dataForExtension = {}) { Logger.debug('Performing scan before saving results'); await this.performScan(browser); - AccessibilityAutomation.pendingAllyReq++; const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); - AccessibilityAutomation.pendingAllyReq--; Logger.debug(util.inspect(results)); } From 7d99495a802fad58df4048e9ed1616f4bd409fcb Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 16:02:53 +0530 Subject: [PATCH 37/46] fix for wrong product map --- nightwatch/globals.js | 2 +- src/utils/helper.js | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 2abb361..0a0c8fb 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -301,7 +301,7 @@ module.exports = { if (!settings.desiredCapabilities['bstack:options']) { settings.desiredCapabilities['bstack:options'] = {}; } - + process.env.BROWSERSTACK_APP_AUTOMATE = helper.checkTestEnvironmentForAppAutomate(testEnvSettings); // Plugin identifier settings.desiredCapabilities['bstack:options']['browserstackSDK'] = `nightwatch-plugin/${helper.getAgentVersion()}`; diff --git a/src/utils/helper.js b/src/utils/helper.js index 23bd35b..ba103d6 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -72,6 +72,21 @@ exports.getObservabilityKey = (config, bstackOptions={}) => { return process.env.BROWSERSTACK_ACCESS_KEY || config?.key || bstackOptions?.accessKey; }; +exports.isAppAutomate = () => { + return process.env.BROWSERSTACK_APP_AUTOMATE === 'true'; +}; + +exports.checkTestEnvironmentForAppAutomate = (testEnvSettings) => { + + const firstEnvKey = Object.keys(testEnvSettings)[0]; + const firstEnv = testEnvSettings[firstEnvKey]; + if (firstEnv?.desiredCapabilities?.['appium:options']?.app) { + return true; + } + + return false; +}; + exports.isAccessibilitySession = () => { return process.env.BROWSERSTACK_ACCESSIBILITY === 'true'; }; @@ -637,7 +652,7 @@ exports.getObservabilityLinkedProductName = (caps, hostname) => { if (hostname) { if (hostname.includes('browserstack.com') && !hostname.includes('hub-ft')) { - if (this.isUndefined(caps.browserName)) { + if (this.isAppAutomate()) { product = 'app-automate'; } else { product = 'automate'; From 18258039448208e96135f9b54da9d77a880dacbe Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 16:36:31 +0530 Subject: [PATCH 38/46] minor changes --- nightwatch/globals.js | 11 +++++++---- src/accessibilityAutomation.js | 7 ------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 9f57d1d..46ada9e 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -259,7 +259,7 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - process.env.VALID_ALLY_PLATFORM = accessibilityAutomation.validateA11yCaps(browser); + process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); @@ -272,7 +272,6 @@ module.exports = { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); if (TestMap.hasTestFinished(uuid)) { Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); - return; } try { @@ -356,7 +355,9 @@ module.exports = { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - helper.patchBrowserTerminateCommand(); + if(!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand() + }; } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); @@ -533,7 +534,9 @@ module.exports = { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - helper.patchBrowserTerminateCommand(); + if(!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand() + }; } } catch (err){ Logger.debug(`Exception while setting Accessibility Automation capabilities. Error ${err}`); diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index 4458854..6431d65 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -165,7 +165,6 @@ class AccessibilityAutomation { validateAppA11yCaps(capabilities = {}) { /* Check if the current driver platform is eligible for AppAccessibility scan */ - Logger.debug(`capabilities ${JSON.stringify(capabilities)}`); if ( capabilities?.platformName && String(capabilities?.platformName).toLowerCase() === 'android' && @@ -286,16 +285,10 @@ class AccessibilityAutomation { if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); - return; } if (this.currentTest.shouldScanTestForAccessibility === false) { - if(commandName && commandName !== '') { - Logger.debug(`Skipping Accessibility scan for command ${commandName} as the test is excluded from accessibility scanning.`); - } else { - Logger.debug('Skipping Accessibility scan as the test is excluded from accessibility scanning.'); - } return; } From e3d9c59bac6f3fcc255c9f0c1f30be3051d15444 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 17:55:06 +0530 Subject: [PATCH 39/46] minor change --- src/accessibilityAutomation.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index c225644..d8f75c1 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -301,7 +301,8 @@ class AccessibilityAutomation { Logger.debug('Performing scan before saving results'); await this.performScan(browser); const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); - Logger.debug(util.inspect(results)); + + return results; } async commandWrapper() { From c52b4b8dd68a084f98216adf1ed55cedd2d3da47 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 1 Dec 2025 22:50:38 +0530 Subject: [PATCH 40/46] review changes pt.1 & lint fixes --- nightwatch/globals.js | 16 ++++---- src/accessibilityAutomation.js | 70 ++++++++++++++++++++-------------- src/utils/helper.js | 9 ++++- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 46ada9e..a62adb2 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -272,6 +272,7 @@ module.exports = { const uuid = process.env.TEST_RUN_UUID || TestMap.getUUID(test); if (TestMap.hasTestFinished(uuid)) { Logger.debug(`Test with UUID ${uuid} already marked as finished, skipping duplicate TestRunFinished event`); + return; } try { @@ -355,8 +356,8 @@ module.exports = { if (helper.isAccessibilitySession() && !settings.parallel_mode) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - if(!process.env.BROWSERSTACK_APP_AUTOMATE){ - helper.patchBrowserTerminateCommand() + if (!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand(); }; } } catch (err){ @@ -486,10 +487,9 @@ module.exports = { async beforeEach(settings) { if (helper.isAppAccessibilitySession()){ - browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js - browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; // [TODO] these yet to be added in accessibilityAutomation.js - } - else{ + browser.getAccessibilityResults = () => { return accessibilityAutomation.getAppAccessibilityResults(browser) }; + browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAppAccessibilityResultsSummary(browser) }; + } else { browser.getAccessibilityResults = () => { return accessibilityAutomation.getAccessibilityResults() }; browser.getAccessibilityResultsSummary = () => { return accessibilityAutomation.getAccessibilityResultsSummary() }; } @@ -534,8 +534,8 @@ module.exports = { if (helper.isAccessibilitySession()) { accessibilityAutomation.setAccessibilityCapabilities(settings); accessibilityAutomation.commandWrapper(); - if(!process.env.BROWSERSTACK_APP_AUTOMATE){ - helper.patchBrowserTerminateCommand() + if (!process.env.BROWSERSTACK_APP_AUTOMATE){ + helper.patchBrowserTerminateCommand(); }; } } catch (err){ diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index a1b8e5f..e2feb3f 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -1,7 +1,7 @@ const path = require('path'); const helper = require('./utils/helper'); const Logger = require('./utils/logger'); -const {APP_ALLY_ENDPOINT,APP_ALLY_ISSUES_SUMMARY_ENDPOINT,APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); +const {APP_ALLY_ENDPOINT, APP_ALLY_ISSUES_SUMMARY_ENDPOINT, APP_ALLY_ISSUES_ENDPOINT} = require('./utils/constants'); const util = require('util'); const AccessibilityScripts = require('./scripts/accessibilityScripts'); @@ -166,16 +166,18 @@ class AccessibilityAutomation { validateAppA11yCaps(capabilities = {}) { /* Check if the current driver platform is eligible for AppAccessibility scan */ if ( - capabilities?.platformName && + capabilities?.platformName && String(capabilities?.platformName).toLowerCase() === 'android' && capabilities?.platformVersion && parseInt(capabilities?.platformVersion?.toString()) < 11 ) { - Logger.warn( - 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' - ); - return false; + Logger.warn( + 'App Accessibility Automation tests are supported on OS version 11 and above for Android devices.' + ); + + return false; } + return true; } @@ -285,6 +287,7 @@ class AccessibilityAutomation { if (!this.isAccessibilityAutomationSession() || !this._isAccessibilitySession) { Logger.warn('Not an Accessibility Automation session, cannot perform Accessibility scan.'); + return; } @@ -307,6 +310,7 @@ class AccessibilityAutomation { {} ); Logger.debug(util.inspect(results)); + return results; } AccessibilityAutomation.pendingAllyReq++; @@ -333,20 +337,23 @@ class AccessibilityAutomation { } if (!helper.isAppAccessibilitySession()) { - Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') - return []; + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results.'); + + return []; } try { - const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; - const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); - const result = apiRespone?.data?.data?.issues; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); - return result; + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.issues; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + + return result; } catch (error) { - Logger.error('No accessibility summary was found.'); - Logger.debug(`getAppA11yResults Failed. Error: ${error}`); - return []; + Logger.error('No accessibility results were found.'); + Logger.debug(`getAppAccessibilityResults Failed. Error: ${error}`); + + return []; } } @@ -357,18 +364,22 @@ class AccessibilityAutomation { } if (!helper.isAppAccessibilitySession()) { - Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.') - return {} + Logger.warn('Not an Accessibility Automation session, cannot retrieve Accessibility results summary.'); + + return {}; } try { - const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; - const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); - const result = apiRespone?.data?.data?.summary; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); - return result; - } catch { - Logger.error('No accessibility summary was found.'); - return {}; + const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; + const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); + const result = apiRespone?.data?.data?.summary; + Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + + return result; + } catch (error) { + Logger.error('No accessibility result summary were found.'); + Logger.debug(`getAppAccessibilityResultsSummary Failed. Error: ${error}`); + + return {}; } } @@ -377,10 +388,11 @@ class AccessibilityAutomation { await this.performScan(browser); const upperTimeLimit = process.env.BSTACK_A11Y_POLLING_TIMEOUT ? Date.now() + parseInt(process.env.BSTACK_A11Y_POLLING_TIMEOUT) * 1000 : Date.now() + 30000; - const params = { test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now() }; // Query params to pass - const header = { Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}` }; + const params = {test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now()}; // Query params to pass + const header = {Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}`}; const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + return apiRespone; } @@ -390,7 +402,7 @@ class AccessibilityAutomation { Logger.debug('Performing scan before saving results'); await this.performScan(browser); if (helper.isAppAccessibilitySession()){ - return; + return; } const results = await browser.executeAsyncScript(AccessibilityScripts.saveTestResults, dataForExtension); diff --git a/src/utils/helper.js b/src/utils/helper.js index 23d5b8a..8f5b519 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1410,8 +1410,10 @@ exports.formatString = (template, ...values) => { if (template === null) { return ''; } + return template.replace(/%s/g, () => { const value = values[i++]; + return value !== null && value !== undefined ? value : ''; }); }; @@ -1440,6 +1442,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }); const responseData = JSON.parse(response.body); + return { data: responseData, headers: response.headers, @@ -1452,6 +1455,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( if (isNaN(nextPollTime)) { Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); + return { data: {}, headers: error.response.headers, @@ -1467,6 +1471,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( // Stop polling if the upper time limit is reached if (nextPollTime > upperLimit) { Logger.warn('Polling stopped due to upper time limit.'); + return { data: {}, headers: error.response.headers, @@ -1478,6 +1483,7 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( // Wait for the specified time and poll again await new Promise((resolve) => setTimeout(resolve, elapsedTime)); + return exports.pollApi(url, params, headers, upperLimit, startTime); } else if (error.response) { throw { @@ -1487,7 +1493,8 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }; } else { Logger.error(`Unexpected error occurred: ${error}`); - return { data: {}, headers: {}, message: 'Unexpected error occurred.' }; + + return {data: {}, headers: {}, message: 'Unexpected error occurred.'}; } } }; From 3b795f6af380203805146a05178241fb182f2295 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 2 Dec 2025 14:20:25 +0530 Subject: [PATCH 41/46] minor change --- .eslintrc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index 09780ca..9ee7cbc 100644 --- a/.eslintrc +++ b/.eslintrc @@ -108,6 +108,8 @@ "by": "readonly", "expect": "readonly", "browser": "readonly", - "Key": "readonly" + "Key": "readonly", + "URLSearchParams": "readonly" + } } From 143d0723f7e1eae02b16c09c5147f3eec47a8b87 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Tue, 2 Dec 2025 17:47:45 +0530 Subject: [PATCH 42/46] lint fix --- nightwatch/globals.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index a62adb2..19908d4 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -259,7 +259,7 @@ module.exports = { }); eventBroadcaster.on('TestRunStarted', async (test) => { - process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); + process.env.VALID_ALLY_PLATFORM = process.env.BROWSERSTACK_APP_AUTOMATE ? accessibilityAutomation.validateAppA11yCaps(test.metadata.sessionCapabilities) : accessibilityAutomation.validateA11yCaps(browser); await accessibilityAutomation.beforeEachExecution(test); if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); From 6956b301c809445cda87801d238caf2a45f92355 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Wed, 3 Dec 2025 19:28:30 +0530 Subject: [PATCH 43/46] added handling of a edge case --- nightwatch/globals.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 19908d4..ab03384 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -21,6 +21,7 @@ const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; let testRunner = ''; +let testEventPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -264,7 +265,7 @@ module.exports = { if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); } }); @@ -278,7 +279,7 @@ module.exports = { try { await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner !== 'cucumber'){ - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); TestMap.markTestFinished(uuid); } @@ -474,6 +475,10 @@ module.exports = { await helper.deleteRerunFile(); } try { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } await testObservability.stopBuildUpstream(); if (process.env.BROWSERSTACK_TESTHUB_UUID) { Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); @@ -543,6 +548,13 @@ module.exports = { } addProductMapAndbuildUuidCapability(settings); + }, + + async afterChildProcess() { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } } }; From d36ada5d9cf4d87906714ec15168860070972e1d Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Thu, 4 Dec 2025 15:39:20 +0530 Subject: [PATCH 44/46] fixed the polling logic --- src/accessibilityAutomation.js | 6 ++--- src/utils/helper.js | 40 ++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/accessibilityAutomation.js b/src/accessibilityAutomation.js index e2feb3f..7e914eb 100644 --- a/src/accessibilityAutomation.js +++ b/src/accessibilityAutomation.js @@ -346,7 +346,7 @@ class AccessibilityAutomation { const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_ENDPOINT}`; const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); const result = apiRespone?.data?.data?.issues; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + Logger.debug(`Results: ${JSON.stringify(result)}`); return result; } catch (error) { @@ -372,7 +372,7 @@ class AccessibilityAutomation { const apiUrl = `${APP_ALLY_ENDPOINT}/${APP_ALLY_ISSUES_SUMMARY_ENDPOINT}`; const apiRespone = await this.getAppA11yResultResponse(apiUrl, browser, browser.sessionId); const result = apiRespone?.data?.data?.summary; - Logger.debug(`Polling Result: ${JSON.stringify(result)}`); + Logger.debug(`Results Summary: ${JSON.stringify(result)}`); return result; } catch (error) { @@ -391,7 +391,7 @@ class AccessibilityAutomation { const params = {test_run_uuid: process.env.TEST_RUN_UUID, session_id: sessionId, timestamp: Date.now()}; // Query params to pass const header = {Authorization: `Bearer ${process.env.BSTACK_A11Y_JWT}`}; const apiRespone = await helper.pollApi(apiUrl, params, header, upperTimeLimit); - Logger.debug(`Polling Result: ${JSON.stringify(apiRespone)}`); + Logger.debug(`Polling Result: ${JSON.stringify(apiRespone.message)}`); return apiRespone; diff --git a/src/utils/helper.js b/src/utils/helper.js index 8f5b519..acd643d 100644 --- a/src/utils/helper.js +++ b/src/utils/helper.js @@ -1442,50 +1442,52 @@ exports.pollApi = async (url, params, headers, upperLimit, startTime = Date.now( }); const responseData = JSON.parse(response.body); - - return { - data: responseData, - headers: response.headers, - message: 'Polling succeeded.' - }; - } catch (error) { - if (error.response && error.response.statusCode === 404) { - const nextPollTime = parseInt(error.response.headers.next_poll_time, 10) * 1000; - Logger.debug(`timeInMillis ${nextPollTime}`); + + if (response.statusCode === 404) { + const nextPollTime = parseInt(response.headers?.next_poll_time, 10) * 1000; + Logger.debug(`nextPollTime: ${nextPollTime}`); if (isNaN(nextPollTime)) { Logger.warn('Invalid or missing `nextPollTime` header. Stopping polling.'); return { data: {}, - headers: error.response.headers, + headers: response.headers || {}, message: 'Invalid nextPollTime header value. Polling stopped.' }; } - const elapsedTime = nextPollTime - Date.now(); - Logger.debug( - `elapsedTime ${elapsedTime} timeInMillis ${nextPollTime} upperLimit ${upperLimit}` - ); - // Stop polling if the upper time limit is reached if (nextPollTime > upperLimit) { Logger.warn('Polling stopped due to upper time limit.'); return { data: {}, - headers: error.response.headers, + headers: response.headers || {}, message: 'Polling stopped due to upper time limit.' }; } - Logger.debug(`Polling again in ${elapsedTime}ms with params:`, params); + const elapsedTime = Math.max(0, nextPollTime - Date.now()); + Logger.debug( + `elapsedTime ${elapsedTime} nextPollTimes ${nextPollTime} upperLimit ${upperLimit}` + ); + + Logger.debug(`Polling for results again in ${elapsedTime}ms`); // Wait for the specified time and poll again await new Promise((resolve) => setTimeout(resolve, elapsedTime)); return exports.pollApi(url, params, headers, upperLimit, startTime); - } else if (error.response) { + } + + return { + data: responseData, + headers: response.headers, + message: 'Polling succeeded.' + }; + } catch (error) { + if (error.response) { throw { data: {}, headers: {}, From c68ae0bc906a63898f09bc07ee8eec8e19d25026 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Fri, 5 Dec 2025 16:28:33 +0530 Subject: [PATCH 45/46] fix for handling a edge case --- nightwatch/globals.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 0a0c8fb..465250a 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -21,6 +21,7 @@ const _testCasesData = {}; let currentTestUUID = ''; let workerList = {}; let testRunner = ''; +let testEventPromises = []; eventHelper.eventEmitter.on(EVENTS.LOG_INIT, (loggingData) => { const testCaseStartedId = loggingData.message.replace('TEST-OBSERVABILITY-PID-TESTCASE-MAPPING-', '').slice(1, -1); @@ -264,7 +265,7 @@ module.exports = { if (testRunner !== 'cucumber'){ const uuid = TestMap.storeTestDetails(test); process.env.TEST_RUN_UUID = uuid; - await testObservability.sendTestRunEvent('TestRunStarted', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunStarted', test, uuid)); } }); @@ -278,7 +279,7 @@ module.exports = { try { await accessibilityAutomation.afterEachExecution(test, uuid); if (testRunner !== 'cucumber'){ - await testObservability.sendTestRunEvent('TestRunFinished', test, uuid); + testEventPromises.push(testObservability.sendTestRunEvent('TestRunFinished', test, uuid)); TestMap.markTestFinished(uuid); } @@ -472,6 +473,10 @@ module.exports = { await helper.deleteRerunFile(); } try { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } await testObservability.stopBuildUpstream(); if (process.env.BROWSERSTACK_TESTHUB_UUID) { Logger.info(`\nVisit https://automation.browserstack.com/builds/${process.env.BROWSERSTACK_TESTHUB_UUID} to view build report, insights, and many more debugging information all at one place!\n`); @@ -533,6 +538,13 @@ module.exports = { } addProductMapAndbuildUuidCapability(settings); + }, + + async afterChildProcess() { + if (testEventPromises.length > 0) { + await Promise.all(testEventPromises); + testEventPromises.length = 0; // Clear the array + } } }; From a7f66fd2bf6b05cb9d47317b5d2d7ea9f2f9f974 Mon Sep 17 00:00:00 2001 From: Bhargavi-BS Date: Mon, 8 Dec 2025 19:00:40 +0530 Subject: [PATCH 46/46] fix for the timeout issue --- nightwatch/globals.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nightwatch/globals.js b/nightwatch/globals.js index 465250a..ecc9e65 100644 --- a/nightwatch/globals.js +++ b/nightwatch/globals.js @@ -541,6 +541,8 @@ module.exports = { }, async afterChildProcess() { + + await helper.shutDownRequestHandler(); if (testEventPromises.length > 0) { await Promise.all(testEventPromises); testEventPromises.length = 0; // Clear the array