MQE-2272: run:failed only runs last failed test when multiple tests are run with run:test

Author: jilu1

src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

@@ -54,11 +54,11 @@ class BaseGenerateCommand extends Command
     private $testsOutputDir = null;
 
     /**
-     * File handle for 'failed_all' file
+     *  String contains all 'failed' tests
      *
-     * @var resource
+     * @var string
      */
-    private $fhFailedAll;
+    private $allFailed;
 
     /**
      * Console output style
@@ -314,20 +314,7 @@ protected function getTestsOutputDir()
     }
 
     /**
-     * Initialize 'failed_all' file
-     *
-     * @return void
-     * @throws TestFrameworkException
-     */
-    protected function initializeFailedAllFile()
-    {
-        // Initialize 'failed_all' file
-        $allFailedFile = $this->getTestsOutputDir() . self::FAILED_ALL_FILE;
-        $this->fhFailedAll = fopen($allFailedFile, 'w+');
-    }
-
-    /**
-     * Appends content of file 'failed' to file 'failed_all'
+     * Save 'failed' tests
      *
      * @return void
      */
@@ -339,46 +326,41 @@ protected function appendRunFailed()
             }
 
             if (file_exists($this->testsFailedFile)) {
-                // Append 'failed' into 'failed_all'
+                // Save 'failed' tests
                 $contents = file_get_contents($this->testsFailedFile);
                 if ($contents !== false && !empty($contents)) {
-                    fwrite($this->fhFailedAll, trim($contents) . PHP_EOL);
+                    $this->allFailed .= trim($contents) . PHP_EOL;
                 }
             }
         } catch (TestFrameworkException $e) {
         }
     }
 
     /**
-     * Replace content of file 'failed' with content in file 'failed_all'
+     * Apply 'allFailed' in 'failed' file
      *
      * @return void
      * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
-    protected function updateRunFailedWithFailedAll()
+    protected function applyAllFailed()
     {
         try {
             if (!$this->testsFailedFile) {
                 $this->testsFailedFile = $this->getTestsOutputDir() . self::FAILED_FILE;
             }
 
-            if (file_exists($this->getTestsOutputDir() . self::FAILED_ALL_FILE)) {
+            if (!empty($this->allFailed)) {
                 // Update 'failed' with content from 'failed_all'
-                $contents = file_get_contents($this->getTestsOutputDir() . self::FAILED_ALL_FILE);
-                if ($contents !== false && !empty($contents)) {
-                    if (file_exists($this->testsFailedFile)) {
-                        rename($this->testsFailedFile, $this->testsFailedFile . '.copy');
-                    }
-                    if (file_put_contents($this->testsFailedFile, $contents) === false
-                        && file_exists($this->testsFailedFile . '.copy')) {
-                        rename($this->testsFailedFile . '.copy', $this->testsFailedFile);
-                    }
-                    if (file_exists($this->testsFailedFile . '.copy')) {
-                        unlink($this->testsFailedFile . '.copy');
-                    }
+                if (file_exists($this->testsFailedFile)) {
+                    rename($this->testsFailedFile, $this->testsFailedFile . '.copy');
+                }
+                if (file_put_contents($this->testsFailedFile, $this->allFailed) === false
+                    && file_exists($this->testsFailedFile . '.copy')) {
+                    rename($this->testsFailedFile . '.copy', $this->testsFailedFile);
+                }
+                if (file_exists($this->testsFailedFile . '.copy')) {
+                    unlink($this->testsFailedFile . '.copy');
                 }
-                fclose($this->fhFailedAll);
-                unlink($this->getTestsOutputDir() . self::FAILED_ALL_FILE);
             }
         } catch (TestFrameworkException $e) {
         }

src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

@@ -109,9 +109,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $testConfigArray = json_decode($testConfiguration, true);
 
-        // Initialize `failed_all` file
-        $this->initializeFailedAllFile();
-
         if (isset($testConfigArray['tests'])) {
             $this->runTests($testConfigArray['tests'], $output);
         }
@@ -120,8 +117,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $this->runTestsInSuite($testConfigArray['suites'], $output);
         }
 
-        // Update `failed` with contents from `failed_all`
-        $this->updateRunFailedWithFailedAll();
+        // Add all failed tests in 'failed' file
+        $this->applyAllFailed();
 
         return max($this->returnCode, $generationErrorCode);
     }
@@ -168,7 +165,7 @@ private function runTests(array $tests, OutputInterface $output)
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
             }
 
-            // Append `failed` to `failed_all`
+            // Save failed tests
             $this->appendRunFailed();
         }
     }
@@ -206,7 +203,7 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
                 $this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));
             }
 
-            // Append `failed` to `failed_all`
+            // Save failed tests
             $this->appendRunFailed();
         }
     }

src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

@@ -103,9 +103,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             }
         }
 
-        // Initialize `failed_all` file
-        $this->initializeFailedAllFile();
-
         if ($this->pauseEnabled()) {
             $commandString = self::CODECEPT_RUN_FUNCTIONAL . '--verbose --steps --debug';
         } else {
@@ -134,12 +131,12 @@ function ($type, $buffer) use ($output) {
                 );
             }
 
-            // Append `failed` to `failed_all`
+            // Save failed tests
             $this->appendRunFailed();
         }
 
-        // Update `failed` with contents from `failed_all`
-        $this->updateRunFailedWithFailedAll();
+        // Add all failed tests in 'failed' file
+        $this->applyAllFailed();
 
         foreach ($returnCodes as $returnCode) {
             if ($returnCode != 0) {