MQE-1800: Allow generate:tests to continue running even if one test fails generation

jilu1 · jilu1 · commit 5134007f120f · 2020-09-24T09:36:19.000-05:00
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Suite/SuiteGeneratorTest.php
@@ -51,7 +51,6 @@ public function setUp(): void
 
     /**
      * Tests generating a single suite given a set of parsed test data
-     * @throws \Exception
      */
     public function testGenerateSuite()
     {
@@ -87,7 +86,6 @@ public function testGenerateSuite()
 
     /**
      * Tests generating all suites given a set of parsed test data
-     * @throws \Exception
      */
     public function testGenerateAllSuites()
     {
@@ -124,18 +122,23 @@ public function testGenerateAllSuites()
 
     /**
      * Tests attempting to generate a suite with no included/excluded tests and no hooks
-     * @throws \Exception
      */
     public function testGenerateEmptySuite()
     {
+        $testDataArrayBuilder = new TestDataArrayBuilder();
+        $mockTestData = $testDataArrayBuilder
+            ->withName('test')
+            ->withAnnotations()
+            ->withTestActions()
+            ->build();
+
         $suiteDataArrayBuilder = new SuiteDataArrayBuilder();
         $mockData = $suiteDataArrayBuilder
             ->withName('basicTestSuite')
             ->build();
         unset($mockData['suites']['basicTestSuite'][TestObjectExtractor::TEST_BEFORE_HOOK]);
         unset($mockData['suites']['basicTestSuite'][TestObjectExtractor::TEST_AFTER_HOOK]);
 
-        $mockTestData = null;
         $this->setMockTestAndSuiteParserOutput($mockTestData, $mockData);
 
         // set expected error message
@@ -146,6 +149,9 @@ public function testGenerateEmptySuite()
         $mockSuiteGenerator->generateSuite("basicTestSuite");
     }
 
+    /**
+     * Tests generating all suites with a suite containing invalid test reference
+     */
     public function testInvalidSuiteTestPair()
     {
         // Mock Suite1 => Test1 and Suite2 => Test2
@@ -191,6 +197,9 @@ public function testInvalidSuiteTestPair()
         $this->assertArrayHasKey('Suite2', $suiteErrors);
     }
 
+    /**
+     * Tests generating all suites with a non-existing suite
+     */
     public function testNonExistentSuiteTestPair()
     {
         $testDataArrayBuilder = new TestDataArrayBuilder();
@@ -221,6 +230,7 @@ public function testNonExistentSuiteTestPair()
      * Function used to set mock for parser return and force init method to run between tests.
      *
      * @param array $testData
+     * @param array $suiteData
      * @throws \Exception
      */
     private function setMockTestAndSuiteParserOutput($testData, $suiteData)
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php
@@ -118,8 +118,7 @@ public function testGetTestObject()
      */
     public function testGetTestWithFileName()
     {
-        $this->markTestIncomplete();
-        //TODO
+        $this->markTestIncomplete('TODO');
     }
 
     /**
diff --git a/dev/tests/unit/Magento/FunctionalTestFramework/Util/ModuleResolverTest.php b/dev/tests/unit/Magento/FunctionalTestFramework/Util/ModuleResolverTest.php
@@ -10,6 +10,7 @@
 use AspectMock\Test as AspectMock;
 
 use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
+use Magento\FunctionalTestingFramework\Exceptions\FastFailException;
 use Magento\FunctionalTestingFramework\Exceptions\TestFrameworkException;
 use Magento\FunctionalTestingFramework\ObjectManager;
 use Magento\FunctionalTestingFramework\ObjectManagerFactory;
@@ -777,7 +778,7 @@ public function testGetModulePathsNoAdminToken()
         $this->setMockResolverProperties($resolver, null, null);
 
         // Cannot Generate if no --force was passed in and no Admin Token is returned succesfully
-        $this->expectException(TestFrameworkException::class);
+        $this->expectException(FastFailException::class);
         $resolver->getModulesPath();
     }
 
@@ -816,7 +817,7 @@ public function testGetAdminTokenWithMissingEnv()
         $resolver = ModuleResolver::getInstance();
 
         // Expect exception
-        $this->expectException(TestFrameworkException::class);
+        $this->expectException(FastFailException::class);
         $resolver->getModulesPath();
     }
 
@@ -833,7 +834,7 @@ public function testGetAdminTokenWithBadResponse()
         $resolver = ModuleResolver::getInstance();
 
         // Expect exception
-        $this->expectException(TestFrameworkException::class);
+        $this->expectException(FastFailException::class);
         $resolver->getModulesPath();
     }
 
diff --git a/dev/tests/verification/ResilientGenerationModule/Suite/NotGenerateEmptySuite.xml b/dev/tests/verification/ResilientGenerationModule/Suite/NotGenerateEmptySuite.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+
+<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="urn:magento:mftf:Suite/etc/suiteSchema.xsd">
+    <suite name="NotGenerateEmptySuite">
+    </suite>
+</suites>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/ActionGroupTest/NotGenerateActionGroupHasBeforeOrAfterTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/ActionGroupTest/NotGenerateActionGroupHasBeforeOrAfterTest.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateActionGroupHasBeforeOrAfterTest">
+        <actionGroup ref="FunctionalActionGroup" stepKey="ag" before="N"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/ActionGroupTest/NotGenerateNonExistingActionGroupTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/ActionGroupTest/NotGenerateNonExistingActionGroupTest.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateNonExistingActionGroupTest">
+        <actionGroup ref="NNNFunctionalActionGroup" stepKey="ag"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/ExtendTest/NotGenerateExtendNonExistingParentTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/ExtendTest/NotGenerateExtendNonExistingParentTest.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateExtendNonExistingParentTest" extends="N">
+        <comment stepKey="comment" userInput="comment"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/ExtendTest/NotGenerateExtendSelfTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/ExtendTest/NotGenerateExtendSelfTest.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateExtendSelfTest" extends="NotGenerateExtendSelfTest">
+        <comment stepKey="comment" userInput="comment"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/NotGenerateActionHasBothBeforeAndAfterTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/NotGenerateActionHasBothBeforeAndAfterTest.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateActionHasBothBeforeAndAfterTest">
+        <amOnPage url="{{AdminLoginPage.url}}" stepKey="navigateToAdmin" before="B" after="A"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/ResilientGenerationModule/Test/NotGenerateActionHasInvalidBeforeOrAfterTest.xml b/dev/tests/verification/ResilientGenerationModule/Test/NotGenerateActionHasInvalidBeforeOrAfterTest.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+<tests xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="urn:magento:mftf:Test/etc/testSchema.xsd">
+    <test name="NotGenerateActionHasInvalidBeforeOrAfterTest">
+        <amOnPage url="{{AdminLoginPage.url}}" stepKey="navigateToAdmin" after="N"/>
+        <click stepKey="a" selector="{{$a.b$}}" userInput="a"/>
+    </test>
+</tests>
diff --git a/dev/tests/verification/Tests/ResilientGenerationTest.php b/dev/tests/verification/Tests/ResilientGenerationTest.php
@@ -38,7 +38,10 @@ class ResilientGenerationTest extends MftfTestCase
         ],
         'NotGenerateHookAfterSuite' => [
             '/Suite NotGenerateHookAfterSuite is not defined in xml or is invalid./'
-        ]
+        ],
+        'NotGenerateEmptySuite' => [
+            '/Suite NotGenerateEmptySuite is not defined in xml or is invalid./'
+        ],
     ];
 
     /**
@@ -213,6 +216,34 @@ function () use ($groupName) {
         }
     }
 
+    /**
+     * Test generate an empty suite
+     */
+    public function testGenerateEmptySuites()
+    {
+        $groupName = 'NotGenerateEmptySuite';
+
+        // Validate exception is thrown
+        $this->assertExceptionRegex(
+            \Exception::class,
+            self::$exceptionGrpLogs[$groupName],
+            function () use ($groupName) {
+                SuiteGenerator::getInstance()->generateSuite($groupName);
+            }
+        );
+
+        // Validate suite is not generated
+        $dirContents = array_diff(scandir(self::GENERATE_RESULT_DIR), ['..', '.']);
+        $this->assertFalse(in_array($groupName, $dirContents));
+
+        // Validate log message
+        $type = 'error';
+        $message = self::$exceptionGrpLogs[$groupName][0];
+        $context = [];
+
+        TestLoggingUtil::getInstance()->validateMockLogStatmentRegex($type, $message, $context);
+    }
+
     /**
      *
      * revert any changes made to config.yml
diff --git a/dev/tests/verification/Tests/SuiteTestReferences.php b/dev/tests/verification/Tests/SuiteTestReferences.php
@@ -61,5 +61,7 @@ class SuiteTestReferences
             'DeprecationCheckDeprecatedTestCest.php',
             'DeprecationCheckTestCest.php'
         ],
+        'NotGenerateEmptySuite' => [
+        ],
     ];
 }
diff --git a/docs/commands/mftf.md b/docs/commands/mftf.md
@@ -89,6 +89,14 @@ vendor/bin/mftf run:failed
 This command cleans up the previously generated tests; generates and runs the tests listed in `dev/tests/acceptance/tests/_output/failed`.
 For more details about `failed`, refer to [Reporting][].
 
+## Error tolerance during generation
+
+Starting from version 3.2.0, when generation errors are encountered, MFTF will not fail right away. 
+Instead MFTF will try to generate as many tests and suites as it can, log errors in `mftf.log` file and exit with non-zero generation status.
+
+Note that not all errors are tolerable at generation. 
+For example, schema validation error, parser error, WebApi authentication error will still cause `hard` failure with no test or suite generated.
+
 ## Reference
 
 ### `build:project`
diff --git a/src/Magento/FunctionalTestingFramework/DataTransport/Auth/WebApiAuth.php b/src/Magento/FunctionalTestingFramework/DataTransport/Auth/WebApiAuth.php
@@ -44,6 +44,8 @@ class WebApiAuth
      * @param string $password
      * @return string
      * @throws FastFailException
+     *
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     public static function getAdminToken($username = null, $password = null)
     {
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -127,9 +127,9 @@ public function getAllObjects()
         if ($errCount > 0
             && MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
             print(
-                "ERROR in TestObjectHandler::getAllObjects(): "
+                "ERROR: "
                 . strval($errCount)
-                . " Test(s) cannot to be extended. See mftf.log for details."
+                . " Test(s) cannot to be extended in TestObjectHandler::getAllObjects(). See mftf.log for details."
             );
         }
 
@@ -176,9 +176,10 @@ public function getTestsByGroup($groupName)
         if ($errCount > 0
             && MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
             print(
-                "ERROR in TestObjectHandler::getTestsByGroup(): "
+                "ERROR: "
                 . strval($errCount)
-                . " Test(s) cannot be referenced for group {$groupName}. See mftf.log for details."
+                . " Test(s) cannot be referenced for group {$groupName} in TestObjectHandler::getTestsByGroup()."
+                . " See mftf.log for details."
             );
         }
 
@@ -204,10 +205,13 @@ public function sanitizeTests($testsToRemove)
     /**
      * This method reads all Test.xml files into objects and stores them in an array for future access.
      *
+     * @param boolean $validateAnnotations
      * @return void
-     * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
      * @throws FastFailException
      * @throws TestFrameworkException
+     *
+     * @SuppressWarnings(PHPMD.UnusedPrivateMethod)
+     * @SuppressWarnings(PHPMD.CyclomaticComplexity)
      */
     private function initTestData($validateAnnotations = true)
     {
diff --git a/src/Magento/FunctionalTestingFramework/Util/GenerationErrorHandler.php b/src/Magento/FunctionalTestingFramework/Util/GenerationErrorHandler.php
@@ -113,8 +113,8 @@ public function printErrorSummary()
                 $totalErrors = count($this->getErrorsByType($type));
                 $totalAnnotationErrors = 0;
                 foreach ($this->getErrorsByType($type) as $entity => $error) {
-                    if ( (is_array($error['generated']) && $error['generated'][0] === true)
-                        || ($error['generated'] === true) ) {
+                    if ((is_array($error['generated']) && $error['generated'][0] === true)
+                        || ($error['generated'] === true)) {
                         $totalAnnotationErrors++;
                     }
                 }
diff --git a/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php b/src/Magento/FunctionalTestingFramework/Util/TestGenerator.php
@@ -356,9 +356,14 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
                     } catch (TestReferenceException $e) {
                         TestObjectHandler::getInstance()->sanitizeTests([$test->getName()]);
                         if (MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
-                            print("ERROR: {$test->getName()} will not be generated. Parent {$e->getMessage()} \n");
-                            LoggingUtil::getInstance()->getLogger(self::class)->error(
-                                "{$test->getName()} will not be generated. Parent does not exist."
+                            $errMessage = "{$test->getName()} will not be generated. "
+                                . "Parent test {$test->getParentName()} not defined in xml.";
+                            print("ERROR: {$errMessage}");
+                            LoggingUtil::getInstance()->getLogger(self::class)->error($errMessage);
+                            GenerationErrorHandler::getInstance()->addError(
+                                'test',
+                                $test->getName(),
+                                self::class . ': ' . $errMessage
                             );
                         }
                         continue;

Original file line number	Diff line number	Diff line change
`@@ -118,8 +118,7 @@ public function testGetTestObject()`
`118`	`118`	`*/`
`119`	`119`	`public function testGetTestWithFileName()`
`120`	`120`	`{`
`121`		`- $this->markTestIncomplete();`
`122`		`- //TODO`
	`121`	`+ $this->markTestIncomplete('TODO');`
`123`	`122`	`}`
`124`	`123`
`125`	`124`	`/**`