MQE-893: Add flag to robo generate: tests which accepts a specific set of tests to execute

- add validation around custom test configuration

Author: imeron2433

src/Magento/FunctionalTestingFramework/Suite/SuiteGenerator.php

@@ -6,6 +6,7 @@
 
 namespace Magento\FunctionalTestingFramework\Suite;
 
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
 use Magento\FunctionalTestingFramework\Suite\Generators\GroupClassGenerator;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Magento\FunctionalTestingFramework\Suite\Objects\SuiteObject;
@@ -156,6 +157,7 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
 
         $relevantTests = [];
         if (!empty($tests)) {
+            $this->validateTestsReferencedInSuite($suiteName, $tests, $originalSuiteName);
             foreach ($tests as $testName) {
                 $relevantTests[$testName] = TestObjectHandler::getInstance()->getObject($testName);
             }
@@ -170,6 +172,34 @@ private function generateSuiteFromTest($suiteName, $tests = [], $originalSuiteNa
         print "Suite ${suiteName} generated to ${relativePath}.\n";
     }
 
+    /**
+     * Function which validates tests passed in as custom configuration against the configuration defined by the user to
+     * prevent possible invalid test configurations from executing.
+     *
+     * @param string $suiteName
+     * @param array $testsReferenced
+     * @param string $originalSuiteName
+     * @return void
+     * @throws TestReferenceException
+     */
+    private function validateTestsReferencedInSuite($suiteName, $testsReferenced, $originalSuiteName)
+    {
+        $suiteRef = $originalSuiteName ?? $suiteName;
+        $possibleTestRef = SuiteObjectHandler::getInstance()->getObject($suiteRef)->getTests();
+        $invalidTestRef = null;
+        $errorMsg = "Cannot reference tests not declared as part of {$suiteRef}:\n ";
+
+        array_walk($testsReferenced, function ($value) use (&$invalidTestRef, $possibleTestRef, &$errorMsg) {
+            if (!array_key_exists($value, $possibleTestRef)) {
+                $invalidTestRef.= "\t{$value}\n";
+            }
+        });
+
+        if ($invalidTestRef != null) {
+            throw new TestReferenceException($errorMsg . $invalidTestRef);
+        }
+    }
+
     /**
      * Function for generating split groups of tests (following a parallel execution). Takes a paralle suite config
      * and generates applicable suites.

src/Magento/FunctionalTestingFramework/Util/Manifest/DefaultTestManifest.php

@@ -45,8 +45,6 @@ public function __construct($suiteConfiguration, $testPath)
         $this->manifestPath = dirname($testPath) . DIRECTORY_SEPARATOR . 'testManifest.txt';
         $this->cleanManifest($this->manifestPath);
         parent::__construct($testPath, self::DEFAULT_CONFIG, $suiteConfiguration);
-//        $fileResource = fopen($this->manifestPath, 'a');
-//        fclose($fileResource);
     }
 
     /**

src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

@@ -217,6 +217,17 @@ private function assembleAllTestPhp($testManifest, array $testsToIgnore)
     {
         /** @var TestObject[] $testObjects */
         $testObjects = $this->loadAllTestObjects();
+
+        // We need to check the tests passed in to insure that we can generate them in the current context.
+        $invalidTestObjects = array_intersect_key($testObjects, $testsToIgnore);
+        if (!empty($invalidTestObjects)) {
+            $errorMsg = "Cannot reference the following tests for generation without accompanying suite:\n";
+            array_walk($invalidTestObjects, function ($value, $key) use (&$errorMsg) {
+                $errorMsg.= "\t{$key}\n";
+            });
+            throw new TestReferenceException($errorMsg);
+        }
+
         $testObjects = array_diff_key($testObjects, $testsToIgnore);
         $cestPhpArray = [];