MQE-1027: Allure report requires a unique story/title for proper reporting

KevinBKozan · web-flow · commit 5dc4c0dfbb0e · 2018-06-11T16:38:08.000+03:00
- Added validation of Story/Title combinations for uniqueness
diff --git a/.gitignore b/.gitignore
@@ -14,4 +14,5 @@ coverage/
 codeception.yml
 dev/tests/functional/MFTF.suite.yml
 dev/tests/functional/_output
+dev/mftf.log
 dev/tests/mftf.log
diff --git a/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ActionGroupWithDataOverrideTestCest
diff --git a/dev/tests/verification/Resources/ActionGroupWithDataTest.txt b/dev/tests/verification/Resources/ActionGroupWithDataTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ActionGroupWithDataTestCest
diff --git a/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt b/dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ActionGroupWithNoDefaultTestCest
diff --git a/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithPersistedData.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ActionGroupWithPersistedDataCest
diff --git a/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt b/dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ActionGroupWithTopLevelPersistedDataCest
diff --git a/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt b/dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class ArgumentWithSameNameAsElementCest
diff --git a/dev/tests/verification/Resources/BasicActionGroupTest.txt b/dev/tests/verification/Resources/BasicActionGroupTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class BasicActionGroupTestCest
diff --git a/dev/tests/verification/Resources/BasicMergeTest.txt b/dev/tests/verification/Resources/BasicMergeTest.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
+ * @Title("BasicMergeTest")
  * @group functional
  * @group mergeTest
  */
diff --git a/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt b/dev/tests/verification/Resources/ChildExtendedTestAddHooks.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Parent")
+ * @Title("ChildExtendedTestAddHooks")
  * @group Parent
  */
 class ChildExtendedTestAddHooksCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestMerging.txt b/dev/tests/verification/Resources/ChildExtendedTestMerging.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestMerging")
  * @group Child
  */
 class ChildExtendedTestMergingCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt b/dev/tests/verification/Resources/ChildExtendedTestNoParent.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestNoParent")
  * @group Child
  * @group skip
  */
diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveAction.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestRemoveAction")
  * @group Child
  */
 class ChildExtendedTestRemoveActionCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt b/dev/tests/verification/Resources/ChildExtendedTestRemoveHookAction.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestRemoveHookAction")
  * @group Child
  */
 class ChildExtendedTestRemoveHookActionCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplace.txt b/dev/tests/verification/Resources/ChildExtendedTestReplace.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestReplace")
  * @group Child
  */
 class ChildExtendedTestReplaceCest
diff --git a/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt b/dev/tests/verification/Resources/ChildExtendedTestReplaceHook.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Child")
+ * @Title("ChildExtendedTestReplaceHook")
  * @group Child
  */
 class ChildExtendedTestReplaceHookCest
diff --git a/dev/tests/verification/Resources/MergedActionGroupTest.txt b/dev/tests/verification/Resources/MergedActionGroupTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class MergedActionGroupTestCest
diff --git a/dev/tests/verification/Resources/MergedReferencesTest.txt b/dev/tests/verification/Resources/MergedReferencesTest.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
+ * @Title("MergedReferencesTest")
  * @group functional
  */
 class MergedReferencesTestCest
diff --git a/dev/tests/verification/Resources/MultipleActionGroupsTest.txt b/dev/tests/verification/Resources/MultipleActionGroupsTest.txt
@@ -16,7 +16,6 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("A Functional Cest")
  * @group functional
  */
 class MultipleActionGroupsTestCest
diff --git a/dev/tests/verification/Resources/ParentExtendedTest.txt b/dev/tests/verification/Resources/ParentExtendedTest.txt
@@ -16,7 +16,7 @@ use Yandex\Allure\Adapter\Model\SeverityLevel;
 use Yandex\Allure\Adapter\Annotation\TestCaseId;
 
 /**
- * @Title("Parent")
+ * @Title("ParentExtendedTest")
  * @group Parent
  */
 class ParentExtendedTestCest
diff --git a/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml b/dev/tests/verification/TestModule/Test/ActionGroupFunctionalTest.xml
@@ -11,7 +11,6 @@
     <test name="BasicActionGroupTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -27,7 +26,6 @@
     <test name="ActionGroupWithDataTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -46,7 +44,6 @@
     <test name="ActionGroupWithDataOverrideTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -67,7 +64,6 @@
     <test name="ActionGroupWithNoDefaultTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -88,7 +84,6 @@
     <test name="ActionGroupWithPersistedData">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -108,7 +103,6 @@
     <test name="ActionGroupWithTopLevelPersistedData">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -127,7 +121,6 @@
     <test name="MultipleActionGroupsTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -149,7 +142,6 @@
     <test name="MergedActionGroupTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
@@ -168,7 +160,6 @@
     <test name="ArgumentWithSameNameAsElement">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
             <group value="functional"/>
             <features value="Action Group Functional Cest"/>
             <stories value="MQE-433"/>
diff --git a/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml b/dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml
@@ -11,7 +11,7 @@
     <test name="ParentExtendedTest">
         <annotations>
             <severity value="AVERAGE"/>
-            <title value="Parent"/>
+            <title value="ParentExtendedTest"/>
             <group value="Parent"/>
             <features value="Parent"/>
             <stories value="Parent"/>
@@ -28,7 +28,7 @@
     <test name="ChildExtendedTestReplace" extends="ParentExtendedTest">
         <annotations>
             <severity value="MINOR"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestReplace"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
@@ -39,7 +39,7 @@
     <test name="ChildExtendedTestReplaceHook" extends="ParentExtendedTest">
         <annotations>
             <severity value="MINOR"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestReplaceHook"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
@@ -52,7 +52,7 @@
     <test name="ChildExtendedTestMerging" extends="ParentExtendedTest">
         <annotations>
             <severity value="MINOR"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestMerging"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
@@ -69,7 +69,7 @@
     <test name="ChildExtendedTestRemoveAction" extends="ParentExtendedTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestRemoveAction"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
@@ -80,7 +80,7 @@
     <test name="ParentExtendedTestNoHooks">
         <annotations>
             <severity value="AVERAGE"/>
-            <title value="Parent"/>
+            <title value="ParentExtendedTestNoHooks"/>
             <group value="Parent"/>
             <features value="Parent"/>
             <stories value="Parent"/>
@@ -91,7 +91,7 @@
     <test name="ChildExtendedTestAddHooks">
         <annotations>
             <severity value="AVERAGE"/>
-            <title value="Parent"/>
+            <title value="ChildExtendedTestAddHooks"/>
             <group value="Parent"/>
             <features value="Parent"/>
             <stories value="Parent"/>
@@ -107,7 +107,7 @@
     <test name="ChildExtendedTestRemoveHookAction" extends="ParentExtendedTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestRemoveHookAction"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
@@ -120,7 +120,7 @@
     <test name="ChildExtendedTestNoParent" extends="ThisTestDoesNotExist">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="Child"/>
+            <title value="ChildExtendedTestNoParent"/>
             <group value="Child"/>
             <features value="Child"/>
             <stories value="Child"/>
diff --git a/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml b/dev/tests/verification/TestModule/Test/MergeFunctionalTest.xml
@@ -11,7 +11,7 @@
     <test name="BasicMergeTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
+            <title value="BasicMergeTest"/>
             <group value="functional"/>
             <features value="Merge Functional Cest"/>
             <stories value="MQE-433"/>
@@ -31,7 +31,7 @@
     <test name="MergedReferencesTest">
         <annotations>
             <severity value="CRITICAL"/>
-            <title value="A Functional Cest"/>
+            <title value="MergedReferencesTest"/>
             <group value="functional"/>
             <features value="Merge Functional Cest"/>
             <stories value="MQE-433"/>
diff --git a/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php b/src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php
@@ -13,6 +13,7 @@
 use Magento\FunctionalTestingFramework\Test\Parsers\TestDataParser;
 use Magento\FunctionalTestingFramework\Test\Util\ObjectExtensionUtil;
 use Magento\FunctionalTestingFramework\Test\Util\TestObjectExtractor;
+use Magento\FunctionalTestingFramework\Test\Util\AnnotationExtractor;
 
 /**
  * Class TestObjectHandler
@@ -142,6 +143,7 @@ private function initTestData()
 
             $this->tests[$testName] = $testObjectExtractor->extractTestData($testData);
         }
+        $testObjectExtractor->getAnnotationExtractor()->validateStoryTitleUniqueness();
     }
 
     /**
diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/AnnotationExtractor.php
diff --git a/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php b/src/Magento/FunctionalTestingFramework/Test/Util/TestObjectExtractor.php
