Skip to content

Commit 11bb8e0

Browse files
jilu1jilu1
committed
MQE-1800: Allow generate:tests to continue running even if one test fails generation
1 parent d9bee7b commit 11bb8e0

File tree

8 files changed

+122
-109
lines changed

8 files changed

+122
-109
lines changed

‎src/Magento/FunctionalTestingFramework/Console/GenerateSuiteCommand.php‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,19 +83,18 @@ protected function execute(InputInterface $input, OutputInterface $output)
8383

8484
if (empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
8585
if ($generated > 0) {
86-
$output->writeln("Suites Generated");
86+
$output->writeln("Suites Generated" . PHP_EOL);
8787
return 0;
8888
}
8989
} else {
9090
GenerationErrorHandler::getInstance()->printErrorSummary();
91-
GenerationErrorHandler::getInstance()->reset();
9291
if ($generated > 0) {
93-
$output->writeln("Suites Generated (with errors)");
92+
$output->writeln("Suites Generated (with errors)" . PHP_EOL);
9493
return 1;
9594
}
9695
}
9796

98-
$output->writeln("No Suite Generated");
97+
$output->writeln("No Suite Generated" . PHP_EOL);
9998
return 1;
10099
}
101100
}

‎src/Magento/FunctionalTestingFramework/Console/GenerateTestsCommand.php‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,12 @@ protected function execute(InputInterface $input, OutputInterface $output)
136136
$testManifest = TestManifestFactory::makeManifest($config, $testConfiguration['suites']);
137137

138138
try {
139-
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
139+
if (empty($tests) || !empty($testConfiguration['tests'])) {
140+
// $testConfiguration['tests'] cannot be empty if $tests is not empty
141+
TestGenerator::getInstance(null, $testConfiguration['tests'])->createAllTestFiles($testManifest);
142+
} elseif (empty($testConfiguration['suites'])) {
143+
throw new FastFailException('Invalid input');
144+
}
140145
} catch (FastFailException $e) {
141146
throw $e;
142147
} catch (\Exception $e) {
@@ -164,12 +169,11 @@ protected function execute(InputInterface $input, OutputInterface $output)
164169
}
165170

166171
if (empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
167-
$output->writeln("Generate Tests Command Run");
172+
$output->writeln("Generate Tests Command Run" . PHP_EOL);
168173
return 0;
169174
} else {
170175
GenerationErrorHandler::getInstance()->printErrorSummary();
171-
GenerationErrorHandler::getInstance()->reset();
172-
$output->writeln("Generate Tests Command Run (with errors)");
176+
$output->writeln("Generate Tests Command Run (with errors)" . PHP_EOL);
173177
return 1;
174178
}
175179
}
@@ -200,18 +204,16 @@ private function createTestConfiguration(
200204
foreach ($testConfiguration['tests'] as $test) {
201205
try {
202206
$testObjects[$test] = TestObjectHandler::getInstance()->getObject($test);
203-
} catch (TestReferenceException $e) {
204-
$message = "Unable to find test {$test} from test configuration";
205-
LoggingUtil::getInstance()->getLogger(self::class)->error(
206-
$message. PHP_EOL . $e->getMessage()
207-
);
207+
} catch (FastFailException $e) {
208+
throw $e;
209+
} catch (\Exception $e) {
210+
$message = "Unable to create test object {$test} from test configuration. " . $e->getMessage();
211+
LoggingUtil::getInstance()->getLogger(self::class)->error($message);
208212
if (MftfApplicationConfig::getConfig()->verboseEnabled()
209213
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
210-
print($message . PHP_EOL);
211-
}
212-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
213-
GenerationErrorHandler::getInstance()->addError('test', $test, $message);
214+
print($message);
214215
}
216+
GenerationErrorHandler::getInstance()->addError('test', $test, $message);
215217
}
216218
}
217219

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
namespace Magento\FunctionalTestingFramework\Console;
99

1010
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
11+
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
12+
use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
1113
use Magento\FunctionalTestingFramework\Util\Path\FilePathFormatter;
1214
use Magento\FunctionalTestingFramework\Util\TestGenerator;
1315
use Symfony\Component\Console\Input\ArrayInput;
@@ -86,6 +88,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8688

8789
$testConfiguration = $this->getTestAndSuiteConfiguration($tests);
8890

91+
$generationErrorCode = 0;
92+
8993
if (!$skipGeneration) {
9094
$command = $this->getApplication()->find('generate:tests');
9195
$args = [
@@ -97,6 +101,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
97101
'-v' => $verbose
98102
];
99103
$command->run(new ArrayInput($args), $output);
104+
105+
if (!empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
106+
$generationErrorCode = 1;
107+
}
100108
}
101109

102110
$testConfigArray = json_decode($testConfiguration, true);
@@ -109,7 +117,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
109117
$this->runTestsInSuite($testConfigArray['suites'], $output);
110118
}
111119

112-
return $this->returnCode;
120+
return max($this->returnCode, $generationErrorCode);
113121
}
114122

115123
/**

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
1111
use Magento\FunctionalTestingFramework\Config\MftfApplicationConfig;
1212
use Magento\FunctionalTestingFramework\Test\Handlers\TestObjectHandler;
13+
use Magento\FunctionalTestingFramework\Util\GenerationErrorHandler;
1314
use Symfony\Component\Console\Input\ArrayInput;
1415
use Symfony\Component\Console\Input\InputArgument;
1516
use Symfony\Component\Console\Input\InputInterface;
@@ -80,6 +81,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8081
$allowSkipped
8182
);
8283

84+
$generationErrorCode = 0;
85+
8386
if (!$skipGeneration) {
8487
$testConfiguration = $this->getGroupAndSuiteConfiguration($groups);
8588
$command = $this->getApplication()->find('generate:tests');
@@ -93,6 +96,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int
9396
];
9497

9598
$command->run(new ArrayInput($args), $output);
99+
100+
if (!empty(GenerationErrorHandler::getInstance()->getAllErrors())) {
101+
$generationErrorCode = 1;
102+
}
96103
}
97104

98105
if ($this->pauseEnabled()) {
@@ -130,6 +137,6 @@ function ($type, $buffer) use ($output) {
130137
}
131138
$exitCode = 0;
132139
}
133-
return $exitCode;
140+
return max($exitCode, $generationErrorCode);
134141
}
135142
}

‎src/Magento/FunctionalTestingFramework/Suite/Util/SuiteObjectExtractor.php‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -109,13 +109,13 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
109109
LoggingUtil::getInstance()->getLogger(self::class)->error(
110110
"Unable to parse suite " . $parsedSuite[self::NAME] . ". Suite must not be empty."
111111
);
112-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
113-
GenerationErrorHandler::getInstance()->addError(
114-
'suite',
115-
$parsedSuite[self::NAME],
116-
self::class . ': ' . 'Suite must not be empty.'
117-
);
118-
}
112+
113+
GenerationErrorHandler::getInstance()->addError(
114+
'suite',
115+
$parsedSuite[self::NAME],
116+
self::class . ': ' . 'Suite must not be empty.'
117+
);
118+
119119
continue;
120120
};
121121

@@ -130,14 +130,13 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
130130
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
131131
print($includeMessage);
132132
}
133-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
134-
GenerationErrorHandler::getInstance()->addError(
135-
'suite',
136-
$parsedSuite[self::NAME],
137-
self::class . ': ' . $includeMessage,
138-
true
139-
);
140-
}
133+
134+
GenerationErrorHandler::getInstance()->addError(
135+
'suite',
136+
$parsedSuite[self::NAME],
137+
self::class . ': ' . $includeMessage,
138+
true
139+
);
141140
}
142141
} catch (FastFailException $e) {
143142
throw $e;
@@ -149,13 +148,13 @@ public function parseSuiteDataIntoObjects($parsedSuiteData)
149148
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
150149
print("ERROR: Unable to parse suite " . $parsedSuite[self::NAME] . "\n");
151150
}
152-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
153-
GenerationErrorHandler::getInstance()->addError(
154-
'suite',
155-
$parsedSuite[self::NAME],
156-
self::class . ': Unable to parse suite ' . $e->getMessage()
157-
);
158-
}
151+
152+
GenerationErrorHandler::getInstance()->addError(
153+
'suite',
154+
$parsedSuite[self::NAME],
155+
self::class . ': Unable to parse suite ' . $e->getMessage()
156+
);
157+
159158
continue;
160159
}
161160

‎src/Magento/FunctionalTestingFramework/Test/Handlers/TestObjectHandler.php‎

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -114,13 +114,11 @@ public function getAllObjects()
114114
LoggingUtil::getInstance()->getLogger(self::class)->error(
115115
"Unable to extend test " . $testName . "\n" . $exception->getMessage()
116116
);
117-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
118-
GenerationErrorHandler::getInstance()->addError(
119-
'test',
120-
$testName,
121-
self::class . ': Unable to extend test ' . $exception->getMessage()
122-
);
123-
}
117+
GenerationErrorHandler::getInstance()->addError(
118+
'test',
119+
$testName,
120+
self::class . ': Unable to extend test ' . $exception->getMessage()
121+
);
124122
}
125123
}
126124

@@ -164,13 +162,12 @@ public function getTestsByGroup($groupName)
164162
. " for group {$groupName}\n"
165163
. $exception->getMessage();
166164
LoggingUtil::getInstance()->getLogger(self::class)->error($message);
167-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
168-
GenerationErrorHandler::getInstance()->addError(
169-
'test',
170-
$test->getName(),
171-
self::class . ': ' . $message
172-
);
173-
}
165+
166+
GenerationErrorHandler::getInstance()->addError(
167+
'test',
168+
$test->getName(),
169+
self::class . ': ' . $message
170+
);
174171
}
175172
}
176173

@@ -254,13 +251,11 @@ private function initTestData($validateAnnotations = true)
254251
&& MftfApplicationConfig::getConfig()->getPhase() == MftfApplicationConfig::GENERATION_PHASE) {
255252
print("ERROR: Unable to parse test " . $testName . "\n");
256253
}
257-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
258-
GenerationErrorHandler::getInstance()->addError(
259-
'test',
260-
$testName,
261-
self::class . ': Unable to parse test ' . $exception->getMessage()
262-
);
263-
}
254+
GenerationErrorHandler::getInstance()->addError(
255+
'test',
256+
$testName,
257+
self::class . ': Unable to parse test ' . $exception->getMessage()
258+
);
264259
}
265260
}
266261
$testNameValidator->summarize(NameValidationUtil::TEST_NAME);

‎src/Magento/FunctionalTestingFramework/Util/GenerationErrorHandler.php‎

Lines changed: 35 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,16 +57,14 @@ private function __construct()
5757
*/
5858
public function addError($type, $entityName, $message, $generated = false)
5959
{
60-
if (MftfApplicationConfig::getConfig()->getPhase() != MftfApplicationConfig::EXECUTION_PHASE) {
61-
$error[$entityName] = [
62-
'message' => $message,
63-
'generated' => $generated,
64-
];
65-
if (isset($this->errors[$type])) {
66-
$this->errors[$type] = array_merge_recursive($this->errors[$type], $error);
67-
} else {
68-
$this->errors[$type] = $error;
69-
}
60+
$error[$entityName] = [
61+
'message' => $message,
62+
'generated' => $generated,
63+
];
64+
if (isset($this->errors[$type])) {
65+
$this->errors[$type] = array_merge_recursive($this->errors[$type], $error);
66+
} else {
67+
$this->errors[$type] = $error;
7068
}
7169
}
7270

@@ -108,39 +106,48 @@ public function reset()
108106
*/
109107
public function printErrorSummary()
110108
{
111-
if (is_array(array_keys($this->errors))) {
112-
foreach (array_keys($this->errors) as $type) {
113-
$totalErrors = count($this->getErrorsByType($type));
114-
$totalAnnotationErrors = 0;
115-
foreach ($this->getErrorsByType($type) as $entity => $error) {
116-
if ((is_array($error['generated']) && $error['generated'][0] === true)
117-
|| ($error['generated'] === true)) {
118-
$totalAnnotationErrors++;
119-
}
109+
foreach (array_keys($this->errors) as $type) {
110+
$totalErrors = count($this->getErrorsByType($type));
111+
$totalAnnotationErrors = 0;
112+
foreach ($this->getErrorsByType($type) as $entity => $error) {
113+
if ((is_array($error['generated']) && $error['generated'][0] === true)
114+
|| ($error['generated'] === true)) {
115+
$totalAnnotationErrors++;
120116
}
121-
$totalNotGenErrors = $totalErrors - $totalAnnotationErrors;
122-
if ($totalNotGenErrors > 0) {
117+
}
118+
$totalNotGenErrors = $totalErrors - $totalAnnotationErrors;
119+
if ($totalNotGenErrors > 0) {
120+
print(
121+
'ERROR: '
122+
. strval($totalNotGenErrors)
123+
. ' '
124+
. ucfirst($type)
125+
. "(s) failed to generate. See mftf.log for details."
126+
. PHP_EOL
127+
);
128+
}
129+
if ($totalAnnotationErrors > 0) {
130+
if ($type !== 'suite') {
123131
print(
124132
'ERROR: '
125-
. strval($totalNotGenErrors)
133+
. strval($totalAnnotationErrors)
126134
. ' '
127135
. ucfirst($type)
128-
. "(s) failed to generate. See mftf.log for details."
136+
. "(s) generated with annotation errors. See mftf.log for details."
129137
. PHP_EOL
130138
);
131-
}
132-
if ($totalAnnotationErrors > 0) {
139+
} else {
133140
print(
134-
'ERROR: '
141+
'ERROR: '
135142
. strval($totalAnnotationErrors)
136143
. ' '
137144
. ucfirst($type)
138-
. "(s) generated with annotation errors. See mftf.log for details."
145+
. "(s) has(have) tests with annotation errors. See mftf.log for details."
139146
. PHP_EOL
140147
);
141148
}
142149
}
143-
print(PHP_EOL);
144150
}
151+
print(PHP_EOL);
145152
}
146153
}

0 commit comments

Comments
 (0)