Skip to content

Revert "[AJDA-1176] Skip orphaned manifests" #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ondrajodas merged 1 commit into from
Oct 6, 2025
Merged

Revert "[AJDA-1176] Skip orphaned manifests" #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions phpstan-baseline.neon
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,11 @@ parameters:
count: 1
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php

-
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:assertManifestContainsColumns\\(\\) has parameter \\$expected with no value type specified in iterable type array\\.$#"
count: 1
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php

-
message: "#^Method Keboola\\\\GoogleAnalyticsExtractor\\\\ApplicationTest\\:\\:getConfig\\(\\) return type has no value type specified in iterable type array\\.$#"
count: 1
Expand All @@ -850,6 +855,11 @@ parameters:
count: 1
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php

-
message: "#^Parameter \\#1 \\$json of function json_decode expects string, string\\|false given\\.$#"
count: 1
path: tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php

-
message: "#^Parameter \\#2 \\$array of static method PHPUnit\\\\Framework\\\\Assert\\:\\:assertArrayHasKey\\(\\) expects array\\|ArrayAccess, mixed given\\.$#"
count: 2
Expand Down
27 changes: 4 additions & 23 deletions src/Extractor/Extractor.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,10 @@ public function runProfiles(array $query, array $profiles): array

if (isset($query['query'])) {
$outputCsv = $this->output->createReport($query);
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true);
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));

$downloadedProfiles = false;
$manifestCreated = false;
foreach ($profiles as $profile) {
$this->logger->info(sprintf('Profile "%s" export started.', $profile['id']));
$apiQuery = $query;
Expand Down Expand Up @@ -130,16 +130,7 @@ public function runProfiles(array $query, array $profiles): array
}
}

$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
if ($rowCount > 0 && !$manifestCreated) {
$this->output->createManifest(
$outputCsv->getFilename(),
$query,
['id'],
true,
);
$manifestCreated = true;
}
$paginator->paginate($apiQuery, $report, $outputCsv);

$status[$query['outputTable']][$profile['id']] = 'ok';
}
Expand Down Expand Up @@ -168,10 +159,10 @@ public function runProperties(array $query, array $properties): array
$query['query']['endpoint'] = 'properties';

$outputCsv = $this->output->createReport($query);
$this->output->createManifest($outputCsv->getFilename(), $query, ['id'], true, 'idProperty');
$this->logger->info(sprintf("Running query '%s'", $query['outputTable']));

$downloadedProperties = false;
$manifestCreated = false;
foreach ($properties as $property) {
$this->logger->info(sprintf('Property "%s" export started.', $property['propertyName']));
if (!empty($query['query']['viewId'])
Expand Down Expand Up @@ -208,17 +199,7 @@ public function runProperties(array $query, array $properties): array
continue;
}

$rowCount = $paginator->paginate($apiQuery, $report, $outputCsv);
if ($rowCount > 0 && !$manifestCreated) {
$this->output->createManifest(
$outputCsv->getFilename(),
$query,
['id'],
true,
'idProperty',
);
$manifestCreated = true;
}
$paginator->paginate($apiQuery, $report, $outputCsv);

$status[$query['outputTable']][$property['propertyKey']] = 'ok';
}
Expand Down
2 changes: 1 addition & 1 deletion src/Extractor/Paginator/IPaginator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ interface IPaginator
{
public function getOutput(): Output;
public function getClient(): Client;
public function paginate(array $query, array $report, CsvFile $csvFile): int;
public function paginate(array $query, array $report, CsvFile $csvFile): void;
}
4 changes: 1 addition & 3 deletions src/Extractor/Paginator/ProfilesPaginator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public function getClient(): Client
return $this->client;
}

public function paginate(array $query, array $report, CsvFile $csvFile): int
public function paginate(array $query, array $report, CsvFile $csvFile): void
{
$counter = 0;
do {
Expand All @@ -60,8 +60,6 @@ public function paginate(array $query, array $report, CsvFile $csvFile): int
}
$query = $nextQuery;
} while ($query);

return $counter;
}

private function getStartIndex(string $link): string
Expand Down
4 changes: 1 addition & 3 deletions src/Extractor/Paginator/PropertiesPaginator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public function setProperty(array $property): self
return $this;
}

public function paginate(array $query, array $report, CsvFile $csvFile): int
public function paginate(array $query, array $report, CsvFile $csvFile): void
{
$localCounter = 0;
do {
Expand All @@ -66,7 +66,5 @@ public function paginate(array $query, array $report, CsvFile $csvFile): int

$query = $nextQuery;
} while ($report['totals'] > $localCounter);

return $localCounter;
}
}
73 changes: 73 additions & 0 deletions tests/Keboola/GoogleAnalyticsExtractor/ApplicationTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,72 @@ private function getConfig(string $suffix = ''): array
return $config;
}

public function testAppRunDailyWalk(): void
{
$this->config = $this->getConfig('_antisampling');
$this->runProcess();

$dailyWalk = $this->getManifestFiles('dailyWalk');
Assert::assertEquals(1, count($dailyWalk));

foreach ($dailyWalk as $file) {
/** @var $file SplFileInfo */
$this->assertManifestContainsColumns($file->getPathname(), [
'id',
'idProfile',
'date',
'sourceMedium',
'landingPagePath',
'pageviews',
]);
}
}

public function testAppRunAdaptive(): void
{
$this->config = $this->getConfig('_antisampling_adaptive');
$this->runProcess();

$adaptive = $this->getManifestFiles('adaptive');
Assert::assertEquals(1, count($adaptive));

foreach ($adaptive as $file) {
/** @var $file SplFileInfo */
$this->assertManifestContainsColumns($file->getPathname(), [
'id',
'idProfile',
'date',
'sourceMedium',
'landingPagePath',
'pageviews',
]);
}
}

public function testAppRunMCF(): void
{
$this->config = $this->getConfig('_mcf');
$this->runProcess();

$funnelFiles = $this->getManifestFiles('funnel');
Assert::assertEquals(1, count($funnelFiles));

foreach ($funnelFiles as $file) {
/** @var $file SplFileInfo */
$this->assertManifestContainsColumns($file->getPathname(), [
'id',
'idProfile',
'mcf:conversionDate',
'mcf:sourcePath',
'mcf:mediumPath',
'mcf:sourceMedium',
'mcf:totalConversions',
'mcf:totalConversionValue',
'mcf:assistedConversions',
]);
}
}

public function testAppProfilesProperties(): void
{
$this->config = $this->getConfig('_empty');
Expand Down Expand Up @@ -117,6 +183,13 @@ private function getManifestFiles(string $queryName): Finder
;
}

private function assertManifestContainsColumns(string $pathname, array $expected): void
{
$manifest = (array) json_decode(file_get_contents($pathname), true, 512, JSON_THROW_ON_ERROR);
Assert::assertArrayHasKey('columns', $manifest);
Assert::assertEquals($expected, $manifest['columns']);
}

public function appRunDataProvider(): Generator
{
yield 'configRow' => [
Expand Down
Loading