Skip to content

Commit f14fb17

Browse files
ppavlovicppavlovic
authored
Merge pull request #56 from ppavlovic/master
Truncate long fields in a log
2 parents 123b2a8 + 29bc3ee commit f14fb17

10 files changed

Lines changed: 213 additions & 29 deletions

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Add your rules here #
22
#######################
33
composer.lock
4+
.phpunit.result.cache
5+
test/.phpunit.result.cache
46

57
# Default GIT ignore rules #
68
############################

‎composer.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,10 @@
2929
},
3030
"require-dev": {
3131
"jakub-onderka/php-parallel-lint": "^0.9.2",
32-
"phpunit/phpunit" : "5.*"
32+
"phpunit/phpunit": "9.*"
3333
},
3434
"require": {
35-
"php" : ">=5.6",
35+
"php" : ">=7.3",
3636
"ext-json" : "*",
3737
"ext-curl" : "*",
3838
"g4/runner" : ">=0.22.0",
@@ -47,6 +47,9 @@
4747
"scripts": {
4848
"lint": [
4949
"./vendor/bin/parallel-lint ./ --exclude vendor"
50+
],
51+
"unit-test": [
52+
"php7.3 ./vendor/bin/phpunit -c test/phpunit.xml"
5053
]
5154
}
5255
}

‎src/Buffer/FieldsTruncator.php‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
3+
namespace G4\Log\Buffer;
4+
5+
use G4\ValueObject\StringLiteral;
6+
7+
/**
8+
* The settings for the truncatedFields are set in the config file.
9+
* The example config is:
10+
*
11+
* truncated_fields.enabled = 1
12+
* truncated_fields.truncate_above = 50000 ; truncate fields longer than characters
13+
* truncated_fields.truncate_to = 5000 ; truncate fields to this many
14+
* ; which fields to truncate in which logs
15+
* truncated_fields.nd_requests[] = resource
16+
* truncated_fields.nd_requests[] = params
17+
*/
18+
19+
class FieldsTruncator
20+
{
21+
private const TRUNCATE_ENABLED = 'enabled';
22+
private const TRUNCATE_ABOVE = 'truncate_above';
23+
private const TRUNCATE_TO = 'truncate_to';
24+
private const TRUNCATE_SUFFIX = '...[truncated, change the truncated_fields.enable=1]';
25+
26+
/**
27+
* @var string
28+
*/
29+
private $logType;
30+
31+
/**
32+
* @var array
33+
*/
34+
private $config;
35+
36+
public function __construct(StringLiteral $logType, array $configTruncatedFields)
37+
{
38+
$this->logType = (string) $logType;
39+
$this->config = $configTruncatedFields;
40+
}
41+
42+
public function truncate(array $logData)
43+
{
44+
if (!$this->enabled()) {
45+
return $logData;
46+
}
47+
foreach ($logData as $key => $value) {
48+
$logData[$key] = $this->truncateValue($key, $value);
49+
}
50+
return $logData;
51+
}
52+
53+
public function enabled(): bool
54+
{
55+
if (!isset($this->config[self::TRUNCATE_ENABLED])) {
56+
return true;
57+
}
58+
return (bool) $this->config[self::TRUNCATE_ENABLED];
59+
}
60+
61+
private function truncateValue(string $fieldName, ?string $value):? string
62+
{
63+
if (!$this->shouldTruncateField($fieldName) || !$value) {
64+
return $value;
65+
}
66+
if (strlen($value) < $this->config[self::TRUNCATE_ABOVE]) {
67+
return $value;
68+
}
69+
return substr($value, 0, (int) $this->config[self::TRUNCATE_TO]) . self::TRUNCATE_SUFFIX;
70+
}
71+
72+
public function shouldTruncateField(string $fieldName): bool
73+
{
74+
if (!isset($this->config[$this->logType])) {
75+
return true;
76+
}
77+
return in_array($fieldName, $this->config[$this->logType]);
78+
}
79+
}

‎src/Buffer/RedisToElastic.php‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ class RedisToElastic
4141
*/
4242
private $prettyPrintFields;
4343

44+
private $truncatedFields;
45+
4446
/**
4547
* RedisToElastic constructor.
4648
*
@@ -54,6 +56,7 @@ public function __construct(Redis $redisClient, RedisElasticsearchCurl $elasticC
5456
$this->elasticClient = $elasticClient;
5557
$this->batchsize = $batchsize;
5658
$this->prettyPrintFields = [];
59+
$this->truncatedFields = [];
5760
}
5861

5962
public function setPrettyPrintFields(array $fields)
@@ -62,15 +65,23 @@ public function setPrettyPrintFields(array $fields)
6265
return $this;
6366
}
6467

68+
public function setTruncatedFields(array $fields)
69+
{
70+
$this->truncatedFields = $fields;
71+
return $this;
72+
}
73+
6574
public function transferData()
6675
{
6776
$data = $this->redisClient->fetchAndClear($this->batchsize);
6877
$this->countFromRedis = count($data);
6978
if (!empty($data)) {
7079
foreach ($data as $key => $log) {
80+
$docLength = strlen($log);
7181
$logData = json_decode($log, 1);
72-
$this->data[$key] = $this->prettify($logData);
73-
$this->data[$key]['doc_length'] = strlen($log);
82+
$prettyLogData = $this->prettify($logData);
83+
$this->data[$key] = $this->truncateLongFields($prettyLogData);
84+
$this->data[$key]['doc_length'] = $docLength;
7485
}
7586
}
7687

@@ -83,6 +94,12 @@ private function prettify(array $logData)
8394
->prettify($logData);
8495
}
8596

97+
private function truncateLongFields(array $logData)
98+
{
99+
return (new FieldsTruncator($this->redisClient->getKey(), $this->truncatedFields))
100+
->truncate($logData);
101+
}
102+
86103
public function insertIntoES()
87104
{
88105
if (empty($this->data)) {

‎test/phpunit.xml‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
1-
<phpunit bootstrap="bootstrap.php" colors="true">
2-
3-
<testsuite name="Test Suite">
4-
<directory>./src</directory>
5-
</testsuite>
6-
7-
<filter>
8-
<whitelist>
9-
<directory suffix=".php">../src/</directory>
10-
</whitelist>
11-
</filter>
12-
13-
</phpunit>
1+
<?xml version="1.0"?>
2+
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" bootstrap="bootstrap.php" colors="true" xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/9.3/phpunit.xsd">
3+
<coverage>
4+
<include>
5+
<directory suffix=".php">../src/</directory>
6+
</include>
7+
</coverage>
8+
<testsuite name="Test Suite">
9+
<directory>./src</directory>
10+
</testsuite>
11+
</phpunit>

‎test/src/Adapter/RedisToEsBuildBulkDataTest.php‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
namespace Adapter;
44

55
use G4\Log\Adapter\RedisToEsBuildBulkData;
6-
use PHPUnit_Framework_TestCase;
76

8-
class RedisToEsBuildBulkDataTest extends PHPUnit_Framework_TestCase
7+
class RedisToEsBuildBulkDataTest extends \PHPUnit\Framework\TestCase
98
{
109
public function testEmptyData()
1110
{
@@ -26,7 +25,7 @@ public function testMissingIndex()
2625
ob_end_clean();
2726

2827
$this->assertEquals("\n", $result);
29-
$this->assertContains("Undefined _index or _type", $output);
28+
$this->assertStringContainsString("Undefined _index or _type", $output);
3029
}
3130

3231
public function testMissingType()
@@ -41,7 +40,7 @@ public function testMissingType()
4140
ob_end_clean();
4241

4342
$this->assertEquals("\n", $result);
44-
$this->assertContains("Undefined _index or _type", $output);
43+
$this->assertStringContainsString("Undefined _index or _type", $output);
4544
}
4645

4746
/** @dataProvider dataFromRedis */

‎test/src/Buffer/FieldsTruncatorTest.php‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<?php
2+
3+
namespace Buffer;
4+
5+
use G4\Log\Buffer\FieldsTruncator;
6+
use G4\ValueObject\StringLiteral;
7+
8+
class FieldsTruncatorTest extends \PHPUnit\Framework\TestCase
9+
{
10+
/** @var array[] */
11+
private $config;
12+
protected function setUp(): void
13+
{
14+
$this->config = [
15+
'enabled' => 1,
16+
'truncate_above' => 100,
17+
'truncate_to' => 50,
18+
'nd_requests' => [
19+
'resource',
20+
'params',
21+
],
22+
];
23+
}
24+
25+
public function testEnabled(): void
26+
{
27+
$config = $this->config;
28+
$fieldsTruncator = new FieldsTruncator(new StringLiteral('nd_requests'), $config);
29+
self::assertTrue($fieldsTruncator->enabled());
30+
$config = $this->config;
31+
$config['enabled'] = 0;
32+
$fieldsTruncator = new FieldsTruncator(new StringLiteral('nd_requests'), $config);
33+
self::assertFalse($fieldsTruncator->enabled());
34+
}
35+
36+
public function testShouldTruncateField(): void
37+
{
38+
$fieldsTruncator = new FieldsTruncator(new StringLiteral('nd_requests'), $this->config);
39+
self::assertTrue($fieldsTruncator->shouldTruncateField('resource'));
40+
self::assertTrue($fieldsTruncator->shouldTruncateField('params'));
41+
self::assertFalse($fieldsTruncator->shouldTruncateField('app_message'));
42+
}
43+
44+
public function testTruncate(): void
45+
{
46+
$fieldsTruncator = new FieldsTruncator(new StringLiteral('nd_requests'), $this->config);
47+
$logData = [
48+
'app_message' => str_repeat('x', 200),
49+
'resource' => str_repeat('y', 200),
50+
'params' => str_repeat('z', 200),
51+
];
52+
53+
$suffix = '...[truncated, change the truncated_fields.enable=1]';
54+
$expected = [
55+
'app_message' => str_repeat('x', 200),
56+
'resource' => str_repeat('y', 50) . $suffix,
57+
'params' => str_repeat('z', 50) . $suffix,
58+
];
59+
60+
self::assertSame($expected, $fieldsTruncator->truncate($logData));
61+
}
62+
}

‎test/src/Data/ExcludeTest.php‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
namespace Data;
44

55
use G4\Log\Data\Exclude;
6-
use PHPUnit_Framework_TestCase;
76

8-
class ExcludeTest extends PHPUnit_Framework_TestCase
7+
class ExcludeTest extends \PHPUnit\Framework\TestCase
98
{
109
public function testGetExclude()
1110
{

‎test/src/Data/LoggerAbstractTest.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,8 @@
66
use G4\CleanCore\Application;
77
use G4\Log\Data\Exclude;
88
use G4\Log\Data\Response;
9-
use PHPUnit_Framework_TestCase;
109

11-
class LoggerAbstractTest extends PHPUnit_Framework_TestCase
10+
class LoggerAbstractTest extends \PHPUnit\Framework\TestCase
1211
{
1312
private $app;
1413
private $profiler;
@@ -17,10 +16,12 @@ class LoggerAbstractTest extends PHPUnit_Framework_TestCase
1716
const EXCLUDED = 'EXCLUDED';
1817

1918

20-
protected function setUp()
19+
protected function setUp(): void
2120
{
21+
$_SERVER['REQUEST_METHOD'] = null;
2222
$this->profiler = $this->createMock(\G4\Runner\Profiler::class);
2323
$this->profiler->method('getProfilerOutput')->willReturn([]);
24+
$this->profiler->method('getProfilerSummary')->willReturn([]);
2425

2526
$this->request = new \G4\CleanCore\Request\Request();
2627
$response = new \G4\CleanCore\Response\Response();

‎test/src/LoggerTest.php‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
<?php
22

3+
namespace test\src;
4+
35
use G4\CleanCore\Application;
46
use G4\Log\Adapter\Redis;
57
use G4\Log\Data\Exclude;
68
use G4\Log\Data\Response;
79

8-
class LoggerTest extends PHPUnit_Framework_TestCase
10+
class LoggerTest extends \PHPUnit\Framework\TestCase
911
{
1012
const EXCLUDED = 'EXCLUDED';
1113

@@ -15,6 +17,7 @@ public function testLog()
1517

1618
$profiler = $this->createMock(\G4\Runner\Profiler::class);
1719
$profiler->method('getProfilerOutput')->willReturn(['test_profiler' => 'example']);
20+
$profiler->method('getProfilerSummary')->willReturn([]);
1821

1922
$request = new \G4\CleanCore\Request\Request();
2023
$request->setResourceName('test_service');
@@ -38,11 +41,32 @@ public function testLog()
3841
'app_message' => null,
3942
'elapsed_time' => self::EXCLUDED,
4043
'elapsed_time_ms' => self::EXCLUDED,
41-
'profiler' => '{"test_profiler":"example"}'
44+
'profiler' => '{"test_profiler":"example"}',
45+
'app_version' => '1.2.3.test',
46+
'cpu_load_1' => 0.1,
47+
'cpu_load_5' => 0.2,
48+
'cpu_load_15' => 0.3,
49+
'cpu_process' => 0.4,
4250
]
4351
);
4452

45-
$response = new Response();
53+
// extend Response class to mock the return value of getCpuLoad method
54+
$response = new class extends Response {
55+
protected function getCpuLoad()
56+
{
57+
return [
58+
'cpu_load_1' => 0.1,
59+
'cpu_load_5' => 0.2,
60+
'cpu_load_15' => 0.3,
61+
'cpu_process' => 0.4,
62+
];
63+
}
64+
65+
public function getAppVersionNumber()
66+
{
67+
return '1.2.3.test';
68+
}
69+
};
4670
$response->setApplication($app);
4771
$response->setProfiler($profiler);
4872

0 commit comments

Comments
 (0)