Skip to content

Commit dc91480

Browse files
CCPCookiesCCPCookies
authored
Merge pull request #9 from CCPCookies/skipCompressionOption
Changes compression calculations from mandatory to optional, defaulting to on. Option to toggle the compression calculation is exposed through library interface and CLI through optional flag.
2 parents bbf93b0 + 40f6a84 commit dc91480

16 files changed

Lines changed: 240 additions & 76 deletions

‎CMakeLists.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
# Copyright © 2025 CCP ehf.
22

33
cmake_minimum_required(VERSION 3.30.1)
4-
project(resources VERSION 3.0.3)
5-
4+
project(resources VERSION 3.1.0)
65

76
include(cmake/CcpGlobalSettings.cmake)
87
include(cmake/CcpTargetConfigurations.cmake)

‎cli/src/CreatePatchCliOperation.cpp‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ CreatePatchCliOperation::CreatePatchCliOperation() :
2323
m_patchFileRelativePathPrefixArgumentId( "--patch-prefix" ),
2424
m_maxInputChunkSizeArgumentId( "--chunk-size" ),
2525
m_downloadRetrySecondsArgumentId( "--download-retry" ),
26-
m_indexFolderArgumentId( "--index-folder" )
26+
m_indexFolderArgumentId( "--index-folder" ),
27+
m_skipCompressionCalculation( "--skip-compression" )
2728
{
2829

2930
AddRequiredPositionalArgument( m_previousResourceGroupPathArgumentId, "Filename to previous resourceGroup." );
@@ -62,6 +63,8 @@ CreatePatchCliOperation::CreatePatchCliOperation() :
6263
AddArgument( m_downloadRetrySecondsArgumentId, "The number of seconds before attempt to download a resource fails with a network related error", false, false, SecondsToString( defaultParams.downloadRetrySeconds ) );
6364

6465
AddArgument( m_indexFolderArgumentId, "The folder in which to place indexes generated for patch files.", false, false, defaultParams.indexFolder.string() );
66+
67+
AddArgumentFlag( m_skipCompressionCalculation, "Set skip compression calculations on patches." );
6568
}
6669

6770
bool CreatePatchCliOperation::Execute( std::string& returnErrorMessage ) const
@@ -175,13 +178,24 @@ bool CreatePatchCliOperation::Execute( std::string& returnErrorMessage ) const
175178

176179
createPatchParams.downloadRetrySeconds = std::chrono::seconds( retrySeconds );
177180

181+
createPatchParams.indexFolder = m_argumentParser->get( m_indexFolderArgumentId );
182+
183+
bool skipCompressionCalculation = m_argumentParser->get<bool>( m_skipCompressionCalculation );
184+
185+
if (skipCompressionCalculation && createPatchParams.resourcePatchBinaryDestinationSettings.destinationType == CarbonResources::ResourceDestinationType::REMOTE_CDN)
186+
{
187+
returnErrorMessage = "Cannot skip compression when patch desination type is REMOTE_CDN.";
188+
189+
return false;
190+
}
191+
192+
createPatchParams.calculateCompressions = !skipCompressionCalculation;
193+
178194
if( s_verbosityLevel != CarbonResources::StatusLevel::OFF )
179195
{
180196
PrintStartBanner( previousResourceGroupParams, nextResourceGroupParams, createPatchParams );
181197
}
182198

183-
createPatchParams.indexFolder = m_argumentParser->get( m_indexFolderArgumentId );
184-
185199
return CreatePatch( previousResourceGroupParams, nextResourceGroupParams, createPatchParams );
186200
}
187201

@@ -229,6 +243,15 @@ void CreatePatchCliOperation::PrintStartBanner( const CarbonResources::ResourceG
229243

230244
std::cout << "Index File Folder: " << createPatchParams.indexFolder << std::endl;
231245

246+
if( createPatchParams.calculateCompressions )
247+
{
248+
std::cout << "Calculate Compression: Off" << std::endl;
249+
}
250+
else
251+
{
252+
std::cout << "Calculate Compression: On" << std::endl;
253+
}
254+
232255
std::cout << "----------------------------\n"
233256
<< std::endl;
234257
}

‎cli/src/CreatePatchCliOperation.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ class CreatePatchCliOperation : public CliOperation
5555
std::string m_downloadRetrySecondsArgumentId;
5656

5757
std::string m_indexFolderArgumentId;
58+
59+
std::string m_skipCompressionCalculation;
5860
};
5961

6062
#endif // CreatePatchCliOperation_H

‎cli/src/CreateResourceGroupCliOperation.cpp‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ CreateResourceGroupCliOperation::CreateResourceGroupCliOperation() :
1111
m_createResourceGroupPathArgumentId( "input-directory" ),
1212
m_createResourceGroupOutputFileArgumentId( "--output-file" ),
1313
m_createResourceGroupDocumentVersionArgumentId( "--document-version" ),
14-
m_createResourceGroupResourcePrefixArgumentId( "--resource-prefix" )
14+
m_createResourceGroupResourcePrefixArgumentId( "--resource-prefix" ),
15+
m_createResourceGroupSkipCompressionCalculation( "--skip-compression" )
1516
{
1617

1718
AddRequiredPositionalArgument( m_createResourceGroupPathArgumentId, "Base directory to create resource group from." );
@@ -28,6 +29,8 @@ CreateResourceGroupCliOperation::CreateResourceGroupCliOperation() :
2829
AddArgument( m_createResourceGroupDocumentVersionArgumentId, "Document version for created resource group.", false, false, VersionToString( defaultImportParams.outputDocumentVersion ) );
2930

3031
AddArgument( m_createResourceGroupResourcePrefixArgumentId, R"(Optional resource path prefix, such as "res" or "app")", false, false, "" );
32+
33+
AddArgumentFlag( m_createResourceGroupSkipCompressionCalculation, "Set skip compression calculations on resources." );
3134
}
3235

3336
bool CreateResourceGroupCliOperation::Execute( std::string& returnErrorMessage ) const
@@ -39,10 +42,12 @@ bool CreateResourceGroupCliOperation::Execute( std::string& returnErrorMessage )
3942
std::string version = m_argumentParser->get( m_createResourceGroupDocumentVersionArgumentId );
4043

4144
std::string resourcePrefix = m_argumentParser->get( m_createResourceGroupResourcePrefixArgumentId );
45+
46+
bool skipCompressionCalculation = m_argumentParser->get<bool>( m_createResourceGroupSkipCompressionCalculation );
4247

4348
CarbonResources::Version documentVersion;
4449

45-
PrintStartBanner( inputDirectory, outputFile, version, resourcePrefix );
50+
PrintStartBanner( inputDirectory, outputFile, version, resourcePrefix, skipCompressionCalculation );
4651

4752
bool versionIsValid = ParseDocumentVersion( version, documentVersion );
4853

@@ -52,10 +57,10 @@ bool CreateResourceGroupCliOperation::Execute( std::string& returnErrorMessage )
5257

5358
return false;
5459
}
55-
return CreateResourceGroup( inputDirectory, outputFile, documentVersion, resourcePrefix );
60+
return CreateResourceGroup( inputDirectory, outputFile, documentVersion, resourcePrefix, skipCompressionCalculation );
5661
}
5762

58-
void CreateResourceGroupCliOperation::PrintStartBanner( const std::filesystem::path& inputDirectory, const std::filesystem::path& outputFile, const std::string& version, const std::string& resourcePrefix ) const
63+
void CreateResourceGroupCliOperation::PrintStartBanner( const std::filesystem::path& inputDirectory, const std::filesystem::path& outputFile, const std::string& version, const std::string& resourcePrefix, bool skipCompressionCalculation ) const
5964
{
6065
if( s_verbosityLevel == CarbonResources::StatusLevel::OFF )
6166
{
@@ -74,11 +79,20 @@ void CreateResourceGroupCliOperation::PrintStartBanner( const std::filesystem::p
7479

7580
std::cout << "Resource Prefix: " << resourcePrefix << std::endl;
7681

82+
if (skipCompressionCalculation)
83+
{
84+
std::cout << "Calculate Compression: Off" << std::endl;
85+
}
86+
else
87+
{
88+
std::cout << "Calculate Compression: On" << std::endl;
89+
}
90+
7791
std::cout << "----------------------------\n"
7892
<< std::endl;
7993
}
8094

81-
bool CreateResourceGroupCliOperation::CreateResourceGroup( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputFile, CarbonResources::Version documentVersion, const std::string& resourcePrefix ) const
95+
bool CreateResourceGroupCliOperation::CreateResourceGroup( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputFile, CarbonResources::Version documentVersion, const std::string& resourcePrefix, bool skipCompressionCalculation ) const
8296
{
8397
CarbonResources::ResourceGroup resourceGroup;
8498

@@ -92,6 +106,7 @@ bool CreateResourceGroupCliOperation::CreateResourceGroup( const std::filesystem
92106

93107
createResourceGroupParams.statusCallback = GetStatusCallback();
94108

109+
createResourceGroupParams.calculateCompressions = !skipCompressionCalculation;
95110

96111
if( createResourceGroupParams.statusCallback )
97112
{

‎cli/src/CreateResourceGroupCliOperation.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ class CreateResourceGroupCliOperation : public CliOperation
1616
virtual bool Execute( std::string& returnErrorMessage ) const final;
1717

1818
private:
19-
void PrintStartBanner( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputDirectory, const std::string& version, const std::string& resourcePrefix ) const;
20-
bool CreateResourceGroup( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputFile, CarbonResources::Version documentVersion, const std::string& resourcePrefix ) const;
19+
void PrintStartBanner( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputDirectory, const std::string& version, const std::string& resourcePrefix, bool skipCompressionCalculation ) const;
20+
bool CreateResourceGroup( const std::filesystem::path& inputDirectory, const std::filesystem::path& resourceGroupOutputFile, CarbonResources::Version documentVersion, const std::string& resourcePrefix, bool skipCompressionCalculation ) const;
2121

2222
private:
2323
std::string m_createResourceGroupPathArgumentId;
@@ -27,6 +27,8 @@ class CreateResourceGroupCliOperation : public CliOperation
2727
std::string m_createResourceGroupDocumentVersionArgumentId;
2828

2929
std::string m_createResourceGroupResourcePrefixArgumentId;
30+
31+
std::string m_createResourceGroupSkipCompressionCalculation;
3032
};
3133

3234
#endif // CreateResourceGroupCliOperation_H

‎include/ResourceGroup.h‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ struct ResourceDestinationSettings
6666
* Where to save the resulting BundleResourceGroup
6767
* @var BundleCreateParams::statusCallback
6868
* Optional status function callback. Callback is triggered at key status update events.
69+
* @var BundleCreateParams::downloadRetrySeconds
70+
* Delay before a failed download is retried (seconds)
71+
* @var BundleCreateParams::calculateCompressions
72+
* Specifies if compression will be calculated for the generated bundle chunks
6973
*/
7074
struct BundleCreateParams
7175
{
@@ -86,6 +90,8 @@ struct BundleCreateParams
8690
StatusCallback statusCallback = nullptr;
8791

8892
std::chrono::seconds downloadRetrySeconds{ 120 };
93+
94+
bool calculateCompressions = true;
8995
};
9096

9197
/** @struct PatchCreateParams
@@ -110,6 +116,12 @@ struct BundleCreateParams
110116
* Where the produced PatchResourceGroup will be saved.
111117
* @var PatchCreateParams::statusCallback
112118
* Optional status function callback. Callback is triggered at key status update events.
119+
* @var PatchCreateParams::downloadRetrySeconds
120+
* Delay before a failed download is retried (seconds)
121+
* @var PatchCreateParams::indexFolder
122+
* Directory to store index calculation files during patch creation.
123+
* @var BundleCreateParams::calculateCompressions
124+
* Specifies if compression will be calculated for the generated bundle chunks
113125
*/
114126
struct PatchCreateParams
115127
{
@@ -136,6 +148,8 @@ struct PatchCreateParams
136148
std::chrono::seconds downloadRetrySeconds{ 120 };
137149

138150
std::filesystem::path indexFolder = std::filesystem::temp_directory_path() / "carbonResources" / "chunkIndexes";
151+
152+
bool calculateCompressions = true;
139153
};
140154

141155
/** @struct ResourceGroupImportFromFileParams
@@ -180,6 +194,10 @@ struct ResourceGroupExportToFileParams
180194
* Document version to output. By default this will be latest supported by the library.
181195
* @var CreateResourceGroupFromDirectoryParams::statusCallback
182196
* Optional status function callback. Callback is triggered at key status update events.
197+
* @var CreateResourceGroupFromDirectoryParams::resourcePrefix
198+
* Resource prefix setting, e.g. res.
199+
* @var BundleCreateParams::calculateCompressions
200+
* Specifies if compression will be calculated for the generated bundle chunks
183201
*/
184202
struct CreateResourceGroupFromDirectoryParams
185203
{
@@ -191,7 +209,9 @@ struct CreateResourceGroupFromDirectoryParams
191209

192210
StatusCallback statusCallback = nullptr;
193211

194-
std::string resourcePrefix;
212+
std::string resourcePrefix = "";
213+
214+
bool calculateCompressions = true;
195215
};
196216

197217
/** @struct ResourceGroupMergeParams

‎src/ResourceGroupImpl.cpp‎

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour
111111
return getResourceDataResult;
112112
}
113113

114-
Result setParametersFromDataResult = resource->SetParametersFromData( resourceData );
114+
Result setParametersFromDataResult = resource->SetParametersFromData( resourceData, params.calculateCompressions );
115115

116116
if( setParametersFromDataResult.type != ResultType::SUCCESS )
117117
{
@@ -135,10 +135,13 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour
135135

136136
ResourceTools::FileDataStreamIn fileStreamIn( params.resourceStreamThreshold );
137137

138-
if( !compressionStream.Start() )
139-
{
140-
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
141-
}
138+
if (params.calculateCompressions)
139+
{
140+
if( !compressionStream.Start() )
141+
{
142+
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
143+
}
144+
}
142145

143146
if( !fileStreamIn.StartRead( entry.path() ) )
144147
{
@@ -168,22 +171,28 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour
168171
return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM };
169172
}
170173

171-
if( !( compressionStream << &fileData ) )
174+
if( params.calculateCompressions )
172175
{
173-
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
176+
if( !( compressionStream << &fileData ) )
177+
{
178+
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
179+
}
174180
}
175181

176182
compressedDataSize += compressedData.size();
177183
compressedData.clear();
178184
}
179185

180-
if( !compressionStream.Finish() )
186+
if( params.calculateCompressions )
181187
{
182-
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
183-
}
188+
if( !compressionStream.Finish() )
189+
{
190+
return Result{ ResultType::FAILED_TO_COMPRESS_DATA };
191+
}
184192

185-
compressedDataSize += compressedData.size();
186-
compressedData.clear();
193+
compressedDataSize += compressedData.size();
194+
compressedData.clear();
195+
}
187196

188197
std::string checksum;
189198

@@ -228,6 +237,11 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour
228237
}
229238
}
230239

240+
if (!params.calculateCompressions)
241+
{
242+
m_totalResourcesSizeCompressed.Reset();
243+
}
244+
231245
if( params.statusCallback )
232246
{
233247
params.statusCallback( CarbonResources::StatusLevel::PROCEDURE, CarbonResources::StatusProgressType::PERCENTAGE, 100, "Resource group successfully created from directory" );
@@ -671,7 +685,7 @@ Result ResourceGroup::ResourceGroupImpl::ImportFromYaml( YAML::Node& resourceGro
671685
YAML::Node totalResourceSizeCompressedNode = resourceGroupFile[m_totalResourcesSizeCompressed.GetTag()];
672686
if( !totalResourceSizeCompressedNode.IsDefined() )
673687
{
674-
return Result{ ResultType::MALFORMED_RESOURCE_GROUP };
688+
m_totalResourcesSizeCompressed.Reset();
675689
}
676690

677691
YAML::Node totalResourceSizeUncompressedNode = resourceGroupFile[m_totalResourcesSizeUncompressed.GetTag()];
@@ -777,8 +791,14 @@ Result ResourceGroup::ResourceGroupImpl::ExportYaml( const VersionInternal& outp
777791
out << YAML::Key << m_numberOfResources.GetTag();
778792
out << YAML::Value << m_numberOfResources.GetValue();
779793

780-
out << YAML::Key << m_totalResourcesSizeCompressed.GetTag();
781-
out << YAML::Value << m_totalResourcesSizeCompressed.GetValue();
794+
if (m_totalResourcesSizeCompressed.HasValue())
795+
{
796+
uintmax_t compressedSize = m_totalResourcesSizeCompressed.GetValue();
797+
798+
out << YAML::Key << m_totalResourcesSizeCompressed.GetTag();
799+
out << YAML::Value << compressedSize;
800+
801+
}
782802

783803
out << YAML::Key << m_totalResourcesSizeUncompressed.GetTag();
784804
out << YAML::Value << m_totalResourcesSizeUncompressed.GetValue();
@@ -1620,7 +1640,7 @@ Result ResourceGroup::ResourceGroupImpl::CreatePatch( const PatchCreateParams& p
16201640
patchSourceOffset += patchSourceOffsetDelta;
16211641
if( !patchData.empty() )
16221642
{
1623-
Result setParametersFromDataResult = patchResource->SetParametersFromData( patchData );
1643+
Result setParametersFromDataResult = patchResource->SetParametersFromData( patchData, params.calculateCompressions );
16241644

16251645
if( setParametersFromDataResult.type != ResultType::SUCCESS )
16261646
{
@@ -1771,13 +1791,11 @@ Result ResourceGroup::ResourceGroupImpl::AddResource( ResourceInfo* resource )
17711791

17721792
Result resourceGetCompressedSizeResult = resource->GetCompressedSize( resourceCompressedSize );
17731793

1774-
if( resourceGetCompressedSizeResult.type != ResultType::SUCCESS )
1794+
if( resourceGetCompressedSizeResult.type == ResultType::SUCCESS )
17751795
{
1776-
return resourceGetCompressedSizeResult;
1796+
m_totalResourcesSizeCompressed = m_totalResourcesSizeCompressed.GetValue() + resourceCompressedSize;
17771797
}
17781798

1779-
m_totalResourcesSizeCompressed = m_totalResourcesSizeCompressed.GetValue() + resourceCompressedSize;
1780-
17811799
return Result{ ResultType::SUCCESS };
17821800
}
17831801

0 commit comments

Comments
 (0)