diff --git a/crates/core/common/src/config.rs b/crates/core/common/src/config.rs index 50f9903b1..7762f400f 100644 --- a/crates/core/common/src/config.rs +++ b/crates/core/common/src/config.rs @@ -147,20 +147,19 @@ impl Default for CompactorConfig { pub struct CompactionAlgorithmConfig { /// Base cooldown duration in seconds (default: 1024.0) pub cooldown_duration: ConfigDuration<1024>, - /// Eager compaction limits (flattened fields: overflow, bytes, rows) - #[serde( - flatten, - default = "SizeLimitConfig::default_eager_limit", - deserialize_with = "SizeLimitConfig::deserialize_eager_limit" - )] - pub eager_compaction_limit: SizeLimitConfig, + /// Maximum generation for eager compaction, that is, cooldowns only apply + /// for generations above this value. + /// + /// Default: 0. To disable eager compaction, set to -1. + pub max_eager_generation: i64, } +#[allow(clippy::derivable_impls)] impl Default for CompactionAlgorithmConfig { fn default() -> Self { Self { cooldown_duration: ConfigDuration::default(), - eager_compaction_limit: SizeLimitConfig::default_eager_limit(), + max_eager_generation: 0, } } } @@ -200,31 +199,6 @@ struct SizeLimitHelper { } impl SizeLimitConfig { - fn default_eager_limit() -> Self { - Self { - bytes: 0, - blocks: 0, - ..Default::default() - } - } - - fn deserialize_eager_limit<'de, D>(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let helper = SizeLimitHelper::deserialize(deserializer)?; - - let mut this = Self::default_eager_limit(); - - helper - .overflow - .inspect(|overflow| this.overflow = *overflow); - helper.bytes.inspect(|bytes| this.bytes = *bytes); - helper.rows.inspect(|rows| this.rows = *rows); - - Ok(this) - } - fn default_upper_limit() -> Self { Self { ..Default::default() diff --git a/crates/core/dump/src/compaction/algorithm.md b/crates/core/dump/src/compaction/algorithm.md index b264b8364..a61f47c67 100644 --- a/crates/core/dump/src/compaction/algorithm.md +++ b/crates/core/dump/src/compaction/algorithm.md @@ -16,12 +16,7 @@ The limit to the size of data permitted in a single partition file. Any number o - **_Eager Compaction Limit_** †: - The upper limit for eager compaction. To activate, at least one of the sub parameters/limits must be set. A candidate partition is considered to exceed this limit if any of the set limits are met or exceeded. This configuration only defines when eager compaction is complete for a partition. The sub parameters are: - - bytes † - - blocks † - - rows † - - file count † - - generation † + The upper generational limit for eager compaction. This is set in the `max_eager_generation` config value. Default of `0`, meaning only newly written files are eagerly compacted. \* Required † Optional @@ -31,32 +26,26 @@ The limit to the size of data permitted in a single partition file. Any number o With 1 required and 2 optional configurable top-level parameters, users can define 5 distinct compaction schemes: -1. **_[Strict Generationally-Tiered Compaction](#generationally-tiered-compaction)_**: - The default compaction configuration when the compactor is enabled. All candidate segments and compaction groups are considered _Hot_ in this scheme. +1. **Strict Eager Compaction**: + To simply compact as eagerly as possible, set a target partition size, disable `max_eager_generation` by setting it to `-1` and disable cooldown by setting a `cooldown_duration` of zero. All candidate segments and compaction groups are considered _Cold_ in this scheme. It is configured by setting: - _Target Partition Size_ -2. **_[Relaxed Generationally-Tiered Compaction (Cooldown)](#scaled-cooldown-tiered-compaction)_**: -Introduces _Scaled, Cooldown-Tiered Compaction_ as described above. Candidate segments and compaction groups are conditionally considered _Hot_ or _Cold_ depending on the expiration of their cooldown period. + - `max_eager_generation = -1` + - Cooldown to zero +2. **Cooldown Compaction**: +Compaction is controlled based on a set cooldown value. Candidate segments and compaction groups are conditionally considered _Hot_ or _Cold_ depending on the expiration of their cooldown period. When compacting, segments will be scanned in chain-time order. When a cold segment is found, a compaction group is initiated and following segments will be added to it up to the partition size. It is configured by setting: - _Target Partition Size_ - _Base Cooldown Duration_ -3. **_[Relaxed Generationally-Tiered Compaction (Eager)](#tiered-eager-compaction)_**: -Introduces _Eager Compaction_ for compaction groups smaller than the eager compaction limit. Candidate segments and compaction groups are conditionally considered _Hot_ or _Live_ depending on their size at the time of consideration. Effectively raises the average size of the smallest file at the end of any given compaction cycle. -It is configured by setting: - - _Target Partition Size_ - - _Eager Compaction Limit_ -4. **_Hybrid Compaction_**: -A combination of both _Relaxed Generationally-Tiered Compaction_ schemes. Candidate segments and compaction groups are conditionally considered _Live_, _Hot_, or _Cold_ depending on their size and age at the time of consideration. Setting +3. **_Hybrid Compaction_**: +While cooldown is good to prevent frequently re-compacting the same data, if chain head files are immediately set on cooldown that might mean a lot of tiny files waiting to be compacted. To eagerly compact recent files while still respecting a cooldown for older ones, combine the cooldown with `max_eager_generation`. Candidate segments and compaction groups are conditionally considered _Live_, _Hot_, or _Cold_ depending on their generation and age at the time of consideration. Settings: - _Target Partition Size_ - _Base Cooldown Duration_ - - _Eager Compaction Limit_ -5. **_Strict Eager Compaction_**: -A special case of either of the previous two schemes where the _Eager Compaction Limit_ is set greater than or equal to _Target Partition Size_. All candidate segments are considered _Live_ in this scheme. -It is configured by setting: - - _Target Partition Size_ - - _Eager Compaction Limit_ + - `max_eager_generation` -### Generationally-Tiered Compaction +### Why not Generationally-Tiered Compaction? + +Generationally tiered compaction is typically employed by LSMs, and could be described as: >Where membership is predicated on all candidate segment files to be contiguous and share the same Generation value. @@ -64,27 +53,9 @@ The file writing and compaction facilities track the _Generation_ of each new se The base logic of the compaction algorithm will only compact candidate segment files if they are both contiguous and share the same _Generation_ value. This organizes compaction operations into a binary tree structure when the compaction ratio $k = 2$. -### Scaled, Cooldown-Tiered Compaction - -> Where membership is predicated on a segment being promoted to accumulator after some _cooldown_ period has elapsed since it was written; _cooldown_ is an interval multiple of the segment's Generation value. - The binary tree structure of generationally tiered compaction provides excellent scaling but at the cost of leftover, not compacted segments. To illustrate how the number of files scales at chain head we can represent a segment and its generation like `[ g ]` for example we visualize the state of table files after 81,935 compaction cycles (where $k=2$ and the target size is $2^{15}$ blocks) like: `[ 15 ] [ 15 ] [ 14 ] [ 3 ] [ 2 ] [ 0 ]`. In this example the overall un-compacted file length is 4 with a generational height of 14. We can calculate the number of un-compacted files at any compaction cycle $n$ and target generation $g$ as $popcount\left(n\mod2^{g+1}\right)$ which gives us a worse case scenario every $2^g-1$ cycles where the remaining un-compacted file count is equal to $g$. This is not ideal if the goal is to minimize the number of small files. -The new compaction algorithm introduces a cooldown period for all segments after which the file is promoted as an "accumulator" that may compact all segments between it and the chain head. The cooldown for a segment is calculated by multiplying a configured `base_duration` by the generation of the segment. - -This relaxation of Generationally-Tiered Compaction mitigates its worst case scenario while maintaining overall $\log_2(N)$ scaling when configured properly. It also enables the compaction of segments that are ingested to bridge gaps between non-contiguous segment chains which is useful for restarting historical data dumps for example. - -### Tiered Eager Compaction - -> Where segments eagerly compact into an accumulator segment until a size threshold is met after which the compaction scheme described above takes precedence. - -Eager compaction is not necessarily pathological and must be available as a compaction strategy. The new compaction algorithm introduces an optional, configurable, secondary size threshold for eager compaction. When set, all segments eagerly compact into an accumulator until it exceeds the threshold. - -This extends the compaction algorithm with the following scenarios based on the value of the secondary threshold: - -- if set greater than or equal to the target partition file size, the algorithm is effectively Eager Compaction -- else if set to a non-unbounded value, increases the minimum segment size while performing the above described compaction strategy. -- else if not set, just perform the above described compaction strategy. +The cooldown-based compaction algorithm avoids this worst-case behaviour by introducing a period for all segments after which the file is promoted as an "accumulator" that may compact all segments between it and the chain head. It also enables the compaction of segments that are ingested to bridge gaps between non-contiguous segment chains which is useful for restarting historical data dumps for example. ## Addtional Terms @@ -92,8 +63,8 @@ This extends the compaction algorithm with the following scenarios based on the By combining the cooldown and eager compaction tier concepts, useful categories emerge to describe the state of a given segment: -- **_Live_**: a recently ingested file or collection thereof (i.e. smaller than the secondary threshold) -- **_Hot_**: a segment that has been recently compacted and needs to stabilize before compacting again +- **_Live_**: a recently ingested file or collection thereof (i.e. smaller than the maximum eager generation). +- **_Hot_**: a segment that has been recently compacted and needs to stabilize before compacting again. - **_Cold_**: a historical or previously compacted segment that may act as an accumulator. -Each category or tier corresponds to a different compaction strategy, providing (I think) the necessary flexibility to handle all kinds of use cases. +Each category or tier corresponds to a different compaction strategy, hopefully providing the necessary flexibility to handle all kinds of use cases. diff --git a/crates/core/dump/src/compaction/algorithm.rs b/crates/core/dump/src/compaction/algorithm.rs index 5e6f545f0..85261b638 100644 --- a/crates/core/dump/src/compaction/algorithm.rs +++ b/crates/core/dump/src/compaction/algorithm.rs @@ -1,6 +1,6 @@ use std::{ fmt::{Debug, Display, Formatter}, - ops::{Deref, Not}, + ops::Not, time::Duration, }; @@ -23,9 +23,8 @@ use crate::compaction::{compactor::CompactionGroup, plan::CompactionFile}; /// - `target_partition_size`: The upper bound for segment size limits. /// Files exceeding this limit will not be compacted together. This /// value must be non-unbounded. -/// - `eager_compaction_limit`: The lower bound for segment size limits. This -/// value can be unbounded, indicating no lower limit for compaction. -#[derive(Clone, Copy)] +/// - `max_eager_generation`: Segments up to this generation will not be subject to cooldowns. +#[derive(Clone, Copy, Debug)] pub struct CompactionAlgorithm { /// The amount of time a file must wait before it can be /// compacted with files of different generations. @@ -33,126 +32,63 @@ pub struct CompactionAlgorithm { /// The upper bound for segment size limits. Files exceeding this limit /// will not be compacted together. This value must be non-unbounded. pub target_partition_size: SegmentSizeLimit, - pub eager_compaction_limit: SegmentSizeLimit, + + /// Segments up to this generation will not be subject to cooldowns + pub max_eager_generation: Option, } impl CompactionAlgorithm { - pub fn kind(&self) -> &'static str { - if self.eager_compaction_limit.0 >= self.target_partition_size.0 { - return "Strict Eager Compaction"; - } - - let unbounded_lower = self.eager_compaction_limit.is_unbounded(); - - if unbounded_lower && self.cooldown_duration.is_zero() { - "Strict Generationally-Tiered Compaction" - } else if unbounded_lower { - "Relaxed Generationally-Tiered Compaction" - } else { - "Hybrid Compaction" - } - } - - fn is_live(&self, segment: &SegmentSize) -> TestResult { - self.eager_compaction_limit.is_live(segment) + fn is_live(&self, segment: &SegmentSize) -> bool { + Some(segment.generation) <= self.max_eager_generation } fn is_hot(&self, segment: &SegmentSize) -> TestResult { Cooldown::new(self.cooldown_duration).is_hot(segment.created_at) } - /// Determines the state of a file based on its size and age. - /// - `Live`: The file is within the lower bound limits, if any. - /// - `Hot`: The file has exceeded lower bound limits (if any) - /// but is still within its cooldown period. - /// - `Cold`: The file has exceeded lower bound limits and is outside - /// its cooldown period. + /// Determines the state of a file: + /// - `Live`: Cooldown does not apply to this file, it can be compacted immediately. + /// - `Hot`: The file is still within its cooldown period. + /// - `Cold`: The file is outside its cooldown period. fn file_state(&self, segment: &SegmentSize) -> FileState { let is_live = self.is_live(segment); - if *is_live { + if is_live { return FileState::Live; } let is_hot = self.is_hot(segment); match is_hot { - TestResult::Skipped => FileState::Hot, + // If there is no cooldown, file is always cold. + TestResult::Skipped => FileState::Cold, TestResult::Activated(true) => FileState::Hot, TestResult::Activated(false) => FileState::Cold, } } - /// Predicate function to determine if two files can be compacted together. + /// Predicate function to determine if: + /// - When a group is empty, if the candidate can start a new group. + /// - When a group is started, if the candidate can be added to it. /// - /// Returns `true` if the candidate file can be compacted with the group, - /// `false` otherwise. The decision is based on the combined size of the - /// files, their states (Live, Hot, Cold), and their generations. + /// The current algorithm is: + /// - If the file is `Hot`, it cannot start a new group. + /// - If a group has been started, it will accept files up to the target size, regardless of file state. pub fn predicate(&self, group: &CompactionGroup, candidate: &CompactionFile) -> bool { - let state = self - .file_state(&group.size) - .max(self.file_state(&candidate.size)); + if group.is_empty() && self.file_state(&candidate.size) == FileState::Hot { + return false; + } // Check if combining sizes exceeds upper bound. - let (size_exceeded, _, _) = self + let size_exceeded = self .target_partition_size .is_exceeded(&(candidate.size + group.size)); - if state == FileState::Live { - // For live files, only compact if size limit is not exceeded. - // If it's the tail file, also require length limit to be exceeded - // (for cases where a minimum number of segments is desired before compaction). - *size_exceeded - } else if state == FileState::Hot { - // For hot files, only compact if size limit is not exceeded, - // and both files share the same generation. - group.size.generation == candidate.size.generation && !*size_exceeded - } else { - // For cold files, compact regardless of generation, - // as long as size limit is not exceeded. - !*size_exceeded - } - } -} - -impl Debug for CompactionAlgorithm { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct(self.kind()) - .field("base_duration", &self.cooldown_duration) - .field("upper_bound", &self.target_partition_size) - .field("lower_bound", &self.eager_compaction_limit) - .finish() - } -} + match size_exceeded { + TestResult::Activated(exceeded) => !exceeded, -impl Display for CompactionAlgorithm { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let kind = self.kind().trim_end_matches(" Compaction"); - match kind { - "Eager" => write!( - f, - "{{ kind: {}, upper_bound: {} }}", - kind, self.target_partition_size - ), - "Exponential" => write!( - f, - "{{ kind: {}, upper_bound: {} }}", - kind, self.target_partition_size - ), - "Relaxed Exponential" => write!( - f, - "{{ kind: {}, base_duration: {:?}, upper_bound: {} }}", - kind, self.cooldown_duration, self.target_partition_size - ), - "Hybrid" => write!( - f, - "{{ kind: {}, base_duration: {:?}, upper_bound: {}, lower_bound: {} }}", - kind, - self.cooldown_duration, - self.target_partition_size, - self.eager_compaction_limit - ), - _ => unreachable!("Unexpected compaction algorithm kind"), + // If all limits are zero, assume compaction is disabled. + TestResult::Skipped => false, } } } @@ -162,9 +98,14 @@ impl<'a> From<&'a ParquetConfig> for CompactionAlgorithm { CompactionAlgorithm { cooldown_duration: config.compactor.algorithm.cooldown_duration.clone().into(), target_partition_size: SegmentSizeLimit::from(&config.target_size), - eager_compaction_limit: SegmentSizeLimit::from( - &config.compactor.algorithm.eager_compaction_limit, - ), + max_eager_generation: { + let generation = config.compactor.algorithm.max_eager_generation; + if generation < 0 { + None + } else { + Some(Generation::from(generation as u64)) + } + }, } } } @@ -217,49 +158,6 @@ pub enum FileState { Cold, } -impl Ord for FileState { - /// Defines a partial ordering for `FileState`: - /// - `Cold` is greater than both `Live` and `Hot`. - /// - `Hot` is greater than `Live` but less than `Cold`. - /// - `Live` is less than both `Hot` and `Cold`. - /// - States of the same type are considered equal. - /// - /// This ordering is used to determine the strictest state - /// when comparing two files, where `Cold` is the strictest - /// and `Live` is the least strict. - /// - /// # Examples - /// ``` - /// use dump::compaction::algorithm::FileState; - /// assert!(FileState::Cold > FileState::Hot); - /// assert!(FileState::Cold > FileState::Live); - /// assert!(FileState::Hot > FileState::Live); - /// assert!(FileState::Hot < FileState::Cold); - /// assert!(FileState::Live < FileState::Hot); - /// assert!(FileState::Live < FileState::Cold); - /// assert_eq!(FileState::Cold, FileState::Cold); - /// assert_eq!(FileState::Hot, FileState::Hot); - /// assert_eq!(FileState::Live, FileState::Live); - /// ``` - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - match (self, other) { - (FileState::Cold, FileState::Cold) - | (FileState::Live, FileState::Live) - | (FileState::Hot, FileState::Hot) => std::cmp::Ordering::Equal, - (FileState::Cold, _) => std::cmp::Ordering::Greater, - (_, FileState::Cold) => std::cmp::Ordering::Less, - (FileState::Hot, _) => std::cmp::Ordering::Greater, - (_, FileState::Hot) => std::cmp::Ordering::Less, - } - } -} - -impl PartialOrd for FileState { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - /// Represents configurable size limits for file compaction operations. /// /// `SegmentSizeLimit` wraps a `SegmentSize` to define thresholds that determine when files @@ -332,11 +230,8 @@ impl SegmentSizeLimit { /// - `segment`: [`SegmentSize`] - The segment to check against the limits /// /// ## Returns - /// [`SizeCheckResult`] - A tuple of three [`TestResult`] values: - /// 1. Combined result for blocks, bytes, and rows dimensions - /// 2. Result for the length (file count) dimension - /// 3. Result for the generation dimension - pub fn is_exceeded(&self, segment: &SegmentSize) -> SizeCheckResult { + /// [`TestResult`] for combined `or` result of blocks, bytes, and rows limits + pub fn is_exceeded(&self, segment: &SegmentSize) -> TestResult { let blocks_ge: TestResult = self .0 .blocks @@ -361,32 +256,7 @@ impl SegmentSizeLimit { .then_some(segment.rows.ge(&self.1.soft_limit(self.0.rows))) .into(); - let generation_ge: TestResult = self - .0 - .generation - .is_compacted() - .then_some(segment.generation.ge(&self.1.soft_limit(self.0.generation))) - .into(); - - let length_ge: TestResult = self - .0 - .length - .ne(&0) - .then_some(segment.length.ge(&self.1.soft_limit(self.0.length))) - .into(); - - (blocks_ge.or(bytes_ge).or(rows_ge), length_ge, generation_ge) - } - - pub fn is_live(&self, segment: &SegmentSize) -> TestResult { - let (size_exceeded, length_exceeded, generation_exceeded) = - Self::is_exceeded(self, segment); - // A segment is considered live if it does not exceed size limits, - // length limits, and generation limits (if any are set). - size_exceeded - .or(length_exceeded) - .or(generation_exceeded) - .not() + blocks_ge.or(bytes_ge).or(rows_ge) } } @@ -421,14 +291,6 @@ impl<'a> From<&'a SizeLimitConfig> for SegmentSizeLimit { } } -/// The result of checking if a segment exceeds size limits. -/// -/// This is a tuple of three [`TestResult`] values: -/// 1. Combined result for blocks, bytes, and rows dimensions -/// 2. Result for the length (file count) dimension -/// 3. Result for the generation dimension -pub type SizeCheckResult = (TestResult, TestResult, TestResult); - /// Three-valued logic to represent boolean tests that may be skipped. /// /// It differs from kleene algebra in that the skipped value is @@ -504,17 +366,6 @@ impl TestResult { } } -impl Deref for TestResult { - type Target = bool; - - fn deref(&self) -> &Self::Target { - match self { - TestResult::Activated(a) => a, - TestResult::Skipped => &false, - } - } -} - impl From> for TestResult { fn from(value: Option) -> Self { match value { @@ -636,12 +487,4 @@ mod tests { assert_eq!(A(false).not(), A(true)); assert_eq!(S.not(), S); } - - #[test] - fn test_result_deref() { - use super::TestResult::{Activated as A, Skipped as S}; - assert!(*A(true)); - assert!(!*A(false)); - assert!(!*S); - } } diff --git a/crates/core/dump/src/compaction/compactor.rs b/crates/core/dump/src/compaction/compactor.rs index 8654f7200..4806d7c58 100644 --- a/crates/core/dump/src/compaction/compactor.rs +++ b/crates/core/dump/src/compaction/compactor.rs @@ -61,9 +61,9 @@ impl Debug for Compactor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "Compactor {{ table: {}, algorithm: {} }}", + "Compactor {{ table: {}, algorithm: {:?} }}", self.table.table_ref(), - self.opts.compactor.algorithm.kind() + self.opts.compactor.algorithm, ) } } diff --git a/crates/core/dump/src/compaction/plan.rs b/crates/core/dump/src/compaction/plan.rs index 90505f6d6..baa6d620c 100644 --- a/crates/core/dump/src/compaction/plan.rs +++ b/crates/core/dump/src/compaction/plan.rs @@ -44,7 +44,6 @@ pub struct CompactionFile { pub range: BlockRange, pub sendable_stream: SendableRecordBatchStream, pub size: SegmentSize, - pub is_tail: bool, } impl CompactionFile { @@ -52,7 +51,6 @@ impl CompactionFile { reader_factory: Arc, partition_index: usize, segment: &Segment, - is_tail: bool, ) -> CompactionResult { let file_id = segment.id; let range = segment.range.clone(); @@ -85,7 +83,6 @@ impl CompactionFile { range, size, sendable_stream, - is_tail, }; Ok(compaction_item) @@ -119,8 +116,6 @@ pub struct CompactionPlan<'a> { table: Arc, /// The current group of files being built for compaction. current_group: CompactionGroup, - /// The file currently being added to the current group. - current_file: Option, /// The next candidate file to consider adding to the current group. current_candidate: Option, /// Indicates whether the stream has been fully processed. @@ -151,8 +146,7 @@ impl<'a> CompactionPlan<'a> { .enumerate() .map(move |(partition_index, segment)| { let reader_factory = Arc::clone(&reader_factory); - let is_tail = partition_index == size - 1; - CompactionFile::try_new(reader_factory, partition_index, segment, is_tail) + CompactionFile::try_new(reader_factory, partition_index, segment) .map_err(CompactorError::from) }) .buffered(opts.compactor.metadata_concurrency) @@ -165,7 +159,6 @@ impl<'a> CompactionPlan<'a> { metrics: metrics.as_ref().map(Arc::clone), table: Arc::clone(table.physical_table()), current_group, - current_file: None, current_candidate: None, done: false, group_count: 0, @@ -188,20 +181,18 @@ impl<'a> Stream for CompactionPlan<'a> { // If we're done processing files, return None. if this.done { break None; - // If we have a current file, add it to the current group and continue. - } else if let Some(current_file) = this.current_file.take() { - this.current_group.push(current_file); // If we have a current candidate, check if it can be added to the current group. } else if let Some(candidate) = this.current_candidate.take() { // If it can, update the current file and continue. if algorithm.predicate(&this.current_group, &candidate) { - this.current_file = Some(candidate); + this.current_group.push(candidate); // If it can't, and the current group is empty or has a single file, // start a new group with the candidate as the current file. } else if this.current_group.is_empty_or_singleton() { - this.current_file = Some(candidate); this.current_group = CompactionGroup::new_empty(&this.opts, &this.metrics, &this.table); + // Requeue the candidate so the predicate is re-evaluated against the fresh group. + this.current_candidate = Some(candidate); // If it can't, and the current group has multiple files, // yield the current group and start a new group with the // candidate as the current file. diff --git a/docs/config.sample.toml b/docs/config.sample.toml index 40cb6cd88..7a4f045a5 100644 --- a/docs/config.sample.toml +++ b/docs/config.sample.toml @@ -84,10 +84,11 @@ url = "postgres://" # metadata_concurrency = 2 # Max concurrent metadata operations (default: 2) # write_concurrency = 2 # Max concurrent compaction write operations (default: 2) # min_interval = 1.0 # Interval in seconds to run the compactor (default: 1.0) -# cooldown_duration = 1024.0 # Base cooldown duration in seconds (default: 1024.0) -# overflow = "1" # Eager compaction overflow (default: "1") -# bytes = 0 # Eager compaction byte threshold (default: 0) -# rows = 0 # Eager compaction row threshold (default: 0) +# cooldown_duration = 1024.0 # Cooldown to wait before re-compacting a file, in seconds (default: 1024.0) +# +# Maximum generation for eager compaction, that is, cooldowns only apply for generations above this value. +# Default: 0. To disable eager compaction, set to -1. +# max_eager_generation = 0 # Garbage collector [writer.collector]