diff --git a/include/silk/util/init.h b/include/silk/util/init.h new file mode 100644 index 0000000..d87404a --- /dev/null +++ b/include/silk/util/init.h @@ -0,0 +1,16 @@ +#pragma once + +namespace silk +{ + +/** + * Initialize all silk singletons in the right order. Call once at process startup before any silk calls + */ +void initialize() noexcept; + +/** + * Tear down silk singletons in reverse order. + */ +void destroy() noexcept; + +} // namespace silk diff --git a/include/silk/util/platform.h b/include/silk/util/platform.h index 945eec5..9be73f5 100644 --- a/include/silk/util/platform.h +++ b/include/silk/util/platform.h @@ -129,10 +129,4 @@ static constexpr uint64_t intHash(uint64_t key) noexcept return key; } -/** - * Initialize librseq. Must be called before any rseq critical sections run. - * Idempotent -- safe to call multiple times. - */ -void initRseq() noexcept; - } // namespace silk diff --git a/include/silk/util/tsc.h b/include/silk/util/tsc.h index 3116fae..28fbafc 100644 --- a/include/silk/util/tsc.h +++ b/include/silk/util/tsc.h @@ -8,7 +8,7 @@ namespace silk /** * TSC frequency query and cycle/nanosecond conversion. * - * Frequency is queried once via CPUID on first use and cached. + * Frequency is queried once via CPUID at initialization and cached. * Conversions use fixed-point multiply+shift - no division on the hot path. */ class Tsc @@ -38,41 +38,24 @@ class Tsc #endif } + /** + * Query TSC frequency, populate the fixed-point conversion factors, and + * assert that the running CPU advertises an invariant TSC. Idempotent. + */ + static void initialize() noexcept; + /** Return the TSC frequency in Hz. */ - static uint64_t getFrequency() noexcept - { - if (frequency == 0) [[unlikely]] - { - init(); - } - return frequency; - } + static uint64_t getFrequency() noexcept { return frequency; } /** Convert TSC cycles to nanoseconds. */ - static uint64_t cyclesToNanoseconds(uint64_t cycles) noexcept - { - if (nsPerCycleFp == 0) [[unlikely]] - { - init(); - } - return (cycles * nsPerCycleFp) >> SHIFT; - } + static uint64_t cyclesToNanoseconds(uint64_t cycles) noexcept { return (cycles * nsPerCycleFp) >> SHIFT; } /** Convert nanoseconds to TSC cycles. */ - static uint64_t nanosecondsToCycles(uint64_t ns) noexcept - { - if (cyclesPerNsFp == 0) [[unlikely]] - { - init(); - } - return (ns * cyclesPerNsFp) >> SHIFT; - } + static uint64_t nanosecondsToCycles(uint64_t ns) noexcept { return (ns * cyclesPerNsFp) >> SHIFT; } private: static constexpr uint32_t SHIFT = 20; - static void init() noexcept; - inline static uint64_t frequency = 0; ///< Hz inline static uint64_t nsPerCycleFp = 0; ///< (1 << SHIFT) * 1'000'000'000 / frequency inline static uint64_t cyclesPerNsFp = 0; ///< frequency * (1 << SHIFT) / 1'000'000'000 diff --git a/src/fibers/benchmarks/main.cpp b/src/fibers/benchmarks/main.cpp index 8336d57..cd8a0d0 100644 --- a/src/fibers/benchmarks/main.cpp +++ b/src/fibers/benchmarks/main.cpp @@ -1,14 +1,11 @@ #include -#include -#include +#include #include int main(int argc, char ** argv) { - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); benchmark::Initialize(&argc, argv); @@ -16,7 +13,6 @@ int main(int argc, char ** argv) benchmark::Shutdown(); silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); + silk::destroy(); return 0; } diff --git a/src/fibers/tests/main.cpp b/src/fibers/tests/main.cpp index 312668f..b0d4d46 100644 --- a/src/fibers/tests/main.cpp +++ b/src/fibers/tests/main.cpp @@ -1,28 +1,22 @@ #include -#include -#include +#include #include int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); - // Skip scheduler initialization if (::testing::GTEST_FLAG(list_tests)) { return RUN_ALL_TESTS(); } - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); - int result = RUN_ALL_TESTS(); + int r = RUN_ALL_TESTS(); silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); - - return result; + silk::destroy(); + return r; } diff --git a/src/gdb/tests/gdb-test.cpp b/src/gdb/tests/gdb-test.cpp index 28fd7d9..422b7f8 100644 --- a/src/gdb/tests/gdb-test.cpp +++ b/src/gdb/tests/gdb-test.cpp @@ -33,9 +33,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -106,9 +106,7 @@ __attribute__((noinline, used)) static void gdb_ready() int main() { - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); // Spinner stays RUNNING on a CPU. @@ -157,8 +155,7 @@ int main() sem_destroy(&ready); silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); + silk::destroy(); return 0; } diff --git a/src/perf/file-perf.cpp b/src/perf/file-perf.cpp index 01aa400..62e8dbc 100644 --- a/src/perf/file-perf.cpp +++ b/src/perf/file-perf.cpp @@ -2,10 +2,10 @@ #include #include +#include #include #include #include -#include #include #include @@ -406,9 +406,7 @@ int main(int argc, char ** argv) sigset_t mask = blockSignals(); - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); LOG_INFO( @@ -445,8 +443,7 @@ int main(int argc, char ** argv) printJson(allLat, cfg); silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); + silk::destroy(); ::close(fd); return 0; diff --git a/src/perf/http-perf.cpp b/src/perf/http-perf.cpp index 4fe98c3..90c2599 100644 --- a/src/perf/http-perf.cpp +++ b/src/perf/http-perf.cpp @@ -4,10 +4,10 @@ #include #include #include +#include #include #include #include -#include #include #include @@ -282,11 +282,9 @@ static void runClient(int argc, char ** argv) sigset_t mask = blockSignals(); bool signalled = false; + silk::initialize(); if (!cfg.useThreads) { - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); silk::FiberScheduler::initialize(); } @@ -323,9 +321,8 @@ static void runClient(int argc, char ** argv) if (!cfg.useThreads) { silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); } + silk::destroy(); } /** diff --git a/src/perf/net-perf-asio.cpp b/src/perf/net-perf-asio.cpp index eb1112a..0ca8fbb 100644 --- a/src/perf/net-perf-asio.cpp +++ b/src/perf/net-perf-asio.cpp @@ -1,6 +1,7 @@ #include "common.h" #include +#include #include #include #include @@ -523,8 +524,7 @@ int main(int argc, char ** argv) return 1; } - silk::initRseq(); - silk::Perf::initialize(); + silk::initialize(); const char * subcmd = argv[1]; if (strcmp(subcmd, "server") == 0) @@ -542,6 +542,6 @@ int main(int argc, char ** argv) return 1; } - silk::Perf::destroy(); + silk::destroy(); return 0; } diff --git a/src/perf/net-perf.cpp b/src/perf/net-perf.cpp index 178b484..271d0ca 100644 --- a/src/perf/net-perf.cpp +++ b/src/perf/net-perf.cpp @@ -2,11 +2,11 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -738,9 +738,7 @@ static void runServer(int argc, char ** argv) sigset_t mask = blockSignals(); - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); LOG_INFO("starting server on {}:{}", cfg.host, cfg.port); @@ -756,8 +754,7 @@ static void runServer(int argc, char ** argv) server.stop(); silk::FiberScheduler::destroy(); - silk::QueueBase::destroy(); - silk::Perf::destroy(); + silk::destroy(); } /** @@ -812,8 +809,7 @@ static void runClient(int argc, char ** argv) sigset_t mask = blockSignals(); - silk::initRseq(); - silk::Perf::initialize(); + silk::initialize(); silk::FiberScheduler::initialize(); LOG_INFO("starting client on {}:{}", cfg.host, cfg.port); @@ -844,7 +840,7 @@ static void runClient(int argc, char ** argv) printJson(allLat, cfg); silk::FiberScheduler::destroy(); - silk::Perf::destroy(); + silk::destroy(); } /** diff --git a/src/perf/s3-perf.cpp b/src/perf/s3-perf.cpp index 65a042c..03c25f9 100644 --- a/src/perf/s3-perf.cpp +++ b/src/perf/s3-perf.cpp @@ -4,10 +4,10 @@ #include #include #include +#include #include #include #include -#include #include #include @@ -756,9 +756,7 @@ int main(int argc, char ** argv) std::shared_ptr executor; - silk::initRseq(); - silk::Perf::initialize(); - silk::QueueBase::initialize(); + silk::initialize(); if (!cfg.useThreads) { silk::FiberScheduler::initialize(); @@ -814,8 +812,7 @@ int main(int argc, char ** argv) { silk::FiberScheduler::destroy(); } - silk::QueueBase::destroy(); - silk::Perf::destroy(); + silk::destroy(); return 0; } diff --git a/src/util/benchmarks/main.cpp b/src/util/benchmarks/main.cpp index 71d9f3e..8051997 100644 --- a/src/util/benchmarks/main.cpp +++ b/src/util/benchmarks/main.cpp @@ -1,13 +1,15 @@ -#include +#include #include int main(int argc, char ** argv) { - silk::initRseq(); + silk::initialize(); benchmark::Initialize(&argc, argv); benchmark::RunSpecifiedBenchmarks(); benchmark::Shutdown(); + + silk::destroy(); return 0; } diff --git a/src/util/platform.cpp b/src/util/init.cpp similarity index 52% rename from src/util/platform.cpp rename to src/util/init.cpp index 7d13131..d96f152 100644 --- a/src/util/platform.cpp +++ b/src/util/init.cpp @@ -1,4 +1,9 @@ +#include + #include +#include +#include +#include // Suppress warnings emitted by librseq headers: volatile assignment in rseq_cs // and unused parameters in the asm stubs. @@ -11,10 +16,20 @@ namespace silk { -void initRseq() noexcept +void initialize() noexcept +{ + int r = rseq_init(); + ASSERT(r == RSEQ_INIT_OK); + + Tsc::initialize(); + Perf::initialize(); + QueueBase::initialize(); +} + +void destroy() noexcept { - int ret = rseq_init(); - ASSERT(ret == RSEQ_INIT_OK); + QueueBase::destroy(); + Perf::destroy(); } } // namespace silk diff --git a/src/util/perf.cpp b/src/util/perf.cpp index 8c0b310..9600c61 100644 --- a/src/util/perf.cpp +++ b/src/util/perf.cpp @@ -12,6 +12,12 @@ namespace silk void Perf::initialize() noexcept { + if (processorState) + { + // Skip the second initialization. + return; + } + uint32_t processorCount = getProcessorCount(); simpleCounters = std::make_unique(NUM_SIMPLE_COUNTERS); diff --git a/src/util/queue.cpp b/src/util/queue.cpp index 9d97ad2..ca83ff9 100644 --- a/src/util/queue.cpp +++ b/src/util/queue.cpp @@ -7,7 +7,11 @@ namespace silk void QueueBase::initialize() noexcept { - ASSERT(!pool); + if (pool) + { + // Skip the second initialization. + return; + } pool = new MemoryPool(); ASSERT(pool); @@ -15,8 +19,6 @@ void QueueBase::initialize() noexcept void QueueBase::destroy() noexcept { - ASSERT(pool); - delete pool; pool = nullptr; } diff --git a/src/util/tests/main.cpp b/src/util/tests/main.cpp index da9b7f2..a684c6f 100644 --- a/src/util/tests/main.cpp +++ b/src/util/tests/main.cpp @@ -1,10 +1,19 @@ -#include +#include #include int main(int argc, char ** argv) { ::testing::InitGoogleTest(&argc, argv); - silk::initRseq(); - return RUN_ALL_TESTS(); + if (::testing::GTEST_FLAG(list_tests)) + { + return RUN_ALL_TESTS(); + } + + silk::initialize(); + + int r = RUN_ALL_TESTS(); + + silk::destroy(); + return r; } diff --git a/src/util/tsc.cpp b/src/util/tsc.cpp index de0ed02..123859f 100644 --- a/src/util/tsc.cpp +++ b/src/util/tsc.cpp @@ -13,6 +13,23 @@ namespace silk { #if defined(__x86_64__) + +// CPUID leaf 0x80000007 EDX bit 8 advertises the Invariant TSC feature: +// TSC runs at a constant rate across frequency transitions and remains synchronized +// across cores. Silk reads TSC on one CPU and compares the result on another +// (sleep deadlines, work-stealing budgets) -- if TSC is not invariant those +// comparisons are unsound. Older CPUs without this bit are not supported. +static bool hasInvariantTsc() noexcept +{ + uint32_t eax, ebx, ecx, edx; + if (__get_cpuid(0x80000000, &eax, &ebx, &ecx, &edx) == 0 || eax < 0x80000007) + { + return false; + } + __cpuid(0x80000007, eax, ebx, ecx, edx); + return (edx & (1u << 8)) != 0; +} + static uint64_t getTscFrequencyCpuid() noexcept { uint32_t eax, ebx, ecx, edx; @@ -84,11 +101,20 @@ static uint64_t getTscFrequencyCpuid() noexcept } #endif // __x86_64__ -void Tsc::init() noexcept +void Tsc::initialize() noexcept { + if (frequency) + { + // Skip the second initialization. + return; + } + #if defined(__x86_64__) + ASSERT(hasInvariantTsc(), "CPU does not advertise Invariant TSC; silk requires a stable cross-core counter"); frequency = getTscFrequencyCpuid(); #elif defined(__aarch64__) + // ARMv8 cntvct_el0 is anchored to a system counter that is invariant by + // architecture, so no equivalent capability check is required. __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(frequency)); #else # error Unsupported platform