From 4520f6170d233291550f2f4da60feff273257488 Mon Sep 17 00:00:00 2001 From: Mauricio Junior Date: Mon, 4 Mar 2024 16:01:35 -0300 Subject: [PATCH 1/3] feat: pprof profile section and monitoring system --- command/server/config/config.go | 60 ++++++++++++++++--------- command/server/init.go | 14 ++++++ command/server/params.go | 18 ++++++++ command/server/server.go | 35 +++++++++++++++ monitoring/pprof.go | 79 +++++++++++++++++++++++++++++++++ monitoring/stats.go | 67 ++++++++++++++++++++++++++++ server/config.go | 10 +++++ server/server.go | 37 +++++++++++++-- 8 files changed, 295 insertions(+), 25 deletions(-) create mode 100644 monitoring/pprof.go create mode 100644 monitoring/stats.go diff --git a/command/server/config/config.go b/command/server/config/config.go index bc64e99b62..dad6f2c23e 100644 --- a/command/server/config/config.go +++ b/command/server/config/config.go @@ -13,28 +13,29 @@ import ( // Config defines the server configuration params type Config struct { - GenesisPath string `json:"chain_config" yaml:"chain_config"` - SecretsConfigPath string `json:"secrets_config" yaml:"secrets_config"` - DataDir string `json:"data_dir" yaml:"data_dir"` - BlockGasTarget string `json:"block_gas_target" yaml:"block_gas_target"` - GRPCAddr string `json:"grpc_addr" yaml:"grpc_addr"` - JSONRPCAddr string `json:"jsonrpc_addr" yaml:"jsonrpc_addr"` - Telemetry *Telemetry `json:"telemetry" yaml:"telemetry"` - Network *Network `json:"network" yaml:"network"` - DataFeed *DataFeed `json:"data_feed" yaml:"data_feed"` - ShouldSeal bool `json:"seal" yaml:"seal"` - TxPool *TxPool `json:"tx_pool" yaml:"tx_pool"` - LogLevel string `json:"log_level" yaml:"log_level"` - RestoreFile string `json:"restore_file" yaml:"restore_file"` - BlockTime uint64 `json:"block_time_s" yaml:"block_time_s"` - Headers *Headers `json:"headers" yaml:"headers"` - LogFilePath string `json:"log_to" yaml:"log_to"` - JSONRPCBatchRequestLimit uint64 `json:"json_rpc_batch_request_limit" yaml:"json_rpc_batch_request_limit"` - JSONRPCBlockRangeLimit uint64 `json:"json_rpc_block_range_limit" yaml:"json_rpc_block_range_limit"` - JSONLogFormat bool `json:"json_log_format" yaml:"json_log_format"` - RPCNrAppName string `json:"rpc_nr_app_name" yaml:"rpc_nr_app_name"` - RPCNrLicenseKey string `json:"rpc_nr_license_key" yaml:"rpc_nr_license_key"` - GasPriceBlockUtilizationMinimum float64 `json:"gasprice_block_utilization_threshold" yaml:"gasprice_block_utilization_threshold"` //nolint:lll + GenesisPath string `json:"chain_config" yaml:"chain_config"` + SecretsConfigPath string `json:"secrets_config" yaml:"secrets_config"` + DataDir string `json:"data_dir" yaml:"data_dir"` + BlockGasTarget string `json:"block_gas_target" yaml:"block_gas_target"` + GRPCAddr string `json:"grpc_addr" yaml:"grpc_addr"` + JSONRPCAddr string `json:"jsonrpc_addr" yaml:"jsonrpc_addr"` + Telemetry *Telemetry `json:"telemetry" yaml:"telemetry"` + Network *Network `json:"network" yaml:"network"` + DataFeed *DataFeed `json:"data_feed" yaml:"data_feed"` + Monitoring *Monitoring `json:"monitoring" yaml:"monitoring"` + ShouldSeal bool `json:"seal" yaml:"seal"` + TxPool *TxPool `json:"tx_pool" yaml:"tx_pool"` + LogLevel string `json:"log_level" yaml:"log_level"` + RestoreFile string `json:"restore_file" yaml:"restore_file"` + BlockTime uint64 `json:"block_time_s" yaml:"block_time_s"` + Headers *Headers `json:"headers" yaml:"headers"` + LogFilePath string `json:"log_to" yaml:"log_to"` + JSONRPCBatchRequestLimit uint64 `json:"json_rpc_batch_request_limit" yaml:"json_rpc_batch_request_limit"` + JSONRPCBlockRangeLimit uint64 `json:"json_rpc_block_range_limit" yaml:"json_rpc_block_range_limit"` + JSONLogFormat bool `json:"json_log_format" yaml:"json_log_format"` + RPCNrAppName string `json:"rpc_nr_app_name" yaml:"rpc_nr_app_name"` + RPCNrLicenseKey string `json:"rpc_nr_license_key" yaml:"rpc_nr_license_key"` + GasPriceBlockUtilizationMinimum float64 `json:"gasprice_block_utilization_threshold" yaml:"gasprice_block_utilization_threshold"` //nolint:lll } // Telemetry holds the config details for metric services. @@ -64,6 +65,14 @@ type DataFeed struct { SXNodeAddress string `json:"sx_node_address" yaml:"sx_node_address"` } +type Monitoring struct { + IsEnable bool `json:"is_enable" yaml:"is_enable"` + DelayInSeconds uint64 `json:"delay_in_seconds" yaml:"delay_in_seconds"` + IsMemStressTestEnable bool `json:"is_mem_stress_test_enable" yaml:"is_mem_stress_test_enable"` + TickerInSeconds uint64 `json:"ticker_in_seconds" yaml:"ticker_in_seconds"` + Threshold float64 `json:"threshold" yaml:"threshold"` +} + // TxPool defines the TxPool configuration params type TxPool struct { PriceLimit uint64 `json:"price_limit" yaml:"price_limit"` @@ -125,6 +134,13 @@ func DefaultConfig() *Config { OutcomeReporterAddress: "", SXNodeAddress: "", }, + Monitoring: &Monitoring{ + IsEnable: false, + DelayInSeconds: 60, + IsMemStressTestEnable: false, + TickerInSeconds: 60, + Threshold: 0.8, // 80% + }, LogLevel: "INFO", RestoreFile: "", BlockTime: DefaultBlockTime, diff --git a/command/server/init.go b/command/server/init.go index f5c9f939df..dde653de05 100644 --- a/command/server/init.go +++ b/command/server/init.go @@ -232,6 +232,10 @@ func (p *serverParams) initAddresses() error { return err } + if err := p.initMonitoringParams(); err != nil { + return err + } + // if err := p.initCustomContractAddress(); err != nil { // return err // } @@ -326,6 +330,16 @@ func (p *serverParams) initDataFeedParams() error { return nil } +func (p *serverParams) initMonitoringParams() error { + p.isEnable = p.rawConfig.Monitoring.IsEnable + p.delayInSeconds = p.rawConfig.Monitoring.DelayInSeconds + p.isMemStressTestEnable = p.rawConfig.Monitoring.IsMemStressTestEnable + p.tickerInSeconds = p.rawConfig.Monitoring.TickerInSeconds + p.threshold = p.rawConfig.Monitoring.Threshold + + return nil +} + func (p *serverParams) initGRPCAddress() error { var parseErr error diff --git a/command/server/params.go b/command/server/params.go index 7ae6a4eabd..f92aca593e 100644 --- a/command/server/params.go +++ b/command/server/params.go @@ -47,6 +47,11 @@ const ( verifyOutcomeAPIURLFlag = "verify-outcome-api-url" outcomeReporterAddressFlag = "outcome-reporter-address" sxNodeAddressFlag = "sx-node-address" + isEnableFlag = "is-enable" + delayInSecondsFlag = "delay-in-seconds" + isMemStressTestEnableFlag = "is-mem-stress-test-enable" + tickerInSecondsFlag = "ticker-in-seconds" + thresholdFlag = "threshold" ) // Flags that are deprecated, but need to be preserved for @@ -102,6 +107,12 @@ type serverParams struct { dataFeedOutcomeReporterAddress string dataFeedSXNodeAddress string + isEnable bool + delayInSeconds uint64 + isMemStressTestEnable bool + tickerInSeconds uint64 + threshold float64 + ibftBaseTimeoutLegacy uint64 genesisConfig *chain.Chain @@ -197,6 +208,13 @@ func (p *serverParams) generateConfig() *server.Config { OutcomeReporterAddress: p.dataFeedOutcomeReporterAddress, SXNodeAddress: p.dataFeedSXNodeAddress, }, + Monitoring: &server.Monitoring{ + IsEnable: p.isEnable, + DelayInSeconds: p.delayInSeconds, + IsMemStressTestEnable: p.isMemStressTestEnable, + TickerInSeconds: p.tickerInSeconds, + Threshold: p.threshold, + }, DataDir: p.rawConfig.DataDir, Seal: p.rawConfig.ShouldSeal, PriceLimit: p.rawConfig.TxPool.PriceLimit, diff --git a/command/server/server.go b/command/server/server.go index c346ff2d65..0ffbe0f275 100644 --- a/command/server/server.go +++ b/command/server/server.go @@ -292,6 +292,41 @@ func setFlags(cmd *cobra.Command) { "the address of the SXNode contract, used by DataFeedService reporting txs", ) + cmd.Flags().BoolVar( + ¶ms.isEnable, + isEnableFlag, + defaultConfig.Monitoring.IsEnable, + "indicates when pprof should start the snapshot profile routine", + ) + + cmd.Flags().Uint64Var( + ¶ms.delayInSeconds, + delayInSecondsFlag, + defaultConfig.Monitoring.DelayInSeconds, + "indicates the delay time the prof will get from the snapshot profile", + ) + + cmd.Flags().BoolVar( + ¶ms.isMemStressTestEnable, + isMemStressTestEnableFlag, + defaultConfig.Monitoring.IsMemStressTestEnable, + "flag that triggers the memory stress test function", + ) + + cmd.Flags().Uint64Var( + ¶ms.tickerInSeconds, + tickerInSecondsFlag, + defaultConfig.Monitoring.TickerInSeconds, + "indicates the delay time that the memory stress test function will allocate memory", + ) + + cmd.Flags().Float64Var( + ¶ms.threshold, + thresholdFlag, + defaultConfig.Monitoring.Threshold, + "the threshold for memory usage, specified as a percentage between 0 and 1", + ) + setLegacyFlags(cmd) setDevFlags(cmd) diff --git a/monitoring/pprof.go b/monitoring/pprof.go new file mode 100644 index 0000000000..2d4bc4eaca --- /dev/null +++ b/monitoring/pprof.go @@ -0,0 +1,79 @@ +package monitoring + +import ( + "fmt" + "os" + "path/filepath" + "runtime/pprof" + "time" + + "github.com/hashicorp/go-hclog" +) + +type Profile struct { + Logger hclog.Logger + IsEnable bool + DelayInSeconds uint64 + Goroutine *pprof.Profile + Heap *pprof.Profile +} + +/* + SetupPprofProfiles configures the pprof profiles for monitoring different aspects of the program. + It creates a directory to store the profile files and starts collecting + the specified profiles in separate goroutines +*/ +func (profile *Profile) SetupPprofProfiles() { + // Get the directory of the executable + exeDir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + profile.Logger.Error("Error getting executable directory", "error", err) + return + } + + // Directory to store profiling files + profileDir := filepath.Join(exeDir, "../pprof") + if err := os.MkdirAll(profileDir, 0755); err != nil { + profile.Logger.Error("Error creating profile directory", "error", err) + return + } + + // Start collecting profiles + if profile.IsEnable { + go collectProfile(profile.Heap, filepath.Join(profileDir, "heap"), profile.Logger, profile.DelayInSeconds) + go collectProfile(profile.Goroutine, filepath.Join(profileDir, "goroutine"), profile.Logger, profile.DelayInSeconds) + } +} + +/* + Collects the specified pprof profile data at regular intervals + and writes it to a new file in the specified profile directory +*/ +func collectProfile(profile *pprof.Profile, profileDir string, logger hclog.Logger, delayInSeconds uint64) { + for { + // Generate a unique filename for the profile + profileFileName := fmt.Sprintf(filepath.Join(profileDir, "%s_%s.prof"), profile.Name(), time.Now().Format("20060102-1504")) + + // Create the profile directory if it does not exist + if err := os.MkdirAll(profileDir, 0755); err != nil { + logger.Error("Error creating profile directory:", err) + return + } + + // Create a new file to write the profile data + profileFile, err := os.Create(profileFileName) + if err != nil { + logger.Error("Error creating profile file:", err) + return + } + + // Write the profile data to the file + if err := profile.WriteTo(profileFile, 0); err != nil { + logger.Error("Error writing profile data:", err) + } + + profileFile.Close() + logger.Info(fmt.Sprintf("Profile file created: %s", profileFileName)) + time.Sleep(time.Second * time.Duration(delayInSeconds)) + } +} diff --git a/monitoring/stats.go b/monitoring/stats.go new file mode 100644 index 0000000000..078dde0adc --- /dev/null +++ b/monitoring/stats.go @@ -0,0 +1,67 @@ +package monitoring + +import ( + "fmt" + "os" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/shirou/gopsutil/mem" +) + +type Stats struct { + Logger hclog.Logger + IsMemStressTestEnable bool + TickerInSeconds uint64 + Threshold float64 +} + +func (stats *Stats) TrackMemoryUsage() { + if stats.IsMemStressTestEnable { + go func() { + heapMemoryStressTest(stats.Logger) + }() + } + + for { + // Get the virtual memory stats + vm, err := mem.VirtualMemory() + if err != nil { + stats.Logger.Error("Error getting VirtualMemory", err) + time.Sleep(time.Second * time.Duration(stats.TickerInSeconds)) // Wait for the specified interval + continue + } + + // Calculate the memory usage percentage + memUsage := float64(vm.Used) / float64(vm.Total) + stats.Logger.Info(fmt.Sprintf("Memory usage: %.2f%% (%v bytes), Total Memory: %v bytes, Threshold: %.2f%%", memUsage*100, vm.Used, vm.Total, stats.Threshold*100)) + + // Check if memory usage exceeds the threshold + if memUsage > stats.Threshold { + stats.Logger.Info("Memory usage exceeds threshold. Performing graceful shutdown...") + gracefulShutdown(stats.Logger) + return + } + + time.Sleep(time.Second * time.Duration(stats.TickerInSeconds)) // Wait for the specified interval + } +} + +func gracefulShutdown(logger hclog.Logger) { + logger.Warn("Graceful shutdown completed") + os.Exit(0) +} + +func heapMemoryStressTest(logger hclog.Logger) { + var memorySlice [][]byte + + // Loop 1000 times to force memory allocations + for i := 0; i < 1000; i++ { + // Allocate a large slice of bytes (100 MB) + memory := make([]byte, 1024*1024*1000) // 100 MB + memorySlice = append(memorySlice, memory) + + logger.Info(fmt.Sprintf("Iteration %d - Allocated %d MB", i+1, len(memorySlice)*100)) + time.Sleep(time.Second * 5) + } +} diff --git a/server/config.go b/server/config.go index f7925af8c4..f441d09368 100644 --- a/server/config.go +++ b/server/config.go @@ -38,6 +38,8 @@ type Config struct { DataFeed *DataFeed + Monitoring *Monitoring + Seal bool SecretsManager *secrets.SecretsManagerConfig @@ -71,3 +73,11 @@ type DataFeed struct { OutcomeReporterAddress string SXNodeAddress string } + +type Monitoring struct { + IsEnable bool + DelayInSeconds uint64 + IsMemStressTestEnable bool + TickerInSeconds uint64 + Threshold float64 +} diff --git a/server/server.go b/server/server.go index c46cd0d2ba..51c1c6ff25 100644 --- a/server/server.go +++ b/server/server.go @@ -21,6 +21,7 @@ import ( configHelper "github.com/0xPolygon/polygon-edge/helper/config" "github.com/0xPolygon/polygon-edge/helper/progress" "github.com/0xPolygon/polygon-edge/jsonrpc" + "github.com/0xPolygon/polygon-edge/monitoring" "github.com/0xPolygon/polygon-edge/network" "github.com/0xPolygon/polygon-edge/secrets" "github.com/0xPolygon/polygon-edge/server/proto" @@ -34,6 +35,9 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "google.golang.org/grpc" + + _ "net/http/pprof" + "runtime/pprof" ) // Server is the central manager of the blockchain client @@ -279,6 +283,11 @@ func NewServer(config *Config) (*Server, error) { return nil, err } + // setup and start monitoring consumer + if err := m.setupMonitoring(); err != nil { + return nil, err + } + // setup and start grpc server if err := m.setupGRPC(); err != nil { return nil, err @@ -718,9 +727,9 @@ func (s *Server) setupDataFeedService() error { QueueName: s.config.DataFeed.DataFeedAMQPQueueName, }, }, - VerifyOutcomeURI: s.config.DataFeed.VerifyOutcomeURI, - OutcomeReporterAddress: s.config.DataFeed.OutcomeReporterAddress, - SXNodeAddress: s.config.DataFeed.SXNodeAddress, + VerifyOutcomeURI: s.config.DataFeed.VerifyOutcomeURI, + OutcomeReporterAddress: s.config.DataFeed.OutcomeReporterAddress, + SXNodeAddress: s.config.DataFeed.SXNodeAddress, } datafeedService, err := datafeed.NewDataFeedService( @@ -738,6 +747,28 @@ func (s *Server) setupDataFeedService() error { return nil } +func (s *Server) setupMonitoring() error { + profile := &monitoring.Profile{ + Logger: s.logger.Named("monitoring.pprof"), + IsEnable: s.config.Monitoring.IsEnable, + DelayInSeconds: s.config.Monitoring.DelayInSeconds, + Goroutine: pprof.Lookup("goroutine"), + Heap: pprof.Lookup("heap"), + } + + stats := &monitoring.Stats{ + Logger: s.logger.Named("monitoring.stats"), + IsMemStressTestEnable: s.config.Monitoring.IsMemStressTestEnable, + TickerInSeconds: s.config.Monitoring.TickerInSeconds, + Threshold: s.config.Monitoring.Threshold, + } + + go profile.SetupPprofProfiles() + go stats.TrackMemoryUsage() + + return nil +} + // setupGRPC sets up the grpc server and listens on tcp func (s *Server) setupGRPC() error { proto.RegisterSystemServer(s.grpcServer, &systemService{server: s}) From 0383bdfc248ddba40558f0f031693980b8a94a70 Mon Sep 17 00:00:00 2001 From: Mauricio Junior Date: Mon, 4 Mar 2024 16:57:58 -0300 Subject: [PATCH 2/3] refact: server param names --- command/server/config/config.go | 8 ++++---- command/server/init.go | 4 ++-- command/server/params.go | 12 ++++++------ command/server/server.go | 12 ++++++------ consensus/ibft/verifier.go | 4 ++-- monitoring/pprof.go | 18 +++++++++--------- monitoring/stats.go | 6 +++--- server/config.go | 4 ++-- server/server.go | 12 ++++++------ 9 files changed, 40 insertions(+), 40 deletions(-) diff --git a/command/server/config/config.go b/command/server/config/config.go index dad6f2c23e..f9f7b30e32 100644 --- a/command/server/config/config.go +++ b/command/server/config/config.go @@ -67,9 +67,9 @@ type DataFeed struct { type Monitoring struct { IsEnable bool `json:"is_enable" yaml:"is_enable"` - DelayInSeconds uint64 `json:"delay_in_seconds" yaml:"delay_in_seconds"` + DelayInSecondsProfile uint64 `json:"delay_in_seconds_profile" yaml:"delay_in_seconds_profile"` IsMemStressTestEnable bool `json:"is_mem_stress_test_enable" yaml:"is_mem_stress_test_enable"` - TickerInSeconds uint64 `json:"ticker_in_seconds" yaml:"ticker_in_seconds"` + DelayInSecondsStats uint64 `json:"delay_in_seconds_stats" yaml:"delay_in_seconds_stats"` Threshold float64 `json:"threshold" yaml:"threshold"` } @@ -136,9 +136,9 @@ func DefaultConfig() *Config { }, Monitoring: &Monitoring{ IsEnable: false, - DelayInSeconds: 60, + DelayInSecondsProfile: 60, IsMemStressTestEnable: false, - TickerInSeconds: 60, + DelayInSecondsStats: 60, Threshold: 0.8, // 80% }, LogLevel: "INFO", diff --git a/command/server/init.go b/command/server/init.go index dde653de05..21dc5c7e95 100644 --- a/command/server/init.go +++ b/command/server/init.go @@ -332,9 +332,9 @@ func (p *serverParams) initDataFeedParams() error { func (p *serverParams) initMonitoringParams() error { p.isEnable = p.rawConfig.Monitoring.IsEnable - p.delayInSeconds = p.rawConfig.Monitoring.DelayInSeconds + p.delayInSecondsProfile = p.rawConfig.Monitoring.DelayInSecondsProfile p.isMemStressTestEnable = p.rawConfig.Monitoring.IsMemStressTestEnable - p.tickerInSeconds = p.rawConfig.Monitoring.TickerInSeconds + p.delayInSecondsStats = p.rawConfig.Monitoring.DelayInSecondsStats p.threshold = p.rawConfig.Monitoring.Threshold return nil diff --git a/command/server/params.go b/command/server/params.go index f92aca593e..57fd633f0a 100644 --- a/command/server/params.go +++ b/command/server/params.go @@ -48,9 +48,9 @@ const ( outcomeReporterAddressFlag = "outcome-reporter-address" sxNodeAddressFlag = "sx-node-address" isEnableFlag = "is-enable" - delayInSecondsFlag = "delay-in-seconds" + delayInSecondsProfileFlag = "delay-in-seconds" isMemStressTestEnableFlag = "is-mem-stress-test-enable" - tickerInSecondsFlag = "ticker-in-seconds" + delayInSecondsStatsFlag = "ticker-in-seconds" thresholdFlag = "threshold" ) @@ -108,9 +108,9 @@ type serverParams struct { dataFeedSXNodeAddress string isEnable bool - delayInSeconds uint64 + delayInSecondsProfile uint64 isMemStressTestEnable bool - tickerInSeconds uint64 + delayInSecondsStats uint64 threshold float64 ibftBaseTimeoutLegacy uint64 @@ -210,9 +210,9 @@ func (p *serverParams) generateConfig() *server.Config { }, Monitoring: &server.Monitoring{ IsEnable: p.isEnable, - DelayInSeconds: p.delayInSeconds, + DelayInSecondsProfile: p.delayInSecondsProfile, IsMemStressTestEnable: p.isMemStressTestEnable, - TickerInSeconds: p.tickerInSeconds, + DelayInSecondsStats: p.delayInSecondsStats, Threshold: p.threshold, }, DataDir: p.rawConfig.DataDir, diff --git a/command/server/server.go b/command/server/server.go index 0ffbe0f275..6744cd5abf 100644 --- a/command/server/server.go +++ b/command/server/server.go @@ -300,9 +300,9 @@ func setFlags(cmd *cobra.Command) { ) cmd.Flags().Uint64Var( - ¶ms.delayInSeconds, - delayInSecondsFlag, - defaultConfig.Monitoring.DelayInSeconds, + ¶ms.delayInSecondsProfile, + delayInSecondsProfileFlag, + defaultConfig.Monitoring.DelayInSecondsProfile, "indicates the delay time the prof will get from the snapshot profile", ) @@ -314,9 +314,9 @@ func setFlags(cmd *cobra.Command) { ) cmd.Flags().Uint64Var( - ¶ms.tickerInSeconds, - tickerInSecondsFlag, - defaultConfig.Monitoring.TickerInSeconds, + ¶ms.delayInSecondsStats, + delayInSecondsStatsFlag, + defaultConfig.Monitoring.DelayInSecondsStats, "indicates the delay time that the memory stress test function will allocate memory", ) diff --git a/consensus/ibft/verifier.go b/consensus/ibft/verifier.go index 3761075616..3311e60875 100644 --- a/consensus/ibft/verifier.go +++ b/consensus/ibft/verifier.go @@ -124,14 +124,14 @@ func (i *backendIBFT) IsProposer(id []byte, height, round uint64) bool { return false } - + nextProposer := CalcProposer( i.currentValidators, round, previousProposer, ) - // store nextProposer in case we need to ID them as offline + // store nextProposer in case we need to ID them as offline i.nextProposer = nextProposer.Addr() return types.BytesToAddress(id) == nextProposer.Addr() diff --git a/monitoring/pprof.go b/monitoring/pprof.go index 2d4bc4eaca..4fd3e4b9c1 100644 --- a/monitoring/pprof.go +++ b/monitoring/pprof.go @@ -11,11 +11,11 @@ import ( ) type Profile struct { - Logger hclog.Logger - IsEnable bool - DelayInSeconds uint64 - Goroutine *pprof.Profile - Heap *pprof.Profile + Logger hclog.Logger + IsEnable bool + DelayInSecondsProfile uint64 + Goroutine *pprof.Profile + Heap *pprof.Profile } /* @@ -40,8 +40,8 @@ func (profile *Profile) SetupPprofProfiles() { // Start collecting profiles if profile.IsEnable { - go collectProfile(profile.Heap, filepath.Join(profileDir, "heap"), profile.Logger, profile.DelayInSeconds) - go collectProfile(profile.Goroutine, filepath.Join(profileDir, "goroutine"), profile.Logger, profile.DelayInSeconds) + go collectProfile(profile.Heap, filepath.Join(profileDir, "heap"), profile.Logger, profile.DelayInSecondsProfile) + go collectProfile(profile.Goroutine, filepath.Join(profileDir, "goroutine"), profile.Logger, profile.DelayInSecondsProfile) } } @@ -49,7 +49,7 @@ func (profile *Profile) SetupPprofProfiles() { Collects the specified pprof profile data at regular intervals and writes it to a new file in the specified profile directory */ -func collectProfile(profile *pprof.Profile, profileDir string, logger hclog.Logger, delayInSeconds uint64) { +func collectProfile(profile *pprof.Profile, profileDir string, logger hclog.Logger, delayInSecondsProfile uint64) { for { // Generate a unique filename for the profile profileFileName := fmt.Sprintf(filepath.Join(profileDir, "%s_%s.prof"), profile.Name(), time.Now().Format("20060102-1504")) @@ -74,6 +74,6 @@ func collectProfile(profile *pprof.Profile, profileDir string, logger hclog.Logg profileFile.Close() logger.Info(fmt.Sprintf("Profile file created: %s", profileFileName)) - time.Sleep(time.Second * time.Duration(delayInSeconds)) + time.Sleep(time.Second * time.Duration(delayInSecondsProfile)) } } diff --git a/monitoring/stats.go b/monitoring/stats.go index 078dde0adc..e7a91f7131 100644 --- a/monitoring/stats.go +++ b/monitoring/stats.go @@ -12,7 +12,7 @@ import ( type Stats struct { Logger hclog.Logger IsMemStressTestEnable bool - TickerInSeconds uint64 + DelayInSecondsStats uint64 Threshold float64 } @@ -28,7 +28,7 @@ func (stats *Stats) TrackMemoryUsage() { vm, err := mem.VirtualMemory() if err != nil { stats.Logger.Error("Error getting VirtualMemory", err) - time.Sleep(time.Second * time.Duration(stats.TickerInSeconds)) // Wait for the specified interval + time.Sleep(time.Second * time.Duration(stats.DelayInSecondsStats)) // Wait for the specified interval continue } @@ -43,7 +43,7 @@ func (stats *Stats) TrackMemoryUsage() { return } - time.Sleep(time.Second * time.Duration(stats.TickerInSeconds)) // Wait for the specified interval + time.Sleep(time.Second * time.Duration(stats.DelayInSecondsStats)) // Wait for the specified interval } } diff --git a/server/config.go b/server/config.go index f441d09368..11a5ef2083 100644 --- a/server/config.go +++ b/server/config.go @@ -76,8 +76,8 @@ type DataFeed struct { type Monitoring struct { IsEnable bool - DelayInSeconds uint64 + DelayInSecondsProfile uint64 IsMemStressTestEnable bool - TickerInSeconds uint64 + DelayInSecondsStats uint64 Threshold float64 } diff --git a/server/server.go b/server/server.go index 51c1c6ff25..7c792e1298 100644 --- a/server/server.go +++ b/server/server.go @@ -749,17 +749,17 @@ func (s *Server) setupDataFeedService() error { func (s *Server) setupMonitoring() error { profile := &monitoring.Profile{ - Logger: s.logger.Named("monitoring.pprof"), - IsEnable: s.config.Monitoring.IsEnable, - DelayInSeconds: s.config.Monitoring.DelayInSeconds, - Goroutine: pprof.Lookup("goroutine"), - Heap: pprof.Lookup("heap"), + Logger: s.logger.Named("monitoring.pprof"), + IsEnable: s.config.Monitoring.IsEnable, + DelayInSecondsProfile: s.config.Monitoring.DelayInSecondsProfile, + Goroutine: pprof.Lookup("goroutine"), + Heap: pprof.Lookup("heap"), } stats := &monitoring.Stats{ Logger: s.logger.Named("monitoring.stats"), IsMemStressTestEnable: s.config.Monitoring.IsMemStressTestEnable, - TickerInSeconds: s.config.Monitoring.TickerInSeconds, + DelayInSecondsStats: s.config.Monitoring.DelayInSecondsStats, Threshold: s.config.Monitoring.Threshold, } From 05861ab7382a4de98ec8445f73df7e4f709ca389 Mon Sep 17 00:00:00 2001 From: Mauricio Junior Date: Mon, 4 Mar 2024 17:12:41 -0300 Subject: [PATCH 3/3] refact: change to warn type shutdown message --- monitoring/stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/stats.go b/monitoring/stats.go index e7a91f7131..3b2b906ee5 100644 --- a/monitoring/stats.go +++ b/monitoring/stats.go @@ -38,7 +38,7 @@ func (stats *Stats) TrackMemoryUsage() { // Check if memory usage exceeds the threshold if memUsage > stats.Threshold { - stats.Logger.Info("Memory usage exceeds threshold. Performing graceful shutdown...") + stats.Logger.Warn("Memory usage exceeds threshold. Performing graceful shutdown...") gracefulShutdown(stats.Logger) return }