From 29599c31f4d38d818b188ef35f2505776bc93666 Mon Sep 17 00:00:00 2001 From: Dana Rubin Date: Wed, 13 Dec 2023 18:21:23 +0200 Subject: [PATCH] NSOF-8274 receiver/hostmetricsreceiver:processscraper: Workaround process createTime bogus value Process createTime collected using gopsutils/process module and uses as startTime of metric sequence. When running inside lxc, function that calculates create time mixes host and guest stats. See https://github.com/shirou/gopsutil/issues/1562 for more details This leads to two issues: 1. Calculated createTime has future value eventually causing scrapper to skip process collection. 2. dp series startTime is set to future in time value, the consequences are unknown. In order to overcome this issue, we will set createTime to system boot time. In case of lxc, container boot time --- .../scraper/processscraper/process_scraper.go | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go index 6edb1d90ae8a..bf5fedc3f48c 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go +++ b/receiver/hostmetricsreceiver/internal/scraper/processscraper/process_scraper.go @@ -11,6 +11,7 @@ import ( "time" "github.com/shirou/gopsutil/v3/common" + "github.com/shirou/gopsutil/v3/host" "github.com/shirou/gopsutil/v3/process" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/pdata/pcommon" @@ -54,6 +55,7 @@ type scraper struct { getProcessHandles func(context.Context) (processHandles, error) handleCountManager handlecount.Manager + bootTime int64 } // newProcessScraper creates a Process Scraper @@ -87,8 +89,15 @@ func newProcessScraper(settings receiver.CreateSettings, cfg *Config) (*scraper, return scraper, nil } -func (s *scraper) start(context.Context, component.Host) error { +func (s *scraper) start(ctx context.Context, _ component.Host) error { s.mb = metadata.NewMetricsBuilder(s.config.MetricsBuilderConfig, s.settings) + ctx = context.WithValue(ctx, common.EnvKey, s.config.EnvMap) + bt, err := host.BootTimeWithContext(ctx) + if err == nil { + s.bootTime = int64(bt) * 1000 + } else { + s.bootTime = time.Now().UnixMilli() + } return nil } @@ -225,9 +234,17 @@ func (s *scraper) getProcessMetadata() ([]*processMetadata, error) { createTime, err := s.getProcessCreateTime(handle, ctx) if err != nil { errs.AddPartial(0, fmt.Errorf("error reading create time for process %q (pid %v): %w", executable.name, pid, err)) - // set the start time to now to avoid including this when a scrape_process_delay is set - createTime = time.Now().UnixMilli() + // set the start time to boot time + createTime = s.bootTime } + + if time.Now().UnixMilli() < createTime { + // getProcessCreateTime is bogus for lxc env + // see https://github.com/shirou/gopsutil/issues/1562 + // use container boot time as createTime => resource startTime + createTime = s.bootTime + } + if s.scrapeProcessDelay.Milliseconds() > (time.Now().UnixMilli() - createTime) { continue }