Merge pull request #54 from rock-core/size_in_reports

fix: handling of the progress bar with compressed log data
rock-core · Jan 22, 2025 · 095ea7c · 095ea7c
2 parents 1e3cafc + c63e8f5
commit 095ea7c
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 5 deletions.
diff --git a/lib/syskit/log.rb b/lib/syskit/log.rb
@@ -78,6 +78,10 @@ def self.open_in_stream(path, &block)
             end
         end
 
+        def self.io_disk_size(io)
+            File.stat(io.path).size
+        end
+
         def self.open_out_stream(path, &block)
             return path.open("w", &block) unless path.extname == ".zst"
 

diff --git a/lib/syskit/log/datastore/normalize.rb b/lib/syskit/log/datastore/normalize.rb
@@ -248,7 +248,7 @@ def normalize_logfile(logfile_path, output_path, reporter: NullReporter.new)
                 reporter.warn "#{logfile_path.basename} looks truncated or contains "\
                               "garbage (#{e.message}), stopping processing but keeping "\
                               "the samples processed so far"
-                reporter.current = in_io.size + reporter_offset
+                reporter.current = Syskit::Log.io_disk_size(in_io) + reporter_offset
             ensure
                 state.out_io_streams.each(&:flush)
                 in_block_stream&.close
@@ -316,7 +316,7 @@ def normalize_logfile_init(logfile_path, in_io, reporter:)
             rescue Pocolog::InvalidFile
                 reporter.warn "#{logfile_path.basename} does not seem to be "\
                                 "a valid pocolog file, skipping"
-                reporter.current += in_io.size
+                reporter.current += Syskit::Log.io_disk_size(in_io)
                 nil
             end
 

diff --git a/test/datastore/normalize_test.rb b/test/datastore/normalize_test.rb
@@ -173,9 +173,9 @@ class Datastore
 
             describe "#normalize_logfile" do
                 it "skips invalid files" do
-                    write_logfile "file0.0.log", "INVALID"
+                    content = Random.hex(ZstdIO::DECOMPRESS_READ_SIZE + 1)
+                    write_logfile "file0.0.log", content
                     reporter = flexmock(NullReporter.new)
-                    flexmock(reporter).should_receive(:current).and_return(10)
                     ext = ".zst" if compress?
                     reporter
                         .should_receive(:warn)
@@ -186,6 +186,9 @@ class Datastore
                         logfile_pathname("file0.0.log"),
                         logfile_pathname("normalized"), reporter: reporter
                     )
+
+                    file_size = logfile_pathname("file0.0.log").stat.size
+                    assert_equal file_size, reporter.current
                 end
                 it "handles truncated files" do
                     create_logfile "file0.0.log", truncate: 1 do
@@ -201,7 +204,10 @@ class Datastore
                     file0_path = logfile_pathname("file0.0.log")
                     logdir_pathname("normalized").mkpath
                     reporter = flexmock(NullReporter.new)
-                    flexmock(reporter).should_receive(:current).and_return(10)
+                    reporter.should_receive(:current).and_return(10)
+                    reporter.should_receive(:current=)
+                            .with(10 + logfile_pathname("file0.0.log").stat.size)
+                            .once
                     ext = ".zst" if compress?
                     reporter.should_receive(:warn)
                             .with(/^file0.0.log#{ext} looks truncated/)