Skip to content

Commit

Permalink
adapt the unit tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@doudou
doudou committed May 31, 2021
1 parent 85a347a commit bfc1c3d
Show file tree
Hide file tree
Showing 16 changed files with 357 additions and 305 deletions.
2 changes: 1 addition & 1 deletion lib/syskit/log.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ module Log

module Syskit
module Log # rubocop:disable Style/Documentation
# Returns the paths of the log files in a given directory
# Returns the paths of the pocolog log files in a given directory
#
# The returned paths are sorted in 'pocolog' order, i.e. multi-IO files are
# following each other in the order of their place in the overall IO
Expand Down
127 changes: 65 additions & 62 deletions lib/syskit/log/cli/datastore.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,19 +153,42 @@ def show_dataset_pocolog(pastel, store, dataset)
end
end

def import_dataset(path, reporter, datastore, metadata, merge: false)
# @api private
#
# Parse a metadata option such as --set some=value some-other=value
def parse_metadata_option(hash)
hash.each_with_object({}) do |arg, metadata|
key, value = arg.split('=')
unless value
raise ArgumentError,
"metadata setters need to be specified as "\
"key=value (got #{arg})"
end
(metadata[key] ||= Set.new) << value
end
end

def import_dataset?(datastore, path, reporter:)
last_import_digest, last_import_time =
Syskit::Log::Datastore::Import.find_import_info(path)
already_imported = last_import_digest &&
datastore.has?(last_import_digest)
if already_imported && !options[:force]
reporter.info(
"#{path} already seem to have been imported as "\
"#{last_import_digest} at #{last_import_time}. Give "\
"--force to import again"
)
return
end
already_imported =
last_import_digest && datastore.has?(last_import_digest)
return true if !already_imported || options[:force]

reporter.info(
"#{path} already seem to have been imported as "\
"#{last_import_digest} at #{last_import_time}. Give "\
"--force to import again"
)
false
end

def dataset_duration(dataset)
dataset.each_pocolog_stream.map(&:duration_lg).max || 0
end

def import_dataset(path, reporter, datastore, metadata, merge: false)
return unless import_dataset?(datastore, path, reporter: reporter)

paths =
if merge
Expand All @@ -177,45 +200,40 @@ def import_dataset(path, reporter, datastore, metadata, merge: false)
datastore.in_incoming do |core_path, cache_path|
importer = Syskit::Log::Datastore::Import.new(datastore)
dataset = importer.normalize_dataset(
paths, core_path, cache_path: cache_path,
reporter: reporter
paths, core_path,
cache_path: cache_path, reporter: reporter
)
metadata.each { |k, v| dataset.metadata_set(k, *v) }
dataset.metadata_write_to_file
stream_duration = dataset.each_pocolog_stream
.map(&:duration_lg)
.max
stream_duration ||= 0

if already_imported
# --force is implied as otherwise we would have
# skipped earlier
dataset_duration = dataset_duration(dataset)
unless dataset_duration >= options[:min_duration]
reporter.info(
"#{path} seem to have already been imported but --force "\
"is given, overwriting"
"#{path} lasts only %.1fs, ignored" % [dataset_duration]
)
datastore.delete(last_import_digest)
break
end

if stream_duration >= options[:min_duration]
begin
final_core_dir = importer.move_dataset_to_store(
path, dataset,
force: options[:force], reporter: reporter
)
puts File.basename(final_core_dir)
rescue Syskit::Log::Datastore::Import::DatasetAlreadyExists
reporter.info(
"#{path} already seem to have been imported as "\
"#{dataset.compute_dataset_digest}. Give "\
"--force to import again"
)
end
else
begin
importer.validate_dataset_import(
dataset, force: options[:force], reporter: reporter
)
rescue Syskit::Log::Datastore::Import::DatasetAlreadyExists
reporter.info(
"#{path} lasts only %.1fs, ignored" % [stream_duration]
"#{path} already seem to have been imported as "\
"#{dataset.compute_dataset_digest}. Give "\
"--force to import again"
)
break
end

dataset = importer.move_dataset_to_store(dataset)
t = Time.now
paths.each do |p|
Syskit::Log::Datastore::Import.save_import_info(
p, dataset, time: t
)
end
dataset
end
end

Expand Down Expand Up @@ -370,21 +388,14 @@ def import(root_path, description = nil)
datastore = create_store

metadata = {}
metadata['description'] = description if description
metadata['tags'] = options[:tags]
options[:metadata].each do |pair|
k, v = pair.split('=')
unless v
raise ArgumentError,
'expected key=value pair as argument to '\
"--metadata but got '#{pair}'"
end
(metadata[k] ||= []) << v
end
metadata["description"] = description if description
metadata["tags"] = options[:tags]
metadata.merge!(parse_metadata_option(options[:metadata]))

paths.each do |p|
import_dataset(p, reporter, datastore, metadata,
merge: options[:merge])
dataset = import_dataset(p, reporter, datastore, metadata,
merge: options[:merge])
puts dataset.digest if dataset
end
end

Expand Down Expand Up @@ -483,15 +494,7 @@ def metadata(*query)
end

if options[:set]
setters = Hash.new
options[:set].map do |arg|
key, value = arg.split('=')
if !value
raise ArgumentError, "metadata setters need to be specified as key=value (got #{arg})"
end
(setters[key] ||= Set.new) << value
end

setters = parse_metadata_option(options[:set])
datasets.each do |set|
setters.each do |k, v|
set.metadata_set(k, *v)
Expand Down
4 changes: 2 additions & 2 deletions lib/syskit/log/datastore/dataset.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,10 +255,10 @@ def compute_dataset_digest(
def each_important_file
return enum_for(__method__) unless block_given?

Pathname.glob(dataset_path + 'pocolog' + '*.*.log') do |path|
Pathname.glob(dataset_path + "pocolog" + "*.*.log") do |path|
yield(path)
end
Pathname.glob(dataset_path + '*-events.log') do |path|
Pathname.glob(dataset_path + "roby-events.*.log") do |path|
yield(path)
end
end
Expand Down
70 changes: 42 additions & 28 deletions lib/syskit/log/datastore/import.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,30 +46,41 @@ def prepare_import(dir_path)
# @param [Pathname] dir_path the input directory
# @return [Pathname] the directory of the imported dataset in the store
def import(
in_path, in_dataset_paths,
force: false, reporter: Pocolog::CLI::NullReporter.new
in_dataset_paths, force: false, reporter: Pocolog::CLI::NullReporter.new
)
datastore.in_incoming do |core_path, cache_path|
dataset = normalize_dataset(
in_dataset_paths, core_path,
cache_path: cache_path, reporter: reporter
)
move_dataset_to_store(
in_path, dataset, force: force, reporter: reporter
validate_dataset_import(
dataset, force: force, reporter: reporter
)
move_dataset_to_store(dataset)
end
end

# Find if a directory has already been imported
#
# @return [(String,Time),nil] if the directory has already been
# imported, the time and digest of the import. Otherwise, returns nil
# @param [Pathname] path
# @return [(String,Time)] the digest and time of the last import
def self.find_import_info(path)
info_path = (path + BASENAME_IMPORT_TAG)
return unless info_path.exist?

info = YAML.safe_load(info_path.read, [Time])
[info['sha2'], info['time']]
[info["digest"], info["time"]]
end

# Save import info, used by {.find_import_info}
#
# @param [Pathname] path
# @param [ImportInfo] info
def self.save_import_info(path, dataset, time: Time.now)
(path + BASENAME_IMPORT_TAG).open("w") do |io|
h = { "digest" => dataset.digest, "time" => time }
YAML.dump(h, io)
end
end

# Move the given dataset to the store
Expand All @@ -80,38 +91,41 @@ def self.find_import_info(path)
# @param [Boolean] force if force (the default), the method will fail if
# the dataset is already in the store. Otherwise, it will erase the
# existing dataset with the new one
# @return [Pathname] the path to the new dataset in the store
# @return [Dataset] the dataset at its final place
# @raise DatasetAlreadyExists if a dataset already exists with the same
# ID than the new one and 'force' is false
def move_dataset_to_store(in_path, dataset,
force: false,
reporter: Pocolog::CLI::NullReporter.new)
dataset_digest = dataset.compute_dataset_digest

if datastore.has?(dataset_digest)
if force
datastore.delete(dataset_digest)
reporter.warn "Replacing existing dataset #{dataset_digest} "\
'with new one'
else
raise DatasetAlreadyExists,
"a dataset identical to #{dataset.dataset_path} already "\
"exists in the store (computed digest is #{dataset_digest})"
end
end

def move_dataset_to_store(dataset)
dataset_digest = dataset.digest
final_core_dir = datastore.core_path_of(dataset_digest)
FileUtils.mv dataset.dataset_path, final_core_dir
final_cache_dir = datastore.cache_path_of(dataset_digest)
if final_core_dir != final_cache_dir
FileUtils.mv dataset.cache_path, final_cache_dir
end

(in_path + BASENAME_IMPORT_TAG).open('w') do |io|
YAML.dump(Hash['sha2' => dataset_digest, 'time' => Time.now], io)
Dataset.new(final_core_dir,
digest: dataset_digest,
cache: final_cache_dir)
end

# @api private
#
# Verifies that the given data should be imported
def validate_dataset_import(
dataset, force: false, reporter: Pocolog::CLI::NullReporter.new
)
return unless datastore.has?(dataset.digest)

if force
datastore.delete(dataset.digest)
reporter.warn "Replacing existing dataset #{dataset.digest} "\
"with new one"
return
end

final_core_dir
raise DatasetAlreadyExists,
"a dataset identical to #{dataset.dataset_path} already "\
"exists in the store (computed digest is #{dataset.digest})"
end

# Import Roby's info.yml information into the dataset metadata
Expand Down
12 changes: 9 additions & 3 deletions lib/syskit/log/datastore/index_build.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def rebuild_pocolog_indexes(
def rebuild_roby_index(force: false, reporter: Pocolog::CLI::NullReporter.new)
dataset.cache_path.mkpath
event_logs = Syskit::Log.logfiles_in_dir(dataset.dataset_path)
event_logs.each do |roby_log_path|
event_logs = event_logs.find_all do |roby_log_path|
rebuild_roby_own_index(
roby_log_path, force: force, reporter: reporter
)
Expand All @@ -89,28 +89,34 @@ def rebuild_roby_index(force: false, reporter: Pocolog::CLI::NullReporter.new)
# @api private
#
# Rebuild Roby's own index file
#
# @return [Boolean] true if the log file is valid and has a valid index,
# false otherwise (e.g. if the log file format is too old)
def rebuild_roby_own_index(
roby_log_path, force: false, reporter: Pocolog::CLI::NullReporter.new
)
roby_index_path = dataset.cache_path + roby_log_path.sub_ext(".idx")
roby_index_path =
dataset.cache_path + roby_log_path.basename.sub_ext(".idx")
needs_rebuild =
force ||
!Roby::DRoby::Logfile::Index.valid_file?(
roby_log_path, roby_index_path
)
unless needs_rebuild
reporter.log " up-to-date: #{roby_log_path.basename}"
return
return true
end

reporter.log " rebuilding: #{roby_log_path.basename}"
begin
Roby::DRoby::Logfile::Index.rebuild_file(
roby_log_path, roby_index_path
)
true
rescue Roby::DRoby::Logfile::InvalidFormatVersion
reporter.warn " #{roby_log_path.basename} is in an obsolete Roby "\
"log file format, skipping"
false
end
end

Expand Down
Loading

0 comments on commit bfc1c3d

Please sign in to comment.