diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..10537ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +/.bundle/ +/.yardoc +/Gemfile.lock +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +*.sw? diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1380895 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +language: ruby +rvm: + - 2.1.6 +before_install: gem install bundler -v 1.11.2 diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..b2674f2 --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in syskit-pocolog.gemspec +gemspec diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..142429e --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sylvain Joyeux + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..abb7978 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# Syskit::Pocolog + +Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/syskit/pocolog`. To experiment with that code, run `bin/console` for an interactive prompt. + +TODO: Delete this and the text above, and describe your gem + +## Installation + +Add this line to your application's Gemfile: + +```ruby +gem 'syskit-pocolog' +``` + +And then execute: + + $ bundle + +Or install it yourself as: + + $ gem install syskit-pocolog + +## Usage + +TODO: Write usage instructions here + +## Development + +After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. + +To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/syskit-pocolog. + + +## License + +The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). + diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..912ee38 --- /dev/null +++ b/Rakefile @@ -0,0 +1,10 @@ +require "bundler/gem_tasks" +require "rake/testtask" + +Rake::TestTask.new(:test) do |t| + t.libs << "test" + t.libs << "lib" + t.test_files = FileList['test/**/*_test.rb'] +end + +task :default => :spec diff --git a/lib/syskit-pocolog.rb b/lib/syskit-pocolog.rb new file mode 100644 index 0000000..4d32bb1 --- /dev/null +++ b/lib/syskit-pocolog.rb @@ -0,0 +1 @@ +require 'syskit/pocolog' diff --git a/lib/syskit/pocolog.rb b/lib/syskit/pocolog.rb new file mode 100644 index 0000000..8e289ee --- /dev/null +++ b/lib/syskit/pocolog.rb @@ -0,0 +1,9 @@ +require 'pocolog' +require 'syskit' + +require 'metaruby/dsls/find_through_method_missing' +require "syskit/pocolog/version" +require "syskit/pocolog/streams" +require "syskit/pocolog/task_streams" +require "syskit/pocolog/rock_stream_matcher" + diff --git a/lib/syskit/pocolog/data_replay_task.rb b/lib/syskit/pocolog/data_replay_task.rb new file mode 100644 index 0000000..9a21233 --- /dev/null +++ b/lib/syskit/pocolog/data_replay_task.rb @@ -0,0 +1,15 @@ +module Syskit + module Pocolog + # A single task in the replay environment + class DataReplayTask < Syskit::RubyTaskContext + # Setup a task to replay the given streams + # + # @overload for(streams) + # Replay all tasks + # @param [ + def self.for(stream_mappings) + end + end + end +end + diff --git a/lib/syskit/pocolog/rock_stream_matcher.rb b/lib/syskit/pocolog/rock_stream_matcher.rb new file mode 100644 index 0000000..8c1cf66 --- /dev/null +++ b/lib/syskit/pocolog/rock_stream_matcher.rb @@ -0,0 +1,60 @@ +module Syskit + module Pocolog + # A stream matcher for Rock's standard metadata + # + # It can be used as parameter to {Streams#query} + # + # The final query is a logical AND of all the characteristics + class RockStreamMatcher + attr_reader :query + + def initialize + @query = Hash.new + end + + def add_regex(key, rx) + if existing = query[key] + query[key] = Regexp.union(existing, rx) + else + query[key] = rx + end + self + end + + # Match only ports + def ports + add_regex('rock_stream_type', /^port$/) + end + + # Match only properties + def properties + add_regex('rock_stream_type', /^property$/) + end + + # Match the object (port/property) name + def object_name(name) + add_regex('rock_task_object_name', name) + end + + # Match the task name + def task_name(name) + add_regex('rock_task_name', name) + end + + # Match the task model + def task_model(model) + add_regex('rock_task_model', model.orogen_model.name) + end + + # Tests whether a stream matches this query + def ===(stream) + query.all? do |key, matcher| + if metadata = stream.metadata[key] + matcher === metadata + end + end + end + end + end +end + diff --git a/lib/syskit/pocolog/streams.rb b/lib/syskit/pocolog/streams.rb new file mode 100644 index 0000000..dac6ed2 --- /dev/null +++ b/lib/syskit/pocolog/streams.rb @@ -0,0 +1,78 @@ +module Syskit + module Pocolog + # A set of log streams + class Streams + # Load the set of streams available from a directory + # + # Note that in each directory, a stream's identity (task name, + # port/property name and type) must be unique. If you need to mix + # log streams, load files in separate {Streams} objects + def self.from_dir(path) + streams = new + streams.add_dir(Pathname(path)) + streams + end + + # Load the set of streams available from a file + def self.from_file(file) + streams = new + streams.add_file(Pathname(file)) + streams + end + + # The list of streams that are available + attr_reader :streams + + def initialize(streams = Array.new) + @streams = streams + end + + def each_stream(&block) + streams.each(&block) + end + + # Load all log files from a directory + def add_dir(path) + path.children.each do |file_or_dir| + if file_or_dir.file? && file_or_dir.to_s =~ /\.\d+\.log$/ + add_file(file_or_dir) + end + end + end + + # Load the streams from a log file + def add_file(file) + ::Pocolog::Logfiles.open(file).streams.each do |s| + add_stream(s) + end + end + + # Add a new stream + # + # @param [Pocolog::DataStream] s + def add_stream(s) + streams << s + end + + # Find all streams whose metadata match the given query + def find_all_streams(query) + streams.find_all { |s| query === s } + end + + # Find all streams that belong to a task + def find_task_by_name(name) + streams = find_all_streams(RockStreamMatcher.new.task_name(name)) + if !streams.empty? + TaskStreams.new(streams) + end + end + + # Give access to the streams per-task by calling _task + def method_missing(m, *args, &block) + MetaRuby::DSLs.find_through_method_missing( + self, m, args, 'task' => "find_task_by_name") || super + end + end + end +end + diff --git a/lib/syskit/pocolog/task_streams.rb b/lib/syskit/pocolog/task_streams.rb new file mode 100644 index 0000000..11f87cd --- /dev/null +++ b/lib/syskit/pocolog/task_streams.rb @@ -0,0 +1,38 @@ +module Syskit + module Pocolog + # Stream accessor for streams that have already be narrowed down to a + # single task + # + # It is returned from the main stream pool by + # {Streams#find_task_by_name) + class TaskStreams < Streams + # Find a port stream that matches the given name + def find_port_by_name(name) + objects = find_all_streams(RockStreamMatcher.new.ports.object_name(name)) + if objects.size > 1 + raise Ambiguous, "there are multiple ports with the name #{name}" + else objects.first + end + end + + # Find a property stream that matches the given name + def find_property_by_name(name) + objects = find_all_streams(RockStreamMatcher.new.properties.object_name(name)) + if objects.size > 1 + raise Ambiguous, "there are multiple properties with the name #{name}" + else objects.first + end + end + + # Syskit-looking accessors for ports (_port) and properties + # (_property) + def method_missing(m, *args) + MetaRuby::DSLs.find_through_method_missing( + self, m, args, + "port" => "find_port_by_name", + "property" => "find_property_by_name") || super + end + end + end +end + diff --git a/lib/syskit/pocolog/version.rb b/lib/syskit/pocolog/version.rb new file mode 100644 index 0000000..eacb90a --- /dev/null +++ b/lib/syskit/pocolog/version.rb @@ -0,0 +1,5 @@ +module Syskit + module Pocolog + VERSION = "0.1.0" + end +end diff --git a/syskit-pocolog.gemspec b/syskit-pocolog.gemspec new file mode 100644 index 0000000..d0b9ca0 --- /dev/null +++ b/syskit-pocolog.gemspec @@ -0,0 +1,27 @@ +# coding: utf-8 +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'syskit/pocolog/version' + +Gem::Specification.new do |spec| + spec.name = "syskit-pocolog" + spec.version = Syskit::Pocolog::VERSION + spec.authors = ["Sylvain Joyeux"] + spec.email = ["sylvain.joyeux@m4x.org"] + + spec.summary = "A Syskit plugin that allows to replay log files generated by pocolog" + spec.description = "Adds the APIs necessary to transform component networks to replay log files" + spec.homepage = "https://github.com/rock-core/syskit-pocolog" + spec.license = "MIT" + + spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + spec.bindir = "exe" + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.require_paths = ["lib"] + + spec.add_dependency 'syskit' + spec.add_dependency 'metaruby' + spec.add_development_dependency "bundler", "~> 1.11" + spec.add_development_dependency "rake", "~> 10.0" + spec.add_development_dependency "minitest", "~> 5.0" +end diff --git a/test/syskit/pocolog/rock_stream_matcher_test.rb b/test/syskit/pocolog/rock_stream_matcher_test.rb new file mode 100644 index 0000000..682f433 --- /dev/null +++ b/test/syskit/pocolog/rock_stream_matcher_test.rb @@ -0,0 +1,67 @@ +require 'test_helper' + +module Syskit + module Pocolog + describe RockStreamMatcher do + attr_reader :streams + before do + path, _ = create_log_file 'test' + create_log_stream 'task.port', '/double', rock_stream_type: 'port', rock_task_object_name: 'port', rock_task_name: 'task' + create_log_stream 'task.property', '/int', rock_stream_type: 'property', rock_task_object_name: 'property', rock_task_name: 'task' + create_log_stream 'stream_without_properties', '/int' + create_log_stream 'other_task.port', '/int', rock_task_name: 'other_task' + create_log_stream 'stream_with_task_model', '/int', rock_task_model: 'orogen_model::Test' + flush_log_file + + @streams = Streams.new + streams.add_file path + end + subject { RockStreamMatcher.new } + + def assert_finds_streams(query, *stream_names) + assert_equal stream_names, streams.find_all_streams(query).map(&:name) + end + + describe "matching the stream type" do + it "matches against ports" do + assert_finds_streams subject.ports, 'task.port' + end + it "matches against properties" do + assert_finds_streams subject.properties, 'task.property' + end + it "never matches streams that do not have the property" do + end + it "ORs ports and properties if both are specified" do + assert_finds_streams subject.ports.properties, 'task.port', 'task.property' + end + end + + describe "matching the task name" do + it "matches tasks that have the name" do + assert_finds_streams subject.task_name('task'), 'task.port', 'task.property' + end + it "ORs the different names" do + assert_finds_streams subject.task_name("task").task_name("other_task"), 'task.port', 'task.property', 'other_task.port' + end + end + + describe "matching the object name" do + it "matches objects that have the name" do + assert_finds_streams subject.object_name('port'), 'task.port' + end + it "ORs the different names" do + assert_finds_streams subject.object_name("port").object_name("property"), 'task.port', 'task.property' + end + end + + describe "the task model" do + it "matches the task model by name" do + task_m = Syskit::TaskContext.new_submodel + flexmock(task_m.orogen_model, :strict, name: 'orogen_model::Test') + assert_finds_streams subject.task_model(task_m), 'stream_with_task_model' + end + end + end + end +end + diff --git a/test/syskit/pocolog/streams_test.rb b/test/syskit/pocolog/streams_test.rb new file mode 100644 index 0000000..042c044 --- /dev/null +++ b/test/syskit/pocolog/streams_test.rb @@ -0,0 +1,111 @@ +require 'test_helper' + +module Syskit + module Pocolog + describe Streams do + subject { Streams.new } + + describe "#add_file" do + it "adds the file's streams to the object" do + logfile_path, logfile = create_log_file 'test' + create_log_stream '/task.file', '/double' + flush_log_file + subject.add_file(logfile_path) + assert_equal ['/task.file'], subject.each_stream.map(&:name) + end + + it "raises if the file does not exist" do + assert_raises(ArgumentError) { subject.add_file(Pathname('does_not_exist')) } + end + end + + describe ".from_dir" do + it "creates a new streams object and adds the dir converted to pathname" do + flexmock(Streams).new_instances.should_receive(:add_dir).once.with(Pathname.new('test')) + assert_kind_of Streams, Streams.from_dir('test') + end + end + + describe ".from_file" do + it "creates a new streams object and adds the file converted to pathname" do + flexmock(Streams).new_instances.should_receive(:add_file).once.with(Pathname.new('test.0.log')) + assert_kind_of Streams, Streams.from_file('test.0.log') + end + end + + describe "#add_dir" do + it "ignores the files that do not match the .NUM.log pattern" do + create_log_dir + FileUtils.touch((created_log_dir + "a.file").to_s) + flexmock(subject).should_receive(:add_file).never + subject.add_dir(created_log_dir) + end + it "adds files that match the .NUM.log pattern" do + create_log_file 'test0' + create_log_file 'test1' + create_log_file 'test2' + flexmock(subject).should_receive(:add_file). + with(created_log_dir + 'test0.0.log').once + flexmock(subject).should_receive(:add_file). + with(created_log_dir + 'test1.0.log').once + flexmock(subject).should_receive(:add_file). + with(created_log_dir + 'test2.0.log').once + subject.add_dir(created_log_dir) + end + end + + describe "#find_all_streams" do + it "returns the streams that match the object" do + logfile_path, _ = create_log_file 'test' + create_log_stream '/task.file', '/double' + create_log_stream '/other.task.file', '/double' + create_log_stream '/does.not.match', '/double' + flush_log_file + subject.add_dir(created_log_dir) + + streams = subject.streams + + query = flexmock + query.should_receive(:===). + with(->(s) { streams.include?(s) }). + and_return { |s| s != streams[2] } + assert_equal streams[0, 2], subject.find_all_streams(query) + end + end + + describe "#find_task_by_name" do + before do + logfile_path, _ = create_log_file 'test' + create_log_stream '/test0', '/double', 'rock_task_name' => "task" + create_log_stream '/test1', '/double', 'rock_task_name' => "task" + create_log_stream '/does.not.match', '/double', 'rock_task_name' => 'another_task' + flush_log_file + subject.add_dir(created_log_dir) + end + + it "returns nil if there are no matching tasks" do + assert !subject.find_task_by_name('does_not_exist') + end + + it "returns a TaskStreams object with the matching streams" do + streams = subject.find_task_by_name('task') + assert_kind_of TaskStreams, streams + assert_equal Set['/test0', '/test1'], streams.each_stream.map(&:name).to_set + end + + describe "method_missing accessor" do + it "returns the streams" do + streams = subject.task_task + assert_kind_of TaskStreams, streams + assert_equal Set['/test0', '/test1'], streams.each_stream.map(&:name).to_set + end + it "raises NoMethodError if no task exists" do + assert_raises(NoMethodError) do + subject.does_not_exist_task + end + end + end + end + end + end +end diff --git a/test/syskit/pocolog/task_streams_test.rb b/test/syskit/pocolog/task_streams_test.rb new file mode 100644 index 0000000..868d369 --- /dev/null +++ b/test/syskit/pocolog/task_streams_test.rb @@ -0,0 +1,104 @@ +require 'test_helper' + +module Syskit + module Pocolog + describe TaskStreams do + attr_reader :subject + before do + create_log_file 'test' + create_log_stream '/port0', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object0', + 'rock_stream_type' => 'port' + create_log_stream '/port1_1', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object1', + 'rock_stream_type' => 'port' + create_log_stream '/port1_2', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object1', + 'rock_stream_type' => 'port' + create_log_stream '/property0', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object0', + 'rock_stream_type' => 'property' + create_log_stream '/property1_1', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object1', + 'rock_stream_type' => 'property' + create_log_stream '/property1_2', '/double', + 'rock_task_name' => "task", + 'rock_task_object_name' => 'object1', + 'rock_stream_type' => 'property' + flush_log_file + streams = Streams.from_dir(created_log_dir) + @subject = streams.find_task_by_name('task') + end + + describe "#find_port_by_name" do + it "returns nil if there are no matches" do + assert !subject.find_port_by_name('does_not_exist') + end + it "returns the matching port stream" do + object = subject.find_port_by_name('object0') + assert_kind_of ::Pocolog::DataStream, object + assert_equal '/port0', object.name + end + it "raises Ambiguous if there are more than one port with the given name" do + assert_raises(Ambiguous) do + subject.find_port_by_name('object1') + end + end + + describe "access through #method_missing" do + it "returns a single match if there is one" do + assert_equal '/port0', subject.object0_port.name + end + it "raises Ambiguous for multiple matches" do + assert_raises(Ambiguous) do + subject.object1_port + end + end + it "raises NoMethodError if there are no matches" do + assert_raises(NoMethodError) do + subject.does_not_exist_port + end + end + end + end + + describe "#find_property_by_name" do + it "returns nil if there are no matches" do + assert !subject.find_property_by_name('does_not_exist') + end + it "returns the matching port stream" do + object = subject.find_property_by_name('object0') + assert_kind_of ::Pocolog::DataStream, object + assert_equal '/property0', object.name + end + it "raises Ambiguous if there are more than one port with the given name" do + assert_raises(Ambiguous) do + subject.find_property_by_name('object1') + end + end + + describe "access through #method_missing" do + it "returns a single match if there is one" do + assert_equal '/property0', subject.object0_property.name + end + it "raises Ambiguous for multiple matches" do + assert_raises(Ambiguous) do + subject.object1_property + end + end + it "raises NoMethodError if there are no matches" do + assert_raises(NoMethodError) do + subject.does_not_exist_property + end + end + end + end + end + end +end + diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..0550cb6 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,63 @@ +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) +require 'syskit/test/self' +require 'syskit/pocolog' +require 'minitest/autorun' + +module Syskit + module Pocolog + module Test + attr_reader :created_log_dir + attr_reader :created_log_file + + def setup + @pocolog_log_level = ::Pocolog.logger.level + ::Pocolog.logger.level = Logger::WARN + super + end + + def teardown + if @pocolog_log_level + ::Pocolog.logger.level = @pocolog_log_level + end + if created_log_dir + created_log_dir.rmtree + end + super + end + + # Create a temporary directory to be used by the other log-related + # helpers + def create_log_dir + @created_log_dir ||= Pathname.new(Dir.mktmpdir('syskit-pocolog-test')) + end + + # Create a log file in a temporary directory + # + # @return [Pathname,Pocolog::Logfiles] the path to the file and the + # file object + def create_log_file(filename, *typenames) + create_log_dir + path = created_log_dir + filename + + registry = Typelib::Registry.new + + @created_log_file = ::Pocolog::Logfiles.create(path.to_s, registry) + return path.sub_ext(".0.log"), created_log_file + end + + # Write all pending changes done to {#created_log_file} on disk + def flush_log_file + created_log_file.io.each(&:flush) + end + + # Create a log stream on the last file created with + # {#create_log_file} + def create_log_stream(name, typename, metadata = Hash.new) + registry = Typelib::Registry.new + type = registry.create_null typename + created_log_file.create_stream name, type, metadata + end + end + end +end +Minitest::Test.include Syskit::Pocolog::Test