diff --git a/README.md b/README.md index c3113de..bca01b2 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Add `tools/syskit-log` in `autoproj/manifest` ## Usage +See [the rock-and-syskit page](https://www.rock-robotics.org/rock-and-syskit/log_management/) + ## Contributing Bug reports and pull requests are welcome on GitHub at https://github.com/rock-core/tools-syskit-log. diff --git a/lib/syskit/log/cli/datastore.rb b/lib/syskit/log/cli/datastore.rb index 3ce10c8..06d3412 100644 --- a/lib/syskit/log/cli/datastore.rb +++ b/lib/syskit/log/cli/datastore.rb @@ -8,6 +8,7 @@ require "syskit/log/datastore/normalize" require "syskit/log/datastore/import" require "syskit/log/datastore/index_build" +require "syskit/log/cli/reports" require "tty-progressbar" require "tty-prompt" require "pocolog/cli/null_reporter" @@ -19,6 +20,9 @@ module CLI class Datastore < Thor # rubocop:disable Metrics/ClassLength namespace "datastore" + desc "reports", "Generation of HTML reports using Jupyter notebooks" + subcommand "reports", Reports + class_option :silent, type: :boolean, default: false class_option :colors, type: :boolean, default: TTY::Color.color? class_option :progress, type: :boolean, default: TTY::Color.color? diff --git a/lib/syskit/log/cli/reports.rb b/lib/syskit/log/cli/reports.rb new file mode 100644 index 0000000..16eb1c7 --- /dev/null +++ b/lib/syskit/log/cli/reports.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +require "syskit/log/reports" + +module Syskit + module Log + module CLI + # Subcommand that allow to generate HTML reports from datasets using + # Jupyter notebooks + class Reports < Thor + no_commands do # rubocop:disable Metrics/BlockLength + # Generate an output name from the dataset digest + def default_output_name(dataset) + if dataset.respond_to?(:to_str) + dataset = Syskit::Log::Datastore.default.get(dataset) + end + + description = + dataset.metadata_fetch("description", "No description") + date = dataset.timestamp.strftime("%Y-%m-%d") + "#{date} - #{description} (#{dataset.digest[0, 10]})" + end + + def parse_query(vars) + query = {} + vars.each do |str| + key, op, value = match_single_query(str) + + value = Regexp.new(value) if op == "~" + + (query[key] ||= []) << value + end + query + end + + def match_single_query(str) + unless (match = /^([^~=]+)([~=])(.*)$/.match(str)) + raise ArgumentError, + "metadata entries must be given as key=value or "\ + "key~value" + end + + [match[1], match[2], match[3]] + end + + def html_write_metadata(output, **metadata) + output.write JSON.generate(metadata) + end + + def auto_html_processed?(output) + json = output.sub_ext(".json") + return false unless json.file? + + json = JSON.parse(json.read) + !json.key?("error") + end + + def auto_html_generate(template, dataset, output) + name = "#{default_output_name(dataset)}.html" + puts "Processing of #{dataset.digest}: #{name}" + + html(template, dataset.digest, output.sub_ext(".html"), + log: output.sub_ext(".log")) + name + end + + def auto_html_save_result(dataset, output, name) + output.sub_ext(".json").write( + JSON.generate({ digest: dataset.digest, name: name }) + ) + end + + def auto_html_save_error(dataset, output, name, error) + output.sub_ext(".json").write( + JSON.generate( + digest: dataset.digest, + name: name, + error: { message: error.message, + backtrace: error.backtrace } + ) + ) + end + + def auto_html_dataset(template, dataset, output_dir) + output = output_dir / dataset.digest + return if !options[:force] && auto_html_processed?(output) + + name = auto_html_generate(template, dataset, output) + auto_html_save_result(dataset, output, name) + rescue StandardError => e + puts " Failed: #{e.message}" + puts " #{e.backtrace.join("\n ")}" + auto_html_save_error(dataset, output, name, e) + end + + def render_single_notebook(output, path, contents) + output_path = output / path.basename + + contents.each_with_index do |(_, c), i| + final_output = + if contents.size > 1 + output_path.sub_ext(".#{i}#{output_path.extname}") + else + output_path + end + + final_output.write(JSON.dump(c)) + end + end + end + + desc "auto-html TEMPLATE OUTPUT QUERY", + "render this template to HTML for every dataset "\ + "that has not been generated yet" + option :force, type: :boolean, default: false + def auto_html(template, output_dir, *query) + query = parse_query(query) + + output_dir = Pathname.new(output_dir) + output_dir.mkpath + datastore = Syskit::Log::Datastore.default + datastore.find_all(query).each do |dataset| + auto_html_dataset(template, dataset, output_dir) + end + end + + desc "html TEMPLATE DATASET [OUTPUT]", + "render this template to HTML using data from the given dataset" + def html(template, dataset_digest, output = nil, log: nil) + description = + Syskit::Log::Reports::ReportDescription + .load(Pathname.new(template), dataset_id: dataset_digest) + + output = Pathname(output) if output + if !output || output.directory? + output = + (output || Pathname.pwd) / + "#{default_output_name(dataset_digest)}.html" + end + description.to_html(Pathname.new(output), log: log) + end + + desc "render-notebooks REPORT DATASET [OUTPUT]", + "interpret each notebook from the REPORT report "\ + "and save them in the OUTPUT directory" + def render_notebooks(report, dataset_digest, output = nil) + description = Syskit::Log::Reports::ReportDescription + .load(Pathname.new(report), dataset_id: dataset_digest) + + output = Pathname.new(output) if output + if !output || output.directory? + output_dir = output || Pathname.pwd + output = output_dir / default_output_name(dataset_digest) + end + + output.mkpath + notebooks = description.each_loaded_notebook.group_by do |path, _| + path + end + + notebooks.each do |path, contents| + render_single_notebook(output, path, contents) + end + end + end + end + end +end diff --git a/lib/syskit/log/reports.rb b/lib/syskit/log/reports.rb new file mode 100644 index 0000000..81b2c4b --- /dev/null +++ b/lib/syskit/log/reports.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require "erb" +require "json" +require "pathname" + +require "syskit/log/dsl" + +require "syskit/log/reports/notebooks" +require "syskit/log/reports/report_description" + +module Syskit + module Log + # Tooling related to generating reports from log datasets + module Reports + # Exception raised in {ReportDescription#to_json} if no notebooks + # were added + class EmptyReport < RuntimeError + end + end + end +end diff --git a/lib/syskit/log/reports/notebooks.rb b/lib/syskit/log/reports/notebooks.rb new file mode 100644 index 0000000..219f7cb --- /dev/null +++ b/lib/syskit/log/reports/notebooks.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Syskit + module Log + module Reports # :nodoc: + # Load Jupyter notebooks and generate a single notebook with the result + # + # The loading process interprets the notebooks as ERB templates, + # passing "vars" as local variables + # + # @param [Array] notebook_paths + def self.notebooks_load_and_concatenate(*notebook_paths, **vars) + notebooks = notebook_paths.map do |path| + notebook_load(path, **vars) + end + notebooks_concatenate(notebooks) + end + + # Generate a single notebook that is the concatenation of all the + # given notebooks + # + # @param [Array] notebooks + # @return [Hash] + def self.notebooks_concatenate(notebooks) + result = notebooks.shift.dup + result["cells"] = + notebooks.inject(result["cells"]) { |cells, nb| cells + nb["cells"] } + result + end + + # Load the notebook's JSON + # + # @param [Pathname] path + # @return [Hash] + def self.notebook_load(path, **vars) + data = path.read + JSON.parse(ERB.new(data).result_with_hash(vars)) + end + end + end +end diff --git a/lib/syskit/log/reports/report_description.rb b/lib/syskit/log/reports/report_description.rb new file mode 100644 index 0000000..484e40a --- /dev/null +++ b/lib/syskit/log/reports/report_description.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +module Syskit + module Log + module Reports + # Representation of a report before it gets processed + class ReportDescription + # Load a report template from its description + # + # @param [Pathname] path + # @param vars set of variables that should be set while evaluating the + # report description file + # @return [ReportTemplate] + def self.load(path, dataset_id: nil, **vars) + template = new + template.set(:dataset_id, dataset_id) + vars.each { |k, v| template.set(k, v) } + template.load(path, dataset_id: dataset_id) + template + end + + def initialize + @notebooks = [] + @vars = {} + end + + # Append a template description + def load(path, dataset_id: @dataset_id) + @template_path = [path.dirname, global_notebook_path] + context = EvaluationContext.new(self, @vars) + context.dataset_select dataset_id if dataset_id + context.instance_eval(path.read, path.to_s, 1) + end + + # Set a variable to be passed to the notebook templates + def set(name, value) + @vars[name.to_sym] = value + end + + Notebook = Struct.new :path, :vars + + # Append a notebook to the report + # + # @param [Pathname] notebook_path the notebook path. Relative paths are + # resolved w.r.t. this package's templates folder + def add_notebook(notebook_path, **vars) + @notebooks << Notebook.new( + resolve_notebook_path(Pathname.new(notebook_path)), + vars + ) + end + + # Render this report to JSON + # + # @raise EmptyReport if this report does not have any notebooks + def to_json(*) + notebooks = each_loaded_notebook.map { |_, json| json } + if notebooks.empty? + raise EmptyReport, "cannot generate a report without notebooks" + end + + Reports.notebooks_concatenate(notebooks) + end + + # Render this report to HTML + # + # @param [Pathname,String] output path to the generated HTML + # + # @raise EmptyReport if this report does not have any notebooks + def to_html(output, log: nil) + json = to_json + redirect = { out: log.to_s, err: log.to_s } if log + + IO.popen( + ["jupyter-nbconvert", "--execute", "--allow-errors", "--stdin", + "--output=#{output}", "--no-input"], "w", **(redirect || {}) + ) do |io| + io.write JSON.dump(json) + end + end + + # Enumerate the notebooks that are part of this report + # + # @yieldparam [Pathname] path the path to the notebook on disk + # @yieldparam [Hash] vars the notebook variables + def each_notebook(&block) + @notebooks.each(&block) + end + + # Load the notebooks that are part of this report and yield them + # + # @yieldparam [Pathname] path the file path + # @yieldparam [Hash] contents the notebook contents + def each_loaded_notebook + return enum_for(__method__) unless block_given? + + each_notebook do |nb| + loaded = Reports.notebook_load(nb.path, **@vars.merge(nb.vars)) + yield nb.path, loaded + end + end + + # @api private + # + # Resolve a notebook path against this report's search path + # + # The method returns the path as-is if absolute. If relative, it + # will check the report's search path, which is first the directory + # from which the report description file was loaded, and second the + # global template dir (this repository's 'template' folder) + # + # @param [Pathname] notebook_path + # @return [Pathname] + # @raise ArgumentError + def resolve_notebook_path(notebook_path) + if notebook_path.absolute? + return notebook_path if notebook_path.file? + + raise ArgumentError, "#{notebook_path} does not exist" + end + + @template_path.each do |ref_path| + absolute = notebook_path.expand_path(ref_path) + return absolute if absolute.file? + end + + raise ArgumentError, "cannot find #{notebook_path}" + end + + # Path to the templates that are within this package + # + # @return [Pathname] + def global_notebook_path + Pathname.new(__dir__) / ".." / ".." / ".." / "templates" + end + + # @api private + # + # Context object used to evaluate report description files + class EvaluationContext < Object + include Syskit::Log::DSL + + def initialize(template, vars) + @template = template + @vars = vars + + __syskit_log_dsl_initialize + end + + def respond_to_missing?(name, include_private) + super || (@vars.key?(name) || @template.respond_to?(name)) + end + + def method_missing(name, *args, &block) # rubocop:disable Style/MethodMissingSuper + if @vars.key?(name) + unless args.empty? + raise ArgumentError, + "expected zero argument, got #{args.size}" + end + return @vars[name] + end + + @template.send(name, *args, &block) + end + end + end + end + end +end diff --git a/test/reports/helpers.rb b/test/reports/helpers.rb new file mode 100644 index 0000000..e772add --- /dev/null +++ b/test/reports/helpers.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "minitest/spec" +require "minitest/autorun" +require "tmpdir" + +require "syskit/log/reports" + +# Helper functions for tests in tw/logs/reporting +module TestHelpers + def setup + super + + @tmpdirs = [] + end + + def teardown + super + + @tmpdirs.each(&:rmtree) + end + + def make_tmppath + dir = Pathname.new(Dir.mktmpdir) + @tmpdirs << dir + dir + end + + def create_notebook(dir, name, cells: [], **metadata) + (dir / name).open("w") do |io| + io << JSON.generate( + { + "cells" => cells, + "metadata" => metadata, + "nbformat" => 4, + "nbformat_minor" => 4 + } + ) + end + end +end diff --git a/test/reports/test_notebooks.rb b/test/reports/test_notebooks.rb new file mode 100644 index 0000000..7bfe504 --- /dev/null +++ b/test/reports/test_notebooks.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require_relative "helpers" + +module Syskit + module Log + module Reports + describe "notebooks" do + describe ".notebooks_load_and_concatenate" do + include TestHelpers + + before do + @tmp = make_tmppath + end + + it "interprets the givens paths as ERB and concatenates the cells" do + one = make_template("1.txt", { "cells" => ["<%= 1 %>"] }) + two = make_template("2.txt", { "cells" => ["<%= 2 %>"] }) + + result = Reports.notebooks_load_and_concatenate(one, two) + assert_equal %w[1 2], result["cells"] + end + + it "uses the metadata from the first" do + one = make_template( + "1.txt", + { "cells" => ["<%= 1 %>"], + "metadata" => { "some" => "thing" } } + ) + two = make_template( + "2.txt", + { "cells" => ["<%= 2 %>"], + "metadata" => { "some" => "thingelse" } } + ) + + result = Reports.notebooks_load_and_concatenate(one, two) + assert_equal({ "some" => "thing" }, result["metadata"]) + end + + it "passes the given variables to the template" do + one = make_template("1.txt", { "cells" => ["<%= one %>"] }) + two = make_template("2.txt", { "cells" => ["<%= two %>"] }) + + result = Reports.notebooks_load_and_concatenate( + one, two, one: 1, two: 2 + ) + assert_equal %w[1 2], result["cells"] + end + + it "errors if some variables do not exist" do + one = make_template("1.txt", { "cells" => ["<%= one %>"] }) + + e = assert_raises(NameError) do + Reports.notebooks_load_and_concatenate(one) + end + assert_equal :one, e.name + end + + # Create a temporary template file and return its full path + def make_template(name, json) + path = @tmp / name + path.write(JSON.dump(json)) + path + end + end + end + end + end +end diff --git a/test/reports/test_report_description.rb b/test/reports/test_report_description.rb new file mode 100644 index 0000000..f3ba162 --- /dev/null +++ b/test/reports/test_report_description.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require_relative "helpers" + +module Syskit + module Log + module Reports + describe ReportDescription do + include TestHelpers + + before do + @tmpdir = make_tmppath + end + + it "reports an error if there are no notebooks" do + report = ReportDescription.new + assert_raises(EmptyReport) do + report.to_json + end + end + + it "concatenates the added notebooks" do + report = ReportDescription.new + create_notebook @tmpdir, "report1", cells: ["1"] + create_notebook @tmpdir, "report2", cells: ["2"] + + report.add_notebook @tmpdir / "report1" + report.add_notebook @tmpdir / "report2" + result = report.to_json + + assert_equal %w[1 2], result["cells"] + end + + it "evaluates the added notebooks with ERB "\ + "using variables passed to 'set'" do + report = ReportDescription.new + report.set "some", "data" + create_notebook @tmpdir, "report1", cells: ["<%= some %>"] + report.add_notebook @tmpdir / "report1" + result = report.to_json + + assert_equal %w[data], result["cells"] + end + + it "overrides variables passed to 'set' using the variables passed "\ + "to add_notebook" do + report = ReportDescription.new + report.set "some", "data" + create_notebook @tmpdir, "report1", cells: ["<%= some %>"] + report.add_notebook @tmpdir / "report1", some: "42" + result = report.to_json + + assert_equal %w[42], result["cells"] + end + + it "raises on evaluation if a variable does not exist" do + report = ReportDescription.new + create_notebook @tmpdir, "report1", cells: ["<%= does_not_exist %>"] + report.add_notebook @tmpdir / "report1" + assert_raises(NameError) do + report.to_json + end + end + + it "allows to load the report description from a DSL-like file" do + create_notebook @tmpdir, "report1", cells: ["<%= everything %>"] + create_notebook @tmpdir, "report2", cells: ["<%= half_of_it %>"] + (@tmpdir / "report.rb").write <<~REPORT + set "everything", "42" + add_notebook "report1" + add_notebook "report2" + REPORT + + report = ReportDescription.new + report.set :half_of_it, 21 + report.load(@tmpdir / "report.rb") + result = report.to_json + assert_equal %w[42 21], result["cells"] + end + + it "provides with a class method to create and load the report object" do + create_notebook @tmpdir, "report1", cells: ["<%= everything %>"] + create_notebook @tmpdir, "report2", cells: ["<%= half_of_it %>"] + (@tmpdir / "report.rb").write <<~REPORT + set "everything", "42" + add_notebook "report1" + add_notebook "report2" + REPORT + + report = ReportDescription.load( + @tmpdir / "report.rb", half_of_it: "21" + ) + result = report.to_json + assert_equal %w[42 21], result["cells"] + end + end + end + end +end