feat: move reporting support from a private repo to syskit-log directly

README.md

@@ -8,6 +8,8 @@ Add `tools/syskit-log` in `autoproj/manifest`
 
 ## Usage
 
+See [the rock-and-syskit page](https://www.rock-robotics.org/rock-and-syskit/log_management/)
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/rock-core/tools-syskit-log.

lib/syskit/log/cli/datastore.rb

@@ -8,6 +8,7 @@
 require "syskit/log/datastore/normalize"
 require "syskit/log/datastore/import"
 require "syskit/log/datastore/index_build"
+require "syskit/log/cli/reports"
 require "tty-progressbar"
 require "tty-prompt"
 require "pocolog/cli/null_reporter"
@@ -19,6 +20,9 @@ module CLI
         class Datastore < Thor # rubocop:disable Metrics/ClassLength
             namespace "datastore"
 
+            desc "reports", "Generation of HTML reports using Jupyter notebooks"
+            subcommand "reports", Reports
+
             class_option :silent, type: :boolean, default: false
             class_option :colors, type: :boolean, default: TTY::Color.color?
             class_option :progress, type: :boolean, default: TTY::Color.color?

lib/syskit/log/cli/reports.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "syskit/log/reports"
+
+module Syskit
+    module Log
+        module CLI
+            # Subcommand that allow to generate HTML reports from datasets using
+            # Jupyter notebooks
+            class Reports < Thor
+                no_commands do # rubocop:disable Metrics/BlockLength
+                    # Generate an output name from the dataset digest
+                    def default_output_name(dataset)
+                        if dataset.respond_to?(:to_str)
+                            dataset = Syskit::Log::Datastore.default.get(dataset)
+                        end
+
+                        description =
+                            dataset.metadata_fetch("description", "No description")
+                        date = dataset.timestamp.strftime("%Y-%m-%d")
+                        "#{date} - #{description} (#{dataset.digest[0, 10]})"
+                    end
+
+                    def parse_query(vars)
+                        query = {}
+                        vars.each do |str|
+                            key, op, value = match_single_query(str)
+
+                            value = Regexp.new(value) if op == "~"
+
+                            (query[key] ||= []) << value
+                        end
+                        query
+                    end
+
+                    def match_single_query(str)
+                        unless (match = /^([^~=]+)([~=])(.*)$/.match(str))
+                            raise ArgumentError,
+                                  "metadata entries must be given as key=value or "\
+                                  "key~value"
+                        end
+
+                        [match[1], match[2], match[3]]
+                    end
+
+                    def html_write_metadata(output, **metadata)
+                        output.write JSON.generate(metadata)
+                    end
+
+                    def auto_html_processed?(output)
+                        json = output.sub_ext(".json")
+                        return false unless json.file?
+
+                        json = JSON.parse(json.read)
+                        !json.key?("error")
+                    end
+
+                    def auto_html_generate(template, dataset, output)
+                        name = "#{default_output_name(dataset)}.html"
+                        puts "Processing of #{dataset.digest}: #{name}"
+
+                        html(template, dataset.digest, output.sub_ext(".html"),
+                             log: output.sub_ext(".log"))
+                        name
+                    end
+
+                    def auto_html_save_result(dataset, output, name)
+                        output.sub_ext(".json").write(
+                            JSON.generate({ digest: dataset.digest, name: name })
+                        )
+                    end
+
+                    def auto_html_save_error(dataset, output, name, error)
+                        output.sub_ext(".json").write(
+                            JSON.generate(
+                                digest: dataset.digest,
+                                name: name,
+                                error: { message: error.message,
+                                         backtrace: error.backtrace }
+                            )
+                        )
+                    end
+
+                    def auto_html_dataset(template, dataset, output_dir)
+                        output = output_dir / dataset.digest
+                        return if !options[:force] && auto_html_processed?(output)
+
+                        name = auto_html_generate(template, dataset, output)
+                        auto_html_save_result(dataset, output, name)
+                    rescue StandardError => e
+                        puts "  Failed: #{e.message}"
+                        puts "    #{e.backtrace.join("\n    ")}"
+                        auto_html_save_error(dataset, output, name, e)
+                    end
+
+                    def render_single_notebook(output, path, contents)
+                        output_path = output / path.basename
+
+                        contents.each_with_index do |(_, c), i|
+                            final_output =
+                                if contents.size > 1
+                                    output_path.sub_ext(".#{i}#{output_path.extname}")
+                                else
+                                    output_path
+                                end
+
+                            final_output.write(JSON.dump(c))
+                        end
+                    end
+                end
+
+                desc "auto-html TEMPLATE OUTPUT QUERY",
+                     "render this template to HTML for every dataset "\
+                     "that has not been generated yet"
+                option :force, type: :boolean, default: false
+                def auto_html(template, output_dir, *query)
+                    query = parse_query(query)
+
+                    output_dir = Pathname.new(output_dir)
+                    output_dir.mkpath
+                    datastore = Syskit::Log::Datastore.default
+                    datastore.find_all(query).each do |dataset|
+                        auto_html_dataset(template, dataset, output_dir)
+                    end
+                end
+
+                desc "html TEMPLATE DATASET [OUTPUT]",
+                     "render this template to HTML using data from the given dataset"
+                def html(template, dataset_digest, output = nil, log: nil)
+                    description =
+                        Syskit::Log::Reports::ReportDescription
+                        .load(Pathname.new(template), dataset_id: dataset_digest)
+
+                    output = Pathname(output) if output
+                    if !output || output.directory?
+                        output =
+                            (output || Pathname.pwd) /
+                            "#{default_output_name(dataset_digest)}.html"
+                    end
+                    description.to_html(Pathname.new(output), log: log)
+                end
+
+                desc "render-notebooks REPORT DATASET [OUTPUT]",
+                     "interpret each notebook from the REPORT report "\
+                     "and save them in the OUTPUT directory"
+                def render_notebooks(report, dataset_digest, output = nil)
+                    description = Syskit::Log::Reports::ReportDescription
+                                  .load(Pathname.new(report), dataset_id: dataset_digest)
+
+                    output = Pathname.new(output) if output
+                    if !output || output.directory?
+                        output_dir = output || Pathname.pwd
+                        output = output_dir / default_output_name(dataset_digest)
+                    end
+
+                    output.mkpath
+                    notebooks = description.each_loaded_notebook.group_by do |path, _|
+                        path
+                    end
+
+                    notebooks.each do |path, contents|
+                        render_single_notebook(output, path, contents)
+                    end
+                end
+            end
+        end
+    end
+end

lib/syskit/log/reports.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "erb"
+require "json"
+require "pathname"
+
+require "syskit/log/dsl"
+
+require "syskit/log/reports/notebooks"
+require "syskit/log/reports/report_description"
+
+module Syskit
+    module Log
+        # Tooling related to generating reports from log datasets
+        module Reports
+            # Exception raised in {ReportDescription#to_json} if no notebooks
+            # were added
+            class EmptyReport < RuntimeError
+            end
+        end
+    end
+end

lib/syskit/log/reports/notebooks.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Syskit
+    module Log
+        module Reports # :nodoc:
+            # Load Jupyter notebooks and generate a single notebook with the result
+            #
+            # The loading process interprets the notebooks as ERB templates,
+            # passing "vars" as local variables
+            #
+            # @param [Array<Pathname>] notebook_paths
+            def self.notebooks_load_and_concatenate(*notebook_paths, **vars)
+                notebooks = notebook_paths.map do |path|
+                    notebook_load(path, **vars)
+                end
+                notebooks_concatenate(notebooks)
+            end
+
+            # Generate a single notebook that is the concatenation of all the
+            # given notebooks
+            #
+            # @param [Array<Hash>] notebooks
+            # @return [Hash]
+            def self.notebooks_concatenate(notebooks)
+                result = notebooks.shift.dup
+                result["cells"] =
+                    notebooks.inject(result["cells"]) { |cells, nb| cells + nb["cells"] }
+                result
+            end
+
+            # Load the notebook's JSON
+            #
+            # @param [Pathname] path
+            # @return [Hash]
+            def self.notebook_load(path, **vars)
+                data = path.read
+                JSON.parse(ERB.new(data).result_with_hash(vars))
+            end
+        end
+    end
+end