rest: add transformation and disaggregation functions

Changes

@@ -1,5 +1,9 @@
 This file documents the revision history for the Monitoring Webinterface Thruk.
 
+next:
+          - Rest:
+            - add transformation and disaggregation functions
+
 3.14.2   Sat Mar 23 19:21:39 CET 2024
           - fix embedding 3rd party tools
           - fix inline search

docs/documentation/rest.asciidoc

@@ -269,6 +269,12 @@ Rename columns by appending `:name` like this:
   %> thruk r /hosts?columns=name:host_name,state:status
 ------
 
+or by adding `as aliasname`
+
+------
+  %> thruk r '/hosts?columns=name:host_name,state as status'
+------
+
 
 === Performance Data Columns
 
@@ -369,7 +375,9 @@ Those columns can be used for filtering or to only select specific custom values
 ]
 ------
 
-== Transformation Functions
+== Functions
+
+=== Transformation Functions
 
 Transformation functions can be used to adjust result data.
 
@@ -391,7 +399,7 @@ ex.: return first 3 characters from upper case host name.
 ------
 
 
-== Aggregation Functions
+=== Aggregation Functions
 
 Aggregation functions can be used to get statistical information.
 
@@ -440,6 +448,38 @@ Aggregation functions can be used in filter (alias cannot be used in filter):
   %> thruk r '/services?columns=avg(execution_time):exec_time&avg(execution_time)[gt]=0'
 ------
 
+=== Disaggregation Functions
+
+Disaggregation functions create multiple result rows from a single attribute.
+
+Available disaggregation functions are:
+
+  * `to_rows`: allocate result rows from list column
+  * `as_rows`: alias for to_rows
+
+------
+  %> thruk r '/hosts?columns=name,to_rows(services) as services'
+------
+
+Of course, this example could be achieved easier but creating a query on the services
+table, but you get the idea.
+
+=== Mixing Functions
+
+When mixing aggregation and transformation functions, apply the aggregation function
+as the outer most function, as for example in:
+
+------
+  %> thruk r '/hosts/localhost?columns=avg(unit(calc(latency,/,1000), "ms"))'
+------
+
+When mixing disaggregation and transformation functions, apply the disaggregation function
+as the inner most function, as for example in:
+
+------
+  %> thruk r '/hosts/localhost?columns=upper(name),upper(to_rows(services))'
+------
+
 == Backends / Sites
 
 If you have multiple sites connected to Thruk, you may want to talk only to

lib/Thruk/Controller/rest_v1.pm

@@ -38,6 +38,10 @@ our $rest_paths = [];
 
 my $reserved_query_parameters  = [qw/limit offset sort columns headers backend backends q CSRFtoken/];
 my $aggregation_function_names = [qw/count sum avg min max/];
+my $disaggregation_function_names  = [qw/as_rows to_rows/];
+my $aggregation_functions      = Thruk::Base::array2hash($aggregation_function_names);
+my $disaggregation_functions   = Thruk::Base::array2hash($disaggregation_function_names);
+
 my $op_translation_words       = {
     'eq'     => '=',
     'ne'     => '!=',
@@ -484,6 +488,9 @@ sub _post_processing {
             $data = _apply_filter($c, $data, PRE_STATS);
             return($data) if _is_failed($data);
 
+            # expand rows (disaggregation functions)
+            $data = _apply_disaggregation($c, $data);
+
             # calculate statistics
             $data = _apply_stats($c, $data);
         }
@@ -631,13 +638,13 @@ sub _apply_stats {
     my($c, $data) = @_;
     return($data) unless $c->req->parameters->{'columns'};
 
-    my $aggregation_functions = Thruk::Base::array2hash($aggregation_function_names);
     my $new_columns   = [];
     my $stats_columns = [];
     my $group_columns = [];
     my $all_columns   = _parse_columns_data(get_request_columns($c, RAW));
     for my $col (@{$all_columns}) {
         # check for aggregation function which must be the outer most one
+        # TODO: check, functions are in reverse order, why check for 0?
         if(scalar @{$col->{'func'}} > 0 && $aggregation_functions->{$col->{'func'}->[0]->[0]}) {
             push @{$stats_columns}, { op => $col->{'func'}->[0]->[0], col => $col->{'column'} };
             push @{$new_columns}, $col;
@@ -760,6 +767,48 @@ sub _apply_stats {
     return($data);
 }
 
+##########################################################
+sub _apply_disaggregation {
+    my($c, $data) = @_;
+    return($data) unless $c->req->parameters->{'columns'};
+
+    my $disaggregation_columns = [];
+    my $all_columns   = _parse_columns_data(get_request_columns($c, RAW));
+    for my $col (@{$all_columns}) {
+        # check for disaggregation function which must be the inner most one
+        if(scalar @{$col->{'func'}} > 0 && $disaggregation_functions->{$col->{'func'}->[0]->[0]}) {
+            push @{$disaggregation_columns}, $col;
+        }
+    }
+
+    return($data) unless scalar @{$disaggregation_columns} > 0;
+    die("too many disaggregation function, do not use more than 2") if scalar @{$disaggregation_columns} >= 2;
+
+    my $expanded_data = [];
+    for my $col (@{$disaggregation_columns}) {
+        for my $f (@{$col->{'func'}}) {
+            next unless $disaggregation_functions->{$f->[0]};
+            if($f->[0] eq 'to_rows' || $f->[0] eq 'as_rows') {
+                for my $d (@{$data}) {
+                    my $val = $d->{$col->{'column'}};
+                    if(ref $val eq 'ARRAY') {
+                        for my $s (@{$val}) {
+                            my $cloned_row = Thruk::Utils::IO::dclone($d);
+                            $cloned_row->{$col->{'column'}} = $s;
+                            push @{$expanded_data}, $cloned_row;
+                        }
+                    }
+                }
+            } else {
+                die("unsupported function: ".$f->[0]);
+            }
+        }
+        $data = $expanded_data;
+    }
+
+    return($data);
+}
+
 ##########################################################
 
 =head2 get_request_columns
@@ -924,7 +973,6 @@ sub _apply_columns {
     my $columns = _parse_columns_data(get_request_columns($c, RAW));
 
     # remove aggregation functions
-    my $aggregation_functions = Thruk::Base::array2hash($aggregation_function_names);
     for my $col (@{$columns}) {
         if(scalar @{$col->{'func'}} > 0 && $aggregation_functions->{$col->{'func'}->[0]->[0]}) {
             $col->{'func'} = [];
@@ -1013,6 +1061,9 @@ sub _apply_data_function {
         return $val;
     }
 
+    return $val if $aggregation_functions->{$name};
+    return $val if $disaggregation_functions->{$name};
+
     die("unknown function: ".$name);
 }
 

t/scenarios/rest_api/t/local/rest_api_misc.t

@@ -8,7 +8,7 @@ BEGIN {
     require TestUtils;
     import TestUtils;
 }
-plan tests => 20;
+plan tests => 40;
 
 ###########################################################
 # test thruks script path
@@ -36,3 +36,29 @@ $ENV{'THRUK_TEST_AUTH_USER'} = "omdadmin";
         like => ['/"loc"/'],
     });
 };
+
+###########################################################
+# mixed stats and transformation
+{
+    TestUtils::test_command({
+        cmd  => '/usr/bin/env thruk r \'/hosts/localhost?columns=avg(unit(calc(last_check,/,1000), "ms")) as testcheck&headers=wrapped_json\'',
+        like => ['/"ms"/', '/"testcheck"/'],
+    });
+};
+
+###########################################################
+# disaggregation function
+{
+    TestUtils::test_command({
+        cmd  => '/usr/bin/env thruk r \'/hosts?columns=to_rows(services) as service&headers=wrapped_json\'',
+        like => ['/"Users"/', '/"service"/'],
+    });
+    TestUtils::test_command({
+        cmd  => '/usr/bin/env thruk r \'/hosts?columns=name,to_rows(services) as svc&headers=wrapped_json\'',
+        like => ['/"Users"/', '/"svc"/'],
+    });
+    TestUtils::test_command({
+        cmd  => '/usr/bin/env thruk r \'/hosts?columns=name,upper(to_rows(services)) as svc&headers=wrapped_json\'',
+        like => ['/"USERS"/', '/"svc"/'],
+    });
+};