Allow listing nodes and Volumes across the Pools

Signed-off-by: Aravinda Vishwanathapura <[email protected]>

clients/crystal/src/moana_client.cr

@@ -23,6 +23,14 @@ module MoanaClient
       Pool.list(self)
     end
 
+    def list_nodes(state = false)
+      Node.list(self, state)
+    end
+
+    def list_volumes(state = false)
+      Volume.list(self, state)
+    end
+
     def pool(name : String)
       Pool.new(self, name)
     end

clients/crystal/src/nodes.cr

@@ -34,5 +34,18 @@ module MoanaClient
         MoanaClient.error_response(response)
       end
     end
+
+    def self.list(client : Client, state = false)
+      url = "#{client.url}/api/v1/nodes?state=#{state ? 1 : 0}"
+      response = MoanaClient.http_get(
+        url,
+        headers: client.auth_header
+      )
+      if response.status_code == 200
+        Array(MoanaTypes::Node).from_json(response.body)
+      else
+        MoanaClient.error_response(response)
+      end
+    end
   end
 end

clients/crystal/src/volumes.cr

@@ -67,6 +67,19 @@ module MoanaClient
       end
     end
 
+    def self.list(client : Client, state = false)
+      url = "#{client.url}/api/v1/volumes?state=#{state ? 1 : 0}"
+      response = MoanaClient.http_get(
+        url,
+        headers: client.auth_header
+      )
+      if response.status_code == 200
+        Array(MoanaTypes::Volume).from_json(response.body)
+      else
+        MoanaClient.error_response(response)
+      end
+    end
+
     def start_stop_volume(action)
       url = "#{@client.url}/api/v1/pools/#{@pool_name}/volumes/#{@name}/#{action}"
 

mgr/src/cmds/nodes.cr

@@ -50,13 +50,13 @@ end
 
 handler "node.list" do |args|
   args.pool_name, _ = pool_and_node_name(args.pos_args.size < 1 ? "" : args.pos_args[0])
-  if args.pool_name == ""
-    STDERR.puts "Pool name is required."
-    exit 1
-  end
 
   api_call(args, "Failed to get list of nodes") do |client|
-    nodes = client.pool(args.pool_name).list_nodes(state: args.node_args.status)
+    if args.pool_name == ""
+      nodes = client.list_nodes(state: args.node_args.status)
+    else
+      nodes = client.pool(args.pool_name).list_nodes(state: args.node_args.status)
+    end
     puts "No nodes added to the Pool. Run `kadalu node add #{args.pool_name}/<node-name>` to add a node." if nodes.size == 0
 
     if args.node_args.status
@@ -69,9 +69,9 @@ handler "node.list" do |args|
 
     nodes.each do |node|
       if args.node_args.status
-        table.record(node.name, node.id, node.state, node.endpoint)
+        table.record("#{node.pool.name}/#{node.name}", node.id, node.state, node.endpoint)
       else
-        table.record(node.name, node.id, node.endpoint)
+        table.record("#{node.pool.name}/#{node.name}", node.id, node.endpoint)
       end
     end
 

mgr/src/cmds/volume_create_parser.cr

@@ -246,7 +246,7 @@ module VolumeRequestParser
 
       case token.kind
       when TokenKind::VolumeName  then req.name = token.value
-      when TokenKind::PoolName    then req.pool_name = token.value
+      when TokenKind::PoolName    then req.pool.name = token.value
       when TokenKind::StorageUnit then all_storage_units << token.value
       when TokenKind::TypeKeyword
         keyword = token.value
@@ -296,7 +296,7 @@ module VolumeRequestParser
   # Validate the Volume create request after parsing
   def self.validate(req)
     raise InvalidVolumeRequest.new("Volume name not specified (Example: mypool/vol1)") if req.name == ""
-    raise InvalidVolumeRequest.new("Pool name not specified (Example: mypool/vol1)") if req.pool_name == ""
+    raise InvalidVolumeRequest.new("Pool name not specified (Example: mypool/vol1)") if req.pool.name == ""
     raise InvalidVolumeRequest.new("Atleast one Storage unit is required") if req.distribute_groups.size == 0
 
     # TODO: Volume name validations

mgr/src/cmds/volumes.cr

@@ -25,7 +25,7 @@ handler "volume.create" do |args|
   begin
     req = VolumeRequestParser.parse(args.pos_args)
     req.no_start = args.volume_args.no_start
-    args.pool_name = req.pool_name
+    args.pool_name = req.pool.name
     api_call(args, "Failed to Create Volume") do |client|
       volume = client.pool(args.pool_name).create_volume(req)
       puts "Volume #{req.name} created successfully"
@@ -85,11 +85,12 @@ command "volume.list", "Volumes list of a Kadalu Storage Pool" do |parser, args|
 end
 
 def volume_detail(volume, status = false)
-  puts "Name                    : #{volume.name}"
+  health = volume.state == "Started" && status ? "#{volume.state} (#{volume.metrics.health})" : volume.state
+
+  puts "Name                    : #{volume.pool.name}/#{volume.name}"
   puts "Type                    : #{volume.type}"
   puts "ID                      : #{volume.id}"
-  puts "Status                  : #{volume.state}"
-  puts "Health                  : #{volume.metrics.health}" if status
+  puts "State                   : #{health}"
   puts "Size                    : #{(volume.metrics.size_used_bytes + volume.metrics.size_free_bytes).humanize_bytes}"
   puts "Inodes                  : #{(volume.metrics.inodes_used_count + volume.metrics.inodes_free_count).humanize}"
   puts "Utilization             : #{volume.metrics.size_used_bytes.humanize_bytes}/#{(volume.metrics.size_used_bytes + volume.metrics.size_free_bytes).humanize_bytes}" if status
@@ -120,13 +121,11 @@ end
 
 handler "volume.list" do |args|
   args.pool_name, args.volume_args.name = pool_and_volume_name(args.pos_args.size < 1 ? "" : args.pos_args[0])
-  if args.pool_name == ""
-    STDERR.puts "Pool name is required."
-    exit 1
-  end
 
   api_call(args, "Failed to get list of volumes") do |client|
-    if args.volume_args.name == ""
+    if args.pool_name == ""
+      volumes = client.list_volumes(state: args.volume_args.status)
+    elsif args.volume_args.name == ""
       volumes = client.pool(args.pool_name).list_volumes(state: args.volume_args.status)
     else
       volumes = [client.pool(args.pool_name).volume(args.volume_args.name).get(state: args.volume_args.status)]
@@ -153,7 +152,7 @@ handler "volume.list" do |args|
     volumes.each do |volume|
       if args.volume_args.status
         table.record(
-          volume.name,
+          "#{volume.pool.name}/#{volume.name}",
           volume.id,
           volume.state == "Started" ? "#{volume.state} (#{volume.metrics.health})" : volume.state,
           volume.type,
@@ -162,7 +161,7 @@ handler "volume.list" do |args|
         )
       else
         table.record(
-          volume.name,
+          "#{volume.pool.name}/#{volume.name}",
           volume.id,
           volume.state,
           volume.type,

mgr/src/server/datastore/nodes.cr

@@ -20,6 +20,10 @@ module Datastore
     "
   end
 
+  def self.nodes_query_order_by
+    " ORDER BY pools.created_on DESC, nodes.created_on DESC "
+  end
+
   private def self.grouped_nodes(nodes)
     nodes.map do |row|
       node = MoanaTypes::Node.new
@@ -34,8 +38,12 @@ module Datastore
     end
   end
 
+  def self.list_nodes
+    grouped_nodes(connection.query_all(nodes_query + nodes_query_order_by, as: NodeView))
+  end
+
   def self.list_nodes(pool_name)
-    grouped_nodes(connection.query_all(nodes_query, as: NodeView))
+    grouped_nodes(connection.query_all(nodes_query + " WHERE pools.name = ?" + nodes_query_order_by, pool_name, as: NodeView))
   end
 
   def self.get_nodes(pool_name, node_names)

mgr/src/server/datastore/volumes.cr

@@ -28,6 +28,8 @@ module Datastore
       volume.state = rows[0].state
       volume.metrics.size_bytes = rows[0].size_bytes
       volume.metrics.inodes_count = rows[0].inodes_count
+      volume.pool.id = rows[0].pool_id
+      volume.pool.name = rows[0].pool_name
 
       dist_grp_data = rows.group_by do |rec|
         [rec.distribute_group_index]
@@ -109,6 +111,13 @@ module Datastore
     " ORDER BY volumes.created_on DESC, distribute_groups.idx ASC, storage_units.idx ASC "
   end
 
+  def self.list_volumes
+    query = volumes_query + volumes_query_order_by
+    group_volumes(
+      connection.query_all(query, as: VolumeView)
+    )
+  end
+
   def self.list_volumes(pool_name)
     query = volumes_query + " WHERE pools.name = ?" + volumes_query_order_by
     group_volumes(

mgr/src/server/plugins/helpers.cr

@@ -13,7 +13,7 @@ post "/_api/v1/:action" do |env|
   rescue ex : Exception
     Log.error &.emit("#{action} Failed", error: "#{ex}")
     env.response.status_code = 500
-    resp = NodeResponse.new(false, {"error": "#{action} Failed"}.to_json)
+    resp = NodeResponse.new(false, ({"error": "#{action} Failed"}).to_json)
   end
 
   resp.status_code = env.response.status_code

mgr/src/server/plugins/nodes.cr

@@ -2,6 +2,26 @@ require "./helpers"
 require "../conf"
 require "../datastore/*"
 
+get "/api/v1/nodes" do |env|
+  state = env.params.query["state"]
+
+  nodes = Datastore.list_nodes
+
+  next nodes.to_json unless state
+
+  resp = dispatch_action(
+    ACTION_PING,
+    "",
+    nodes
+  )
+
+  nodes.each do |node|
+    node.state = resp.node_responses[node.name].ok ? "Up" : "Down"
+  end
+
+  nodes.to_json
+end
+
 get "/api/v1/pools/:pool_name/nodes" do |env|
   pool_name = env.params.url["pool_name"]
   state = env.params.query["state"]

mgr/src/server/plugins/volume_create.cr

@@ -42,15 +42,26 @@ post "/api/v1/pools/:pool_name/volumes" do |env|
 
   # TODO: Validate the request, dist count, storage_units count etc
 
+  nodes = [] of MoanaTypes::Node
+
   # Validate if the nodes are part of the Pool
-  node_names(req).each do |node|
-    unless Datastore.node_exists?(pool_name, node)
-      # TODO: Move this halt out of the Loop
-      halt(env, status_code: 400, response: ({"error": "Node #{node} is not part of the Pool"}.to_json))
+  # Also fetch the full node details
+  invalid_node = false
+  invalid_node_name = ""
+  participating_nodes(pool_name, req).each do |n|
+    node = Datastore.get_node(pool_name, n.name)
+    if node.nil?
+      invalid_node = true
+      invalid_node_name = n.name
+      break
     end
+    nodes << node
+  end
+
+  if invalid_node
+    halt(env, status_code: 400, response: ({"error": "Node #{invalid_node_name} is not part of the Pool"}.to_json))
   end
 
-  nodes = participating_nodes(pool_name, req)
   node_details_add_to_volume(req, nodes)
 
   # Validate if all the nodes are reachable.

mgr/src/server/plugins/volume_list_status.cr

@@ -80,7 +80,9 @@ def volume_status_node_request_prepare(pool_name, volumes)
 end
 
 def volume_list_detail_status(env, pool_name, volume_name, state)
-  if volume_name.nil?
+  if pool_name == ""
+    volumes = Datastore.list_volumes
+  elsif volume_name.nil?
     volumes = Datastore.list_volumes(pool_name)
   else
     vol = Datastore.get_volume(pool_name, volume_name)
@@ -121,7 +123,13 @@ def volume_list_detail_status(env, pool_name, volume_name, state)
     set_volume_metrics(volume)
   end
 
-  volumes.to_json
+  volume_name.nil? ? volumes.to_json : volumes[0].to_json
+end
+
+get "/api/v1/volumes" do |env|
+  state = env.params.query["state"]
+
+  volume_list_detail_status(env, "", nil, state ? true : false)
 end
 
 get "/api/v1/pools/:pool_name/volumes" do |env|

mgr/src/server/plugins/volume_utils.cr

@@ -28,19 +28,25 @@ end
 def participating_nodes(pool_name, req)
   case req
   when MoanaTypes::Volume
-    nodes = [] of String
+    nodes = [] of MoanaTypes::Node
     req.distribute_groups.each do |dist_grp|
       dist_grp.storage_units.each do |storage_unit|
-        nodes << storage_unit.node.name
+        nodes << storage_unit.node
       end
     end
-    nodes.uniq!
-    Datastore.get_nodes(pool_name, nodes)
+    # Shorthand equivalant to
+    # nodes.uniq! do |node|
+    #   node.name
+    # end
+    nodes.uniq!(&.name)
+    nodes
   when Array(MoanaTypes::Volume)
     nodes = [] of MoanaTypes::Node
     req.each do |volume|
-      nodes += participating_nodes(pool_name, volume)
+      nodes += participating_nodes(volume.pool.name, volume)
     end
+
+    nodes.uniq!(&.name)
     nodes
   else
     [] of MoanaTypes::Node

types/src/moana_types.cr

@@ -106,7 +106,7 @@ module MoanaTypes
     property id = "",
       name = "",
       state = "",
-      pool_name = "",
+      pool = Pool.new,
       distribute_groups = [] of DistributeGroup,
       no_start = false,
       options = Hash(String, String).new,