From ef626e3892d1a94bd08e56a65b3dc6139a1124c0 Mon Sep 17 00:00:00 2001 From: Daniel Clavijo Coca Date: Mon, 4 Dec 2023 23:03:00 -0600 Subject: [PATCH] Implement update operation --- src/server/function.rb | 67 +++++++++++++------- src/server/runtime.rb | 126 ++++++++++++++++++++++++++++---------- tests/init.rb | 4 +- tests/lib/auth.rb | 2 +- tests/lib/common.rb | 44 +++++++++++-- tests/lib/crud.rb | 46 ++++++++++++-- tests/lib/crud_invalid.rb | 39 ++++++++---- 7 files changed, 248 insertions(+), 80 deletions(-) diff --git a/src/server/function.rb b/src/server/function.rb index d98b2d3..222c117 100644 --- a/src/server/function.rb +++ b/src/server/function.rb @@ -5,8 +5,14 @@ module ProvisionEngine # class Function < OpenNebula::VirtualMachine + STATES = { + :pending => 'PENDING', + :running => 'RUNNING', + :error => 'ERROR', + :updating => 'UPDATING' + }.freeze STATE_MAP = { - 'PENDING' => [ + STATES[:pending] => [ 'LCM_INIT', 'BOOT', 'PROLOG', @@ -18,8 +24,8 @@ class Function < OpenNebula::VirtualMachine 'PROLOG_UNDEPLOY', 'CLEANUP_RESUBMIT' ], - 'RUNNING' => ['RUNNING'], - 'ERROR' => [ + STATES[:running] => ['RUNNING'], + STATES[:error] => [ 'FAILURE', 'UNKNOWN', 'BOOT_FAILURE', @@ -36,10 +42,10 @@ class Function < OpenNebula::VirtualMachine 'PROLOG_MIGRATE_UNKNOWN_FAILURE' ] }.freeze - FUNCTIONS = ['FAAS', 'DAAS'] + FUNCTIONS = ['FAAS', 'DAAS'].freeze - T = '//TEMPLATE/' - SRF = 'Serverless Runtime Function VM' + T = '//TEMPLATE/'.freeze + SRF = 'Serverless Runtime Function VM'.freeze def id @pe_id.to_i @@ -179,7 +185,7 @@ def to_function function = {} function['VM_ID'] = @pe_id - function['STATE'] = map_state + function['STATE'] = state_function if function['STATE'] == 'ERROR' if error @@ -204,7 +210,7 @@ def to_function # # @return [Array] [Response Code, ''/Error] # - def resize_capacity?(specification) + def resize_capacity?(specification, logger) capacity_template = [] if specification['MEMORY'] != memory[:memory] @@ -221,6 +227,11 @@ def resize_capacity?(specification) return [200, ''] if capacity_template.empty? + capacity_template = capacity_template.join("\n") + + logger.info("Updating #{SRF} #{@id} capacity") + logger.debug(capacity_template) + response = resize(capacity_template, true) if OpenNebula.is_error?(response) @@ -239,10 +250,13 @@ def resize_capacity?(specification) # # @return [Array] [Response Code, ''/Error] # - def resize_disk?(specification) + def resize_disk?(specification, logger) return [200, ''] unless specification['DISK_SIZE'] != size - response = disk_resize(0, new_size) + logger.info("Resizing #{SRF} #{@id} disk") + logger.debug("From: #{size} To: #{specification['DISK_SIZE']}") + + response = disk_resize(0, specification['DISK_SIZE']) if OpenNebula.is_error?(response) rc = ProvisionEngine::Error.map_error_oned(response.errno) @@ -253,24 +267,33 @@ def resize_disk?(specification) [200, ''] end + def pending? + state_function == STATES[:pending] + end + + def running? + state_function == STATES[:running] + end + + def error? + state_function == STATES[:error] + end + + def updating? + state_function == STATES[:updating] + end + # # Maps an OpenNebula VM state to the accepted Function VM states # # @return [String] Serverless Runtime Function state # - def map_state - case state_str - when 'INIT', 'PENDING', 'HOLD' - STATE_MAP.keys[0] - when 'ACTIVE' - STATE_MAP.each do |function_state, lcm_states| - return function_state if lcm_states.include?(lcm_state_str) - end - when 'STOPPED', 'SUSPENDED', 'POWEROFF', 'UNDEPLOYED', 'CLONING' - STATE_MAP.keys[2] - else - STATE_MAP.keys[3] + def state_function + STATE_MAP.each do |function_state, lcm_states| + return function_state if lcm_states.include?(lcm_state_str) end + + return STATES[:updating] end end diff --git a/src/server/runtime.rb b/src/server/runtime.rb index 7a244c0..557627f 100644 --- a/src/server/runtime.rb +++ b/src/server/runtime.rb @@ -9,7 +9,9 @@ class ServerlessRuntime < OpenNebula::DocumentJSON SCHEMA = JSON.load_file('/etc/provision-engine/schemas/serverless_runtime.json').freeze SR = 'Serverless Runtime'.freeze + SRR = 'SERVERLESS_RUNTIME'.freeze SRD = "#{SR} Document".freeze + SRF = 'Serverless Runtime Function VM'.freeze SRS = "#{SR} Service".freeze SRS_NOT_FOUND = "#{SRS} not found".freeze SRS_NO_READ = "Failed to read #{SRS}".freeze @@ -29,7 +31,7 @@ def self.create(client, specification) response = ServerlessRuntime.validate(specification) return response unless response[0] == 200 - specification = specification['SERVERLESS_RUNTIME'] + specification = specification[SRR] response = ServerlessRuntime.to_service(client, specification) return response unless response[0] == 200 @@ -98,20 +100,34 @@ def self.get(client, id) return ProvisionEngine::Error.new(rc, error, message) end - response = ServerlessRuntime.sync(client, document.body) + document.cclient = client + document.update + end + + # + # Syncronizes the Serverless Runtime backing components with the document + # + # @return [Array] [Response Code, ServerlessRuntime/error] + # + def update + cclient? + initial_state = to_hash + + response = ProvisionEngine::ServerlessRuntime.sync(@cclient, @body) return response unless response[0] == 200 - response = document.update + return [200, self] if to_hash == initial_state + + @cclient.logger.info("Updating #{SRD} #{@id}") + response = super() if OpenNebula.is_error?(response) rc = ProvisionEngine::Error.map_error_oned(response.errno) - error = "Failed to update #{SR}" + error = "Failed to update #{SR} #{@id}" return ProvisionEngine::Error.new(rc, error, response.error) end - document.cclient = client - - [200, document] + [200, self] end # @@ -127,45 +143,83 @@ def update_sr(specification) response = ServerlessRuntime.validate(specification) return response unless response[0] == 200 - specification = specification['SERVERLESS_RUNTIME'] - rename?(specification) + specification = specification[SRR] ProvisionEngine::Function::FUNCTIONS.each do |function| + next if specification[function].nil? + vm_id = specification[function]['VM_ID'] + next if vm_id.nil? - response = ProvisionEngine::Function.new_with_id(vm_id, @cclient.client_oned) - rc = response[0] + vm = ProvisionEngine::Function.new_with_id(vm_id, @cclient.client_oned) + response = vm.info - if rc != 200 + if OpenNebula.is_error?(response) + rc = ProvisionEngine::Error.map_error_oned(response.errno) error = "Failed to read #{SRF} #{function}" - return ProvisionEngine::Error.new(rc, error, response[1]) + return ProvisionEngine::Error.new(rc, error, response.message) end - vm = response[1] + # Resize VM hardware + case vm.state_function + when ProvisionEngine::Function::STATES[:updating], ProvisionEngine::Function::STATES[:pending] + rc = 423 + error = "Cannot update #{SRF} #{function} on a transient state" + return ProvisionEngine::Error.new(rc, error, vm.state_function) + when ProvisionEngine::Function::STATES[:error] + vm.recover(2) # retry + error = "Cannot update #{SRF} #{function} on an error state. A recovery was attempted" + return ProvisionEngine::Error.new(500, error, + ProvisionEngine::Function::STATES[:error]) + when ProvisionEngine::Function::STATES[:running] + ['capacity', 'disk'].each do |resource| + response = vm.public_send("resize_#{resource}?", specification[function], + @cclient.logger) + return response unless response[0] == 200 - ['capactiy', 'disk'].each do |resource| - @cclient.logger.info("Updating #{SRF} #{function} #{resource}") + 1.upto(@cclient.conf[:timeout]) do |t| + vm.info + + if t == @cclient.conf[:timeout] + rc = 504 + error = "#{SRF} #{function} stuck while updating capabilities" + return ProvisionEngine::Error.new(rc, error, vm.state_function) + end + + case vm.state_function + when ProvisionEngine::Function::STATES[:running] + break + when ProvisionEngine::Function::STATES[:updating] + sleep 1 + next + else + rc = 500 + error = "#{SRF} #{function} entered unexpected state" + return ProvisionEngine::Error.new(rc, error, vm.state_function) + end + end + end - response = vm.public_send("resize_#{resource}?", specification[function]) - return response unless response[0] == 200 end - end - @cclient.logger.info("Updating #{SRD} #{@id}") + # Update document body and VMs USER_TEMPLATE + ['SCHEDULING', 'DEVICE_INFO'].each do |schevice| + next if specification[function][schevice].nil? + next if specification[function][schevice] == @body[SRR][function][schevice] - response = ProvisionEngine::ServerlessRuntime.sync(@cclient, @body) - return response unless response[0] == 200 + @body[SRR][function][schevice] = specification[function][schevice] + vm.update(specification[function][schevice], true) - response = update + next unless OpenNebula.is_error?(response) - if OpenNebula.is_error?(response) - rc = ProvisionEngine::Error.map_error_oned(response.errno) - error = "Failed to update #{SR}" - return ProvisionEngine::Error.new(rc, error, response.error) + rc = ProvisionEngine::Error.map_error_oned(response.errno) + error = "Failed to update #{SRF} #{schevice}" + return ProvisionEngine::Error.new(rc, error, response.message) + end end - [200, self] + update end # @@ -430,11 +484,9 @@ def self.to_service(client, specification) # # Perform recovery operations on the Serverless Runtime backing components # - # @param [Int] service_id flow service backing the Serverless Runtime - # # @return [Array] [Response Code, Service Document Body/Error] # - def recover(service_id) + def recover response = @cclient.service_recover(service_id) rc = response[0] @@ -471,11 +523,12 @@ def recover(service_id) # @return [Array] [Response Code, ''/Error] # def rename?(specification) - new_name = specification['NAME'] + new_name = specification[SRR]['NAME'] return [200, ''] unless new_name && new_name != name @cclient.logger.info("Renaming #{SRD} #{@id}") + @cclient.logger.debug("From: #{name} To: #{new_name}") response = rename(new_name) @@ -524,12 +577,12 @@ def to_sr load? runtime = { - :SERVERLESS_RUNTIME => { + SRR => { :NAME => name, :ID => id } } - rsr = runtime[:SERVERLESS_RUNTIME] + rsr = runtime[SRR] rsr.merge!(@body) rsr.delete('registration_time') @@ -537,6 +590,11 @@ def to_sr runtime end + def service_id + load? + @body[SRR]['SERVICE_ID'] + end + # # Generates the flow template name for the service instantiation # diff --git a/tests/init.rb b/tests/init.rb index 3aec5c6..4cf51e7 100644 --- a/tests/init.rb +++ b/tests/init.rb @@ -68,11 +68,11 @@ include_context('crud', sr_template) end + + tests.delete('crud') end tests.each do |examples, enabled| - next if examples == 'crud' - if enabled require examples include_context(examples) diff --git a/tests/lib/auth.rb b/tests/lib/auth.rb index 2d35f55..44920be 100644 --- a/tests/lib/auth.rb +++ b/tests/lib/auth.rb @@ -24,7 +24,7 @@ @conf[:auth][:create] = true runtime = JSON.parse(response.body) - @conf[:auth][:id] = runtime['SERVERLESS_RUNTIME']['ID'].to_i + @conf[:auth][:id] = runtime[SRR]['ID'].to_i end it 'missing auth on Create' do diff --git a/tests/lib/common.rb b/tests/lib/common.rb index f9f674c..049cfd9 100644 --- a/tests/lib/common.rb +++ b/tests/lib/common.rb @@ -1,4 +1,11 @@ SR = 'Serverless Runtime'.freeze +SRR = 'SERVERLESS_RUNTIME'.freeze + +HARDWARE = { + 'CPU' => 1, + 'MEMORY' => 64, + 'DISK_SIZE' => 128 +} ############################################################################ # RSpec methods @@ -15,8 +22,8 @@ def verify_sr_spec(specification, runtime) raise response[1] unless response[0] == 200 end - specification = specification['SERVERLESS_RUNTIME'] - runtime = runtime['SERVERLESS_RUNTIME'] + specification = specification[SRR] + runtime = runtime[SRR] # optional name has been applied if given expect(runtime['NAME']).to eq(specification['NAME']) if specification['NAME'] @@ -36,7 +43,12 @@ def verify_sr_spec(specification, runtime) next unless specification[role] && !specification[role]['FLAVOUR'].empty? vm = OpenNebula::VirtualMachine.new_with_id(runtime[role]['VM_ID'], @conf[:client][:oned]) - raise "Error getting #{SR} VM" if OpenNebula.is_error?(vm.info) + + response = vm.info + + if OpenNebula.is_error?(response) + raise "Error getting #{SR} function VM #{role} \n#{response.message}" + end nic = "#{t}NIC[NIC_ID=\"0\"]/" @@ -146,7 +158,7 @@ def generate_faas_minimal(flavour = nil) pp "rolled a random flavour #{flavour}" end { - 'SERVERLESS_RUNTIME' => { + SRR => { 'NAME' => flavour, 'FAAS' => { 'FLAVOUR' => flavour @@ -156,3 +168,27 @@ def generate_faas_minimal(flavour = nil) } } end + +def increase_runtime_hardware(specification, mode = 'multiply') + ProvisionEngine::Function::FUNCTIONS.each do |function| + next unless specification[SRR][function] + + case mode + when 'multiply' + HARDWARE.keys.each do |h| + next if specification[SRR][function][h].nil? + + specification[SRR][function][h] = specification[SRR][function][h] * 2 + end + when 'increase' + HARDWARE.each do |key, value| + next if specification[SRR][function][key].nil? + + specification[SRR][function][key] = specification[SRR][function][key] + value + end + + else + raise "Invalid #{SR} hardware update mode" + end + end +end diff --git a/tests/lib/crud.rb b/tests/lib/crud.rb index 6674814..2f40632 100644 --- a/tests/lib/crud.rb +++ b/tests/lib/crud.rb @@ -10,7 +10,7 @@ runtime = JSON.parse(response.body) - @conf[:id] = runtime['SERVERLESS_RUNTIME']['ID'].to_i + @conf[:id] = runtime[SRR]['ID'].to_i @conf[:create] = true end @@ -22,16 +22,50 @@ runtime = JSON.parse(response.body) verify_sr_spec(@conf[:specification], runtime) + + @conf[:runtime] = runtime end - it "fail to update #{SR}" do + # VM reaches RUNNING state eventually + # only updates the specified functions + # only updates what is different from the existing function + # missing properties will be ignored + # for the time being only: + # updates CPU, MEMORY and DISK + # rename document + # runtime ID, service ID and Function IDs remain the same + it "update #{SR}" do skip "#{SR} creation failed" unless @conf[:create] - response = @conf[:client][:engine].update(@conf[:id], {}) - expect(response.code).to eq(200) + increase_runtime_hardware(@conf[:runtime], 'increase') - runtime = JSON.parse(response.body) - verify_sr_spec(@conf[:specification], runtime) + timeout = @conf[:conf][:tests][:timeouts][:get] + 1.upto(timeout) do |t| + if t == timeout + raise "Timeut reached for #{SR} deployment" + end + + response = @conf[:client][:engine].update(@conf[:id], @conf[:runtime]) + rc = response.code + body = JSON.parse(response.body) + + case rc + when 200 + verify_sr_spec(@conf[:runtime], body) + break + when 423 + pp "Waiting for #{SR} to be RUNNING" + verify_error(body) + + sleep 1 + next + else + pp body + verify_error(body) + + raise "Unexpected error code #{rc}" + end + end end it "delete a #{SR}" do diff --git a/tests/lib/crud_invalid.rb b/tests/lib/crud_invalid.rb index 3e153b3..5afe2ef 100644 --- a/tests/lib/crud_invalid.rb +++ b/tests/lib/crud_invalid.rb @@ -1,14 +1,7 @@ RSpec.shared_context 'crud_invalid' do - it "fail to create a #{SR} with invalid FLAVOUR" do - response = @conf[:client][:engine].create(generate_faas_minimal) - - expect(response.code).to eq(422) - verify_error(response.body) - end - it "fail to create a #{SR} with invalid schema" do specification = { - 'SERVERLESS_RUNTIME' => { + SRR => { 'FAAS' => { 'FLAVOUR' => 'Function' }, @@ -24,10 +17,17 @@ verify_error(response.body) end + it "fail to create a #{SR} with invalid FLAVOUR" do + response = @conf[:client][:engine].create(generate_faas_minimal) + + expect(response.code).to eq(422) + verify_error(response.body) + end + # requires a flow template FAILED_DEPLOY which guarantees service on FAILED_DEPLOY it "fail to create #{SR} if oneflow service enters FAIL_DEPLOY" do specification = { - 'SERVERLESS_RUNTIME' => { + SRR => { 'FAAS' => { 'FLAVOUR' => 'FAILED_DEPLOY' }, @@ -64,6 +64,23 @@ verify_error(response.body) end + it "fail to create a #{SR} with invalid schema" do + end + + it "fail to update a #{SR} with mismatching FLAVOUR" do + end + + it "fail to update a FaaS only #{SR} with DaaS FLAVOUR" do + end + + it "fail to update a #{SR} with hardware decrease" do + end + + it "fail to resize #{SR} beyond memory_max and vcpu_max" do + # Function VMs remain in STATE=RUNNING + # Function has the ERROR=VM_ERROR + end + it "fail to delete a non existing #{SR}" do response = @conf[:client][:engine].delete(@conf[:invalid][:sky]) @@ -78,8 +95,8 @@ expect(response.code).to eq(201) body = JSON.parse(response.body) - @conf[:invalid][:id] = body['SERVERLESS_RUNTIME']['ID'].to_i - @conf[:invalid][:service_id] = body['SERVERLESS_RUNTIME']['SERVICE_ID'].to_i + @conf[:invalid][:id] = body[SRR]['ID'].to_i + @conf[:invalid][:service_id] = body[SRR]['SERVICE_ID'].to_i end ['get', 'delete'].each do |operation|