Skip to content

Commit 13db198

Browse files
leonidcAlexander Indenbaum
leonidc
authored and
Alexander Indenbaum
committed
nvmeof refactoring - remove MAX_SUPPORTED_ANA_GROUPS
Signed-off-by: Alexander Indenbaum <[email protected]> Signed-off-by: Leonid Chernin <[email protected]>
1 parent 0ee9f52 commit 13db198

7 files changed

+286
-212
lines changed

‎src/mon/NVMeofGwMap.cc

+103-75
Large diffs are not rendered by default.

‎src/mon/NVMeofGwMap.h

+2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ class NVMeofGwMap
4949
void handle_removed_subsystems (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, const std::vector<NvmeNqnId> &current_subsystems, bool &propose_pending);
5050
void start_timer (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeAnaGrpId anagrpid, uint8_t value);
5151
private:
52+
void add_grp_id (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, const NvmeAnaGrpId grpid);
53+
void remove_grp_id(const NvmeGwId &gw_id, const NvmeGroupKey& group_key, const NvmeAnaGrpId grpid);
5254
void fsm_handle_gw_down (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, gw_states_per_group_t state, NvmeAnaGrpId grpid, bool &map_modified);
5355
void fsm_handle_gw_delete (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, gw_states_per_group_t state, NvmeAnaGrpId grpid, bool &map_modified);
5456
void fsm_handle_gw_alive (const NvmeGwId &gw_id, const NvmeGroupKey& group_key, NvmeGwMonState & gw_state, gw_states_per_group_t state,

‎src/mon/NVMeofGwMon.cc

+36-37
Original file line numberDiff line numberDiff line change
@@ -24,27 +24,27 @@
2424

2525
using std::string;
2626

27-
void NVMeofGwMon::init(){
27+
void NVMeofGwMon::init() {
2828
dout(10) << "called " << dendl;
2929
}
3030

31-
void NVMeofGwMon::on_restart(){
31+
void NVMeofGwMon::on_restart() {
3232
dout(10) << "called " << dendl;
3333
last_beacon.clear();
3434
last_tick = ceph::coarse_mono_clock::now();
3535
synchronize_last_beacon();
3636
}
3737

3838

39-
void NVMeofGwMon::synchronize_last_beacon(){
39+
void NVMeofGwMon::synchronize_last_beacon() {
4040
dout(10) << "called, is leader : " << mon.is_leader() <<" active " << is_active() << dendl;
4141
// Initialize last_beacon to identify transitions of available GWs to unavailable state
4242
for (const auto& created_map_pair: map.created_gws) {
4343
const auto& group_key = created_map_pair.first;
4444
const NvmeGwMonStates& gw_created_map = created_map_pair.second;
4545
for (const auto& gw_created_pair: gw_created_map) {
4646
const auto& gw_id = gw_created_pair.first;
47-
if (gw_created_pair.second.availability == gw_availability_t::GW_AVAILABLE){
47+
if (gw_created_pair.second.availability == gw_availability_t::GW_AVAILABLE) {
4848
dout(10) << "synchronize last_beacon for GW :" << gw_id << dendl;
4949
LastBeacon lb = {gw_id, group_key};
5050
last_beacon[lb] = last_tick;
@@ -57,8 +57,8 @@ void NVMeofGwMon::on_shutdown() {
5757
dout(10) << "called " << dendl;
5858
}
5959

60-
void NVMeofGwMon::tick(){
61-
if (!is_active() || !mon.is_leader()){
60+
void NVMeofGwMon::tick() {
61+
if (!is_active() || !mon.is_leader()) {
6262
dout(10) << "NVMeofGwMon leader : " << mon.is_leader() << "active : " << is_active() << dendl;
6363
return;
6464
}
@@ -89,10 +89,10 @@ void NVMeofGwMon::tick(){
8989
_propose_pending |= propose;
9090

9191
const auto cutoff = now - nvmegw_beacon_grace;
92-
for (auto &itr : last_beacon){// Pass over all the stored beacons
92+
for (auto &itr : last_beacon) {// Pass over all the stored beacons
9393
auto& lb = itr.first;
9494
auto last_beacon_time = itr.second;
95-
if (last_beacon_time < cutoff){
95+
if (last_beacon_time < cutoff) {
9696
dout(10) << "beacon timeout for GW " << lb.gw_id << dendl;
9797
pending_map.process_gw_map_gw_down(lb.gw_id, lb.group_key, propose);
9898
_propose_pending |= propose;
@@ -106,7 +106,7 @@ void NVMeofGwMon::tick(){
106106
pending_map.handle_abandoned_ana_groups(propose); // Periodic: take care of not handled ANA groups
107107
_propose_pending |= propose;
108108

109-
if (_propose_pending){
109+
if (_propose_pending) {
110110
dout(10) << "propose pending " <<dendl;
111111
propose_pending();
112112
}
@@ -120,14 +120,14 @@ const char **NVMeofGwMon::get_tracked_conf_keys() const
120120
return KEYS;
121121
}
122122

123-
void NVMeofGwMon::create_pending(){
123+
void NVMeofGwMon::create_pending() {
124124

125125
pending_map = map;// deep copy of the object
126126
pending_map.epoch++;
127127
dout(10) << " pending " << pending_map << dendl;
128128
}
129129

130-
void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t){
130+
void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t) {
131131

132132
dout(10) << dendl;
133133
ceph_assert(get_last_committed() + 1 == pending_map.epoch);
@@ -137,7 +137,7 @@ void NVMeofGwMon::encode_pending(MonitorDBStore::TransactionRef t){
137137
put_last_committed(t, pending_map.epoch);
138138
}
139139

140-
void NVMeofGwMon::update_from_paxos(bool *need_bootstrap){
140+
void NVMeofGwMon::update_from_paxos(bool *need_bootstrap) {
141141
version_t version = get_last_committed();
142142

143143
if (version != map.epoch) {
@@ -177,15 +177,15 @@ void NVMeofGwMon::check_subs(bool t)
177177
const std::string type = "NVMeofGw";
178178
dout(10) << "count " << mon.session_map.subs.count(type) << dendl;
179179

180-
if (mon.session_map.subs.count(type) == 0){
180+
if (mon.session_map.subs.count(type) == 0) {
181181
return;
182182
}
183-
for (auto sub : *(mon.session_map.subs[type])){
183+
for (auto sub : *(mon.session_map.subs[type])) {
184184
check_sub(sub);
185185
}
186186
}
187187

188-
bool NVMeofGwMon::preprocess_query(MonOpRequestRef op){
188+
bool NVMeofGwMon::preprocess_query(MonOpRequestRef op) {
189189
dout(20) << dendl;
190190

191191
auto m = op->get_req<PaxosServiceMessage>();
@@ -210,7 +210,7 @@ bool NVMeofGwMon::preprocess_query(MonOpRequestRef op){
210210
return false;
211211
}
212212

213-
bool NVMeofGwMon::prepare_update(MonOpRequestRef op){
213+
bool NVMeofGwMon::prepare_update(MonOpRequestRef op) {
214214
auto m = op->get_req<PaxosServiceMessage>();
215215
switch (m->get_type()) {
216216
case MSG_MNVMEOF_GW_BEACON:
@@ -254,7 +254,7 @@ bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
254254
dout(10) << "MonCommand : "<< prefix << dendl;
255255
string format = cmd_getval_or<string>(cmdmap, "format", "plain");
256256
boost::scoped_ptr<Formatter> f(Formatter::create(format));
257-
if (prefix == "nvme-gw show"){
257+
if (prefix == "nvme-gw show") {
258258
std::string pool, group;
259259
if (!f) {
260260
f.reset(Formatter::create(format, "json-pretty", "json-pretty"));
@@ -264,19 +264,18 @@ bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
264264
auto group_key = std::make_pair(pool, group);
265265
dout(10) <<"nvme-gw show pool "<< pool << " group "<< group << dendl;
266266

267-
if (map.created_gws[group_key].size()){
267+
if (map.created_gws[group_key].size()) {
268268
f->open_object_section("common");
269269
f->dump_unsigned("epoch", map.epoch);
270270
f->dump_string("pool", pool);
271271
f->dump_string("group", group);
272272
f->dump_unsigned("num gws", map.created_gws[group_key].size());
273273
sstrm <<"[ ";
274-
NvmeAnaGrpId anas[MAX_SUPPORTED_ANA_GROUPS];
275-
int i = 0;
274+
NvmeGwId gw_id;
276275
for (auto& gw_created_pair: map.created_gws[group_key]) {
276+
gw_id = gw_created_pair.first;
277277
auto& st = gw_created_pair.second;
278278
sstrm << st.ana_grp_id+1 << " ";
279-
anas[i++] = st.ana_grp_id;
280279
}
281280
sstrm << "]";
282281
f->dump_string("Anagrp list", sstrm.str());
@@ -293,8 +292,8 @@ bool NVMeofGwMon::preprocess_command(MonOpRequestRef op)
293292
sstrm1 << state.availability;
294293
f->dump_string("Availability", sstrm1.str());
295294
sstrm1.str("");
296-
for (size_t i = 0; i < map.created_gws[group_key].size(); i++) {
297-
sstrm1 << " " << anas[i]+1 <<": " << state.sm_state[anas[i]] << ",";
295+
for (auto &state_itr: map.created_gws[group_key][gw_id].sm_state) {
296+
sstrm1 << " " << state_itr.first + 1 << ": " << state.sm_state[state_itr.first] << ",";
298297
}
299298
f->dump_string("ana states", sstrm1.str());
300299
f->close_section();
@@ -345,17 +344,17 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
345344
cmd_getval(cmdmap, "group", group);
346345
auto group_key = std::make_pair(pool, group);
347346
dout(10) << " id "<< id <<" pool "<< pool << " group "<< group << dendl;
348-
if (prefix == "nvme-gw create"){
347+
if (prefix == "nvme-gw create") {
349348
rc = pending_map.cfg_add_gw(id, group_key);
350-
if (rc == -EINVAL){
349+
if (rc == -EINVAL) {
351350
err = rc;
352351
dout (4) << "Error: GW cannot be created " << id << " " << pool << " " << group << " rc " << rc << dendl;
353352
sstrm.str("");
354353
}
355354
}
356355
else{
357356
rc = pending_map.cfg_delete_gw(id, group_key);
358-
if (rc == -EINVAL){
357+
if (rc == -EINVAL) {
359358
dout (4) << "Error: GW not found in the database " << id << " " << pool << " " << group << " rc " << rc << dendl;
360359
err = 0;
361360
sstrm.str("");
@@ -381,7 +380,7 @@ bool NVMeofGwMon::prepare_command(MonOpRequestRef op)
381380
}
382381

383382

384-
bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op){
383+
bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op) {
385384
auto m = op->get_req<MNVMeofGwBeacon>();
386385
const BeaconSubsystems& sub = m->get_subsystems();
387386
dout(15) << "beacon from " << m->get_type() << " GW : " << m->get_gw_id() << " num subsystems " << sub.size() << dendl;
@@ -390,7 +389,7 @@ bool NVMeofGwMon::preprocess_beacon(MonOpRequestRef op){
390389
}
391390

392391

393-
bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
392+
bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op) {
394393
auto m = op->get_req<MNVMeofGwBeacon>();
395394

396395
dout(20) << "availability " << m->get_availability() << " GW : " << m->get_gw_id() <<
@@ -408,20 +407,20 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
408407
auto gw = group_gws.find(gw_id);
409408
const BeaconSubsystems& sub = m->get_subsystems();
410409

411-
if (avail == gw_availability_t::GW_CREATED){
410+
if (avail == gw_availability_t::GW_CREATED) {
412411
if (gw == group_gws.end()) {
413412
gw_created = false;
414413
dout(10) << "Warning: GW " << gw_id << " group_key " << group_key << " was not found in the map.Created_gws "<< map.created_gws <<dendl;
415414
goto set_propose;
416415
}
417416
else {
418417
dout(10) << "GW prepares the full startup " << gw_id << " GW availability: " << pending_map.created_gws[group_key][gw_id].availability << dendl;
419-
if (pending_map.created_gws[group_key][gw_id].availability == gw_availability_t::GW_AVAILABLE){
418+
if (pending_map.created_gws[group_key][gw_id].availability == gw_availability_t::GW_AVAILABLE) {
420419
dout(4) << " Warning :GW marked as Available in the NVmeofGwMon database, performed full startup - Force gw to exit!" << gw_id <<dendl;
421420
avail = gw_availability_t::GW_UNAVAILABLE;
422421
// Monitor performs Force Failover for this GW in process_gw_map_gw_down
423422
}
424-
else if (pending_map.created_gws[group_key][gw_id].performed_full_startup == false){
423+
else if (pending_map.created_gws[group_key][gw_id].performed_full_startup == false) {
425424
pending_map.created_gws[group_key][gw_id].performed_full_startup = true;
426425
propose = true;
427426
goto set_propose;
@@ -444,7 +443,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
444443
}
445444

446445
// At this stage the gw has to be in the Created_gws
447-
if (gw == group_gws.end()){
446+
if (gw == group_gws.end()) {
448447
dout(4) << "Administratively deleted GW sends beacon " << gw_id <<dendl;
449448
goto false_return; // not sending ack to this beacon
450449
}
@@ -474,7 +473,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
474473
nonce_propose = true;
475474
}
476475
pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid = ( map.epoch == m->get_last_gwmap_epoch() );
477-
if (pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid == false){
476+
if (pending_map.created_gws[group_key][gw_id].last_gw_map_epoch_valid == false) {
478477
dout(20) << "map epoch of gw is not up-to-date " << gw_id << " epoch " << map.epoch << " beacon_epoch " << m->get_last_gwmap_epoch() << dendl;
479478
}
480479
if (avail == gw_availability_t::GW_AVAILABLE)
@@ -487,11 +486,11 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
487486
epoch_t last_osd_epoch = m->get_last_osd_epoch();
488487
pending_map.process_gw_map_ka(gw_id, group_key, last_osd_epoch, propose);
489488
}
490-
else if (avail == gw_availability_t::GW_UNAVAILABLE){ // state set by GW client application
489+
else if (avail == gw_availability_t::GW_UNAVAILABLE) { // state set by GW client application
491490
LastBeacon lb = {gw_id, group_key};
492491

493492
auto it = last_beacon.find(lb);
494-
if (it != last_beacon.end()){
493+
if (it != last_beacon.end()) {
495494
last_beacon.erase(lb);
496495
pending_map.process_gw_map_gw_down(gw_id, group_key, propose);
497496
}
@@ -502,7 +501,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
502501

503502
set_propose:
504503
if (!propose) {
505-
if (gw_created){
504+
if (gw_created) {
506505
ack_map.created_gws[group_key][gw_id] = map.created_gws[group_key][gw_id];// respond with a map slice correspondent to the same GW
507506
}
508507
ack_map.epoch = map.epoch;
@@ -514,7 +513,7 @@ bool NVMeofGwMon::prepare_beacon(MonOpRequestRef op){
514513
mon.no_reply(op);
515514
}
516515
false_return:
517-
if (propose){
516+
if (propose) {
518517
dout(10) << "decision in prepare_beacon" <<dendl;
519518
return true;
520519
}

0 commit comments

Comments
 (0)