Exception occured in MaskMMDetectionCallback when calling event after_loss error message for instance segmentation

[aisensiy]

Follow the tutorials https://airctic.com/0.12.0/getting_started_instance_segmentation/ and use coco2017 to have a try. But got the error message below in training:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Input In [24], in <module>
----> 1 learn.fine_tune(3, 0.0004)

File /usr/local/lib/python3.8/site-packages/fastai/callback/schedule.py:161, in fine_tune(self, epochs, base_lr, freeze_epochs, lr_mult, pct_start, div, **kwargs)
    159 "Fine tune with `Learner.freeze` for `freeze_epochs`, then with `Learner.unfreeze` for `epochs`, using discriminative LR."
    160 self.freeze()
--> 161 self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)
    162 base_lr /= 2
    163 self.unfreeze()

File /usr/local/lib/python3.8/site-packages/fastai/callback/schedule.py:116, in fit_one_cycle(self, n_epoch, lr_max, div, div_final, pct_start, wd, moms, cbs, reset_opt)
    113 lr_max = np.array([h['lr'] for h in self.opt.hypers])
    114 scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),
    115           'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}
--> 116 self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:221, in Learner.fit(self, n_epoch, lr, wd, cbs, reset_opt)
    219 self.opt.set_hypers(lr=self.lr if lr is None else lr)
    220 self.n_epoch = n_epoch
--> 221 self._with_events(self._do_fit, 'fit', CancelFitException, self._end_cleanup)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:163, in Learner._with_events(self, f, event_type, ex, final)
    162 def _with_events(self, f, event_type, ex, final=noop):
--> 163     try: self(f'before_{event_type}');  f()
    164     except ex: self(f'after_cancel_{event_type}')
    165     self(f'after_{event_type}');  final()

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:212, in Learner._do_fit(self)
    210 for epoch in range(self.n_epoch):
    211     self.epoch=epoch
--> 212     self._with_events(self._do_epoch, 'epoch', CancelEpochException)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:163, in Learner._with_events(self, f, event_type, ex, final)
    162 def _with_events(self, f, event_type, ex, final=noop):
--> 163     try: self(f'before_{event_type}');  f()
    164     except ex: self(f'after_cancel_{event_type}')
    165     self(f'after_{event_type}');  final()

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:207, in Learner._do_epoch(self)
    205 def _do_epoch(self):
    206     self._do_epoch_train()
--> 207     self._do_epoch_validate()

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:203, in Learner._do_epoch_validate(self, ds_idx, dl)
    201 if dl is None: dl = self.dls[ds_idx]
    202 self.dl = dl
--> 203 with torch.no_grad(): self._with_events(self.all_batches, 'validate', CancelValidException)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:163, in Learner._with_events(self, f, event_type, ex, final)
    162 def _with_events(self, f, event_type, ex, final=noop):
--> 163     try: self(f'before_{event_type}');  f()
    164     except ex: self(f'after_cancel_{event_type}')
    165     self(f'after_{event_type}');  final()

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:169, in Learner.all_batches(self)
    167 def all_batches(self):
    168     self.n_iter = len(self.dl)
--> 169     for o in enumerate(self.dl): self.one_batch(*o)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:194, in Learner.one_batch(self, i, b)
    192 b = self._set_device(b)
    193 self._split(b)
--> 194 self._with_events(self._do_one_batch, 'batch', CancelBatchException)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:163, in Learner._with_events(self, f, event_type, ex, final)
    162 def _with_events(self, f, event_type, ex, final=noop):
--> 163     try: self(f'before_{event_type}');  f()
    164     except ex: self(f'after_cancel_{event_type}')
    165     self(f'after_{event_type}');  final()

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:177, in Learner._do_one_batch(self)
    175     self.loss_grad = self.loss_func(self.pred, *self.yb)
    176     self.loss = self.loss_grad.clone()
--> 177 self('after_loss')
    178 if not self.training or not len(self.yb): return
    179 self('before_backward')

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:141, in Learner.__call__(self, event_name)
--> 141 def __call__(self, event_name): L(event_name).map(self._call_one)

File /usr/local/lib/python3.8/site-packages/fastcore/foundation.py:155, in L.map(self, f, gen, *args, **kwargs)
--> 155 def map(self, f, *args, gen=False, **kwargs): return self._new(map_ex(self, f, *args, gen=gen, **kwargs))

File /usr/local/lib/python3.8/site-packages/fastcore/basics.py:698, in map_ex(iterable, f, gen, *args, **kwargs)
    696 res = map(g, iterable)
    697 if gen: return res
--> 698 return list(res)

File /usr/local/lib/python3.8/site-packages/fastcore/basics.py:683, in bind.__call__(self, *args, **kwargs)
    681     if isinstance(v,_Arg): kwargs[k] = args.pop(v.i)
    682 fargs = [args[x.i] if isinstance(x, _Arg) else x for x in self.pargs] + args[self.maxi+1:]
--> 683 return self.func(*fargs, **kwargs)

File /usr/local/lib/python3.8/site-packages/fastai/learner.py:145, in Learner._call_one(self, event_name)
    143 def _call_one(self, event_name):
    144     if not hasattr(event, event_name): raise Exception(f'missing {event_name}')
--> 145     for cb in self.cbs.sorted('order'): cb(event_name)

File /usr/local/lib/python3.8/site-packages/fastai/callback/core.py:57, in Callback.__call__(self, event_name)
     55 res = None
     56 if self.run and _run:
---> 57     try: res = getattr(self, event_name, noop)()
     58     except (CancelBatchException, CancelEpochException, CancelFitException, CancelStepException, CancelTrainException, CancelValidException): raise
     59     except Exception as e:

File /usr/local/lib/python3.8/site-packages/icevision/models/mmdet/fastai/callbacks.py:45, in MMDetectionCallback.after_loss(self)
     42 self.model.train()
     44 # TODO: implement abstract function
---> 45 self.learn.converted_preds = self.convert_raw_predictions(
     46     batch=self.xb[0], raw_preds=self.raw_preds, records=self.records
     47 )

File /usr/local/lib/python3.8/site-packages/icevision/models/mmdet/common/mask/fastai/callbacks.py:9, in MaskMMDetectionCallback.convert_raw_predictions(self, batch, raw_preds, records)
      8 def convert_raw_predictions(self, batch, raw_preds, records):
----> 9     return convert_raw_predictions(
     10         batch=batch, raw_preds=raw_preds, records=records, detection_threshold=0.0
     11     )

File /usr/local/lib/python3.8/site-packages/icevision/models/mmdet/common/mask/prediction.py:93, in convert_raw_predictions(batch, raw_preds, records, detection_threshold, keep_images)
     87     batch = {
     88         "img": batch["img"][0],
     89         "img_metas": batch["img_metas"][0],
     90     }
     92 batch_list = [dict(zip(batch, t)) for t in zipsafe(*batch.values())]
---> 93 return [
     94     convert_raw_prediction(
     95         sample=sample,
     96         raw_pred=raw_pred,
     97         record=record,
     98         detection_threshold=detection_threshold,
     99         keep_image=keep_images,
    100     )
    101     for sample, raw_pred, record in zip(batch_list, raw_preds, records)
    102 ]

File /usr/local/lib/python3.8/site-packages/icevision/models/mmdet/common/mask/prediction.py:94, in <listcomp>(.0)
     87     batch = {
     88         "img": batch["img"][0],
     89         "img_metas": batch["img_metas"][0],
     90     }
     92 batch_list = [dict(zip(batch, t)) for t in zipsafe(*batch.values())]
     93 return [
---> 94     convert_raw_prediction(
     95         sample=sample,
     96         raw_pred=raw_pred,
     97         record=record,
     98         detection_threshold=detection_threshold,
     99         keep_image=keep_images,
    100     )
    101     for sample, raw_pred, record in zip(batch_list, raw_preds, records)
    102 ]

File /usr/local/lib/python3.8/site-packages/icevision/models/mmdet/common/mask/prediction.py:120, in convert_raw_prediction(sample, raw_pred, record, detection_threshold, keep_image)
    118 keep_labels = labels[keep_mask]
    119 keep_bboxes = [BBox.from_xyxy(*o) for o in bboxes[keep_mask]]
--> 120 keep_masks = MaskArray(np.vstack(raw_masks)[keep_mask])
    122 keep_labels = convert_background_from_last_to_zero(
    123     label_ids=keep_labels, class_map=record.detection.class_map
    124 )
    126 pred = BaseRecord(
    127     (
    128         ScoresRecordComponent(),
   (...)
    133     )
    134 )

File <__array_function__ internals>:180, in vstack(*args, **kwargs)

File /usr/local/lib/python3.8/site-packages/numpy/core/shape_base.py:282, in vstack(tup)
    280 if not isinstance(arrs, list):
    281     arrs = [arrs]
--> 282 return _nx.concatenate(arrs, 0)

File <__array_function__ internals>:180, in concatenate(*args, **kwargs)

ValueError: Exception occured in `MaskMMDetectionCallback` when calling event `after_loss`:
	all the input arrays must have same number of dimensions, but the array at index 0 has 3 dimension(s) and the array at index 1 has 2 dimension(s)
​

I know this quite looks like a dataset error message. But I just use the coco dataset which should be validated. So... I don't know how to solve this problem. The code shows below:

from icevision.all import *

selection = 1

if selection == 0:
    m/openbayes/input/input1/annotations/ask_rcnn
    backbone = model_type.backbones.resnet50_fpn_1x

if selection == 1:
    model_type = models.mmdet.mask_rcnn
    backbone = model_type.backbones.mask_rcnn_swin_t_p4_w7_fpn_1x_coco

if selection == 2:
    model_type = models.mmdet.yolact
    backbone = model_type.backbones.r101_1x8_coco

parser = parsers.COCOMaskParser(
    annotations_filepath="instances_val2017.json",
    img_dir="val2017/",
)

train_rs = parser.parse()

image_size = 640
train_tfms = tfms.A.Adapter([*tfms.A.resize_and_pad(image_size), tfms.A.Normalize()])
valid_tfms = tfms.A.Adapter([*tfms.A.resize_and_pad(image_size), tfms.A.Normalize()])

train_ds = Dataset(train_rs, train_tfms)

train_dl = model_type.train_dl(train_ds, batch_size=8, num_workers=8, shuffle=True)

model = model_type.model(backbone=backbone(pretrained=True), num_classes=len(parser.class_map)) 

metrics = [COCOMetric(metric_type=COCOMetricType.mask)]
learn = model_type.fastai.learner(dls=[train_dl, train_dl], model=model, metrics=metrics)
learn.lr_find()
learn.fine_tune(3, 0.0004)