diff --git a/src/KeyHandler.cpp b/src/KeyHandler.cpp
index 6f0240c..b521866 100644
--- a/src/KeyHandler.cpp
+++ b/src/KeyHandler.cpp
@@ -1125,12 +1125,18 @@ bool KeyHandler::handleNumberInput(Key key,
         std::make_unique<InputStates::NumberInput>(newNumber, candidates);
     stateCallback(std::move(newState));
     return true;
+  } else if (!state->candidates.empty()) {
+    // If the candidate panel is visible, let it handle the key.
+    return false;
   } else if (std::isprint(key.ascii)) {
+    // Reject all other printable, non-numeric keys.
     errorCallback();
     return true;
   }
 
-  return false;
+  // If the buffer is empty, all other keys (e.g. cursor keys) exit the state.
+  stateCallback(std::make_unique<InputStates::EmptyIgnoringPrevious>());
+  return true;
 }
 
 bool KeyHandler::handleBig5(Key key, McBopomofo::InputStates::Big5* state,
diff --git a/src/McBopomofo.cpp b/src/McBopomofo.cpp
index 7224552..73ecc79 100644
--- a/src/McBopomofo.cpp
+++ b/src/McBopomofo.cpp
@@ -716,13 +716,6 @@ void McBopomofoEngine::keyEvent(const fcitx::InputMethodEntry& /*unused*/,
       keyEvent.filterAndAccept();
       return;
     }
-    InputStates::NumberInput* currentNumberInput =
-        dynamic_cast<InputStates::NumberInput*>(state_.get());
-    if (currentNumberInput != nullptr) {
-      if (currentNumberInput->candidates.empty()) {
-        return;
-      }
-    }
   }
 
   if (dynamic_cast<InputStates::ChoosingCandidate*>(state_.get()) != nullptr ||