diff --git a/jwt.go b/jwt.go
index 1e97d30..e34a273 100644
--- a/jwt.go
+++ b/jwt.go
@@ -246,11 +246,11 @@ func (config Config) ToMiddleware() (echo.MiddlewareFunc, error) {
 			} else if lastExtractorErr != nil {
 				err = &TokenExtractionError{Err: lastExtractorErr}
 			}
+			if config.ContinueOnIgnoredError {
+				return next(c)
+			}
 			if config.ErrorHandler != nil {
 				tmpErr := config.ErrorHandler(c, err)
-				if config.ContinueOnIgnoredError && tmpErr == nil {
-					return next(c)
-				}
 				return tmpErr
 			}