diff --git a/changelog/13122.improvement.rst b/changelog/13122.improvement.rst
new file mode 100644
index 00000000000..d7372380e81
--- /dev/null
+++ b/changelog/13122.improvement.rst
@@ -0,0 +1,11 @@
+Improve the ``--stepwise`` flag to not forget the last failed test in case pytest is executed later without the flag.
+
+This enables the following workflow:
+
+1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test;
+2. User iteratively updates the code and runs the test in isolation, without the ``--stepwise`` flag
+   (for example in an IDE), until it is fixed.
+3. At this point, the user can execute pytest with ``--stepwise`` again and pytest will continue from the previously
+   failed test, and if it passes, continue with the next tests.
+
+Previously, at step 3, pytest would start from the beginning, forgetting the failed tests.
diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py
index c7860808c35..82d1fe3aee5 100644
--- a/src/_pytest/stepwise.py
+++ b/src/_pytest/stepwise.py
@@ -47,8 +47,6 @@ def pytest_sessionfinish(session: Session) -> None:
             # Do not update cache if this process is a xdist worker to prevent
             # race conditions (#10641).
             return
-        # Clear the list of failing tests if the plugin is not active.
-        session.config.cache.set(STEPWISE_CACHE_DIR, [])
 
 
 class StepwisePlugin:
diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py
index affdb73375e..02239e8b5f1 100644
--- a/testing/test_stepwise.py
+++ b/testing/test_stepwise.py
@@ -358,3 +358,41 @@ def test_one():
     with stepwise_cache_file.open(encoding="utf-8") as file_handle:
         observed_value = file_handle.readlines()
     assert [expected_value] == observed_value
+
+
+def test_do_not_clear_cache_if_disabled(pytester: Pytester) -> None:
+    """
+    If pytest is run without --step-wise, do not clear the stepwise cache.
+
+    Keeping the cache around is important for this workflow:
+
+    1. Run tests with --stepwise
+    2. Stop at the failing test, and iterate over it changing the code and running it in isolation
+    (in the IDE for example).
+    3. Run tests with --stepwise again - at this point we expect to start from the failing test, which should now pass,
+       and continue with the next tests.
+    """
+    pytester.makepyfile(
+        """
+        def test_1():
+            pass
+        def test_2():
+            assert False
+        def test_3():
+            pass
+        """
+    )
+    result = pytester.runpytest("--stepwise")
+    result.stdout.fnmatch_lines(
+        ["*::test_2 - assert False*", "*failed, continuing from this test next run*"]
+    )
+
+    # Run a specific test without passing `--stepwise`.
+    result = pytester.runpytest("-k", "test_1")
+    result.stdout.fnmatch_lines(["*1 passed*"])
+
+    # Running with `--stepwise` should continue from the last failing test.
+    result = pytester.runpytest("--stepwise")
+    result.stdout.fnmatch_lines(
+        ["*::test_2 - assert False*", "*failed, continuing from this test next run*"]
+    )