Increase xact retry delay and increase per attempt

Bump the base transaction retry from 100ms to 500ms, and increase
linearly between each successive attempt to provide deeper backoff. This
alleviates pressure on smaller connection pools and gives more time to
recover from contentions failures.

lib/oban/peers/postgres.ex

@@ -70,11 +70,13 @@ defmodule Oban.Peers.Postgres do
     if is_reference(timer), do: Process.cancel_timer(timer)
 
     if state.leader? do
+      fun = fn ->
+        delete_self(state)
+        notify_down(state)
+      end
+
       try do
-        Repo.transaction(state.conf, fn ->
-          delete_self(state)
-          notify_down(state)
-        end)
+        Repo.transaction(state.conf, fun, retry: 1)
       catch
         :exit, _reason -> :ok
       end
@@ -102,7 +104,7 @@ defmodule Oban.Peers.Postgres do
           |> upsert_peer()
         end
 
-        case Repo.transaction(state.conf, fun) do
+        case Repo.transaction(state.conf, fun, retry: 1) do
           {:ok, state} ->
             {state, %{meta | leader: state.leader?}}
 

lib/oban/repo.ex

@@ -64,7 +64,7 @@ defmodule Oban.Repo do
     update_all: 3
   ]
 
-  @retry_opts delay: 100, retry: 5, expected_delay: 5, expected_retry: 10
+  @retry_opts delay: 500, retry: 5, expected_delay: 10, expected_retry: 20
 
   for {fun, arity} <- @callbacks_without_opts do
     args = [Macro.var(:conf, __MODULE__) | Macro.generate_arguments(arity, __MODULE__)]
@@ -141,11 +141,11 @@ defmodule Oban.Repo do
 
   Backoff helpers, in addition to the standard transaction options:
 
-  * `delay` — the time to sleep between retries, defaults to `100ms`
+  * `delay` — the time to sleep between retries, defaults to `500ms`
   * `retry` — the number of retries for unexpected errors, defaults to `5`
   * `expected_delay` — the time to sleep between expected errors, e.g. `serialization` or
-    `lock_not_available`, defaults to `5ms`
-  * `expected_retry` — the number of retries for expected errors, defaults to `10`
+    `lock_not_available`, defaults to `10ms`
+  * `expected_retry` — the number of retries for expected errors, defaults to `20`
   """
   @doc since: "2.18.1"
   def transaction(conf, fun_or_multi, opts \\ []) do
@@ -163,7 +163,7 @@ defmodule Oban.Repo do
           jittery_sleep(opts[:expected_delay])
 
         attempt < opts[:retry] ->
-          jittery_sleep(opts[:delay])
+          jittery_sleep(attempt * opts[:delay])
 
         true ->
           reraise error, __STACKTRACE__