Return the value in case of pickle serialization exception instead of failing the whole call

[riyadparvez]

from dogpile.cache import make_region

TIMEOUT_SECONDS = 10 * 60


def my_key_generator(namespace, fn):
    fname = fn.__name__

    def generate_key(*arg):
        # generate a key template:
        # "fname_%d_arg1_arg2_arg3..."
        # key_template = fname + "_" + "%d" + "_".join(str(s) for s in arg[1:])
        key_template = fname + "_" + "_".join(str(s) for s in arg)

        return key_template

    return generate_key


region = make_region(function_key_generator=my_key_generator).configure(
    "dogpile.cache.redis",
    expiration_time=TIMEOUT_SECONDS,
    arguments={
        "host": "localhost",
        "port": 6379,
        "db": 0,
        "redis_expiration_time": TIMEOUT_SECONDS * 2,  # 2 hours
        "distributed_lock": True,
        # thread_local_lock – bool, whether a thread-local Redis lock object should be used.
        # This is the default, but is not compatible with asynchronous runners, as they run in
        # a different thread than the one used to create the lock.
        "thread_local_lock": False,
    },

)


@region.cache_on_arguments()
def load_user_info(user_id):
    log.info(f"Called func {user_id}")
    return lambda: user_id

print(load_user_info(1))

This returns the pickle error, which is to be expected.

Traceback (most recent call last):
  File "/Users/riyad/dev-src/chaos-theory/python-test/dogpile-test.py", line 120, in <module>
    print(load_user_info(1))
          ^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/decorator.py", line 232, in fun
    return caller(func, *(extras + args), **kw)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1577, in get_or_create_for_user_func
    return self.get_or_create(
           ^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1042, in get_or_create
    with Lock(
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/lock.py", line 185, in __enter__
    return self._enter()
           ^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/lock.py", line 94, in _enter
    generated = self._enter_create(value, createdtime)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/lock.py", line 178, in _enter_create
    return self.creator()
           ^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1012, in gen_value
    self._set_cached_value_to_backend(key, value)
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1288, in _set_cached_value_to_backend
    key, self._serialized_cached_value(value)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1258, in _serialized_cached_value
    return self._serialize_cached_value_elements(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/lib/python3.11/site-packages/dogpile/cache/region.py", line 1232, in _serialize_cached_value_elements
    serializer(payload),
    ^^^^^^^^^^^^^^^^^^^
AttributeError: Can't pickle local object 'load_user_info.<locals>.<lambda>'

My question is is there a way when an exception happens on the dogpile side catch the exception, log the exception and return the actual value instead of totally failing the call?

[zzzeek]

not yet. we have a PR for this on the deserialize side at #233. The contributor has not been around. Can you pick up this PR and we can add a similar handler for the serialize side?