From 63d241328a8d569cecd041b898b9a6fcc6c96bb1 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 25 Jun 2024 16:50:53 -0400 Subject: [PATCH 1/5] Add how it works doc. --- CHANGELOG.rst | 9 +++++++++ docs/how_it_works.rst | 26 ++++++++++++++++++++++++++ docs/index.rst | 15 +++++++++------ 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 docs/how_it_works.rst diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9085908..9d88eea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,15 @@ Changelog ######### +next +==== + +Improvements +------------ + +* Improve documentation. (`#92 `_) + + 0.9 (2024-06-03) ================ diff --git a/docs/how_it_works.rst b/docs/how_it_works.rst new file mode 100644 index 0000000..8f8bd04 --- /dev/null +++ b/docs/how_it_works.rst @@ -0,0 +1,26 @@ +How it works +############ + +celery-batches makes no changes to how tasks are created or sent to the broker, +but operates on the workers to process multiple tasks at once. Exactly how tasks +are processed depends on the configuration, but the below assumes usage of the +default `"prefork" configuration`_ of a celery worker (the explanation doesn't +change significantly if the gevent, eventlet, or threads worker pools are used, +but the math is different). + +As background, Celery workers have a "main" process which fetches tasks from the +broker. By default it fetches the ":setting:`worker_prefetch_multiplier` x :setting:`worker_concurrency`" +number of tasks (if available). For example, if the prefetch multiplier is 100 and the +concurrency is 4, it attempts to fetch up to 400 items from the broker's queue. +Once in memory the worker deserializes the messages and runs whatever their +:attr:`~celery.app.task.Task.Strategy` is -- for a normal celery +:class:`~celery.app.task.Task` this passes the tasks to the workers in the +processing pool one at a time. (This is the :func:`~celery.worker.strategy.default` strategy.) + +The :class:`~celery_batches.Batches` task provides a different strategy which instructs +the "main" celery worker process to queue tasks in memory until either +the :attr:`~celery_batches.Batches.flush_interval` or :attr:`~celery_batches.Batches.flush_every` +is reached and passes that list of tasks to the worker in the processing pool +together. + +.. _"prefork" configuration: https://docs.celeryq.dev/en/stable/userguide/workers.html#concurrency diff --git a/docs/index.rst b/docs/index.rst index 5dbaa96..82a4d35 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,8 +14,9 @@ Some potential use-cases for batching of task calls include: * Bulk inserting / updating of data. * Tasks with expensive setup that can run across a range of arguments. -For the ``Batches`` task to work properly you must configure :setting:`worker_prefetch_multiplier` -to zero, or some value where the final multiplied value is higher than ``flush_every``. +For the :class:`~celery_batches.Batches` task to work properly you must configure +:setting:`worker_prefetch_multiplier` to zero, or some value where the final +multiplied value is higher than :attr:`~celery_batches.Batches.flush_every`. .. warning:: @@ -31,8 +32,9 @@ Returning results ################# It is possible to return a result for each task request by calling ``mark_as_done`` -on your results backend. Returning a value from the ``Batches`` task call is only -used to provide values to signals and does not populate into the results backend. +on your results backend. Returning a value from the :class:`~celery_batches.Batches` +task call is only used to provide values to signals and does not populate into the +results backend. .. note:: @@ -48,7 +50,7 @@ Retrying tasks ############## In order to retry a failed task, the task must be re-executed with the original -``task_id``, see the example below: +task :attr:`~celery.worker.request.Request.id`, see the example below: .. code-block:: python @@ -69,7 +71,7 @@ In order to retry a failed task, the task must be re-executed with the original else: app.backend.mark_as_done(request.id, response, request=request) -Note that the retried task is still bound by the flush rules of the ``Batches`` +Note that the retried task is still bound by the flush rules of the :class:`~celery_batches.Batches` task, it is used as a lower-bound and will not run *before* that timeout. In the example above it will run between 10 - 20 seconds from now, assuming no other tasks are in the queue. @@ -79,4 +81,5 @@ tasks are in the queue. examples api + how_it_works history From 6e30fdd96b353d2c592824f2161dac3ddefdbed8 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 25 Jun 2024 20:00:08 -0400 Subject: [PATCH 2/5] Attempt to fix build. --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 7ca1e95..ccc24e0 100644 --- a/tox.ini +++ b/tox.ini @@ -24,8 +24,10 @@ python = [testenv] deps= -r{toxinidir}/requirements/test.txt - celery50: celery>=5.0,<5.1 - celery51: celery>=5.1,<5.2 + # Older celerys have an invalid identifier: pytz (>dev). Use an older pip + # which supports it. + celery50: celery>=5.0,<5.1, pip<24.1 + celery51: celery>=5.1,<5.2, pip<24.1 celery52: celery>=5.2.0,<5.3 celery53: celery>=5.3.0,<5.4 celery54: celery>=5.4.0,<5.5 From a35d098e560592d1e1e68735f849bce494476d74 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 25 Jun 2024 20:15:24 -0400 Subject: [PATCH 3/5] Attempt again. --- tox.ini | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index ccc24e0..4546cb6 100644 --- a/tox.ini +++ b/tox.ini @@ -24,15 +24,18 @@ python = [testenv] deps= -r{toxinidir}/requirements/test.txt - # Older celerys have an invalid identifier: pytz (>dev). Use an older pip - # which supports it. - celery50: celery>=5.0,<5.1, pip<24.1 - celery51: celery>=5.1,<5.2, pip<24.1 + celery50: celery>=5.0,<5.1 + celery51: celery>=5.1,<5.2 celery52: celery>=5.2.0,<5.3 celery53: celery>=5.3.0,<5.4 celery54: celery>=5.4.0,<5.5 celerymain: https://codeload.github.com/celery/celery/zip/main + # Older celerys have an invalid identifier: pytz (>dev). Use an older pip + # which supports it. + celery50: pip<24.1 + celery51: pip<24.1 + # By default celery (via kombu) install py-amqp. redis: celery[redis] sitepackages = False From 2d56f45d226214a3cc55dc198de6ef3adcd31978 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 26 Jun 2024 20:32:38 -0400 Subject: [PATCH 4/5] Maybe? --- tox.ini | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index 4546cb6..7fc7adc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,8 @@ [tox] +# Older celerys have an invalid identifier: pytz (>dev). Use an older pip +# which supports it. +requires = + pip<24.1 envlist = {pypy3,3.8,3.9}-celery{50,51,52,main}-unit, # Celery 5.2 added support for Python 3.10. @@ -31,11 +35,6 @@ deps= celery54: celery>=5.4.0,<5.5 celerymain: https://codeload.github.com/celery/celery/zip/main - # Older celerys have an invalid identifier: pytz (>dev). Use an older pip - # which supports it. - celery50: pip<24.1 - celery51: pip<24.1 - # By default celery (via kombu) install py-amqp. redis: celery[redis] sitepackages = False From 944d92de4382c73375a58538d196feb6caee503e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 26 Jun 2024 20:45:07 -0400 Subject: [PATCH 5/5] Drop support for Celery < 5.2. --- CHANGELOG.rst | 7 ++++++- tox.ini | 10 +--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d88eea..cf22dbc 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,7 +9,12 @@ next Improvements ------------ -* Improve documentation. (`#92 `_) +* Improve documentation. (`#92 `_) + +Maintenance +----------- + +* Drop support for Celery < 5.2. (`#92 `_) 0.9 (2024-06-03) diff --git a/tox.ini b/tox.ini index 7fc7adc..1cfb923 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,6 @@ [tox] -# Older celerys have an invalid identifier: pytz (>dev). Use an older pip -# which supports it. -requires = - pip<24.1 envlist = - {pypy3,3.8,3.9}-celery{50,51,52,main}-unit, - # Celery 5.2 added support for Python 3.10. - 3.10-celery{52,53,main}-unit, + {pypy3,3.8,3.9,3.10}-celery{52,main}-unit, # Celery 5.3 added support for Python 3.11. 3.11-celery{53,main}-unit, # Celery 5.4 added support for Python 3.12. @@ -28,8 +22,6 @@ python = [testenv] deps= -r{toxinidir}/requirements/test.txt - celery50: celery>=5.0,<5.1 - celery51: celery>=5.1,<5.2 celery52: celery>=5.2.0,<5.3 celery53: celery>=5.3.0,<5.4 celery54: celery>=5.4.0,<5.5