From fcbea2050e516e7d7e6ac9f1846d5e4213866d84 Mon Sep 17 00:00:00 2001 From: Pierre Kancir Date: Thu, 29 Aug 2024 22:22:16 +0200 Subject: [PATCH] update.py: split procs per tasks and max it out --- update.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/update.py b/update.py index 08443e95d5..2afdf41c12 100755 --- a/update.py +++ b/update.py @@ -232,7 +232,7 @@ def fetch_ardupilot_generated_data(site_mapping: Dict, base_url: str, sub_url: s executor.map(fetch_and_rename, urls, targetfiles, names, timeout=5*60) -def build_one(wiki, fast): +def build_one(wiki, fast, procs_num): """build one wiki""" print('Using sphinx-build for sphinx: %s' % wiki) @@ -246,7 +246,7 @@ def build_one(wiki, fast): shutil.rmtree(output_dir) app = Sphinx(srcdir=source_dir, confdir=source_dir, outdir=html_dir, doctreedir=doctree_dir, buildername='html', - parallel=2) + parallel=procs_num) app.build() @@ -257,6 +257,17 @@ def sphinx_make(site, parallel, fast): done = set() wikis = set(ALL_WIKIS[:]) procs = [] + num_processors = os.cpu_count() + if parallel == -1: + # illimited parallelism so focus on vehicle wikis to split the load + procs_per_task = num_processors // len(WIKI_NAME_TO_VEHICLE_NAME) + else: + if parallel == 1: + # build one at a time, so max procs minus one. + procs_per_task = max(1, num_processors -1) + else: + # split the load accross the tasks + procs_per_task = num_processors // parallel while len(done) != len(wikis): wiki = list(wikis.difference(done))[0] @@ -267,7 +278,7 @@ def sphinx_make(site, parallel, fast): continue if site is not None and not site == wiki: continue - p = multiprocessing.Process(target=build_one, args=(wiki, fast)) + p = multiprocessing.Process(target=build_one, args=(wiki, fast, procs_per_task)) p.start() procs.append(p) while parallel != -1 and len(procs) >= parallel: