Skip to content

Commit

Permalink
Redefine the calculation of factors
Browse files Browse the repository at this point in the history
  • Loading branch information
Hhhilulu committed Dec 7, 2023
1 parent 2c1a1b6 commit c1e228a
Showing 1 changed file with 4 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -421,17 +421,15 @@ def _determine_worker_count(self):
factors = {
"default_worker_count": self._DEFAULT_WORKER_COUNT,
"row_count": self._nlines,
"estimated_worker_count_based_on_memory_usage": estimated_available_worker_count,
}

if estimated_available_worker_count is not None and estimated_available_worker_count > 0:
factors.update({
"estimated_worker_count_based_on_memory_usage": estimated_available_worker_count,
})
valid_factors = {k: v for k, v in factors.items() if v is not None and v > 0}

# Take the minimum value as the result
worker_count = min(factors.values())
worker_count = min(valid_factors.values())
bulk_logger.info(
f"Set process count to {worker_count} by taking the minimum value among the factors of {factors}.")
f"Set process count to {worker_count} by taking the minimum value among the factors of {valid_factors}.")
return worker_count

def _log_set_worker_count(self, worker_count, estimated_available_worker_count):
Expand Down

0 comments on commit c1e228a

Please sign in to comment.