Merge pull request #866 from altendky/release/v0.5.1

Author: altendky

CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.1] - 2021-07-15
+### Fixed
+- Detects binary-installed Chia plotting processes again after being broken in v0.5.
+  ([#865](https://github.com/ericaltendorf/plotman/pull/865))
+- Wrap archival indexes around when there are fewer disks, rather than just pointing all the "extra" indexes at the last disk.
+  This will distribute the plot transfers better when you have fewer disks than plotters.
+  ([#855](https://github.com/ericaltendorf/plotman/pull/855))
+### Added
+- `path_suffix` option for rsync and rsyncd archive targets.
+  Allows adding suffixes to the destination path such as to separate original vs. pool plots.
+  ([#800](https://github.com/ericaltendorf/plotman/pull/800))
+- `executable` option for each configurable plotter.
+  Allows explicit specification of the plotter executable path if this is preferred over setting the `PATH` environment variable to find the program.
+  Presently does not support executables other than the expected names (`chia`, and `chia_plot`).
+  ([#823](https://github.com/ericaltendorf/plotman/pull/823))
+
 ## [0.5] - 2021-07-07
 ### Fixed
 - `plotman kill` doesn't leave any temporary files behind anymore.

VERSION

@@ -1 +1 @@
-0.5
+0.5.1

src/plotman/archive.py

@@ -240,7 +240,7 @@ def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archivin
     available = [(d, space) for (d, space) in archdir_freebytes.items() if
                  space > (chosen_plot_size + free_space_margin)]
     if len(available) > 0:
-        index = min(arch_cfg.index, len(available) - 1)
+        index = arch_cfg.index % len(available)
         (archdir, freespace) = sorted(available)[index]
 
     if not archdir:

src/plotman/configuration.py

@@ -73,6 +73,8 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi
 
     if loaded.plotting.type == "chia":
         if loaded.plotting.chia is None:
+            # TODO: fix all the `TODO: use the configured executable` so this is not
+            #       needed.
             raise ConfigurationException(
                 "chia selected as plotter but plotting: chia: was not specified in the config",
             )
@@ -81,6 +83,12 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi
             raise ConfigurationException(
                 "Chia Network plotter accepts up to one of plotting: pool_pk: and pool_contract_address: but both are specified",
             )
+
+        executable_name = os.path.basename(loaded.plotting.chia.executable)
+        if executable_name != "chia":
+            raise ConfigurationException(
+                "plotting: chia: executable: must refer to an executable named chia"
+            )
     elif loaded.plotting.type == "madmax":
         if loaded.plotting.madmax is None:
             raise ConfigurationException(
@@ -101,6 +109,14 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi
                 "madMAx plotter accepts only one of plotting: pool_pk: and pool_contract_address: but both are specified",
             )
 
+        executable_name = os.path.basename(loaded.plotting.madmax.executable)
+        if executable_name != "chia_plot":
+            # TODO: fix all the `TODO: use the configured executable` so this is not
+            #       needed.
+            raise ConfigurationException(
+                "plotting: madmax: executable: must refer to an executable named chia_plot"
+            )
+
     if loaded.archiving is not None:
         preset_target_objects = yaml.safe_load(preset_target_definitions_text)
         preset_target_schema = desert.schema(PresetTargetDefinitions)
@@ -312,6 +328,7 @@ class Scheduling:
 
 @attr.frozen
 class ChiaPlotterOptions:
+    executable: str = "chia"
     n_threads: int = 2
     n_buckets: int = 128
     k: Optional[int] = 32
@@ -321,6 +338,7 @@ class ChiaPlotterOptions:
 
 @attr.frozen
 class MadmaxPlotterOptions:
+    executable: str = "chia_plot"
     n_threads: int = 4
     n_buckets: int = 256
 
@@ -369,9 +387,15 @@ class PlotmanConfig:
     @contextlib.contextmanager
     def setup(self) -> Generator[None, None, None]:
         if self.plotting.type == 'chia':
+            if self.plotting.chia is None:
+                message = (
+                    "internal plotman error, please report the full traceback and your"
+                    + " full configuration file"
+                )
+                raise Exception(message)
             if self.plotting.pool_contract_address is not None:
                 completed_process = subprocess.run(
-                    args=['chia', 'version'],
+                    args=[self.plotting.chia.executable, 'version'],
                     capture_output=True,
                     check=True,
                     encoding='utf-8',
@@ -384,9 +408,16 @@ def setup(self) -> Generator[None, None, None]:
                         f' plots but found: {version}'
                     )
         elif self.plotting.type == 'madmax':
+            if self.plotting.madmax is None:
+                message = (
+                    "internal plotman error, please report the full traceback and your"
+                    + " full configuration file"
+                )
+                raise Exception(message)
+
             if self.plotting.pool_contract_address is not None:
                 completed_process = subprocess.run(
-                    args=['chia_plot', '--help'],
+                    args=[self.plotting.madmax.executable, '--help'],
                     capture_output=True,
                     check=True,
                     encoding='utf-8',

src/plotman/job.py

@@ -31,17 +31,22 @@ def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["
     return sorted([j.progress() for j in all_jobs if os.path.normpath(j.dstdir) == os.path.normpath(d)])
 
 def is_plotting_cmdline(cmdline: typing.List[str]) -> bool:
-    if cmdline and 'python' in cmdline[0].lower():  # Stock Chia plotter
-        cmdline = cmdline[1:]
+    if len(cmdline) == 0:
+        return False
+
+    if 'chia_plot' == os.path.basename(cmdline[0].lower()):  # Madmax plotter
+        # TODO: use the configured executable
+        return True
+    else:
+        if 'python' in cmdline[0].lower():  # Stock Chia plotter
+            cmdline = cmdline[1:]
         return (
             len(cmdline) >= 3
+            # TODO: use the configured executable
             and 'chia' in cmdline[0]
             and 'plots' == cmdline[1]
             and 'create' == cmdline[2]
         )
-    elif cmdline and 'chia_plot' == os.path.basename(cmdline[0].lower()):  # Madmax plotter
-        return True
-    return False
 
 def parse_chia_plot_time(s: str) -> pendulum.DateTime:
     # This will grow to try ISO8601 as well for when Chia logs that way
@@ -54,9 +59,16 @@ def parse_chia_plots_create_command_line(
 ) -> "ParsedChiaPlotsCreateCommand":
     command_line = list(command_line)
     # Parse command line args
-    if 'python' in command_line[0].lower():  # Stock Chia plotter
-        command_line = command_line[1:]
+
+    if 'chia_plot' == os.path.basename(command_line[0].lower()):  # Madmax plotter
+        # TODO: use the configured executable
+        all_command_arguments = command_line[1:]
+        command = madmax._cli_c8121b9
+    else:
+        if 'python' in command_line[0].lower():  # Stock Chia plotter
+            command_line = command_line[1:]
         assert len(command_line) >= 3
+        # TODO: use the configured executable
         assert 'chia' in command_line[0]
         assert 'plots' == command_line[1]
         assert 'create' == command_line[2]
@@ -65,10 +77,6 @@ def parse_chia_plots_create_command_line(
         #       associated command.  For now we'll just use the latest one we have
         #       copied.
         command = chia.commands.latest_command()
-    elif 'chia_plot' in command_line[0].lower():  # Madmax plotter
-        command_line = command_line[1:]
-        all_command_arguments = command_line[2:]
-        command = madmax._cli_c8121b9
 
     # nice idea, but this doesn't include -h
     # help_option_names = command.get_help_option_names(ctx=context)
@@ -268,6 +276,7 @@ def __init__(
         #     'nobitfield': False,
         #     'exclude_final_dir': False,
         # }
+        # TODO: use the configured executable
         if proc.name().startswith("chia_plot"): # MADMAX
             self.k = 32
             self.r = self.args['threads']  # type: ignore[assignment]

src/plotman/manager.py

@@ -144,7 +144,8 @@ def key(key: str) -> job.Phase:
                     raise Exception(
                         "madmax plotter selected but not configured, report this as a plotman bug",
                     )
-                plot_args = ['chia_plot',
+                plot_args = [
+                    plotting_cfg.madmax.executable,
                     '-n', str(1),
                     '-r', str(plotting_cfg.madmax.n_threads),
                     '-u', str(plotting_cfg.madmax.n_buckets),
@@ -158,7 +159,7 @@ def key(key: str) -> job.Phase:
                     raise Exception(
                         "chia plotter selected but not configured, report this as a plotman bug",
                     )
-                plot_args = ['chia', 'plots', 'create',
+                plot_args = [plotting_cfg.chia.executable, 'plots', 'create',
                     '-k', str(plotting_cfg.chia.k),
                     '-r', str(plotting_cfg.chia.n_threads),
                     '-u', str(plotting_cfg.chia.n_buckets),

src/plotman/resources/plotman.yaml

@@ -150,14 +150,19 @@ scheduling:
 # See documentation at
 # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create
 plotting:
-        # Your public keys: farmer and pool - Required for madMAx, optional for chia with mnemonic.txt
+        # Your public keys.  Be sure to use the pool contract address for
+        # portable pool plots.  The pool public key is only for original
+        # non-portable plots that can not be used with the official pooling
+        # protocol.
         # farmer_pk: ...
         # pool_pk: ...
-
+        # pool_contract_address: ...
+        
         # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs
         type: chia
         chia:
                 # The stock plotter: https://github.com/Chia-Network/chia-blockchain
+                # executable: /path/to/chia
                 k: 32                # k-size of plot, leave at 32 most of the time
                 e: False             # Use -e plotting option
                 n_threads: 2         # Threads per job
@@ -167,5 +172,6 @@ plotting:
         # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs
         madmax:
                 # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter
+                # executable: /path/to/chia_plot
                 n_threads: 4          # Default is 4, crank up if you have many cores
                 n_buckets: 256        # Default is 256

src/plotman/resources/target_definitions.yaml

@@ -4,6 +4,7 @@ target_definitions:
       command: rsync
       options: --preallocate --remove-source-files --skip-compress plot --whole-file
       site_root: null
+      path_suffix: ""
 
     # The disk space script must return a line for each directory
     # to consider archiving to with the following form.
@@ -24,7 +25,8 @@ target_definitions:
     transfer_script: |
       #!/bin/bash
       set -evx
-      "${command}" ${options} "${source}" "${destination}"
+      full_destination=$(realpath --canonicalize-missing "${destination}/${path_suffix}")
+      "${command}" ${options} "${source}" "${full_destination}/"
     transfer_process_name: "{command}"
     transfer_process_argument_prefix: "{site_root}"
   rsyncd:
@@ -38,6 +40,7 @@ target_definitions:
       host: null
       site_root: null
       site: null
+      path_suffix: ""
     disk_space_script: |
       #!/bin/bash
       set -evx
@@ -49,9 +52,10 @@ target_definitions:
       #!/bin/bash
       set -evx
       echo Launching transfer activity
-      relative_path=$(realpath --canonicalize-missing --relative-to="${site_root}" "${destination}")
+      full_destination=$(realpath --canonicalize-missing "${destination}/${path_suffix}")
+      relative_path=$(realpath --canonicalize-missing --relative-to="${site_root}" "${full_destination}")
       url_root="rsync://${user}@${host}:${rsync_port}/${site}"
-      "${command}" ${options} "${source}" "${url_root}/${relative_path}"
+      "${command}" ${options} "${source}" "${url_root}/${relative_path}/"
     transfer_process_name: "{command}"
     transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}"
 #  external_script: