Hyperparameter tuning, standardization & extended searchspace construction

.gitignore

@@ -2,6 +2,7 @@
 poetry.lock
 noxenv.txt
 noxsettings.toml
+hyperparamtuning/
 
 ### Python ###
 *.pyc
@@ -16,6 +17,8 @@ push_to_pypi.sh
 .nfs*
 *.log
 *.json
+!kernel_tuner/schema/T1/1.0.0/input-schema.json
+!test/test_T1_input.json
 *.csv
 .cache
 *.ipynb_checkpoints

kernel_tuner/__init__.py

@@ -1,5 +1,5 @@
 from kernel_tuner.integration import store_results, create_device_targets
-from kernel_tuner.interface import tune_kernel, run_kernel
+from kernel_tuner.interface import tune_kernel, tune_kernel_T1, run_kernel
 
 from importlib.metadata import version
 

kernel_tuner/backends/backend.py

@@ -1,16 +1,16 @@
-"""This module contains the interface of all kernel_tuner backends"""
+"""This module contains the interface of all kernel_tuner backends."""
 from __future__ import print_function
 
 from abc import ABC, abstractmethod
 
 
 class Backend(ABC):
-    """Base class for kernel_tuner backends"""
+    """Base class for kernel_tuner backends."""
 
     @abstractmethod
     def ready_argument_list(self, arguments):
         """This method must implement the allocation of the arguments on device memory."""
-        pass
+        return arguments
 
     @abstractmethod
     def compile(self, kernel_instance):
@@ -59,7 +59,7 @@ def memcpy_htod(self, dest, src):
 
 
 class GPUBackend(Backend):
-    """Base class for GPU backends"""
+    """Base class for GPU backends."""
 
     @abstractmethod
     def __init__(self, device, iterations, compiler_options, observers):
@@ -82,7 +82,7 @@ def copy_texture_memory_args(self, texmem_args):
 
 
 class CompilerBackend(Backend):
-    """Base class for compiler backends"""
+    """Base class for compiler backends."""
 
     @abstractmethod
     def __init__(self, iterations, compiler_options, compiler):

kernel_tuner/backends/hypertuner.py

@@ -0,0 +1,131 @@
+"""This module contains a 'device' for hyperparameter tuning using the autotuning methodology."""
+
+import platform
+from pathlib import Path
+
+from numpy import mean
+
+from kernel_tuner.backends.backend import Backend
+from kernel_tuner.observers.observer import BenchmarkObserver
+
+try:
+    methodology_available = True
+    from autotuning_methodology.experiments import generate_experiment_file
+    from autotuning_methodology.report_experiments import get_strategy_scores
+except ImportError:
+    methodology_available = False
+
+
+class ScoreObserver(BenchmarkObserver):
+    def __init__(self, dev):
+        self.dev = dev
+        self.scores = []
+
+    def after_finish(self):
+        self.scores.append(self.dev.last_score)
+
+    def get_results(self):
+        results = {'score': mean(self.scores), 'scores': self.scores.copy()}
+        self.scores = []
+        return results
+
+class HypertunerFunctions(Backend):
+    """Class for executing hyperparameter tuning."""
+    units = {}
+
+    def __init__(self, iterations):
+        self.iterations = iterations
+        self.observers = [ScoreObserver(self)]
+        self.name = platform.processor()
+        self.max_threads = 1024
+        self.last_score = None
+
+        # set the environment options
+        env = dict()
+        env["iterations"] = self.iterations
+        self.env = env
+
+        # check for the methodology package
+        if methodology_available is not True:
+            raise ImportError("Unable to import the autotuning methodology, run `pip install autotuning_methodology`.")
+
+    def ready_argument_list(self, arguments):
+        arglist = super().ready_argument_list(arguments)
+        if arglist is None:
+            arglist = []
+        return arglist
+
+    def compile(self, kernel_instance):
+        super().compile(kernel_instance)
+        path = Path(__file__).parent.parent.parent / "hyperparamtuning"
+        path.mkdir(exist_ok=True)
+
+        # TODO get applications & GPUs args from benchmark
+        gpus = ["RTX_3090", "RTX_2080_Ti"]
+        applications = None
+        # applications = [
+        #     {
+        #         "name": "convolution",
+        #         "folder": "./cached_data_used/kernels",
+        #         "input_file": "convolution.json"
+        #     },
+        #     {
+        #         "name": "pnpoly",
+        #         "folder": "./cached_data_used/kernels",
+        #         "input_file": "pnpoly.json"
+        #     }
+        # ]
+
+        # strategy settings
+        strategy: str = kernel_instance.arguments[0]
+        hyperparams = [{'name': k, 'value': v} for k, v in kernel_instance.params.items()]
+        hyperparams_string = "_".join(f"{k}={str(v)}" for k, v in kernel_instance.params.items())
+        searchspace_strategies = [{
+            "autotuner": "KernelTuner",
+            "name": f"{strategy.lower()}_{hyperparams_string}",
+            "display_name": strategy.replace('_', ' ').capitalize(),
+            "search_method": strategy.lower(),
+            'search_method_hyperparameters': hyperparams
+        }]
+
+        # any additional settings
+        override = { 
+            "experimental_groups_defaults": { 
+                "samples": self.iterations 
+            }
+        }
+
+        name = kernel_instance.name if len(kernel_instance.name) > 0 else kernel_instance.kernel_source.kernel_name
+        experiments_filepath = generate_experiment_file(name, path, searchspace_strategies, applications, gpus, 
+                                                        override=override, overwrite_existing_file=True)
+        return str(experiments_filepath)
+
+    def start_event(self):
+        return super().start_event()
+
+    def stop_event(self):
+        return super().stop_event()
+
+    def kernel_finished(self):
+        super().kernel_finished()
+        return True
+
+    def synchronize(self):
+        return super().synchronize()
+
+    def run_kernel(self, func, gpu_args=None, threads=None, grid=None, stream=None):
+        # generate the experiments file
+        experiments_filepath = Path(func)
+
+        # run the methodology to get a fitness score for this configuration
+        scores = get_strategy_scores(str(experiments_filepath))
+        self.last_score = scores[list(scores.keys())[0]]['score']
+
+    def memset(self, allocation, value, size):
+        return super().memset(allocation, value, size)
+
+    def memcpy_dtoh(self, dest, src):
+        return super().memcpy_dtoh(dest, src)
+
+    def memcpy_htod(self, dest, src):
+        return super().memcpy_htod(dest, src)