diff --git a/python/cuml/cuml/multiclass/multiclass.py b/python/cuml/cuml/multiclass/multiclass.py index f50ffa218c..aa5c4c157d 100644 --- a/python/cuml/cuml/multiclass/multiclass.py +++ b/python/cuml/cuml/multiclass/multiclass.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # import cupy as cp +import numpy as np from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base @@ -10,6 +11,67 @@ from cuml.internals.validation import check_inputs +def _fit_weighted_ovo(wrapper, X, y, sample_weight): + """Fit one weighted estimator for each pair of classes.""" + from sklearn.base import clone + from sklearn.utils.multiclass import check_classification_targets + + check_classification_targets(y) + classes = np.unique(y) + if len(classes) < 2: + raise ValueError( + "OneVsOneClassifier can not be fit when only one class is present." + ) + + estimators = [] + pairwise_indices = [] + pairwise = wrapper.__sklearn_tags__().input_tags.pairwise + for i, class_i in enumerate(classes): + for class_j in classes[i + 1 :]: + mask = (y == class_i) | (y == class_j) + indices = np.flatnonzero(mask) + X_binary = X[indices] + if pairwise: + X_binary = X_binary[:, indices] + y_binary = (y[indices] == class_j).astype(np.int32) + estimators.append( + clone(wrapper.estimator).fit( + X_binary, + y_binary, + sample_weight=sample_weight[indices], + ) + ) + pairwise_indices.append(indices) + + wrapper.classes_ = classes + wrapper.estimators_ = estimators + wrapper.pairwise_indices_ = pairwise_indices if pairwise else None + return wrapper + + +def _fit_weighted_ovr(wrapper, X, y, sample_weight): + """Fit one weighted estimator for each class against all other classes.""" + from sklearn.base import clone + from sklearn.preprocessing import LabelBinarizer + from sklearn.utils.multiclass import check_classification_targets + + check_classification_targets(y) + label_binarizer = LabelBinarizer(sparse_output=False) + Y = label_binarizer.fit_transform(y) + + wrapper.label_binarizer_ = label_binarizer + wrapper.classes_ = label_binarizer.classes_ + wrapper.estimators_ = [ + clone(wrapper.estimator).fit( + X, + y_binary, + sample_weight=sample_weight, + ) + for y_binary in Y.T + ] + return wrapper + + class _BaseMulticlassClassifier(ClassifierMixin, Base): """Shared base class for multiclass classifiers""" @@ -33,7 +95,7 @@ def classes_(self): @generate_docstring(y="dense_anydtype") @mlfunc(set_input_type=True) - def fit(self, X, y) -> "_BaseMulticlassClassifier": + def fit(self, X, y, sample_weight=None) -> "_BaseMulticlassClassifier": """ Fit a multiclass classifier. """ @@ -47,10 +109,12 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier": raise ValueError( f"Expected `strategy` to be one of {list(opts)}, got {self.strategy}" ) - X, y = check_inputs( + + X, y, sample_weight = check_inputs( self, X, y, + sample_weight, dtype=("float32", "float64"), y_dtype=None, accept_sparse=True, @@ -59,7 +123,20 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier": ) with exit_internal_context(): - wrapper = cls(self.estimator, n_jobs=None).fit(X, y) + wrapper = cls(self.estimator, n_jobs=None) + if sample_weight is None: + wrapper.fit(X, y) + elif self.strategy == "ovo": + wrapper = _fit_weighted_ovo(wrapper, X, y, sample_weight) + else: + wrapper = _fit_weighted_ovr(wrapper, X, y, sample_weight) + + if hasattr(wrapper.estimators_[0], "n_features_in_"): + wrapper.n_features_in_ = wrapper.estimators_[0].n_features_in_ + if hasattr(wrapper.estimators_[0], "feature_names_in_"): + wrapper.feature_names_in_ = wrapper.estimators_[ + 0 + ].feature_names_in_ self.multiclass_estimator = wrapper return self @@ -114,14 +191,13 @@ def decision_function(self, X): class OneVsRestClassifier(_BaseMulticlassClassifier): """ - Wrapper around Sckit-learn's class with the same name. The input can be - any kind of cuML compatible array, and the output type follows cuML's - output type configuration rules. - - Before passing the data to scikit-learn, it is converted to host (numpy) - array. Under the hood the data is partitioned for binary classification, - and it is transformed back to the device by the cuML estimator. These - copies back and forth the device and the host have some overhead. For more + Fit one binary classifier per class. The input can be any kind of cuML + compatible array, and the output type follows cuML's output type + configuration rules. + + The input is converted to a host (NumPy) array and partitioned into binary + classification problems. Each cuML estimator transforms its partition + back to the device. These host/device copies have some overhead. For more details see issue https://github.com/NVIDIA/cuml/issues/2876. For documentation see `scikit-learn's OneVsRestClassifier @@ -161,14 +237,13 @@ class OneVsRestClassifier(_BaseMulticlassClassifier): class OneVsOneClassifier(_BaseMulticlassClassifier): """ - Wrapper around Sckit-learn's class with the same name. The input can be - any kind of cuML compatible array, and the output type follows cuML's - output type configuration rules. - - Before passing the data to scikit-learn, it is converted to host (numpy) - array. Under the hood the data is partitioned for binary classification, - and it is transformed back to the device by the cuML estimator. These - copies back and forth the device and the host have some overhead. For more + Fit one binary classifier per pair of classes. The input can be any kind + of cuML compatible array, and the output type follows cuML's output type + configuration rules. + + The input is converted to a host (NumPy) array and partitioned into binary + classification problems. Each cuML estimator transforms its partition + back to the device. These host/device copies have some overhead. For more details see issue https://github.com/NVIDIA/cuml/issues/2876. For documentation see `scikit-learn's OneVsOneClassifier diff --git a/python/cuml/cuml/svm/svc.py b/python/cuml/cuml/svm/svc.py index 2cd28201c5..32d54f564c 100644 --- a/python/cuml/cuml/svm/svc.py +++ b/python/cuml/cuml/svm/svc.py @@ -8,7 +8,6 @@ from cuml.common.doc_utils import generate_docstring from cuml.common.sparse import is_sparse from cuml.internals.interop import UnsupportedOnCPU, UnsupportedOnGPU -from cuml.internals.logger import warn from cuml.internals.mixins import ClassifierMixin from cuml.internals.outputs import ClassLabels, mlfunc from cuml.internals.validation import check_inputs, check_is_fitted @@ -289,13 +288,13 @@ def intercept_(self, value): self._intercept_ = value def _fit_multiclass(self, X, y, sample_weight): - if sample_weight is not None: - warn( - "Sample weights are currently ignored for multi class classification" - ) - params = self.get_params() decision_function_shape = params.pop("decision_function_shape") + # ``y`` is label encoded before reaching the multiclass wrapper. + # Passing the original class-weight mapping to the binary estimators + # would incorrectly apply it to their temporary labels 0/1. The + # weights have already been incorporated into ``sample_weight``. + params["class_weight"] = None wrappers = {"ovo": OneVsOneClassifier, "ovr": OneVsRestClassifier} if (multiclass_cls := wrappers.get(decision_function_shape)) is None: raise ValueError( @@ -307,7 +306,7 @@ def _fit_multiclass(self, X, y, sample_weight): verbose=self.verbose, output_type=self.output_type, ) - self._multiclass.fit(X, y) + self._multiclass.fit(X, y, sample_weight=sample_weight) # if using one-vs-one we align support_ indices to those of # full dataset diff --git a/python/cuml/tests/test_svm.py b/python/cuml/tests/test_svm.py index e5b381c9da..1063bd6430 100644 --- a/python/cuml/tests/test_svm.py +++ b/python/cuml/tests/test_svm.py @@ -188,6 +188,85 @@ def test_svm_skl_cmp_multiclass( ) +@pytest.mark.parametrize("sparse", [False, True]) +@pytest.mark.parametrize("label_kind", ["numeric", "string"]) +@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"]) +def test_svc_multiclass_class_weight( + sparse, label_kind, decision_function_shape +): + X = np.array( + [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], + dtype=np.float64, + ) + y = np.repeat(np.arange(3), 2) + class_weight = {0: 0.1, 1: 10.0, 2: 0.5} + X_test = X[[0, 3, 5]] + + if label_kind == "string": + labels = np.array(["a", "b", "c"]) + y = labels.take(y) + class_weight = dict(zip(labels, class_weight.values())) + + if sparse: + X = scipy_sparse.csr_matrix(X) + X_test = scipy_sparse.csr_matrix(X_test) + + params = { + "kernel": "rbf", + "decision_function_shape": decision_function_shape, + "class_weight": class_weight, + } + cu_model = cu_svm.SVC(**params).fit(X, y) + sk_model = svm.SVC(**params).fit(X, y) + + expected = np.array([1, 1, 2]) + if label_kind == "string": + expected = labels.take(expected) + + np.testing.assert_array_equal(cu_model.predict(X_test), expected) + np.testing.assert_array_equal(sk_model.predict(X_test), expected) + np.testing.assert_allclose(cu_model.class_weight_, sk_model.class_weight_) + + +@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"]) +def test_svc_multiclass_sample_weight(decision_function_shape): + X = np.array( + [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], + dtype=np.float64, + ) + y = np.repeat(np.arange(3), 2) + sample_weight = np.array([0.1, 0.1, 10.0, 10.0, 0.5, 0.5]) + X_test = X[[0, 3, 5]] + + params = { + "kernel": "rbf", + "decision_function_shape": decision_function_shape, + } + cu_model = cu_svm.SVC(**params).fit(X, y, sample_weight=sample_weight) + sk_model = svm.SVC(**params).fit(X, y, sample_weight=sample_weight) + + expected = np.array([1, 1, 2]) + np.testing.assert_array_equal(cu_model.predict(X_test), expected) + np.testing.assert_array_equal(sk_model.predict(X_test), expected) + + +@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"]) +def test_svc_multiclass_balanced_class_weight(decision_function_shape): + X = np.arange(14, dtype=np.float64).reshape(7, 2) / 2 + y = np.array([0, 0, 0, 0, 1, 1, 2]) + + params = { + "kernel": "rbf", + "decision_function_shape": decision_function_shape, + "class_weight": "balanced", + } + cu_model = cu_svm.SVC(**params).fit(X, y) + sk_model = svm.SVC(**params).fit(X, y) + + np.testing.assert_array_equal(cu_model.predict(X), sk_model.predict(X)) + np.testing.assert_allclose(cu_model.class_weight_, sk_model.class_weight_) + + def test_svm_skl_cmp_decision_function(): X_train, X_test, y_train, _ = make_dataset("classification1", 4000, 20) y_train = y_train.astype("int32")