Fix multiclass SVC with non-uniform class weights

python/cuml/cuml/multiclass/multiclass.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 import cupy as cp
+import numpy as np
 
 from cuml.common.doc_utils import generate_docstring
 from cuml.internals.base import Base
@@ -10,6 +11,67 @@
 from cuml.internals.validation import check_inputs
 
 
+def _fit_weighted_ovo(wrapper, X, y, sample_weight):
+    """Fit one weighted estimator for each pair of classes."""
+    from sklearn.base import clone
+    from sklearn.utils.multiclass import check_classification_targets
+
+    check_classification_targets(y)
+    classes = np.unique(y)
+    if len(classes) < 2:
+        raise ValueError(
+            "OneVsOneClassifier can not be fit when only one class is present."
+        )
+
+    estimators = []
+    pairwise_indices = []
+    pairwise = wrapper.__sklearn_tags__().input_tags.pairwise
+    for i, class_i in enumerate(classes):
+        for class_j in classes[i + 1 :]:
+            mask = (y == class_i) | (y == class_j)
+            indices = np.flatnonzero(mask)
+            X_binary = X[indices]
+            if pairwise:
+                X_binary = X_binary[:, indices]
+            y_binary = (y[indices] == class_j).astype(np.int32)
+            estimators.append(
+                clone(wrapper.estimator).fit(
+                    X_binary,
+                    y_binary,
+                    sample_weight=sample_weight[indices],
+                )
+            )
+            pairwise_indices.append(indices)
+
+    wrapper.classes_ = classes
+    wrapper.estimators_ = estimators
+    wrapper.pairwise_indices_ = pairwise_indices if pairwise else None
+    return wrapper
+
+
+def _fit_weighted_ovr(wrapper, X, y, sample_weight):
+    """Fit one weighted estimator for each class against all other classes."""
+    from sklearn.base import clone
+    from sklearn.preprocessing import LabelBinarizer
+    from sklearn.utils.multiclass import check_classification_targets
+
+    check_classification_targets(y)
+    label_binarizer = LabelBinarizer(sparse_output=False)
+    Y = label_binarizer.fit_transform(y)
+
+    wrapper.label_binarizer_ = label_binarizer
+    wrapper.classes_ = label_binarizer.classes_
+    wrapper.estimators_ = [
+        clone(wrapper.estimator).fit(
+            X,
+            y_binary,
+            sample_weight=sample_weight,
+        )
+        for y_binary in Y.T
+    ]
+    return wrapper
+
+
 class _BaseMulticlassClassifier(ClassifierMixin, Base):
     """Shared base class for multiclass classifiers"""
 
@@ -33,7 +95,7 @@ def classes_(self):
 
     @generate_docstring(y="dense_anydtype")
     @mlfunc(set_input_type=True)
-    def fit(self, X, y) -> "_BaseMulticlassClassifier":
+    def fit(self, X, y, sample_weight=None) -> "_BaseMulticlassClassifier":
         """
         Fit a multiclass classifier.
         """
@@ -47,10 +109,12 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier":
             raise ValueError(
                 f"Expected `strategy` to be one of {list(opts)}, got {self.strategy}"
             )
-        X, y = check_inputs(
+
+        X, y, sample_weight = check_inputs(
             self,
             X,
             y,
+            sample_weight,
             dtype=("float32", "float64"),
             y_dtype=None,
             accept_sparse=True,
@@ -59,7 +123,20 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier":
         )
 
         with exit_internal_context():
-            wrapper = cls(self.estimator, n_jobs=None).fit(X, y)
+            wrapper = cls(self.estimator, n_jobs=None)
+            if sample_weight is None:
+                wrapper.fit(X, y)
+            elif self.strategy == "ovo":
+                wrapper = _fit_weighted_ovo(wrapper, X, y, sample_weight)
+            else:
+                wrapper = _fit_weighted_ovr(wrapper, X, y, sample_weight)
+
+            if hasattr(wrapper.estimators_[0], "n_features_in_"):
+                wrapper.n_features_in_ = wrapper.estimators_[0].n_features_in_
+            if hasattr(wrapper.estimators_[0], "feature_names_in_"):
+                wrapper.feature_names_in_ = wrapper.estimators_[
+                    0
+                ].feature_names_in_
 
         self.multiclass_estimator = wrapper
         return self
@@ -114,14 +191,13 @@ def decision_function(self, X):
 
 class OneVsRestClassifier(_BaseMulticlassClassifier):
     """
-    Wrapper around Sckit-learn's class with the same name. The input can be
-    any kind of cuML compatible array, and the output type follows cuML's
-    output type configuration rules.
-
-    Before passing the data to scikit-learn, it is converted to host (numpy)
-    array. Under the hood the data is partitioned for binary classification,
-    and it is transformed back to the device by the cuML estimator. These
-    copies back and forth the device and the host have some overhead. For more
+    Fit one binary classifier per class. The input can be any kind of cuML
+    compatible array, and the output type follows cuML's output type
+    configuration rules.
+
+    The input is converted to a host (NumPy) array and partitioned into binary
+    classification problems. Each cuML estimator transforms its partition
+    back to the device. These host/device copies have some overhead. For more
     details see issue https://github.com/NVIDIA/cuml/issues/2876.
 
     For documentation see `scikit-learn's OneVsRestClassifier
@@ -161,14 +237,13 @@ class OneVsRestClassifier(_BaseMulticlassClassifier):
 
 class OneVsOneClassifier(_BaseMulticlassClassifier):
     """
-    Wrapper around Sckit-learn's class with the same name. The input can be
-    any kind of cuML compatible array, and the output type follows cuML's
-    output type configuration rules.
-
-    Before passing the data to scikit-learn, it is converted to host (numpy)
-    array. Under the hood the data is partitioned for binary classification,
-    and it is transformed back to the device by the cuML estimator. These
-    copies back and forth the device and the host have some overhead. For more
+    Fit one binary classifier per pair of classes. The input can be any kind
+    of cuML compatible array, and the output type follows cuML's output type
+    configuration rules.
+
+    The input is converted to a host (NumPy) array and partitioned into binary
+    classification problems. Each cuML estimator transforms its partition
+    back to the device. These host/device copies have some overhead. For more
     details see issue https://github.com/NVIDIA/cuml/issues/2876.
 
     For documentation see `scikit-learn's OneVsOneClassifier

python/cuml/cuml/svm/svc.py

@@ -8,7 +8,6 @@
 from cuml.common.doc_utils import generate_docstring
 from cuml.common.sparse import is_sparse
 from cuml.internals.interop import UnsupportedOnCPU, UnsupportedOnGPU
-from cuml.internals.logger import warn
 from cuml.internals.mixins import ClassifierMixin
 from cuml.internals.outputs import ClassLabels, mlfunc
 from cuml.internals.validation import check_inputs, check_is_fitted
@@ -289,13 +288,13 @@ def intercept_(self, value):
         self._intercept_ = value
 
     def _fit_multiclass(self, X, y, sample_weight):
-        if sample_weight is not None:
-            warn(
-                "Sample weights are currently ignored for multi class classification"
-            )
-
         params = self.get_params()
         decision_function_shape = params.pop("decision_function_shape")
+        # ``y`` is label encoded before reaching the multiclass wrapper.
+        # Passing the original class-weight mapping to the binary estimators
+        # would incorrectly apply it to their temporary labels 0/1. The
+        # weights have already been incorporated into ``sample_weight``.
+        params["class_weight"] = None
         wrappers = {"ovo": OneVsOneClassifier, "ovr": OneVsRestClassifier}
         if (multiclass_cls := wrappers.get(decision_function_shape)) is None:
             raise ValueError(
@@ -307,7 +306,7 @@ def _fit_multiclass(self, X, y, sample_weight):
             verbose=self.verbose,
             output_type=self.output_type,
         )
-        self._multiclass.fit(X, y)
+        self._multiclass.fit(X, y, sample_weight=sample_weight)
 
         # if using one-vs-one we align support_ indices to those of
         # full dataset

python/cuml/tests/test_svm.py

@@ -188,6 +188,85 @@ def test_svm_skl_cmp_multiclass(
         )
 
 
+@pytest.mark.parametrize("sparse", [False, True])
+@pytest.mark.parametrize("label_kind", ["numeric", "string"])
+@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"])
+def test_svc_multiclass_class_weight(
+    sparse, label_kind, decision_function_shape
+):
+    X = np.array(
+        [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]],
+        dtype=np.float64,
+    )
+    y = np.repeat(np.arange(3), 2)
+    class_weight = {0: 0.1, 1: 10.0, 2: 0.5}
+    X_test = X[[0, 3, 5]]
+
+    if label_kind == "string":
+        labels = np.array(["a", "b", "c"])
+        y = labels.take(y)
+        class_weight = dict(zip(labels, class_weight.values()))
+
+    if sparse:
+        X = scipy_sparse.csr_matrix(X)
+        X_test = scipy_sparse.csr_matrix(X_test)
+
+    params = {
+        "kernel": "rbf",
+        "decision_function_shape": decision_function_shape,
+        "class_weight": class_weight,
+    }
+    cu_model = cu_svm.SVC(**params).fit(X, y)
+    sk_model = svm.SVC(**params).fit(X, y)
+
+    expected = np.array([1, 1, 2])
+    if label_kind == "string":
+        expected = labels.take(expected)
+
+    np.testing.assert_array_equal(cu_model.predict(X_test), expected)
+    np.testing.assert_array_equal(sk_model.predict(X_test), expected)
+    np.testing.assert_allclose(cu_model.class_weight_, sk_model.class_weight_)
+
+
+@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"])
+def test_svc_multiclass_sample_weight(decision_function_shape):
+    X = np.array(
+        [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5]],
+        dtype=np.float64,
+    )
+    y = np.repeat(np.arange(3), 2)
+    sample_weight = np.array([0.1, 0.1, 10.0, 10.0, 0.5, 0.5])
+    X_test = X[[0, 3, 5]]
+
+    params = {
+        "kernel": "rbf",
+        "decision_function_shape": decision_function_shape,
+    }
+    cu_model = cu_svm.SVC(**params).fit(X, y, sample_weight=sample_weight)
+    sk_model = svm.SVC(**params).fit(X, y, sample_weight=sample_weight)
+
+    expected = np.array([1, 1, 2])
+    np.testing.assert_array_equal(cu_model.predict(X_test), expected)
+    np.testing.assert_array_equal(sk_model.predict(X_test), expected)
+
+
+@pytest.mark.parametrize("decision_function_shape", ["ovo", "ovr"])
+def test_svc_multiclass_balanced_class_weight(decision_function_shape):
+    X = np.arange(14, dtype=np.float64).reshape(7, 2) / 2
+    y = np.array([0, 0, 0, 0, 1, 1, 2])
+
+    params = {
+        "kernel": "rbf",
+        "decision_function_shape": decision_function_shape,
+        "class_weight": "balanced",
+    }
+    cu_model = cu_svm.SVC(**params).fit(X, y)
+    sk_model = svm.SVC(**params).fit(X, y)
+
+    np.testing.assert_array_equal(cu_model.predict(X), sk_model.predict(X))
+    np.testing.assert_allclose(cu_model.class_weight_, sk_model.class_weight_)
+
+
 def test_svm_skl_cmp_decision_function():
     X_train, X_test, y_train, _ = make_dataset("classification1", 4000, 20)
     y_train = y_train.astype("int32")