From 175e6c9ec73fba3fe2c58429f86f90439593d3d2 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Wed, 18 Sep 2024 10:05:03 -0600 Subject: [PATCH] removing telemetry --- doc/reference/developer.md | 50 ----- src/sklearn_evaluation/__init__.py | 3 - src/sklearn_evaluation/evaluator.py | 2 - .../grid/random_forest_classifier_grid.py | 8 - .../nb/NotebookCollection.py | 4 - src/sklearn_evaluation/plot/calibration.py | 4 - src/sklearn_evaluation/plot/classification.py | 7 - .../plot/classification_report.py | 4 - src/sklearn_evaluation/plot/clustering.py | 7 - .../plot/confusion_matrix_interactive.py | 4 - .../plot/cumulative_gain_lift_curve.py | 3 - .../plot/feature_ranking.py | 3 - src/sklearn_evaluation/telemetry.py | 109 ----------- src/sklearn_evaluation/tracker.py | 13 -- tests/test_skeval_telemetry.py | 185 ------------------ 15 files changed, 406 deletions(-) delete mode 100644 src/sklearn_evaluation/telemetry.py delete mode 100644 tests/test_skeval_telemetry.py diff --git a/doc/reference/developer.md b/doc/reference/developer.md index bc912d20..4d22ae1d 100644 --- a/doc/reference/developer.md +++ b/doc/reference/developer.md @@ -403,53 +403,3 @@ See the [`roc`](https://github.com/ploomber/sklearn-evaluation/blob/8056bc31ec5e The `ax` object must be returned at the end of the function. See the [`roc`](https://github.com/ploomber/sklearn-evaluation/blob/8056bc31ec5e372102d0ee5ada988e380b077c4b/src/sklearn_evaluation/plot/roc.py#L45) function for an example. - -## Telemetry - -Monitoring the state of `sklearn-evaluation` - -Use [`SKLearnEvaluationLogger`](https://github.com/ploomber/sklearn-evaluation/blob/f32c15a43f4a9b4c2e588b3c0f71ba6dc5a71a7e/src/sklearn_evaluation/telemetry.py#L19) decorator to generate logs - -Example: - -```python -@SKLearnEvaluationLogger.log(feature='plot') -def confusion_matrix( - y_true, - y_pred, - target_names=None, - normalize=False, - cmap=None, - ax=None, - **kwargs): -pass -``` - -this will generate the following log: - -```json - { - "metadata": { - "action": "confusion_matrix" - "feature": "plot", - "args": { - "target_names": "None", - "normalize": "False", - "cmap": "None", - "ax": "None" - } - } - } -``` - -\*\* since `y_true` and `y_pred` are positional arguments without default values it won't log them - -### Queries - -1. Run queries and filter out `sklearn-evaluation` events by the event name: `sklearn-evaluation` -2. Break these events by feature ('plot', 'report', 'SQLiteTracker', 'NotebookCollection') -3. Break events by actions/func name (i.e: 'confusion_matrix', 'roc', etc...) - -### Errors - -Failing runnings will be named: `sklearn-evaluation-error` diff --git a/src/sklearn_evaluation/__init__.py b/src/sklearn_evaluation/__init__.py index 59132302..aa317b84 100644 --- a/src/sklearn_evaluation/__init__.py +++ b/src/sklearn_evaluation/__init__.py @@ -5,7 +5,6 @@ from sklearn_evaluation.nb.NotebookCollection import NotebookCollection from sklearn_evaluation.nb.NotebookDatabase import NotebookDatabase from sklearn_evaluation.tracker import SQLiteTracker -from sklearn_evaluation.telemetry import telemetry __all__ = [ "ClassifierEvaluator", @@ -14,5 +13,3 @@ "NotebookCollection", "NotebookDatabase", ] - -telemetry.log_api("imported") diff --git a/src/sklearn_evaluation/evaluator.py b/src/sklearn_evaluation/evaluator.py index 1718999b..e6b2397c 100644 --- a/src/sklearn_evaluation/evaluator.py +++ b/src/sklearn_evaluation/evaluator.py @@ -5,7 +5,6 @@ from sklearn_evaluation.report.report import Report from sklearn_evaluation.util import estimator_type, class_name from sklearn_evaluation import plot -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger class ClassifierEvaluator(object): @@ -153,7 +152,6 @@ def html_serializable(self): """ return EvaluatorHTMLSerializer(self) - @SKLearnEvaluationLogger.log(feature="report") def make_report(self, template=None): """ Make HTML report diff --git a/src/sklearn_evaluation/grid/random_forest_classifier_grid.py b/src/sklearn_evaluation/grid/random_forest_classifier_grid.py index 65de9f92..ac8d1571 100644 --- a/src/sklearn_evaluation/grid/random_forest_classifier_grid.py +++ b/src/sklearn_evaluation/grid/random_forest_classifier_grid.py @@ -5,7 +5,6 @@ from sklearn_evaluation.grid.classifier_grid import AbstractClassifierGrid, GridTypes from sklearn.utils.validation import check_consistent_length import warnings -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger class RandomForestClassifierGrid(AbstractClassifierGrid): @@ -49,7 +48,6 @@ class RandomForestClassifierGrid(AbstractClassifierGrid): } ) - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-init") def __init__(self, grid, cv=3, verbose=0): """ A random forest classifier grid. @@ -96,7 +94,6 @@ def __init__(self, grid, cv=3, verbose=0): verbose=verbose, ) - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-fit") def fit(self, X, y): """ Fit estimator. @@ -121,7 +118,6 @@ def fit(self, X, y): self.grid_search_cv_.fit(X, y, sample_weight=None) return self - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-set-test-data") def set_test_data(self, X_test, y_test) -> None: """ Set the test data @@ -140,7 +136,6 @@ def set_test_data(self, X_test, y_test) -> None: self.X_test = X_test self.y_test = y_test - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-confusion-matrix") def confusion_matrix(self): """ Plots a confusion matrix based on `GridSearchCV.best_estimator_`. @@ -160,7 +155,6 @@ def confusion_matrix(self): y_pred = self.grid_search_cv_.best_estimator_.predict(X_test) return plot.confusion_matrix(y_test, y_pred) - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-roc") def roc(self): """ Plots an ROC based on `GridSearchCV.best_estimator_`. @@ -185,7 +179,6 @@ def roc(self): return plot.roc(y_test, y_pred) - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-feature-importances") def feature_importances(self): """ Plots feature importances based on `GridSearchCV.best_estimator_`. @@ -203,7 +196,6 @@ def feature_importances(self): feature_importances = self.grid_search_cv_.best_estimator_.feature_importances_ return plot.feature_importances(feature_importances) - @SKLearnEvaluationLogger.log("RandomForestClassifierGrid-grid-search-results") def grid_search_results(self, change="n_estimators", kind="line"): """ Plots grid search results based on `GridSearchCV.best_estimator_`. diff --git a/src/sklearn_evaluation/nb/NotebookCollection.py b/src/sklearn_evaluation/nb/NotebookCollection.py index 08141e67..58ccb2fb 100644 --- a/src/sklearn_evaluation/nb/NotebookCollection.py +++ b/src/sklearn_evaluation/nb/NotebookCollection.py @@ -15,7 +15,6 @@ from sklearn_evaluation.nb.NotebookIntrospector import NotebookIntrospector from sklearn_evaluation.nb.sets import differences from sklearn_evaluation.table import Table -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger _env = Environment(loader=PackageLoader("sklearn_evaluation", "assets/nb")) _fm = black.FileMode(string_normalization=False, line_length=40) @@ -41,9 +40,6 @@ class NotebookCollection(Mapping): as identifier (ignores extension) """ # noqa - @SKLearnEvaluationLogger.log( - feature="NotebookCollection", action="init-NotebookCollection" - ) def __init__(self, paths, ids=None, scores=False): if ids is None: ids = paths diff --git a/src/sklearn_evaluation/plot/calibration.py b/src/sklearn_evaluation/plot/calibration.py index be19918b..6b641f9e 100644 --- a/src/sklearn_evaluation/plot/calibration.py +++ b/src/sklearn_evaluation/plot/calibration.py @@ -34,7 +34,6 @@ from sklearn_evaluation import __version__ from sklearn_evaluation.util import isiterofiter from ploomber_core.exceptions import modify_exceptions -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from sklearn_evaluation.plot.plot import AbstractComposedPlot, AbstractPlot from sklearn_evaluation.plot.style import apply_theme, get_color_palette @@ -147,7 +146,6 @@ class CalibrationCurve(AbstractPlot): """ @modify_exceptions - @SKLearnEvaluationLogger.log(feature="plot", action="calibration-curve-init") def __init__( self, mean_predicted_value, @@ -304,7 +302,6 @@ def _get_data(self): class CalibrationCurveAdd(AbstractComposedPlot): @modify_exceptions - @SKLearnEvaluationLogger.log(feature="plot", action="calibration-curve-add-init") def __init__( self, mean_predicted_value_list, @@ -360,7 +357,6 @@ def plot(self, ax=None): @modify_exceptions -@SKLearnEvaluationLogger.log(feature="plot") def calibration_curve( y_true, probabilities, clf_names=None, n_bins=10, cmap="nipy_spectral", ax=None ): diff --git a/src/sklearn_evaluation/plot/classification.py b/src/sklearn_evaluation/plot/classification.py index f59c16e4..4b4f3c65 100644 --- a/src/sklearn_evaluation/plot/classification.py +++ b/src/sklearn_evaluation/plot/classification.py @@ -10,7 +10,6 @@ from sklearn.metrics import confusion_matrix as sk_confusion_matrix from sklearn_evaluation import __version__ -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from sklearn_evaluation.plot.matplotlib import bar from sklearn_evaluation.metrics import precision_at from sklearn_evaluation import compute @@ -93,7 +92,6 @@ class ConfusionMatrix(AbstractPlot): Added ``cmap`` argument """ - @SKLearnEvaluationLogger.log(feature="plot", action="confusion-matrix-init") @modify_exceptions def __init__(self, cm, *, target_names=None, normalize=False, cmap=None): self.cm = cm @@ -112,14 +110,12 @@ def plot(self, ax=None): return self - @SKLearnEvaluationLogger.log(feature="plot", action="confusion-matrix-sub") def __sub__(self, other): cm = self.cm - other.cm obj = ConfusionMatrixSub(cm, self.target_names) obj.plot() return obj - @SKLearnEvaluationLogger.log(feature="plot", action="confusion-matrix-add") def __add__(self, other): obj = ConfusionMatrixAdd(self.cm, other.cm, self.target_names) obj.plot() @@ -178,7 +174,6 @@ def _confusion_matrix(y_true, y_pred, normalize): return cm -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def confusion_matrix( y_true, y_pred, target_names=None, normalize=False, cmap=None, ax=None @@ -314,7 +309,6 @@ def _plot_cm(cm, cmap, ax, target_names, normalize): # http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def feature_importances( data, top_n=None, feature_names=None, orientation="horizontal", ax=None @@ -371,7 +365,6 @@ def feature_importances( return ax -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def precision_at_proportions(y_true, y_score, ax=None): """ diff --git a/src/sklearn_evaluation/plot/classification_report.py b/src/sklearn_evaluation/plot/classification_report.py index 58a58f13..890f9595 100644 --- a/src/sklearn_evaluation/plot/classification_report.py +++ b/src/sklearn_evaluation/plot/classification_report.py @@ -4,7 +4,6 @@ import numpy as np import matplotlib.pyplot as plt from sklearn.metrics import classification_report as sk_classification_report -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from sklearn_evaluation.plot.classification import _add_values_to_matrix from sklearn_evaluation.plot.plot import AbstractPlot, AbstractComposedPlot @@ -85,7 +84,6 @@ class ClassificationReport(AbstractPlot): """ - @SKLearnEvaluationLogger.log(feature="plot", action="classification-report-init") def __init__( self, matrix, @@ -109,13 +107,11 @@ def plot(self, ax=None): return self - @SKLearnEvaluationLogger.log(feature="plot", action="classification-report-sub") def __sub__(self, other): return ClassificationReportSub( self.matrix, other.matrix, self.keys, target_names=self.target_names ).plot() - @SKLearnEvaluationLogger.log(feature="plot", action="classification-report-add") def __add__(self, other): return ClassificationReportAdd( self.matrix, other.matrix, keys=self.keys, target_names=self.target_names diff --git a/src/sklearn_evaluation/plot/clustering.py b/src/sklearn_evaluation/plot/clustering.py index 1572e26b..09665c1f 100644 --- a/src/sklearn_evaluation/plot/clustering.py +++ b/src/sklearn_evaluation/plot/clustering.py @@ -36,8 +36,6 @@ from sklearn.preprocessing import LabelEncoder from joblib import Parallel, delayed -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger - from ploomber_core.exceptions import modify_exceptions from sklearn_evaluation.plot.style import apply_theme @@ -61,7 +59,6 @@ def _generate_axes(cluster, figsize, ax): return ax -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def elbow_curve( X, @@ -140,7 +137,6 @@ def elbow_curve( @apply_theme() -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def elbow_curve_from_results(n_clusters, sum_of_squares, times, ax=None): """ @@ -198,7 +194,6 @@ def _clone_and_score_clusterer(clf, X, n_clusters): return clf.fit(X).score(X), time.time() - start -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def silhouette_analysis( X, @@ -298,7 +293,6 @@ def silhouette_analysis( @apply_theme(cmap_style="gradient") -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def _silhouette_analysis_one_model( X, @@ -386,7 +380,6 @@ def _silhouette_analysis_one_model( return ax -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def silhouette_analysis_from_results( X, diff --git a/src/sklearn_evaluation/plot/confusion_matrix_interactive.py b/src/sklearn_evaluation/plot/confusion_matrix_interactive.py index 19b98b1d..ef24d656 100644 --- a/src/sklearn_evaluation/plot/confusion_matrix_interactive.py +++ b/src/sklearn_evaluation/plot/confusion_matrix_interactive.py @@ -14,7 +14,6 @@ from sklearn.metrics import confusion_matrix as sk_confusion_matrix from sklearn_evaluation import __version__ -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from sklearn_evaluation.plot.plot import AbstractPlot from ploomber_core.dependencies import requires from ploomber_core.exceptions import modify_exceptions @@ -264,9 +263,6 @@ class InteractiveConfusionMatrix(AbstractPlot): .. versionadded:: 0.11.3 """ - @SKLearnEvaluationLogger.log( - feature="plot", action="interactive-confusion-matrix-init" - ) @modify_exceptions def __init__(self, cm, *, target_names=None, interactive_data=None): self.cm = cm diff --git a/src/sklearn_evaluation/plot/cumulative_gain_lift_curve.py b/src/sklearn_evaluation/plot/cumulative_gain_lift_curve.py index e07d2a10..ac20b512 100644 --- a/src/sklearn_evaluation/plot/cumulative_gain_lift_curve.py +++ b/src/sklearn_evaluation/plot/cumulative_gain_lift_curve.py @@ -28,7 +28,6 @@ import numpy as np import matplotlib.pyplot as plt -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from ploomber_core.exceptions import modify_exceptions from sklearn_evaluation.plot.style import apply_theme @@ -94,7 +93,6 @@ def _cumulative_gain_curve(y_true, y_score, pos_label=None): @apply_theme() -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def cumulative_gain( y_true, @@ -178,7 +176,6 @@ def cumulative_gain( @apply_theme() -@SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def lift_curve( y_true, diff --git a/src/sklearn_evaluation/plot/feature_ranking.py b/src/sklearn_evaluation/plot/feature_ranking.py index 8ae3f34f..7b62e030 100644 --- a/src/sklearn_evaluation/plot/feature_ranking.py +++ b/src/sklearn_evaluation/plot/feature_ranking.py @@ -21,7 +21,6 @@ from scipy.stats import shapiro from scipy.stats import spearmanr from scipy.stats import kendalltau as sp_kendalltau -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from ploomber_core.exceptions import modify_exceptions import matplotlib.pyplot as plt @@ -153,7 +152,6 @@ def _derive_features_from_ranks(self, ranks): ) self.features_ = np.array(self.features) - @SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def feature_ranks(self, X): """ @@ -174,7 +172,6 @@ def feature_ranks(self, X): return self.ax - @SKLearnEvaluationLogger.log(feature="plot") @modify_exceptions def feature_ranks_custom_algorithm(self, ranks): """ diff --git a/src/sklearn_evaluation/telemetry.py b/src/sklearn_evaluation/telemetry.py deleted file mode 100644 index 92669fd3..00000000 --- a/src/sklearn_evaluation/telemetry.py +++ /dev/null @@ -1,109 +0,0 @@ -from functools import wraps -import inspect -from inspect import signature - -try: - from importlib.metadata import version -except ModuleNotFoundError: - from importlib_metadata import version - -from ploomber_core.telemetry.telemetry import Telemetry - -telemetry = Telemetry( - api_key="phc_P9SpSeypyPwxrMdFn2edOOEooQioF2axppyEeDwtMSP", - package_name="sklearn-evaluation", - version=version("sklearn_evaluation"), -) - - -class SKLearnEvaluationLogger: - def flags(): - return ["is_report"] - - @classmethod - def log(self, action=None, feature=None): - """Logs the function and then runs it - - Parameters - ---------- - action : string, default=None - The desired action to be logged (i.e: 'confusion_matrix', 'roc'). - If `action=None` it will log the function's name. - - feature: string, default=None - The main feature (i.e: 'plot', 'report', - 'SQLiteTracker', 'NotebookCollection') - """ - - def wrapper(func): - @wraps(func) - def inner(*args, **kwargs): - metadata = self._prepare_metadata( - self, func, action, feature, *args, **kwargs - ) - telemetry.log_api("sklearn-evaluation", metadata=metadata) - - try: - result = func(*args, **kwargs) - except Exception as e: - metadata["exception"] = str(e) - telemetry.log_api("sklearn-evaluation-error", metadata=metadata) - raise e - - return result - - return inner - - return wrapper - - def _get_func_arguments_to_log(self, func, *args, **kwargs): - args_to_log = dict() - flags = dict({}) - - sig = signature(func) - bound = sig.bind(*args, **kwargs) - bound.apply_defaults() - arguments = bound.arguments - args_with_default_values = [] - - for tupple in sig.parameters.items(): - param_name = tupple[0] - param_value = tupple[1] - if param_value.default is not inspect._empty: - args_with_default_values.append(param_name) - - # extract only args with default values - for key, value in arguments.items(): - if key in args_with_default_values: - args_to_log[key] = value - - elif key == "kwargs": - flags = self._extract_flags(self, **value) - - return args_to_log, flags - - def _extract_flags(self, **kwargs): - flags = dict({}) - for key, value in kwargs.items(): - if self._is_flag(self, key): - flags[key] = value - - return flags - - def _is_flag(self, key): - flags = self.flags() - return key in flags - - def _prepare_metadata(self, func, action, feature, *args, **kwargs): - _action = action or func.__name__ - _args, _flags = self._get_func_arguments_to_log(self, func, *args, **kwargs) - - metadata = {"action": _action, "feature": feature} - - if len(_args) > 0: - metadata["args"] = _args - - if len(_flags) > 0: - metadata["flags"] = _flags - - return metadata diff --git a/src/sklearn_evaluation/tracker.py b/src/sklearn_evaluation/tracker.py index 9504d2e8..73eb497b 100644 --- a/src/sklearn_evaluation/tracker.py +++ b/src/sklearn_evaluation/tracker.py @@ -9,7 +9,6 @@ from sklearn_evaluation.table import Table from sklearn_evaluation.report.serialize import try_serialize_figures, figure2html -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger from sklearn_evaluation.nb.NotebookCollection import ( add_compare_tab, tabs_html_from_content, @@ -264,7 +263,6 @@ def _json_loads(s): return df - @SKLearnEvaluationLogger.log(feature="SQLiteTracker") def query(self, code, as_frame=True, render_plots=False): """Query the database @@ -322,7 +320,6 @@ def query(self, code, as_frame=True, render_plots=False): rows = cursor.fetchall() return Results(columns, rows, render_plots=render_plots) - @SKLearnEvaluationLogger.log(feature="SQLiteTracker") def new(self): """Create a new experiment, returns a uuid""" uuid = str(uuid4())[:8] @@ -338,12 +335,10 @@ def new(self): self.conn.commit() return uuid - @SKLearnEvaluationLogger.log(feature="SQLiteTracker") def new_experiment(self): """Returns an experiment instance""" return Experiment.new(self) - @SKLearnEvaluationLogger.log(feature="SQLiteTracker") def update(self, uuid, parameters, allow_overwrite=False): """Update the parameters of a experiment given its uuid""" if not allow_overwrite: @@ -361,7 +356,6 @@ def update(self, uuid, parameters, allow_overwrite=False): cur.close() self.conn.commit() - @SKLearnEvaluationLogger.log(action="upsert", feature="SQLiteTracker") def upsert(self, uuid, parameters): """Modify the stored parameters of an existing experiment""" existing = self.get(uuid, unserialize_plots=False)._data @@ -379,7 +373,6 @@ def upsert(self, uuid, parameters): cur.close() self.conn.commit() - @SKLearnEvaluationLogger.log(action="upsert_append", feature="SQLiteTracker") def upsert_append(self, uuid, parameters): """Append the parameters to an existing experiment @@ -432,7 +425,6 @@ def upsert_append(self, uuid, parameters): cur.close() self.conn.commit() - @SKLearnEvaluationLogger.log("SQLiteTracker") def insert(self, uuid, parameters): """Insert a new experiment""" # serialize matplotlib.figure.Figure, if any @@ -449,7 +441,6 @@ def insert(self, uuid, parameters): cur.close() self.conn.commit() - @SKLearnEvaluationLogger.log("SQLiteTracker") def insert_many(self, parameters_all): """Insert many experiments at once""" cur = self.conn.cursor() @@ -468,7 +459,6 @@ def insert_many(self, parameters_all): cur.close() self.conn.commit() - @SKLearnEvaluationLogger.log("SQLiteTracker") def comment(self, uuid, comment): """Add a comment to an experiment given its uuid""" # TODO: add overwrite (false by default) and append options @@ -547,7 +537,6 @@ def _can_update(self, uuid): "not exist".format(uuid) ) - @SKLearnEvaluationLogger.log("SQLiteTracker") def get_parameters_keys(self, limit=100): """ Return the keys in the parameters column by randomly sampling records @@ -571,7 +560,6 @@ def get_parameters_keys(self, limit=100): return extract_if_length_one(sorted(keys)) - @SKLearnEvaluationLogger.log("SQLiteTracker") def get_sample_query(self, compatibility_mode=True): keys = self.get_parameters_keys() @@ -583,7 +571,6 @@ def get_sample_query(self, compatibility_mode=True): template = Template(TEMPLATE) return template.render(keys=collapse(keys)) - @SKLearnEvaluationLogger.log("SQLiteTracker") def get(self, uuid, unserialize_plots=True): """Get an experiment given its UUID diff --git a/tests/test_skeval_telemetry.py b/tests/test_skeval_telemetry.py deleted file mode 100644 index 24928161..00000000 --- a/tests/test_skeval_telemetry.py +++ /dev/null @@ -1,185 +0,0 @@ -import pytest -import numpy as np -from unittest.mock import Mock, call - -from ploomber_core.telemetry import telemetry -from sklearn_evaluation.telemetry import SKLearnEvaluationLogger -from sklearn_evaluation.plot.pca import pca -from sklearn_evaluation.plot.calibration import CalibrationCurve - -MOCK_API_KEY = "phc_P1dsjk20bijsabdaib2eu" - - -@pytest.fixture -def mock_telemetry(monkeypatch): - telemetry.Telemetry("ploomber", "0.14.0", MOCK_API_KEY) - - mock = Mock() - mock_dt = Mock() - mock_dt.now.side_effect = [1, 2] - - monkeypatch.setattr(telemetry.Telemetry, "log_api", mock) - monkeypatch.setattr(telemetry.datetime, "datetime", mock_dt) - monkeypatch.setattr(telemetry.sys, "argv", ["/path/to/bin", "arg"]) - - yield mock - - -@pytest.mark.parametrize("action", ["some_action", None]) -@pytest.mark.parametrize("feature", ["report", None]) -@pytest.mark.parametrize("x", [1, None]) -@pytest.mark.parametrize("y", [2, None]) -@pytest.mark.parametrize( - "kwargs", - [ - {"is_report": True, "not-a-flag": True}, - {"not-a-flag": "random-value"}, - {"is_report": True}, - {}, - ], -) -def test_logger(mock_telemetry, action, feature, x, y, kwargs): - flags = dict() - function_arguments = dict({"x": x, "y": y}) - _action = action or "my_function" - - @SKLearnEvaluationLogger.log(feature=feature, action=action) - def my_function(a, b, x=None, y=None, **kwargs): - pass - - my_function(1, 2, x=x, y=y, **kwargs) - - if len(kwargs) > 0: - for key, value in kwargs.items(): - if key in SKLearnEvaluationLogger.flags(): - flags[key] = value - - expected_metadata = dict( - {"action": _action, "feature": feature, "args": function_arguments} - ) - - if len(flags) > 0: - expected_metadata["flags"] = flags - - mock_telemetry.assert_has_calls( - [call("sklearn-evaluation", metadata=expected_metadata)] - ) - - -@pytest.mark.parametrize("action", ["some_action", None]) -@pytest.mark.parametrize("feature", ["report", None]) -@pytest.mark.parametrize("x", [1, None]) -@pytest.mark.parametrize("y", [2, None]) -@pytest.mark.parametrize( - "kwargs", - [ - {"is_report": True, "not-a-flag": True}, - {"not-a-flag": "random-value"}, - {"is_report": True}, - {}, - ], -) -def test_logger_with_errors(mock_telemetry, action, feature, x, y, kwargs): - _exception = "Failed to run function" - flags = dict() - function_arguments = dict({"x": x, "y": y}) - _action = action or "my_function" - - @SKLearnEvaluationLogger.log(feature=feature, action=action) - def my_function(a, b, x=None, y=None, **kwargs): - raise Exception(_exception) - - try: - my_function(1, 2, x=x, y=y, **kwargs) - except Exception: - pass - - if len(kwargs) > 0: - for key, value in kwargs.items(): - if key in SKLearnEvaluationLogger.flags(): - flags[key] = value - - expected_metadata = dict( - { - "action": _action, - "feature": feature, - "args": function_arguments, - "exception": _exception, - } - ) - - if len(flags) > 0: - expected_metadata["flags"] = flags - - mock_telemetry.assert_has_calls( - [call("sklearn-evaluation-error", metadata=expected_metadata)] - ) - - -def test_pca(mock_telemetry): - pca(np.array([[1, 3, 2], [3, 0, 6]])) - function_arguments = dict( - {"y": None, "target_names": None, "n_components": 2, "colors": None, "ax": None} - ) - expected_metadata = dict( - {"action": "pca", "feature": "plot", "args": function_arguments} - ) - - mock_telemetry.assert_has_calls( - [call("sklearn-evaluation", metadata=expected_metadata)] - ) - - -def test_calibration_curve(mock_telemetry): - CalibrationCurve([0.5, 0.9], [0.45, 0.89], label=["Classifier 1"]).plot() - function_arguments = dict({"label": ["Classifier 1"], "cmap": None}) - expected_metadata = dict( - { - "action": "calibration-curve-init", - "feature": "plot", - "args": function_arguments, - } - ) - mock_telemetry.assert_has_calls( - [call("sklearn-evaluation", metadata=expected_metadata)] - ) - - -def test_calibration_curve_add(mock_telemetry): - cc1 = CalibrationCurve([0.5, 0.9], [0.45, 0.89], label=["Classifier 1"]).plot() - cc2 = CalibrationCurve([0.49, 0.92], [0.52, 0.88], label=["Classifier 2"]).plot() - cc1 + cc2 - function_arguments_one = dict({"label": ["Classifier 1"], "cmap": None}) - expected_metadata_one = dict( - { - "action": "calibration-curve-init", - "feature": "plot", - "args": function_arguments_one, - } - ) - - function_arguments_two = dict({"label": ["Classifier 2"], "cmap": None}) - expected_metadata_two = dict( - { - "action": "calibration-curve-init", - "feature": "plot", - "args": function_arguments_two, - } - ) - - function_arguments_add = dict({"cmaps": [None, None]}) - expected_metadata_add = dict( - { - "action": "calibration-curve-add-init", - "feature": "plot", - "args": function_arguments_add, - } - ) - - mock_telemetry.assert_has_calls( - [ - call("sklearn-evaluation", metadata=expected_metadata_one), - call("sklearn-evaluation", metadata=expected_metadata_two), - call("sklearn-evaluation", metadata=expected_metadata_add), - ] - )