diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66e535ea..a416ad8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,7 +158,6 @@ stages: echo "" cat cicd-release/values.yaml echo "" - FILE_NAME=$(ls cicd-release/ | grep neptune-docs) helm upgrade -i \ neptune-docs \ diff --git a/docs/_static/images/others/skorch_neptuneai.png b/docs/_static/images/others/skorch_neptuneai.png new file mode 100644 index 00000000..0ba87685 Binary files /dev/null and b/docs/_static/images/others/skorch_neptuneai.png differ diff --git a/docs/_static/images/skorch/skorch_monitoring.gif b/docs/_static/images/skorch/skorch_monitoring.gif new file mode 100644 index 00000000..07b1b380 Binary files /dev/null and b/docs/_static/images/skorch/skorch_monitoring.gif differ diff --git a/docs/index.rst b/docs/index.rst index 5b420762..a33ade88 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -123,6 +123,7 @@ Documentation contents PyTorchLightning Catalyst PyTorch Ignite + Skorch Neptune Contrib .. External links diff --git a/docs/integrations/skorch.rst b/docs/integrations/skorch.rst new file mode 100644 index 00000000..99f2db6e --- /dev/null +++ b/docs/integrations/skorch.rst @@ -0,0 +1,220 @@ +Log Skorch metrics to neptune +============================= +.. image:: ../_static/images/others/skorch_neptuneai.png + :target: ../_static/images/others/skorch_neptuneai.png + :alt: Skorch neptune.ai integration + +Prerequisites +------------- +Integration with |Skorch| framework is introduced as a part of logging module so just need to have |neptune-client| installed. + +.. code-block:: bash + + pip install neptune-client + + +Create an **Experiment** +------------------------ + +.. note:: I am using a shared, anonymous token but you can change to your user api token. + +.. code-block:: python3 + + import neptune + + neptune.init( + api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vdWkubmVwdHVuZS5haSIsImFwaV9rZXkiOiJiNzA2YmM4Zi03NmY5LTRjMmUtOTM5ZC00YmEwMzZmOTMyZTQifQ==', + project_qualified_name='shared/skorch-integration') + + experiment = neptune.create_experiment(name='skorch-basic-example', + params={'max_epochs': 20, + 'lr': 0.1}, + upload_source_files=['skorch_example.py']) + +Create **NeptuneLogger** callback +-------------------------------- +Pass the experiment object as first argument. + +.. note:: To be able to log information after the .fit() method finishes remember to pass ``close_after_train=False`` + +.. code-block:: python3 + + from skorch.callbacks.logging import NeptuneLogger + neptune_logger = NeptuneLogger(experiment, close_after_train=False) + +Pass **neptune_logger** to **NeuralNetClassifier** +------------------------------------------------- +.. code-block:: python3 + + net = NeuralNetClassifier( + ClassifierModule, + max_epochs=20, + lr=0.1, + callbacks=[neptune_logger]) + + net.fit(X, y) + +Log additional information +-------------------------- + +**Log test metrics after training** + +.. code-block:: python3 + + from sklearn.metrics import roc_auc_score + + y_pred = net.predict_proba(X) + auc = roc_auc_score(y, y_pred[:, 1]) + + neptune_logger.experiment.log_metric('roc_auc_score', auc) + +**Log performance charts** + +.. code-block:: python3 + + from scikitplot.metrics import plot_roc + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(16, 12)) + plot_roc(y, y_pred, ax=ax) + + neptune_logger.experiment.log_image('roc_curve', fig) + +**Log trained model** + +.. code-block:: python3 + + net.save_params(f_params='basic_model.pkl') + neptune_logger.experiment.log_artifact('basic_model.pkl') + +Monitor your Skorch training in Neptune +--------------------------------------- +Now you can watch your Skorch model training in neptune! + +Check out this |example experiment|. + +.. image:: ../_static/images/skorch/skorch_monitoring.gif + :target: ../_static/images/skorch/skorch_monitoring.gif + :alt: Skorch monitoring in neptune + +Close experiment +---------------- +If you passed ``close_after_train=False`` to ``NeptuneLogger`` you may want to close your experiment when you are done logging. + +.. code-block:: python3 + + neptune_logger.experiment.stop() + +Full Skorch monitoring script +----------------------------- +Simply copy and paste it to ``skorch_example.py`` and run. +Remember to change your credentials in **neptune.init()**: + +.. code-block:: python3 + + neptune.init(api_token=os.getenv('NEPTUNE_API_TOKEN'), + project_qualified_name='USER_NAME/PROJECT_NAME') + +.. code-block:: python3 + + import torch + from torch import nn + import torch.nn.functional as F + + torch.manual_seed(0) + + # create data + import numpy as np + from sklearn.datasets import make_classification + + X, y = make_classification(1000, 20, n_informative=10, random_state=0) + X = X.astype(np.float32) + + + # create pytorch module + class ClassifierModule(nn.Module): + def __init__( + self, + num_units=10, + nonlin=F.relu, + dropout=0.5, + ): + super(ClassifierModule, self).__init__() + self.num_units = num_units + self.nonlin = nonlin + self.dropout = dropout + + self.dense0 = nn.Linear(20, num_units) + self.nonlin = nonlin + self.dropout = nn.Dropout(dropout) + self.dense1 = nn.Linear(num_units, 10) + self.output = nn.Linear(10, 2) + + def forward(self, X, **kwargs): + X = self.nonlin(self.dense0(X)) + X = self.dropout(X) + X = F.relu(self.dense1(X)) + X = F.softmax(self.output(X), dim=-1) + return X + + + # create neptune logger and pass it to NeuralNetClassifier + from skorch import NeuralNetClassifier + import neptune + from skorch.callbacks.logging import NeptuneLogger + + neptune.init('neptune-ai/skorch-integration') + experiment = neptune.create_experiment( + name='skorch-basic-example', + params={'max_epochs': 20, + 'lr': 0.1}, + upload_source_files=['skorch_example.py']) + neptune_logger = NeptuneLogger(experiment, close_after_train=False) + + net = NeuralNetClassifier( + ClassifierModule, + max_epochs=20, + lr=0.1, + callbacks=[neptune_logger] + ) + + # run training + net.fit(X, y) + + # log score after training + from sklearn.metrics import roc_auc_score + + y_pred = net.predict_proba(X) + auc = roc_auc_score(y, y_pred[:, 1]) + + neptune_logger.experiment.log_metric('roc_auc_score', auc) + + # log charts like ROC curve + from scikitplot.metrics import plot_roc + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(figsize=(16, 12)) + plot_roc(y, y_pred, ax=ax) + neptune_logger.experiment.log_image('roc_curve', fig) + + # log model after training + net.save_params(f_params='basic_model.pkl') + neptune_logger.experiment.log_artifact('basic_model.pkl') + + # close experiment + neptune_logger.experiment.stop() + + +.. External links + +.. |Skorch| raw:: html + + Skorch + +.. |example experiment| raw:: html + + example experiment + +.. |neptune-client| raw:: html + + neptune-client \ No newline at end of file diff --git a/docs/notebooks/configuration.rst b/docs/notebooks/configuration.rst index 10756a2a..66605819 100644 --- a/docs/notebooks/configuration.rst +++ b/docs/notebooks/configuration.rst @@ -1,7 +1,7 @@ Connecting the Jupyter Extension to Your Neptune Account ======================================================== -After you have successfully `installed the Jupyter extension for Neptune `_, +After you have successfully `installed the Jupyter extension for Neptune `_, you connect it to your Neptune account. **Procedure** @@ -13,7 +13,7 @@ you connect it to your Neptune account. :alt: image -The **Configure your connection to Neptune** dialog appears. +The **Configure your connection to Neptune** dialog appears. .. image:: ../_static/images/notebooks/configure_connect.png :target: ../_static/images/notebooks/configure_connect.png @@ -25,7 +25,7 @@ The **Configure your connection to Neptune** dialog appears. 3. In the Neptune UI, obtain your API Token and copy it to the clipboard. a. In the upper right corner, click the avatar, and then click **Get API Token**. - + .. image:: ../_static/images/notebooks/get_api_token.png :target: ../_static/images/notebooks/get_api_token.png :alt: image @@ -39,6 +39,6 @@ The **Configure your connection to Neptune** dialog appears. .. warning:: Your *API Token* is private and unique. Never share it. It's like sharing password. -5. To conclude, to see experiments that you will run associated with this Notebook, click **Activate**. +5. To conclude, to see experiments that you will run associated with this Notebook, click **Activate**. In the dialog that appears, click **Activate**. diff --git a/docs/notebooks/installation.rst b/docs/notebooks/installation.rst index ad558ff9..a1709eef 100644 --- a/docs/notebooks/installation.rst +++ b/docs/notebooks/installation.rst @@ -1,12 +1,15 @@ -Installing neptune-notebooks + +Installing neptune-notebooks ============================ -This page provides instructions for installing neptune-notebooks -- the Neptune Jupyter extension. + +This page provides instructions for installing neptune-notebooks -- the Neptune Jupyter extension. The extension enables the integration of Neptune and Jupyter or JupyterLab. -When the extension is successfully installed, -you will be able to upload notebooks to Neptune, check out previously uploaded Notebooks, -and track experiments and metrics directly from the Jupyter UI. +When the extension is successfully installed, +you will be able to upload notebooks to Neptune, check out previously uploaded Notebooks, +and track experiments and metrics directly from the Jupyter UI. + Neptune versions the Notebook automatically once an experiment has started. About neptune-notebooks @@ -22,7 +25,7 @@ neptune-notebooks is a Python package hosted `here `_. +.. tip:: When installing Python packages, it is best practice to work in a `virtual environment `_. Procedure @@ -48,7 +51,8 @@ Procedure Installation for JupyterLab --------------------------- -JupyterLab is the next-generation web-based UI for Project Jupyter. +JupyterLab is the next-generation web-based UI for Project Jupyter. + When you install JupyterLab using ``pip install jupyterlab``, the standard Jupyter is also installed, as a dependency. @@ -67,11 +71,12 @@ Details for these methods follow below. Irrespective of which method you use, the following must be preinstalled on your system before you begin. -- `Node.js `_ + +- `Node.js `_ - `npm `_ - `JupyterLab `_ -.. tip:: When installing Python packages, it is best practice to work in a `virtual environment `_. +.. tip:: When installing Python packages, it is best practice to work in a `virtual environment `_. Method 1: Install the full neptune-notebooks package through the command line @@ -81,13 +86,13 @@ Method 1: Install the full neptune-notebooks package through the command line This is the most comprehensive type of installation: -- It includes extensions for both standard Jupyter and JupyterLab. +- It includes extensions for both standard Jupyter and JupyterLab. - It also installs a CLI for uploading Notebook files. - The the neptune-client package is a dependency. **Cons** -- After the installation, you still have to perform some manual actions in JupyterLab. +- After the installation, you still have to perform some manual actions in JupyterLab. - This method may not be the easiest way of installing extensions in JupyterLab. **Procedure** @@ -123,7 +128,8 @@ The neptune-notebooks pip package, neptune-client pip package and extension to t **Pros** -This method gives you more flexibility. For example, the extension can be running on a + +This method gives you more flexibility. For example, the extension can be running on a different machine than your code that is executed from the Notebook. **Procedure** diff --git a/docs/notebooks/integrations.rst b/docs/notebooks/integrations.rst index 4ea2f4b5..b583d2bc 100644 --- a/docs/notebooks/integrations.rst +++ b/docs/notebooks/integrations.rst @@ -65,7 +65,7 @@ You can run Neptune and track experiments that you run on AWS cloud. 5. Define your secrets. - a. Go to Neptune, `get your NEPTUNE_API_TOKEN `_ and copy it to the clipboard. + a. Go to Neptune, `get your NEPTUNE_API_TOKEN `_ and copy it to the clipboard. b. Create a password for your JupyterLab server. c. Set the following two secrets to your environment variables, ``NEPTUNE_API_TOKEN`` and ``JUPYTERLAB_PASSWORD``: @@ -133,10 +133,10 @@ You can run Neptune and track experiments that you run on AWS cloud. 9. Open the JupyterLab server in your browser. Go to `localhost:8888`. - + Enjoy your JupyterLab server with Neptune. -Neptune extensions are enabled and ``NEPTUNE_API_TOKEN`` is already in the environment variable so you can work with +Neptune extensions are enabled and ``NEPTUNE_API_TOKEN`` is already in the environment variable so you can work with Notebooks and run experiments with no problem. Setting up a Neptune-enabled AWS SageMaker Jupyter instance @@ -158,7 +158,8 @@ You can use Neptune to track experiments that you run on AWS SageMaker. .. image:: ../_static/images/how-to/ht-sagemaker-create_configuration.png :target: ../_static/images/how-to/ht-sagemaker-create_configuration.png :alt: image - | + + | You can choose whatever name you want -- just make sure to remember it. @@ -169,7 +170,7 @@ You can use Neptune to track experiments that you run on AWS SageMaker. :alt: image | - + c. Copy and paste the script below to your **Create Notebook** tab. In the **PARAMETERS** section, choose in which environments you want to install neptune-client. @@ -201,7 +202,6 @@ You can use Neptune to track experiments that you run on AWS SageMaker. jupyter nbextension enable --py neptune-notebooks --sys-prefix jupyter labextension install neptune-notebooks source /home/ec2-user/anaconda3/bin/deactivate - EOF 3. Create a Notebook instance. @@ -230,16 +230,16 @@ You can use Neptune to track experiments that you run on AWS SageMaker. 4. Start Notebook. - If everything went well, your AWS SageMaker instance should have *InService* status and you can open a Jupyter Notebook or JupyterLab + If everything went well, your AWS SageMaker instance should have *InService* status and you can open a Jupyter Notebook or JupyterLab with Neptune Notebook versioning enabled. .. image:: ../_static/images/how-to/ht-sagemaker-notebook_run.png :target: ../_static/images/how-to/ht-sagemaker-notebook_run.png :alt: image - You can now version your Notebooks and track experiments in Amazon SageMaker with Neptune. .. image:: ../_static/images/how-to/ht-sagemaker-notebook_runs_01.png :target: ../_static/images/how-to/ht-sagemaker-notebook_runs_01.png + :alt: image \ No newline at end of file diff --git a/docs/notebooks/introduction.rst b/docs/notebooks/introduction.rst index 13328105..68775f97 100644 --- a/docs/notebooks/introduction.rst +++ b/docs/notebooks/introduction.rst @@ -4,12 +4,11 @@ Using Jupyter Notebooks in Neptune .. image:: ../_static/images/others/notebooks_neptuneml.png :target: ../_static/images/others/notebooks_neptuneml.png :alt: Jupyter Notebooks neptune.ai integration - -Jupyter Notebooks are a useful and popular tool for data scientists, regardless of their area of specialization. -They allow data scientists to work interactively, keeping code and results - like visualizations - in a single document. +Jupyter Notebooks are a useful and popular tool for data scientists, regardless of their area of specialization. +They allow data scientists to work interactively, keeping code and results - like visualizations - in a single document. -While Neptune is essentially a platform for tracking experiments, it provides Jupyter and JupyterLab extensions that also let +While Neptune is essentially a platform for tracking experiments, it provides Jupyter and JupyterLab extensions that also let you track Jupyter Notebooks. Key Features @@ -18,17 +17,18 @@ Key Features * In Neptune, each Notebook consists of a collection of checkpoints that you upload directly from the Jupyter user interface. * In any project, an unlimited number of Notebooks and checkpoints is allowed. * You can browse checkpoints history across all `Notebooks in the project `_. -* You can `share `_ a notebook as a link. +* You can `share `_ a Notebook as a link. * You can `compare `_ two Notebooks side-by-side, like source code. Quick Start ----------- -To start working with Notebooks in Neptune, `install `_ and `configure `_ the open + +To start working with Notebooks in Neptune, `install `_ and `configure `_ the open source extension for Jupyter or JupyterLab. When you are done, you can start working with Notebooks immediately. -To try it now, without registering to Neptune, look at the sample Notebooks in the public project `onboarding `_. -Use the public user's API token that appears below, and the username *neptuner* to upload some snapshots to this project. +To try it now, without registering to Neptune, look at the sample Notebooks in the public project `onboarding `_. +Use the public user's API token that appears below, and the username *neptuner* to upload some snapshots to this project. You still need to `install `_ and `configure `_ Jupyter extension. **Public user’s API token**: @@ -57,7 +57,8 @@ The Notebook data is arranged in the following columns: In addition, for each Notebook, there are buttons for downloading the Notebook, comparing it with another Notebook, or for sharing a link to it. -A **Compare** button at the top right displays a Notebooks Comparison pane. See `Compare Notebooks `_. +A **Compare** button at the top right displays a Notebooks Comparison pane. See `Compare Notebooks `_. + Notebook contents ~~~~~~~~~~~~~~~~~ @@ -66,14 +67,16 @@ Once you select a Notebook, you can see all its contents, that is: code and mark There are two tabs on the right: - **Details**: Here are shown the ID, size, creation date, latest checkpoint, owner, description and associated experiments of the selected Notebook. -- **Checkpoints**: Here are listed all the checkpoints of the Notebook. Click a checkpoint to see the details in the main pane. From this tab, you can also access the experiments that are associated with the checkpoint. +- **Checkpoints**: Here are listed all the checkpoints of the Notebook. Click a checkpoint to see the details in the main pane. From this tab, you can also access the experiments that are associated with the checkpoint. + You can also view snapshots of the work with the Notebook, as well as download, share or compare this checkpoint with another checkpoint. -.. image:: ../_static/images/notebooks/nb-view-22.png +.. image:: ../_static/images/notebooks/nb-view-22.png :target: ../_static/images/notebooks/nb-view-22.png :alt: image -Compare Notebooks +Compare Notebooks + ~~~~~~~~~~~~~~~~~ The Notebooks Comparison pane lets you compare Notebook checkpoints. @@ -84,7 +87,8 @@ You display the pane by clicking the **Compare** button anywhere it is visible i :target: ../_static/images/notebooks/compare.png :alt: image -In the Notebooks Comparison pane, select two Notebook checkpoints, then click **Compare** to see a side-by-side comparison, just like source code. +In the Notebooks Comparison pane, select two Notebook checkpoints, then click **Compare** to see a side-by-side comparison, just like source code. + .. Commented out. Doesn't seem to be working. .. `Compare view `_ let you look at the difference between checkpoints of the same Notebook, or two entirely different Notebooks (Try yourself `here `_). @@ -99,30 +103,34 @@ In the Notebooks Comparison pane, select two Notebook checkpoints, then click ** Uploading and Downloading Notebook Checkpoints -------------------------------------------- +---------------------------------------------- + Notebooks are stored as files on your computer. Each Notebook file (.ipynb) is a JSON containing everything that the user can see in a Notebook and some metadata. -Neptune uses metadata to associate particular files with Notebook entities on Neptune servers. That means that after a Notebook + +Neptune uses metadata to associate particular files with Notebook entities on Neptune servers. That means that after a Notebook + is uploaded to Neptune, the file on disk is changed to include the ID of the entity on the Neptune server. **Name changes** -If you copy a Notebook file (let’s call it "Notebook A") and -edit it with the intention of creating something completely separate from Notebook A, -the association with Notebook A on the Neptune server remains. If the name of the Notebook changes from "Notebook A", -you will be warned. +If you copy a Notebook file (let’s call it "Notebook A") and +edit it with the intention of creating something completely separate from Notebook A, +the association with Notebook A on the Neptune server remains. If the name of the Notebook changes from "Notebook A", +you will be warned. **Global accessibility** -When you download a Notebook checkpoint, the ID in the metadata is preserved, so that when, after some work, +When you download a Notebook checkpoint, the ID in the metadata is preserved, so that when, after some work, you click **Upload**, Neptune knows that this may be another checkpoint in a particular Notebook. -You can do some work, upload some intermediate snapshot, go to another computer +You can do some work, upload some intermediate snapshot, go to another computer (or another SageMaker instance, and so on), download the Notebook and keep on working on it. -The capability is comparable to Google Docs in that there’s a place where you store your work and you can access +The capability is comparable to Google Docs in that there’s a place where you store your work and you can access + it easily from wherever you choose. **Collaboration** @@ -132,8 +140,8 @@ Depending on their roles, members of a project can view and download all Noteboo - Viewers can download Notebooks. - Contributors and Owners can also upload them. -When uploading a new Notebook, a user becomes the owner of this Notebook. Only the owner of a Notebook can upload -new checkpoints of this Notebook. +When uploading a new Notebook, a user becomes the owner of this Notebook. Only the owner of a Notebook can upload +new checkpoints of this Notebook. Uploading a Notebook ~~~~~~~~~~~~~~~~~~~~ @@ -148,7 +156,7 @@ You can upload Notebook checkpoints from Jupyter to Neptune. :target: ../_static/images/notebooks/upload_dialog.png :width: 450 :alt: Upload Notebook dialog - + 2. In the dialog that is displayed, select a project from the list. 3. (Optional) Type in a checkpoint name and description. 4. Click **Upload checkpoint**. @@ -170,7 +178,10 @@ You can download a specific Notebook checkpoint from Neptune to Jupyter. :alt: Download Notebook dialog 2. In the dialog that is displayed, select the following from the respective lists: + - Project - Notebook - Checkpoint -3. Click **Download**. + + +3. Click **Download**. \ No newline at end of file diff --git a/docs/notebooks/troubleshoot.rst b/docs/notebooks/troubleshoot.rst index 2e9e7d09..6f331c11 100644 --- a/docs/notebooks/troubleshoot.rst +++ b/docs/notebooks/troubleshoot.rst @@ -1,6 +1,8 @@ Troubleshoot ============ -You may experience issues while working with Jupyter Notebooks and Neptune. + +You may experience issues while working with Jupyter Notebooks and Neptune. + The following presents possible solutions to some of the issues. .. contents:: @@ -22,7 +24,7 @@ then enable the extension for your Jupyter: jupyter nbextension enable --py neptune-notebooks -Don't forget to install Neptune client: +Don't forget to install Neptune client: .. code-block:: bash @@ -57,14 +59,14 @@ Where is *NEPTUNE_API_TOKEN*? My integration does not work, but it worked well previously. What do I do? ------------------------------------------------------------------------- -Most likely, you restarted the kernel. +-------------------------------------------------------------------------- +Most likely, you restarted the kernel. -If that is the case, the experiments are not associated with the notebook. +If that is the case, the experiments are not associated with the notebook. In Jupyter, click **Activate**. .. image:: ../_static/images/notebooks/activate_button.png :target: ../_static/images/notebooks/activate_button.png - :alt: image + :alt: image \ No newline at end of file