diff --git a/README.md b/README.md
index c0120a7b..4b2f5b9b 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,7 @@ supports tensorflow versions 2.x.
| **Tutorial Name** | Notebook |
| :-------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| Getting Started 1 - Creating a 1-Lipschitz neural network | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/Getting_started_1.ipynb) |
+| Getting Started 2 - Training an adversarially robust 1-Lipschitz neural network | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/Getting_started_2.ipynb) |
| Wasserstein distance estimation on toy example | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo1.ipynb) |
| HKR Classifier on toy dataset | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo2.ipynb) |
| HKR classifier on MNIST dataset | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo3.ipynb) |
diff --git a/deel/lip/losses.py b/deel/lip/losses.py
index 50cdba25..8f6843f6 100644
--- a/deel/lip/losses.py
+++ b/deel/lip/losses.py
@@ -16,6 +16,10 @@
Reduction,
)
from tensorflow.keras.utils import register_keras_serializable
+from deel.lip.layers.base_layer import LipschitzLayer
+import logging
+
+logging.basicConfig(level=logging.WARNING)
@register_keras_serializable("deel-lip", "_kr")
@@ -561,3 +565,323 @@ def get_config(self):
config = {"tau": self.tau.numpy()}
base_config = super().get_config()
return dict(list(base_config.items()) + list(config.items()))
+
+
+@register_keras_serializable("deel-lip", "Certificate_Binary")
+class Certificate_Binary(Loss):
+ def __init__(
+ self, model, K=None, reduction=Reduction.AUTO, name="Certificate_Binary"
+ ):
+ r"""
+ Certificate-based loss in the context of binary classification,
+ which is inspired from the paper
+ 'Improved deterministic l2 robustness on CIFAR-10 and CIFAR-100'
+ (https://openreview.net/forum?id=tD7eCtaSkR).
+
+ Our formula is as follow:
+
+ $$\text{certificate} = |\hat{y}|/K$$
+
+ where $\hat{y}$ is the single logit predicted by our model,
+ and $K$ is the Lipschitz constant associated with the model.
+
+ Note that `y_true` and `y_pred` must be of rank 2:
+ (batch_size, 1).
+
+
+ Args:
+ model: A tensorflow multi-layered model.
+ K: The Lipschitz constant of the model.
+ It is calculated using the model if
+ not provided by a user upon instanciation.
+ reduction: passed to tf.keras.Loss constructor
+ name (str): passed to tf.keras.Loss constructor
+
+ """
+ self.model = model
+ check_last_layer(model)
+ if K is None:
+ self.K = tf.constant(get_K_(get_layers(self.model)), dtype=tf.float32)
+ else:
+ self.K = tf.constant(K, dtype=tf.float32)
+ super(Certificate_Binary, self).__init__(reduction=reduction, name=name)
+
+ @tf.function
+ def call(self, y_true, y_pred):
+ return tf.abs(y_pred[:, 0]) / self.K
+
+ def get_config(self):
+ config = {"model": self.model, "K": self.K.numpy()}
+ base_config = super(Certificate_Binary, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+
+@register_keras_serializable("deel-lip", "Certificate_Multiclass")
+class Certificate_Multiclass(Loss):
+ def __init__(
+ self,
+ model,
+ K_n_minus_1=None,
+ reduction=Reduction.AUTO,
+ name="Certificate_Multiclass",
+ ):
+ r"""
+ Certificate-based loss, which is inspired from the paper
+ 'Improved deterministic l2 robustness on CIFAR-10 and CIFAR-100'
+ (https://openreview.net/forum?id=tD7eCtaSkR).
+
+ Our formula is as follow:
+
+ $$\text{certificate} = \min_i \frac{ \hat{y}_{\pi_1} -
+ \hat{y}_{\pi_i}}{||\mathbf{W}_{n,\pi_1}-\mathbf{W}_{n,\pi_i}||K_{n-1}}
+ $$
+
+ where $\pi$ is the permutation of {1, ..., C} - C being the number of labels
+ that sorts the elements of $\hat{y}$ from most likely to least likely,
+ where $|\mathbf{W}_{n,k}$ designates the weights of the k-th
+ perceptron of the last layer (n being the number of layers),
+ and $K_{n-1}$ is the Lipschitz constant associated with the
+ first n-1 layers of the model.
+
+ Note that `y_true` and `y_pred` must be of rank 2:
+ (batch_size, C) for multilabel classification with C categories
+
+
+ Args:
+ model: A tensorflow multi-layered model.
+ K_n_minus_1 : The Lipschitz constant of the n - 1 first layers,
+ where n is the total number of layers.
+ It is calculated using the model if not provided by
+ a user upon instanciation.
+ reduction: passed to tf.keras.Loss constructor
+ name (str): passed to tf.keras.Loss constructor
+
+ """
+ self.model = model
+ check_last_layer(model)
+ if K_n_minus_1 is None:
+ self.K_n_minus_1 = tf.constant(
+ get_K_(get_layers(self.model)[:-1]), dtype=tf.float32
+ )
+ else:
+ self.K_n_minus_1 = tf.constant(K_n_minus_1, dtype=tf.float32)
+ super(Certificate_Multiclass, self).__init__(reduction=reduction, name=name)
+
+ @tf.function
+ def call(self, y_true, y_pred):
+ last_layer_weights = get_last_layer(self.model).kernel
+ num_classes = tf.shape(y_true)[1]
+ sorted_indices = get_sorted_logits_indices_tensor(y_pred)
+ certificate = tf.zeros(len(y_pred), dtype=tf.float32)
+ num_classes = tf.shape(y_true)[1]
+ indices_batch_size = tf.range(len(y_pred), dtype=tf.int32)
+ indices_num_classes = tf.range(1, num_classes, dtype=tf.int32)
+ for j in indices_batch_size:
+ min_value = tf.constant(
+ float("inf"), dtype=tf.float32
+ ) # Initialize with positive infinity
+ for i in indices_num_classes:
+ numerator = (
+ y_pred[j][sorted_indices[j][0]] - y_pred[j][sorted_indices[j][i]]
+ )
+ denominator = tf.norm(
+ last_layer_weights[:, sorted_indices[j][0]]
+ - last_layer_weights[:, sorted_indices[j][i]]
+ )
+ formula_value = numerator / (denominator * self.K_n_minus_1)
+ min_value = tf.minimum(min_value, formula_value)
+ certificate = tf.tensor_scatter_nd_update(
+ certificate,
+ indices=tf.expand_dims(tf.expand_dims(j, axis=0), axis=0),
+ updates=tf.expand_dims(min_value, axis=0),
+ )
+ return certificate
+
+ def get_config(self):
+ config = {"model": self.model, "K_n_minus_1": self.K_n_minus_1.numpy()}
+ base_config = super(Certificate_Multiclass, self).get_config()
+ return dict(list(base_config.items()) + list(config.items()))
+
+
+def get_K_(layers):
+ check_is_Lipschitz(layers)
+ K_ = 1
+ for layer in layers: # Print information about each layer
+ if isinstance(layer, LipschitzLayer):
+ K_ = layer.k_coef_lip * K_
+ else:
+ pass
+ return K_
+
+
+GLOBAL_CONSTANTS = {
+ "supported_neutral_layers": ["Flatten", "InputLayer"],
+ "not_deel": [
+ "dense",
+ "average_pooling2d",
+ "global_average_pooling2d",
+ "conv2d",
+ ], # We don't use layer.__class__.__name__ to \
+ # find these as for Conv2D and GlobalAveragePooling2D, \
+ # it results in 'type'
+ "not_Lipschitz": [
+ "Dropout",
+ "ELU",
+ "LeakyReLU",
+ "ThresholdedReLU",
+ "BatchNormalization",
+ ],
+ "unrecommended_activation_functions": [
+ tf.keras.activations.relu,
+ tf.keras.activations.softmax,
+ tf.keras.activations.exponential,
+ tf.keras.activations.elu,
+ tf.keras.activations.selu,
+ tf.keras.activations.tanh,
+ tf.keras.activations.sigmoid,
+ tf.keras.activations.softplus,
+ tf.keras.activations.softsign,
+ ],
+ "min_max_norm": tf.keras.constraints.MinMaxNorm,
+ "recommended_activation_names": [
+ "group_sort2",
+ "full_sort",
+ "group_sort",
+ "householder",
+ "max_min",
+ "p_re_lu",
+ ],
+ "unrecommended_activation_names": ["ReLU"],
+ "no_activation": tf.keras.activations.linear,
+}
+
+
+def get_sorted_logits_indices_tensor(model_output):
+ # Sort the model outputs model.predict(x)
+ sorted_indices = tf.argsort(model_output, axis=1, direction="DESCENDING")
+ return sorted_indices
+
+
+def get_layers(model):
+ return model.layers
+
+
+def get_weights_last_layer(model):
+ return get_last_layer(model).get_weights()[0]
+
+
+def get_sorted_logits_indices(model_output):
+ # Sort the model outputs model.predict(x)
+ sorted_indices = np.argsort(model_output, axis=1)[:, ::-1]
+ return sorted_indices
+
+
+def get_last_layer(model):
+ return get_layers(model)[-1]
+
+
+def check_last_layer(model):
+ last_layer = get_last_layer(model)
+ if not hasattr(
+ last_layer, "get_weights"
+ ): # check last layer is a layer with weights
+ raise BadLastLayerError(
+ "The last layer '%s' must have a set of \
+weights to calculate the certificate."
+ % last_layer.name
+ )
+ if last_layer.get_weights() == []:
+ raise BadLastLayerError(
+ "The last layer '%s' must have a set of weights \
+to calculate the certificate."
+ % last_layer.name
+ )
+ # check last layer has no activation function set
+ activation = getattr(last_layer, "activation")
+ if activation != GLOBAL_CONSTANTS["no_activation"]:
+ logging.warning(
+ "We recommend avoiding using an activation \
+function for the last layer (here the '%s' activation function of the layer '%s').\n"
+ % (activation, last_layer.name)
+ )
+
+
+class NotLipschtzLayerError(Exception):
+ pass
+
+
+class BadLastLayerError(Exception):
+ pass
+
+
+def check_is_Lipschitz(layers):
+ for i, layer in enumerate(layers):
+ check_activation_layer(layer)
+ if layer.__class__.__name__ in GLOBAL_CONSTANTS["supported_neutral_layers"]:
+ pass
+ elif isinstance(layer, LipschitzLayer):
+ pass
+ elif (
+ layer.__class__.__name__ in GLOBAL_CONSTANTS["not_Lipschitz"]
+ ): # triggers when using none Lipschitz layers such as "batch_normalization"
+ raise NotLipschtzLayerError("The layer '%s' is not supported" % layer.name)
+ print("ok")
+ elif any(
+ layer.name.startswith(substring)
+ for substring in GLOBAL_CONSTANTS["not_deel"]
+ ):
+ logging.warning(
+ "A deel equivalent exists for '%s'. For practical \
+purposes, we will assume that the layer is 1-Lipschitz."
+ % layer.name
+ )
+ elif (
+ layer.__class__.__name__
+ in GLOBAL_CONSTANTS["unrecommended_activation_names"]
+ ):
+ logging.warning(
+ "The layer '%s' is not recommended. \
+For practical purposes, we recommend to use deel lip \
+activation layer instead such as GroupSort2.\n"
+ % (layer.name)
+ )
+ else:
+ logging.warning(
+ "Unknown layer '%s' used. For practical purposes, \
+we will assume that the layer is 1-Lipschitz."
+ % layer.name
+ )
+
+
+def check_activation_layer(layer):
+ if hasattr(layer, "activation"):
+ activation = getattr(layer, "activation")
+ if activation != GLOBAL_CONSTANTS["no_activation"]:
+ if activation in GLOBAL_CONSTANTS["unrecommended_activation_functions"]:
+ logging.warning(
+ "The '%s' activation function of the layer '%s' is not recommended.\
+For practical purposes, we recommend to use deel lip activation \
+functions instead such as GroupSort2.\n"
+ % (activation, layer.name)
+ )
+ return None
+ if isinstance(activation, GLOBAL_CONSTANTS["min_max_norm"]):
+ return None
+ elif hasattr(activation, "name"):
+ n = activation.name
+ if (
+ layer.activation.__class__.__name__
+ in GLOBAL_CONSTANTS["recommended_activation_names"]
+ ):
+ return None
+ else:
+ print(
+ "The '%s' activation function of the layer '%s' is unknown. \
+We will assume it is 1-Lipschitz.\n"
+ % (n, layer.name)
+ )
+ else:
+ logging.warning(
+ "The '%s' activation function of the layer '%s' is unknown.\n"
+ % (activation, layer.name)
+ )
diff --git a/docs/assets/noise.PNG b/docs/assets/noise.PNG
new file mode 100644
index 00000000..ec1c51a0
Binary files /dev/null and b/docs/assets/noise.PNG differ
diff --git a/docs/assets/pigs.png b/docs/assets/pigs.png
new file mode 100644
index 00000000..2617f8da
Binary files /dev/null and b/docs/assets/pigs.png differ
diff --git a/docs/index.md b/docs/index.md
index 4621a07b..eff7a217 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -66,6 +66,7 @@ supports tensorflow versions 2.x.
| **Tutorial Name** | Notebook |
| :-------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------: |
| Getting started 1 - Creating a 1-Lipschitz neural network | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/Getting_started_1.ipynb) |
+| Getting started 2 - Training an adversarially robust 1-Lipschitz neural network | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/Getting_started_2.ipynb) |
| Wasserstein distance estimation on toy example | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo1.ipynb) |
| HKR Classifier on toy dataset | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo2.ipynb) |
| HKR classifier on MNIST dataset | [](https://colab.research.google.com/github/deel-ai/deel-lip/blob/master/docs/notebooks/demo3.ipynb) |
diff --git a/docs/notebooks/Getting_started_2.ipynb b/docs/notebooks/Getting_started_2.ipynb
new file mode 100644
index 00000000..b147109b
--- /dev/null
+++ b/docs/notebooks/Getting_started_2.ipynb
@@ -0,0 +1,666 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "86557797-36de-44bb-b277-2ee7a22b1458",
+ "metadata": {},
+ "source": [
+ "# ๐ Getting started 2: Training adversarially robust 1-Lipschitz neural networks for classification"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "23c0071a-2546-4143-af7e-496d9ffa2fd4",
+ "metadata": {},
+ "source": [
+ "The goal of this series of tutorials is to show the different usages of `deel-lip`.\n",
+ "\n",
+ "In the first notebook, we have shown how to create 1-Lipschitz neural networks with `deel-lip`. \n",
+ "In this second notebook, we will show how to train adversarially robust 1-Lipschitz neural networks with `deel-lip`. \n",
+ "\n",
+ "In particular, we will cover the following: \n",
+ "1. [๐ Theoretical background](#theoretical_background) \n",
+ "A brief theoretical background on adversarial robustness. This section can be safely skipped if one is not interested in the theory.\n",
+ "\n",
+ "2. [๐ช Training provable adversarially robust 1-Lipschitz neural networks on the MNIST dataset](#deel_keras) \n",
+ "Using the MNIST dataset, we will show examples of training adversarially robust 1-Lipschitz neural networks using `deel-lip` loss functions `TauCategoricalCrossentropy` and `MulticlassHKR`.\n",
+ "\n",
+ "We will also see that:\n",
+ "- when training robust models, there is an accuracy-robustness trade-off\n",
+ "- the `MulticlassKR` loss function can be used to assess the adversarial robustness of the resulting models\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "## ๐ Theoretical background \n",
+ "### Adversarial attacks\n",
+ "In the context of classification problems, an adversarial attack is the result of adding an *adversarial perturbation* $\\epsilon$ to the input data point $x$ of a trained predictive model $A$, with the intent to change its prediction (for simplicity, $A$ returns a class as opposed to a set of logits in the formalism used below).\n",
+ "\n",
+ "In simple mathematical terms, an adversarial example (i.e. a succesful adversarial attack) can be transcribed as below:\n",
+ "\n",
+ "$$A(x)=y_1,$$\n",
+ "$$A(x+\\epsilon)=y_{\\epsilon},$$\n",
+ "where:\n",
+ "$$y_1\\neq y_\\epsilon.$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f8769d17-2c47-4491-ad22-7785324f88ec",
+ "metadata": {},
+ "source": [
+ "### An adversarial example\n",
+ "\n",
+ "The following example is directly taken from https://adversarial-ml-tutorial.org/introduction/.\n",
+ "\n",
+ "\n",
+ "\n",
+ "The first image is correctly classified as a **pig** by a classifier. The second image is incorrectly classified as an **airplane** by the same classifier. \n",
+ "\n",
+ "While both images cannot be distinguished from our (human) perspective, the second image is in fact the result of surimposing \"noise\" (i.e. adding an adversarial perturbation) to the original first image."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "84e3cb53-98f7-44cb-bb9b-29b94e4fc6bd",
+ "metadata": {},
+ "source": [
+ "Below is a visualization of the added noise, zoomed-in by a factor of 50 so that we can see it:\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d4125128-de5d-4267-83ae-892e9d8ced69",
+ "metadata": {},
+ "source": [
+ "### Adversarial robustness of 1-Lipschitz neural network\n",
+ "The adversarial robustness of a predictive model is its ability to remain accurate and reliable when subjected to adversarial perturbations. \n",
+ "\n",
+ "A major advantage of 1-Lipschitz neural networks is that they can offer provable guarantees on their robustness for any particular input $x$, by providing a *certificate* $\\epsilon_x$. \n",
+ "Such a guarantee can be understood by using the following terminology:\n",
+ "\n",
+ "> \"For an input $x$, we can certify that there are no adversarial perturbations constrained to be under the certificate $\\epsilon_x$ that will change our model's prediction.\"\n",
+ "\n",
+ "In simple mathematical terms: \n",
+ "\n",
+ "For a given $x$, $\\forall \\epsilon$ such that $||\\epsilon||<\\epsilon_x$, we obtain that:\n",
+ "$$A(x)=y,$$\n",
+ "$$A(x+\\epsilon)=y_{\\epsilon},$$\n",
+ "then:\n",
+ "$$y_{\\epsilon}=y.$$\n",
+ "\n",
+ "๐ก We will use certificates in this notebook as a metric to evaluate the provable adversarial robustness of deep learning 1-Lispchitz models.\n",
+ "\n",
+ "๐ก Depending on the type of norm you choose (e.g. L1 or L2), the guarantee you can offer will differ, as $||\\epsilon||_{L2}<\\epsilon_x$ and $||\\epsilon||_{L1}<\\epsilon_x$ are not equivalent. \n",
+ "\n",
+ "๐จ **Note**: *`deel-lip` only deals with L2 norm, as previously said in the first notebook 'Getting started 1'*\n",
+ "\n",
+ "As such, an additional example of guarantee that could be obtained with `deel-lip` with a more precise formulation would be:\n",
+ "> \"For an input $x$, we can certify that are no adversarial perturbations constrained to be within a $\\text{L2}$-norm ball of certificate $\\epsilon_{x,\\text{ L2}}$ that will change our model's prediction.\"\n",
+ "\n",
+ "For a given $x$, $\\forall \\epsilon$ such that $||\\epsilon||_{L2}<\\epsilon_{x,\\text{ L2}}$, we obtain that:\n",
+ "$$A(x)=y,$$\n",
+ "$$A(x+\\epsilon)=y_{\\epsilon},$$\n",
+ "then:\n",
+ "$$y_{\\epsilon}=y.$$\n",
+ "\n",
+ "\n",
+ "## ๐ช Training provable adversarially robust 1-Lipschitz neural networks on the MNIST dataset \n",
+ "\n",
+ "\n",
+ "### ๐พ MNIST dataset\n",
+ "MNIST dataset contains a large number of 28x28 handwritten digit images to which are associated digit labels."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "d3cf25c6-1691-4935-bdac-9f182a87ef32",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from tensorflow.keras.datasets import mnist\n",
+ "from tensorflow.keras.utils import to_categorical\n",
+ "import numpy as np"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "90da93e3-9c59-4e72-b817-01f97b324c51",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load MNIST Database\n",
+ "(X_train, y_train_ord), (X_test, y_test_ord) = mnist.load_data()\n",
+ "\n",
+ "# standardize and reshape the data\n",
+ "X_train = np.expand_dims(X_train, -1) / 255\n",
+ "X_test = np.expand_dims(X_test, -1) / 255\n",
+ "\n",
+ "# one hot encode the labels\n",
+ "y_train = to_categorical(y_train_ord)\n",
+ "y_test = to_categorical(y_test_ord)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "df14bc65-3ebb-4f3c-925b-9a21cefa7e23",
+ "metadata": {},
+ "source": [
+ "### ๐ฎ Control over the accuracy-robustness trade-off with `deel-lip`'s loss functions.\n",
+ "\n",
+ "When training neural networks, there is always a compromise between the robustness and the accuracy of the models. In simple terms, achieving stronger robustness often involves sacrificing some performance (at the extreme point, the most robust function being the constant function).\n",
+ "\n",
+ "In this section, we will show the pivotal role of `deel-lip`'s loss functions in training 1-Lipschitz networks. Each of these functions comes with its own set of hyperparameters, enabling you to precisely navigate and adjust the balance between accuracy and robustness.\n",
+ "\n",
+ "We show two cases. In the first case, we use `deel-lip`'s `TauCategoricalCrossentropy` from the `losses` submodule. In the second case, we use another loss function from `deel-lip`: `MulticlassHKR`.\n",
+ "\n",
+ "\n",
+ "#### ๐ฎ Prediction Model\n",
+ "Since we will be instantiating the same model four times within our examples, we encapsulate the code for creating the model within a function to enhance conciseness:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "359fa47e-86f3-440d-971f-0a49fe7dcdc7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from deel import lip\n",
+ "\n",
+ "from tensorflow.keras.optimizers import Adam\n",
+ "from tensorflow.keras.layers import Input, Flatten\n",
+ "\n",
+ "\n",
+ "def create_conv_model(name_model, input_shape, output_shape):\n",
+ " \"\"\"\n",
+ " A simple convolutional neural network, made to be 1-Lipschitz.\n",
+ " \"\"\"\n",
+ " model= lip.Sequential(\n",
+ " [\n",
+ " Input(shape=input_shape),\n",
+ " \n",
+ " lip.layers.SpectralConv2D(\n",
+ " filters=16,\n",
+ " kernel_size=(3, 3),\n",
+ " use_bias=True,\n",
+ " kernel_initializer=\"orthogonal\",\n",
+ " ),\n",
+ "\n",
+ " lip.layers.GroupSort2(), \n",
+ " \n",
+ " lip.layers.ScaledL2NormPooling2D(pool_size=(2, 2), data_format=\"channels_last\"),\n",
+ " \n",
+ " lip.layers.SpectralConv2D(\n",
+ " filters=32,\n",
+ " kernel_size=(3, 3),\n",
+ " use_bias=True,\n",
+ " kernel_initializer=\"orthogonal\",\n",
+ " ),\n",
+ "\n",
+ " lip.layers.GroupSort2(),\n",
+ " \n",
+ " lip.layers.ScaledL2NormPooling2D(pool_size=(2, 2), data_format=\"channels_last\"),\n",
+ " \n",
+ " Flatten(),\n",
+ " \n",
+ " lip.layers.SpectralDense(\n",
+ " 64,\n",
+ " use_bias=True,\n",
+ " kernel_initializer=\"orthogonal\",\n",
+ " ),\n",
+ " \n",
+ " lip.layers.GroupSort2(),\n",
+ " \n",
+ " lip.layers.SpectralDense(\n",
+ " output_shape, \n",
+ " activation=None, \n",
+ " use_bias=False, \n",
+ " kernel_initializer=\"orthogonal\"\n",
+ " ),\n",
+ " ],\n",
+ "\n",
+ " name=name_model,\n",
+ " )\n",
+ "\n",
+ " return model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "b2d7a4ce-df5e-4bf0-b461-5049fc749680",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "input_shape=X_train.shape[1:]\n",
+ "output_shape=y_train.shape[-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d4657a28-95c5-4f35-90bd-53a48bb44a5e",
+ "metadata": {},
+ "source": [
+ "#### Cross-entropy loss: `TauCategoricalCrossentropy`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c8aa1c60-0271-4b76-a0b7-7d2bf89f4550",
+ "metadata": {},
+ "source": [
+ "Similar to the classes we have seen in \"Getting started 1\", the `TauCategoricalCrossentropy` class inherits from its equivalent in `keras`, but it comes with an additional settable parameter named 'temperature' and denoted as: `tau`. This parameter will allow to adjust the robustness of our model. The lower the temperature is, the more robust our model becomes, but it also becomes less accurate.\n",
+ "\n",
+ "To show the impact of the parameter `tau` on both the performance and robustness of our model, we will train two models on the MNIST dataset. The first model will have a temperature of 100, the second model will have a temperature of 3."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "fc8a3115-ccb4-4ce3-86a2-b3c0a22866b3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# high-temperature model\n",
+ "model_1 = create_conv_model(\"cross_entropy_model_1\", input_shape, output_shape)\n",
+ "\n",
+ "temperature_1=100.\n",
+ "\n",
+ "model_1.compile(\n",
+ " loss=lip.losses.TauCategoricalCrossentropy(tau=temperature_1),\n",
+ " optimizer=Adam(1e-4),\n",
+ " # notice the use of lip.losses.Certificate_Multiclass, \n",
+ " # to assess adversarial robustness\n",
+ " metrics=[\"accuracy\", lip.losses.Certificate_Multiclass(model_1)],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "795bf0ad-5495-4b55-aa24-baa6205415b5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# low-temperature model\n",
+ "model_2 = create_conv_model(\"cross_entropy_model_2\", input_shape, output_shape)\n",
+ "\n",
+ "temperature_2=3.\n",
+ "\n",
+ "model_2.compile(\n",
+ " loss=lip.losses.TauCategoricalCrossentropy(tau=temperature_2),\n",
+ " optimizer=Adam(1e-4),\n",
+ " metrics=[\"accuracy\", lip.losses.Certificate_Multiclass(model_2)],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "13859a2a-cd4a-4cc5-bc90-f61a07cd22ad",
+ "metadata": {},
+ "source": [
+ "๐ก Notice that we use the accuracy metric to measure the performance, and we use the `Certificate_Multiclass` loss to measure adversarial robustness. The latter is a measure of our model's average certificates: **the higher this measure is, the more robust our model is**. \n",
+ "\n",
+ "**๐จ Note:** *This is true only for 1-Lipschitz neural networks*"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f878ac98-e80a-4d9a-a2a4-ef607e8b8001",
+ "metadata": {},
+ "source": [
+ "We fit both our models and observe the results."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "bba315c1-c4df-47aa-ae13-202831555355",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Epoch 1/2\n",
+ "235/235 [==============================] - 42s 168ms/step - loss: 0.0104 - accuracy: 0.7811 - Certificate_Multiclass: 0.0324 - val_loss: 0.0029 - val_accuracy: 0.9181 - val_Certificate_Multiclass: 0.0408\n",
+ "Epoch 2/2\n",
+ "235/235 [==============================] - 39s 166ms/step - loss: 0.0024 - accuracy: 0.9294 - Certificate_Multiclass: 0.0426 - val_loss: 0.0017 - val_accuracy: 0.9447 - val_Certificate_Multiclass: 0.0444\n"
+ ]
+ }
+ ],
+ "source": [
+ "# fit the high-temperature model\n",
+ "result_1=model_1.fit(\n",
+ " X_train,\n",
+ " y_train,\n",
+ " batch_size=256,\n",
+ " epochs=2,\n",
+ " validation_data=(X_test, y_test),\n",
+ " shuffle=True,\n",
+ " #verbose=1,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "914ac450-e529-406e-8233-79facc4c1190",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Epoch 1/2\n",
+ "235/235 [==============================] - 40s 163ms/step - loss: 0.3460 - accuracy: 0.7822 - Certificate_Multiclass: 0.2839 - val_loss: 0.1574 - val_accuracy: 0.8994 - val_Certificate_Multiclass: 0.5042\n",
+ "Epoch 2/2\n",
+ "235/235 [==============================] - 40s 169ms/step - loss: 0.1348 - accuracy: 0.9094 - Certificate_Multiclass: 0.5874 - val_loss: 0.1075 - val_accuracy: 0.9265 - val_Certificate_Multiclass: 0.6748\n"
+ ]
+ }
+ ],
+ "source": [
+ "# fit the low-temperature model\n",
+ "result_2=model_2.fit(\n",
+ " X_train,\n",
+ " y_train,\n",
+ " batch_size=256,\n",
+ " epochs=2,\n",
+ " validation_data=(X_test, y_test),\n",
+ " shuffle=True,\n",
+ " #verbose=1,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "af5c9695-023b-4fd4-a1f8-d630f3298f43",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Model accuracy: 0.9447\n",
+ "Model's mean certificate: 0.0444\n",
+ "Loss' temperature: 100.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# metrics for the high-temperature model => performance-oriented \n",
+ "print(f\"Model accuracy: {result_1.history['val_accuracy'][-1]:.4f}\")\n",
+ "print(f\"Model's mean certificate: {result_1.history['val_Certificate_Multiclass'][-1]:.4f}\")\n",
+ "print(f\"Loss' temperature: {model_1.loss.tau.numpy():.1f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "f5ed957c-4308-4f75-9a41-dcbc88dcc341",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Model accuracy: 0.9265\n",
+ "Model's mean certificate: 0.6748\n",
+ "Loss' temperature: 3.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# metrics for the low-temperature model => robustness-oriented\n",
+ "print(f\"Model accuracy: {result_2.history['val_accuracy'][-1]:.4f}\")\n",
+ "print(f\"Model's mean certificate: {result_2.history['val_Certificate_Multiclass'][-1]:.4f}\")\n",
+ "print(f\"Loss' temperature: {model_2.loss.tau.numpy():.1f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4c77b7cd-3d8c-4030-9063-d1e142c47b2f",
+ "metadata": {},
+ "source": [
+ "When decreasing the temperature, we observe a large increase in robustness, but a slight decrease in accuracy."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e8353a6a-4591-45d6-aeb5-eb9c49231f09",
+ "metadata": {},
+ "source": [
+ "#### Hinge-KantorovichโRubinstein loss: `MulticlassHKR`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "cbea6020-4102-4956-bbb4-87b169caaf39",
+ "metadata": {},
+ "source": [
+ "We work in the same way as in the previous section. The difference lies in the parameters that control the robustness.\n",
+ "\n",
+ "We count two of them: `min_margin` (minimal margin) and `alpha` (regularization factor).\n",
+ "\n",
+ "As will be shown in the following, a higher minimal margin and a lower alpha increases robustness. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "a964983d-4347-4f57-8489-98e6a05cfd0c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# performance-oriented model\n",
+ "model_3 = create_conv_model(\"HKR_model_3\", input_shape, output_shape)\n",
+ "\n",
+ "min_margin_3=0.1\n",
+ "alpha_3=50\n",
+ "\n",
+ "model_3.compile(\n",
+ " loss=lip.losses.MulticlassHKR(min_margin=min_margin_3,alpha=alpha_3),\n",
+ " optimizer=Adam(1e-4),\n",
+ " metrics=[\"accuracy\", lip.losses.Certificate_Multiclass(model_3)],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "5b57e8ad-19fc-4791-8e28-d0e613cbddc5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# robustness-oriented model\n",
+ "model_4 = create_conv_model(\"HKR_model_4\", input_shape, output_shape)\n",
+ "\n",
+ "min_margin_4=1\n",
+ "alpha_4=30\n",
+ "\n",
+ "model_4.compile(\n",
+ " loss=lip.losses.MulticlassHKR(min_margin=min_margin_4,alpha=alpha_4),\n",
+ " optimizer=Adam(1e-4),\n",
+ " metrics=[\"accuracy\", lip.losses.Certificate_Multiclass(model_4)],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3b9f6ce2-ec2b-4f4d-b282-918d434eb9b3",
+ "metadata": {},
+ "source": [
+ "We fit both our models and observe the results."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "b976968d-4494-4935-8338-06ebd99b6c78",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Epoch 1/2\n",
+ "235/235 [==============================] - 39s 155ms/step - loss: 0.8067 - accuracy: 0.8190 - Certificate_Multiclass: 0.0768 - val_loss: 0.1725 - val_accuracy: 0.9286 - val_Certificate_Multiclass: 0.1178\n",
+ "Epoch 2/2\n",
+ "235/235 [==============================] - 36s 154ms/step - loss: 0.0459 - accuracy: 0.9326 - Certificate_Multiclass: 0.1501 - val_loss: -0.1157 - val_accuracy: 0.9483 - val_Certificate_Multiclass: 0.1858\n"
+ ]
+ }
+ ],
+ "source": [
+ "# fit the model\n",
+ "result_3=model_3.fit(\n",
+ " X_train,\n",
+ " y_train,\n",
+ " batch_size=256,\n",
+ " epochs=2,\n",
+ " validation_data=(X_test, y_test),\n",
+ " shuffle=True,\n",
+ " #verbose=1,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "8f19da69-9869-463e-8fac-b7f179e00315",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Epoch 1/2\n",
+ "235/235 [==============================] - 38s 154ms/step - loss: 10.4566 - accuracy: 0.7053 - Certificate_Multiclass: 0.2512 - val_loss: 4.1848 - val_accuracy: 0.8783 - val_Certificate_Multiclass: 0.4431\n",
+ "Epoch 2/2\n",
+ "235/235 [==============================] - 36s 154ms/step - loss: 3.2332 - accuracy: 0.8920 - Certificate_Multiclass: 0.5451 - val_loss: 2.1959 - val_accuracy: 0.9128 - val_Certificate_Multiclass: 0.6435\n"
+ ]
+ }
+ ],
+ "source": [
+ "# fit the model\n",
+ "result_4=model_4.fit(\n",
+ " X_train,\n",
+ " y_train,\n",
+ " batch_size=256,\n",
+ " epochs=2,\n",
+ " validation_data=(X_test, y_test),\n",
+ " shuffle=True,\n",
+ " #verbose=1,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "5bfe7641-f7af-4ac3-9ff4-fca3300448ff",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Model accuracy: 0.9483\n",
+ "Model's mean certificate: 0.1858\n",
+ "Loss' minimum margin: 0.1\n",
+ "Loss' alpha: 50.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# performance-oriented model\n",
+ "print(f\"Model accuracy: {result_3.history['val_accuracy'][-1]:.4f}\")\n",
+ "print(f\"Model's mean certificate: {result_3.history['val_Certificate_Multiclass'][-1]:.4f}\")\n",
+ "print(f\"Loss' minimum margin: {model_3.loss.min_margin.numpy():.1f}\")\n",
+ "print(f\"Loss' alpha: {model_3.loss.alpha.numpy():.1f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "7ce20cb8-797b-4e03-8600-d0030adfccc1",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Model accuracy: 0.9128\n",
+ "Model's mean certificate: 0.6435\n",
+ "Loss' minimum margin: 1.0\n",
+ "Loss' alpha: 30.0\n"
+ ]
+ }
+ ],
+ "source": [
+ "# robustness-oriented model\n",
+ "print(f\"Model accuracy: {result_4.history['val_accuracy'][-1]:.4f}\")\n",
+ "print(f\"Model's mean certificate: {result_4.history['val_Certificate_Multiclass'][-1]:.4f}\")\n",
+ "print(f\"Loss' minimum margin: {model_4.loss.min_margin.numpy():.1f}\")\n",
+ "print(f\"Loss' alpha: {model_4.loss.alpha.numpy():.1f}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "00e39a0d-a145-4a44-85c3-5f67e64ce444",
+ "metadata": {},
+ "source": [
+ "We confirmed experimentally the accuracy-robustness trade-off: a higher minimal margin and a lower alpha increases robustness, but also decreases accuracy."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c4857464-3ed3-436e-8853-9ed67d0ce5c6",
+ "metadata": {},
+ "source": [
+ "## ๐ Congratulations\n",
+ "You now know how to train provable adversarially robust 1-Lipschitz neural networks!\n",
+ "\n",
+ "๐ Interested readers can learn more about the role of loss functions and the accuracy-robustness trade-off which occurs when training adversarially robust 1-Lipschitz neural network in the following paper: \n",
+ " [Pay attention to your loss: understanding misconceptions about 1-Lipschitz neural networks](https://arxiv.org/abs/2104.05097)."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.9"
+ },
+ "toc": {
+ "base_numbering": 1,
+ "nav_menu": {},
+ "number_sections": true,
+ "sideBar": true,
+ "skip_h1_title": false,
+ "title_cell": "Table of Contents",
+ "title_sidebar": "Contents",
+ "toc_cell": false,
+ "toc_position": {},
+ "toc_section_display": true,
+ "toc_window_display": true
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/mkdocs.yml b/mkdocs.yml
index 4d05594f..501678c2 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -15,6 +15,7 @@ nav:
- deel.lip.utils: api/utils.md
- Tutorials:
- "Getting started 1 - Creating a 1-Lipschitz neural network": notebooks/Getting_started_1.ipynb
+ - "Getting started 2 - Training an adversarially robust 1-Lipschitz neural network": notebooks/Getting_started_2.ipynb
- "Demo 0: Example & Usage": notebooks/demo0.ipynb
- "Demo 1: Wasserstein distance estimation on toy example": notebooks/demo1.ipynb
- "Demo 2: HKR Classifier on toy dataset": notebooks/demo2.ipynb