[WIP] Generative regression (workflow, scores, cross validation, prediction type)

.travis.yml

@@ -4,6 +4,7 @@ cache: pip
 branches:
   only:
   - master
+  - advanced
 env:
   - PYTHON_VERSION=3.6
   - PYTHON_VERSION=3.7

README.rst

@@ -1,6 +1,8 @@
 RAMP workflow
 =============
 
+The advanced branch contains advanced features and may be less stable than the master. It works with the advanced branch of ramp-board_.
+
 RAMP workflow allows to define and run machine learning pipeline, documentations available here_.
 
 .. _here: https://paris-saclay-cds.github.io/ramp-docs/ramp-workflow/stable/

ci_tools/circle/push_doc.sh

@@ -6,6 +6,8 @@
 # We have three possibily workflows:
 #   If the git branch is 'master' then we want to commit and merge the dev/
 #       docs on gh-pages
+#   If the git branch is 'advanced' then we want to commit and merge the
+#       advanced docs on gh-pages
 #   If the git branch is [0-9].[0.9].X (i.e. 0.9.X, 1.0.X, 1.2.X, 41.21.X) then
 #        we want to commit and merge the major.minor/ docs on gh-pages
 #   If the git branch is anything else then we just want to test that committing
@@ -45,6 +47,13 @@ then
     doc_clone_commit
     git push origin $DOC_BRANCH
     echo "Push complete"
+elif [ "$CIRCLE_BRANCH" = "advanced" ]
+then
+    # Changes are made to advanced/ directory
+    DIR="ramp-workflow/advanced"
+    doc_clone_commit
+    git push origin $DOC_BRANCH
+    echo "Push complete"
 elif [[ "$CIRCLE_BRANCH" =~ ^[0-9]+\.[0-9]+\.X$ ]]
 then
     # Strip off .X from branch name, so changes will go to 0.1/, 91.235/, etc

doc/index.rst

@@ -19,9 +19,9 @@ and `prediction types <https://github.com/paris-saclay-cds/ramp-workflow/tree/ma
 Installation
 ************
 
-Ramp-workflow is available on PyPI and can be installed via `pip`::
+The advanced branch of ramp-workflow can be installed via `pip`::
 
-    $ pip install ramp-workflow
+    $ pip install git+https://github.com/paris-saclay-cds/ramp-workflow.git@advanced
 
 .. toctree::
    :maxdepth: 2

rampwf/cvs/__init__.py

@@ -1,7 +1,13 @@
 from .clustering import Clustering
-from .time_series import TimeSeries
+from .time_series import (
+	InsideEpisode, KFoldPerEpisode, RollingPerEpisode, ShufflePerEpisode,
+	TimeSeries)
 
 __all__ = [
     'Clustering',
+    'InsideEpisode',
+    'KFoldPerEpisode',
+    'RollingPerEpisode',
+    'ShufflePerEpisode',
     'TimeSeries',
 ]

rampwf/cvs/tests/test_time_series.py

@@ -0,0 +1,68 @@
+import numpy as np
+from numpy.testing import assert_allclose
+import pandas as pd
+
+from rampwf.cvs.time_series import KFoldPerEpisode
+from rampwf.cvs.time_series import RollingPerEpisode
+from rampwf.cvs.time_series import ShufflePerEpisode
+
+
+def test_get_episode_starts():
+    restart_name = 'restart'
+    X_data = np.random.randn(10, 2)
+    data = np.concatenate(
+        (X_data,
+         np.array([[1], [0], [0], [0], [1], [0], [0], [1], [0], [0]])), axis=1)
+    X_df = pd.DataFrame(
+        columns=['X_1', 'X_2', 'restart'], data=data)
+
+    cv = KFoldPerEpisode(restart_name, 0)
+    episode_starts = cv._get_episode_starts(X_df)
+    assert_allclose(episode_starts, np.array([0, 4, 7]))
+
+    cv = KFoldPerEpisode(restart_name, 2)
+    episode_starts = cv._get_episode_starts(X_df)
+    assert_allclose(episode_starts, np.array([0, 2, 3]))
+
+
+def test_per_restart():
+    restart_name = 'restart'
+    X_data = np.random.randn(10, 2)
+    y = np.array([[1], [0], [0], [0], [1], [0], [0], [1], [0], [0]])
+    data = np.concatenate(
+        (X_data,
+         y), axis=1)
+    X_df = pd.DataFrame(
+        columns=['X_1', 'X_2', 'restart'], data=data)
+
+    cv = KFoldPerEpisode(restart_name, 0)
+    # adding the virtual next episode start index
+    gen = cv.get_cv(X_df, y)
+    train_is, test_is = next(gen)
+    assert train_is == list(range(4, 10))
+    assert test_is == list(range(4))
+
+    cv = KFoldPerEpisode(restart_name, 2)
+    gen = cv.get_cv(X_df, np.arange(3))
+    train_is, test_is = next(gen)
+    assert train_is == [2]
+    assert test_is == list(range(2))
+
+    cv = ShufflePerEpisode(restart_name, random_state=2)
+    gen = cv.get_cv(X_df, y)
+    train_is, test_is = next(gen)
+    assert train_is == [0, 1, 2, 3, 4, 5, 6]
+    assert test_is == [7, 8, 9]
+    train_is, test_is = next(gen)
+    train_is, test_is = next(gen)
+    assert train_is == [4, 5, 6, 7, 8, 9]
+    assert test_is == [0, 1, 2, 3]
+
+    cv = RollingPerEpisode(restart_name)
+    gen = cv.get_cv(X_df, y)
+    train_is, test_is = next(gen)
+    assert train_is == [0, 1, 2, 3]
+    assert test_is == [4, 5, 6]
+    train_is, test_is = next(gen)
+    assert train_is == [0, 1, 2, 3, 4, 5, 6]
+    assert test_is == [7, 8, 9]