Merge pull request #104 from ESMValGroup/instructions-data-download-o…

…wn-machine-54

Instructions data download own machine 54

.github/workflows/recipes.yml

@@ -0,0 +1,48 @@
+name: Test recipes
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache conda
+        uses: actions/cache@v1
+        env:
+          # Increase this value to reset cache if data/recipe_example.yml has not changed
+          CACHE_NUMBER: 1
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('data/recipe_example.yml') }}
+      - uses: goanpeca/setup-miniconda@v1
+      - name: Install esmvaltool
+        run: conda install -n test -y -c esmvalgroup -c conda-forge esmvaltool-python
+      - name: Setup config
+        run: |
+          $CONDA/envs/test/bin/esmvaltool config get_config_user
+          patch ~/.esmvaltool/config-user.yml << EOF
+          44c44
+          < #rootpath:
+          ---
+          > rootpath:
+          48c48
+          < #  default: ~/default_inputpath
+          ---
+          >   default: ~/default_inputpath
+          EOF
+      - name: Cache datasets
+        uses: actions/cache@v1
+        env:
+          # Increase this value to reset cache if data/dataset.urls has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/default_inputpath
+          key: ${{ runner.os }}-datasets-${{ env.CACHE_NUMBER }}-${{ hashFiles('data/dataset.urls') }}
+      - name: Download dataset files for data/recipe_example.yml
+        run: |
+          head -4 data/dataset.urls | grep -v '#' | wget --input-file - --no-clobber --directory-prefix $HOME/default_inputpath/
+      - name: Run data/recipe_example.yml
+        run: $CONDA/envs/test/bin/esmvaltool run $PWD/data/recipe_example.yml

data/dataset.urls

@@ -0,0 +1,47 @@
+# Dataset urls required fo tutorial, described at https://esmvalgroup.github.io/ESMValTool_Tutorial/setup.html#using-your-own-machine
+# For Running a recipe episode
+# Variable = thetaoga, model = HadGEM2-ES
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/ocean/Omon/r1i1p1/v20110916/thetaoga/thetaoga_Omon_HadGEM2-ES_historical_r1i1p1_185912-200512.nc
+# Variable = ts, model = HadGEM2-ES
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ts/ts_Amon_HadGEM2-ES_historical_r1i1p1_188412-190911.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ts/ts_Amon_HadGEM2-ES_historical_r1i1p1_190912-193411.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ts/ts_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/ts/ts_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+# Variable = tas, model =  HadGEM2-AO
+http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/NIMR-KMA/HadGEM2-AO/historical/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
+# Variable = tas, model = HadGEM2-ES
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/tas/tas_Amon_HadGEM2-ES_historical_r1i1p1_188412-190911.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/tas/tas_Amon_HadGEM2-ES_historical_r1i1p1_190912-193411.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/tas/tas_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/v20120928/tas/tas_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
+# Variable = tos, model = HadGEM2-AO
+http://aims3.llnl.gov/thredds/fileServer/cmip5_css02_data/cmip5/output1/NIMR-KMA/HadGEM2-AO/historical/mon/ocean/Omon/r1i1p1/tos/1/tos_Omon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc
+# Variable = tos, model = HadGEM2-CC
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/historical/mon/ocean/Omon/r1i1p1/v20110930/tos/tos_Omon_HadGEM2-CC_historical_r1i1p1_185912-195911.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-CC/historical/mon/ocean/Omon/r1i1p1/v20110930/tos/tos_Omon_HadGEM2-CC_historical_r1i1p1_195912-200511.nc
+# Variable = tos, model = HadGEM2-ES
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/ocean/Omon/r1i1p1/v20110916/tos/tos_Omon_HadGEM2-ES_historical_r1i1p1_185912-195911.nc
+http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/ocean/Omon/r1i1p1/v20110916/tos/tos_Omon_HadGEM2-ES_historical_r1i1p1_195912-200512.nc
+# For Working with preprocessors episode
+# Model = UKESM1-0-LL, project = CMIP6, Experiment = historical, ensemble= r1i1p1f2, variable = thetao, pr and tas, 1970-2000
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r1i1p1f2/Omon/thetao/gn/v20190627/thetao_Omon_UKESM1-0-LL_historical_r1i1p1f2_gn_200001-201412.nc
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r1i1p1f2/Omon/thetao/gn/v20190627/thetao_Omon_UKESM1-0-LL_historical_r1i1p1f2_gn_195001-199912.nc
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r1i1p1f2/Amon/tas/gn/v20190406/tas_Amon_UKESM1-0-LL_historical_r1i1p1f2_gn_195001-201412.nc
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r1i1p1f2/Amon/pr/gn/v20190406/pr_Amon_UKESM1-0-LL_historical_r1i1p1f2_gn_195001-201412.nc
+# Model = CanESM2, ensemble: "r(1:4)i1p1", project: CMIP5, variable = tas
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esg_dataroot/AR5/CMIP5/output/CCCma/CanESM2/historical/mon/atmos/tas/r1i1p1/tas_Amon_CanESM2_historical_r1i1p1_185001-200512.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esg_dataroot/AR5/CMIP5/output/CCCma/CanESM2/historical/mon/atmos/tas/r2i1p1/tas_Amon_CanESM2_historical_r2i1p1_185001-200512.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esg_dataroot/AR5/CMIP5/output/CCCma/CanESM2/historical/mon/atmos/tas/r3i1p1/tas_Amon_CanESM2_historical_r3i1p1_185001-200512.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esg_dataroot/AR5/CMIP5/output/CCCma/CanESM2/historical/mon/atmos/tas/r4i1p1/tas_Amon_CanESM2_historical_r4i1p1_185001-200512.nc
+# Model = MPI-ESM-LR, ensemble: "r(1:2)i1p1", project: CMIP5, variable = tas
+http://esgf1.dkrz.de/thredds/fileServer/cmip5/cmip5/output1/MPI-M/MPI-ESM-LR/historical/mon/atmos/Amon/r1i1p1/v20120315/tas/tas_Amon_MPI-ESM-LR_historical_r1i1p1_185001-200512.nc
+http://esgf1.dkrz.de/thredds/fileServer/cmip5/cmip5/output1/MPI-M/MPI-ESM-LR/historical/mon/atmos/Amon/r2i1p1/v20120315/tas/tas_Amon_MPI-ESM-LR_historical_r2i1p1_185001-200512.nc
+# Model = UKESM1-0-LL, ensemble: "r(1:4)i1p1f2", grid: gn, project: CMIP6, variable = tas
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r2i1p1f2/Amon/tas/gn/v20190502/tas_Amon_UKESM1-0-LL_historical_r2i1p1f2_gn_195001-201412.nc
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r3i1p1f2/Amon/tas/gn/v20190502/tas_Amon_UKESM1-0-LL_historical_r3i1p1f2_gn_195001-201412.nc
+http://esgf-data3.ceda.ac.uk/thredds/fileServer/esg_cmip6/CMIP6/CMIP/MOHC/UKESM1-0-LL/historical/r4i1p1f2/Amon/tas/gn/v20190502/tas_Amon_UKESM1-0-LL_historical_r4i1p1f2_gn_195001-201412.nc
+# Model = CanESM5, ensemble: "r(1:4)i1p2f1", grid: gn, project: CMIP6, variable = tas
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esgC_dataroot/AR6/CMIP6/CMIP/CCCma/CanESM5/historical/r1i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_historical_r1i1p2f1_gn_185001-201412.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esgC_dataroot/AR6/CMIP6/CMIP/CCCma/CanESM5/historical/r2i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_historical_r2i1p2f1_gn_185001-201412.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esgC_dataroot/AR6/CMIP6/CMIP/CCCma/CanESM5/historical/r3i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_historical_r3i1p2f1_gn_185001-201412.nc
+http://crd-esgf-drc.ec.gc.ca/thredds/fileServer/esgC_dataroot/AR6/CMIP6/CMIP/CCCma/CanESM5/historical/r4i1p2f1/Amon/tas/gn/v20190429/tas_Amon_CanESM5_historical_r4i1p2f1_gn_185001-201412.nc

setup.md

@@ -68,7 +68,8 @@ If necessary, data can be downloaded using the
 
 ### CEDA-Jasmin
 
-Please skip this section if you are not going to use JASMIN and go [here](#Github-account-(Advanced)).
+Please skip this section if you are not going to use JASMIN
+and go [here](#Github-account-(Advanced)).
 
 If you do not already have an account on JASMIN, then request an account as soon
 as possible. Please follow [these instructions on how to create a Jasmin
@@ -253,30 +254,37 @@ local machine and go [here](#gitHub-account-(advanced)).
 If you are planning on running ESMValTool on your own machine, please make sure
 that you are able to download CMIP data and that you have several GB of space
 available to install conda & ESMValTool, but also enough to make a copy of some
-data.
+data (12Gb).
 
-You will also need to able to use:
-- git
-- conda
-- synda
+We can download a single data file following the instructions as described below:
 
-#### Linux/Unix
+1. Go to the [CMIP5 search page on the DKRZ ESGF node][cmip5-search]
+2. Perform the following search constraints
 
-For Linux/Unix systems, please follow the instructions of the [Installation episode]({{
-page.root}}{% link _episodes/02-installation.md  %}).
+    - Model = HadGEM2-ES
+    - Experiment = historical
+    - Time frequency = mon
+    - Ensemble =  r1i1p1
+    - Variable = thetaoga
 
+3. Press search button
+4. On single search result press `List files`
+5. At bottom of page click on `Show All Files`
+6. Find the file which starts with `thetaoga` in the list
+7. Use `HTTP Download` link to download
+   the [thetaoga_Omon_HadGEM2-ES_historical_r1i1p1_185912-200512.nc][theatoga.nc] file
+   to `~/default_inputpath/` directory.
 
-#### Mac OSx
+However, the tutorial needs more data files. The [dataset.urls][ds] file contains all data set URLs and in comments the used search constraints. All of the data files should be downloaded
+to `~/default_inputpath/` directory.
 
-Also, for Mac OSx systems, please follow the instructions of the [Installation episode]({{
-page.root}}{% link _episodes/02-installation.md  %}).
+To download the data, run the following command using [wget](https://en.wikipedia.org/wiki/Wget):
 
-#### Windows
-
-ESMValTool does not directly support Windows,
-but successful usage has been reported through the
-[Windows Subsystem for Linux(WSL)](https://docs.microsoft.com/en-us/windows/wsl/),
-available in Windows 10.
+~~~shell
+wget --no-clobber --input-file \
+  https://github.com/ESMValGroup/ESMValTool_Tutorial/raw/master/data/dataset.urls \
+  --directory-prefix $HOME/default_inputpath/
+~~~
 
 ## GitHub account (Advanced)
 
@@ -310,13 +318,8 @@ A GitHub pull request is the act of requesting that a branch is merged with anot
 This is an advanced feature of GitHub, and will generally be performed by the
 ESMValTool development team.
 
-## Install conda
-
-The python package manager Conda (anaconda or miniconda) needs to be installed
-on your system before the tutorial starts. In some cases, your system may have a
-central version installed already.
-
-More details on this process are available in the [Installation episode]({{
-page.root}}{% link _episodes/02-installation.md  %}).
-
 {% include links.md %}
+
+[cmip5-search]: https://esgf-data.dkrz.de/search/cmip5-dkrz/
+[theatoga.nc]: http://esgf-data1.ceda.ac.uk/thredds/fileServer/esg_dataroot/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/ocean/Omon/r1i1p1/v20110916/thetaoga/thetaoga_Omon_HadGEM2-ES_historical_r1i1p1_185912-200512.nc
+[ds]: https://github.com/ESMValGroup/ESMValTool_Tutorial/raw/master/data/dataset.urls