Merge pull request #62 from kreczko/testing-PR-60

Docker for HTCondor testing

.dockerignore

@@ -0,0 +1,105 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Eclipse project files
+.*project
+
+# Vagrant folder
+.vagrant
+
+# common editor backup files
+~*
+.*sw[po]
+
+# Custom ignores
+external
+workspace
+data
+benchmark

README.md

@@ -42,15 +42,15 @@ make setup
 
 ### running tests
 Tests can be run either on an SL 6 machine or in the Vagrant box:
-```
+```bash
 make test
 # if a grid proxy is provided (e.g. via voms-proxy-init --voms cms)
 # you can also run tests that require grid access:
 make test-all
 ```
 
 ### running benchmark
-```
+```bash
 # install python requirements
 pip install -r requirements.txt --user
 make benchmark
@@ -72,6 +72,7 @@ make docs-latex # produces output in docs/_build/latex
 # you might need to
 # export PATH:/cvmfs/sft.cern.ch/lcg/external/texlive/2014/bin/x86_64-linux:$PATH
 # for docs-latex
+```
 
 #### Prerequisites
 ```bash
@@ -88,11 +89,35 @@ You can see an example of an analyzer at: `cmsl1t/analyzers/demo_analyzer.py`.
 To implement your own analyzer, all you need to do is make a new class in a file under `cmsl1t/analyzers/` which inherits from `cmsl1t.analyzers.BaseAnalyzer.BaseAnalyzer`.  You then need to implement two or three methods: `prepare_for_event`, ` fill_histograms`, `write_histograms`, and `make_plots`.  See the BaseAnalyzer class and the demo_analyzer for examples and documentation of these methods.
 
 Once you have implemented an analyzer and written a simple configuration for it, you can run it with `cmsl1t` command:
-```
+```bash
 cmsl1t -c config/demo.yaml -n 1000
 ```
 
 Get help on the command line options by doing:
-```
+```bash
 cmsl1t --help
 ```
+
+### Testing HTCondor submission
+
+For HTCondor we have an all-in-one Docker container. From the code repo:
+```bash
+docker-compose up -d
+docker exec -ti cmsl1tanalysis_cmsl1t_1 cdw
+# do your tests
+
+# logout once done
+
+# shut down the container(s)
+docker-compose down
+```
+
+**NOTE**: If you are on Linux you have to install `docker-compose` by hand:
+```bash
+sudo curl -L https://github.com/docker/compose/releases/download/1.15.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+```
+to update you have to `sudo rm -f /usr/local/bin/docker-compose` first.
+
+
+To build the docker container: `docker-compose build` or `docker build -t kreczko/cms-l1t-analysis -f docker/Dockerfile .`.

bin/env.sh

@@ -39,17 +39,41 @@ export PATH=~/.local/bin:$PATH
 unset old_projectbase
 unset envscript
 
-# this gives you voms-proxy-*, xrdcp and other grid tools
-source /cvmfs/grid.cern.ch/etc/profile.d/setup-cvmfs-ui.sh
-# ROOT 6, voms-proxy-init and other things
-source /cvmfs/sft.cern.ch/lcg/views/LCG_latest/x86_64-slc6-gcc62-opt/setup.sh
-
-# to fix java for the hadoop commands:
-unset JAVA_HOME
-
+if [[ -z "${NO_CVMFS}" ]]
+then
+  # this gives you voms-proxy-*, xrdcp and other grid tools
+  source /cvmfs/grid.cern.ch/etc/profile.d/setup-cvmfs-ui.sh
+  # ROOT 6, voms-proxy-init and other things
+  source /cvmfs/sft.cern.ch/lcg/views/LCG_latest/x86_64-slc6-gcc62-opt/setup.sh
+  # to fix java for the hadoop commands:
+  unset JAVA_HOME
+  pip install --user -r requirements.txt
+else
+  echo "No CVMFS available, setting up Anaconda Python"
+  if [ ! -d "${CMSL1T_CONDA_PATH}" ]
+  then
+    wget -nv https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O /tmp/miniconda.sh
+    bash /tmp/miniconda.sh -b -p ${CMSL1T_CONDA_PATH}
+    PATH=${CMSL1T_CONDA_PATH}/bin:$PATH; export PATH
+    rm -f miniconda.sh
+    echo "Finished conda installation, creating new conda environment"
+    conda update conda -yq
+    conda update pip -yq
+    conda install psutil -yq
+    conda config --add channels http://conda.anaconda.org/NLeSC
+    conda config --set show_channel_urls yes
+    conda create -n cms python=2.7 -yq
+    source activate cms
+    conda install root=6 root-numpy numpy matplotlib nose \
+    sphinx pytables rootpy pandas -yq
+    echo "Created conda environment, installing basic dependencies"
+    pip install -r requirements.txt
+    conda clean -t -y
+  fi
+  source activate cms
+fi
 
 git submodule init
 git submodule update
-pip install --user -r requirements.txt
 
 echo "Environment for ${PROJECT_NAME} is ready"

conda_requirements.txt

@@ -0,0 +1,7 @@
+root>=6.04
+root-numpy
+numpy
+matplotlib
+pytables
+rootpy
+pandas

docker-compose.yaml

@@ -0,0 +1,16 @@
+version: '3'
+services:
+  cmsl1t:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+    image: kreczko/cms-l1t-analysis
+    volumes:
+      - .:/code
+    environment:
+      DISPLAY: ${DISPLAY}
+      TERM: ${TERM}
+    working_dir: /code
+    tty: true
+    domainname: foo.bar
+    hostname: cmsl1t

docker/Dockerfile

@@ -0,0 +1,40 @@
+FROM kreczko/htcondor-in-a-box
+
+ENV CMSL1T_CONDA_PATH /software/cmsl1t/miniconda
+
+# ROOT requirements
+RUN sudo apt install -y libxpm-dev libxft-dev
+
+RUN sudo mkdir -p ${CMSL1T_CONDA_PATH};\
+  cd ${CMSL1T_CONDA_PATH};\
+  cd ..;\
+  sudo chown `whoami` $PWD;\
+  sudo rmdir ${CMSL1T_CONDA_PATH};
+
+RUN wget -nv https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O /tmp/miniconda.sh && \
+  bash /tmp/miniconda.sh -b -p ${CMSL1T_CONDA_PATH} && \
+  rm -f miniconda.sh
+ENV PATH="${CMSL1T_CONDA_PATH}/bin:$PATH"
+
+RUN echo "Finished conda installation, updating conda and pip" && \
+  conda update conda -yq && \
+  conda update pip -yq && \
+  conda install psutil -yq && \
+  conda config --add channels http://conda.anaconda.org/NLeSC && \
+  conda config --set show_channel_urls yes && \
+  conda clean -t -y
+
+COPY conda_requirements.txt /tmp/conda_requirements.txt
+
+RUN conda create -n cms python=2.7 -yq &&\
+  echo "Created conda environment, installing basic dependencies" && \
+  /bin/bash -c "source activate cms && \
+  conda install -yq --file /tmp/conda_requirements.txt" && \
+  conda clean -t -y
+
+COPY requirements.txt /tmp/requirements.txt
+
+RUN /bin/bash -c "source activate cms && \
+    pip install --no-cache-dir -r /tmp/requirements.txt"
+
+COPY docker/cdw /usr/local/bin/cdw

docker/cdw

@@ -0,0 +1,6 @@
+#!/bin/bash
+cd /code
+export NO_CVMFS=1
+source bin/env.sh
+stty rows 256 cols 256
+bash

requirements.txt

@@ -1,5 +1,4 @@
 memory_profiler
-#tqdm?
 flake8
 pep8
 nose
@@ -10,3 +9,4 @@ pyfakefs
 htcondor
 pyaml
 rootpy
+dill