deleted requirements_developers.txt

.gitignore

@@ -6,6 +6,11 @@ __pycache__/
 # C extensions
 *.so
 
+# Cython modules translated into C
+# Eveyrtime you create a new one, remember to add it to the gitignore.
+*create_shuffles.c
+*overlap_regions.c
+
 # Distribution / packaging
 .Python
 build/
@@ -24,6 +29,7 @@ wheels/
 .installed.cfg
 *.egg
 MANIFEST
+release_in_progress
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Makefile

@@ -44,11 +44,12 @@ help:
 	@echo -e "\tbats_cmd Usage: make bats_cmd CMD=select_by_key"
 	@echo ""
 	@echo -e "\ttest_para Usage: make test_para -j 10"
+	@echo -e "\release_all Usage: make release_all VER=1.0.0"
 
 doc:
 	@rm -Rf docs/build/
 	@rm -Rf  dist build *.egg*
-	@cd docs/; make html; cd ../..;
+	@cd docs/; make html -j 4; cd ../..;
 	@echo ">>>> Docs is available in: docs/build/html/index.html"
 
 pylint:
@@ -59,10 +60,10 @@ pylintshort:
 	@find . -name "*.py"  -exec pylint $(PYLINT_ARGS) {} \; 2>/dev/null |perl -ne  "print if(/(Your code has been rated)|(\*\*\*\* Module)/)"
 
 nose:
-	@cd /tmp; mkdir -p gtftk_test; cd gtftk_test; a=`python -c "import os,pygtftk; print(os.path.dirname(pygtftk.__file__))"`; cd $$a ; for i in `find . -name "*.py" | perl -ne  'print unless(/(setup)|(plugin)|(bwig)|(libgtftk.py)|(__)/)'`; do echo "================="; echo $$i; nosetests --with-doctest $$i;   done
+	@cd /tmp; mkdir -p gtftk_test; cd gtftk_test; a=`python -c "import os,pygtftk; print(os.path.dirname(pygtftk.__file__))"`; cd $$a ; for i in `find . -name "*.py*" | perl -ne  'print unless(/(setup)|(plugin)|(bwig)|(libgtftk.py)|(__)/)'`; do echo "================="; echo $$i; nosetests --with-doctest $$i --doctest-extension=pyx;   done
 
 nose_travis:
-	@ source activate pygtftk_py3k; mkdir -p ~/tmp; cd ~/tmp ; mkdir -p gtftk_test; cd gtftk_test; a=`python -c "import os,pygtftk; print(os.path.dirname(pygtftk.__file__))"`; echo $$a; cd $$a ; for i in `find . -name "*.py" | perl -ne  'print unless(/(setup)|(plugin)||(bwig)|(libgtftk.py)|(__)/)'`; do echo "================="; echo $$i; nosetests --with-doctest $$i;   done
+	@ source activate pygtftk_py3k; mkdir -p ~/tmp; cd ~/tmp ; mkdir -p gtftk_test; cd gtftk_test; a=`python -c "import os,pygtftk; print(os.path.dirname(pygtftk.__file__))"`; echo $$a; cd $$a ; for i in `find . -name "*.py*" | perl -ne  'print unless(/(setup)|(plugin)||(bwig)|(libgtftk.py)|(__)/)'`; do echo "================="; echo $$i; nosetests --with-doctest $$i --doctest-extension=pyx;   done
 
 
 install:
@@ -121,7 +122,7 @@ test_para_travis: $(OUTPUT4)
 
 clean:
 	@make bats_cmd CMD=clean
-	@git checkout docs/source/conf.py pygtftk/version.py; rm -rf mk_matrix_6 expected_s* ids* diff_fasta.py chr1_hg38_10M.fa* observed_s* order_fasta.py simple* control_list_reference.txt control_list_data.txt add_attr_to_pos.tab test.py pygtftk.egg-info build airway_love.txt* ENCFF630HEX_Total_RNAseq_K562_count_mini.txt STDIN.e* closest_1.tsv STDIN.o* dist cmd_list.txt example_list.txt tmp_list.txt simple.chromInfo prgm_list.txt test_list.txt *.bats *.completed *mini_real* heatmap_* tx_classes* *~ \#* hh profile_* toto tott;  cd docs/; make clean; cd ..; find . -type f -name '*~' -exec rm -f '{}' \;
+	@git checkout docs/source/conf.py pygtftk/version.py; rm -rf mk_matrix_6 ologram_output* expected_s* ids* diff_fasta.py chr1_hg38_10M.fa* observed_s* order_fasta.py simple* control_list_reference.txt control_list_data.txt add_attr_to_pos.tab test.py pygtftk.egg-info build airway_love.txt* ENCFF630HEX_Total_RNAseq_K562_count_mini.txt STDIN.e* closest_1.tsv STDIN.o* dist cmd_list.txt example_list.txt tmp_list.txt simple.chromInfo prgm_list.txt test_list.txt *.bats *.completed *mini_real* heatmap_* tx_classes* *~ \#* hh profile_* toto tott;  cd docs/; make clean; cd ..; find . -type f -name '*~' -exec rm -f '{}' \;
 
 check_cmd_has_example:
 	@for i in $$(gtftk -l); do if grep -q  "^$$i" docs/source/*.rst; then echo "" >/dev/null; else echo $$i; fi; done
@@ -134,8 +135,105 @@ check_example_has_cmd:
 
 nb_test:
 	@gtftk -p| perl -ne 'BEGIN{$$/="{"}{/\@test\s+"(\w+)_\d+"/; print $$1,"\n"}'| sort | uniq -c | sort -nr
+
+
+
+#------------------------------------------------------------------
+# Creating a release
+#------------------------------------------------------------------
+
+
 release:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Starting the release $(VER)                   #"
+	@ echo "#-----------------------------------------------#"
 	@touch release_in_progress
 
+release_bump: release
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Bumping the program version                   #"
+	@ echo "#-----------------------------------------------#"
+	@ git checkout docs/source/conf.py
+	@ git checkout setup.cfg
+	@ git checkout pygtftk/version.py
+	@ python setup.py install
+	@ cat pygtftk/version.py | perl -npe "s/='(.*)'/='$(VER)'/" > /tmp/pygtftk.bump
+	@ mv /tmp/pygtftk.bump pygtftk/version.py
+	@ cat setup.cfg | perl -npe 's/^version = .*/version = $(VER)/' > /tmp/pygtftk.bump
+	@ mv /tmp/pygtftk.bump setup.cfg
+	@ cat docs/source/conf.py | perl -npe "s/version = u'\d+\.\d+\.\d+'$$/version = u'$(VER)'/" | perl -npe "s/release = u'\d+\.\d+\.\d+'$$/release = u'$(VER)'/"  > /tmp/pygtftk.bump
+	@ mv /tmp/pygtftk.bump docs/source/conf.py
+	@ echo "Version was bump to $(VER)"
+	@ make install
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Check gtftk version                           #"
+	@ echo "#-----------------------------------------------#"
+	@ echo `gtftk -v`
+	@ git add docs/source/conf.py pygtftk/version.py setup.cfg
+	@ git commit -m 'Bumped version $(VER)'
+
+release_test:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Performing tests (bats)                       #"
+	@ echo "#-----------------------------------------------#"
+	@ make test_para -j 6
+
+release_nose:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Performing tests  (nose)                      #"
+	@ echo "#-----------------------------------------------#"
+	@ make nose
+
+release_doc:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Building doc                                  #"
+	@ echo "#-----------------------------------------------#"
+	@ make doc
+	@ git pull
+	@ git add docs/source/example*png
+	@ git add docs/source/example*pdf
+	@ cp docs/source/example*png docs/source/_static
+	@ cp docs/source/example*pdf docs/source/_static
+	@ git add docs/source/_static/*png
+	@ git add docs/source/_static/*pdf
+	@ git add docs/source/example*png
+	@ git add docs/source/example*pdf
+	@ git commit -m "Updated img in source and source/_static "
+	@ git push
+
+
+release_pip_unix:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Creating manylinux compliant package (pip)    #"
+	@ echo "#-----------------------------------------------#"
+	@rm -rf /tmp/tmp                                            ; \
+ 	rm -f manylinux/pygtftk-*whl                                ; \
+	cd manylinux                                                ; \
+	docker rmi -f manylinux                                     ; \
+	docker stop imanylinux  || true && docker rm -f imanylinux || true  ; \                                      ;\
+	docker build -t manylinux .                                 ; \
+	docker create  -t --name imanylinux  manylinux /bin/bash    ; \
+	docker cp  imanylinux:/tmp/ /tmp                            ; \
+	rm -rf ../wheels                                            ; \
+	mkdir -p ../wheels                                          ; \
+	cp /tmp/tmp/wheelhouse_manylinux/pygtftk-*whl ../wheels     ; \
+	cp /tmp/tmp/log ../wheels/log_unix.txt                      ; \
+	echo "Manylinux wheels should be in wheels folder."         ; \
+	echo "Have a look at log in wheels/log and upload user twine if OK."
+
+release_pip_osx:
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Creating osx compliant package (pip)          #"
+	@ echo "#-----------------------------------------------#"
+	@rm -rf dist build; python setup.py bdist_wheel             ; \
+	cd dist                                                     ; \
+	mv *whl ../wheels                                           ; \
+	cd ..; rm -rf dist build
+
+release_pip: release_pip_unix release_pip_osx
+	@ echo "#-----------------------------------------------#"
+	@ echo "# Creating unix/osx pip compliant package       #"
+	@ echo "#-----------------------------------------------#"
+
 unrelease:
 	@rm -f release_in_progress

README.rst

@@ -104,7 +104,8 @@ Installation through pip can be done as follow. ::
     pip install -r requirements.txt
     pip install pygtftk
     # It is important to call gtftk -h
-    # to find and dump plugin parsers
+    # to look for plugins and their
+    # CLI in ~/.gtftk
     # before going further
     gtftk -h     
 

conda/env.yaml

@@ -25,4 +25,4 @@ dependencies:
 - matplotlib >=2.0.2
 - plotnine >=0.4.0
 - pyBigWig >=0.3.12
-- nose
+- cython

conda/env_read_the_doc.yaml

@@ -17,3 +17,4 @@ dependencies:
     - pyyaml
     - nose
     - numpy >=1.10.0
+    - cython

conda/meta.yaml

@@ -37,8 +37,8 @@ requirements:
   - biopython >=1.69
   - matplotlib >=2.0.2
   - plotnine >=0.4.0
-  - future
   - nose
+  - cython
 
 test:
   imports:
@@ -55,4 +55,3 @@ about:
   home: http://github.com/dputhier/pygtftk
   license: MIT
   summary: 'The gtftk suite providing facilities to manipulate genomic annotations in gtf format.'
-

docs/source/annotation.rst

@@ -125,3 +125,123 @@ intron_sizes
 
 .. command-output:: gtftk intron_sizes -h
 	:shell:
+
+
+------------------------------------------------------------------------------------------------------------------
+
+
+ologram
+~~~~~~~~~~~~~~~~~~~~~~
+
+**Description:** Annotate peaks (in bed format) with region sets computed on the fly from a GTF file  (e.g promoter, tts, gene body, UTR...). By default, the midpoint of
+ each peak is considered and intersected iteratively with region sets. A binomial p-value is computed based on hypothesized probability of success p (fraction of genome covered by the
+ feature f), the number of trials (number of peaks) and the number of successes (number of intersections).
+
+
+We will first request a lightweight example dataset.
+
+
+.. command-output:: gtftk get_example -d mini_real -f '*'
+	:shell:
+
+
+**Example:** Perform a basic annotation. We are searching whether H3K4me3 peaks tends to be enriched in some specific genomic elements.
+
+
+.. command-output:: gtftk get_example -d mini_real | gtftk ologram -p ENCFF112BHN_H3K4me3_K562_sub.bed -c hg38.genome -u 1500 -d 1500 -D  -if example_pa_01.pdf -k 4
+	:shell:
+
+Download :download:`pdf <_static/example_pa_01.pdf>`.
+
+
+
+.. raw:: html
+
+  <br>
+  <table>
+  <tr>
+  <td valign="top">
+  <iframe src="_static/example_pa_01.pdf" title="your_title" align="top" width="500" height="620" width="50%" frameborder="0" scrolling="auto" target="Message">
+  </iframe>
+  </td>
+  </tr>
+  </table>
+  <br>
+  <br>
+
+**Example:** Now we are using the gene_biotype key (note that a list of keys can be provided). This will tell us whether H3K4me3 tends to be located in particular transcripts (protein coding, LncRNAs...). The --no-basic-feature argument tells ologram not to test basic genomic elements (gene, transcripts...).
+
+.. command-output:: gtftk get_example -d mini_real | gtftk ologram -m gene_biotype -p ENCFF112BHN_H3K4me3_K562_sub.bed -c hg38.genome -D -n  -if example_pa_02.pdf
+	:shell:
+
+Download :download:`pdf <_static/example_pa_02.pdf>`.
+
+
+
+.. raw:: html
+
+  <br>
+  <table>
+  <tr>
+  <td valign="top">
+  <iframe src="_static/example_pa_02.pdf" title="your_title" align="top" width="500" height="620" width="50%" frameborder="0" scrolling="auto" target="Message">
+  </iframe>
+  </td>
+  </tr>
+  </table>
+  <br>
+  <br>
+
+**Example:** A more complex example where the key is created on the fly. Expression data are loaded as a novel key using the join_attr command and associated to gene features. This novel key (exprs) is then discretized to created 6 classes of genes with increasing expression (based on percentiles, -p) which are tested for enrichment in H3K4me3.
+
+.. command-output:: gtftk get_example -d mini_real |  gtftk join_attr -H -j mini_real_counts_ENCFF630HEX.tsv -k gene_name -n exprs -t gene | gtftk discretize_key -k exprs -p -d exprs_class -n 6   | gtftk ologram -p ENCFF112BHN_H3K4me3_K562_sub.bed -c hg38.genome -D -n -m exprs_class -if example_pa_03.pdf
+	:shell:
+
+Download :download:`pdf <_static/example_pa_03.pdf>`.
+
+
+
+.. raw:: html
+
+  <br>
+  <table>
+  <tr>
+  <td valign="top">
+  <iframe src="_static/example_pa_03.pdf" title="your_title" align="top" width="500" height="620" width="50%" frameborder="0" scrolling="auto" target="Message">
+  </iframe>
+  </td>
+  </tr>
+  </table>
+  <br>
+  <br>
+
+**Example:** Using the add_exon_nb, we add the exon number transcript-wise (numbering from 5' to 3') and discretize this novel key into 5 classes tested for enrichment.
+
+.. command-output:: gtftk get_example -d mini_real | gtftk add_exon_nb -k exon_nbr | gtftk discretize_key -p -d exon_nbr_cat -n 5  -k exon_nbr | gtftk ologram -p ENCFF112BHN_H3K4me3_K562_sub.bed -c hg38.genome -D -n -m exon_nbr_cat -if example_pa_04.pdf
+	:shell:
+
+Download :download:`pdf <_static/example_pa_04.pdf>`.
+
+
+
+.. raw:: html
+
+  <br>
+  <table>
+  <tr>
+  <td valign="top">
+  <iframe src="_static/example_pa_04.pdf" title="your_title" align="top" width="500" height="620" width="50%" frameborder="0" scrolling="auto" target="Message">
+  </iframe>
+  </td>
+  </tr>
+  </table>
+  <br>
+  <br>
+
+
+**Arguments:**
+
+.. command-output:: gtftk ologram -h
+	:shell:
+
+

docs/source/conf.py

@@ -55,10 +55,10 @@
 # built documents.
 #
 # The short X.Y version.
-version = u'0.9.10'
+version = u'1.0.0'
 
 # The full version, including alpha/beta/rc tags.
-release = u'0.9.10'
+release = u'1.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

manylinux/Dockerfile

@@ -0,0 +1,4 @@
+FROM quay.io/pypa/manylinux1_x86_64
+MAINTAINER dputhier
+COPY ./install.sh /
+RUN bash /install.sh

manylinux/install.sh

@@ -0,0 +1,14 @@
+echo "Creating working directory."
+mkdir -p /io && cd /io
+
+echo "Cloning git repository."
+git clone --branch develop --single-branch  https://github.com/dputhier/pygtftk.git .
+
+echo "Instructing pygtftk we are doing a release."
+touch release_in_progress
+
+echo "Building manylinux compliant pip packages."
+cd manylinux
+chmod u+x build_wheels.sh
+bash ./build_wheels.sh &> /tmp/log
+cp -r ./* /tmp

pygtftk/Line.py

@@ -9,9 +9,6 @@
 
 """
 
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
 from builtins import object
 from builtins import range
 from builtins import str