Merge pull request #394 from Nesvilab/dev

Dev

.github/workflows/go.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.18
+    - name: Set up Go 1.19
       uses: actions/setup-go@v1
       with:
-        go-version: 1.18
+        go-version: 1.19.3
       id: go
 
     - name: Check out code into the Go module directory
@@ -38,17 +38,8 @@ jobs:
         unzip -o lib/dat/bindata.go.zip -d lib/dat/
         unzip -o lib/obo/unimod/bindata.go.zip -d lib/obo/unimod/
 
-    - name: Download 01_CPTAC_TMTS1-NCI7_Z_JHUZ_20170502_LUMOS.mzML
-      run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/01_CPTAC_TMTS1-NCI7_Z_JHUZ_20170502_LUMOS.mzML
-
-    - name: Download b1906_293T_proteinID_01A_QE3_122212.mzML
-      run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/b1906_293T_proteinID_01A_QE3_122212.mzML
-
-    - name: Download b1922_293T_proteinID_02A_QE3_122212.mzML
-      run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/b1922_293T_proteinID_02A_QE3_122212.mzML
-
-    - name: Download b1923_293T_proteinID_03A_QE3_122212.mzML
-      run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/b1923_293T_proteinID_03A_QE3_122212.mzML
+    - name: Download z04397_tc-o238g-setB_MS3.mzML
+      run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/z04397_tc-o238g-setB_MS3.mzML
 
     - name: Download interact.pep.xml
       run: wget -P test/wrksp https://github.com/prvst/philosopher-test/releases/download/0.0.1/interact.pep.xml

Changelog

@@ -1,2 +1,17 @@
+### Added
+- Added  new flag `--removecontam` to the report command to remove contaminants from the reports
+- Added a new combined_psm report to abacus (beta)
+
+### Changed
+- Updated the TMT-Integrator to match with the version v4.0.0
+- Updated the pickedFDR filter
+- Replaced the cRAP contaminant database with the Universal Protein Contaminants from Frankenfield et al. (10.1021/acs.jproteome.2c00145)
+- The pipeline command now accepts annotation files with prefixes
+- Updated PTMProphet to the latest release
+- Updated the database fecth with the new UniProt API (The new API is still unstable, people might experience hicups with certain cases, specially non-model organisms without reviewed sequences)
+
 ### Fixed
-- Issue with the database command only fetching the human FASTA.
+- Contaminant tags are now added to Human sequences as well
+- Several minor fixes and updates
+- Added a fix to fix errorrs causes by division-by-zero errors (PR by @chhh)
+- Fixed the protein coverage calculations to take into acount all peptides

Dockerfile

@@ -5,7 +5,7 @@ LABEL version="1"
 LABEL software="Philosopher"
 LABEL description="A complete toolkit for shotgun proteomics data analysis"
 LABEL website="https://philosopher.nesvilab.org"
-LABEL documentation="https://philosopher/wiki"
+LABEL documentation="https://github.com/Nesvilab/philosopher/wiki"
 LABEL license="GPL-3.0"
 LABEL BIOTOOLS=""
 LABEL tags="Proteomics"

Makefile

@@ -6,7 +6,8 @@ BINARY = philosopher
 VERSION = $(shell date +%Y%m%d)
 BUILD = $(shell  date +%Y%m%d%H%M)
 
-TAG = v4.4.0
+TAG = v4.6.0
+RC = RC-1
 
 LDFLAGS = -ldflags "-w -s -extldflags -static -X main.version=${TAG} -X main.build=${BUILD}"
 
@@ -23,9 +24,19 @@ deploy:
 	unzip -o lib/ext/cdhit/win/bindata.go.zip -d lib/ext/cdhit/win/
 
 	unzip -o lib/ext/comet/unix/bindata.go.zip -d lib/ext/comet/unix/
-	unzip -o lib/ext/comet/win/bindata.go.zip -d lib/ext/comet/win/rm -rf /usr/local/go && tar -C /usr/local -xzf go1.18.2.linux-amd64.tar.gz
-	unzip -o lib/ext/rawfilereader/reh64/bindata.go.zip -d lib/ext/rawfilereader/reh64/
-	unzip -o lib/ext/rawfilereader/win/bindata.go.zip -d lib/ext/rawfilereader/win/
+	unzip -o lib/ext/comet/win/bindata.go.zip -d lib/ext/comet/win/
+
+	unzip -o lib/ext/interprophet/unix/bindata.go.zip -d lib/ext/interprophet/unix/
+	unzip -o lib/ext/interprophet/win/bindata.go.zip -d lib/ext/interprophet/win/
+
+	unzip -o lib/ext/peptideprophet/unix/bindata.go.zip -d lib/ext/peptideprophet/unix/
+	unzip -o lib/ext/peptideprophet/win/bindata.go.zip -d lib/ext/peptideprophet/win/
+
+	unzip -o lib/ext/proteinprophet/unix/bindata.go.zip -d lib/ext/proteinprophet/unix/
+	unzip -o lib/ext/proteinprophet/win/bindata.go.zip -d lib/ext/proteinprophet/win/
+
+	unzip -o lib/ext/ptmprophet/unix/bindata.go.zip -d lib/ext/ptmprophet/unix/
+	unzip -o lib/ext/ptmprophet/win/bindata.go.zip -d lib/ext/ptmprophet/win/
 
 	unzip -o lib/pip/bindata.go.zip -d  lib/pip/
 
@@ -39,23 +50,25 @@ test:
 
 .PHONY: rc
 rc:
-	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}-RC
-	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}-RC
-	mv philosopher-${TAG}-RC ~/bin/
-	mv philosopher-${TAG}-RC.exe ~/bin/
+	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}-${RC}
+	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}-${RC}
+	mv philosopher-${TAG}-${RC} ~/bin/
+	mv philosopher-${TAG}-${RC}.exe ~/bin/
 
 .PHONY: linux
 linux:
-	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher
+	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}
+	mv philosopher-${TAG} ~/bin/
 
 .PHONY: windows
 windows:
-	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher
+	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}
+	mv philosopher-${TAG}-${RC}.exe ~/bin/
 
 .PHONY: all
 all:
-	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher
-	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher
+	env CGO_ENABLED=0 gox -os="linux" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}
+	env CGO_ENABLED=0 gox -os="windows" ${LDFLAGS} -arch=amd64 -output philosopher-${TAG}
 
 .PHONY: push
 push:
@@ -64,8 +77,8 @@ push:
 
 .PHONY: draft
 draft:
-	goreleaser --skip-publish --snapshot --release-notes=Changelog --rm-dist
+	goreleaser --skip-publish --snapshot --release-notes=Changelog
 
 .PHONY: release
 release:
-	goreleaser --release-notes=Changelog --rm-dist
+	goreleaser --release-notes=Changelog

README.md

@@ -14,7 +14,7 @@
 
 - Database downloading and formatting.
 
-- Peptide-spectrum matching with Comet.
+- Peptide-spectrum matching with MSFragger and Comet.
 
 - Peptide assignment validation with PeptideProphet.
 

cmd/abacus.go

@@ -45,6 +45,7 @@ func init() {
 		m.Restore(sys.Meta())
 
 		abacusCmd.Flags().StringVarP(&m.Abacus.Tag, "tag", "", "rev_", "decoy tag")
+		abacusCmd.Flags().StringVarP(&m.Abacus.Plex, "plex", "", "10", "number of channels")
 		abacusCmd.Flags().Float64VarP(&m.Abacus.ProtProb, "prtProb", "", 0.9, "minimum protein probability")
 		abacusCmd.Flags().Float64VarP(&m.Abacus.PepProb, "pepProb", "", 0.5, "minimum peptide probability")
 		abacusCmd.Flags().BoolVarP(&m.Abacus.Protein, "protein", "", false, "global level protein report")
@@ -54,7 +55,7 @@ func init() {
 		abacusCmd.Flags().BoolVarP(&m.Abacus.Unique, "uniqueonly", "", false, "report TMT quantification based on only unique peptides")
 		abacusCmd.Flags().BoolVarP(&m.Abacus.Labels, "labels", "", false, "indicates whether the data sets includes TMT labels or not")
 		abacusCmd.Flags().BoolVarP(&m.Abacus.Reprint, "reprint", "", false, "create abacus reports using the Reprint format")
-		abacusCmd.Flags().BoolVarP(&m.Abacus.Full, "full", "", true, "generates combined tables with extra information")
+		abacusCmd.Flags().BoolVarP(&m.Abacus.Full, "full", "", false, "generates combined tables with extra information")
 	}
 
 	RootCmd.AddCommand(abacusCmd)

cmd/filter.go

@@ -69,6 +69,7 @@ func init() {
 		filterCmd.Flags().Float64VarP(&m.Filter.PepProb, "pepProb", "", 0.7, "top peptide probability threshold for the FDR filtering")
 		filterCmd.Flags().Float64VarP(&m.Filter.ProtProb, "protProb", "", 0.5, "protein probability threshold for the FDR filtering (not used with the razor algorithm)")
 		filterCmd.Flags().Float64VarP(&m.Filter.Weight, "weight", "", 1, "threshold for defining peptide uniqueness")
+		filterCmd.Flags().BoolVarP(&m.Filter.Delta, "delta", "", false, "applies a stratification to PSMs based on Delta mass profile")
 		filterCmd.Flags().BoolVarP(&m.Filter.Seq, "sequential", "", false, "alternative algorithm that estimates FDR using both filtered PSM and protein lists")
 		filterCmd.Flags().BoolVarP(&m.Filter.TwoD, "2d", "", false, "two-dimensional FDR filtering")
 		filterCmd.Flags().BoolVarP(&m.Filter.Model, "models", "", false, "print model distribution")
@@ -77,6 +78,7 @@ func init() {
 		filterCmd.Flags().BoolVarP(&m.Filter.Mapmods, "mapmods", "", false, "map modifications")
 		filterCmd.Flags().BoolVarP(&m.Filter.Inference, "inference", "", false, "extremely fast and efficient protein inference compatible with 2D and Sequential filters")
 		filterCmd.Flags().MarkHidden("mods")
+		filterCmd.Flags().MarkHidden("delta")
 		filterCmd.Flags().MarkHidden("razorbin")
 	}
 

cmd/report.go

@@ -41,9 +41,11 @@ func init() {
 		m.Restore(sys.Meta())
 
 		reportCmd.Flags().BoolVarP(&m.Report.Decoys, "decoys", "", false, "add decoy observations to reports")
+		reportCmd.Flags().BoolVarP(&m.Report.RemoveContam, "removecontam", "", false, "remove contaminant sequences from the reports")
 		reportCmd.Flags().BoolVarP(&m.Report.MSstats, "msstats", "", false, "create an output compatible with MSstats")
 		reportCmd.Flags().BoolVarP(&m.Report.MZID, "mzid", "", false, "create a mzID output")
 		reportCmd.Flags().BoolVarP(&m.Report.IonMob, "ionmobility", "", false, "forces the printing of the ion mobility column")
+		reportCmd.Flags().BoolVarP(&m.Report.Prefix, "prefix", "", false, "add the project (folder) name as a prefix to the output files")
 	}
 
 	RootCmd.AddCommand(reportCmd)

go.sum

@@ -200,8 +200,6 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

lib/aba/aba.go

@@ -2,13 +2,7 @@
 package aba
 
 import (
-	"bufio"
-	"errors"
-	"os"
-	"strings"
-
 	"philosopher/lib/met"
-	"philosopher/lib/msg"
 )
 
 // DataSetLabelNames maps all custom names to each TMT tags
@@ -21,9 +15,11 @@ type DataSetLabelNames struct {
 // TODO update error methos on the abacus function
 func Run(m met.Data, args []string) {
 
-	if !m.Abacus.Peptide && !m.Abacus.Protein {
-		msg.Custom(errors.New("you need to specify a peptide or protein combined file for the Abacus analysis"), "fatal")
-	}
+	// if !m.Abacus.Peptide && !m.Abacus.Protein {
+	// 	msg.Custom(errors.New("you need to specify a peptide or protein combined file for the Abacus analysis"), "fatal")
+	// }
+
+	psmLevelAbacus(m, args)
 
 	if m.Abacus.Peptide {
 		peptideLevelAbacus(m, args)
@@ -35,25 +31,28 @@ func Run(m met.Data, args []string) {
 }
 
 // addCustomNames adds to the label structures user-defined names to be used on the TMT labels
-func getLabelNames(annot string) map[string]string {
+// func getLabelNames(dataSet, annot string) map[string]string {
 
-	var labels = make(map[string]string)
+// 	var labels = make(map[string]string)
 
-	file, e := os.Open(annot)
-	if e != nil {
-		msg.ReadFile(errors.New("cannot open annotation file"), "error")
-	}
-	defer file.Close()
+// 	file, e := os.Open(annot)
+// 	if e != nil {
+// 		msg.ReadFile(errors.New("cannot open annotation file"), "error")
+// 	}
+// 	defer file.Close()
 
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		names := strings.Split(scanner.Text(), " ")
-		labels[names[0]] = names[1]
-	}
+// 	scanner := bufio.NewScanner(file)
+// 	for scanner.Scan() {
+// 		names := strings.Fields(scanner.Text())
 
-	if e = scanner.Err(); e != nil {
-		msg.Custom(errors.New("the annotation file looks to be empty"), "fatal")
-	}
+// 		name := dataSet + " " + names[0]
 
-	return labels
-}
+// 		labels[name] = names[1]
+// 	}
+
+// 	if e = scanner.Err(); e != nil {
+// 		msg.Custom(errors.New("the annotation file looks to be empty"), "fatal")
+// 	}
+
+// 	return labels
+// }

lib/aba/pep.go

@@ -2,6 +2,7 @@
 package aba
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"io"
@@ -29,7 +30,7 @@ func peptideLevelAbacus(m met.Data, args []string) {
 	var names []string
 	//var xmlFiles []string
 	var datasets = make(map[string]rep.PSMEvidenceList)
-	var labelList []DataSetLabelNames
+	var labels = make(map[string]string)
 
 	// restoring combined file
 	logrus.Info("Processing combined file")
@@ -49,9 +50,6 @@ func peptideLevelAbacus(m met.Data, args []string) {
 		var evi rep.Evidence
 		rep.RestorePSM(&evi.PSM)
 
-		var labels DataSetLabelNames
-		labels.LabelName = make(map[string]string)
-
 		files, _ := ioutil.ReadDir(i)
 
 		// collect interact full file names
@@ -63,10 +61,24 @@ func peptideLevelAbacus(m met.Data, args []string) {
 			//}
 			if strings.Contains(f.Name(), "annotation") {
 				var annot = fmt.Sprintf("%s%s%s", i, string(filepath.Separator), f.Name())
-				labels.Name = annot
 
-				if len(m.Quantify.Annot) > 0 {
-					labels.LabelName = getLabelNames(annot)
+				file, e := os.Open(annot)
+				if e != nil {
+					msg.ReadFile(errors.New("cannot open annotation file"), "error")
+				}
+				defer file.Close()
+
+				scanner := bufio.NewScanner(file)
+				for scanner.Scan() {
+					names := strings.Fields(scanner.Text())
+
+					name := i + " " + names[0]
+
+					labels[name] = names[1]
+				}
+
+				if e = scanner.Err(); e != nil {
+					msg.Custom(errors.New("the annotation file looks to be empty"), "fatal")
 				}
 			}
 
@@ -79,8 +91,6 @@ func peptideLevelAbacus(m met.Data, args []string) {
 			prjName = strings.Replace(filepath.Base(prjName), string(filepath.Separator), "", -1)
 		}
 
-		labelList = append(labelList, labels)
-
 		// unique list and map of datasets
 		datasets[prjName] = evi.PSM
 		names = append(names, prjName)
@@ -98,7 +108,7 @@ func peptideLevelAbacus(m met.Data, args []string) {
 
 	os.Chdir(local)
 
-	savePeptideAbacusResult(m.Temp, evidences, datasets, names, m.Abacus.Unique, false, labelList)
+	savePeptideAbacusResult(m.Temp, evidences, datasets, names, m.Abacus.Unique, false, labels)
 
 }
 
@@ -119,7 +129,7 @@ func processPeptideCombinedFile(a met.Abacus) {
 		uniqPeps := fil.GetUniquePeptides(pepID)
 
 		//filteredPSMs, _ := fil.PepXMLFDRFilter(uniqPsms, 0.01, "PSM", a.Tag)
-		filteredPeptides, _ := fil.PepXMLFDRFilter(uniqPeps, 0.01, "Peptide", a.Tag)
+		filteredPeptides, _ := fil.PepXMLFDRFilter(uniqPeps, 0.01, "Peptide", a.Tag, "test")
 		filteredPeptides.Serialize("pep")
 
 	}
@@ -254,7 +264,7 @@ func SummarizeAttributes(evidences rep.CombinedPeptideEvidenceList, datasets map
 }
 
 // savePeptideAbacusResult creates a single report using 1 or more philosopher result files
-func savePeptideAbacusResult(session string, evidences rep.CombinedPeptideEvidenceList, datasets map[string]rep.PSMEvidenceList, namesList []string, uniqueOnly, hasTMT bool, labelsList []DataSetLabelNames) {
+func savePeptideAbacusResult(session string, evidences rep.CombinedPeptideEvidenceList, datasets map[string]rep.PSMEvidenceList, namesList []string, uniqueOnly, hasTMT bool, labelsList map[string]string) {
 
 	// create result file
 	output := fmt.Sprintf("%s%scombined_peptide.tsv", session, string(filepath.Separator))