Merge pull request #400 from Nesvilab/dev

Dev

Changelog

@@ -1,8 +1,12 @@
 ### Added
-
+- Adjusted the peptide-to-protein mapping due to I/L replacement rules
+- Adjusted the protein FDR caclulation rules.
 
 ### Changed
-
+- Removed the normalization done at the protein level in labelquant
+- Improved the protein filter for small scale analyses
 
 ### Fixed
-- Adjusted the msstats file format, added Purity to all plex sizes.
+- Adjusted the msstats file format, added Purity to all plex sizes
+- Fixed the protein coverage calculations
+- Fixed repeated isobaric values that would eventually show up in the protein table

Makefile

@@ -6,8 +6,8 @@ BINARY = philosopher
 VERSION = $(shell date +%Y%m%d)
 BUILD = $(shell  date +%Y%m%d%H%M)
 
-TAG = v4.6.1
-RC = RC-1
+TAG = v4.7.0
+RC = RC-6
 
 LDFLAGS = -ldflags "-w -s -extldflags -static -X main.version=${TAG} -X main.build=${BUILD}"
 

README.md

@@ -7,8 +7,7 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/Nesvilab/philosopher?style=for-the-badge&color=red&logo=appveyor)](https://goreportcard.com/report/github.com/Nesvilab/philosopher)
 ![GitHub](https://img.shields.io/github/license/Nesvilab/philosopher?style=for-the-badge)
 ![](https://img.shields.io/github/downloads/Nesvilab/philosopher/total.svg?color=red&style=for-the-badge)
-![GitHub Workflow Status](https://img.shields.io/github/workflow/status/Nesvilab/philosopher/philosopher-build?style=for-the-badge)
-<!--[![DOI](https://img.shields.io/badge/DOI-doi.org%2F10.5281%2Fzenodo.3909842-9cf?style=for-the-badge)](https://zenodo.org/record/3909842#.XvZXn5YpCAk)-->
+![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Nesvilab/philosopher/go.yml?style=for-the-badge)
 
 #### Philosopher is fast, easy-to-use, scalable, and versatile data analysis software for mass spectrometry-based proteomics. Philosopher is dependency-free and can analyze both traditional database searches and open searches for post-translational modification (PTM) discovery. 
 

cmd/inspect.go

@@ -8,10 +8,10 @@ import (
 	"path/filepath"
 
 	"philosopher/lib/dat"
-	"philosopher/lib/fil"
 	"philosopher/lib/met"
 	"philosopher/lib/msg"
 	"philosopher/lib/qua"
+	"philosopher/lib/raz"
 	"philosopher/lib/rep"
 	"philosopher/lib/sys"
 
@@ -71,7 +71,7 @@ var inspectCmd = &cobra.Command{
 			spew.Dump(o.Intensities)
 
 		case "razor":
-			var o fil.RazorMap
+			var o raz.RazorMap
 
 			target := fmt.Sprintf(".meta%srazor.bin", string(filepath.Separator))
 			sys.Restore(&o, target, false)

go.mod

@@ -3,30 +3,26 @@ module philosopher
 go 1.13
 
 require (
-	github.com/ajstarks/svgo v0.0.0-20200204031535-0cbcf57ea1d8 // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/go-resty/resty/v2 v2.7.0
-	github.com/gorilla/websocket v1.4.1 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jpillora/go-ogle-analytics v0.0.0-20161213085824-14b04e0594ef
-	github.com/jung-kurt/gofpdf v1.16.2 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/mattn/go-colorable v0.1.6
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/mattn/go-colorable v0.1.13
 	github.com/nlopes/slack v0.6.0
 	github.com/pierrre/archivefile v0.0.0-20170218184037-e2d100bc74f5
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31
 	github.com/ryanskidmore/parallel v0.0.1
 	github.com/satori/go.uuid v1.2.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/cobra v0.0.6
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/sirupsen/logrus v1.9.0
+	github.com/spf13/cobra v1.6.1
+	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/vmihailenco/msgpack/v5 v5.3.5
-	golang.org/x/exp v0.0.0-20200228211341-fcea875c7e85 // indirect
-	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
-	gonum.org/v1/netlib v0.0.0-20200229103305-d71f404090bf // indirect
-	gonum.org/v1/plot v0.7.0
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
-	gopkg.in/yaml.v2 v2.2.8
+	golang.org/x/exp v0.0.0-20221217163422-3c43f8badb15 // indirect
+	golang.org/x/image v0.2.0 // indirect
+	golang.org/x/net v0.4.0 // indirect
+	gonum.org/v1/plot v0.12.0
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/yaml.v2 v2.4.0
 )

lib/dat/dat_test.go

@@ -73,8 +73,8 @@ func TestBase_ProcessDB(t *testing.T) {
 			}
 			d.ProcessDB(tt.args.file, tt.args.decoyTag)
 
-			if len(d.Records) != 20385 {
-				t.Errorf("Number of FASTA entries is incorrect, got %d, want %d", len(d.Records), 20383)
+			if len(d.Records) != 20389 {
+				t.Errorf("Number of FASTA entries is incorrect, got %d, want %d", len(d.Records), 20389)
 			}
 		})
 	}

lib/fil/fdr.go

@@ -10,6 +10,7 @@ import (
 	"philosopher/lib/cla"
 	"philosopher/lib/id"
 	"philosopher/lib/msg"
+	"philosopher/lib/raz"
 
 	"github.com/sirupsen/logrus"
 )
@@ -203,12 +204,12 @@ func PickedFDR(p id.ProtXML) id.ProtXML {
 }
 
 // RazorCandidateMap is a list of razor candidates
-type RazorCandidateMap map[string]RazorCandidate
+//type RazorCandidateMap map[string]RazorCandidate
 
 // RazorFilter classifies peptides as razor
 func RazorFilter(p id.ProtXML) id.ProtXML {
 
-	var r RazorMap = make(map[string]RazorCandidate)
+	var r raz.RazorMap = make(map[string]raz.RazorCandidate)
 	var rList []string
 
 	// perform a test load of the razor assingment, if there's a file, then the assignment is skipped, and the current file used.
@@ -223,7 +224,7 @@ func RazorFilter(p id.ProtXML) id.ProtXML {
 					v, ok := r[k.PeptideSequence]
 					if !ok {
 
-						var rc RazorCandidate
+						var rc raz.RazorCandidate
 						rc.Sequence = k.PeptideSequence
 						rc.MappedProteinsW = make(map[string]float64)
 						rc.MappedProteinsGW = make(map[string]float64)
@@ -454,18 +455,16 @@ func ProtXMLFilter(p id.ProtXML, targetFDR, pepProb, protProb float64, isPicked,
 			} else {
 
 				if isPicked {
-					if p.Groups[i].Proteins[j].Probability >= protProb && p.Groups[i].Proteins[j].Picked == 1 {
+					if p.Groups[i].Proteins[j].TopPepProb >= protProb && p.Groups[i].Proteins[j].Picked == 1 {
 						list = append(list, p.Groups[i].Proteins[j])
 					}
 
 				} else {
-					if p.Groups[i].Proteins[j].TopPepProb >= pepProb && p.Groups[i].Proteins[j].Probability >= protProb {
+					if p.Groups[i].Proteins[j].TopPepProb >= pepProb && p.Groups[i].Proteins[j].TopPepProb >= protProb {
 						list = append(list, p.Groups[i].Proteins[j])
 					}
 				}
-
 			}
-
 		}
 	}
 
@@ -482,10 +481,15 @@ func ProtXMLFilter(p id.ProtXML, targetFDR, pepProb, protProb float64, isPicked,
 	// from botttom to top, classify every protein block with a given fdr score
 	// the score is only calculates to the first (last) protein in each block
 	// proteins with the same score, get the same fdr value.
-	var scoreMap = make(map[float64]float64)
+	var FDRMap = make(map[float64]float64)
+
 	for j := (len(list) - 1); j >= 0; j-- {
 
-		scoreMap[list[j].TopPepProb] = float64(decoys) / float64(targets)
+		_, ok := FDRMap[list[j].TopPepProb]
+		if !ok {
+			FDRMap[list[j].TopPepProb] = float64(decoys) / float64(targets)
+			//fmt.Println("probability:", list[j].TopPepProb, "targets:", targets, "decoys:", decoys, "Current FDR:", uti.Round(FDRMap[list[j].TopPepProb]*100, 5, 2))
+		}
 
 		if cla.IsDecoyProtein(list[j], p.DecoyTag) {
 			decoys--
@@ -494,90 +498,76 @@ func ProtXMLFilter(p id.ProtXML, targetFDR, pepProb, protProb float64, isPicked,
 		}
 	}
 
-	var keys []float64
-	for k := range scoreMap {
-		keys = append(keys, k)
+	var topPepProb []float64
+	for k := range FDRMap {
+		topPepProb = append(topPepProb, k)
 	}
 
-	sort.Sort(sort.Reverse(sort.Float64Slice(keys)))
-
-	for i := range keys {
-		if scoreMap[keys[i]] <= targetFDR {
-			minProb = keys[i]
-			calcFDR = scoreMap[keys[i]]
-		}
-	}
+	sort.Sort(sort.Reverse(sort.Float64Slice(topPepProb)))
 
-	var curProb = 10.0
-	var curScore = 0.0
+	var currentTopPepProb = 10.0
+	var currentFDR = 0.0
 	var probArray []float64
+	var probIndex int
 	var probList = make(map[float64]uint8)
 
-	for i := range keys {
+	for i := range topPepProb {
 
 		// for inspections
-		//f := uti.Round(scoreMap[keys[i]]*100, 5, 2)
-		//fmt.Println(keys[i], "\t", scoreMap[keys[i]], "\t", uti.ToFixed(scoreMap[keys[i]], 4), "\t", f)
-		//fmt.Println(keys[i], "\t", scoreMap[keys[i]], "\t", uti.ToFixed(scoreMap[keys[i]], 4), "\t", f, "\t", targetFDR)
+		// fmt.Println(topPepProb[i], "\t", FDRMap[topPepProb[i]], "\t", uti.ToFixed(FDRMap[topPepProb[i]], 4), "\t", uti.Round(FDRMap[topPepProb[i]]*100, 5, 2))
 
-		probArray = append(probArray, keys[i])
+		probArray = append(probArray, topPepProb[i])
 
-		if scoreMap[keys[i]] <= targetFDR {
+		if FDRMap[topPepProb[i]] <= targetFDR {
 
-			probList[keys[i]] = 0
-			minProb = keys[i]
-			calcFDR = scoreMap[keys[i]]
+			probList[topPepProb[i]] = 0
+			minProb = topPepProb[i]
+			calcFDR = FDRMap[topPepProb[i]]
 
-			if keys[i] < curProb {
-				curProb = keys[i]
+			if topPepProb[i] < currentTopPepProb {
+				currentTopPepProb = topPepProb[i]
+				probIndex = i
 			}
 
-			if scoreMap[keys[i]] > curScore {
-				curScore = scoreMap[keys[i]]
+			if FDRMap[topPepProb[i]] > currentFDR {
+				currentFDR = FDRMap[topPepProb[i]]
 			}
 		}
 
 	}
 
-	if curProb == 10 {
+	if currentTopPepProb == 10 {
 		msg.Custom(errors.New("the protein FDR filter didn't reach the desired threshold, try a higher threshold using the --prot parameter"), "error")
 	}
 
-	// for inspections
-	//fmt.Println("curscore:", curScore, "\t", "fmtScore:", fmtScore, "\t", "targetfdr:", targetFDR)
-
-	if curScore < targetFDR && probArray[len(probArray)-1] != curProb {
+	// if the current protein block threshold is below the threshold, we look for the next one
+	if currentFDR < targetFDR && probArray[len(probArray)-1] != currentTopPepProb {
 
-		for i := 0; i <= len(probArray); i++ {
+		// test if the next protein block surpasses the threshold by 10%
+		thresh := currentFDR + (0.01 * currentFDR)
 
-			if probArray[i] == curProb {
+		if FDRMap[probArray[probIndex+1]] <= thresh {
 
-				if i+1 > len(probArray) {
-					msg.Custom(errors.New("the protein FDR filter didn't reach the desired threshold, try a higher threshold using the --prot parameter"), "warning")
-					break
-				}
+			probList[probArray[probIndex+1]] = 0
 
-				probList[probArray[i+1]] = 0
-				minProb = probArray[i+1]
-				calcFDR = scoreMap[probArray[i+1]]
-				break
-			}
+			minProb = probArray[probIndex+1]
+			calcFDR = FDRMap[probArray[probIndex+1]]
 
 		}
-
 	}
 
 	// for inspections
-	//fmt.Println("curscore:", curScore, "\t", "fmtScore:", fmtScore, "\t", "targetfdr:", targetFDR)
+	// fmt.Println("index ", probIndex, probArray[probIndex], "minProb ", minProb, "calcFDR ", calcFDR)
 
-	var cleanlist id.ProtIDList
+	var finalList id.ProtIDList
 	decoys = 0
 	targets = 0
 
 	for i := range list {
-		_, ok := probList[list[i].TopPepProb]
-		if ok {
-			cleanlist = append(cleanlist, list[i])
+		if list[i].TopPepProb >= currentTopPepProb {
+
+			finalList = append(finalList, list[i])
+
 			if cla.IsDecoyProtein(list[i], p.DecoyTag) {
 				decoys++
 			} else {
@@ -593,7 +583,7 @@ func ProtXMLFilter(p id.ProtXML, targetFDR, pepProb, protProb float64, isPicked,
 		"threshold": minProb,
 	}).Info(msg)
 
-	return cleanlist
+	return finalList
 }
 
 // sequentialFDRControl estimates FDR levels by applying a second filter where all
@@ -668,30 +658,6 @@ func twoDFDRFilter(pep id.PepIDList, pro id.ProtIDList, psm, peptide, ion float6
 	filteredIons.Serialize("ion")
 }
 
-// correctRazorAssignment updates the razor assignment for the PSMs recovered from the 2D filter
-func correctRazorAssignment(list id.PepIDList) id.PepIDList {
-
-	var rm RazorMap = make(map[string]RazorCandidate)
-	rm.Restore(false)
-
-	for i := range list {
-		if v, ok := rm[list[i].Peptide]; ok {
-
-			if list[i].Protein != v.MappedProtein {
-
-				list[i].AlternativeProteins[list[i].Protein]++
-				delete(list[i].AlternativeProteins, v.MappedProtein)
-
-				list[i].Protein = v.MappedProtein
-			}
-		}
-	}
-
-	//rm.Serialize()
-
-	return list
-}
-
 // mirrorProteinList takes a filtered list and regenerate the correspondedn decoys
 func mirrorProteinList(p id.ProtIDList, decoyTag string) id.ProtIDList {