From c968e2d28add8472a552ae73e43a260e6d457677 Mon Sep 17 00:00:00 2001 From: Dmitry Avtonomov Date: Sat, 5 Jan 2019 23:06:51 -0800 Subject: [PATCH] Fix some issues with protein prophet --- build.gradle | 40 +++--- build.xml | 101 +++++++++++++++ .../com/dmtavt/deltamass/Bundle.properties | 2 +- .../com/dmtavt/deltamass/Bundle.properties | 2 +- .../com/dmtavt/deltamass/DeltaMassInfo.java | 7 +- .../dmtavt/deltamass/logic/LogicClean.java | 119 +++++++----------- .../deltamass/logic/LogicInputFiles.java | 12 +- .../deltamass/parsers/PepxmlParser.java | 42 ++++--- .../deltamass/ui/DeltaMassOptionsForm.java | 13 ++ .../com/dmtavt/deltamass/utils/OsUtils.java | 92 ++++++++++++++ 10 files changed, 312 insertions(+), 118 deletions(-) create mode 100644 build.xml create mode 100644 src/main/java/com/dmtavt/deltamass/utils/OsUtils.java diff --git a/build.gradle b/build.gradle index 72f2e63..aabec3f 100644 --- a/build.gradle +++ b/build.gradle @@ -9,7 +9,7 @@ plugins { } group = 'com.dmtavt' -version = '1.0' +version = '1.1' description = """DeltaMass - detection and visualization of common mass shifts in proteomic searches.""" @@ -29,7 +29,12 @@ jar { shadowJar { classifier = null - minimize() + minimize { + exclude(dependency('javax.xml.bind:jaxb-api:.*')) + exclude(dependency('org.glassfish.jaxb:jaxb-runtime:.*')) + exclude(dependency('javax.activation:javax.activation-api:.*')) + exclude(dependency('com.sun.istack:istack-commons-runtime:.*')) + } } launch4j { @@ -46,19 +51,19 @@ repositories { } dependencies { // our projects - implementation("com.github.chhh:dmtavt-utils:0.4.4") - implementation("com.github.chhh:msftbx:1.8.6") - implementation("com.github.chhh:ptm-factory:1.0") - implementation("edu.umich.andykong:mscalibrator:1.1") + compile("com.github.chhh:dmtavt-utils:0.4.4") + compile("com.github.chhh:msftbx:1.8.6") + compile("com.github.chhh:ptm-factory:1.0") + compile("edu.umich.andykong:mscalibrator:1.1") // basic - implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' - implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' - implementation group: 'com.beust', name: 'jcommander', version: '1.72' + compile group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.3' + compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' + compile group: 'com.beust', name: 'jcommander', version: '1.72' // commons - implementation group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' - implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.8.1' + compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' + compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.8.1' compile 'commons-io:commons-io:2.6' // other @@ -67,17 +72,22 @@ dependencies { compile 'com.miglayout:miglayout-swing:5.2' compile 'org.jfree:jfreechart:1.5.0' - implementation group: 'com.edwardraff', name: 'JSAT', version: '0.0.9' - implementation(group: 'com.github.haifengl', name: 'smile-core', version: '1.5.2') { + compile(group: 'com.edwardraff', name: 'JSAT', version: '0.0.9') + compile(group: 'com.github.haifengl', name: 'smile-core', version: '1.5.2') { exclude(module: 'smile-data') exclude(module: 'smile-graph') } + // JAXB stuff for parsing mods from databases + compile 'javax.xml.bind:jaxb-api:2.3.1' + compile 'org.glassfish.jaxb:jaxb-runtime:2.3.1' + compile 'javax.activation:javax.activation-api:1.2.0' + // downloaded / precompiled jars compile fileTree(dir: 'libs', include: '*.jar') // test - testImplementation(group: 'org.apache.maven', name: 'maven-model', version: '3.6.0') - testImplementation group: 'junit', name: 'junit', version: '4.12' + testCompile(group: 'org.apache.maven', name: 'maven-model', version: '3.6.0') + testCompile group: 'junit', name: 'junit', version: '4.12' testCompile("org.assertj:assertj-core:3.11.1") } diff --git a/build.xml b/build.xml new file mode 100644 index 0000000..5bc5c76 --- /dev/null +++ b/build.xml @@ -0,0 +1,101 @@ + + + + + + + + + + + Builds, tests, and runs the project DeltaMass. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/master/src/main/java/com/dmtavt/deltamass/Bundle.properties b/master/src/main/java/com/dmtavt/deltamass/Bundle.properties index a112905..4f8c8d0 100644 --- a/master/src/main/java/com/dmtavt/deltamass/Bundle.properties +++ b/master/src/main/java/com/dmtavt/deltamass/Bundle.properties @@ -1 +1 @@ -deltamass.version.current=0.1-SNAPSHOT +deltamass.version.current=1.1 diff --git a/src/main/java/com/dmtavt/deltamass/Bundle.properties b/src/main/java/com/dmtavt/deltamass/Bundle.properties index 266e39b..35438c1 100644 --- a/src/main/java/com/dmtavt/deltamass/Bundle.properties +++ b/src/main/java/com/dmtavt/deltamass/Bundle.properties @@ -1 +1 @@ -deltamass.version.current=1.0 +deltamass.version.current=1.1 diff --git a/src/main/java/com/dmtavt/deltamass/DeltaMassInfo.java b/src/main/java/com/dmtavt/deltamass/DeltaMassInfo.java index a99ee42..bd49eb1 100644 --- a/src/main/java/com/dmtavt/deltamass/DeltaMassInfo.java +++ b/src/main/java/com/dmtavt/deltamass/DeltaMassInfo.java @@ -20,15 +20,14 @@ public final class DeltaMassInfo { private static final EventBus bus = EventBus.getDefault(); public static final String Name = "DeltaMass"; - public static final String Ver = "1.0"; + public static final String Ver = "1.1"; private static final String PROP_VER = "deltamass.version.current"; private static final String PROP_DOWNLOAD_URL = "deltamass.download.url"; private static final String DEFAULT_DOWNLOAD_URL = "https://github.com/chhh/deltamass/releases"; - private static final String BUNDLE_PATH = "com/dmtavt/deltamass/Bundle"; + private static final String BUNDLE_PATH = "com/dmtavt/deltamass/Bundle.properties"; private static final String BUNDLE_URL = - "https://raw.githubusercontent.com/chhh/deltamass/master/src/main/java/" + BUNDLE_PATH - + ".properties"; + "https://raw.githubusercontent.com/chhh/deltamass/master/src/main/java/" + BUNDLE_PATH; private static final URI BUNDLE_URI = URI.create(BUNDLE_URL); private DeltaMassInfo() { diff --git a/src/main/java/com/dmtavt/deltamass/logic/LogicClean.java b/src/main/java/com/dmtavt/deltamass/logic/LogicClean.java index c428d78..c16b564 100644 --- a/src/main/java/com/dmtavt/deltamass/logic/LogicClean.java +++ b/src/main/java/com/dmtavt/deltamass/logic/LogicClean.java @@ -25,37 +25,45 @@ public LogicClean(CommandClean cmd) { public void run() { log.info("Running cleanup"); - if (cmd.dryRun) log.info("Dry-run, won't take any action."); - int stepsPerformed = 0; + final List scheduledForDeletion = getScheduledForDeletion(); - if (!cmd.noPep) { - stepsPerformed++; - log.info("Cleaning up peptide identification files."); - DeletionStats stats = deleteMatchingFilesInSubtrees(cmd.inputFiles, - CacheLocator::isPepCacheFile, cmd.dryRun); - log.info("Pep ID cleanup summary: scheduled: {}, deleted: {}, skipped: {}, error: {}", - stats.scheduled, stats.deleted, stats.skipped.size(), stats.error.size()); - } + final String list = scheduledForDeletion.stream().map(Path::toString) + .collect(Collectors.joining("\n")); + log.info("Files to delete:\n" + list); - if (!cmd.noLcms) { - stepsPerformed++; - log.info("Cleaning up LC-MS calibration files."); - DeletionStats stats = deleteMatchingFilesInSubtrees(cmd.inputFiles, - CacheLocator::isCalCacheFile, cmd.dryRun); - log.info("Calibration cleanup summary: scheduled: {}, deleted: {}, skipped: {}, error: {}", - stats.scheduled, stats.deleted, stats.skipped.size(), stats.error.size()); + if (cmd.dryRun) { + log.info("Dry-run, won't take any action."); + return; } - if (stepsPerformed == 0) log.info("You made it clear that nothing should be cleaned up. Doing nothing."); + DeletionStats stats = delete(scheduledForDeletion); + log.info("Cleanup: deleted {}/{}, errors: {}", stats.deleted.size(), stats.scheduled.size(), stats.error.size()); + } + + private static List modifyInputPathsForCacheDeletion(List paths) { + List possibleCachePaths = new ArrayList<>(); + for (Path inputFile : paths) { + if (Files.isDirectory(inputFile)) { + possibleCachePaths.add(inputFile); + } else { + Path parent = inputFile.getParent(); + Path fn = inputFile.getFileName(); + possibleCachePaths.add(parent.resolve(fn.toString() + CacheLocator.CACHE_EXT_PEP)); + possibleCachePaths.add(parent.resolve(fn.toString() + CacheLocator.CACHE_EXT_CAL)); + } + } + return possibleCachePaths; } public List getScheduledForDeletion() { List delete = new ArrayList<>(); + final List putativeCachePaths = modifyInputPathsForCacheDeletion(cmd.inputFiles); + if (!cmd.noPep) { - delete.addAll(getScheduledForDeletion(cmd.inputFiles, CacheLocator::isPepCacheFile)); + delete.addAll(getScheduledForDeletion(putativeCachePaths, CacheLocator::isPepCacheFile)); } if (!cmd.noLcms) { - delete.addAll(getScheduledForDeletion(cmd.inputFiles, CacheLocator::isCalCacheFile)); + delete.addAll(getScheduledForDeletion(putativeCachePaths, CacheLocator::isCalCacheFile)); } return delete; } @@ -89,7 +97,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { log.warn("Error collecting files for deletion in subtree: {}" + path, e); } } else { - if (condition.test(path)) { + if (Files.exists(path) && condition.test(path)) { delete.add(path); } } @@ -97,65 +105,26 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { return new ArrayList<>(delete); } - private class DeletionStats { - int scheduled = 0; - int deleted = 0; - ConcurrentLinkedQueue skipped = new ConcurrentLinkedQueue<>(); + private static class DeletionStats { + ConcurrentLinkedQueue scheduled = new ConcurrentLinkedQueue<>(); + ConcurrentLinkedQueue deleted = new ConcurrentLinkedQueue<>(); ConcurrentLinkedQueue error = new ConcurrentLinkedQueue<>(); } - private DeletionStats deleteMatchingFilesInSubtrees(Iterable paths, Predicate condition, boolean dryRun) { - final ConcurrentLinkedQueue delete = new ConcurrentLinkedQueue<>(); - final DeletionStats stats = new DeletionStats(); - - for (Path path : paths) { - if (Files.isDirectory(path)) { - try { - Files.walkFileTree(path, new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { - if (!Files.isReadable(dir)) return FileVisitResult.SKIP_SUBTREE; - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - if (!Files.isWritable(file)) { - log.warn("File not writeable, skipping: {}", file); - stats.skipped.add(file); - return FileVisitResult.CONTINUE; - } - if (condition.test(file)) delete.add(file); - return FileVisitResult.CONTINUE; - } - }); - } catch (IOException e) { - log.error("Error collecting files for deletion in subtree: {}" + path, e); - } - } else { - if (CacheLocator.isPepCacheFile(path)) - delete.add(path); - } - } - - if (!delete.isEmpty()) { - log.info("Files scheduled for deletion:\n {}", - delete.stream().map(Path::toString).collect(Collectors.joining("\n "))); - } else { - log.info("Found no files to delete."); - } - stats.scheduled = delete.size(); - - if (!dryRun) delete.forEach(path -> { + public static DeletionStats delete(List pathsToDelete) { + DeletionStats ds = new DeletionStats(); + ds.scheduled.addAll(pathsToDelete); + for (Path path : pathsToDelete) { try { - Files.deleteIfExists(path); - stats.deleted++; + if (Files.deleteIfExists(path)) { + ds.deleted.add(path); + } } catch (IOException e) { - log.error("Could not delete file: " + path, e); - stats.error.add(path); + log.error("Error deleting file: " + path.toString() + "\nDetails:\n" + e.getMessage()); + ds.error.add(path); } - }); - - return stats; + } + return ds; } + } diff --git a/src/main/java/com/dmtavt/deltamass/logic/LogicInputFiles.java b/src/main/java/com/dmtavt/deltamass/logic/LogicInputFiles.java index c454f78..dac1463 100644 --- a/src/main/java/com/dmtavt/deltamass/logic/LogicInputFiles.java +++ b/src/main/java/com/dmtavt/deltamass/logic/LogicInputFiles.java @@ -259,6 +259,11 @@ private List inputFiles(List inputPaths, final Pattern regex, boolea return matched; } + private boolean isPepFileSupported(Path pepFilePath) { + IParserFactory parser = PepidParserRegistry.find(pepFilePath); + return parser != null; + } + private List collectSupportedFiles(Iterable paths, boolean recursive) { final ConcurrentLinkedQueue supported = new ConcurrentLinkedQueue<>(); @@ -280,8 +285,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { - IParserFactory parser = PepidParserRegistry.find(file); - if (parser != null) { + if (isPepFileSupported(file)) { supported.add(file); } return FileVisitResult.CONTINUE; @@ -292,8 +296,10 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { log.warn("Error collecting files for deletion in subtree: {}" + path, e); } } else { - if (CacheLocator.isPepCacheFile(path)) + // given a regular file, not a directory + if (isPepFileSupported(path)) { supported.add(path); + } } } return new ArrayList<>(supported); diff --git a/src/main/java/com/dmtavt/deltamass/parsers/PepxmlParser.java b/src/main/java/com/dmtavt/deltamass/parsers/PepxmlParser.java index b1b1bed..6c82308 100644 --- a/src/main/java/com/dmtavt/deltamass/parsers/PepxmlParser.java +++ b/src/main/java/com/dmtavt/deltamass/parsers/PepxmlParser.java @@ -156,7 +156,6 @@ public PepSearchFile parse(final Pattern decoyRegex) throws IOException { if (charge != 0) { // TODO: is this value really not present in pepxml and need to be computed? spm.mzObs = (precMassZ0 + charge * PROTON_MASS) / charge; - int a = 1; } spm.spectrumId = spectrumId; spm.seq = sh.getPeptide(); @@ -206,23 +205,28 @@ public PepSearchFile parse(final Pattern decoyRegex) throws IOException { + "spectrum query `%s`", i, sq.getSpectrum())); } } - if (pepProphIdx > -1) { - Object analysis = sh.getAnalysisResult().stream() - .flatMap(analysisResult -> analysisResult.getAny().stream()) - .filter(o -> o instanceof PeptideprophetResult).findAny() - .orElseThrow(() -> new IllegalStateException(String.format( - "PeptideProphet score could not be found in file `%s`, SpectrumQuery `%s`", - path, sq.getSpectrum()))); - spm.scores[pepProphIdx] = ((PeptideprophetResult) analysis).getProbability(); - } - if (iProphIdx> -1) { - Object analysis = sh.getAnalysisResult().stream() - .flatMap(analysisResult -> analysisResult.getAny().stream()) - .filter(o -> o instanceof InterprophetResult).findAny() - .orElseThrow(() -> new IllegalStateException(String.format( - "iProphet score could not be found in file `%s`, SpectrumQuery `%s`", path, - sq.getSpectrum()))); - spm.scores[iProphIdx] = ((InterprophetResult) analysis).getProbability(); + try { + if (pepProphIdx > -1) { + Object analysis = sh.getAnalysisResult().stream() + .flatMap(analysisResult -> analysisResult.getAny().stream()) + .filter(o -> o instanceof PeptideprophetResult).findAny() + .orElseThrow(() -> new IllegalStateException(String.format( + "PeptideProphet score could not be found in file `%s`, SpectrumQuery `%s`", + path, sq.getSpectrum()))); + spm.scores[pepProphIdx] = ((PeptideprophetResult) analysis).getProbability(); + } + if (iProphIdx > -1) { + Object analysis = sh.getAnalysisResult().stream() + .flatMap(analysisResult -> analysisResult.getAny().stream()) + .filter(o -> o instanceof InterprophetResult).findAny() + .orElseThrow(() -> new IllegalStateException(String.format( + "iProphet score could not be found in file `%s`, SpectrumQuery `%s`", path, + sq.getSpectrum()))); + spm.scores[iProphIdx] = ((InterprophetResult) analysis).getProbability(); + } + } catch (IllegalStateException e) { + log.warn(e.getMessage()); + throw e; } spms.add(spm); } @@ -254,7 +258,7 @@ private HashMap mapScoreNames(List queries) thro scoreMapping.put(scoreName, scoreIndex); } - // try get peptide prophet score + // try get peptidepprophet/iprophet score List analysisResults = sh.getAnalysisResult(); for (AnalysisResult analysisResult : analysisResults) { String analysis = analysisResult.getAnalysis(); diff --git a/src/main/java/com/dmtavt/deltamass/ui/DeltaMassOptionsForm.java b/src/main/java/com/dmtavt/deltamass/ui/DeltaMassOptionsForm.java index 98fb9ba..2762be6 100644 --- a/src/main/java/com/dmtavt/deltamass/ui/DeltaMassOptionsForm.java +++ b/src/main/java/com/dmtavt/deltamass/ui/DeltaMassOptionsForm.java @@ -18,6 +18,7 @@ import com.dmtavt.deltamass.messages.MsgRunPlot; import com.dmtavt.deltamass.messages.MsgStop; import com.dmtavt.deltamass.messages.MsgVersionUpdateInfo; +import com.dmtavt.deltamass.utils.OsUtils; import com.github.chhh.utils.StringUtils; import com.github.chhh.utils.SwingUtils; import com.github.chhh.utils.ser.SwingCachePropsStore; @@ -56,6 +57,7 @@ import javax.swing.JPanel; import javax.swing.JScrollPane; import javax.swing.JTable; +import javax.swing.SwingUtilities; import javax.swing.border.TitledBorder; import javax.swing.table.DefaultTableModel; import net.java.balloontip.BalloonTip; @@ -332,11 +334,22 @@ private void init() { bus.post(new MsgStop(uiBtnStop)); }); + + bus.register(this); // Try to fetch new version info from remote. Runs in a separate thread. DeltaMassInfo.checkForNewVersions(); cache.load(this); + logSysInfo(); + } + + /** Log system info. */ + private void logSysInfo() { + final StringBuilder sb = new StringBuilder("System info:\n"); + sb.append(OsUtils.osInfo()).append("\n"); + sb.append(OsUtils.javaInfo()); + log.info(sb.toString()); } @Subscribe diff --git a/src/main/java/com/dmtavt/deltamass/utils/OsUtils.java b/src/main/java/com/dmtavt/deltamass/utils/OsUtils.java new file mode 100644 index 0000000..0c5c06b --- /dev/null +++ b/src/main/java/com/dmtavt/deltamass/utils/OsUtils.java @@ -0,0 +1,92 @@ +package com.dmtavt.deltamass.utils; + +import com.github.chhh.utils.StringUtils; +import java.util.Arrays; +import java.util.List; + +public class OsUtils { + + private OsUtils() { + } + + public static String osInfo() { + String os = OsUtils.getOsName(); + os = StringUtils.isNullOrWhitespace(os) ? "?" : os; + String arch = OsUtils.getSystemArch(); + arch = StringUtils.isNullOrWhitespace(arch) ? "?" : arch; + return "System OS: " + os + ", Architecture: " + arch; + } + + public static String javaInfo() { + List propNames = Arrays.asList( + "java.version", + "java.vm.name", + "java.vm.vendor" + ); + StringBuilder sb = new StringBuilder("Java Info: "); + for (int i = 0; i < propNames.size(); i++) { + String p = propNames.get(i); + String val = System.getProperty(p); + if (!StringUtils.isNullOrWhitespace(val)) { + sb.append(val); + } + if (i < propNames.size() - 1) { + sb.append(", "); + } + } + + return sb.toString(); + } + + public static boolean isWindows() { + String osName = System.getProperty("os.name"); + if (osName == null) { + return true; // just the default + } + return osName.toLowerCase().startsWith("win"); + } + + /** + * OS name. E.g. 'Linux' or 'Windows XP'. + * + * @return + */ + public static String getOsName() { + String osName = System.getProperty("os.name"); + return osName; + } + + /** + * Tries to determine processor architecture from system properties. + * + * @return null if could not determine. + */ + public static String getSystemArch() { + + try { + if (isWindows()) { + String wow64 = System.getenv("PROCESSOR_ARCHITEW6432"); + if (wow64 != null) { + return wow64; + } + String arch = System.getenv("PROCESSOR_ARCHITECTURE"); + if (arch != null) { + return arch; + } + + } else { + String osArch = System.getProperty("os.arch"); + if (osArch != null) { + return osArch; + } + } + } catch (IllegalArgumentException e) { + // could not map os.arch or whatever to our enum values, not a biggie + } + + // if all else fails, at least try to get the bitness of the JRE + // which might still be null + String osArch = System.getProperty("os.arch"); + return osArch; + } +}