meteor: Add some warnings around Meteor usage to help troubleshoot (#65)

* meteor: Add some warnings around Meteor usage to help troubleshoot
memory issues.
Helps with #64.
* numpy: Allow pickle.
* python 2: Lower scikit-learn version.

README.md

@@ -139,6 +139,9 @@ gives
     VectorExtremaCosineSimilarity: 0.568696
     GreedyMatchingScore: 0.784205
 
+## Troubleshooting
+If you have issues with Meteor then you can try lowering the `mem` variable in meteor.py
+
 ## Important Note ##
 CIDEr by default (with idf parameter set to "corpus" mode) computes IDF values using the reference sentences provided. Thus,
 CIDEr score for a reference dataset with only 1 image (or example for NLG) will be zero. When evaluating using one (or few)

nlgeval/pycocoevalcap/meteor/meteor.py

@@ -1,21 +1,26 @@
 #!/usr/bin/env python
 
 # Python wrapper for METEOR implementation, by Xinlei Chen
-# Acknowledge Michael Denkowski for the generous discussion and help 
+# Acknowledge Michael Denkowski for the generous discussion and help
+from __future__ import division
 
 import atexit
-import sys
+import logging
 import os
 import subprocess
+import sys
 import threading
 
+import psutil
+
 # Assumes meteor-1.5.jar is in the same directory as meteor.py.  Change as needed.
 METEOR_JAR = 'meteor-1.5.jar'
 
 
 def enc(s):
     return s.encode('utf-8')
 
+
 def dec(s):
     return s.decode('utf-8')
 
@@ -26,7 +31,16 @@ def __init__(self):
         # Used to guarantee thread safety
         self.lock = threading.Lock()
 
-        meteor_cmd = ['java', '-jar', '-Xmx2G', METEOR_JAR,
+        mem = '2G'
+        mem_available_G = psutil.virtual_memory().available / 1E9
+        if mem_available_G < 2:
+            logging.warning("There is less than 2GB of available memory.\n"
+                            "Will try with limiting Meteor to 1GB of memory but this might cause issues.\n"
+                            "If you have problems using Meteor, "
+                            "then you can try to lower the `mem` variable in meteor.py")
+            mem = '1G'
+
+        meteor_cmd = ['java', '-jar', '-Xmx{}'.format(mem), METEOR_JAR,
                       '-', '-', '-stdio', '-l', 'en', '-norm']
         env = os.environ.copy()
         env['LC_ALL'] = "C"
@@ -65,7 +79,17 @@ def compute_score(self, gts, res):
             self.meteor_p.stdin.write(enc('{}\n'.format(eval_line)))
             self.meteor_p.stdin.flush()
             for i in range(0, len(imgIds)):
-                scores.append(float(dec(self.meteor_p.stdout.readline().strip())))
+                v = self.meteor_p.stdout.readline()
+                try:
+                    scores.append(float(dec(v.strip())))
+                except:
+                    sys.stderr.write("Error handling value: {}\n".format(v))
+                    sys.stderr.write("Decoded value: {}\n".format(dec(v.strip())))
+                    sys.stderr.write("eval_line: {}\n".format(eval_line))
+                    # You can try uncommenting the next code line to show stderr from the Meteor JAR.
+                    # If the Meteor JAR is not writing to stderr, then the line will just hang.
+                    # sys.stderr.write("Error from Meteor:\n{}".format(self.meteor_p.stderr.read()))
+                    raise
             score = float(dec(self.meteor_p.stdout.readline()).strip())
 
         return score, scores

nlgeval/skipthoughts/skipthoughts.py

@@ -76,8 +76,8 @@ def load_tables():
     Load the tables
     """
     words = []
-    utable = numpy.load(os.path.join(path_to_tables, 'utable.npy'), encoding='bytes')
-    btable = numpy.load(os.path.join(path_to_tables, 'btable.npy'), encoding='bytes')
+    utable = numpy.load(os.path.join(path_to_tables, 'utable.npy'), allow_pickle=True, encoding='bytes')
+    btable = numpy.load(os.path.join(path_to_tables, 'btable.npy'), allow_pickle=True,  encoding='bytes')
     f = open(os.path.join(path_to_tables, 'dictionary.txt'), 'rb')
     for line in f:
         words.append(line.decode('utf-8').strip())

requirements.txt

@@ -1,6 +1,7 @@
 click>=6.3
 nltk>=3.1
 numpy>=1.11.0
+psutil>=5.6.2
 requests>=2.19
 six>=1.11
 scipy>=0.17.0

requirements_py2.txt

@@ -1,10 +1,11 @@
 click>=6.3
 nltk>=3.1
 numpy>=1.11.0
+psutil>=5.6.2
 requests>=2.19
 six>=1.11
 scipy>=0.17.0
-scikit-learn>=0.17
+scikit-learn<0.21
 gensim<1
 Theano>=0.8.1
 tqdm>=4.24