diff --git a/.travis.yml b/.travis.yml
index 80a1cfb..70fcad1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -62,6 +62,8 @@ script:
           coverage run --branch --source tlsfuzzer -m unittest discover;
       fi
   - coverage report -m
+  # check if all the scripts are included in the json file
+  - python tests/verify-scripts-json.py tests/tlslite-ng-random-subset.json
   - travis_wait 30 python tests/scripts_retention.py tests/tlslite-ng-random-subset.json `which tls.py`
   # pylint doesn't work on 2.6: https://bitbucket.org/logilab/pylint/issue/390/py26-compatiblity-broken
   # diff-quality doesn't work on 3.2: https://github.com/edx/diff-cover/issues/94
diff --git a/tests/verify-scripts-json.py b/tests/verify-scripts-json.py
new file mode 100755
index 0000000..ffb78e2
--- /dev/null
+++ b/tests/verify-scripts-json.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python
+
+import os
+from os import listdir, path
+import json
+from os.path import isfile, join, dirname, abspath
+from sys import argv
+
+parent_dir = dirname(dirname(abspath(__file__)))
+scriptlist = [f for f in listdir(os.path.join(parent_dir,'scripts')) if isfile(join(os.path.join(parent_dir,'scripts'), f))]
+status = 0
+for arg in argv[1:]:
+    missing = []
+    try:
+        jsonfile = json.load(open(arg))
+    except IOError:
+        print("Please check the input file name, it doesn't appear to exist")
+        status =1
+        continue
+    except ValueError:
+        print("Wrong file type: The input must be a json file")
+        status =1
+        continue
+    except IndexError:
+        print("No input file was provided")
+        status =1
+        continue
+    for f in scriptlist:
+        if f not in str(jsonfile):
+            missing.append(f)
+    if not missing:
+        print(" All scripts are in the json file")
+    else:
+        print("There are {0} scripts that are missing from {1}:".format(str(len(missing)), str(arg)))
+        print("\n".join(missing))
+        status = 1
+exit(status)