deploy package with custom function and the softwares.josn are read f…

…rom the function

analysis/Dockerfile_scanner

@@ -14,7 +14,8 @@ RUN pip install -r requirements.txt
 
 COPY pyFinder/ /code
 
-COPY deploy-package/analysis.py /code/pyfinder
+#deploy-packe is the folder given by an user with the  custom analysis function
+COPY deploy-package /code/pyfinder/deploy
 
 ENTRYPOINT ["python","/code/entryScanner.py"]
 CMD ["--help"]

analysis/deploy-package/analysis.py

@@ -1,7 +1,77 @@
+import json
+import docker
+import re
+import os
 
+client_docker= docker.DockerClient(base_url="unix://var/run/docker.sock")
 
+def on_message(repo_name, context):
 
+    logger = context['logger']
+    docker_hub = context['hub']
+    client_images = context['images']
 
-def handler_analysis(image_name, context):
-    print ("Hello world "+ image_name)
-    print (context.get_images())
+    logger.info("Received name " + repo_name)
+
+    my_image = dict()
+
+    tag = "latest"
+    try:
+        # download the Docker image
+        list_tags = docker_hub.get_all_tags(repo_name)
+        tag = ""
+        if "latest" in list_tags:
+            tag = "latest"
+        elif len(list_tags)>0:
+            tag = list_tags[0] # take the first tag
+
+
+
+        logger.info("Pulling image {0}:{1} ...".format(repo_name, tag))
+        image = client_docker.images.pull(name=repo_name, tag=tag)
+        tag = image.tags[0]
+        logger.info("{} is  pulled locally.".format(tag))
+
+        my_image['name'] = tag
+
+        # create a infinite running container
+        container = client_docker.containers.create(tag, entrypoint="sleep 1000000000")
+        # start the container
+        container.start()
+
+
+        # list of software distributions found in the image.
+        softwares = []
+        with open(os.path.join(os.path.dirname(__file__),'softwares.json')) as json_data:
+        #with open('pyfinder/deploy/softwares.json') as json_data:
+            software= json.load(json_data)
+            for sw in software:# [{'opt':'python --version','regex':'[0-9]+[.][0-9]*[.0-9]*'}]:
+                command = sw['name']+" " + sw['cmd']
+                # create an exec instance
+                res = container.exec_run(cmd=command)
+                output = res.decode()
+                prog = re.compile(sw['regex'])
+                match = prog.search(output)
+                if match:
+                      version = match.group(0)
+                      softwares.append({'software': sw['name'], 'ver': version})
+                      logger.debug("{0} {1} found.".format(sw['name'], version))
+                else:
+                      logger.debug("[{0}] NOT found in ".format(sw['name']))
+
+        #logger.info("{0} {1} found.".format(sw['name'], version))
+        logger.info('['+''.join('{} {},'.format(s['software'],s['ver']) for s in softwares)+"]")
+
+        my_image['softwares']  = softwares
+
+        client_images.post_image(my_image)
+
+        container.stop(timeout=2) # after 2 second it stops the container with SIGKILL
+        container.remove()
+        client_docker.images.remove(image=tag, force=True)
+        logger.info("Removed image {0}".format(tag))
+
+    except  docker.errors.APIError as e:
+        logger.error(str(e))
+    except  docker.errors.ImageNotFound as e:
+        logger.error(str(e))

analysis/deploy-package/softwares.json

@@ -0,0 +1,67 @@
+[{
+		"name": "python",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "java",
+		"cmd": "-version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "perl",
+		"cmd": "-version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "curl",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "nano",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "php",
+		"cmd": "-v",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	},{
+		"name": "ruby",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "scala",
+		"cmd": "-version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "groovy",
+		"cmd": "-version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	},{
+		"name": "httpd",
+		"cmd": "-V",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "nginx",
+		"cmd": "-v",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "pip",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "node",
+		"cmd": "-v",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "npm",
+		"cmd": "-v",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}, {
+		"name": "gunicorn",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	},
+	{
+		"name": "wget",
+		"cmd": "--version",
+		"regex": "[0-9]+[.][0-9]*[.0-9]*"
+	}
+]

analysis/pyFinder/entryChecker.py

@@ -1,4 +1,4 @@
-from pyfinder import Checker
+from pyfinder.checker import Checker
 from docopt import docopt
 import time
 from os import path

analysis/pyFinder/entryCrawler.py

@@ -1,5 +1,8 @@
-from pyfinder import Crawler
-from pyfinder import Tester
+from pyfinder.crawler import Crawler
+
+
+#from pyfinder.tester import Tester
+
 from docopt import docopt
 from os import path
 import logging.config
@@ -50,11 +53,11 @@
                     force_from_page= True if args['--force-page'] =="True" else False
                     )
 
-    if args['test']:
-        tester = Tester(path_file_images=args['--pf'])
-        if args['build']:
-            tester.build_test(num_images_test=int(args['--ni']), from_page=int(args['--fp']))
-
-        if args['send']:
-            tester.push_test(amqp_url=args['--amqp-url'], exchange=args['--ex'], queue=args['--queue'],
-                             route_key=args['--key'])
+    # if args['test']:
+    #     tester = Tester(path_file_images=args['--pf'])
+    #     if args['build']:
+    #         tester.build_test(num_images_test=int(args['--ni']), from_page=int(args['--fp']))
+    #
+    #     if args['send']:
+    #         tester.push_test(amqp_url=args['--amqp-url'], exchange=args['--ex'], queue=args['--queue'],
+    #                          route_key=args['--key'])

analysis/pyFinder/entryScanner.py

@@ -1,12 +1,12 @@
-from pyfinder import Scanner
+from pyfinder.scanner import Scanner
 from docopt import docopt
 import time
 from os import path
 import logging.config
-__doc__= """Scanner.
+__doc__ = """Scanner.
 
 Usage:
-  entryScanner.py run [--amqp-url=<amqp://guest:guest@rabbitmq:5672>] [--ex=<dofinder>] [--queue=<images>] [--key=<images.scan>] [--images-url=<http://images_server:3000/api/images>] [--software-url=<http://software_server:3001/api/software>] [--hub-url=<https://hub.docker.com/>] [--rmi]
+  entryScanner.py run [--amqp-url=<amqp://guest:guest@rabbitmq:5672>] [--ex=<dofinder>] [--queue=<images>] [--key=<images.scan>] [--images-url=<http://images_server:3000/api/images>]  [--hub-url=<https://hub.docker.com/>]
   entryScanner.py scan <name> [--tag=<latest>] [--software-url=<http://127.0.0.1:3001/api/software>]
   entryScanner.py exec <name> --p=<program>  --opt=<option>  --regex=<regex>
   entryScanner.py (-h | --help)
@@ -19,9 +19,7 @@
   --queue==QUEUE        Queue name of the rabbitMQ server       [default: images]
   --key=KEY             Routing key used by the rabbitMQ server [default: images.scan]
   --images-url=IMAGES_URL      The url of the images service    [default: http://127.0.0.1:3000/api/images]
-  --software-url=SOFTWARE-URL  THe url of the software service  [default: http://127.0.0.1:3001/api/software]
   --hub-url=HUB-URL            The url of the DockerHub          [default: https://hub.docker.com]
-  --rmi                 If True remove the images after the scan.
   --tag=TAG             TAG  of the image to scan                  [default: latest]
   --p=PROGRAM           The program name to pass to the container.
   --opt=OPTION          Option of the command to run in the contianer
@@ -30,35 +28,41 @@
 """
 
 # interactive mode for scanner
-#docker run -it --net=core-net --entrypoint=/bin/sh dofinder/scanner:latest
+# docker run -it --net=core-net --entrypoint=/bin/sh dofinder/scanner:latest
 
 if __name__ == '__main__':
     args = docopt(__doc__, version='Scanner 0.0.1')
-    #print(args)
-    log_file_path = path.dirname(path.abspath(__file__))+ '/pyfinder/resources/logging.conf'
+    # print(args)
+    log_file_path = path.dirname(path.abspath(
+        __file__)) + '/pyfinder/resources/logging.conf'
     logging.config.fileConfig(log_file_path)
     logger = logging.getLogger()
-    logger.info("Logging conf: "+ log_file_path)
+    logger.info("Logging conf: " + log_file_path)
 
-    scanner = Scanner(amqp_url=args['--amqp-url'], exchange=args['--ex'], queue=args['--queue'], route_key=args['--key'],
+    scanner = Scanner(amqp_url=args['--amqp-url'],
+                      exchange=args['--ex'],
+                      queue=args['--queue'],
+                      route_key=args['--key'],
                       images_url=args['--images-url'],
-                      software_url=args['--software-url'],
-                      hub_url=args['--hub-url'],
-                      rmi=args['--rmi'])
+                      hub_url=args['--hub-url']
+                      )
 
     if args['scan']:
         image_name = args['<name>']
         tag = args['--tag']
         dict_image = scanner.scan(image_name, tag=tag)
 
-        for sw_json in dict_image['softwares']: # list of json software :{"software": <name>,  "ver":<version>}
-             print(sw_json['software'] + " " + sw_json['ver'])
-        print(str(len(dict_image['softwares']))+" software found in "+ image_name)
+        # list of json software :{"software": <name>,  "ver":<version>}
+        for sw_json in dict_image['softwares']:
+            print(sw_json['software'] + " " + sw_json['ver'])
+        print(str(len(dict_image['softwares'])) +
+              " software found in " + image_name)
         # print(dict_image['distro'] + " distribution of " + image_name)
 
     if args['exec']:
         print(args)
-        out = scanner.version_from_regex(args['<name>'],args['--p'],args['--opt'], args['--regex'])
+        out = scanner.version_from_regex(
+            args['<name>'], args['--p'], args['--opt'], args['--regex'])
         print(out)
 
     if args['run']:

analysis/pyFinder/entryTester.py

@@ -1,5 +1,4 @@
-from pyfinder import Tester
-from pyfinder import ClientSoftware
+from pyfinder.tester import Tester
 
 from docopt import docopt
 
@@ -27,7 +26,6 @@
 
 if __name__ == '__main__':
     args = docopt(__doc__, version='Tester 0.0.1')
-    sw = ClientSoftware(api_url="http://127.0.0.1:3001/api/software")
     tester = Tester(path_file_images=args['--pf'])
     if args['build']:
         tester.build_test(num_images_test=int(args['--ni']), from_page=int(args['--fp']))

analysis/pyFinder/pyfinder/__init__.py

@@ -1,11 +1,7 @@
-from .scanner import Scanner
-from .checker import Checker
-from .crawler import Crawler
-from .model.image import Image
-from .client_images_service import ClientImages
-from .client_dockerhub import ClientHub
-from .client_software import ClientSoftware
-from .tester import Tester
-from .client_software import ClientSoftware
-
-__all__ = [Scanner, Crawler, ClientImages, ClientHub, Tester, Checker, Image]
+# from .scanner import Scanner
+# from .checker import Checker
+# from .crawler import Crawler
+# from .model.image import Image
+# from .tester import Tester
+#
+# __all__ = [Scanner, Crawler, Checker]

analysis/pyFinder/pyfinder/checker.py

@@ -1,6 +1,6 @@
-from .client_images_service import ClientImages
-from .client_dockerhub import ClientHub
-from .publisher_rabbit import PublisherRabbit
+
+from pyfinder.core import ClientImages, PublisherRabbit, ClientHub
+
 import logging
 from logging.handlers import TimedRotatingFileHandler
 import json

analysis/pyFinder/pyfinder/core/__init__.py

@@ -0,0 +1,8 @@
+from .client_images_service import ClientImages
+from .client_dockerhub import ClientHub
+from .client_software import ClientSoftware
+from .consumer_rabbit import ConsumerRabbit
+from .publisher_rabbit import PublisherRabbit
+
+
+__all__ = [ClientImages, ClientHub, ConsumerRabbit, PublisherRabbit, ClientSoftware]

analysis/pyFinder/pyfinder/consumer_rabbit.py → analysis/pyFinder/pyfinder/core/consumer_rabbit.py

@@ -3,7 +3,6 @@
 import pika
 import json
 import time
-from pyfinder import utils
 
 class ConsumerRabbit(object):
     """This is an example consumer that will handle unexpected interactions

analysis/pyFinder/pyfinder/publisher_rabbit.py → analysis/pyFinder/pyfinder/core/publisher_rabbit.py

@@ -1,5 +1,4 @@
 import pika
-from pyfinder import utils
 import logging
 
 

analysis/pyFinder/pyfinder/crawler.py

@@ -1,8 +1,8 @@
 import json
 import pickle
-from .publisher_rabbit import PublisherRabbit
-from .client_images_service import ClientImages
-from .client_dockerhub import ClientHub
+
+from pyfinder.core import ClientImages, PublisherRabbit, ClientHub
+
 import logging
 import random
 

analysis/pyFinder/pyfinder/deploy/__init__.py

@@ -0,0 +1,4 @@
+# The folder will contains the deploy package of the scanner
+#     -- analysis.py
+#     --requirements.txt
+#      (any othe file needed)

analysis/pyFinder/pyfinder/model/__init__.py

@@ -0,0 +1,4 @@
+from .image import Image
+
+
+__all__ = [Image]