Create Device42 to FreshService Script

Author: RomanNyschuk1

LICENSE

@@ -0,0 +1,13 @@
+Copyright [2019] [Device42, Inc.]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README.md

@@ -0,0 +1,55 @@
+[Device42](http://www.device42.com/) is a Continuous Discovery software for your IT Infrastructure. It helps you automatically maintain an up-to-date inventory of your physical, virtual, and cloud servers and containers, network components, software/services/applications, and their inter-relationships and inter-dependencies.
+
+
+This repository contains script that helps you sync data from Device42 to FreshService.
+
+### Download and Installation
+-----------------------------
+To utilize the Device42_freshservice_mapping script, Python 3.5+ is required. The following Python Packages are required as well:
+
+* pycrypto==2.6.1
+* pyparsing==2.1.10
+* pyzmq==16.0.2
+* requests==2.13.0
+* xmljson==0.2.0
+
+These can all be installed by running `pip install -r requirements.txt`.
+
+Once installed, the script itself is run by this command: `python d42_sd_sync.py`.
+
+### Configuration
+-----------------------------
+Prior to using the script, it must be configured to connect to your Device42 instance and your FreshService instance.
+* Save a copy of mapping.xml.sample as mapping.xml. 
+* Enter your URL, User, Password, API Key in the FreshService and Device42 sections (lines 2-10).
+API Key can be obtained from FreshService profile page
+
+Below the credential settings, you’ll see a Tasks section. 
+Multiple Tasks can be setup to synchronize various CIs from Device42 to FreshService.
+In the <api> section of each task, there will be a <resource> section that queries Device42 to obtain the desired CIs. 
+Full documentation of the Device42 API and endpoints is available at https://api.device42.com. 
+Individual tasks within a mapping.xml file can be enabled or disabled at will by changing the `enable="true"` to `enable="false"` in the <task> section.
+
+Once the Device42 API resource and FreshService Target are entered, the <mapping> section is where fields from Device42 (the `resource` value) can be mapped to fields in FreshService (the `target` value).
+It is very important to adjust the list of default values in accordance between freshservice and device 42 (for example, service_level).
+
+After configuring the fields to map as needed, the script should be ready to run. 
+
+### Compatibility
+-----------------------------
+* Script runs on Linux and Windows
+
+### Info
+-----------------------------
+* mapping.xml - file from where we get fields relations between D42 and FreshService
+* devicd42.py - file with integration device42 instance
+* freshservice.py - file with integration freshservice instance
+* d42_sd_sync.py - initialization and processing file, where we prepare API calls
+
+### Support
+-----------------------------
+We will support any issues you run into with the script and help answer any questions you have. Please reach out to us at [email protected]
+
+###Version
+-----------------------------
+1.0.0.190411

d42_sd_sync.py

@@ -0,0 +1,295 @@
+__author__ = 'Roman Nyschuk'
+
+import os
+import sys
+import logging
+import json
+import argparse
+import datetime
+from device42 import Device42
+from freshservice import FreshService
+import xml.etree.ElementTree as eTree
+from xmljson import badgerfish as bf
+import time
+
+logger = logging.getLogger('log')
+logger.setLevel(logging.INFO)
+ch = logging.StreamHandler(sys.stdout)
+ch.setFormatter(logging.Formatter('%(asctime)-15s\t%(levelname)s\t %(message)s'))
+logger.addHandler(ch)
+CUR_DIR = os.path.dirname(os.path.abspath(__file__))
+
+parser = argparse.ArgumentParser(description="freshservice")
+
+parser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
+parser.add_argument('-q', '--quiet', action='store_true', help='Quiet mode - outputs only errors')
+parser.add_argument('-c', '--config', help='Config file', default='mapping.xml')
+parser.add_argument('-l', '--logfolder', help='log folder path', default='.')
+
+freshservice = None
+
+
+class JSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, datetime):
+            return o.strftime("%Y %m %d %H:%M:%S")
+        return json.JSONEncoder.default(self, o)
+
+
+def find_object_by_name(assets, name):
+    for asset in assets:
+        if asset["name"] == name:
+            return asset
+
+    return None
+
+
+def get_asset_type_field(asset_type_fields, map_info):
+    for section in asset_type_fields:
+        if section["field_header"] == map_info["@target-header"]:
+            for field in section["fields"]:
+                name = map_info["@target"]
+                if "@target-field" in map_info:
+                    name = map_info["@target-field"]
+                if field["asset_type_id"] is not None:
+                    name += "_" + str(field["asset_type_id"])
+                if field["name"] == name:
+                    return field
+
+    return None
+
+
+def get_map_value_from_device42(source, map_info, b_add=False, asset_type_id=None):
+    d42_value = source[map_info["@resource"]]
+    if d42_value is None and "@resource-secondary" in map_info:
+        d42_value = source[map_info["@resource-secondary"]]
+    if "@is-array" in map_info and map_info["@is-array"]:
+        d42_vals = d42_value
+        d42_value = None
+        for d42_val in d42_vals:
+            if map_info["@sub-key"] in d42_val:
+                d42_value = d42_val[map_info["@sub-key"]]
+                break
+    else:
+        if "value-mapping" in map_info:
+            d42_val = None
+            if isinstance(map_info["value-mapping"]["item"], list):
+                items = map_info["value-mapping"]["item"]
+            else:
+                items = [map_info["value-mapping"]["item"]]
+            for item in items:
+                if item["@key"] == d42_value:
+                    d42_val = item["@value"]
+            if d42_val is None and "@default" in map_info["value-mapping"]:
+                d42_val = map_info["value-mapping"]["@default"]
+
+            d42_value = d42_val
+        else:
+            pass
+
+    if "@target-foregin-key" in map_info:
+        value = freshservice.get_id_by_name(map_info["@target-foregin"], d42_value)
+        if b_add and value is None and "@not-null" in map_info and map_info[
+            "@not-null"] and "@required" in map_info and map_info["@required"]:
+            name = d42_value
+            id = freshservice.insert_and_get_id_by_name(map_info["@target-foregin"], name, asset_type_id)
+            d42_value = id
+        else:
+            d42_value = value
+
+    return d42_value
+
+
+def update_objects_from_server(sources, _target, mapping, doql=False):
+    global freshservice
+
+    logger.info("Getting all existing devices in FS.")
+    existing_objects = freshservice.request(_target["@path"] + "?include=type_fields", "GET", _target["@model"])
+    logger.info("finished getting all existing devices in FS.")
+
+    asset_type = freshservice.get_ci_type_by_name(_target["@asset-type"])
+
+    asset_type_fields = freshservice.get_asset_type_fields(asset_type["id"])
+
+    for source in sources:
+        try:
+            existing_object = find_object_by_name(existing_objects, source["name"])
+            data = dict()
+            data["type_fields"] = dict()
+            for map_info in mapping["field"]:
+                asset_type_field = get_asset_type_field(asset_type_fields, map_info)
+                if asset_type_field is None:
+                    continue
+
+                value = get_map_value_from_device42(source, map_info)
+
+                if asset_type_field["asset_type_id"] is not None:
+                    data["type_fields"][asset_type_field["name"]] = value
+                else:
+                    data[map_info["@target"]] = value
+
+            # validation
+            for map_info in mapping["field"]:
+                asset_type_field = get_asset_type_field(asset_type_fields, map_info)
+                if asset_type_field is None:
+                    continue
+
+                if asset_type_field["asset_type_id"] is not None:
+                    value = data["type_fields"][asset_type_field["name"]]
+                else:
+                    value = data[map_info["@target"]]
+
+                is_valid = True
+                if value is not None and "@min-length" in map_info and len(value) < map_info["@min-length"]:
+                    is_valid = False
+                if value is None and "@not-null" in map_info and map_info["@not-null"]:
+                    is_valid = False
+                if not is_valid and "@required" in map_info and map_info["@required"]:
+                    value = get_map_value_from_device42(source, map_info, True, data["asset_type_id"])
+                    if value is not None:
+                        is_valid = True
+                if "@target-type" in map_info and value is not None:
+                    target_type = map_info["@target-type"]
+                    if target_type == "integer":
+                        try:
+                            value = int(value)
+                        except:
+                            is_valid = False
+
+                if not is_valid:
+                    logger.debug("argument '%s' is invalid." % map_info["@target"])
+                    if asset_type_field["asset_type_id"] is not None:
+                        data["type_fields"].pop(asset_type_field["name"], None)
+                    else:
+                        data.pop(map_info["@target"], None)
+                if is_valid:
+                    if asset_type_field["asset_type_id"] is not None:
+                        data["type_fields"][asset_type_field["name"]] = value
+                    else:
+                        data[map_info["@target"]] = value
+
+            if existing_object is None:
+                logger.info("adding device %s" % source["name"])
+                new_asset_id = freshservice.insert_asset(data)
+                logger.info("added new asset %d" % new_asset_id)
+            else:
+                logger.info("updating device %s" % source["name"])
+                updated_asset_id = freshservice.update_asset(data, existing_object["display_id"])
+                logger.info("updated new asset %d" % updated_asset_id)
+        except Exception as e:
+            logger.exception("Error (%s) updating device %s" % (type(e), source["name"]))
+
+
+def delete_objects_from_server(sources, _target, mapping):
+    global freshservice
+
+    logger.info("Getting all existing devices in FS.")
+    existing_objects = freshservice.request(_target["@path"] + "?include=type_fields", "GET", _target["@model"])
+    logger.info("finished getting all existing devices in FS.")
+
+    for existing_object in existing_objects:
+        exist = False
+        for source in sources:
+            if source[mapping["@key"]] == existing_object[mapping["@key"]]:
+                exist = True
+                break
+
+        if not exist:
+            try:
+                logger.info("deleting device %s" % existing_object["name"])
+                freshservice.delete_asset(existing_object["display_id"])
+                logger.info("deleted asset %s" % existing_object["name"])
+            except Exception as e:
+                logger.exception("Error (%s) deleting device %s" % (type(e), existing_object["name"]))
+
+
+def parse_config(url):
+    config = eTree.parse(url)
+    meta = config.getroot()
+    config_json = bf.data(meta)
+
+    return config_json
+
+
+def task_execute(task, device42):
+    if "@description" in task:
+        logger.info("Execute task - %s" % task["@description"])
+
+    _resource = task["api"]["resource"]
+    _target = task["api"]["target"]
+
+    method = _resource['@method']
+    if "@doql" in _resource:
+        doql = _resource['@doql']
+    else:
+        doql = None
+
+    source_url = _resource['@path']
+    if "@extra-filter" in _resource:
+        source_url += _resource["@extra-filter"] + "&amp;"
+
+    _type = None
+    if "@type" in task:
+        _type = task["@type"]
+
+    mapping = task['mapping']
+
+    if doql is not None and doql:
+        sources = device42.doql(source_url, method, query=doql)
+    else:
+        sources = device42.request(source_url, method, _resource["@model"])
+
+    if "@delete" in _target and _target["@delete"]:
+        delete_objects_from_server(sources, _target, mapping)
+        return
+
+    update_objects_from_server(sources, _target, mapping, doql=False)
+
+
+def main():
+    global freshservice
+
+    args = parser.parse_args()
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    if args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    try:
+        log_file = "%s/d42_fs_sync_%d.log" % (args.logfolder, int(time.time()))
+        logging.basicConfig(filename=log_file)
+    except Exception as e:
+        print("Error in config log: %s" % str(e))
+        return -1
+
+    config = parse_config(args.config)
+    logger.debug("configuration info: %s" % (json.dumps(config)))
+
+    settings = config["meta"]["settings"]
+    device42 = Device42(settings['device42']['@url'], settings['device42']['@user'], settings['device42']['@pass'])
+    freshservice = FreshService(settings['freshservice']['@url'], settings['freshservice']['@api_key'], logger)
+
+    if not "task" in config["meta"]["tasks"]:
+        logger.debug("No task")
+        return 0
+
+    if isinstance(config["meta"]["tasks"]["task"], list):
+        tasks = config["meta"]["tasks"]["task"]
+    else:
+        tasks = [config["meta"]["tasks"]["task"]]
+
+    for task in tasks:
+        if not task["@enable"]:
+            continue
+
+        task_execute(task, device42)
+
+    print("Completed! View log at %s" % log_file)
+    return 0
+
+
+if __name__ == "__main__":
+    print('Running...')
+    ret_val = main()
+    print('Done')
+    sys.exit(ret_val)