Merge branch 'master' into fix-964

lib/jnpr/junos/command/__init__.py

@@ -5,6 +5,8 @@
 
 from jnpr.junos.factory.factory_loader import FactoryLoader
 
+import yamlordereddictloader
+
 __all__ = []
 
 
@@ -28,7 +30,7 @@ def load_module(self, fullname):
                 as stream:
             try:
                 modules = FactoryLoader().load(yaml.load(stream,
-                                                         Loader=yaml.FullLoader))
+                                                         Loader=yamlordereddictloader.Loader))
             except yaml.YAMLError as exc:
                 raise ImportError("%s is not loaded" % mod)
         for k, v in modules.items():

lib/jnpr/junos/console.py

@@ -122,6 +122,7 @@ def __init__(self, **kvargs):
         self._attempts = kvargs.get('attempts', 10)
         self._gather_facts = kvargs.get('gather_facts', False)
         self._fact_style = kvargs.get('fact_style', 'new')
+        self._huge_tree = kvargs.get('huge_tree', False)
         if self._fact_style != 'new':
             warnings.warn('fact-style %s will be removed in '
                           'a future release.' %
@@ -275,7 +276,8 @@ def _rpc_reply(self, rpc_cmd_e, *args, **kwargs):
             if isinstance(rpc_cmd_e, etree._Element) else rpc_cmd_e
         reply = self._tty.nc.rpc(rpc_cmd)
         rpc_rsp_e = NCElement(reply,
-                              self.junos_dev_handler.transform_reply()
+                              self.junos_dev_handler.transform_reply(),
+                              self._huge_tree
                               )._NCElement__doc
         return rpc_rsp_e
 
@@ -290,6 +292,7 @@ def _tty_login(self):
         tty_args['timeout'] = float(self._timeout)
         tty_args['attempts'] = int(self._attempts)
         tty_args['baud'] = self._baud
+        tty_args['huge_tree'] = self._huge_tree
         if self._mode and self._mode.upper() == 'TELNET':
             tty_args['host'] = self._hostname
             tty_args['port'] = self._port

lib/jnpr/junos/device.py

@@ -1147,6 +1147,9 @@ def __init__(self, *vargs, **kvargs):
             default is ``False`` to use DOM.
             Select ``True`` to use SAX (if SAX input is provided).
 
+        :param bool huge_tree:
+            *OPTIONAL* parse XML with very deep trees and long text content.
+            default is ``False``.
         """
 
         # ----------------------------------------
@@ -1162,6 +1165,7 @@ def __init__(self, *vargs, **kvargs):
         self._auto_probe = kvargs.get('auto_probe', self.__class__.auto_probe)
         self._fact_style = kvargs.get('fact_style', 'new')
         self._use_filter = kvargs.get('use_filter', False)
+        self._huge_tree = kvargs.get('huge_tree', False)
         if self._fact_style != 'new':
             warnings.warn('fact-style %s will be removed in a future '
                           'release.' %
@@ -1341,6 +1345,8 @@ def open(self, *vargs, **kvargs):
             cnx_err._orig = err
             raise cnx_err
 
+        if self._huge_tree:
+            self._conn.huge_tree = True
         self.connected = True
 
         self._nc_transform = self.transform

lib/jnpr/junos/factory/cmdtable.py

@@ -95,11 +95,12 @@ def get(self, *vargs, **kvargs):
                                                                 str) else \
                 kvargs['filters']
 
+        cmd_args = self.CMD_ARGS.copy()
         if 'args' in kvargs and isinstance(kvargs['args'], dict):
-            self.CMD_ARGS.update(kvargs['args'])
+            cmd_args.update(kvargs['args'])
 
-        if len(self.CMD_ARGS) > 0:
-            self.GET_CMD = Template(self.GET_CMD).render(**self.CMD_ARGS)
+        if len(cmd_args) > 0:
+            self.GET_CMD = Template(self.GET_CMD).render(**cmd_args)
 
         # execute the Junos RPC to retrieve the table
         if hasattr(self, 'TARGET'):

lib/jnpr/junos/factory/optable.py

@@ -78,7 +78,6 @@ def get(self, *vargs, **kvargs):
             rpc_args.update(kvargs.pop('args'))
         rpc_args.update(kvargs)           # copy caller provided args
 
-
         if hasattr(self, 'GET_KEY') and argkey is not None:
             rpc_args.update({self.GET_KEY: argkey})
 
@@ -98,11 +97,10 @@ def generate_sax_parser_input(obj):
     Returns: lxml etree object to be used as sax parser input
 
     """
+    item_tags = []
     if '/' in obj.ITEM_XPATH:
-        tags = obj.ITEM_XPATH.split('/')
-        parser_ingest = E(tags.pop(-1), E(obj.ITEM_NAME_XPATH))
-        for tag in tags[::-1]:
-            parser_ingest = E(tag, parser_ingest)
+        item_tags = obj.ITEM_XPATH.split('/')
+        parser_ingest = E(item_tags.pop(-1), E(obj.ITEM_NAME_XPATH))
     else:
         parser_ingest = E(obj.ITEM_XPATH, E(obj.ITEM_NAME_XPATH))
     local_field_dict = deepcopy(obj.VIEW.FIELDS)
@@ -117,6 +115,7 @@ def generate_sax_parser_input(obj):
             for key, val in group_field_dict.items():
                 group_ele.append(E(val.get('xpath')))
             parser_ingest.append(group_ele)
+    map_multilayer_fields = dict()
     for i, item in enumerate(local_field_dict.items()):
         # i is the index and item will be taple of field key and value
         field_dict = item[1]
@@ -129,10 +128,32 @@ def generate_sax_parser_input(obj):
             # xpath can be multi level, for ex traffic-statistics/input-pps
             if '/' in xpath:
                 tags = xpath.split('/')
-                obj = E(tags[0])
-                for tag in tags[1:]:
-                    obj.append(E(tag))
+                if tags[0] in map_multilayer_fields:
+                    # cases where multiple fields got same parents
+                    # fields:
+                    #    input-bytes: traffic-statistics/input-bytes
+                    #    output-bytes: traffic-statistics/output-bytes
+                    existing_elem = parser_ingest.xpath(tags[0])
+                    if existing_elem:
+                        obj = existing_elem[0]
+                        for tag in tags[1:]:
+                            obj.append(E(tag))
+                    else:
+                        continue
+                else:
+                    obj = E(tags[0])
+                    for tag in tags[1:]:
+                        obj.append(E(tag))
+                    map_multilayer_fields[tags[0]] = obj
                 parser_ingest.insert(i + 1, obj)
             else:
                 parser_ingest.insert(i + 1, E(xpath))
+    # cases where item is something like
+    # item: task-memory-malloc-usage-report/task-malloc-list/task-malloc
+    # created filter from last item task-malloc
+    # Now add all the tags if present
+    for item_tag in item_tags[::-1]:
+        parser_ingest = E(item_tag, parser_ingest)
+    logger.debug("Generated filter XML is: %s" % etree.tostring(parser_ingest))
+
     return parser_ingest

lib/jnpr/junos/factory/state_machine.py

@@ -711,18 +711,15 @@ def parse_using_regex(self, event):
             # checking index as there can be blank line at position 0 and 2
             if line.strip() == '':
                 if index > 2:
-                    if self.is_row_column():
-                        try:
-                            if len(_regex.scanString(self._lines[self._lines.index(
-                                    line) + 1])[0]) == 0:
-                                break
-                        except IndexError:
-                            break
-                    elif self.is_regex_data():
+                    if self.is_row_column() or self.is_regex_data():
                         try:
+                            match = []
+                            # check if line after new line matches condition
+                            # There can be data set where there are new line in between.
                             for result, start, end in _regex.scanString(
-                                    self._lines[self._lines.index(
-                                        line) + 1]):
+                                    self._lines[index + 1]):
+                                match.extend(result)
+                            if match:
                                 continue
                             else:
                                 break

lib/jnpr/junos/factory/table.py

@@ -99,6 +99,9 @@ def _tkey(self, this, key_list):
             try:
                 keys.append(this.xpath(k)[0].text)
             except:
+                # Case where key is provided like key: re-name | Null
+                if ' | ' in k and 'Null' in k:
+                    continue
                 keys.append(None)
         return tuple(keys)
 
@@ -134,8 +137,9 @@ def _keys(self):
             # Check if pipe is in the key_value, if so append xpath
             # to each value
             if ' | ' in key_value:
-                return self._keys_simple(' | '.join([xpath + '/' + x for x in
-                                                     key_value.split(' | ')]))
+                return self._keys_simple(' | '.join(
+                    [xpath + '/' + x for x in key_value.split(' | ') if
+                     x != 'Null']))
             return self._keys_simple(xpath + '/' + key_value)
 
         # user explicitly passed key as Null in Table

lib/jnpr/junos/factory/view.py

@@ -85,23 +85,53 @@ def name(self):
         if isinstance(self.ITEM_NAME_XPATH, str):
             # xpath union key
             if ' | ' in self.ITEM_NAME_XPATH:
+                if 'Null' in self.ITEM_NAME_XPATH:
+                    return self._check_key_delimiter_null(self._xml, self.ITEM_NAME_XPATH)
                 return self._xml.xpath(self.ITEM_NAME_XPATH)[0].text.strip()
             # simple key
             return self._xml.findtext(self.ITEM_NAME_XPATH).strip()
         else:
             # composite key
             # keys with missing XPATH nodes are set to None
             keys = []
-            for i in self.ITEM_NAME_XPATH:
-                try:
-                    keys.append(self.xml.xpath(i)[0].text.strip())
-                except:
-                    keys.append(None)
-            return tuple(keys)
+            for item_name_xpath in self.ITEM_NAME_XPATH:
+                if ' | ' in item_name_xpath:
+                    key_with_null_cleaned = \
+                        self._check_key_delimiter_null(self._xml,
+                                                       item_name_xpath)
+                    if key_with_null_cleaned:
+                        keys.append(key_with_null_cleaned)
+                else:
+                    try:
+                        keys.append(self.xml.xpath(item_name_xpath)[0].text.strip())
+                    except:
+                        keys.append(None)
+            if keys:
+                return tuple(keys)
+            else:
+                return keys
 
     # ALIAS key <=> name
     key = name
 
+    def _check_key_delimiter_null(self, xml, item_name_xpath):
+        """
+        Case where key is provided like key: re-name | Null
+
+        :param xml: xml object retrieved from device
+        :param item_name_xpath: key xpath
+        :return: key if fetched else []
+        """
+        if 'Null' in item_name_xpath:
+            # Let try get value for valid xpath key
+            xpath_key = [x for x in item_name_xpath.split(' | ') if x != 'Null']
+            if xpath_key:
+                val = xml.xpath(xpath_key[0])
+                if val:
+                    return val[0].text.strip()
+                else:
+                    # To handle Null key
+                    return []
     @property
     def xml(self):
         """ returns the XML associated to the item """

lib/jnpr/junos/op/__init__.py

@@ -10,10 +10,11 @@
 
 class MetaPathFinder(object):
     def find_module(self, fullname, path=None):
-        mod = fullname.split('.')[-1]
-        if mod in [os.path.splitext(i)[0] for i in os.listdir(
-                os.path.dirname(__file__))]:
-            return MetaPathLoader()
+        if fullname.startswith('jnpr.junos'):
+            mod = fullname.split('.')[-1]
+            if mod in [os.path.splitext(i)[0] for i in os.listdir(
+                    os.path.dirname(__file__))]:
+                return MetaPathLoader()
 
 
 class MetaPathLoader(object):

lib/jnpr/junos/transport/tty.py

@@ -82,6 +82,7 @@ def __init__(self, **kvargs):
         self.cs_passwd = kvargs.get('cs_passwd')
         self.login_attempts = kvargs.get('attempts') or self.LOGIN_RETRY
         self.console_has_banner = kvargs.get('console_has_banner') or False
+        self._huge_tree = kvargs.get('huge_tree', False)
 
         # misc setup
         self.nc = tty_netconf(self)

lib/jnpr/junos/transport/tty_netconf.py

@@ -119,7 +119,7 @@ def rpc(self, cmd):
 
         rsp = self._receive()
         rsp = rsp.decode('utf-8') if isinstance(rsp, bytes) else rsp
-        reply = RPCReply(rsp)
+        reply = RPCReply(rsp, huge_tree=self._tty._huge_tree)
         errors = reply.errors
         if len(errors) > 1:
             raise RPCError(to_ele(reply._raw), errs=errors)
@@ -161,8 +161,10 @@ def _receive(self):
             rxbuf = [i.strip() for i in rxbuf if i.strip() != PY6.EMPTY_STR]
             rcvd_data = PY6.NEW_LINE.join(rxbuf)
             logger.debug('Received: \n%s' % rcvd_data)
+            parser = etree.XMLParser(remove_blank_text=True,
+                                     huge_tree=self._tty._huge_tree)
             try:
-                etree.XML(rcvd_data)
+                etree.XML(rcvd_data, parser)
             except XMLSyntaxError:
                 if _NETCONF_EOM in rcvd_data:
                     rcvd_data = rcvd_data[:rcvd_data.index(_NETCONF_EOM)]

lib/jnpr/junos/utils/config.py

@@ -208,7 +208,7 @@ def commit_check(self):
     # show | compare rollback <number|0*>
     # -------------------------------------------------------------------------
 
-    def diff(self, rb_id=0):
+    def diff(self, rb_id=0, ignore_warning=False):
         """
         Retrieve a diff (patch-format) report of the candidate config against
         either the current active config, or a different rollback.
@@ -224,9 +224,9 @@ def diff(self, rb_id=0):
             raise ValueError("Invalid rollback #" + str(rb_id))
 
         try:
-            rsp = self.rpc.get_configuration(dict(
-                compare='rollback', rollback=str(rb_id), format='text'
-            ))
+            rsp = self.rpc.get_configuration(
+                    dict(compare='rollback', rollback=str(rb_id), format='text'),
+                    ignore_warning=ignore_warning)
         except RpcTimeoutError:
             raise 
         except RpcError as err:

lib/jnpr/junos/utils/sw.py

@@ -814,6 +814,23 @@ def _progress(report):
 
         self.log = _progress
 
+        # ---------------------------------------------------------------------
+        # Before doing anything, Do check if any pending install exists.
+        # ---------------------------------------------------------------------
+        try:
+            pending_install = self._dev.rpc.request_package_checks_pending_install()
+            msg = pending_install.text
+            if msg and msg.strip() != '' and pending_install.getparent().findtext(
+                    'package-result').strip() == '1':
+                _progress(msg)
+                return False
+        except RpcError:
+            _progress("request-package-check-pending-install rpc is not "
+                      "supported on given device")
+        except Exception as ex:
+            _progress("check pending install failed with exception: %s" % ex)
+            # Continue with software installation
+
         # ---------------------------------------------------------------------
         # perform a 'safe-copy' of the image to the remote device
         # ---------------------------------------------------------------------

lib/jnpr/junos/version.py

@@ -1,4 +1,4 @@
-VERSION = "2.3.1"
+VERSION = "2.3.2.dev0"
 DATE = "2019-Dec-10"
 
 # Augment with the internal version if present

setup.py

@@ -11,7 +11,7 @@
 setup(
     name="junos-eznc",
     namespace_packages=['jnpr'],
-    version="2.3.1",
+    version="2.3.2.dev0",
     author="Jeremy Schulman, Nitin Kumar, Rick Sherman, Stacy Smith",
     author_email="[email protected]",
     description=("Junos 'EZ' automation for non-programmers"),

tests/unit/factory/rpc-reply/show-utmd-status-use-Null.xml

@@ -0,0 +1,5 @@
+<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/19.4I0/junos" xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="urn:uuid:d002acd0-0f28-4ba2-a78c-042aeb5cc910">
+<utmd-status xmlns="http://xml.juniper.net/junos/19.4I0/junos-utmd">
+<running/>
+</utmd-status>
+</rpc-reply>

tests/unit/factory/rpc-reply/show-utmd-status.xml

@@ -0,0 +1,21 @@
+<rpc-reply xmlns="urn:ietf:params:xml:ns:netconf:base:1.0" xmlns:junos="http://xml.juniper.net/junos/20.2I0/junos" xmlns:nc="urn:ietf:params:xml:ns:netconf:base:1.0" message-id="urn:uuid:c9c57ecd-dc9f-44cf-803b-5487182b9475">
+<multi-routing-engine-results>
+
+<multi-routing-engine-item>
+
+<re-name>node0</re-name>
+
+<utmd-status xmlns="http://xml.juniper.net/junos/20.2I0/junos-utmd"><running/>
+</utmd-status>
+</multi-routing-engine-item>
+
+<multi-routing-engine-item>
+
+<re-name>node1</re-name>
+
+<utmd-status xmlns="http://xml.juniper.net/junos/20.2I0/junos-utmd"><running/>
+</utmd-status>
+</multi-routing-engine-item>
+
+</multi-routing-engine-results>
+</rpc-reply>