Merge branch 'Juniper:master' into bugfix/serial_connection

Dockerfile

@@ -1,26 +1,29 @@
-FROM alpine:3.12
+FROM python:3.12-alpine
 
 LABEL net.juniper.description="Junos PyEZ library for Python in a lightweight container." \
-      net.juniper.maintainer="Stephen Steiner <ssteiner@juniper.net>"
+      net.juniper.maintainer="jnpr-community-netdev@juniper.net"
 
 WORKDIR /source
 
-## Copy project inside the container
+## Copy project inside the containers
 ADD setup.* ./
 ADD versioneer.py .
 ADD requirements.txt .
 ADD lib lib 
 ADD entrypoint.sh /usr/local/bin/.
 
 ## Install dependancies and PyEZ
-RUN apk add --no-cache build-base python3-dev py-lxml \
+RUN apk add --no-cache build-base python3-dev \
     libxslt-dev libxml2-dev libffi-dev openssl-dev curl \
-    ca-certificates py3-pip bash \
-    && pip install -U pip \
-    && pip install -r requirements.txt \
-    && apk del -r --purge gcc make g++ \
-    && ln -s /usr/bin/python3 /usr/bin/python \
-    && pip install . \
+    ca-certificates py3-pip bash
+
+RUN pip install --upgrade pip \
+    && pip install pipdeptree \
+    && python3 -m pip install -r requirements.txt \
+    && pip install .
+
+## Clean up and start init
+RUN apk del -r --purge gcc make g++ \
     && rm -rf /source/* \
     && chmod +x /usr/local/bin/entrypoint.sh
 

README.md

@@ -1,5 +1,5 @@
 [![PyPi Version](https://img.shields.io/pypi/v/junos-eznc.svg)](https://pypi.python.org/pypi/junos-eznc/)
-[![Documentation Status](https://readthedocs.org/projects/junos-pyez/badge/?version=stable)](http://junos-pyez.readthedocs.io)
+[![Documentation Status](https://readthedocs.org/projects/junos-pyez/badge/?version=stable)](https://junos-pyez.readthedocs.io/en/latest/)
 [![Coverage Status](https://img.shields.io/coveralls/Juniper/py-junos-eznc.svg)](https://coveralls.io/r/Juniper/py-junos-eznc)
 [![UnitTest Status](https://travis-ci.org/Juniper/py-junos-eznc.svg?branch=master)](https://travis-ci.org/Juniper/py-junos-eznc)
 [![](https://images.microbadger.com/badges/image/juniper/pyez.svg)](https://microbadger.com/images/juniper/pyez)
@@ -29,15 +29,11 @@ This means that "non-programmers", for example the _Network Engineer_, can use t
 
 ## For "Programmers" - Open and Extensible
 
-There is a growing interest and need to automate the network infrastructure into larger IT systems.  To do so, traditional software programmers, DevOps, "hackers", etc. need an abstraction library of code to further those activities.  _Junos PyEZ_ is designed for extensibility so that the programmer can quickly and easily add new widgets to the library in support of their specific project requirements.  There is no need to "wait on the vendor" to provide new functionality.   _Junos PyEZ_ is not specifically tied to any version of Junos or any Junos product family.
+There is a growing interest and need to automate the network infrastructure into larger IT systems.  To do so, traditional software programmers, DevOps, "hackers", etc. need an abstraction library of code to further those activities.  _Junos PyEZ_ is designed for extensibility so that the programmer can quickly and easily add new widgets to the library in support of their specific project requirements.  There is no need to "wait on the vendor" to provide new functionality.   _Junos PyEZ_ is not specifically tied to any version of Junos (or Junos Evolved) or any Junos (or Junos Evolved) product family.
 
 # SUPPORT
 
-For questions and general support, please visit our [Google Group](https://groups.google.com/forum/#!forum/junos-python-ez)
-
-You can also post your query on stackoverflow with __pyez__ [tag](http://stackoverflow.com/questions/tagged/pyez)
-
-For documentation and more usage examples, please visit the _Junos PyEZ_ project page, [here](http://forums.juniper.net/t5/Automation/Where-can-I-learn-more-about-Junos-PyEZ/ta-p/280496).
+For documentation and more usage examples, please visit the _Junos PyEZ_ project page, [here](http://www.juniper.net/techpubs/en_US/release-independent/junos-pyez/information-products/pathway-pages/index.html).
 
 Issues and bugs can be opened in the repository.
 
@@ -54,7 +50,7 @@ _Junos PyEZ_ is designed to provide the same capabilities as a user would have o
 
 # NOTICES
 
-- As of release 2.0.0, _Junos PyEZ_ requires [ncclient](https://pypi.python.org/pypi/ncclient) version 0.5.2 or later.
+- As of release 2.7.1, _Junos PyEZ_ requires [ncclient](https://pypi.python.org/pypi/ncclient) version 0.6.15 or later.
 - When using the `ssh_private_key_file` argument of the Device constructor on MacOS Mojave and higher, ensure that the SSH keys are in the RSA format, and not the newer OPENSSH format.
   - New key: `ssh-keygen -p -m PEM -f ~/.ssh/id_rsa`
   - Convert an existing OPENSSH key: ``ssh-keygen -p -m PEM -f ~/.ssh/private_key`
@@ -88,11 +84,16 @@ Move to the local directory which contains your script(s) and run the container.
 
 Your local scripts will be mounted to /scripts in the container.
 
+As per PEP 668, use Virtual Env in the container (after you enter the container). The docker container has virtual environment installed in the /scripts folder.
+
+Use `source .venv/bin/activate` to activate the virtual environment.
+
+
 ### Microservice Usage
 
 This image can also be used as a Python "executable" with the required Python PyEZ libraries pre-installed. To use the image in this way, mount the volume which contains the Python script and pass the script name as an argument to `docker run`. Optionally, you may also pass in a `requirements.txt` file to install additional python packages via `pip`. To add OS packages (Alpine Linux), provide a file with a list of packages --one per line-- and either reference it as an env var (`$APK`) or mount it to the container `/extras/apk.txt`. To add additional Python packages (via pip), provide a `requirements.txt` file and pass it in as an env var (`$REQ`) or mount it to the container at `/extras/requirements.txt`.
 
-`Usage: `docker run -it [ --rm ] -v some/dir:/scripts juniper/pyez [ myscript.py ]`
+Usage: `docker run -it [ --rm ] -v some/dir:/scripts juniper/pyez [ myscript.py ]`
 
 Example:
 
@@ -153,8 +154,11 @@ Juniper Networks is actively contributing to and maintaining this repo. Please c
 
 *Contributors:*
 
-[Nitin Kumar](https://github.com/vnitinv), [Stacy Smith](https://github.com/stacywsmith), [Stephen Steiner](https://github.com/ntwrkguru)
-
+* v2.7.2: [Dinesh Babu](https://github.com/dineshbaburam91), [Chidanand Pujar](https://github.com/chidanandpujar)
+* v2.7.1: [Dinesh Babu](https://github.com/dineshbaburam91)
+* v2.6.4: [Chidanand Pujar](https://github.com/chidanandpujar)
+* v2.6.3: [Rahul Kumar](https://github.com/rahkumar651991)
+* v2.5.0: [Rahul Kumar](https://github.com/rahkumar651991)
 * v2.4.1: [Nitin Kumar](https://github.com/vnitinv)
 * v2.4.0: [Nitin Kumar](https://github.com/vnitinv)
 * v2.3.0: [Nitin Kumar](https://github.com/vnitinv), [Raja Shekar Mekala](https://github.com/rsmekala), [Dinesh Babu](https://github.com/dineshbaburam91), [Chris Jenn](https://github.com/ipmonk), [Shigechika](https://github.com/shigechika)
@@ -167,4 +171,4 @@ Juniper Networks is actively contributing to and maintaining this repo. Please c
 
 *Former Contributors:*
 
-[Jeremy Schulman](https://github.com/jeremyschulman), [Rick Sherman](https://github.com/shermdog), [Edward Arcuri](https://github.com/sdndude)
+[Jeremy Schulman](https://github.com/jeremyschulman), [Rick Sherman](https://github.com/shermdog), [Edward Arcuri](https://github.com/sdndude), [Nitin Kumar](https://github.com/vnitinv), [Stacy Smith](https://github.com/stacywsmith), [Stephen Steiner](https://github.com/ntwrkguru)

RELEASE-NOTES.md

@@ -1,8 +1,29 @@
-## Release 2.7.2 - 12 APR 2024
+## Release 2.7.3.dev0 - 23 OCT 2024
 ## Features Added
 
 ## Bugs Fixed
 
+## Release 2.7.2 - 23 OCT 2024
+## Features Added
+- Introduced bind_addr parameter in Device() API #1279
+- Introduced vmhost paramater in dev.facts #1333
+- Introduced hostkey_verify paramater in Device() API #1321
+- Updated Docker file to use the latest Alpine #1316
+
+## Bugs Fixed
+
+- Fixed the missing key to EthernetSwitchingTable #1228
+- Fixed error handling on HelloHandler #1339
+- Fixed the version check #1338
+- Removed Google and Stackflow link from the ReadME #1337
+- Fixed SystemStorageTable tables and views to handles multiple routing-engine file system storage information.#1244
+- Fixed Console' object has no attribute '_use_filter' error when executed Table/View script #1335
+- Fixed cli function to get full RPC response #1315
+- Fixed sw.install to set no_validate option when validate=False for NSSU and ISSU upgrade #1323
+- Fixed UT framework mock to use built-in unittest.mock #1311
+- Fixed specific VC member reboot handling #1308 #1310
+- Supported latest paramiko version which supports aes128-gcm and aes256-gcm cipher
+
 ## Release 2.7.1 - 12 APR 2024
 ## Features Added
 - Added customer juniper paramiko module as a dependency  which supported aes128 and aes257 cipher #1299 

development.txt

@@ -6,4 +6,4 @@ pep8	     # https://github.com/jcrocholl/pep8
 pyflakes	 # https://launchpad.net/pyflakes
 coveralls	 # https://coveralls.io/
 ntc_templates # user needs to explicitly install this
-cryptography==3.2
+cryptography

docs/.readthedocs.yaml

@@ -0,0 +1,22 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: docs/docreq.txt

docs/docreq.txt

@@ -0,0 +1,12 @@
+git+https://github.com/ryan-roemer/sphinx-bootstrap-theme.git#egg=sphinx-bootstrap-theme
+lxml>=3.2.4
+ncclient==0.6.15
+scp>=0.7.0
+jinja2>=2.7.1
+PyYAML>=5.1
+paramiko>=3.5.0
+six
+pyserial
+yamlordereddictloader
+pyparsing
+transitions

entrypoint.sh

@@ -18,8 +18,11 @@ function pip_install {
 }
 
 function run_scripts {
-    echo "Executing defined script"
-    python3 $1 ${@:2}
+    if [ "$1" = "python3" ] || [ "$1" = "python" ]; then python3
+    else
+      echo "Executing defined script"
+      $1 ${@:2}
+    fi
 }
 
 ## Manually defined variables will take precedence

lib/jnpr/junos/console.py

@@ -114,6 +114,7 @@ def __init__(self, **kvargs):
         self._attempts = kvargs.get("attempts", 10)
         self._gather_facts = kvargs.get("gather_facts", False)
         self._fact_style = kvargs.get("fact_style", "new")
+        self._use_filter = False
         self._huge_tree = kvargs.get("huge_tree", False)
         if self._fact_style != "new":
             warnings.warn(

lib/jnpr/junos/device.py

@@ -735,6 +735,8 @@ def cli(self, command, format="text", warning=True):
             if rsp is True:
                 return ""
             if rsp.tag in ["output", "rpc-reply"]:
+                if rsp.tag == "output" and rsp.getparent() is not None:
+                    rsp = rsp.getparent()
                 encode = None if sys.version < "3" else "unicode"
                 return etree.tostring(
                     rsp, method="text", with_tail=False, encoding=encode
@@ -868,9 +870,8 @@ def execute(self, rpc_cmd, ignore_warning=False, **kvargs):
             "JSON",
         ]:
             ver_info = self.facts.get("version_info")
-            if (
-                ver_info
-                and ver_info.major[0] >= 15
+            if ver_info and (
+                ver_info.major[0] >= 15
                 or (ver_info.major[0] == 14 and ver_info.major[1] >= 2)
             ):
                 try:
@@ -1216,6 +1217,14 @@ def __init__(self, *vargs, **kvargs):
             *OPTIONAL* To disable public key authentication.
             default is ``None``.
 
+        :param str bind_addr:
+            *OPTIONAL* To use (local) source IP address.
+            default is ``None``.
+
+        :param bool hostkey_verify:
+            *OPTIONAL* To enable ssh_known hostkey verify
+            default is ``False``.
+
         """
 
         # ----------------------------------------
@@ -1234,6 +1243,8 @@ def __init__(self, *vargs, **kvargs):
         self._huge_tree = kvargs.get("huge_tree", False)
         self._conn_open_timeout = kvargs.get("conn_open_timeout", 30)
         self._look_for_keys = kvargs.get("look_for_keys", None)
+        self._bind_addr = kvargs.get("bind_addr", None)
+        self._hostkey_verify = kvargs.get("hostkey_verify", False)
         if self._fact_style != "new":
             warnings.warn(
                 "fact-style %s will be removed in a future "
@@ -1367,18 +1378,27 @@ def open(self, *vargs, **kvargs):
             else:
                 look_for_keys = self._look_for_keys
 
+            # option to enable ssh_known hosts key verification
+            # using hostkey_verify=True
+            # Default is disabled with hostkey_verify=False
+            if self._hostkey_verify is None:
+                hostkey_verify = False
+            else:
+                hostkey_verify = self._hostkey_verify
+
             # open connection using ncclient transport
             self._conn = netconf_ssh.connect(
                 host=self._hostname,
                 port=self._port,
                 sock_fd=self._sock_fd,
                 username=self._auth_user,
                 password=self._auth_password,
-                hostkey_verify=False,
+                hostkey_verify=hostkey_verify,
                 key_filename=self._ssh_private_key_file,
                 allow_agent=allow_agent,
                 look_for_keys=look_for_keys,
                 ssh_config=self._sshconf_lkup(),
+                bind_addr=self._bind_addr,
                 timeout=self._conn_open_timeout,
                 device_params={
                     "name": "junos",

lib/jnpr/junos/factory/optable.py

@@ -68,8 +68,8 @@ def get(self, *vargs, **kvargs):
                 rpc_args["filter_xml"] = filter_xml
             except Exception as ex:
                 logger.debug("Not able to create SAX parser input due to " "'%s'" % ex)
+            self.D.transform = lambda: remove_namespaces_and_spaces
 
-        self.D.transform = lambda: remove_namespaces_and_spaces
         rpc_args.update(self.GET_ARGS)  # copy default args
         # saltstack get_table pass args as named keyword
         if "args" in kvargs and isinstance(kvargs["args"], dict):

lib/jnpr/junos/facts/__init__.py

@@ -48,6 +48,7 @@
 import jnpr.junos.facts.personality
 import jnpr.junos.facts.swver
 import jnpr.junos.facts.is_linux
+import jnpr.junos.facts.vmhost
 
 
 def _build_fact_callbacks_and_doc_strings():

lib/jnpr/junos/facts/vmhost.py

@@ -0,0 +1,40 @@
+from jnpr.junos.exception import RpcError
+import re
+from lxml import etree
+
+
+def provides_facts():
+    """
+    Returns a dictionary keyed on the facts provided by this module. The value
+    of each key is the doc string describing the fact.
+    """
+    return {
+        "vmhost": "A boolean indicating if the device is vmhost.",
+    }
+
+
+def get_facts(device):
+    """
+    Gathers facts from the sysctl command.
+    """
+    SYSCTL_VMHOST_MODE = "sysctl -n hw.re.vmhost_mode"
+    vmhost = None
+
+    if device.facts["_is_linux"]:
+        vmhost = False
+    else:
+        try:
+            rsp = device.rpc.request_shell_execute(command=SYSCTL_VMHOST_MODE)
+            if rsp.tag == "rpc-error":
+                raise RpcError()
+            result = re.sub("<[^<]+>", "", etree.tostring(rsp).decode())
+            if result.strip() == "1":
+                vmhost = True
+            else:
+                vmhost = False
+        except RpcError:
+            pass
+
+    return {
+        "vmhost": vmhost,
+    }

lib/jnpr/junos/op/ethernetswitchingtable.yml

@@ -3,6 +3,7 @@ EthernetSwitchingTable:
     rpc: get-ethernet-switching-table-information
     args:
         detail: True
+    key: mac-table-count
     item: ethernet-switching-table
     view: EthernetSwitchingView
 

lib/jnpr/junos/op/systemstorage.yml

@@ -1,12 +1,23 @@
----
+  ---
 SystemStorageTable:
   rpc: get-system-storage
-  key: filesystem-name
-  item: filesystem
+  key: re-name | Null
+  item: //multi-routing-engine-item | //rpc-reply/system-storage-information
   view: SystemStorageView
 
 SystemStorageView:
   fields:
+    re-name: re-name
+    filesystems: _FsTable
+
+_FsTable:
+  item: system-storage-information/filesystem | //rpc-reply/system-storage-information/filesystem
+  key: filesystem-name
+  view: _FsView
+
+_FsView:
+  fields:
+    name: filesystem-name
     total_blocks: total-blocks
     used_blocks: used-blocks
     available_blocks: available-blocks

lib/jnpr/junos/transport/tty.py

@@ -52,7 +52,7 @@ class Terminal(object):
         "(?P<badpasswd>ogin incorrect)",
         r"(?P<netconf_closed><!-- session end at .*-->\s*)",
         r"(?P<shell>%|#|(~\$)\s*$)",
-        '(?P<cli>[^\\-"]>\s*$)',
+        r'(?P<cli>[^\-"]>\s*$)',
         r"(?P<option>Enter your option:\s*$)",
         "(?P<hotkey>connection: <CTRL>Z)",
     ]

lib/jnpr/junos/transport/tty_netconf.py

@@ -68,7 +68,9 @@ def open(self, at_shell):
             raise RuntimeError("Error: netconf not responding")
 
         self.hello = self._receive()
-        self._session_id, _ = HelloHandler.parse(self.hello.decode("utf-8"))
+        self._session_id, _ = HelloHandler.parse(
+            self.hello.decode("utf-8") if isinstance(self.hello, bytes) else self.hello
+        )
 
     def close(self, force=False):
         """issue the XML API to close the session"""