diff --git a/Dockerfile b/Dockerfile index e5a3afdfa..567130a2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM alpine:3.12 +FROM python:3.12-alpine LABEL net.juniper.description="Junos PyEZ library for Python in a lightweight container." \ - net.juniper.maintainer="Stephen Steiner " + net.juniper.maintainer="jnpr-community-netdev@juniper.net" WORKDIR /source -## Copy project inside the container +## Copy project inside the containers ADD setup.* ./ ADD versioneer.py . ADD requirements.txt . @@ -13,14 +13,17 @@ ADD lib lib ADD entrypoint.sh /usr/local/bin/. ## Install dependancies and PyEZ -RUN apk add --no-cache build-base python3-dev py-lxml \ +RUN apk add --no-cache build-base python3-dev \ libxslt-dev libxml2-dev libffi-dev openssl-dev curl \ - ca-certificates py3-pip bash \ - && pip install -U pip \ - && pip install -r requirements.txt \ - && apk del -r --purge gcc make g++ \ - && ln -s /usr/bin/python3 /usr/bin/python \ - && pip install . \ + ca-certificates py3-pip bash + +RUN pip install --upgrade pip \ + && pip install pipdeptree \ + && python3 -m pip install -r requirements.txt \ + && pip install . + +## Clean up and start init +RUN apk del -r --purge gcc make g++ \ && rm -rf /source/* \ && chmod +x /usr/local/bin/entrypoint.sh diff --git a/README.md b/README.md index bc1f48c9e..8747559a5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![PyPi Version](https://img.shields.io/pypi/v/junos-eznc.svg)](https://pypi.python.org/pypi/junos-eznc/) -[![Documentation Status](https://readthedocs.org/projects/junos-pyez/badge/?version=stable)](http://junos-pyez.readthedocs.io) +[![Documentation Status](https://readthedocs.org/projects/junos-pyez/badge/?version=stable)](https://junos-pyez.readthedocs.io/en/latest/) [![Coverage Status](https://img.shields.io/coveralls/Juniper/py-junos-eznc.svg)](https://coveralls.io/r/Juniper/py-junos-eznc) [![UnitTest Status](https://travis-ci.org/Juniper/py-junos-eznc.svg?branch=master)](https://travis-ci.org/Juniper/py-junos-eznc) [![](https://images.microbadger.com/badges/image/juniper/pyez.svg)](https://microbadger.com/images/juniper/pyez) @@ -29,15 +29,11 @@ This means that "non-programmers", for example the _Network Engineer_, can use t ## For "Programmers" - Open and Extensible -There is a growing interest and need to automate the network infrastructure into larger IT systems. To do so, traditional software programmers, DevOps, "hackers", etc. need an abstraction library of code to further those activities. _Junos PyEZ_ is designed for extensibility so that the programmer can quickly and easily add new widgets to the library in support of their specific project requirements. There is no need to "wait on the vendor" to provide new functionality. _Junos PyEZ_ is not specifically tied to any version of Junos or any Junos product family. +There is a growing interest and need to automate the network infrastructure into larger IT systems. To do so, traditional software programmers, DevOps, "hackers", etc. need an abstraction library of code to further those activities. _Junos PyEZ_ is designed for extensibility so that the programmer can quickly and easily add new widgets to the library in support of their specific project requirements. There is no need to "wait on the vendor" to provide new functionality. _Junos PyEZ_ is not specifically tied to any version of Junos (or Junos Evolved) or any Junos (or Junos Evolved) product family. # SUPPORT -For questions and general support, please visit our [Google Group](https://groups.google.com/forum/#!forum/junos-python-ez) - -You can also post your query on stackoverflow with __pyez__ [tag](http://stackoverflow.com/questions/tagged/pyez) - -For documentation and more usage examples, please visit the _Junos PyEZ_ project page, [here](http://forums.juniper.net/t5/Automation/Where-can-I-learn-more-about-Junos-PyEZ/ta-p/280496). +For documentation and more usage examples, please visit the _Junos PyEZ_ project page, [here](http://www.juniper.net/techpubs/en_US/release-independent/junos-pyez/information-products/pathway-pages/index.html). Issues and bugs can be opened in the repository. @@ -54,7 +50,7 @@ _Junos PyEZ_ is designed to provide the same capabilities as a user would have o # NOTICES -- As of release 2.0.0, _Junos PyEZ_ requires [ncclient](https://pypi.python.org/pypi/ncclient) version 0.5.2 or later. +- As of release 2.7.1, _Junos PyEZ_ requires [ncclient](https://pypi.python.org/pypi/ncclient) version 0.6.15 or later. - When using the `ssh_private_key_file` argument of the Device constructor on MacOS Mojave and higher, ensure that the SSH keys are in the RSA format, and not the newer OPENSSH format. - New key: `ssh-keygen -p -m PEM -f ~/.ssh/id_rsa` - Convert an existing OPENSSH key: ``ssh-keygen -p -m PEM -f ~/.ssh/private_key` @@ -88,11 +84,16 @@ Move to the local directory which contains your script(s) and run the container. Your local scripts will be mounted to /scripts in the container. +As per PEP 668, use Virtual Env in the container (after you enter the container). The docker container has virtual environment installed in the /scripts folder. + +Use `source .venv/bin/activate` to activate the virtual environment. + + ### Microservice Usage This image can also be used as a Python "executable" with the required Python PyEZ libraries pre-installed. To use the image in this way, mount the volume which contains the Python script and pass the script name as an argument to `docker run`. Optionally, you may also pass in a `requirements.txt` file to install additional python packages via `pip`. To add OS packages (Alpine Linux), provide a file with a list of packages --one per line-- and either reference it as an env var (`$APK`) or mount it to the container `/extras/apk.txt`. To add additional Python packages (via pip), provide a `requirements.txt` file and pass it in as an env var (`$REQ`) or mount it to the container at `/extras/requirements.txt`. -`Usage: `docker run -it [ --rm ] -v some/dir:/scripts juniper/pyez [ myscript.py ]` +Usage: `docker run -it [ --rm ] -v some/dir:/scripts juniper/pyez [ myscript.py ]` Example: @@ -153,8 +154,11 @@ Juniper Networks is actively contributing to and maintaining this repo. Please c *Contributors:* -[Nitin Kumar](https://github.com/vnitinv), [Stacy Smith](https://github.com/stacywsmith), [Stephen Steiner](https://github.com/ntwrkguru) - +* v2.7.2: [Dinesh Babu](https://github.com/dineshbaburam91), [Chidanand Pujar](https://github.com/chidanandpujar) +* v2.7.1: [Dinesh Babu](https://github.com/dineshbaburam91) +* v2.6.4: [Chidanand Pujar](https://github.com/chidanandpujar) +* v2.6.3: [Rahul Kumar](https://github.com/rahkumar651991) +* v2.5.0: [Rahul Kumar](https://github.com/rahkumar651991) * v2.4.1: [Nitin Kumar](https://github.com/vnitinv) * v2.4.0: [Nitin Kumar](https://github.com/vnitinv) * v2.3.0: [Nitin Kumar](https://github.com/vnitinv), [Raja Shekar Mekala](https://github.com/rsmekala), [Dinesh Babu](https://github.com/dineshbaburam91), [Chris Jenn](https://github.com/ipmonk), [Shigechika](https://github.com/shigechika) @@ -167,4 +171,4 @@ Juniper Networks is actively contributing to and maintaining this repo. Please c *Former Contributors:* -[Jeremy Schulman](https://github.com/jeremyschulman), [Rick Sherman](https://github.com/shermdog), [Edward Arcuri](https://github.com/sdndude) +[Jeremy Schulman](https://github.com/jeremyschulman), [Rick Sherman](https://github.com/shermdog), [Edward Arcuri](https://github.com/sdndude), [Nitin Kumar](https://github.com/vnitinv), [Stacy Smith](https://github.com/stacywsmith), [Stephen Steiner](https://github.com/ntwrkguru) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index da92bb139..222c12309 100755 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,8 +1,29 @@ -## Release 2.7.2 - 12 APR 2024 +## Release 2.7.3.dev0 - 23 OCT 2024 ## Features Added ## Bugs Fixed +## Release 2.7.2 - 23 OCT 2024 +## Features Added +- Introduced bind_addr parameter in Device() API #1279 +- Introduced vmhost paramater in dev.facts #1333 +- Introduced hostkey_verify paramater in Device() API #1321 +- Updated Docker file to use the latest Alpine #1316 + +## Bugs Fixed + +- Fixed the missing key to EthernetSwitchingTable #1228 +- Fixed error handling on HelloHandler #1339 +- Fixed the version check #1338 +- Removed Google and Stackflow link from the ReadME #1337 +- Fixed SystemStorageTable tables and views to handles multiple routing-engine file system storage information.#1244 +- Fixed Console' object has no attribute '_use_filter' error when executed Table/View script #1335 +- Fixed cli function to get full RPC response #1315 +- Fixed sw.install to set no_validate option when validate=False for NSSU and ISSU upgrade #1323 +- Fixed UT framework mock to use built-in unittest.mock #1311 +- Fixed specific VC member reboot handling #1308 #1310 +- Supported latest paramiko version which supports aes128-gcm and aes256-gcm cipher + ## Release 2.7.1 - 12 APR 2024 ## Features Added - Added customer juniper paramiko module as a dependency which supported aes128 and aes257 cipher #1299 diff --git a/development.txt b/development.txt index 24a995026..84be0bed3 100644 --- a/development.txt +++ b/development.txt @@ -6,4 +6,4 @@ pep8 # https://github.com/jcrocholl/pep8 pyflakes # https://launchpad.net/pyflakes coveralls # https://coveralls.io/ ntc_templates # user needs to explicitly install this -cryptography==3.2 +cryptography diff --git a/docreq.txt b/docreq.txt deleted file mode 100644 index e57488aa1..000000000 --- a/docreq.txt +++ /dev/null @@ -1 +0,0 @@ -git+https://github.com/ryan-roemer/sphinx-bootstrap-theme.git#egg=sphinx-bootstrap-theme diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 000000000..6c4ddaa5a --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,22 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.8" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# We recommend specifying your dependencies to enable reproducible builds: +# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/docreq.txt diff --git a/docs/docreq.txt b/docs/docreq.txt new file mode 100644 index 000000000..f95fb895b --- /dev/null +++ b/docs/docreq.txt @@ -0,0 +1,12 @@ +git+https://github.com/ryan-roemer/sphinx-bootstrap-theme.git#egg=sphinx-bootstrap-theme +lxml>=3.2.4 +ncclient==0.6.15 +scp>=0.7.0 +jinja2>=2.7.1 +PyYAML>=5.1 +paramiko>=3.5.0 +six +pyserial +yamlordereddictloader +pyparsing +transitions diff --git a/entrypoint.sh b/entrypoint.sh index af403b5f7..ecf2b2fbc 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -18,8 +18,11 @@ function pip_install { } function run_scripts { - echo "Executing defined script" - python3 $1 ${@:2} + if [ "$1" = "python3" ] || [ "$1" = "python" ]; then python3 + else + echo "Executing defined script" + $1 ${@:2} + fi } ## Manually defined variables will take precedence diff --git a/lib/jnpr/junos/console.py b/lib/jnpr/junos/console.py index 5c96614e7..5fe232a35 100644 --- a/lib/jnpr/junos/console.py +++ b/lib/jnpr/junos/console.py @@ -114,6 +114,7 @@ def __init__(self, **kvargs): self._attempts = kvargs.get("attempts", 10) self._gather_facts = kvargs.get("gather_facts", False) self._fact_style = kvargs.get("fact_style", "new") + self._use_filter = False self._huge_tree = kvargs.get("huge_tree", False) if self._fact_style != "new": warnings.warn( diff --git a/lib/jnpr/junos/device.py b/lib/jnpr/junos/device.py index 641806803..27133661c 100644 --- a/lib/jnpr/junos/device.py +++ b/lib/jnpr/junos/device.py @@ -735,6 +735,8 @@ def cli(self, command, format="text", warning=True): if rsp is True: return "" if rsp.tag in ["output", "rpc-reply"]: + if rsp.tag == "output" and rsp.getparent() is not None: + rsp = rsp.getparent() encode = None if sys.version < "3" else "unicode" return etree.tostring( rsp, method="text", with_tail=False, encoding=encode @@ -868,9 +870,8 @@ def execute(self, rpc_cmd, ignore_warning=False, **kvargs): "JSON", ]: ver_info = self.facts.get("version_info") - if ( - ver_info - and ver_info.major[0] >= 15 + if ver_info and ( + ver_info.major[0] >= 15 or (ver_info.major[0] == 14 and ver_info.major[1] >= 2) ): try: @@ -1216,6 +1217,14 @@ def __init__(self, *vargs, **kvargs): *OPTIONAL* To disable public key authentication. default is ``None``. + :param str bind_addr: + *OPTIONAL* To use (local) source IP address. + default is ``None``. + + :param bool hostkey_verify: + *OPTIONAL* To enable ssh_known hostkey verify + default is ``False``. + """ # ---------------------------------------- @@ -1234,6 +1243,8 @@ def __init__(self, *vargs, **kvargs): self._huge_tree = kvargs.get("huge_tree", False) self._conn_open_timeout = kvargs.get("conn_open_timeout", 30) self._look_for_keys = kvargs.get("look_for_keys", None) + self._bind_addr = kvargs.get("bind_addr", None) + self._hostkey_verify = kvargs.get("hostkey_verify", False) if self._fact_style != "new": warnings.warn( "fact-style %s will be removed in a future " @@ -1367,6 +1378,14 @@ def open(self, *vargs, **kvargs): else: look_for_keys = self._look_for_keys + # option to enable ssh_known hosts key verification + # using hostkey_verify=True + # Default is disabled with hostkey_verify=False + if self._hostkey_verify is None: + hostkey_verify = False + else: + hostkey_verify = self._hostkey_verify + # open connection using ncclient transport self._conn = netconf_ssh.connect( host=self._hostname, @@ -1374,11 +1393,12 @@ def open(self, *vargs, **kvargs): sock_fd=self._sock_fd, username=self._auth_user, password=self._auth_password, - hostkey_verify=False, + hostkey_verify=hostkey_verify, key_filename=self._ssh_private_key_file, allow_agent=allow_agent, look_for_keys=look_for_keys, ssh_config=self._sshconf_lkup(), + bind_addr=self._bind_addr, timeout=self._conn_open_timeout, device_params={ "name": "junos", diff --git a/lib/jnpr/junos/factory/optable.py b/lib/jnpr/junos/factory/optable.py index 4e22cd318..8078bd8a8 100644 --- a/lib/jnpr/junos/factory/optable.py +++ b/lib/jnpr/junos/factory/optable.py @@ -68,8 +68,8 @@ def get(self, *vargs, **kvargs): rpc_args["filter_xml"] = filter_xml except Exception as ex: logger.debug("Not able to create SAX parser input due to " "'%s'" % ex) + self.D.transform = lambda: remove_namespaces_and_spaces - self.D.transform = lambda: remove_namespaces_and_spaces rpc_args.update(self.GET_ARGS) # copy default args # saltstack get_table pass args as named keyword if "args" in kvargs and isinstance(kvargs["args"], dict): diff --git a/lib/jnpr/junos/facts/__init__.py b/lib/jnpr/junos/facts/__init__.py index 7ac4317ae..db28fdfed 100644 --- a/lib/jnpr/junos/facts/__init__.py +++ b/lib/jnpr/junos/facts/__init__.py @@ -48,6 +48,7 @@ import jnpr.junos.facts.personality import jnpr.junos.facts.swver import jnpr.junos.facts.is_linux +import jnpr.junos.facts.vmhost def _build_fact_callbacks_and_doc_strings(): diff --git a/lib/jnpr/junos/facts/vmhost.py b/lib/jnpr/junos/facts/vmhost.py new file mode 100644 index 000000000..89e59b339 --- /dev/null +++ b/lib/jnpr/junos/facts/vmhost.py @@ -0,0 +1,40 @@ +from jnpr.junos.exception import RpcError +import re +from lxml import etree + + +def provides_facts(): + """ + Returns a dictionary keyed on the facts provided by this module. The value + of each key is the doc string describing the fact. + """ + return { + "vmhost": "A boolean indicating if the device is vmhost.", + } + + +def get_facts(device): + """ + Gathers facts from the sysctl command. + """ + SYSCTL_VMHOST_MODE = "sysctl -n hw.re.vmhost_mode" + vmhost = None + + if device.facts["_is_linux"]: + vmhost = False + else: + try: + rsp = device.rpc.request_shell_execute(command=SYSCTL_VMHOST_MODE) + if rsp.tag == "rpc-error": + raise RpcError() + result = re.sub("<[^<]+>", "", etree.tostring(rsp).decode()) + if result.strip() == "1": + vmhost = True + else: + vmhost = False + except RpcError: + pass + + return { + "vmhost": vmhost, + } diff --git a/lib/jnpr/junos/op/ethernetswitchingtable.yml b/lib/jnpr/junos/op/ethernetswitchingtable.yml index 1cfc7601b..c5b25e30a 100644 --- a/lib/jnpr/junos/op/ethernetswitchingtable.yml +++ b/lib/jnpr/junos/op/ethernetswitchingtable.yml @@ -3,6 +3,7 @@ EthernetSwitchingTable: rpc: get-ethernet-switching-table-information args: detail: True + key: mac-table-count item: ethernet-switching-table view: EthernetSwitchingView diff --git a/lib/jnpr/junos/op/systemstorage.yml b/lib/jnpr/junos/op/systemstorage.yml index 86531ae64..d61125e37 100644 --- a/lib/jnpr/junos/op/systemstorage.yml +++ b/lib/jnpr/junos/op/systemstorage.yml @@ -1,12 +1,23 @@ ---- + --- SystemStorageTable: rpc: get-system-storage - key: filesystem-name - item: filesystem + key: re-name | Null + item: //multi-routing-engine-item | //rpc-reply/system-storage-information view: SystemStorageView SystemStorageView: fields: + re-name: re-name + filesystems: _FsTable + +_FsTable: + item: system-storage-information/filesystem | //rpc-reply/system-storage-information/filesystem + key: filesystem-name + view: _FsView + +_FsView: + fields: + name: filesystem-name total_blocks: total-blocks used_blocks: used-blocks available_blocks: available-blocks diff --git a/lib/jnpr/junos/transport/tty.py b/lib/jnpr/junos/transport/tty.py index 89dc53214..f16a9a920 100644 --- a/lib/jnpr/junos/transport/tty.py +++ b/lib/jnpr/junos/transport/tty.py @@ -52,7 +52,7 @@ class Terminal(object): "(?Pogin incorrect)", r"(?P\s*)", r"(?P%|#|(~\$)\s*$)", - '(?P[^\\-"]>\s*$)', + r'(?P[^\-"]>\s*$)', r"(?P