From f76fc35a7b8f63f4565bec9797f4ee410bc514c8 Mon Sep 17 00:00:00 2001 From: Michael Weinstein Date: Thu, 24 Dec 2020 18:24:30 -0800 Subject: [PATCH 1/4] Appears to be working for a connection to the new Optum gateway. At least for testing purposes. --- requirements.txt | 3 +- zymoTransmit.py | 35 +++++++++++++++++-- zymoTransmitSupport/config.py | 8 +++++ zymoTransmitSupport/inputOutput/connection.py | 20 ++++++++--- 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index c74dda3..b010ff0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pyOpenSSL==19.1.0 zipcodes==1.1.0 zeep==3.4.0 -pytk==0.0.2.0 \ No newline at end of file +pytk==0.0.2.0 +pathlib==1.0.1 \ No newline at end of file diff --git a/zymoTransmit.py b/zymoTransmit.py index 0dc45bb..59ea011 100644 --- a/zymoTransmit.py +++ b/zymoTransmit.py @@ -3,6 +3,7 @@ import sys import traceback import csv + import pathlib contentRoot = os.path.split(os.path.abspath(__file__))[0] @@ -198,6 +199,35 @@ def processRejects(resultList:typing.List[zymoTransmitSupport.inputOutput.result file.close() +def selectCertificatePath(): + if config.Connection.usingOptum: + if config.Configuration.productionReady: + certFile = config.Connection.optumProductionCertificate + keyFile = config.Connection.optumProductionKey + else: + certFile = config.Connection.optumTestingCertificate + keyFile = config.Connection.optumTestingKey + certPath = os.path.join(contentRoot, config.Connection.certificateFolder, certFile) + keyPath = os.path.join(contentRoot, config.Connection.certificateFolder, keyFile) + certificateFilePath = (certPath, keyPath) + else: + certificateFilePath = os.path.join(contentRoot, config.Connection.certificateFolder, config.Connection.certificateFileName) + return certificateFilePath + + +def selectURLForWSDL(): + if config.Connection.usingOptum: + if config.Configuration.productionReady: + relativePath = os.path.join(config.Connection.localWSDLFolder, config.Connection.optumProductionWSDL) + else: + relativePath = os.path.join(config.Connection.localWSDLFolder, config.Connection.optumTestingWSDL) + absolutePath = os.path.abspath(relativePath) + wsdlURL = pathlib.Path(absolutePath).as_uri() + else: + wsdlURL = config.Connection.wsdlURL + return wsdlURL + + def prepareAndSendResults(args:CheckArgs): if not args.hl7Directory: if os.path.abspath(args.input) == os.path.join(contentRoot, "config.txt"): @@ -216,9 +246,10 @@ def prepareAndSendResults(args:CheckArgs): quit() skippedData = [] resultList = [] - certificateFilePath = os.path.join(contentRoot, config.Connection.certificateFolder, config.Connection.certificateFileName) + certificateFilePath = selectCertificatePath() + wsdlURL = selectURLForWSDL() client, session = zymoTransmitSupport.inputOutput.connection.getSOAPClient( - config.Connection.wsdlURL, certificateFilePath, dumpClientInfo=False) + wsdlURL, certificateFilePath, dumpClientInfo=False, testOnly=False) if args.hl7Directory: if not os.path.isdir(args.input): raise NotADirectoryError("%s was given as a directory for raw HL7 block files, but it is not a directory.") diff --git a/zymoTransmitSupport/config.py b/zymoTransmitSupport/config.py index d194053..d3cfbb3 100644 --- a/zymoTransmitSupport/config.py +++ b/zymoTransmitSupport/config.py @@ -97,6 +97,14 @@ class Connection: certificateFileName = configDict.setdefault("certificate file name", "") userName = configDict.setdefault("gateway user", "") password = configDict.setdefault("gateway password", "") + usingOptum = configDict.setdefault("using optum", False) + localWSDLFolder = configDict.setdefault("local wsdl folder", "") + optumTestingWSDL = configDict.setdefault("optum testing wsdl file", "") + optumProductionWSDL = configDict.setdefault("optum production wsdl file", "") + optumTestingCertificate = configDict.setdefault("optum testing certificate", "") + optumTestingKey = configDict.setdefault("optum testing key", "") + optumProductionCertificate = configDict.setdefault("optum production certificate", "") + optumProductionKey = configDict.setdefault("optum production key", "") class Configuration: diff --git a/zymoTransmitSupport/inputOutput/connection.py b/zymoTransmitSupport/inputOutput/connection.py index 51fcf84..ca21cf4 100644 --- a/zymoTransmitSupport/inputOutput/connection.py +++ b/zymoTransmitSupport/inputOutput/connection.py @@ -26,11 +26,23 @@ def getSOAPClient(wsdlURL:str, clientCertificatePath:str=None, dumpClientInfo:bo if strongSSLCipherSuiteEnforcement: session.mount("https://", adapter=TLSEnforcer()) if clientCertificatePath: - if not os.path.isfile(clientCertificatePath): - raise FileNotFoundError("Unable to find client certificate path at %s" %clientCertificatePath) + if type(clientCertificatePath) == str: + if not os.path.isfile(clientCertificatePath): + raise FileNotFoundError("Unable to find client certificate path at %s" %clientCertificatePath) + else: + if type(clientCertificatePath) in [tuple, list]: + if not len(clientCertificatePath) == 2: + raise ValueError("List/tuple certificate paths should have exactly two elements with the first being the certificate and second being key.") + certPath, keyPath = clientCertificatePath + if not os.path.isfile(certPath) or not os.path.isfile(keyPath): + raise FileNotFoundError("Unable to find client certificate and/or key path.\nCert: %s\nKey: %s" %(certPath, keyPath)) session.cert = clientCertificatePath transport = zeep.transports.Transport(session=session, timeout=30) - client = zeep.Client(wsdlURL, transport=transport) + try: + client = zeep.Client(wsdlURL, transport=transport) + except OSError: + wsdlURL = wsdlURL.replace("file:///", "file://") # Seems like a weird quirk on windows where it wants to add a root / to the URI. Might be a bug somewhere I need to report. + client = zeep.Client(wsdlURL, transport=transport) #testUpload = client.service.connectivityTest("ping") if testOnly or dumpClientInfo: print("Server returned:") @@ -38,4 +50,4 @@ def getSOAPClient(wsdlURL:str, clientCertificatePath:str=None, dumpClientInfo:bo if testOnly: input("Press ENTER key to quit.") quit() - return client, session \ No newline at end of file + return client, session From 2e6efc12a1f0ed291f11d4ebd25beeab055b97d3 Mon Sep 17 00:00:00 2001 From: Michael Weinstein Date: Mon, 28 Dec 2020 11:40:43 -0800 Subject: [PATCH 2/4] Polishing up for production. Ready to test with real data. --- requirements.txt | 2 +- templateSubmission.csv | 2 +- zymoTransmitSupport/configClean.txt | 8 ++++++++ zymoTransmitSupport/inputOutput/connection.py | 2 +- zymoTransmitSupport/supportData.py | 4 ++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index b010ff0..cd91698 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pyOpenSSL==19.1.0 zipcodes==1.1.0 -zeep==3.4.0 +zeep==4.0.0 pytk==0.0.2.0 pathlib==1.0.1 \ No newline at end of file diff --git a/templateSubmission.csv b/templateSubmission.csv index fa0ff84..8e1ef2a 100644 --- a/templateSubmission.csv +++ b/templateSubmission.csv @@ -1,2 +1,2 @@ #PatientID,Patient Last Name,Patient First Name,Patient Middle Name,Patient Date of Birth,Patient Sex,Patient Street Address,Patient City,Patient State,Patient Zip,Patient Phone,Provider Last Name,Provider First Name,Provider Middle Name,Provider Street,Provider City,Provider State,Provider Zip,Provider Phone,Specimen ID,Collection Date,Collection Time,Received Date,Received Time,Specimen SNOMED Code,Test LOINC Code,Analysis Date,Analysis Time,Result,Reported Date,Reported Time,Notes,Race,Ethnicity,Accession ID,Test Ordered Date,Test Ordered Time,Test Equipment Code,Test Equipment Description,Test Equipment ID Number,Provider NPI -1K94U,Weinstein,Andrew,P,8/15/2017,M,17062 Murphy Ave. #1,Irvine,CA,92614,888-882-9682,Rain,Leo,,13842 Tustin East Drive,Tustin,CA,92780,(818) 344-3434,1234567,6/8/2020,0:00,6/8/2020,13:00,258529004,94306-8,6/9/2020,11:00,Negative,6/10/2020,14:00,"Phone is pretty flexible for formatting, just be reasonable. Email address is also an acceptable entry there, if needed. Please stick to mm/dd/yyyy formatting for dates. Having a # at the start of a line will cause that line to be ignored as a comment or header.",unknown,unk,1A2B3C4D5E,6/8/2020,0:00,qPCR,Zymo Research Quick SARS-CoV-2 rRT-PCR Kit,R3011, +#1K94U,Weinstein,Andrew,P,8/15/2017,M,17062 Murphy Ave. #1,Irvine,CA,92614,888-882-9682,Rain,Leo,,13842 Tustin East Drive,Tustin,CA,92780,(818) 344-3434,1234567,6/8/2020,0:00,6/8/2020,13:00,258529004,94306-8,6/9/2020,11:00,Negative,6/10/2020,14:00,"Phone is pretty flexible for formatting, just be reasonable. Email address is also an acceptable entry there, if needed. Please stick to mm/dd/yyyy formatting for dates. Having a # at the start of a line will cause that line to be ignored as a comment or header.",unknown,unk,1A2B3C4D5E,6/8/2020,0:00,qPCR,Zymo Research Quick SARS-CoV-2 rRT-PCR Kit,R3011, diff --git a/zymoTransmitSupport/configClean.txt b/zymoTransmitSupport/configClean.txt index 12f7121..c49e183 100644 --- a/zymoTransmitSupport/configClean.txt +++ b/zymoTransmitSupport/configClean.txt @@ -64,6 +64,14 @@ WSDL URL: https://hiegateway.cdph.ca.gov/submit/CDPH_transfer.wsdl Submission URL: https://hiegateway.cdph.ca.gov/submit/services/CDPH_transfer.CDPH_transferHttpsSoap12Endpoint Certificate Folder: certificates Certificate File Name: certificate.pem +Using Optum: FALSE +Local WSDL Folder: wsdl +Optum Testing WSDL File: ca_impl_wsdl.xml +Optum Production WSDL File: ca_prod_wsdl.xml +Optum Testing Certificate: impl-cie-ws.cer +Optum Testing Key: impl-cie-ws.key +Optum Production Certificate: odxi-ws.cer +Optum Production Key: odxi-ws.key ## Message Header Values Receiver Name: CDPH CA REDIE diff --git a/zymoTransmitSupport/inputOutput/connection.py b/zymoTransmitSupport/inputOutput/connection.py index ca21cf4..0773c99 100644 --- a/zymoTransmitSupport/inputOutput/connection.py +++ b/zymoTransmitSupport/inputOutput/connection.py @@ -41,7 +41,7 @@ def getSOAPClient(wsdlURL:str, clientCertificatePath:str=None, dumpClientInfo:bo try: client = zeep.Client(wsdlURL, transport=transport) except OSError: - wsdlURL = wsdlURL.replace("file:///", "file://") # Seems like a weird quirk on windows where it wants to add a root / to the URI. Might be a bug somewhere I need to report. + wsdlURL = wsdlURL.replace("file:///", "file://") # Seems like a weird quirk on windows where it wants to add a root / to the URI. Might be a bug somewhere I need to report. Actually, fixed in the latest zeep major release. client = zeep.Client(wsdlURL, transport=transport) #testUpload = client.service.connectivityTest("ping") if testOnly or dumpClientInfo: diff --git a/zymoTransmitSupport/supportData.py b/zymoTransmitSupport/supportData.py index b887cca..5dd48e8 100644 --- a/zymoTransmitSupport/supportData.py +++ b/zymoTransmitSupport/supportData.py @@ -1,4 +1,4 @@ -softwareVersion = "1.4.0" +softwareVersion = "2.0.0" -softwareDate = "20201201" \ No newline at end of file +softwareDate = "20201228" \ No newline at end of file From 9a3bbdd96f9476fc3aea20b2a7f8fb5b5c6143c7 Mon Sep 17 00:00:00 2001 From: Michael Weinstein Date: Tue, 5 Jan 2021 21:47:41 -0800 Subject: [PATCH 3/4] Upgrade ready for public release after talking to CDPH. Still needs directions for those looking to upgrade. --- README.md | 13 +++++++--- configUpgrade.txt | 11 +++++++++ wsdl/ca_impl_wsdl.xml | 57 +++++++++++++++++++++++++++++++++++++++++++ wsdl/ca_prod_wsdl.xml | 57 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 configUpgrade.txt create mode 100644 wsdl/ca_impl_wsdl.xml create mode 100644 wsdl/ca_prod_wsdl.xml diff --git a/README.md b/README.md index 716388a..3a337c1 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ It is with these principles guiding our efforts that we offer this software pack At present, there is not publication planned for this software. If anybody in the public health/epidemiology field wishes to collaborate on one (especially the Fielding School of Public Health), please contact us. ## Quick Start Guide -This Program was written in Python 3.6.4. It should work with other version of 3.6 and most likely later versions as well. +This Program was written in Python 3.8. It has also been tested with version 3.6 and most likely will work with other versions as well. Earlier versions may have some difficulties. Please report any compatibility problems you may have and I will see about addressing them if you really must run on a different version of Python. If you do not already have the Python language installed on your computer, you can download the web-based installer from this page on [python.org](https://www.python.org/downloads/release/python-364/) or directly from [here](https://www.python.org/ftp/python/3.6.4/python-3.6.4-amd64-webinstall.exe) on a Windows computer. **During Python installation, be sure to enable Python in your PATH variables, as that will make this program much easier to run in the future. Also be sure to approve installation of Tcl/Tk and PIP, as those will also contribute to making this easier to run in the future.** @@ -81,6 +81,12 @@ If you need to edit this file in the future, it can be found at ```zymoTransmit/ ``` python zymoTransmit.py ``` +**If NOT using a PFX certificate (you have CER and KEY files available)** + +You should have files called impl-cie-ws (for testing) and odxi-ws (for production) with endings of .cer and .key for each. Simply copy these credential files into the certificates folder in your Zymo Research Transmit directory. No further attention to these files should be required. + +**If using a PFX certificate for the legacy CDPH gateway** + Convert your certificate for use (if you do not supply a file name, a file browser window will appear if GUI functionality is available) ``` python zymoTransmit.py -c [fileName.pfx] @@ -179,9 +185,9 @@ Patch release: No changes to parameters ## Current Version -The current major release contains all initial functionality plus some additional features designed to help the California Department of Public Health clear backlogged data using their existing form. Many of these extended features will also help other users with their submissions. Essential features covered include transmission of data from this program's preferred table format (either CSV or tab-delimited text), transmission of a large block of HL7 data, transmission of a folder with individual HL7 data blocks, transmission of a CDPH-formatted CSV file, and handling of results that fail to transmit for various reasons. +The current version contains all functionality from the previous version with the new key feature being the ability to connect to the new Optum API for reporting results in California. -This major release has been nicknamed Imahara's Pudding Cup after [Grant Imahara](https://en.wikipedia.org/wiki/Grant_Imahara), a popular advocate of STEAM education and performing random acts of kindness for others. +This major release has been nicknamed Geoff Peterson's Metal Mohawk in remembrance of [Grant Imahara](https://en.wikipedia.org/wiki/Grant_Imahara), a popular advocate of STEAM education and performing random acts of kindness for others. ## Authors @@ -202,6 +208,7 @@ We would like to thank the following, without whom this would not have happened: * The Python Foundation * The staff at Zymo Research * The UC System + - Special thanks to UCLA's departments of Molecular, Cell, and Developmental Biology as well as Computational Medicine * Pangea Laboratory * The California Department of Public Health * Our customers diff --git a/configUpgrade.txt b/configUpgrade.txt new file mode 100644 index 0000000..71cb087 --- /dev/null +++ b/configUpgrade.txt @@ -0,0 +1,11 @@ +## Copy the text below into your config file if you are upgrading from version 1.x of this program. +## The location is not important, but adding it to the end of the connection section would be ideal location + +Using Optum: FALSE +Local WSDL Folder: wsdl +Optum Testing WSDL File: ca_impl_wsdl.xml +Optum Production WSDL File: ca_prod_wsdl.xml +Optum Testing Certificate: impl-cie-ws.cer +Optum Testing Key: impl-cie-ws.key +Optum Production Certificate: odxi-ws.cer +Optum Production Key: odxi-ws.key \ No newline at end of file diff --git a/wsdl/ca_impl_wsdl.xml b/wsdl/ca_impl_wsdl.xml new file mode 100644 index 0000000..7a742b2 --- /dev/null +++ b/wsdl/ca_impl_wsdl.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/wsdl/ca_prod_wsdl.xml b/wsdl/ca_prod_wsdl.xml new file mode 100644 index 0000000..1a0cde3 --- /dev/null +++ b/wsdl/ca_prod_wsdl.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 10098111a5cc3a9fbe0b8151d3afa8554a45a7fb Mon Sep 17 00:00:00 2001 From: Michael Weinstein Date: Wed, 6 Jan 2021 11:05:34 -0800 Subject: [PATCH 4/4] Update race codes for unknown --- raceEthnicityCodes.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raceEthnicityCodes.txt b/raceEthnicityCodes.txt index 595bc10..6dcba9f 100644 --- a/raceEthnicityCodes.txt +++ b/raceEthnicityCodes.txt @@ -9,4 +9,4 @@ PHC1369 not obtainable PHC1367 refused ASKU asked but unknown -UNK unknown u +U unknown unk