From 16a9d6b2e31545e5b94de68863131885d03f9286 Mon Sep 17 00:00:00 2001 From: Warren Pettee Date: Thu, 8 Mar 2018 09:06:23 -0600 Subject: [PATCH 1/4] Adds (very) basic IEM ASOS capabilities, more work to come. First attempt at Unidata/siphon#157. --- siphon/simplewebservice/iem.py | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 siphon/simplewebservice/iem.py diff --git a/siphon/simplewebservice/iem.py b/siphon/simplewebservice/iem.py new file mode 100644 index 000000000..c225ae80a --- /dev/null +++ b/siphon/simplewebservice/iem.py @@ -0,0 +1,55 @@ +""" Requests data from IEM """ +# ASOS : +# http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?station=LNK&data=all&year1=2018&month1=3&day1=8&year2=2018&month2=3&day2=8&tz=Etc%2FUTC&format=onlycomma&latlon=no&direct=no&report_type=1&report_type=2 + +import requests +import datetime + +from ..http_util import create_http_session + + +class IemAsos: + """ + IEM ASOS data object. Handles data collection. + """ + def __init__(self, state, sites, startDate=None, endDate=None): + if startDate is None: + self.startDate = datetime.datetime.now() + self.endDate = datetime.datetime.now() + elif endDate is None: + self.endDate = datetime.datetime(today) + + self.state = state + self.sites = sites + self.getData() + self.parseData() + + def getData(self): + """ + Downloads IEM ASOS data + """ + URL = 'http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?' + + for site in self.sites: + URL = URL + '&station=' + site + + URL = URL + '&data=all' + + URL = URL + '&year1='+str(self.startDate.year) + URL = URL + '&month1='+str(self.startDate.month) + URL = URL + '&day1='+str(self.startDate.day) + + URL = URL + '&year2='+str(self.endDate.year) + URL = URL + '&month2='+str(self.endDate.month) + URL = URL + '&day2='+str(self.endDate.day) + + URL = URL + '&tz=Etc%2FUTC&format=onlycomma&latlon=yes&direct=no&report_type=1&report_type=2' + print(URL) + response = create_http_session().post(URL) + self.rawData = response.text + + def parseData(self): + """ + Parses IEM ASOS data returned by getData method. + """ + print(self.rawData) From adab1760ea4f30d4764843b76dce01f0fcc8ef0c Mon Sep 17 00:00:00 2001 From: Warren Pettee Date: Mon, 19 Mar 2018 16:02:35 -0500 Subject: [PATCH 2/4] Adds a roughcut version of parsing the response text from IEM into lists. --- siphon/simplewebservice/iem.py | 54 +++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/siphon/simplewebservice/iem.py b/siphon/simplewebservice/iem.py index c225ae80a..c6e9eaa8e 100644 --- a/siphon/simplewebservice/iem.py +++ b/siphon/simplewebservice/iem.py @@ -12,14 +12,13 @@ class IemAsos: """ IEM ASOS data object. Handles data collection. """ - def __init__(self, state, sites, startDate=None, endDate=None): + def __init__(self, sites, startDate=None, endDate=None): if startDate is None: self.startDate = datetime.datetime.now() self.endDate = datetime.datetime.now() elif endDate is None: - self.endDate = datetime.datetime(today) + self.endDate = datetime.datetime.now() - self.state = state self.sites = sites self.getData() self.parseData() @@ -52,4 +51,51 @@ def parseData(self): """ Parses IEM ASOS data returned by getData method. """ - print(self.rawData) + + splitData = self.rawData.split('\n') + + i = 0 + data = [] + head = [] + for row in splitData: + subRow = row.split(',') + + if i == 0: + headCount = 0 + for element in subRow: + if element == 'valid': + timeSlot = headCount + head.append(element.lstrip()) + headCount += 1 + i += 1 + continue + + if len(subRow) < len(head): + continue + + entry = [] + elemCount = 0 + for element in subRow: + eType = head[elemCount] + if elemCount == timeSlot: + element = datetime.datetime.strptime(element, '%Y-%m-%d %H:%M') + + if eType == 'station' or eType[:4] == 'skyc' or eType[:4] == 'skyl' or eType == 'wxcodes' or eType == 'metar' or eType == 'valid': + entry.append(element) + else: + if element == 'M': + entry.append(None) + else: + entry.append(float(element)) + + elemCount += 1 + data.append(entry) + + i += 1 + + self.asosData = data + self.headers = head + + print(self.headers) + print(len(self.asosData[0])) + print(self.asosData[1]) From 9a24766cec9b4adcf7d45f2da6bf2f7108b371ee Mon Sep 17 00:00:00 2001 From: Warren Pettee Date: Fri, 26 Oct 2018 10:12:17 -0500 Subject: [PATCH 3/4] Adds Pandas dataframe ability to IemAsos class. Also adds unit test for IemAsos Class. --- siphon/simplewebservice/iem.py | 134 +++++++++++++++--------------- siphon/tests/fixtures/iem_request | 28 +++++++ siphon/tests/test_iem.py | 25 ++++++ 3 files changed, 122 insertions(+), 65 deletions(-) create mode 100644 siphon/tests/fixtures/iem_request create mode 100644 siphon/tests/test_iem.py diff --git a/siphon/simplewebservice/iem.py b/siphon/simplewebservice/iem.py index c6e9eaa8e..1ae3530af 100644 --- a/siphon/simplewebservice/iem.py +++ b/siphon/simplewebservice/iem.py @@ -1,32 +1,64 @@ -""" Requests data from IEM """ -# ASOS : -# http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?station=LNK&data=all&year1=2018&month1=3&day1=8&year2=2018&month2=3&day2=8&tz=Etc%2FUTC&format=onlycomma&latlon=no&direct=no&report_type=1&report_type=2 +""" Requests data from IEM asos.py""" import requests import datetime +import pandas as pd + +from io import StringIO from ..http_util import create_http_session class IemAsos: - """ - IEM ASOS data object. Handles data collection. + """Handles data collection of ASOS data from IEM. + + This handles the collection of ASOS data from the Iowa + Environmental Mesonet, via their asos.py request URL. + + Attributes + ---------- + startDate : datetime + The starting date for the dataset + endDate : datetime + The ending date for the dataset + sites : list + Station IDs in the dataset + data : pandas.DataFrame + Pandas dataframe containing the IEM ASOS data + """ def __init__(self, sites, startDate=None, endDate=None): + """Initialize the IemAsos object. + + Initialization will set the datetime objects and + start the data call + + Parameters + ---------- + sites : list + List of station ID's to request data for + startDate : datetime + The start date as a datetime object + endDate : datetime + The end date as a datetime object + """ if startDate is None: self.startDate = datetime.datetime.now() self.endDate = datetime.datetime.now() elif endDate is None: self.endDate = datetime.datetime.now() + self.startDate = startDate + else: + self.startDate = startDate + self.endDate = endDate self.sites = sites self.getData() - self.parseData() def getData(self): - """ - Downloads IEM ASOS data - """ + """ Downloads IEM ASOS data """ + + # Build the URL URL = 'http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?' for site in self.sites: @@ -43,59 +75,31 @@ def getData(self): URL = URL + '&day2='+str(self.endDate.day) URL = URL + '&tz=Etc%2FUTC&format=onlycomma&latlon=yes&direct=no&report_type=1&report_type=2' - print(URL) - response = create_http_session().post(URL) - self.rawData = response.text - - def parseData(self): - """ - Parses IEM ASOS data returned by getData method. - """ - - splitData = self.rawData.split('\n') - - i = 0 - data = [] - head = [] - for row in splitData: - subRow = row.split(',') - - if i == 0: - headCount = 0 - for element in subRow: - if element == 'valid': - timeSlot = headCount - head.append(element.lstrip()) - headCount += 1 - i += 1 - continue - - if len(subRow) < len(head): - continue - - entry = [] - elemCount = 0 - for element in subRow: - eType = head[elemCount] - if elemCount == timeSlot: - element = datetime.datetime.strptime(element, '%Y-%m-%d %H:%M') - - if eType == 'station' or eType[:4] == 'skyc' or eType[:4] == 'skyl' or eType == 'wxcodes' or eType == 'metar' or eType == 'valid': - entry.append(element) - else: - if element == 'M': - entry.append(None) - else: - entry.append(float(element)) - - elemCount += 1 - data.append(entry) - - i += 1 - - self.asosData = data - self.headers = head - - print(self.headers) - print(len(self.asosData[0])) - print(self.asosData[1]) + + # Collect the data + try: + response = create_http_session().post(URL) + csvData = StringIO(response.text) + except requests.exceptions.Timeout: + raise IemAsosException('Connection Timeout') + + # Process the data into a dataframe + ''' + Convert the response text into a DataFrame. The index_col ensures that the first + column isn't used as a row identifier. This prevents the station IDs from being used + as row indices. + ''' + df = pd.read_csv(csvData, header=0, sep=',', index_col=False) + + # Strip whitespace from the column names + df.columns = df.columns.str.strip() + + df['valid'] = pd.to_datetime(df['valid'], format="%Y-%m-%d %H:%M:%S") + + self.data = df + + +class IemAsosException(Exception): + """This class handles exceptions raised by the IemAsos class.""" + + pass diff --git a/siphon/tests/fixtures/iem_request b/siphon/tests/fixtures/iem_request new file mode 100644 index 000000000..2cf3e716a --- /dev/null +++ b/siphon/tests/fixtures/iem_request @@ -0,0 +1,28 @@ +interactions: +- request: + body: null + headers: + Accept: ['*/*'] + Accept-Encoding: ['gzip, deflate'] + Connection: [keep-alive] + Content-Length: ['0'] + User-Agent: [Siphon (0.6.0+62.gadab176.dirty)] + method: POST + uri: http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?&station=K&station=L&station=N&station=K&station=,&station=%20&station=K&station=G&station=S&station=O&station=,&station=%20&station=K&station=B&station=D&station=U&data=all&year1=2018&month1=1&day1=1&year2=2018&month2=1&day2=1&tz=Etc%2FUTC&format=onlycomma&latlon=yes&direct=no&report_type=1&report_type=2 + response: + body: {string: 'station,valid,lon,lat,tmpf, dwpf, relh, drct, sknt, p01i, alti, + mslp, vsby, gust, skyc1, skyc2, skyc3, skyc4, skyl1, skyl2, skyl3, skyl4, + wxcodes, metar + + '} + headers: + Access-Control-Allow-Origin: ['*'] + Connection: [Keep-Alive] + Content-Type: [text/plain; charset=UTF-8] + Date: ['Fri, 26 Oct 2018 15:01:31 GMT'] + Keep-Alive: ['timeout=5, max=100'] + Server: [Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips mod_fcgid/2.3.9 + mod_wsgi/4.6.4 Python/3.6] + X-IEM-ServerID: [iemvs103.local] + status: {code: 200, message: OK} +version: 1 diff --git a/siphon/tests/test_iem.py b/siphon/tests/test_iem.py new file mode 100644 index 000000000..78d3cb2c5 --- /dev/null +++ b/siphon/tests/test_iem.py @@ -0,0 +1,25 @@ +"""Test IEM Asos Data Downloading.""" +from datetime import datetime + +from siphon.simplewebservice.iem import IemAsos +from siphon.testing import get_recorder + +recorder = get_recorder(__file__) + + +@recorder.use_cassette('iem_request') + +def test_iem_download(): + """Test the downloading of IEM ASOS data""" + start = datetime(2018,1,1) + end = datetime(2018,1,1) + + asosCall = IemAsos(['KLNK', 'KGSO', 'KBDU'], start, end) + + stn1 = asosCall.data[(asosCall.data.station == 'GSO')] + stn2 = asosCall.data[(asosCall.data.station == 'LNK')] + stn3 = asosCall.data[(asosCall.data.station == 'BDU')] + + assert stn1['station'][1] == 'GSO' + assert stn2['station'][1] == 'LNK' + assert stn3['station'][1] == 'BDU' From 38b98922ee189977cca1677a4c4f1134e03ad3cf Mon Sep 17 00:00:00 2001 From: Warren Pettee Date: Fri, 26 Oct 2018 10:26:26 -0500 Subject: [PATCH 4/4] Fixes PEP8 formatting issues. Ready for PR to discuss addressing Unidata#157. --- siphon/simplewebservice/iem.py | 23 ++++++++++++----------- siphon/tests/test_iem.py | 11 +++++------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/siphon/simplewebservice/iem.py b/siphon/simplewebservice/iem.py index 1ae3530af..c8f55d78d 100644 --- a/siphon/simplewebservice/iem.py +++ b/siphon/simplewebservice/iem.py @@ -11,10 +11,10 @@ class IemAsos: """Handles data collection of ASOS data from IEM. - + This handles the collection of ASOS data from the Iowa Environmental Mesonet, via their asos.py request URL. - + Attributes ---------- startDate : datetime @@ -25,14 +25,14 @@ class IemAsos: Station IDs in the dataset data : pandas.DataFrame Pandas dataframe containing the IEM ASOS data - + """ def __init__(self, sites, startDate=None, endDate=None): """Initialize the IemAsos object. - + Initialization will set the datetime objects and start the data call - + Parameters ---------- sites : list @@ -57,7 +57,7 @@ def __init__(self, sites, startDate=None, endDate=None): def getData(self): """ Downloads IEM ASOS data """ - + # Build the URL URL = 'http://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?' @@ -74,15 +74,16 @@ def getData(self): URL = URL + '&month2='+str(self.endDate.month) URL = URL + '&day2='+str(self.endDate.day) - URL = URL + '&tz=Etc%2FUTC&format=onlycomma&latlon=yes&direct=no&report_type=1&report_type=2' - + URL = URL + '&tz=Etc%2FUTC&format=onlycomma&latlon=yes&direct=no' \ + '&report_type=1&report_type=2' + # Collect the data try: response = create_http_session().post(URL) csvData = StringIO(response.text) except requests.exceptions.Timeout: raise IemAsosException('Connection Timeout') - + # Process the data into a dataframe ''' Convert the response text into a DataFrame. The index_col ensures that the first @@ -90,9 +91,9 @@ def getData(self): as row indices. ''' df = pd.read_csv(csvData, header=0, sep=',', index_col=False) - + # Strip whitespace from the column names - df.columns = df.columns.str.strip() + df.columns = df.columns.str.strip() df['valid'] = pd.to_datetime(df['valid'], format="%Y-%m-%d %H:%M:%S") diff --git a/siphon/tests/test_iem.py b/siphon/tests/test_iem.py index 78d3cb2c5..88752036d 100644 --- a/siphon/tests/test_iem.py +++ b/siphon/tests/test_iem.py @@ -8,18 +8,17 @@ @recorder.use_cassette('iem_request') - def test_iem_download(): """Test the downloading of IEM ASOS data""" - start = datetime(2018,1,1) - end = datetime(2018,1,1) - + start = datetime(2018, 1, 1) + end = datetime(2018, 1, 1) + asosCall = IemAsos(['KLNK', 'KGSO', 'KBDU'], start, end) - + stn1 = asosCall.data[(asosCall.data.station == 'GSO')] stn2 = asosCall.data[(asosCall.data.station == 'LNK')] stn3 = asosCall.data[(asosCall.data.station == 'BDU')] - + assert stn1['station'][1] == 'GSO' assert stn2['station'][1] == 'LNK' assert stn3['station'][1] == 'BDU'