From 19aef1ae2844141a36745aea44968a8afb770926 Mon Sep 17 00:00:00 2001 From: Reid Miller Date: Mon, 4 Sep 2017 09:07:32 -0400 Subject: [PATCH 1/2] #130 Split test_ercot.py into unit test and integration tests. This commit splits test_ercot.py into unit tests, which have no external dependencies, and integration tests, which test pyiso's integration with external systems (e.g. ERCOT's web servers). --- .../ercot/real_time_system_conditions.html | 88 +++++++++ tests/integration/integration_test_ercot.py | 61 ++++++ tests/test_ercot.py | 187 ------------------ tests/unit/test_ercot.py | 51 +++++ 4 files changed, 200 insertions(+), 187 deletions(-) create mode 100644 tests/fixtures/ercot/real_time_system_conditions.html create mode 100644 tests/integration/integration_test_ercot.py delete mode 100644 tests/test_ercot.py create mode 100644 tests/unit/test_ercot.py diff --git a/tests/fixtures/ercot/real_time_system_conditions.html b/tests/fixtures/ercot/real_time_system_conditions.html new file mode 100644 index 0000000..133f4d0 --- /dev/null +++ b/tests/fixtures/ercot/real_time_system_conditions.html @@ -0,0 +1,88 @@ + + + +Real-Time System Conditions + + + + + + + + +
+
+
Real-Time System Conditions
+
Help?
+
+
+
Last Updated: Apr 14, 2016 18:38:40
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Frequency
Current Frequency59.998
Instantaneous Time Error-17.468
Real-Time Data
Actual System Demand38850
Total System Capacity (not including Ancillary Services)42514
Total Wind Output5242
DC Tie Flows
DC_E (East)-31
DC_L (Laredo VFT)0
DC_N (North)0
DC_R (Railroad)0
DC_S (Eagle Pass)1
+
+ + diff --git a/tests/integration/integration_test_ercot.py b/tests/integration/integration_test_ercot.py new file mode 100644 index 0000000..8ddad21 --- /dev/null +++ b/tests/integration/integration_test_ercot.py @@ -0,0 +1,61 @@ +from pyiso import client_factory +from unittest import TestCase +import pytz +from datetime import datetime, timedelta +import pandas as pd + + +class IntegrationTestERCOT(TestCase): + def setUp(self): + self.c = client_factory('ERCOT') + + def test_request_report_gen_hrly(self): + # get data as list of dicts + df = self.c._request_report('gen_hrly') + + # test for expected data + self.assertEqual(len(df), 1) + for key in ['SE_EXE_TIME_DST', 'SE_EXE_TIME', 'SE_MW']: + self.assertIn(key, df.columns) + + def test_request_report_wind_hrly(self): + # get data as list of dicts + df = self.c._request_report('wind_hrly') + + # test for expected data + self.assertLessEqual(len(df), 96) + for key in ['DSTFlag', 'ACTUAL_SYSTEM_WIDE', 'HOUR_ENDING']: + self.assertIn(key, df.columns) + + def test_request_report_load_7day(self): + # get data as list of dicts + df = self.c._request_report('load_7day') + + # test for expected data + # subtract 1 hour for DST + self.assertGreaterEqual(len(df), 8 * 24 - 1) + for key in ['SystemTotal', 'HourEnding', 'DSTFlag', 'DeliveryDate']: + self.assertIn(key, df.columns) + + def test_request_report_dam_hrl_lmp(self): + now = datetime.now(pytz.timezone(self.c.TZ_NAME)) + df = self.c._request_report('dam_hrly_lmp', now) + s = pd.to_datetime(df['DeliveryDate'], utc=True) + + self.assertEqual(s.min().date(), now.date()) + self.assertEqual(s.max().date(), now.date()) + + node_counts = range(612, 630) + self.assertIn(len(df), [n * 24 for n in node_counts]) # nodes * 24 hrs/day + + def test_request_report_rt5m_lmp(self): + now = datetime.now(pytz.utc) - timedelta(minutes=5) + df = self.c._request_report('rt5m_lmp', now) + df.index = df['SCEDTimestamp'] + s = pd.to_datetime(df.index).tz_localize('Etc/GMT+5').tz_convert('utc') + + self.assertLess((s.min() - now).total_seconds(), 8 * 60) + self.assertLess((s.max() - now).total_seconds(), 8 * 60) + + node_counts = range(612, 630) + self.assertIn(len(df), node_counts) diff --git a/tests/test_ercot.py b/tests/test_ercot.py deleted file mode 100644 index 4d01075..0000000 --- a/tests/test_ercot.py +++ /dev/null @@ -1,187 +0,0 @@ -from pyiso import client_factory -from unittest import TestCase -import pytz -from datetime import datetime, timedelta -import pandas as pd - - -class TestERCOT(TestCase): - def setUp(self): - self.c = client_factory('ERCOT') - - self.rtm_html = """ - - - -Real-Time System Conditions - - - - - - - - -
-
-
Real-Time System Conditions
-
Help?
-
-
-
Last Updated: Apr 14, 2016 18:38:40
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Frequency
Current Frequency59.998
Instantaneous Time Error-17.468
Real-Time Data
Actual System Demand38850
Total System Capacity (not including Ancillary Services)42514
Total Wind Output5242
DC Tie Flows
DC_E (East)-31
DC_L (Laredo VFT)0
DC_N (North)0
DC_R (Railroad)0
DC_S (Eagle Pass)1
-
- - -""" - - def test_utcify(self): - ts_str = '05/03/2014 02:00' - ts = self.c.utcify(ts_str) - self.assertEqual(ts.year, 2014) - self.assertEqual(ts.month, 5) - self.assertEqual(ts.day, 3) - self.assertEqual(ts.hour, 2+5) - self.assertEqual(ts.minute, 0) - self.assertEqual(ts.tzinfo, pytz.utc) - - def test_parse_load(self): - self.c.handle_options(data='load', latest=True) - data = self.c.parse_rtm(self.rtm_html) - self.assertEqual(len(data), 1) - expected_keys = ['timestamp', 'ba_name', 'load_MW', 'freq', 'market'] - self.assertEqual(sorted(data[0].keys()), sorted(expected_keys)) - self.assertEqual(data[0]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) - self.assertEqual(data[0]['load_MW'], 38850.0) - - def test_parse_genmix(self): - self.c.handle_options(data='gen', latest=True) - data = self.c.parse_rtm(self.rtm_html) - - self.assertEqual(len(data), 2) - expected_keys = ['timestamp', 'ba_name', 'gen_MW', 'fuel_name', 'freq', 'market'] - self.assertEqual(sorted(data[0].keys()), sorted(expected_keys)) - - self.assertEqual(data[0]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) - self.assertEqual(data[0]['gen_MW'], 5242.0) - self.assertEqual(data[0]['fuel_name'], 'wind') - - self.assertEqual(data[1]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) - self.assertEqual(data[1]['gen_MW'], 38850 - 5242 + 31 - 1) - self.assertEqual(data[1]['fuel_name'], 'nonwind') - - def test_request_report_gen_hrly(self): - # get data as list of dicts - df = self.c._request_report('gen_hrly') - - # test for expected data - self.assertEqual(len(df), 1) - for key in ['SE_EXE_TIME_DST', 'SE_EXE_TIME', 'SE_MW']: - self.assertIn(key, df.columns) - - def test_request_report_wind_hrly(self): - # get data as list of dicts - df = self.c._request_report('wind_hrly') - - # test for expected data - self.assertLessEqual(len(df), 96) - for key in ['DSTFlag', 'ACTUAL_SYSTEM_WIDE', 'HOUR_ENDING']: - self.assertIn(key, df.columns) - - def test_request_report_load_7day(self): - # get data as list of dicts - df = self.c._request_report('load_7day') - - # test for expected data - # subtract 1 hour for DST - self.assertGreaterEqual(len(df), 8*24-1) - for key in ['SystemTotal', 'HourEnding', 'DSTFlag', 'DeliveryDate']: - self.assertIn(key, df.columns) - - def test_request_report_dam_hrl_lmp(self): - now = datetime.now(pytz.timezone(self.c.TZ_NAME)) - df = self.c._request_report('dam_hrly_lmp', now) - s = pd.to_datetime(df['DeliveryDate'], utc=True) - - self.assertEqual(s.min().date(), now.date()) - self.assertEqual(s.max().date(), now.date()) - - node_counts = range(612, 630) - self.assertIn(len(df), [n*24 for n in node_counts]) # nodes * 24 hrs/day - - def test_request_report_rt5m_lmp(self): - now = datetime.now(pytz.utc) - timedelta(minutes=5) - df = self.c._request_report('rt5m_lmp', now) - df.index = df['SCEDTimestamp'] - s = pd.to_datetime(df.index).tz_localize('Etc/GMT+5').tz_convert('utc') - - self.assertLess((s.min() - now).total_seconds(), 8*60) - self.assertLess((s.max() - now).total_seconds(), 8*60) - - node_counts = range(612, 630) - self.assertIn(len(df), node_counts) diff --git a/tests/unit/test_ercot.py b/tests/unit/test_ercot.py new file mode 100644 index 0000000..feb30e7 --- /dev/null +++ b/tests/unit/test_ercot.py @@ -0,0 +1,51 @@ +import os + +from pyiso import client_factory +from unittest import TestCase +import pytz +from datetime import datetime + + +FIXTURES_DIR = os.path.join(os.path.dirname(__file__), '../fixtures/ercot') + + +class TestERCOT(TestCase): + + def setUp(self): + self.c = client_factory('ERCOT') + self.rtm_html = open(FIXTURES_DIR + '/real_time_system_conditions.html').read().encode('utf8') + + def test_utcify(self): + ts_str = '05/03/2014 02:00' + ts = self.c.utcify(ts_str) + self.assertEqual(ts.year, 2014) + self.assertEqual(ts.month, 5) + self.assertEqual(ts.day, 3) + self.assertEqual(ts.hour, 2+5) + self.assertEqual(ts.minute, 0) + self.assertEqual(ts.tzinfo, pytz.utc) + + def test_parse_load(self): + self.c.handle_options(data='load', latest=True) + data = self.c.parse_rtm(self.rtm_html) + self.assertEqual(len(data), 1) + expected_keys = ['timestamp', 'ba_name', 'load_MW', 'freq', 'market'] + self.assertEqual(sorted(data[0].keys()), sorted(expected_keys)) + self.assertEqual(data[0]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) + self.assertEqual(data[0]['load_MW'], 38850.0) + + def test_parse_genmix(self): + self.c.handle_options(data='gen', latest=True) + data = self.c.parse_rtm(self.rtm_html) + + self.assertEqual(len(data), 2) + expected_keys = ['timestamp', 'ba_name', 'gen_MW', 'fuel_name', 'freq', 'market'] + self.assertEqual(sorted(data[0].keys()), sorted(expected_keys)) + + self.assertEqual(data[0]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) + self.assertEqual(data[0]['gen_MW'], 5242.0) + self.assertEqual(data[0]['fuel_name'], 'wind') + + self.assertEqual(data[1]['timestamp'], pytz.utc.localize(datetime(2016, 4, 14, 23, 38, 40))) + self.assertEqual(data[1]['gen_MW'], 38850 - 5242 + 31 - 1) + self.assertEqual(data[1]['fuel_name'], 'nonwind') From cb111fb7120ebcb89fd7b7cccf82f0c8961e00de Mon Sep 17 00:00:00 2001 From: Reid Miller Date: Mon, 4 Sep 2017 10:52:48 -0400 Subject: [PATCH 2/2] #130 Fixed LMP node count in integration_test_ercot.py --- tests/integration/integration_test_ercot.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/integration/integration_test_ercot.py b/tests/integration/integration_test_ercot.py index 8ddad21..09344d0 100644 --- a/tests/integration/integration_test_ercot.py +++ b/tests/integration/integration_test_ercot.py @@ -8,6 +8,7 @@ class IntegrationTestERCOT(TestCase): def setUp(self): self.c = client_factory('ERCOT') + self.node_counts = range(612, 633) # 632 LMP nodes as of 2017-09-04. TODO Why is this a range? def test_request_report_gen_hrly(self): # get data as list of dicts @@ -44,9 +45,7 @@ def test_request_report_dam_hrl_lmp(self): self.assertEqual(s.min().date(), now.date()) self.assertEqual(s.max().date(), now.date()) - - node_counts = range(612, 630) - self.assertIn(len(df), [n * 24 for n in node_counts]) # nodes * 24 hrs/day + self.assertIn(len(df), [n * 24 for n in self.node_counts]) # nodes * 24 hrs/day def test_request_report_rt5m_lmp(self): now = datetime.now(pytz.utc) - timedelta(minutes=5) @@ -56,6 +55,4 @@ def test_request_report_rt5m_lmp(self): self.assertLess((s.min() - now).total_seconds(), 8 * 60) self.assertLess((s.max() - now).total_seconds(), 8 * 60) - - node_counts = range(612, 630) - self.assertIn(len(df), node_counts) + self.assertIn(len(df), self.node_counts)