diff --git a/README.md b/README.md index fa68565..ccf53dd 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ A Python 3.x module to turn data from the PurpleAir/ThingSpeak API into a Pandas * Install development requirements with `pip install -r requirements/dev.txt` * Install example file requirements with `pip install -r requirements/examples.txt` +## Frequently Asked Questions + +Before opening a new ticket, please refer to the [FAQ document](docs/faq.md). + ## Example code For detailed documentation, see the [docs](docs/documentation.md) file. @@ -31,7 +35,7 @@ For detailed documentation, see the [docs](docs/documentation.md) file. ```python from purpleair.network import SensorList -p = SensorList() # Initialized 10,812 sensors! +p = SensorList() # Initialized 11,220 sensors! print(len(p.useful_sensors)) # 10047, List of sensors with no defects ``` @@ -47,7 +51,7 @@ print(s) # Sensor 2891 at 10834, Canyon Road, Omaha, Douglas County, Nebraska, ```python from purpleair.network import SensorList -p = SensorList() # Initialized 10,812 sensors! +p = SensorList() # Initialized 11,220 sensors! # Other sensor filters include 'outside', 'useful', 'family', and 'no_child' df = p.to_dataframe(sensor_filter='all', channel='parent') @@ -69,14 +73,12 @@ id ```python from purpleair.network import SensorList -p = SensorList() # Initialized 10,812 sensors! +p = SensorList() # Initialized 11,220 sensors! # If `sensor_filter` is set to 'column' then we must also provide a value for `column` -df_1 = p.to_dataframe(sensor_filter='all', - channel='parent') -df_2 = p.to_dataframe(sensor_filter='column', - channel='parent', - column='m10avg') # See Channel docs for all column options -print(len(df_1), len(df_2)) # 11,071 10,723 +df = p.to_dataframe(sensor_filter='column', + channel='parent', + column='m10avg') # See Channel docs for all column options +print(len(df)) # 10,723 ``` ### Get historical data for parent sensor secondary channel @@ -107,7 +109,7 @@ entry_id from purpleair.sensor import Sensor se = Sensor(2890) df = se.child.get_historical(weeks_to_get=1, - thingspeak_field='secondary') + thingspeak_field='primary') print(df.head()) ``` diff --git a/docs/faq.md b/docs/faq.md index e4a3e0a..4c1d78e 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,24 +1,63 @@ # Frequently Asked Questions and Common Problems -## ValueError: Invalid JSON data returned from network +If your question is not answered in this document, please open a new [issue](https://github.com/ReagentX/purple_air_api/issues). -Occasionally, PurpleAir's API will return invalid JSON. Usually, this means a double quote char or some other delimiter is missing. There is nothing we can do, so this library raise a ValueError. +Please refer to the [PurpleAir FAQ](https://www2.purpleair.com/community/faq) for questions related to the sensor network. + +## How do I do ___ + +Please see the sample code section of the [readme](/README.md#example-code). + +## What is the relationship between `SensorList`, `Sensor`, `Channel`, and `ThingSpeak` + +This is a pseudo-JSON representation of the relationships between these data: + +```log +[ + sensor_1: { + 'parent': { + 'primary': thingspeak.Channel, + 'secondary: thingspeak.Channel + }, + 'child': { + 'primary': thingspeak.Channel, + 'secondary: thingspeak.Channel + } + }, + sensor_2: {...}, + ... +] +``` + +The outer list represents the `SensorList`. `sensor_1` is an instance of `Sensor` and has two `Channel`s, `'parent'` and `'child'`. Each `Channel` has two ThingSpeak fields, `'primary'` and `'secondary'`. + +## Why is `get_location()` slow + +The `Sensor.get_location()` method returns the approximate street address from a sensor given the latitude and longitude value. Since we make the conversion using `geopy`'s `nominatim` API, we are limited to [1 request per second maximum](https://operations.osmfoundation.org/policies/nominatim/). + +*** + +## Python Environment and PurpleAir Network problems + +### ValueError: Invalid JSON data returned from network + +Occasionally, PurpleAir's API will return invalid JSON. Usually, this means a double quote character or some other delimiter is missing. There is nothing we can do, so this library raise a `ValueError`. The only fix is to try again or invalidate or delete the cache that `requests_cache` creates. If the invalid response is cached (it shouldn’t be), you can delete the cache by removing the `cache.sqlite` file it creates in the project’s root directory. -## Unable to open cache or purge cache database, requests will not be cached +### Unable to open cache or purge cache database, requests will not be cached This error means there is a problem connecting to the `cache.sqlite` file created by `requests_cache`. The program will still run, but results of API calls will not be cached, so affected programs may hit rate limits. -## Invalid ThingSpeak key +*** -Provided key does not exist on ThingSpeak. Refer to the [purpleair docs](/docs/purpleair_documentation.md#Field%20descriptions) for valid columns and their meanings. +## Network and SensorList problems -## Child `{child_sensor_id}` lists parent `{parent_sensor_id}`, but parent does not exist +### Child `{child_sensor_id}` lists parent `{parent_sensor_id}`, but parent does not exist The child sensor requested lists a parent, but the parent does not exist on the PurpleAir network. This is a problem with PurpleAir, not this program. Try removing the `cache.sqlite` file it creates in the project’s root directory. -## No column name provided to filter on +### No column name provided to filter on `to_dataframe` was invoked with `sensor_filter` set to `'column'` but no value for the `column` parameter was provided. It should be invoked like this: @@ -29,45 +68,55 @@ p.to_dataframe(sensor_filter='column', column='m10avg') # See Channel docs for all column options ``` -## Column name provided does not exist in sensor data +### Column name provided does not exist in sensor data `to_dataframe` was invoked with `sensor_filter` set to `'column'` and the value for the `column` parameter does not exist as a column. Please only use properties of a [Channel](/docs/documentation.md#Channel). -## No data for filter set: Column `{column}`, value filter: `{value_filter}` +### No data for filter set: Column `{column}`, value filter: `{value_filter}` `to_dataframe` was invoked with `sensor_filter` set to `'column'` and none of the values in the column denoted by the `column` parameter match the given `value_filter`. This means the DataFrame would be empty, so we raise an error here before the user attempts to transform data. -## Invalid sensor channel: `{channel}`. Must be in `{"a", "b"}` +*** + +## Sensor Problems -A function that requires a `channel` parameter can only look at channels `a` and `b`. Since there are no other channels, we raise an error when this occurs. +### Invalid ThingSpeak key -## Invalid sensor: no configuration found for `{identifer}` +Provided key does not exist on ThingSpeak. Refer to the [purpleair docs](/docs/purpleair_documentation.md#Field%20descriptions) for valid columns and their meanings. + +### Invalid sensor channel: `{channel}`. Must be in `{"parent", "child"}` + +A function that requires a `channel` parameter can only look at channels `parent` and `child`. Since there are no other channels, we raise an error when this occurs. + +### Invalid sensor: no configuration found for `{identifer}` The requested sensor does not have any data on PurpleAir. -## Sensor `{identifier}` created without valid data +### Sensor `{identifier}` created without valid data A `Sensor` was created with the `json_data` parameter filled, but the json is malformed. -## Invalid sensor ID +### Invalid sensor ID The given `Sensor()`'s ID is not in valid integer form. -## More than 2 channels found for `{identifier}` +### More than 2 channels found for `{identifier}` PurpleAir reports that a sensor has more than one child. This is a problem with PurpleAir, not this program. Try removing the `cache.sqlite` file it creates in the project’s root directory. -## No sensor data returned from PurpleAir +### No sensor data returned from PurpleAir This error happens if the API fails to return data with a `results` key, where `results` is mapped to a JSON blob of sensors. -### Rate Limit Error +#### Rate Limit Error If this error includes a rate limit message, try again when the rate limit is expired. -### Other Message +#### Other Message + +If the error message is not a rate limit error, try to delete the cache by removing the `cache.sqlite` file it creates in the project’s root directory. If this does not solve the problem, please open a new [issue](https://github.com/ReagentX/purple_air_api/issues) with the full traceback and message. -If the error message is not a rate limit error, please open a new [issue](https://github.com/ReagentX/purple_air_api/issues) with the full traceback and message. +*** ## Other Crashes and Errors diff --git a/maps/sensor_map.png b/maps/sensor_map.png index a9e7372..7edda17 100644 Binary files a/maps/sensor_map.png and b/maps/sensor_map.png differ diff --git a/purpleair/channel.py b/purpleair/channel.py index c51849e..6705802 100644 --- a/purpleair/channel.py +++ b/purpleair/channel.py @@ -123,7 +123,7 @@ def setup(self) -> None: last_seen = self.channel_data.get('LastSeen') if last_seen is not None: self.last_seen: Optional[datetime] = datetime.utcfromtimestamp( - int(last_seen) / 1000) + int(last_seen)) else: self.last_seen = last_seen self.model: Optional[str] = self.channel_data.get('Type') diff --git a/scripts/plot_map.py b/scripts/plot_map.py index a181484..f51425a 100644 --- a/scripts/plot_map.py +++ b/scripts/plot_map.py @@ -15,6 +15,7 @@ # Get PurpleAir data p = SensorList() df = p.to_dataframe('all', 'parent') +df = df[df['temp_c'] <= 50] # Store the lat and lon coords to plot lat = df['lat'].values diff --git a/setup.py b/setup.py index b3c41bd..95e0d94 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='purpleair', - version='1.2', + version='1.2.1', description='Python API Client to get and transform PurpleAir data.', long_description=LONG_DESCRIPTION, long_description_content_type="text/markdown", diff --git a/tests/test_channel.py b/tests/test_channel.py index 6067130..a710264 100644 --- a/tests/test_channel.py +++ b/tests/test_channel.py @@ -8,12 +8,19 @@ class TestChannelMethods(unittest.TestCase): Tests for Sensor class """ + def test_can_repr(self): + """ + Test that we properly generate a Channel's string representation + """ + se = sensor.Sensor(2891) + self.assertEqual(se.child.__repr__(), 'Sensor 2891, child of 2890') + def test_get_historical(self): """ Test that we properly get a sensor's historical data """ se = sensor.Sensor(2891) - se.parent.get_historical(1, 'primary') + se.parent.get_historical(2, 'primary') se.parent.get_historical(1, 'secondary') se.child.get_historical(1, 'primary') se.child.get_historical(1, 'secondary') @@ -141,7 +148,3 @@ def test_as_flat_dict(self): result = se.child.as_flat_dict() for key in expected_shape: self.assertIn(key, result) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_purpleair.py b/tests/test_purpleair.py index 61477c0..5137230 100644 --- a/tests/test_purpleair.py +++ b/tests/test_purpleair.py @@ -66,22 +66,27 @@ class TestPurpleAirColumnFilters(unittest.TestCase): """ Test that we can initialize the PurpleAir network """ + def test_to_dataframe_filtering_no_column(self): """ Test that not providing a column fails """ + p = network.SensorList() with self.assertRaises(ValueError): - p = network.SensorList() p.to_dataframe('column', 'parent') + + with self.assertRaises(ValueError): p.to_dataframe('column', 'child') def test_to_dataframe_filtering_bad_column(self): """ Test that providing a bad column fails """ + p = network.SensorList() with self.assertRaises(ValueError): - p = network.SensorList() p.to_dataframe('column', 'parent', 'fake_col_name') + + with self.assertRaises(ValueError): p.to_dataframe('column', 'child', 'fake_col_name') def test_to_dataframe_filtering_no_value(self): @@ -105,10 +110,9 @@ def test_to_dataframe_filtering_bad_value(self): """ Test that providing a bad value fails """ + p = network.SensorList() with self.assertRaises(ValueError): - p = network.SensorList() p.to_dataframe('column', 'parent', 'location_type', 1234) - p.to_dataframe('column', 'child', 'location_type', 1234) -if __name__ == '__main__': - unittest.main() + with self.assertRaises(ValueError): + p.to_dataframe('column', 'child', 'location_type', 1234) diff --git a/tests/test_sensor.py b/tests/test_sensor.py index 8779f79..2871eb5 100644 --- a/tests/test_sensor.py +++ b/tests/test_sensor.py @@ -18,12 +18,29 @@ def test_create_sensor_location(self): 'Sensor 2891 at 10834, Canyon Road, Omaha, Douglas County, Nebraska, 68112, United States of America' ) + def test_can_get_field(self): + """ + Test that we can get data from the ThingSpeak API + """ + se = sensor.Sensor(7423, parse_location=True) + se.get_field('field3') + self.assertIn('field3', se.thingspeak_data) + self.assertIn('primary', se.thingspeak_data['field3']) + self.assertIn('feeds', se.thingspeak_data['field3']['primary']['channel_a']) + self.assertIn('feeds', se.thingspeak_data['field3']['primary']['channel_b']) + self.assertIn('feeds', se.thingspeak_data['field3']['secondary']['channel_a']) + self.assertIn('feeds', se.thingspeak_data['field3']['secondary']['channel_b']) + self.assertGreater( + len(se.thingspeak_data['field3']['primary']['channel_a']['feeds']), + 0 + ) + def test_cannot_create_sensor_bad_id(self): """ Test that we cannot create a sensor without an integer ID """ with self.assertRaises(ValueError): - se = sensor.Sensor('parent') + se = sensor.Sensor('a') def test_cannot_create_sensor_bad_json(self): """ @@ -246,7 +263,3 @@ def test_as_flat_dict(self): self.assertIn(data_category, src) for data in src: self.assertNotIsInstance(src[data], dict) - - -if __name__ == '__main__': - unittest.main()