From 607b5590c8487422cda48446c7e338d80e04ff90 Mon Sep 17 00:00:00 2001 From: Xavier Frankline Odhiambo <49587182+thepsalmist@users.noreply.github.com> Date: Thu, 14 Nov 2024 20:23:04 +0300 Subject: [PATCH] Chore/setup update api v2 (#138) * chore: update Dockerfile and requirements, cleanup settings * chore: add /env template * fix: fix error TypeError 'bool' object is not callable * chore: add Now api endpoint, cleanup router * chore: update v1 route to only push sensors data, add statistics view * update: fix push-sensors-url * update: update NodesView * update: update NodesView * Update README docker setup * update: modify NodeView permissions * chore: update views docstrings * chore: modify data stats url * chore: add .dockerignore file --------- Co-authored-by: Xavier Frankline --- .dockerignore | 77 ++++++++++++++++++ .env.template | 6 ++ Dockerfile | 23 +++--- README.md | 8 +- docker-compose.yml | 30 ++++--- requirements.txt | 2 +- sensorsafrica/api/v1/router.py | 6 +- sensorsafrica/api/v1/views.py | 4 +- sensorsafrica/api/v2/router.py | 42 ++++------ sensorsafrica/api/v2/views.py | 141 +++++++++++++++++++++++++++++---- sensorsafrica/settings.py | 7 -- sensorsafrica/urls.py | 4 +- 12 files changed, 268 insertions(+), 82 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.template diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..69a711c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,77 @@ +# Git +.git +.gitignore + +# Docker +docker-compose.yml +.docker + +# Byte-compiled / optimized / DLL files +__pycache__/ +*/__pycache__/ +*/*/__pycache__/ +*/*/*/__pycache__/ +*.py[cod] +*/*.py[cod] +*/*/*.py[cod] +*/*/*/*.py[cod] + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Virtual environment +.env/ +.venv/ +venv/ + +# PyCharm +.idea + +# Python mode for VIM +.ropeproject +*/.ropeproject +*/*/.ropeproject +*/*/*/.ropeproject + +# Vim swap files +*.swp +*/*.swp +*/*/*.swp +*/*/*/*.swp diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..9c920db --- /dev/null +++ b/.env.template @@ -0,0 +1,6 @@ +SENSORSAFRICA_DATABASE_URL="" +SENSORSAFRICA_DEBUG="" +SENSORSAFRICA_FLOWER_ADMIN_PASSWORD="" +SENSORSAFRICA_FLOWER_ADMIN_USERNAME="" +SENSORSAFRICA_RABBITMQ_URL="" +SENSORSAFRICA_SENTRY_DSN="" diff --git a/Dockerfile b/Dockerfile index 0ccc619..aaef287 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,25 +2,21 @@ FROM python:3.6.3 ENV PYTHONUNBUFFERED 1 -# Create root directory for our project in the container -RUN mkdir /src - -# Create application subdirectories +# Create application root directory WORKDIR /src + RUN mkdir media static logs VOLUME [ "/src/logs" ] -# Copy the current directory contents into the container at sensorsafrica -ADD . /src/ - -# Upgrade pip and setuptools -RUN pip install -q -U pip setuptools +# Upgrade pip and setuptools with trusted hosts +RUN python -m pip install --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip setuptools -# Install feinstaub from opendata-stuttgart -RUN pip install -q git+https://github.com/opendata-stuttgart/feinstaub-api +# Copy the current directory contents into the container at sensorsafrica +COPY . /src/ -# Install sensors.AFRICA-api and its dependencies -RUN pip install -q -U . +# Install dependencies +RUN pip install -q git+https://github.com/opendata-stuttgart/feinstaub-api && \ + pip install -q . # Expose port server EXPOSE 8000 @@ -31,3 +27,4 @@ COPY ./contrib/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] CMD [ "/start.sh" ] + diff --git a/README.md b/README.md index 1f70360..5f7cd95 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ API to save and access data from deployed sensors in cities all around Africa. ## Documentation -The API is documented [here.](https://github.com/CodeForAfricaLabs/sensors.AFRICA-api/wiki/API-Documentation) +The API is documented [here.](https://github.com/CodeForAfricaLabs/sensors.AFRICA-api/wiki/API-Documentation) ## Development @@ -29,11 +29,17 @@ GRANT ALL PRIVILEGES ON DATABASE sensorsafrica TO sensorsafrica; - Migrate the database; `python manage.py migrate` - Run the server; `python manage.py runserver` +- Create super user for admin login; `python manage.py createsuperuser` + + username: `` + email: blank + password: `` ### Docker Using docker compose: +- Create a `.env` file using `.env.template` . ***docker-compose has some default values for these variables*** - Build the project; `docker-compose build` or `make build` - Run the project; `docker-compose up -d` or `make up` diff --git a/docker-compose.yml b/docker-compose.yml index 49c12ef..8a1c12a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,15 +2,20 @@ version: '3.3' services: rabbitmq: - image: rabbitmq:3.5.1 + image: rabbitmq:3.12.7-management ports: - - 4369:4369 - - 5672:5672 - - 25672:25672 - - 15672:15672 + - "5672:5672" + # GUI port + - "15672:15672" environment: - - RABBITMQ_USERNAME=sensorsafrica - - RABBITMQ_PASSWORD=sensorsafrica + - RABBITMQ_DEFAULT_USER=sensorsafrica + - RABBITMQ_DEFAULT_PASS=sensorsafrica + healthcheck: + test: [ "CMD-SHELL", "rabbitmq-diagnostics -q ping" ] + interval: 10s + timeout: 5s + retries: 2 + postgres: image: postgres:13.7 ports: @@ -25,12 +30,11 @@ services: build: context: . environment: - SENSORSAFRICA_DATABASE_URL: postgres://sensorsafrica:sensorsafrica@postgres:5432/sensorsafrica - SENSORSAFRICA_READ_DATABASE_URLS: postgres://sensorsafrica:sensorsafrica@postgres:5432/sensorsafrica - SENSORSAFRICA_RABBITMQ_URL: amqp://sensorsafrica:sensorsafrica@rabbitmq// - SENSORSAFRICA_FLOWER_ADMIN_USERNAME: admin - SENSORSAFRICA_FLOWER_ADMIN_PASSWORD: password - DOKKU_APP_NAME: sensorsafrica + SENSORSAFRICA_DATABASE_URL: ${SENSORSAFRICA_DATABASE_URL:-postgres://sensorsafrica:sensorsafrica@postgres:5432/sensorsafrica} + SENSORSAFRICA_RABBITMQ_URL: ${SENSORSAFRICA_RABBITMQ_URL:-amqp://sensorsafrica:sensorsafrica@rabbitmq/} + SENSORSAFRICA_FLOWER_ADMIN_USERNAME: ${SENSORSAFRICA_FLOWER_ADMIN_USERNAME:-admin} + SENSORSAFRICA_FLOWER_ADMIN_PASSWORD: ${SENSORSAFRICA_FLOWER_ADMIN_PASSWORD:-password} + DOKKU_APP_NAME: ${DOKKU_APP_NAME:-sensorsafrica} depends_on: - postgres - rabbitmq diff --git a/requirements.txt b/requirements.txt index 92145ee..086b243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ ckanapi==4.1 celery-slack==0.3.0 -urllib3<1.25,>=1.21.1 #requests 2.21.0 +urllib3<2 django-cors-headers==3.0.2 diff --git a/sensorsafrica/api/v1/router.py b/sensorsafrica/api/v1/router.py index c923328..74a49be 100644 --- a/sensorsafrica/api/v1/router.py +++ b/sensorsafrica/api/v1/router.py @@ -18,7 +18,6 @@ from rest_framework import routers router = routers.DefaultRouter() -router.register(r"push-sensor-data", PostSensorDataView) router.register(r"node", NodeView) router.register(r"sensor", SensorView) router.register(r"data", VerboseSensorDataView) @@ -31,3 +30,8 @@ router.register(r"filter", FilterView, basename="filter") api_urls = router.urls + +push_sensor_data_router = routers.DefaultRouter() +push_sensor_data_router.register(r"push-sensor-data", PostSensorDataView) + +push_sensor_data_urls = push_sensor_data_router.urls diff --git a/sensorsafrica/api/v1/views.py b/sensorsafrica/api/v1/views.py index 6c6a83a..182a446 100644 --- a/sensorsafrica/api/v1/views.py +++ b/sensorsafrica/api/v1/views.py @@ -54,7 +54,7 @@ class NodeView( filter_class = NodeFilter def get_queryset(self): - if self.request.user.is_authenticated(): + if self.request.user.is_authenticated: if self.request.user.groups.filter(name="show_me_everything").exists(): return Node.objects.all() @@ -92,7 +92,7 @@ class PostSensorDataView(mixins.CreateModelMixin, permission_classes = tuple() serializer_class = LastNotifySensorDataSerializer queryset = SensorData.objects.all() - + class VerboseSensorDataView(SensorDataView): filter_class = SensorFilter diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py index 46f4bc4..2edb377 100644 --- a/sensorsafrica/api/v2/router.py +++ b/sensorsafrica/api/v2/router.py @@ -4,42 +4,28 @@ from .views import ( CitiesView, NodesView, + NowView, SensorDataStatsView, SensorDataView, SensorLocationsView, SensorTypesView, SensorsView, + StatisticsView, meta_data, ) -stat_data_router = routers.DefaultRouter() -stat_data_router.register(r"", SensorDataStatsView) - -data_router = routers.DefaultRouter() -data_router.register(r"", SensorDataView) - -cities_router = routers.DefaultRouter() -cities_router.register(r"", CitiesView, basename="cities") - -nodes_router = routers.DefaultRouter() -nodes_router.register(r"", NodesView, basename="map") - -sensors_router = routers.DefaultRouter() -sensors_router.register(r"", SensorsView, basename="sensors") - -sensor_locations_router = routers.DefaultRouter() -sensor_locations_router.register(r"", SensorLocationsView, basename="locations") - -sensor_types_router = routers.DefaultRouter() -sensor_types_router.register(r"", SensorTypesView, basename="sensor_types") +router = routers.DefaultRouter() +router.register(r"data", SensorDataView, basename="sensor-data") +router.register(r"data/stats/(?P[air]+)", SensorDataStatsView, basename="sensor-data-stats") +router.register(r"cities", CitiesView, basename="cities") +router.register(r"nodes", NodesView, basename="nodes") +router.register(r"now", NowView, basename="now") +router.register(r"locations", SensorLocationsView, basename="sensor-locations") +router.register(r"sensors", SensorsView, basename="sensors") +router.register(r"sensor-types", SensorTypesView, basename="sensor-types") +router.register(r"statistics", StatisticsView, basename="statistics") api_urls = [ - url(r"data/(?P[air]+)/", include(stat_data_router.urls)), - url(r"data/", include(data_router.urls)), - url(r"cities/", include(cities_router.urls)), - url(r"nodes/", include(nodes_router.urls)), - url(r"locations/", include(sensor_locations_router.urls)), - url(r"sensors/", include(sensors_router.urls)), - url(r"sensor-types/", include(sensor_types_router.urls)), - url(r"meta/", meta_data), + url(r"^", include(router.urls)), + url(r"^meta/", meta_data), ] diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py index a2a1758..caca2f2 100644 --- a/sensorsafrica/api/v2/views.py +++ b/sensorsafrica/api/v2/views.py @@ -9,7 +9,7 @@ from django.conf import settings from django.utils import timezone from django.db import connection -from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q +from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q, Count from django.db.models.functions import Cast, TruncHour, TruncDay, TruncMonth from django.utils.decorators import method_decorator from django.utils.text import slugify @@ -17,13 +17,14 @@ from rest_framework import mixins, pagination, viewsets from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.decorators import action from rest_framework.exceptions import ValidationError from rest_framework.response import Response from rest_framework.permissions import IsAuthenticated, AllowAny from rest_framework.decorators import api_view, authentication_classes from feinstaub.sensors.views import SensorFilter, StandardResultsSetPagination - +from feinstaub.sensors.serializers import NowSerializer from feinstaub.sensors.models import ( Node, Sensor, @@ -129,17 +130,13 @@ class CitiesView(mixins.ListModelMixin, viewsets.GenericViewSet): class NodesView(viewsets.ViewSet): + """Create and list nodes, with the option to list authenticated user's nodes.""" authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] - def get_permissions(self): - if self.action == "create": - permission_classes = [IsAuthenticated] - else: - permission_classes = [AllowAny] - - return [permission() for permission in permission_classes] - - def list(self, request): + @action(detail=False, methods=["get"], url_path="list-nodes", url_name="list_nodes") + def list_nodes(self, request): + """List all public nodes with active sensors.""" nodes = [] # Loop through the last active nodes for last_active in LastActiveNodes.objects.iterator(): @@ -219,7 +216,24 @@ def list(self, request): return Response(nodes) - def create(self, request): + @action(detail=False, methods=["get"], url_path="my-nodes", url_name="my_nodes") + def list_my_nodes(self, request): + """List only the nodes owned by the authenticated user.""" + if request.user.is_authenticated: + queryset = Node.objects.filter( + Q(owner=request.user) + | Q( + owner__groups__name__in=[ + g.name for g in request.user.groups.all() + ] + ) + ) + serializer = NodeSerializer(queryset, many=True) + return Response(serializer.data) + return Response({"detail": "Authentication credentials were not provided."}, status=403) + + @action(detail=False, methods=["post"], url_path="register-node", url_name="register_node") + def register_node(self, request): serializer = NodeSerializer(data=request.data) if serializer.is_valid(): serializer.save() @@ -237,7 +251,17 @@ class SensorDataPagination(pagination.CursorPagination): class SensorDataView( mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet ): - """This endpoint is to download sensor data from the api.""" + """ + View for retrieving and downloading detailed sensor data records, with access controlled based on + user permissions and ownership. + + This endpoint allows authenticated users to retrieve sensor data records, with the following access rules: + - Users in the `show_me_everything` group have access to all sensor data records. + - Other users can access data from sensors they own, sensors owned by members of their groups, or public sensors. + - Non-authenticated users can only access public sensor data. + """ + + authentication_classes = [SessionAuthentication, TokenAuthentication] queryset = SensorData.objects.all() @@ -248,7 +272,7 @@ class SensorDataView( serializer_class = SensorDataSerializer def get_queryset(self): - if self.request.user.is_authenticated(): + if self.request.user.is_authenticated: if self.request.user.groups.filter(name="show_me_everything").exists(): return SensorData.objects.all() @@ -265,9 +289,40 @@ def get_queryset(self): class SensorDataStatsView(mixins.ListModelMixin, viewsets.GenericViewSet): + """ + View to retrieve summarized statistics for specific sensor types (e.g., air quality) within a defined date range, + filtered by city and grouped by specified intervals (hourly, daily, or monthly). + + **URL Parameters:** + - `sensor_type` (str): The type of sensor data to retrieve (e.g., air_quality). + + **Query Parameters:** + - `city` (str, optional): Comma-separated list of city slugs to filter data by location. + - `from` (str, optional): Start date in "YYYY-MM-DD" format. Required if `to` is specified. + - `to` (str, optional): End date in "YYYY-MM-DD" format. Defaults to 24 hours before `to_date` if unspecified. + - `interval` (str, optional): Aggregation interval for results - either "hour", "day", or "month". Defaults to "day". + - `value_type` (str, optional): Comma-separated list of value types to filter (e.g., "PM2.5, PM10"). + + **Caching:** + - Results are cached for 1 hour (`@cache_page(3600)`) to reduce server load. + + **Returns:** + - A list of sensor data statistics, grouped by city, value type, and specified interval. + - Each entry includes: + - `value_type` (str): Type of sensor value (e.g., PM2.5). + - `city_slug` (str): City identifier. + - `truncated_timestamp` (datetime): Timestamp truncated to the specified interval. + - `start_datetime` (datetime): Start of the aggregated time period. + - `end_datetime` (datetime): End of the aggregated time period. + - `calculated_average` (float): Weighted average of sensor values. + - `calculated_minimum` (float): Minimum recorded value within the period. + - `calculated_maximum` (float): Maximum recorded value within the period. + """ queryset = SensorDataStat.objects.none() serializer_class = SensorDataStatSerializer pagination_class = CustomPagination + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] @method_decorator(cache_page(3600)) def dispatch(self, request, *args, **kwargs): @@ -353,6 +408,9 @@ def get_queryset(self): class SensorLocationsView(viewsets.ViewSet): + """ + View for retrieving and creating sensor entries. + """ authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination @@ -372,6 +430,9 @@ def create(self, request): class SensorTypesView(viewsets.ViewSet): + """ + View for retrieving and creating sensor type entries. + """ authentication_classes = [SessionAuthentication, TokenAuthentication] permission_classes = [IsAuthenticated] pagination_class = StandardResultsSetPagination @@ -471,3 +532,55 @@ def get_database_last_updated(): sensor_data_value = SensorDataValue.objects.latest('created') if sensor_data_value: return sensor_data_value.modified + + +class NowView(mixins.ListModelMixin, viewsets.GenericViewSet): + """Show all public sensors active in the last 5 minutes with newest value""" + + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] + serializer_class = NowSerializer + + def get_queryset(self): + now = timezone.now() + startdate = now - datetime.timedelta(minutes=5) + return SensorData.objects.filter( + sensor__public=True, modified__range=[startdate, now] + ) + + +class StatisticsView(viewsets.ViewSet): + authentication_classes = [SessionAuthentication, TokenAuthentication] + permission_classes = [IsAuthenticated] + + def list(self, request): + user_count = User.objects.aggregate(count=Count('id'))['count'] + sensor_count = Sensor.objects.aggregate(count=Count('id'))['count'] + sensor_data_count = SensorData.objects.aggregate(count=Count('id'))['count'] + sensor_data_value_count = SensorDataValue.objects.aggregate(count=Count('id'))['count'] + sensor_type_count = SensorType.objects.aggregate(count=Count('id'))['count'] + sensor_type_list = list(SensorType.objects.order_by('uid').values_list('name', flat=True)) + location_count = SensorLocation.objects.aggregate(count=Count('id'))['count'] + + stats = { + 'user': { + 'count': user_count, + }, + 'sensor': { + 'count': sensor_count, + }, + 'sensor_data': { + 'count': sensor_data_count, + }, + 'sensor_data_value': { + 'count': sensor_data_value_count, + }, + 'sensor_type': { + 'count': sensor_type_count, + 'list': sensor_type_list, + }, + 'location': { + 'count': location_count, + } + } + return Response(stats) diff --git a/sensorsafrica/settings.py b/sensorsafrica/settings.py index fcbbf17..2377373 100644 --- a/sensorsafrica/settings.py +++ b/sensorsafrica/settings.py @@ -113,13 +113,6 @@ DATABASES = {"default": dj_database_url.parse(DATABASE_URL), } -READ_DATABASE_URLS = os.getenv("SENSORSAFRICA_READ_DATABASE_URLS", DATABASE_URL).split(",") - -for index, read_database_url in enumerate(READ_DATABASE_URLS,start=1): - DATABASES[f"read_replica_{index}"] = dj_database_url.parse(read_database_url) - -DATABASE_ROUTERS = ["sensorsafrica.router.ReplicaRouter", ] - # Password validation # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators diff --git a/sensorsafrica/urls.py b/sensorsafrica/urls.py index 4e9f7d5..e8df713 100644 --- a/sensorsafrica/urls.py +++ b/sensorsafrica/urls.py @@ -21,13 +21,13 @@ from rest_framework.authtoken.views import obtain_auth_token from rest_framework.documentation import include_docs_urls -from .api.v1.router import api_urls as sensors_api_v1 +from .api.v1.router import push_sensor_data_urls from .api.v2.router import api_urls as sensors_api_v2 urlpatterns = [ url(r"^$", RedirectView.as_view(url="/docs/", permanent=False)), url(r"^admin/", admin.site.urls), - url(r"^v1/", include(sensors_api_v1)), + url(r"^v1/", include(push_sensor_data_urls)), url(r"^v2/", include(sensors_api_v2)), url(r"^get-auth-token/", obtain_auth_token), url(r"^auth/", include("rest_framework.urls", namespace="rest_framework")),