From de02a0280b4ef6644c4fe62c0fbb726abf093336 Mon Sep 17 00:00:00 2001 From: Ming <36557541+minghao912@users.noreply.github.com> Date: Sun, 20 Nov 2022 15:20:08 -0800 Subject: [PATCH] feat: Add endpoint which returns 500 if any critical errors in sendpost.py occur (#390) * chore: allow print calls to show in docker log * feat: set up health check endpoint * fix: correct import --- docker-compose.yml | 2 ++ meow/meow/urls.py | 2 ++ meow/meow/views.py | 24 ++++++++++++++ .../management/commands/sendposts.py | 3 ++ meow/scheduler/management/healthcheck.py | 32 +++++++++++++++++++ package-lock.json | 14 ++++---- 6 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 meow/scheduler/management/healthcheck.py diff --git a/docker-compose.yml b/docker-compose.yml index d2fedd4c6..74e170cfa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: - redis volumes: - .:/meow + environment: + PYTHONUNBUFFERED: 1 db: image: postgres:latest environment: diff --git a/meow/meow/urls.py b/meow/meow/urls.py index b241966e3..db031385d 100644 --- a/meow/meow/urls.py +++ b/meow/meow/urls.py @@ -6,5 +6,7 @@ urlpatterns = [ url(r'^admin/', admin.site.urls), url(r'^api/v1/', include('urls.urls')), + url(r'healthcheck/', views.healthcheck, name='healthcheck'), + url(r'healthchecktest/', views.healthchecktest, name='healthchecktest'), url(r'', views.base, name='base') ] diff --git a/meow/meow/views.py b/meow/meow/views.py index 44a2184a4..e99e16755 100644 --- a/meow/meow/views.py +++ b/meow/meow/views.py @@ -2,7 +2,31 @@ import re from django.shortcuts import redirect, render, get_object_or_404 from django.template.loader import get_template +from django.http import JsonResponse + +# For health check +from scheduler.management.healthcheck import HealthCheck def base(request): #print("BASE") return render(request, 'base.html') + +def healthcheck(request): + (healthy, message) = HealthCheck.healthCheck() + + # Set up JSON Response + data = { + 'healthy': str(healthy), + 'message': message if message else '' + } + + return JsonResponse(data, status=(500 if not healthy else 200)) + +# Test only, sets program to unhealthy state +def healthchecktest(request): + msg = request.GET.get('msg', '') + + print('Health Check Test: ' + msg, flush=True) + + HealthCheck.setUnhealthy(msg if msg else None) + return JsonResponse({'message': 'ok'}) \ No newline at end of file diff --git a/meow/scheduler/management/commands/sendposts.py b/meow/scheduler/management/commands/sendposts.py index 0b8641cb2..18c8cfc69 100644 --- a/meow/scheduler/management/commands/sendposts.py +++ b/meow/scheduler/management/commands/sendposts.py @@ -18,6 +18,7 @@ import logging from scheduler.models import MeowSetting, SMPost +from scheduler.management.healthcheck import HealthCheck logger = logging.getLogger('scheduler') @@ -125,6 +126,7 @@ def handle(self, *args, **options): except: logger.critical("Something is very wrong in sendpost.py ") logger.critical("Something is very wrong in sendpost.py " + str(traceback.format_exc())) + HealthCheck.setUnhealthy("Something is very wrong in sendpost.py " + str(traceback.format_exc())) post.log(traceback.format_exc()) logger.info("sendpost.py: Post {}-{} failed to send because it would been late. ".format(post.slug, post.id)) @@ -190,5 +192,6 @@ def handle(self, *args, **options): post.save() except (Exception) as e: logger.critical("Something is very wrong" + traceback.format_exc()) + HealthCheck.setUnhealthy("Something is very wrong" + traceback.format_exc()) post.log(traceback.format_exc()) post.log_error(e, post.section, True) diff --git a/meow/scheduler/management/healthcheck.py b/meow/scheduler/management/healthcheck.py new file mode 100644 index 000000000..93df48d8b --- /dev/null +++ b/meow/scheduler/management/healthcheck.py @@ -0,0 +1,32 @@ +from typing import Optional, Tuple, List + +''' +Utils class to check app health. +Used for Kubernetes to automatically restart app when critical errors occur. + +Process: + 1. A critical error occurs + 2. HealthCheck receives notification of this via setUnhealthy(msg: Optional[str]) method + 3. HealthCheck keeps changes program state to unhealthy, and keeps track of the message + 4. A routine call to the healthcheck API occurs + a. API calls healthCheck() which returns current status + b. If program not in healthy state, API returns 500 status code along with messages + c. If program is healthy, API returns 200 OK +''' +class HealthCheck(object): + # Static var + _healthy = True + _message: List[str] = [] + + # Used to set global healthiness, with optional message + @staticmethod + def setUnhealthy(message: Optional[str] = None) -> None: + HealthCheck._healthy = False + + if message: + HealthCheck._message.append(message) + + # Returns global healthiness + @staticmethod + def healthCheck() -> Tuple[bool, str]: + return (HealthCheck._healthy, HealthCheck._message) \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index aabcaa77c..3f58dc104 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2124,7 +2124,7 @@ "async-foreach": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz", - "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI=", + "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", "dev": true }, "async-limiter": { @@ -2508,7 +2508,7 @@ "babel-plugin-syntax-jsx": { "version": "6.18.0", "resolved": "https://registry.npmjs.org/babel-plugin-syntax-jsx/-/babel-plugin-syntax-jsx-6.18.0.tgz", - "integrity": "sha1-CvMqmm4Tyno/1QaeYtew9Y0NiUY=" + "integrity": "sha512-qrPaCSo9c8RHNRHIotaufGbuOBN8rtdC4QrrFFc43vyWCCz7Kl7GL1PGaXtMGQZUXrkCjNEgxDfmAuAabr/rlw==" }, "babel-plugin-syntax-object-rest-spread": { "version": "6.13.0", @@ -4121,7 +4121,7 @@ "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", "dev": true }, "constants-browserify": { @@ -4352,7 +4352,7 @@ "css-color-keywords": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/css-color-keywords/-/css-color-keywords-1.0.0.tgz", - "integrity": "sha1-/qJhbcZ2spYmhrOvjb2+GAskTgU=" + "integrity": "sha512-FyyrDHZKEjXDpNJYvVsV960FiqQyXc/LlYmsxl2BcdMb2WPx0OGRVgTg55rPSyLSNMqP52R9r8geSp7apN3Ofg==" }, "css-color-names": { "version": "0.0.4", @@ -4983,7 +4983,7 @@ "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", "dev": true }, "depd": { @@ -8473,7 +8473,7 @@ "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "dev": true }, "has-value": { @@ -11455,7 +11455,7 @@ "lodash.tail": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.tail/-/lodash.tail-4.1.1.tgz", - "integrity": "sha1-0jM6NtnncXyK0vfKyv7HwytERmQ=", + "integrity": "sha512-+7y6zfkH4TqgS5DYKIqJuxmL5xT3WUUumVMZVRpDUo0UqJREwZqKmGo9wluj12FbPGl1UjRf2TnAImbw/bKtdw==", "dev": true }, "lodash.template": {