feat: Add endpoint which returns 500 if any critical errors in sendpost.py occur (#390)

* chore: allow print calls to show in docker log

* feat: set up health check endpoint

* fix: correct import

Author: minghao912

docker-compose.yml

@@ -14,6 +14,8 @@ services:
       - redis
     volumes:
       - .:/meow
+    environment:
+      PYTHONUNBUFFERED: 1
   db:
     image: postgres:latest
     environment:

meow/meow/urls.py

@@ -6,5 +6,7 @@
 urlpatterns = [
     url(r'^admin/', admin.site.urls),
     url(r'^api/v1/', include('urls.urls')),
+    url(r'healthcheck/', views.healthcheck, name='healthcheck'),
+    url(r'healthchecktest/', views.healthchecktest, name='healthchecktest'),
     url(r'', views.base, name='base')
 ]

meow/meow/views.py

@@ -2,7 +2,31 @@
 import re
 from django.shortcuts import redirect, render, get_object_or_404
 from django.template.loader import get_template
+from django.http import JsonResponse
+
+# For health check
+from scheduler.management.healthcheck import HealthCheck
 
 def base(request):
     #print("BASE")
     return render(request, 'base.html')
+
+def healthcheck(request):
+    (healthy, message) = HealthCheck.healthCheck()
+
+    # Set up JSON Response
+    data = {
+            'healthy': str(healthy),
+            'message': message if message else ''
+        }
+
+    return JsonResponse(data, status=(500 if not healthy else 200))
+
+# Test only, sets program to unhealthy state
+def healthchecktest(request):
+    msg = request.GET.get('msg', '')
+
+    print('Health Check Test: ' + msg, flush=True)
+
+    HealthCheck.setUnhealthy(msg if msg else None)
+    return JsonResponse({'message': 'ok'})

meow/scheduler/management/commands/sendposts.py

@@ -18,6 +18,7 @@
 import logging
 
 from scheduler.models import MeowSetting, SMPost
+from scheduler.management.healthcheck import HealthCheck
 
 logger = logging.getLogger('scheduler')
 
@@ -125,6 +126,7 @@ def handle(self, *args, **options):
                     except:
                         logger.critical("Something is very wrong in sendpost.py ")
                         logger.critical("Something is very wrong in sendpost.py " + str(traceback.format_exc()))
+                        HealthCheck.setUnhealthy("Something is very wrong in sendpost.py " + str(traceback.format_exc()))
                         post.log(traceback.format_exc())
 
                     logger.info("sendpost.py: Post {}-{} failed to send because it would been late. ".format(post.slug, post.id))
@@ -190,5 +192,6 @@ def handle(self, *args, **options):
                 post.save()
             except (Exception) as e:
                 logger.critical("Something is very wrong" + traceback.format_exc())
+                HealthCheck.setUnhealthy("Something is very wrong" + traceback.format_exc())
                 post.log(traceback.format_exc())
                 post.log_error(e, post.section, True)

meow/scheduler/management/healthcheck.py

@@ -0,0 +1,32 @@
+from typing import Optional, Tuple, List
+
+'''
+Utils class to check app health.
+Used for Kubernetes to automatically restart app when critical errors occur.
+
+Process:
+    1. A critical error occurs
+    2. HealthCheck receives notification of this via setUnhealthy(msg: Optional[str]) method
+    3. HealthCheck keeps changes program state to unhealthy, and keeps track of the message
+    4. A routine call to the healthcheck API occurs
+        a. API calls healthCheck() which returns current status
+        b. If program not in healthy state, API returns 500 status code along with messages
+        c. If program is healthy, API returns 200 OK
+'''
+class HealthCheck(object):
+    # Static var
+    _healthy = True
+    _message: List[str] = []
+
+    # Used to set global healthiness, with optional message
+    @staticmethod
+    def setUnhealthy(message: Optional[str] = None) -> None:
+        HealthCheck._healthy = False
+
+        if message:
+            HealthCheck._message.append(message)
+
+    # Returns global healthiness
+    @staticmethod
+    def healthCheck() -> Tuple[bool, str]:
+        return (HealthCheck._healthy, HealthCheck._message)