Async release

app/configs/logging/__init__.py

@@ -0,0 +1,7 @@
+from .filters import ErrorAndAboveFilter
+from .queue_handler import QueueListenerHandler
+
+__all__ = [
+    "ErrorAndAboveFilter",
+    "QueueListenerHandler"
+]

app/configs/logging/filters.py

@@ -0,0 +1,6 @@
+import logging
+
+
+class ErrorAndAboveFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool | logging.LogRecord:
+        return record.levelno < logging.ERROR

app/configs/logging/formatters.py

@@ -0,0 +1,105 @@
+import datetime as dt
+import json
+import logging
+
+LOG_RECORD_BUILTIN_ATTRS = {
+    "args",
+    "asctime",
+    "created",
+    "exc_info",
+    "exc_text",
+    "filename",
+    "funcName",
+    "levelname",
+    "levelno",
+    "lineno",
+    "module",
+    "msecs",
+    "message",
+    "msg",
+    "name",
+    "pathname",
+    "process",
+    "processName",
+    "relativeCreated",
+    "stack_info",
+    "thread",
+    "threadName",
+    "taskName",
+}
+
+"""
+    This isn't being used since the app will be run on gcloud run but this can be used for future apps.
+    If you want to test it:
+     
+    formatters:
+    
+    "json": {
+      "()": "json_formatter.JSONFormatter",
+      "fmt_keys": {
+        "level": "levelname",
+        "message": "message",
+        "timestamp": "timestamp",
+        "logger": "name",
+        "module": "module",
+        "function": "funcName",
+        "line": "lineno",
+        "thread_name": "threadName"
+      }
+    } 
+    
+    handlers:
+    
+    "file_json": {
+      "class": "logging.handlers.RotatingFileHandler",
+      "level": "DEBUG",
+      "formatter": "json",
+      "filename": "logs/log",
+      "maxBytes": 1000000,
+      "backupCount": 3
+    }
+    
+    and add "cfg://handlers.file_json" to queue handler
+"""
+
+# From this video https://www.youtube.com/watch?v=9L77QExPmI0
+# Src here: https://github.com/mCodingLLC/VideosSampleCode/blob/master/videos/135_modern_logging/mylogger.py
+class JSONFormatter(logging.Formatter):
+    def __init__(
+        self,
+        *,
+        fmt_keys: dict[str, str] | None = None,
+    ):
+        super().__init__()
+        self.fmt_keys = fmt_keys if fmt_keys is not None else {}
+
+    def format(self, record: logging.LogRecord) -> str:
+        message = self._prepare_log_dict(record)
+        return json.dumps(message, default=str)
+
+    def _prepare_log_dict(self, record: logging.LogRecord):
+        always_fields = {
+            "message": record.getMessage(),
+            "timestamp": dt.datetime.fromtimestamp(
+                record.created, tz=dt.timezone.utc
+            ).isoformat(),
+        }
+        if record.exc_info is not None:
+            always_fields["exc_info"] = self.formatException(record.exc_info)
+
+        if record.stack_info is not None:
+            always_fields["stack_info"] = self.formatStack(record.stack_info)
+
+        message = {
+            key: msg_val
+            if (msg_val := always_fields.pop(val, None)) is not None
+            else getattr(record, val)
+            for key, val in self.fmt_keys.items()
+        }
+        message.update(always_fields)
+
+        for key, val in record.__dict__.items():
+            if key not in LOG_RECORD_BUILTIN_ATTRS:
+                message[key] = val
+
+        return message

app/configs/logging/logging_config.json

@@ -0,0 +1,53 @@
+{
+  "version": 1,
+  "objects": {
+    "queue": {
+      "class": "queue.Queue",
+      "maxsize": 1000
+    }
+  },
+  "disable_existing_loggers": false,
+  "formatters": {
+    "simple": {
+      "format": "[%(levelname)s] (%(module)s|L: %(lineno)d) %(asctime)s: %(message)s",
+      "datefmt": "%Y-%m-%dT%H:%M:%S%z"
+    }
+  },
+  "filters": {
+    "error_and_above": {
+      "()": "app.configs.logging.ErrorAndAboveFilter"
+    }
+  },
+  "handlers": {
+    "console": {
+      "class": "logging.StreamHandler",
+      "level": "INFO",
+      "formatter": "simple",
+      "stream": "ext://sys.stdout",
+      "filters": ["error_and_above"]
+    },
+    "error": {
+      "class": "logging.StreamHandler",
+      "level": "ERROR",
+      "formatter": "simple",
+      "stream": "ext://sys.stderr"
+    },
+    "queue_handler": {
+      "class": "app.configs.logging.QueueListenerHandler",
+      "handlers": [
+        "cfg://handlers.console",
+        "cfg://handlers.error"
+      ],
+      "queue": "cfg://objects.queue",
+      "respect_handler_level": true
+    }
+  },
+  "loggers": {
+    "root": {
+      "level": "DEBUG",
+      "handlers": [
+        "queue_handler"
+      ]
+    }
+  }
+}

app/configs/logging/queue_handler.py

@@ -0,0 +1,61 @@
+from logging.config import ConvertingList, ConvertingDict, valid_ident
+from logging.handlers import QueueHandler, QueueListener
+from queue import Queue
+import atexit
+
+
+class QueueHnadlerHelper:
+
+    @staticmethod
+    def resolve_handlers(l):
+        if not isinstance(l, ConvertingList):
+            return l
+
+        # Indexing the list performs the evaluation.
+        return [l[i] for i in range(len(l))]
+
+    @staticmethod
+    def resolve_queue(q):
+        if not isinstance(q, ConvertingDict):
+            return q
+        if '__resolved_value__' in q:
+            return q['__resolved_value__']
+
+        cname = q.pop('class')
+        klass = q.configurator.resolve(cname)
+        props = q.pop('.', None)
+        kwargs = {k: q[k] for k in q if valid_ident(k)}
+        result = klass(**kwargs)
+        if props:
+            for name, value in props.items():
+                setattr(result, name, value)
+
+        q['__resolved_value__'] = result
+        return result
+
+
+# The guy from this video https://www.youtube.com/watch?v=9L77QExPmI0 is using logging features only available in 3.12
+# This article had the class required to build the queue handler in 3.11
+# https://rob-blackbourn.medium.com/how-to-use-python-logging-queuehandler-with-dictconfig-1e8b1284e27a
+class QueueListenerHandler(QueueHandler):
+
+    def __init__(self, handlers, respect_handler_level=False, auto_run=True, queue=Queue(-1)):
+        queue = QueueHnadlerHelper.resolve_queue(queue)
+        super().__init__(queue)
+        handlers = QueueHnadlerHelper.resolve_handlers(handlers)
+        self._listener = QueueListener(
+            self.queue,
+            *handlers,
+            respect_handler_level=respect_handler_level)
+        if auto_run:
+            self.start()
+            atexit.register(self.stop)
+
+    def start(self):
+        self._listener.start()
+
+    def stop(self):
+        self._listener.stop()
+
+    def emit(self, record):
+        return super().emit(record)