adding backed logger feature ...

Author: sadayamuthu

logmetrics_sdk/__init__.py

@@ -1,7 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""Top-level package for logmetrics_sdk."""
-
-__author__ = """devxchange.io"""
-__email__ = 'info@devxchange.io'
-__version__ = '1.0.0'

logmetrics_sdk/common/__init__.py

@@ -0,0 +1,21 @@
+
+class LogMetricsConfig:
+
+    enable_logmetrics = True
+    enable_frontend_request = True
+    enable_frontend_response = True
+    enable_backend = True
+    enable_backend_request = True
+    enable_backend_response = True
+
+    @classmethod
+    def set_internal_state(cls, enable_logmetrics=True,
+                           enable_frontend_request=True, enable_frontend_response=True,
+                           enable_backend=True, enable_backend_request=True, enable_backend_response=True):
+        cls.enable_logmetrics = enable_logmetrics
+        cls.enable_frontend_request = enable_frontend_request
+        cls.enable_frontend_response = enable_frontend_response
+        cls.enable_backend = enable_backend
+        cls.enable_backend_request = enable_backend_request
+        cls.enable_backend_response = enable_backend_response
+

logmetrics_sdk/common/logmetrics_config.py

@@ -1,4 +1,5 @@
 message: dict = {
+    'TrackingID': str,
     'MessageType': str,
     'Duration': int,
     'Host': str,
@@ -13,7 +14,7 @@
     'StartDateTime': str,
     'EndDateTime': str,
     'Aspects': str,
-    'Query': str,
+    'QueryParams': str,
     'Path': str,
     'RequestBody': str,
     'ResponseBody': str,
@@ -29,9 +30,7 @@
 
 
 def clear_message():
-    """
-
-    """
+    message['TrackingID'] = None
     message['MessageType'] = "LOGMETRICS_MESSAGE"
     message['Duration'] = 0
     message['Host'] = None
@@ -45,7 +44,8 @@ def clear_message():
     message['ServiceVersion'] = None
     message['StartDateTime'] = None
     message['EndDateTime'] = None
-    message['Query'] = None
+    message['Aspects'] = None
+    message['QueryParams'] = None
     message['Path'] = None
     message['RequestBody'] = None
     message['ResponseBody'] = None

logmetrics_sdk/utils/message.py logmetrics_sdk/common/message.pylogmetrics_sdk/utils/message.py renamed to logmetrics_sdk/common/message.py

@@ -5,32 +5,49 @@
 from functools import wraps
 from json import JSONDecodeError
 from typing import Dict
+import uuid
 
 from logmetrics_sdk.utils.importer import LazyImport
-from logmetrics_sdk.utils.message import message, clear_message
+from logmetrics_sdk.common.message import message, clear_message
 from logmetrics_sdk.utils.constants import *
 from logmetrics_sdk.utils.timer import TimeUtil, preferred_clock
+from logmetrics_sdk.patch.patch import patch_backend
+from logmetrics_sdk.common.logmetrics_config import LogMetricsConfig
 
 flask = LazyImport('flask')
 logging.basicConfig(level=logging.INFO)
 
 
-def LogMetrics(_fn: object = None) -> object:
-    """
-    """
+def LogMetrics(_fn=None, *, enable_logmetrics=True, enable_frontend_request=True,enable_frontend_response=True,
+               enable_backend=True, enable_backend_request=True, enable_backend_response = True):
+
+    LogMetricsConfig.set_internal_state(enable_logmetrics,
+                                        enable_frontend_request,
+                                        enable_frontend_response,
+                                        enable_backend,
+                                        enable_backend_request,
+                                        enable_backend_response)
+    patch_backend()
 
     def new_func(fn):
         @wraps(fn)
         def wrap(*args, **kwargs):
             start = preferred_clock()
             clear_message()
+            message['TrackingID'] = str(uuid.uuid4())
             message['Action'] = FRONTEND
             message['MessageType'] = LOGMETRICS_MESSAGE
             message['StartDateTime'] = str(datetime.datetime.now())
 
             if flask.has_request_context():
-                message['Query'] = flask.request.query_string
-                message['RequestBody'] = flask.request.data
+
+                if enable_frontend_request:
+                    if flask.request.data:
+                        message['RequestBody'] = flask.request.data
+
+                    if flask.request.query_string:
+                        message['QueryParams'] = flask.request.query_string
+
                 message['ClientHost'] = flask.request.remote_addr
                 message['Host'] = flask.request.host
                 message['HttpMethod'] = flask.request.method
@@ -48,10 +65,12 @@ def wrap(*args, **kwargs):
                     status = int(response.status_code)
                     message['HttpStatus'] = status
                     message['Fault'] = status not in (200, 300)
-                    message['ResponseBody'] = json.dumps(data)
+                    if enable_frontend_response and data:
+                        message['ResponseBody'] = json.dumps(data)
                     message['ContentType'] = response.content_type
                 except (JSONDecodeError, TypeError):
-                    message['ResponseBody'] = json.dumps(data)
+                    if enable_frontend_response and data:
+                        message['ResponseBody'] = json.dumps(data)
 
             elif isinstance(return_value, Dict):
                 if 'statusCode' in return_value:
@@ -60,9 +79,11 @@ def wrap(*args, **kwargs):
                     message['Fault'] = not (status == 200 or status == 300)
                     if 'body' in return_value:
                         try:
-                            message['ResponseBody'] = json.dumps(json.loads(return_value['body']))
+                            if enable_frontend_response:
+                                message['ResponseBody'] = json.dumps(json.loads(return_value['body']))
                         except (JSONDecodeError, TypeError):
-                            message['ResponseBody'] = return_value['body']
+                            if enable_frontend_response:
+                                message['ResponseBody'] = return_value['body']
                 else:
                     try:
                         status = int(getattr(return_value, 'status_code', 200))
@@ -72,7 +93,12 @@ def wrap(*args, **kwargs):
                         pass
             message['Duration'] = TimeUtil.get_duration(start, end)
 
-            logging.info(message)
+            if enable_logmetrics:
+                try:
+                    logging.info(json.dumps(message))
+                except Exception as ex:
+                    logging.error(f'{ex}')
+
             return return_value
 
         wrap._original = fn

logmetrics_sdk/logmetrics.py

@@ -0,0 +1,50 @@
+
+import logging
+import importlib
+
+log = logging.getLogger(__name__)
+
+
+SUPPORTED_MODULES = (
+    'requests'
+)
+
+_PATCHED_MODULES = set()
+
+
+def patch_backend():
+    patch(SUPPORTED_MODULES, raise_errors=False)
+
+
+def patch(modules_to_patch, raise_errors=True):
+    if isinstance(modules_to_patch, str):
+        modules_to_patch = [modules_to_patch]
+    modules = set()
+    for module_to_patch in modules_to_patch:
+            modules.add(module_to_patch)
+    for m in modules:
+        _patch_module(m, raise_errors)
+
+
+def _patch_module(module_to_patch, raise_errors=True):
+    try:
+        _patch(module_to_patch)
+    except KeyError:
+        if raise_errors:
+            raise
+        log.debug('failed to patch module %s', module_to_patch)
+
+
+def _patch(module_to_patch):
+
+    path = 'logmetrics_sdk.patch.%s' % module_to_patch
+
+    if module_to_patch in _PATCHED_MODULES:
+        log.debug('%s already patched', module_to_patch)
+        return
+
+    imported_module = importlib.import_module(path)
+    imported_module.patch()
+
+    _PATCHED_MODULES.add(module_to_patch)
+    log.info('successfully patched module %s', module_to_patch)

logmetrics_sdk/patch/__init__.py

@@ -0,0 +1,3 @@
+from .patch import patch_requests_session as patch
+
+__all__ = ['patch']

logmetrics_sdk/patch/patch.py

@@ -0,0 +1,148 @@
+
+import json
+import logging
+import requests.sessions
+import chardet
+import datetime
+
+from functools import wraps
+from urllib.parse import urlsplit
+from json import JSONDecodeError
+from xml.etree.ElementTree import ParseError
+from requests.structures import CaseInsensitiveDict
+
+from logmetrics_sdk.common.logmetrics_config import LogMetricsConfig
+from logmetrics_sdk.common.message import message
+from logmetrics_sdk.utils.parse import parse_qs
+from logmetrics_sdk.utils.constants import *
+from logmetrics_sdk.utils.json_util import to_object
+
+
+CONTENT_TYPE = 'content-type'
+
+log = logging.getLogger(__name__)
+
+
+def patch_requests_session():
+    _patch_request()
+
+
+def _patch_request():
+
+    def time_requests_session_request(f):
+
+        @wraps(f)
+        def wrap(self, method, url, **kwargs):
+            start_time = str(datetime.datetime.now())
+            collect_payload_for_host = True
+            parsed_url = urlsplit(url)
+            query_params = (requests.sessions.merge_setting(kwargs.get('params'), self.params)
+                            or parse_qs(parsed_url.query))
+            request_data = kwargs.get('data') or kwargs.get('json')
+
+            _message = {
+                'TrackingID': message.get('TrackingID'),
+                'MessageType': LOGMETRICS_MESSAGE,
+                'FrontendMethod': message.get('FrontendMethod', None),
+                'ApplicationName': message.get('ApplicationName', None),
+                'BackendServiceName': None,
+                'BackendSystem': parsed_url.netloc,
+                'BackendMethod': parsed_url.path,
+                'HttpMethod': method,
+                'Action': BACKEND,
+                'StartDateTime': start_time
+            }
+
+            if collect_payload_for_host and query_params:
+                _message['QueryParams'] = query_params
+
+            if request_data:
+                request_headers = requests.sessions.merge_setting(
+                    kwargs.get('headers'), self.headers, dict_class=CaseInsensitiveDict
+                )
+                if collect_payload_for_host and LogMetricsConfig.enable_backend_request:
+                    payload = _process_payload_data(
+                        payload=request_data,
+                        payload_type='request',
+                        content_type=request_headers.get(CONTENT_TYPE)
+                    )
+                    _message.update(payload)
+            response = None
+            try:
+                response = f(self, method, url, **kwargs)
+            except Exception as e:
+                _message['Fault'] = True
+                _message['ErrorMessage'] = getattr(e, 'message', str(e))
+                _message['ErrorCode'] = type(e).__name__
+                _message['HttpStatus'] = None
+                raise e
+            finally:
+                if response:
+                    if response.encoding is None:
+                        response.encoding = chardet.detect(response.content)['encoding']
+                        if collect_payload_for_host and LogMetricsConfig.enable_backend_response:
+                            payload = _process_payload_data(
+                                payload=response.text,
+                                payload_type='response',
+                                content_type=response.headers.get(CONTENT_TYPE))
+                            _message.update(payload)
+                    _message['HttpStatus'] = response.status_code
+                    _message['Duration'] = 0
+                if LogMetricsConfig.enable_backend_response:
+                    logging.info(json.dumps(_message))
+            return response
+        return wrap
+
+    requests.sessions.Session.request = time_requests_session_request(
+        requests.sessions.Session.request
+    )
+
+
+def _process_payload_data(payload: str, payload_type: str, content_type: str):
+
+    _message = {}
+    metric_fieldname = payload_type.capitalize() + 'Body'
+    content_is_json, content_is_xml, content_is_plaintext = False, False, False
+
+    if content_type:
+        content_is_json = '/json' in content_type
+        content_is_xml = '/xml' in content_type
+        content_is_plaintext = 'text/' in content_type
+
+    if content_is_json:
+        try:
+            py_obj_data = to_object(payload)
+            _message['ResponseBody'] = json.dumps(py_obj_data)
+        except JSONDecodeError:
+            _message[metric_fieldname] = payload.strip()
+            _message['ErrorMessage'] = ('Error occurred while parsing JSON data for '
+                                      + metric_fieldname)
+            _message['ErrorCode'] = 'JSONParseError'
+
+    elif content_is_xml:
+        try:
+            _message[metric_fieldname] = "{\"XML\": true}"
+        except ParseError:
+            _message[metric_fieldname] = payload.strip()
+            _message['ErrorMessage'] = ('Error occurred while parsing XML data for '
+                                      + metric_fieldname)
+            _message['ErrorCode'] = 'XMLParseError'
+
+    elif content_is_plaintext:
+        _message[metric_fieldname] = payload.strip()
+
+    else:
+        try:
+            py_obj_data = to_object(payload)
+            _message[metric_fieldname] = json.dumps(py_obj_data)
+        except JSONDecodeError:
+            try:
+                message[metric_fieldname] = "{\"XML\": true}"
+
+            except ParseError:
+                _message[metric_fieldname] = payload.strip()
+                _message['ErrorMessage'] = ('Error occurred while parsing data as JSON & XML for '
+                                          + metric_fieldname)
+                _message['ErrorCode'] = 'ParseError'
+
+    return _message