Python Logging - Configuring Rotating File Handler From Config file - python-3.x

I have the below config file which I am using to configure logging in my python modules , the issue with this is that it does not append the timestamp to the file-name , right now the log file created is file_handler.log but I want it to be file_handler-timestamp.log
{
"version": 1,
"disable_existing_loggers": true,
"formatters": {
"simple": {
"format": "%(asctime)s %(levelname)s %(filename)s %(lineno)s %(message)s",
"datefmt": "%Y-%m-%d %H:%M:%S"
}
},
"handlers": {
"file_handler": {
"level": "INFO",
"class": "logging.handlers.TimedRotatingFileHandler",
"formatter": "simple",
"filename": "error.log",
"backupCount": 10,
"interval" : 1,
"when": "s",
"encoding": "utf8"
}
},
"loggers": { },
"root": {
"handlers": [
"file_handler"
],
"level": "DEBUG"
}
}
Working code without config
import logging
import time
from logging.handlers import RotatingFileHandler
logger = logging.getLogger('log_1')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', '%Y-%m-%d %H:%M:%S')
handler = RotatingFileHandler('log' + time.strftime('_%Y%m%d-%H%M') + '.log', maxBytes=2000, backupCount=10)
handler.setFormatter(formatter)
logger.addHandler(handler)
Workaround code which updated the config dynamically when program is initiated
import logging.config
import json
import time
fp = open("sample_config.json")
config = json.load(fp)
fp.close()
config['handlers']['file_handler']['filename'] += time.strftime('_%Y%m%d-%H%M') + '.log'
logging.config.dictConfig(config)

timestamp of when? When the file is rotated or when it is created?
If you use TimedRotatingFileHandler, then when the file is rotated, the timestamp is appended to the filename.
If you want the timestamp appended to the log filename, you have to subclass RotatingFileHandler and modify the __init__ method. Then point the "class" item of "file_handler" of the config file to your new class.
It is easy to do, but it may not be what you need.
Just let the timestamp be appended to the rotated log filename or if you need to limit the size too, read this answer

Related

Creating CustomConnector connection (GoogleAds) using AWS Lambda | AppFlow

Does Boto3 client support connectors for GoogleAds and FacebookAds? According to documentation we can use Custom Connector but when i try to use it in the code i get the below error saying it should be one of the built in types.
[ERROR] ParamValidationError: Parameter validation failed:
Unknown parameter in connectorProfileConfig.connectorProfileProperties: "CustomConnector", must be one of: Amplitude, Datadog, Dynatrace, GoogleAnalytics, Honeycode, InforNexus, Marketo, Redshift, Salesforce, ServiceNow, Singular, Slack, Snowflake, Trendmicro, Veeva, Zendesk, SAPOData
Unknown parameter in connectorProfileConfig.connectorProfileCredentials: "CustomConnector", must be one of: Amplitude, Datadog, Dynatrace, GoogleAnalytics, Honeycode, InforNexus, Marketo, Redshift, Salesforce, ServiceNow, Singular, Slack, Snowflake, Trendmicro, Veeva, Zendesk, SAPOData
Traceback (most recent call last):
  File "/var/task/lambda_function.py", line 34, in lambda_handler
    response = client.create_connector_profile(
  File "/var/runtime/botocore/client.py", line 391, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/var/runtime/botocore/client.py", line 691, in _make_api_call
    request_dict = self._convert_to_request_dict(
  File "/var/runtime/botocore/client.py", line 739, in _convert_to_request_dict
    request_dict = self._serializer.serialize_to_request(
  File "/var/runtime/botocore/validate.py", line 360, in serialize_to_request
    raise ParamValidationError(report=report.generate_report())
Code in Lambda :
import json
import boto3
def lambda_handler(event, context):
client = boto3.client('appflow')
### Google Ads
response = client.create_connector_profile(
connectorProfileName='GoogleAdsConn',
connectorType='CustomConnector',
# connectorLabel='GoogleAds',
connectionMode='Public',
connectorProfileConfig= {
"connectorProfileProperties": {
'CustomConnector': {
# 'profileProperties': {
# 'string': 'string'
# },
'oAuth2Properties': {
'tokenUrl': 'https://oauth2.googleapis.com/token',
'oAuth2GrantType': 'AUTHORIZATION_CODE'
# ,'tokenUrlCustomProperties': {
# 'string': 'string'
# }
}
}
},
"connectorProfileCredentials": {
"CustomConnector": {
"authenticationType": "OAUTH2",
"oauth2": {
"accessToken": "myaccesstoken",
"clientId": "myclientid",
"clientSecret": "myclientsecret",
"oAuthRequest": {
"authCode": "string",
"redirectUri": "string"
},
"refreshToken": "myrefreshtoken"
}
}
}
}
)
return {
'response': response
}
Any leads on this will be appreciated.
Thanks!
The issue was with older boto3 version. Adding a new lambda layer with latest boto3 version(1.24.70) and updating code with profileProperties it worked seamlessly. Below is the complete working code.
import json
import boto3
def lambda_handler(event, context):
client = boto3.client('appflow')
### Google Ads
response = client.create_connector_profile(
connectorProfileName='GoogleAdsConnThruLambda',
connectorType='CustomConnector',
connectorLabel='GoogleAds',
connectionMode='Public',
connectorProfileConfig= {
"connectorProfileProperties": {
'CustomConnector': {
'profileProperties': {
'developerToken': 'developerToken',
'instanceUrl':'https://googleads.googleapis.com',
'managerID':'managerID',
'apiVersion':'v11'
},
'oAuth2Properties': {
'tokenUrl': 'https://oauth2.googleapis.com/token',
'oAuth2GrantType': 'AUTHORIZATION_CODE'
# ,'tokenUrlCustomProperties': {
# "string":"string"
# }
}
}
},
"connectorProfileCredentials": {
"CustomConnector": {
"authenticationType": "OAUTH2",
"oauth2": {
"accessToken": "accessToken",
"clientId": "clientId",
"clientSecret": "clientSecret",
"oAuthRequest": {
"authCode": "authCode",
"redirectUri": "https://<your_region>.console.aws.amazon.com/appflow/oauth"
},
"refreshToken": "refreshToken"
}
}
}
}
)
return {
'response': response
}

Django Logging - Unable to set propagate to False

What I'm experiencing is getting duplicate entries in my logs. I've looked around, and it seems that I have to set 'propagate': False in my loggers configuration which I have already done. However when I printout the logger.propagate it returns True. I even tried to manually set logger.propagate = False but it still returns True and I'm receiving duplicate entries in my logs.
What might be the cause of problem?
import logging
logger = logging.getLogger(__name__)
logger.propagate = False
class PostsListAPIView(ListAPIView):
def get_queryset(self):
# Getting both twice
logger.error('Something went wrong!')
logger.error(logger.propagate) # Returns True
queryset = ...
return queryset
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"simple": {
"format": "{levelname} {message}",
"style": "{",
}
},
"handlers": {
"console": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "simple",
},
},
"loggers": {
"app": {
"level": "DEBUG",
"handlers": ["console"],
'propagate': False,
}
},
}
I also tried setting "disable_existing_loggers": True, but it has no effects.
As for getting duplicate log entries, I found out that the cause is get_queryset method being called twice in order to display the forms as my BrowsableAPI is enabled.
However I still have not Idea why logger.propagate returns True.

How to save log file into subdirectory using dictConfig configuration?

I have the following config in my Python 3.6 application:
config = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"standard": {
"format": ("[%(asctime)s] %(levelname)s - %(threadName)s - %(name)s - %(message)s")
},
},
"handlers": {
"default": {
"class": "logging.StreamHandler",
"formatter": "standard",
"level": DEFAULT_LOG_LEVEL,
},
"fh": {
"class": "logging.handlers.RotatingFileHandler",
"formatter": "standard",
"level": FILE_LOG_LEVEL,
"filename": "myfile.log",
"maxBytes": 1024*1000, # 1 MB
"backupCount": 20
}
},
"loggers": {
'': {
"handlers": ["default", "fh"],
'level': DEFAULT_LOG_LEVEL,
'propagate': True
}
}
}
This currently works; however, it saves all the .log files to the current working directory (where the executable is on Windows).
What I'd like to do is have it save this to the "logs" directory which is in the same level as the executable itself and already existing.
My thought was to modify the "filename" property int he log configuration, but I'm not sure how that impacts my script files that use:
logger = logging.getLogger(__name__)
Would that need to be modified as well to get it to log to the correct location?
Desired Structure:
app.exe
logs
myfile.log
My initial thought seems do what I need after testing. I simply put the directory name in front of the file name in the config's "filename" property value. No modification was needed to the script's logger line.

Returning large data from RPC (Crossbar + Autobahn|Python)

I am trying to transmit large data through websockets using crossbar/autobahn's RPC. My setup is as follow:
Python 2.7
A crossbar router (version 17.8.1.post1)
A back-end that will try to send a large pandas DataFrame as a json string
A front-end that will want to receive this string
In essence my front-end is trying to call a function that will return a large string.
class MyComponent(ApplicationSession):
#inlineCallbacks
def onJoin(self, details):
print("session ready")
try:
res = yield self.call(u'data.get')
And I get this error:
2017-08-09T16:38:10+0200 session closed with reason wamp.close.transport_lost [WAMP transport was lost without closing the session before]
2017-08-09T16:38:10+0200 Cancelling 1 outstanding requests
2017-08-09T16:38:10+0200 call error: ApplicationError(error=<wamp.close.transport_lost>, args=[u'WAMP transport was lost without closing the session before'], kwargs={}, enc_algo=None)
It seems crossbar is kicking me out because my client session looks dead to him, but I thought that autobahn would chunk my data and that the call would not block the client reactor.
I enabled a few things in my crossbar configuration to improve websocket treatment; thanks to that I was able to transmit larger amount of data but eventually I would hit a limit (config file largely copied and pasted from sam & max).
"options": {
"enable_webstatus": false,
"max_frame_size": 16777216,
"auto_fragment_size": 65536,
"fail_by_drop": true,
"open_handshake_timeout": 2500,
"close_handshake_timeout": 1000,
"auto_ping_interval": 10000,
"auto_ping_timeout": 5000,
"auto_ping_size": 4,
"compression": {
"deflate": {
"request_no_context_takeover": false,
"request_max_window_bits": 11,
"no_context_takeover": false,
"max_window_bits": 11,
"memory_level": 4
}
}
}
Any ideas, takes, things that I am doing wrong?
Thank you,
Client code:
from __future__ import print_function
import pandas as pd
from autobahn.twisted.wamp import ApplicationSession
from twisted.internet.defer import inlineCallbacks
class MyComponent(ApplicationSession):
#inlineCallbacks
def onJoin(self, details):
print("session ready")
try:
res = yield self.call(u'data.get')
print('Got the data')
data = pd.read_json(res)
print("call result: {}".format(data.head()))
print("call result shape: {0}, {1}".format(*data.shape))
except Exception as e:
print("call error: {0}".format(e))
if __name__ == "__main__":
from autobahn.twisted.wamp import ApplicationRunner
runner = ApplicationRunner(url=u"ws://127.0.0.1:8080/ws", realm=u"realm1")
runner.run(MyComponent)
Backend code
from __future__ import absolute_import, division, print_function
from twisted.internet.defer import inlineCallbacks
from autobahn.twisted.wamp import ApplicationSession
from twisted.internet import reactor, defer, threads
# Imports
import pandas as pd
def get_data():
"""Returns a DataFrame of stuff as a JSON
:return: str, data as a JSON string
"""
data = pd.DataFrame({
'col1': pd.np.arange(1000000),
'col2': "I'm big",
'col3': 'Like really big',
})
print("call result shape: {0}, {1}".format(*data.shape))
print(data.memory_usage().sum())
print(data.head())
return data.to_json()
class MyBackend(ApplicationSession):
def __init__(self, config):
ApplicationSession.__init__(self, config)
#inlineCallbacks
def onJoin(self, details):
# Register a procedure for remote calling
#inlineCallbacks
def async_daily_price(eqt_list):
res = yield threads.deferToThread(get_data)
defer.returnValue(res)
yield self.register(async_daily_price, u'data.get')
if __name__ == "__main__":
from autobahn.twisted.wamp import ApplicationRunner
runner = ApplicationRunner(url=u"ws://127.0.0.1:8080/ws", realm=u"realm1")
runner.run(MyBackend)
Configuration
{
"version": 2,
"controller": {},
"workers": [
{
"type": "router",
"realms": [
{
"name": "realm1",
"roles": [
{
"name": "anonymous",
"permissions": [
{
"uri": "",
"match": "prefix",
"allow": {
"call": true,
"register": true,
"publish": true,
"subscribe": true
},
"disclose": {
"caller": false,
"publisher": false
},
"cache": true
}
]
}
]
}
],
"transports": [
{
"type": "universal",
"endpoint": {
"type": "tcp",
"port": 8080
},
"rawsocket": {
},
"websocket": {
"ws": {
"type": "websocket",
"options": {
"enable_webstatus": false,
"max_frame_size": 16777216,
"auto_fragment_size": 65536,
"fail_by_drop": true,
"open_handshake_timeout": 2500,
"close_handshake_timeout": 1000,
"auto_ping_interval": 10000,
"auto_ping_timeout": 5000,
"auto_ping_size": 4,
"compression": {
"deflate": {
"request_no_context_takeover": false,
"request_max_window_bits": 11,
"no_context_takeover": false,
"max_window_bits": 11,
"memory_level": 4
}
}
}
}
},
"web": {
"paths": {
"/": {
"type": "static",
}
}
}
}
]
}
]
}
The solution suggested by the crossbar.io group was to use the progressive result option of the RPC.
A full working example is located at https://github.com/crossbario/autobahn-python/tree/master/examples/twisted/wamp/rpc/progress
In my code I had to add a the chunking of the result in the backend
step = 10000
if details.progress and len(res) > step:
for i in xrange(0, len(res), step):
details.progress(res[i:i+step])
else:
defer.returnValue(res)
And to the caller
res = yield self.call(
u'data.get'
options=CallOptions(
on_progress=partial(on_progress, res=res_list)
)
)
Where my function on_progress adds the chunks to a result list
def on_progress(x, res):
res.append(x)
Picking the right chunk size will do the trick.

Logging RotatingFileHandler not rotating

I have implemented a custom RotatingFileHandler:
class FreezeAwareFileHandler(RotatingFileHandler):
def emit(self, record):
try:
msg = self.format(record)
stream = self.stream
stream.write(msg)
stream.write('\n')
self.flush()
except (KeyboardInterrupt, SystemExit): #pragma: no cover
raise
except:
self.handleError(record)
And I have this configuration json file (I have tried even with yaml and specifying the configuration through the class methods):
{
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"standard": {
"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
"datefmt": "%Y-%m-%d %H:%M:%S"
}
},
"handlers": {
"freeze_aware_file_handler": {
"class": "Logging.MyHandler",
"formatter": "standard",
"level": "INFO",
"filename": "logs\\MyLog.log",
"maxBytes": 1024,
"backupCount": 10,
"encoding": "utf8"
}
},
"loggers": {
"my_module": {
"level": "INFO",
"handlers": ["my_handler"],
"propagate": "no"
}
},
"root": {
"level": "INFO",
"handlers": ["my_handler"]
}
}
And this is the code I use for initializing:
if os.path.exists(path):
with open(path, 'rt') as f:
config_json = json.load(f)
logging.config.dictConfig(config_json)
logger = logging.getLogger("my_handler")
I can normally log to the specified file but it is never rotating.
Someone know why I'm having this behavior?
I'm using Python 3.5
It turned out the problem was really simple.
For the rotating file handler it is needed to explicitly call the shouldRollver and doRollover methods in the emit of the base class.
def emit(self, record):
try:
if self.shouldRollover(record):
self.doRollover()
...
do any custom actions here
...
except:
self.handleError(record)

Resources