Source code for ruffus.proxy_logger

#!/usr/bin/env python
################################################################################
#
#   proxy_logger.py
#
#
#   Copyright (c) 10/9/2009 Leo Goodstadt
#
#   Permission is hereby granted, free of charge, to any person obtaining a copy
#   of this software and associated documentation files (the "Software"), to deal
#   in the Software without restriction, including without limitation the rights
#   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the Software is
#   furnished to do so, subject to the following conditions:
#
#   The above copyright notice and this permission notice shall be included in
#   all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#   THE SOFTWARE.
#################################################################################
"""
****************************************************************************
Create proxy for logging for use with multiprocessing
****************************************************************************

These can be safely sent (marshalled) across process boundaries


===========
Example 1
===========

    Set up logger from config file::

        from proxy_logger import *
        args={}
        args["config_file"] = "/my/config/file"

        (logger_proxy,
         logging_mutex) = make_shared_logger_and_proxy (setup_std_shared_logger,
                                                        "my_logger", args)


===========
Example 2
===========

    Log to file ``"/my/lg.log"`` in the specified format (Time / Log name / Event type / Message).

    Delay file creation until first log.

    Only log ``Debug`` messages

        Other alternatives for the logging threshold (``args["level"]``) include

            * ``logging.DEBUG``
            * ``logging.INFO``
            * ``logging.WARNING``
            * ``logging.ERROR``
            * ``logging.CRITICAL``

    ::

        from proxy_logger import *
        args={}
        args["file_name"] = "/my/lg.log"
        args["formatter"] = "%(asctime)s - %(name)s - %(levelname)6s - %(message)s"
        args["delay"]     = True
        args["level"]     = logging.DEBUG

        (logger_proxy,
         logging_mutex) = make_shared_logger_and_proxy (setup_std_shared_logger,
                                                        "my_logger", args)

===========
Example 3
===========

    Rotate log files every 20 Kb, with up to 10 backups.
    ::

        from proxy_logger import *
        args={}
        args["file_name"] = "/my/lg.log"
        args["rotating"] = True
        args["maxBytes"]=20000
        args["backupCount"]=10
        (logger_proxy,
         logging_mutex) = make_shared_logger_and_proxy (setup_std_shared_logger,
                                                        "my_logger", args)



==============
To use:
==============

    ::

        (logger_proxy,
         logging_mutex) = make_shared_logger_and_proxy (setup_std_shared_logger,
                                                        "my_logger", args)

        with logging_mutex:
            my_log.debug('This is a debug message')
            my_log.info('This is an info message')
            my_log.warning('This is a warning message')
            my_log.error('This is an error message')
            my_log.critical('This is a critical error message')
            my_log.log(logging.DEBUG, 'This is a debug message')

    Note that the logging function ``exception()`` is not included because python
    stack trace information is not well-marshalled
    (`pickle <http://docs.python.org/library/pickle.html>`_\ d) across processes.

"""





#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

#   imports


#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

import sys,os


#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

#   Shared logging


#88888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

import multiprocessing
import multiprocessing.managers


import logging
import logging.handlers



#
#   setup_logger
#
[docs]def setup_std_shared_logger(logger_name, args): """ This function is a simple around wrapper around the python `logging <http://docs.python.org/library/logging.html>`_ module. This *logger_factory* example creates logging objects which can then be managed by proxy via ``ruffus.proxy_logger.make_shared_logger_and_proxy()`` This can be: * a `disk log file <http://docs.python.org/library/logging.html#filehandler>`_ * a automatically backed-up `(rotating) log <http://docs.python.org/library/logging.html#rotatingfilehandler>`_. * any log specified in a `configuration file <http://docs.python.org/library/logging.html#configuration-file-format>`_ These are specified in the ``args`` dictionary forwarded by ``make_shared_logger_and_proxy()`` :param logger_name: name of log :param args: a dictionary of parameters forwarded from ``make_shared_logger_and_proxy()`` Valid entries include: .. describe:: "level" Sets the `threshold <http://docs.python.org/library/logging.html#logging.Handler.setLevel>`_ for the logger. .. describe:: "config_file" The logging object is configured from this `configuration file <http://docs.python.org/library/logging.html#configuration-file-format>`_. .. describe:: "file_name" Sets disk log file name. .. describe:: "rotating" Chooses a `(rotating) log <http://docs.python.org/library/logging.html#rotatingfilehandler>`_. .. describe:: "maxBytes" Allows the file to rollover at a predetermined size .. describe:: "backupCount" If backupCount is non-zero, the system will save old log files by appending the extensions ``.1``, ``.2``, ``.3`` etc., to the filename. .. describe:: "delay" Defer file creation until the log is written to. .. describe:: "formatter" `Converts <http://docs.python.org/library/logging.html#formatter-objects>`_ the message to a logged entry string. For example, :: "%(asctime)s - %(name)s - %(levelname)6s - %(message)s" """ # # Log file name with logger level # new_logger = logging.getLogger(logger_name) if "level" in args: new_logger.setLevel(args["level"]) if "config_file" in args: logging.config.fileConfig(args["config_file"]) else: if "file_name" not in args: raise Exception("Missing file name for log. Remember to set 'file_name'") log_file_name = args["file_name"] if "rotating" in args: rotating_args = {} # override default rotating_args["maxBytes"]=args.get("maxBytes", 100000) rotating_args["backupCount"]=args.get("backupCount", 5) handler = logging.handlers.RotatingFileHandler( log_file_name, **rotating_args) else: defer_loggin = "delay" in args handler = logging.handlers.RotatingFileHandler( log_file_name, delay=defer_loggin) # %(name)s # %(levelno)s # %(levelname)s # %(pathname)s # %(filename)s # %(module)s # %(funcName)s # %(lineno)d # %(created)f # %(relativeCreated)d # %(asctime)s # %(msecs)d # %(thread)d # %(threadName)s # %(process)d # %(message)s # # E.g.: "%(asctime)s - %(name)s - %(levelname)6s - %(message)s" # if "formatter" in args: my_formatter = logging.Formatter(args["formatter"]) handler.setFormatter(my_formatter) new_logger.addHandler(handler) # # This log object will be wrapped in proxy # return new_logger # # Proxy object for logging # Logging messages will be marshalled (forwarded) to the process where the # shared log lives #
class LoggerProxy(multiprocessing.managers.BaseProxy): def debug(self, *args, **kwargs): return self._callmethod('debug', args, kwargs) def log(self, *args, **kwargs): return self._callmethod('log', args, kwargs) def info(self, *args, **kwargs): return self._callmethod('info', args, kwargs) def warning(self, *args, **kwargs): return self._callmethod('warning', args, kwargs) def error(self, *args, **kwargs): return self._callmethod('error', args, kwargs) def critical(self, *args, **kwargs): return self._callmethod('critical', args, kwargs) def log(self, *args, **kwargs): return self._callmethod('log', args, kwargs) def __str__ (self): return "<LoggingProxy>" def __repr__ (self): return 'LoggerProxy()' # # Register the setup_logger function as a proxy for setup_logger # # We use SyncManager as a base class so we can get a lock proxy for synchronising # logging later on # class LoggingManager(multiprocessing.managers.SyncManager): """ Logging manager sets up its own process and will create the real Log object there We refer to this (real) log via proxies """ pass
[docs]def make_shared_logger_and_proxy (logger_factory, logger_name, args): """ Make a `logging <http://docs.python.org/library/logging.html>`_ object called "\ ``logger_name``\ " by calling ``logger_factory``\ (``args``\ ) This function will return a proxy to the shared logger which can be copied to jobs in other processes, as well as a mutex which can be used to prevent simultaneous logging from happening. :param logger_factory: functions which creates and returns an object with the `logging <http://docs.python.org/library/logging.html>`_ interface. ``setup_std_shared_logger()`` is one example of a logger factory. :param logger_name: name of log :param args: parameters passed (as a single argument) to ``logger_factory`` :returns: a proxy to the shared logger which can be copied to jobs in other processes :returns: a mutex which can be used to prevent simultaneous logging from happening """ # # make shared log and proxy # manager = LoggingManager() manager.register( 'setup_logger', logger_factory, proxytype=LoggerProxy, exposed = ( 'critical', 'log', 'info', 'debug', 'warning', 'error')) manager.start() logger_proxy = manager.setup_logger(logger_name, args) # # make sure we are not logging at the same time in different processes # logging_mutex = manager.Lock() return logger_proxy, logging_mutex