Usually in the process of log writing, every time the log is written, it is written directly to the file. When the amount of data is not large, it can be done in this way without any problem. However, when the amount of data is large, each write of data will cause a large performance problem due to IO bottleneck.

In order to solve this problem, this paper designs a set of buffer mechanism based on multithreading, which can add logs to memory and then write files.

Basic principles

1. Memory log list
   First, create a buffer list to save the log. When the log is generated, it is written to the buffer first, so that the data can be written at a very high speed using the high-speed memory.
2. Multithreaded write
   Multithreading is used to collect and write log data to files. This mechanism uses an independent thread to add log information to the specified file according to whether there is new data in the log list at a certain time.

source code

Use example

static void Main(string[] args)
{
    LogManager.Log(LogType.L101001_UserLogin, "10232");
    LogManager.Log(LogType.L101002_UserLogout, "10232", DateTime.Now);
    LogManager.Log(LogType.L415001_Debug, "MyLog.Program", true, 135.2);

    // Simulates this program running for 5 seconds.
    Thread.Sleep(5000);
}

Results written

1136515010100110232
11365150101002102322019-05-12 12:59:10
11365150415001MyLog.ProgramTrue135.2

source code

LogManager class

    /// <summary>
    /// This class is used to manage logs from the same program.
    /// It has a key method AddLog to receive log. This is a 
    /// multi-threading class to save logs into a file in specified 
    /// seconds to avoid save logs too frequently. 
    /// </summary>
    public class LogManager
    {
        /// <summary>
        /// The file path to store log.
        /// </summary>
        public string LogPath { get; private set; } = "system.log";

        /// <summary>
        /// The interval (in second) of saving logs into file.
        /// </summary>
        public int SaveInterval { get; set; } = 1;

        // To contronl the running status of the thread
        bool isRunning = false;

        // Teporary log list for holding current logs.
        List<Log> logs = new List<Log>();

        // The thread to perform method process() 
        Thread thread;

        /// <summary>
        /// Construct a LogManager object.
        /// </summary>
        public LogManager()
        {

        }

        /// <summary>
        /// Construct a LogManager object.
        /// </summary>
        /// <param name="logPath">The file path to store logs.</param>
        /// <param name="saveInterval">Writing intervl.</param>
        public LogManager(string logPath, int saveInterval = 10)
        {
            LogPath = logPath;
            SaveInterval = saveInterval;
        }

        /// <summary>
        /// Add a new log.
        /// </summary>
        /// <param name="logType">The type of this log.</param>
        /// <param name="args">The arguments of this log.</param>
        public void AddLog(LogType logType, object[] args)
        {
            AddLog(new Log(logType, args));
        }

        /// <summary>
        /// Add a new log.
        /// </summary>
        /// <param name="log">The log to be added.</param>
        public void AddLog(Log log)
        {
            lock (this)
            {
                logs.Add(log);
            }
        }

        /// <summary>
        /// main method to process logs.
        /// </summary>
        private void process()
        {
            isRunning = true;
            while (isRunning)
            {
                if (logs.Count > 0)
                {
                    // Obtain all logs and release lock
                    List<Log> loglist = new List<Log>();
                    lock (this)
                    {
                        loglist.AddRange(logs);
                        logs.Clear();
                    }

                    // append new logs into log file.
                    using (StreamWriter fw = new StreamWriter(LogPath, true, Encoding.UTF8))
                    {
                        foreach (Log log in loglist)
                            fw.WriteLine(log);
                    }

                    System.Console.WriteLine(DateTime.Now + " Saved.");
                }

                // Sleep SaveInterval seconds for no waisting CPU resource.
                Thread.Sleep(SaveInterval);
            }
        }

        /// <summary>
        /// Starts thread of this log manager. 
        /// </summary>
        public void Start()
        {
            if (thread != null)
                return;

            thread = new Thread(process);
            thread.IsBackground = true;
            thread.Start();
        }

        /// <summary>
        /// Stops the thread of this log manager. 
        /// </summary>
        public void Stop()
        {
            isRunning = false;
        }

        // used in static
        static LogManager logManager;
        // a static method to save logs.
        public static void Log(LogType logType, params object[] args)
        {
            if (logManager == null)
            {
                logManager = new LogManager();
                logManager.Start();
            }

            logManager.AddLog(logType, args);
        }
    }

Class Log

    public class Log
    {
        /// <summary>
        /// The occuring time.
        /// </summary>
        public DateTime Time { get; set; } = DateTime.Now;

        /// <summary>
        /// The type of the log.
        /// </summary>
        public LogType LogType { get; set; }

        /// <summary>
        /// Arguements of a log.
        /// </summary>
        public string[] Arguments { get; set; }


        // Used to separate data fields of a log when serializing.
        char separator = (char)6;

        /// <summary>
        /// Construct a Log object.
        /// </summary>
        /// <param name="logType">The type of log.</param>
        /// <param name="args">Argument list.</param>
        public Log(LogType logType, object[] args)
        {
            LogType = logType;
            Arguments = new string[args == null ? 0 : args.Length];
            for (int i = 0; i < Arguments.Length; i++)
                Arguments[i] = args[i] == null ? "" : args[i].ToString();
        }

        /// <summary>
        /// Format: yyyyMMddHHmmss|LogType|Arguments
        /// </summary>
        /// <returns></returns>
        public override string ToString()
        {
            StringBuilder sb = new StringBuilder();
            sb.Append((int)(Time - new DateTime(2019,1,1)).TotalSeconds);
            sb.Append(separator);
            sb.Append((int)LogType);
            sb.Append(separator);
            sb.Append(string.Join(separator.ToString(), Arguments));
            return sb.ToString();
        }

        /// <summary>
        /// Parses a string into a Log object.
        /// </summary>
        /// <param name="str">Input string.</param>
        /// <returns></returns>
        public Log Parse(string str)
        {

            return null;
        }
    }

Class LogType

    /// <summary>
    /// Used to represent different types of logs based on Integers.
    /// </summary>
    public enum LogType : int
    {
        L101001_UserLogin = 101001,
        L101002_UserLogout = 101002,
        L415001_Debug = 415001,
    }