From 4cb6aa90b2cba0eba682af8a7319de11f5b47a4f Mon Sep 17 00:00:00 2001 From: gabi Date: Mon, 24 Nov 2014 01:29:09 +0200 Subject: [PATCH] New async_logger class and API --- example/bench.cpp | 28 ++++++-- example/example.cpp | 14 +++- include/spdlog/async_logger.h | 66 ++++++++++++++++++ include/spdlog/details/async_logger_impl.h | 78 ++++++++++++++++++++++ include/spdlog/details/log_msg.h | 14 +++- include/spdlog/details/logger_impl.h | 43 ++++++++---- include/spdlog/details/registry.h | 23 ++++++- include/spdlog/details/spdlog_impl.h | 11 +++ include/spdlog/logger.h | 24 ++++--- include/spdlog/sinks/async_sink.h | 20 +++--- include/spdlog/spdlog.h | 32 +++++++-- 11 files changed, 305 insertions(+), 48 deletions(-) create mode 100644 include/spdlog/async_logger.h create mode 100644 include/spdlog/details/async_logger_impl.h diff --git a/example/bench.cpp b/example/bench.cpp index feda61da..980493ce 100644 --- a/example/bench.cpp +++ b/example/bench.cpp @@ -31,8 +31,8 @@ #include #include #include "spdlog/spdlog.h" +#include "spdlog/async_logger.h" #include "spdlog/sinks/file_sinks.h" -#include "spdlog/sinks/async_sink.h" #include "spdlog/sinks/null_sink.h" #include "utils.h" @@ -50,8 +50,8 @@ void bench_mt(int howmany, std::shared_ptr log, int thread_count int main(int argc, char* argv[]) { - int howmany = 250000; - int threads = 4; + int howmany = 1000000; + int threads = 10; int flush_interval = 1000; int file_size = 30 * 1024 * 1024; int rotating_files = 5; @@ -70,11 +70,8 @@ int main(int argc, char* argv[]) auto rotating_st = spdlog::rotating_logger_st("rotating_st", "logs/rotating_st", file_size, rotating_files, flush_interval); bench(howmany, rotating_st); - - auto daily_st = spdlog::daily_logger_st("daily_st", "logs/daily_st", flush_interval); bench(howmany, daily_st); - bench(howmany, spdlog::create("null_st")); cout << "\n*******************************************************************************\n"; @@ -87,8 +84,25 @@ int main(int argc, char* argv[]) auto daily_mt = spdlog::daily_logger_mt("daily_mt", "logs/daily_mt", flush_interval); bench_mt(howmany, daily_mt, threads); + bench(howmany, spdlog::create("null_mt")); + + + cout << "\n*******************************************************************************\n"; + cout << "async logging.. " << threads << " threads sharing same logger, " << format(howmany) << " iterations, flush every " << flush_interval << " lines" << endl; + cout << "*******************************************************************************\n"; + + spdlog::set_async_mode(howmany); + auto rotating_st_async = spdlog::rotating_logger_st("rotating_st_Async", "logs/rotating_st_async", file_size, rotating_files, flush_interval); + bench(howmany, rotating_st_async); + auto daily_st_async = spdlog::daily_logger_st("daily_st_async", "logs/daily_st_async", flush_interval); + bench(howmany, daily_st_async); + bench(howmany, spdlog::create("null_st_async")); + + + spdlog::stop(); + cin.ignore(); + - bench_mt(howmany, spdlog::create("null_mt"), threads); } catch (std::exception &ex) { diff --git a/example/example.cpp b/example/example.cpp index 10a0a93e..5e8c37f4 100644 --- a/example/example.cpp +++ b/example/example.cpp @@ -29,7 +29,7 @@ #include "spdlog/spdlog.h" -int main(int, char* []) +int main_(int, char* []) { namespace spd = spdlog; @@ -63,7 +63,19 @@ int main(int, char* []) SPDLOG_TRACE(file_logger, "This is a trace message (only #ifdef _DEBUG)", 123); + // + // Asynchronous logging is easy.. + // Just call spdlog::set_async_mode(max_q_size) and all created loggers from now on will be asynchronous.. + // + size_t max_q_size = 100000; + spdlog::set_async_mode(max_q_size); + auto async_file= spd::daily_logger_st("async_file_logger", "async_" + filename); + async_file->info() << "This is async log.." << "Should be very fast!"; + + // + // syslog example + // #ifdef __linux__ auto syslog_logger = spd::syslog_logger("syslog"); syslog_logger->warn("This is warning that will end up in syslog. This is Linux only!"); diff --git a/include/spdlog/async_logger.h b/include/spdlog/async_logger.h new file mode 100644 index 00000000..ddc8e4b6 --- /dev/null +++ b/include/spdlog/async_logger.h @@ -0,0 +1,66 @@ +/*************************************************************************/ +/* spdlog - an extremely fast and easy to use c++11 logging library. */ +/* Copyright (c) 2014 Gabi Melman. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#pragma once + +// Async logger +// Upon each log write the logger: +// 1. Checks if its log level is enough to log the message +// 2. Push a new copy of the message to a queue (uses sinks::async_sink for this) + +#include +#include "common.h" +#include "logger.h" + + +namespace spdlog +{ + +namespace sinks { +class async_sink; +} + +class async_logger :public logger +{ +public: + template + async_logger(const std::string& name, const It& begin, const It& end, size_t queue_size, const log_clock::duration& shutdown_duration); + async_logger(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const log_clock::duration& shutdown_duration); + async_logger(const std::string& logger_name, sink_ptr single_sink, size_t queue_size, const log_clock::duration& shutdown_duration); + + +protected: + void _log_msg(details::log_msg& msg) override; + void _set_formatter(spdlog::formatter_ptr msg_formatter) override; + void _set_pattern(const std::string& pattern) override; + void _stop() override; + +private: + std::unique_ptr _as; + log_clock::duration _shutdown_duration; +}; +} + + +#include "./details/async_logger_impl.h" diff --git a/include/spdlog/details/async_logger_impl.h b/include/spdlog/details/async_logger_impl.h new file mode 100644 index 00000000..c4f032ea --- /dev/null +++ b/include/spdlog/details/async_logger_impl.h @@ -0,0 +1,78 @@ +/*************************************************************************/ +/* spdlog - an extremely fast and easy to use c++11 logging library. */ +/* Copyright (c) 2014 Gabi Melman. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#pragma once + + +#include +#include "../sinks/async_sink.h" + +// +// Async Logger implementation +// Use single async_sink (queue) to perform the logging in a worker thread +// + + +template +inline spdlog::async_logger::async_logger(const std::string& name, const It& begin, const It& end, size_t queue_size, const log_clock::duration& shutdown_duration) : + logger(name, begin, end), + _shutdown_duration(shutdown_duration), + _as(std::unique_ptr(new sinks::async_sink(queue_size))) +{ + _as->set_formatter(_formatter); + for (auto &s : _sinks) + _as->add_sink(s); +}; + +inline spdlog::async_logger::async_logger(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const log_clock::duration& shutdown_duration) : + async_logger(logger_name, sinks.begin(), sinks.end(), queue_size, shutdown_duration) {} + +inline spdlog::async_logger::async_logger(const std::string& logger_name, sink_ptr single_sink, size_t queue_size, const log_clock::duration& shutdown_duration) : + async_logger(logger_name, { single_sink }, queue_size, shutdown_duration) {} + + +inline void spdlog::async_logger::_log_msg(details::log_msg& msg) +{ + _as->log(msg); +} + +inline void spdlog::async_logger::_set_formatter(spdlog::formatter_ptr msg_formatter) +{ + _formatter = msg_formatter; + _as->set_formatter(_formatter); +} + +inline void spdlog::async_logger::_set_pattern(const std::string& pattern) +{ + _formatter = std::make_shared(pattern); + _as->set_formatter(_formatter); +} + + + +inline void spdlog::async_logger::_stop() +{ + set_level(level::OFF); + _as->shutdown(_shutdown_duration); +} \ No newline at end of file diff --git a/include/spdlog/details/log_msg.h b/include/spdlog/details/log_msg.h index 76ca9b89..19abcfda 100644 --- a/include/spdlog/details/log_msg.h +++ b/include/spdlog/details/log_msg.h @@ -58,7 +58,19 @@ struct log_msg raw(std::move(other.raw)), formatted(std::move(other.formatted)) {} - log_msg& operator=(log_msg&& other) = delete; + log_msg& operator=(log_msg&& other) + { + if (this == &other) + return *this; + + logger_name = std::move(other.logger_name); + level = other.level; + time = std::move(other.time); + tm_time = other.tm_time; + raw = std::move(other.raw); + formatted = std::move(other.formatted); + return *this; + } diff --git a/include/spdlog/details/logger_impl.h b/include/spdlog/details/logger_impl.h index 25fb121b..ccf1886e 100644 --- a/include/spdlog/details/logger_impl.h +++ b/include/spdlog/details/logger_impl.h @@ -30,7 +30,7 @@ #include "./line_logger.h" - +/* public functions */ template inline spdlog::logger::logger(const std::string& logger_name, const It& begin, const It& end) : _name(logger_name), @@ -51,20 +51,18 @@ inline spdlog::logger::logger(const std::string& logger_name, spdlog::sink_ptr s {} +inline spdlog::logger::~logger() {} + inline void spdlog::logger::set_formatter(spdlog::formatter_ptr msg_formatter) { - _formatter = msg_formatter; + _set_formatter(msg_formatter); } inline void spdlog::logger::set_pattern(const std::string& pattern) { - _formatter = std::make_shared(pattern); + _set_pattern(pattern); } -inline spdlog::formatter_ptr spdlog::logger::get_formatter() const -{ - return _formatter; -} template @@ -158,11 +156,33 @@ inline bool spdlog::logger::should_log(spdlog::level::level_enum msg_level) cons } inline void spdlog::logger::stop() +{ + _stop(); +} + +/* protected virtual */ +inline void spdlog::logger::_log_msg(details::log_msg& msg) +{ + _formatter->format(msg); + for (auto &sink : _sinks) + sink->log(msg); +} + +inline void spdlog::logger::_set_pattern(const std::string& pattern) +{ + _formatter = std::make_shared(pattern); +} +inline void spdlog::logger::_set_formatter(formatter_ptr msg_formatter) +{ + _formatter = msg_formatter; +} + +inline void spdlog::logger::_stop() { set_level(level::OFF); } - +/* private functions */ inline void spdlog::logger::_variadic_log(spdlog::details::line_logger&) {} template @@ -181,10 +201,5 @@ inline void spdlog::logger::_variadic_log(spdlog::details::line_logger& l, const _variadic_log(l, rest...); } -inline void spdlog::logger::_log_msg(details::log_msg& msg) -{ - _formatter->format(msg); - for (auto &sink : _sinks) - sink->log(msg); -} + diff --git a/include/spdlog/details/registry.h b/include/spdlog/details/registry.h index d2988d5a..9cf14352 100644 --- a/include/spdlog/details/registry.h +++ b/include/spdlog/details/registry.h @@ -33,6 +33,7 @@ #include #include "../logger.h" +#include "../async_logger.h" #include "../common.h" namespace spdlog @@ -58,8 +59,12 @@ public: auto found = _loggers.find(logger_name); if (found != _loggers.end()) return found->second; + std::shared_ptr new_logger; + if (_async_mode) + new_logger = std::make_shared(logger_name, sinks_begin, sinks_end, _async_q_size, _async_shutdown_duration); + else + new_logger = std::make_shared(logger_name, sinks_begin, sinks_end); - auto new_logger = std::make_shared(logger_name, sinks_begin, sinks_end); if (_formatter) new_logger->set_formatter(_formatter); new_logger->set_level(_level); @@ -102,7 +107,20 @@ public: std::lock_guard lock(_mutex); for (auto& l : _loggers) l.second->set_level(log_level); + } + void set_async_mode(size_t q_size, const log_clock::duration& shutdown_duration) + { + std::lock_guard lock(_mutex); + _async_mode = true; + _async_q_size = q_size; + _async_shutdown_duration = shutdown_duration; + } + + void set_sync_mode() + { + std::lock_guard lock(_mutex); + _async_mode = false; } void stop_all() @@ -128,6 +146,9 @@ private: std::unordered_map > _loggers; formatter_ptr _formatter; level::level_enum _level = level::INFO; + bool _async_mode = false; + size_t _async_q_size = 0; + log_clock::duration _async_shutdown_duration; }; } } diff --git a/include/spdlog/details/spdlog_impl.h b/include/spdlog/details/spdlog_impl.h index 127722de..a0f4917d 100644 --- a/include/spdlog/details/spdlog_impl.h +++ b/include/spdlog/details/spdlog_impl.h @@ -128,6 +128,17 @@ inline void spdlog::set_level(level::level_enum log_level) return details::registry::instance().set_level(log_level); } + +inline void spdlog::set_async_mode(size_t queue_size, const log_clock::duration& shutdown_duration) +{ + details::registry::instance().set_async_mode(queue_size, shutdown_duration); +} + +inline void spdlog::set_sync_mode() +{ + details::registry::instance().set_sync_mode(); +} + inline void spdlog::stop() { return details::registry::instance().stop_all(); diff --git a/include/spdlog/logger.h b/include/spdlog/logger.h index 3052c39b..d52f1665 100644 --- a/include/spdlog/logger.h +++ b/include/spdlog/logger.h @@ -34,6 +34,7 @@ #include #include #include "sinks/base_sink.h" +#include "sinks/async_sink.h" #include "common.h" namespace spdlog @@ -47,17 +48,12 @@ class line_logger; class logger { public: - logger(const std::string& logger_name, sink_ptr single_sink); logger(const std::string& name, sinks_init_list); template logger(const std::string& name, const It& begin, const It& end); - void set_pattern(const std::string&); - void set_formatter(formatter_ptr); - formatter_ptr get_formatter() const; - - + virtual ~logger(); logger(const logger&) = delete; logger& operator=(const logger&) = delete; @@ -83,18 +79,30 @@ public: template details::line_logger emerg(const Args&... args); -private: + void set_pattern(const std::string&); + void set_formatter(formatter_ptr); + + +protected: + virtual void _log_msg(details::log_msg& msg); + virtual void _set_pattern(const std::string&); + virtual void _set_formatter(formatter_ptr); + virtual void _stop(); + + friend details::line_logger; std::string _name; std::vector _sinks; formatter_ptr _formatter; std::atomic_int _level; + +private: void _variadic_log(details::line_logger& l); template inline void _variadic_log(spdlog::details::line_logger& l, const Last& last); template void _variadic_log(details::line_logger&l, const First& first, const Rest&... rest); - void _log_msg(details::log_msg& msg); + }; } diff --git a/include/spdlog/sinks/async_sink.h b/include/spdlog/sinks/async_sink.h index 767c6529..54ff89b3 100644 --- a/include/spdlog/sinks/async_sink.h +++ b/include/spdlog/sinks/async_sink.h @@ -35,7 +35,6 @@ #include #include #include -#include #include "./base_sink.h" #include "../logger.h" @@ -43,14 +42,14 @@ #include "../details/null_mutex.h" #include "../details/log_msg.h" -#include namespace spdlog { namespace sinks { -class async_sink : public base_sink < details::null_mutex > + +class async_sink : public base_sink < details::null_mutex > //single worker thread so null_mutex { public: using q_type = details::blocking_queue < std::unique_ptr > ; @@ -61,9 +60,9 @@ public: ~async_sink(); void add_sink(sink_ptr sink); void remove_sink(sink_ptr sink_ptr); - q_type& q(); + void set_formatter(formatter_ptr); //Wait to remaining items (if any) in the queue to be written and shutdown - void shutdown(const std::chrono::milliseconds& timeout); + void shutdown(const log_clock::duration& timeout); @@ -80,6 +79,7 @@ private: //Last exception thrown from the back thread std::shared_ptr _last_backthread_ex; + formatter_ptr _formatter; //will throw last back thread exception or if backthread no active void _push_sentry(); @@ -124,13 +124,13 @@ inline void spdlog::sinks::async_sink::_thread_loop() { if (!_active) return; + _formatter->format(*msg); for (auto &s : _sinks) { try { s->log(*msg); } - catch (const std::exception& ex) { _last_backthread_ex = std::make_shared(ex.what()); @@ -139,7 +139,6 @@ inline void spdlog::sinks::async_sink::_thread_loop() { _last_backthread_ex = std::make_shared("Unknown exception"); } - } } } @@ -159,13 +158,14 @@ inline void spdlog::sinks::async_sink::remove_sink(spdlog::sink_ptr s) } -inline spdlog::sinks::async_sink::q_type& spdlog::sinks::async_sink::q() +inline void spdlog::sinks::async_sink::set_formatter(formatter_ptr msg_formatter) { - return _q; + _formatter = msg_formatter; } -inline void spdlog::sinks::async_sink::shutdown(const std::chrono::milliseconds& timeout) + +inline void spdlog::sinks::async_sink::shutdown(const log_clock::duration& timeout) { if (timeout > std::chrono::milliseconds::zero()) { diff --git a/include/spdlog/spdlog.h b/include/spdlog/spdlog.h index ddad0fda..35fd4323 100644 --- a/include/spdlog/spdlog.h +++ b/include/spdlog/spdlog.h @@ -43,37 +43,59 @@ namespace spdlog std::shared_ptr get(const std::string& name); +// // Set global formatting -// spdlog::set_pattern("%Y-%m-%d %H:%M:%S.%e %l : %v"); +// e.g. spdlog::set_pattern("%Y-%m-%d %H:%M:%S.%e %l : %v"); +// void set_pattern(const std::string& format_string); +void set_formatter(formatter_ptr f); - -//Set global logging level +// +// Set global logging level +// void set_level(level::level_enum log_level); +// +// Async mode - off by default. +// + +// Turn on async mode and set the queue size for each async_logger +// shutdown_duration indicates max time to wait for the worker thread to log its messages before terminating. + +void set_async_mode(size_t queue_size, const log_clock::duration& shutdown_duration = std::chrono::seconds(5)); +// Turn off async mode +void set_sync_mode(); + +// // Create multi/single threaded rotating file logger +// std::shared_ptr rotating_logger_mt(const std::string& logger_name, const std::string& filename, size_t max_file_size, size_t max_files, size_t flush_inverval = 0); std::shared_ptr rotating_logger_st(const std::string& logger_name, const std::string& filename, size_t max_file_size, size_t max_files, size_t flush_inverval = 0); +// // Create file logger which creates new file at midnight): +// std::shared_ptr daily_logger_mt(const std::string& logger_name, const std::string& filename, size_t flush_inverval = 0); std::shared_ptr daily_logger_st(const std::string& logger_name, const std::string& filename, size_t flush_inverval = 0); +// // Create stdout/stderr loggers +// std::shared_ptr stdout_logger_mt(const std::string& logger_name); std::shared_ptr stdout_logger_st(const std::string& logger_name); std::shared_ptr stderr_logger_mt(const std::string& logger_name); std::shared_ptr stderr_logger_st(const std::string& logger_name); +// // Create a syslog logger +// #ifdef __linux__ std::shared_ptr syslog_logger(const std::string& logger_name); #endif - // // Create a logger with multiple sinks // @@ -90,8 +112,6 @@ std::shared_ptr create(const std::string& logger_name, const Arg -// Set global formatter object -void set_formatter(formatter_ptr f); // Stop logging by setting all the loggers to log level OFF void stop();