Loading ifs/CMakeLists.txt +6 −0 Original line number Diff line number Diff line Loading @@ -139,6 +139,12 @@ set_target_properties(RocksDB INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} ) add_library(spdlog INTERFACE) # we cannot use target_include_directories with CMake < 3.11 set_target_properties(spdlog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/extern" ) set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") Loading ifs/include/daemon/adafs_daemon.hpp +1 −3 Original line number Diff line number Diff line Loading @@ -4,13 +4,11 @@ // std libs #include <string> #include <spdlog/spdlog.h> // adafs config #include <global/configure.hpp> #include <global/global_defs.hpp> // third party libs #include <extern/spdlog/spdlog.h> #include <extern/spdlog/fmt/fmt.h> // margo extern "C" { #include <abt.h> Loading ifs/include/daemon/classes/fs_data.hpp +1 −0 Original line number Diff line number Diff line Loading @@ -10,6 +10,7 @@ class ChunkStorage; class Distributor; #include <unordered_map> #include <map> #include <functional> //std::hash class FsData { Loading ifs/include/extern/spdlog/async.h 0 → 100644 +87 −0 Original line number Diff line number Diff line // // Copyright(c) 2018 Gabi Melman. // Distributed under the MIT License (http://opensource.org/licenses/MIT) // #pragma once // // Async logging using global thread pool // All loggers created here share same global thread pool. // Each log message is pushed to a queue along withe a shared pointer to the // logger. // If a logger deleted while having pending messages in the queue, it's actual // destruction will defer // until all its messages are processed by the thread pool. // This is because each message in the queue holds a shared_ptr to the // originating logger. #include "spdlog/async_logger.h" #include "spdlog/details/registry.h" #include "spdlog/details/thread_pool.h" #include <memory> #include <mutex> namespace spdlog { namespace details { static const size_t default_async_q_size = 8192; } // async logger factory - creates async loggers backed with thread pool. // if a global thread pool doesn't already exist, create it with default queue // size of 8192 items and single thread. template<async_overflow_policy OverflowPolicy = async_overflow_policy::block> struct async_factory_impl { template<typename Sink, typename... SinkArgs> static std::shared_ptr<async_logger> create(const std::string &logger_name, SinkArgs &&... args) { auto ®istry_inst = details::registry::instance(); // create global thread pool if not already exists.. std::lock_guard<std::recursive_mutex> tp_lock(registry_inst.tp_mutex()); auto tp = registry_inst.get_tp(); if (tp == nullptr) { tp = std::make_shared<details::thread_pool>(details::default_async_q_size, 1); registry_inst.set_tp(tp); } auto sink = std::make_shared<Sink>(std::forward<SinkArgs>(args)...); auto new_logger = std::make_shared<async_logger>(logger_name, std::move(sink), std::move(tp), OverflowPolicy); registry_inst.register_and_init(new_logger); return new_logger; } }; using async_factory = async_factory_impl<async_overflow_policy::block>; using async_factory_nonblock = async_factory_impl<async_overflow_policy::overrun_oldest>; template<typename Sink, typename... SinkArgs> inline std::shared_ptr<spdlog::logger> create_async(const std::string &logger_name, SinkArgs &&... sink_args) { return async_factory::create<Sink>(logger_name, std::forward<SinkArgs>(sink_args)...); } template<typename Sink, typename... SinkArgs> inline std::shared_ptr<spdlog::logger> create_async_nb(const std::string &logger_name, SinkArgs &&... sink_args) { return async_factory_nonblock::create<Sink>(logger_name, std::forward<SinkArgs>(sink_args)...); } // set global thread pool. inline void init_thread_pool(size_t q_size, size_t thread_count) { auto tp = std::make_shared<details::thread_pool>(q_size, thread_count); details::registry::instance().set_tp(std::move(tp)); } // get the global thread pool. inline std::shared_ptr<spdlog::details::thread_pool> thread_pool() { return details::registry::instance().get_tp(); } } // namespace spdlog ifs/include/extern/spdlog/async_logger.h +40 −46 Original line number Diff line number Diff line Loading @@ -5,73 +5,67 @@ #pragma once // Very fast asynchronous logger (millions of logs per second on an average desktop) // Uses pre allocated lockfree queue for maximum throughput even under large number of threads. // Very fast asynchronous logger (millions of logs per second on an average // desktop) // Uses pre allocated lockfree queue for maximum throughput even under large // number of threads. // Creates a single back thread to pop messages from the queue and log them. // // Upon each log write the logger: // 1. Checks if its log level is enough to log the message // 2. Push a new copy of the message to a queue (or block the caller until space is available in the queue) // 2. Push a new copy of the message to a queue (or block the caller until // space is available in the queue) // 3. will throw spdlog_ex upon log exceptions // Upon destruction, logs all remaining messages in the queue before destructing.. // Upon destruction, logs all remaining messages in the queue before // destructing.. #include <extern/spdlog/common.h> #include <extern/spdlog/logger.h> #include "spdlog/common.h" #include "spdlog/logger.h" #include <chrono> #include <functional> #include <string> #include <memory> #include <string> namespace spdlog { namespace spdlog { namespace details // Async overflow policy - block by default. enum class async_overflow_policy { class async_log_helper; block, // Block until message can be enqueued overrun_oldest // Discard oldest message in the queue if full when trying to // add new item. }; namespace details { class thread_pool; } class async_logger :public logger class async_logger SPDLOG_FINAL : public std::enable_shared_from_this<async_logger>, public logger { friend class details::thread_pool; public: template<class It> async_logger(const std::string& name, const It& begin, const It& end, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); template<typename It> async_logger(std::string logger_name, const It &begin, const It &end, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); async_logger(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); async_logger(std::string logger_name, sinks_init_list sinks_list, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); async_logger(const std::string& logger_name, sink_ptr single_sink, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); async_logger(std::string logger_name, sink_ptr single_sink, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); //Wait for the queue to be empty, and flush synchronously //Warning: this can potentialy last forever as we wait it to complete void flush() override; protected: void _sink_it(details::log_msg& msg) override; void _set_formatter(spdlog::formatter_ptr msg_formatter) override; void _set_pattern(const std::string& pattern) override; void sink_it_(details::log_msg &msg) override; void flush_() override; void backend_log_(details::log_msg &incoming_log_msg); void backend_flush_(); private: std::unique_ptr<details::async_log_helper> _async_log_helper; std::weak_ptr<details::thread_pool> thread_pool_; async_overflow_policy overflow_policy_; }; } } // namespace spdlog #include <extern/spdlog/details/async_logger_impl.h> #include "details/async_logger_impl.h" Loading
ifs/CMakeLists.txt +6 −0 Original line number Diff line number Diff line Loading @@ -139,6 +139,12 @@ set_target_properties(RocksDB INTERFACE_INCLUDE_DIRECTORIES ${ROCKSDB_INCLUDE_DIRS} ) add_library(spdlog INTERFACE) # we cannot use target_include_directories with CMake < 3.11 set_target_properties(spdlog PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/extern" ) set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") Loading
ifs/include/daemon/adafs_daemon.hpp +1 −3 Original line number Diff line number Diff line Loading @@ -4,13 +4,11 @@ // std libs #include <string> #include <spdlog/spdlog.h> // adafs config #include <global/configure.hpp> #include <global/global_defs.hpp> // third party libs #include <extern/spdlog/spdlog.h> #include <extern/spdlog/fmt/fmt.h> // margo extern "C" { #include <abt.h> Loading
ifs/include/daemon/classes/fs_data.hpp +1 −0 Original line number Diff line number Diff line Loading @@ -10,6 +10,7 @@ class ChunkStorage; class Distributor; #include <unordered_map> #include <map> #include <functional> //std::hash class FsData { Loading
ifs/include/extern/spdlog/async.h 0 → 100644 +87 −0 Original line number Diff line number Diff line // // Copyright(c) 2018 Gabi Melman. // Distributed under the MIT License (http://opensource.org/licenses/MIT) // #pragma once // // Async logging using global thread pool // All loggers created here share same global thread pool. // Each log message is pushed to a queue along withe a shared pointer to the // logger. // If a logger deleted while having pending messages in the queue, it's actual // destruction will defer // until all its messages are processed by the thread pool. // This is because each message in the queue holds a shared_ptr to the // originating logger. #include "spdlog/async_logger.h" #include "spdlog/details/registry.h" #include "spdlog/details/thread_pool.h" #include <memory> #include <mutex> namespace spdlog { namespace details { static const size_t default_async_q_size = 8192; } // async logger factory - creates async loggers backed with thread pool. // if a global thread pool doesn't already exist, create it with default queue // size of 8192 items and single thread. template<async_overflow_policy OverflowPolicy = async_overflow_policy::block> struct async_factory_impl { template<typename Sink, typename... SinkArgs> static std::shared_ptr<async_logger> create(const std::string &logger_name, SinkArgs &&... args) { auto ®istry_inst = details::registry::instance(); // create global thread pool if not already exists.. std::lock_guard<std::recursive_mutex> tp_lock(registry_inst.tp_mutex()); auto tp = registry_inst.get_tp(); if (tp == nullptr) { tp = std::make_shared<details::thread_pool>(details::default_async_q_size, 1); registry_inst.set_tp(tp); } auto sink = std::make_shared<Sink>(std::forward<SinkArgs>(args)...); auto new_logger = std::make_shared<async_logger>(logger_name, std::move(sink), std::move(tp), OverflowPolicy); registry_inst.register_and_init(new_logger); return new_logger; } }; using async_factory = async_factory_impl<async_overflow_policy::block>; using async_factory_nonblock = async_factory_impl<async_overflow_policy::overrun_oldest>; template<typename Sink, typename... SinkArgs> inline std::shared_ptr<spdlog::logger> create_async(const std::string &logger_name, SinkArgs &&... sink_args) { return async_factory::create<Sink>(logger_name, std::forward<SinkArgs>(sink_args)...); } template<typename Sink, typename... SinkArgs> inline std::shared_ptr<spdlog::logger> create_async_nb(const std::string &logger_name, SinkArgs &&... sink_args) { return async_factory_nonblock::create<Sink>(logger_name, std::forward<SinkArgs>(sink_args)...); } // set global thread pool. inline void init_thread_pool(size_t q_size, size_t thread_count) { auto tp = std::make_shared<details::thread_pool>(q_size, thread_count); details::registry::instance().set_tp(std::move(tp)); } // get the global thread pool. inline std::shared_ptr<spdlog::details::thread_pool> thread_pool() { return details::registry::instance().get_tp(); } } // namespace spdlog
ifs/include/extern/spdlog/async_logger.h +40 −46 Original line number Diff line number Diff line Loading @@ -5,73 +5,67 @@ #pragma once // Very fast asynchronous logger (millions of logs per second on an average desktop) // Uses pre allocated lockfree queue for maximum throughput even under large number of threads. // Very fast asynchronous logger (millions of logs per second on an average // desktop) // Uses pre allocated lockfree queue for maximum throughput even under large // number of threads. // Creates a single back thread to pop messages from the queue and log them. // // Upon each log write the logger: // 1. Checks if its log level is enough to log the message // 2. Push a new copy of the message to a queue (or block the caller until space is available in the queue) // 2. Push a new copy of the message to a queue (or block the caller until // space is available in the queue) // 3. will throw spdlog_ex upon log exceptions // Upon destruction, logs all remaining messages in the queue before destructing.. // Upon destruction, logs all remaining messages in the queue before // destructing.. #include <extern/spdlog/common.h> #include <extern/spdlog/logger.h> #include "spdlog/common.h" #include "spdlog/logger.h" #include <chrono> #include <functional> #include <string> #include <memory> #include <string> namespace spdlog { namespace spdlog { namespace details // Async overflow policy - block by default. enum class async_overflow_policy { class async_log_helper; block, // Block until message can be enqueued overrun_oldest // Discard oldest message in the queue if full when trying to // add new item. }; namespace details { class thread_pool; } class async_logger :public logger class async_logger SPDLOG_FINAL : public std::enable_shared_from_this<async_logger>, public logger { friend class details::thread_pool; public: template<class It> async_logger(const std::string& name, const It& begin, const It& end, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); template<typename It> async_logger(std::string logger_name, const It &begin, const It &end, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); async_logger(const std::string& logger_name, sinks_init_list sinks, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); async_logger(std::string logger_name, sinks_init_list sinks_list, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); async_logger(const std::string& logger_name, sink_ptr single_sink, size_t queue_size, const async_overflow_policy overflow_policy = async_overflow_policy::block_retry, const std::function<void()>& worker_warmup_cb = nullptr, const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(), const std::function<void()>& worker_teardown_cb = nullptr); async_logger(std::string logger_name, sink_ptr single_sink, std::weak_ptr<details::thread_pool> tp, async_overflow_policy overflow_policy = async_overflow_policy::block); //Wait for the queue to be empty, and flush synchronously //Warning: this can potentialy last forever as we wait it to complete void flush() override; protected: void _sink_it(details::log_msg& msg) override; void _set_formatter(spdlog::formatter_ptr msg_formatter) override; void _set_pattern(const std::string& pattern) override; void sink_it_(details::log_msg &msg) override; void flush_() override; void backend_log_(details::log_msg &incoming_log_msg); void backend_flush_(); private: std::unique_ptr<details::async_log_helper> _async_log_helper; std::weak_ptr<details::thread_pool> thread_pool_; async_overflow_policy overflow_policy_; }; } } // namespace spdlog #include <extern/spdlog/details/async_logger_impl.h> #include "details/async_logger_impl.h"