external apps, migrating, dependencies

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)

set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)

# Rocksdb dependencies

find_package(LZ4 REQUIRED)

find_package(ZLIB REQUIRED)

find_package(BZip2 REQUIRED)

find_package(GFlags REQUIRED)

find_package(snappy REQUIRED)

find_package(ZStd REQUIRED)

find_package(RocksDB REQUIRED)

# margo dependencies

find_package(Libev REQUIRED)

#find_package(CCI REQUIRED)

find_package(BMI REQUIRED)

find_package(Mercury REQUIRED)

find_package(MercuryUtil REQUIRED)

find_package(Abt REQUIRED)

find_package(Abt-Snoozer REQUIRED)

find_package(Margo REQUIRED)

# boost dependencies, system is required for filesystem #TODO VERSION UNTESTED. I USE 1.62

find_package(Boost 1.58 REQUIRED COMPONENTS system filesystem serialization)

include_directories(include ${ROCKSDB_INCLUDE_DIR}

        # margo paths

        ${MARGO_INCLUDE_DIR} ${ABT_INCLUDE_DIR} ${ABT_SNOOZER_INCLUDE_DIR} ${MERCURY_INCLUDE_DIR} ${LIBEV_INCLUDE_DIRS} ${ABT_IO_INCLUDE_DIRS}

        )

include_directories(include)

set(SOURCE_FILES main.cpp)

set(SOURCE_FILES main.cpp main.hpp)

add_executable(adafs_daemon ${SOURCE_FILES})

target_link_libraries(adafs_daemon ${ROCKSDB_LIBRARIES}

        # rocksdb libs

        ${snappy_LIBRARIES} ${ZLIB_LIBRARIES} ${BZIP2_LIBRARIES} ${ZSTD_LIBRARIES} ${gflags_LIBRARIES} ${LZ4_LIBRARY}

        # margo libs

        ${BMI_LIBRARIES} ${MERCURY_LIBRARIES} ${MERCURY_UTIL_LIBRARIES} ${ABT_LIBRARIES} ${ABT_SNOOZER_LIBRARIES} ${MARGO_LIBRARIES}

        -lpthread -lboost_system -lboost_filesystem -lboost_serialization -lboost_program_options -pg)

add_subdirectory(src/preload)

 No newline at end of file

//

// Created by evie on 3/17/17.

//

#ifndef FS_CONFIGURE_H

#define FS_CONFIGURE_H

// To enabled logging with info level

#define LOG_INFO

//#define LOG_DEBUG

//#define LOG_TRACE

// If ACM time should be considered

#define ACMtime

// If access permissions should be checked while opening a file

//#define CHECK_ACCESS

// Write-ahead logging of rocksdb

//#define RDB_WOL

// RPC configuration

#define RPCPORT 4433

#define RPC_TIMEOUT 15000

// Debug configurations

//#define RPC_TEST

#endif //FS_CONFIGURE_H

/*

 * LRUCache11 - a templated C++11 based LRU cache class that allows

 * specification of

 * key, value and optionally the map container type (defaults to

 * std::unordered_map)

 * By using the std::map and a linked list of keys it allows O(1) insert, delete

 * and

 * refresh operations.

 *

 * This is a header-only library and all you need is the LRUCache11.hpp file

 *

 * Github: https://github.com/mohaps/lrucache11

 *

 * This is a follow-up to the LRUCache project -

 * https://github.com/mohaps/lrucache

 *

 * Copyright (c) 2012-22 SAURAV MOHAPATRA <mohaps@gmail.com>

 *

 * Permission to use, copy, modify, and distribute this software for any

 * purpose with or without fee is hereby granted, provided that the above

 * copyright notice and this permission notice appear in all copies.

 *

 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES

 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF

 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR

 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN

 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF

 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

 */

#pragma once

#include <algorithm>

#include <cstdint>

#include <list>

#include <mutex>

#include <stdexcept>

#include <thread>

#include <unordered_map>

namespace lru11 {

/**

 * base class to prevent copy

 * use as ClassName : private NoCopy {}

 * to prevent copy constructor of ClassName and assignment by copy

 */

    class NoCopy {

    public:

        virtual ~NoCopy() = default;

    protected:

        NoCopy() = default;

    private:

        NoCopy(const NoCopy&) = delete;

        const NoCopy& operator=(const NoCopy&) = delete;

    };

/*

 * a noop lockable concept that can be used in place of std::mutex

 */

    class NullLock {

    public:

        void lock() {}

        void unlock() {}

        bool try_lock() { return true; }

    };

/**

 * error raised when a key not in cache is passed to get()

 */

    class KeyNotFound : public std::invalid_argument {

    public:

        KeyNotFound() : std::invalid_argument("key_not_found") {}

    };

    template<typename K, typename V>

    struct KeyValuePair {

    public:

        K key;

        V value;

        KeyValuePair(const K& k, const V& v) : key(k), value(v) {}

    };

/**

 *	The LRU Cache class templated by

 *		Key - key type

 *		Value - value type

 *		MapType - an associative container like std::unordered_map

 *		LockType - a lock type derived from the Lock class (default:

 *NullLock = no synchronization)

 *

 *	The default NullLock based template is not thread-safe, however passing

 *Lock=std::mutex will make it

 *	thread-safe

 */

    template<class Key, class Value, class Lock = NullLock,

            class Map = std::unordered_map<

                    Key, typename std::list<KeyValuePair<Key, Value>>::iterator>>

    class Cache : private NoCopy {

    public:

        typedef KeyValuePair<Key, Value> node_type;

        typedef std::list<KeyValuePair<Key, Value>> list_type;

        typedef Map map_type;

        typedef Lock lock_type;

        using Guard = std::lock_guard<lock_type>;

        /**

         * the max size is the hard limit of keys and (maxSize + elasticity) is the

         * soft limit

         * the cache is allowed to grow till maxSize + elasticity and is pruned back

         * to maxSize keys

         * set maxSize = 0 for an unbounded cache (but in that case, you're better off

         * using a std::unordered_map

         * directly anyway! :)

         */

        explicit Cache(size_t maxSize = 64, size_t elasticity = 10)

                : maxSize_(maxSize), elasticity_(elasticity) {}

        virtual ~Cache() = default;

        size_t size() const {

            Guard g(lock_);

            return cache_.size();

        }

        bool empty() const {

            Guard g(lock_);

            return cache_.empty();

        }

        void clear() {

            Guard g(lock_);

            cache_.clear();

            keys_.clear();

        }

        void insert(const Key& k, const Value& v) {

            Guard g(lock_);

            const auto iter = cache_.find(k);

            if (iter != cache_.end()) {

                iter->second->value = v;

                keys_.splice(keys_.begin(), keys_, iter->second);

                return;

            }

            keys_.emplace_front(k, v);

            cache_[k] = keys_.begin();

            prune();

        }

        bool tryGet(const Key& kIn, Value& vOut) {

            Guard g(lock_);

            const auto iter = cache_.find(kIn);

            if (iter == cache_.end()) {

                return false;

            }

            keys_.splice(keys_.begin(), keys_, iter->second);

            vOut = iter->second->value;

            return true;

        }

        const Value& get(const Key& k) {

            Guard g(lock_);

            const auto iter = cache_.find(k);

            if (iter == cache_.end()) {

                throw KeyNotFound();

            }

            keys_.splice(keys_.begin(), keys_, iter->second);

            return iter->second->value;

        }

        bool remove(const Key& k) {

            Guard g(lock_);

            auto iter = cache_.find(k);

            if (iter == cache_.end()) {

                return false;

            }

            keys_.erase(iter->second);

            cache_.erase(iter);

            return true;

        }

        bool contains(const Key& k) {

            Guard g(lock_);

            return cache_.find(k) != cache_.end();

        }

        size_t getMaxSize() const { return maxSize_; }

        size_t getElasticity() const { return elasticity_; }

        size_t getMaxAllowedSize() const { return maxSize_ + elasticity_; }

        template<typename F>

        void cwalk(F& f) const {

            Guard g(lock_);

            std::for_each(keys_.begin(), keys_.end(), f);

        }

    protected:

        size_t prune() {

            size_t maxAllowed = maxSize_ + elasticity_;

            if (maxSize_ == 0 || cache_.size() < maxAllowed) {

                return 0;

            }

            size_t count = 0;

            while (cache_.size() > maxSize_) {

                cache_.erase(keys_.back().key);

                keys_.pop_back();

                ++count;

            }

            return count;

        }

    private:

        mutable Lock lock_;

        Map cache_;

        list_type keys_;

        size_t maxSize_;

        size_t elasticity_;

    };

}  // namespace LRUCache11

//

// Copyright(c) 2015 Gabi Melman.

// Distributed under the MIT License (http://opensource.org/licenses/MIT)

//

#pragma once

// Very fast asynchronous logger (millions of logs per second on an average desktop)

// Uses pre allocated lockfree queue for maximum throughput even under large number of threads.

// Creates a single back thread to pop messages from the queue and log them.

//

// Upon each log write the logger:

//    1. Checks if its log level is enough to log the message

//    2. Push a new copy of the message to a queue (or block the caller until space is available in the queue)

//    3. will throw spdlog_ex upon log exceptions

// Upon destruction, logs all remaining messages in the queue before destructing..

#include <extern/spdlog/common.h>

#include <extern/spdlog/logger.h>

#include <chrono>

#include <functional>

#include <string>

#include <memory>

namespace spdlog

{

namespace details

{

class async_log_helper;

}

class async_logger :public logger

{

public:

    template<class It>

    async_logger(const std::string& name,

                 const It& begin,

                 const It& end,

                 size_t queue_size,

                 const async_overflow_policy overflow_policy =  async_overflow_policy::block_retry,

                 const std::function<void()>& worker_warmup_cb = nullptr,

                 const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(),

                 const std::function<void()>& worker_teardown_cb = nullptr);

    async_logger(const std::string& logger_name,

                 sinks_init_list sinks,

                 size_t queue_size,

                 const async_overflow_policy overflow_policy = async_overflow_policy::block_retry,

                 const std::function<void()>& worker_warmup_cb = nullptr,

                 const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(),

                 const std::function<void()>& worker_teardown_cb = nullptr);

    async_logger(const std::string& logger_name,

                 sink_ptr single_sink,

                 size_t queue_size,

                 const async_overflow_policy overflow_policy =  async_overflow_policy::block_retry,

                 const std::function<void()>& worker_warmup_cb = nullptr,

                 const std::chrono::milliseconds& flush_interval_ms = std::chrono::milliseconds::zero(),

                 const std::function<void()>& worker_teardown_cb = nullptr);

    //Wait for the queue to be empty, and flush synchronously

    //Warning: this can potentialy last forever as we wait it to complete

    void flush() override;

protected:

    void _sink_it(details::log_msg& msg) override;

    void _set_formatter(spdlog::formatter_ptr msg_formatter) override;

    void _set_pattern(const std::string& pattern) override;

private:

    std::unique_ptr<details::async_log_helper> _async_log_helper;

};

}

#include <extern/spdlog/details/async_logger_impl.h>

//

// Copyright(c) 2015 Gabi Melman.

// Distributed under the MIT License (http://opensource.org/licenses/MIT)

//

#pragma once

#include <string>

#include <initializer_list>

#include <chrono>

#include <memory>

#include <atomic>

#include <exception>

#include<functional>

#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)

#include <codecvt>

#include <locale>

#endif

#include <extern/spdlog/details/null_mutex.h>

//visual studio upto 2013 does not support noexcept nor constexpr

#if defined(_MSC_VER) && (_MSC_VER < 1900)

#define SPDLOG_NOEXCEPT throw()

#define SPDLOG_CONSTEXPR

#else

#define SPDLOG_NOEXCEPT noexcept

#define SPDLOG_CONSTEXPR constexpr

#endif

#if defined(__GNUC__)  || defined(__clang__)

#define SPDLOG_DEPRECATED __attribute__((deprecated))

#elif defined(_MSC_VER)

#define SPDLOG_DEPRECATED __declspec(deprecated)

#else

#define SPDLOG_DEPRECATED

#endif

#include <extern/spdlog/fmt/fmt.h>

namespace spdlog

{

class formatter;

namespace sinks

{

class sink;

}

using log_clock = std::chrono::system_clock;

using sink_ptr = std::shared_ptr < sinks::sink >;

using sinks_init_list = std::initializer_list < sink_ptr >;

using formatter_ptr = std::shared_ptr<spdlog::formatter>;

#if defined(SPDLOG_NO_ATOMIC_LEVELS)

using level_t = details::null_atomic_int;

#else

using level_t = std::atomic<int>;

#endif

using log_err_handler = std::function<void(const std::string &err_msg)>;

//Log level enum

namespace level

{

typedef enum

{

    trace = 0,

    debug = 1,

    info = 2,

    warn = 3,

    err = 4,

    critical = 5,

    off = 6

} level_enum;

static const char* level_names[] { "trace", "debug", "info",  "warning", "error", "critical", "off" };

static const char* short_level_names[] { "T", "D", "I", "W", "E", "C", "O" };

inline const char* to_str(spdlog::level::level_enum l)

{

    return level_names[l];

}

inline const char* to_short_str(spdlog::level::level_enum l)

{

    return short_level_names[l];

}

} //level

//

// Async overflow policy - block by default.

//

enum class async_overflow_policy

{

    block_retry, // Block / yield / sleep until message can be enqueued

    discard_log_msg // Discard the message it enqueue fails

};

//

// Log exception

//

namespace details

{

namespace os

{

std::string errno_str(int err_num);

}

}

class spdlog_ex: public std::exception

{

public:

    spdlog_ex(const std::string& msg):_msg(msg)

    {}

    spdlog_ex(const std::string& msg, int last_errno)

    {

        _msg = msg + ": " + details::os::errno_str(last_errno);

    }

    const char* what() const SPDLOG_NOEXCEPT override

    {

        return _msg.c_str();

    }

private:

    std::string _msg;

};

//

// wchar support for windows file names (SPDLOG_WCHAR_FILENAMES must be defined)

//

#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES)

using filename_t = std::wstring;

#else

using filename_t = std::string;

#endif

} //spdlog