Commit f5a24bdd authored by Marc Vef's avatar Marc Vef
Browse files

Sanitizing --dbbackend input, refactor metadata db path usage

parent be7a9eab

include/config.hpp

+3 −2
Original line number Diff line number Diff line
@@ -57,6 +57,9 @@ constexpr auto daemon_log_level = 4; // info 
} // namespace log



namespace metadata {

// directory name where the metadata db instance is placed

constexpr auto dir = "metadata";



// which metadata should be considered apart from size and mode

constexpr auto use_atime = false;

constexpr auto use_ctime = false;
@@ -98,8 +101,6 @@ constexpr auto daemon_handler_xstreams = 4; 
namespace rocksdb {

// Write-ahead logging of rocksdb

constexpr auto use_write_ahead_log = false;

// directory name where the rocksdb instance is placed

constexpr auto data_dir = "rocksdb";

} // namespace rocksdb



} // namespace gkfs::config

include/daemon/backend/metadata/db.hpp

+3 −1
Original line number Diff line number Diff line
@@ -47,6 +47,9 @@ 


namespace gkfs::metadata {



constexpr auto rocksdb_backend = "rocksdb";

constexpr auto parallax_backend = "parallaxdb";





class MetadataDB {

private:
@@ -54,7 +57,6 @@ private: 
    std::shared_ptr<spdlog::logger> log_;

    std::unique_ptr<AbstractMetadataBackend> backend_;





public:

    MetadataDB(const std::string& path, const std::string_view database);

src/daemon/backend/metadata/db.cpp

+17 −15
Original line number Diff line number Diff line
@@ -34,11 +34,12 @@ 
#include <common/metadata.hpp>

#include <common/path_util.hpp>

#include <iostream>

#include <filesystem>



extern "C" {

#include <sys/stat.h>

}



namespace fs = std::filesystem;



namespace gkfs::metadata {
@@ -51,23 +52,24 @@ struct MetadataDBFactory { 
    static std::unique_ptr<AbstractMetadataBackend>

    create(const std::string& path, const std::string_view id) {





        if(id == "parallaxdb") {

        if(id == gkfs::metadata::parallax_backend) {

#ifdef GKFS_ENABLE_PARALLAX

            return std::make_unique<ParallaxBackend>(path);

#else

            GKFS_METADATA_MOD->log()->error("PARALLAX not compiled");

            exit(EXIT_FAILURE);

            auto metadata_path = fmt::format("{}/{}", path,

                                             gkfs::metadata::parallax_backend);

            GKFS_METADATA_MOD->log()->trace("Using Parallax file '{}'",

                                            metadata_path);

            return std::make_unique<ParallaxBackend>(metadata_path);

#endif

        } else if(id == "rocksdb") {

        } else if(id == gkfs::metadata::rocksdb_backend) {

#ifdef GKFS_ENABLE_ROCKSDB

            return std::make_unique<RocksDBBackend>(path);

#else

            GKFS_METADATA_MOD->log()->error("ROCKSDB not compiled");

            exit(EXIT_FAILURE);

            auto metadata_path =

                    fmt::format("{}/{}", path, gkfs::metadata::rocksdb_backend);

            fs::create_directories(metadata_path);

            GKFS_METADATA_MOD->log()->trace("Using RocksDB directory '{}'",

                                            metadata_path);

            return std::make_unique<RocksDBBackend>(metadata_path);

#endif

        }



        GKFS_METADATA_MOD->log()->error("No valid metadata backend selected");

        exit(EXIT_FAILURE);

    }
@@ -83,13 +85,13 @@ struct MetadataDBFactory { 
MetadataDB::MetadataDB(const std::string& path, const std::string_view database)

    : path_(path) {



    backend_ = MetadataDBFactory::create(path, database);



    /* Get logger instance and set it for data module and chunk storage */

    GKFS_METADATA_MOD->log(spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME));

    assert(GKFS_METADATA_MOD->log());

    log_ = spdlog::get(GKFS_METADATA_MOD->LOGGER_NAME);

    assert(log_);



    backend_ = MetadataDBFactory::create(path, database);

}



MetadataDB::~MetadataDB() {

src/daemon/backend/metadata/parallax_backend.cpp

+4 −5
Original line number Diff line number Diff line
@@ -58,7 +58,8 @@ ParallaxBackend::~ParallaxBackend() { 
 * Called when the daemon is started: Connects to the KV store

 * @param path where KV store data is stored

 */

ParallaxBackend::ParallaxBackend(const std::string& path) {

ParallaxBackend::ParallaxBackend(const std::string& path)

    : par_path_(std::move(path)) {



    // We try to open options.yml if it exists, if not we create it by default

    int options = open("options.yml", O_RDWR | O_CREAT, 0644);
@@ -71,9 +72,6 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { 
    }



    close(options);



    // Kreon

    par_path_ = path + "x"; // file is rocksdb (add an x)

    int64_t size;



    int fd = open(par_path_.c_str(), O_RDWR | O_CREAT, 0644);
@@ -99,7 +97,8 @@ ParallaxBackend::ParallaxBackend(const std::string& path) { 
        write(fd, tmp.c_str(), 1);

        close(fd);



        // We format the database

        // We format the database TODO this doesn't work kv_format.parallax is

        // not in path

        std::string cmd = "kv_format.parallax --device " + par_path_ +

                          " --max_regions_num 1 ";

        system(cmd.c_str());

src/daemon/daemon.cpp

+34 −14
Original line number Diff line number Diff line
@@ -243,7 +243,8 @@ void 
init_environment() {

    // Initialize metadata db

    auto metadata_path = fmt::format("{}/{}", GKFS_DATA->metadir(),

                                     gkfs::config::rocksdb::data_dir);

                                     gkfs::config::metadata::dir);

    fs::create_directories(metadata_path);

    GKFS_DATA->spdlogger()->debug("{}() Initializing metadata DB: '{}'",

                                  __func__, metadata_path);

    try {
@@ -413,13 +414,9 @@ destroy_enviroment() { 


    // Delete rootdir/metadir if requested

    if(!GKFS_DATA->keep_rootdir()) {

        GKFS_DATA->spdlogger()->info("{}() Removing RootDir/MetaDir/MetaFile",

        GKFS_DATA->spdlogger()->info("{}() Removing rootdir and metadir ...",

                                     __func__);

        fs::remove_all(GKFS_DATA->metadir(), ecode);

#ifdef GKFS_ENABLE_PARALLAX

        // some metadata backends uses a file instead of a dir.

        fs::remove_all(GKFS_DATA->metadir() + "x", ecode);

#endif

        fs::remove_all(GKFS_DATA->rootdir(), ecode);

    }

}
@@ -551,7 +548,7 @@ parse_input(const cli_options& opts, const CLI::App& desc) { 
    auto rootdir_path = fs::path(rootdir);

    if(desc.count("--rootdir-suffix")) {

        if(opts.rootdir_suffix == gkfs::config::data::chunk_dir ||

           opts.rootdir_suffix == gkfs::config::rocksdb::data_dir)

           opts.rootdir_suffix == gkfs::config::metadata::dir)

            throw runtime_error(fmt::format(

                    "rootdir_suffix '{}' is reserved and not allowed.",

                    opts.rootdir_suffix));
@@ -609,10 +606,33 @@ parse_input(const cli_options& opts, const CLI::App& desc) { 
    }



    if(desc.count("--dbbackend")) {

        auto dbbackend = opts.dbbackend;

        GKFS_DATA->dbbackend(dbbackend);

        if(opts.dbbackend == gkfs::metadata::rocksdb_backend ||

           opts.dbbackend == gkfs::metadata::parallax_backend) {

#ifndef GKFS_ENABLE_PARALLAX

            if(opts.dbbackend == gkfs::metadata::parallax_backend) {

                throw runtime_error(fmt::format(

                        "dbbackend '{}' was not compiled and is disabled. "

                        "Pass -DGKFS_ENABLE_PARALLAX:BOOL=ON to CMake to enable.",

                        opts.dbbackend));

            }

#endif

#ifndef GKFS_ENABLE_ROCKSDB

            if(opts.dbbackend == gkfs::metadata::rocksdb_backend) {

                throw runtime_error(fmt::format(

                        "dbbackend '{}' was not compiled and is disabled. "

                        "Pass -DGKFS_ENABLE_ROCKSDB:BOOL=ON to CMake to enable.",

                        opts.dbbackend));

            }

#endif

            GKFS_DATA->dbbackend(opts.dbbackend);

        } else {

            throw runtime_error(

                    fmt::format("dbbackend '{}' is not valid. Consult `--help`",

                                opts.dbbackend));

        }



    } else

        GKFS_DATA->dbbackend("rocksdb");

        GKFS_DATA->dbbackend(gkfs::metadata::rocksdb_backend);



    if(desc.count("--kreonsize")) { // Size in GB

        GKFS_DATA->kreon_size_md(stoi(opts.kreonsize));
@@ -678,11 +698,11 @@ main(int argc, const char* argv[]) { 
                "Cleans Rootdir >after< the deamon finishes");

    desc.add_option(

                "--dbbackend,-d", opts.dbbackend,

                "Database Backend to use. If not set, rocksdb is used. For parallaxdb, a file called rocksdbx with 8GB will be created in metadir");

   

   

                "Metadata database backend to use. Available: {rocksdb, parallaxdb}'\n"

                "RocksDB is default if not set. Parallax support is experimental.\n"

                "Note, parallaxdb creates a file called rocksdbx with 8GB created in metadir.");

    desc.add_option("--kreonsize",opts.kreonsize,

                    "parallaxdb - Metatada file size in GB (default 8), "

                    "parallaxdb - metadata file size in GB (default 8GB), "

                    "used only with new files");

    desc.add_flag("--version", "Print version and exit.");

    // clang-format on