Commit 01cd5eba authored by Marc Vef's avatar Marc Vef
Browse files

Init_rocksdb moved. DB Options optimized (44k creates/sec) for 4m files

parent b85e35e4

lfs/src/db/util.cpp

+69 −0
Original line number Diff line number Diff line
@@ -6,6 +6,75 @@ 


using namespace std;



bool init_rocksdb() {

    rocksdb::DB* db;

    ADAFS_DATA->rdb_path(ADAFS_DATA->rootdir() + "/meta/rocksdb"s);

    rocksdb::Options options;

    // Optimize RocksDB. This is the easiest way to get RocksDB to perform well

    options.IncreaseParallelism();

    options.OptimizeLevelStyleCompaction();

    // create the DB if it's not already present

    options.create_if_missing = true;





    optimize_rocksdb(options);



    ADAFS_DATA->rdb_options(options);

    rocksdb::OptimisticTransactionDB* txn_db;

    rocksdb::OptimisticTransactionOptions txn_options{};

    ADAFS_DATA->txn_rdb_options(txn_options);

    ADAFS_DATA->spdlogger()->info("RocksDB options set. About to connect...");

    // open DB

    auto s_txn = rocksdb::OptimisticTransactionDB::Open(ADAFS_DATA->rdb_options(), ADAFS_DATA->rdb_path(), &txn_db);



    if (s_txn.ok()) {

        db = txn_db->GetBaseDB(); // db connection for db operations without transactions

        shared_ptr<rocksdb::DB> s_db(db);

        ADAFS_DATA->rdb(s_db);

        shared_ptr<rocksdb::OptimisticTransactionDB> s_txn_db(txn_db);

        ADAFS_DATA->txn_rdb(s_txn_db);

        ADAFS_DATA->spdlogger()->info("RocksDB connection established.");

        return true;

    } else {

        ADAFS_DATA->spdlogger()->info("[ERROR] RocksDB connection FAILURE. Exiting...");

        return false;

    }

}



void optimize_rocksdb(rocksdb::Options& options) {



    //    rocksdb::BlockBasedTableOptions block_options{};

//    block_options.block_size = 16384 * 2;

//    options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options));

    // experimental settings

//    options.write_buffer_size = 512;

//    options.max_write_buffer_number = 16;

//    options.min_write_buffer_number_to_merge = 4;

    // These 4 below have the most impact

    options.max_bytes_for_level_base = 2048;

    options.max_bytes_for_level_multiplier = 10;

    options.target_file_size_base = 256;

    options.target_file_size_multiplier = 1;

    //

    options.max_background_flushes = 1;

    options.max_background_compactions = 48;

    options.level0_file_num_compaction_trigger = 1;

    options.level0_slowdown_writes_trigger = 48;

    options.level0_stop_writes_trigger = 56;

//    options.arena_block_size = 1024 * 8;

//    options.compression = rocksdb::kNoCompression; // doesnt do anything



    // Disable Write-Ahead Logging if configured

    rocksdb::WriteOptions write_options{};

#ifndef RDB_WOL

    write_options.disableWAL = true;

#endif

    ADAFS_DATA->rdb_write_options(write_options);

}









/**

 * Build dentry key of form <d_ParentInode_filename>

 * @param inode

lfs/src/db/util.hpp

+4 −0
Original line number Diff line number Diff line
@@ -23,6 +23,10 @@ const std::array<std::string, 10> md_field_map = { 
        "_atime"s, "_mtime"s, "_ctime"s, "_uid"s, "_gid"s, "_mode"s, "_inodeno"s, "_lnkcnt"s, "_size"s, "_blkcnt"s

};



bool init_rocksdb();



void optimize_rocksdb(rocksdb::Options& options);



std::string db_build_dentry_key(const fuse_ino_t inode, const std::string& name);



std::string db_build_dentry_prefix(const fuse_ino_t inode);

lfs/src/main.cpp

+0 −36
Original line number Diff line number Diff line
@@ -8,42 +8,6 @@ static struct fuse_lowlevel_ops adafs_ops; 


using namespace std;



bool init_rocksdb() {

    rocksdb::DB* db;

    ADAFS_DATA->rdb_path(ADAFS_DATA->rootdir() + "/meta/rocksdb"s);

    rocksdb::Options options;

    // Optimize RocksDB. This is the easiest way to get RocksDB to perform well

    options.IncreaseParallelism();

    options.OptimizeLevelStyleCompaction();

    // create the DB if it's not already present

    options.create_if_missing = true;

    rocksdb::WriteOptions write_options{};

#ifndef RDB_WOL

    write_options.disableWAL = true;

#endif

    ADAFS_DATA->rdb_options(options);

    ADAFS_DATA->rdb_write_options(write_options);

    rocksdb::OptimisticTransactionDB* txn_db;

    rocksdb::OptimisticTransactionOptions txn_options{};

    ADAFS_DATA->txn_rdb_options(txn_options);

    ADAFS_DATA->spdlogger()->info("RocksDB options set. About to connect...");

    // open DB

    auto s_txn = rocksdb::OptimisticTransactionDB::Open(ADAFS_DATA->rdb_options(), ADAFS_DATA->rdb_path(), &txn_db);



    if (s_txn.ok()) {

        db = txn_db->GetBaseDB(); // db connection for db operations without transactions

        shared_ptr<rocksdb::DB> s_db(db);

        ADAFS_DATA->rdb(s_db);

        shared_ptr<rocksdb::OptimisticTransactionDB> s_txn_db(txn_db);

        ADAFS_DATA->txn_rdb(s_txn_db);

        ADAFS_DATA->spdlogger()->info("RocksDB connection established.");

        return true;

    } else {

        ADAFS_DATA->spdlogger()->info("[ERROR] RocksDB connection FAILURE. Exiting...");

        return false;

    }

}



/**

 * Initialize filesystem

 *