Merge branch 'cleanup' into 'master'

// Daemon path to auxiliary files

#define DAEMON_AUX_PATH "/tmp"

// If ACM time should be considered

#define ACMtime //unused

// XXX Should blocksize and chunksize be merged?

#define BLOCKSIZE 524288 // in bytes 512KB

#define CHUNKSIZE 524288 // in bytes 512KB

// What metadata is used TODO this has to be parametrized or put into a configuration file

#define MDATA_USE_GID false

#define MDATA_USE_LINK_CNT false

#define MDATA_USE_BLOCKS false

#define MDATA_USE_SIZE true // XXX to be added in ADAFS_DATA. currently on by default

// should permissions be checked when access() is called or discarded (disabled by default)

//#define CHECK_ACCESS

//#define ZERO_BUFFER_BEFORE_READ

// Write-ahead logging of rocksdb

//#define KV_WOL

// Optimize Key-Value store. Eventually, different modes will be available for different workloads. TODO

//#define KV_OPTIMIZE

// Optimize Key-Value store for tmpfs/ramdisk usage

//#define KV_OPTIMIZE_RAMDISK

#define KV_WOL false

// Buffer size for Rocksdb. A high number means that all entries are held in memory.

// However, when full the application blocks until **all** entries are flushed to disk.

//size of preallocated buffer to hold directory entries in rpc call

#define RPC_DIRENTS_BUFF_SIZE (8 * 1024 * 1024) // 8 mega

// Debug configurations

//#define RPC_TEST //unused

// environment prefixes

#define ENV_PREFIX "ADAFS_"

    options.create_if_missing = true;

    options.merge_operator.reset(new MetadataMergeOperator);

    MetadataDB::optimize_rocksdb_options(options);

#if !defined(KV_WOL)

    write_opts.disableWAL = true;

#endif

    write_opts.disableWAL = !(KV_WOL);

    rdb::DB * rdb_ptr;

    auto s = rocksdb::DB::Open(options, path, &rdb_ptr);

    if (!s.ok()) {

void MetadataDB::optimize_rocksdb_options(rdb::Options& options) {

    options.max_successive_merges = 128;

#if defined(KV_OPTIMIZE_RAMDISK)

    // as described at https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide

    // use mmap read

    options.allow_mmap_reads = true;

    // disable block cache, enable blook filters and reduce the delta encoding restart interval

    rocksdb::BlockBasedTableOptions table_options{};

    table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));

    table_options.no_block_cache = true;

    table_options.block_restart_interval = 4;

    options.table_factory.reset(NewBlockBasedTableFactory(table_options));

    // enable lightweight compression (snappy or lz4). We use lz4 for now

    options.compression = rocksdb::CompressionType::kLZ4Compression;

    // set up compression more aggressively and allocate more threads for flush and compaction

    options.level0_file_num_compaction_trigger = 1;

    options.max_background_flushes = 8;

    options.max_background_compactions = 8;

    options.max_subcompactions = 4;

    // keep all the files open

    options.max_open_files = -1;

#elif defined(KV_OPTIMIZE)

    // rocksdb::BlockBasedTableOptions block_options{};

    // block_options.block_size = 16384 * 2;

    // options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options));

    // experimental settings

    // options.write_buffer_size = 512;

    // options.max_write_buffer_number = 16;

    // options.min_write_buffer_number_to_merge = 4;

    // These 4 below have the most impact

    options.max_bytes_for_level_base = 2048;

    options.max_bytes_for_level_multiplier = 10;

    options.target_file_size_base = 256;

    options.target_file_size_multiplier = 1;

    options.max_background_flushes = 1;

    options.max_background_compactions = 48;

    options.level0_file_num_compaction_trigger = 1;

    options.level0_slowdown_writes_trigger = 48;

    options.level0_stop_writes_trigger = 56;

    // options.arena_block_size = 1024 * 8;

    // options.compression = rocksdb::kNoCompression; // doesnt do anything

#endif

#if defined(KV_WRITE_BUFFER)

    // write_buffer_size is multiplied by the write_buffer_number to get the amount of data hold in memory.

    // at min_write_buffer_number_to_merge rocksdb starts to flush entries out to disk

    attr.st_uid = CTX->fs_conf()->uid;

    attr.st_gid = CTX->fs_conf()->gid;

    attr.st_rdev = 0;

    attr.st_blksize = BLOCKSIZE;

    attr.st_blksize = CHUNKSIZE;

    attr.st_blocks = 0;

    memset(&attr.st_atim, 0, sizeof(timespec));