Commit cff17b2f authored by Ramon Nou's avatar Ramon Nou
Browse files

Moving reads extended log to normal log module. 

Minor corrections.
parent df7d54dd

CHANGELOG.md

+2 −0
Original line number Diff line number Diff line
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).



## [Unreleased]

- Created a Guided Distributor using a mapping file to map chunks to specific

  nodes.



## [0.8.0] - 2020-09-15

## New

CMakeLists.txt

+0 −3
Original line number Diff line number Diff line
@@ -148,9 +148,6 @@ set(USE_GUIDED_PATH "~/guided.txt" CACHE STRING "File Path for guided distributo 
set_property(CACHE USE_GUIDED_PATH PROPERTY STRINGS)

message(STATUS "[gekkofs] Guided data distributor input file path: ${USE_GUIDED_PATH}")



option(TRACE_GUIDED "Output at INFO level information for guided distributor generation: " OFF)

message(STATUS "[gekkofs] Generate log line at INFO level for guided distributor: ${TRACE_GUIDED}")



configure_file(include/global/cmake_configure.hpp.in include/global/cmake_configure.hpp)



# Imported target

README.md

+24 −26
Original line number Diff line number Diff line
@@ -227,6 +227,7 @@ The following modules are available: 
   module will only be available if the client library is built in `Debug`

   mode.

 - `all`: All previous options combined.

 - `traces_reads`: Generate log line with extra information in read operations for guided distributor

 - `help`: Print a help message and exit.



When tracing sytem calls, specific syscalls can be removed from log messages by
@@ -255,11 +256,9 @@ Guided distributor distributes chunks using a shared file with the next format: 


Chunks not specified, are distributed using the Simple Hash distributor.



To generate such file we need to follow a first execution, using the next compilation options:

* `TRACE_GUIDED` ON

* `USE_GUIDED` OFF

To generate such file we need to follow a first execution, using the trace_reads log option



This will enable a `INFO` level log at the clients offering several lines that can be used to generate the input file. 

This will enable a `TRACE_READS` level log at the clients offering several lines that can be used to generate the input file.

In this stage, each node should generate a separated file this can be done in SLURM using the next line :

`srun -N 10 -n 320 --export="ALL" /bin/bash -c "export LIBGKFS_LOG_OUTPUT=${HOME}/test/GLOBAL.txt;LD_PRELOAD=${GKFS_PRLD} <app>"`
@@ -270,7 +269,6 @@ This should work if the nodes are sorted in alphabetical order, which is the usu 


```

Finally, enable the distributor using the next compilation flags:

* `TRACE_GUIDED` OFF

* `USE_GUIDED` ON

* `USE_GUIDED_PATH` `<path to guided.txt>`

examples/distributors/guided/generate.py

0 → 100644
+40 −0
Original line number Diff line number Diff line 
###

#  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain

#  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany



#  This software was partially supported by the

#  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).



#  This software was partially supported by the

#  ADA-FS project under the SPPEXA project funded by the DFG.



#  SPDX-License-Identifier: MIT

###



import re

import sys

import collections



file = sys.argv[1]



pattern = re.compile(r".+(read )(.*)( host: )(\d+).+(path: )(.+),.+(chunk_start: )(\d+).+(chunk_end: )(\d+)")



d = collections.OrderedDict()



with open(file) as f:

    for line in f:

        result = pattern.match(line)

        if result:

            d[result[2]] = 1

keys = sorted(d.keys())

i = 0

for key in keys:

    d[key] = i

    i = i + 1



with open(file) as f:

    for line in f:

        result = pattern.match(line)

        if result:

            for i in range(int(result[8]), int(result[10])+1):

                print (result[6], i, d[result[2]])

include/client/logging.hpp

+64 −53
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ 
namespace gkfs {

namespace log {



enum class log_level : short {

enum class log_level : u32 {

    print_syscalls       = 1 << 0,

    print_syscalls_entry = 1 << 1,

    print_info           = 1 << 2,
@@ -42,6 +42,7 @@ enum class log_level : short { 
    print_hermes         = 1 << 6,

    print_mercury        = 1 << 7,

    print_debug          = 1 << 8,

    print_trace_reads    = 1 << 9,



    // for internal use

    print_none           = 0,
@@ -105,6 +106,7 @@ static const auto constexpr warning = log_level::print_warnings; 
static const auto constexpr hermes           = log_level::print_hermes;

static const auto constexpr mercury          = log_level::print_mercury;

static const auto constexpr debug            = log_level::print_debug;

static const auto constexpr trace_reads      = log_level::print_trace_reads;

static const auto constexpr none             = log_level::print_none;

static const auto constexpr most             = log_level::print_most;

static const auto constexpr all              = log_level::print_all;
@@ -120,7 +122,8 @@ static const auto constexpr level_names = 
        "warning",

        "hermes",

        "mercury",

        "debug"

        "debug",

        "trace_reads"

);



inline constexpr auto
@@ -478,6 +481,7 @@ static_buffer::grow(std::size_t size) { 
#define LOG_MERCURY(...) do {} while(0);

#define LOG_SYSCALL(...) do {} while(0);

#define LOG_DEBUG(...) do {} while(0);

#define LOG_TRACE_READS(...) do {} while(0);



#else // !GKFS_ENABLE_LOGGING
@@ -523,6 +527,13 @@ static_buffer::grow(std::size_t size) { 
    }                                                                   \

} while(0);



#define LOG_TRACE_READS(...) do {                                             \

    if(gkfs::log::get_global_logger()) {                                \

        gkfs::log::get_global_logger()->log(                            \

                gkfs::log::trace_reads, __func__, __LINE__, __VA_ARGS__);     \

    }                                                                   \

} while(0);



#ifdef GKFS_DEBUG_BUILD



#define LOG_SYSCALL(...) do {                                           \