Commit f43f9e90 authored by Ramon Nou's avatar Ramon Nou
Browse files

Added hostname in the TRACE_GUIDED logs

parent 9bb75e26

README.md

+3 −13
Original line number Diff line number Diff line
@@ -257,23 +257,13 @@ To generate such file we need to follow a first execution, using the next compil 


This will enable a `INFO` level log at the clients offering several lines that can be used to generate the input file. 

In this stage, each node should generate a separated file this can be done in SLURM using the next line :

`srun -N 10 -n 320 --export="ALL" /bin/bash -c "export LIBGKFS_LOG_OUTPUT=${HOME}/test/GUIDED-\${SLURMD_NODENAME}.txt;LD_PRELOAD=${GKFS_PRLD} <app>"`

`srun -N 10 -n 320 --export="ALL" /bin/bash -c "export LIBGKFS_LOG_OUTPUT=${HOME}/test/GLOBAL.txt;LD_PRELOAD=${GKFS_PRLD} <app>"`



Then, use the `utils/generate.py` to create the output file. 

* `python utils/generate.py <numnode> ~/test/GUIDED-node<x> >> guided.txt`

* `python utils/generate.py ~/test/GLOBAL.txt >> guided.txt`



The next script could be used, normally `SLURM` nodes are set in alphabetical order:

This should work if the nodes are sorted in alphabetical order, which is the usual scenario.



```

rm guided.txt

#/bin/bash

index=0

for i in ~/test/GUIDED-*.txt;

do

echo $i $index

python utils/generate.py $index $i >> guided.txt

index=$((index+1))

done

```

Finally, enable the distributor using the next compilation flags:

* `TRACE_GUIDED` OFF

include/client/preload_context.hpp

+7 −1
Original line number Diff line number Diff line
@@ -20,6 +20,7 @@ 
#include <memory>

#include <vector>

#include <string>

#include <config.hpp>



#include <bitset>
@@ -92,7 +93,9 @@ private: 
    mutable std::mutex internal_fds_mutex_;

    bool internal_fds_must_relocate_;

    std::bitset<MAX_USER_FDS> protected_fds_;



#ifdef TRACE_GUIDED

    std::string hostname = "";

#endif

public:

    static PreloadContext* getInstance() {

        static PreloadContext instance;
@@ -167,6 +170,9 @@ public: 
    void protect_user_fds();



    void unprotect_user_fds();

#ifdef TRACE_GUIDED

    std::string get_hostname();

#endif

};



} // namespace preload

src/client/preload_context.cpp

+12 −1
Original line number Diff line number Diff line
@@ -383,5 +383,16 @@ PreloadContext::unprotect_user_fds() { 
    internal_fds_must_relocate_ = true;

}



#ifdef TRACE_GUIDED

std::string PreloadContext::get_hostname() {

    if (hostname == "") {

        char host[255];

        gethostname(host,255);

        hostname = host;

    }

    return hostname;

}

#endif



} // namespace preload

} // namespace gkfs

src/client/rpc/forward_data.cpp

+2 −3
Original line number Diff line number Diff line
@@ -226,7 +226,6 @@ pair<int, ssize_t> forward_read(const string& path, void* buf, const off64_t off 
    // contains the recipient ids, used to access the target_chnks map.

    // First idx is chunk with potential offset

    std::vector<uint64_t> targets{};



    // targets for the first and last chunk as they need special treatment

    uint64_t chnk_start_target = 0;

    uint64_t chnk_end_target = 0;
@@ -325,8 +324,8 @@ pair<int, ssize_t> forward_read(const string& path, void* buf, const off64_t off 
                target, path, chnk_start, chnk_end, in.chunk_n(), total_chunk_size, in.offset());

		

#ifdef TRACE_GUIDED

	LOG(INFO, "read host: {}, path: {}, chunk_start: {}, chunk_end: {}",

                target, path, chnk_start, chnk_end);

	LOG(INFO, "read {} host: {}, path: {}, chunk_start: {}, chunk_end: {}",

                CTX->get_hostname(), target, path, chnk_start, chnk_end);

#endif



        } catch (const std::exception& ex) {

utils/generate.py

+22 −6
Original line number Diff line number Diff line 
import re

import sys

import collections

#[2020-03-03 10:50:45.513741 CET] [151530] [debug] <read():315> host: 9, path: /fluidda-XFIEL.00000001.00000119.mpio.bin, chunks: 1, size: 524288, offset: 295200

#(read).+(host: )(\d+).+(path: )(.+),.+(chunks: )(\d+).+(size: )(\d+).+(offset: )(\d+)

# Original host # path # chunkid # size # offset 

destination = sys.argv[1]

file = sys.argv[2]

#destination = sys.argv[1]

file = sys.argv[1]

pattern = re.compile(r".+(read).+(host: )(\d+).+(path: )(.+),.+(chunks: )(\d+).+(size: )(\d+).+(offset: )(\d+)")



# [2020-03-30 14:58:09.938632 CEST] [81763] [info] host: 6, path: /fluidda-XFIEL.00000001.00000181.mpio.bin, chunks: 1, size: 8, offset: 168
@@ -12,11 +13,26 @@ pattern = re.compile(r".+(read).+(host: )(\d+).+(path: )(.+),.+(chunks: )(\d+).+ 


pattern = re.compile(r".*(host: )(\d+).+(path: )(.+),.+(chunks: )(\d+).+(size: )(\d+).+(offset: )(\d+)")



pattern = re.compile(r".+(read).+(host: )(\d+).+(path: )(.+),.+(chunk_start: )(\d+).+(chunk_end: )(\d+)")

pattern = re.compile(r".+(read )(.*)( host: )(\d+).+(path: )(.+),.+(chunk_start: )(\d+).+(chunk_end: )(\d+)")



d = collections.OrderedDict()



with open(file) as f:

	for line in f:

		result = pattern.match(line)

		if result:

			d[result[2]] = 1

keys = sorted(d.keys())

i = 0

for key in keys:

#	print (key)

	d[key] = i

	i = i + 1



with open(file) as f:

        for line in f:

                result = pattern.match(line)

                if result:

			#split = re.split(":\,",line)

			for i in range(int(result[7]), int(result[9])+1):

				print (result[5], i, destination)
 
                        for i in range(int(result[8]), int(result[10])+1):

                               print (result[6], i, d[result[2]])