LCOV - code coverage report
Current view: top level - src/common/rpc - distributor.cpp (source / functions) Hit Total Coverage
Test: coverage.info Lines: 76 127 59.8 %
Date: 2024-04-23 00:09:24 Functions: 13 32 40.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*
       2             :   Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain
       3             :   Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany
       4             : 
       5             :   This software was partially supported by the
       6             :   EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
       7             : 
       8             :   This software was partially supported by the
       9             :   ADA-FS project under the SPPEXA project funded by the DFG.
      10             : 
      11             :   This file is part of GekkoFS.
      12             : 
      13             :   GekkoFS is free software: you can redistribute it and/or modify
      14             :   it under the terms of the GNU General Public License as published by
      15             :   the Free Software Foundation, either version 3 of the License, or
      16             :   (at your option) any later version.
      17             : 
      18             :   GekkoFS is distributed in the hope that it will be useful,
      19             :   but WITHOUT ANY WARRANTY; without even the implied warranty of
      20             :   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      21             :   GNU General Public License for more details.
      22             : 
      23             :   You should have received a copy of the GNU General Public License
      24             :   along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.
      25             : 
      26             :   SPDX-License-Identifier: GPL-3.0-or-later
      27             : */
      28             : 
      29             : #include <common/rpc/distributor.hpp>
      30             : 
      31             : using namespace std;
      32             : 
      33             : namespace gkfs {
      34             : 
      35             : namespace rpc {
      36             : 
      37           0 : SimpleHashDistributor::SimpleHashDistributor(host_t localhost,
      38           0 :                                              unsigned int hosts_size)
      39           0 :     : localhost_(localhost), hosts_size_(hosts_size), all_hosts_(hosts_size) {
      40           0 :     ::iota(all_hosts_.begin(), all_hosts_.end(), 0);
      41           0 : }
      42             : 
      43           0 : SimpleHashDistributor::SimpleHashDistributor() {}
      44             : 
      45             : host_t
      46           0 : SimpleHashDistributor::localhost() const {
      47           0 :     return localhost_;
      48             : }
      49             : 
      50             : unsigned int
      51           0 : SimpleHashDistributor::hosts_size() const {
      52           0 :     return hosts_size_;
      53             : }
      54             : 
      55             : host_t
      56           0 : SimpleHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id,
      57             :                                    const int num_copy) const {
      58           0 :     return (str_hash(path + ::to_string(chnk_id)) + num_copy) % hosts_size_;
      59             : }
      60             : 
      61             : host_t
      62           0 : SimpleHashDistributor::locate_data(const string& path, const chunkid_t& chnk_id,
      63             :                                    unsigned int hosts_size,
      64             :                                    const int num_copy) {
      65           0 :     if(hosts_size_ != hosts_size) {
      66           0 :         hosts_size_ = hosts_size;
      67           0 :         all_hosts_ = std::vector<unsigned int>(hosts_size);
      68           0 :         ::iota(all_hosts_.begin(), all_hosts_.end(), 0);
      69             :     }
      70             : 
      71           0 :     return (str_hash(path + ::to_string(chnk_id)) + num_copy) % hosts_size_;
      72             : }
      73             : 
      74             : host_t
      75           0 : SimpleHashDistributor::locate_file_metadata(const string& path,
      76             :                                             const int num_copy) const {
      77           0 :     return (str_hash(path) + num_copy) % hosts_size_;
      78             : }
      79             : 
      80             : ::vector<host_t>
      81           0 : SimpleHashDistributor::locate_directory_metadata(const string& path) const {
      82           0 :     return all_hosts_;
      83             : }
      84             : 
      85           0 : LocalOnlyDistributor::LocalOnlyDistributor(host_t localhost)
      86           0 :     : localhost_(localhost) {}
      87             : 
      88             : host_t
      89           0 : LocalOnlyDistributor::localhost() const {
      90           0 :     return localhost_;
      91             : }
      92             : 
      93             : unsigned int
      94           0 : LocalOnlyDistributor::hosts_size() const {
      95           0 :     return hosts_size_;
      96             : }
      97             : 
      98             : host_t
      99           0 : LocalOnlyDistributor::locate_data(const string& path, const chunkid_t& chnk_id,
     100             :                                   const int num_copy) const {
     101           0 :     return localhost_;
     102             : }
     103             : 
     104             : host_t
     105           0 : LocalOnlyDistributor::locate_file_metadata(const string& path,
     106             :                                            const int num_copy) const {
     107           0 :     return localhost_;
     108             : }
     109             : 
     110             : ::vector<host_t>
     111           0 : LocalOnlyDistributor::locate_directory_metadata(const string& path) const {
     112           0 :     return {localhost_};
     113             : }
     114             : 
     115          21 : ForwarderDistributor::ForwarderDistributor(host_t fwhost,
     116          21 :                                            unsigned int hosts_size)
     117          21 :     : fwd_host_(fwhost), hosts_size_(hosts_size), all_hosts_(hosts_size) {
     118          21 :     ::iota(all_hosts_.begin(), all_hosts_.end(), 0);
     119          21 : }
     120             : 
     121             : host_t
     122           0 : ForwarderDistributor::localhost() const {
     123           0 :     return fwd_host_;
     124             : }
     125             : 
     126             : unsigned int
     127           0 : ForwarderDistributor::hosts_size() const {
     128           0 :     return hosts_size_;
     129             : }
     130             : 
     131             : host_t
     132           9 : ForwarderDistributor::locate_data(const std::string& path,
     133             :                                   const chunkid_t& chnk_id,
     134             :                                   const int num_copy) const {
     135           9 :     return fwd_host_;
     136             : }
     137             : 
     138             : host_t
     139           0 : ForwarderDistributor::locate_data(const std::string& path,
     140             :                                   const chunkid_t& chnk_id,
     141             :                                   unsigned int host_size, const int num_copy) {
     142           0 :     return fwd_host_;
     143             : }
     144             : 
     145             : host_t
     146          40 : ForwarderDistributor::locate_file_metadata(const std::string& path,
     147             :                                            const int num_copy) const {
     148          40 :     return (str_hash(path) + num_copy) % hosts_size_;
     149             : }
     150             : 
     151             : 
     152             : std::vector<host_t>
     153           3 : ForwarderDistributor::locate_directory_metadata(const std::string& path) const {
     154           3 :     return all_hosts_;
     155             : }
     156             : 
     157             : void
     158          11 : IntervalSet::Add(chunkid_t smaller, chunkid_t bigger) {
     159          11 :     const auto next = _intervals.upper_bound(smaller);
     160          11 :     if(next != _intervals.cbegin()) {
     161           4 :         const auto prev = std::prev(next);
     162           4 :         if(next != _intervals.cend() && next->first <= bigger + 1) {
     163           0 :             bigger = next->second;
     164           0 :             _intervals.erase(next);
     165             :         }
     166           4 :         if(prev->second + 1 >= smaller) {
     167           4 :             smaller = prev->first;
     168           4 :             _intervals.erase(prev);
     169             :         }
     170             :     }
     171          11 :     _intervals[smaller] = bigger;
     172          11 : }
     173             : 
     174             : bool
     175          18 : IntervalSet::IsInsideInterval(unsigned int v) const {
     176          18 :     const auto suspectNext = _intervals.upper_bound(v);
     177          18 :     const auto suspect = std::prev(suspectNext);
     178          18 :     return suspect->first <= v && v <= suspect->second;
     179             : }
     180             : 
     181             : bool
     182         249 : GuidedDistributor::init_guided() {
     183         249 :     unsigned int destination_host;
     184         249 :     chunkid_t chunk_id;
     185         498 :     string path;
     186         498 :     std::ifstream mapfile;
     187         249 :     mapfile.open(GKFS_USE_GUIDED_DISTRIBUTION_PATH);
     188         249 :     if((mapfile.rdstate() & std::ifstream::failbit) != 0)
     189             :         return false; // If it fails, the mapping will be as the SimpleHash
     190             : 
     191          12 :     while(mapfile >> path >> chunk_id >> destination_host) {
     192             :         // We need destination+1, as 0 has an special meaning in the interval
     193             :         // map.
     194          11 :         if(path.size() > 0 and path[0] == '#') {
     195             :             // Path that has this prefix will have metadata and data in the same
     196             :             // place  i.e. #/mdtest-hard/ 10 10 chunk_id and destination_host
     197             :             // are not used
     198           0 :             prefix_list.emplace_back(path.substr(1, path.size()));
     199           0 :             continue;
     200             :         }
     201             : 
     202          11 :         auto I = map_interval.find(path);
     203          11 :         if(I == map_interval.end()) {
     204          14 :             auto tmp = IntervalSet();
     205           7 :             tmp.Add(chunk_id, chunk_id + 1);
     206           7 :             map_interval[path] = make_pair(tmp, destination_host + 1);
     207           4 :         } else if(I->second.first.IsInsideInterval(chunk_id)) {
     208           8 :             auto is = I->second.first;
     209           4 :             is.Add(chunk_id, chunk_id + 1);
     210           4 :             I->second = (make_pair(is, destination_host + 1));
     211             :         }
     212             :     }
     213           1 :     mapfile.close();
     214             :     return true;
     215             : }
     216             : 
     217           1 : GuidedDistributor::GuidedDistributor() {
     218           1 :     init_guided();
     219           1 : }
     220             : 
     221         248 : GuidedDistributor::GuidedDistributor(host_t localhost,
     222         248 :                                      unsigned int hosts_size) {
     223         248 :     if(hosts_size_ != hosts_size) {
     224         248 :         hosts_size_ = hosts_size;
     225         248 :         localhost_ = localhost;
     226         248 :         all_hosts_ = std::vector<unsigned int>(hosts_size);
     227         248 :         ::iota(all_hosts_.begin(), all_hosts_.end(), 0);
     228             :     }
     229         248 :     init_guided();
     230         248 : }
     231             : 
     232             : host_t
     233           0 : GuidedDistributor::localhost() const {
     234           0 :     return localhost_;
     235             : }
     236             : 
     237             : unsigned int
     238           0 : GuidedDistributor::hosts_size() const {
     239           0 :     return hosts_size_;
     240             : }
     241             : 
     242             : host_t
     243          14 : GuidedDistributor::locate_data(const string& path, const chunkid_t& chnk_id,
     244             :                                unsigned int hosts_size, const int num_copy) {
     245          14 :     if(hosts_size_ != hosts_size) {
     246           1 :         hosts_size_ = hosts_size;
     247           1 :         all_hosts_ = std::vector<unsigned int>(hosts_size);
     248           1 :         ::iota(all_hosts_.begin(), all_hosts_.end(), 0);
     249             :     }
     250             : 
     251          14 :     return (locate_data(path, chnk_id, num_copy));
     252             : }
     253             : 
     254             : host_t
     255         205 : GuidedDistributor::locate_data(const string& path, const chunkid_t& chnk_id,
     256             :                                const int num_copy) const {
     257         205 :     auto it = map_interval.find(path);
     258         205 :     if(it != map_interval.end()) {
     259          14 :         auto it_f = it->second.first.IsInsideInterval(chnk_id);
     260          14 :         if(it_f) {
     261           7 :             return (it->second.second -
     262           7 :                     1); // Decrement destination host from the interval_map
     263             :         }
     264             :     }
     265             : 
     266         198 :     for(auto const& it : prefix_list) {
     267           0 :         if(0 == path.compare(0, min(it.length(), path.length()), it, 0,
     268           0 :                              min(it.length(), path.length()))) {
     269             :         }
     270           0 :         return str_hash(path) % hosts_size_;
     271             :     }
     272             : 
     273         403 :     auto locate = path + ::to_string(chnk_id);
     274         198 :     return (str_hash(locate) + num_copy) % hosts_size_;
     275             : }
     276             : 
     277             : host_t
     278        2496 : GuidedDistributor::locate_file_metadata(const string& path,
     279             :                                         const int num_copy) const {
     280        2496 :     return (str_hash(path) + num_copy) % hosts_size_;
     281             : }
     282             : 
     283             : 
     284             : ::vector<host_t>
     285          29 : GuidedDistributor::locate_directory_metadata(const string& path) const {
     286          29 :     return all_hosts_;
     287             : }
     288             : 
     289             : } // namespace rpc
     290             : } // namespace gkfs

Generated by: LCOV version 1.16