Commit ce0df018 authored by Ramon Nou's avatar Ramon Nou
Browse files

Added directory reading for FS Plugins, still bug with gekkofs writting, debugging

parent c47cd466

src/master.cpp

+10 −10
Original line number Diff line number Diff line
@@ -66,7 +66,7 @@ make_message(std::uint64_t tid, std::uint32_t seqno, 
    }



    return std::make_tuple(

            static_cast<int>(cargo::tag::seq_mixed),

            static_cast<int>(cargo::tag::sequential),

            cargo::transfer_message{tid, seqno, input.path(),

                                    static_cast<uint32_t>(input.get_type()),

                                    output.path(),
@@ -290,15 +290,15 @@ master_server::transfer_datasets(const network::request& req, 
        // bbb/xxx -> ttt/xxx

        const auto& p = s.path();



        std::vector<std::filesystem::path> files;

        if(std::filesystem::is_directory(p)) {

        std::vector<std::string> files;

        // Check stat of p using FSPlugin class      

        auto fs = FSPlugin::make_fs(static_cast<cargo::FSPlugin::type>(s.get_type()));

        struct stat buf;

        fs->stat(p, &buf);

        if(buf.st_mode & S_IFDIR) {

            LOGGER_INFO("Expanding input directory {}", p);

            for(const auto& f :

                std::filesystem::recursive_directory_iterator(p)) {

                if(std::filesystem::is_regular_file(f)) {

                    files.push_back(f.path());

                }

            }

            files = fs->readdir(p);

            



            /*

            We have all the files expanded. Now create a new
@@ -319,7 +319,7 @@ master_server::transfer_datasets(const network::request& req, 
                }



                d_new.path(d.path() / std::filesystem::path(

                                              f.string().substr(leading + 1)));

                                              f.substr(leading + 1)));



                LOGGER_DEBUG("Expanded file {} -> {}", s_new.path(),

                             d_new.path());

src/posix_file/posix_file/fs_plugin/fs_plugin.cpp

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ FSPlugin::make_fs(type t) { 


    switch(t) {

        case type::posix:

        case type::parallel:

            return std::make_unique<cargo::posix_plugin>();

#ifdef GEKKOFS_PLUGIN

        case type::gekkofs:

src/posix_file/posix_file/fs_plugin/fs_plugin.hpp

+7 −0
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 
#define FS_PLUGIN_HPP



#include <string>

#include <vector>

#include <filesystem>

#include <utility>

#include <fcntl.h>
@@ -36,6 +37,12 @@ public: 
    lseek(int fd, off_t offset, int whence) = 0;

    virtual off_t

    fallocate(int fd, int mode, off_t offset, off_t len) = 0;

    virtual std::vector<std::string>

    readdir(const std::string& path) = 0;

    virtual int

    unlink(const std::string& path) = 0;

    virtual int

    stat(const std::string& path, struct stat* buf) = 0;

};

} // namespace cargo

#endif // FS_PLUGIN_HPP
 
 No newline at end of file

src/posix_file/posix_file/fs_plugin/gekko_plugin.cpp

+39 −3
Original line number Diff line number Diff line
@@ -62,4 +62,40 @@ gekko_plugin::fallocate(int fd, int mode, off_t offset, off_t len) { 
    (void) len;

    return len;

}



int

gekko_plugin::unlink(const std::string& path) {

    return gkfs::syscall::gkfs_remove(path);

}





std::vector<std::string>

gekko_plugin::readdir(const std::string& path) {

    // Fill recursively the files, checking if the file is a directory

    std::vector<std::string> files;

    std::vector<std::string> final_list;

    files = gkfs::syscall::gkfs_get_file_list(path);



    for(auto& file : files) {

        file = "/" + file;

        struct stat buf;

        stat(file, &buf);

        if(S_ISDIR(buf.st_mode)) {

            std::vector<std::string> subfiles = readdir(file);

            final_list.insert(final_list.end(), subfiles.begin(),

                              subfiles.end());

        } else {

            final_list.push_back(file);

        }

    }

    return final_list;

}



// stat

int

gekko_plugin::stat(const std::string& path, struct stat* buf) {

    return gkfs::syscall::gkfs_stat(path, buf);

}





} // namespace cargo

src/posix_file/posix_file/fs_plugin/gekko_plugin.hpp

+6 −0
Original line number Diff line number Diff line
@@ -25,6 +25,12 @@ public: 
    // Fallocate is not needed in GekkoFS as pwrite takes care of it.

    off_t

    fallocate(int fd, int mode, off_t offset, off_t len) final;

    std::vector<std::string>

    readdir(const std::string& path) final;

    int

    unlink(const std::string& path) final;

    int

    stat(const std::string& path, struct stat* buf) final;

};

}; // namespace cargo