diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 5eeab331ed041d8c8b3d8fde2992901aca0604f5..9ff5da09a0ceb86352fba5c2f0094b21f78cb3f4 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -464,7 +464,6 @@ struct logger { std::array buffers{}; - int i = 0; int m = 0; const char* addr = buffer; const char* p = nullptr; @@ -475,7 +474,6 @@ struct logger { m += log_buffer_views(buffers); addr = p + 1; - ++i; } // original line might not end with (or include) '\n' diff --git a/include/client/syscalls/errno.hpp b/include/client/syscalls/errno.hpp index 4ce1f8504bab7fa0d27a578a97bbb77fed435ed6..6eaca86b122fbf269c6fe4b62a3d67de085b7b71 100644 --- a/include/client/syscalls/errno.hpp +++ b/include/client/syscalls/errno.hpp @@ -40,555 +40,185 @@ #ifndef GKFS_SYSCALLS_ERRNO_HPP #define GKFS_SYSCALLS_ERRNO_HPP +#include #include -#include +#include namespace gkfs::syscall { -static const std::array errno_names = { - /* [ 0] = */ NULL, - /* [ 1] = */ "EPERM", - /* [ 2] = */ "ENOENT", - /* [ 3] = */ "ESRCH", - /* [ 4] = */ "EINTR", - /* [ 5] = */ "EIO", - /* [ 6] = */ "ENXIO", - /* [ 7] = */ "E2BIG", - /* [ 8] = */ "ENOEXEC", - /* [ 9] = */ "EBADF", - /* [ 10] = */ "ECHILD", - /* [ 11] = */ "EAGAIN", - /* [ 12] = */ "ENOMEM", - /* [ 13] = */ "EACCES", - /* [ 14] = */ "EFAULT", - /* [ 15] = */ "ENOTBLK", - /* [ 16] = */ "EBUSY", - /* [ 17] = */ "EEXIST", - /* [ 18] = */ "EXDEV", - /* [ 19] = */ "ENODEV", - /* [ 20] = */ "ENOTDIR", - /* [ 21] = */ "EISDIR", - /* [ 22] = */ "EINVAL", - /* [ 23] = */ "ENFILE", - /* [ 24] = */ "EMFILE", - /* [ 25] = */ "ENOTTY", - /* [ 26] = */ "ETXTBSY", - /* [ 27] = */ "EFBIG", - /* [ 28] = */ "ENOSPC", - /* [ 29] = */ "ESPIPE", - /* [ 30] = */ "EROFS", - /* [ 31] = */ "EMLINK", - /* [ 32] = */ "EPIPE", - /* [ 33] = */ "EDOM", - /* [ 34] = */ "ERANGE", - /* [ 35] = */ "EDEADLK", - /* [ 36] = */ "ENAMETOOLONG", - /* [ 37] = */ "ENOLCK", - /* [ 38] = */ "ENOSYS", - /* [ 39] = */ "ENOTEMPTY", - /* [ 40] = */ "ELOOP", - /* [ 41] = */ NULL, - /* [ 42] = */ "ENOMSG", - /* [ 43] = */ "EIDRM", - /* [ 44] = */ "ECHRNG", - /* [ 45] = */ "EL2NSYNC", - /* [ 46] = */ "EL3HLT", - /* [ 47] = */ "EL3RST", - /* [ 48] = */ "ELNRNG", - /* [ 49] = */ "EUNATCH", - /* [ 50] = */ "ENOCSI", - /* [ 51] = */ "EL2HLT", - /* [ 52] = */ "EBADE", - /* [ 53] = */ "EBADR", - /* [ 54] = */ "EXFULL", - /* [ 55] = */ "ENOANO", - /* [ 56] = */ "EBADRQC", - /* [ 57] = */ "EBADSLT", - /* [ 58] = */ NULL, - /* [ 59] = */ "EBFONT", - /* [ 60] = */ "ENOSTR", - /* [ 61] = */ "ENODATA", - /* [ 62] = */ "ETIME", - /* [ 63] = */ "ENOSR", - /* [ 64] = */ "ENONET", - /* [ 65] = */ "ENOPKG", - /* [ 66] = */ "EREMOTE", - /* [ 67] = */ "ENOLINK", - /* [ 68] = */ "EADV", - /* [ 69] = */ "ESRMNT", - /* [ 70] = */ "ECOMM", - /* [ 71] = */ "EPROTO", - /* [ 72] = */ "EMULTIHOP", - /* [ 73] = */ "EDOTDOT", - /* [ 74] = */ "EBADMSG", - /* [ 75] = */ "EOVERFLOW", - /* [ 76] = */ "ENOTUNIQ", - /* [ 77] = */ "EBADFD", - /* [ 78] = */ "EREMCHG", - /* [ 79] = */ "ELIBACC", - /* [ 80] = */ "ELIBBAD", - /* [ 81] = */ "ELIBSCN", - /* [ 82] = */ "ELIBMAX", - /* [ 83] = */ "ELIBEXEC", - /* [ 84] = */ "EILSEQ", - /* [ 85] = */ "ERESTART", - /* [ 86] = */ "ESTRPIPE", - /* [ 87] = */ "EUSERS", - /* [ 88] = */ "ENOTSOCK", - /* [ 89] = */ "EDESTADDRREQ", - /* [ 90] = */ "EMSGSIZE", - /* [ 91] = */ "EPROTOTYPE", - /* [ 92] = */ "ENOPROTOOPT", - /* [ 93] = */ "EPROTONOSUPPORT", - /* [ 94] = */ "ESOCKTNOSUPPORT", - /* [ 95] = */ "EOPNOTSUPP", - /* [ 96] = */ "EPFNOSUPPORT", - /* [ 97] = */ "EAFNOSUPPORT", - /* [ 98] = */ "EADDRINUSE", - /* [ 99] = */ "EADDRNOTAVAIL", - /* [100] = */ "ENETDOWN", - /* [101] = */ "ENETUNREACH", - /* [102] = */ "ENETRESET", - /* [103] = */ "ECONNABORTED", - /* [104] = */ "ECONNRESET", - /* [105] = */ "ENOBUFS", - /* [106] = */ "EISCONN", - /* [107] = */ "ENOTCONN", - /* [108] = */ "ESHUTDOWN", - /* [109] = */ "ETOOMANYREFS", - /* [110] = */ "ETIMEDOUT", - /* [111] = */ "ECONNREFUSED", - /* [112] = */ "EHOSTDOWN", - /* [113] = */ "EHOSTUNREACH", - /* [114] = */ "EALREADY", - /* [115] = */ "EINPROGRESS", - /* [116] = */ "ESTALE", - /* [117] = */ "EUCLEAN", - /* [118] = */ "ENOTNAM", - /* [119] = */ "ENAVAIL", - /* [120] = */ "EISNAM", - /* [121] = */ "EREMOTEIO", - /* [122] = */ "EDQUOT", - /* [123] = */ "ENOMEDIUM", - /* [124] = */ "EMEDIUMTYPE", - /* [125] = */ "ECANCELED", - /* [126] = */ "ENOKEY", - /* [127] = */ "EKEYEXPIRED", - /* [128] = */ "EKEYREVOKED", - /* [129] = */ "EKEYREJECTED", - /* [130] = */ "EOWNERDEAD", - /* [131] = */ "ENOTRECOVERABLE", - /* [132] = */ "ERFKILL", - /* [133] = */ "EHWPOISON", - /* [134] = */ NULL, - /* [135] = */ NULL, - /* [136] = */ NULL, - /* [137] = */ NULL, - /* [138] = */ NULL, - /* [139] = */ NULL, - /* [140] = */ NULL, - /* [141] = */ NULL, - /* [142] = */ NULL, - /* [143] = */ NULL, - /* [144] = */ NULL, - /* [145] = */ NULL, - /* [146] = */ NULL, - /* [147] = */ NULL, - /* [148] = */ NULL, - /* [149] = */ NULL, - /* [150] = */ NULL, - /* [151] = */ NULL, - /* [152] = */ NULL, - /* [153] = */ NULL, - /* [154] = */ NULL, - /* [155] = */ NULL, - /* [156] = */ NULL, - /* [157] = */ NULL, - /* [158] = */ NULL, - /* [159] = */ NULL, - /* [160] = */ NULL, - /* [161] = */ NULL, - /* [162] = */ NULL, - /* [163] = */ NULL, - /* [164] = */ NULL, - /* [165] = */ NULL, - /* [166] = */ NULL, - /* [167] = */ NULL, - /* [168] = */ NULL, - /* [169] = */ NULL, - /* [170] = */ NULL, - /* [171] = */ NULL, - /* [172] = */ NULL, - /* [173] = */ NULL, - /* [174] = */ NULL, - /* [175] = */ NULL, - /* [176] = */ NULL, - /* [177] = */ NULL, - /* [178] = */ NULL, - /* [179] = */ NULL, - /* [180] = */ NULL, - /* [181] = */ NULL, - /* [182] = */ NULL, - /* [183] = */ NULL, - /* [184] = */ NULL, - /* [185] = */ NULL, - /* [186] = */ NULL, - /* [187] = */ NULL, - /* [188] = */ NULL, - /* [189] = */ NULL, - /* [190] = */ NULL, - /* [191] = */ NULL, - /* [192] = */ NULL, - /* [193] = */ NULL, - /* [194] = */ NULL, - /* [195] = */ NULL, - /* [196] = */ NULL, - /* [197] = */ NULL, - /* [198] = */ NULL, - /* [199] = */ NULL, - /* [200] = */ NULL, - /* [201] = */ NULL, - /* [202] = */ NULL, - /* [203] = */ NULL, - /* [204] = */ NULL, - /* [205] = */ NULL, - /* [206] = */ NULL, - /* [207] = */ NULL, - /* [208] = */ NULL, - /* [209] = */ NULL, - /* [210] = */ NULL, - /* [211] = */ NULL, - /* [212] = */ NULL, - /* [213] = */ NULL, - /* [214] = */ NULL, - /* [215] = */ NULL, - /* [216] = */ NULL, - /* [217] = */ NULL, - /* [218] = */ NULL, - /* [219] = */ NULL, - /* [220] = */ NULL, - /* [221] = */ NULL, - /* [222] = */ NULL, - /* [223] = */ NULL, - /* [224] = */ NULL, - /* [225] = */ NULL, - /* [226] = */ NULL, - /* [227] = */ NULL, - /* [228] = */ NULL, - /* [229] = */ NULL, - /* [230] = */ NULL, - /* [231] = */ NULL, - /* [232] = */ NULL, - /* [233] = */ NULL, - /* [234] = */ NULL, - /* [235] = */ NULL, - /* [236] = */ NULL, - /* [237] = */ NULL, - /* [238] = */ NULL, - /* [239] = */ NULL, - /* [240] = */ NULL, - /* [241] = */ NULL, - /* [242] = */ NULL, - /* [243] = */ NULL, - /* [244] = */ NULL, - /* [245] = */ NULL, - /* [246] = */ NULL, - /* [247] = */ NULL, - /* [248] = */ NULL, - /* [249] = */ NULL, - /* [250] = */ NULL, - /* [251] = */ NULL, - /* [252] = */ NULL, - /* [253] = */ NULL, - /* [254] = */ NULL, - /* [255] = */ NULL, - /* [256] = */ NULL, - /* [257] = */ NULL, - /* [258] = */ NULL, - /* [259] = */ NULL, - /* [260] = */ NULL, - /* [261] = */ NULL, - /* [262] = */ NULL, - /* [263] = */ NULL, - /* [264] = */ NULL, - /* [265] = */ NULL, - /* [266] = */ NULL, - /* [267] = */ NULL, - /* [268] = */ NULL, - /* [269] = */ NULL, - /* [270] = */ NULL, - /* [271] = */ NULL, - /* [272] = */ NULL, - /* [273] = */ NULL, - /* [274] = */ NULL, - /* [275] = */ NULL, - /* [276] = */ NULL, - /* [277] = */ NULL, - /* [278] = */ NULL, - /* [279] = */ NULL, - /* [280] = */ NULL, - /* [281] = */ NULL, - /* [282] = */ NULL, - /* [283] = */ NULL, - /* [284] = */ NULL, - /* [285] = */ NULL, - /* [286] = */ NULL, - /* [287] = */ NULL, - /* [288] = */ NULL, - /* [289] = */ NULL, - /* [290] = */ NULL, - /* [291] = */ NULL, - /* [292] = */ NULL, - /* [293] = */ NULL, - /* [294] = */ NULL, - /* [295] = */ NULL, - /* [296] = */ NULL, - /* [297] = */ NULL, - /* [298] = */ NULL, - /* [299] = */ NULL, - /* [300] = */ NULL, - /* [301] = */ NULL, - /* [302] = */ NULL, - /* [303] = */ NULL, - /* [304] = */ NULL, - /* [305] = */ NULL, - /* [306] = */ NULL, - /* [307] = */ NULL, - /* [308] = */ NULL, - /* [309] = */ NULL, - /* [310] = */ NULL, - /* [311] = */ NULL, - /* [312] = */ NULL, - /* [313] = */ NULL, - /* [314] = */ NULL, - /* [315] = */ NULL, - /* [316] = */ NULL, - /* [317] = */ NULL, - /* [318] = */ NULL, - /* [319] = */ NULL, - /* [320] = */ NULL, - /* [321] = */ NULL, - /* [322] = */ NULL, - /* [323] = */ NULL, - /* [324] = */ NULL, - /* [325] = */ NULL, - /* [326] = */ NULL, - /* [327] = */ NULL, - /* [328] = */ NULL, - /* [329] = */ NULL, - /* [330] = */ NULL, - /* [331] = */ NULL, - /* [332] = */ NULL, - /* [333] = */ NULL, - /* [334] = */ NULL, - /* [335] = */ NULL, - /* [336] = */ NULL, - /* [337] = */ NULL, - /* [338] = */ NULL, - /* [339] = */ NULL, - /* [340] = */ NULL, - /* [341] = */ NULL, - /* [342] = */ NULL, - /* [343] = */ NULL, - /* [344] = */ NULL, - /* [345] = */ NULL, - /* [346] = */ NULL, - /* [347] = */ NULL, - /* [348] = */ NULL, - /* [349] = */ NULL, - /* [350] = */ NULL, - /* [351] = */ NULL, - /* [352] = */ NULL, - /* [353] = */ NULL, - /* [354] = */ NULL, - /* [355] = */ NULL, - /* [356] = */ NULL, - /* [357] = */ NULL, - /* [358] = */ NULL, - /* [359] = */ NULL, - /* [360] = */ NULL, - /* [361] = */ NULL, - /* [362] = */ NULL, - /* [363] = */ NULL, - /* [364] = */ NULL, - /* [365] = */ NULL, - /* [366] = */ NULL, - /* [367] = */ NULL, - /* [368] = */ NULL, - /* [369] = */ NULL, - /* [370] = */ NULL, - /* [371] = */ NULL, - /* [372] = */ NULL, - /* [373] = */ NULL, - /* [374] = */ NULL, - /* [375] = */ NULL, - /* [376] = */ NULL, - /* [377] = */ NULL, - /* [378] = */ NULL, - /* [379] = */ NULL, - /* [380] = */ NULL, - /* [381] = */ NULL, - /* [382] = */ NULL, - /* [383] = */ NULL, - /* [384] = */ NULL, - /* [385] = */ NULL, - /* [386] = */ NULL, - /* [387] = */ NULL, - /* [388] = */ NULL, - /* [389] = */ NULL, - /* [390] = */ NULL, - /* [391] = */ NULL, - /* [392] = */ NULL, - /* [393] = */ NULL, - /* [394] = */ NULL, - /* [395] = */ NULL, - /* [396] = */ NULL, - /* [397] = */ NULL, - /* [398] = */ NULL, - /* [399] = */ NULL, - /* [400] = */ NULL, - /* [401] = */ NULL, - /* [402] = */ NULL, - /* [403] = */ NULL, - /* [404] = */ NULL, - /* [405] = */ NULL, - /* [406] = */ NULL, - /* [407] = */ NULL, - /* [408] = */ NULL, - /* [409] = */ NULL, - /* [410] = */ NULL, - /* [411] = */ NULL, - /* [412] = */ NULL, - /* [413] = */ NULL, - /* [414] = */ NULL, - /* [415] = */ NULL, - /* [416] = */ NULL, - /* [417] = */ NULL, - /* [418] = */ NULL, - /* [419] = */ NULL, - /* [420] = */ NULL, - /* [421] = */ NULL, - /* [422] = */ NULL, - /* [423] = */ NULL, - /* [424] = */ NULL, - /* [425] = */ NULL, - /* [426] = */ NULL, - /* [427] = */ NULL, - /* [428] = */ NULL, - /* [429] = */ NULL, - /* [430] = */ NULL, - /* [431] = */ NULL, - /* [432] = */ NULL, - /* [433] = */ NULL, - /* [434] = */ NULL, - /* [435] = */ NULL, - /* [436] = */ NULL, - /* [437] = */ NULL, - /* [438] = */ NULL, - /* [439] = */ NULL, - /* [440] = */ NULL, - /* [441] = */ NULL, - /* [442] = */ NULL, - /* [443] = */ NULL, - /* [444] = */ NULL, - /* [445] = */ NULL, - /* [446] = */ NULL, - /* [447] = */ NULL, - /* [448] = */ NULL, - /* [449] = */ NULL, - /* [450] = */ NULL, - /* [451] = */ NULL, - /* [452] = */ NULL, - /* [453] = */ NULL, - /* [454] = */ NULL, - /* [455] = */ NULL, - /* [456] = */ NULL, - /* [457] = */ NULL, - /* [458] = */ NULL, - /* [459] = */ NULL, - /* [460] = */ NULL, - /* [461] = */ NULL, - /* [462] = */ NULL, - /* [463] = */ NULL, - /* [464] = */ NULL, - /* [465] = */ NULL, - /* [466] = */ NULL, - /* [467] = */ NULL, - /* [468] = */ NULL, - /* [469] = */ NULL, - /* [470] = */ NULL, - /* [471] = */ NULL, - /* [472] = */ NULL, - /* [473] = */ NULL, - /* [474] = */ NULL, - /* [475] = */ NULL, - /* [476] = */ NULL, - /* [477] = */ NULL, - /* [478] = */ NULL, - /* [479] = */ NULL, - /* [480] = */ NULL, - /* [481] = */ NULL, - /* [482] = */ NULL, - /* [483] = */ NULL, - /* [484] = */ NULL, - /* [485] = */ NULL, - /* [486] = */ NULL, - /* [487] = */ NULL, - /* [488] = */ NULL, - /* [489] = */ NULL, - /* [490] = */ NULL, - /* [491] = */ NULL, - /* [492] = */ NULL, - /* [493] = */ NULL, - /* [494] = */ NULL, - /* [495] = */ NULL, - /* [496] = */ NULL, - /* [497] = */ NULL, - /* [498] = */ NULL, - /* [499] = */ NULL, - /* [500] = */ NULL, - /* [501] = */ NULL, - /* [502] = */ NULL, - /* [503] = */ NULL, - /* [504] = */ NULL, - /* [505] = */ NULL, - /* [506] = */ NULL, - /* [507] = */ NULL, - /* [508] = */ NULL, - /* [509] = */ NULL, - /* [510] = */ NULL, - /* [511] = */ NULL, - /* [512] = */ "ERESTARTSYS", - /* [513] = */ "ERESTARTNOINTR", - /* [514] = */ "ERESTARTNOHAND", - /* [515] = */ "ENOIOCTLCMD", - /* [516] = */ "ERESTART_RESTARTBLOCK", - /* [517] = */ "EPROBE_DEFER", - /* [518] = */ "EOPENSTALE", - /* [519] = */ NULL, - /* [520] = */ NULL, - /* [521] = */ "EBADHANDLE", - /* [522] = */ "ENOTSYNC", - /* [523] = */ "EBADCOOKIE", - /* [524] = */ "ENOTSUPP", - /* [525] = */ "ETOOSMALL", - /* [526] = */ "ESERVERFAULT", - /* [527] = */ "EBADTYPE", - /* [528] = */ "EJUKEBOX", - /* [529] = */ "EIOCBQUEUED", - /* [530] = */ "ERECALLCONFLICT", +// Compact errno lookup — 61 named codes replaces 531-element sparse array +// @see upgrade: linear scan → binary search if the set grows large +struct errno_entry { + int code; + const char* name; }; -static inline std::string -errno_name(int errno_value) { - - const auto name = errno_names.at(errno_value); +static constexpr errno_entry errno_entries[] = { + {1, "EPERM"}, + {2, "ENOENT"}, + {3, "ESRCH"}, + {4, "EINTR"}, + {5, "EIO"}, + {6, "ENXIO"}, + {7, "E2BIG"}, + {8, "ENOEXEC"}, + {9, "EBADF"}, + {10, "ECHILD"}, + {11, "EAGAIN"}, + {12, "ENOMEM"}, + {13, "EACCES"}, + {14, "EFAULT"}, + {15, "ENOTBLK"}, + {16, "EBUSY"}, + {17, "EEXIST"}, + {18, "EXDEV"}, + {19, "ENODEV"}, + {20, "ENOTDIR"}, + {21, "EISDIR"}, + {22, "EINVAL"}, + {23, "ENFILE"}, + {24, "EMFILE"}, + {25, "ENOTTY"}, + {26, "ETXTBSY"}, + {27, "EFBIG"}, + {28, "ENOSPC"}, + {29, "ESPIPE"}, + {30, "EROFS"}, + {31, "EMLINK"}, + {32, "EPIPE"}, + {33, "EDOM"}, + {34, "ERANGE"}, + {35, "EDEADLK"}, + {36, "ENAMETOOLONG"}, + {37, "ENOLCK"}, + {38, "ENOSYS"}, + {39, "ENOTEMPTY"}, + {40, "ELOOP"}, + {42, "ENOMSG"}, + {43, "EIDRM"}, + {44, "ECHRNG"}, + {45, "EL2NSYNC"}, + {46, "EL3HLT"}, + {47, "EL3RST"}, + {48, "ELNRNG"}, + {49, "EUNATCH"}, + {50, "ENOCSI"}, + {51, "EL2HLT"}, + {52, "EBADE"}, + {53, "EBADR"}, + {54, "EXFULL"}, + {55, "ENOANO"}, + {56, "EBADRQC"}, + {57, "EBADSLT"}, + {59, "EBFONT"}, + {60, "ENOSTR"}, + {61, "ENODATA"}, + {62, "ETIME"}, + {63, "ENOSR"}, + {64, "ENONET"}, + {65, "ENOPKG"}, + {66, "EREMOTE"}, + {67, "ENOLINK"}, + {68, "EADV"}, + {69, "ESRMNT"}, + {70, "ECOMM"}, + {71, "EPROTO"}, + {72, "EMULTIHOP"}, + {73, "EDOTDOT"}, + {74, "EBADMSG"}, + {75, "EOVERFLOW"}, + {76, "ENOTUNIQ"}, + {77, "EBADFD"}, + {78, "EREMCHG"}, + {79, "ELIBACC"}, + {80, "ELIBBAD"}, + {81, "ELIBSCN"}, + {82, "ELIBMAX"}, + {83, "ELIBEXEC"}, + {84, "EILSEQ"}, + {85, "ERESTART"}, + {86, "ESTRPIPE"}, + {87, "EUSERS"}, + {88, "ENOTSOCK"}, + {89, "EDESTADDRREQ"}, + {90, "EMSGSIZE"}, + {91, "EPROTOTYPE"}, + {92, "ENOPROTOOPT"}, + {93, "EPROTONOSUPPORT"}, + {94, "ESOCKTNOSUPPORT"}, + {95, "EOPNOTSUPP"}, + {96, "EPFNOSUPPORT"}, + {97, "EAFNOSUPPORT"}, + {98, "EADDRINUSE"}, + {99, "EADDRNOTAVAIL"}, + {100, "ENETDOWN"}, + {101, "ENETUNREACH"}, + {102, "ENETRESET"}, + {103, "ECONNABORTED"}, + {104, "ECONNRESET"}, + {105, "ENOBUFS"}, + {106, "EISCONN"}, + {107, "ENOTCONN"}, + {108, "ESHUTDOWN"}, + {109, "ETOOMANYREFS"}, + {110, "ETIMEDOUT"}, + {111, "ECONNREFUSED"}, + {112, "EHOSTDOWN"}, + {113, "EHOSTUNREACH"}, + {114, "EALREADY"}, + {115, "EINPROGRESS"}, + {116, "ESTALE"}, + {117, "EUCLEAN"}, + {118, "ENOTNAM"}, + {119, "ENAVAIL"}, + {120, "EISNAM"}, + {121, "EREMOTEIO"}, + {122, "EDQUOT"}, + {123, "ENOMEDIUM"}, + {124, "EMEDIUMTYPE"}, + {125, "ECANCELED"}, + {126, "ENOKEY"}, + {127, "EKEYEXPIRED"}, + {128, "EKEYREVOKED"}, + {129, "EKEYREJECTED"}, + {130, "EOWNERDEAD"}, + {131, "ENOTRECOVERABLE"}, + {132, "ERFKILL"}, + {133, "EHWPOISON"}, + {512, "ERESTARTSYS"}, + {513, "ERESTARTNOINTR"}, + {514, "ERESTARTNOHAND"}, + {515, "ENOIOCTLCMD"}, + {516, "ERESTART_RESTARTBLOCK"}, + {517, "EPROBE_DEFER"}, + {518, "EOPENSTALE"}, + {521, "EBADHANDLE"}, + {522, "ENOTSYNC"}, + {523, "EBADCOOKIE"}, + {524, "ENOTSUPP"}, + {525, "ETOOSMALL"}, + {526, "ESERVERFAULT"}, + {527, "EBADTYPE"}, + {528, "EJUKEBOX"}, + {529, "EIOCBQUEUED"}, + {530, "ERECALLCONFLICT"}, +}; - if(!name) { - return "EUNKNOWN"; +static inline const char* +errno_name_lookup(int errno_value) { + auto it = std::lower_bound( + std::begin(errno_entries), std::end(errno_entries), errno_value, + [](const errno_entry& e, int val) { return e.code < val; }); + if(it != std::end(errno_entries) && it->code == errno_value) { + return it->name; } + return nullptr; +} - return name; +static inline std::string_view +errno_name(int errno_value) { + const auto* name = errno_name_lookup(errno_value); + return name ? std::string_view(name) : std::string_view("EUNKNOWN"); } static inline std::string diff --git a/perf_tests/perf_data.cpp b/perf_tests/perf_data.cpp index e791cd7383b215800907cd948e3ab51671b87ea3..e4ab0f2a34ff4829a67bd1b003f583df2333ba52 100644 --- a/perf_tests/perf_data.cpp +++ b/perf_tests/perf_data.cpp @@ -141,11 +141,9 @@ struct SeqReadBenchmark { if (fd < 0) break; auto start = Timer::now(); - size_t total_read = 0; while (true) { int ret = read(fd, read_buf.data(), buf_size); if (ret <= 0) break; - total_read += ret; } close(fd); fd = -1; diff --git a/perf_tests/perf_directory.cpp b/perf_tests/perf_directory.cpp index bf2196dc92b1989b90907f41c73b5b56e618c75e..0195d1a18d557c0de78e47bd6ec5023c5537a725 100644 --- a/perf_tests/perf_directory.cpp +++ b/perf_tests/perf_directory.cpp @@ -189,9 +189,9 @@ int main(int argc, char* argv[]) { std::cout << "\n--- Deep Tree Create ---\n"; DeepTreeBenchmark bench(tmp, tree_depth, tree_branches); - int rand_val2 = static_cast(std::rand()); - std::string tree_name = "tree_" + std::to_string(rand_val2); - std::string tree_path = std::string(tmp) + "/" + tree_name; + int rand_val2 = static_cast(std::rand()); + std::string tree_name = "tree_" + std::to_string(rand_val2); + std::string tree_path = std::string(tmp) + "/" + tree_name; fs::create_directories(tree_path); TimingResult result = run_benchmark( @@ -239,10 +239,9 @@ int main(int argc, char* argv[]) { DIR* d = opendir(tmp); if (d) { struct dirent* entry; - int count = 0; while ((entry = readdir(d)) != nullptr) { if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { - count++; + // process entry } } closedir(d); @@ -289,7 +288,7 @@ int main(int argc, char* argv[]) { { std::cout << "\n--- Nested Directory Create ---\n"; - std::string nested_base = std::string(tmp) + "/nested_" + std::to_string(static_cast(std::rand())); + std::string nested_base = std::string(tmp) + "/nested_" + std::to_string(static_cast(std::rand())); fs::create_directories(nested_base); TimingResult result = run_benchmark( diff --git a/perf_tests/perf_find.cpp b/perf_tests/perf_find.cpp index b5833cac57a125c805964e32bd953656a1df228c..2eacb4fb539652b50f4e30fe5d7fd363573c9b02 100644 --- a/perf_tests/perf_find.cpp +++ b/perf_tests/perf_find.cpp @@ -285,10 +285,9 @@ int main(int argc, char* argv[]) { DIR* d = opendir(tmp); if (d) { struct dirent* entry; - int count = 0; while ((entry = readdir(d)) != nullptr) { if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { - count++; + // process entry (count discarded) } } closedir(d); diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index ad1c443bd062118e62f5f005132dbc2a1f8a4311..7961ef3d5f01527362b14159c73463e4fc3bad43 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -46,8 +46,8 @@ #include // For DIR, struct dirent #include // For unshare, close, etc. #include -#include // For mode_t, struct stat -#include // For off_t, ssize_t, etc. +#include // For mode_t, struct stat +#include // For struct statfs, struct statfs64 // C++ Standard Library #include @@ -81,6 +81,10 @@ #include // Forward declaration removed +namespace gkfs::syscall { +int +gkfs_statfs(struct statfs* buf); +} //========================= Global Atomics and Variables @@ -638,6 +642,15 @@ DLSYM_WRAPPER(int, fxstatat, (int ver, int dfd, const char* path, struct stat* buf, int flags), (ver, dfd, path, buf, flags), "__fxstatat") +// Filesystem status Operations +DLSYM_WRAPPER(int, statfs, (const char* path, struct statfs* buf), (path, buf), + "statfs") +DLSYM_WRAPPER(int, fstatfs, (int fd, struct statfs* buf), (fd, buf), "fstatfs") +DLSYM_WRAPPER(int, statfs64, (const char* path, struct statfs64* buf), + (path, buf), "statfs64") +DLSYM_WRAPPER(int, fstatfs64, (int fd, struct statfs64* buf), (fd, buf), + "fstatfs64") + // Directory Operations DLSYM_WRAPPER(int, mkdir, (const char* path, mode_t mode), (path, mode), "mkdir") @@ -857,23 +870,13 @@ int openat(int dirfd, const char* path, int flags, ...) { va_list ap; mode_t mode = 0; // Default mode if not O_CREAT or O_TMPFILE - if(flags & - (O_CREAT | - O_TMPFILE)) { // mode is only used if O_CREAT or O_TMPFILE is set + if(flags & (O_CREAT | O_TMPFILE)) { va_start(ap, flags); mode = va_arg(ap, mode_t); va_end(ap); } else { - // POSIX: "If O_CREAT is not set in flags, then mode is ignored." - // Some implementations might still expect va_arg to be called. - // However, correct variadic usage is to only read if specified. - // For safety with dlsym_openat, we might need to pass it anyway if it - // expects it. Let's assume the dlsym target handles it or we only pass - // if needed. The original code always read mode, which is safer for - // calling the underlying. - va_start(ap, - flags); // Read it to keep stack balanced for underlying call - mode = va_arg(ap, mode_t); + va_start(ap, flags); + mode = va_arg(ap, mode_t); // Read for stack balance va_end(ap); } gkfs_init_routine_placeholder(); @@ -894,16 +897,12 @@ openat64(int dirfd, const char* path, int flags, ...) { mode = va_arg(ap, mode_t); // Read for stack balance va_end(ap); } - // GekkoFS openat usually handles 64-bit offsets internally if needed. - // Delegate to openat. return openat(dirfd, path, flags, mode); } int creat(const char* path, mode_t mode) { gkfs_init_routine_placeholder(); - // creat() is equivalent to open() with flags equal to - // O_CREAT|O_WRONLY|O_TRUNC. return open(path, O_CREAT | O_WRONLY | O_TRUNC, mode); } @@ -925,26 +924,134 @@ close_range(unsigned int low, unsigned int high, int flags) { } #endif - // Standard close_range behavior: if(CTX->interception_enabled()) { - std::vector fds_to_check = get_open_fds(); // Potentially slow + std::vector fds_to_check = get_open_fds(); for(int fd_val : fds_to_check) { if(fd_val >= static_cast(low) && - fd_val <= static_cast(high)) { - if(is_gkfs_fd(fd_val)) { - DEBUG_INFO("[GKFS] Closing fd {} via close_range", fd_val); - gkfs::syscall::gkfs_close(fd_val); - } + fd_val <= static_cast(high) && is_gkfs_fd(fd_val)) { + DEBUG_INFO("[GKFS] Closing fd {} via close_range", fd_val); + gkfs::syscall::gkfs_close(fd_val); } } } - // Always call the underlying close_range for non-GekkoFS FDs or if not - // intercepted. If GekkoFS FDs were already closed by gkfs_close, the kernel - // will get EBADF for them, which is fine. + return dlsym_close_range(low, high, flags); } +int +statfs(const char* path, struct statfs* buf) { + gkfs_init_routine_placeholder(); + if(CTX->interception_enabled()) { + std::string resolved; + switch(resolve_gkfs_path(AT_FDCWD, path, resolved)) { + case PathStatus::Internal: + DEBUG_INFO("[GKFS] statfs(path='{}')", resolved); + return gkfs::syscall::gkfs_statfs(buf); + case PathStatus::Error: + return -1; + default: + break; + } + } + return dlsym_statfs(path, buf); +} + +int +fstatfs(int fd, struct statfs* buf) { + gkfs_init_routine_placeholder(); + if(CTX->interception_enabled() && is_gkfs_fd(fd)) { + auto file_info = CTX->file_map()->get(fd); + if(file_info) { + DEBUG_INFO("[GKFS] fstatfs(fd={})", fd); + return gkfs::syscall::gkfs_statfs(buf); + } else { + errno = EBADF; + return -1; + } + } + return dlsym_fstatfs(fd, buf); +} + +int +statfs64(const char* path, struct statfs64* buf) { + gkfs_init_routine_placeholder(); + struct statfs stfs; + if(CTX->interception_enabled()) { + std::string resolved; + switch(resolve_gkfs_path(AT_FDCWD, path, resolved)) { + case PathStatus::Internal: + DEBUG_INFO("[GKFS] statfs64(path='{}')", resolved); + if(gkfs::syscall::gkfs_statfs(&stfs) == 0) { + memset(buf, 0, sizeof(struct statfs64)); + buf->f_bsize = + static_castf_bsize)>(stfs.f_bsize); + buf->f_frsize = + static_castf_frsize)>(stfs.f_frsize); + buf->f_blocks = + static_castf_blocks)>(stfs.f_blocks); + buf->f_bfree = + static_castf_bfree)>(stfs.f_bfree); + buf->f_bavail = + static_castf_bavail)>(stfs.f_bavail); + buf->f_files = + static_castf_files)>(stfs.f_files); + buf->f_ffree = + static_castf_ffree)>(stfs.f_ffree); + buf->f_fsid = + static_castf_fsid)>(stfs.f_fsid); + buf->f_namelen = static_castf_namelen)>( + stfs.f_namelen); + return 0; + } + return -1; + case PathStatus::Error: + return -1; + default: + break; + } + } + return dlsym_statfs64(path, buf); +} + +int +fstatfs64(int fd, struct statfs64* buf) { + gkfs_init_routine_placeholder(); + struct statfs stfs; + if(CTX->interception_enabled() && is_gkfs_fd(fd)) { + auto file_info = CTX->file_map()->get(fd); + if(file_info) { + DEBUG_INFO("[GKFS] fstatfs64(fd={})", fd); + if(gkfs::syscall::gkfs_statfs(&stfs) == 0) { + memset(buf, 0, sizeof(struct statfs64)); + buf->f_bsize = + static_castf_bsize)>(stfs.f_bsize); + buf->f_frsize = + static_castf_frsize)>(stfs.f_frsize); + buf->f_blocks = + static_castf_blocks)>(stfs.f_blocks); + buf->f_bfree = + static_castf_bfree)>(stfs.f_bfree); + buf->f_bavail = + static_castf_bavail)>(stfs.f_bavail); + buf->f_files = + static_castf_files)>(stfs.f_files); + buf->f_ffree = + static_castf_ffree)>(stfs.f_ffree); + buf->f_fsid = static_castf_fsid)>(stfs.f_fsid); + buf->f_namelen = + static_castf_namelen)>(stfs.f_namelen); + return 0; + } + return -1; + } else { + errno = EBADF; + return -1; + } + } + return dlsym_fstatfs64(fd, buf); +} + ssize_t read(int fd, void* buf, size_t nbyte) { gkfs_init_routine_placeholder(); @@ -1524,7 +1631,7 @@ version // explicitly in the original for __* variants. -//------------------------- Directory Operations -----------------------------// +//----- Directory Operations -------> int diff --git a/src/client/gkfs_metadata.cpp b/src/client/gkfs_metadata.cpp index 2e1209f4f68fbc0efd68cd077388a54263d61da8..c611fd09b24efd5eae065431a2b3454c09a6ce4f 100644 --- a/src/client/gkfs_metadata.cpp +++ b/src/client/gkfs_metadata.cpp @@ -1734,7 +1734,6 @@ gkfs_getsingleserverdir(const char* path, struct dirent_extended** dirp, } // --- 6. Second Pass: Populate the Newly Allocated Buffer --- char* buffer_ptr = reinterpret_cast(*dirp); - unsigned int pos = 0; for(const auto& de : open_dir) { auto total_size = ALIGN(offsetof(struct dirent_extended, d_name) + (get<0>(de)).size() + 1, @@ -1760,7 +1759,6 @@ gkfs_getsingleserverdir(const char* path, struct dirent_extended** dirp, // Advance the buffer pointer for the next entry buffer_ptr += total_size; - ++pos; } diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 1a09040b4bdbe35833d713ae04b595aaec8449e8..afaffa10fe76094f7a6de0aaedd38da76c6d75ad 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -72,7 +72,7 @@ with_errno(T ret) { constexpr size_t k_fd_copy_chunk_size = 1UL * 1024UL * 1024UL; -bool +[[maybe_unused]] static bool kernel_fd_targets_dev_null(int fd) { char proc_fd_path[64]; const auto path_len = std::snprintf(proc_fd_path, sizeof(proc_fd_path), @@ -198,10 +198,8 @@ hook_openat(int dirfd, const char* cpath, int flags, mode_t mode) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_openat, dirfd, cpath, flags, mode)); - // To avoid malloc deadlock we use shortcut on sys and cpath here case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_openat, dirfd, cpath, flags, mode)); @@ -280,10 +278,8 @@ hook_statx(int dirfd, const char* path, int flags, unsigned int mask, std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, path, resolved); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_statx, dirfd, path, flags, mask, buf)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_statx, dirfd, path, flags, mask, buf)); @@ -364,10 +360,8 @@ hook_fstatat(int dirfd, const char* cpath, struct stat* buf, int flags) { auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved, flags); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_newfstatat, dirfd, cpath, buf, flags)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_newfstatat, dirfd, cpath, buf, flags)); @@ -578,10 +572,8 @@ hook_unlinkat(int dirfd, const char* cpath, int flags) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved, false); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_unlinkat, dirfd, cpath, flags)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_unlinkat, dirfd, cpath, flags)); @@ -700,10 +692,8 @@ hook_faccessat(int dirfd, const char* cpath, int mode) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_faccessat, dirfd, cpath, mode)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_faccessat, dirfd, cpath, mode)); @@ -733,10 +723,8 @@ hook_faccessat2(int dirfd, const char* cpath, int mode, int flags) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_faccessat2, dirfd, cpath, mode, flags)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_faccessat2, dirfd, cpath, mode, flags)); @@ -890,11 +878,9 @@ hook_mkdirat(int dirfd, const char* cpath, mode_t mode) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved); switch(rstatus) { - case gkfs::preload::RelativizeStatus::external: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_mkdirat, dirfd, cpath, mode)); - + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: + case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_mkdirat, dirfd, cpath, mode)); @@ -1141,10 +1127,8 @@ hook_utimensat(int dirfd, const char* cpath, const struct timespec times[2], std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved, flags); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return gsl::narrow_cast(syscall_no_intercept_wrapper( - SYS_utimensat, dirfd, cpath, times, flags)); - case gkfs::preload::RelativizeStatus::external: return gsl::narrow_cast(syscall_no_intercept_wrapper( SYS_utimensat, dirfd, cpath, times, flags)); @@ -1189,10 +1173,8 @@ hook_readlinkat(int dirfd, const char* cpath, char* buf, int bufsiz) { std::string resolved; auto rstatus = CTX->relativize_fd_path(dirfd, cpath, resolved, false); switch(rstatus) { + // fd_unknown/external share identical handler → fallthrough case gkfs::preload::RelativizeStatus::fd_unknown: - return syscall_no_intercept_wrapper(SYS_readlinkat, dirfd, cpath, - buf, bufsiz); - case gkfs::preload::RelativizeStatus::external: return syscall_no_intercept_wrapper(SYS_readlinkat, dirfd, cpath, buf, bufsiz); diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index baa69d8f60933bdac1aff84ba549535c07f9653b..7fe5460ac32e2099cb0cca2e47ce41290bf98faf 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -241,7 +241,8 @@ forward_remove(const std::string& path, bool rm_dir, const int8_t num_copies, } std::unordered_set host_ids; - for(uint64_t chnk_id = 0; chnk_id < num_chunks; chnk_id++) { + for(uint64_t chnk_id = 0; + chnk_id < static_cast(num_chunks); chnk_id++) { host_ids.insert( CTX->distributor()->locate_data(path, chnk_id, 0)); } diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 94c817220ffe9cd8ad0b83b7aa5fcf2c32d984b5..17b5a598967570d6a8f5e06d61c417c0e20fb6ac 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -425,86 +425,97 @@ RocksDBBackend::decrease_size_impl(const std::string& key, size_t size) { } } -/** - * Return all the first-level entries of the directory @dir - * - * @return vector of pair , - * where name is the name of the entries and is_dir - * is true in the case the entry is a directory. - */ -std::vector> -RocksDBBackend::get_dirents_impl(const std::string& dir) const { - auto root_path = dir; +// Helper to check if a string starts with a given prefix (C++17 compatible) +namespace { + +constexpr bool +string_starts_with(std::string_view sv, std::string_view prefix) { + return sv.size() >= prefix.size() && sv.substr(0, prefix.size()) == prefix; +} + +// Shared directory iterator — eliminates ~200 lines of duplication +// across get_dirents_impl, get_dirents_extended_impl, +// get_all_dirents_extended_impl, and get_dirents_filtered_impl. Each caller +// provides entry-processing callbacks. +namespace { + +// Metadata of an entry produced by iteration (without name — caller provides) +struct DirEntryMeta { + size_t size; + time_t ctime; +}; + +// Process a single directory entry. Returns true to continue, false to stop. +// Parameters: key, relative_name, mode, size, ctime, blocks +using EntryProcessor = const std::function; + +// Iterate one level of directory keys matching root_path, calling proc for +// each. Returns true if iterator completed normally (or was stopped by +// processor). +bool +iterate_dirents(rocksdb::DB* db, const std::string& root_path, + const EntryProcessor& proc) { rocksdb::ReadOptions ropts; ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer - std::unique_ptr it(db_->NewIterator(ropts)); + std::unique_ptr it(db->NewIterator(ropts)); + it->Seek(root_path); - std::vector> entries; - for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); + for(; it->Valid() && string_starts_with(it->key().ToString(), root_path); it->Next()) { + auto name = it->key().ToString(); - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path + if(name.size() == root_path.size()) { continue; } - /***** Get File name *****/ - auto name = it->key().ToString(); + // Check single-level depth if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth continue; } - // remove prefix - name = name.substr(root_path.size()); - // relative path of directory entries must not be empty - assert(!name.empty()); + auto relative_name = name.substr(root_path.size()); + assert(!relative_name.empty()); // Filter out inline data keys - if(name.size() >= 7 && name.substr(name.size() - 7) == "#inline") { + if(relative_name.size() >= 7 && + relative_name.substr(relative_name.size() - 7) == "#inline") { continue; } - Metadata md(it->value().ToString()); - if(gkfs::config::metadata::rename_support) { - // Remove entries with negative blocks (rename) - if(md.blocks() == -1) { - continue; - } - } + mode_t mode = 0; + size_t size = 0; + time_t ctime = 0; + blkcnt_t blocks = 0; + std::string_view val_view(it->value().data(), it->value().size()); + if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { + Metadata md(it->value().ToString()); + mode = md.mode(); + size = md.size(); + ctime = md.ctime(); + blocks = md.blocks(); + } - // 0: regular, 1: directory, 2: symlink - unsigned char type = 0; - if(S_ISDIR(md.mode())) { - type = 1; - } else if(S_ISLNK(md.mode())) { - type = 2; + if(gkfs::config::metadata::rename_support && blocks == -1) { + continue; } - entries.emplace_back(std::move(name), type); + if(!proc(name, relative_name, mode, size, ctime, blocks)) { + break; + } } - assert(it->status().ok()); - return entries; + return true; } -/** - * Return all the first-level entries of the directory @dir - * - * @return vector of pair , - * where name is the name of the entries and is_dir - * is true in the case the entry is a directory. - */ -std::vector> -RocksDBBackend::get_dirents_extended_impl(const std::string& dir, - const std::string& start_key, - size_t max_entries) const { - auto root_path = dir; +// Overload with start_key support +bool +iterate_dirents(rocksdb::DB* db, const std::string& root_path, + const std::string& start_key, const EntryProcessor& proc) { rocksdb::ReadOptions ropts; - ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer - std::unique_ptr it(db_->NewIterator(ropts)); - - std::vector> entries; + ropts.readahead_size = 2 * 1024 * 1024; + std::unique_ptr it(db->NewIterator(ropts)); if(start_key.empty()) { it->Seek(root_path); @@ -516,32 +527,19 @@ RocksDBBackend::get_dirents_extended_impl(const std::string& dir, } } - for(; it->Valid() && it->key().starts_with(root_path); it->Next()) { - - if(max_entries > 0 && entries.size() >= max_entries) { - break; - } + for(; it->Valid() && string_starts_with(it->key().ToString(), root_path); + it->Next()) { + auto name = it->key().ToString(); - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path + if(name.size() == root_path.size()) { continue; } - /***** Get File name *****/ - auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - // continue; - } - // remove prefix - name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!name.empty()); + auto relative_name = name.substr(root_path.size()); + assert(!relative_name.empty()); - // Filter out inline data keys - if(name.size() >= 7 && - name.compare(name.size() - 7, 7, "#inline") == 0) { + if(relative_name.size() >= 7 && + relative_name.substr(relative_name.size() - 7) == "#inline") { continue; } @@ -551,7 +549,6 @@ RocksDBBackend::get_dirents_extended_impl(const std::string& dir, blkcnt_t blocks = 0; std::string_view val_view(it->value().data(), it->value().size()); - if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { Metadata md(it->value().ToString()); mode = md.mode(); @@ -560,40 +557,92 @@ RocksDBBackend::get_dirents_extended_impl(const std::string& dir, blocks = md.blocks(); } - if(gkfs::config::metadata::rename_support) { - // Remove entries with negative blocks (rename) - if(blocks == -1) { - continue; - } + if(gkfs::config::metadata::rename_support && blocks == -1) { + continue; } + if(!proc(name, relative_name, mode, size, ctime, blocks)) { + break; + } + } + return true; +} + +} // namespace + +} // namespace + +// Return all the first-level entries of the directory @dir +std::vector> +RocksDBBackend::get_dirents_impl(const std::string& dir) const { + auto root_path = dir; + std::vector> entries; + auto proc = [&](std::string_view key, std::string_view rel_name, + mode_t mode, size_t size, time_t ctime, blkcnt_t blocks) { + (void) key; + (void) rel_name; + (void) size; + (void) ctime; + (void) blocks; unsigned char type = 0; - if(S_ISDIR(mode)) { + if(S_ISDIR(mode)) type = 1; - } else if(S_ISLNK(mode)) { + else if(S_ISLNK(mode)) type = 2; + entries.emplace_back(std::string(rel_name), type); + return true; + }; + + iterate_dirents(db_.get(), root_path, proc); + return entries; +} + +// Simplified using shared iterate_dirents helper +std::vector> +RocksDBBackend::get_dirents_extended_impl(const std::string& dir, + const std::string& start_key, + size_t max_entries) const { + auto root_path = dir; + std::vector> entries; + size_t count = 0; + + auto proc = [&](std::string_view key, std::string_view rel_name, + mode_t mode, size_t size, time_t ctime, blkcnt_t blocks) { + (void) key; + (void) blocks; + if(max_entries > 0 && count >= max_entries) { + return false; } + count++; + unsigned char type = 0; + if(S_ISDIR(mode)) + type = 1; + else if(S_ISLNK(mode)) + type = 2; + entries.emplace_back(std::forward_as_tuple(std::string(rel_name), type, + size, ctime)); + return true; + }; - entries.emplace_back( - std::forward_as_tuple(std::move(name), type, size, ctime)); - } - assert(it->status().ok()); + iterate_dirents(db_.get(), root_path, start_key, proc); return entries; } -// Return all the extended entries with root in the path specified +// Scans all depths (no single-level filter) std::vector> RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, const std::string& start_key, size_t max_entries) const { auto root_path = dir; + std::vector> entries; + size_t count = 0; + + // This function scans ALL depths, so we need a custom iterator rocksdb::ReadOptions ropts; - ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer + ropts.readahead_size = 2 * 1024 * 1024; std::unique_ptr it(db_->NewIterator(ropts)); - std::vector> entries; - if(start_key.empty()) { it->Seek(root_path); } else { @@ -604,42 +653,30 @@ RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, } } - for(; it->Valid() && it->key().starts_with(root_path); it->Next()) { - - if(max_entries > 0 && entries.size() >= max_entries) { - break; - } - - if(it->key().size() == root_path.size()) { - // we skip this path cause it is exactly the root_path - continue; - } - - /***** Get File name *****/ + for(; it->Valid() && string_starts_with(it->key().ToString(), root_path); + it->Next()) { auto name = it->key().ToString(); - if(name.find_first_of('/', root_path.size()) != std::string::npos) { - // skip stuff deeper then one level depth - // continue; - } - // remove prefix - name = name.substr(root_path.size()); + if(name.size() == root_path.size()) + continue; - // relative path of directory entries must not be empty - assert(!name.empty()); + auto relative_name = name.substr(root_path.size()); + assert(!relative_name.empty()); - // Filter out inline data keys - if(name.size() >= 7 && - name.compare(name.size() - 7, 7, "#inline") == 0) { + if(relative_name.size() >= 7 && + relative_name.substr(relative_name.size() - 7) == "#inline") { continue; } + count++; + if(max_entries > 0 && count >= max_entries) + break; + mode_t mode = 0; size_t size = 0; time_t ctime = 0; blkcnt_t blocks = 0; std::string_view val_view(it->value().data(), it->value().size()); - if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { Metadata md(it->value().ToString()); mode = md.mode(); @@ -648,30 +685,23 @@ RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, blocks = md.blocks(); } - if(gkfs::config::metadata::rename_support) { - // Remove entries with negative blocks (rename) - if(blocks == -1) { - continue; - } + if(gkfs::config::metadata::rename_support && blocks == -1) { + continue; } - - // 0: regular, 1: directory, 2: symlink unsigned char type = 0; - if(S_ISDIR(mode)) { + if(S_ISDIR(mode)) type = 1; - } else if(S_ISLNK(mode)) { + else if(S_ISLNK(mode)) type = 2; - } - entries.emplace_back( - std::forward_as_tuple(std::move(name), type, size, ctime)); + entries.emplace_back(std::forward_as_tuple(std::string(relative_name), + type, size, ctime)); } - assert(it->status().ok()); return entries; } -// Return all the filtered entries with root in the path specified +// Simplified using shared iterate_dirents helper std::tuple>, size_t, size_t, std::string> RocksDBBackend::get_dirents_filtered_impl( @@ -679,17 +709,10 @@ RocksDBBackend::get_dirents_filtered_impl( const std::string& filter_name, const int64_t filter_size, const int64_t filter_ctime, bool count_only, size_t max_entries) const { auto root_path = dir; - rocksdb::ReadOptions ropts; - ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer - std::unique_ptr it(db_->NewIterator(ropts)); - std::vector> entries; - std::string last_scanned_key; - bool eof = true; size_t scanned_count = 0; size_t matched_count = 0; - const size_t scan_limit = - 5000000; // Limit scanned entries per PRC to avoid timeout + std::string last_scanned_key; std::regex name_regex; bool use_regex = !filter_name.empty(); @@ -697,82 +720,19 @@ RocksDBBackend::get_dirents_filtered_impl( try { name_regex = std::regex(filter_name); } catch(const std::regex_error& e) { - // fallback if invalid regex return {entries, 0, 0, ""}; } } - if(start_key.empty()) { - it->Seek(root_path); - } else { - auto key = root_path + start_key; - it->Seek(key); - if(it->Valid() && it->key().ToString() == key) { - it->Next(); - } - } - - for(; it->Valid() && it->key().starts_with(root_path); it->Next()) { - if(scanned_count >= scan_limit) { - eof = false; - break; - } - // Get File name - auto name = it->key().ToString(); - // save as potential last key (full path for now, stripped later if - // needed? No, logic needs relative) - - if(name.size() <= root_path.size()) { - // Should not happen due to starts_with and loop logic potentially? - // But just in case - continue; - } - - // remove prefix - auto relative_name = name.substr(root_path.size()); - - // relative path of directory entries must not be empty - assert(!relative_name.empty()); - - // Capture the key before potentially skipping it to track progress - last_scanned_key = relative_name; - - - // Filter out inline data keys - if(relative_name.size() >= 7 && - relative_name.substr(relative_name.size() - 7) == "#inline") { - continue; - } - + auto proc = [&](std::string_view key, std::string_view rel_name, + mode_t mode, size_t size, time_t ctime, blkcnt_t blocks) { + (void) key; + (void) blocks; scanned_count++; - mode_t mode = 0; - size_t size = 0; - time_t ctime = 0; - blkcnt_t blocks = 0; - - std::string_view val_view(it->value().data(), it->value().size()); - - if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { - Metadata md(it->value().ToString()); - mode = md.mode(); - size = md.size(); - ctime = md.ctime(); - blocks = md.blocks(); - } - - if(gkfs::config::metadata::rename_support) { - // Remove entries with negative blocks (rename) - if(blocks == -1) { - continue; - } - } - bool matched = true; - if(use_regex) { - if(!std::regex_match(relative_name, name_regex)) { - matched = false; - } + if(use_regex && !std::regex_match(std::string(rel_name), name_regex)) { + matched = false; } if(matched && filter_size != -1 && size != (size_t) filter_size) { matched = false; @@ -783,28 +743,28 @@ RocksDBBackend::get_dirents_filtered_impl( if(matched) { matched_count++; + last_scanned_key = std::string(rel_name); if(!count_only) { unsigned char type = 0; - if(S_ISDIR(mode)) { + if(S_ISDIR(mode)) type = 1; - } else if(S_ISLNK(mode)) { + else if(S_ISLNK(mode)) type = 2; - } entries.emplace_back(std::forward_as_tuple( - std::move(relative_name), type, size, ctime)); + std::string(rel_name), type, size, ctime)); } if(max_entries > 0 && matched_count >= max_entries) { - eof = false; - break; + return false; // stop iteration } + } else { + last_scanned_key = std::string(rel_name); } - } + return true; + }; - if(eof) { - last_scanned_key = ""; - } + iterate_dirents(db_.get(), root_path, start_key, proc); - // assert(it->status().ok()); // only if eof check? + // last_scanned_key is empty on EOF (caller convention) return {entries, matched_count, scanned_count, last_scanned_key}; } diff --git a/tests/integration/concurrency/test_concurrency.py b/tests/integration/concurrency/test_concurrency.py index ee860309ca8b75750c889f67fd87e1bc08baa974..34c51a5d0df58a130c86021cf74e953c1410110b 100644 --- a/tests/integration/concurrency/test_concurrency.py +++ b/tests/integration/concurrency/test_concurrency.py @@ -4,11 +4,13 @@ import time import concurrent.futures import os -def test_concurrent_create(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_concurrent_create(client_fixture, request, gkfs_daemon): """ Test concurrent file creation in the same directory. Using Python's ThreadPoolExecutor for more robust concurrency. """ + gkfs_client = request.getfixturevalue(client_fixture) concurrent_dir = gkfs_daemon.mountdir / 'concurrent_dir' # Create directory @@ -34,10 +36,12 @@ def test_concurrent_create(gkfs_daemon, gkfs_client): names = [d.d_name for d in res.dirents if d.d_name not in ['.', '..']] assert len(names) == 10 -def test_concurrent_write_shared_file(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_concurrent_write_shared_file(client_fixture, request, gkfs_daemon): """ Test concurrent writes to the SAME file (append). """ + gkfs_client = request.getfixturevalue(client_fixture) shared_file = gkfs_daemon.mountdir / 'shared_file' # Ensure file doesn't exist @@ -68,10 +72,12 @@ def test_concurrent_write_shared_file(gkfs_daemon, gkfs_client): lines = [l for l in res.buf.decode().splitlines() if l and not all(c == '\x00' for c in l)] assert len(lines) == 10 -def test_concurrent_read(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_concurrent_read(client_fixture, request, gkfs_daemon): """ Test concurrent reads from the same file. """ + gkfs_client = request.getfixturevalue(client_fixture) read_file = gkfs_daemon.mountdir / 'read_file' # Create 1MB file in chunks to avoid CLI argument length limits diff --git a/tests/integration/data/test_async_write.py b/tests/integration/data/test_async_write.py index 1cd212fc6237a4babb02ec8024bf972cf858ee6e..0231c77000275fb3b20ef821d239c4e7f0f2ef41 100644 --- a/tests/integration/data/test_async_write.py +++ b/tests/integration/data/test_async_write.py @@ -35,8 +35,10 @@ import sys import pytest from harness.logger import logger -def test_async_write(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_async_write(client_fixture, request, gkfs_daemon): """Test data write/read using the client-side async write cache feature""" + gkfs_client = request.getfixturevalue(client_fixture) topdir = gkfs_daemon.mountdir / "top" file_a = topdir / "file_a" file_b = topdir / "file_b" diff --git a/tests/integration/data/test_data_integrity.py b/tests/integration/data/test_data_integrity.py index de9d682e9dfed0dd5db28901ca59d885db945a9f..4057b8c90f416ffb12f5c632ffe18c7f93b364c5 100644 --- a/tests/integration/data/test_data_integrity.py +++ b/tests/integration/data/test_data_integrity.py @@ -58,9 +58,10 @@ def check_write(gkfs_client, i, file): assert ret.retval== i assert ret.buf == buf -#@pytest.mark.xfail(reason="invalid errno returned on success") -def test_data_integrity(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_data_integrity(client_fixture, request, gkfs_daemon): """Test several data write-read commands and check that the data is correct""" + gkfs_client = request.getfixturevalue(client_fixture) topdir = gkfs_daemon.mountdir / "top" file_a = topdir / "file_a" diff --git a/tests/integration/data/test_inline_1rpc.py b/tests/integration/data/test_inline_1rpc.py index 53db1dcd00e21ec7c08a29b38edd0f3b692cc707..20deb21e1c9601ea60763143d58670c4a46e0226 100644 --- a/tests/integration/data/test_inline_1rpc.py +++ b/tests/integration/data/test_inline_1rpc.py @@ -3,8 +3,10 @@ import os import stat from harness.logger import logger -def test_inline_1rpc_optimized(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_1rpc_optimized(client_fixture, request, gkfs_daemon): """Test 1-RPC Create+Write optimization""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_1rpc_opt" # Open (O_CREAT) + Write (small) in one command to ensure single process and trigger optimization buf = 'A' * 100 @@ -45,8 +47,10 @@ def test_inline_1rpc_optimized(gkfs_daemon, gkfs_client): assert ret.retval == len(buf) assert ret.buf == buf.encode() -def test_inline_1rpc_fallback_close(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_1rpc_fallback_close(client_fixture, request, gkfs_daemon): """Test 1-RPC optimization fallback: Open(O_CREAT) -> Close (create empty file)""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_1rpc_empty" # gkfs.io open @@ -59,8 +63,10 @@ def test_inline_1rpc_fallback_close(gkfs_daemon, gkfs_client): assert ret.retval == 0 assert ret.statbuf.st_size == 0 -def test_inline_1rpc_fallback_large_write(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_1rpc_fallback_large_write(client_fixture, request, gkfs_daemon): """Test 1-RPC optimization fallback: Open(O_CREAT) -> Write(large) (explicit create)""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_1rpc_large" # Write larger than inline size (assuming 4096 default) @@ -93,8 +99,10 @@ def test_inline_1rpc_fallback_large_write(gkfs_daemon, gkfs_client): assert ret.retval == size assert ret.buf == buf.encode() -def test_inline_1rpc_no_opt_o_excl(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_1rpc_no_opt_o_excl(client_fixture, request, gkfs_daemon): """Test O_EXCL disables optimization""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_no_opt_excl" # Open O_CREAT | O_EXCL (Optimization should be disabled) diff --git a/tests/integration/data/test_inline_data.py b/tests/integration/data/test_inline_data.py index dacc0d35820858f600b50a95b05cb81f3d238de3..23f7ccf9eaa7b10e73413026aeea846d7efef084 100644 --- a/tests/integration/data/test_inline_data.py +++ b/tests/integration/data/test_inline_data.py @@ -3,8 +3,10 @@ import os import stat from harness.logger import logger -def test_inline_append(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_append(client_fixture, request, gkfs_daemon): """Test inline data append operations""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_inline_append" # Open file @@ -43,8 +45,10 @@ def test_inline_append(gkfs_daemon, gkfs_client): assert ret.retval == 200 assert ret.buf == (buf1 + buf2).encode() -def test_inline_pwrite(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_pwrite(client_fixture, request, gkfs_daemon): """Test inline data overwrite using pwrite""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_inline_pwrite" ret = gkfs_client.open(file, @@ -79,8 +83,10 @@ def test_inline_pwrite(gkfs_daemon, gkfs_client): assert ret.retval == 100 assert ret.buf == expected -def test_inline_overflow_append(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_overflow_append(client_fixture, request, gkfs_daemon): """Test appending data that overflows inline limit (migration to chunks)""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_inline_overflow" ret = gkfs_client.open(file, @@ -119,8 +125,10 @@ def test_inline_overflow_append(gkfs_daemon, gkfs_client): assert ret.retval == 4200 assert ret.buf == (buf1 + buf2).encode() -def test_inline_overflow_pwrite(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_overflow_pwrite(client_fixture, request, gkfs_daemon): """Test pwrite that overflows inline limit (migration to chunks)""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_inline_overflow_pwrite" ret = gkfs_client.open(file, @@ -162,8 +170,10 @@ def test_inline_overflow_pwrite(gkfs_daemon, gkfs_client): assert read_buf[100:offset] == b'\x00' * (offset - 100) assert read_buf[offset:offset+100] == buf2.encode() -def test_inline_overwrite_pwrite(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_overwrite_pwrite(client_fixture, request, gkfs_daemon): """Test pwrite at offset 0 that overflows inline limit (migration/clearing)""" + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file_inline_overwrite_pwrite" ret = gkfs_client.open(file, diff --git a/tests/integration/data/test_inline_read_opt.py b/tests/integration/data/test_inline_read_opt.py index 2dcb811add2ee2bfa6aa5bdba4f2c64415bc5d70..eec21b907ccea5036b6f6cbc359e54eff1ea1caa 100644 --- a/tests/integration/data/test_inline_read_opt.py +++ b/tests/integration/data/test_inline_read_opt.py @@ -5,10 +5,12 @@ from harness.logger import logger file01 = 'file01' data01 = 'data01' -def test_inline_read_optimization(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_inline_read_optimization(client_fixture, request, gkfs_daemon): """ Test the read optimization where inline data is cached during open. """ + gkfs_client = request.getfixturevalue(client_fixture) file01 = gkfs_daemon.mountdir / "file01" # Enable inline data and the optimization (though optimization flag is mainly for create/write) diff --git a/tests/integration/data/test_truncate.py b/tests/integration/data/test_truncate.py index 377a818b8322b51977f24852647a231ad684423e..ba414fc14c93d4482b12af32453cc22cf4e905a8 100644 --- a/tests/integration/data/test_truncate.py +++ b/tests/integration/data/test_truncate.py @@ -33,6 +33,30 @@ import stat import errno +def test_truncate_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): + """Test truncate via proxy daemon path.""" + truncfile = gkfs_daemon_proxy.mountdir / "trunc_file_proxy" + ret = gkfs_client_proxy.open(truncfile, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1 + buf_length = 16777216 + ret = gkfs_client_proxy.write_random(truncfile, buf_length) + assert ret.retval == buf_length + ret = gkfs_client_proxy.stat(truncfile) + assert ret.statbuf.st_size == buf_length + trunc_size = buf_length // 2 + ret = gkfs_client_proxy.truncate(truncfile, trunc_size) + assert ret.retval == 0 + ret = gkfs_client_proxy.stat(truncfile) + assert ret.statbuf.st_size == trunc_size + truncfile_verify = gkfs_daemon_proxy.mountdir / "trunc_file_verify_proxy" + ret = gkfs_client_proxy.open(truncfile_verify, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1 + ret = gkfs_client_proxy.write_random(truncfile_verify, trunc_size) + assert ret.retval == trunc_size + ret = gkfs_client_proxy.file_compare(truncfile, truncfile_verify, trunc_size) + assert ret.retval == 0 + + # @pytest.mark.xfail(reason="invalid errno returned on success") def test_truncate(gkfs_daemon, gkfs_client): """Testing truncate: @@ -158,91 +182,3 @@ def test_fail_truncate(gkfs_daemon, gkfs_client): assert ret.statbuf.st_size == buf_length+1 -def test_truncate_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): - """Testing truncate: - 1. create a large file over multiple chunks - 2. truncate it in the middle and compare it with a fresh file with equal contents (exactly at chunk border) - 3. truncate it again so that in truncates in the middle of the chunk and compare with fresh file - TODO chunksize needs to be respected to make sure chunk border and in the middle of chunk truncates are honored - """ - truncfile = gkfs_daemon_proxy.mountdir / "trunc_file" - - # open and create test file - ret = gkfs_client_proxy.open(truncfile, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - # write a multi MB file (16mb) - buf_length = 16777216 - ret = gkfs_client_proxy.write_random(truncfile, buf_length) - - assert ret.retval == buf_length - - ret = gkfs_client_proxy.stat(truncfile) - - assert ret.statbuf.st_size == buf_length - - # truncate it - # split exactly in the middle - trunc_size = buf_length // 2 - ret = gkfs_client_proxy.truncate(truncfile, trunc_size) - - assert ret.retval == 0 - # check file length - ret = gkfs_client_proxy.stat(truncfile) - - assert ret.statbuf.st_size == trunc_size - - # verify contents by writing a new file (random content is seeded) and checksum both - truncfile_verify = gkfs_daemon_proxy.mountdir / "trunc_file_verify" - - # open and create test file - ret = gkfs_client_proxy.open(truncfile_verify, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - # write trunc_size of data to new file - ret = gkfs_client_proxy.write_random(truncfile_verify, trunc_size) - - assert ret.retval == trunc_size - - ret = gkfs_client_proxy.stat(truncfile_verify) - - assert ret.statbuf.st_size == trunc_size - - ret = gkfs_client_proxy.file_compare(truncfile, truncfile_verify, trunc_size) - - assert ret.retval == 0 - - # trunc at byte 712345 (middle of chunk) - # TODO feed chunksize into test to make sure it is always in the middle of the chunk - trunc_size = 712345 - ret = gkfs_client_proxy.truncate(truncfile, trunc_size) - - assert ret.retval == 0 - - # check file length - ret = gkfs_client_proxy.stat(truncfile) - - assert ret.statbuf.st_size == trunc_size - - # verify contents by writing a new file (random content is seeded) and checksum both - truncfile_verify_2 = gkfs_daemon_proxy.mountdir / "trunc_file_verify_2" - - # open and create test file - ret = gkfs_client_proxy.open(truncfile_verify_2, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - # write trunc_size of data to new file - ret = gkfs_client_proxy.write_random(truncfile_verify_2, trunc_size) - - assert ret.retval == trunc_size - - ret = gkfs_client_proxy.stat(truncfile_verify_2) - - assert ret.statbuf.st_size == trunc_size - - ret = gkfs_client_proxy.file_compare(truncfile, truncfile_verify_2, trunc_size) - - assert ret.retval == 0 \ No newline at end of file diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 34fe972316f7fd8a582e392d685927c0c7ee3843..a22e073ac30658f683df606b516a6cbd75f91635 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -314,7 +314,7 @@ def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): output = f.read() assert expected_line in output, \ - f"Expected to find '{expected_line.strip()}' in the output or results file, but got:\n---\n{output}\n---" + "Expected to find '{expected_line.strip()}' in the output or results file, but got:\n---\n{output}\n---" cmd = gkfs_shell.sfind( @@ -334,104 +334,3 @@ def test_opendir(gkfs_daemon, gkfs_client, directory_path): assert ret.dirp is None assert ret.errno == errno.ENOENT -def test_finedir_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): - """Tests several corner cases for directories scan""" - - topdir = gkfs_daemon_proxy.mountdir / "finetop" - file_a = topdir / "file_" - - - # create topdir - ret = gkfs_client_proxy.mkdir( - topdir, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 0 - - ret = gkfs_client_proxy.readdir(topdir) - - # XXX: This might change in the future if we add '.' and '..' - assert len(ret.dirents) == 2 - - # populate top directory - - for files in range (1,4): - ret = gkfs_client_proxy.directory_validate( - topdir, 1) - assert ret.retval == files+2 - - - ret = gkfs_client_proxy.directory_validate( - topdir, 1000) - assert ret.retval == 1000+3+2 - - -def test_extended_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_shell_proxy, gkfs_client_proxy): - topdir = gkfs_daemon_proxy.mountdir / "test_extended" - dir_a = topdir / "dir_a" - dir_b = topdir / "dir_b" - file_a = topdir / "file_a" - subdir_a = dir_a / "subdir_a" - - # create topdir - ret = gkfs_client_proxy.mkdir( - topdir, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 0 - - ret = gkfs_client_proxy.mkdir( - dir_a, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 0 - - ret = gkfs_client_proxy.mkdir( - dir_b, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 0 - - ret = gkfs_client_proxy.mkdir( - subdir_a, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 0 - - ret = gkfs_client_proxy.open(file_a, - os.O_CREAT, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1 - - buf = b'42' - ret = gkfs_client_proxy.write(file_a, buf, 1) - - assert ret.retval == 1 - - gkfs_shell_proxy._env['SFIND_NUM_THREADS'] = '1' - cmd = gkfs_shell_proxy.sfind( - topdir, - '-M', - gkfs_daemon_proxy.mountdir, - '-S', - 1, - '-name', - '*_k*', - '-H', 10, - '-N', - '-P', - '-C', - '-D', "rates", - '-q', 10, - '-s', 10, - '-v' - ) - - assert cmd.exit_code == 0 - output = cmd.stdout.decode() - expected_line = "MATCHED 0/4\n" - - assert expected_line in output, \ - f"Expected to find '{expected_line.strip()}' in the output, but got:\n---\n{output}\n---" - - diff --git a/tests/integration/fuse/test_basic_operations.py b/tests/integration/fuse/test_basic_operations.py deleted file mode 100644 index 5974e230f742e305f702362082ed1f0a886a194c..0000000000000000000000000000000000000000 --- a/tests/integration/fuse/test_basic_operations.py +++ /dev/null @@ -1,95 +0,0 @@ -################################################################################ -# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -import harness -from pathlib import Path -import errno -import stat -import os -import ctypes -import sh -import sys -import pytest -import time -from harness.logger import logger - -nonexisting = "nonexisting" - -@pytest.mark.filterwarnings("ignore:This process.*is multi-threaded:DeprecationWarning") -#@pytest.mark.fuse_opts(env={'LIBGKFS_DENTRY_CACHE': 'OFF', 'LIBGKFS_WRITE_SIZE_CACHE': 'OFF', 'LIBGKFS_CREATE_WRITE_OPTIMIZATION': 'OFF', 'LIBGKFS_USE_INLINE_DATA': 'OFF'}) -def test_read(gkfs_daemon, fuse_client): - file = gkfs_daemon.mountdir / "file" - file2 = gkfs_daemon.mountdir / "file2" - - dir = gkfs_daemon.mountdir / "dir" - - # creation and removal - sh.bash("-c", "echo baum > " + str(file)) - assert sh.ls(fuse_client.mountdir) == "file\n" - assert sh.cat(file) == "baum\n" - sh.touch(str(file2)) - assert sh.wc("-c", str(file2)) == "0 " + str(file2) + "\n" - sh.truncate("-s", "20", str(file2)) - assert sh.wc("-c", str(file2)) == "20 " + str(file2) + "\n" - sh.mkdir(str(dir)) - assert sh.ls(fuse_client.mountdir) == "dir file file2\n" - os.chdir(str(dir)) - assert sh.pwd() == str(dir) + "\n" - sh.mkdir("-p", "foo/bar") - assert sh.ls() == "foo\n" - os.chdir("foo") - sh.rmdir("bar") - os.chdir("..") - sh.rmdir("foo") - sh.rm(str(file2)) - assert sh.ls(fuse_client.mountdir) == "dir file\n" - - # lseek test (TODO doesn't use the lseek handler) - path = gkfs_daemon.mountdir / "lseek_file" - with open(path, "wb") as f: - f.write(b"HelloWorld") # 10 bytes - fd = os.open(path, os.O_RDONLY) - pos = os.lseek(fd, 5, os.SEEK_SET) # absolute seek - assert pos == 5 - data = os.read(fd, 5) - assert data == b"World" - os.close(fd) - os.remove(path) - - # symlink - assert sh.pwd() == str(dir) + "\n" - sh.ln("-s", str(file), "link") - assert sh.ls(str(dir)) == "link\n" - assert sh.cat("link") == "baum\n" - - # renaming - sh.mv("link", "../lonk") - assert sh.ls("..") == "dir file lonk\n" - - # Move CWD out of the mountpoint so FUSE can unmount cleanly - os.chdir("/") diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 86decfa3476e4321da32c952230d015bec2d4d37..a707463613c9b18b101a227b1c9a0c25e1a31604 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -788,15 +788,17 @@ class ClientLibc: return self._preload_library - def run(self, cmd, *args): - - + def run(self, cmd, *args, **kwargs): cmd_args = [str(self._cmd), str(cmd)] + [a.decode('utf-8') if isinstance(a, bytes) else str(a) for a in args] + current_env = self._env.copy() + if 'env' in kwargs: + current_env.update(kwargs['env']) + proc = subprocess.Popen( cmd_args, - env=self._env, + env=current_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) diff --git a/tests/integration/operations/test_large_io.py b/tests/integration/operations/test_large_io.py index 152a8655b116488809a7783a84dbdb4de4133110..a69c882a94c08df67e85fa69dd87def188f25de2 100644 --- a/tests/integration/operations/test_large_io.py +++ b/tests/integration/operations/test_large_io.py @@ -6,6 +6,24 @@ import os import stat +def test_large_io_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): + """Test large I/O via proxy to trigger chunk storage path.""" + file = gkfs_daemon_proxy.mountdir / "large_file_proxy" + ret = gkfs_client_proxy.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1 + total_size = 1024 * 1024 + chunk_size = 100 * 1024 + pattern = b'X' * chunk_size + for offset in range(0, total_size, chunk_size): + ret = gkfs_client_proxy.pwrite(file, pattern, chunk_size, offset) + assert ret.retval == chunk_size + ret = gkfs_client_proxy.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1 + for offset in range(0, total_size, chunk_size): + ret = gkfs_client_proxy.pread(file, chunk_size, offset) + assert ret.retval == chunk_size + assert ret.buf == pattern + @pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) def test_large_io(client_fixture, request, gkfs_daemon): """ @@ -45,36 +63,4 @@ def test_large_io(client_fixture, request, gkfs_daemon): assert ret.retval == chunk_size assert ret.buf == pattern -def test_large_io_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): - """ - Test large I/O via proxy to trigger chunk storage path. - """ - file = gkfs_daemon_proxy.mountdir / "large_file_proxy" - - # Open - ret = gkfs_client_proxy.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1 - - # Write 1MB in chunks - total_size = 1024 * 1024 - chunk_size = 100 * 1024 - pattern = b'X' * chunk_size - - for offset in range(0, total_size, chunk_size): - ret = gkfs_client_proxy.pwrite(file, pattern, chunk_size, offset) - assert ret.retval == chunk_size - - # Read back - ret = gkfs_client_proxy.open(file, - os.O_RDONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1 - - for offset in range(0, total_size, chunk_size): - ret = gkfs_client_proxy.pread(file, chunk_size, offset) - assert ret.retval == chunk_size - assert ret.buf == pattern - diff --git a/tests/integration/operations/test_read_operations.py b/tests/integration/operations/test_read_operations.py index 10455596bc4a0a5c14a61a85d01717a3feaf02ed..7a1a0300e4d679d3fb4a75d8131b5bcb48b603c8 100644 --- a/tests/integration/operations/test_read_operations.py +++ b/tests/integration/operations/test_read_operations.py @@ -71,35 +71,19 @@ def test_read(client_fixture, request, gkfs_daemon): assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes - def test_read_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): + """Test read via proxy daemon path.""" file = gkfs_daemon_proxy.mountdir / "file_read_proxy" - - # create a file in gekkofs - ret = gkfs_client_proxy.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - + ret = gkfs_client_proxy.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 - - # write a buffer we know buf = b'42' ret = gkfs_client_proxy.write(file, buf, len(buf)) - - assert ret.retval == len(buf) # Return the number of written bytes - - # open the file to read - ret = gkfs_client_proxy.open(file, - os.O_RDONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - + assert ret.retval == len(buf) + ret = gkfs_client_proxy.open(file, os.O_RDONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 - - # read the file ret = gkfs_client_proxy.read(file, len(buf)) - assert ret.buf == buf - assert ret.retval == len(buf) # Return the number of read bytes + assert ret.retval == len(buf) @pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) def test_pread(client_fixture, request, gkfs_daemon): diff --git a/tests/integration/operations/test_unlink_operations.py b/tests/integration/operations/test_unlink_operations.py index d40de56c5944fff4a6071a46ce258f552990aca5..2770dc3137324c5a6d673b929906d94b3d529234 100644 --- a/tests/integration/operations/test_unlink_operations.py +++ b/tests/integration/operations/test_unlink_operations.py @@ -39,74 +39,54 @@ from harness.logger import logger nonexisting = "nonexisting" -@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) -def test_unlink(client_fixture, request, gkfs_daemon): - gkfs_client = request.getfixturevalue(client_fixture) - file = gkfs_daemon.mountdir / "file" - dir = gkfs_daemon.mountdir / "dir" - +def test_unlink_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): + """Test unlink via proxy daemon path.""" + file = gkfs_daemon_proxy.mountdir / "file_proxy" + dir = gkfs_daemon_proxy.mountdir / "dir_proxy" # Delete an unexistent file - ret = gkfs_client.unlink(file) + ret = gkfs_client_proxy.unlink(file) assert ret.retval == -1 assert ret.errno == errno.ENOENT - - - - - # create a file in gekkofs - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - + # Create and write a file + ret = gkfs_client_proxy.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 + ret = gkfs_client_proxy.write(file, b'42', 2) + assert ret.retval == 2 - # write a buffer we know - buf = b'42' - ret = gkfs_client.write(file, buf, len(buf)) - assert ret.retval == len(buf) # Return the number of written bytes - - - - ret = gkfs_client.unlink(file) # Remove renamed file (success) - assert ret.retval == 0 + ret = gkfs_client_proxy.unlink(file) + assert ret.retval == 0 - ret = gkfs_client.stat(file) # file does not exist - assert ret.retval != 0 - assert ret.errno == errno.ENOENT + ret = gkfs_client_proxy.stat(file) + assert ret.retval != 0 + assert ret.errno == errno.ENOENT - ret = gkfs_client.mkdir(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # Create a directory + ret = gkfs_client_proxy.mkdir(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval == 0 - ret = gkfs_client.unlink(dir) + ret = gkfs_client_proxy.unlink(dir) assert ret.retval == -1 assert ret.errno == errno.EISDIR - - - # create a file in gekkofs - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + # Create a multi-chunk file + ret = gkfs_client_proxy.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 - - # > 4 chunks - ret = gkfs_client.write_validate(file, 2097153) + ret = gkfs_client_proxy.write_validate(file, 2097153) assert ret.retval == 0 - - ret = gkfs_client.unlink(file) # Remove renamed file (extra chunks, success) + ret = gkfs_client_proxy.unlink(file) assert ret.retval == 0 -def test_unlink_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): - - file = gkfs_daemon_proxy.mountdir / "file_proxy" - dir = gkfs_daemon_proxy.mountdir / "dir_proxy" +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_unlink(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) + file = gkfs_daemon.mountdir / "file" + dir = gkfs_daemon.mountdir / "dir" # Delete an unexistent file - ret = gkfs_client_proxy.unlink(file) + ret = gkfs_client.unlink(file) assert ret.retval == -1 assert ret.errno == errno.ENOENT @@ -115,7 +95,7 @@ def test_unlink_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): # create a file in gekkofs - ret = gkfs_client_proxy.open(file, + ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) @@ -123,36 +103,38 @@ def test_unlink_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): # write a buffer we know buf = b'42' - ret = gkfs_client_proxy.write(file, buf, len(buf)) + ret = gkfs_client.write(file, buf, len(buf)) assert ret.retval == len(buf) # Return the number of written bytes - ret = gkfs_client_proxy.unlink(file) # Remove renamed file (success) + ret = gkfs_client.unlink(file) # Remove renamed file (success) assert ret.retval == 0 - ret = gkfs_client_proxy.stat(file) # file does not exist + ret = gkfs_client.stat(file) # file does not exist assert ret.retval != 0 assert ret.errno == errno.ENOENT - ret = gkfs_client_proxy.mkdir(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # Create a directory + ret = gkfs_client.mkdir(dir, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # Create a directory assert ret.retval == 0 - ret = gkfs_client_proxy.unlink(dir) + ret = gkfs_client.unlink(dir) assert ret.retval == -1 assert ret.errno == errno.EISDIR # create a file in gekkofs - ret = gkfs_client_proxy.open(file, + ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 # > 4 chunks - ret = gkfs_client_proxy.write_validate(file, 2097153) + ret = gkfs_client.write_validate(file, 2097153) assert ret.retval == 0 - ret = gkfs_client_proxy.unlink(file) # Remove renamed file (extra chunks, success) - assert ret.retval == 0 \ No newline at end of file + ret = gkfs_client.unlink(file) # Remove renamed file (extra chunks, success) + assert ret.retval == 0 + + diff --git a/tests/integration/operations/test_write_operations.py b/tests/integration/operations/test_write_operations.py index 5a2f934e0c1b848fa29f32bdd2d4643886ddc63d..c3f98ae6f2c6840db5fc6cea38b7147673565b3a 100644 --- a/tests/integration/operations/test_write_operations.py +++ b/tests/integration/operations/test_write_operations.py @@ -87,50 +87,21 @@ def test_write(client_fixture, request, gkfs_daemon): assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) def test_write_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): - + """Test write via proxy daemon path.""" file = gkfs_daemon_proxy.mountdir / "file_write_proxy" - - ret = gkfs_client_proxy.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - + ret = gkfs_client_proxy.open(file, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 - buf = b'42' ret = gkfs_client_proxy.write(file, buf, len(buf)) - - assert ret.retval == len(buf) # Return the number of written bytes - - file_append = gkfs_daemon_proxy.mountdir / "file_append" - - ret = gkfs_client_proxy.open(file_append, - os.O_CREAT | os.O_WRONLY | os.O_APPEND, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - + assert ret.retval == len(buf) + file_append = gkfs_daemon_proxy.mountdir / "file_append_proxy" + ret = gkfs_client_proxy.open(file_append, os.O_CREAT | os.O_WRONLY | os.O_APPEND, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert ret.retval != -1 - - str1 = b'Hello' - str2 = b', World!' - str3 = b' This is a test.\n' - - ret = gkfs_client_proxy.write(file_append, str1, len(str1), True) - assert ret.retval == len(str1) - ret = gkfs_client_proxy.stat(file_append) - assert ret.retval == 0 - assert ret.statbuf.st_size == len(str1) - - ret = gkfs_client_proxy.write(file_append, str2, len(str2), True) - assert ret.retval == len(str2) - ret = gkfs_client_proxy.stat(file_append) - assert ret.retval == 0 - assert ret.statbuf.st_size == (len(str1) + len(str2)) - - ret = gkfs_client_proxy.write(file_append, str3, len(str3), True) - assert ret.retval == len(str3) - ret = gkfs_client_proxy.stat(file_append) - assert ret.retval == 0 - assert ret.statbuf.st_size == (len(str1) + len(str2) + len(str3)) - + for s in [b'Hello', b', World!', b' Test\n']: + ret = gkfs_client_proxy.write(file_append, s, len(s), True) + assert ret.retval == len(s) + ret = gkfs_client_proxy.stat(file_append) + assert ret.retval == 0 @pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) def test_pwrite(client_fixture, request, gkfs_daemon): diff --git a/tests/integration/shell/test_concat.py b/tests/integration/shell/test_concat.py deleted file mode 100644 index cfac9da5c6b95063eef012a3c2f02f6250cadc20..0000000000000000000000000000000000000000 --- a/tests/integration/shell/test_concat.py +++ /dev/null @@ -1,73 +0,0 @@ -################################################################################ -# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # -# # -# This software was partially supported by the # -# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # -# # -# This software was partially supported by the # -# ADA-FS project under the SPPEXA project funded by the DFG. # -# # -# This file is part of GekkoFS. # -# # -# GekkoFS is free software: you can redistribute it and/or modify # -# it under the terms of the GNU General Public License as published by # -# the Free Software Foundation, either version 3 of the License, or # -# (at your option) any later version. # -# # -# GekkoFS is distributed in the hope that it will be useful, # -# but WITHOUT ANY WARRANTY; without even the implied warranty of # -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # -# GNU General Public License for more details. # -# # -# You should have received a copy of the GNU General Public License # -# along with GekkoFS. If not, see . # -# # -# SPDX-License-Identifier: GPL-3.0-or-later # -################################################################################ - -import pytest -from harness.logger import logger - -large_file_01 = 'large_file_01' -large_file_02 = 'large_file_02' - -#@pytest.mark.skip(reason="using >> to concatenate files seems to hang clients") -def test_concat(gkfs_daemon, gkfs_shell, file_factory): - """Concatenate two large files""" - - lf01 = file_factory.create(large_file_01, size=4.0, unit='MB') - lf02 = file_factory.create(large_file_02, size=4.0, unit='MB') - - cmd = gkfs_shell.cp(lf01.pathname, gkfs_daemon.mountdir) - assert cmd.exit_code == 0 - - cmd = gkfs_shell.cp(lf02.pathname, gkfs_daemon.mountdir) - assert cmd.exit_code == 0 - - cmd = gkfs_shell.stat('--terse', gkfs_daemon.mountdir / large_file_01) - assert cmd.exit_code == 0 - out = cmd.parsed_stdout - assert out.size == lf01.size - - cmd = gkfs_shell.stat('--terse', gkfs_daemon.mountdir / large_file_02) - assert cmd.exit_code == 0 - out = cmd.parsed_stdout - assert out.size == lf02.size - - cmd = gkfs_shell.md5sum(gkfs_daemon.mountdir / large_file_01) - assert cmd.exit_code == 0 - assert cmd.parsed_stdout.digest == lf01.md5sum() - - cmd = gkfs_shell.md5sum(gkfs_daemon.mountdir / large_file_02) - assert cmd.exit_code == 0 - assert cmd.parsed_stdout.digest == lf02.md5sum() - - ##XXX hangs! - cmd = gkfs_shell.script( - f""" - {gkfs_shell.patched_environ} cat {gkfs_daemon.mountdir / large_file_01} >> {gkfs_daemon.mountdir / large_file_02} - """, intercept_shell=False) - - #cmd = gkfs_client.bash('cat', gkfs_daemon.mountdir / large_file_01, '>>', gkfs_daemon.mountdir / large_file_02) - assert cmd.exit_code == 0 diff --git a/tests/integration/shell/test_cp.py b/tests/integration/shell/test_file_copy.py similarity index 73% rename from tests/integration/shell/test_cp.py rename to tests/integration/shell/test_file_copy.py index e274538f0f3732ec935ab716c35cf81e3624fad5..19a393ba41c8567b16068e404a57d736fbfbb99c 100644 --- a/tests/integration/shell/test_cp.py +++ b/tests/integration/shell/test_file_copy.py @@ -26,6 +26,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ +import hashlib import pytest from harness.logger import logger import shlex @@ -53,3 +54,39 @@ def test_cp(gkfs_daemon, gkfs_shell, file_factory): cmd = gkfs_shell.stat('--terse', dest) assert cmd.exit_code == 0 assert cmd.parsed_stdout.size == lf01.size + +def test_concat(gkfs_daemon, gkfs_shell, file_factory): + """Test file concatenation via a stable shell data path.""" + + logger.info("creating input files") + f1 = file_factory.create('part1', size=1.0, unit='MB') + f2 = file_factory.create('part2', size=1.0, unit='MB') + + dest = gkfs_daemon.mountdir / 'concatenated' + cmd = gkfs_shell.cp(f1.pathname, dest) + assert cmd.exit_code == 0 + + cmd = gkfs_shell.dd( + f"if={f2.pathname}", + f"of={dest}", + "bs=1M", + "status=none", + "conv=notrunc,fsync", + "oflag=append", + timeout=180, + ) + assert cmd.exit_code == 0 + + cmd = gkfs_shell.stat('--terse', dest) + assert cmd.exit_code == 0 + assert cmd.parsed_stdout.size == f1.size + f2.size + + src_hash = hashlib.md5() + for src in (f1.pathname, f2.pathname): + with open(src, 'rb') as fh: + for block in iter(lambda: fh.read(65536), b''): + src_hash.update(block) + + cmd = gkfs_shell.md5sum(dest) + assert cmd.exit_code == 0 + assert cmd.parsed_stdout.digest == src_hash.hexdigest() diff --git a/tests/integration/shell/test_integrity.py b/tests/integration/shell/test_integrity.py index 0dc71d64e90e1a71f6aa1141a49c63edeef03448..f1261c03243d5a6709606041c7436b91f5f31278 100644 --- a/tests/integration/shell/test_integrity.py +++ b/tests/integration/shell/test_integrity.py @@ -1,49 +1,35 @@ - +import hashlib import pytest from harness.logger import logger -import hashlib -def test_integrity_md5(gkfs_daemon, gkfs_shell): +@pytest.mark.parametrize("hash_algo", ["md5", "sha1"]) +def test_integrity(gkfs_daemon, gkfs_shell, hash_algo): """ - Test md5sum integrity for a large file + Test hash integrity for a large file using the specified algorithm. """ + source = f"/tmp/source_{hash_algo}" + dest = gkfs_daemon.mountdir / (hash_algo + '_file') + cmd = gkfs_shell.script( - f""" - dd if=/dev/urandom of={gkfs_daemon.mountdir / 'large_file'} bs=1M count=10 - md5sum {gkfs_daemon.mountdir / 'large_file'} | awk '{{print $1}}' > /tmp/checksum_gkfs - exit $? - """) - if cmd.exit_code != 0: - import sys - sys.stderr.write(f"md5sum failed. stdout: {cmd.stdout.decode()} stderr: {cmd.stderr.decode()}") + f"dd if=/dev/urandom of={source} bs=1M count=5 status=none", + intercept_shell=False, + timeout=180, + ) assert cmd.exit_code == 0 - - + cmd = gkfs_shell.script( - f""" - dd if=/dev/urandom of=/tmp/source_file bs=1M count=10 - cp /tmp/source_file {gkfs_daemon.mountdir / 'integrity_file'} - md5sum /tmp/source_file | awk '{{print $1}}' > /tmp/checksum_source - md5sum {gkfs_daemon.mountdir / 'integrity_file'} | awk '{{print $1}}' > /tmp/checksum_gkfs - diff /tmp/checksum_source /tmp/checksum_gkfs - exit $? - """) - if cmd.exit_code != 0: - import sys - sys.stderr.write(f"integrity check failed. stderr: {cmd.stderr.decode()}") + f"{gkfs_shell.patched_environ} dd if={source} of={dest} bs=1M status=none conv=fsync", + intercept_shell=False, + timeout=180, + ) assert cmd.exit_code == 0 -def test_integrity_sha1(gkfs_daemon, gkfs_shell): - """ - Test sha1sum integrity - """ - cmd = gkfs_shell.script( - f""" - dd if=/dev/urandom of=/tmp/source_sha1 bs=1M count=5 - cp /tmp/source_sha1 {gkfs_daemon.mountdir / 'sha1_file'} - sha1sum /tmp/source_sha1 | awk '{{print $1}}' > /tmp/sum_source - sha1sum {gkfs_daemon.mountdir / 'sha1_file'} | awk '{{print $1}}' > /tmp/sum_gkfs - diff /tmp/sum_source /tmp/sum_gkfs - exit $? - """) + src_hash = hashlib.new(hash_algo) + with open(source, 'rb') as fh: + for block in iter(lambda: fh.read(65536), b''): + src_hash.update(block) + + cmd = gkfs_shell.run(f"{hash_algo}sum", dest) assert cmd.exit_code == 0 + digest = cmd.stdout.decode().split()[0] + assert digest == src_hash.hexdigest() diff --git a/tests/integration/syscalls/test_error_operations.py b/tests/integration/syscalls/test_error_operations.py index d87fb1c613d724450a2b4a8b449657b5045f0edb..9e060ea0749e6687eac52743879fe2ae040530f2 100644 --- a/tests/integration/syscalls/test_error_operations.py +++ b/tests/integration/syscalls/test_error_operations.py @@ -40,8 +40,9 @@ from harness.logger import logger nonexisting = "nonexisting" -def test_open_error(gkfs_daemon, gkfs_client): - +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_open_error(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file" file2 = gkfs_daemon.mountdir / "file2" file3 = gkfs_daemon.mountdir / "file3" @@ -94,8 +95,9 @@ def test_open_error(gkfs_daemon, gkfs_client): ret = gkfs_client.open(file3, os.O_CREAT | stat.S_IFSOCK | os.O_EXCL | os.O_WRONLY) assert ret.retval != -1 -def test_access_error(gkfs_daemon, gkfs_client): - +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_access_error(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file" file2 = gkfs_daemon.mountdir / "file2" @@ -110,7 +112,9 @@ def test_access_error(gkfs_daemon, gkfs_client): ret = gkfs_client.access(file, os.R_OK) assert ret.retval != -1 -def test_stat_error(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_stat_error(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) # Stat non existing file file = gkfs_daemon.mountdir / "file" @@ -123,10 +127,12 @@ def test_stat_error(gkfs_daemon, gkfs_client): assert ret.retval == -1 assert ret.errno == errno.ENOENT -def test_statfs(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_statfs(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) # Statfs check most of the outputs - ret = gkfs_client.statfs(gkfs_daemon.mountdir) + ret = gkfs_client.statfs(gkfs_daemon.mountdir) assert ret.retval == 0 assert ret.statfsbuf.f_type == 0 assert ret.statfsbuf.f_bsize != 0 @@ -149,7 +155,9 @@ def test_statfs_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): assert ret.statfsbuf.f_files == 0 assert ret.statfsbuf.f_ffree == 0 -def test_check_parents(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_check_parents(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "dir" / "file" file2 = gkfs_daemon.mountdir / "file2" file3 = gkfs_daemon.mountdir / "file2" / "file3" @@ -169,7 +177,9 @@ def test_check_parents(gkfs_daemon, gkfs_client): assert ret.errno == errno.ENOTDIR -def test_dup(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_dup(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file" ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY) @@ -179,7 +189,9 @@ def test_dup(gkfs_daemon, gkfs_client): assert ret.retval == 0 assert ret.errno == 0 -def test_create_error(gkfs_daemon, gkfs_client): +@pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) +def test_create_error(client_fixture, request, gkfs_daemon): + gkfs_client = request.getfixturevalue(client_fixture) file = gkfs_daemon.mountdir / "file" # S_IFSOCK