diff --git a/include/config.hpp b/include/config.hpp index 6f2f7cfb5672c3067f5de3c21e752481fcee8ad1..00679ccefa2ec1690b54bafa60c6fab3c20e3ef5 100644 --- a/include/config.hpp +++ b/include/config.hpp @@ -83,6 +83,7 @@ namespace io { * If buffer is not zeroed, sparse regions contain invalid data. */ constexpr auto zero_buffer_before_read = false; +constexpr auto enable_chunk_read_before_write = true; /* * When the daemon handler serves a read request, it starts tasklets (for each * chunk) from the io pool to read all chunks of that read request in parallel. diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 962dee8ec856c0481b2f14015868a37ce9ecfdf9..b5ff0fb9f7324e62b4631de47cbd272aad24afc4 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1104,7 +1104,62 @@ gkfs_do_write(gkfs::filemap::OpenFile& file, const char* buf, size_t count, count > gkfs::config::proxy::fwd_io_count_threshold) { ret_write = gkfs::rpc::forward_write_proxy(*path, buf, offset, count); } else { - ret_write = gkfs::rpc::forward_write(*path, buf, offset, count, 0); + // If we are going to write chunk 0 (0 to chunksize), with an offset > + // 0, and we are using ofi+verbs Do a forward_read from 0 to offset, + // concat read buffer (0..offset) with the rest of the chunk and write + // the first chunk. Then do a forward write of the rest of the data. + if(gkfs::config::io::enable_chunk_read_before_write && + offset > 0 && + offset < gkfs::config::rpc::chunksize && + gkfs::config::rpc::chunksize < count) { + // Read the first chunk + char* read_buf = new char[offset]; + std::set failed; + auto ret_read = gkfs::rpc::forward_read(*path, read_buf, 0, offset, + 0, failed); + if(ret_read.first != 0) { + LOG(ERROR, "Failed to read first chunk for RMW: {}", + ret_read.first); + delete[] read_buf; + errno = ret_read.first; + return -1; + } + + // Create a new buffer for the first chunk write + char* first_chunk_write_buf = + new char[gkfs::config::rpc::chunksize]; + // Copy existing data up to offset + memcpy(first_chunk_write_buf, read_buf, offset); + // Copy new data into the first chunk + memcpy(first_chunk_write_buf + offset, buf, + gkfs::config::rpc::chunksize - offset); + + // Write the first chunk + ret_write = + gkfs::rpc::forward_write(*path, first_chunk_write_buf, 0, + gkfs::config::rpc::chunksize, 0); + delete[] read_buf; + delete[] first_chunk_write_buf; + + if(ret_write.first != 0) { + LOG(ERROR, "Failed to write first chunk during RMW: {}", + ret_write.first); + errno = ret_write.first; + return -1; + } + + // Write the rest of the data + if(count > gkfs::config::rpc::chunksize) { + ret_write = gkfs::rpc::forward_write( + *path, buf + (gkfs::config::rpc::chunksize - offset), + gkfs::config::rpc::chunksize, + count - (gkfs::config::rpc::chunksize - offset), 0); + } + } else { + + + ret_write = gkfs::rpc::forward_write(*path, buf, offset, count, 0); + } } err = ret_write.first; write_size = ret_write.second; diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 35df1cf9a8bc6594cf1592d1b9191a880935afad..e58fba13c3f03b018c3312875db5652f50892041 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -1023,6 +1023,13 @@ socketcall_wrapper(long syscall_number, long& arg0, long& arg1, long& arg2, void hook_forwarded_syscall(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, long arg5, long result) { + + pthread_once(&key_once_control, make_key); + if (pthread_getspecific(reentrance_guard_key) != NULL) { + return; + } + pthread_setspecific(reentrance_guard_key, (void*) 1); + pthread_once(&key_once_control, make_key); if(pthread_getspecific(reentrance_guard_key) != NULL) {