Line data Source code
1 : /*
2 : Copyright 2018-2024, Barcelona Supercomputing Center (BSC), Spain
3 : Copyright 2015-2024, Johannes Gutenberg Universitaet Mainz, Germany
4 :
5 : This software was partially supported by the
6 : EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
7 :
8 : This software was partially supported by the
9 : ADA-FS project under the SPPEXA project funded by the DFG.
10 :
11 : This file is part of GekkoFS.
12 :
13 : GekkoFS is free software: you can redistribute it and/or modify
14 : it under the terms of the GNU General Public License as published by
15 : the Free Software Foundation, either version 3 of the License, or
16 : (at your option) any later version.
17 :
18 : GekkoFS is distributed in the hope that it will be useful,
19 : but WITHOUT ANY WARRANTY; without even the implied warranty of
20 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 : GNU General Public License for more details.
22 :
23 : You should have received a copy of the GNU General Public License
24 : along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.
25 :
26 : SPDX-License-Identifier: GPL-3.0-or-later
27 : */
28 : /**
29 : * @brief Chunk storage definitions handles all interactions with the node-local
30 : * storage system.
31 : */
32 :
33 : #include <daemon/backend/data/data_module.hpp>
34 : #include <daemon/backend/data/chunk_storage.hpp>
35 : #include <daemon/backend/data/file_handle.hpp>
36 : #include <common/path_util.hpp>
37 :
38 : #include <cerrno>
39 :
40 : #include <filesystem>
41 : #include <spdlog/spdlog.h>
42 :
43 : extern "C" {
44 : #include <sys/statfs.h>
45 : #include <sys/stat.h>
46 : #include <fcntl.h>
47 : }
48 :
49 : namespace fs = std::filesystem;
50 : using namespace std;
51 :
52 : namespace gkfs::data {
53 :
54 : // private functions
55 :
56 : string
57 284 : ChunkStorage::absolute(const string& internal_path) const {
58 284 : assert(gkfs::path::is_relative(internal_path));
59 284 : return fmt::format("{}/{}", root_path_, internal_path);
60 : }
61 :
62 : /**
63 : * @internal
64 : * All GekkoFS files are placed within the rootdir directory, each GekkoFS file
65 : * is represented by a directory on the local file system.
66 : * We do not mirror a directory hierarchy on the local file system.
67 : * Therefore the path /mnt/gekkofs_mount/foo/bar has the internal path
68 : * /tmp/rootdir/<pid>/data/chunks/foo:bar. Each chunk is then its own file,
69 : * numbered by its index, e.g., /tmp/rootdir/<pid>/data/chunks/foo:bar/0 for the
70 : * first chunk file.
71 : * @endinternal
72 : */
73 : string
74 283 : ChunkStorage::get_chunks_dir(const string& file_path) {
75 283 : assert(gkfs::path::is_absolute(file_path));
76 284 : string chunk_dir = file_path.substr(1);
77 284 : ::replace(chunk_dir.begin(), chunk_dir.end(), '/', ':');
78 284 : return chunk_dir;
79 : }
80 :
81 : string
82 176 : ChunkStorage::get_chunk_path(const string& file_path,
83 : gkfs::rpc::chnk_id_t chunk_id) {
84 352 : return fmt::format("{}/{}", get_chunks_dir(file_path), chunk_id);
85 : }
86 :
87 : void
88 100 : ChunkStorage::init_chunk_space(const string& file_path) const {
89 100 : auto chunk_dir = absolute(get_chunks_dir(file_path));
90 100 : auto err = mkdir(chunk_dir.c_str(), 0750);
91 100 : if(err == -1 && errno != EEXIST) {
92 0 : auto err_str = fmt::format(
93 : "{}() Failed to create chunk directory. File: '{}', Error: '{}'",
94 0 : __func__, file_path, errno);
95 0 : throw ChunkStorageException(errno, err_str);
96 : }
97 100 : }
98 :
99 : // public functions
100 :
101 33 : ChunkStorage::ChunkStorage(string& path, const size_t chunksize)
102 33 : : root_path_(path), chunksize_(chunksize) {
103 : /* Get logger instance and set it for data module and chunk storage */
104 66 : GKFS_DATA_MOD->log(spdlog::get(GKFS_DATA_MOD->LOGGER_NAME));
105 33 : assert(GKFS_DATA_MOD->log());
106 33 : log_ = spdlog::get(GKFS_DATA_MOD->LOGGER_NAME);
107 33 : assert(log_);
108 33 : assert(gkfs::path::is_absolute(root_path_));
109 : // Verify that we have sufficient access for chunk directories
110 33 : if(access(root_path_.c_str(), W_OK | R_OK) != 0) {
111 0 : auto err_str = fmt::format(
112 : "{}() Insufficient permissions to create chunk directories in path '{}'",
113 0 : __func__, root_path_);
114 0 : throw ChunkStorageException(EPERM, err_str);
115 : }
116 33 : log_->debug("{}() Chunk storage initialized with path: '{}'", __func__,
117 33 : root_path_);
118 33 : }
119 :
120 : void
121 6 : ChunkStorage::destroy_chunk_space(const string& file_path) const {
122 6 : auto chunk_dir = absolute(get_chunks_dir(file_path));
123 6 : try {
124 : // Note: remove_all does not throw an error when path doesn't exist.
125 6 : auto n = fs::remove_all(chunk_dir);
126 12 : log_->debug("{}() Removed '{}' files and directories from '{}'",
127 6 : __func__, n, chunk_dir);
128 0 : } catch(const fs::filesystem_error& e) {
129 0 : auto err_str = fmt::format(
130 : "{}() Failed to remove chunk directory. Path: '{}', Error: '{}'",
131 0 : __func__, chunk_dir, e.what());
132 0 : throw ChunkStorageException(e.code().value(), err_str);
133 : }
134 6 : }
135 :
136 : /**
137 : * @internal
138 : * Refer to
139 : * https://www.gnu.org/software/libc/manual/html_node/I_002fO-Primitives.html
140 : * for pwrite behavior.
141 : * @endinternal
142 : */
143 : ssize_t
144 100 : ChunkStorage::write_chunk(const string& file_path,
145 : gkfs::rpc::chnk_id_t chunk_id, const char* buf,
146 : size_t size, off64_t offset) const {
147 :
148 100 : assert((offset + size) <= chunksize_);
149 : // may throw ChunkStorageException on failure
150 100 : init_chunk_space(file_path);
151 :
152 100 : auto chunk_path = absolute(get_chunk_path(file_path, chunk_id));
153 :
154 100 : FileHandle fh(open(chunk_path.c_str(), O_WRONLY | O_CREAT, 0640),
155 200 : chunk_path);
156 100 : if(!fh.valid()) {
157 0 : auto err_str = fmt::format(
158 : "{}() Failed to open chunk file for write. File: '{}', Error: '{}'",
159 0 : __func__, chunk_path, ::strerror(errno));
160 0 : throw ChunkStorageException(errno, err_str);
161 : }
162 :
163 : size_t wrote_total{};
164 100 : ssize_t wrote{};
165 :
166 100 : do {
167 200 : wrote = pwrite(fh.native(), buf + wrote_total, size - wrote_total,
168 100 : offset + wrote_total);
169 :
170 100 : if(wrote < 0) {
171 : // retry if a signal or anything else has interrupted the read
172 : // system call
173 0 : if(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
174 0 : continue;
175 0 : auto err_str = fmt::format(
176 : "{}() Failed to write chunk file. File: '{}', size: '{}', offset: '{}', Error: '{}'",
177 0 : __func__, chunk_path, size, offset, ::strerror(errno));
178 0 : throw ChunkStorageException(errno, err_str);
179 : }
180 100 : wrote_total += wrote;
181 100 : } while(wrote_total != size);
182 :
183 : // file is closed via the file handle's destructor.
184 200 : return wrote_total;
185 : }
186 :
187 : /**
188 : * @internal
189 : * Refer to
190 : * https://www.gnu.org/software/libc/manual/html_node/I_002fO-Primitives.html
191 : * for pread behavior
192 : * @endinternal
193 : */
194 : ssize_t
195 74 : ChunkStorage::read_chunk(const string& file_path, gkfs::rpc::chnk_id_t chunk_id,
196 : char* buf, size_t size, off64_t offset) const {
197 74 : assert((offset + size) <= chunksize_);
198 74 : auto chunk_path = absolute(get_chunk_path(file_path, chunk_id));
199 :
200 134 : FileHandle fh(open(chunk_path.c_str(), O_RDONLY), chunk_path);
201 74 : if(!fh.valid()) {
202 14 : auto err_str = fmt::format(
203 : "{}() Failed to open chunk file for read. File: '{}', Error: '{}'",
204 28 : __func__, chunk_path, ::strerror(errno));
205 14 : throw ChunkStorageException(errno, err_str);
206 : }
207 60 : size_t read_total = 0;
208 60 : ssize_t read = 0;
209 :
210 61 : do {
211 122 : read = pread64(fh.native(), buf + read_total, size - read_total,
212 61 : offset + read_total);
213 61 : if(read == 0) {
214 : /*
215 : * A value of zero indicates end-of-file (except if the value of the
216 : * size argument is also zero). This is not considered an error. If
217 : * you keep calling read while at end-of-file, it will keep
218 : * returning zero and doing nothing else. Hence, we break here.
219 : */
220 : break;
221 : }
222 :
223 50 : if(read < 0) {
224 : // retry if a signal or anything else has interrupted the read
225 : // system call
226 0 : if(errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
227 0 : continue;
228 0 : auto err_str = fmt::format(
229 : "Failed to read chunk file. File: '{}', size: '{}', offset: '{}', Error: '{}'",
230 0 : chunk_path, size, offset, ::strerror(errno));
231 0 : throw ChunkStorageException(errno, err_str);
232 : }
233 :
234 : #ifndef NDEBUG
235 50 : if(read_total + read < size) {
236 1 : log_->debug(
237 : "Read less bytes than requested: '{}'/{}. Total read was '{}'. This is not an error!",
238 16 : read, size - read_total, size);
239 : }
240 : #endif
241 50 : assert(read > 0);
242 50 : read_total += read;
243 50 : } while(read_total != size);
244 :
245 : // file is closed via the file handle's destructor.
246 120 : return read_total;
247 : }
248 :
249 : /**
250 : * @internal
251 : * Note eventual consistency here: While chunks are removed, there is no lock
252 : * that prevents other processes from modifying anything in that directory. It
253 : * is the application's responsibility to stop modifying the file while truncate
254 : * is executed.
255 : *
256 : * If an error is encountered when removing a chunk file, the function will
257 : * still remove all files and report the error afterwards with
258 : * ChunkStorageException.
259 : * @endinternal
260 : */
261 : void
262 2 : ChunkStorage::trim_chunk_space(const string& file_path,
263 : gkfs::rpc::chnk_id_t chunk_start) {
264 :
265 2 : auto chunk_dir = absolute(get_chunks_dir(file_path));
266 2 : const fs::directory_iterator end;
267 2 : auto err_flag = false;
268 39 : for(fs::directory_iterator chunk_file(chunk_dir); chunk_file != end;
269 33 : ++chunk_file) {
270 66 : auto chunk_path = chunk_file->path();
271 99 : auto chunk_id = std::stoul(chunk_path.filename().c_str());
272 33 : if(chunk_id >= chunk_start) {
273 30 : auto err = unlink(chunk_path.c_str());
274 30 : if(err == -1 && errno != ENOENT) {
275 0 : err_flag = true;
276 0 : log_->warn(
277 : "{}() Failed to remove chunk file. File: '{}', Error: '{}'",
278 0 : __func__, chunk_path.native(), ::strerror(errno));
279 : }
280 : }
281 : }
282 2 : if(err_flag)
283 0 : throw ChunkStorageException(
284 : EIO,
285 0 : fmt::format(
286 : "{}() One or more errors occurred when truncating '{}'",
287 0 : __func__, file_path));
288 2 : }
289 :
290 : void
291 2 : ChunkStorage::truncate_chunk_file(const string& file_path,
292 : gkfs::rpc::chnk_id_t chunk_id, off_t length) {
293 2 : auto chunk_path = absolute(get_chunk_path(file_path, chunk_id));
294 2 : assert(length > 0 &&
295 : static_cast<gkfs::rpc::chnk_id_t>(length) <= chunksize_);
296 2 : auto ret = truncate(chunk_path.c_str(), length);
297 2 : if(ret == -1) {
298 0 : auto err_str = fmt::format(
299 : "Failed to truncate chunk file. File: '{}', Error: '{}'",
300 0 : chunk_path, ::strerror(errno));
301 0 : throw ChunkStorageException(errno, err_str);
302 : }
303 2 : }
304 :
305 : /**
306 : * @internal
307 : * Return ChunkStat with following fields:
308 : * unsigned long chunk_size;
309 : unsigned long chunk_total;
310 : unsigned long chunk_free;
311 : * @endinternal
312 : */
313 : ChunkStat
314 2 : ChunkStorage::chunk_stat() const {
315 2 : struct statfs sfs {};
316 :
317 2 : if(statfs(root_path_.c_str(), &sfs) != 0) {
318 0 : auto err_str = fmt::format(
319 : "Failed to get filesystem statistic for chunk directory. Error: '{}'",
320 0 : ::strerror(errno));
321 0 : throw ChunkStorageException(errno, err_str);
322 : }
323 :
324 2 : log_->debug("Chunksize '{}', total '{}', free '{}'", sfs.f_bsize,
325 2 : sfs.f_blocks, sfs.f_bavail);
326 2 : auto bytes_total = static_cast<unsigned long long>(sfs.f_bsize) *
327 2 : static_cast<unsigned long long>(sfs.f_blocks);
328 2 : auto bytes_free = static_cast<unsigned long long>(sfs.f_bsize) *
329 2 : static_cast<unsigned long long>(sfs.f_bavail);
330 2 : return {chunksize_, bytes_total / chunksize_, bytes_free / chunksize_};
331 : }
332 :
333 : } // namespace gkfs::data
|