Reformat examples (94e84f40) · Commits · hpc / gekkofs

examples/gfind/gfind.cpp

+410 −404

Original line number	Diff line number	Diff line
		@@ -58,10 +58,9 @@ struct dirent_extended {

		/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with
		* -fPIC, if not will segfault */
		extern "C" int gkfs_getsingleserverdir(const char *path,
		struct dirent_extended *dirp,
		unsigned int count, int server)
		__attribute__((weak));
		extern "C" int
		gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp,
		unsigned int count, int server) __attribute__((weak));

		/* PFIND OPTIONS EXTENDED We need to add the GekkoFS mount dir and the number of
		* servers */
		@@ -85,7 +84,8 @@ typedef struct {
		int max_entries_per_iter;
		int steal_from_next; // if true, then steal from the next process
		int parallel_single_dir_access; // if 1, use hashing to parallelize single
		// directory access, if 2 sequential increment
		// directory access, if 2 sequential
		// increment

		int verbosity;
		} pfind_options_t;
		@@ -103,12 +103,14 @@ int pfind_rank;

		static pfind_options_t* opt;

		void pfind_abort(const string str) {
		void
		pfind_abort(const string str) {
		printf("%s", str.c_str());
		exit(1);
		}

		static void pfind_print_help(pfind_options_t *res) {
		static void
		pfind_print_help(pfind_options_t* res) {
		printf("pfind \nSynopsis:\n"
		"pfind <workdir> [-newer <timestamp file>] [-size <size>c] [-name "
		"<substr>] [-regex <regex>] [-S <numserver>] [-M <mountdir>]\n"
		@@ -120,13 +122,13 @@ static void pfind_print_help(pfind_options_t *res) {
		"Optional flags\n"
		"\t-h: prints the help\n"
		"\t--help: prints the help without initializing MPI\n",
		res->workdir, res->timestamp_file, res->name_pattern, res->num_servers,
		res->mountdir);
		res->workdir, res->timestamp_file, res->name_pattern,
		res->num_servers, res->mountdir);
		}
		MPI_Comm pfind_com;
		int pfind_size;
		pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help,
		MPI_Comm com) {
		pfind_options_t*
		pfind_parse_args(int argc, char** argv, int force_print_help, MPI_Comm com) {
		MPI_Comm_rank(com, &pfind_rank);
		MPI_Comm_size(com, &pfind_size);
		pfind_com = com;
		@@ -231,7 +233,8 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help,
		res->steal_from_next = 1;
		break;
		case 'x':
		/* ignore fake arg that we added when we processed the extra args */
		/* ignore fake arg that we added when we processed the extra
		* args */
		break;
		case 'P':
		res->print_by_process = 1;
		@@ -262,12 +265,6 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help,
		case 's':
		res->stonewall_timer = atol(optarg);
		break;
		// case 'S':
		// res->num_servers = atoi(optarg);
		// break;
		// case 'M':
		// res->mountdir = strdup(optarg);
		// break;
		case 'v':
		res->verbosity++;
		break;
		@@ -299,14 +296,16 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help,
		}

		/* Master send a new path to the workers */
		void send_newPath(string path) {
		void
		send_newPath(string path) {
		auto count = path.size() + 1;
		MPI_Bcast(&count, 1, MPI_INT, 0, MPI_COMM_WORLD);
		MPI_Bcast((void*) path.c_str(), count, MPI_CHAR, 0, MPI_COMM_WORLD);
		}

		/* Clients get a new path, getting a "0" size char means there is no new path*/
		string recv_newPath() {
		string
		recv_newPath() {
		int count;
		MPI_Bcast(&count, 1, MPI_INT, 0, MPI_COMM_WORLD);
		if(count == 0)
		@@ -323,15 +322,16 @@ string recv_newPath() {
		* server, which is enough for most cases
		*
		*/
		void dirProcess(const string path, unsigned long long &checked,
		void
		dirProcess(const string path, unsigned long long& checked,
		unsigned long long& found, queue<string>& dirs,
		unsigned int world_rank, unsigned int world_size,
		pfind_options_t* opt) {
		struct dirent_extended* getdir = (struct dirent_extended*) malloc(
		(sizeof(struct dirent_extended) + 255) * 1024 * 100);
		memset(getdir, 0, (sizeof(struct dirent_extended) + 255) * 1024 * 100);
		// cout << "PROCESSING " << world_rank << "/"<< world_size << " = " << path <<
		// endl;
		// cout << "PROCESSING " << world_rank << "/"<< world_size << " = " << path
		// << endl;


		int servers_per_node = ceil(opt->num_servers / (world_size - 1));
		@@ -359,8 +359,8 @@ void dirProcess(const string path, unsigned long long &checked,
		slash = "/";
		checked++;
		dirs.push(path + slash + temp->d_name);
		temp = reinterpret_cast<dirent_extended >(reinterpret_cast<char >(temp) +
		temp->d_reclen);
		temp = reinterpret_cast<dirent_extended*>(
		reinterpret_cast<char*>(temp) + temp->d_reclen);
		continue;
		}
		/* Find filtering */
		@@ -369,18 +369,20 @@ void dirProcess(const string path, unsigned long long &checked,
		if((uint64_t) temp->ctime < runtime.ctime_min)
		timeOK = false;
		}
		if (timeOK and (temp->size == opt->size or opt->size == std::numeric_limits<uint64_t>::max()))
		if(timeOK and (temp->size == opt->size or
		opt->size == std::numeric_limits<uint64_t>::max()))
		if(!(opt->name_pattern &&
		regexec(&opt->name_regex, temp->d_name, 0, nullptr, 0)))
		found++;
		checked++;
		temp =
		reinterpret_cast<dirent_extended >(reinterpret_cast<char >(temp) + temp->d_reclen);
		temp = reinterpret_cast<dirent_extended*>(
		reinterpret_cast<char*>(temp) + temp->d_reclen);
		}
		}
		}

		int process(char *processor_name, int world_rank, int world_size,
		int
		process(char* processor_name, int world_rank, int world_size,
		pfind_options_t* opt) {
		// Print off a hello world message

		@@ -427,7 +429,8 @@ int process(char *processor_name, int world_rank, int world_size,
		MPI_Get_count(&mpistatus, MPI_CHAR, &count);

		std::vector<char> buf(count);
		MPI_Recv(buf.data(), count, MPI_CHAR, i, 0, MPI_COMM_WORLD, &mpistatus);
		MPI_Recv(buf.data(), count, MPI_CHAR, i, 0, MPI_COMM_WORLD,
		&mpistatus);

		if(count == 0) {
		continue;
		@@ -445,10 +448,10 @@ int process(char *processor_name, int world_rank, int world_size,

		MPI_Barrier(MPI_COMM_WORLD);

		unsigned long long *Array_checked =
		(unsigned long long )malloc(sizeof(unsigned long long) world_size);
		unsigned long long *Array_found =
		(unsigned long long )malloc(sizeof(unsigned long long) world_size);
		unsigned long long* Array_checked = (unsigned long long*) malloc(
		sizeof(unsigned long long) * world_size);
		unsigned long long* Array_found = (unsigned long long*) malloc(
		sizeof(unsigned long long) * world_size);
		unsigned long long checked = 0;
		unsigned long long found = 0;

		@@ -474,10 +477,12 @@ int process(char *processor_name, int world_rank, int world_size,
		if(toProcess == "Terminate") {
		break;
		}
		// cout << "REceived " << toProcess << " --- " << world_rank << endl;
		// cout << "REceived " << toProcess << " --- " << world_rank <<
		// endl;
		queue<string> dirs;

		dirProcess(toProcess, checked, found, dirs, world_rank, world_size, opt);
		dirProcess(toProcess, checked, found, dirs, world_rank, world_size,
		opt);
		// Send NEW DIRS to master
		while(!dirs.empty()) {
		string s = dirs.front();
		@@ -500,7 +505,8 @@ int process(char *processor_name, int world_rank, int world_size,
		return 0;
		}

		int main(int argc, char **argv) {
		int
		main(int argc, char** argv) {

		for(int i = 0; i < argc; i++) {
		if(strcmp(argv[i], "--help") == 0) {

examples/gfind/sfind.cpp

+315 −313

Original line number	Diff line number	Diff line
		@@ -57,10 +57,9 @@ struct dirent_extended {

		/* Function exported from GekkoFS LD_PRELOAD, code needs to be compiled with
		* -fPIC */
		extern "C" int gkfs_getsingleserverdir(const char *path,
		struct dirent_extended *dirp,
		unsigned int count, int server)
		__attribute__((weak));
		extern "C" int
		gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp,
		unsigned int count, int server) __attribute__((weak));

		/* PFIND OPTIONS EXTENDED We need to add the GekkoFS mount dir and the number of
		* servers */
		@@ -84,7 +83,8 @@ typedef struct {
		int max_entries_per_iter;
		int steal_from_next; // if true, then steal from the next process
		int parallel_single_dir_access; // if 1, use hashing to parallelize single
		// directory access, if 2 sequential increment
		// directory access, if 2 sequential
		// increment

		int verbosity;
		} pfind_options_t;
		@@ -102,12 +102,14 @@ int pfind_rank;

		static pfind_options_t* opt;

		void pfind_abort(const std::string str) {
		void
		pfind_abort(const std::string str) {
		printf("%s", str.c_str());
		exit(1);
		}

		static void pfind_print_help(pfind_options_t *res) {
		static void
		pfind_print_help(pfind_options_t* res) {
		printf("pfind \nSynopsis:\n"
		"pfind <workdir> [-newer <timestamp file>] [-size <size>c] [-name "
		"<substr>] [-regex <regex>] [-S <numserver>] [-M <mountdir>]\n"
		@@ -118,12 +120,13 @@ static void pfind_print_help(pfind_options_t *res) {
		"\t-M: mountdir = \"%s\"\n"
		"Optional flags\n"
		"\t-h: prints the help\n"
		"\t--help: prints the help without initializing MPI\n",res->workdir,
		res->timestamp_file, res->name_pattern, res->num_servers,
		res->mountdir );
		"\t--help: prints the help without initializing MPI\n",
		res->workdir, res->timestamp_file, res->name_pattern,
		res->num_servers, res->mountdir);
		}
		int pfind_size;
		pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help){
		pfind_options_t*
		pfind_parse_args(int argc, char** argv, int force_print_help) {

		pfind_rank = 0;
		pfind_size = 1;
		@@ -240,7 +243,8 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help){
		res->steal_from_next = 1;
		break;
		case 'x':
		/* ignore fake arg that we added when we processed the extra args */
		/* ignore fake arg that we added when we processed the extra
		* args */
		break;
		case 'P':
		res->print_by_process = 1;
		@@ -271,12 +275,6 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help){
		case 's':
		res->stonewall_timer = atol(optarg);
		break;
		// case 'S':
		// res->num_servers = atoi(optarg);
		// break;
		// case 'M':
		// res->mountdir = strdup(optarg);
		// break;
		case 'v':
		res->verbosity++;
		break;
		@@ -309,15 +307,16 @@ pfind_options_t pfind_parse_args(int argc, char *argv, int force_print_help){
		* server, which is enough for most cases
		*
		*/
		void dirProcess(const string path, unsigned long long &checked,
		void
		dirProcess(const string path, unsigned long long& checked,
		unsigned long long& found, queue<string>& dirs,
		unsigned int world_rank, unsigned int world_size,
		pfind_options_t* opt) {
		struct dirent_extended* getdir = (struct dirent_extended*) malloc(
		(sizeof(struct dirent_extended) + 255) * 1024 * 100);
		memset(getdir, 0, (sizeof(struct dirent_extended) + 255) * 1024 * 100);
		// cout << "PROCESSING " << world_rank << "/"<< world_size << " = " << path <<
		// endl;
		// cout << "PROCESSING " << world_rank << "/"<< world_size << " = " << path
		// << endl;

		for(auto server = 0; server < opt->num_servers; server++) {
		unsigned long long total_size = 0;
		@@ -337,8 +336,8 @@ void dirProcess(const string path, unsigned long long &checked,
		slash = "/";
		checked++;
		dirs.push(path + slash + temp->d_name);
		temp =
		reinterpret_cast<dirent_extended >(reinterpret_cast<char >(temp) + temp->d_reclen);
		temp = reinterpret_cast<dirent_extended*>(
		reinterpret_cast<char*>(temp) + temp->d_reclen);
		continue;
		}
		/* Find filtering */
		@@ -347,17 +346,20 @@ void dirProcess(const string path, unsigned long long &checked,
		if((uint64_t) temp->ctime < runtime.ctime_min)
		timeOK = false;
		}
		if (timeOK and (temp->size == opt->size or opt->size == std::numeric_limits<uint64_t>::max()))
		if(timeOK and (temp->size == opt->size or
		opt->size == std::numeric_limits<uint64_t>::max()))
		if(!(opt->name_pattern &&
		regexec(&opt->name_regex, temp->d_name, 0, nullptr, 0)))
		found++;
		checked++;
		temp = reinterpret_cast<dirent_extended >(reinterpret_cast<char >(temp) + temp->d_reclen);
		temp = reinterpret_cast<dirent_extended*>(
		reinterpret_cast<char*>(temp) + temp->d_reclen);
		}
		}
		}

		int process(pfind_options_t *opt) {
		int
		process(pfind_options_t* opt) {
		// Print off a hello world message
		unsigned long long found, checked;
		// INIT PFIND
		@@ -398,7 +400,8 @@ int process(pfind_options_t *opt) {
		return 0;
		}

		int main(int argc, char **argv) {
		int
		main(int argc, char** argv) {

		for(auto i = 0; i < argc; i++) {
		if(strcmp(argv[i], "--help") == 0) {
		@@ -412,5 +415,4 @@ int main(int argc, char **argv) {
		opt = pfind_parse_args(argc, argv, 0);

		process(opt);

		}