Merge branch 'python3-compatibility' into 'master' (e4559ade) · Commits · hpc / gekkofs

ifs/scripts/shutdown_adafs.py

100644 → 100755

+24 −23

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		# -- coding: utf-8 --

		from __future__ import print_function
		import argparse
		import time

		@@ -28,7 +29,7 @@ def check_dependencies():
		if pssh_path != '':
		PSSH_PATH = pssh_path
		return
		print '[ERR] parallel-ssh/pssh executable cannot be found. Please add it to the parameter list'
		print('[ERR] parallel-ssh/pssh executable cannot be found. Please add it to the parameter list')
		exit(1)


		@@ -64,39 +65,39 @@ def shutdown_system(daemon_pid_path, nodelist, sigkill):
		else:
		cmd_str = '%s "pkill -SIGTERM --pidfile \"%s\""' % (pssh, daemon_pid_path)
		if PRETEND:
		print 'Pretending: %s' % cmd_str
		print('Pretending: {}'.format(cmd_str))
		else:
		print 'Running: %s' % cmd_str
		print('Running: {}'.format(cmd_str))
		pssh_ret = util.exec_shell(cmd_str, True)
		err = False
		for line in pssh_ret:
		if 'FAILURE' in line.strip()[:30]:
		err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line[line.find('FAILURE'):].strip().split(' ')[1])
		print line
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line[line.find('FAILURE'):].strip().split(' ')[1]))
		print(line)
		if not err:
		if sigkill:
		print 'pssh daemon launch successfully executed. FS daemons have been force killed ...'
		print('pssh daemon launch successfully executed. FS daemons have been force killed ...')
		exit(1)
		else:
		print 'pssh daemon launch successfully executed. Checking for FS shutdown errors ...\n'
		print('pssh daemon launch successfully executed. Checking for FS shutdown errors ...\n')
		else:
		print '[ERR] with pssh. Aborting...'
		print('[ERR] with pssh. Aborting...')
		exit(1)

		if not PRETEND:
		print 'Give it some time (%d second) to finish up ...' % WAITTIME
		print('Give it some time ({} second) to finish up ...'.format(WAITTIME))
		for i in range(WAITTIME):
		print '%d\r' % (WAITTIME - i),
		print('{}\r'.format(WAITTIME - i))
		time.sleep(1)
		print 'Checking logs ...\n'
		print('Checking logs ...\n')

		cmd_chk_str = '%s "tail -4 /tmp/adafs_daemon.log"' % pssh
		if PRETEND:
		print 'Pretending: %s' % cmd_chk_str
		print('Pretending: {}'.format(cmd_chk_str))
		else:
		print 'Running: %s' % cmd_chk_str
		print('Running: {}'.format(cmd_chk_str))
		pssh_ret = util.exec_shell(cmd_chk_str, True)
		err = False
		fs_err = False
		@@ -105,21 +106,21 @@ def shutdown_system(daemon_pid_path, nodelist, sigkill):
		continue
		if 'Failure' in line.strip()[:30]:
		err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line[line.find('FAILURE'):].strip().split(' ')[1])
		print line
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line[line.find('FAILURE'):].strip().split(' ')[1]))
		print(line)
		else:
		# check for errors in log
		if not 'All services shut down.' in line[line.strip().find('\n') + 1:]:
		fs_err = True
		print '------------------------- WARN pssh -- Host "%s" -------------------------' % \
		(line.strip().split(' ')[3].split('\n')[0])
		print '%s' % line[line.find('\n') + 1:]
		print('------------------------- WARN pssh -- Host "{}" -------------------------'.format(\
		line.strip().split(' ')[3].split('\n')[0]))
		print('{}'.format(line[line.find('\n') + 1:]))

		if not err and not fs_err:
		print 'pssh logging check successfully executed. Looks prime.'
		print('pssh logging check successfully executed. Looks prime.')
		else:
		print '[WARN] while checking fs logs. Something might went wrong when shutting down'
		print('[WARN] while checking fs logs. Something might went wrong when shutting down')
		exit(1)


		@@ -160,4 +161,4 @@ Defaults to /tmp/hostfile_pssh''')
		WAITTIME = 5
		shutdown_system(args.daemonpidpath, args.nodelist, args.sigkill)

		print '\nNothing left to do; exiting. :)'
		print('\nNothing left to do; exiting. :)')

ifs/scripts/startup_adafs.py

100644 → 100755

+29 −28

Original line number	Diff line number	Diff line
		#!/usr/bin/env python
		# -- coding: utf-8 --

		from __future__ import print_function
		import argparse
		import time

		@@ -28,7 +29,7 @@ def check_dependencies():
		if pssh_path != '':
		PSSH_PATH = pssh_path
		return
		print '[ERR] parallel-ssh/pssh executable cannot be found. Please add it to the parameter list'
		print('[ERR] parallel-ssh/pssh executable cannot be found. Please add it to the parameter list')
		exit(1)


		@@ -74,21 +75,21 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu
		if cleanroot:
		cmd_rm_str = '%s "rm -rf %s/* %s/* && truncate -s 0 /tmp/adafs_daemon.log /tmp/adafs_preload.log"' % (pssh, rootdir, metadir)
		if PRETEND:
		print 'Pretending: %s' % cmd_rm_str
		print('Pretending: {}'.format(cmd_rm_str))
		else:
		print 'Running: %s' % cmd_rm_str
		print('Running: {}'.format(cmd_rm_str))
		pssh_ret = util.exec_shell(cmd_rm_str, True)
		err = False
		for line in pssh_ret:
		if 'FAILURE' in line.strip()[:30]:
		err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line[line.find('FAILURE'):].strip().split(' ')[1])
		print line
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line[line.find('FAILURE'):].strip().split(' ')[1]))
		print(line)
		if not err:
		print 'pssh daemon launch successfully executed. Root and Metadata dir are cleaned.\n'
		print('pssh daemon launch successfully executed. Root and Metadata dir are cleaned.\n')
		else:
		print '[ERR] with pssh. Aborting!'
		print('[ERR] with pssh. Aborting!')
		exit(1)

		# Start deamons
		@@ -109,57 +110,57 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu
		% (pssh, numactl, daemon_path, rootdir, metadir, mountdir, nodelist)

		if PRETEND:
		print 'Pretending: %s' % cmd_str
		print('Pretending: {}'.format(cmd_str))
		else:
		print 'Running: %s' % cmd_str
		print('Running: {}'.format(cmd_str))
		pssh_ret = util.exec_shell(cmd_str, True)
		err = False
		for line in pssh_ret:
		if 'FAILURE' in line.strip()[:30]:
		err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line[line.find('FAILURE'):].strip().split(' ')[1])
		print line
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line[line.find('FAILURE'):].strip().split(' ')[1]))
		print(line)
		if not err:
		print 'pssh daemon launch successfully executed. Checking for FS startup errors ...\n'
		print('pssh daemon launch successfully executed. Checking for FS startup errors ...\n')
		else:
		print '[ERR] with pssh. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!'
		print('[ERR] with pssh. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!')
		exit(1)

		if not PRETEND:
		print 'Give it some time (%d second) to startup ...' % WAITTIME
		print('Give it some time ({} second) to startup ...'.format(WAITTIME))
		for i in range(WAITTIME):
		print '%d\r' % (WAITTIME - i),
		print('{}\r'.format(WAITTIME - i)),
		time.sleep(1)

		# Check adafs logs for errors
		cmd_chk_str = '%s "head -5 /tmp/adafs_daemon.log"' % pssh
		if PRETEND:
		print 'Pretending: %s' % cmd_chk_str
		print('Pretending: {}'.format(cmd_chk_str))
		else:
		print 'Running: %s' % cmd_chk_str
		print('Running: {}'.format(cmd_chk_str))
		pssh_ret = util.exec_shell(cmd_chk_str, True)
		err = False
		fs_err = False
		for line in pssh_ret:
		if 'Failure' in line.strip()[:30]:
		err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line[line.find('FAILURE'):].strip().split(' ')[1])
		print line
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line[line.find('FAILURE'):].strip().split(' ')[1]))
		print(line)
		else:
		# check for errors in log
		if '[E]' in line[line.strip().find('\n') + 1:] or 'Assertion `err\'' in line[
		line.strip().find('\n') + 1:]:
		fs_err = True
		print '------------------------- ERROR pssh -- Host "%s" -------------------------' % \
		(line.strip().split(' ')[3].split('\n')[0])
		print '%s' % line[line.find('\n') + 1:]
		print('------------------------- ERROR pssh -- Host "{}" -------------------------'.format(\
		line.strip().split(' ')[3].split('\n')[0]))
		print('{}'.format(line[line.find('\n') + 1:]))

		if not err and not fs_err:
		print 'pssh logging check successfully executed. Looks prime.'
		print('pssh logging check successfully executed. Looks prime.')
		else:
		print '[ERR] while checking fs logs. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!'
		print('[ERR] while checking fs logs. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!')
		exit(1)


		@@ -208,4 +209,4 @@ Defaults to /tmp/hostfile_pssh''')
		WAITTIME = 5
		init_system(args.daemonpath, args.rootdir, args.metadir, args.mountdir, args.nodelist, args.cleanroot, args.numactl)

		print '\nNothing left to do; exiting. :)'
		print('\nNothing left to do; exiting. :)')

ifs/scripts/util/util.py

100644 → 100755

+13 −12

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@
		# Marc-Andre Vef
		# Version 0.2 (06/19/2015)

		from __future__ import print_function
		import collections
		import shutil
		import sys
		@@ -24,8 +25,8 @@ def create_dir(path):
		if not os.path.exists(path):
		os.makedirs(path)
		except OSError as e:
		print 'Error: Output directory could not be created.'
		print e.strerror
		print('Error: Output directory could not be created.')
		print(e.strerror)
		sys.exit(1)


		@@ -34,8 +35,8 @@ def rm_rf(path):
		try:
		shutil.rmtree(path)
		except shutil.Error as e:
		print 'Warning: Could not delete path %s' % path
		print e.strerror
		print('Warning: Could not delete path {}'.format(path))
		print(e.strerror)


		def rm_file(path):
		@@ -43,8 +44,8 @@ def rm_file(path):
		try:
		os.remove(path)
		except OSError as e:
		print 'Warning: Could not delete file %s' % path
		print e.strerror
		print('Warning: Could not delete file {}'.format(path))
		print(e.strerror)


		def tprint(toprint, nobreak=False):
		@@ -57,9 +58,9 @@ def tprint(toprint, nobreak=False):
		"""
		curr_time = time.strftime('[%H:%M:%S]')
		if nobreak:
		print '%s\t%s' % (curr_time, toprint),
		print('{}\t{}'.format(curr_time, toprint))
		else:
		print '%s\t%s' % (curr_time, toprint)
		print('{}\t{}'.format(curr_time, toprint))


		def exec_shell(cmd, suppress_output=False):
		@@ -92,8 +93,8 @@ def exec_shell(cmd, suppress_output=False):
		out_tuple = collections.namedtuple('shell_out', ['err', 'output'])
		return out_tuple(err=stderr.strip(), output=shell_out.strip())
		except OSError as e:
		print 'ERR when executing shell command'
		print e.strerror
		print('ERR when executing shell command')
		print(e.strerror)


		def check_shell_out(msg):
		@@ -135,7 +136,7 @@ def create_pssh_hostfile(hostfile, hostfile_pssh):
		with open(hostfile_pssh, 'a') as wf:
		wf.write(line.strip().split(' ')[0] + '\n')
		except IOError as e:
		print 'ERR while creating pssh compatible hostfile'
		print e.strerror
		print('ERR while creating pssh compatible hostfile')
		print(e.strerror)
		return False
		return True