Import random slicing from DaDiSi (5258e268) · Commits · David Auer / gekkofs

external/flat_segment_tree/flat_segment_tree.hpp

0 → 100644

+1221 −0

File added.

Preview size limit exceeded, changes collapsed.

external/flat_segment_tree/flat_segment_tree_node.hpp

0 → 100644

+321 −0

Original line number	Diff line number	Diff line
		/*************************************************************************
		*
		* Copyright (c) 2008-2010 Kohei Yoshida
		*
		* Permission is hereby granted, free of charge, to any person
		* obtaining a copy of this software and associated documentation
		* files (the "Software"), to deal in the Software without
		* restriction, including without limitation the rights to use,
		* copy, modify, merge, publish, distribute, sublicense, and/or sell
		* copies of the Software, and to permit persons to whom the
		* Software is furnished to do so, subject to the following
		* conditions:
		*
		* The above copyright notice and this permission notice shall be
		* included in all copies or substantial portions of the Software.
		*
		* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
		* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
		* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
		* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
		* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
		* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
		* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
		* OTHER DEALINGS IN THE SOFTWARE.
		*
		************************************************************************/

		#ifndef __MDDS_NODE_HXX__
		#define __MDDS_NODE_HXX__

		#include <iostream>
		#include <list>
		#include <cassert>

		#include <boost/intrusive_ptr.hpp>

		namespace mdds {

		#ifdef DEBUG_NODE_BASE
		size_t node_instance_count = 0;
		#endif

		template<typename _NodePtr, typename _NodeType>
		struct node_base
		{
		static size_t get_instance_count()
		{
		#ifdef DEBUG_NODE_BASE
		return node_instance_count;
		#else
		return 0;
		#endif
		}
		size_t refcount;

		_NodePtr parent; /// parent node
		_NodePtr left; /// left child node or previous sibling if it's a leaf node.
		_NodePtr right; /// right child node or next sibling if it's aleaf node.
		bool is_leaf;

		node_base(bool _is_leaf) :
		refcount(0),
		is_leaf(_is_leaf)
		{
		#ifdef DEBUG_NODE_BASE
		++node_instance_count;
		#endif
		}

		/**
		* When copying node, only the stored values should be copied.
		* Connections to the parent, left and right nodes must not be copied.
		*/
		node_base(const node_base& r) :
		refcount(0),
		is_leaf(r.is_leaf)
		{
		#ifdef DEBUG_NODE_BASE
		++node_instance_count;
		#endif
		}

		/**
		* Like the copy constructor, only the stored values should be copied.
		*/
		node_base& operator=(const node_base& r)
		{
		if (this == &r)
		// assignment to self.
		return *this;

		is_leaf = r.is_leaf;
		return *this;
		}

		~node_base()
		{
		#ifdef DEBUG_NODE_BASE
		--node_instance_count;
		#endif
		static_cast<_NodeType*>(this)->dispose();
		}
		};

		template<typename _NodePtr>
		inline void intrusive_ptr_add_ref(_NodePtr p)
		{
		++p->refcount;
		}

		template<typename _NodePtr>
		inline void intrusive_ptr_release(_NodePtr p)
		{
		--p->refcount;
		if (!p->refcount)
		delete p;
		}

		template<typename _NodePtr>
		void disconnect_all_nodes(_NodePtr p)
		{
		if (!p)
		return;

		p->left.reset();
		p->right.reset();
		p->parent.reset();
		}

		template<typename _NodePtr>
		void disconnect_leaf_nodes(_NodePtr left_node, _NodePtr right_node)
		{
		if (!left_node \|\| !right_node)
		return;

		// Go through all leaf nodes, and disconnect their links.
		_NodePtr cur_node = left_node;
		do
		{
		_NodePtr next_node = cur_node->right.get();
		disconnect_all_nodes(cur_node);
		cur_node = next_node;
		}
		while (cur_node != right_node);

		disconnect_all_nodes(right_node);
		}

		template<typename _NodePtr>
		void link_nodes(_NodePtr& left, _NodePtr& right)
		{
		left->right = right;
		right->left = left;
		}

		/**
		* Disconnect all non-leaf nodes so that their ref-counted instances will
		* all get destroyed afterwards.
		*/
		template<typename _NodePtr>
		void clear_tree(_NodePtr node)
		{
		if (!node)
		// Nothing to do.
		return;

		if (node->is_leaf)
		{
		node->parent.reset();
		return;
		}

		clear_tree(node->left.get());
		clear_tree(node->right.get());
		disconnect_all_nodes(node);
		}

		template<typename _NodePtr, typename _NodeType>
		_NodePtr make_parent_node(const _NodePtr& node1, const _NodePtr& node2)
		{
		_NodePtr parent_node(new _NodeType(false));
		node1->parent = parent_node;
		parent_node->left = node1;
		if (node2)
		{
		node2->parent = parent_node;
		parent_node->right = node2;
		}

		parent_node->fill_nonleaf_value(node1, node2);
		return parent_node;
		}

		template<typename _NodePtr, typename _NodeType>
		_NodePtr build_tree_non_leaf(const ::std::list<_NodePtr>& node_list)
		{
		size_t node_count = node_list.size();
		if (node_count == 1)
		{
		return node_list.front();
		}
		else if (node_count == 0)
		return _NodePtr();

		::std::list<_NodePtr> new_node_list;
		_NodePtr node_pair[2];
		typename ::std::list<_NodePtr>::const_iterator itr = node_list.begin();
		typename ::std::list<_NodePtr>::const_iterator itr_end = node_list.end();
		for (bool even_itr = false; itr != itr_end; ++itr, even_itr = !even_itr)
		{
		node_pair[even_itr] = *itr;
		if (even_itr)
		{
		_NodePtr parent_node = make_parent_node<_NodePtr, _NodeType>(node_pair[0], node_pair[1]);
		node_pair[0].reset();
		node_pair[1].reset();
		new_node_list.push_back(parent_node);
		}
		}

		if (node_pair[0])
		{
		// Un-paired node still needs a parent...
		_NodePtr parent_node = make_parent_node<_NodePtr, _NodeType>(node_pair[0], _NodePtr());
		node_pair[0].reset();
		node_pair[1].reset();
		new_node_list.push_back(parent_node);
		}

		// Move up one level, and do the same procedure until the root node is reached.
		return build_tree_non_leaf<_NodePtr, _NodeType>(new_node_list);
		}

		template<typename _NodePtr, typename _NodeType>
		_NodePtr build_tree(const _NodePtr& left_leaf_node)
		{
		if (!left_leaf_node)
		// The left leaf node is empty. Nothing to build.
		return _NodePtr();

		_NodePtr node1, node2;
		node1 = left_leaf_node;

		::std::list<_NodePtr> node_list;
		while (true)
		{
		node2 = node1->right;
		_NodePtr parent_node = make_parent_node<_NodePtr, _NodeType>(node1, node2);
		node_list.push_back(parent_node);

		if (!node2 \|\| !node2->right)
		// no more nodes. Break out of the loop.
		break;

		node1 = node2->right;
		}

		return build_tree_non_leaf<_NodePtr, _NodeType>(node_list);
		}

		#ifdef UNIT_TEST
		template<typename _NodePtr>
		size_t dump_tree_layer(const ::std::list<_NodePtr>& node_list, unsigned int level)
		{
		using ::std::cout;
		using ::std::endl;

		if (node_list.empty())
		return 0;

		size_t node_count = node_list.size();

		bool isLeaf = node_list.front()->is_leaf;
		cout << "level " << level << " (" << (isLeaf?"leaf":"non-leaf") << ")" << endl;

		::std::list<_NodePtr> newList;
		typename ::std::list<_NodePtr>::const_iterator itr = node_list.begin(), itrEnd = node_list.end();
		for (; itr != itrEnd; ++itr)
		{
		const _NodePtr& p = *itr;
		if (!p)
		{
		cout << "(x) ";
		continue;
		}

		p->dump_value();

		if (p->is_leaf)
		continue;

		if (p->left)
		{
		newList.push_back(p->left.get());
		if (p->right)
		newList.push_back(p->right.get());
		}
		}
		cout << endl;

		if (!newList.empty())
		node_count += dump_tree_layer(newList, level+1);

		return node_count;
		}

		template<typename _NodePtr>
		size_t dump_tree(_NodePtr root_node)
		{
		if (!root_node)
		return 0;

		::std::list<_NodePtr> node_list;
		node_list.push_back(root_node);
		return dump_tree_layer(node_list, 0);
		}
		#endif

		}

		#endif

include/global/random_slicing/Disk.h

0 → 100644

+190 −0

Original line number	Diff line number	Diff line
		// Adapted from https://sourceforge.net/p/dadisi/code/8/tree/trunk/dadisi/
		/*
		* File: Disk.h
		* Author: fermat
		*
		* Created on 20. Januar 2010, 10:39
		*/

		#ifndef _DISK_H
		#define _DISK_H

		#define __STDC_LIMIT_MACROS // required for limit macros
		#include<stdint.h>
		#include<xercesc/dom/DOMElement.hpp>
		#include<list>
		#include<string>

		#include "helper.h"

		namespace VDRIVE {

		/**
		* This class represents a Disk for the Distributor. Each Disk consists of
		* an unique ID (in the list of Disks used by this lib), a capacity and a
		* reference to some data, that can be used by the person instantiating a
		* Disk.
		*
		* @author Sascha Effert <fermat@uni-paderborn.de>
		*/
		class Disk {
		public:

		/**
		* Create a new instance from the data in the given XML Element.
		*
		* @param data An XML-Element containing the description of a disk.
		*/
		Disk(xercesc::DOMElement* data);

		/**
		* instantiates a new Disk with the given values.
		*
		* @param id ID of the disk. Has to be unique all over this library.
		* @param capacity Capacity of the Disk in bytes. (This can also be in
		* any other scale, but has to be same for ExtentSize
		* of Distributor)
		* @param data Pointer to be used by developer instantiating this disk.
		* May not be changed by Distributors!
		*/
		Disk(int64_t id, int64_t capacity, void* data);

		/**
		* copy constructor
		*
		* @param orig original Disk
		*/
		Disk(const Disk& orig);

		/**
		* Destructor
		*/
		virtual ~Disk();

		/**
		* Get the ID of the disk. Has to be unique all over this library.
		*
		* @return ID of the disk. Has to be unique all over this library.
		*/
		int64_t getId() const;

		/**
		* Set the ID of the disk. Has to be unique all over this library.
		*
		* @param id ID of the disk. Has to be unique all over this library.
		*/
		void setId(int64_t id);

		/**
		* Get the Capacity of the Disk in bytes. (This can also be in any
		* other scale, but has to be same for ExtentSize of Distributor)
		*
		* @return Capacity of the Disk in bytes. (This can also be in any
		* other scale, but has to be same for ExtentSize of
		* Distributor)
		*/
		int64_t getCapacity() const;

		/**
		* Set the Capacity of the Disk in bytes. (This can also be in any
		* other scale, but has to be same for ExtentSize of Distributor)
		*
		* @param capacity Capacity of the Disk in bytes. (This can also be in
		* any other scale, but has to be same for ExtentSize of
		* Distributor)
		*/
		void setCapacity(int64_t capacity);

		/**
		* Get the Pointer to be used by developer instantiating this disk.
		* May not be changed by Distributors!
		*
		* @return Pointer to be used by developer instantiating this disk.
		* May not be changed by Distributors!
		*/
		void* getData() const;

		/**
		* Set the Pointer to be used by developer instantiating this disk. May not be changed by Distributors!
		*
		* @param data Pointer to be used by developer instantiating this disk.
		* May not be changed by Distributors!
		*/
		void setData(void* data);

		/**
		* build an XML-Version of this object
		*
		* @param doc the document needed to create new XML Elements
		* @return a new Element containing the description of this object.
		*/
		virtual xercesc::DOMElement* toXML(xercesc::DOMDocument* doc);

		/**
		* Get the Root-Type of XML-Elements representing this class.
		*
		* @return the Root-Type of XML-Elements representing this class.
		*/
		static std::string getXMLRootType() {
		return std::string("Disk");
		}

		/**
		* Using this method it is possible to store a list of disks in a file.
		*
		* @param disks The disks to be stored
		* @param filename The name of the file the disks shall be stored in.
		*/
		static void storeDiskList(std::list<Disk> disks, std::string filename);

		/**
		* Using this method it is possible to read a list of disks out of a
		* file.
		*
		* @param filename The name of the file containing the disks.
		*
		* @return A list with the disks readen out of the file.
		*/
		static std::list<Disk> loadDiskList(std::string filename);

		#ifndef no_sqlite
		/**
		* Using this method it is possible to store a list of disks in a sqlite db file.
		*
		* @param disks The disks to be stored
		* @param filename The name of the file the disks shall be stored in.
		*/
		static void storeDiskListDBFile(std::list<Disk> disks, std::string filename);

		/**
		* Using this method it is possible to read a list of disks out of a sqlite db
		* file.
		*
		* @param filename The name of the file containing the disks.
		*
		* @return A list with the disks readen out of the file.
		*/
		static std::list<Disk> loadDiskListDBFile(std::string filename);
		#endif
		private:

		/**
		* ID of the disk. Has to be unique all over this library.
		*/
		int64_t id;

		/**
		* Capacity of the Disk in bytes. (This can also be in any other
		* scale, but has to be same for ExtentSize of Distributor)
		*/
		int64_t capacity;

		/**
		* Pointer to be used by developer instantiating this disk. May not be
		* changed by Distributors!
		*/
		void* data;
		};
		}
		#endif /* _DISK_H */

include/global/random_slicing/DistRandSlice.h

0 → 100644

+223 −0

Original line number	Diff line number	Diff line
		// Adapted from https://sourceforge.net/p/dadisi/code/8/tree/trunk/dadisi/
		/*
		* File: DistRandSlice.h
		* Author: amiranda
		*
		* Created on 8, November 2010, 12:24
		*/

		#ifndef _DISTRANDSLICE_H
		#define _DISTRANDSLICE_H


		#include <tr1/unordered_map> //FIXME?
		#define __STDC_LIMIT_MACROS// required for UINT64_MAX macro
		#include <stdint.h>
		#include "Distributor.h"
		#include "flat_segment_tree.hpp"


		#define DEBUG
		#ifdef DEBUG
		# define DUMP_COPIES
		//# define DUMP_DISKS
		//# define DUMP_PARTITIONS
		//# define DUMP_FREE_SPACE_COLLECTION
		//# define DUMP_FREE_SPACE_ASSIMILATION
		//# define DUMP_FREE_SPACE
		//# define DUMP_INTERVALS
		//# define DUMP_INTERVALS_VERBOSE
		#endif


		namespace VDRIVE {

		/**
		* XXX: Place comment describing the distribution algorithm here
		*
		*
		*
		*
		*/
		class DistRandSlice : public Distributor {
		public:

		// constants declaring the min and max values for the
		// virtual allocation space.
		// right now, it spans from 0 to 2^64-1 blocks/files/whatever
		static const uint64_t VSPACE_MIN = 0;
		static const uint64_t VSPACE_MAX = UINT64_MAX;


		/**
		* Create a new instance from the data in the given XML Element.
		*
		* @param data An XML-Element containing the description of a DistRandSlice.
		*/
		DistRandSlice(xercesc::DOMElement* data);

		/**
		* generate a new, uninitialized Rand Slice Implementation.
		*/
		DistRandSlice(int argc, char** argv);

		/**
		* copy constructor
		*
		* @param orig original DistRandSlice
		*/
		DistRandSlice(const DistRandSlice& orig);

		/**
		* Destructor
		*/
		virtual ~DistRandSlice();

		/**
		* @see Distributor::placeExtent
		*/
		virtual std::list<Disk> placeExtent(int64_t virtualVolumeId, int64_t position);

		/**
		* @see Distributor::setConfiguration
		*/
		virtual void setConfiguration(std::list<Disk> disks, int64_t extentsize, int32_t copies);

		/**
		* @see Distributor::setDisks
		*/
		virtual void setDisks(std::list<Disk> disks);

		/**
		* @see Distributor::toXML
		*/
		virtual xercesc::DOMElement* toXML(xercesc::DOMDocument* doc) const;

		/**
		* @see Distributor::getDisks
		*/
		virtual std::list<Disk> getDisks() const;

		/**
		* @see Distributor::getExtentsize
		*/
		virtual int64_t getExtentsize() const {
		return m_extentsize;
		}

		/**
		* @see Distributor::getCopies
		*/
		virtual int32_t getCopies() const {
		return m_copies;
		}

		/**
		* @return the number of partitions currently in the system
		*/
		uint64_t getNumPartitions() const {
		return m_num_partitions;
		}

		/**
		* @return the number of intervals currently in the system
		*/
		uint64_t getNumIntervals() const;

		/**
		* Get the Root-Type of XML-Elements representing this class.
		*
		* @return the Root-Type of XML-Elements representing this class.
		*/
		static std::string getXMLRootType() {
		return std::string("RandSlice");
		}

		private:

		void cleanup(void);

		void create_partitions(std::list<Disk> disks);

		void add_partitions(std::list<Disk> disks);

		void redistribute(std::tr1::unordered_map<uint64_t, uint64_t>& old_partitions,
		const std::list< std::pair<uint64_t, uint64_t> >& new_partitions);

		void collect_free_space(std::tr1::unordered_map<uint64_t, uint64_t>& old_partitions,
		std::list< std::pair<uint64_t, uint64_t> >& free_space);

		void collect_free_space_even_odd(std::tr1::unordered_map<uint64_t, uint64_t>& old_partitions,
		std::list< std::pair<uint64_t, uint64_t> >& free_space);

		void reuse_free_space(std::list< std::pair<uint64_t, uint64_t> >& free_space,
		const std::list< std::pair<uint64_t, uint64_t> >& new_partitions);

		void reuse_free_space_sort(std::list< std::pair<uint64_t, uint64_t> >& free_space,
		const std::list< std::pair<uint64_t, uint64_t> >& new_partitions);


		#if defined DEBUG
		void dump_intervals(void);
		void dump_free_space(const std::list< std::pair<uint64_t, uint64_t> >& l) const;
		void verify_partitions(void);
		std::vector<uint64_t> compute_interval_sizes(void);
		#endif

		/**
		* ExtenSize as given to setConfiguration.
		*/
		int64_t m_extentsize;

		/**
		* number of copies to be distributed.
		*/
		int32_t m_copies;

		/**
		* the disks as given to setDisks (or setConfiguration)
		*/
		std::tr1::unordered_map<uint64_t, Disk> m_disks;

		/**
		* number of Disks contained by this Distributor.
		*/
		uint64_t m_num_disks;

		/**
		* partition info: id -> capacity
		*/
		std::tr1::unordered_map<uint64_t, uint64_t>* m_partitions;

		/**
		* number of partitions
		*/
		uint64_t m_num_partitions;

		/**
		* interval tree for searches
		*/
		typedef ::mdds::flat_segment_tree<uint64_t, uint64_t> flat_segment_tree;
		flat_segment_tree* m_interval_tree;

		/**
		* capacity of the system
		*/
		uint64_t m_capacity;


		/**
		* use Even-Odd strategy when collecting free space
		*/
		bool m_use_even_odd_collection;

		/**
		* sort free intervals decreasingly when assimilating free space
		*/
		bool m_use_sorted_assimilation;


		};
		}
		#endif /* _DISTRANDSLICE_H */

src/global/random_slicing/Disk.cpp

0 → 100644

+323 −0

File added.

Preview size limit exceeded, changes collapsed.