Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
hpc
gekkofs
Commits
86599ac4
Commit
86599ac4
authored
Feb 22, 2022
by
Ramon Nou
Browse files
updated CMake
Sampling output Thread First stats test Added Stats argument to enable output
parent
4fb3a7b7
Changes
10
Hide whitespace changes
Inline
Side-by-side
CHANGELOG.md
View file @
86599ac4
...
...
@@ -7,6 +7,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
-
Added Stats gathering in servers
### New
-
Added new experimental metadata backend:
...
...
README.md
View file @
86599ac4
...
...
@@ -109,6 +109,7 @@ Options:
RocksDB is default
if
not set. Parallax support is experimental.
Note, parallaxdb creates a file called rocksdbx with 8GB created
in
metadir.
--parallaxsize
TEXT parallaxdb - metadata file size
in
GB
(
default 8GB
)
, used only with new files
--output-stats
Enables the output of the stats on the stdout
(
each 10s
)
for
debug
--version
Print version and exit.
```
...
...
include/common/statistics/stats.hpp
View file @
86599ac4
...
...
@@ -37,11 +37,13 @@
#include
<deque>
#include
<chrono>
#include
<initializer_list>
#include
<thread>
#include
<iostream>
/**
* Provides storage capabilities to provide stats about GekkoFS
* The information is per server.
* The information is per server.
* We do not provide accurate stats for 1-5-10 minute stats
*
*
*/
namespace
gkfs
::
utils
{
...
...
@@ -56,13 +58,14 @@ namespace gkfs::utils {
5 minute mean
10 minute mean
To provide the stats that we need,
To provide the stats that we need,
we need to store the info and the timestamp to calculate it
A vector should work, with a maximum of elements,
A vector should work, with a maximum of elements,
The stats will only be calculated when requested
a cached value will be send (with a deadline)
*/
class
Stats
{
class
Stats
{
public:
enum
class
IOPS_OP
{
IOPS_CREATE
,
IOPS_WRITE
,
...
...
@@ -72,102 +75,139 @@ class Stats{
IOPS_REMOVE
,
};
constexpr
static
const
std
::
initializer_list
<
Stats
::
IOPS_OP
>
all_IOPS_OP
=
{
IOPS_OP
::
IOPS_CREATE
,
IOPS_OP
::
IOPS_WRITE
,
IOPS_OP
::
IOPS_READ
,
IOPS_OP
::
IOPS_MKDIR
,
IOPS_OP
::
IOPS_RMDIR
,
IOPS_OP
::
IOPS_REMOVE
};
enum
class
SIZE_OP
{
METADATA_SIZE
,
WRITE_SIZE
,
READ_SIZE
,
DATA_SIZE
};
enum
class
SIZE_OP
{
METADATA_SIZE
,
WRITE_SIZE
,
READ_SIZE
,
DATA_SIZE
};
constexpr
static
const
std
::
initializer_list
<
Stats
::
SIZE_OP
>
all_SIZE_OP
=
{
SIZE_OP
::
METADATA_SIZE
,
SIZE_OP
::
DATA_SIZE
,
SIZE_OP
::
WRITE_SIZE
,
SIZE_OP
::
READ_SIZE
};
private:
constexpr
static
const
std
::
initializer_list
<
Stats
::
IOPS_OP
>
all_IOPS_OP
=
{
IOPS_OP
::
IOPS_CREATE
,
IOPS_OP
::
IOPS_WRITE
,
IOPS_OP
::
IOPS_READ
,
IOPS_OP
::
IOPS_MKDIR
,
IOPS_OP
::
IOPS_RMDIR
,
IOPS_OP
::
IOPS_REMOVE
};
constexpr
static
const
std
::
initializer_list
<
Stats
::
SIZE_OP
>
all_SIZE_OP
=
{
SIZE_OP
::
METADATA_SIZE
,
SIZE_OP
::
DATA_SIZE
,
SIZE_OP
::
WRITE_SIZE
,
SIZE_OP
::
READ_SIZE
};
const
std
::
vector
<
std
::
string
>
IOPS_OP_S
=
{
"IOPS_CREATE"
,
"IOPS_WRITE"
,
"IOPS_READ"
,
"IOPS_MKDIR"
,
"IOPS_RMDIR"
,
"IOPS_REMOVE"
};
const
std
::
vector
<
std
::
string
>
SIZE_OP_S
=
{
"METADATA_SIZE"
,
"WRITE_SIZE"
,
"READ_SIZE"
,
"DATA_SIZE"
};
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>
last_cached
;
/* Measures when we started the server */
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>
start
;
// How many stats will be stored
const
unsigned
int
MAX_STATS
=
1000000
;
// How many stats will be stored
const
unsigned
int
MAX_STATS
=
1000000
;
// Stores total value for global mean
std
::
map
<
IOPS_OP
,
unsigned
long
>
IOPS
;
std
::
map
<
SIZE_OP
,
unsigned
long
>
SIZE
;
std
::
map
<
IOPS_OP
,
unsigned
long
>
IOPS
;
std
::
map
<
SIZE_OP
,
unsigned
long
>
SIZE
;
// Stores timestamp when an operation comes
// removes if first operation if > 10 minutes
// removes if first operation if > 10 minutes
// Different means will be stored and cached 1 minuted
std
::
map
<
IOPS_OP
,
std
::
deque
<
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>
>
>
TIME_IOPS
;
std
::
map
<
IOPS_OP
,
std
::
deque
<
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>>>
TIME_IOPS
;
// We will store 1, 5, and 10 minute mean;
std
::
map
<
IOPS_OP
,
std
::
vector
<
double
>
>
CACHED_IOPS
;
std
::
map
<
IOPS_OP
,
std
::
vector
<
double
>>
CACHED_IOPS
;
// For size operations we need to store the timestamp and
// the size
std
::
map
<
enum
SIZE_OP
,
std
::
deque
<
std
::
pair
<
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>
,
unsigned
long
long
>
>
>
TIME_SIZE
;
std
::
map
<
enum
SIZE_OP
,
std
::
deque
<
std
::
pair
<
std
::
chrono
::
time_point
<
std
::
chrono
::
steady_clock
>
,
unsigned
long
long
>>>
TIME_SIZE
;
// We will store 1, 5, and 10 minute mean;
std
::
map
<
enum
SIZE_OP
,
std
::
vector
<
double
>
>
CACHED_SIZE
;
/**
* @brief Starts the Stats module and initializes structures
*
*/
public:
Stats
();
/**
* Add a new value for a IOPS, that does not involve any size
* No value needed as they are simple (1 create, 1 read...)
* Size operations internally call this operation (read,write)
*
* @param IOPS_OP Which operation to add
*/
void
add_value_iops
(
enum
IOPS_OP
);
/**
* @brief Store a new stat point, with a size value.
* If it involves a IO operations it will call the corresponding
* operation
*
* @param SIZE_OP Which operation we refer
* @param value to store (SIZE_OP)
*/
void
add_value_size
(
enum
SIZE_OP
,
unsigned
long
long
value
);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
get_mean
(
enum
IOPS_OP
);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
get_mean
(
enum
SIZE_OP
);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a SIZE_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std
::
vector
<
double
>
get_four_means
(
enum
SIZE_OP
);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a IOPS_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std
::
vector
<
double
>
get_four_means
(
enum
IOPS_OP
);
std
::
map
<
enum
SIZE_OP
,
std
::
vector
<
double
>>
CACHED_SIZE
;
// Thread that outputs stats info
std
::
thread
t_output
;
bool
output_thread_
;
// Controls the destruction of the class/stops the thread
bool
running
=
true
;
/**
* @brief Sends all the stats to the screen
* Debug Function
*
* @param d is the time between output
*/
void
output
(
std
::
chrono
::
seconds
d
);
public:
/**
* @brief Starts the Stats module and initializes structures
*
*/
Stats
(
bool
output_thread
);
/**
* @brief Destroys the class, and any associated thread
*
*/
~
Stats
();
/**
* Add a new value for a IOPS, that does not involve any size
* No value needed as they are simple (1 create, 1 read...)
* Size operations internally call this operation (read,write)
*
* @param IOPS_OP Which operation to add
*/
void
add_value_iops
(
enum
IOPS_OP
);
/**
* @brief Store a new stat point, with a size value.
* If it involves a IO operations it will call the corresponding
* operation
*
* @param SIZE_OP Which operation we refer
* @param value to store (SIZE_OP)
*/
void
add_value_size
(
enum
SIZE_OP
,
unsigned
long
long
value
);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
get_mean
(
enum
IOPS_OP
);
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
get_mean
(
enum
SIZE_OP
);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a SIZE_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std
::
vector
<
double
>
get_four_means
(
enum
SIZE_OP
);
/**
* @brief Get all the means (total, 1,5 and 10 minutes) for a IOPS_OP
* Returns precalculated values if we just calculated them 1 minute ago
*
* @return std::vector< double > with 4 means
*/
std
::
vector
<
double
>
get_four_means
(
enum
IOPS_OP
);
/**
* @brief Dumps all the means from the stats
*
*/
void
dump
();
};
}
// namespace gkfs::utils
...
...
include/daemon/classes/fs_data.hpp
View file @
86599ac4
...
...
@@ -92,6 +92,7 @@ private:
// Statistics
std
::
shared_ptr
<
gkfs
::
utils
::
Stats
>
stats_
;
bool
output_stats_
=
false
;
public:
static
FsData
*
...
...
@@ -224,8 +225,17 @@ public:
void
stats
(
const
std
::
shared_ptr
<
gkfs
::
utils
::
Stats
>&
stats
);
void
close_stats
();
bool
output_stats
()
const
;
void
output_stats
(
bool
output_stats
);
};
}
// namespace daemon
}
// namespace gkfs
...
...
src/common/CMakeLists.txt
View file @
86599ac4
...
...
@@ -39,6 +39,15 @@ target_sources(distributor
${
CMAKE_CURRENT_LIST_DIR
}
/rpc/distributor.cpp
)
add_library
(
statistics STATIC
)
set_property
(
TARGET statistics PROPERTY POSITION_INDEPENDENT_CODE ON
)
target_sources
(
statistics
PUBLIC
${
INCLUDE_DIR
}
/common/statistics/stats.hpp
PRIVATE
${
CMAKE_CURRENT_LIST_DIR
}
/statistics/stats.cpp
)
if
(
GKFS_ENABLE_CODE_COVERAGE
)
target_code_coverage
(
distributor AUTO
)
endif
()
...
...
src/common/statistics/stats.cpp
View file @
86599ac4
...
...
@@ -27,86 +27,107 @@
*/
#include
"/home/rnou/gekkofs/include/
common/statistics/stats.hpp
"
#include
<
common/statistics/stats.hpp
>
using
namespace
std
;
namespace
gkfs
::
utils
{
namespace
gkfs
::
utils
{
Stats
::
Stats
(
)
{
Stats
::
Stats
(
bool
output_thread
)
{
// Init clocks
start
=
std
::
chrono
::
steady_clock
::
now
();
last_cached
=
std
::
chrono
::
steady_clock
::
now
();
// Init cached (4 mean values)
for
(
auto
e
:
all_IOPS_OP
)
for
(
int
i
=
0
;
i
<
4
;
i
++
)
CACHED_IOPS
[
e
].
push_back
(
0.0
);
// Init clocks
start
=
std
::
chrono
::
steady_clock
::
now
();
last_cached
=
std
::
chrono
::
steady_clock
::
now
();
// Init cached (4 mean values)
for
(
auto
e
:
all_
SIZE
_OP
)
for
(
int
i
=
0
;
i
<
4
;
i
++
)
CACHED_
SIZE
[
e
].
push_back
(
0.0
);
for
(
auto
e
:
all_
IOPS
_OP
)
for
(
int
i
=
0
;
i
<
4
;
i
++
)
CACHED_
IOPS
[
e
].
push_back
(
0.0
);
for
(
auto
e
:
all_SIZE_OP
)
for
(
int
i
=
0
;
i
<
4
;
i
++
)
CACHED_SIZE
[
e
].
push_back
(
0.0
);
// To simplify the control we add an element into the different maps
// Statistaclly will be negligible... and we get a faster flow
for
(
auto
e
:
all_IOPS_OP
)
{
IOPS
[
e
]
=
0
;
TIME_IOPS
[
e
].
push_back
(
std
::
chrono
::
steady_clock
::
now
());
}
// To simplify the control we add an element into the different maps
// Statistaclly will be negligible... and we get a faster flow
for
(
auto
e
:
all_SIZE_OP
)
{
SIZE
[
e
]
=
0
;
TIME_SIZE
[
e
].
push_back
(
pair
(
std
::
chrono
::
steady_clock
::
now
(),
0.0
));
}
for
(
auto
e
:
all_IOPS_OP
)
{
IOPS
[
e
]
=
0
;
TIME_IOPS
[
e
].
push_back
(
std
::
chrono
::
steady_clock
::
now
());
}
void
Stats
::
add_value_iops
(
enum
IOPS_OP
iop
){
IOPS
[
iop
]
++
;
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
for
(
auto
e
:
all_SIZE_OP
)
{
SIZE
[
e
]
=
0
;
TIME_SIZE
[
e
].
push_back
(
pair
(
std
::
chrono
::
steady_clock
::
now
(),
0.0
));
}
if
(
(
now
-
TIME_IOPS
[
iop
].
front
())
>
std
::
chrono
::
duration
(
10s
)
)
{
TIME_IOPS
[
iop
].
pop_front
();
}
else
if
(
TIME_IOPS
[
iop
].
size
()
>=
MAX_STATS
)
TIME_IOPS
[
iop
].
pop_front
();
output_thread_
=
output_thread
;
TIME_IOPS
[
iop
].
push_back
(
std
::
chrono
::
steady_clock
::
now
());
if
(
output_thread_
)
{
t_output
=
std
::
thread
([
this
]
{
output
(
std
::
chrono
::
duration
(
10s
));
});
}
}
void
Stats
::
add_value_size
(
enum
SIZE_OP
iop
,
unsigned
long
long
value
){
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
SIZE
[
iop
]
+=
value
;
if
(
(
now
-
TIME_SIZE
[
iop
].
front
().
first
)
>
std
::
chrono
::
duration
(
10s
)
)
{
TIME_SIZE
[
iop
].
pop_front
();
}
else
if
(
TIME_SIZE
[
iop
].
size
()
>=
MAX_STATS
)
TIME_SIZE
[
iop
].
pop_front
();
TIME_SIZE
[
iop
].
push_back
(
pair
(
std
::
chrono
::
steady_clock
::
now
(),
value
)
);
if
(
iop
==
SIZE_OP
::
READ_SIZE
)
IOPS
[
IOPS_OP
::
IOPS_READ
]
++
;
else
if
(
iop
==
SIZE_OP
::
WRITE_SIZE
)
IOPS
[
IOPS_OP
::
IOPS_WRITE
]
++
;
Stats
::~
Stats
()
{
// We do not need a mutex for that
if
(
output_thread_
)
{
running
=
false
;
t_output
.
join
();
}
}
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
Stats
::
get_mean
(
enum
SIZE_OP
sop
){
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
now
-
start
);
double
value
=
(
double
)
SIZE
[
sop
]
/
(
double
)
duration
.
count
();
return
value
;
void
Stats
::
add_value_iops
(
enum
IOPS_OP
iop
)
{
IOPS
[
iop
]
++
;
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
}
double
Stats
::
get_mean
(
enum
IOPS_OP
iop
){
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
now
-
start
);
double
value
=
(
double
)
IOPS
[
iop
]
/
(
double
)
duration
.
count
();
return
value
;
}
if
((
now
-
TIME_IOPS
[
iop
].
front
())
>
std
::
chrono
::
duration
(
10s
))
{
TIME_IOPS
[
iop
].
pop_front
();
}
else
if
(
TIME_IOPS
[
iop
].
size
()
>=
MAX_STATS
)
TIME_IOPS
[
iop
].
pop_front
();
TIME_IOPS
[
iop
].
push_back
(
std
::
chrono
::
steady_clock
::
now
());
}
void
Stats
::
add_value_size
(
enum
SIZE_OP
iop
,
unsigned
long
long
value
)
{
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
SIZE
[
iop
]
+=
value
;
if
((
now
-
TIME_SIZE
[
iop
].
front
().
first
)
>
std
::
chrono
::
duration
(
10s
))
{
TIME_SIZE
[
iop
].
pop_front
();
}
else
if
(
TIME_SIZE
[
iop
].
size
()
>=
MAX_STATS
)
TIME_SIZE
[
iop
].
pop_front
();
TIME_SIZE
[
iop
].
push_back
(
pair
(
std
::
chrono
::
steady_clock
::
now
(),
value
));
if
(
iop
==
SIZE_OP
::
READ_SIZE
)
IOPS
[
IOPS_OP
::
IOPS_READ
]
++
;
else
if
(
iop
==
SIZE_OP
::
WRITE_SIZE
)
IOPS
[
IOPS_OP
::
IOPS_WRITE
]
++
;
}
/**
* @brief Get the total mean value of the asked stat
* This can be provided inmediately without cost
* @return mean value
*/
double
Stats
::
get_mean
(
enum
SIZE_OP
sop
)
{
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
now
-
start
);
double
value
=
(
double
)
SIZE
[
sop
]
/
(
double
)
duration
.
count
();
return
value
;
}
double
Stats
::
get_mean
(
enum
IOPS_OP
iop
)
{
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
seconds
>
(
now
-
start
);
double
value
=
(
double
)
IOPS
[
iop
]
/
(
double
)
duration
.
count
();
return
value
;
}
/**
...
...
@@ -115,50 +136,92 @@ namespace gkfs::utils{
* // TODO: cache
* @return std::vector< double > with 4 means
*/
std
::
vector
<
double
>
Stats
::
get_four_means
(
enum
SIZE_OP
sop
){
std
::
vector
<
double
>
results
=
{
0
,
0
,
0
,
0
};
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
for
(
auto
e
:
TIME_SIZE
[
sop
])
{
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
minutes
>
(
now
-
e
.
first
).
count
();
if
(
duration
>
10
)
break
;
results
[
3
]
+=
e
.
second
;
if
(
duration
>
5
)
continue
;
results
[
2
]
+=
e
.
second
;
if
(
duration
>
1
)
continue
;
results
[
1
]
+=
e
.
second
;
}
results
[
0
]
=
get_mean
(
sop
);
results
[
3
]
/=
10
*
60
;
results
[
2
]
/=
5
*
60
;
results
[
1
]
/=
60
;
std
::
vector
<
double
>
Stats
::
get_four_means
(
enum
SIZE_OP
sop
)
{
std
::
vector
<
double
>
results
=
{
0
,
0
,
0
,
0
};
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
for
(
auto
e
:
TIME_SIZE
[
sop
])
{
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
minutes
>
(
now
-
e
.
first
)
.
count
();
if
(
duration
>
10
)
break
;
results
[
3
]
+=
e
.
second
;
if
(
duration
>
5
)
continue
;
results
[
2
]
+=
e
.
second
;
if
(
duration
>
1
)
continue
;
results
[
1
]
+=
e
.
second
;
}
return
results
;
results
[
0
]
=
get_mean
(
sop
);
results
[
3
]
/=
10
*
60
;
results
[
2
]
/=
5
*
60
;
results
[
1
]
/=
60
;
return
results
;
}
std
::
vector
<
double
>
Stats
::
get_four_means
(
enum
IOPS_OP
iop
)
{
std
::
vector
<
double
>
results
=
{
0
,
0
,
0
,
0
};
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
for
(
auto
e
:
TIME_IOPS
[
iop
])
{
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
minutes
>
(
now
-
e
)
.
count
();
if
(
duration
>
10
)
break
;
results
[
3
]
++
;
if
(
duration
>
5
)
continue
;
results
[
2
]
++
;
if
(
duration
>
1
)
continue
;
results
[
1
]
++
;
}
results
[
0
]
=
get_mean
(
iop
);
results
[
3
]
/=
10
*
60
;
results
[
2
]
/=
5
*
60
;
results
[
1
]
/=
60
;
std
::
vector
<
double
>
Stats
::
get_four_means
(
enum
IOPS_OP
iop
){
std
::
vector
<
double
>
results
=
{
0
,
0
,
0
,
0
};
auto
now
=
std
::
chrono
::
steady_clock
::
now
();
for
(
auto
e
:
TIME_IOPS
[
iop
])
{
auto
duration
=
std
::
chrono
::
duration_cast
<
std
::
chrono
::
minutes
>
(
now
-
e
).
count
();
if
(
duration
>
10
)
break
;
return
results
;
}
results
[
3
]
++
;
if
(
duration
>
5
)
continue
;
results
[
2
]
++
;
if
(
duration
>
1
)
continue
;
results
[
1
]
++
;
}
void
Stats
::
dump
()
{
for
(
auto
e
:
all_IOPS_OP
)
{
auto
tmp
=
get_four_means
(
e
);
results
[
0
]
=
get_mean
(
iop
);
results
[
3
]
/=
10
*
60
;
results
[
2
]
/=
5
*
60
;
results
[
1
]
/=
60
;
std
::
cout
<<
"Stats "
<<
IOPS_OP_S
[
static_cast
<
int
>
(
e
)]
<<
" "
;
for
(
auto
mean
:
tmp
)
{
std
::
cout
<<
mean
<<
" - "
;
}
std
::
cout
<<
std
::
endl
;
}
for
(
auto
e
:
all_SIZE_OP
)
{
auto
tmp
=
get_four_means
(
e
);
return
results
;
std
::
cout
<<
"Stats "
<<
SIZE_OP_S
[
static_cast
<
int
>
(
e
)]
<<
" "
;
for
(
auto
mean
:
tmp
)
{
std
::
cout
<<
mean
<<
" - "
;
}
std
::
cout
<<
std
::
endl
;
}
}
void
Stats
::
output
(
std
::
chrono
::
seconds
d
)
{