static int lfs_pcc_state(int argc, char **argv);
static int lfs_pcc(int argc, char **argv);
-enum stats_flag {
- STATS_ON,
- STATS_OFF,
-};
-
static int lfs_migrate_to_dom(int fd_src, int fd_dst, char *name,
__u64 migration_flags,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec);
struct pool_to_id_cbdata {
{ .pc_name = "extend", .pc_func = lfs_mirror_extend,
.pc_help = "Extend a mirrored file.\n"
"usage: lfs mirror extend "
- "{--mirror-count|-N[MIRROR_COUNT]} [--no-verify] "
- "[SETSTRIPE_OPTIONS|-f VICTIM_FILE] ... FILENAME ...\n" },
+ "{--mirror-count|-N[MIRROR_COUNT]} [--no-verify]|\n"
+ "\t\t--stats|--stats-interval=<sec>|\n"
+ "\t\t--W <bandwidth>|--bandwidth-limit=<bandwidth>\n"
+ "\t\t[SETSTRIPE_OPTIONS|-f VICTIM_FILE] ... FILENAME ...\n" },
{ .pc_name = "split", .pc_func = lfs_mirror_split,
.pc_help = "Split a mirrored file.\n"
"usage: lfs mirror split {--mirror-id MIRROR_ID |\n"
"\t\t{--write-mirror|-o MIRROR_ID1[,...]} <mirrored_file>\n" },
{ .pc_name = "resync", .pc_func = lfs_mirror_resync,
.pc_help = "Resynchronizes out-of-sync mirrored file(s).\n"
- "usage: lfs mirror resync [--only MIRROR_ID[,...]>]\n"
+ "usage: lfs mirror resync [--only MIRROR_ID[,...]>]|\n"
+ "\t\t--stats|--stats-interval=<sec>|\n"
+ "\t\t--W <bandwidth>|--bandwidth-limit=<bandwidth>\n"
"\t\t<mirrored_file> [<mirrored_file2>...]\n" },
{ .pc_name = "verify", .pc_func = lfs_mirror_verify,
.pc_help = "Verify mirrored file(s).\n"
}
static void stats_log(struct timespec *now, struct timespec *start_time,
- enum stats_flag stats_flag,
ssize_t read_bytes, size_t write_bytes,
off_t file_size_bytes)
{
struct timespec diff = timespec_sub(start_time, now);
- if (stats_flag == STATS_ON && ((diff.tv_sec != 0) ||
- (diff.tv_nsec != 0)) && file_size_bytes != 0)
- printf("- { seconds: %li, rmbps: %5.2g, wmbps: %5.2g, copied: %lu, size: %lu, pct: %lu%% }\n",
- diff.tv_sec,
- (double) read_bytes/((ONE_MB * diff.tv_sec) +
- ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
- (double) write_bytes/((ONE_MB * diff.tv_sec) +
- ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
- write_bytes/ONE_MB,
- file_size_bytes/ONE_MB,
- ((write_bytes*100)/file_size_bytes));
+ if (file_size_bytes == 0)
+ return;
+
+ if (diff.tv_sec == 0 && diff.tv_nsec == 0)
+ return;
+
+ printf("- { seconds: %li, rmbps: %5.2g, wmbps: %5.2g, copied: %lu, size: %lu, pct: %lu%% }\n",
+ diff.tv_sec,
+ (double) read_bytes/((ONE_MB * diff.tv_sec) +
+ ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
+ (double) write_bytes/((ONE_MB * diff.tv_sec) +
+ ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
+ write_bytes/ONE_MB, file_size_bytes/ONE_MB,
+ ((write_bytes*100)/file_size_bytes));
}
static int migrate_copy_data(int fd_src, int fd_dst, int (*check_file)(int),
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec, off_t file_size_bytes)
{
struct llapi_layout *layout;
} while (rc < 0 && errno == EINTR);
if (rc < 0) {
- if (stats_flag == STATS_OFF)
+ if (stats_interval_sec)
fprintf(stderr,
"error %s: delay for bandwidth control failed: %s\n",
progname,
}
clock_gettime(CLOCK_REALTIME, &now);
- if ((write_bytes != file_size_bytes) &&
+ if (stats_interval_sec && (write_bytes != file_size_bytes) &&
(now.tv_sec >= last_bw_print.tv_sec +
stats_interval_sec)) {
- stats_log(&now, &start_time, stats_flag,
+ stats_log(&now, &start_time,
read_bytes, write_bytes,
file_size_bytes);
last_bw_print = now;
}
/* Output at least one log, regardless of stats_interval */
- clock_gettime(CLOCK_REALTIME, &now);
- stats_log(&now, &start_time, stats_flag,
- read_bytes, write_bytes,
- file_size_bytes);
+ if (stats_interval_sec) {
+ clock_gettime(CLOCK_REALTIME, &now);
+ stats_log(&now, &start_time, read_bytes, write_bytes,
+ file_size_bytes);
+ }
rc = fsync(fd_dst);
if (rc < 0)
static int migrate_block(int fd_src, int fd_dst,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec)
{
struct stat st;
}
rc = migrate_copy_data(fd_src, fd_dst, NULL, bandwidth_bytes_sec,
- stats_flag, stats_interval_sec,
- st.st_size);
+ stats_interval_sec, st.st_size);
if (rc < 0) {
error_loc = "data copy failed";
goto out_unlock;
static int migrate_nonblock(int fd_src, int fd_dst,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec)
{
struct stat st;
}
rc = migrate_copy_data(fd_src, fd_dst, check_lease,
- bandwidth_bytes_sec, stats_flag,
+ bandwidth_bytes_sec,
stats_interval_sec, st.st_size);
if (rc < 0) {
error_loc = "data copy failed";
struct llapi_stripe_param *param,
struct llapi_layout *layout,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag, long stats_interval_sec)
+ long stats_interval_sec)
{
struct llapi_layout *existing;
uint64_t dom_new, dom_cur;
* if new layout used bigger DOM size, then mirroring is used
*/
if (dom_new > dom_cur) {
- rc = lfs_migrate_to_dom(fd_src, fd_dst, name, migration_flags,
- bandwidth_bytes_sec, stats_flag,
+ rc = lfs_migrate_to_dom(fd_src, fd_dst, name,
+ migration_flags,
+ bandwidth_bytes_sec,
stats_interval_sec);
if (rc)
error_loc = "cannot migrate to DOM layout";
goto out_closed;
}
- if (stats_flag == STATS_ON)
+ if (stats_interval_sec)
printf("%s:\n", name);
if (!(migration_flags & LLAPI_MIGRATION_NONBLOCK)) {
* atomic swap/close (LU-6785)
*/
rc = migrate_block(fd_src, fd_dst, bandwidth_bytes_sec,
- stats_flag, stats_interval_sec);
+ stats_interval_sec);
goto out;
}
goto out;
}
- rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec, stats_flag,
+ rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec,
stats_interval_sec);
if (rc < 0) {
llapi_lease_release(fd_src);
static int mirror_extend_layout(char *name, struct llapi_layout *m_layout,
bool inherit, uint32_t flags,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec)
{
struct llapi_layout *f_layout = NULL;
goto out;
}
- if (stats_flag)
+ if (stats_interval_sec)
printf("%s:\n", name);
- rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec, stats_flag,
+ rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec,
stats_interval_sec);
if (rc < 0) {
llapi_lease_release(fd_src);
static int mirror_extend(char *fname, struct mirror_args *mirror_list,
enum mirror_flags mirror_flags,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag, long stats_interval_sec)
+ long stats_interval_sec)
{
int rc = 0;
mirror_list->m_inherit,
mirror_list->m_flags,
bandwidth_bytes_sec,
- stats_flag,
stats_interval_sec);
if (rc)
break;
static inline
int lfs_mirror_resync_file(const char *fname, struct ll_ioc_lease *ioc,
- __u16 *mirror_ids, int ids_nr);
+ __u16 *mirror_ids, int ids_nr,
+ long stats_interval_sec, long bandwidth_bytes_sec);
+
static int lfs_migrate_to_dom(int fd_src, int fd_dst, char *name,
__u64 migration_flags,
unsigned long long bandwidth_bytes_sec,
- enum stats_flag stats_flag,
long stats_interval_sec)
{
struct ll_ioc_lease *data = NULL;
goto out_close;
}
- if (stats_flag)
+ if (stats_interval_sec)
printf("%s:\n", name);
- rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec, stats_flag,
+ rc = migrate_nonblock(fd_src, fd_dst, bandwidth_bytes_sec,
stats_interval_sec);
if (rc < 0)
goto out_release;
close(fd_src);
close(fd_dst);
- rc = lfs_mirror_resync_file(name, data, NULL, 0);
+ rc = lfs_mirror_resync_file(name, data, NULL, 0,
+ stats_interval_sec,
+ bandwidth_bytes_sec);
if (rc) {
error_loc = "cannot resync file";
goto out;
mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
unsigned long long bandwidth_bytes_sec = 0;
unsigned long long bandwidth_unit = ONE_MB;
- enum stats_flag stats_flag = STATS_OFF;
- long stats_interval_sec = 5;
+ long stats_interval_sec = 0;
struct option long_opts[] = {
/* find { .val = '0', .name = "null", .has_arg = no_argument }, */
template = optarg;
break;
case LFS_STATS_OPT:
- stats_flag = STATS_ON;
+ stats_interval_sec = 5;
break;
case LFS_STATS_INTERVAL_OPT:
- stats_flag = STATS_ON;
stats_interval_sec = strtol(optarg, &end, 0);
- if (stats_interval_sec == 0)
- stats_interval_sec = 5;
+ if (stats_interval_sec == 0 && errno) {
+ fprintf(stderr,
+ "%s %s: invalid stats interval %s\n",
+ progname, argv[0], optarg);
+ goto usage_error;
+ }
break;
case 'b':
if (!migrate_mode) {
result = lfs_migrate(fname, migration_flags, param,
layout, bandwidth_bytes_sec,
- stats_flag, stats_interval_sec);
+ stats_interval_sec);
} else if (comp_set != 0) {
result = lfs_component_set(fname, comp_id,
lsa.lsa_pool_name,
result = mirror_extend(fname, mirror_list,
mirror_flags,
bandwidth_bytes_sec,
- stats_flag, stats_interval_sec);
+ stats_interval_sec);
} else if (opc == SO_MIRROR_SPLIT || opc == SO_MIRROR_DELETE) {
if (!mirror_id && !comp_id && !lsa.lsa_pool_name) {
fprintf(stderr,
static inline
int lfs_mirror_resync_file(const char *fname, struct ll_ioc_lease *ioc,
- __u16 *mirror_ids, int ids_nr)
+ __u16 *mirror_ids, int ids_nr,
+ long stats_interval_sec, long bandwidth_bytes_sec)
{
struct llapi_resync_comp comp_array[1024] = { { 0 } };
struct llapi_layout *layout;
goto free_layout;
}
- rc = llapi_mirror_resync_many(fd, layout, comp_array, comp_size,
- start, end);
+ rc = llapi_mirror_resync_many_params(fd, layout, comp_array, comp_size,
+ start, end, stats_interval_sec,
+ bandwidth_bytes_sec);
if (rc < 0)
fprintf(stderr, "%s: '%s' llapi_mirror_resync_many: %s.\n",
progname, fname, strerror(-rc));
struct option long_opts[] = {
{ .val = 'h', .name = "help", .has_arg = no_argument },
{ .val = 'o', .name = "only", .has_arg = required_argument },
+ { .val = 'W', .name = "bandwidth", .has_arg = required_argument },
+ { .val = LFS_STATS_OPT,
+ .name = "stats", .has_arg = no_argument},
+ { .val = LFS_STATS_INTERVAL_OPT,
+ .name = "stats-interval",
+ .has_arg = required_argument},
{ .name = NULL } };
struct ll_ioc_lease *ioc = NULL;
__u16 mirror_ids[128] = { 0 };
+ unsigned int stats_interval_sec = 0;
+ unsigned long long bandwidth_bytes_sec = 0;
+ unsigned long long bandwidth_unit = ONE_MB;
int ids_nr = 0;
int c;
int rc = 0;
- while ((c = getopt_long(argc, argv, "ho:", long_opts, NULL)) >= 0) {
+ while ((c = getopt_long(argc, argv, "ho:W:", long_opts, NULL)) >= 0) {
+ char *end;
switch (c) {
case 'o':
rc = parse_mirror_ids(mirror_ids,
}
ids_nr = rc;
break;
+ case 'W':
+ if (llapi_parse_size(optarg, &bandwidth_bytes_sec,
+ &bandwidth_unit, 0) < 0) {
+ fprintf(stderr,
+ "error: %s: bad value for bandwidth '%s'\n",
+ argv[0], optarg);
+ goto error;
+ }
+ break;
+ case LFS_STATS_OPT:
+ stats_interval_sec = 5;
+ break;
+ case LFS_STATS_INTERVAL_OPT:
+ stats_interval_sec = strtol(optarg, &end, 0);
+ break;
default:
fprintf(stderr, "%s: unrecognized option '%s'\n",
progname, argv[optind - 1]);
for (; optind < argc; optind++) {
rc = lfs_mirror_resync_file(argv[optind], ioc,
- mirror_ids, ids_nr);
+ mirror_ids, ids_nr,
+ stats_interval_sec,
+ bandwidth_bytes_sec);
/* ignore previous file's error, continue with next file */
/* reset ioc */
#include <assert.h>
#include <sys/xattr.h>
#include <sys/param.h>
+#include <sys/time.h>
+#include <time.h>
#include <libcfs/util/list.h>
#include <lustre/lustreapi.h>
return mirror_id;
}
-int llapi_mirror_resync_many(int fd, struct llapi_layout *layout,
- struct llapi_resync_comp *comp_array,
- int comp_size, uint64_t start, uint64_t end)
+#ifndef NSEC_PER_SEC
+# define NSEC_PER_SEC 1000000000UL
+#endif
+#define ONE_MB 0x100000
+static struct timespec timespec_sub(struct timespec *before,
+ struct timespec *after)
+{
+ struct timespec ret;
+
+ ret.tv_sec = after->tv_sec - before->tv_sec;
+ if (after->tv_nsec < before->tv_nsec) {
+ ret.tv_sec--;
+ ret.tv_nsec = NSEC_PER_SEC + after->tv_nsec - before->tv_nsec;
+ } else {
+ ret.tv_nsec = after->tv_nsec - before->tv_nsec;
+ }
+
+ return ret;
+}
+
+static void stats_log(struct timespec *now, struct timespec *start_time,
+ ssize_t read_bytes, size_t write_bytes,
+ off_t file_size_bytes)
+{
+ struct timespec diff = timespec_sub(start_time, now);
+
+ if (file_size_bytes == 0)
+ return;
+
+ if (diff.tv_sec == 0 && diff.tv_nsec == 0)
+ return;
+
+ llapi_printf(LLAPI_MSG_NORMAL,
+ "- { seconds: %li, rmbps: %5.2g, wmbps: %5.2g, copied: %lu, size: %lu, pct: %lu%% }\n",
+ diff.tv_sec,
+ (double) read_bytes/((ONE_MB * diff.tv_sec) +
+ ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
+ (double) write_bytes/((ONE_MB * diff.tv_sec) +
+ ((ONE_MB * diff.tv_nsec)/NSEC_PER_SEC)),
+ write_bytes/ONE_MB,
+ file_size_bytes/ONE_MB,
+ ((write_bytes*100)/file_size_bytes));
+}
+
+int llapi_mirror_resync_many_params(int fd, struct llapi_layout *layout,
+ struct llapi_resync_comp *comp_array,
+ int comp_size, uint64_t start,
+ uint64_t end,
+ unsigned long stats_interval_sec,
+ unsigned long bandwidth_bytes_sec)
{
size_t page_size = sysconf(_SC_PAGESIZE);
- const size_t buflen = 4 << 20; /* 4M */
+ size_t buflen = 64 << 20; /* 64M */
void *buf;
uint64_t pos = start;
uint64_t data_off = pos, data_end = pos;
int i;
int rc;
int rc2 = 0;
+ struct timespec start_time;
+ struct timespec now;
+ struct timespec last_bw_print;
+ size_t total_bytes_read = 0;
+ size_t total_bytes_written = 0;
+ off_t write_estimation_bytes = 0;
+
+ if (bandwidth_bytes_sec > 0 || stats_interval_sec) {
+ struct stat st;
+ rc = fstat(fd, &st);
+ if (rc < 0)
+ return -errno;
+ write_estimation_bytes = st.st_size * comp_size;
+ }
+
+ /* limit transfer size to what can be sent in one second */
+ if (bandwidth_bytes_sec && bandwidth_bytes_sec < buflen)
+ buflen = (bandwidth_bytes_sec + ONE_MB - 1) & ~(ONE_MB - 1);
rc = posix_memalign(&buf, page_size, buflen);
if (rc)
return -rc;
+ clock_gettime(CLOCK_MONOTONIC, &start_time);
+ now = last_bw_print = start_time;
+
while (pos < end) {
ssize_t bytes_read;
size_t to_read;
rc = bytes_read;
break;
}
+ total_bytes_read += bytes_read;
/* round up to page align to make direct IO happy. */
to_write = ((bytes_read - 1) | (page_size - 1)) + 1;
for (i = 0; i < comp_size; i++) {
+ unsigned long long write_target;
+ struct timespec diff;
ssize_t written;
off_t pos2 = pos;
size_t to_write2 = to_write;
continue;
}
assert(written == to_write2);
+ total_bytes_written += written;
+
+ if (bandwidth_bytes_sec == 0)
+ continue;
+
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ diff = timespec_sub(&start_time, &now);
+ write_target = ((bandwidth_bytes_sec * diff.tv_sec) +
+ ((bandwidth_bytes_sec *
+ diff.tv_nsec)/NSEC_PER_SEC));
+
+ if (write_target < total_bytes_written) {
+ unsigned long long excess;
+ struct timespec delay = { 0, 0 };
+
+ excess = total_bytes_written - write_target;
+
+ if (excess == 0)
+ continue;
+
+ delay.tv_sec = excess / bandwidth_bytes_sec;
+ delay.tv_nsec = (excess % bandwidth_bytes_sec) *
+ NSEC_PER_SEC / bandwidth_bytes_sec;
+
+ do {
+ rc = clock_nanosleep(CLOCK_MONOTONIC, 0,
+ &delay, &delay);
+ } while (rc < 0 && errno == EINTR);
+
+ if (rc < 0) {
+ llapi_error(LLAPI_MSG_ERROR, rc,
+ "errors: delay for bandwidth control failed: %s\n",
+ strerror(-rc));
+ rc = 0;
+ }
+ }
+
+ if (stats_interval_sec) {
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ if ((total_bytes_written != end - start) &&
+ (now.tv_sec >= last_bw_print.tv_sec +
+ stats_interval_sec)) {
+ stats_log(&now, &start_time,
+ total_bytes_read,
+ total_bytes_written,
+ write_estimation_bytes);
+ last_bw_print = now;
+ }
+ }
}
pos += bytes_read;
}
return rc;
}
+ /* Output at least one log, regardless of stats_interval */
+ if (stats_interval_sec) {
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ stats_log(&now, &start_time, total_bytes_read,
+ total_bytes_written,
+ write_estimation_bytes);
+ }
+
/**
* no fatal error happens, each lrc_synced tells whether the component
* has been resync successfully (note: we'd reverse the value to
return rc2;
}
+int llapi_mirror_resync_many(int fd, struct llapi_layout *layout,
+ struct llapi_resync_comp *comp_array,
+ int comp_size, uint64_t start, uint64_t end)
+{
+ return llapi_mirror_resync_many_params(fd, layout, comp_array,
+ comp_size, start, end, 0, 0);
+}
+
enum llapi_layout_comp_sanity_error {
LSE_OK,
LSE_INCOMPLETE_MIRROR,
git://git.whamcloud.com / fs / lustre-release.git / commitdiff