X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=lustre%2Futils%2Flustre_rsync.c;h=1830ac1ecfc307c893dddf2c556bf64fb256867b;hb=f6995cf04407dff15d6ca79ca44cfa97dc6eb014;hp=43fa7b4cb7dd54af983b4382cc60adaafcb4c260;hpb=7efcf4cd0e261389bcd1b5c222325c823845da7c;p=fs%2Flustre-release.git diff --git a/lustre/utils/lustre_rsync.c b/lustre/utils/lustre_rsync.c index 43fa7b4..1830ac1 100644 --- a/lustre/utils/lustre_rsync.c +++ b/lustre/utils/lustre_rsync.c @@ -26,6 +26,8 @@ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2012, 2014, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -119,9 +121,9 @@ #include #include -#include -#include -#include +#include +#include +#include #include #include "lustre_rsync.h" @@ -200,6 +202,8 @@ char rsync[PATH_MAX]; char rsync_ver[PATH_MAX]; struct lr_parent_child_list *parents; +FILE *debug_log; + /* Command line options */ struct option long_opts[] = { {"source", required_argument, 0, 's'}, @@ -217,7 +221,8 @@ struct option long_opts[] = { {"start-recno", required_argument, 0, 'n'}, {"abort-on-err",no_argument, 0, 'a'}, {"debug", required_argument, 0, 'd'}, - {0, 0, 0, 0} + {"debuglog", required_argument, 0, 'D'}, + {0, 0, 0, 0} }; /* Command line usage */ @@ -235,18 +240,31 @@ void lr_usage() "\t--dry-run don't write anything\n"); } +#define DEBUG_ENTRY(info) \ + lr_debug(D_TRACE, "***** Start %lld %s (%d) %s %s %s *****\n", \ + (info)->recno, changelog_type2str((info)->type), \ + (info)->type, (info)->tfid, (info)->pfid, (info)->name); + +#define DEBUG_EXIT(info, rc) \ + lr_debug(D_TRACE, "##### End %lld %s (%d) %s %s %s rc=%d #####\n", \ + (info)->recno, changelog_type2str((info)->type), \ + (info)->type, (info)->tfid, (info)->pfid, (info)->name, rc); + /* Print debug information. This is controlled by the value of the global variable 'debug' */ void lr_debug(int level, const char *fmt, ...) { - va_list ap; + va_list ap; - if (level > debug) - return; + if (level > debug) + return; - va_start(ap, fmt); - vprintf(fmt, ap); - va_end(ap); + va_start(ap, fmt); + if (debug_log != NULL) + vfprintf(debug_log, fmt, ap); + else + vfprintf(stdout, fmt, ap); + va_end(ap); } @@ -552,10 +570,9 @@ int lr_get_symlink(struct lr_info *info) } else { link = info->linktmp; } - strncpy(info->link, link, PATH_MAX); - info->link[PATH_MAX] = '\0'; + strlcpy(info->link, link, sizeof(info->link)); - return rc; + return rc; } /* Create file/directory/device file/symlink. */ @@ -611,18 +628,29 @@ int lr_mkfile(struct lr_info *info) int lr_add_pc(const char *pfid, const char *tfid, const char *name) { - struct lr_parent_child_list *p; - - p = calloc(1, sizeof(*p)); - if (!p) - return -ENOMEM; - strcpy(p->pc_log.pcl_pfid, pfid); - strcpy(p->pc_log.pcl_tfid, tfid); - strcpy(p->pc_log.pcl_name, name); - - p->pc_next = parents; - parents = p; - return 0; + struct lr_parent_child_list *p; + size_t len; + + p = calloc(1, sizeof(*p)); + if (p == NULL) + return -ENOMEM; + len = strlcpy(p->pc_log.pcl_pfid, pfid, sizeof(p->pc_log.pcl_pfid)); + if (len >= sizeof(p->pc_log.pcl_pfid)) + goto out_err; + len = strlcpy(p->pc_log.pcl_tfid, tfid, sizeof(p->pc_log.pcl_tfid)); + if (len >= sizeof(p->pc_log.pcl_tfid)) + goto out_err; + len = strlcpy(p->pc_log.pcl_name, name, sizeof(p->pc_log.pcl_name)); + if (len >= sizeof(p->pc_log.pcl_name)) + goto out_err; + + p->pc_next = parents; + parents = p; + return 0; + +out_err: + free(p); + return -E2BIG; } void lr_cascade_move(const char *fid, const char *dest, struct lr_info *info) @@ -647,11 +675,11 @@ void lr_cascade_move(const char *fid, const char *dest, struct lr_info *info) info->src, d, errno); errors++; } - lr_cascade_move(curr->pc_log.pcl_tfid, d, info); if (curr == parents) parents = curr->pc_next; else prev->pc_next = curr->pc_next; + lr_cascade_move(curr->pc_log.pcl_tfid, d, info); free(curr); prev = curr = parents; @@ -816,7 +844,7 @@ int lr_create(struct lr_info *info) if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate a file remove (rmdir/unlink) operation */ @@ -857,7 +885,7 @@ int lr_remove(struct lr_info *info) continue; } } - return rc; + return rc; } /* Replicate a rename/move operation. */ @@ -895,47 +923,56 @@ int lr_move(struct lr_info *info) status->ls_targets[info->target_no], info->path, info->name); } + lr_debug(DINFO, "dest path %s rc_dest=%d\n", info->dest, + rc_dest); } if (rc_dest == -ENOENT) { snprintf(info->dest, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], SPECIAL_DIR, info->sfid); special_dest = 1; + lr_debug(DINFO, "special dest %s\n", info->dest); } - if (!rc_src) - snprintf(info->src, PATH_MAX, "%s/%s/%s", - status->ls_targets[info->target_no], + if (!rc_src) { + snprintf(info->src, PATH_MAX, "%s/%s/%s", + status->ls_targets[info->target_no], srcpath, info->sname); + lr_debug(DINFO, "src path %s rc_src=%d\n", info->src, + rc_src); + } if (rc_src == -ENOENT || (access(info->src, F_OK) != 0 && errno == ENOENT)) { snprintf(info->src, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], SPECIAL_DIR, info->sfid); special_src = 1; + lr_debug(DINFO, "special src %s\n", info->src); } rc1 = 0; + errno = 0; if (strcmp(info->src, info->dest) != 0) { rc1 = rename(info->src, info->dest); if (rc1 == -1) rc1 = -errno; + lr_debug(DINFO, "rename returns %d\n", rc1); } if (special_src) { - lr_remove_pc(info->spfid, info->sfid); + rc1 = lr_remove_pc(info->spfid, info->sfid); if (!special_dest) lr_cascade_move(info->sfid, info->dest, info); } if (special_dest) - lr_add_pc(info->pfid, info->sfid, info->name); + rc1 = lr_add_pc(info->pfid, info->sfid, info->name); lr_debug(DINFO, "move: %s [to] %s rc1=%d, errno=%d\n", info->src, info->dest, rc1, errno); if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate a hard link */ @@ -969,7 +1006,7 @@ int lr_link(struct lr_info *info) break; len = strlen(info->path) - strlen(info->name); - if (len > 0 && strcmp(info->path + len, + if (len >= 0 && strcmp(info->path + len, info->name) == 0) snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[info->target_no], @@ -1007,7 +1044,7 @@ int lr_link(struct lr_info *info) if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate file attributes */ @@ -1039,7 +1076,7 @@ int lr_setattr(struct lr_info *info) if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate xattrs */ @@ -1069,37 +1106,49 @@ int lr_setxattr(struct lr_info *info) rc = rc1; } - return rc; + return rc; } /* Parse a line of changelog entry */ int lr_parse_line(void *priv, struct lr_info *info) { - struct changelog_ext_rec *rec; - - if (llapi_changelog_recv(priv, &rec) != 0) - return -1; - - info->is_extended = CHANGELOG_REC_EXTENDED(rec); - info->recno = rec->cr_index; - info->type = rec->cr_type; - sprintf(info->tfid, DFID, PFID(&rec->cr_tfid)); - sprintf(info->pfid, DFID, PFID(&rec->cr_pfid)); - strncpy(info->name, rec->cr_name, rec->cr_namelen); - - if (fid_is_sane(&rec->cr_sfid)) { - sprintf(info->sfid, DFID, PFID(&rec->cr_sfid)); - sprintf(info->spfid, DFID, PFID(&rec->cr_spfid)); - strncpy(info->sname, changelog_rec_sname(rec), - changelog_rec_snamelen(rec)); - info->sname[changelog_rec_snamelen(rec)] = '\0'; + struct changelog_rec *rec; + struct changelog_ext_rename *rnm; + size_t namelen; + size_t copylen = sizeof(info->name); + + if (llapi_changelog_recv(priv, &rec) != 0) + return -1; + + info->is_extended = !!(rec->cr_flags & CLF_RENAME); + info->recno = rec->cr_index; + info->type = rec->cr_type; + snprintf(info->tfid, sizeof(info->tfid), DFID, PFID(&rec->cr_tfid)); + snprintf(info->pfid, sizeof(info->pfid), DFID, PFID(&rec->cr_pfid)); + + namelen = strnlen(changelog_rec_name(rec), rec->cr_namelen); + if (copylen > namelen + 1) + copylen = namelen + 1; + strlcpy(info->name, changelog_rec_name(rec), copylen); + + /* Don't use rnm if CLF_RENAME isn't set */ + rnm = changelog_rec_rename(rec); + if (rec->cr_flags & CLF_RENAME && !fid_is_zero(&rnm->cr_sfid)) { + copylen = sizeof(info->sname); + + snprintf(info->sfid, sizeof(info->sfid), DFID, + PFID(&rnm->cr_sfid)); + snprintf(info->spfid, sizeof(info->spfid), DFID, + PFID(&rnm->cr_spfid)); + namelen = changelog_rec_snamelen(rec); + if (copylen > namelen + 1) + copylen = namelen + 1; + strlcpy(info->sname, changelog_rec_sname(rec), copylen); if (verbose > 1) printf("Rec %lld: %d %s %s\n", info->recno, info->type, info->name, info->sname); } else { - info->name[rec->cr_namelen] = '\0'; - if (verbose > 1) printf("Rec %lld: %d %s\n", info->recno, info->type, info->name); @@ -1114,7 +1163,7 @@ int lr_parse_line(void *priv, struct lr_info *info) /* Initialize the replication parameters */ int lr_init_status() { - size_t size = sizeof(struct lustre_rsync_status) + PATH_MAX; + size_t size = sizeof(struct lustre_rsync_status) + PATH_MAX + 1; if (status != NULL) return 0; @@ -1193,38 +1242,57 @@ int lr_read_log() struct lustre_rsync_status *s; int fd = -1; size_t size; - size_t read_size = sizeof(struct lustre_rsync_status) + PATH_MAX; + size_t read_size = sizeof(struct lustre_rsync_status) + PATH_MAX + 1; int rc = 0; if (statuslog == NULL) return 0; s = calloc(1, read_size); - if (s == NULL) - GOTO(out, rc = -ENOMEM); - - fd = open(statuslog, O_RDONLY); - if (fd == -1) - GOTO(out, rc = -errno); - size = read(fd, s, read_size); - if (size != read_size) - GOTO(out, rc = -EINVAL); - if (read_size < s->ls_size) { - read_size = s->ls_size; - s = lr_grow_buf(s, read_size); - if (s == NULL) - GOTO(out, rc = -ENOMEM); - if (lseek(fd, 0, SEEK_SET) == -1) - GOTO(out, rc = -errno); - size = read(fd, s, read_size); - if (size != read_size) - GOTO(out, rc = -EINVAL); - } + if (s == NULL) { + rc = -ENOMEM; + goto out; + } + + fd = open(statuslog, O_RDONLY); + if (fd == -1) { + rc = -errno; + goto out; + } + + size = read(fd, s, read_size); + if (size != read_size) { + rc = -EINVAL; + goto out; + } + + if (read_size < s->ls_size) { + read_size = s->ls_size; + s = lr_grow_buf(s, read_size); + if (s == NULL) { + rc = -ENOMEM; + goto out; + } + + if (lseek(fd, 0, SEEK_SET) == -1) { + rc = -ENOMEM; + goto out; + } + + size = read(fd, s, read_size); + if (size != read_size) { + rc = -EINVAL; + goto out; + } + } + + while (read(fd, &rec, sizeof(rec)) != 0) { + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) { + rc = -ENOMEM; + goto out; + } - while (read(fd, &rec, sizeof(rec)) != 0) { - tmp = calloc(1, sizeof(*tmp)); - if (!tmp) - GOTO(out, rc = -ENOMEM); tmp->pc_log = rec; tmp->pc_next = parents; parents = tmp; @@ -1234,31 +1302,35 @@ int lr_read_log() if (status->ls_num_targets == 0) { if (status->ls_size != s->ls_size) { status = lr_grow_buf(status, s->ls_size); - if (status == NULL) - GOTO(out, rc = -ENOMEM); + if (status == NULL) { + rc = -ENOMEM; + goto out; + } + status->ls_size = s->ls_size; } status->ls_num_targets = s->ls_num_targets; memcpy(status->ls_targets, s->ls_targets, - PATH_MAX * s->ls_num_targets); + (PATH_MAX + 1) * s->ls_num_targets); } if (status->ls_last_recno == -1) status->ls_last_recno = s->ls_last_recno; - if (status->ls_registration[0] == '\0') - strncpy(status->ls_registration, s->ls_registration, - LR_NAME_MAXLEN); + if (status->ls_registration[0] == '\0') + strlcpy(status->ls_registration, s->ls_registration, + sizeof(status->ls_registration)); - if (status->ls_mdt_device[0] == '\0') - strncpy(status->ls_mdt_device, s->ls_mdt_device, - LR_NAME_MAXLEN); + if (status->ls_mdt_device[0] == '\0') + strlcpy(status->ls_mdt_device, s->ls_mdt_device, + sizeof(status->ls_mdt_device)); - if (status->ls_source_fs[0] == '\0') - strncpy(status->ls_source_fs, s->ls_source_fs, - LR_NAME_MAXLEN); + if (status->ls_source_fs[0] == '\0') + strlcpy(status->ls_source_fs, s->ls_source_fs, + sizeof(status->ls_source_fs)); - if (status->ls_source[0] == '\0') - strncpy(status->ls_source, s->ls_source, PATH_MAX); + if (status->ls_source[0] == '\0') + strlcpy(status->ls_source, s->ls_source, + sizeof(status->ls_source)); out: if (fd != -1) @@ -1272,9 +1344,9 @@ int lr_read_log() processing. */ int lr_clear_cl(struct lr_info *info, int force) { - char mdt_device[LR_NAME_MAXLEN + 1]; - long long rec; - int rc = 0; + char mdt_device[LR_NAME_MAXLEN + 1]; + long long rec; + int rc = 0; if (force || info->recno > status->ls_last_recno + CLEAR_INTERVAL) { if (info->type == CL_RENAME) @@ -1286,8 +1358,8 @@ int lr_clear_cl(struct lr_info *info, int force) * device name so make a copy of it until this * is fixed. */ - strncpy(mdt_device, status->ls_mdt_device, - LR_NAME_MAXLEN); + strlcpy(mdt_device, status->ls_mdt_device, + sizeof(mdt_device)); rc = llapi_changelog_clear(mdt_device, status->ls_registration, rec); @@ -1386,7 +1458,7 @@ int lr_replicate() { void *changelog_priv; struct lr_info *info; - struct lr_info *ext; + struct lr_info *ext = NULL; time_t start; int xattr_not_supp; int i; @@ -1402,15 +1474,17 @@ int lr_replicate() if (rc) { fprintf(stderr, "Source path is not a valid Lustre client " "mountpoint.\n"); - return rc; + goto out; } if (status->ls_mdt_device[0] == '\0') snprintf(status->ls_mdt_device, LR_NAME_MAXLEN, "%s%s", status->ls_source_fs, DEFAULT_MDT); ext = calloc(1, sizeof(struct lr_info)); - if (ext == NULL) - return -ENOMEM; + if (ext == NULL) { + rc = -ENOMEM; + goto out; + } for (i = 0, xattr_not_supp = 0; i < status->ls_num_targets; i++) { snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[i], @@ -1419,7 +1493,8 @@ int lr_replicate() if (rc == -1 && errno != EEXIST) { fprintf(stderr, "Error writing to target path %s.\n", status->ls_targets[i]); - return -errno; + rc = -errno; + goto out; } rc = llistxattr(info->src, info->xlist, info->xsize); if (rc == -1 && errno == ENOTSUP) { @@ -1434,13 +1509,14 @@ int lr_replicate() lr_print_status(info); - /* Open changelogs for consumption*/ - rc = llapi_changelog_start(&changelog_priv, CHANGELOG_FLAG_BLOCK, - status->ls_source_fs, status->ls_last_recno); + /* Open changelogs for consumption*/ + rc = llapi_changelog_start(&changelog_priv, + CHANGELOG_FLAG_BLOCK | CHANGELOG_FLAG_JOBID, + status->ls_source_fs, status->ls_last_recno); if (rc < 0) { fprintf(stderr, "Error opening changelog file for fs %s.\n", status->ls_source_fs); - return rc; + goto out; } while (!quit && lr_parse_line(changelog_priv, info) == 0) { @@ -1456,14 +1532,16 @@ int lr_replicate() memcpy(info->spfid, info->pfid, sizeof(info->spfid)); memcpy(info->tfid, ext->tfid, sizeof(info->tfid)); memcpy(info->pfid, ext->pfid, sizeof(info->pfid)); - strncpy(info->sname, info->name, sizeof(info->sname)); - strncpy(info->name, ext->name, sizeof(info->name)); + strlcpy(info->sname, info->name, sizeof(info->sname)); + strlcpy(info->name, ext->name, sizeof(info->name)); info->is_extended = 1; } if (dryrun) continue; + DEBUG_ENTRY(info); + switch(info->type) { case CL_CREATE: case CL_MKDIR: @@ -1488,15 +1566,17 @@ int lr_replicate() case CL_XATTR: rc = lr_setxattr(info); break; - case CL_CLOSE: - case CL_EXT: - case CL_OPEN: - case CL_IOCTL: - case CL_MARK: - /* Nothing needs to be done for these entries */ - default: - break; - } + case CL_CLOSE: + case CL_EXT: + case CL_OPEN: + case CL_LAYOUT: + case CL_MARK: + /* Nothing needs to be done for these entries */ + /* fallthrough */ + default: + break; + } + DEBUG_EXIT(info, rc); if (rc && rc != -ENOENT) { lr_print_failure(info, rc); errors++; @@ -1523,7 +1603,15 @@ int lr_replicate() printf("Changelog records consumed: %lld\n", rec_count); } - return 0; + rc = 0; + +out: + if (info != NULL) + free(info); + if (ext != NULL) + free(ext); + + return rc; } void @@ -1543,8 +1631,8 @@ int main(int argc, char *argv[]) if ((rc = lr_init_status()) != 0) return rc; - while ((rc = getopt_long(argc, argv, "as:t:m:u:l:vx:zc:ry:n:d:", - long_opts, NULL)) >= 0) { + while ((rc = getopt_long(argc, argv, "as:t:m:u:l:vx:zc:ry:n:d:D:", + long_opts, NULL)) >= 0) { switch (rc) { case 'a': /* Assume absolute paths */ @@ -1552,7 +1640,8 @@ int main(int argc, char *argv[]) break; case 's': /* Assume absolute paths */ - strncpy(status->ls_source, optarg, PATH_MAX); + strlcpy(status->ls_source, optarg, + sizeof(status->ls_source)); break; case 't': status->ls_num_targets++; @@ -1566,23 +1655,23 @@ int main(int argc, char *argv[]) status->ls_num_targets = numtargets; } newsize = sizeof (struct lustre_rsync_status) + - (status->ls_num_targets * PATH_MAX); + (status->ls_num_targets * (PATH_MAX + 1)); if (status->ls_size != newsize) { status->ls_size = newsize; status = lr_grow_buf(status, newsize); if (status == NULL) return -ENOMEM; } - strncpy(status->ls_targets[status->ls_num_targets - 1], - optarg, - PATH_MAX); - break; - case 'm': - strncpy(status->ls_mdt_device, optarg, LR_NAME_MAXLEN); - break; - case 'u': - strncpy(status->ls_registration, optarg, - LR_NAME_MAXLEN); + strlcpy(status->ls_targets[status->ls_num_targets - 1], + optarg, sizeof(status->ls_targets[0])); + break; + case 'm': + strlcpy(status->ls_mdt_device, optarg, + sizeof(status->ls_mdt_device)); + break; + case 'u': + strlcpy(status->ls_registration, optarg, + sizeof(status->ls_registration)); break; case 'l': statuslog = optarg; @@ -1634,6 +1723,17 @@ int main(int argc, char *argv[]) if (debug < 0 || debug > 2) debug = 0; break; + case 'D': + /* Undocumented option debug log file */ + if (debug_log != NULL) + fclose(debug_log); + debug_log = fopen(optarg, "a"); + if (debug_log == NULL) { + printf("Cannot open %s for debug log\n", + optarg); + return -1; + } + break; default: fprintf(stderr, "error: %s: option '%s' " "unrecognized.\n", argv[0], argv[optind - 1]); @@ -1680,5 +1780,7 @@ int main(int argc, char *argv[]) rc = lr_replicate(); - return rc; + if (debug_log != NULL) + fclose(debug_log); + return rc; }