X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;ds=sidebyside;f=lustre%2Futils%2Flustre_rsync.c;h=361d2ad06fc0f1d91f2e3de7c5f1465215784f08;hb=9afa14529b35de066b395abba448e4d2e9c2999f;hp=5ef329712eaec9a2a91a2bb6ae7f397c9d80bc2e;hpb=b53302d92dfe124c75d54b05937bc2940aadf1c9;p=fs%2Flustre-release.git diff --git a/lustre/utils/lustre_rsync.c b/lustre/utils/lustre_rsync.c index 5ef3297..361d2ad 100644 --- a/lustre/utils/lustre_rsync.c +++ b/lustre/utils/lustre_rsync.c @@ -1,6 +1,4 @@ -/* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- - * vim:expandtab:shiftwidth=8:tabstop=8: - * +/* * GPL HEADER START * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. @@ -17,17 +15,15 @@ * * You should have received a copy of the GNU General Public License * version 2 along with this program; If not, see - * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf - * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * http://www.gnu.org/licenses/gpl-2.0.html * * GPL HEADER END */ /* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved + * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. + * + * Copyright (c) 2012, 2016, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -62,7 +58,11 @@ * tfid - The FID of the target file * pfid - The FID of the parent of the target file (at the time of * the operation) - * name - The name of the target file (at the time of the operation) + * sfid - The FID of the source file + * spfid - The FID of the parent of the source file + * name - The name of the target file (at the time of the operation), the name + * of the source file is appended (delimited with '\0') if this + * operation involves a source * * With just this information, it is not alwasy possible to determine * the file paths for each operation. For instance, if pfid does not @@ -72,7 +72,7 @@ * changelog are replayed, all the files in this special directory * will get moved to the location as in the source-fs. * - * Shorthand used: f2p(tfid) = fid2path(tfid) + * Shorthand used: f2p(fid) = fid2path(fid) * * The following are the metadata operations of interest. * 1. creat @@ -90,19 +90,20 @@ * rm .lustrerepl/[tfid] * Else if pfid is present on the source-fs, * if f2p(pfid)+name is present, - * rm f2p(pfid)+name(pfid,name) + * rm f2p(pfid)+name * - * 3. move (pfid1,name1) to (pfid2,name2) - * If pfid2 is present - * if pfid1 is also present, mv (pfid1,name1) to (pfid2,name2) - * else mv .lustrerepl/[tfid] to (pfid2,name2) - * If pfid2 is not present, - * if pfid1 is present, mv (pfid1,name1) .lustrerepl/[tfid] + * 3. move (spfid,sname) to (pfid,name) + * If pfid is present + * if spfid is also present, mv (spfid,sname) to (pfid,name) + * else mv .lustrerepl/[sfid] to (pfid,name) + * Else if pfid is not present, + * if spfid is present, mv (spfid,sname) .lustrerepl/[sfid] * If moving out of .lustrerepl * move out all its children in .lustrerepl. * [pfid,tfid,name] tracked from (1) is used for this. */ +#include #include #include #include @@ -110,17 +111,20 @@ #include #include #include +#include #include #include #include #include #include +#include #include +#include -#include -#include -#include -#include +#include +#include +#include +#include #include "lustre_rsync.h" #define REPLICATE_STATUS_VER 1 @@ -148,10 +152,14 @@ extern int obd_initialize(int argc, char **argv); struct lr_info { long long recno; int target_no; + unsigned int is_extended:1; enum changelog_rec_type type; - char pfid[LR_FID_STR_LEN]; - char tfid[LR_FID_STR_LEN]; - char name[PATH_MAX + 1]; + char tfid[LR_FID_STR_LEN]; + char pfid[LR_FID_STR_LEN]; + char sfid[LR_FID_STR_LEN]; + char spfid[LR_FID_STR_LEN]; + char sname[NAME_MAX + 1]; + char name[NAME_MAX + 1]; char src[PATH_MAX + 1]; char dest[PATH_MAX + 1]; char path[PATH_MAX + 1]; @@ -194,25 +202,28 @@ char rsync[PATH_MAX]; char rsync_ver[PATH_MAX]; struct lr_parent_child_list *parents; +FILE *debug_log; + /* Command line options */ struct option long_opts[] = { - {"source", required_argument, 0, 's'}, - {"target", required_argument, 0, 't'}, - {"mdt", required_argument, 0, 'm'}, - {"user", required_argument, 0, 'u'}, - {"statuslog", required_argument, 0, 'l'}, - {"verbose", no_argument, 0, 'v'}, - {"xattr", required_argument, 0, 'x'}, - {"dry-run", no_argument, 0, 'z'}, - /* Undocumented options follow */ - {"cl-clear", required_argument, 0, 'c'}, - {"use-rsync", no_argument, 0, 'r'}, - {"rsync-threshold", required_argument, 0, 'y'}, - {"start-recno", required_argument, 0, 'n'}, - {"abort-on-err",no_argument, 0, 'a'}, - {"debug", required_argument, 0, 'd'}, - {0, 0, 0, 0} -}; + { .val = 'l', .name = "statuslog", .has_arg = required_argument }, + { .val = 'm', .name = "mdt", .has_arg = required_argument }, + { .val = 's', .name = "source", .has_arg = required_argument }, + { .val = 't', .name = "target", .has_arg = required_argument }, + { .val = 'u', .name = "user", .has_arg = required_argument }, + { .val = 'v', .name = "verbose", .has_arg = no_argument }, + { .val = 'x', .name = "xattr", .has_arg = required_argument }, + { .val = 'z', .name = "dry-run", .has_arg = no_argument }, + /* Undocumented options follow */ + { .val = 'a', .name = "abort-on-err", .has_arg = no_argument }, + { .val = 'c', .name = "cl-clear", .has_arg = required_argument }, + { .val = 'd', .name = "debug", .has_arg = required_argument }, + { .val = 'D', .name = "debuglog", .has_arg = required_argument }, + { .val = 'n', .name = "start-recno", .has_arg = required_argument }, + { .val = 'r', .name = "use-rsync", .has_arg = no_argument }, + { .val = 'y', .name = "rsync-threshold", + .has_arg = required_argument }, + { .name = NULL } }; /* Command line usage */ void lr_usage() @@ -229,18 +240,31 @@ void lr_usage() "\t--dry-run don't write anything\n"); } +#define DEBUG_ENTRY(info) \ + lr_debug(D_TRACE, "***** Start %lld %s (%d) %s %s %s *****\n", \ + (info)->recno, changelog_type2str((info)->type), \ + (info)->type, (info)->tfid, (info)->pfid, (info)->name); + +#define DEBUG_EXIT(info, rc) \ + lr_debug(D_TRACE, "##### End %lld %s (%d) %s %s %s rc=%d #####\n", \ + (info)->recno, changelog_type2str((info)->type), \ + (info)->type, (info)->tfid, (info)->pfid, (info)->name, rc); + /* Print debug information. This is controlled by the value of the global variable 'debug' */ void lr_debug(int level, const char *fmt, ...) { - va_list ap; + va_list ap; - if (level > debug) - return; + if (level > debug) + return; - va_start(ap, fmt); - vprintf(fmt, ap); - va_end(ap); + va_start(ap, fmt); + if (debug_log != NULL) + vfprintf(debug_log, fmt, ap); + else + vfprintf(stdout, fmt, ap); + va_end(ap); } @@ -366,31 +390,39 @@ int lr_copy_data(struct lr_info *info) info->bufsize = bufsize; } - while (1) { - rsize = read(fd_src, info->buf, bufsize); - if (rsize == 0) { - break; - } else if (rsize < 0) { - rc = -errno; - goto out; - } - errno = 0; - if (write(fd_dest, info->buf, rsize) != rsize) { - if (errno != 0) - rc = -errno; - else - rc = -EINTR; - } - } - fsync(fd_dest); + while (1) { + char *buf; + int wsize; + + buf = info->buf; + rsize = read(fd_src, buf, bufsize); + if (rsize == 0) { + rc = 0; + break; + } + if (rsize < 0) { + rc = -errno; + break; + } + do { + wsize = write(fd_dest, buf, rsize); + if (wsize <= 0) { + rc = -errno; + break; + } + rsize -= wsize; + buf += wsize; + } while (rsize > 0); + } + fsync(fd_dest); out: - if (fd_src != -1) - close(fd_src); - if (fd_dest != -1) - close(fd_dest); + if (fd_src != -1) + close(fd_src); + if (fd_dest != -1) + close(fd_dest); - return rc; + return rc; } /* Copy data from source to destination */ @@ -546,10 +578,9 @@ int lr_get_symlink(struct lr_info *info) } else { link = info->linktmp; } - strncpy(info->link, link, PATH_MAX); - info->link[PATH_MAX] = '\0'; + strlcpy(info->link, link, sizeof(info->link)); - return rc; + return rc; } /* Create file/directory/device file/symlink. */ @@ -579,8 +610,13 @@ int lr_mkfile(struct lr_info *info) } else { rc = mknod(info->dest, S_IFREG | 0777, 0); } - if (rc) - return -errno; + + if (rc < 0) { + if (errno == EEXIST) + rc = 0; + else + return -errno; + } /* Sync data and attributes */ if (info->type == CL_CREATE || info->type == CL_MKDIR) { @@ -605,18 +641,29 @@ int lr_mkfile(struct lr_info *info) int lr_add_pc(const char *pfid, const char *tfid, const char *name) { - struct lr_parent_child_list *p; - - p = calloc(1, sizeof(*p)); - if (!p) - return -ENOMEM; - strcpy(p->pc_log.pcl_pfid, pfid); - strcpy(p->pc_log.pcl_tfid, tfid); - strcpy(p->pc_log.pcl_name, name); - - p->pc_next = parents; - parents = p; - return 0; + struct lr_parent_child_list *p; + size_t len; + + p = calloc(1, sizeof(*p)); + if (p == NULL) + return -ENOMEM; + len = strlcpy(p->pc_log.pcl_pfid, pfid, sizeof(p->pc_log.pcl_pfid)); + if (len >= sizeof(p->pc_log.pcl_pfid)) + goto out_err; + len = strlcpy(p->pc_log.pcl_tfid, tfid, sizeof(p->pc_log.pcl_tfid)); + if (len >= sizeof(p->pc_log.pcl_tfid)) + goto out_err; + len = strlcpy(p->pc_log.pcl_name, name, sizeof(p->pc_log.pcl_name)); + if (len >= sizeof(p->pc_log.pcl_name)) + goto out_err; + + p->pc_next = parents; + parents = p; + return 0; + +out_err: + free(p); + return -E2BIG; } void lr_cascade_move(const char *fid, const char *dest, struct lr_info *info) @@ -641,11 +688,11 @@ void lr_cascade_move(const char *fid, const char *dest, struct lr_info *info) info->src, d, errno); errors++; } - lr_cascade_move(curr->pc_log.pcl_tfid, d, info); if (curr == parents) parents = curr->pc_next; else prev->pc_next = curr->pc_next; + lr_cascade_move(curr->pc_log.pcl_tfid, d, info); free(curr); prev = curr = parents; @@ -658,7 +705,7 @@ void lr_cascade_move(const char *fid, const char *dest, struct lr_info *info) free(d); } -/* remove [info->pfid, ext->tfid] from parents */ +/* remove [info->spfid, info->sfid] from parents */ int lr_remove_pc(const char *pfid, const char *tfid) { struct lr_parent_child_list *curr, *prev; @@ -708,6 +755,19 @@ int lr_rmfile(struct lr_info *info) return rc; } +/* Recursively remove directory and its contents */ +int lr_rm_recursive(struct lr_info *info) +{ + int rc; + + snprintf(info->cmd, PATH_MAX, "rm -rf %s", info->dest); + rc = system(info->cmd); + if (rc == -1) + rc = -errno; + + return rc; +} + /* Remove a file under SPECIAL_DIR with its tfid as its name. */ int lr_rm_special(struct lr_info *info) { @@ -756,7 +816,9 @@ int lr_create(struct lr_info *info) /* Is f2p(pfid)+name != f2p(tfid)? If not the file has moved. */ len = strlen(info->path); - if (len - 1 >= 0 && info->path[len - 1] == '/') + if (len == 1 && info->path[0] == '/') + snprintf(info->dest, PATH_MAX, "%s", info->name); + else if (len - 1 > 0 && info->path[len - 1] == '/') snprintf(info->dest, PATH_MAX, "%s%s", info->path, info->name); else snprintf(info->dest, PATH_MAX, "%s/%s", info->path, info->name); @@ -772,14 +834,17 @@ int lr_create(struct lr_info *info) /* Is f2p(pfid) present on the target? If not, the parent has moved */ if (!mkspecial) { - snprintf(info->dest, PATH_MAX, "%s%s", status->ls_targets[0], - info->path); - if (access(info->dest, F_OK) != 0) - mkspecial = 1; + snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[0], + info->path); + if (access(info->dest, F_OK) != 0) { + lr_debug(DTRACE, "create: parent %s not found\n", + info->dest); + mkspecial = 1; + } } for (info->target_no = 0; info->target_no < status->ls_num_targets; info->target_no++) { - snprintf(info->dest, PATH_MAX, "%s%s", + snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[info->target_no], info->savedpath); lr_get_FID_PATH(status->ls_source, info->tfid, info->src, PATH_MAX); @@ -792,7 +857,7 @@ int lr_create(struct lr_info *info) if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate a file remove (rmdir/unlink) operation */ @@ -818,101 +883,115 @@ int lr_remove(struct lr_info *info) rc = rc1; continue; } - snprintf(info->dest, PATH_MAX, "%s%s/%s", + snprintf(info->dest, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], info->path, info->name); rc1 = lr_rmfile(info); lr_debug(DINFO, "remove: %s; rc1=%d, errno=%d\n", info->dest, rc1, errno); + if (rc1 == -ENOTEMPTY) + rc1 = lr_rm_recursive(info); + if (rc1) { rc = rc1; continue; } } - return rc; + return rc; } -/* Replicate a rename/move operation. This operations are tracked by - two changelog records. */ -int lr_move(struct lr_info *info, struct lr_info *ext) +/* Replicate a rename/move operation. */ +int lr_move(struct lr_info *info) { int rc = 0; int rc1; int rc_dest, rc_src; int special_src = 0; int special_dest = 0; + char srcpath[PATH_MAX + 1] = ""; + + assert(info->is_extended); - rc_dest = lr_get_path(ext, ext->pfid); - if (rc_dest < 0 && rc_dest != -ENOENT) - return rc_dest; + rc_src = lr_get_path(info, info->spfid); + if (rc_src < 0 && rc_src != -ENOENT) + return rc_src; + memcpy(srcpath, info->path, strlen(info->path)); - rc_src = lr_get_path(info, info->pfid); - if (rc_src < 0 && rc_src != -ENOENT) - return rc_src; + rc_dest = lr_get_path(info, info->pfid); + if (rc_dest < 0 && rc_dest != -ENOENT) + return rc_dest; for (info->target_no = 0; info->target_no < status->ls_num_targets; info->target_no++) { if (!rc_dest) { - snprintf(info->dest, PATH_MAX, "%s%s", + snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[info->target_no], - ext->path); + info->path); if (access(info->dest, F_OK) != 0) { rc_dest = -errno; } else { - snprintf(info->dest, PATH_MAX, "%s%s/%s", + snprintf(info->dest, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], - ext->path, ext->name); + info->path, info->name); } + lr_debug(DINFO, "dest path %s rc_dest=%d\n", info->dest, + rc_dest); } if (rc_dest == -ENOENT) { snprintf(info->dest, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], - SPECIAL_DIR, info->tfid); + SPECIAL_DIR, info->sfid); special_dest = 1; + lr_debug(DINFO, "special dest %s\n", info->dest); } - if (!rc_src) - snprintf(info->src, PATH_MAX, "%s%s/%s", - status->ls_targets[info->target_no], - info->path, info->name); + if (!rc_src) { + snprintf(info->src, PATH_MAX, "%s/%s/%s", + status->ls_targets[info->target_no], + srcpath, info->sname); + lr_debug(DINFO, "src path %s rc_src=%d\n", info->src, + rc_src); + } if (rc_src == -ENOENT || (access(info->src, F_OK) != 0 && errno == ENOENT)) { snprintf(info->src, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], - SPECIAL_DIR, info->tfid); + SPECIAL_DIR, info->sfid); special_src = 1; + lr_debug(DINFO, "special src %s\n", info->src); } rc1 = 0; + errno = 0; if (strcmp(info->src, info->dest) != 0) { rc1 = rename(info->src, info->dest); if (rc1 == -1) rc1 = -errno; + lr_debug(DINFO, "rename returns %d\n", rc1); } - if (special_src) { - lr_remove_pc(info->pfid, info->tfid); - if (!special_dest) - lr_cascade_move(info->tfid, info->dest, info); - } - if (special_dest) - lr_add_pc(ext->pfid, info->tfid, ext->name); + if (special_src) + rc1 = lr_remove_pc(info->spfid, info->sfid); + + if (!special_dest) + lr_cascade_move(info->sfid, info->dest, info); + else + rc1 = lr_add_pc(info->pfid, info->sfid, info->name); lr_debug(DINFO, "move: %s [to] %s rc1=%d, errno=%d\n", info->src, info->dest, rc1, errno); if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate a hard link */ int lr_link(struct lr_info *info) { int i; - int len; int rc; int rc1; struct stat st; @@ -925,42 +1004,57 @@ int lr_link(struct lr_info *info) for (info->target_no = 0; info->target_no < status->ls_num_targets; info->target_no++) { - info->src[0] = 0; - info->dest[0] = 0; - rc1 = 0; - - /* Search through the hardlinks to get the src and dest */ - for (i = 0; i < st.st_nlink && (info->src[0] == 0 || - info->dest[0] == 0); i++) { + info->src[0] = 0; + info->dest[0] = 0; + rc1 = 0; + + /* + * The changelog record has the new parent directory FID and + * name of the target file. So info->dest can be constructed + * by getting the path of the new parent directory and + * appending the target file name. + */ + rc1 = lr_get_path(info, info->pfid); + lr_debug(rc1 ? 0 : DTRACE, "\tparent fid2path %s, %s, rc=%d\n", + info->path, info->name, rc1); + + if (rc1 == 0) { + snprintf(info->dest, sizeof(info->dest), "%s/%s/%s", + status->ls_targets[info->target_no], + info->path, info->name); + lr_debug(DINFO, "link destination is %s\n", info->dest); + } + + /* Search through the hardlinks to get the src */ + for (i = 0; i < st.st_nlink && info->src[0] == 0; i++) { rc1 = lr_get_path_ln(info, info->tfid, i); lr_debug(rc1 ? 0:DTRACE, "\tfid2path %s, %s, %d rc=%d\n", info->path, info->name, i, rc1); if (rc1) break; - len = strlen(info->path) - strlen(info->name); - if (len > 0 && strcmp(info->path + len, - info->name) == 0) - snprintf(info->dest, PATH_MAX, "%s%s", - status->ls_targets[info->target_no], - info->path); - else if (info->src[0] == 0) - snprintf(info->src, PATH_MAX, "%s%s", - status->ls_targets[info->target_no], - info->path); - } + /* + * Compare the path of target FID with info->dest + * to find out info->src. + */ + char srcpath[PATH_MAX]; + + snprintf(srcpath, sizeof(srcpath), "%s/%s", + status->ls_targets[info->target_no], + info->path); + + if (strcmp(srcpath, info->dest) != 0) { + strlcpy(info->src, srcpath, sizeof(info->src)); + lr_debug(DINFO, "link source is %s\n", + info->src); + } + } if (rc1) { rc = rc1; continue; } - if (info->src[0] == 0 || info->dest[0] == 0) - /* Could not find the source or destination. - This can happen when some links don't exist - anymore. */ - return -EINVAL; - if (info->src[0] == 0) snprintf(info->src, PATH_MAX, "%s/%s/%s", status->ls_targets[info->target_no], @@ -970,14 +1064,15 @@ int lr_link(struct lr_info *info) status->ls_targets[info->target_no], SPECIAL_DIR, info->tfid); - rc1 = link(info->src, info->dest); - lr_debug(rc1?0:DINFO, "link: %s [to] %s; rc1=%d %s\n", - info->src, info->dest, rc1, strerror(errno)); + rc1 = link(info->src, info->dest); + lr_debug(DINFO, "link: %s [to] %s; rc1=%d %s\n", + info->src, info->dest, rc1, + strerror(rc1 ? errno : 0)); if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate file attributes */ @@ -998,7 +1093,7 @@ int lr_setattr(struct lr_info *info) for (info->target_no = 0; info->target_no < status->ls_num_targets; info->target_no++) { - snprintf(info->dest, PATH_MAX, "%s%s", + snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[info->target_no], info->path); lr_debug(DINFO, "setattr: %s %s %s", info->src, info->dest, info->tfid); @@ -1009,7 +1104,7 @@ int lr_setattr(struct lr_info *info) if (rc1) rc = rc1; } - return rc; + return rc; } /* Replicate xattrs */ @@ -1029,7 +1124,7 @@ int lr_setxattr(struct lr_info *info) for (info->target_no = 0; info->target_no < status->ls_num_targets; info->target_no++) { - snprintf(info->dest, PATH_MAX, "%s%s", + snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[info->target_no], info->path); lr_debug(DINFO, "setxattr: %s %s %s\n", info->src, info->dest, info->tfid); @@ -1039,26 +1134,53 @@ int lr_setxattr(struct lr_info *info) rc = rc1; } - return rc; + return rc; } /* Parse a line of changelog entry */ int lr_parse_line(void *priv, struct lr_info *info) { - struct changelog_rec *rec; - - if (llapi_changelog_recv(priv, &rec) != 0) - return -1; - - info->recno = rec->cr_index; - info->type = rec->cr_type; - sprintf(info->tfid, DFID, PFID(&rec->cr_tfid)); - sprintf(info->pfid, DFID, PFID(&rec->cr_pfid)); - strncpy(info->name, rec->cr_name, rec->cr_namelen); - info->name[rec->cr_namelen] = '\0'; - - if (verbose > 1) - printf("Rec %lld: %d %s\n", info->recno, info->type,info->name); + struct changelog_rec *rec; + struct changelog_ext_rename *rnm; + size_t namelen; + size_t copylen = sizeof(info->name); + + if (llapi_changelog_recv(priv, &rec) != 0) + return -1; + + info->is_extended = !!(rec->cr_flags & CLF_RENAME); + info->recno = rec->cr_index; + info->type = rec->cr_type; + snprintf(info->tfid, sizeof(info->tfid), DFID, PFID(&rec->cr_tfid)); + snprintf(info->pfid, sizeof(info->pfid), DFID, PFID(&rec->cr_pfid)); + + namelen = strnlen(changelog_rec_name(rec), rec->cr_namelen); + if (copylen > namelen + 1) + copylen = namelen + 1; + strlcpy(info->name, changelog_rec_name(rec), copylen); + + /* Don't use rnm if CLF_RENAME isn't set */ + rnm = changelog_rec_rename(rec); + if (rec->cr_flags & CLF_RENAME && !fid_is_zero(&rnm->cr_sfid)) { + copylen = sizeof(info->sname); + + snprintf(info->sfid, sizeof(info->sfid), DFID, + PFID(&rnm->cr_sfid)); + snprintf(info->spfid, sizeof(info->spfid), DFID, + PFID(&rnm->cr_spfid)); + namelen = changelog_rec_snamelen(rec); + if (copylen > namelen + 1) + copylen = namelen + 1; + strlcpy(info->sname, changelog_rec_sname(rec), copylen); + + if (verbose > 1) + printf("Rec %lld: %d %s %s\n", info->recno, info->type, + info->name, info->sname); + } else { + if (verbose > 1) + printf("Rec %lld: %d %s\n", info->recno, info->type, + info->name); + } llapi_changelog_free(&rec); @@ -1069,7 +1191,7 @@ int lr_parse_line(void *priv, struct lr_info *info) /* Initialize the replication parameters */ int lr_init_status() { - size_t size = sizeof(struct lustre_rsync_status) + PATH_MAX; + size_t size = sizeof(struct lustre_rsync_status) + PATH_MAX + 1; if (status != NULL) return 0; @@ -1148,38 +1270,57 @@ int lr_read_log() struct lustre_rsync_status *s; int fd = -1; size_t size; - size_t read_size = sizeof(struct lustre_rsync_status) + PATH_MAX; + size_t read_size = sizeof(struct lustre_rsync_status) + PATH_MAX + 1; int rc = 0; if (statuslog == NULL) return 0; s = calloc(1, read_size); - if (s == NULL) - GOTO(out, rc = -ENOMEM); - - fd = open(statuslog, O_RDONLY); - if (fd == -1) - GOTO(out, rc = -errno); - size = read(fd, s, read_size); - if (size != read_size) - GOTO(out, rc = -EINVAL); - if (read_size < s->ls_size) { - read_size = s->ls_size; - s = lr_grow_buf(s, read_size); - if (s == NULL) - GOTO(out, rc = -ENOMEM); - if (lseek(fd, 0, SEEK_SET) == -1) - GOTO(out, rc = -errno); - size = read(fd, s, read_size); - if (size != read_size) - GOTO(out, rc = -EINVAL); - } + if (s == NULL) { + rc = -ENOMEM; + goto out; + } + + fd = open(statuslog, O_RDONLY); + if (fd == -1) { + rc = -errno; + goto out; + } + + size = read(fd, s, read_size); + if (size != read_size) { + rc = -EINVAL; + goto out; + } + + if (read_size < s->ls_size) { + read_size = s->ls_size; + s = lr_grow_buf(s, read_size); + if (s == NULL) { + rc = -ENOMEM; + goto out; + } + + if (lseek(fd, 0, SEEK_SET) == -1) { + rc = -ENOMEM; + goto out; + } + + size = read(fd, s, read_size); + if (size != read_size) { + rc = -EINVAL; + goto out; + } + } + + while (read(fd, &rec, sizeof(rec)) != 0) { + tmp = calloc(1, sizeof(*tmp)); + if (!tmp) { + rc = -ENOMEM; + goto out; + } - while (read(fd, &rec, sizeof(rec)) != 0) { - tmp = calloc(1, sizeof(*tmp)); - if (!tmp) - GOTO(out, rc = -ENOMEM); tmp->pc_log = rec; tmp->pc_next = parents; parents = tmp; @@ -1189,31 +1330,35 @@ int lr_read_log() if (status->ls_num_targets == 0) { if (status->ls_size != s->ls_size) { status = lr_grow_buf(status, s->ls_size); - if (status == NULL) - GOTO(out, rc = -ENOMEM); + if (status == NULL) { + rc = -ENOMEM; + goto out; + } + status->ls_size = s->ls_size; } status->ls_num_targets = s->ls_num_targets; memcpy(status->ls_targets, s->ls_targets, - PATH_MAX * s->ls_num_targets); + (PATH_MAX + 1) * s->ls_num_targets); } if (status->ls_last_recno == -1) status->ls_last_recno = s->ls_last_recno; - if (status->ls_registration[0] == '\0') - strncpy(status->ls_registration, s->ls_registration, - LR_NAME_MAXLEN); + if (status->ls_registration[0] == '\0') + strlcpy(status->ls_registration, s->ls_registration, + sizeof(status->ls_registration)); - if (status->ls_mdt_device[0] == '\0') - strncpy(status->ls_mdt_device, s->ls_mdt_device, - LR_NAME_MAXLEN); + if (status->ls_mdt_device[0] == '\0') + strlcpy(status->ls_mdt_device, s->ls_mdt_device, + sizeof(status->ls_mdt_device)); - if (status->ls_source_fs[0] == '\0') - strncpy(status->ls_source_fs, s->ls_source_fs, - LR_NAME_MAXLEN); + if (status->ls_source_fs[0] == '\0') + strlcpy(status->ls_source_fs, s->ls_source_fs, + sizeof(status->ls_source_fs)); - if (status->ls_source[0] == '\0') - strncpy(status->ls_source, s->ls_source, PATH_MAX); + if (status->ls_source[0] == '\0') + strlcpy(status->ls_source, s->ls_source, + sizeof(status->ls_source)); out: if (fd != -1) @@ -1227,9 +1372,9 @@ int lr_read_log() processing. */ int lr_clear_cl(struct lr_info *info, int force) { - char mdt_device[LR_NAME_MAXLEN + 1]; - long long rec; - int rc = 0; + char mdt_device[LR_NAME_MAXLEN + 1]; + long long rec; + int rc = 0; if (force || info->recno > status->ls_last_recno + CLEAR_INTERVAL) { if (info->type == CL_RENAME) @@ -1241,8 +1386,8 @@ int lr_clear_cl(struct lr_info *info, int force) * device name so make a copy of it until this * is fixed. */ - strncpy(mdt_device, status->ls_mdt_device, - LR_NAME_MAXLEN); + strlcpy(mdt_device, status->ls_mdt_device, + sizeof(mdt_device)); rc = llapi_changelog_clear(mdt_device, status->ls_registration, rec); @@ -1319,7 +1464,8 @@ void lr_print_status(struct lr_info *info) if (statuslog != NULL) printf("Statuslog: %s\n", statuslog); printf("Changelog registration: %s\n", status->ls_registration); - printf("Starting changelog record: %lld\n", status->ls_last_recno); + printf("Starting changelog record: %jd\n", + (uintmax_t)status->ls_last_recno); if (noxattr) printf("Replicate xattrs: no\n"); if (noclear) @@ -1341,7 +1487,7 @@ int lr_replicate() { void *changelog_priv; struct lr_info *info; - struct lr_info *ext; + struct lr_info *ext = NULL; time_t start; int xattr_not_supp; int i; @@ -1357,16 +1503,17 @@ int lr_replicate() if (rc) { fprintf(stderr, "Source path is not a valid Lustre client " "mountpoint.\n"); - return rc; + goto out; } if (status->ls_mdt_device[0] == '\0') snprintf(status->ls_mdt_device, LR_NAME_MAXLEN, "%s%s", status->ls_source_fs, DEFAULT_MDT); ext = calloc(1, sizeof(struct lr_info)); - if (ext == NULL) - return -ENOMEM; - memcpy(ext, info, sizeof(struct lr_info)); + if (ext == NULL) { + rc = -ENOMEM; + goto out; + } for (i = 0, xattr_not_supp = 0; i < status->ls_num_targets; i++) { snprintf(info->dest, PATH_MAX, "%s/%s", status->ls_targets[i], @@ -1375,7 +1522,8 @@ int lr_replicate() if (rc == -1 && errno != EEXIST) { fprintf(stderr, "Error writing to target path %s.\n", status->ls_targets[i]); - return -errno; + rc = -errno; + goto out; } rc = llistxattr(info->src, info->xlist, info->xsize); if (rc == -1 && errno == ENOTSUP) { @@ -1390,25 +1538,39 @@ int lr_replicate() lr_print_status(info); - /* Open changelogs for consumption*/ - rc = llapi_changelog_start(&changelog_priv, CHANGELOG_FLAG_BLOCK, - status->ls_source_fs, status->ls_last_recno); + /* Open changelogs for consumption*/ + rc = llapi_changelog_start(&changelog_priv, + CHANGELOG_FLAG_BLOCK | CHANGELOG_FLAG_JOBID, + status->ls_mdt_device, status->ls_last_recno); if (rc < 0) { fprintf(stderr, "Error opening changelog file for fs %s.\n", status->ls_source_fs); - return rc; + goto out; } while (!quit && lr_parse_line(changelog_priv, info) == 0) { rc = 0; - if (info->type == CL_RENAME) - /* Rename operations have an additional changelog - record of information. */ - lr_parse_line(changelog_priv, ext); + if (info->type == CL_RENAME && !info->is_extended) { + /* Newer rename operations extends changelog to store + * source file information, but old changelog has + * another record. + */ + if (lr_parse_line(changelog_priv, ext) != 0) + break; + memcpy(info->sfid, info->tfid, sizeof(info->sfid)); + memcpy(info->spfid, info->pfid, sizeof(info->spfid)); + memcpy(info->tfid, ext->tfid, sizeof(info->tfid)); + memcpy(info->pfid, ext->pfid, sizeof(info->pfid)); + strlcpy(info->sname, info->name, sizeof(info->sname)); + strlcpy(info->name, ext->name, sizeof(info->name)); + info->is_extended = 1; + } if (dryrun) continue; + DEBUG_ENTRY(info); + switch(info->type) { case CL_CREATE: case CL_MKDIR: @@ -1421,7 +1583,7 @@ int lr_replicate() rc = lr_remove(info); break; case CL_RENAME: - rc = lr_move(info, ext); + rc = lr_move(info); break; case CL_HARDLINK: rc = lr_link(info); @@ -1433,15 +1595,17 @@ int lr_replicate() case CL_XATTR: rc = lr_setxattr(info); break; - case CL_CLOSE: - case CL_EXT: - case CL_OPEN: - case CL_IOCTL: - case CL_MARK: - /* Nothing needs to be done for these entries */ - default: - break; - } + case CL_CLOSE: + case CL_EXT: + case CL_OPEN: + case CL_LAYOUT: + case CL_MARK: + /* Nothing needs to be done for these entries */ + /* fallthrough */ + default: + break; + } + DEBUG_EXIT(info, rc); if (rc && rc != -ENOENT) { lr_print_failure(info, rc); errors++; @@ -1468,7 +1632,15 @@ int lr_replicate() printf("Changelog records consumed: %lld\n", rec_count); } - return 0; + rc = 0; + +out: + if (info != NULL) + free(info); + if (ext != NULL) + free(ext); + + return rc; } void @@ -1488,8 +1660,8 @@ int main(int argc, char *argv[]) if ((rc = lr_init_status()) != 0) return rc; - while ((rc = getopt_long(argc, argv, "as:t:m:u:l:vx:zc:ry:n:d:", - long_opts, NULL)) >= 0) { + while ((rc = getopt_long(argc, argv, "as:t:m:u:l:vx:zc:ry:n:d:D:", + long_opts, NULL)) >= 0) { switch (rc) { case 'a': /* Assume absolute paths */ @@ -1497,7 +1669,8 @@ int main(int argc, char *argv[]) break; case 's': /* Assume absolute paths */ - strncpy(status->ls_source, optarg, PATH_MAX); + strlcpy(status->ls_source, optarg, + sizeof(status->ls_source)); break; case 't': status->ls_num_targets++; @@ -1511,23 +1684,23 @@ int main(int argc, char *argv[]) status->ls_num_targets = numtargets; } newsize = sizeof (struct lustre_rsync_status) + - (status->ls_num_targets * PATH_MAX); + (status->ls_num_targets * (PATH_MAX + 1)); if (status->ls_size != newsize) { status->ls_size = newsize; status = lr_grow_buf(status, newsize); if (status == NULL) return -ENOMEM; } - strncpy(status->ls_targets[status->ls_num_targets - 1], - optarg, - PATH_MAX); - break; - case 'm': - strncpy(status->ls_mdt_device, optarg, LR_NAME_MAXLEN); - break; - case 'u': - strncpy(status->ls_registration, optarg, - LR_NAME_MAXLEN); + strlcpy(status->ls_targets[status->ls_num_targets - 1], + optarg, sizeof(status->ls_targets[0])); + break; + case 'm': + strlcpy(status->ls_mdt_device, optarg, + sizeof(status->ls_mdt_device)); + break; + case 'u': + strlcpy(status->ls_registration, optarg, + sizeof(status->ls_registration)); break; case 'l': statuslog = optarg; @@ -1579,6 +1752,17 @@ int main(int argc, char *argv[]) if (debug < 0 || debug > 2) debug = 0; break; + case 'D': + /* Undocumented option debug log file */ + if (debug_log != NULL) + fclose(debug_log); + debug_log = fopen(optarg, "a"); + if (debug_log == NULL) { + printf("Cannot open %s for debug log\n", + optarg); + return -1; + } + break; default: fprintf(stderr, "error: %s: option '%s' " "unrecognized.\n", argv[0], argv[optind - 1]); @@ -1625,5 +1809,7 @@ int main(int argc, char *argv[]) rc = lr_replicate(); - return rc; + if (debug_log != NULL) + fclose(debug_log); + return rc; }