X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Futils%2Fliblustreapi_hsm.c;h=c8a68ef851bd6047b167b05bd0eea6ee916c9de8;hb=875252d59924ad09db8de9f0fbb611788a0b9c78;hp=78ab4d87f61bd1d87c4c0db64f63a23653e871a0;hpb=b80d263bed6d4756ecc471129822612104f01487;p=fs%2Flustre-release.git diff --git a/lustre/utils/liblustreapi_hsm.c b/lustre/utils/liblustreapi_hsm.c index 78ab4d8..c8a68ef 100644 --- a/lustre/utils/liblustreapi_hsm.c +++ b/lustre/utils/liblustreapi_hsm.c @@ -6,7 +6,7 @@ * (C) Copyright 2012 Commissariat a l'energie atomique et aux energies * alternatives * - * Copyright (c) 2013, 2014, Intel Corporation. + * Copyright (c) 2013, 2017, Intel Corporation. * * All rights reserved. This program and the accompanying materials * are made available under the terms of the GNU Lesser General Public License @@ -32,6 +32,7 @@ * Author: Henri Doreau */ +#include #include #include #include @@ -45,10 +46,10 @@ #include #include #include +#include #include #include #include -#include #include #ifdef HAVE_LINUX_UNISTD_H #include @@ -56,8 +57,7 @@ #include #endif -#include -#include +#include #include #include "lustreapi_internal.h" @@ -66,18 +66,18 @@ /****** HSM Copytool API ********/ #define CT_PRIV_MAGIC 0xC0BE2001 struct hsm_copytool_private { - int magic; - char *mnt; - struct kuc_hdr *kuch; - int mnt_fd; - int open_by_fid_fd; - struct lustre_kernelcomm kuc; - __u32 archives; + int magic; + char *mnt; + struct kuc_hdr *kuch; + int mnt_fd; + int open_by_fid_fd; + struct lustre_kernelcomm *kuc; }; #define CP_PRIV_MAGIC 0x19880429 struct hsm_copyaction_private { __u32 magic; + __u32 source_fd; __s32 data_fd; const struct hsm_copytool_private *ct_priv; struct hsm_copy copy; @@ -281,7 +281,7 @@ static int llapi_hsm_log_ct_registration(struct hsm_copytool_private **priv, goto err; rc = llapi_json_add_item(&json_items, "archive", LLAPI_JSON_INTEGER, - &ct->archives); + &ct->kuc->lk_data_count); if (rc < 0) goto err; @@ -428,11 +428,12 @@ static int llapi_hsm_log_ct_progress(struct hsm_copyaction_private **phcp, goto err; } - if (progress_type == CT_RUNNING) + if (progress_type == CT_RUNNING) { rc = llapi_json_add_item(&json_items, "current_bytes", LLAPI_JSON_BIGNUM, ¤t); if (rc < 0) goto err; + } cancel: rc = llapi_json_add_item(&json_items, "event_type", LLAPI_JSON_STRING, @@ -473,6 +474,8 @@ int llapi_hsm_register_event_fifo(const char *path) { int read_fd; struct stat statbuf; + struct sigaction ignore_action; + int rc; /* Create the FIFO if necessary. */ if ((mkfifo(path, 0644) < 0) && (errno != EEXIST)) { @@ -507,19 +510,24 @@ int llapi_hsm_register_event_fifo(const char *path) /* Open the FIFO for writes, but don't block on waiting * for a reader. */ llapi_hsm_event_fd = open(path, O_WRONLY | O_NONBLOCK); - if (llapi_hsm_event_fd < 0) { - llapi_error(LLAPI_MSG_ERROR, errno, - "cannot open(%s) for write", path); - return -errno; - } + rc = -errno; /* Now close the reader. An external monitoring process can * now open the FIFO for reads. If no reader comes along the * events are lost. NOTE: Only one reader at a time! */ close(read_fd); + if (llapi_hsm_event_fd < 0) { + llapi_error(LLAPI_MSG_ERROR, -rc, + "cannot open(%s) for write", path); + return rc; + } + /* Ignore SIGPIPEs -- can occur if the reader goes away. */ - signal(SIGPIPE, SIG_IGN); + memset(&ignore_action, 0, sizeof(ignore_action)); + ignore_action.sa_handler = SIG_IGN; + sigemptyset(&ignore_action.sa_mask); + sigaction(SIGPIPE, &ignore_action, NULL); return 0; } @@ -678,11 +686,14 @@ int llapi_hsm_copytool_register(struct hsm_copytool_private **priv, return -EINVAL; } - if (archive_count > LL_HSM_MAX_ARCHIVE) { - llapi_err_noerrno(LLAPI_MSG_ERROR, "%d requested when maximum " - "of %zu archives supported", archive_count, - LL_HSM_MAX_ARCHIVE); - return -EINVAL; + for (rc = 0; rc < archive_count; rc++) { + /* in the list we have an all archive wildcard + * so move to all archives mode + */ + if (archives[rc] == 0) { + archive_count = 0; + break; + } } ct = calloc(1, sizeof(*ct)); @@ -692,8 +703,6 @@ int llapi_hsm_copytool_register(struct hsm_copytool_private **priv, ct->magic = CT_PRIV_MAGIC; ct->mnt_fd = -1; ct->open_by_fid_fd = -1; - ct->kuc.lk_rfd = LK_NOFD; - ct->kuc.lk_wfd = LK_NOFD; ct->mnt = strdup(mnt); if (ct->mnt == NULL) { @@ -701,7 +710,7 @@ int llapi_hsm_copytool_register(struct hsm_copytool_private **priv, goto out_err; } - ct->kuch = malloc(HAL_MAXSIZE + sizeof(*ct->kuch)); + ct->kuch = calloc(1, HAL_MAXSIZE + sizeof(*ct->kuch)); if (ct->kuch == NULL) { rc = -ENOMEM; goto out_err; @@ -719,34 +728,34 @@ int llapi_hsm_copytool_register(struct hsm_copytool_private **priv, goto out_err; } - /* no archives specified means "match all". */ - ct->archives = 0; + ct->kuc = malloc(sizeof(*ct) + archive_count * sizeof(__u32)); + if (ct->kuc == NULL) { + rc = -ENOMEM; + goto out_err; + } + + ct->kuc->lk_rfd = LK_NOFD; + ct->kuc->lk_wfd = LK_NOFD; + + rc = libcfs_ukuc_start(ct->kuc, KUC_GRP_HSM, rfd_flags); + if (rc < 0) + goto out_free_kuc; + + ct->kuc->lk_flags = LK_FLG_DATANR; + ct->kuc->lk_data_count = archive_count; for (rc = 0; rc < archive_count; rc++) { - if ((archives[rc] > LL_HSM_MAX_ARCHIVE) || (archives[rc] < 0)) { + if (archives[rc] < 0) { llapi_err_noerrno(LLAPI_MSG_ERROR, "%d requested when " - "archive id [0 - %zu] is supported", - archives[rc], LL_HSM_MAX_ARCHIVE); + "archive id >= 0 is supported", + archives[rc]); rc = -EINVAL; - goto out_err; - } - /* in the list we have an all archive wildcard - * so move to all archives mode - */ - if (archives[rc] == 0) { - ct->archives = 0; - archive_count = 0; - break; + goto out_kuc; } - ct->archives |= (1 << (archives[rc] - 1)); - } - rc = libcfs_ukuc_start(&ct->kuc, KUC_GRP_HSM, rfd_flags); - if (rc < 0) - goto out_err; + ct->kuc->lk_data[rc] = archives[rc]; + } - /* Storing archive(s) in lk_data; see mdc_ioc_hsm_ct_start */ - ct->kuc.lk_data = ct->archives; - rc = ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, &ct->kuc); + rc = ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, ct->kuc); if (rc < 0) { rc = -errno; llapi_error(LLAPI_MSG_ERROR, rc, @@ -757,15 +766,18 @@ int llapi_hsm_copytool_register(struct hsm_copytool_private **priv, llapi_hsm_log_ct_registration(&ct, CT_REGISTER); /* Only the kernel reference keeps the write side open */ - close(ct->kuc.lk_wfd); - ct->kuc.lk_wfd = LK_NOFD; + close(ct->kuc->lk_wfd); + ct->kuc->lk_wfd = LK_NOFD; *priv = ct; return 0; out_kuc: /* cleanup the kuc channel */ - libcfs_ukuc_stop(&ct->kuc); + libcfs_ukuc_stop(ct->kuc); + +out_free_kuc: + free(ct->kuc); out_err: if (!(ct->mnt_fd < 0)) @@ -799,12 +811,16 @@ int llapi_hsm_copytool_unregister(struct hsm_copytool_private **priv) if (ct->magic != CT_PRIV_MAGIC) return -EINVAL; - /* Tell the kernel to stop sending us messages */ - ct->kuc.lk_flags = LK_FLG_STOP; - ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, &ct->kuc); + /* Close the read side of the KUC pipe. This should be done + * before unregistering to avoid deadlock: a ldlm_cb thread + * enters libcfs_kkuc_group_put() acquires kg_sem and blocks + * in pipe_write() due to full pipe; then we attempt to + * unregister and block on kg_sem. */ + libcfs_ukuc_stop(ct->kuc); - /* Shut down the kernelcomms */ - libcfs_ukuc_stop(&ct->kuc); + /* Tell the kernel to stop sending us messages */ + ct->kuc->lk_flags = LK_FLG_STOP; + ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, ct->kuc); llapi_hsm_log_ct_registration(&ct, CT_UNREGISTER); @@ -812,6 +828,7 @@ int llapi_hsm_copytool_unregister(struct hsm_copytool_private **priv) close(ct->mnt_fd); free(ct->mnt); free(ct->kuch); + free(ct->kuc); free(ct); *priv = NULL; @@ -828,7 +845,7 @@ int llapi_hsm_copytool_get_fd(struct hsm_copytool_private *ct) if (ct == NULL || ct->magic != CT_PRIV_MAGIC) return -EINVAL; - return libcfs_ukuc_get_rfd(&ct->kuc); + return libcfs_ukuc_get_rfd(ct->kuc); } /** Wait for the next hsm_action_list @@ -856,7 +873,7 @@ int llapi_hsm_copytool_recv(struct hsm_copytool_private *ct, kuch = ct->kuch; repeat: - rc = libcfs_ukuc_msg_get(&ct->kuc, (char *)kuch, + rc = libcfs_ukuc_msg_get(ct->kuc, (char *)kuch, HAL_MAXSIZE + sizeof(*kuch), KUC_TRANSPORT_HSM); if (rc < 0) @@ -892,15 +909,16 @@ repeat: /* Check that we have registered for this archive # * if 0 registered, we serve any archive */ - if (ct->archives && - ((1 << (hal->hal_archive_id - 1)) & ct->archives) == 0) { - llapi_err_noerrno(LLAPI_MSG_INFO, - "This copytool does not service archive #%d," - " ignoring this request." - " Mask of served archive is 0x%.8X", - hal->hal_archive_id, ct->archives); - - goto repeat; + if (ct->kuc != NULL && ct->kuc->lk_data_count != 0) { + int i; + + for (i = 0; i < ct->kuc->lk_data_count; i++) { + if (hal->hal_archive_id == ct->kuc->lk_data[i]) + break; + } + + if (i >= ct->kuc->lk_data_count) + goto repeat; } *halh = hal; @@ -921,7 +939,7 @@ out_err: * \param parent_len Destination buffer size. * \return 0 on success. */ -static int fid_parent(const char *mnt, const lustre_fid *fid, char *parent, +static int fid_parent(const char *mnt, const struct lu_fid *fid, char *parent, size_t parent_len) { int rc; @@ -981,32 +999,30 @@ static int ct_md_getattr(const struct hsm_copytool_private *ct, lstat_t *st) { struct lov_user_mds_data *lmd; + char fname[FID_NOBRACE_LEN + 1] = ""; size_t lmd_size; int rc; + rc = snprintf(fname, sizeof(fname), DFID_NOBRACE, PFID(fid)); + if (rc < 0) + return rc; + if (rc >= sizeof(fname) || rc == 0) + return -EINVAL; + lmd_size = sizeof(lmd->lmd_st) + lov_user_md_size(LOV_MAX_STRIPE_COUNT, LOV_USER_MAGIC_V3); if (lmd_size < sizeof(lmd->lmd_st) + XATTR_SIZE_MAX) lmd_size = sizeof(lmd->lmd_st) + XATTR_SIZE_MAX; - if (lmd_size < FID_NOBRACE_LEN + 1) - lmd_size = FID_NOBRACE_LEN + 1; - lmd = malloc(lmd_size); if (lmd == NULL) return -ENOMEM; - snprintf((char *)lmd, lmd_size, DFID_NOBRACE, PFID(fid)); - - rc = ioctl(ct->open_by_fid_fd, IOC_MDC_GETFILEINFO, lmd); - if (rc != 0) { - rc = -errno; - llapi_error(LLAPI_MSG_ERROR, rc, - "cannot get metadata attributes of "DFID" in '%s'", - PFID(fid), ct->mnt); + rc = get_lmd_info_fd(fname, ct->open_by_fid_fd, -1, + lmd, lmd_size, GET_LMD_INFO); + if (rc) goto out; - } *st = lmd->lmd_st; out: @@ -1087,13 +1103,15 @@ int llapi_hsm_action_begin(struct hsm_copyaction_private **phcp, int restore_mdt_index, int restore_open_flags, bool is_error) { - struct hsm_copyaction_private *hcp; - int rc; + struct hsm_copyaction_private *hcp; + int fd; + int rc; hcp = calloc(1, sizeof(*hcp)); if (hcp == NULL) return -ENOMEM; + hcp->source_fd = -1; hcp->data_fd = -1; hcp->ct_priv = ct; hcp->copy.hc_hai = *hai; @@ -1102,7 +1120,16 @@ int llapi_hsm_action_begin(struct hsm_copyaction_private **phcp, if (is_error) goto ok_out; - if (hai->hai_action == HSMA_RESTORE) { + if (hai->hai_action == HSMA_ARCHIVE) { + fd = ct_open_by_fid(hcp->ct_priv, &hai->hai_dfid, + O_RDONLY | O_NOATIME | O_NOFOLLOW | O_NONBLOCK); + if (fd < 0) { + rc = fd; + goto err_out; + } + + hcp->source_fd = fd; + } else if (hai->hai_action == HSMA_RESTORE) { rc = ct_md_getattr(hcp->ct_priv, &hai->hai_fid, &hcp->stat); if (rc < 0) goto err_out; @@ -1111,6 +1138,40 @@ int llapi_hsm_action_begin(struct hsm_copyaction_private **phcp, restore_open_flags); if (rc < 0) goto err_out; + } else if (hai->hai_action == HSMA_REMOVE) { + /* Since remove is atomic there is no need to send an + * initial MDS_HSM_PROGRESS RPC. + * RW-PCC uses Lustre HSM mechanism for data synchronization. + * At the beginning of RW-PCC attach, the client tries to + * exclusively open the file by using a lease lock. A + * successful lease open ensures that the current attach + * process is the unique opener for the file. + * After taking the lease, the file data is then copied from + * OSTs into PCC and then the client closes the lease with + * with a PCC attach intent. + * However, for a file with HSM exists, archived state (i.e. a + * cached file just was detached from PCC and restore into + * OST), a HSM REMOVE request may delete the above PCC copy + * during RW-PCC attach wrongly. + * Thus, a open/close on the corresponding Lustre file is added + * for HSMA_REMOVE here to solve this conflict. + */ + fd = ct_open_by_fid(hcp->ct_priv, &hai->hai_fid, + O_RDONLY | O_NOATIME | O_NOFOLLOW | O_NONBLOCK); + if (fd < 0) { + rc = fd; + /* ignore the error in case of Remove Archive on Last + * Unlink (RAoLU). + */ + if (rc == -ENOENT) { + rc = 0; + goto out_log; + } + goto err_out; + } + + hcp->source_fd = fd; + goto out_log; } rc = ioctl(ct->mnt_fd, LL_IOC_HSM_COPY_START, &hcp->copy); @@ -1119,6 +1180,7 @@ int llapi_hsm_action_begin(struct hsm_copyaction_private **phcp, goto err_out; } +out_log: llapi_hsm_log_ct_progress(&hcp, hai, CT_START, 0, 0); ok_out: @@ -1127,6 +1189,9 @@ ok_out: return 0; err_out: + if (!(hcp->source_fd < 0)) + close(hcp->source_fd); + if (!(hcp->data_fd < 0)) close(hcp->data_fd); @@ -1163,19 +1228,23 @@ int llapi_hsm_action_end(struct hsm_copyaction_private **phcp, hai = &hcp->copy.hc_hai; if (hai->hai_action == HSMA_RESTORE && errval == 0) { - struct timeval tv[2]; - - /* Set {a,m}time of volatile file to that of original. */ - tv[0].tv_sec = hcp->stat.st_atime; - tv[0].tv_usec = 0; - tv[1].tv_sec = hcp->stat.st_mtime; - tv[1].tv_usec = 0; - if (futimes(hcp->data_fd, tv) < 0) { + struct ll_futimes_3 lfu = { + .lfu_atime_sec = hcp->stat.st_atim.tv_sec, + .lfu_atime_nsec = hcp->stat.st_atim.tv_nsec, + .lfu_mtime_sec = hcp->stat.st_mtim.tv_sec, + .lfu_mtime_nsec = hcp->stat.st_mtim.tv_nsec, + .lfu_ctime_sec = hcp->stat.st_ctim.tv_sec, + .lfu_ctime_nsec = hcp->stat.st_ctim.tv_nsec, + }; + + rc = fsync(hcp->data_fd); + if (rc < 0) { errval = -errno; goto end; } - rc = fsync(hcp->data_fd); + /* Set {a,m,c}time of volatile file to that of original. */ + rc = ioctl(hcp->data_fd, LL_IOC_FUTIMES_3, &lfu); if (rc < 0) { errval = -errno; goto end; @@ -1204,6 +1273,9 @@ end: llapi_hsm_log_ct_progress(&hcp, hai, CT_FINISH, 0, 0); err_cleanup: + if (!(hcp->source_fd < 0)) + close(hcp->source_fd); + if (!(hcp->data_fd < 0)) close(hcp->data_fd); @@ -1258,7 +1330,7 @@ int llapi_hsm_action_progress(struct hsm_copyaction_private *hcp, * @return error code if the action is not a copy operation. */ int llapi_hsm_action_get_dfid(const struct hsm_copyaction_private *hcp, - lustre_fid *fid) + struct lu_fid *fid) { const struct hsm_action_item *hai = &hcp->copy.hc_hai; @@ -1289,8 +1361,8 @@ int llapi_hsm_action_get_fd(const struct hsm_copyaction_private *hcp) return -EINVAL; if (hai->hai_action == HSMA_ARCHIVE) { - return ct_open_by_fid(hcp->ct_priv, &hai->hai_dfid, - O_RDONLY | O_NOATIME | O_NOFOLLOW | O_NONBLOCK); + fd = dup(hcp->source_fd); + return fd < 0 ? -errno : fd; } else if (hai->hai_action == HSMA_RESTORE) { fd = dup(hcp->data_fd); return fd < 0 ? -errno : fd; @@ -1315,7 +1387,7 @@ int llapi_hsm_action_get_fd(const struct hsm_copyaction_private *hcp) int llapi_hsm_import(const char *dst, int archive, const struct stat *st, unsigned long long stripe_size, int stripe_offset, int stripe_count, int stripe_pattern, char *pool_name, - lustre_fid *newfid) + struct lu_fid *newfid) { struct hsm_user_import hui; int fd;