4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * (C) Copyright 2012 Commissariat a l'energie atomique et aux energies
9 * Copyright (c) 2013, 2015, Intel Corporation.
11 * All rights reserved. This program and the accompanying materials
12 * are made available under the terms of the GNU Lesser General Public License
13 * (LGPL) version 2.1 or (at your discretion) any later version.
14 * (LGPL) version 2.1 accompanies this distribution, and is available at
15 * http://www.gnu.org/licenses/lgpl-2.1.html
17 * This library is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Lesser General Public License for more details.
25 * lustre/utils/liblustreapi_hsm.c
27 * lustreapi library for hsm calls
29 * Author: Aurelien Degremont <aurelien.degremont@cea.fr>
30 * Author: JC Lafoucriere <jacques-charles.lafoucriere@cea.fr>
31 * Author: Thomas Leibovici <thomas.leibovici@cea.fr>
32 * Author: Henri Doreau <henri.doreau@cea.fr>
40 #include <sys/ioctl.h>
48 #include <sys/types.h>
51 #include <sys/syscall.h>
54 #ifdef HAVE_LINUX_UNISTD_H
55 #include <linux/unistd.h>
60 #include <lnet/lnetctl.h>
61 #include <lustre/lustre_idl.h>
62 #include <lustre/lustreapi.h>
63 #include "lustreapi_internal.h"
65 #define OPEN_BY_FID_PATH dot_lustre_name"/fid"
67 /****** HSM Copytool API ********/
68 #define CT_PRIV_MAGIC 0xC0BE2001
69 struct hsm_copytool_private {
75 struct lustre_kernelcomm kuc;
79 #define CP_PRIV_MAGIC 0x19880429
80 struct hsm_copyaction_private {
83 const struct hsm_copytool_private *ct_priv;
88 enum ct_progress_type {
99 CT_ARCHIVE_START = HSMA_ARCHIVE,
100 CT_ARCHIVE_RUNNING = HSMA_ARCHIVE + CT_RUNNING,
101 CT_ARCHIVE_FINISH = HSMA_ARCHIVE + CT_FINISH,
102 CT_ARCHIVE_CANCEL = HSMA_ARCHIVE + CT_CANCEL,
103 CT_ARCHIVE_ERROR = HSMA_ARCHIVE + CT_ERROR,
104 CT_RESTORE_START = HSMA_RESTORE,
105 CT_RESTORE_RUNNING = HSMA_RESTORE + CT_RUNNING,
106 CT_RESTORE_FINISH = HSMA_RESTORE + CT_FINISH,
107 CT_RESTORE_CANCEL = HSMA_RESTORE + CT_CANCEL,
108 CT_RESTORE_ERROR = HSMA_RESTORE + CT_ERROR,
109 CT_REMOVE_START = HSMA_REMOVE,
110 CT_REMOVE_RUNNING = HSMA_REMOVE + CT_RUNNING,
111 CT_REMOVE_FINISH = HSMA_REMOVE + CT_FINISH,
112 CT_REMOVE_CANCEL = HSMA_REMOVE + CT_CANCEL,
113 CT_REMOVE_ERROR = HSMA_REMOVE + CT_ERROR,
117 /* initialized in llapi_hsm_register_event_fifo() */
118 static int llapi_hsm_event_fd = -1;
119 static bool created_hsm_event_fifo;
121 static inline const char *llapi_hsm_ct_ev2str(int type)
128 case CT_ARCHIVE_START:
129 return "ARCHIVE_START";
130 case CT_ARCHIVE_RUNNING:
131 return "ARCHIVE_RUNNING";
132 case CT_ARCHIVE_FINISH:
133 return "ARCHIVE_FINISH";
134 case CT_ARCHIVE_CANCEL:
135 return "ARCHIVE_CANCEL";
136 case CT_ARCHIVE_ERROR:
137 return "ARCHIVE_ERROR";
138 case CT_RESTORE_START:
139 return "RESTORE_START";
140 case CT_RESTORE_RUNNING:
141 return "RESTORE_RUNNING";
142 case CT_RESTORE_FINISH:
143 return "RESTORE_FINISH";
144 case CT_RESTORE_CANCEL:
145 return "RESTORE_CANCEL";
146 case CT_RESTORE_ERROR:
147 return "RESTORE_ERROR";
148 case CT_REMOVE_START:
149 return "REMOVE_START";
150 case CT_REMOVE_RUNNING:
151 return "REMOVE_RUNNING";
152 case CT_REMOVE_FINISH:
153 return "REMOVE_FINISH";
154 case CT_REMOVE_CANCEL:
155 return "REMOVE_CANCEL";
156 case CT_REMOVE_ERROR:
157 return "REMOVE_ERROR";
159 llapi_err_noerrno(LLAPI_MSG_ERROR,
160 "Unknown event type: %d", type);
166 * Writes a JSON event to the monitor FIFO. Noop if no FIFO has been
169 * \param event A list of llapi_json_items comprising a
170 * single JSON-formatted event.
172 * \retval 0 on success.
173 * \retval -errno on error.
175 static int llapi_hsm_write_json_event(struct llapi_json_item_list **event)
178 char time_string[40];
179 char json_buf[PIPE_BUF];
181 time_t event_time = time(0);
182 struct tm time_components;
183 struct llapi_json_item_list *json_items;
185 /* Noop unless the event fd was initialized */
186 if (llapi_hsm_event_fd < 0)
189 if (event == NULL || *event == NULL)
194 localtime_r(&event_time, &time_components);
196 if (strftime(time_string, sizeof(time_string), "%Y-%m-%d %T %z",
197 &time_components) == 0) {
199 llapi_error(LLAPI_MSG_ERROR, rc, "strftime() failed");
203 rc = llapi_json_add_item(&json_items, "event_time", LLAPI_JSON_STRING,
206 llapi_error(LLAPI_MSG_ERROR, -rc, "error in "
207 "llapi_json_add_item()");
211 buf_file = fmemopen(json_buf, sizeof(json_buf), "w");
212 if (buf_file == NULL)
215 rc = llapi_json_write_list(event, buf_file);
223 if (write(llapi_hsm_event_fd, json_buf, strlen(json_buf)) < 0) {
224 /* Ignore write failures due to missing reader. */
233 * Hook for llapi_hsm_copytool_register and llapi_hsm_copytool_unregister
234 * to generate JSON events suitable for consumption by a copytool
235 * monitoring process.
237 * \param priv Opaque private control structure.
238 * \param event_type The type of event (register or unregister).
240 * \retval 0 on success.
241 * \retval -errno on error.
243 static int llapi_hsm_log_ct_registration(struct hsm_copytool_private **priv,
247 char agent_uuid[UUID_MAX];
248 struct hsm_copytool_private *ct;
249 struct llapi_json_item_list *json_items;
251 /* Noop unless the event fd was initialized */
252 if (llapi_hsm_event_fd < 0)
255 if (priv == NULL || *priv == NULL)
259 if (ct->magic != CT_PRIV_MAGIC)
262 if (event_type != CT_REGISTER && event_type != CT_UNREGISTER)
265 rc = llapi_json_init_list(&json_items);
269 rc = llapi_get_agent_uuid(ct->mnt, agent_uuid, sizeof(agent_uuid));
272 llapi_chomp_string(agent_uuid);
274 rc = llapi_json_add_item(&json_items, "uuid", LLAPI_JSON_STRING,
279 rc = llapi_json_add_item(&json_items, "mount_point", LLAPI_JSON_STRING,
284 rc = llapi_json_add_item(&json_items, "archive", LLAPI_JSON_INTEGER,
289 rc = llapi_json_add_item(&json_items, "event_type", LLAPI_JSON_STRING,
290 (char *)llapi_hsm_ct_ev2str(event_type));
294 rc = llapi_hsm_write_json_event(&json_items);
301 llapi_error(LLAPI_MSG_ERROR, rc, "error in "
302 "llapi_hsm_log_ct_registration()");
305 if (json_items != NULL)
306 llapi_json_destroy_list(&json_items);
312 * Given a copytool progress update, construct a JSON event suitable for
313 * consumption by a copytool monitoring process.
315 * Examples of various events generated here and written by
316 * llapi_hsm_write_json_event:
318 * Copytool registration and deregistration:
319 * {"event_time": "2014-02-26 14:58:01 -0500", "event_type": "REGISTER",
320 * "archive": 0, "mount_point": "/mnt/lustre",
321 * "uuid": "80379a60-1f8a-743f-daf2-307cde793ec2"}
322 * {"event_time": "2014-02-26 14:58:01 -0500", "event_type": "UNREGISTER",
323 * "archive": 0, "mount_point": "/mnt/lustre",
324 * "uuid": "80379a60-1f8a-743f-daf2-307cde793ec2"}
326 * An archive action, start to completion:
327 * {"event_time": "2014-02-26 14:50:13 -0500", "event_type": "ARCHIVE_START",
328 * "total_bytes": 0, "lustre_path": "d71.sanity-hsm/f71.sanity-hsm",
329 * "source_fid": "0x2000013a1:0x2:0x0", "data_fid": "0x2000013a1:0x2:0x0"}
330 * {"event_time": "2014-02-26 14:50:18 -0500", "event_type": "ARCHIVE_RUNNING",
331 * "current_bytes": 5242880, "total_bytes": 39000000,
332 * "lustre_path": "d71.sanity-hsm/f71.sanity-hsm",
333 * "source_fid": "0x2000013a1:0x2:0x0", "data_fid": "0x2000013a1:0x2:0x0"}
334 * {"event_time": "2014-02-26 14:50:50 -0500", "event_type": "ARCHIVE_FINISH",
335 * "source_fid": "0x2000013a1:0x2:0x0", "data_fid": "0x2000013a1:0x2:0x0"}
338 * {"event_time": "2014-02-26 14:50:13 -0500", "event_type": "LOGGED_MESSAGE",
340 * "message": "lhsmtool_posix[42]: copytool fs=lustre archive#=2 item_count=1"}
342 * \param hcp Opaque action handle returned by
343 * llapi_hsm_action_start.
344 * \param hai The hsm_action_item describing the request.
345 * \param progress_type The ct_progress_type describing the update.
346 * \param total The total expected bytes for the request.
347 * \param current The current copied byte count for the request.
349 * \retval 0 on success.
350 * \retval -errno on error.
352 static int llapi_hsm_log_ct_progress(struct hsm_copyaction_private **phcp,
353 const struct hsm_action_item *hai,
355 __u64 total, __u64 current)
359 long long recno = -1;
360 char lustre_path[PATH_MAX];
361 char strfid[FID_NOBRACE_LEN + 1];
362 struct hsm_copyaction_private *hcp;
363 struct llapi_json_item_list *json_items;
365 /* Noop unless the event fd was initialized */
366 if (llapi_hsm_event_fd < 0)
369 if (phcp == NULL || *phcp == NULL)
374 rc = llapi_json_init_list(&json_items);
378 snprintf(strfid, sizeof(strfid), DFID_NOBRACE, PFID(&hai->hai_dfid));
379 rc = llapi_json_add_item(&json_items, "data_fid",
380 LLAPI_JSON_STRING, strfid);
384 snprintf(strfid, sizeof(strfid), DFID_NOBRACE, PFID(&hai->hai_fid));
385 rc = llapi_json_add_item(&json_items, "source_fid",
386 LLAPI_JSON_STRING, strfid);
390 if (hcp->copy.hc_errval == ECANCELED) {
391 progress_type = CT_CANCEL;
395 if (hcp->copy.hc_errval != 0) {
396 progress_type = CT_ERROR;
398 rc = llapi_json_add_item(&json_items, "errno",
400 &hcp->copy.hc_errval);
404 rc = llapi_json_add_item(&json_items, "error",
406 strerror(hcp->copy.hc_errval));
413 /* lustre_path isn't available after a restore completes */
414 /* total_bytes isn't available after a restore or archive completes */
415 if (progress_type != CT_FINISH) {
416 rc = llapi_fid2path(hcp->ct_priv->mnt, strfid, lustre_path,
417 sizeof(lustre_path), &recno, &linkno);
421 rc = llapi_json_add_item(&json_items, "lustre_path",
422 LLAPI_JSON_STRING, lustre_path);
426 rc = llapi_json_add_item(&json_items, "total_bytes",
427 LLAPI_JSON_BIGNUM, &total);
432 if (progress_type == CT_RUNNING)
433 rc = llapi_json_add_item(&json_items, "current_bytes",
434 LLAPI_JSON_BIGNUM, ¤t);
439 rc = llapi_json_add_item(&json_items, "event_type", LLAPI_JSON_STRING,
440 (char *)llapi_hsm_ct_ev2str(hai->hai_action +
445 rc = llapi_hsm_write_json_event(&json_items);
452 llapi_error(LLAPI_MSG_ERROR, rc, "error in "
453 "llapi_hsm_log_ct_progress()");
456 if (json_items != NULL)
457 llapi_json_destroy_list(&json_items);
463 * Given a path to a FIFO, create a filehandle for nonblocking writes to it.
464 * Intended to be used for copytool monitoring processes that read an
465 * event stream from the FIFO. Events written in the absence of a reader
468 * \param path Path to monitor FIFO.
470 * \retval 0 on success.
471 * \retval -errno on error.
473 int llapi_hsm_register_event_fifo(const char *path)
479 /* Create the FIFO if necessary. */
480 if ((mkfifo(path, 0644) < 0) && (errno != EEXIST)) {
481 llapi_error(LLAPI_MSG_ERROR, errno, "mkfifo(%s) failed", path);
484 if (errno == EEXIST) {
485 if (stat(path, &statbuf) < 0) {
486 llapi_error(LLAPI_MSG_ERROR, errno, "mkfifo(%s) failed",
490 if (!S_ISFIFO(statbuf.st_mode) ||
491 ((statbuf.st_mode & 0777) != 0644)) {
492 llapi_error(LLAPI_MSG_ERROR, errno, "%s exists but is "
493 "not a pipe or has a wrong mode", path);
497 created_hsm_event_fifo = true;
500 /* Open the FIFO for read so that the subsequent open for write
501 * doesn't immediately fail. */
502 read_fd = open(path, O_RDONLY | O_NONBLOCK);
504 llapi_error(LLAPI_MSG_ERROR, errno,
505 "cannot open(%s) for read", path);
509 /* Open the FIFO for writes, but don't block on waiting
511 llapi_hsm_event_fd = open(path, O_WRONLY | O_NONBLOCK);
514 /* Now close the reader. An external monitoring process can
515 * now open the FIFO for reads. If no reader comes along the
516 * events are lost. NOTE: Only one reader at a time! */
519 if (llapi_hsm_event_fd < 0) {
520 llapi_error(LLAPI_MSG_ERROR, -rc,
521 "cannot open(%s) for write", path);
525 /* Ignore SIGPIPEs -- can occur if the reader goes away. */
526 signal(SIGPIPE, SIG_IGN);
532 * Given a path to a FIFO, close its filehandle and delete the FIFO.
534 * \param path Path to monitor FIFO.
536 * \retval 0 on success.
537 * \retval -errno on error.
539 int llapi_hsm_unregister_event_fifo(const char *path)
541 /* Noop unless the event fd was initialized */
542 if (llapi_hsm_event_fd < 0)
545 if (close(llapi_hsm_event_fd) < 0)
548 if (created_hsm_event_fifo) {
550 created_hsm_event_fifo = false;
553 llapi_hsm_event_fd = -1;
559 * Custom logging callback to be used when a monitoring FIFO has been
560 * registered. Formats log entries as JSON events suitable for
561 * consumption by a copytool monitoring process.
563 * \param level The message loglevel.
564 * \param _rc The returncode associated with the message.
565 * \param fmt The message format string.
566 * \param args Arguments to be formatted by the format string.
570 void llapi_hsm_log_error(enum llapi_message_level level, int _rc,
571 const char *fmt, va_list args)
578 struct llapi_json_item_list *json_items;
580 /* Noop unless the event fd was initialized */
581 if (llapi_hsm_event_fd < 0)
584 rc = llapi_json_init_list(&json_items);
588 if ((level & LLAPI_MSG_NO_ERRNO) == 0) {
589 rc = llapi_json_add_item(&json_items, "errno",
595 rc = llapi_json_add_item(&json_items, "error",
602 va_copy(args2, args);
603 msg_len = vsnprintf(NULL, 0, fmt, args2) + 1;
606 msg = (char *) alloca(msg_len);
612 rc = vsnprintf(msg, msg_len, fmt, args);
616 rc = llapi_json_add_item(&json_items, "message",
622 rc = llapi_json_add_item(&json_items, "message",
624 "INTERNAL ERROR: message failed");
629 real_level = level & LLAPI_MSG_NO_ERRNO;
630 real_level = real_level > 0 ? level - LLAPI_MSG_NO_ERRNO : level;
632 rc = llapi_json_add_item(&json_items, "level", LLAPI_JSON_STRING,
633 (void *)llapi_msg_level2str(real_level));
637 rc = llapi_json_add_item(&json_items, "event_type", LLAPI_JSON_STRING,
642 rc = llapi_hsm_write_json_event(&json_items);
649 /* Write directly to stderr to avoid llapi_error, which now
650 * emits JSON event messages. */
651 fprintf(stderr, "\nFATAL ERROR IN llapi_hsm_log_error(): rc %d,", rc);
654 if (json_items != NULL)
655 llapi_json_destroy_list(&json_items);
660 /** Register a copytool
661 * \param[out] priv Opaque private control structure
662 * \param mnt Lustre filesystem mount point
663 * \param archive_count Number of valid archive IDs in \a archives
664 * \param archives Which archive numbers this copytool is
666 * \param rfd_flags flags applied to read fd of pipe
669 * \retval 0 on success.
670 * \retval -errno on error.
672 int llapi_hsm_copytool_register(struct hsm_copytool_private **priv,
673 const char *mnt, int archive_count,
674 int *archives, int rfd_flags)
676 struct hsm_copytool_private *ct;
679 if (archive_count > 0 && archives == NULL) {
680 llapi_err_noerrno(LLAPI_MSG_ERROR,
681 "NULL archive numbers");
685 if (archive_count > LL_HSM_MAX_ARCHIVE) {
686 llapi_err_noerrno(LLAPI_MSG_ERROR, "%d requested when maximum "
687 "of %zu archives supported", archive_count,
692 ct = calloc(1, sizeof(*ct));
696 ct->magic = CT_PRIV_MAGIC;
698 ct->open_by_fid_fd = -1;
699 ct->kuc.lk_rfd = LK_NOFD;
700 ct->kuc.lk_wfd = LK_NOFD;
702 ct->mnt = strdup(mnt);
703 if (ct->mnt == NULL) {
708 ct->kuch = malloc(HAL_MAXSIZE + sizeof(*ct->kuch));
709 if (ct->kuch == NULL) {
714 ct->mnt_fd = open(ct->mnt, O_RDONLY);
715 if (ct->mnt_fd < 0) {
720 ct->open_by_fid_fd = openat(ct->mnt_fd, OPEN_BY_FID_PATH, O_RDONLY);
721 if (ct->open_by_fid_fd < 0) {
726 /* no archives specified means "match all". */
728 for (rc = 0; rc < archive_count; rc++) {
729 if ((archives[rc] > LL_HSM_MAX_ARCHIVE) || (archives[rc] < 0)) {
730 llapi_err_noerrno(LLAPI_MSG_ERROR, "%d requested when "
731 "archive id [0 - %zu] is supported",
732 archives[rc], LL_HSM_MAX_ARCHIVE);
736 /* in the list we have an all archive wildcard
737 * so move to all archives mode
739 if (archives[rc] == 0) {
744 ct->archives |= (1 << (archives[rc] - 1));
747 rc = libcfs_ukuc_start(&ct->kuc, KUC_GRP_HSM, rfd_flags);
751 /* Storing archive(s) in lk_data; see mdc_ioc_hsm_ct_start */
752 ct->kuc.lk_data = ct->archives;
753 rc = ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, &ct->kuc);
756 llapi_error(LLAPI_MSG_ERROR, rc,
757 "cannot start copytool on '%s'", mnt);
761 llapi_hsm_log_ct_registration(&ct, CT_REGISTER);
763 /* Only the kernel reference keeps the write side open */
764 close(ct->kuc.lk_wfd);
765 ct->kuc.lk_wfd = LK_NOFD;
771 /* cleanup the kuc channel */
772 libcfs_ukuc_stop(&ct->kuc);
775 if (!(ct->mnt_fd < 0))
778 if (!(ct->open_by_fid_fd < 0))
779 close(ct->open_by_fid_fd);
790 /** Deregister a copytool
791 * Note: under Linux, until llapi_hsm_copytool_unregister is called
792 * (or the program is killed), the libcfs module will be referenced
793 * and unremovable, even after Lustre services stop.
795 int llapi_hsm_copytool_unregister(struct hsm_copytool_private **priv)
797 struct hsm_copytool_private *ct;
799 if (priv == NULL || *priv == NULL)
803 if (ct->magic != CT_PRIV_MAGIC)
806 /* Close the read side of the KUC pipe. This should be done
807 * before unregistering to avoid deadlock: a ldlm_cb thread
808 * enters libcfs_kkuc_group_put() acquires kg_sem and blocks
809 * in pipe_write() due to full pipe; then we attempt to
810 * unregister and block on kg_sem. */
811 libcfs_ukuc_stop(&ct->kuc);
813 /* Tell the kernel to stop sending us messages */
814 ct->kuc.lk_flags = LK_FLG_STOP;
815 ioctl(ct->mnt_fd, LL_IOC_HSM_CT_START, &ct->kuc);
817 llapi_hsm_log_ct_registration(&ct, CT_UNREGISTER);
819 close(ct->open_by_fid_fd);
829 /** Returns a file descriptor to poll/select on.
830 * \param ct Opaque private control structure
831 * \retval -EINVAL on error
832 * \retval the file descriptor for reading HSM events from the kernel
834 int llapi_hsm_copytool_get_fd(struct hsm_copytool_private *ct)
836 if (ct == NULL || ct->magic != CT_PRIV_MAGIC)
839 return libcfs_ukuc_get_rfd(&ct->kuc);
842 /** Wait for the next hsm_action_list
843 * \param ct Opaque private control structure
844 * \param halh Action list handle, will be allocated here
845 * \param msgsize Number of bytes in the message, will be set here
846 * \return 0 valid message received; halh and msgsize are set
848 * Note: The application must not call llapi_hsm_copytool_recv until it has
849 * cleared the data in ct->kuch from the previous call.
851 int llapi_hsm_copytool_recv(struct hsm_copytool_private *ct,
852 struct hsm_action_list **halh, int *msgsize)
854 struct kuc_hdr *kuch;
855 struct hsm_action_list *hal;
858 if (ct == NULL || ct->magic != CT_PRIV_MAGIC)
861 if (halh == NULL || msgsize == NULL)
867 rc = libcfs_ukuc_msg_get(&ct->kuc, (char *)kuch,
868 HAL_MAXSIZE + sizeof(*kuch),
873 /* Handle generic messages */
874 if (kuch->kuc_transport == KUC_TRANSPORT_GENERIC &&
875 kuch->kuc_msgtype == KUC_MSG_SHUTDOWN) {
880 if (kuch->kuc_transport != KUC_TRANSPORT_HSM ||
881 kuch->kuc_msgtype != HMT_ACTION_LIST) {
882 llapi_err_noerrno(LLAPI_MSG_ERROR,
883 "Unknown HSM message type %d:%d\n",
884 kuch->kuc_transport, kuch->kuc_msgtype);
889 if (kuch->kuc_msglen < sizeof(*kuch) + sizeof(*hal)) {
890 llapi_err_noerrno(LLAPI_MSG_ERROR, "Short HSM message %d",
896 /* Our message is a hsm_action_list. Use pointer math to skip
897 * kuch_hdr and point directly to the message payload.
899 hal = (struct hsm_action_list *)(kuch + 1);
901 /* Check that we have registered for this archive #
902 * if 0 registered, we serve any archive */
904 ((1 << (hal->hal_archive_id - 1)) & ct->archives) == 0) {
905 llapi_err_noerrno(LLAPI_MSG_INFO,
906 "This copytool does not service archive #%d,"
907 " ignoring this request."
908 " Mask of served archive is 0x%.8X",
909 hal->hal_archive_id, ct->archives);
915 *msgsize = kuch->kuc_msglen - sizeof(*kuch);
924 /** Get parent path from mount point and fid.
926 * \param mnt Filesystem root path.
927 * \param fid Object FID.
928 * \param parent Destination buffer.
929 * \param parent_len Destination buffer size.
930 * \return 0 on success.
932 static int fid_parent(const char *mnt, const lustre_fid *fid, char *parent,
937 long long recno = -1;
939 char strfid[FID_NOBRACE_LEN + 1];
942 snprintf(strfid, sizeof(strfid), DFID_NOBRACE, PFID(fid));
944 rc = llapi_fid2path(mnt, strfid, file, sizeof(file),
949 /* fid2path returns a relative path */
950 rc = snprintf(parent, parent_len, "%s/%s", mnt, file);
951 if (rc >= parent_len)
952 return -ENAMETOOLONG;
954 /* remove file name */
955 ptr = strrchr(parent, '/');
956 if (ptr == NULL || ptr == parent) {
966 static int ct_open_by_fid(const struct hsm_copytool_private *ct,
967 const struct lu_fid *fid, int open_flags)
969 char fid_name[FID_NOBRACE_LEN + 1];
972 snprintf(fid_name, sizeof(fid_name), DFID_NOBRACE, PFID(fid));
974 fd = openat(ct->open_by_fid_fd, fid_name, open_flags);
975 return fd < 0 ? -errno : fd;
979 * Get metadata attributes of file by FID.
981 * Use the IOC_MDC_GETFILEINFO ioctl (to send a MDS_GETATTR_NAME RPC)
982 * to get the attributes of the file identified by \a fid. This
983 * returns only the attributes stored on the MDT and avoids taking
984 * layout locks or accessing OST objects. It also bypasses the inode
985 * cache. Attributes are returned in \a st.
987 static int ct_md_getattr(const struct hsm_copytool_private *ct,
988 const struct lu_fid *fid,
991 struct lov_user_mds_data *lmd;
995 lmd_size = sizeof(lmd->lmd_st) +
996 lov_user_md_size(LOV_MAX_STRIPE_COUNT, LOV_USER_MAGIC_V3);
998 if (lmd_size < sizeof(lmd->lmd_st) + XATTR_SIZE_MAX)
999 lmd_size = sizeof(lmd->lmd_st) + XATTR_SIZE_MAX;
1001 if (lmd_size < FID_NOBRACE_LEN + 1)
1002 lmd_size = FID_NOBRACE_LEN + 1;
1004 lmd = malloc(lmd_size);
1008 snprintf((char *)lmd, lmd_size, DFID_NOBRACE, PFID(fid));
1010 rc = ioctl(ct->open_by_fid_fd, IOC_MDC_GETFILEINFO, lmd);
1013 llapi_error(LLAPI_MSG_ERROR, rc,
1014 "cannot get metadata attributes of "DFID" in '%s'",
1015 PFID(fid), ct->mnt);
1026 /** Create the destination volatile file for a restore operation.
1028 * \param hcp Private copyaction handle.
1029 * \param mdt_index MDT index where to create the volatile file.
1030 * \param flags Volatile file creation flags.
1031 * \return 0 on success.
1033 static int create_restore_volatile(struct hsm_copyaction_private *hcp,
1034 int mdt_index, int open_flags)
1038 char parent[PATH_MAX + 1];
1039 const char *mnt = hcp->ct_priv->mnt;
1040 struct hsm_action_item *hai = &hcp->copy.hc_hai;
1042 rc = fid_parent(mnt, &hai->hai_fid, parent, sizeof(parent));
1044 /* fid_parent() failed, try to keep on going */
1045 llapi_error(LLAPI_MSG_ERROR, rc,
1046 "cannot get parent path to restore "DFID" "
1047 "using '%s'", PFID(&hai->hai_fid), mnt);
1048 snprintf(parent, sizeof(parent), "%s", mnt);
1051 fd = llapi_create_volatile_idx(parent, mdt_index, open_flags);
1055 rc = fchown(fd, hcp->stat.st_uid, hcp->stat.st_gid);
1059 rc = llapi_fd2fid(fd, &hai->hai_dfid);
1074 /** Start processing an HSM action.
1075 * Should be called by copytools just before starting handling a request.
1076 * It could be skipped if copytool only want to directly report an error,
1077 * \see llapi_hsm_action_end().
1079 * \param hcp Opaque action handle to be passed to
1080 * llapi_hsm_action_progress and llapi_hsm_action_end.
1081 * \param ct Copytool handle acquired at registration.
1082 * \param hai The hsm_action_item describing the request.
1083 * \param restore_mdt_index On restore: MDT index where to create the volatile
1084 * file. Use -1 for default.
1085 * \param restore_open_flags On restore: volatile file creation mode. Use
1086 * O_LOV_DELAY_CREATE to manually set the LOVEA
1088 * \param is_error Whether this call is just to report an error.
1090 * \return 0 on success.
1092 int llapi_hsm_action_begin(struct hsm_copyaction_private **phcp,
1093 const struct hsm_copytool_private *ct,
1094 const struct hsm_action_item *hai,
1095 int restore_mdt_index, int restore_open_flags,
1098 struct hsm_copyaction_private *hcp;
1101 hcp = calloc(1, sizeof(*hcp));
1107 hcp->copy.hc_hai = *hai;
1108 hcp->copy.hc_hai.hai_len = sizeof(*hai);
1113 if (hai->hai_action == HSMA_RESTORE) {
1114 rc = ct_md_getattr(hcp->ct_priv, &hai->hai_fid, &hcp->stat);
1118 rc = create_restore_volatile(hcp, restore_mdt_index,
1119 restore_open_flags);
1124 rc = ioctl(ct->mnt_fd, LL_IOC_HSM_COPY_START, &hcp->copy);
1130 llapi_hsm_log_ct_progress(&hcp, hai, CT_START, 0, 0);
1133 hcp->magic = CP_PRIV_MAGIC;
1138 if (!(hcp->data_fd < 0))
1139 close(hcp->data_fd);
1146 /** Terminate an HSM action processing.
1147 * Should be called by copytools just having finished handling the request.
1148 * \param hdl[in,out] Handle returned by llapi_hsm_action_start.
1149 * \param he[in] The final range of copied data (for copy actions).
1150 * \param errval[in] The status code of the operation.
1151 * \param flags[in] The flags about the termination status (HP_FLAG_RETRY if
1152 * the error is retryable).
1154 * \return 0 on success.
1156 int llapi_hsm_action_end(struct hsm_copyaction_private **phcp,
1157 const struct hsm_extent *he, int hp_flags, int errval)
1159 struct hsm_copyaction_private *hcp;
1160 struct hsm_action_item *hai;
1163 if (phcp == NULL || *phcp == NULL || he == NULL)
1168 if (hcp->magic != CP_PRIV_MAGIC)
1171 hai = &hcp->copy.hc_hai;
1173 if (hai->hai_action == HSMA_RESTORE && errval == 0) {
1174 struct ll_futimes_3 lfu = {
1175 .lfu_atime_sec = hcp->stat.st_atim.tv_sec,
1176 .lfu_atime_nsec = hcp->stat.st_atim.tv_nsec,
1177 .lfu_mtime_sec = hcp->stat.st_mtim.tv_sec,
1178 .lfu_mtime_nsec = hcp->stat.st_mtim.tv_nsec,
1179 .lfu_ctime_sec = hcp->stat.st_ctim.tv_sec,
1180 .lfu_ctime_nsec = hcp->stat.st_ctim.tv_nsec,
1183 /* Set {a,m,c}time of volatile file to that of original. */
1184 if (ioctl(hcp->data_fd, LL_IOC_FUTIMES_3, &lfu) < 0) {
1189 rc = fsync(hcp->data_fd);
1197 /* In some cases, like restore, 2 FIDs are used.
1198 * Set the right FID to use here. */
1199 if (hai->hai_action == HSMA_ARCHIVE || hai->hai_action == HSMA_RESTORE)
1200 hai->hai_fid = hai->hai_dfid;
1202 /* Fill the last missing data that will be needed by
1203 * kernel to send a hsm_progress. */
1204 hcp->copy.hc_flags = hp_flags;
1205 hcp->copy.hc_errval = abs(errval);
1207 hcp->copy.hc_hai.hai_extent = *he;
1209 rc = ioctl(hcp->ct_priv->mnt_fd, LL_IOC_HSM_COPY_END, &hcp->copy);
1215 llapi_hsm_log_ct_progress(&hcp, hai, CT_FINISH, 0, 0);
1218 if (!(hcp->data_fd < 0))
1219 close(hcp->data_fd);
1227 /** Notify a progress in processing an HSM action.
1228 * \param hdl[in,out] handle returned by llapi_hsm_action_start.
1229 * \param he[in] the range of copied data (for copy actions).
1230 * \param total[in] the expected total of copied data (for copy actions).
1231 * \param hp_flags[in] HSM progress flags.
1232 * \return 0 on success.
1234 int llapi_hsm_action_progress(struct hsm_copyaction_private *hcp,
1235 const struct hsm_extent *he, __u64 total,
1239 struct hsm_progress hp;
1240 struct hsm_action_item *hai;
1242 if (hcp == NULL || he == NULL)
1245 if (hcp->magic != CP_PRIV_MAGIC)
1248 hai = &hcp->copy.hc_hai;
1250 memset(&hp, 0, sizeof(hp));
1252 hp.hp_cookie = hai->hai_cookie;
1253 hp.hp_flags = hp_flags;
1255 /* Progress is made on the data fid */
1256 hp.hp_fid = hai->hai_dfid;
1259 rc = ioctl(hcp->ct_priv->mnt_fd, LL_IOC_HSM_PROGRESS, &hp);
1263 llapi_hsm_log_ct_progress(&hcp, hai, CT_RUNNING, total, he->length);
1268 /** Get the fid of object to be used for copying data.
1269 * @return error code if the action is not a copy operation.
1271 int llapi_hsm_action_get_dfid(const struct hsm_copyaction_private *hcp,
1274 const struct hsm_action_item *hai = &hcp->copy.hc_hai;
1276 if (hcp->magic != CP_PRIV_MAGIC)
1279 if (hai->hai_action != HSMA_RESTORE && hai->hai_action != HSMA_ARCHIVE)
1282 *fid = hai->hai_dfid;
1288 * Get a file descriptor to be used for copying data. It's up to the
1289 * caller to close the FDs obtained from this function.
1291 * @retval a file descriptor on success.
1292 * @retval a negative error code on failure.
1294 int llapi_hsm_action_get_fd(const struct hsm_copyaction_private *hcp)
1296 const struct hsm_action_item *hai = &hcp->copy.hc_hai;
1299 if (hcp->magic != CP_PRIV_MAGIC)
1302 if (hai->hai_action == HSMA_ARCHIVE) {
1303 return ct_open_by_fid(hcp->ct_priv, &hai->hai_dfid,
1304 O_RDONLY | O_NOATIME | O_NOFOLLOW | O_NONBLOCK);
1305 } else if (hai->hai_action == HSMA_RESTORE) {
1306 fd = dup(hcp->data_fd);
1307 return fd < 0 ? -errno : fd;
1314 * Import an existing hsm-archived file into Lustre.
1316 * Caller must access file by (returned) newfid value from now on.
1318 * \param dst path to Lustre destination (e.g. /mnt/lustre/my/file).
1319 * \param archive archive number.
1320 * \param st struct stat buffer containing file ownership, perm, etc.
1321 * \param stripe_* Striping options. Currently ignored, since the restore
1322 * operation will set the striping. In V2, this striping might
1324 * \param newfid[out] Filled with new Lustre fid.
1326 int llapi_hsm_import(const char *dst, int archive, const struct stat *st,
1327 unsigned long long stripe_size, int stripe_offset,
1328 int stripe_count, int stripe_pattern, char *pool_name,
1331 struct hsm_user_import hui;
1335 if (stripe_pattern == 0)
1336 stripe_pattern = LOV_PATTERN_RAID0;
1338 /* Create a non-striped file */
1339 fd = llapi_file_open_pool(dst, O_CREAT | O_WRONLY, st->st_mode,
1340 stripe_size, stripe_offset, stripe_count,
1341 stripe_pattern | LOV_PATTERN_F_RELEASED,
1344 llapi_error(LLAPI_MSG_ERROR, fd,
1345 "cannot create '%s' for import", dst);
1349 /* Get the new fid in Lustre. Caller needs to use this fid
1351 rc = llapi_fd2fid(fd, newfid);
1353 llapi_error(LLAPI_MSG_ERROR, rc,
1354 "cannot get fid of '%s' for import", dst);
1358 hui.hui_uid = st->st_uid;
1359 hui.hui_gid = st->st_gid;
1360 hui.hui_mode = st->st_mode;
1361 hui.hui_size = st->st_size;
1362 hui.hui_archive_id = archive;
1363 hui.hui_atime = st->st_atime;
1364 hui.hui_atime_ns = st->st_atim.tv_nsec;
1365 hui.hui_mtime = st->st_mtime;
1366 hui.hui_mtime_ns = st->st_mtim.tv_nsec;
1367 rc = ioctl(fd, LL_IOC_HSM_IMPORT, &hui);
1370 llapi_error(LLAPI_MSG_ERROR, rc, "cannot import '%s'", dst);
1383 * Return the current HSM states and HSM requests related to file pointed by \a
1386 * \param hus Should be allocated by caller. Will be filled with current file
1389 * \retval 0 on success.
1390 * \retval -errno on error.
1392 int llapi_hsm_state_get_fd(int fd, struct hsm_user_state *hus)
1396 rc = ioctl(fd, LL_IOC_HSM_STATE_GET, hus);
1397 /* If error, save errno value */
1398 rc = rc ? -errno : 0;
1404 * Return the current HSM states and HSM requests related to file pointed by \a
1407 * see llapi_hsm_state_get_fd() for args use and return
1409 int llapi_hsm_state_get(const char *path, struct hsm_user_state *hus)
1414 fd = open(path, O_RDONLY | O_NONBLOCK);
1418 rc = llapi_hsm_state_get_fd(fd, hus);
1425 * Set HSM states of file pointed by \a fd
1427 * Using the provided bitmasks, the current HSM states for this file will be
1428 * changed. \a archive_id could be used to change the archive number also. Set
1429 * it to 0 if you do not want to change it.
1431 * \param setmask Bitmask for flag to be set.
1432 * \param clearmask Bitmask for flag to be cleared.
1433 * \param archive_id Archive number identifier to use. 0 means no change.
1435 * \retval 0 on success.
1436 * \retval -errno on error.
1438 int llapi_hsm_state_set_fd(int fd, __u64 setmask, __u64 clearmask,
1441 struct hsm_state_set hss;
1444 hss.hss_valid = HSS_SETMASK|HSS_CLEARMASK;
1445 hss.hss_setmask = setmask;
1446 hss.hss_clearmask = clearmask;
1447 /* Change archive_id if provided. We can only change
1448 * to set something different than 0. */
1449 if (archive_id > 0) {
1450 hss.hss_valid |= HSS_ARCHIVE_ID;
1451 hss.hss_archive_id = archive_id;
1453 rc = ioctl(fd, LL_IOC_HSM_STATE_SET, &hss);
1454 /* If error, save errno value */
1455 rc = rc ? -errno : 0;
1461 * Set HSM states of file pointed by \a path.
1463 * see llapi_hsm_state_set_fd() for args use and return
1465 int llapi_hsm_state_set(const char *path, __u64 setmask, __u64 clearmask,
1471 fd = open(path, O_WRONLY | O_LOV_DELAY_CREATE | O_NONBLOCK);
1475 rc = llapi_hsm_state_set_fd(fd, setmask, clearmask, archive_id);
1482 * Return the current HSM request related to file pointed by \a path.
1484 * \param hca Should be allocated by caller. Will be filled with current file
1487 * \retval 0 on success.
1488 * \retval -errno on error.
1490 int llapi_hsm_current_action(const char *path, struct hsm_current_action *hca)
1495 fd = open(path, O_RDONLY | O_NONBLOCK);
1499 rc = ioctl(fd, LL_IOC_HSM_ACTION, hca);
1500 /* If error, save errno value */
1501 rc = rc ? -errno : 0;
1508 * Allocate a hsm_user_request with the specified carateristics.
1509 * This structure should be freed with free().
1511 * \return an allocated structure on success, NULL otherwise.
1513 struct hsm_user_request *llapi_hsm_user_request_alloc(int itemcount,
1518 len += sizeof(struct hsm_user_request);
1519 len += sizeof(struct hsm_user_item) * itemcount;
1522 return (struct hsm_user_request *)malloc(len);
1526 * Send a HSM request to Lustre, described in \param request.
1528 * \param path Fullpath to the file to operate on.
1529 * \param request The request, allocated with llapi_hsm_user_request_alloc().
1531 * \return 0 on success, an error code otherwise.
1533 int llapi_hsm_request(const char *path, const struct hsm_user_request *request)
1538 rc = get_root_path(WANT_FD, NULL, &fd, (char *)path, -1);
1542 rc = ioctl(fd, LL_IOC_HSM_REQUEST, request);
1543 /* If error, save errno value */
1544 rc = rc ? -errno : 0;