lustre/utils/lsnapshot.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.  A copy is
  14  * included in the COPYING file that accompanied this code.
  15
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2017, Intel Corporation.
  24  *
  25  * lustre/utils/lsnapshot.c
  26  *
  27  * Author: Fan, Yong <fan.yong@intel.com>
  28  */
  29
  30 #include <stdio.h>
  31 #include <string.h>
  32 #include <getopt.h>
  33 #include <unistd.h>
  34 #include <stdlib.h>
  35 #include <errno.h>
  36 #include <string.h>
  37 #include <sys/types.h>
  38 #include <sys/wait.h>
  39 #include <sys/time.h>
  40 #include <sys/file.h>
  41 #include <time.h>
  42 #include <limits.h>
  43 #include <ctype.h>
  44
  45 #include <libcfs/util/list.h>
  46 #include <libcfs/util/ioctl.h>
  47 #include <linux/lustre/lustre_ioctl.h>
  48 #include <linux/lustre/lustre_barrier_user.h>
  49
  50 #include "obdctl.h"
  51
  52 #define SNAPSHOT_CONF_DIR       "/etc/lsnapshot"
  53 #define LDEV_CONF               "/etc/ldev.conf"
  54 #define SNAPSHOT_LOG            "/var/log/lsnapshot.log"
  55 #define SNAPSHOT_MAGIC          "0x14F711B9"
  56 #define MAX_BUF_SIZE            4096
  57
  58 enum snapshot_role {
  59         SR_MGS  = 0x0001,
  60         SR_MDT  = 0x0002,
  61         SR_OST  = 0x0004,
  62 };
  63
  64 struct snapshot_target {
  65         struct list_head         st_list;
  66         /* Target node name. */
  67         char                    *st_host;
  68         /* Where the pool is */
  69         char                    *st_dir;
  70         /* The target pool name on the target node. */
  71         char                    *st_pool;
  72         /* The backend filesystem name against the target pool. */
  73         char                    *st_filesystem;
  74         int                      st_role;
  75         unsigned int             st_index;
  76         unsigned int             st_gen;
  77         int                      st_line;
  78         int                      st_status;
  79         pid_t                    st_pid;
  80         bool                     st_ignored;
  81 };
  82
  83 struct snapshot_instance {
  84         struct list_head         si_mdts_list;
  85         struct list_head         si_osts_list;
  86         struct snapshot_target  *si_mgs;
  87         struct snapshot_target  *si_mdt0;
  88         FILE                    *si_log_fp;
  89         char                    *si_rsh;
  90         char                    *si_fsname;
  91         char                    *si_ssname;
  92         char                    *si_new_ssname;
  93         char                    *si_comment;
  94         int                      si_conf_fd;
  95         int                      si_timeout;
  96         bool                     si_barrier;
  97         bool                     si_detail;
  98         bool                     si_force;
  99 };
 100
 101 static const char snapshot_rsh_default[] = "ssh";
 102
 103 static char *snapshot_role2name(char *name, enum snapshot_role role,
 104                                 __u32 index)
 105 {
 106         if (role & SR_MDT)
 107                 snprintf(name, 8, "MDT%04x", index);
 108         else if (role & SR_MGS)
 109                 snprintf(name, 4, "MGS");
 110         else
 111                 snprintf(name, 8, "OST%04x", index);
 112
 113         return name;
 114 }
 115
 116 #define SNAPSHOT_ADD_LOG(si, format, ...)                               \
 117 do {                                                                    \
 118         char buf[MAX_BUF_SIZE];                                         \
 119         char *ptr;                                                      \
 120         time_t tt;                                                      \
 121                                                                         \
 122         memset(buf, 0, sizeof(buf));                                    \
 123         time(&tt);                                                      \
 124         snprintf(buf, sizeof(buf) - 1, "%s", ctime(&tt));               \
 125         ptr = strrchr(buf, '\n');                                       \
 126         if (ptr)                                                        \
 127                 *ptr = '\0';                                            \
 128                                                                         \
 129         fprintf(si->si_log_fp, "%s (%d:%s:%d:%s:%s): "format, buf,      \
 130                 getpid(), __func__, __LINE__, si->si_fsname,            \
 131                 si->si_rsh, ## __VA_ARGS__);                            \
 132 } while (0)
 133
 134 char *snapshot_fgets(FILE *fp, char *buf, int buflen)
 135 {
 136         char *ptr;
 137
 138         memset(buf, 0, buflen);
 139         if (fgets(buf, buflen, fp) == NULL)
 140                 return NULL;
 141
 142         ptr = strchr(buf, '\n');
 143         if (ptr)
 144                 *ptr = '\0';
 145
 146         return buf;
 147 }
 148
 149 static int snapshot_exec(const char *cmd)
 150 {
 151         int rc;
 152
 153         errno = 0;
 154
 155         /* system() return value depends on both the system() general framework,
 156          * such as whether fork()/exec() success or fail, and the real @cmd exec
 157          * result. Especially, if the @cmd is remote command, we may cannot know
 158          * the real failure. */
 159         rc = system(cmd);
 160         /* fork()/exec() error */
 161         if (rc == -1)
 162                 return errno != 0 ? -errno : -1;
 163
 164         if (WIFEXITED(rc)) {
 165                 rc = WEXITSTATUS(rc);
 166                 if (rc > 0)
 167                         rc = -rc;
 168         } else if (WIFSIGNALED(rc)) {
 169                 rc = -EINTR;
 170         } else {
 171                 /* all other known or unknown cases. */
 172                 rc = -EFAULT;
 173         }
 174
 175         return rc;
 176 }
 177
 178 static int snapshot_load_conf_ldev(struct snapshot_instance *si, char *buf,
 179                                    struct snapshot_target *st, char **role)
 180 {
 181         char *label = NULL;
 182         char *device = NULL;
 183         char *ignore = NULL;
 184         char *ptr;
 185         char *ptr1;
 186         int len;
 187         int rc;
 188
 189         rc = sscanf(buf, "%ms %ms %ms %ms",
 190                     &st->st_host, &ignore, &label, &device);
 191         if (rc < 4) {
 192                 rc = -EINVAL;
 193                 goto out;
 194         }
 195
 196         free(ignore);
 197
 198         /* Format of device:
 199          * [md|zfs:][pool_dir/]<pool>/<filesystem> */
 200         ptr = strchr(device, ':');
 201         if (ptr) {
 202                 ptr++;
 203                 if (strncmp(device, "zfs:", strlen("zfs:")) != 0) {
 204                         rc = -EINVAL;
 205                         goto out;
 206                 }
 207         } else {
 208                         ptr = device;
 209         }
 210
 211         ptr1 = strrchr(ptr, '/');
 212         /* "ptr1 - ptr + 1 == strlen(ptr)" means '/' is at the tail. */
 213         if (!ptr1 || ptr1 == ptr || ptr1 - ptr + 1 == strlen(ptr)) {
 214                 rc = -EINVAL;
 215                 goto out;
 216         }
 217
 218         len = strlen(ptr1);
 219         st->st_filesystem = malloc(len);
 220         if (!st->st_filesystem) {
 221                 rc = -ENOMEM;
 222                 goto out;
 223         }
 224
 225         *ptr1 = '\0';
 226         strncpy(st->st_filesystem, ptr1 + 1, len - 1);
 227         st->st_filesystem[len - 1] = '\0';
 228
 229         if (*ptr == '/') {
 230                 ptr1 = strrchr(ptr, '/');
 231                 *ptr1 = '\0';
 232                 len = strlen(ptr);
 233                 st->st_dir = malloc(len + 1);
 234                 if (!st->st_dir) {
 235                         rc = -ENOMEM;
 236                         goto out;
 237                 }
 238
 239                 strncpy(st->st_dir, ptr, len);
 240                 st->st_dir[len] = '\0';
 241                 ptr = ptr1 + 1;
 242         }
 243
 244         len = strlen(ptr);
 245         st->st_pool = malloc(len + 1);
 246         if (!st->st_pool) {
 247                 rc = -ENOMEM;
 248                 goto out;
 249         }
 250
 251         strncpy(st->st_pool, ptr, len);
 252         st->st_pool[len] = '\0';
 253
 254         /* Format of label:
 255          * fsname-<role><index> or <role><index> */
 256         ptr = strchr(label, '-');
 257         if (ptr) {
 258                 if (strncmp(si->si_fsname, label, ptr - label) != 0) {
 259                         /* This line is NOT for current filesystem .*/
 260                         rc = -EAGAIN;
 261                         goto out;
 262                 }
 263
 264                 ptr++;
 265         } else {
 266                 ptr = label;
 267         }
 268
 269         if (strlen(ptr) < 3 || strlen(ptr) > 7) {
 270                 rc = -EINVAL;
 271                 goto out;
 272         }
 273
 274         *role = malloc(4);
 275         if (!*role) {
 276                 rc = -ENOMEM;
 277                 goto out;
 278         }
 279
 280         strncpy(*role, ptr, 3);
 281         (*role)[3] = 0;
 282         ptr += 3;
 283         len = 0;
 284         while (isxdigit(ptr[len])) {
 285                 if (isdigit(ptr[len]))
 286                         st->st_index =
 287                                 st->st_index * 16 + ptr[len] - '0';
 288                 else if (isupper(ptr[len]))
 289                         st->st_index =
 290                                 st->st_index * 16 + ptr[len] - 'A' + 10;
 291                 else
 292                         st->st_index =
 293                                 st->st_index * 16 + ptr[len] - 'a' + 10;
 294                 len++;
 295         }
 296
 297         if (len == 0) {
 298                 if (strncasecmp(*role, "MGS", 3) != 0)
 299                         rc = -EINVAL;
 300                 else
 301                         rc = 0;
 302
 303                 goto out;
 304         }
 305
 306         if (!isxdigit(ptr[len]) && ptr[len] != '\0') {
 307                 rc = -EINVAL;
 308                 goto out;
 309         }
 310
 311 out:
 312         if (label)
 313                 free(label);
 314         if (device)
 315                 free(device);
 316
 317         return rc;
 318 }
 319
 320 /**
 321  * For old snasphot tools, the configration is in /etc/lsnapshot/${fsname}.conf,
 322  * the format is:
 323  * <host> <pool_dir> <pool> <local_fsname> <role(,s)> <index>
 324  *
 325  * For example:
 326  *
 327  * host-mdt1 /tmp myfs-mdt1 mdt1 MGS,MDT 0
 328  * host-mdt2 /tmp myfs-mdt2 mdt2 MDT 1
 329  * host-ost1 /tmp myfs-ost1 ost1 OST 0
 330  * host-ost2 /tmp myfs-ost2 ost2 OST 1
 331  *
 332  *
 333  * For new snasphot tools, the configration is in /etc/ldev.conf, which is not
 334  * only for snapshot, but also for other purpose. The format is:
 335  * <host> foreign/- <label> <device> [journal-path]/- [raidtab]
 336  *
 337  * The format of <label> is:
 338  * fsname-<role><index> or <role><index>
 339  *
 340  * The format of <device> is:
 341  * [md|zfs:][pool_dir/]<pool>/<filesystem>
 342  *
 343  * Snapshot only uses the fields <host>, <label> and <device>.
 344  *
 345  * For example:
 346  *
 347  * host-mdt1 - myfs-MDT0000 zfs:/tmp/myfs-mdt1/mdt1
 348  *
 349  *
 350  * \retval       0      for success
 351  * \retval      +ve     the line# with which the current line is conflict
 352  * \retval      -EAGAIN skip current line
 353  * \retval      -ve     other failures
 354  */
 355 static int snapshot_load_conf_one(struct snapshot_instance *si,
 356                                   char *buf, int line_num, bool is_ldev)
 357 {
 358         struct snapshot_target *st;
 359         char *role = NULL;
 360         int rc = 0;
 361
 362         /* filter out space */
 363         while (isspace(*buf))
 364                 buf++;
 365
 366         /* skip empty line */
 367         if (*buf == '\0')
 368                 return 0;
 369
 370         /* skip comment line */
 371         if (*buf == '#')
 372                 return 0;
 373
 374         st = malloc(sizeof(*st));
 375         if (!st)
 376                 return -ENOMEM;
 377
 378         memset(st, 0, sizeof(*st));
 379         INIT_LIST_HEAD(&st->st_list);
 380
 381         if (is_ldev) {
 382                 rc = snapshot_load_conf_ldev(si, buf, st, &role);
 383         } else {
 384                 rc = sscanf(buf, "%ms %ms %ms %ms %ms %d",
 385                             &st->st_host, &st->st_dir, &st->st_pool,
 386                             &st->st_filesystem, &role, &st->st_index);
 387                 if (rc < 6)
 388                         rc = -EINVAL;
 389         }
 390
 391         if (rc < 0)
 392                 goto out;
 393         rc = 0;
 394
 395         if (strncasecmp(role, "MGS", 3) == 0) {
 396                 st->st_role = SR_MGS;
 397                 if (role[3] == ',') {
 398                         /* MGS,MDT */
 399                         if (strncasecmp(&role[4], "MDT", 3) != 0) {
 400                                 rc = -EINVAL;
 401                                 goto out;
 402                         }
 403
 404                         st->st_role |= SR_MDT;
 405                 }
 406         } else if (strncasecmp(role, "MDT", 3) == 0) {
 407                 st->st_role = SR_MDT;
 408                 if (role[3] == ',') {
 409                         /* MDT,MGS */
 410                         if (strncasecmp(&role[4], "MGS", 3) != 0) {
 411                                 rc = -EINVAL;
 412                                 goto out;
 413                         }
 414
 415                         st->st_role |= SR_MGS;
 416                 }
 417         } else if (strncasecmp(role, "OST", 3) == 0) {
 418                 st->st_role = SR_OST;
 419         } else {
 420                 rc = -EINVAL;
 421                 goto out;
 422         }
 423
 424         st->st_line = line_num;
 425         if (st->st_role & SR_MDT) {
 426                 /* MGS is the first, MDT0 is just after the MGS
 427                  * if they are not combined together. */
 428                 if (st->st_role & SR_MGS) {
 429                         if (si->si_mgs) {
 430                                 rc = si->si_mgs->st_line;
 431                                 goto out;
 432                         }
 433
 434                         si->si_mgs = st;
 435                         list_add(&st->st_list, &si->si_mdts_list);
 436                 }
 437
 438                 if (st->st_index == 0) {
 439                         if (si->si_mdt0) {
 440                                 rc = si->si_mdt0->st_line;
 441                                 goto out;
 442                         }
 443
 444                         si->si_mdt0 = st;
 445                         if (list_empty(&st->st_list)) {
 446                                 if (list_empty(&si->si_mdts_list) ||
 447                                     !si->si_mgs)
 448                                         list_add(&st->st_list,
 449                                                  &si->si_mdts_list);
 450                                 else
 451                                         list_add(&st->st_list,
 452                                                  &si->si_mgs->st_list);
 453                         }
 454                 } else if (list_empty(&st->st_list)) {
 455                         list_add_tail(&st->st_list, &si->si_mdts_list);
 456                 }
 457         } else if (st->st_role & SR_MGS) {
 458                 if (si->si_mgs) {
 459                         rc = si->si_mgs->st_line;
 460                         goto out;
 461                 }
 462
 463                 si->si_mgs = st;
 464                 list_add(&st->st_list, &si->si_mdts_list);
 465         } else {
 466                 list_add_tail(&st->st_list, &si->si_osts_list);
 467         }
 468
 469 out:
 470         if (role)
 471                 free(role);
 472
 473         if (rc) {
 474                 if (st->st_host)
 475                         free(st->st_host);
 476                 if (st->st_dir)
 477                         free(st->st_dir);
 478                 if (st->st_pool)
 479                         free(st->st_pool);
 480                 if (st->st_filesystem)
 481                         free(st->st_filesystem);
 482                 free(st);
 483         }
 484
 485         return rc;
 486 }
 487
 488 static int snapshot_load_conf(struct snapshot_instance *si, int lock_mode)
 489 {
 490         FILE *fp;
 491         char buf[MAX_BUF_SIZE];
 492         char conf_name[32];
 493         int line_num = 1;
 494         int fd = -1;
 495         int rc = 0;
 496         bool is_ldev = true;
 497
 498         memset(conf_name, 0, sizeof(conf_name));
 499         strncpy(conf_name, LDEV_CONF, sizeof(conf_name) - 1);
 500         fd = open(conf_name, O_RDONLY);
 501         if (fd < 0) {
 502                 if (errno != ENOENT) {
 503                         fprintf(stderr,
 504                                 "Can't open the snapshot config file %s: %s\n",
 505                                 conf_name, strerror(errno));
 506
 507                         return fd;
 508                 }
 509
 510                 snprintf(conf_name, sizeof(conf_name) - 1, "%s/%s.conf",
 511                          SNAPSHOT_CONF_DIR, si->si_fsname);
 512                 fd = open(conf_name, O_RDONLY);
 513                 if (fd < 0) {
 514                         fprintf(stderr,
 515                                 "Can't open the snapshot config file %s: %s\n",
 516                                 conf_name, strerror(errno));
 517
 518                         return fd;
 519                 }
 520
 521                 is_ldev = false;
 522         }
 523
 524         rc = flock(fd, lock_mode | LOCK_NB);
 525         if (rc < 0) {
 526                 fprintf(stderr,
 527                         "Can't lock the snapshot config file %s (%d): %s\n",
 528                         conf_name, lock_mode, strerror(errno));
 529                 close(fd);
 530                 return rc;
 531         }
 532
 533         fp = fdopen(fd, "r");
 534         if (!fp) {
 535                 fprintf(stderr,
 536                         "Can't fdopen the snapshot config file %s: %s\n",
 537                         conf_name, strerror(errno));
 538                 rc = -1;
 539                 goto out;
 540         }
 541
 542         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
 543                 rc = snapshot_load_conf_one(si, buf, line_num, is_ldev);
 544                 if (rc == -EINVAL) {
 545                         fprintf(stderr,
 546                                 "Invalid snapshot config file %s at the line "
 547                                 "%d '%s'\n", conf_name, line_num, buf);
 548                 } else if (rc == -EAGAIN) {
 549                         rc = 0;
 550                 } else if (rc > 0) {
 551                         fprintf(stderr,
 552                                 "The config role has been specified repeatedly "
 553                                 "at the lines %d/%d in %s\n",
 554                                 rc, line_num, conf_name);
 555                         rc = -EINVAL;
 556                 }
 557
 558                 if (rc)
 559                         goto out;
 560
 561                 line_num++;
 562         }
 563
 564         if (!si->si_mdt0) {
 565                 fprintf(stderr,
 566                         "Miss MDT0 in the config file %s\n",
 567                         conf_name);
 568                 rc = -1;
 569                 goto out;
 570         }
 571
 572         /* By default, the MGS is on the MDT0 if it is not specified. */
 573         if (!si->si_mgs) {
 574                 si->si_mgs = si->si_mdt0;
 575                 si->si_mgs->st_role |= SR_MGS;
 576         }
 577
 578         if (list_empty(&si->si_osts_list)) {
 579                 fprintf(stderr,
 580                         "Miss OST(s) in the config file %s\n",
 581                         conf_name);
 582                 rc = -1;
 583                 goto out;
 584         }
 585
 586 out:
 587         if (fd >= 0) {
 588                 if (rc < 0) {
 589                         flock(fd, LOCK_UN);
 590                         close(fd);
 591                 } else {
 592                         si->si_conf_fd = fd;
 593                 }
 594         }
 595
 596         return rc;
 597 }
 598
 599 static void snapshot_unload_conf(struct snapshot_instance *si)
 600 {
 601         struct snapshot_target *st;
 602
 603         while (!list_empty(&si->si_mdts_list)) {
 604                 st = list_entry(si->si_mdts_list.next,
 605                                 struct snapshot_target, st_list);
 606                 list_del(&st->st_list);
 607                 free(st->st_host);
 608                 free(st->st_dir);
 609                 free(st->st_pool);
 610                 free(st->st_filesystem);
 611                 free(st);
 612         }
 613
 614         while (!list_empty(&si->si_osts_list)) {
 615                 st = list_entry(si->si_osts_list.next,
 616                                 struct snapshot_target, st_list);
 617                 list_del(&st->st_list);
 618                 free(st->st_host);
 619                 free(st->st_dir);
 620                 free(st->st_pool);
 621                 free(st->st_filesystem);
 622                 free(st);
 623         }
 624
 625         si->si_mgs = NULL;
 626         si->si_mdt0 = NULL;
 627
 628         if (si->si_conf_fd >= 0) {
 629                 flock(si->si_conf_fd, LOCK_UN);
 630                 close(si->si_conf_fd);
 631                 si->si_conf_fd = -1;
 632         }
 633 }
 634
 635 static int snapshot_handle_string_option(char **dst, const char *option,
 636                                          const char *opt_name)
 637 {
 638         int len;
 639
 640         if (*dst && *dst != snapshot_rsh_default) {
 641                 fprintf(stderr, "specify the %s repeatedly.\n", opt_name);
 642                 return -EINVAL;
 643         }
 644
 645         len = strlen(option);
 646         *dst = malloc(len + 1);
 647         if (!*dst)
 648                 return -ENOMEM;
 649
 650         strncpy(*dst, option, len);
 651         (*dst)[len] = '\0';
 652         return 0;
 653 }
 654
 655 static void snapshot_fini(struct snapshot_instance *si)
 656 {
 657         snapshot_unload_conf(si);
 658
 659         if (si->si_log_fp)
 660                 fclose(si->si_log_fp);
 661
 662         if (si->si_rsh && si->si_rsh != snapshot_rsh_default)
 663                 free(si->si_rsh);
 664         if (si->si_fsname)
 665                 free(si->si_fsname);
 666         if (si->si_ssname)
 667                 free(si->si_ssname);
 668         if (si->si_new_ssname)
 669                 free(si->si_new_ssname);
 670         if (si->si_comment)
 671                 free(si->si_comment);
 672
 673         free(si);
 674 }
 675
 676 static struct snapshot_instance *
 677 snapshot_init(int argc, char * const argv[], const struct option *longopts,
 678               const char *optstring, void (*usage)(void),
 679               int lock_mode, int *err)
 680 {
 681         struct snapshot_instance *si;
 682         int idx;
 683         int opt;
 684
 685         *err = 0;
 686         si = malloc(sizeof(*si));
 687         if (!si) {
 688                 fprintf(stderr,
 689                         "No enough memory to initialize snapshot instance.\n");
 690                 *err = -ENOMEM;
 691                 return NULL;
 692         }
 693
 694         memset(si, 0, sizeof(*si));
 695         INIT_LIST_HEAD(&si->si_mdts_list);
 696         INIT_LIST_HEAD(&si->si_osts_list);
 697         si->si_rsh = (char *)snapshot_rsh_default;
 698         si->si_conf_fd = -1;
 699         si->si_timeout = BARRIER_TIMEOUT_DEFAULT;
 700         si->si_barrier = true;
 701         si->si_detail = false;
 702         si->si_force = false;
 703
 704         while ((opt = getopt_long(argc, argv, optstring,
 705                                   longopts, &idx)) != EOF) {
 706                 switch (opt) {
 707                 case 'b':
 708                         if (!optarg || strcmp(optarg, "on") == 0) {
 709                                 si->si_barrier = true;
 710                         } else if (strcmp(optarg, "off") == 0) {
 711                                 si->si_barrier = false;
 712                         } else {
 713                                 usage();
 714                                 *err = -EINVAL;
 715                                 goto out;
 716                         }
 717                         break;
 718                 case 'c':
 719                         *err = snapshot_handle_string_option(&si->si_comment,
 720                                                              optarg, "comment");
 721                         if (*err != 0)
 722                                 goto out;
 723                         break;
 724                 case 'd':
 725                         si->si_detail = true;
 726                         break;
 727                 case 'f':
 728                         si->si_force = true;
 729                         break;
 730                 case 'F':
 731                         *err = snapshot_handle_string_option(&si->si_fsname,
 732                                                              optarg, "fsname");
 733                         if (*err != 0)
 734                                 goto out;
 735                         break;
 736                 case 'n':
 737                         *err = snapshot_handle_string_option(&si->si_ssname,
 738                                                              optarg, "ssname");
 739                         if (*err != 0)
 740                                 goto out;
 741                         break;
 742                 case 'N':
 743                         *err = snapshot_handle_string_option(&si->si_new_ssname,
 744                                                              optarg,
 745                                                              "new ssname");
 746                         if (*err != 0)
 747                                 goto out;
 748                         break;
 749                 case 'r':
 750                         *err = snapshot_handle_string_option(&si->si_rsh,
 751                                                              optarg,
 752                                                              "remote shell");
 753                         if (*err != 0)
 754                                 goto out;
 755                         break;
 756                 case 't':
 757                         si->si_timeout = atoi(optarg);
 758                         break;
 759                 default:
 760                         *err = -EINVAL;
 761                         usage();
 762                         goto out;
 763                 case 'h':
 764                         usage();
 765                         snapshot_fini(si);
 766                         *err = 0;
 767                         return NULL;
 768                 }
 769         }
 770
 771         if (!si->si_fsname) {
 772                 fprintf(stderr, "The fsname must be specified\n");
 773                 usage();
 774                 *err = -EINVAL;
 775                 goto out;
 776         }
 777
 778         if (strlen(si->si_fsname) > 8) {
 779                 fprintf(stderr, "Invalid fsname %s\n", si->si_fsname);
 780                 *err = -EINVAL;
 781                 goto out;
 782         }
 783
 784         si->si_log_fp = fopen(SNAPSHOT_LOG, "a");
 785         if (!si->si_log_fp) {
 786                 *err = -errno;
 787                 fprintf(stderr,
 788                         "Can't open the snapshot log file %s: %s\n",
 789                         SNAPSHOT_LOG, strerror(errno));
 790                 goto out;
 791         }
 792
 793         *err = snapshot_load_conf(si, lock_mode);
 794
 795 out:
 796         if (*err != 0 && si) {
 797                 snapshot_fini(si);
 798                 si = NULL;
 799         }
 800
 801         return si;
 802 }
 803
 804 static int __snapshot_wait(struct snapshot_instance *si,
 805                            struct list_head *head, int *err)
 806 {
 807         struct snapshot_target *st;
 808         int count = 0;
 809         int rc = 0;
 810
 811         list_for_each_entry(st, head, st_list) {
 812                 if (st->st_pid == 0)
 813                         continue;
 814
 815                 rc = waitpid(st->st_pid, &st->st_status, 0);
 816                 if (rc < 0) {
 817                         SNAPSHOT_ADD_LOG(si, "Can't wait child (%d) operation "
 818                                          "on the target <%s:%x:%d>: %s\n",
 819                                          st->st_pid, st->st_host, st->st_role,
 820                                          st->st_index, strerror(errno));
 821                         count++;
 822                         if (*err == 0)
 823                                 *err = rc;
 824
 825                         st->st_pid = 0;
 826                         /* continue to wait for next */
 827                         continue;
 828                 }
 829
 830                 if (WIFEXITED(st->st_status)) {
 831                         rc = WEXITSTATUS(st->st_status);
 832                         if (rc > 0)
 833                                 rc -= 256;
 834
 835                         if (rc == -ESRCH) {
 836                                 st->st_ignored = true;
 837                         } else if (rc) {
 838                                 count++;
 839                                 if (*err == 0)
 840                                         *err = rc;
 841                         }
 842                 } else if (WIFSIGNALED(st->st_status)) {
 843                         SNAPSHOT_ADD_LOG(si, "The child (%d) operation on the "
 844                                          "target <%s:%x:%d> was killed by "
 845                                          "signal %d\n",
 846                                          st->st_pid, st->st_host, st->st_role,
 847                                          st->st_index, WTERMSIG(st->st_status));
 848                         count++;
 849                         if (*err == 0)
 850                                 *err = -EINTR;
 851                 } else {
 852                         SNAPSHOT_ADD_LOG(si, "The child (%d) operation on the "
 853                                          "target <%s:%x:%d> failed for "
 854                                          "unknown reason\n",
 855                                          st->st_pid, st->st_host, st->st_role,
 856                                          st->st_index);
 857                         count++;
 858                         if (*err == 0)
 859                                 *err = -EFAULT;
 860                 }
 861
 862                 st->st_pid = 0;
 863         }
 864
 865         return count;
 866 }
 867
 868 static int snapshot_wait(struct snapshot_instance *si, int *err)
 869 {
 870         int count;
 871
 872         count = __snapshot_wait(si, &si->si_mdts_list, err);
 873         count += __snapshot_wait(si, &si->si_osts_list, err);
 874
 875         return count;
 876 }
 877
 878 static char *snapshot_first_skip_blank(char *buf)
 879 {
 880         char *ptr;
 881
 882         ptr = strchr(buf, ' ');
 883         if (!ptr) {
 884                 ptr = strchr(buf, '\t');
 885                 if (!ptr)
 886                         return NULL;
 887         }
 888
 889         while (*ptr == ' ' || *ptr == '\t')
 890                 ptr++;
 891
 892         if (*ptr == '\0')
 893                 ptr = NULL;
 894
 895         return ptr;
 896 }
 897
 898 static int mdt0_is_lustre_snapshot(struct snapshot_instance *si)
 899 {
 900         char buf[MAX_BUF_SIZE];
 901         FILE *fp;
 902         int rc;
 903
 904         memset(buf, 0, sizeof(buf));
 905         snprintf(buf, sizeof(buf) - 1,
 906                  "%s %s 'zpool import -d %s %s > /dev/null 2>&1; "
 907                  "zfs get -H -o value lustre:magic %s/%s@%s'",
 908                  si->si_rsh, si->si_mdt0->st_host,
 909                  si->si_mdt0->st_dir ? si->si_mdt0->st_dir :
 910                         "/dev -d /tmp",
 911                  si->si_mdt0->st_pool, si->si_mdt0->st_pool,
 912                  si->si_mdt0->st_filesystem, si->si_ssname);
 913         fp = popen(buf, "r");
 914         if (!fp) {
 915                 SNAPSHOT_ADD_LOG(si, "Popen fail to check snapshot "
 916                                  "on mdt0: %s\n", strerror(errno));
 917                 return -errno;
 918         }
 919
 920         if (snapshot_fgets(fp, buf, strlen(SNAPSHOT_MAGIC) + 1) == NULL) {
 921                 rc = -EINVAL;
 922         } else if (strcmp(buf, SNAPSHOT_MAGIC) == 0) {
 923                 rc = 0;
 924         } else {
 925                 fprintf(stderr,
 926                         "The target %s is not Lustre snapshot "
 927                         "or it does not exists\n", si->si_ssname);
 928                 rc = -EPERM;
 929         }
 930
 931         pclose(fp);
 932         return rc;
 933 }
 934
 935 static int target_is_mounted(struct snapshot_instance *si,
 936                              struct snapshot_target *st, const char *ssname)
 937 {
 938         char buf[MAX_BUF_SIZE];
 939         char fullname[MAX_BUF_SIZE];
 940         FILE *fp;
 941         char *ptr;
 942         int rc = 0;
 943
 944         memset(buf, 0, sizeof(buf));
 945         snprintf(buf, sizeof(buf) - 1, "%s %s 'mount'",
 946                  si->si_rsh, st->st_host);
 947         fp = popen(buf, "r");
 948         if (!fp) {
 949                 SNAPSHOT_ADD_LOG(si, "Popen fail to check target mount: %s\n",
 950                                  strerror(errno));
 951                 return -errno;
 952         }
 953
 954         memset(fullname, 0, sizeof(fullname));
 955         if (ssname)
 956                 snprintf(fullname, sizeof(fullname) - 1, "%s/%s@%s on ",
 957                          st->st_pool, st->st_filesystem, ssname);
 958         else
 959                 snprintf(fullname, sizeof(fullname) - 1, "%s/%s on ",
 960                          st->st_pool, st->st_filesystem);
 961
 962         while (snapshot_fgets(fp, buf, sizeof(buf)) != NULL) {
 963                 ptr = strstr(buf, fullname);
 964                 if (!ptr)
 965                         continue;
 966
 967                 ptr += strlen(fullname) + 1; /* mount point */
 968                 if (ptr >= buf + strlen(buf))
 969                         continue;
 970
 971                 ptr = strstr(ptr, "type lustre");
 972                 if (ptr) {
 973                         rc = 1;
 974                         break;
 975                 }
 976         }
 977
 978         pclose(fp);
 979         return rc;
 980 }
 981
 982 static int snapshot_get_fsname(struct snapshot_instance *si,
 983                                char *fsname, int fslen)
 984 {
 985         char buf[MAX_BUF_SIZE];
 986         FILE *fp;
 987         int rc = 0;
 988
 989         memset(buf, 0, sizeof(buf));
 990         snprintf(buf, sizeof(buf) - 1,
 991                  "%s %s 'zpool import -d %s %s > /dev/null 2>&1; "
 992                  "zfs get -H -o value lustre:fsname %s/%s@%s'",
 993                  si->si_rsh, si->si_mdt0->st_host,
 994                  si->si_mdt0->st_dir ? si->si_mdt0->st_dir :
 995                         "/dev -d /tmp",
 996                  si->si_mdt0->st_pool, si->si_mdt0->st_pool,
 997                  si->si_mdt0->st_filesystem, si->si_ssname);
 998         fp = popen(buf, "r");
 999         if (!fp) {
1000                 SNAPSHOT_ADD_LOG(si, "Popen fail to get fsname: %s\n",
1001                                  strerror(errno));
1002                 return -errno;
1003         }
1004
1005         if (snapshot_fgets(fp, fsname, fslen) == NULL)
1006                 rc = -EINVAL;
1007
1008         pclose(fp);
1009         return rc;
1010 }
1011
1012 static int snapshot_get_mgsnode(struct snapshot_instance *si,
1013                                 char *node, int size)
1014 {
1015         char buf[MAX_BUF_SIZE];
1016         struct snapshot_target *st;
1017         FILE *fp;
1018         int rc = 0;
1019
1020         st = list_entry(si->si_osts_list.next, struct snapshot_target,
1021                         st_list);
1022         memset(buf, 0, sizeof(buf));
1023         snprintf(buf, sizeof(buf) - 1,
1024                  "%s %s 'zfs get -H -o value lustre:mgsnode %s/%s'",
1025                  si->si_rsh, st->st_host, st->st_pool, st->st_filesystem);
1026         fp = popen(buf, "r");
1027         if (!fp) {
1028                 SNAPSHOT_ADD_LOG(si, "Popen fail to get mgsnode: %s\n",
1029                                  strerror(errno));
1030                 return -errno;
1031         }
1032
1033         if (snapshot_fgets(fp, node, size) == NULL)
1034                 rc = -EINVAL;
1035
1036         pclose(fp);
1037         return rc;
1038 }
1039
1040 static int snapshot_exists_check(struct snapshot_instance *si,
1041                                  struct snapshot_target *st)
1042 {
1043         char buf[MAX_BUF_SIZE];
1044         FILE *fp;
1045         int rc = 0;
1046
1047         memset(buf, 0, sizeof(buf));
1048         snprintf(buf, sizeof(buf) - 1,
1049                  "%s %s 'zfs list %s/%s@%s 2>/dev/null'",
1050                  si->si_rsh, st->st_host, st->st_pool,
1051                  st->st_filesystem, si->si_ssname);
1052         fp = popen(buf, "r");
1053         if (!fp) {
1054                 SNAPSHOT_ADD_LOG(si, "Popen fail to create check: %s\n",
1055                                  strerror(errno));
1056                 return -errno;
1057         }
1058
1059         if (snapshot_fgets(fp, buf, sizeof(buf)) != NULL)
1060                 rc = -EEXIST;
1061
1062         pclose(fp);
1063         return rc;
1064 }
1065
1066 static int snapshot_general_check(struct snapshot_instance *si)
1067 {
1068         return mdt0_is_lustre_snapshot(si);
1069 }
1070
1071 static void snapshot_create_usage(void)
1072 {
1073         fprintf(stderr,
1074                 "Create snapshot for the given filesystem.\n"
1075                 "Usage:\n"
1076                 "snapshot_create [-b | --barrier [on | off]] "
1077                                 "[-c | --comment comment] "
1078                                 "<-F | --fsname fsname> "
1079                                 "[-h | --help] <-n | --name ssname> "
1080                                 "[-r | --rsh remote_shell]"
1081                                 "[-t | --timeout timeout]\n"
1082                 "Options:\n"
1083                 "-b: set write barrier before creating snapshot, "
1084                         "the default value is 'on'.\n"
1085                 "-c: describe what the snapshot is for, and so on.\n"
1086                 "-F: the filesystem name.\n"
1087                 "-h: for help information.\n"
1088                 "-n: the snapshot's name.\n"
1089                 "-r: the remote shell used for communication with remote "
1090                         "target, the default value is 'ssh'.\n"
1091                 "-t: the life cycle (seconds) for write barrier, "
1092                         "the default value is %d seconds.\n",
1093                 BARRIER_TIMEOUT_DEFAULT);
1094 }
1095
1096 static int snapshot_create_check(struct snapshot_instance *si)
1097 {
1098         int rc;
1099
1100         rc = snapshot_exists_check(si, si->si_mdt0);
1101         if (rc == -EEXIST)
1102                 fprintf(stderr, "The snapshot %s exists\n", si->si_ssname);
1103
1104         return rc;
1105 }
1106
1107 static int snapshot_inherit_prop(struct snapshot_instance *si,
1108                                  struct snapshot_target *st,
1109                                  char *cmd, int size)
1110 {
1111         char buf[MAX_BUF_SIZE];
1112         FILE *fp;
1113         int len = 0;
1114         int rc = 0;
1115
1116         memset(buf, 0, sizeof(buf));
1117         snprintf(buf, sizeof(buf) - 1,
1118                  "%s %s \"zpool import -d %s %s > /dev/null 2>&1; "
1119                  "zfs get all %s/%s | grep lustre: | grep local$ | "
1120                  "awk '{ \\$1=\\\"\\\"; \\$NF=\\\"\\\"; print \\$0 }' | "
1121                  "sed -e 's/^ //'\"",
1122                  si->si_rsh, st->st_host,
1123                  st->st_dir ? st->st_dir : "/dev -d /tmp",
1124                  st->st_pool, st->st_pool, st->st_filesystem);
1125         fp = popen(buf, "r");
1126         if (!fp) {
1127                 SNAPSHOT_ADD_LOG(si, "Popen fail to list one: %s\n",
1128                                  strerror(errno));
1129                 return -errno;
1130         }
1131
1132         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
1133                 char *ptr;
1134                 char *end;
1135
1136                 if (strncmp(buf, "lustre:fsname",
1137                             strlen("lustre:fsname")) == 0)
1138                         continue;
1139
1140                 if (strncmp(buf, "lustre:magic",
1141                             strlen("lustre:magic")) == 0)
1142                         continue;
1143
1144                 if (strncmp(buf, "lustre:ctime",
1145                             strlen("lustre:ctime")) == 0)
1146                         continue;
1147
1148                 if (strncmp(buf, "lustre:mtime",
1149                             strlen("lustre:mtime")) == 0)
1150                         continue;
1151
1152                 if (strncmp(buf, "lustre:comment",
1153                             strlen("lustre:comment")) == 0)
1154                         continue;
1155
1156                 if (strncmp(buf, "lustre:svname",
1157                             strlen("lustre:svname")) == 0)
1158                         continue;
1159
1160                 if (strncmp(buf, "lustre:mgsnode",
1161                             strlen("lustre:mgsnode")) == 0)
1162                         continue;
1163
1164                 ptr = strchr(buf, ' ');
1165                 if (!ptr) {
1166                         ptr = strchr(buf, '\t');
1167                         if (!ptr)
1168                                 continue;
1169                 }
1170
1171                 *ptr = '\0';
1172                 ptr++;
1173                 while (*ptr == ' ' || *ptr == '\t')
1174                         ptr++;
1175
1176                 if (*ptr == '\0')
1177                         continue;
1178
1179                 end = strchr(ptr, ' ');
1180                 if (!end)
1181                         end = strchr(buf, '\t');
1182                 if (end)
1183                         *end = '\0';
1184
1185                 rc = snprintf(cmd + len, size - len - 1,
1186                               "-o %s=\"%s\" ", buf, ptr);
1187                 if (rc <= 0)
1188                         return -EOVERFLOW;
1189
1190                 len += rc;
1191         }
1192
1193         pclose(fp);
1194         return len;
1195 }
1196
1197 static int __snapshot_create(struct snapshot_instance *si,
1198                              struct list_head *head, const char *fsname,
1199                              const char *mgsnode, __u64 xtime)
1200 {
1201         struct snapshot_target *st;
1202         pid_t pid;
1203         int rc;
1204
1205         list_for_each_entry(st, head, st_list) {
1206                 st->st_status = 0;
1207                 st->st_ignored = 0;
1208                 st->st_pid = 0;
1209
1210                 pid = fork();
1211                 if (pid < 0) {
1212                         SNAPSHOT_ADD_LOG(si, "Can't fork for create snapshot "
1213                                          "(%s@%s <%s>) on the target "
1214                                          "(%s:%x:%d): %s\n",
1215                                          fsname, si->si_ssname,
1216                                          si->si_comment, st->st_host,
1217                                          st->st_role, st->st_index,
1218                                          strerror(errno));
1219                         return pid;
1220                 }
1221
1222                 /* child */
1223                 if (pid == 0) {
1224                         char cmd[MAX_BUF_SIZE];
1225                         int len;
1226
1227                         memset(cmd, 0, sizeof(cmd));
1228                         len = snprintf(cmd, sizeof(cmd) - 1,
1229                                        "%s %s 'zfs snapshot "
1230                                        "-o lustre:fsname=%s "
1231                                        "-o lustre:magic=%s "
1232                                        "-o lustre:ctime=%llu "
1233                                        "-o lustre:mtime=%llu ",
1234                                        si->si_rsh, st->st_host, fsname,
1235                                        SNAPSHOT_MAGIC, xtime, xtime);
1236                         if (len <= 0)
1237                                 exit(-EOVERFLOW);
1238
1239                         if (si->si_comment) {
1240                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1241                                               "-o lustre:comment=\"%s\" ",
1242                                               si->si_comment);
1243                                 if (rc <= 0)
1244                                         exit(-EOVERFLOW);
1245
1246                                 len += rc;
1247                         }
1248
1249                         /* Make the inherited properties as local ones,
1250                          * then even if others changed (or removed) the
1251                          * property of the parent dataset, the snapshot
1252                          * will not be affected. */
1253                         rc = snapshot_inherit_prop(si, st, cmd + len,
1254                                                    MAX_BUF_SIZE - len - 1);
1255                         if (rc < 0) {
1256                                 SNAPSHOT_ADD_LOG(si, "Can't filter property on "
1257                                                  "target (%s:%x:%d): rc = %d\n",
1258                                                  st->st_host, st->st_role,
1259                                                  st->st_index, rc);
1260
1261                                 exit(rc);
1262                         }
1263
1264                         len += rc;
1265                         if (st->st_role & SR_OST)
1266                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1267                                               "-o lustre:svname=%s-OST%04x "
1268                                               "-o lustre:mgsnode=%s %s/%s@%s'",
1269                                               fsname, st->st_index, mgsnode,
1270                                               st->st_pool, st->st_filesystem,
1271                                               si->si_ssname);
1272                         else if (!(st->st_role & SR_MGS) ||
1273                                 /* MGS is on MDT0 */
1274                                  si->si_mdt0 == si->si_mgs)
1275                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1276                                               "-o lustre:svname=%s-MDT%04x "
1277                                               "-o lustre:mgsnode=%s %s/%s@%s'",
1278                                               fsname, st->st_index, mgsnode,
1279                                               st->st_pool, st->st_filesystem,
1280                                               si->si_ssname);
1281                         else
1282                                 /* separated MGS */
1283                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1284                                               "%s/%s@%s'", st->st_pool,
1285                                               st->st_filesystem, si->si_ssname);
1286                         if (rc <= 0)
1287                                 exit(-EOVERFLOW);
1288
1289                         rc = snapshot_exec(cmd);
1290                         if (rc)
1291                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1292                                                  "target (%s:%x:%d): rc = %d\n",
1293                                                  cmd, st->st_host, st->st_role,
1294                                                  st->st_index, rc);
1295
1296                         exit(rc);
1297                 } /* end of child */
1298
1299                 /* parent continue to run more snapshot commands in parallel. */
1300                 st->st_pid = pid;
1301         }
1302
1303         return 0;
1304 }
1305
1306 static int __snapshot_destroy(struct snapshot_instance *si,
1307                               struct list_head *head);
1308
1309 static int snapshot_create(struct snapshot_instance *si)
1310 {
1311         char *__argv[3];
1312         char buf[MAX_BUF_SIZE];
1313         struct timeval tv;
1314         char new_fsname[9];
1315         int rc = 0;
1316         int rc1 = 0;
1317         int rc2 = 0;
1318
1319         rc = snapshot_create_check(si);
1320         if (rc)
1321                 return rc;
1322
1323         rc = gettimeofday(&tv, NULL);
1324         if (rc)
1325                 return rc;
1326
1327         srandom(tv.tv_usec);
1328         snprintf(new_fsname, sizeof(new_fsname), "%08x", (__u32)random());
1329
1330         rc = snapshot_get_mgsnode(si, buf, sizeof(buf));
1331         if (rc)
1332                 return rc;
1333
1334         __argv[1] = si->si_fsname;
1335         /* 1. Get barrier */
1336         if (si->si_barrier) {
1337                 char tbuf[8];
1338
1339                 memset(tbuf, 0, sizeof(tbuf));
1340                 snprintf(tbuf, 7, "%u", si->si_timeout);
1341                 __argv[0] = "barrier_freeze";
1342                 __argv[2] = tbuf;
1343                 rc = jt_barrier_freeze(3, __argv);
1344                 if (rc) {
1345                         SNAPSHOT_ADD_LOG(si, "Can't set barrier within %u "
1346                                          "seconds on %s: rc = %d\n",
1347                                          si->si_timeout, si->si_fsname, rc);
1348
1349                         return rc;
1350                 }
1351         }
1352
1353         /* 2. Fork config llog on MGS */
1354         __argv[0] = "fork_lcfg";
1355         __argv[2] = new_fsname;
1356         rc = jt_lcfg_fork(3, __argv);
1357         if (rc) {
1358                 SNAPSHOT_ADD_LOG(si, "Can't fork config log for create "
1359                                  "snapshot %s from %s to %s: rc = %d\n",
1360                                  si->si_ssname, si->si_fsname, new_fsname, rc);
1361                 goto out;
1362         }
1363
1364         /* 3.1 Create snapshot on every MDT */
1365         rc = __snapshot_create(si, &si->si_mdts_list, new_fsname, buf,
1366                                tv.tv_sec);
1367         if (!rc)
1368                 /* 3.2 Create snapshot on every OST */
1369                 rc = __snapshot_create(si, &si->si_osts_list, new_fsname, buf,
1370                                        tv.tv_sec);
1371
1372         /* 4. Wait for all children, even though part of them maybe failed */
1373         snapshot_wait(si, &rc1);
1374
1375 out:
1376         /* 5. Put barrier */
1377         if (si->si_barrier) {
1378                 if (!rc && !rc1) {
1379                         struct barrier_ctl bc;
1380
1381                         rc = __jt_barrier_stat(__argv[1], &bc);
1382                         if (rc) {
1383                                 SNAPSHOT_ADD_LOG(si, "Can't get barrier status "
1384                                                  "on %s: rc = %d\n",
1385                                                  si->si_fsname, rc);
1386                         } else if (bc.bc_status != BS_FROZEN ||
1387                                    bc.bc_timeout <= 0) {
1388                                 SNAPSHOT_ADD_LOG(si, "The barrier expired "
1389                                                  "on %s\n", si->si_fsname);
1390                                 rc = -ETIMEDOUT;
1391                         }
1392                 }
1393
1394                 __argv[0] = "barrier_thaw";
1395                 rc2 = jt_barrier_thaw(2, __argv);
1396                 if (rc2)
1397                         SNAPSHOT_ADD_LOG(si, "Can't release barrier on %s: "
1398                                          "rc = %d\n", si->si_fsname, rc2);
1399         }
1400
1401         /* cleanup */
1402         if (rc || rc1) {
1403                 si->si_force = true;
1404                 __snapshot_destroy(si, &si->si_osts_list);
1405                 __snapshot_destroy(si, &si->si_mdts_list);
1406                 snapshot_wait(si, &rc2);
1407
1408                 __argv[0] = "erase_lcfg";
1409                 __argv[1] = new_fsname;
1410                 __argv[2] = "-q";
1411                 jt_lcfg_erase(3, __argv);
1412         }
1413
1414         return rc ? rc : (rc1 ? rc1 : rc2);
1415 }
1416
1417 int jt_snapshot_create(int argc, char **argv)
1418 {
1419         struct snapshot_instance *si;
1420         struct option long_opts[] = {
1421         { .val = 'b',   .name = "barrier",      .has_arg = optional_argument },
1422         { .val = 'c',   .name = "comment",      .has_arg = required_argument },
1423         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1424         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1425         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1426         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1427         { .val = 't',   .name = "timeout",      .has_arg = required_argument },
1428         { .name = NULL } };
1429         int rc = 0;
1430
1431         si = snapshot_init(argc, argv, long_opts, "b::c:F:hn:r:t:",
1432                            snapshot_create_usage, LOCK_EX, &rc);
1433         if (!si)
1434                 return rc;
1435
1436         if (!si->si_ssname) {
1437                 fprintf(stderr,
1438                         "Miss the snapshot name to be created\n");
1439                 snapshot_create_usage();
1440                 snapshot_fini(si);
1441                 return -EINVAL;
1442         }
1443
1444         rc = snapshot_create(si);
1445         if (rc) {
1446                 fprintf(stderr,
1447                         "Can't create the snapshot %s\n", si->si_ssname);
1448                 SNAPSHOT_ADD_LOG(si, "Can't create snapshot %s with "
1449                                  "comment <%s> barrier <%s>, timeout "
1450                                  "<%d>: %d\n",
1451                                  si->si_ssname, si->si_comment,
1452                                  si->si_barrier ? "enable" : "disable",
1453                                  si->si_barrier ? si->si_timeout : -1, rc);
1454         } else {
1455                 SNAPSHOT_ADD_LOG(si, "Create snapshot %s successfully "
1456                                  "with comment <%s>, barrier <%s>, "
1457                                  "timeout <%d>\n",
1458                                  si->si_ssname, si->si_comment,
1459                                  si->si_barrier ? "enable" : "disable",
1460                                  si->si_barrier ? si->si_timeout : -1);
1461         }
1462
1463         snapshot_fini(si);
1464         return rc;
1465 }
1466
1467 static void snapshot_destroy_usage(void)
1468 {
1469         fprintf(stderr,
1470                 "Destroy the specified snapshot.\n"
1471                 "Usage:\n"
1472                 "snapshot_destroy [-f | --force] "
1473                                  "<-F | --fsname fsname> [-h | --help] "
1474                                  "<-n | --name ssname> "
1475                                  "[-r | --rsh remote_shell]\n"
1476                 "Options:\n"
1477                 "-f: destroy the snapshot by force.\n"
1478                 "-F: the filesystem name.\n"
1479                 "-h: for help information.\n"
1480                 "-n: the snapshot's name.\n"
1481                 "-r: the remote shell used for communication with remote "
1482                         "target, the default value is 'ssh'.\n");
1483 }
1484
1485 static int snapshot_destroy_check(struct snapshot_instance *si)
1486 {
1487         struct list_head *head = &si->si_osts_list;
1488         struct snapshot_target *st;
1489         pid_t pid;
1490         int rc = 0;
1491
1492 again1:
1493         list_for_each_entry(st, head, st_list) {
1494                 st->st_status = 0;
1495                 st->st_ignored = 0;
1496                 st->st_pid = 0;
1497
1498                 pid = fork();
1499                 if (pid < 0) {
1500                         SNAPSHOT_ADD_LOG(si, "Can't fork for check snapshot "
1501                                          "%s on the target (%s:%x:%d): %s\n",
1502                                          si->si_ssname, st->st_host,
1503                                          st->st_role, st->st_index,
1504                                          strerror(errno));
1505                         return pid;
1506                 }
1507
1508                 /* child */
1509                 if (pid == 0) {
1510                         rc = snapshot_exists_check(si, st);
1511                         if (!rc)
1512                                 /* The snapshot piece does not exist */
1513                                 exit(-ESRCH);
1514
1515                         exit(rc == -EEXIST ? 0: rc);
1516                 } /* end of child */
1517
1518                 /* parent continue to run more snapshot commands in parallel. */
1519                 st->st_pid = pid;
1520         }
1521
1522         if (head == &si->si_osts_list) {
1523                 head = &si->si_mdts_list;
1524                 goto again1;
1525         }
1526
1527         snapshot_wait(si, &rc);
1528         if (rc)
1529                 return rc;
1530
1531         head = &si->si_osts_list;
1532
1533 again2:
1534         list_for_each_entry(st, head, st_list) {
1535                 if (st->st_ignored && !si->si_force) {
1536                         char name[8];
1537
1538                         snapshot_role2name(name, st->st_role, st->st_index);
1539                         fprintf(stderr,
1540                                 "Miss snapshot piece on the %s. Use '-f' "
1541                                 "option if want to destroy it by force.\n",
1542                                 name);
1543
1544                         return -ENOENT;
1545                 }
1546         }
1547
1548         if (head == &si->si_osts_list) {
1549                 head = &si->si_mdts_list;
1550                 goto again2;
1551         }
1552
1553         if (!si->si_force)
1554                 rc = snapshot_general_check(si);
1555
1556         return rc;
1557 }
1558
1559 static int __snapshot_destroy(struct snapshot_instance *si,
1560                               struct list_head *head)
1561 {
1562         struct snapshot_target *st;
1563         pid_t pid;
1564         int rc;
1565
1566         list_for_each_entry(st, head, st_list) {
1567                 if (st->st_ignored)
1568                         continue;
1569
1570                 st->st_status = 0;
1571                 st->st_pid = 0;
1572
1573                 pid = fork();
1574                 if (pid < 0) {
1575                         SNAPSHOT_ADD_LOG(si, "Can't fork for destroy snapshot "
1576                                          "%s on the target (%s:%x:%d): %s\n",
1577                                          si->si_ssname, st->st_host,
1578                                          st->st_role, st->st_index,
1579                                          strerror(errno));
1580                         return pid;
1581                 }
1582
1583                 /* child */
1584                 if (pid == 0) {
1585                         char cmd[MAX_BUF_SIZE];
1586
1587                         memset(cmd, 0, sizeof(cmd));
1588                         if (si->si_force)
1589                                 snprintf(cmd, sizeof(cmd) - 1,
1590                                          "%s %s 'umount -f %s/%s@%s "
1591                                          "> /dev/null 2>&1; "
1592                                          "zfs destroy -f %s/%s@%s'",
1593                                          si->si_rsh, st->st_host, st->st_pool,
1594                                          st->st_filesystem, si->si_ssname,
1595                                          st->st_pool, st->st_filesystem,
1596                                          si->si_ssname);
1597                         else
1598                                 snprintf(cmd, sizeof(cmd) - 1,
1599                                          "%s %s 'zfs destroy %s/%s@%s'",
1600                                          si->si_rsh, st->st_host, st->st_pool,
1601                                          st->st_filesystem, si->si_ssname);
1602                         rc = snapshot_exec(cmd);
1603                         if (rc)
1604                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1605                                                  "target (%s:%x:%d): rc = %d\n",
1606                                                  cmd, st->st_host, st->st_role,
1607                                                  st->st_index, rc);
1608
1609                         exit(rc);
1610                 } /* end of child */
1611
1612                 /* parent continue to run more snapshot commands in parallel. */
1613                 st->st_pid = pid;
1614         }
1615
1616         return 0;
1617 }
1618
1619 static int snapshot_destroy(struct snapshot_instance *si)
1620 {
1621         char fsname[9];
1622         int rc = 0;
1623         int rc1 = 0;
1624         int rc2 = 0;
1625         int rc3 = 0;
1626
1627         rc = snapshot_destroy_check(si);
1628         if (rc)
1629                 return rc;
1630
1631         rc = snapshot_get_fsname(si, fsname, sizeof(fsname));
1632         if (rc)
1633                 return rc;
1634
1635         /* 1.1 Destroy snapshot on every OST */
1636         rc = __snapshot_destroy(si, &si->si_osts_list);
1637         if (!si->si_force) {
1638                 if (rc)
1639                         return rc;
1640
1641                 __snapshot_wait(si, &si->si_osts_list, &rc);
1642                 if (rc)
1643                         return rc;
1644         }
1645
1646         /* 1.2 Destroy snapshot on every MDT */
1647         rc1 = __snapshot_destroy(si, &si->si_mdts_list);
1648
1649         /* 2 Wait for all children, even though part of them maybe failed */
1650         snapshot_wait(si, &rc2);
1651         if (rc2 == -ENOENT && si->si_force)
1652                 rc2 = 0;
1653
1654         /* 3. Erase config llog from MGS */
1655         if ((!rc && !rc1 && !rc2) || si->si_force) {
1656                 char *__argv[3];
1657
1658                 __argv[0] = "erase_lcfg";
1659                 __argv[1] = fsname;
1660                 __argv[2] = "-q";
1661                 rc3 = jt_lcfg_erase(3, __argv);
1662                 if (rc3 && errno == ENOENT)
1663                         rc3 = 0;
1664                 if (rc3)
1665                         SNAPSHOT_ADD_LOG(si, "Can't erase config for destroy "
1666                                          "snapshot %s, fsname %s: rc = %d\n",
1667                                          si->si_ssname, fsname, rc3);
1668         }
1669
1670         return rc ? rc : (rc1 ? rc1 : (rc2 ? rc2 : rc3));
1671 }
1672
1673 int jt_snapshot_destroy(int argc, char **argv)
1674 {
1675         struct snapshot_instance *si;
1676         struct option long_opts[] = {
1677         { .val = 'f',   .name = "force",        .has_arg = no_argument },
1678         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1679         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1680         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1681         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1682         { .name = NULL } };
1683         int rc = 0;
1684
1685         si = snapshot_init(argc, argv, long_opts, "fF:hn:r:",
1686                            snapshot_destroy_usage, LOCK_EX, &rc);
1687         if (!si)
1688                 return rc;
1689
1690         if (!si->si_ssname) {
1691                 fprintf(stderr,
1692                         "Miss the snapshot name to be destroyed\n");
1693                 snapshot_destroy_usage();
1694                 snapshot_fini(si);
1695                 return -EINVAL;
1696         }
1697
1698         rc = snapshot_destroy(si);
1699         if (rc) {
1700                 fprintf(stderr,
1701                         "Can't destroy the snapshot %s\n", si->si_ssname);
1702                 SNAPSHOT_ADD_LOG(si, "Can't destroy snapshot %s with "
1703                                  "force <%s>: %d\n", si->si_ssname,
1704                                  si->si_force ? "enable" : "disable", rc);
1705         } else {
1706                 SNAPSHOT_ADD_LOG(si, "Destroy snapshot %s successfully "
1707                                  "with force <%s>\n", si->si_ssname,
1708                                  si->si_force ? "enable" : "disable");
1709         }
1710
1711         snapshot_fini(si);
1712         return rc;
1713 }
1714
1715 static void snapshot_modify_usage(void)
1716 {
1717         fprintf(stderr,
1718                 "Change the specified snapshot's name and/or comment.\n"
1719                 "Usage:\n"
1720                 "snapshot_modify [-c | --comment comment] "
1721                                 "<-F | --fsname fsname> [-h | --help] "
1722                                 "<-n | --name ssname> [-N | --new new_ssname] "
1723                                 "[-r | --rsh remote_shell]\n"
1724                 "Options:\n"
1725                 "-c: update the snapshot's comment.\n"
1726                 "-F: the filesystem name.\n"
1727                 "-h: for help information.\n"
1728                 "-n: the snapshot's name.\n"
1729                 "-N: rename the snapshot's name as the new_ssname.\n"
1730                 "-r: the remote shell used for communication with remote "
1731                         "target, the default value is 'ssh'.\n");
1732 }
1733
1734 static int snapshot_modify_check(struct snapshot_instance *si)
1735 {
1736         int rc;
1737
1738         if (si->si_new_ssname &&
1739             strcmp(si->si_ssname, si->si_new_ssname) == 0) {
1740                 fprintf(stderr, "The new snapshot's name is the same as "
1741                         "the old one %s %s.\n",
1742                         si->si_ssname, si->si_new_ssname);
1743                 return -EPERM;
1744         }
1745
1746         if (!si->si_new_ssname && !si->si_comment) {
1747                 fprintf(stderr, "Miss options, nothing to be changed.\n");
1748                 return -EINVAL;
1749         }
1750
1751         rc = mdt0_is_lustre_snapshot(si);
1752         if (!rc && si->si_new_ssname) {
1753                 rc = target_is_mounted(si, si->si_mdt0, si->si_ssname);
1754                 if (rc > 0) {
1755                         fprintf(stderr,
1756                                 "snapshot %s is mounted, can't be renamed.\n",
1757                                 si->si_ssname);
1758                         rc = -EBUSY;
1759                 }
1760         }
1761
1762         return rc;
1763 }
1764
1765 static int __snapshot_modify(struct snapshot_instance *si,
1766                              struct list_head *head, __u64 xtime)
1767 {
1768         struct snapshot_target *st;
1769         pid_t pid;
1770         int rc;
1771
1772         list_for_each_entry(st, head, st_list) {
1773                 st->st_status = 0;
1774                 st->st_ignored = 0;
1775                 st->st_pid = 0;
1776
1777                 pid = fork();
1778                 if (pid < 0) {
1779                         SNAPSHOT_ADD_LOG(si, "Can't fork for modify snapshot "
1780                                          "(%s|%s, <%s>) on the target "
1781                                          "(%s:%x:%d): %s\n",
1782                                          si->si_ssname, si->si_new_ssname,
1783                                          si->si_comment, st->st_host,
1784                                          st->st_role, st->st_index,
1785                                          strerror(errno));
1786                         return pid;
1787                 }
1788
1789                 /* child */
1790                 if (pid == 0) {
1791                         char cmd[MAX_BUF_SIZE];
1792
1793                         memset(cmd, 0, sizeof(cmd));
1794                         if (si->si_new_ssname && si->si_comment)
1795                                 snprintf(cmd, sizeof(cmd) - 1,
1796                                          "%s %s 'zpool import -d %s %s > "
1797                                          "/dev/null 2>&1; "
1798                                          "zfs rename %s/%s@%s %s/%s@%s && "
1799                                          "zfs set lustre:comment=\"%s\" "
1800                                          "%s/%s@%s && "
1801                                          "zfs set lustre:mtime=%llu %s/%s@%s'",
1802                                          si->si_rsh, st->st_host,
1803                                          st->st_dir ? st->st_dir :
1804                                                 "/dev -d /tmp",
1805                                          st->st_pool, st->st_pool,
1806                                          st->st_filesystem, si->si_ssname,
1807                                          st->st_pool, st->st_filesystem,
1808                                          si->si_new_ssname, si->si_comment,
1809                                          st->st_pool, st->st_filesystem,
1810                                          si->si_new_ssname, xtime,
1811                                          st->st_pool, st->st_filesystem,
1812                                          si->si_new_ssname);
1813                         else if (si->si_new_ssname)
1814                                 snprintf(cmd, sizeof(cmd) - 1,
1815                                          "%s %s 'zpool import -d %s %s > "
1816                                          "/dev/null 2>&1; "
1817                                          "zfs rename %s/%s@%s %s/%s@%s && "
1818                                          "zfs set lustre:mtime=%llu %s/%s@%s'",
1819                                          si->si_rsh, st->st_host,
1820                                          st->st_dir ? st->st_dir :
1821                                                 "/dev -d /tmp",
1822                                          st->st_pool, st->st_pool,
1823                                          st->st_filesystem, si->si_ssname,
1824                                          st->st_pool, st->st_filesystem,
1825                                          si->si_new_ssname, xtime, st->st_pool,
1826                                          st->st_filesystem, si->si_new_ssname);
1827                         else if (si->si_comment)
1828                                 snprintf(cmd, sizeof(cmd) - 1,
1829                                          "%s %s 'zpool import -d %s %s > "
1830                                          "/dev/null 2>&1; zfs set "
1831                                          "lustre:comment=\"%s\" %s/%s@%s && "
1832                                          "zfs set lustre:mtime=%llu %s/%s@%s'",
1833                                          si->si_rsh, st->st_host,
1834                                          st->st_dir ? st->st_dir :
1835                                                 "/dev -d /tmp",
1836                                          st->st_pool, si->si_comment,
1837                                          st->st_pool, st->st_filesystem,
1838                                          si->si_ssname, xtime, st->st_pool,
1839                                          st->st_filesystem, si->si_ssname);
1840                         else
1841                                 exit(-EINVAL);
1842
1843                         rc = snapshot_exec(cmd);
1844                         if (rc)
1845                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1846                                                  "target (%s:%x:%d): rc = %d\n",
1847                                                  cmd, st->st_host, st->st_role,
1848                                                  st->st_index, rc);
1849
1850                         exit(rc);
1851                 } /* end of child */
1852
1853                 /* parent continue to run more snapshot commands in parallel. */
1854                 st->st_pid = pid;
1855         }
1856
1857         return 0;
1858 }
1859
1860 static int snapshot_modify(struct snapshot_instance *si)
1861 {
1862         time_t tt;
1863         int rc = 0;
1864         int rc1 = 0;
1865
1866         rc = snapshot_modify_check(si);
1867         if (rc)
1868                 return rc;
1869
1870         time(&tt);
1871
1872         /* Modify snapshot on every MDT */
1873         rc = __snapshot_modify(si, &si->si_mdts_list, (__u64)tt);
1874         if (!rc)
1875                 /* Modify snapshot on every OST */
1876                 rc = __snapshot_modify(si, &si->si_osts_list, (__u64)tt);
1877
1878         /* Wait for all children, even though part of them maybe failed */
1879         snapshot_wait(si, &rc1);
1880
1881         return rc ? rc : rc1;
1882 }
1883
1884 int jt_snapshot_modify(int argc, char **argv)
1885 {
1886         struct snapshot_instance *si;
1887         struct option long_opts[] = {
1888         { .val = 'c',   .name = "comment",      .has_arg = required_argument },
1889         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1890         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1891         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1892         { .val = 'N',   .name = "new",          .has_arg = required_argument },
1893         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1894         { .name = NULL } };
1895         int rc = 0;
1896
1897         si = snapshot_init(argc, argv, long_opts, "c:F:hn:N:r:",
1898                            snapshot_modify_usage, LOCK_EX, &rc);
1899         if (!si)
1900                 return rc;
1901
1902         if (!si->si_ssname) {
1903                 fprintf(stderr,
1904                         "Miss the snapshot name to be modified\n");
1905                 snapshot_modify_usage();
1906                 snapshot_fini(si);
1907                 return -EINVAL;
1908         }
1909
1910         rc = snapshot_modify(si);
1911         if (rc) {
1912                 fprintf(stderr,
1913                         "Can't modify the snapshot %s\n", si->si_ssname);
1914                 SNAPSHOT_ADD_LOG(si, "Can't modify snapshot %s with "
1915                                  "name <%s>, comment <%s>: %d\n",
1916                                  si->si_ssname, si->si_new_ssname,
1917                                  si->si_comment, rc);
1918         } else {
1919                 SNAPSHOT_ADD_LOG(si, "Modify snapshot %s successfully "
1920                                  "with name <%s>, comment <%s>\n",
1921                                  si->si_ssname, si->si_new_ssname,
1922                                  si->si_comment);
1923         }
1924
1925         snapshot_fini(si);
1926         return rc;
1927 }
1928
1929 static void snapshot_list_usage(void)
1930 {
1931         fprintf(stderr,
1932                 "List the specified snapshot or all snapshots.\n"
1933                 "Usage:\n"
1934                 "snapshot_list [-d | --detail] "
1935                               "<-F | --fsname fsname> [-h | --help] "
1936                               "[-n | --name ssname] [-r | --rsh remote_shell]\n"
1937                 "Options:\n"
1938                 "-d: list every piece for the specified snapshot.\n"
1939                 "-F: the filesystem name.\n"
1940                 "-h: for help information.\n"
1941                 "-n: the snapshot's name.\n"
1942                 "-r: the remote shell used for communication with remote "
1943                         "target, the default value is 'ssh'.\n");
1944 }
1945
1946 static int snapshot_list_one(struct snapshot_instance *si,
1947                              struct snapshot_target *st)
1948 {
1949         char buf[MAX_BUF_SIZE];
1950         FILE *fp;
1951         int rc;
1952
1953         memset(buf, 0, sizeof(buf));
1954         snprintf(buf, sizeof(buf) - 1,
1955                  "%s %s \"zpool import -d %s %s > /dev/null 2>&1; "
1956                  "zfs get all %s/%s@%s | grep lustre: | grep local$ | "
1957                  "awk '{ \\$1=\\\"\\\"; \\$NF=\\\"\\\"; print \\$0 }' | "
1958                  "sed -e 's/^ //'\"",
1959                  si->si_rsh, st->st_host,
1960                  st->st_dir ? st->st_dir : "/dev -d /tmp",
1961                  st->st_pool, st->st_pool, st->st_filesystem, si->si_ssname);
1962         fp = popen(buf, "r");
1963         if (!fp) {
1964                 SNAPSHOT_ADD_LOG(si, "Popen fail to list one: %s\n",
1965                                  strerror(errno));
1966                 return -errno;
1967         }
1968
1969         if (si->si_detail) {
1970                 char name[8];
1971
1972                 snapshot_role2name(name, st->st_role, st->st_index);
1973                 printf("\nsnapshot_role: %s\n", name);
1974         }
1975
1976         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
1977                 __u64 xtime;
1978                 char *ptr;
1979
1980                 if (strncmp(buf, "lustre:fsname",
1981                             strlen("lustre:fsname")) == 0) {
1982                         ptr = snapshot_first_skip_blank(buf);
1983                         if (ptr)
1984                                 printf("snapshot_fsname: %s\n", ptr);
1985                         continue;
1986                 }
1987
1988                 if (strncmp(buf, "lustre:comment",
1989                             strlen("lustre:comment")) == 0) {
1990                         ptr = snapshot_first_skip_blank(buf);
1991                         if (ptr)
1992                                 printf("comment: %s\n", ptr);
1993                         continue;
1994                 }
1995
1996                 if (strncmp(buf, "lustre:ctime",
1997                             strlen("lustre:ctime")) == 0) {
1998                         ptr = snapshot_first_skip_blank(buf);
1999                         if (ptr) {
2000                                 sscanf(ptr, "%llu", &xtime);
2001                                 printf("create_time: %s",
2002                                        ctime((time_t *)&xtime));
2003                         }
2004                         continue;
2005                 }
2006
2007                 if (strncmp(buf, "lustre:mtime",
2008                             strlen("lustre:mtime")) == 0) {
2009                         ptr = snapshot_first_skip_blank(buf);
2010                         if (ptr) {
2011                                 sscanf(ptr, "%llu", &xtime);
2012                                 printf("modify_time: %s",
2013                                        ctime((time_t *)&xtime));
2014                         }
2015                         continue;
2016                 }
2017         }
2018
2019         pclose(fp);
2020         rc = target_is_mounted(si, st, si->si_ssname);
2021         if (rc < 0)
2022                 printf("status: unknown\n");
2023         else if (!rc)
2024                 printf("status: not mount\n");
2025         else
2026                 printf("status: mounted\n");
2027
2028         return rc;
2029 }
2030
2031 static int __snapshot_list(struct snapshot_instance *si,
2032                            struct list_head *head)
2033 {
2034         struct snapshot_target *st;
2035         int rc = 0;
2036
2037         list_for_each_entry(st, head, st_list) {
2038                 int rc1;
2039
2040                 rc1 = snapshot_list_one(si, st);
2041                 if (rc1 < 0 || rc >= 0)
2042                         rc = rc1;
2043         }
2044
2045         return rc;
2046 }
2047
2048 static int snapshot_list(struct snapshot_instance *si)
2049 {
2050         int rc = 0;
2051
2052         rc = snapshot_general_check(si);
2053         if (rc)
2054                 return rc;
2055
2056         printf("\nfilesystem_name: %s\nsnapshot_name: %s\n",
2057                si->si_fsname, si->si_ssname);
2058
2059         if (!si->si_detail) {
2060                 rc = snapshot_list_one(si, si->si_mdt0);
2061         } else {
2062                 int rc1;
2063
2064                 rc = __snapshot_list(si, &si->si_mdts_list);
2065                 rc1 = __snapshot_list(si, &si->si_osts_list);
2066                 if (rc >= 0)
2067                         rc = rc1;
2068         }
2069
2070         return rc < 0 ? rc : 0;
2071 }
2072
2073 static int snapshot_list_all(struct snapshot_instance *si)
2074 {
2075         struct list_sub_item {
2076                 struct list_head lsi_list;
2077                 char lsi_ssname[0];
2078         };
2079
2080         struct list_head list_sub_items;
2081         struct list_sub_item *lsi;
2082         char buf[MAX_BUF_SIZE];
2083         FILE *fp;
2084         int rc = 0;
2085
2086         INIT_LIST_HEAD(&list_sub_items);
2087         memset(buf, 0, sizeof(buf));
2088         snprintf(buf, sizeof(buf) - 1,
2089                  "%s %s \"zfs get -H -r lustre:magic %s/%s | "
2090                  "grep %s | awk '{ print \\$1 }' | cut -d@ -f2\"",
2091                  si->si_rsh, si->si_mdt0->st_host, si->si_mdt0->st_pool,
2092                  si->si_mdt0->st_filesystem, SNAPSHOT_MAGIC);
2093         fp = popen(buf, "r");
2094         if (!fp) {
2095                 SNAPSHOT_ADD_LOG(si, "Popen fail to list ssnames: %s\n",
2096                                  strerror(errno));
2097                 return -errno;
2098         }
2099
2100         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
2101                 int len = strlen(buf);
2102
2103                 lsi = malloc(len + 1 + sizeof(struct list_sub_item));
2104                 if (!lsi) {
2105                         SNAPSHOT_ADD_LOG(si, "NOT enough memory\n");
2106                         rc = -ENOMEM;
2107                         break;
2108                 }
2109
2110                 strncpy(lsi->lsi_ssname, buf, len);
2111                 lsi->lsi_ssname[len] = '\0';
2112                 list_add(&lsi->lsi_list, &list_sub_items);
2113         }
2114
2115         pclose(fp);
2116         while (!list_empty(&list_sub_items)) {
2117                 lsi = list_entry(list_sub_items.next,
2118                                  struct list_sub_item, lsi_list);
2119                 list_del(&lsi->lsi_list);
2120                 if (!rc) {
2121                         si->si_ssname = lsi->lsi_ssname;
2122                         rc = snapshot_list(si);
2123                         si->si_ssname = NULL;
2124                 }
2125
2126                 free(lsi);
2127         }
2128
2129         return rc;
2130 }
2131
2132 int jt_snapshot_list(int argc, char **argv)
2133 {
2134         struct snapshot_instance *si;
2135         struct option long_opts[] = {
2136         { .val = 'd',   .name = "detail",       .has_arg = no_argument },
2137         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2138         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2139         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2140         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2141         { .name = NULL } };
2142         int rc = 0;
2143
2144         si = snapshot_init(argc, argv, long_opts, "dF:hn:r:",
2145                            snapshot_list_usage, LOCK_SH, &rc);
2146         if (!si)
2147                 return rc;
2148
2149         if (si->si_ssname)
2150                 rc = snapshot_list(si);
2151         else
2152                 rc = snapshot_list_all(si);
2153
2154         if (rc) {
2155                 fprintf(stderr,
2156                         "Can't list the snapshot %s\n", si->si_ssname);
2157                 SNAPSHOT_ADD_LOG(si, "Can't list snapshot %s with detail "
2158                                  "<%s>: %d\n", si->si_ssname,
2159                                  si->si_detail ? "yes" : "no", rc);
2160         }
2161
2162         snapshot_fini(si);
2163         return rc;
2164 }
2165
2166 static void snapshot_mount_usage(void)
2167 {
2168         fprintf(stderr,
2169                 "Mount the specified snapshot.\n"
2170                 "Usage:\n"
2171                 "snapshot_mount <-F | --fsname fsname> [-h | --help] "
2172                                "<-n | --name ssname> "
2173                                "[-r | --rsh remote_shell]\n"
2174                 "Options:\n"
2175                 "-F: the filesystem name.\n"
2176                 "-h: for help information.\n"
2177                 "-n: the snapshot's name.\n"
2178                 "-r: the remote shell used for communication with remote "
2179                         "target, the default value is 'ssh'.\n");
2180 }
2181
2182 static int snapshot_mount_check(struct snapshot_instance *si, char *fsname,
2183                                 int fslen, bool *mgs_running)
2184 {
2185         int rc;
2186
2187         rc = mdt0_is_lustre_snapshot(si);
2188         if (rc)
2189                 return rc;
2190
2191         rc = snapshot_get_fsname(si, fsname, fslen);
2192         if (rc < 0)
2193                 return rc;
2194
2195         rc = target_is_mounted(si, si->si_mgs, NULL);
2196         if (rc > 0) {
2197                 *mgs_running = true;
2198                 rc = 0;
2199         }
2200
2201         return rc;
2202 }
2203
2204 static int snapshot_mount_target(struct snapshot_instance *si,
2205                                  struct snapshot_target *st, const char *optstr)
2206 {
2207         char cmd[MAX_BUF_SIZE];
2208         char name[8];
2209         int rc;
2210
2211         rc = target_is_mounted(si, st, si->si_ssname);
2212         if (rc < 0)
2213                 return rc;
2214
2215         if (rc > 0)
2216                 return -ESRCH;
2217
2218         memset(cmd, 0, sizeof(cmd));
2219         memset(name, 0, sizeof(name));
2220         snapshot_role2name(name, st->st_role, st->st_index);
2221         snprintf(cmd, sizeof(cmd) - 1,
2222                  "%s %s 'zpool import -d %s %s > /dev/null 2>&1; "
2223                  "mkdir -p /mnt/%s_%s && mount -t lustre "
2224                  "-o rdonly_dev%s %s/%s@%s /mnt/%s_%s'",
2225                  si->si_rsh, st->st_host,
2226                  st->st_dir ? st->st_dir : "/dev -d /tmp",
2227                  st->st_pool, si->si_ssname, name,
2228                  st != si->si_mdt0 ? "" : optstr,
2229                  st->st_pool, st->st_filesystem, si->si_ssname,
2230                  si->si_ssname, name);
2231         rc = snapshot_exec(cmd);
2232         if (rc)
2233                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on the target "
2234                                  "(%s:%x:%d): rc = %d\n", cmd, st->st_host,
2235                                  st->st_role, st->st_index, rc);
2236
2237         return rc;
2238 }
2239
2240 static int __snapshot_mount(struct snapshot_instance *si,
2241                             struct list_head *head)
2242 {
2243         struct snapshot_target *st;
2244         pid_t pid;
2245         int rc;
2246
2247         list_for_each_entry(st, head, st_list) {
2248                 if (st == si->si_mgs || st == si->si_mdt0)
2249                         continue;
2250
2251                 st->st_status = 0;
2252                 st->st_ignored = 0;
2253                 st->st_pid = 0;
2254
2255                 pid = fork();
2256                 if (pid < 0) {
2257                         SNAPSHOT_ADD_LOG(si, "Can't fork for mount snapshot "
2258                                          "%s on target (%s:%x:%d): %s\n",
2259                                          si->si_ssname, st->st_host,
2260                                          st->st_role, st->st_index,
2261                                          strerror(errno));
2262                         return pid;
2263                 }
2264
2265                 /* child */
2266                 if (pid == 0) {
2267                         rc = snapshot_mount_target(si, st, "");
2268                         exit(rc);
2269                 }
2270
2271                 /* parent continue to run more snapshot commands in parallel. */
2272                 st->st_pid = pid;
2273         }
2274
2275         return 0;
2276 }
2277
2278 static int __snapshot_umount(struct snapshot_instance *si,
2279                              struct list_head *head);
2280
2281 static int snapshot_mount(struct snapshot_instance *si)
2282 {
2283         struct snapshot_target *st;
2284         int needed = 0;
2285         int failed = 0;
2286         int rc = 0;
2287         int rc1 = 0;
2288         char fsname[9];
2289         bool mdt0_mounted = false;
2290         bool mgs_running = false;
2291
2292         rc = snapshot_mount_check(si, fsname, sizeof(fsname), &mgs_running);
2293         if (rc < 0) {
2294                 fprintf(stderr,
2295                         "Can't mount the snapshot %s: %s\n",
2296                         si->si_ssname, strerror(-rc));
2297                 return rc;
2298         }
2299
2300         /* 1. MGS is not mounted yet, mount the MGS firstly */
2301         si->si_mgs->st_ignored = 0;
2302         si->si_mgs->st_pid = 0;
2303         if (!mgs_running) {
2304                 rc = snapshot_mount_target(si, si->si_mgs, "");
2305                 if (rc == -ESRCH) {
2306                         si->si_mgs->st_ignored = 1;
2307                         rc = 0;
2308                 }
2309
2310                 if (rc < 0) {
2311                         fprintf(stderr,
2312                                 "Can't mount the snapshot %s: %s\n",
2313                                 si->si_ssname, strerror(-rc));
2314                         return rc;
2315                 }
2316
2317                 if (si->si_mgs == si->si_mdt0)
2318                         mdt0_mounted = true;
2319         }
2320
2321         /* 2. Mount MDT0 if it is not combined with the MGS. */
2322         if (!mdt0_mounted) {
2323                 si->si_mdt0->st_ignored = 0;
2324                 si->si_mdt0->st_pid = 0;
2325                 rc = snapshot_mount_target(si, si->si_mdt0, ",nomgs");
2326                 if (rc)
2327                         goto cleanup;
2328         }
2329
2330         /* 3.1 Mount other MDTs in parallel */
2331         rc = __snapshot_mount(si, &si->si_mdts_list);
2332         if (!rc)
2333                 /* 3.2 Mount OSTs in parallel */
2334                 rc = __snapshot_mount(si, &si->si_osts_list);
2335
2336         /* Wait for all children, even though part of them maybe failed */
2337         failed = snapshot_wait(si, &rc1);
2338
2339         list_for_each_entry(st, &si->si_mdts_list, st_list) {
2340                 if (!st->st_ignored)
2341                         needed++;
2342         }
2343
2344         list_for_each_entry(st, &si->si_osts_list, st_list) {
2345                 if (!st->st_ignored)
2346                         needed++;
2347         }
2348
2349 cleanup:
2350         if (rc || rc1) {
2351                 int rc2 = 0;
2352
2353                 __snapshot_umount(si, &si->si_mdts_list);
2354                 __snapshot_umount(si, &si->si_osts_list);
2355                 snapshot_wait(si, &rc2);
2356
2357                 if (rc)
2358                         fprintf(stderr,
2359                                 "Can't mount the snapshot %s: %s\n",
2360                                 si->si_ssname, strerror(-rc));
2361                 else
2362                         fprintf(stderr,
2363                                 "%d of %d pieces of the snapshot %s "
2364                                 "can't be mounted: %s\n",
2365                                 failed, needed, si->si_ssname, strerror(-rc1));
2366
2367                 return rc ? rc : rc1;
2368         }
2369
2370         if (needed == 0) {
2371                 fprintf(stderr,
2372                         "The snapshot %s has already been mounted by other\n",
2373                         si->si_ssname);
2374                 return -EALREADY;
2375         }
2376
2377         fprintf(stdout, "mounted the snapshot %s with fsname %s\n",
2378                 si->si_ssname, fsname);
2379
2380         return 0;
2381 }
2382
2383 int jt_snapshot_mount(int argc, char **argv)
2384 {
2385         struct snapshot_instance *si;
2386         struct option long_opts[] = {
2387         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2388         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2389         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2390         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2391         { .name = NULL } };
2392         int rc = 0;
2393
2394         si = snapshot_init(argc, argv, long_opts, "F:hn:r:",
2395                            snapshot_mount_usage, LOCK_EX, &rc);
2396         if (!si)
2397                 return rc;
2398
2399         if (!si->si_ssname) {
2400                 fprintf(stderr,
2401                         "Miss the snapshot name to be mounted\n");
2402                 snapshot_mount_usage();
2403                 snapshot_fini(si);
2404                 return -EINVAL;
2405         }
2406
2407         rc = snapshot_mount(si);
2408         if (rc)
2409                 SNAPSHOT_ADD_LOG(si, "Can't mount snapshot %s: %d\n",
2410                                  si->si_ssname, rc);
2411         else
2412                 SNAPSHOT_ADD_LOG(si, "The snapshot %s is mounted\n",
2413                                  si->si_ssname);
2414
2415         snapshot_fini(si);
2416         return rc;
2417
2418 }
2419
2420 static void snapshot_umount_usage(void)
2421 {
2422         fprintf(stderr,
2423                 "Umount the specified snapshot.\n"
2424                 "Usage:\n"
2425                 "snapshot_umount <-F | --fsname fsname> [-h | --help] "
2426                                 "<-n | --name ssname> "
2427                                 "[-r | --rsh remote_shell]\n"
2428                 "Options:\n"
2429                 "-F: the filesystem name.\n"
2430                 "-h: for help information.\n"
2431                 "-n: the snapshot's name.\n"
2432                 "-r: the remote shell used for communication with remote "
2433                         "target, the default value is 'ssh'.\n");
2434 }
2435
2436 static int __snapshot_umount(struct snapshot_instance *si,
2437                              struct list_head *head)
2438 {
2439         struct snapshot_target *st;
2440         pid_t pid;
2441         int rc;
2442
2443         list_for_each_entry(st, head, st_list) {
2444                 st->st_status = 0;
2445                 st->st_ignored = 0;
2446                 st->st_pid = 0;
2447
2448                 pid = fork();
2449                 if (pid < 0) {
2450                         SNAPSHOT_ADD_LOG(si, "Can't fork for umount snapshot "
2451                                          "%s on target (%s:%x:%d): %s\n",
2452                                          si->si_ssname, st->st_host,
2453                                          st->st_role, st->st_index,
2454                                          strerror(errno));
2455                         return pid;
2456                 }
2457
2458                 /* child */
2459                 if (pid == 0) {
2460                         char cmd[MAX_BUF_SIZE];
2461
2462                         rc = target_is_mounted(si, st, si->si_ssname);
2463                         if (rc < 0)
2464                                 exit(rc);
2465
2466                         if (!rc)
2467                                 exit(-ESRCH);
2468
2469                         memset(cmd, 0, sizeof(cmd));
2470                         snprintf(cmd, sizeof(cmd) - 1,
2471                                  "%s %s 'umount %s/%s@%s'",
2472                                  si->si_rsh, st->st_host, st->st_pool,
2473                                  st->st_filesystem, si->si_ssname);
2474                         rc = snapshot_exec(cmd);
2475
2476                         exit(rc);
2477                 }
2478
2479                 /* parent continue to run more snapshot commands in parallel. */
2480                 st->st_pid = pid;
2481         }
2482
2483         return 0;
2484 }
2485
2486 static int snapshot_umount(struct snapshot_instance *si)
2487 {
2488         struct snapshot_target *st;
2489         int needed = 0;
2490         int failed;
2491         int rc = 0;
2492         int rc1 = 0;
2493         int rc2 = 0;
2494
2495         rc = snapshot_general_check(si);
2496         if (rc < 0) {
2497                 fprintf(stderr,
2498                         "Can't umount the snapshot %s: %s\n",
2499                         si->si_ssname, strerror(-rc));
2500                 return rc;
2501         }
2502
2503         rc = __snapshot_umount(si, &si->si_mdts_list);
2504         rc1 = __snapshot_umount(si, &si->si_osts_list);
2505         failed = snapshot_wait(si, &rc2);
2506
2507         list_for_each_entry(st, &si->si_mdts_list, st_list) {
2508                 if (!st->st_ignored)
2509                         needed++;
2510         }
2511
2512         list_for_each_entry(st, &si->si_osts_list, st_list) {
2513                 if (!st->st_ignored)
2514                         needed++;
2515         }
2516
2517         if (needed == 0) {
2518                 fprintf(stderr,
2519                         "The snapshot %s has not been mounted\n",
2520                         si->si_ssname);
2521                 return -EALREADY;
2522         }
2523
2524         if (failed != 0)
2525                 fprintf(stderr,
2526                         "%d of %d pieces of the snapshot %s "
2527                         "can't be umounted: %s\n",
2528                         failed, needed, si->si_ssname, strerror(-rc2));
2529
2530         return rc ? rc : (rc1 ? rc1 : rc2);
2531 }
2532
2533 int jt_snapshot_umount(int argc, char **argv)
2534 {
2535         struct snapshot_instance *si;
2536         struct option long_opts[] = {
2537         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2538         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2539         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2540         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2541         { .name = NULL } };
2542         int rc = 0;
2543
2544         si = snapshot_init(argc, argv, long_opts, "F:hn:r:",
2545                            snapshot_umount_usage, LOCK_EX, &rc);
2546         if (!si)
2547                 return rc;
2548
2549         if (!si->si_ssname) {
2550                 fprintf(stderr,
2551                         "Miss the snapshot name to be umounted\n");
2552                 snapshot_umount_usage();
2553                 snapshot_fini(si);
2554                 return -EINVAL;
2555         }
2556
2557         rc = snapshot_umount(si);
2558         if (rc < 0)
2559                 SNAPSHOT_ADD_LOG(si, "Can't umount snapshot %s: %d\n",
2560                                  si->si_ssname, rc);
2561         else
2562                 SNAPSHOT_ADD_LOG(si, "the snapshot %s have been umounted\n",
2563                                  si->si_ssname);
2564
2565         snapshot_fini(si);
2566         return rc;
2567 }