lustre/utils/lsnapshot.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License version 2 for more details.  A copy is
  14  * included in the COPYING file that accompanied this code.
  15
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  19  *
  20  * GPL HEADER END
  21  */
  22 /*
  23  * Copyright (c) 2017, Intel Corporation.
  24  *
  25  * lustre/utils/lsnapshot.c
  26  *
  27  * Author: Fan, Yong <fan.yong@intel.com>
  28  */
  29
  30 #include <stdio.h>
  31 #include <string.h>
  32 #include <getopt.h>
  33 #include <unistd.h>
  34 #include <stdlib.h>
  35 #include <errno.h>
  36 #include <string.h>
  37 #include <sys/types.h>
  38 #include <sys/wait.h>
  39 #include <sys/time.h>
  40 #include <sys/file.h>
  41 #include <time.h>
  42 #include <limits.h>
  43 #include <ctype.h>
  44
  45 #include <libcfs/util/list.h>
  46 #include <libcfs/util/ioctl.h>
  47 #include <libcfs/util/string.h>
  48 #include <linux/lustre/lustre_ioctl.h>
  49 #include <linux/lustre/lustre_barrier_user.h>
  50
  51 #include "obdctl.h"
  52
  53 #define SNAPSHOT_CONF_DIR       "/etc/lsnapshot"
  54 #define LDEV_CONF               "/etc/ldev.conf"
  55 #define SNAPSHOT_LOG            "/var/log/lsnapshot.log"
  56 #define SNAPSHOT_MAGIC          "0x14F711B9"
  57 #define MAX_BUF_SIZE            4096
  58
  59 enum snapshot_role {
  60         SR_MGS  = 0x0001,
  61         SR_MDT  = 0x0002,
  62         SR_OST  = 0x0004,
  63 };
  64
  65 struct snapshot_target {
  66         struct list_head         st_list;
  67         /* Target node name. */
  68         char                    *st_host;
  69         /* Where the pool is */
  70         char                    *st_dir;
  71         /* The target pool name on the target node. */
  72         char                    *st_pool;
  73         /* The backend filesystem name against the target pool. */
  74         char                    *st_filesystem;
  75         int                      st_role;
  76         unsigned int             st_index;
  77         unsigned int             st_gen;
  78         int                      st_line;
  79         int                      st_status;
  80         pid_t                    st_pid;
  81         bool                     st_ignored;
  82 };
  83
  84 struct snapshot_instance {
  85         struct list_head         si_mdts_list;
  86         struct list_head         si_osts_list;
  87         struct snapshot_target  *si_mgs;
  88         struct snapshot_target  *si_mdt0;
  89         FILE                    *si_log_fp;
  90         char                    *si_rsh;
  91         char                    *si_fsname;
  92         char                    *si_ssname;
  93         char                    *si_new_ssname;
  94         char                    *si_comment;
  95         int                      si_conf_fd;
  96         int                      si_timeout;
  97         bool                     si_barrier;
  98         bool                     si_detail;
  99         bool                     si_force;
 100 };
 101
 102 static const char snapshot_rsh_default[] = "ssh";
 103 static char snapshot_path[MAX_BUF_SIZE];
 104
 105 static char *snapshot_role2name(char *name, enum snapshot_role role,
 106                                 __u32 index)
 107 {
 108         if (role & SR_MDT)
 109                 snprintf(name, 8, "MDT%04x", index);
 110         else if (role & SR_MGS)
 111                 snprintf(name, 4, "MGS");
 112         else
 113                 snprintf(name, 8, "OST%04x", index);
 114
 115         return name;
 116 }
 117
 118 #define SNAPSHOT_ADD_LOG(si, format, ...)                               \
 119 do {                                                                    \
 120         char buf[MAX_BUF_SIZE];                                         \
 121         char *ptr;                                                      \
 122         time_t tt;                                                      \
 123                                                                         \
 124         memset(buf, 0, sizeof(buf));                                    \
 125         time(&tt);                                                      \
 126         snprintf(buf, sizeof(buf) - 1, "%s", ctime(&tt));               \
 127         ptr = strrchr(buf, '\n');                                       \
 128         if (ptr)                                                        \
 129                 *ptr = '\0';                                            \
 130                                                                         \
 131         fprintf(si->si_log_fp, "%s (%d:%s:%d:%s:%s): "format, buf,      \
 132                 getpid(), __func__, __LINE__, si->si_fsname,            \
 133                 si->si_rsh, ## __VA_ARGS__);                            \
 134 } while (0)
 135
 136 #define DRSH "%s %s"
 137 #define DFSNAME "%s/%s"
 138 #define DSSNAME "%s/%s@%s"
 139 #define DZFS "%s zfs"
 140 #define DIMPORT "%s zpool import -d %s %s > /dev/null 2>&1"
 141
 142 #define PRSH(si, st) (si)->si_rsh, (st)->st_host
 143 #define PFSNAME(st) (st)->st_pool, (st)->st_filesystem
 144 #define PSSNAME(si, st) PFSNAME(st), (si)->si_ssname
 145 #define PSS_NEW(si, st) PFSNAME(st), (si)->si_new_ssname
 146 #define PZFS(st) snapshot_path
 147 #define PIMPORT(st) snapshot_path, \
 148                 (st)->st_dir ? (st)->st_dir : "/dev -d /tmp", (st)->st_pool
 149
 150 char *snapshot_fgets(FILE *fp, char *buf, int buflen)
 151 {
 152         char *ptr;
 153
 154         memset(buf, 0, buflen);
 155         if (fgets(buf, buflen, fp) == NULL)
 156                 return NULL;
 157
 158         ptr = strchr(buf, '\n');
 159         if (ptr)
 160                 *ptr = '\0';
 161
 162         return buf;
 163 }
 164
 165 static int snapshot_exec(const char *cmd)
 166 {
 167         int rc;
 168
 169         errno = 0;
 170
 171         /* system() return value depends on both the system() general framework,
 172          * such as whether fork()/exec() success or fail, and the real @cmd exec
 173          * result. Especially, if the @cmd is remote command, we may cannot know
 174          * the real failure. */
 175         rc = system(cmd);
 176         /* fork()/exec() error */
 177         if (rc == -1)
 178                 return errno != 0 ? -errno : -1;
 179
 180         if (WIFEXITED(rc)) {
 181                 rc = WEXITSTATUS(rc);
 182                 if (rc > 0)
 183                         rc = -rc;
 184         } else if (WIFSIGNALED(rc)) {
 185                 rc = -EINTR;
 186         } else {
 187                 /* all other known or unknown cases. */
 188                 rc = -EFAULT;
 189         }
 190
 191         return rc;
 192 }
 193
 194 static int snapshot_load_conf_ldev(struct snapshot_instance *si, char *buf,
 195                                    struct snapshot_target *st, char **role)
 196 {
 197         char *label = NULL;
 198         char *device = NULL;
 199         char *ignore = NULL;
 200         char *ptr;
 201         char *ptr1;
 202         int len;
 203         int rc;
 204
 205         rc = sscanf(buf, "%ms %ms %ms %ms",
 206                     &st->st_host, &ignore, &label, &device);
 207         if (rc < 4) {
 208                 rc = -EINVAL;
 209                 goto out;
 210         }
 211
 212         free(ignore);
 213
 214         /* Format of device:
 215          * [md|zfs:][pool_dir/]<pool>/<filesystem> */
 216         ptr = strchr(device, ':');
 217         if (ptr) {
 218                 ptr++;
 219                 if (strncmp(device, "zfs:", strlen("zfs:")) != 0) {
 220                         rc = -EINVAL;
 221                         goto out;
 222                 }
 223         } else {
 224                         ptr = device;
 225         }
 226
 227         ptr1 = strrchr(ptr, '/');
 228         /* "ptr1 - ptr + 1 == strlen(ptr)" means '/' is at the tail. */
 229         if (!ptr1 || ptr1 == ptr || ptr1 - ptr + 1 == strlen(ptr)) {
 230                 rc = -EINVAL;
 231                 goto out;
 232         }
 233
 234         len = strlen(ptr1);
 235         st->st_filesystem = malloc(len);
 236         if (!st->st_filesystem) {
 237                 rc = -ENOMEM;
 238                 goto out;
 239         }
 240
 241         *ptr1 = '\0';
 242         strncpy(st->st_filesystem, ptr1 + 1, len - 1);
 243         st->st_filesystem[len - 1] = '\0';
 244
 245         if (*ptr == '/') {
 246                 ptr1 = strrchr(ptr, '/');
 247                 *ptr1 = '\0';
 248                 st->st_dir = strdup(ptr);
 249                 if (!st->st_dir) {
 250                         rc = -ENOMEM;
 251                         goto out;
 252                 }
 253                 ptr = ptr1 + 1;
 254         }
 255
 256         st->st_pool = strdup(ptr);
 257         if (!st->st_pool) {
 258                 rc = -ENOMEM;
 259                 goto out;
 260         }
 261
 262         /* Format of label:
 263          * fsname-<role><index> or <role><index> */
 264         ptr = strrchr(label, '-');
 265         if (ptr) {
 266                 if (strncmp(si->si_fsname, label, ptr - label) != 0) {
 267                         /* This line is NOT for current filesystem .*/
 268                         rc = -EAGAIN;
 269                         goto out;
 270                 }
 271
 272                 ptr++;
 273         } else {
 274                 ptr = label;
 275         }
 276
 277         if (strlen(ptr) < 3 || strlen(ptr) > 7) {
 278                 rc = -EINVAL;
 279                 goto out;
 280         }
 281
 282         *role = malloc(4);
 283         if (!*role) {
 284                 rc = -ENOMEM;
 285                 goto out;
 286         }
 287
 288         strncpy(*role, ptr, 3);
 289         (*role)[3] = 0;
 290         ptr += 3;
 291         len = 0;
 292         while (isxdigit(ptr[len])) {
 293                 if (isdigit(ptr[len]))
 294                         st->st_index =
 295                                 st->st_index * 16 + ptr[len] - '0';
 296                 else if (isupper(ptr[len]))
 297                         st->st_index =
 298                                 st->st_index * 16 + ptr[len] - 'A' + 10;
 299                 else
 300                         st->st_index =
 301                                 st->st_index * 16 + ptr[len] - 'a' + 10;
 302                 len++;
 303         }
 304
 305         if (len == 0) {
 306                 if (strncasecmp(*role, "MGS", 3) != 0)
 307                         rc = -EINVAL;
 308                 else
 309                         rc = 0;
 310
 311                 goto out;
 312         }
 313
 314         if (!isxdigit(ptr[len]) && ptr[len] != '\0') {
 315                 rc = -EINVAL;
 316                 goto out;
 317         }
 318
 319 out:
 320         if (label)
 321                 free(label);
 322         if (device)
 323                 free(device);
 324
 325         return rc;
 326 }
 327
 328 /**
 329  * For old snasphot tools, the configration is in /etc/lsnapshot/${fsname}.conf,
 330  * the format is:
 331  * <host> <pool_dir> <pool> <local_fsname> <role(,s)> <index>
 332  *
 333  * For example:
 334  *
 335  * host-mdt1 /tmp myfs-mdt1 mdt1 MGS,MDT 0
 336  * host-mdt2 /tmp myfs-mdt2 mdt2 MDT 1
 337  * host-ost1 /tmp myfs-ost1 ost1 OST 0
 338  * host-ost2 /tmp myfs-ost2 ost2 OST 1
 339  *
 340  *
 341  * For new snasphot tools, the configration is in /etc/ldev.conf, which is not
 342  * only for snapshot, but also for other purpose. The format is:
 343  * <host> foreign/- <label> <device> [journal-path]/- [raidtab]
 344  *
 345  * The format of <label> is:
 346  * fsname-<role><index> or <role><index>
 347  *
 348  * The format of <device> is:
 349  * [md|zfs:][pool_dir/]<pool>/<filesystem>
 350  *
 351  * Snapshot only uses the fields <host>, <label> and <device>.
 352  *
 353  * For example:
 354  *
 355  * host-mdt1 - myfs-MDT0000 zfs:/tmp/myfs-mdt1/mdt1
 356  *
 357  *
 358  * \retval       0      for success
 359  * \retval      +ve     the line# with which the current line is conflict
 360  * \retval      -EAGAIN skip current line
 361  * \retval      -ve     other failures
 362  */
 363 static int snapshot_load_conf_one(struct snapshot_instance *si,
 364                                   char *buf, int line_num, bool is_ldev)
 365 {
 366         struct snapshot_target *st;
 367         char *role = NULL;
 368         char *path;
 369         int rc = 0;
 370
 371         path = getenv("PATH");
 372         if (!path)
 373                 return -EINVAL;
 374
 375         memset(snapshot_path, 0, sizeof(snapshot_path));
 376         snprintf(snapshot_path, sizeof(snapshot_path) - 1, "PATH='%s'", path);
 377
 378         /* filter out space */
 379         while (isspace(*buf))
 380                 buf++;
 381
 382         /* skip empty line */
 383         if (*buf == '\0')
 384                 return 0;
 385
 386         /* skip comment line */
 387         if (*buf == '#')
 388                 return 0;
 389
 390         st = malloc(sizeof(*st));
 391         if (!st)
 392                 return -ENOMEM;
 393
 394         memset(st, 0, sizeof(*st));
 395         INIT_LIST_HEAD(&st->st_list);
 396
 397         if (is_ldev) {
 398                 rc = snapshot_load_conf_ldev(si, buf, st, &role);
 399         } else {
 400                 rc = sscanf(buf, "%ms %ms %ms %ms %ms %d",
 401                             &st->st_host, &st->st_dir, &st->st_pool,
 402                             &st->st_filesystem, &role, &st->st_index);
 403                 if (rc < 6)
 404                         rc = -EINVAL;
 405         }
 406
 407         if (rc < 0)
 408                 goto out;
 409         rc = 0;
 410
 411         if (strncasecmp(role, "MGS", 3) == 0) {
 412                 st->st_role = SR_MGS;
 413                 if (role[3] == ',') {
 414                         /* MGS,MDT */
 415                         if (strncasecmp(&role[4], "MDT", 3) != 0) {
 416                                 rc = -EINVAL;
 417                                 goto out;
 418                         }
 419
 420                         st->st_role |= SR_MDT;
 421                 }
 422         } else if (strncasecmp(role, "MDT", 3) == 0) {
 423                 st->st_role = SR_MDT;
 424                 if (role[3] == ',') {
 425                         /* MDT,MGS */
 426                         if (strncasecmp(&role[4], "MGS", 3) != 0) {
 427                                 rc = -EINVAL;
 428                                 goto out;
 429                         }
 430
 431                         st->st_role |= SR_MGS;
 432                 }
 433         } else if (strncasecmp(role, "OST", 3) == 0) {
 434                 st->st_role = SR_OST;
 435         } else {
 436                 rc = -EINVAL;
 437                 goto out;
 438         }
 439
 440         st->st_line = line_num;
 441         if (st->st_role & SR_MDT) {
 442                 /* MGS is the first, MDT0 is just after the MGS
 443                  * if they are not combined together. */
 444                 if (st->st_role & SR_MGS) {
 445                         if (si->si_mgs) {
 446                                 rc = si->si_mgs->st_line;
 447                                 goto out;
 448                         }
 449
 450                         si->si_mgs = st;
 451                         list_add(&st->st_list, &si->si_mdts_list);
 452                 }
 453
 454                 if (st->st_index == 0) {
 455                         if (si->si_mdt0) {
 456                                 rc = si->si_mdt0->st_line;
 457                                 goto out;
 458                         }
 459
 460                         si->si_mdt0 = st;
 461                         if (list_empty(&st->st_list)) {
 462                                 if (list_empty(&si->si_mdts_list) ||
 463                                     !si->si_mgs)
 464                                         list_add(&st->st_list,
 465                                                  &si->si_mdts_list);
 466                                 else
 467                                         list_add(&st->st_list,
 468                                                  &si->si_mgs->st_list);
 469                         }
 470                 } else if (list_empty(&st->st_list)) {
 471                         list_add_tail(&st->st_list, &si->si_mdts_list);
 472                 }
 473         } else if (st->st_role & SR_MGS) {
 474                 if (si->si_mgs) {
 475                         rc = si->si_mgs->st_line;
 476                         goto out;
 477                 }
 478
 479                 si->si_mgs = st;
 480                 list_add(&st->st_list, &si->si_mdts_list);
 481         } else {
 482                 list_add_tail(&st->st_list, &si->si_osts_list);
 483         }
 484
 485 out:
 486         if (role)
 487                 free(role);
 488
 489         if (rc) {
 490                 if (st->st_host)
 491                         free(st->st_host);
 492                 if (st->st_dir)
 493                         free(st->st_dir);
 494                 if (st->st_pool)
 495                         free(st->st_pool);
 496                 if (st->st_filesystem)
 497                         free(st->st_filesystem);
 498                 free(st);
 499         }
 500
 501         return rc;
 502 }
 503
 504 static int snapshot_load_conf(struct snapshot_instance *si, int lock_mode)
 505 {
 506         FILE *fp;
 507         char buf[MAX_BUF_SIZE];
 508         char conf_name[32];
 509         int line_num = 1;
 510         int fd = -1;
 511         int rc = 0;
 512         bool is_ldev = true;
 513
 514         memset(conf_name, 0, sizeof(conf_name));
 515         strncpy(conf_name, LDEV_CONF, sizeof(conf_name) - 1);
 516         fd = open(conf_name, O_RDONLY);
 517         if (fd < 0) {
 518                 if (errno != ENOENT) {
 519                         fprintf(stderr,
 520                                 "Can't open the snapshot config file %s: %s\n",
 521                                 conf_name, strerror(errno));
 522
 523                         return fd;
 524                 }
 525
 526                 snprintf(conf_name, sizeof(conf_name) - 1, "%s/%s.conf",
 527                          SNAPSHOT_CONF_DIR, si->si_fsname);
 528                 fd = open(conf_name, O_RDONLY);
 529                 if (fd < 0) {
 530                         fprintf(stderr,
 531                                 "Can't open the snapshot config file %s: %s\n",
 532                                 conf_name, strerror(errno));
 533
 534                         return fd;
 535                 }
 536
 537                 is_ldev = false;
 538         }
 539
 540         rc = flock(fd, lock_mode | LOCK_NB);
 541         if (rc < 0) {
 542                 fprintf(stderr,
 543                         "Can't lock the snapshot config file %s (%d): %s\n",
 544                         conf_name, lock_mode, strerror(errno));
 545                 close(fd);
 546                 return rc;
 547         }
 548
 549         fp = fdopen(fd, "r");
 550         if (!fp) {
 551                 fprintf(stderr,
 552                         "Can't fdopen the snapshot config file %s: %s\n",
 553                         conf_name, strerror(errno));
 554                 rc = -1;
 555                 goto out;
 556         }
 557
 558         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
 559                 rc = snapshot_load_conf_one(si, buf, line_num, is_ldev);
 560                 if (rc == -EINVAL) {
 561                         fprintf(stderr,
 562                                 "Invalid snapshot config file %s at the line "
 563                                 "%d '%s'\n", conf_name, line_num, buf);
 564                 } else if (rc == -EAGAIN) {
 565                         rc = 0;
 566                 } else if (rc > 0) {
 567                         fprintf(stderr,
 568                                 "The config role has been specified repeatedly "
 569                                 "at the lines %d/%d in %s\n",
 570                                 rc, line_num, conf_name);
 571                         rc = -EINVAL;
 572                 }
 573
 574                 if (rc)
 575                         goto out;
 576
 577                 line_num++;
 578         }
 579
 580         if (!si->si_mdt0) {
 581                 fprintf(stderr,
 582                         "Miss MDT0 in the config file %s\n",
 583                         conf_name);
 584                 rc = -1;
 585                 goto out;
 586         }
 587
 588         /* By default, the MGS is on the MDT0 if it is not specified. */
 589         if (!si->si_mgs) {
 590                 si->si_mgs = si->si_mdt0;
 591                 si->si_mgs->st_role |= SR_MGS;
 592         }
 593
 594         if (list_empty(&si->si_osts_list)) {
 595                 fprintf(stderr,
 596                         "Miss OST(s) in the config file %s\n",
 597                         conf_name);
 598                 rc = -1;
 599                 goto out;
 600         }
 601
 602 out:
 603         if (fd >= 0) {
 604                 if (rc < 0) {
 605                         flock(fd, LOCK_UN);
 606                         close(fd);
 607                 } else {
 608                         si->si_conf_fd = fd;
 609                 }
 610         }
 611
 612         return rc;
 613 }
 614
 615 static void snapshot_unload_conf(struct snapshot_instance *si)
 616 {
 617         struct snapshot_target *st;
 618
 619         while (!list_empty(&si->si_mdts_list)) {
 620                 st = list_entry(si->si_mdts_list.next,
 621                                 struct snapshot_target, st_list);
 622                 list_del(&st->st_list);
 623                 free(st->st_host);
 624                 free(st->st_dir);
 625                 free(st->st_pool);
 626                 free(st->st_filesystem);
 627                 free(st);
 628         }
 629
 630         while (!list_empty(&si->si_osts_list)) {
 631                 st = list_entry(si->si_osts_list.next,
 632                                 struct snapshot_target, st_list);
 633                 list_del(&st->st_list);
 634                 free(st->st_host);
 635                 free(st->st_dir);
 636                 free(st->st_pool);
 637                 free(st->st_filesystem);
 638                 free(st);
 639         }
 640
 641         si->si_mgs = NULL;
 642         si->si_mdt0 = NULL;
 643
 644         if (si->si_conf_fd >= 0) {
 645                 flock(si->si_conf_fd, LOCK_UN);
 646                 close(si->si_conf_fd);
 647                 si->si_conf_fd = -1;
 648         }
 649 }
 650
 651 static int snapshot_handle_string_option(char **dst, const char *option,
 652                                          const char *opt_name)
 653 {
 654         if (*dst && *dst != snapshot_rsh_default) {
 655                 fprintf(stderr,
 656                         "%s option has been specified repeatedly.\n", opt_name);
 657                 return -EINVAL;
 658         }
 659
 660         *dst = strdup(option);
 661         if (!*dst)
 662                 return -ENOMEM;
 663         return 0;
 664 }
 665
 666 static void snapshot_fini(struct snapshot_instance *si)
 667 {
 668         snapshot_unload_conf(si);
 669
 670         if (si->si_log_fp)
 671                 fclose(si->si_log_fp);
 672
 673         if (si->si_rsh && si->si_rsh != snapshot_rsh_default)
 674                 free(si->si_rsh);
 675         if (si->si_fsname)
 676                 free(si->si_fsname);
 677         if (si->si_ssname)
 678                 free(si->si_ssname);
 679         if (si->si_new_ssname)
 680                 free(si->si_new_ssname);
 681         if (si->si_comment)
 682                 free(si->si_comment);
 683
 684         free(si);
 685 }
 686
 687 static struct snapshot_instance *
 688 snapshot_init(int argc, char * const argv[], const struct option *longopts,
 689               const char *optstring, void (*usage)(void),
 690               int lock_mode, int *err)
 691 {
 692         struct snapshot_instance *si;
 693         int idx;
 694         int opt;
 695
 696         *err = 0;
 697         si = malloc(sizeof(*si));
 698         if (!si) {
 699                 fprintf(stderr,
 700                         "No enough memory to initialize snapshot instance.\n");
 701                 *err = -ENOMEM;
 702                 return NULL;
 703         }
 704
 705         memset(si, 0, sizeof(*si));
 706         INIT_LIST_HEAD(&si->si_mdts_list);
 707         INIT_LIST_HEAD(&si->si_osts_list);
 708         si->si_rsh = (char *)snapshot_rsh_default;
 709         si->si_conf_fd = -1;
 710         si->si_timeout = BARRIER_TIMEOUT_DEFAULT;
 711         si->si_barrier = true;
 712         si->si_detail = false;
 713         si->si_force = false;
 714
 715         while ((opt = getopt_long(argc, argv, optstring,
 716                                   longopts, &idx)) != EOF) {
 717                 switch (opt) {
 718                 case 'b':
 719                         if (!optarg || strcmp(optarg, "on") == 0) {
 720                                 si->si_barrier = true;
 721                         } else if (strcmp(optarg, "off") == 0) {
 722                                 si->si_barrier = false;
 723                         } else {
 724                                 usage();
 725                                 *err = -EINVAL;
 726                                 goto out;
 727                         }
 728                         break;
 729                 case 'c':
 730                         *err = snapshot_handle_string_option(&si->si_comment,
 731                                                              optarg, "comment");
 732                         if (*err != 0)
 733                                 goto out;
 734                         break;
 735                 case 'd':
 736                         si->si_detail = true;
 737                         break;
 738                 case 'f':
 739                         si->si_force = true;
 740                         break;
 741                 case 'F':
 742                         *err = snapshot_handle_string_option(&si->si_fsname,
 743                                                              optarg, "fsname");
 744                         if (*err != 0)
 745                                 goto out;
 746                         break;
 747                 case 'n':
 748                         *err = snapshot_handle_string_option(&si->si_ssname,
 749                                                              optarg, "ssname");
 750                         if (*err != 0)
 751                                 goto out;
 752                         break;
 753                 case 'N':
 754                         *err = snapshot_handle_string_option(&si->si_new_ssname,
 755                                                              optarg,
 756                                                              "new ssname");
 757                         if (*err != 0)
 758                                 goto out;
 759                         break;
 760                 case 'r':
 761                         *err = snapshot_handle_string_option(&si->si_rsh,
 762                                                              optarg,
 763                                                              "remote shell");
 764                         if (*err != 0)
 765                                 goto out;
 766                         break;
 767                 case 't':
 768                         si->si_timeout = atoi(optarg);
 769                         break;
 770                 default:
 771                         *err = -EINVAL;
 772                         usage();
 773                         goto out;
 774                 case 'h':
 775                         usage();
 776                         snapshot_fini(si);
 777                         *err = 0;
 778                         return NULL;
 779                 }
 780         }
 781
 782         if (!si->si_fsname) {
 783                 fprintf(stderr, "The fsname must be specified\n");
 784                 usage();
 785                 *err = -EINVAL;
 786                 goto out;
 787         }
 788
 789         if (strlen(si->si_fsname) > 8) {
 790                 fprintf(stderr, "Invalid fsname %s\n", si->si_fsname);
 791                 *err = -EINVAL;
 792                 goto out;
 793         }
 794
 795         si->si_log_fp = fopen(SNAPSHOT_LOG, "a");
 796         if (!si->si_log_fp) {
 797                 *err = -errno;
 798                 fprintf(stderr,
 799                         "Can't open the snapshot log file %s: %s\n",
 800                         SNAPSHOT_LOG, strerror(errno));
 801                 goto out;
 802         }
 803
 804         *err = snapshot_load_conf(si, lock_mode);
 805
 806 out:
 807         if (*err != 0 && si) {
 808                 snapshot_fini(si);
 809                 si = NULL;
 810         }
 811
 812         return si;
 813 }
 814
 815 static int __snapshot_wait(struct snapshot_instance *si,
 816                            struct list_head *head, int *err)
 817 {
 818         struct snapshot_target *st;
 819         int count = 0;
 820         int rc = 0;
 821
 822         list_for_each_entry(st, head, st_list) {
 823                 if (st->st_pid == 0)
 824                         continue;
 825
 826                 rc = waitpid(st->st_pid, &st->st_status, 0);
 827                 if (rc < 0) {
 828                         SNAPSHOT_ADD_LOG(si, "Can't wait child (%d) operation "
 829                                          "on the target <%s:%x:%d>: %s\n",
 830                                          st->st_pid, st->st_host, st->st_role,
 831                                          st->st_index, strerror(errno));
 832                         count++;
 833                         if (*err == 0)
 834                                 *err = rc;
 835
 836                         st->st_pid = 0;
 837                         /* continue to wait for next */
 838                         continue;
 839                 }
 840
 841                 if (WIFEXITED(st->st_status)) {
 842                         rc = WEXITSTATUS(st->st_status);
 843                         if (rc > 0)
 844                                 rc -= 256;
 845
 846                         if (rc == -ESRCH) {
 847                                 st->st_ignored = true;
 848                         } else if (rc) {
 849                                 count++;
 850                                 if (*err == 0)
 851                                         *err = rc;
 852                         }
 853                 } else if (WIFSIGNALED(st->st_status)) {
 854                         SNAPSHOT_ADD_LOG(si, "The child (%d) operation on the "
 855                                          "target <%s:%x:%d> was killed by "
 856                                          "signal %d\n",
 857                                          st->st_pid, st->st_host, st->st_role,
 858                                          st->st_index, WTERMSIG(st->st_status));
 859                         count++;
 860                         if (*err == 0)
 861                                 *err = -EINTR;
 862                 } else {
 863                         SNAPSHOT_ADD_LOG(si, "The child (%d) operation on the "
 864                                          "target <%s:%x:%d> failed for "
 865                                          "unknown reason\n",
 866                                          st->st_pid, st->st_host, st->st_role,
 867                                          st->st_index);
 868                         count++;
 869                         if (*err == 0)
 870                                 *err = -EFAULT;
 871                 }
 872
 873                 st->st_pid = 0;
 874         }
 875
 876         return count;
 877 }
 878
 879 static int snapshot_wait(struct snapshot_instance *si, int *err)
 880 {
 881         int count;
 882
 883         count = __snapshot_wait(si, &si->si_mdts_list, err);
 884         count += __snapshot_wait(si, &si->si_osts_list, err);
 885
 886         return count;
 887 }
 888
 889 static char *snapshot_first_skip_blank(char *buf)
 890 {
 891         char *ptr;
 892
 893         ptr = strchr(buf, ' ');
 894         if (!ptr) {
 895                 ptr = strchr(buf, '\t');
 896                 if (!ptr)
 897                         return NULL;
 898         }
 899
 900         while (*ptr == ' ' || *ptr == '\t')
 901                 ptr++;
 902
 903         if (*ptr == '\0')
 904                 ptr = NULL;
 905
 906         return ptr;
 907 }
 908
 909 static int mdt0_is_lustre_snapshot(struct snapshot_instance *si)
 910 {
 911         struct snapshot_target *st = si->si_mdt0;
 912         char buf[MAX_BUF_SIZE * 3];
 913         FILE *fp;
 914         int rc;
 915
 916         memset(buf, 0, sizeof(buf));
 917         snprintf(buf, sizeof(buf) - 1,
 918                  DRSH" '"DIMPORT"; "DZFS
 919                  " get -H -o value lustre:magic "DSSNAME"'",
 920                  PRSH(si, st), PIMPORT(st), PZFS(st), PSSNAME(si, st));
 921         fp = popen(buf, "r");
 922         if (!fp) {
 923                 SNAPSHOT_ADD_LOG(si, "Popen fail to check snapshot "
 924                                  "on mdt0: %s\n", strerror(errno));
 925                 return -errno;
 926         }
 927
 928         if (snapshot_fgets(fp, buf, strlen(SNAPSHOT_MAGIC) + 1) == NULL) {
 929                 rc = -EINVAL;
 930         } else if (strcmp(buf, SNAPSHOT_MAGIC) == 0) {
 931                 rc = 0;
 932         } else {
 933                 fprintf(stderr,
 934                         "The target %s is not Lustre snapshot "
 935                         "or it does not exists\n", si->si_ssname);
 936                 rc = -EPERM;
 937         }
 938
 939         pclose(fp);
 940         return rc;
 941 }
 942
 943 static int target_is_mounted(struct snapshot_instance *si,
 944                              struct snapshot_target *st, const char *ssname)
 945 {
 946         char buf[MAX_BUF_SIZE];
 947         char fullname[MAX_BUF_SIZE];
 948         FILE *fp;
 949         char *ptr;
 950         int rc = 0;
 951
 952         memset(buf, 0, sizeof(buf));
 953         snprintf(buf, sizeof(buf) - 1,
 954                  DRSH" 'mount'",
 955                  PRSH(si, st));
 956         fp = popen(buf, "r");
 957         if (!fp) {
 958                 SNAPSHOT_ADD_LOG(si, "Popen fail to check target mount: %s\n",
 959                                  strerror(errno));
 960                 return -errno;
 961         }
 962
 963         memset(fullname, 0, sizeof(fullname));
 964         if (ssname)
 965                 snprintf(fullname, sizeof(fullname) - 1,
 966                          DFSNAME"@%s on ",
 967                          PFSNAME(st), ssname);
 968         else
 969                 snprintf(fullname, sizeof(fullname) - 1,
 970                          DFSNAME" on ",
 971                          PFSNAME(st));
 972
 973         while (snapshot_fgets(fp, buf, sizeof(buf)) != NULL) {
 974                 ptr = strstr(buf, fullname);
 975                 if (!ptr)
 976                         continue;
 977
 978                 ptr += strlen(fullname) + 1; /* mount point */
 979                 if (ptr >= buf + strlen(buf))
 980                         continue;
 981
 982                 ptr = strstr(ptr, "type lustre");
 983                 if (ptr) {
 984                         rc = 1;
 985                         break;
 986                 }
 987         }
 988
 989         pclose(fp);
 990         return rc;
 991 }
 992
 993 static int snapshot_get_fsname(struct snapshot_instance *si,
 994                                char *fsname, int fslen)
 995 {
 996         struct snapshot_target *st = si->si_mdt0;
 997         char buf[MAX_BUF_SIZE * 3];
 998         FILE *fp;
 999         int rc = 0;
1000
1001         memset(buf, 0, sizeof(buf));
1002         snprintf(buf, sizeof(buf) - 1,
1003                  DRSH" '"DIMPORT"; "DZFS
1004                  " get -H -o value lustre:fsname "DSSNAME"'",
1005                  PRSH(si, st), PIMPORT(st), PZFS(st), PSSNAME(si, st));
1006         fp = popen(buf, "r");
1007         if (!fp) {
1008                 SNAPSHOT_ADD_LOG(si, "Popen fail to get fsname: %s\n",
1009                                  strerror(errno));
1010                 return -errno;
1011         }
1012
1013         if (snapshot_fgets(fp, fsname, fslen) == NULL)
1014                 rc = -EINVAL;
1015
1016         pclose(fp);
1017         return rc;
1018 }
1019
1020 static int snapshot_get_mgsnode(struct snapshot_instance *si,
1021                                 char *node, int size)
1022 {
1023         char buf[MAX_BUF_SIZE * 2];
1024         struct snapshot_target *st;
1025         FILE *fp;
1026         int rc = 0;
1027
1028         st = list_entry(si->si_osts_list.next, struct snapshot_target,
1029                         st_list);
1030         memset(buf, 0, sizeof(buf));
1031         snprintf(buf, sizeof(buf) - 1,
1032                  DRSH" '"DZFS" get -H -o value lustre:mgsnode "DFSNAME"'",
1033                  PRSH(si, st), PZFS(st), PFSNAME(st));
1034         fp = popen(buf, "r");
1035         if (!fp) {
1036                 SNAPSHOT_ADD_LOG(si, "Popen fail to get mgsnode: %s\n",
1037                                  strerror(errno));
1038                 return -errno;
1039         }
1040
1041         if (snapshot_fgets(fp, node, size) == NULL)
1042                 rc = -EINVAL;
1043
1044         pclose(fp);
1045         return rc;
1046 }
1047
1048 static int snapshot_exists_check(struct snapshot_instance *si,
1049                                  struct snapshot_target *st)
1050 {
1051         char buf[MAX_BUF_SIZE * 2];
1052         FILE *fp;
1053         int rc = 0;
1054
1055         memset(buf, 0, sizeof(buf));
1056         snprintf(buf, sizeof(buf) - 1,
1057                  DRSH" '"DZFS" list "DSSNAME" 2>/dev/null'",
1058                  PRSH(si, st), PZFS(st), PSSNAME(si, st));
1059         fp = popen(buf, "r");
1060         if (!fp) {
1061                 SNAPSHOT_ADD_LOG(si, "Popen fail to create check: %s\n",
1062                                  strerror(errno));
1063                 return -errno;
1064         }
1065
1066         if (snapshot_fgets(fp, buf, sizeof(buf)) != NULL)
1067                 rc = -EEXIST;
1068
1069         pclose(fp);
1070         return rc;
1071 }
1072
1073 static int snapshot_general_check(struct snapshot_instance *si)
1074 {
1075         return mdt0_is_lustre_snapshot(si);
1076 }
1077
1078 static void snapshot_create_usage(void)
1079 {
1080         fprintf(stderr,
1081                 "Create snapshot for the given filesystem.\n"
1082                 "Usage:\n"
1083                 "snapshot_create [-b | --barrier [on | off]] "
1084                                 "[-c | --comment comment] "
1085                                 "<-F | --fsname fsname> "
1086                                 "[-h | --help] <-n | --name ssname> "
1087                                 "[-r | --rsh remote_shell]"
1088                                 "[-t | --timeout timeout]\n"
1089                 "Options:\n"
1090                 "-b: set write barrier before creating snapshot, "
1091                         "the default value is 'on'.\n"
1092                 "-c: describe what the snapshot is for, and so on.\n"
1093                 "-F: the filesystem name.\n"
1094                 "-h: for help information.\n"
1095                 "-n: the snapshot's name.\n"
1096                 "-r: the remote shell used for communication with remote "
1097                         "target, the default value is 'ssh'.\n"
1098                 "-t: the life cycle (seconds) for write barrier, "
1099                         "the default value is %d seconds.\n",
1100                 BARRIER_TIMEOUT_DEFAULT);
1101 }
1102
1103 static int snapshot_create_check(struct snapshot_instance *si)
1104 {
1105         int rc;
1106
1107         rc = snapshot_exists_check(si, si->si_mdt0);
1108         if (rc == -EEXIST)
1109                 fprintf(stderr, "The snapshot %s exists\n", si->si_ssname);
1110
1111         return rc;
1112 }
1113
1114 static int snapshot_inherit_prop(struct snapshot_instance *si,
1115                                  struct snapshot_target *st,
1116                                  char *cmd, int size)
1117 {
1118         char buf[MAX_BUF_SIZE * 3];
1119         FILE *fp;
1120         int len = 0;
1121         int rc = 0;
1122
1123         memset(buf, 0, sizeof(buf));
1124         snprintf(buf, sizeof(buf) - 1,
1125                  DRSH" \""DIMPORT"; "DZFS
1126                  " get all "DFSNAME" | grep lustre: | grep local$ | "
1127                  "awk '{ \\$1=\\\"\\\"; \\$NF=\\\"\\\"; print \\$0 }' | "
1128                  "sed -e 's/^ //'\"",
1129                  PRSH(si, st), PIMPORT(st), PZFS(st), PFSNAME(st));
1130         fp = popen(buf, "r");
1131         if (!fp) {
1132                 SNAPSHOT_ADD_LOG(si, "Popen fail to list one: %s\n",
1133                                  strerror(errno));
1134                 return -errno;
1135         }
1136
1137         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
1138                 char *ptr;
1139                 char *end;
1140
1141                 if (strncmp(buf, "lustre:fsname",
1142                             strlen("lustre:fsname")) == 0)
1143                         continue;
1144
1145                 if (strncmp(buf, "lustre:magic",
1146                             strlen("lustre:magic")) == 0)
1147                         continue;
1148
1149                 if (strncmp(buf, "lustre:ctime",
1150                             strlen("lustre:ctime")) == 0)
1151                         continue;
1152
1153                 if (strncmp(buf, "lustre:mtime",
1154                             strlen("lustre:mtime")) == 0)
1155                         continue;
1156
1157                 if (strncmp(buf, "lustre:comment",
1158                             strlen("lustre:comment")) == 0)
1159                         continue;
1160
1161                 if (strncmp(buf, "lustre:svname",
1162                             strlen("lustre:svname")) == 0)
1163                         continue;
1164
1165                 if (strncmp(buf, "lustre:mgsnode",
1166                             strlen("lustre:mgsnode")) == 0)
1167                         continue;
1168
1169                 ptr = strchr(buf, ' ');
1170                 if (!ptr) {
1171                         ptr = strchr(buf, '\t');
1172                         if (!ptr)
1173                                 continue;
1174                 }
1175
1176                 *ptr = '\0';
1177                 ptr++;
1178                 while (*ptr == ' ' || *ptr == '\t')
1179                         ptr++;
1180
1181                 if (*ptr == '\0')
1182                         continue;
1183
1184                 end = strchr(ptr, ' ');
1185                 if (!end)
1186                         end = strchr(buf, '\t');
1187                 if (end)
1188                         *end = '\0';
1189
1190                 rc = scnprintf(cmd + len, size - len - 1,
1191                                "-o %s=\"%s\" ", buf, ptr);
1192                 if (rc <= 0)
1193                         return -EOVERFLOW;
1194
1195                 len += rc;
1196         }
1197
1198         pclose(fp);
1199         return len;
1200 }
1201
1202 static int __snapshot_create(struct snapshot_instance *si,
1203                              struct list_head *head, const char *fsname,
1204                              const char *mgsnode, __u64 xtime)
1205 {
1206         struct snapshot_target *st;
1207         pid_t pid;
1208         int rc;
1209
1210         list_for_each_entry(st, head, st_list) {
1211                 st->st_status = 0;
1212                 st->st_ignored = 0;
1213                 st->st_pid = 0;
1214
1215                 pid = fork();
1216                 if (pid < 0) {
1217                         SNAPSHOT_ADD_LOG(si, "Can't fork for create snapshot "
1218                                          "(%s@%s <%s>) on the target "
1219                                          "(%s:%x:%d): %s\n",
1220                                          fsname, si->si_ssname,
1221                                          si->si_comment, st->st_host,
1222                                          st->st_role, st->st_index,
1223                                          strerror(errno));
1224                         return pid;
1225                 }
1226
1227                 /* child */
1228                 if (pid == 0) {
1229                         char cmd[MAX_BUF_SIZE];
1230                         int len;
1231
1232                         memset(cmd, 0, sizeof(cmd));
1233                         len = scnprintf(cmd, sizeof(cmd) - 1,
1234                                         DRSH" '"DZFS" snapshot "
1235                                         "-o lustre:fsname=%s "
1236                                         "-o lustre:magic=%s "
1237                                         "-o lustre:ctime=%llu "
1238                                         "-o lustre:mtime=%llu ",
1239                                         PRSH(si, st), PZFS(st), fsname,
1240                                         SNAPSHOT_MAGIC, xtime, xtime);
1241                         if (len <= 0)
1242                                 exit(-EOVERFLOW);
1243
1244                         if (si->si_comment) {
1245                                 rc = scnprintf(cmd + len, sizeof(cmd) - len - 1,
1246                                                "-o lustre:comment=\"%s\" ",
1247                                                si->si_comment);
1248                                 if (rc <= 0)
1249                                         exit(-EOVERFLOW);
1250
1251                                 len += rc;
1252                         }
1253
1254                         /* Make the inherited properties as local ones,
1255                          * then even if others changed (or removed) the
1256                          * property of the parent dataset, the snapshot
1257                          * will not be affected. */
1258                         rc = snapshot_inherit_prop(si, st, cmd + len,
1259                                                    MAX_BUF_SIZE - len - 1);
1260                         if (rc < 0) {
1261                                 SNAPSHOT_ADD_LOG(si, "Can't filter property on "
1262                                                  "target (%s:%x:%d): rc = %d\n",
1263                                                  st->st_host, st->st_role,
1264                                                  st->st_index, rc);
1265
1266                                 exit(rc);
1267                         }
1268
1269                         len += rc;
1270                         if (st->st_role & SR_OST)
1271                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1272                                               "-o lustre:svname=%s-OST%04x "
1273                                               "-o lustre:mgsnode=%s "DSSNAME"'",
1274                                               fsname, st->st_index, mgsnode,
1275                                               PSSNAME(si, st));
1276                         else if (!(st->st_role & SR_MGS) ||
1277                                 /* MGS is on MDT0 */
1278                                  si->si_mdt0 == si->si_mgs)
1279                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1280                                               "-o lustre:svname=%s-MDT%04x "
1281                                               "-o lustre:mgsnode=%s "DSSNAME"'",
1282                                               fsname, st->st_index, mgsnode,
1283                                               PSSNAME(si, st));
1284                         else
1285                                 /* separated MGS */
1286                                 rc = snprintf(cmd + len, sizeof(cmd) - len - 1,
1287                                               DSSNAME"'", PSSNAME(si, st));
1288                         if (rc <= 0)
1289                                 exit(-EOVERFLOW);
1290
1291                         rc = snapshot_exec(cmd);
1292                         if (rc)
1293                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1294                                                  "target (%s:%x:%d): rc = %d\n",
1295                                                  cmd, st->st_host, st->st_role,
1296                                                  st->st_index, rc);
1297
1298                         exit(rc);
1299                 } /* end of child */
1300
1301                 /* parent continue to run more snapshot commands in parallel. */
1302                 st->st_pid = pid;
1303         }
1304
1305         return 0;
1306 }
1307
1308 static int __snapshot_destroy(struct snapshot_instance *si,
1309                               struct list_head *head);
1310
1311 static int snapshot_create(struct snapshot_instance *si)
1312 {
1313         char *__argv[3];
1314         char buf[MAX_BUF_SIZE];
1315         struct timeval tv;
1316         char new_fsname[9];
1317         int rc = 0;
1318         int rc1 = 0;
1319         int rc2 = 0;
1320
1321         rc = snapshot_create_check(si);
1322         if (rc)
1323                 return rc;
1324
1325         rc = gettimeofday(&tv, NULL);
1326         if (rc)
1327                 return rc;
1328
1329         srandom(tv.tv_usec);
1330         snprintf(new_fsname, sizeof(new_fsname), "%08x", (__u32)random());
1331
1332         rc = snapshot_get_mgsnode(si, buf, sizeof(buf));
1333         if (rc)
1334                 return rc;
1335
1336         __argv[1] = si->si_fsname;
1337         /* 1. Get barrier */
1338         if (si->si_barrier) {
1339                 char tbuf[8];
1340
1341                 memset(tbuf, 0, sizeof(tbuf));
1342                 snprintf(tbuf, 7, "%u", si->si_timeout);
1343                 __argv[0] = "barrier_freeze";
1344                 __argv[2] = tbuf;
1345                 rc = jt_barrier_freeze(3, __argv);
1346                 if (rc) {
1347                         SNAPSHOT_ADD_LOG(si, "Can't set barrier within %u "
1348                                          "seconds on %s: rc = %d\n",
1349                                          si->si_timeout, si->si_fsname, rc);
1350
1351                         return rc;
1352                 }
1353         }
1354
1355         /* 2. Fork config llog on MGS */
1356         __argv[0] = "fork_lcfg";
1357         __argv[2] = new_fsname;
1358         rc = jt_lcfg_fork(3, __argv);
1359         if (rc) {
1360                 SNAPSHOT_ADD_LOG(si, "Can't fork config log for create "
1361                                  "snapshot %s from %s to %s: rc = %d\n",
1362                                  si->si_ssname, si->si_fsname, new_fsname, rc);
1363                 goto out;
1364         }
1365
1366         /* 3.1 Create snapshot on every MDT */
1367         rc = __snapshot_create(si, &si->si_mdts_list, new_fsname, buf,
1368                                tv.tv_sec);
1369         if (!rc)
1370                 /* 3.2 Create snapshot on every OST */
1371                 rc = __snapshot_create(si, &si->si_osts_list, new_fsname, buf,
1372                                        tv.tv_sec);
1373
1374         /* 4. Wait for all children, even though part of them maybe failed */
1375         snapshot_wait(si, &rc1);
1376
1377 out:
1378         /* 5. Put barrier */
1379         if (si->si_barrier) {
1380                 if (!rc && !rc1) {
1381                         struct barrier_ctl bc;
1382
1383                         rc = __jt_barrier_stat(__argv[1], &bc);
1384                         if (rc) {
1385                                 SNAPSHOT_ADD_LOG(si, "Can't get barrier status "
1386                                                  "on %s: rc = %d\n",
1387                                                  si->si_fsname, rc);
1388                         } else if (bc.bc_status != BS_FROZEN ||
1389                                    bc.bc_timeout <= 0) {
1390                                 SNAPSHOT_ADD_LOG(si, "The barrier expired "
1391                                                  "on %s\n", si->si_fsname);
1392                                 rc = -ETIMEDOUT;
1393                         }
1394                 }
1395
1396                 __argv[0] = "barrier_thaw";
1397                 rc2 = jt_barrier_thaw(2, __argv);
1398                 if (rc2)
1399                         SNAPSHOT_ADD_LOG(si, "Can't release barrier on %s: "
1400                                          "rc = %d\n", si->si_fsname, rc2);
1401         }
1402
1403         /* cleanup */
1404         if (rc || rc1) {
1405                 si->si_force = true;
1406                 __snapshot_destroy(si, &si->si_osts_list);
1407                 __snapshot_destroy(si, &si->si_mdts_list);
1408                 snapshot_wait(si, &rc2);
1409
1410                 __argv[0] = "erase_lcfg";
1411                 __argv[1] = new_fsname;
1412                 __argv[2] = "-q";
1413                 jt_lcfg_erase(3, __argv);
1414         }
1415
1416         return rc ? rc : (rc1 ? rc1 : rc2);
1417 }
1418
1419 int jt_snapshot_create(int argc, char **argv)
1420 {
1421         struct snapshot_instance *si;
1422         struct option long_opts[] = {
1423         { .val = 'b',   .name = "barrier",      .has_arg = optional_argument },
1424         { .val = 'c',   .name = "comment",      .has_arg = required_argument },
1425         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1426         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1427         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1428         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1429         { .val = 't',   .name = "timeout",      .has_arg = required_argument },
1430         { .name = NULL } };
1431         int rc = 0;
1432
1433         si = snapshot_init(argc, argv, long_opts, "b::c:F:hn:r:t:",
1434                            snapshot_create_usage, LOCK_EX, &rc);
1435         if (!si)
1436                 return rc;
1437
1438         if (!si->si_ssname) {
1439                 fprintf(stderr,
1440                         "Miss the snapshot name to be created\n");
1441                 snapshot_create_usage();
1442                 snapshot_fini(si);
1443                 return -EINVAL;
1444         }
1445
1446         rc = snapshot_create(si);
1447         if (rc) {
1448                 fprintf(stderr,
1449                         "Can't create the snapshot %s\n", si->si_ssname);
1450                 SNAPSHOT_ADD_LOG(si, "Can't create snapshot %s with "
1451                                  "comment <%s> barrier <%s>, timeout "
1452                                  "<%d>: %d\n",
1453                                  si->si_ssname, si->si_comment,
1454                                  si->si_barrier ? "enable" : "disable",
1455                                  si->si_barrier ? si->si_timeout : -1, rc);
1456         } else {
1457                 SNAPSHOT_ADD_LOG(si, "Create snapshot %s successfully "
1458                                  "with comment <%s>, barrier <%s>, "
1459                                  "timeout <%d>\n",
1460                                  si->si_ssname, si->si_comment,
1461                                  si->si_barrier ? "enable" : "disable",
1462                                  si->si_barrier ? si->si_timeout : -1);
1463         }
1464
1465         snapshot_fini(si);
1466         return rc;
1467 }
1468
1469 static void snapshot_destroy_usage(void)
1470 {
1471         fprintf(stderr,
1472                 "Destroy the specified snapshot.\n"
1473                 "Usage:\n"
1474                 "snapshot_destroy [-f | --force] "
1475                                  "<-F | --fsname fsname> [-h | --help] "
1476                                  "<-n | --name ssname> "
1477                                  "[-r | --rsh remote_shell]\n"
1478                 "Options:\n"
1479                 "-f: destroy the snapshot by force.\n"
1480                 "-F: the filesystem name.\n"
1481                 "-h: for help information.\n"
1482                 "-n: the snapshot's name.\n"
1483                 "-r: the remote shell used for communication with remote "
1484                         "target, the default value is 'ssh'.\n");
1485 }
1486
1487 static int snapshot_destroy_check(struct snapshot_instance *si)
1488 {
1489         struct list_head *head = &si->si_osts_list;
1490         struct snapshot_target *st;
1491         pid_t pid;
1492         int rc = 0;
1493
1494 again1:
1495         list_for_each_entry(st, head, st_list) {
1496                 st->st_status = 0;
1497                 st->st_ignored = 0;
1498                 st->st_pid = 0;
1499
1500                 pid = fork();
1501                 if (pid < 0) {
1502                         SNAPSHOT_ADD_LOG(si, "Can't fork for check snapshot "
1503                                          "%s on the target (%s:%x:%d): %s\n",
1504                                          si->si_ssname, st->st_host,
1505                                          st->st_role, st->st_index,
1506                                          strerror(errno));
1507                         return pid;
1508                 }
1509
1510                 /* child */
1511                 if (pid == 0) {
1512                         rc = snapshot_exists_check(si, st);
1513                         if (!rc)
1514                                 /* The snapshot piece does not exist */
1515                                 exit(-ESRCH);
1516
1517                         exit(rc == -EEXIST ? 0: rc);
1518                 } /* end of child */
1519
1520                 /* parent continue to run more snapshot commands in parallel. */
1521                 st->st_pid = pid;
1522         }
1523
1524         if (head == &si->si_osts_list) {
1525                 head = &si->si_mdts_list;
1526                 goto again1;
1527         }
1528
1529         snapshot_wait(si, &rc);
1530         if (rc)
1531                 return rc;
1532
1533         head = &si->si_osts_list;
1534
1535 again2:
1536         list_for_each_entry(st, head, st_list) {
1537                 if (st->st_ignored && !si->si_force) {
1538                         char name[8];
1539
1540                         snapshot_role2name(name, st->st_role, st->st_index);
1541                         fprintf(stderr,
1542                                 "Miss snapshot piece on the %s. Use '-f' "
1543                                 "option if want to destroy it by force.\n",
1544                                 name);
1545
1546                         return -ENOENT;
1547                 }
1548         }
1549
1550         if (head == &si->si_osts_list) {
1551                 head = &si->si_mdts_list;
1552                 goto again2;
1553         }
1554
1555         if (!si->si_force)
1556                 rc = snapshot_general_check(si);
1557
1558         return rc;
1559 }
1560
1561 static int __snapshot_destroy(struct snapshot_instance *si,
1562                               struct list_head *head)
1563 {
1564         struct snapshot_target *st;
1565         pid_t pid;
1566         int rc;
1567
1568         list_for_each_entry(st, head, st_list) {
1569                 if (st->st_ignored)
1570                         continue;
1571
1572                 st->st_status = 0;
1573                 st->st_pid = 0;
1574
1575                 pid = fork();
1576                 if (pid < 0) {
1577                         SNAPSHOT_ADD_LOG(si, "Can't fork for destroy snapshot "
1578                                          "%s on the target (%s:%x:%d): %s\n",
1579                                          si->si_ssname, st->st_host,
1580                                          st->st_role, st->st_index,
1581                                          strerror(errno));
1582                         return pid;
1583                 }
1584
1585                 /* child */
1586                 if (pid == 0) {
1587                         char cmd[MAX_BUF_SIZE * 2];
1588
1589                         memset(cmd, 0, sizeof(cmd));
1590                         if (si->si_force)
1591                                 snprintf(cmd, sizeof(cmd) - 1,
1592                                          DRSH" 'umount -f "DSSNAME
1593                                          " > /dev/null 2>&1; "DZFS
1594                                          " destroy -f "DSSNAME"'",
1595                                          PRSH(si, st), PSSNAME(si, st),
1596                                          PZFS(st), PSSNAME(si, st));
1597                         else
1598                                 snprintf(cmd, sizeof(cmd) - 1,
1599                                          DRSH" '"DZFS" destroy "DSSNAME"'",
1600                                          PRSH(si, st), PZFS(st),
1601                                          PSSNAME(si, st));
1602                         rc = snapshot_exec(cmd);
1603                         if (rc)
1604                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1605                                                  "target (%s:%x:%d): rc = %d\n",
1606                                                  cmd, st->st_host, st->st_role,
1607                                                  st->st_index, rc);
1608
1609                         exit(rc);
1610                 } /* end of child */
1611
1612                 /* parent continue to run more snapshot commands in parallel. */
1613                 st->st_pid = pid;
1614         }
1615
1616         return 0;
1617 }
1618
1619 static int snapshot_destroy(struct snapshot_instance *si)
1620 {
1621         char fsname[9];
1622         int rc = 0;
1623         int rc1 = 0;
1624         int rc2 = 0;
1625         int rc3 = 0;
1626
1627         rc = snapshot_destroy_check(si);
1628         if (rc)
1629                 return rc;
1630
1631         rc = snapshot_get_fsname(si, fsname, sizeof(fsname));
1632         if (rc)
1633                 return rc;
1634
1635         /* 1.1 Destroy snapshot on every OST */
1636         rc = __snapshot_destroy(si, &si->si_osts_list);
1637         if (!si->si_force) {
1638                 if (rc)
1639                         return rc;
1640
1641                 __snapshot_wait(si, &si->si_osts_list, &rc);
1642                 if (rc)
1643                         return rc;
1644         }
1645
1646         /* 1.2 Destroy snapshot on every MDT */
1647         rc1 = __snapshot_destroy(si, &si->si_mdts_list);
1648
1649         /* 2 Wait for all children, even though part of them maybe failed */
1650         snapshot_wait(si, &rc2);
1651         if (rc2 == -ENOENT && si->si_force)
1652                 rc2 = 0;
1653
1654         /* 3. Erase config llog from MGS */
1655         if ((!rc && !rc1 && !rc2) || si->si_force) {
1656                 char *__argv[3];
1657
1658                 __argv[0] = "erase_lcfg";
1659                 __argv[1] = fsname;
1660                 __argv[2] = "-q";
1661                 rc3 = jt_lcfg_erase(3, __argv);
1662                 if (rc3 && errno == ENOENT)
1663                         rc3 = 0;
1664                 if (rc3)
1665                         SNAPSHOT_ADD_LOG(si, "Can't erase config for destroy "
1666                                          "snapshot %s, fsname %s: rc = %d\n",
1667                                          si->si_ssname, fsname, rc3);
1668         }
1669
1670         return rc ? rc : (rc1 ? rc1 : (rc2 ? rc2 : rc3));
1671 }
1672
1673 int jt_snapshot_destroy(int argc, char **argv)
1674 {
1675         struct snapshot_instance *si;
1676         struct option long_opts[] = {
1677         { .val = 'f',   .name = "force",        .has_arg = no_argument },
1678         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1679         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1680         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1681         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1682         { .name = NULL } };
1683         int rc = 0;
1684
1685         si = snapshot_init(argc, argv, long_opts, "fF:hn:r:",
1686                            snapshot_destroy_usage, LOCK_EX, &rc);
1687         if (!si)
1688                 return rc;
1689
1690         if (!si->si_ssname) {
1691                 fprintf(stderr,
1692                         "Miss the snapshot name to be destroyed\n");
1693                 snapshot_destroy_usage();
1694                 snapshot_fini(si);
1695                 return -EINVAL;
1696         }
1697
1698         rc = snapshot_destroy(si);
1699         if (rc) {
1700                 fprintf(stderr,
1701                         "Can't destroy the snapshot %s\n", si->si_ssname);
1702                 SNAPSHOT_ADD_LOG(si, "Can't destroy snapshot %s with "
1703                                  "force <%s>: %d\n", si->si_ssname,
1704                                  si->si_force ? "enable" : "disable", rc);
1705         } else {
1706                 SNAPSHOT_ADD_LOG(si, "Destroy snapshot %s successfully "
1707                                  "with force <%s>\n", si->si_ssname,
1708                                  si->si_force ? "enable" : "disable");
1709         }
1710
1711         snapshot_fini(si);
1712         return rc;
1713 }
1714
1715 static void snapshot_modify_usage(void)
1716 {
1717         fprintf(stderr,
1718                 "Change the specified snapshot's name and/or comment.\n"
1719                 "Usage:\n"
1720                 "snapshot_modify [-c | --comment comment] "
1721                                 "<-F | --fsname fsname> [-h | --help] "
1722                                 "<-n | --name ssname> [-N | --new new_ssname] "
1723                                 "[-r | --rsh remote_shell]\n"
1724                 "Options:\n"
1725                 "-c: update the snapshot's comment.\n"
1726                 "-F: the filesystem name.\n"
1727                 "-h: for help information.\n"
1728                 "-n: the snapshot's name.\n"
1729                 "-N: rename the snapshot's name as the new_ssname.\n"
1730                 "-r: the remote shell used for communication with remote "
1731                         "target, the default value is 'ssh'.\n");
1732 }
1733
1734 static int snapshot_modify_check(struct snapshot_instance *si)
1735 {
1736         int rc;
1737
1738         if (si->si_new_ssname &&
1739             strcmp(si->si_ssname, si->si_new_ssname) == 0) {
1740                 fprintf(stderr, "The new snapshot's name is the same as "
1741                         "the old one %s %s.\n",
1742                         si->si_ssname, si->si_new_ssname);
1743                 return -EPERM;
1744         }
1745
1746         if (!si->si_new_ssname && !si->si_comment) {
1747                 fprintf(stderr, "Miss options, nothing to be changed.\n");
1748                 return -EINVAL;
1749         }
1750
1751         rc = mdt0_is_lustre_snapshot(si);
1752         if (!rc && si->si_new_ssname) {
1753                 rc = target_is_mounted(si, si->si_mdt0, si->si_ssname);
1754                 if (rc > 0) {
1755                         fprintf(stderr,
1756                                 "snapshot %s is mounted, can't be renamed.\n",
1757                                 si->si_ssname);
1758                         rc = -EBUSY;
1759                 }
1760         }
1761
1762         return rc;
1763 }
1764
1765 static int __snapshot_modify(struct snapshot_instance *si,
1766                              struct list_head *head, __u64 xtime)
1767 {
1768         struct snapshot_target *st;
1769         pid_t pid;
1770         int rc;
1771
1772         list_for_each_entry(st, head, st_list) {
1773                 st->st_status = 0;
1774                 st->st_ignored = 0;
1775                 st->st_pid = 0;
1776
1777                 pid = fork();
1778                 if (pid < 0) {
1779                         SNAPSHOT_ADD_LOG(si, "Can't fork for modify snapshot "
1780                                          "(%s|%s, <%s>) on the target "
1781                                          "(%s:%x:%d): %s\n",
1782                                          si->si_ssname, si->si_new_ssname,
1783                                          si->si_comment, st->st_host,
1784                                          st->st_role, st->st_index,
1785                                          strerror(errno));
1786                         return pid;
1787                 }
1788
1789                 /* child */
1790                 if (pid == 0) {
1791                         char cmd[MAX_BUF_SIZE * 5];
1792
1793                         memset(cmd, 0, sizeof(cmd));
1794                         if (si->si_new_ssname && si->si_comment)
1795                                 snprintf(cmd, sizeof(cmd) - 1,
1796                                          DRSH" '"DIMPORT"; "DZFS" rename "
1797                                          DSSNAME" "DSSNAME" && "DZFS
1798                                          " set lustre:comment=\"%s\" "DSSNAME
1799                                          " && "DZFS
1800                                          " set lustre:mtime=%llu "DSSNAME"'",
1801                                          PRSH(si, st), PIMPORT(st), PZFS(st),
1802                                          PSSNAME(si, st), PSS_NEW(si, st),
1803                                          PZFS(st), si->si_comment,
1804                                          PSS_NEW(si, st), PZFS(st), xtime,
1805                                          PSS_NEW(si, st));
1806                         else if (si->si_new_ssname)
1807                                 snprintf(cmd, sizeof(cmd) - 1,
1808                                          DRSH" '"DIMPORT"; "DZFS
1809                                          " rename "DSSNAME" "DSSNAME" && "DZFS
1810                                          " set lustre:mtime=%llu "DSSNAME"'",
1811                                          PRSH(si, st), PIMPORT(st), PZFS(st),
1812                                          PSSNAME(si, st), PSS_NEW(si, st),
1813                                          PZFS(st), xtime, PSS_NEW(si, st));
1814                         else if (si->si_comment)
1815                                 snprintf(cmd, sizeof(cmd) - 1,
1816                                          DRSH" '"DIMPORT"; "DZFS
1817                                          " set lustre:comment=\"%s\" "DSSNAME
1818                                          " && "DZFS
1819                                          " set lustre:mtime=%llu "DSSNAME"'",
1820                                          PRSH(si, st), PIMPORT(st), PZFS(st),
1821                                          si->si_comment, PSSNAME(si, st),
1822                                          PZFS(st), xtime, PSSNAME(si, st));
1823                         else
1824                                 exit(-EINVAL);
1825
1826                         rc = snapshot_exec(cmd);
1827                         if (rc)
1828                                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on "
1829                                                  "target (%s:%x:%d): rc = %d\n",
1830                                                  cmd, st->st_host, st->st_role,
1831                                                  st->st_index, rc);
1832
1833                         exit(rc);
1834                 } /* end of child */
1835
1836                 /* parent continue to run more snapshot commands in parallel. */
1837                 st->st_pid = pid;
1838         }
1839
1840         return 0;
1841 }
1842
1843 static int snapshot_modify(struct snapshot_instance *si)
1844 {
1845         time_t tt;
1846         int rc = 0;
1847         int rc1 = 0;
1848
1849         rc = snapshot_modify_check(si);
1850         if (rc)
1851                 return rc;
1852
1853         time(&tt);
1854
1855         /* Modify snapshot on every MDT */
1856         rc = __snapshot_modify(si, &si->si_mdts_list, (__u64)tt);
1857         if (!rc)
1858                 /* Modify snapshot on every OST */
1859                 rc = __snapshot_modify(si, &si->si_osts_list, (__u64)tt);
1860
1861         /* Wait for all children, even though part of them maybe failed */
1862         snapshot_wait(si, &rc1);
1863
1864         return rc ? rc : rc1;
1865 }
1866
1867 int jt_snapshot_modify(int argc, char **argv)
1868 {
1869         struct snapshot_instance *si;
1870         struct option long_opts[] = {
1871         { .val = 'c',   .name = "comment",      .has_arg = required_argument },
1872         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
1873         { .val = 'h',   .name = "help",         .has_arg = no_argument },
1874         { .val = 'n',   .name = "name",         .has_arg = required_argument },
1875         { .val = 'N',   .name = "new",          .has_arg = required_argument },
1876         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
1877         { .name = NULL } };
1878         int rc = 0;
1879
1880         si = snapshot_init(argc, argv, long_opts, "c:F:hn:N:r:",
1881                            snapshot_modify_usage, LOCK_EX, &rc);
1882         if (!si)
1883                 return rc;
1884
1885         if (!si->si_ssname) {
1886                 fprintf(stderr,
1887                         "Miss the snapshot name to be modified\n");
1888                 snapshot_modify_usage();
1889                 snapshot_fini(si);
1890                 return -EINVAL;
1891         }
1892
1893         rc = snapshot_modify(si);
1894         if (rc) {
1895                 fprintf(stderr,
1896                         "Can't modify the snapshot %s\n", si->si_ssname);
1897                 SNAPSHOT_ADD_LOG(si, "Can't modify snapshot %s with "
1898                                  "name <%s>, comment <%s>: %d\n",
1899                                  si->si_ssname, si->si_new_ssname,
1900                                  si->si_comment, rc);
1901         } else {
1902                 SNAPSHOT_ADD_LOG(si, "Modify snapshot %s successfully "
1903                                  "with name <%s>, comment <%s>\n",
1904                                  si->si_ssname, si->si_new_ssname,
1905                                  si->si_comment);
1906         }
1907
1908         snapshot_fini(si);
1909         return rc;
1910 }
1911
1912 static void snapshot_list_usage(void)
1913 {
1914         fprintf(stderr,
1915                 "List the specified snapshot or all snapshots.\n"
1916                 "Usage:\n"
1917                 "snapshot_list [-d | --detail] "
1918                               "<-F | --fsname fsname> [-h | --help] "
1919                               "[-n | --name ssname] [-r | --rsh remote_shell]\n"
1920                 "Options:\n"
1921                 "-d: list every piece for the specified snapshot.\n"
1922                 "-F: the filesystem name.\n"
1923                 "-h: for help information.\n"
1924                 "-n: the snapshot's name.\n"
1925                 "-r: the remote shell used for communication with remote "
1926                         "target, the default value is 'ssh'.\n");
1927 }
1928
1929 static int snapshot_list_one(struct snapshot_instance *si,
1930                              struct snapshot_target *st)
1931 {
1932         char buf[MAX_BUF_SIZE * 3];
1933         FILE *fp;
1934         int rc;
1935
1936         memset(buf, 0, sizeof(buf));
1937         snprintf(buf, sizeof(buf) - 1,
1938                  DRSH" \""DIMPORT"; "DZFS
1939                  " get all "DSSNAME" | grep lustre: | grep local$ | "
1940                  "awk '{ \\$1=\\\"\\\"; \\$NF=\\\"\\\"; print \\$0 }' | "
1941                  "sed -e 's/^ //'\"",
1942                  PRSH(si, st), PIMPORT(st), PZFS(st), PSSNAME(si, st));
1943         fp = popen(buf, "r");
1944         if (!fp) {
1945                 SNAPSHOT_ADD_LOG(si, "Popen fail to list one: %s\n",
1946                                  strerror(errno));
1947                 return -errno;
1948         }
1949
1950         if (si->si_detail) {
1951                 char name[8];
1952
1953                 snapshot_role2name(name, st->st_role, st->st_index);
1954                 printf("\nsnapshot_role: %s\n", name);
1955         }
1956
1957         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
1958                 __u64 xtime;
1959                 char *ptr;
1960
1961                 if (strncmp(buf, "lustre:fsname",
1962                             strlen("lustre:fsname")) == 0) {
1963                         ptr = snapshot_first_skip_blank(buf);
1964                         if (ptr)
1965                                 printf("snapshot_fsname: %s\n", ptr);
1966                         continue;
1967                 }
1968
1969                 if (strncmp(buf, "lustre:comment",
1970                             strlen("lustre:comment")) == 0) {
1971                         ptr = snapshot_first_skip_blank(buf);
1972                         if (ptr)
1973                                 printf("comment: %s\n", ptr);
1974                         continue;
1975                 }
1976
1977                 if (strncmp(buf, "lustre:ctime",
1978                             strlen("lustre:ctime")) == 0) {
1979                         ptr = snapshot_first_skip_blank(buf);
1980                         if (ptr) {
1981                                 sscanf(ptr, "%llu", &xtime);
1982                                 printf("create_time: %s",
1983                                        ctime((time_t *)&xtime));
1984                         }
1985                         continue;
1986                 }
1987
1988                 if (strncmp(buf, "lustre:mtime",
1989                             strlen("lustre:mtime")) == 0) {
1990                         ptr = snapshot_first_skip_blank(buf);
1991                         if (ptr) {
1992                                 sscanf(ptr, "%llu", &xtime);
1993                                 printf("modify_time: %s",
1994                                        ctime((time_t *)&xtime));
1995                         }
1996                         continue;
1997                 }
1998         }
1999
2000         pclose(fp);
2001         rc = target_is_mounted(si, st, si->si_ssname);
2002         if (rc < 0)
2003                 printf("status: unknown\n");
2004         else if (!rc)
2005                 printf("status: not mount\n");
2006         else
2007                 printf("status: mounted\n");
2008
2009         return rc;
2010 }
2011
2012 static int __snapshot_list(struct snapshot_instance *si,
2013                            struct list_head *head)
2014 {
2015         struct snapshot_target *st;
2016         int rc = 0;
2017
2018         list_for_each_entry(st, head, st_list) {
2019                 int rc1;
2020
2021                 rc1 = snapshot_list_one(si, st);
2022                 if (rc1 < 0 || rc >= 0)
2023                         rc = rc1;
2024         }
2025
2026         return rc;
2027 }
2028
2029 static int snapshot_list(struct snapshot_instance *si)
2030 {
2031         int rc = 0;
2032
2033         rc = snapshot_general_check(si);
2034         if (rc)
2035                 return rc;
2036
2037         printf("\nfilesystem_name: %s\nsnapshot_name: %s\n",
2038                si->si_fsname, si->si_ssname);
2039
2040         if (!si->si_detail) {
2041                 rc = snapshot_list_one(si, si->si_mdt0);
2042         } else {
2043                 int rc1;
2044
2045                 rc = __snapshot_list(si, &si->si_mdts_list);
2046                 rc1 = __snapshot_list(si, &si->si_osts_list);
2047                 if (rc >= 0)
2048                         rc = rc1;
2049         }
2050
2051         return rc < 0 ? rc : 0;
2052 }
2053
2054 static int snapshot_list_all(struct snapshot_instance *si)
2055 {
2056         struct snapshot_target *st = si->si_mdt0;
2057         struct list_sub_item {
2058                 struct list_head lsi_list;
2059                 char lsi_ssname[0];
2060         };
2061
2062         struct list_head list_sub_items;
2063         struct list_sub_item *lsi;
2064         char buf[MAX_BUF_SIZE * 2];
2065         FILE *fp;
2066         int rc = 0;
2067
2068         INIT_LIST_HEAD(&list_sub_items);
2069         memset(buf, 0, sizeof(buf));
2070         snprintf(buf, sizeof(buf) - 1,
2071                  DRSH" \""DZFS" get -H -r lustre:magic "DFSNAME
2072                  " | grep %s | awk '{ print \\$1 }' | cut -d@ -f2\"",
2073                  PRSH(si, st), PZFS(st), PFSNAME(st), SNAPSHOT_MAGIC);
2074         fp = popen(buf, "r");
2075         if (!fp) {
2076                 SNAPSHOT_ADD_LOG(si, "Popen fail to list ssnames: %s\n",
2077                                  strerror(errno));
2078                 return -errno;
2079         }
2080
2081         while (snapshot_fgets(fp, buf, MAX_BUF_SIZE) != NULL) {
2082                 int len = strlen(buf);
2083
2084                 lsi = malloc(len + 1 + sizeof(struct list_sub_item));
2085                 if (!lsi) {
2086                         SNAPSHOT_ADD_LOG(si, "NOT enough memory\n");
2087                         rc = -ENOMEM;
2088                         break;
2089                 }
2090
2091                 memcpy(lsi->lsi_ssname, buf, len + 1);
2092                 list_add(&lsi->lsi_list, &list_sub_items);
2093         }
2094
2095         pclose(fp);
2096         while (!list_empty(&list_sub_items)) {
2097                 lsi = list_entry(list_sub_items.next,
2098                                  struct list_sub_item, lsi_list);
2099                 list_del(&lsi->lsi_list);
2100                 if (!rc) {
2101                         si->si_ssname = lsi->lsi_ssname;
2102                         rc = snapshot_list(si);
2103                         si->si_ssname = NULL;
2104                 }
2105
2106                 free(lsi);
2107         }
2108
2109         return rc;
2110 }
2111
2112 int jt_snapshot_list(int argc, char **argv)
2113 {
2114         struct snapshot_instance *si;
2115         struct option long_opts[] = {
2116         { .val = 'd',   .name = "detail",       .has_arg = no_argument },
2117         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2118         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2119         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2120         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2121         { .name = NULL } };
2122         int rc = 0;
2123
2124         si = snapshot_init(argc, argv, long_opts, "dF:hn:r:",
2125                            snapshot_list_usage, LOCK_SH, &rc);
2126         if (!si)
2127                 return rc;
2128
2129         if (si->si_ssname)
2130                 rc = snapshot_list(si);
2131         else
2132                 rc = snapshot_list_all(si);
2133
2134         if (rc) {
2135                 fprintf(stderr,
2136                         "Can't list the snapshot %s\n", si->si_ssname);
2137                 SNAPSHOT_ADD_LOG(si, "Can't list snapshot %s with detail "
2138                                  "<%s>: %d\n", si->si_ssname,
2139                                  si->si_detail ? "yes" : "no", rc);
2140         }
2141
2142         snapshot_fini(si);
2143         return rc;
2144 }
2145
2146 static void snapshot_mount_usage(void)
2147 {
2148         fprintf(stderr,
2149                 "Mount the specified snapshot.\n"
2150                 "Usage:\n"
2151                 "snapshot_mount <-F | --fsname fsname> [-h | --help] "
2152                                "<-n | --name ssname> "
2153                                "[-r | --rsh remote_shell]\n"
2154                 "Options:\n"
2155                 "-F: the filesystem name.\n"
2156                 "-h: for help information.\n"
2157                 "-n: the snapshot's name.\n"
2158                 "-r: the remote shell used for communication with remote "
2159                         "target, the default value is 'ssh'.\n");
2160 }
2161
2162 static int snapshot_mount_check(struct snapshot_instance *si, char *fsname,
2163                                 int fslen, bool *mgs_running)
2164 {
2165         int rc;
2166
2167         rc = mdt0_is_lustre_snapshot(si);
2168         if (rc)
2169                 return rc;
2170
2171         rc = snapshot_get_fsname(si, fsname, fslen);
2172         if (rc < 0)
2173                 return rc;
2174
2175         rc = target_is_mounted(si, si->si_mgs, NULL);
2176         if (rc > 0) {
2177                 *mgs_running = true;
2178                 rc = 0;
2179         }
2180
2181         return rc;
2182 }
2183
2184 static int snapshot_mount_target(struct snapshot_instance *si,
2185                                  struct snapshot_target *st, const char *optstr)
2186 {
2187         char cmd[MAX_BUF_SIZE * 2];
2188         char name[8];
2189         int rc;
2190
2191         rc = target_is_mounted(si, st, si->si_ssname);
2192         if (rc < 0)
2193                 return rc;
2194
2195         if (rc > 0)
2196                 return -ESRCH;
2197
2198         memset(cmd, 0, sizeof(cmd));
2199         memset(name, 0, sizeof(name));
2200         snapshot_role2name(name, st->st_role, st->st_index);
2201         snprintf(cmd, sizeof(cmd) - 1,
2202                  DRSH" '"DIMPORT"; mkdir -p /mnt/%s_%s && mount -t lustre "
2203                  "-o rdonly_dev%s "DSSNAME" /mnt/%s_%s'",
2204                  PRSH(si, st), PIMPORT(st), si->si_ssname, name,
2205                  st != si->si_mdt0 ? "" : optstr, PSSNAME(si, st),
2206                  si->si_ssname, name);
2207         rc = snapshot_exec(cmd);
2208         if (rc)
2209                 SNAPSHOT_ADD_LOG(si, "Can't execute \"%s\" on the target "
2210                                  "(%s:%x:%d): rc = %d\n", cmd, st->st_host,
2211                                  st->st_role, st->st_index, rc);
2212
2213         return rc;
2214 }
2215
2216 static int __snapshot_mount(struct snapshot_instance *si,
2217                             struct list_head *head)
2218 {
2219         struct snapshot_target *st;
2220         pid_t pid;
2221         int rc;
2222
2223         list_for_each_entry(st, head, st_list) {
2224                 if (st == si->si_mgs || st == si->si_mdt0)
2225                         continue;
2226
2227                 st->st_status = 0;
2228                 st->st_ignored = 0;
2229                 st->st_pid = 0;
2230
2231                 pid = fork();
2232                 if (pid < 0) {
2233                         SNAPSHOT_ADD_LOG(si, "Can't fork for mount snapshot "
2234                                          "%s on target (%s:%x:%d): %s\n",
2235                                          si->si_ssname, st->st_host,
2236                                          st->st_role, st->st_index,
2237                                          strerror(errno));
2238                         return pid;
2239                 }
2240
2241                 /* child */
2242                 if (pid == 0) {
2243                         rc = snapshot_mount_target(si, st, "");
2244                         exit(rc);
2245                 }
2246
2247                 /* parent continue to run more snapshot commands in parallel. */
2248                 st->st_pid = pid;
2249         }
2250
2251         return 0;
2252 }
2253
2254 static int __snapshot_umount(struct snapshot_instance *si,
2255                              struct list_head *head);
2256
2257 static int snapshot_mount(struct snapshot_instance *si)
2258 {
2259         struct snapshot_target *st;
2260         int needed = 0;
2261         int failed = 0;
2262         int rc = 0;
2263         int rc1 = 0;
2264         char fsname[9];
2265         bool mdt0_mounted = false;
2266         bool mgs_running = false;
2267
2268         rc = snapshot_mount_check(si, fsname, sizeof(fsname), &mgs_running);
2269         if (rc < 0) {
2270                 fprintf(stderr,
2271                         "Can't mount the snapshot %s: %s\n",
2272                         si->si_ssname, strerror(-rc));
2273                 return rc;
2274         }
2275
2276         /* 1. MGS is not mounted yet, mount the MGS firstly */
2277         si->si_mgs->st_ignored = 0;
2278         si->si_mgs->st_pid = 0;
2279         if (!mgs_running) {
2280                 rc = snapshot_mount_target(si, si->si_mgs, "");
2281                 if (rc == -ESRCH) {
2282                         si->si_mgs->st_ignored = 1;
2283                         rc = 0;
2284                 }
2285
2286                 if (rc < 0) {
2287                         fprintf(stderr,
2288                                 "Can't mount the snapshot %s: %s\n",
2289                                 si->si_ssname, strerror(-rc));
2290                         return rc;
2291                 }
2292
2293                 if (si->si_mgs == si->si_mdt0)
2294                         mdt0_mounted = true;
2295         }
2296
2297         /* 2. Mount MDT0 if it is not combined with the MGS. */
2298         if (!mdt0_mounted) {
2299                 si->si_mdt0->st_ignored = 0;
2300                 si->si_mdt0->st_pid = 0;
2301                 rc = snapshot_mount_target(si, si->si_mdt0, ",nomgs");
2302                 if (rc)
2303                         goto cleanup;
2304         }
2305
2306         /* 3.1 Mount other MDTs in parallel */
2307         rc = __snapshot_mount(si, &si->si_mdts_list);
2308         if (!rc)
2309                 /* 3.2 Mount OSTs in parallel */
2310                 rc = __snapshot_mount(si, &si->si_osts_list);
2311
2312         /* Wait for all children, even though part of them maybe failed */
2313         failed = snapshot_wait(si, &rc1);
2314
2315         list_for_each_entry(st, &si->si_mdts_list, st_list) {
2316                 if (!st->st_ignored)
2317                         needed++;
2318         }
2319
2320         list_for_each_entry(st, &si->si_osts_list, st_list) {
2321                 if (!st->st_ignored)
2322                         needed++;
2323         }
2324
2325 cleanup:
2326         if (rc || rc1) {
2327                 int rc2 = 0;
2328
2329                 __snapshot_umount(si, &si->si_mdts_list);
2330                 __snapshot_umount(si, &si->si_osts_list);
2331                 snapshot_wait(si, &rc2);
2332
2333                 if (rc)
2334                         fprintf(stderr,
2335                                 "Can't mount the snapshot %s: %s\n",
2336                                 si->si_ssname, strerror(-rc));
2337                 else
2338                         fprintf(stderr,
2339                                 "%d of %d pieces of the snapshot %s "
2340                                 "can't be mounted: %s\n",
2341                                 failed, needed, si->si_ssname, strerror(-rc1));
2342
2343                 return rc ? rc : rc1;
2344         }
2345
2346         if (needed == 0) {
2347                 fprintf(stderr,
2348                         "The snapshot %s has already been mounted by other\n",
2349                         si->si_ssname);
2350                 return -EALREADY;
2351         }
2352
2353         fprintf(stdout, "mounted the snapshot %s with fsname %s\n",
2354                 si->si_ssname, fsname);
2355
2356         return 0;
2357 }
2358
2359 int jt_snapshot_mount(int argc, char **argv)
2360 {
2361         struct snapshot_instance *si;
2362         struct option long_opts[] = {
2363         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2364         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2365         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2366         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2367         { .name = NULL } };
2368         int rc = 0;
2369
2370         si = snapshot_init(argc, argv, long_opts, "F:hn:r:",
2371                            snapshot_mount_usage, LOCK_EX, &rc);
2372         if (!si)
2373                 return rc;
2374
2375         if (!si->si_ssname) {
2376                 fprintf(stderr,
2377                         "Miss the snapshot name to be mounted\n");
2378                 snapshot_mount_usage();
2379                 snapshot_fini(si);
2380                 return -EINVAL;
2381         }
2382
2383         rc = snapshot_mount(si);
2384         if (rc)
2385                 SNAPSHOT_ADD_LOG(si, "Can't mount snapshot %s: %d\n",
2386                                  si->si_ssname, rc);
2387         else
2388                 SNAPSHOT_ADD_LOG(si, "The snapshot %s is mounted\n",
2389                                  si->si_ssname);
2390
2391         snapshot_fini(si);
2392         return rc;
2393
2394 }
2395
2396 static void snapshot_umount_usage(void)
2397 {
2398         fprintf(stderr,
2399                 "Umount the specified snapshot.\n"
2400                 "Usage:\n"
2401                 "snapshot_umount <-F | --fsname fsname> [-h | --help] "
2402                                 "<-n | --name ssname> "
2403                                 "[-r | --rsh remote_shell]\n"
2404                 "Options:\n"
2405                 "-F: the filesystem name.\n"
2406                 "-h: for help information.\n"
2407                 "-n: the snapshot's name.\n"
2408                 "-r: the remote shell used for communication with remote "
2409                         "target, the default value is 'ssh'.\n");
2410 }
2411
2412 static int __snapshot_umount(struct snapshot_instance *si,
2413                              struct list_head *head)
2414 {
2415         struct snapshot_target *st;
2416         pid_t pid;
2417         int rc;
2418
2419         list_for_each_entry(st, head, st_list) {
2420                 st->st_status = 0;
2421                 st->st_ignored = 0;
2422                 st->st_pid = 0;
2423
2424                 pid = fork();
2425                 if (pid < 0) {
2426                         SNAPSHOT_ADD_LOG(si, "Can't fork for umount snapshot "
2427                                          "%s on target (%s:%x:%d): %s\n",
2428                                          si->si_ssname, st->st_host,
2429                                          st->st_role, st->st_index,
2430                                          strerror(errno));
2431                         return pid;
2432                 }
2433
2434                 /* child */
2435                 if (pid == 0) {
2436                         char cmd[MAX_BUF_SIZE];
2437
2438                         rc = target_is_mounted(si, st, si->si_ssname);
2439                         if (rc < 0)
2440                                 exit(rc);
2441
2442                         if (!rc)
2443                                 exit(-ESRCH);
2444
2445                         memset(cmd, 0, sizeof(cmd));
2446                         snprintf(cmd, sizeof(cmd) - 1,
2447                                  DRSH" 'umount "DSSNAME"'",
2448                                  PRSH(si, st), PSSNAME(si, st));
2449                         rc = snapshot_exec(cmd);
2450
2451                         exit(rc);
2452                 }
2453
2454                 /* parent continue to run more snapshot commands in parallel. */
2455                 st->st_pid = pid;
2456         }
2457
2458         return 0;
2459 }
2460
2461 static int snapshot_umount(struct snapshot_instance *si)
2462 {
2463         struct snapshot_target *st;
2464         int needed = 0;
2465         int failed;
2466         int rc = 0;
2467         int rc1 = 0;
2468         int rc2 = 0;
2469
2470         rc = snapshot_general_check(si);
2471         if (rc < 0) {
2472                 fprintf(stderr,
2473                         "Can't umount the snapshot %s: %s\n",
2474                         si->si_ssname, strerror(-rc));
2475                 return rc;
2476         }
2477
2478         rc = __snapshot_umount(si, &si->si_mdts_list);
2479         rc1 = __snapshot_umount(si, &si->si_osts_list);
2480         failed = snapshot_wait(si, &rc2);
2481
2482         list_for_each_entry(st, &si->si_mdts_list, st_list) {
2483                 if (!st->st_ignored)
2484                         needed++;
2485         }
2486
2487         list_for_each_entry(st, &si->si_osts_list, st_list) {
2488                 if (!st->st_ignored)
2489                         needed++;
2490         }
2491
2492         if (needed == 0) {
2493                 fprintf(stderr,
2494                         "The snapshot %s has not been mounted\n",
2495                         si->si_ssname);
2496                 return -EALREADY;
2497         }
2498
2499         if (failed != 0)
2500                 fprintf(stderr,
2501                         "%d of %d pieces of the snapshot %s "
2502                         "can't be umounted: %s\n",
2503                         failed, needed, si->si_ssname, strerror(-rc2));
2504
2505         return rc ? rc : (rc1 ? rc1 : rc2);
2506 }
2507
2508 int jt_snapshot_umount(int argc, char **argv)
2509 {
2510         struct snapshot_instance *si;
2511         struct option long_opts[] = {
2512         { .val = 'F',   .name = "fsname",       .has_arg = required_argument },
2513         { .val = 'h',   .name = "help",         .has_arg = no_argument },
2514         { .val = 'n',   .name = "name",         .has_arg = required_argument },
2515         { .val = 'r',   .name = "rsh",          .has_arg = required_argument },
2516         { .name = NULL } };
2517         int rc = 0;
2518
2519         si = snapshot_init(argc, argv, long_opts, "F:hn:r:",
2520                            snapshot_umount_usage, LOCK_EX, &rc);
2521         if (!si)
2522                 return rc;
2523
2524         if (!si->si_ssname) {
2525                 fprintf(stderr,
2526                         "Miss the snapshot name to be umounted\n");
2527                 snapshot_umount_usage();
2528                 snapshot_fini(si);
2529                 return -EINVAL;
2530         }
2531
2532         rc = snapshot_umount(si);
2533         if (rc < 0)
2534                 SNAPSHOT_ADD_LOG(si, "Can't umount snapshot %s: %d\n",
2535                                  si->si_ssname, rc);
2536         else
2537                 SNAPSHOT_ADD_LOG(si, "the snapshot %s have been umounted\n",
2538                                  si->si_ssname);
2539
2540         snapshot_fini(si);
2541         return rc;
2542 }