lustre/utils/lfs.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lustre/utils/lfs.c
  37  *
  38  * Author: Peter J. Braam <braam@clusterfs.com>
  39  * Author: Phil Schwan <phil@clusterfs.com>
  40  * Author: Robert Read <rread@clusterfs.com>
  41  */
  42
  43 /* for O_DIRECTORY */
  44 #ifndef _GNU_SOURCE
  45 #define _GNU_SOURCE
  46 #endif
  47
  48 #include <stdlib.h>
  49 #include <stdio.h>
  50 #include <getopt.h>
  51 #include <string.h>
  52 #include <mntent.h>
  53 #include <errno.h>
  54 #include <pwd.h>
  55 #include <grp.h>
  56 #include <sys/types.h>
  57 #include <sys/stat.h>
  58 #include <fcntl.h>
  59 #include <dirent.h>
  60 #include <time.h>
  61 #include <ctype.h>
  62 #ifdef HAVE_SYS_QUOTA_H
  63 # include <sys/quota.h>
  64 #endif
  65
  66 /* For dirname() */
  67 #include <libgen.h>
  68 #include <poll.h>
  69
  70 #include <lnet/api-support.h>
  71 #include <lnet/lnetctl.h>
  72
  73 #include <liblustre.h>
  74 #include <lustre/lustre_idl.h>
  75 #include <lustre/liblustreapi.h>
  76 #include <lustre/lustre_user.h>
  77
  78 #include <libcfs/libcfsutil.h>
  79 #include "obdctl.h"
  80
  81 unsigned int libcfs_subsystem_debug = 0;
  82
  83 /* all functions */
  84 static int lfs_setstripe(int argc, char **argv);
  85 static int lfs_find(int argc, char **argv);
  86 static int lfs_getstripe(int argc, char **argv);
  87 static int lfs_osts(int argc, char **argv);
  88 static int lfs_df(int argc, char **argv);
  89 static int lfs_check(int argc, char **argv);
  90 static int lfs_catinfo(int argc, char **argv);
  91 #ifdef HAVE_SYS_QUOTA_H
  92 static int lfs_quotachown(int argc, char **argv);
  93 static int lfs_quotacheck(int argc, char **argv);
  94 static int lfs_quotaon(int argc, char **argv);
  95 static int lfs_quotaoff(int argc, char **argv);
  96 static int lfs_setquota(int argc, char **argv);
  97 static int lfs_quota(int argc, char **argv);
  98 static int lfs_quotainv(int argc, char **argv);
  99 #endif
 100 static int lfs_flushctx(int argc, char **argv);
 101 static int lfs_join(int argc, char **argv);
 102 static int lfs_lsetfacl(int argc, char **argv);
 103 static int lfs_lgetfacl(int argc, char **argv);
 104 static int lfs_rsetfacl(int argc, char **argv);
 105 static int lfs_rgetfacl(int argc, char **argv);
 106 static int lfs_cp(int argc, char **argv);
 107 static int lfs_ls(int argc, char **argv);
 108 static int lfs_poollist(int argc, char **argv);
 109 static int lfs_changelog(int argc, char **argv);
 110 static int lfs_changelog_clear(int argc, char **argv);
 111 static int lfs_fid2path(int argc, char **argv);
 112 static int lfs_path2fid(int argc, char **argv);
 113
 114 /* all avaialable commands */
 115 command_t cmdlist[] = {
 116         {"setstripe", lfs_setstripe, 0,
 117          "Create a new file with a specific striping pattern or\n"
 118          "set the default striping pattern on an existing directory or\n"
 119          "delete the default striping pattern from an existing directory\n"
 120          "usage: setstripe [--size|-s stripe_size] [--offset|-o start_ost]\n"
 121          "                 [--count|-c stripe_count] [--pool|-p pool_name]\n"
 122          "                 <dir|filename>\n"
 123          "       or \n"
 124          "       setstripe -d <dir>   (to delete default striping)\n"
 125          "\tstripe_size:  Number of bytes on each OST (0 filesystem default)\n"
 126          "\t              Can be specified with k, m or g (in KB, MB and GB\n"
 127          "\t              respectively)\n"
 128          "\tstart_ost:    OST index of first stripe (-1 filesystem default)\n"
 129          "\tstripe_count: Number of OSTs to stripe over (0 default, -1 all)\n"
 130          "\tpool_name:    Name of OST pool"},
 131         {"getstripe", lfs_getstripe, 0,
 132          "To list the striping info for a given file or files in a\n"
 133          "directory or recursively for all files in a directory tree.\n"
 134          "usage: getstripe [--obd|-O <uuid>] [--quiet | -q] [--verbose | -v]\n"
 135          "                 [--count | -c ] [--size | -s ] [--index | -i ]\n"
 136          "                 [--offset | -o ] [--pool | -p ]\n"
 137          "                 [--recursive | -r] <dir|file> ..."},
 138         {"pool_list", lfs_poollist, 0,
 139          "List pools or pool OSTs\n"
 140          "usage: pool_list <fsname>[.<poolname>] | <pathname>\n"},
 141         {"find", lfs_find, 0,
 142          "To find files that match given parameters recursively in a directory tree.\n"
 143          "usage: find <dir|file> ... \n"
 144          "     [[!] --atime|-A [+-]N] [[!] --mtime|-M [+-]N] [[!] --ctime|-C [+-]N]\n"
 145          "     [--maxdepth|-D N] [[!] --name|-n <pattern>] [--print0|-P]\n"
 146          "     [--print|-p] [--obd|-O <uuid[s]>] [[!] --size|-s [+-]N[bkMGTP]]\n"
 147          "     [[!] --type|-t <filetype>] [[!] --gid|-g N] [[!] --group|-G <name>]\n"
 148          "     [[!] --uid|-u N] [[!] --user|-U <name>]\n"
 149          "     [[!] --pool <name>]\n"
 150          "\t !: used before an option indicates 'NOT' the requested attribute\n"
 151          "\t -: used before an value indicates 'AT MOST' the requested value\n"
 152          "\t +: used before an option indicates 'AT LEAST' the requested value\n"},
 153         {"check", lfs_check, 0,
 154          "Display the status of MDS or OSTs (as specified in the command)\n"
 155          "or all the servers (MDS and OSTs).\n"
 156          "usage: check <osts|mds|servers>"},
 157         {"catinfo", lfs_catinfo, 0,
 158          "Show information of specified type logs.\n"
 159          "usage: catinfo {keyword} [node name]\n"
 160          "\tkeywords are one of followings: config, deletions.\n"
 161          "\tnode name must be provided when use keyword config."},
 162         {"join", lfs_join, 0,
 163          "join two lustre files into one - join A, B, will be like cat B >> A & del B\n"
 164          "usage: join <filename_A> <filename_B>\n"},
 165         {"osts", lfs_osts, 0, "osts"},
 166         {"df", lfs_df, 0,
 167          "report filesystem disk space usage or inodes usage"
 168          "of each MDS/OSD.\n"
 169          "Usage: df [-i] [-h] [path]"},
 170 #ifdef HAVE_SYS_QUOTA_H
 171         {"quotachown",lfs_quotachown, 0,
 172          "Change files' owner or group on the specified filesystem.\n"
 173          "usage: quotachown [-i] <filesystem>\n"
 174          "\t-i: ignore error if file is not exist\n"},
 175         {"quotacheck", lfs_quotacheck, 0,
 176          "Scan the specified filesystem for disk usage, and create,\n"
 177          "or update quota files.\n"
 178          "usage: quotacheck [ -ug ] <filesystem>"},
 179         {"quotaon", lfs_quotaon, 0, "Turn filesystem quotas on.\n"
 180          "usage: quotaon [ -ugf ] <filesystem>"},
 181         {"quotaoff", lfs_quotaoff, 0, "Turn filesystem quotas off.\n"
 182          "usage: quotaoff [ -ug ] <filesystem>"},
 183         {"setquota", lfs_setquota, 0, "Set filesystem quotas.\n"
 184          "usage: setquota [ -u | -g ] <name> -b <block-softlimit> -B <block-hardlimit> -i <inode-softlimit> -I <inode-hardlimit> <filesystem>\n"
 185          "       setquota -t [ -u | -g ] <block-grace> <inode-grace> <filesystem>\n"
 186          "       setquota [ -u | --user | -g | --group ] <name>\n"
 187          "                [--block-softlimit <block-softlimit>]\n"
 188          "                [--block-hardlimit <block-hardlimit>]\n"
 189          "                [--inode-softlimit <inode-softlimit>]\n"
 190          "                [--inode-hardlimit <inode-hardlimit>] <filesystem>\n"
 191          "       setquota [-t] [ -u | --user | -g | --group ]\n"
 192          "                [--block-grace <block-grace>]\n"
 193          "                [--inode-grace <inode-grace>] <filesystem>\n"
 194          "       -b can be used instead of --block-softlimit/--block-grace\n"
 195          "       -B can be used instead of --block-hardlimit\n"
 196          "       -i can be used instead of --inode-softlimit/--inode-grace\n"
 197          "       -I can be used instead of --inode-hardlimit"},
 198         {"quota", lfs_quota, 0, "Display disk usage and limits.\n"
 199          "usage: quota [-v] [-o obd_uuid|-i mdt_idx|-I ost_idx] [{-u|-g <name>}|-t] <filesystem>"},
 200         {"quotainv", lfs_quotainv, 0, "Invalidate quota data.\n"
 201          "usage: quotainv [-u|-g] <filesystem>"},
 202 #endif
 203         {"flushctx", lfs_flushctx, 0, "Flush security context for current user.\n"
 204          "usage: flushctx [-k] [mountpoint...]"},
 205         {"lsetfacl", lfs_lsetfacl, 0,
 206          "Remote user setfacl for user/group on the same remote client.\n"
 207          "usage: lsetfacl [-bkndRLPvh] [{-m|-x} acl_spec] [{-M|-X} acl_file] file ..."},
 208         {"lgetfacl", lfs_lgetfacl, 0,
 209          "Remote user getfacl for user/group on the same remote client.\n"
 210          "usage: lgetfacl [-dRLPvh] file ..."},
 211         {"rsetfacl", lfs_rsetfacl, 0,
 212          "Remote user setfacl for user/group on other clients.\n"
 213          "usage: rsetfacl [-bkndRLPvh] [{-m|-x} acl_spec] [{-M|-X} acl_file] file ..."},
 214         {"rgetfacl", lfs_rgetfacl, 0,
 215          "Remote user getfacl for user/group on other clients.\n"
 216          "usage: rgetfacl [-dRLPvh] file ..."},
 217         {"cp", lfs_cp, 0,
 218          "Remote user copy files and directories.\n"
 219          "usage: cp [OPTION]... [-T] SOURCE DEST\n\tcp [OPTION]... SOURCE... DIRECTORY\n\tcp [OPTION]... -t DIRECTORY SOURCE..."},
 220         {"ls", lfs_ls, 0,
 221          "Remote user list directory contents.\n"
 222          "usage: ls [OPTION]... [FILE]..."},
 223         {"changelog", lfs_changelog, 0,
 224          "Show the metadata changes on an MDT."
 225          "\nusage: changelog [--follow] <mdtname> [startrec [endrec]]"
 226          "\n(note: --follow is only valid when run on MDT node)"},
 227         {"changelog_clear", lfs_changelog_clear, 0,
 228          "Indicate that old changelog records up to <endrec> are no longer of "
 229          "interest to consumer <id>, allowing the system to free up space.\n"
 230          "An <endrec> of 0 means all records.\n"
 231          "usage: changelog_clear <mdtname> <id> <endrec>"},
 232         {"fid2path", lfs_fid2path, 0,
 233          "Resolve the full path to a given FID. For a specific hardlink "
 234          "specify link number <linkno>.\n"
 235          /* "For a historical name, specify changelog record <recno>.\n" */
 236          "usage: fid2path <fsname|rootpath> <fid> [--link <linkno>]"
 237                 /*[--rec <recno>]*/},
 238         {"path2fid", lfs_path2fid, 0, "Display the fid for a given path.\n"
 239          "usage: path2fid <path>"},
 240         {"help", Parser_help, 0, "help"},
 241         {"exit", Parser_quit, 0, "quit"},
 242         {"quit", Parser_quit, 0, "quit"},
 243         { 0, 0, 0, NULL }
 244 };
 245
 246 static int isnumber(const char *str)
 247 {
 248         const char *ptr;
 249
 250         if (str[0] != '-' && !isdigit(str[0]))
 251                 return 0;
 252
 253         for (ptr = str + 1; *ptr != '\0'; ptr++) {
 254                 if (!isdigit(*ptr))
 255                         return 0;
 256         }
 257
 258         return 1;
 259 }
 260
 261 /* functions */
 262 static int lfs_setstripe(int argc, char **argv)
 263 {
 264         char *fname;
 265         int result;
 266         unsigned long long st_size;
 267         int  st_offset, st_count;
 268         char *end;
 269         int c;
 270         int delete = 0;
 271         char *stripe_size_arg = NULL;
 272         char *stripe_off_arg = NULL;
 273         char *stripe_count_arg = NULL;
 274         char *pool_name_arg = NULL;
 275         unsigned long long size_units;
 276
 277         struct option long_opts[] = {
 278                 {"size",        required_argument, 0, 's'},
 279                 {"count",       required_argument, 0, 'c'},
 280                 {"index",       required_argument, 0, 'i'},
 281                 {"offset",      required_argument, 0, 'o'},
 282                 {"pool",        required_argument, 0, 'p'},
 283                 {"delete",      no_argument,       0, 'd'},
 284                 {0, 0, 0, 0}
 285         };
 286
 287         st_size = 0;
 288         st_offset = -1;
 289         st_count = 0;
 290
 291 #if LUSTRE_VERSION < OBD_OCD_VERSION(2,1,0,0)
 292         if (argc == 5 && argv[1][0] != '-' &&
 293             isnumber(argv[2]) && isnumber(argv[3]) && isnumber(argv[4])) {
 294                 fprintf(stderr, "warning: deprecated usage of setstripe "
 295                         "positional parameters.  Use -c, -i, -s instead.\n");
 296                 /* for compatibility with the existing positional parameter
 297                  * usage */
 298                 fname = argv[1];
 299                 stripe_size_arg = argv[2];
 300                 stripe_off_arg = argv[3];
 301                 stripe_count_arg = argv[4];
 302                 optind = 4;
 303         } else
 304 #else
 305 #warning "remove obsolete positional parameter code"
 306 #endif
 307         {
 308                 optind = 0;
 309                 while ((c = getopt_long(argc, argv, "c:di:o:s:p:",
 310                                         long_opts, NULL)) >= 0) {
 311                         switch (c) {
 312                         case 0:
 313                                 /* Long options. */
 314                                 break;
 315                         case 'c':
 316                                 stripe_count_arg = optarg;
 317                                 break;
 318                         case 'd':
 319                                 /* delete the default striping pattern */
 320                                 delete = 1;
 321                                 break;
 322                         case 'i':
 323                         case 'o':
 324                                 stripe_off_arg = optarg;
 325                                 break;
 326                         case 's':
 327                                 stripe_size_arg = optarg;
 328                                 break;
 329                         case 'p':
 330                                 pool_name_arg = optarg;
 331                                 break;
 332                         case '?':
 333                                 return CMD_HELP;
 334                         default:
 335                                 fprintf(stderr, "error: %s: option '%s' "
 336                                                 "unrecognized\n",
 337                                                 argv[0], argv[optind - 1]);
 338                                 return CMD_HELP;
 339                         }
 340                 }
 341
 342                 fname = argv[optind];
 343
 344                 if (delete &&
 345                     (stripe_size_arg != NULL || stripe_off_arg != NULL ||
 346                      stripe_count_arg != NULL || pool_name_arg != NULL)) {
 347                         fprintf(stderr, "error: %s: cannot specify -d with "
 348                                         "-s, -c -o or -p options\n",
 349                                         argv[0]);
 350                         return CMD_HELP;
 351                 }
 352         }
 353
 354         if (optind == argc) {
 355                 fprintf(stderr, "error: %s: missing filename|dirname\n",
 356                         argv[0]);
 357                 return CMD_HELP;
 358         }
 359
 360         /* get the stripe size */
 361         if (stripe_size_arg != NULL) {
 362                 result = parse_size(stripe_size_arg, &st_size, &size_units, 0);
 363                 if (result) {
 364                         fprintf(stderr, "error: %s: bad size '%s'\n",
 365                                 argv[0], stripe_size_arg);
 366                         return result;
 367                 }
 368         }
 369         /* get the stripe offset */
 370         if (stripe_off_arg != NULL) {
 371                 st_offset = strtoul(stripe_off_arg, &end, 0);
 372                 if (*end != '\0') {
 373                         fprintf(stderr, "error: %s: bad stripe offset '%s'\n",
 374                                 argv[0], stripe_off_arg);
 375                         return CMD_HELP;
 376                 }
 377         }
 378         /* get the stripe count */
 379         if (stripe_count_arg != NULL) {
 380                 st_count = strtoul(stripe_count_arg, &end, 0);
 381                 if (*end != '\0') {
 382                         fprintf(stderr, "error: %s: bad stripe count '%s'\n",
 383                                 argv[0], stripe_count_arg);
 384                         return CMD_HELP;
 385                 }
 386         }
 387
 388         do {
 389                 result = llapi_file_create_pool(fname, st_size, st_offset,
 390                                                 st_count, 0, pool_name_arg);
 391                 if (result) {
 392                         fprintf(stderr,"error: %s: create stripe file '%s' "
 393                                 "failed\n", argv[0], fname);
 394                         break;
 395                 }
 396                 fname = argv[++optind];
 397         } while (fname != NULL);
 398
 399         return result;
 400 }
 401
 402 static int lfs_poollist(int argc, char **argv)
 403 {
 404         if (argc != 2)
 405                 return CMD_HELP;
 406
 407         return llapi_poollist(argv[1]);
 408 }
 409
 410 static int set_time(time_t *time, time_t *set, char *str)
 411 {
 412         time_t t;
 413         int res = 0;
 414
 415         if (str[0] == '+')
 416                 res = 1;
 417         else if (str[0] == '-')
 418                 res = -1;
 419
 420         if (res)
 421                 str++;
 422
 423         t = strtol(str, NULL, 0);
 424         if (*time < t * 24 * 60 * 60) {
 425                 if (res)
 426                         str--;
 427                 fprintf(stderr, "Wrong time '%s' is specified.\n", str);
 428                 return INT_MAX;
 429         }
 430
 431         *set = *time - t * 24 * 60 * 60;
 432         return res;
 433 }
 434
 435 #define USER 0
 436 #define GROUP 1
 437
 438 static int name2id(unsigned int *id, char *name, int type)
 439 {
 440         if (type == USER) {
 441                 struct passwd *entry;
 442
 443                 if (!(entry = getpwnam(name))) {
 444                         if (!errno)
 445                                 errno = ENOENT;
 446                         return -1;
 447                 }
 448
 449                 *id = entry->pw_uid;
 450         } else {
 451                 struct group *entry;
 452
 453                 if (!(entry = getgrnam(name))) {
 454                         if (!errno)
 455                                 errno = ENOENT;
 456                         return -1;
 457                 }
 458
 459                 *id = entry->gr_gid;
 460         }
 461
 462         return 0;
 463 }
 464
 465 static int id2name(char **name, unsigned int id, int type)
 466 {
 467         if (type == USER) {
 468                 struct passwd *entry;
 469
 470                 if (!(entry = getpwuid(id))) {
 471                         if (!errno)
 472                                 errno = ENOENT;
 473                         return -1;
 474                 }
 475
 476                 *name = entry->pw_name;
 477         } else {
 478                 struct group *entry;
 479
 480                 if (!(entry = getgrgid(id))) {
 481                         if (!errno)
 482                                 errno = ENOENT;
 483                         return -1;
 484                 }
 485
 486                 *name = entry->gr_name;
 487         }
 488
 489         return 0;
 490 }
 491
 492 #define FIND_POOL_OPT 3
 493 static int lfs_find(int argc, char **argv)
 494 {
 495         int new_fashion = 1;
 496         int c, ret;
 497         time_t t;
 498         struct find_param param = { .maxdepth = -1 };
 499         char str[1024];
 500         struct option long_opts[] = {
 501                 /* New find options. */
 502                 {"atime",     required_argument, 0, 'A'},
 503                 {"ctime",     required_argument, 0, 'C'},
 504                 {"mtime",     required_argument, 0, 'M'},
 505                 {"maxdepth",  required_argument, 0, 'D'},
 506                 {"gid",       required_argument, 0, 'g'},
 507                 {"group",     required_argument, 0, 'G'},
 508                 {"uid",       required_argument, 0, 'u'},
 509                 {"user",      required_argument, 0, 'U'},
 510                 {"name",      required_argument, 0, 'n'},
 511                 /* no short option for pool, p/P already used */
 512                 {"pool",      required_argument, 0, FIND_POOL_OPT},
 513                 /* --obd is considered as a new option. */
 514                 {"obd",       required_argument, 0, 'O'},
 515                 {"ost",       required_argument, 0, 'O'},
 516                 {"print",     no_argument,       0, 'P'},
 517                 {"print0",    no_argument,       0, 'p'},
 518                 /* Old find options. */
 519                 {"quiet",     no_argument,       0, 'q'},
 520                 {"recursive", no_argument,       0, 'r'},
 521                 {"size",      required_argument, 0, 's'},
 522                 {"type",      required_argument, 0, 't'},
 523                 {"verbose",   no_argument,       0, 'v'},
 524                 {0, 0, 0, 0}
 525         };
 526         int pathstart = -1;
 527         int pathend = -1;
 528         int neg_opt = 0;
 529         time_t *xtime;
 530         int *xsign;
 531         int isoption;
 532         char *endptr;
 533
 534         time(&t);
 535
 536         optind = 0;
 537         /* when getopt_long_only() hits '!' it returns 1, puts "!" in optarg */
 538         while ((c = getopt_long_only(argc,argv,"-A:C:D:g:G:M:n:PpO:qrs:t:u:U:v",
 539                                      long_opts, NULL)) >= 0) {
 540                 xtime = NULL;
 541                 xsign = NULL;
 542                 if (neg_opt)
 543                         --neg_opt;
 544                 /* '!' is part of option */
 545                 /* when getopt_long_only() finds a string which is not
 546                  * an option nor a known option argument it returns 1
 547                  * in that case if we already have found pathstart and pathend
 548                  * (i.e. we have the list of pathnames),
 549                  * the only supported value is "!"
 550                  */
 551                 isoption = (c != 1) || (strcmp(optarg, "!") == 0);
 552                 if (!isoption && pathend != -1) {
 553                         fprintf(stderr, "err: %s: filename|dirname must either "
 554                                         "precede options or follow options\n",
 555                                         argv[0]);
 556                         return CMD_HELP;
 557                 }
 558                 if (!isoption && pathstart == -1)
 559                         pathstart = optind - 1;
 560                 if (isoption && pathstart != -1 && pathend == -1) {
 561                         pathend = optind - 2;
 562                         if ((c == 1 && strcmp(optarg, "!") == 0) ||
 563                             c == 'P' || c == 'p' || c == 'O' ||
 564                             c == 'q' || c == 'r' || c == 'v')
 565                                 pathend = optind - 1;
 566                 }
 567                 switch (c) {
 568                 case 0:
 569                         /* Long options. */
 570                         break;
 571                 case 1:
 572                         /* unknown; opt is "!" or path component,
 573                          * checking done above.
 574                          */
 575                         if (strcmp(optarg, "!") == 0)
 576                                 neg_opt = 2;
 577                         break;
 578                 case 'A':
 579                         xtime = &param.atime;
 580                         xsign = &param.asign;
 581                 case 'C':
 582                         if (c == 'C') {
 583                                 xtime = &param.ctime;
 584                                 xsign = &param.csign;
 585                         }
 586                 case 'M':
 587                         if (c == 'M') {
 588                                 xtime = &param.mtime;
 589                                 xsign = &param.msign;
 590                         }
 591                         new_fashion = 1;
 592                         if (neg_opt) {
 593                                 if (optarg[0] == '-')
 594                                         optarg[0] = '+';
 595                                 else if (optarg[0] == '+')
 596                                         optarg[0] = '-';
 597                                 else {
 598                                         str[0] = '-';
 599                                         str[1] = '\0';
 600                                         strcat(str, optarg);
 601                                         optarg = str;
 602                                 }
 603                         }
 604                         ret = set_time(&t, xtime, optarg);
 605                         if (ret == INT_MAX)
 606                                 return -1;
 607                         if (ret)
 608                                 *xsign = ret;
 609                         break;
 610                 case 'D':
 611                         new_fashion = 1;
 612                         param.maxdepth = strtol(optarg, 0, 0);
 613                         break;
 614                 case 'g':
 615                         new_fashion = 1;
 616                         param.gid = strtol(optarg, &endptr, 10);
 617                         if (optarg == endptr) {
 618                                 fprintf(stderr, "Bad gid: %s\n", optarg);
 619                                 return CMD_HELP;
 620                         }
 621                         param.exclude_gid = !!neg_opt;
 622                         param.check_gid = 1;
 623                         break;
 624                 case 'G':
 625                         new_fashion = 1;
 626                         param.gid = strtol(optarg, &endptr, 10);
 627                         if (optarg == endptr) {
 628                                 ret = name2id(&param.gid, optarg, GROUP);
 629                                 if (ret != 0) {
 630                                         fprintf(stderr, "Group/GID: %s cannot "
 631                                                 "be found.\n", optarg);
 632                                         return -1;
 633                                 }
 634                         }
 635                         param.exclude_gid = !!neg_opt;
 636                         param.check_gid = 1;
 637                         break;
 638                 case 'u':
 639                         new_fashion = 1;
 640                         param.uid = strtol(optarg, &endptr, 10);
 641                         if (optarg == endptr) {
 642                                 fprintf(stderr, "Bad uid: %s\n", optarg);
 643                                 return CMD_HELP;
 644                         }
 645                         param.exclude_uid = !!neg_opt;
 646                         param.check_uid = 1;
 647                         break;
 648                 case 'U':
 649                         new_fashion = 1;
 650                         param.uid = strtol(optarg, &endptr, 10);
 651                         if (optarg == endptr) {
 652                                 ret = name2id(&param.uid, optarg, USER);
 653                                 if (ret != 0) {
 654                                         fprintf(stderr, "User/UID: %s cannot "
 655                                                 "be found.\n", optarg);
 656                                         return -1;
 657                                 }
 658                         }
 659                         param.exclude_uid = !!neg_opt;
 660                         param.check_uid = 1;
 661                         break;
 662                 case FIND_POOL_OPT:
 663                         new_fashion = 1;
 664                         if (strlen(optarg) > LOV_MAXPOOLNAME) {
 665                                 fprintf(stderr,
 666                                         "Pool name %s is too long"
 667                                         " (max is %d)\n", optarg,
 668                                         LOV_MAXPOOLNAME);
 669                                 return -1;
 670                         }
 671                         /* we do check for empty pool because empty pool
 672                          * is used to find V1 lov attributes */
 673                         strncpy(param.poolname, optarg, LOV_MAXPOOLNAME);
 674                         param.poolname[LOV_MAXPOOLNAME] = '\0';
 675                         param.exclude_pool = !!neg_opt;
 676                         param.check_pool = 1;
 677                         break;
 678                 case 'n':
 679                         new_fashion = 1;
 680                         param.pattern = (char *)optarg;
 681                         param.exclude_pattern = !!neg_opt;
 682                         break;
 683                 case 'O': {
 684                         char *buf, *token, *next, *p;
 685                         int len;
 686
 687                         len = strlen((char *)optarg);
 688                         buf = malloc(len+1);
 689                         if (buf == NULL)
 690                                 return -ENOMEM;
 691                         strcpy(buf, (char *)optarg);
 692
 693                         if (param.num_alloc_obds == 0) {
 694                                 param.obduuid = malloc(FIND_MAX_OSTS *
 695                                                        sizeof(struct obd_uuid));
 696                                 if (param.obduuid == NULL)
 697                                         return -ENOMEM;
 698                                 param.num_alloc_obds = INIT_ALLOC_NUM_OSTS;
 699                         }
 700
 701                         for (token = buf; token && *token; token = next) {
 702                                 p = strchr(token, ',');
 703                                 next = 0;
 704                                 if (p) {
 705                                         *p = 0;
 706                                         next = p+1;
 707                                 }
 708                                 strcpy((char *)&param.obduuid[param.num_obds++].uuid,
 709                                        token);
 710                         }
 711
 712                         if (buf)
 713                                 free(buf);
 714                         break;
 715                 }
 716                 case 'p':
 717                         new_fashion = 1;
 718                         param.zeroend = 1;
 719                         break;
 720                 case 'P':
 721                         break;
 722                 case 'q':
 723                         new_fashion = 0;
 724                         param.quiet++;
 725                         param.verbose = 0;
 726                         break;
 727                 case 'r':
 728                         new_fashion = 0;
 729                         param.recursive = 1;
 730                         break;
 731                 case 't':
 732                         param.exclude_type = !!neg_opt;
 733                         switch(optarg[0]) {
 734                         case 'b': param.type = S_IFBLK; break;
 735                         case 'c': param.type = S_IFCHR; break;
 736                         case 'd': param.type = S_IFDIR; break;
 737                         case 'f': param.type = S_IFREG; break;
 738                         case 'l': param.type = S_IFLNK; break;
 739                         case 'p': param.type = S_IFIFO; break;
 740                         case 's': param.type = S_IFSOCK; break;
 741 #ifdef S_IFDOOR /* Solaris only */
 742                         case 'D': param.type = S_IFDOOR; break;
 743 #endif
 744                         default: fprintf(stderr, "error: %s: bad type '%s'\n",
 745                                          argv[0], optarg);
 746                                  return CMD_HELP;
 747                         };
 748                         break;
 749                 case 's':
 750                         if (neg_opt) {
 751                                 if (optarg[0] == '-')
 752                                         optarg[0] = '+';
 753                                 else if (optarg[0] == '+')
 754                                         optarg[0] = '-';
 755                                 else {
 756                                         str[0] = '-';
 757                                         str[1] = '\0';
 758                                         strcat(str, optarg);
 759                                         optarg = str;
 760                                 }
 761                         }
 762                         if (optarg[0] == '+')
 763                                 param.size_sign = -1;
 764                         else if (optarg[0] == '-')
 765                                 param.size_sign = +1;
 766
 767                         if (param.size_sign)
 768                                 optarg++;
 769                         ret = parse_size(optarg, &param.size,
 770                                          &param.size_units, 0);
 771                         if (ret) {
 772                                 fprintf(stderr,"error: bad size '%s'\n",
 773                                         optarg);
 774                                 return ret;
 775                         }
 776                         break;
 777                 case 'v':
 778                         new_fashion = 0;
 779                         param.verbose++;
 780                         param.quiet = 0;
 781                         break;
 782                 case '?':
 783                         return CMD_HELP;
 784                 default:
 785                         fprintf(stderr, "error: %s: option '%s' unrecognized\n",
 786                                 argv[0], argv[optind - 1]);
 787                         return CMD_HELP;
 788                 };
 789         }
 790
 791         if (pathstart == -1) {
 792                 fprintf(stderr, "error: %s: no filename|pathname\n",
 793                         argv[0]);
 794                 return CMD_HELP;
 795         } else if (pathend == -1) {
 796                 /* no options */
 797                 pathend = argc;
 798         }
 799
 800         if (new_fashion) {
 801                 param.quiet = 1;
 802         } else {
 803                 static int deprecated_warning;
 804                 if (!deprecated_warning) {
 805                         fprintf(stderr, "lfs find: -q, -r, -v options "
 806                                 "deprecated.  Use 'lfs getstripe' instead.\n");
 807                         deprecated_warning = 1;
 808                 }
 809                 if (!param.recursive && param.maxdepth == -1)
 810                         param.maxdepth = 1;
 811         }
 812
 813         do {
 814                 if (new_fashion)
 815                         ret = llapi_find(argv[pathstart], &param);
 816                 else
 817                         ret = llapi_getstripe(argv[pathstart], &param);
 818         } while (++pathstart < pathend && !ret);
 819
 820         if (ret)
 821                 fprintf(stderr, "error: %s failed for %s.\n",
 822                         argv[0], argv[optind - 1]);
 823
 824         if (param.obduuid && param.num_alloc_obds)
 825                 free(param.obduuid);
 826
 827         return ret;
 828 }
 829
 830 static int lfs_getstripe(int argc, char **argv)
 831 {
 832         struct option long_opts[] = {
 833                 {"obd", 1, 0, 'O'},
 834                 {"quiet", 0, 0, 'q'},
 835                 {"recursive", 0, 0, 'r'},
 836                 {"count", 0, 0, 'c'},
 837                 {"size", 0, 0, 's'},
 838                 {"index", 0, 0, 'i'},
 839                 {"offset", 0, 0, 'o'},
 840                 {"pool", 0, 0, 'p'},
 841                 {"verbose", 0, 0, 'v'},
 842                 {0, 0, 0, 0}
 843         };
 844         char short_opts[] = "hO:qrvcsiop";
 845         int c, rc;
 846         struct find_param param = { 0 };
 847
 848         optind = 0;
 849         while ((c = getopt_long(argc, argv, short_opts,
 850                                 long_opts, NULL)) != -1) {
 851                 switch (c) {
 852                 case 'O':
 853                         if (param.obduuid) {
 854                                 fprintf(stderr,
 855                                         "error: %s: only one obduuid allowed",
 856                                         argv[0]);
 857                                 return CMD_HELP;
 858                         }
 859                         param.obduuid = (struct obd_uuid *)optarg;
 860                         break;
 861                 case 'q':
 862                         param.quiet++;
 863                         break;
 864                 case 'r':
 865                         param.recursive = 1;
 866                         break;
 867                 case 'v':
 868                         param.verbose = VERBOSE_ALL | VERBOSE_DETAIL;
 869                         param.quiet = 0;
 870                         break;
 871                 case 'c':
 872                         param.verbose |= VERBOSE_COUNT;
 873                         break;
 874                 case 's':
 875                         param.verbose |= VERBOSE_SIZE;
 876                         break;
 877                 case 'i':
 878                 case 'o':
 879                         param.verbose |= VERBOSE_OFFSET;
 880                         break;
 881                 case 'p':
 882                         param.verbose |= VERBOSE_POOL;
 883                         break;
 884                 case '?':
 885                         return CMD_HELP;
 886                 default:
 887                         fprintf(stderr, "error: %s: option '%s' unrecognized\n",
 888                                 argv[0], argv[optind - 1]);
 889                         return CMD_HELP;
 890                 }
 891         }
 892
 893         if (optind >= argc)
 894                 return CMD_HELP;
 895
 896         param.maxdepth = param.recursive ? -1 : 1;
 897
 898         do {
 899                 rc = llapi_getstripe(argv[optind], &param);
 900         } while (++optind < argc && !rc);
 901
 902         if (rc)
 903                 fprintf(stderr, "error: %s failed for %s.\n",
 904                         argv[0], argv[optind - 1]);
 905         return rc;
 906 }
 907
 908 static int lfs_osts(int argc, char **argv)
 909 {
 910         FILE *fp;
 911         struct mntent *mnt = NULL;
 912         struct find_param param;
 913         int rc=0;
 914
 915         if (argc != 1)
 916                 return CMD_HELP;
 917
 918         fp = setmntent(MOUNTED, "r");
 919
 920         if (fp == NULL) {
 921                  fprintf(stderr, "%s: setmntent(%s): %s:", argv[0], MOUNTED,
 922                         strerror (errno));
 923         } else {
 924                 mnt = getmntent(fp);
 925                 while (feof(fp) == 0 && ferror(fp) ==0) {
 926                         memset(&param, 0, sizeof(param));
 927                         if (llapi_is_lustre_mnt(mnt)) {
 928                                 rc = llapi_getstripe(mnt->mnt_dir, &param);
 929                                 if (rc)
 930                                         fprintf(stderr,
 931                                                "error: %s: failed on %s\n",
 932                                                argv[0], mnt->mnt_dir);
 933                         }
 934                         mnt = getmntent(fp);
 935                 }
 936                 endmntent(fp);
 937         }
 938
 939         return rc;
 940 }
 941
 942 #define COOK(value)                                                     \
 943 ({                                                                      \
 944         int radix = 0;                                                  \
 945         while (value > 1024) {                                          \
 946                 value /= 1024;                                          \
 947                 radix++;                                                \
 948         }                                                               \
 949         radix;                                                          \
 950 })
 951 #define UUF     "%-20s"
 952 #define CSF     "%9s"
 953 #define CDF     "%9llu"
 954 #define HSF     "%8s"
 955 #define HDF     "%6.1f"
 956 #define RSF     "%5s"
 957 #define RDF     "%4d%%"
 958
 959 static int path2mnt(char *path, FILE *fp, char *mntdir, int dir_len)
 960 {
 961         char rpath[PATH_MAX] = {'\0'};
 962         struct mntent *mnt;
 963         int rc, len, out_len = 0;
 964
 965         if (!realpath(path, rpath)) {
 966                 rc = -errno;
 967                 fprintf(stderr, "error: lfs df: invalid path '%s': %s\n",
 968                         path, strerror(-rc));
 969                 return rc;
 970         }
 971
 972         len = 0;
 973         mnt = getmntent(fp);
 974         while (feof(fp) == 0 && ferror(fp) == 0) {
 975                 if (llapi_is_lustre_mnt(mnt)) {
 976                         len = strlen(mnt->mnt_dir);
 977                         if (len > out_len &&
 978                             !strncmp(rpath, mnt->mnt_dir, len)) {
 979                                 out_len = len;
 980                                 memset(mntdir, 0, dir_len);
 981                                 strncpy(mntdir, mnt->mnt_dir, dir_len);
 982                         }
 983                 }
 984                 mnt = getmntent(fp);
 985         }
 986
 987         if (out_len > 0)
 988                 return 0;
 989
 990         fprintf(stderr, "error: lfs df: %s isn't mounted on lustre\n", path);
 991         return -EINVAL;
 992 }
 993
 994 static int showdf(char *mntdir, struct obd_statfs *stat,
 995                   char *uuid, int ishow, int cooked,
 996                   char *type, int index, int rc)
 997 {
 998         long long avail, used, total;
 999         double ratio = 0;
1000         char *suffix = "KMGTPEZY";
1001         /* Note if we have >2^64 bytes/fs these buffers will need to be grown */
1002         char tbuf[20], ubuf[20], abuf[20], rbuf[20];
1003
1004         if (!uuid || !stat)
1005                 return -EINVAL;
1006
1007         switch (rc) {
1008         case 0:
1009                 if (ishow) {
1010                         avail = stat->os_ffree;
1011                         used = stat->os_files - stat->os_ffree;
1012                         total = stat->os_files;
1013                 } else {
1014                         int shift = cooked ? 0 : 10;
1015
1016                         avail = (stat->os_bavail * stat->os_bsize) >> shift;
1017                         used  = ((stat->os_blocks - stat->os_bfree) *
1018                                  stat->os_bsize) >> shift;
1019                         total = (stat->os_blocks * stat->os_bsize) >> shift;
1020                 }
1021
1022                 if (total > 0)
1023                         ratio = (double)used / (double)total;
1024
1025                 if (cooked) {
1026                         int i;
1027                         double cook_val;
1028
1029                         cook_val = (double)total;
1030                         i = COOK(cook_val);
1031                         if (i > 0)
1032                                 sprintf(tbuf, HDF"%c", cook_val, suffix[i - 1]);
1033                         else
1034                                 sprintf(tbuf, CDF, total);
1035
1036                         cook_val = (double)used;
1037                         i = COOK(cook_val);
1038                         if (i > 0)
1039                                 sprintf(ubuf, HDF"%c", cook_val, suffix[i - 1]);
1040                         else
1041                                 sprintf(ubuf, CDF, used);
1042
1043                         cook_val = (double)avail;
1044                         i = COOK(cook_val);
1045                         if (i > 0)
1046                                 sprintf(abuf, HDF"%c", cook_val, suffix[i - 1]);
1047                         else
1048                                 sprintf(abuf, CDF, avail);
1049                 } else {
1050                         sprintf(tbuf, CDF, total);
1051                         sprintf(ubuf, CDF, used);
1052                         sprintf(abuf, CDF, avail);
1053                 }
1054
1055                 sprintf(rbuf, RDF, (int)(ratio * 100));
1056                 printf(UUF" "CSF" "CSF" "CSF" "RSF" %-s",
1057                        uuid, tbuf, ubuf, abuf, rbuf, mntdir);
1058                 if (type)
1059                         printf("[%s:%d]\n", type, index);
1060                 else
1061                         printf("\n");
1062
1063                 break;
1064         case -ENODATA:
1065                 printf(UUF": inactive device\n", uuid);
1066                 break;
1067         default:
1068                 printf(UUF": %s\n", uuid, strerror(-rc));
1069                 break;
1070         }
1071
1072         return 0;
1073 }
1074
1075 static int mntdf(char *mntdir, int ishow, int cooked)
1076 {
1077         struct obd_statfs stat_buf, sum = { .os_bsize = 1 };
1078         struct obd_uuid uuid_buf;
1079         __u32 index;
1080         int rc;
1081
1082         if (ishow)
1083                 printf(UUF" "CSF" "CSF" "CSF" "RSF" %-s\n",
1084                        "UUID", "Inodes", "IUsed", "IFree",
1085                        "IUse%", "Mounted on");
1086         else
1087                 printf(UUF" "CSF" "CSF" "CSF" "RSF" %-s\n",
1088                        "UUID", cooked ? "bytes" : "1K-blocks",
1089                        "Used", "Available", "Use%", "Mounted on");
1090
1091         for (index = 0; ; index++) {
1092                 memset(&stat_buf, 0, sizeof(struct obd_statfs));
1093                 memset(&uuid_buf, 0, sizeof(struct obd_uuid));
1094                 rc = llapi_obd_statfs(mntdir, LL_STATFS_MDC, index,
1095                                       &stat_buf, &uuid_buf);
1096                 if (rc == -ENODEV)
1097                         break;
1098
1099                 if (rc == -EAGAIN)
1100                         continue;
1101
1102                 if (rc == -ENOTCONN || rc == -ETIMEDOUT || rc == -EIO ||
1103                     rc == -ENODATA || rc == 0) {
1104                         showdf(mntdir, &stat_buf, obd_uuid2str(&uuid_buf),
1105                                ishow, cooked, "MDT", index, rc);
1106                 } else {
1107                         fprintf(stderr,
1108                                 "error: llapi_obd_statfs(%s): %s (%d)\n",
1109                                 obd_uuid2str(&uuid_buf), strerror(-rc), rc);
1110                         return rc;
1111                 }
1112                 if (rc == 0) {
1113                         sum.os_ffree += stat_buf.os_ffree;
1114                         sum.os_files += stat_buf.os_files;
1115                 }
1116         }
1117
1118         for (index = 0; ; index++) {
1119                 memset(&stat_buf, 0, sizeof(struct obd_statfs));
1120                 memset(&uuid_buf, 0, sizeof(struct obd_uuid));
1121                 rc = llapi_obd_statfs(mntdir, LL_STATFS_LOV, index,
1122                                       &stat_buf, &uuid_buf);
1123                 if (rc == -ENODEV)
1124                         break;
1125
1126                 if (rc == -EAGAIN)
1127                         continue;
1128
1129                 if (rc == -ENOTCONN || rc == -ETIMEDOUT || rc == -EIO ||
1130                     rc == -ENODATA || rc == 0) {
1131                         showdf(mntdir, &stat_buf, obd_uuid2str(&uuid_buf),
1132                                ishow, cooked, "OST", index, rc);
1133                 } else {
1134                         fprintf(stderr,
1135                                 "error: llapi_obd_statfs failed: %s (%d)\n",
1136                                 strerror(-rc), rc);
1137                         return rc;
1138                 }
1139                 if (rc == 0) {
1140                         sum.os_blocks += stat_buf.os_blocks * stat_buf.os_bsize;
1141                         sum.os_bfree  += stat_buf.os_bfree * stat_buf.os_bsize;
1142                         sum.os_bavail += stat_buf.os_bavail * stat_buf.os_bsize;
1143                 }
1144         }
1145
1146         printf("\n");
1147         showdf(mntdir, &sum, "filesystem summary:", ishow, cooked, NULL, 0,0);
1148
1149         return 0;
1150 }
1151
1152 static int lfs_df(int argc, char **argv)
1153 {
1154         FILE *fp;
1155         char *path = NULL;
1156         struct mntent *mnt = NULL;
1157         char *mntdir = NULL;
1158         int ishow = 0, cooked = 0;
1159         int c, rc = 0;
1160
1161         optind = 0;
1162         while ((c = getopt(argc, argv, "ih")) != -1) {
1163                 switch (c) {
1164                 case 'i':
1165                         ishow = 1;
1166                         break;
1167                 case 'h':
1168                         cooked = 1;
1169                         break;
1170                 default:
1171                         return CMD_HELP;
1172                 }
1173         }
1174         if (optind < argc )
1175                 path = argv[optind];
1176
1177         fp = setmntent(MOUNTED, "r");
1178         if (fp == NULL) {
1179                 rc = -errno;
1180                 fprintf(stderr, "error: %s: open %s failed( %s )\n",
1181                         argv[0], MOUNTED, strerror(errno));
1182                 return rc;
1183         }
1184
1185         if ((mntdir = malloc(PATH_MAX)) == NULL) {
1186                 fprintf(stderr, "error: cannot allocate %d bytes\n",
1187                         PATH_MAX);
1188                 return -ENOMEM;
1189         }
1190         memset(mntdir, 0, PATH_MAX);
1191
1192         if (path) {
1193                 rc = path2mnt(path, fp, mntdir, PATH_MAX);
1194                 if (rc) {
1195                         endmntent(fp);
1196                         free(mntdir);
1197                         return rc;
1198                 }
1199
1200                 rc = mntdf(mntdir, ishow, cooked);
1201                 printf("\n");
1202                 endmntent(fp);
1203         } else {
1204                 mnt = getmntent(fp);
1205                 while (feof(fp) == 0 && ferror(fp) == 0) {
1206                         if (llapi_is_lustre_mnt(mnt)) {
1207                                 rc = mntdf(mnt->mnt_dir, ishow, cooked);
1208                                 if (rc)
1209                                         break;
1210                                 printf("\n");
1211                         }
1212                         mnt = getmntent(fp);
1213                 }
1214                 endmntent(fp);
1215         }
1216
1217         free(mntdir);
1218         return rc;
1219 }
1220
1221 static int lfs_check(int argc, char **argv)
1222 {
1223         int rc;
1224         FILE *fp;
1225         struct mntent *mnt = NULL;
1226         int num_types = 1;
1227         char *obd_types[2];
1228         char obd_type1[4];
1229         char obd_type2[4];
1230
1231         if (argc != 2)
1232                 return CMD_HELP;
1233
1234         obd_types[0] = obd_type1;
1235         obd_types[1] = obd_type2;
1236
1237         if (strcmp(argv[1], "osts") == 0) {
1238                 strcpy(obd_types[0], "osc");
1239         } else if (strcmp(argv[1], "mds") == 0) {
1240                 strcpy(obd_types[0], "mdc");
1241         } else if (strcmp(argv[1], "servers") == 0) {
1242                 num_types = 2;
1243                 strcpy(obd_types[0], "osc");
1244                 strcpy(obd_types[1], "mdc");
1245         } else {
1246                 fprintf(stderr, "error: %s: option '%s' unrecognized\n",
1247                                 argv[0], argv[1]);
1248                         return CMD_HELP;
1249         }
1250
1251         fp = setmntent(MOUNTED, "r");
1252         if (fp == NULL) {
1253                  fprintf(stderr, "setmntent(%s): %s:", MOUNTED,
1254                         strerror (errno));
1255         } else {
1256                 mnt = getmntent(fp);
1257                 while (feof(fp) == 0 && ferror(fp) ==0) {
1258                         if (llapi_is_lustre_mnt(mnt))
1259                                 break;
1260                         mnt = getmntent(fp);
1261                 }
1262                 endmntent(fp);
1263         }
1264
1265         if (!mnt) {
1266                 fprintf(stderr, "No suitable Lustre mount found\n");
1267                 return -1;
1268         }
1269
1270         rc = llapi_target_iterate(num_types, obd_types,
1271                                   mnt->mnt_dir, llapi_ping_target);
1272
1273         if (rc)
1274                 fprintf(stderr, "error: %s: %s status failed\n",
1275                                 argv[0],argv[1]);
1276
1277         return rc;
1278
1279 }
1280
1281 static int lfs_catinfo(int argc, char **argv)
1282 {
1283         FILE *fp;
1284         struct mntent *mnt = NULL;
1285         int rc;
1286
1287         if (argc < 2 || (!strcmp(argv[1],"config") && argc < 3))
1288                 return CMD_HELP;
1289
1290         if (strcmp(argv[1], "config") && strcmp(argv[1], "deletions"))
1291                 return CMD_HELP;
1292
1293         fp = setmntent(MOUNTED, "r");
1294         if (fp == NULL) {
1295                  fprintf(stderr, "setmntent(%s): %s:", MOUNTED,
1296                          strerror(errno));
1297         } else {
1298                 mnt = getmntent(fp);
1299                 while (feof(fp) == 0 && ferror(fp) == 0) {
1300                         if (llapi_is_lustre_mnt(mnt))
1301                                 break;
1302                         mnt = getmntent(fp);
1303                 }
1304                 endmntent(fp);
1305         }
1306
1307         if (mnt) {
1308                 if (argc == 3)
1309                         rc = llapi_catinfo(mnt->mnt_dir, argv[1], argv[2]);
1310                 else
1311                         rc = llapi_catinfo(mnt->mnt_dir, argv[1], NULL);
1312         } else {
1313                 fprintf(stderr, "no lustre_lite mounted.\n");
1314                 rc = -1;
1315         }
1316
1317         return rc;
1318 }
1319
1320 int lfs_join(int argc, char **argv)
1321 {
1322         char *name_head, *name_tail;
1323         int fd, rc;
1324         loff_t size;
1325
1326         if (argc != 3)
1327                 return CMD_HELP;
1328         name_head = argv[1];
1329         fd = open(name_head, O_WRONLY);
1330         if (fd < 0) {
1331                 fprintf(stderr, "Can not open name_head %s rc=%d\n",
1332                         name_head, fd);
1333                 return fd;
1334         }
1335         size = lseek(fd, 0, SEEK_END);
1336         if (size % JOIN_FILE_ALIGN) {
1337                 fprintf(stderr,"head file %s size %llu must be mutiple of %d\n",
1338                         name_head, (long long)size, JOIN_FILE_ALIGN);
1339                 rc = -EINVAL;
1340                 goto out;
1341         }
1342         name_tail = argv[2];
1343         rc = ioctl(fd, LL_IOC_JOIN, name_tail);
1344 out:
1345         close(fd);
1346         if (rc) {
1347                 fprintf(stderr, "Lustre joining files: %s, %s, failed\n",
1348                         argv[1], argv[2]);
1349         }
1350         return rc;
1351 }
1352
1353 #ifdef HAVE_SYS_QUOTA_H
1354 static int lfs_quotachown(int argc, char **argv)
1355 {
1356
1357         int c,rc;
1358         int flag = 0;
1359
1360         optind = 0;
1361         while ((c = getopt(argc, argv, "i")) != -1) {
1362                 switch (c) {
1363                 case 'i':
1364                         flag++;
1365                         break;
1366                 default:
1367                         fprintf(stderr, "error: %s: option '-%c' "
1368                                         "unrecognized\n", argv[0], c);
1369                         return CMD_HELP;
1370                 }
1371         }
1372         if (optind == argc)
1373                 return CMD_HELP;
1374         rc = llapi_quotachown(argv[optind], flag);
1375         if(rc)
1376                 fprintf(stderr,"error: change file owner/group failed.\n");
1377         return rc;
1378 }
1379
1380 static int lfs_quotacheck(int argc, char **argv)
1381 {
1382         int c, check_type = 0;
1383         char *mnt;
1384         struct if_quotacheck qchk;
1385         struct if_quotactl qctl;
1386         char *obd_type = (char *)qchk.obd_type;
1387         int rc;
1388
1389         memset(&qchk, 0, sizeof(qchk));
1390
1391         optind = 0;
1392         while ((c = getopt(argc, argv, "ug")) != -1) {
1393                 switch (c) {
1394                 case 'u':
1395                         check_type |= 0x01;
1396                         break;
1397                 case 'g':
1398                         check_type |= 0x02;
1399                         break;
1400                 default:
1401                         fprintf(stderr, "error: %s: option '-%c' "
1402                                         "unrecognized\n", argv[0], c);
1403                         return CMD_HELP;
1404                 }
1405         }
1406
1407         if (check_type)
1408                 check_type--;
1409         else    /* do quotacheck for both user & group quota by default */
1410                 check_type = 0x02;
1411
1412         if (argc == optind)
1413                 return CMD_HELP;
1414
1415         mnt = argv[optind];
1416
1417         rc = llapi_quotacheck(mnt, check_type);
1418         if (rc) {
1419                 fprintf(stderr, "quotacheck failed: %s\n", strerror(errno));
1420                 return rc;
1421         }
1422
1423         rc = llapi_poll_quotacheck(mnt, &qchk);
1424         if (rc) {
1425                 if (*obd_type)
1426                         fprintf(stderr, "%s %s ", obd_type,
1427                                 obd_uuid2str(&qchk.obd_uuid));
1428                 fprintf(stderr, "quota check failed: %s\n", strerror(errno));
1429                 return rc;
1430         }
1431
1432         memset(&qctl, 0, sizeof(qctl));
1433         qctl.qc_cmd = LUSTRE_Q_QUOTAON;
1434         qctl.qc_type = check_type;
1435         rc = llapi_quotactl(mnt, &qctl);
1436         if (rc && errno != EALREADY) {
1437                 if (*obd_type)
1438                         fprintf(stderr, "%s %s ", (char *)qctl.obd_type,
1439                                 obd_uuid2str(&qctl.obd_uuid));
1440                 fprintf(stderr, "%s turn on quota failed: %s\n",
1441                         argv[0], strerror(errno));
1442                 return rc;
1443         }
1444
1445         return 0;
1446 }
1447
1448 static int lfs_quotaon(int argc, char **argv)
1449 {
1450         int c;
1451         char *mnt;
1452         struct if_quotactl qctl;
1453         char *obd_type = (char *)qctl.obd_type;
1454         int rc;
1455
1456         memset(&qctl, 0, sizeof(qctl));
1457         qctl.qc_cmd = LUSTRE_Q_QUOTAON;
1458
1459         optind = 0;
1460         while ((c = getopt(argc, argv, "ugf")) != -1) {
1461                 switch (c) {
1462                 case 'u':
1463                         qctl.qc_type |= 0x01;
1464                         break;
1465                 case 'g':
1466                         qctl.qc_type |= 0x02;
1467                         break;
1468                 case 'f':
1469                         qctl.qc_cmd = LUSTRE_Q_QUOTAOFF;
1470                         break;
1471                 default:
1472                         fprintf(stderr, "error: %s: option '-%c' "
1473                                         "unrecognized\n", argv[0], c);
1474                         return CMD_HELP;
1475                 }
1476         }
1477
1478         if (qctl.qc_type)
1479                 qctl.qc_type--;
1480         else /* by default, enable quota for both user & group */
1481                 qctl.qc_type = 0x02;
1482
1483         if (argc == optind)
1484                 return CMD_HELP;
1485
1486         mnt = argv[optind];
1487
1488         rc = llapi_quotactl(mnt, &qctl);
1489         if (rc) {
1490                 if (errno == EALREADY) {
1491                         fprintf(stderr, "\n%s quotas are enabled already.\n",
1492                                 qctl.qc_type == 0x02 ? "user/group" :
1493                                 (qctl.qc_type == 0x00 ? "user" : "group"));
1494                         rc = 0;
1495                 } else {
1496                         if (*obd_type)
1497                                 fprintf(stderr, "%s %s ", obd_type,
1498                                         obd_uuid2str(&qctl.obd_uuid));
1499                         fprintf(stderr, "%s failed: %s\n", argv[0],
1500                                 strerror(errno));
1501                 }
1502         }
1503
1504         return rc;
1505 }
1506
1507 static int lfs_quotaoff(int argc, char **argv)
1508 {
1509         int c;
1510         char *mnt;
1511         struct if_quotactl qctl;
1512         char *obd_type = (char *)qctl.obd_type;
1513         int rc;
1514
1515         memset(&qctl, 0, sizeof(qctl));
1516         qctl.qc_cmd = LUSTRE_Q_QUOTAOFF;
1517
1518         optind = 0;
1519         while ((c = getopt(argc, argv, "ug")) != -1) {
1520                 switch (c) {
1521                 case 'u':
1522                         qctl.qc_type |= 0x01;
1523                         break;
1524                 case 'g':
1525                         qctl.qc_type |= 0x02;
1526                         break;
1527                 default:
1528                         fprintf(stderr, "error: %s: option '-%c' "
1529                                         "unrecognized\n", argv[0], c);
1530                         return CMD_HELP;
1531                 }
1532         }
1533
1534         if (qctl.qc_type)
1535                 qctl.qc_type--;
1536         else /* by default, disable quota for both user & group */
1537                 qctl.qc_type = 0x02;
1538
1539         if (argc == optind)
1540                 return CMD_HELP;
1541
1542         mnt = argv[optind];
1543
1544         rc = llapi_quotactl(mnt, &qctl);
1545         if (rc) {
1546                 if (errno == EALREADY) {
1547                         fprintf(stderr, "\n%s quotas are disabled already.\n",
1548                                 qctl.qc_type == 0x02 ? "user/group" :
1549                                 (qctl.qc_type == 0x00 ? "user" : "group"));
1550                         rc = 0;
1551                 } else {
1552                         if (*obd_type)
1553                                 fprintf(stderr, "%s %s ", obd_type,
1554                                         obd_uuid2str(&qctl.obd_uuid));
1555                         fprintf(stderr, "quotaoff failed: %s\n",
1556                                 strerror(errno));
1557                 }
1558         }
1559
1560         return rc;
1561 }
1562
1563 static int lfs_quotainv(int argc, char **argv)
1564 {
1565         int c;
1566         char *mnt;
1567         struct if_quotactl qctl;
1568         int rc;
1569
1570         memset(&qctl, 0, sizeof(qctl));
1571         qctl.qc_cmd = LUSTRE_Q_INVALIDATE;
1572
1573         optind = 0;
1574         while ((c = getopt(argc, argv, "ugf")) != -1) {
1575                 switch (c) {
1576                 case 'u':
1577                         qctl.qc_type |= 0x01;
1578                         break;
1579                 case 'g':
1580                         qctl.qc_type |= 0x02;
1581                         break;
1582                 case 'f':
1583                         qctl.qc_cmd = LUSTRE_Q_FINVALIDATE;
1584                         break;
1585                 default:
1586                         fprintf(stderr, "error: %s: option '-%c' "
1587                                         "unrecognized\n", argv[0], c);
1588                         return CMD_HELP;
1589                 }
1590         }
1591
1592         if (qctl.qc_type)
1593                 qctl.qc_type--;
1594         else /* by default, invalidate quota for both user & group */
1595                 qctl.qc_type = 0x02;
1596
1597         if (argc == optind)
1598                 return CMD_HELP;
1599
1600         mnt = argv[optind];
1601
1602         rc = llapi_quotactl(mnt, &qctl);
1603         if (rc) {
1604                 fprintf(stderr, "quotainv failed: %s\n", strerror(errno));
1605                 return rc;
1606         }
1607
1608         return 0;
1609 }
1610
1611 #define ARG2INT(nr, str, msg)                                           \
1612 do {                                                                    \
1613         char *endp;                                                     \
1614         nr = strtol(str, &endp, 0);                                     \
1615         if (*endp) {                                                    \
1616                 fprintf(stderr, "error: bad %s: %s\n", msg, str);       \
1617                 return CMD_HELP;                                        \
1618         }                                                               \
1619 } while (0)
1620
1621 #define ADD_OVERFLOW(a,b) ((a + b) < a) ? (a = ULONG_MAX) : (a = a + b)
1622
1623 /* Convert format time string "XXwXXdXXhXXmXXs" into seconds value
1624  * returns the value or ULONG_MAX on integer overflow or incorrect format
1625  * Notes:
1626  *        1. the order of specifiers is arbitrary (may be: 5w3s or 3s5w)
1627  *        2. specifiers may be encountered multiple times (2s3s is 5 seconds)
1628  *        3. empty integer value is interpreted as 0
1629  */
1630
1631 static unsigned long str2sec(const char* timestr) {
1632         const char spec[] = "smhdw";
1633         const unsigned long mult[] = {1, 60, 60*60, 24*60*60, 7*24*60*60};
1634         unsigned long val = 0;
1635         char *tail;
1636
1637         if (strpbrk(timestr, spec) == NULL) {
1638                 /* no specifiers inside the time string,
1639                    should treat it as an integer value */
1640                 val = strtoul(timestr, &tail, 10);
1641                 return *tail ? ULONG_MAX : val;
1642         }
1643
1644         /* format string is XXwXXdXXhXXmXXs */
1645         while (*timestr) {
1646                 unsigned long v;
1647                 int ind;
1648                 char* ptr;
1649
1650                 v = strtoul(timestr, &tail, 10);
1651                 if (v == ULONG_MAX || *tail == '\0')
1652                         /* value too large (ULONG_MAX or more)
1653                            or missing specifier */
1654                         goto error;
1655
1656                 ptr = strchr(spec, *tail);
1657                 if (ptr == NULL)
1658                         /* unknown specifier */
1659                         goto error;
1660
1661                 ind = ptr - spec;
1662
1663                 /* check if product will overflow the type */
1664                 if (!(v < ULONG_MAX / mult[ind]))
1665                         goto error;
1666
1667                 ADD_OVERFLOW(val, mult[ind] * v);
1668                 if (val == ULONG_MAX)
1669                         goto error;
1670
1671                 timestr = tail + 1;
1672         }
1673
1674         return val;
1675
1676 error:
1677         return ULONG_MAX;
1678 }
1679
1680 #define ARG2ULL(nr, str, defscale)                                      \
1681 do {                                                                    \
1682         unsigned long long limit, units = 0;                            \
1683         int rc;                                                         \
1684                                                                         \
1685         rc = parse_size(str, &limit, &units, 1);                        \
1686         if (rc < 0) {                                                   \
1687                 fprintf(stderr, "error: bad limit value %s\n", str);    \
1688                 return CMD_HELP;                                        \
1689         }                                                               \
1690         nr = ((units == 0) ? (defscale) : 1) * limit;                   \
1691 } while (0)
1692
1693 static inline int has_times_option(int argc, char **argv)
1694 {
1695         int i;
1696
1697         for (i = 1; i < argc; i++)
1698                 if (!strcmp(argv[i], "-t"))
1699                         return 1;
1700
1701         return 0;
1702 }
1703
1704 int lfs_setquota_times(int argc, char **argv)
1705 {
1706         int c, rc;
1707         struct if_quotactl qctl;
1708         char *mnt, *obd_type = (char *)qctl.obd_type;
1709         struct obd_dqblk *dqb = &qctl.qc_dqblk;
1710         struct obd_dqinfo *dqi = &qctl.qc_dqinfo;
1711         struct option long_opts[] = {
1712                 {"user",            no_argument,       0, 'u'},
1713                 {"group",           no_argument,       0, 'g'},
1714                 {"block-grace",     required_argument, 0, 'b'},
1715                 {"inode-grace",     required_argument, 0, 'i'},
1716                 {"times",           no_argument,       0, 't'},
1717                 {0, 0, 0, 0}
1718         };
1719
1720         memset(&qctl, 0, sizeof(qctl));
1721         qctl.qc_cmd  = LUSTRE_Q_SETINFO;
1722         qctl.qc_type = UGQUOTA;
1723
1724         optind = 0;
1725         while ((c = getopt_long(argc, argv, "ugb:i:t", long_opts, NULL)) != -1) {
1726                 switch (c) {
1727                 case 'u':
1728                 case 'g':
1729                         if (qctl.qc_type != UGQUOTA) {
1730                                 fprintf(stderr, "error: -u and -g can't be used "
1731                                                 "more than once\n");
1732                                 return CMD_HELP;
1733                         }
1734                         qctl.qc_type = (c == 'u') ? USRQUOTA : GRPQUOTA;
1735                         break;
1736                 case 'b':
1737                         if ((dqi->dqi_bgrace = str2sec(optarg)) == ULONG_MAX) {
1738                                 fprintf(stderr, "error: bad block-grace: %s\n",
1739                                         optarg);
1740                                 return CMD_HELP;
1741                         }
1742                         dqb->dqb_valid |= QIF_BTIME;
1743                         break;
1744                 case 'i':
1745                         if ((dqi->dqi_igrace = str2sec(optarg)) == ULONG_MAX) {
1746                                 fprintf(stderr, "error: bad inode-grace: %s\n",
1747                                         optarg);
1748                                 return CMD_HELP;
1749                         }
1750                         dqb->dqb_valid |= QIF_ITIME;
1751                         break;
1752                 case 't': /* Yes, of course! */
1753                         break;
1754                 default: /* getopt prints error message for us when opterr != 0 */
1755                         return CMD_HELP;
1756                 }
1757         }
1758
1759         if (qctl.qc_type == UGQUOTA) {
1760                 fprintf(stderr, "error: neither -u nor -g specified\n");
1761                 return CMD_HELP;
1762         }
1763
1764         if (optind != argc - 1) {
1765                 fprintf(stderr, "error: unexpected parameters encountered\n");
1766                 return CMD_HELP;
1767         }
1768
1769         mnt = argv[optind];
1770         rc = llapi_quotactl(mnt, &qctl);
1771         if (rc) {
1772                 if (*obd_type)
1773                         fprintf(stderr, "%s %s ", obd_type,
1774                                 obd_uuid2str(&qctl.obd_uuid));
1775                 fprintf(stderr, "setquota failed: %s\n", strerror(errno));
1776                 return rc;
1777         }
1778
1779         return 0;
1780 }
1781
1782 #define BSLIMIT (1 << 0)
1783 #define BHLIMIT (1 << 1)
1784 #define ISLIMIT (1 << 2)
1785 #define IHLIMIT (1 << 3)
1786
1787 int lfs_setquota(int argc, char **argv)
1788 {
1789         int c, rc;
1790         struct if_quotactl qctl;
1791         char *mnt, *obd_type = (char *)qctl.obd_type;
1792         struct obd_dqblk *dqb = &qctl.qc_dqblk;
1793         struct option long_opts[] = {
1794                 {"user",            required_argument, 0, 'u'},
1795                 {"group",           required_argument, 0, 'g'},
1796                 {"block-softlimit", required_argument, 0, 'b'},
1797                 {"block-hardlimit", required_argument, 0, 'B'},
1798                 {"inode-softlimit", required_argument, 0, 'i'},
1799                 {"inode-hardlimit", required_argument, 0, 'I'},
1800                 {0, 0, 0, 0}
1801         };
1802         unsigned limit_mask = 0;
1803
1804         if (has_times_option(argc, argv))
1805                 return lfs_setquota_times(argc, argv);
1806
1807         memset(&qctl, 0, sizeof(qctl));
1808         qctl.qc_cmd  = LUSTRE_Q_SETQUOTA;
1809         qctl.qc_type = UGQUOTA; /* UGQUOTA makes no sense for setquota,
1810                                  * so it can be used as a marker that qc_type
1811                                  * isn't reinitialized from command line */
1812
1813         optind = 0;
1814         while ((c = getopt_long(argc, argv, "u:g:b:B:i:I:", long_opts, NULL)) != -1) {
1815                 switch (c) {
1816                 case 'u':
1817                 case 'g':
1818                         if (qctl.qc_type != UGQUOTA) {
1819                                 fprintf(stderr, "error: -u and -g can't be used"
1820                                                 " more than once\n");
1821                                 return CMD_HELP;
1822                         }
1823                         qctl.qc_type = (c == 'u') ? USRQUOTA : GRPQUOTA;
1824                         rc = name2id(&qctl.qc_id, optarg,
1825                                      (qctl.qc_type == USRQUOTA) ? USER : GROUP);
1826                         if (rc) {
1827                                 fprintf(stderr, "error: unknown id %s\n",
1828                                         optarg);
1829                                 return CMD_HELP;
1830                         }
1831                         break;
1832                 case 'b':
1833                         ARG2ULL(dqb->dqb_bsoftlimit, optarg, 1024);
1834                         dqb->dqb_bsoftlimit >>= 10;
1835                         limit_mask |= BSLIMIT;
1836                         break;
1837                 case 'B':
1838                         ARG2ULL(dqb->dqb_bhardlimit, optarg, 1024);
1839                         dqb->dqb_bhardlimit >>= 10;
1840                         limit_mask |= BHLIMIT;
1841                         break;
1842                 case 'i':
1843                         ARG2ULL(dqb->dqb_isoftlimit, optarg, 1);
1844                         limit_mask |= ISLIMIT;
1845                         break;
1846                 case 'I':
1847                         ARG2ULL(dqb->dqb_ihardlimit, optarg, 1);
1848                         limit_mask |= IHLIMIT;
1849                         break;
1850                 default: /* getopt prints error message for us when opterr != 0 */
1851                         return CMD_HELP;
1852                 }
1853         }
1854
1855         if (qctl.qc_type == UGQUOTA) {
1856                 fprintf(stderr, "error: neither -u nor -g are specified\n");
1857                 return CMD_HELP;
1858         }
1859
1860         if (limit_mask == 0) {
1861                 fprintf(stderr, "error: at least one limit must be specified\n");
1862                 return CMD_HELP;
1863         }
1864
1865         if (optind != argc - 1) {
1866                 fprintf(stderr, "error: unexpected parameters encountered\n");
1867                 return CMD_HELP;
1868         }
1869
1870         mnt = argv[optind];
1871
1872         if ((!(limit_mask & BHLIMIT) ^ !(limit_mask & BSLIMIT)) ||
1873             (!(limit_mask & IHLIMIT) ^ !(limit_mask & ISLIMIT))) {
1874                 /* sigh, we can't just set blimits/ilimits */
1875                 struct if_quotactl tmp_qctl = {.qc_cmd  = LUSTRE_Q_GETQUOTA,
1876                                                .qc_type = qctl.qc_type,
1877                                                .qc_id   = qctl.qc_id};
1878
1879                 rc = llapi_quotactl(mnt, &tmp_qctl);
1880                 if (rc < 0) {
1881                         fprintf(stderr, "error: getquota failed\n");
1882                         return CMD_HELP;
1883                 }
1884
1885                 if (!(limit_mask & BHLIMIT))
1886                         dqb->dqb_bhardlimit = tmp_qctl.qc_dqblk.dqb_bhardlimit;
1887                 if (!(limit_mask & BSLIMIT))
1888                         dqb->dqb_bsoftlimit = tmp_qctl.qc_dqblk.dqb_bsoftlimit;
1889                 if (!(limit_mask & IHLIMIT))
1890                         dqb->dqb_ihardlimit = tmp_qctl.qc_dqblk.dqb_ihardlimit;
1891                 if (!(limit_mask & ISLIMIT))
1892                         dqb->dqb_isoftlimit = tmp_qctl.qc_dqblk.dqb_isoftlimit;
1893         }
1894
1895         dqb->dqb_valid |= (limit_mask & (BHLIMIT | BSLIMIT)) ? QIF_BLIMITS : 0;
1896         dqb->dqb_valid |= (limit_mask & (IHLIMIT | ISLIMIT)) ? QIF_ILIMITS : 0;
1897
1898         rc = llapi_quotactl(mnt, &qctl);
1899         if (rc) {
1900                 if (*obd_type)
1901                         fprintf(stderr, "%s %s ", obd_type,
1902                                 obd_uuid2str(&qctl.obd_uuid));
1903                 fprintf(stderr, "setquota failed: %s\n", strerror(errno));
1904                 return rc;
1905         }
1906
1907         return 0;
1908 }
1909
1910 static inline char *type2name(int check_type)
1911 {
1912         if (check_type == USRQUOTA)
1913                 return "user";
1914         else if (check_type == GRPQUOTA)
1915                 return "group";
1916         else
1917                 return "unknown";
1918 }
1919
1920
1921 /* Converts seconds value into format string
1922  * result is returned in buf
1923  * Notes:
1924  *        1. result is in descenting order: 1w2d3h4m5s
1925  *        2. zero fields are not filled (except for p. 3): 5d1s
1926  *        3. zero seconds value is presented as "0s"
1927  */
1928 static void sec2str(time_t seconds, char *buf)
1929 {
1930         const char spec[] = "smhdw";
1931         const unsigned long mult[] = {1, 60, 60*60, 24*60*60, 7*24*60*60};
1932         unsigned long c;
1933         char* tail = buf;
1934         int i;
1935
1936         for (i = sizeof(mult) / sizeof(mult[0]) - 1 ; i >= 0; i--) {
1937                 c = seconds / mult[i];
1938
1939                 if (c > 0 || (i == 0 && buf == tail))
1940                         tail += snprintf(tail, 40-(tail-buf), "%lu%c", c, spec[i]);
1941
1942                 seconds %= mult[i];
1943         }
1944 }
1945
1946
1947 static void diff2str(time_t seconds, char *buf, time_t now)
1948 {
1949
1950         buf[0] = 0;
1951         if (!seconds)
1952                 return;
1953         if (seconds <= now) {
1954                 strcpy(buf, "none");
1955                 return;
1956         }
1957         sec2str(seconds - now, buf);
1958 }
1959
1960 static void print_quota_title(char *name, struct if_quotactl *qctl)
1961 {
1962         printf("Disk quotas for %s %s (%cid %u):\n",
1963                type2name(qctl->qc_type), name,
1964                *type2name(qctl->qc_type), qctl->qc_id);
1965         printf("%15s%8s %7s%8s%8s%8s %7s%8s%8s\n",
1966                "Filesystem",
1967                "kbytes", "quota", "limit", "grace",
1968                "files", "quota", "limit", "grace");
1969 }
1970
1971 static void print_quota(char *mnt, struct if_quotactl *qctl, int type, int rc)
1972 {
1973         time_t now;
1974
1975         time(&now);
1976
1977         if (qctl->qc_cmd == LUSTRE_Q_GETQUOTA || qctl->qc_cmd == Q_GETOQUOTA) {
1978                 int bover = 0, iover = 0;
1979                 struct obd_dqblk *dqb = &qctl->qc_dqblk;
1980
1981                 if (dqb->dqb_bhardlimit &&
1982                     toqb(dqb->dqb_curspace) >= dqb->dqb_bhardlimit) {
1983                         bover = 1;
1984                 } else if (dqb->dqb_bsoftlimit &&
1985                            toqb(dqb->dqb_curspace) >= dqb->dqb_bsoftlimit) {
1986                         if (dqb->dqb_btime > now) {
1987                                 bover = 2;
1988                         } else {
1989                                 bover = 3;
1990                         }
1991                 }
1992
1993                 if (dqb->dqb_ihardlimit &&
1994                     dqb->dqb_curinodes >= dqb->dqb_ihardlimit) {
1995                         iover = 1;
1996                 } else if (dqb->dqb_isoftlimit &&
1997                            dqb->dqb_curinodes >= dqb->dqb_isoftlimit) {
1998                         if (dqb->dqb_btime > now) {
1999                                 iover = 2;
2000                         } else {
2001                                 iover = 3;
2002                         }
2003                 }
2004
2005 #if 0           /* XXX: always print quotas even when no usages */
2006                 if (dqb->dqb_curspace || dqb->dqb_curinodes)
2007 #endif
2008                 {
2009                         char numbuf[3][32];
2010                         char timebuf[40];
2011
2012                         if (strlen(mnt) > 15)
2013                                 printf("%s\n%15s", mnt, "");
2014                         else
2015                                 printf("%15s", mnt);
2016
2017                         if (bover)
2018                                 diff2str(dqb->dqb_btime, timebuf, now);
2019                         if (rc == -1 && errno == EREMOTEIO)
2020                                 sprintf(numbuf[0], LPU64"*",
2021                                         toqb(dqb->dqb_curspace));
2022                         else
2023                                 sprintf(numbuf[0],
2024                                         (dqb->dqb_valid & QIF_SPACE) ?
2025                                         LPU64 : "["LPU64"]",
2026                                         toqb(dqb->dqb_curspace));
2027                         if (type == QC_GENERAL)
2028                                 sprintf(numbuf[1], (dqb->dqb_valid & QIF_BLIMITS)
2029                                         ? LPU64 : "["LPU64"]",
2030                                         dqb->dqb_bsoftlimit);
2031                         else
2032                                 sprintf(numbuf[1], "%s", "");
2033                         sprintf(numbuf[2], (dqb->dqb_valid & QIF_BLIMITS)
2034                                 ? LPU64 : "["LPU64"]", dqb->dqb_bhardlimit);
2035                         printf(" %7s%c %6s %7s %7s",
2036                                numbuf[0], bover ? '*' : ' ', numbuf[1],
2037                                numbuf[2], bover > 1 ? timebuf : "");
2038
2039                         if (iover)
2040                                 diff2str(dqb->dqb_itime, timebuf, now);
2041
2042                         sprintf(numbuf[0], (dqb->dqb_valid & QIF_INODES) ?
2043                                 LPU64 : "["LPU64"]", dqb->dqb_curinodes);
2044                        if (type == QC_GENERAL)
2045                                 sprintf(numbuf[1], (dqb->dqb_valid & QIF_ILIMITS)
2046                                         ? LPU64 : "["LPU64"]",
2047                                         dqb->dqb_isoftlimit);
2048                         else
2049                                 sprintf(numbuf[1], "%s", "");
2050                         sprintf(numbuf[2], (dqb->dqb_valid & QIF_ILIMITS) ?
2051                                 LPU64 : "["LPU64"]", dqb->dqb_ihardlimit);
2052                         if (type != QC_OSTIDX)
2053                                 printf(" %7s%c %6s %7s %7s",
2054                                        numbuf[0], iover ? '*' : ' ', numbuf[1],
2055                                        numbuf[2], iover > 1 ? timebuf : "");
2056                         printf("\n");
2057                 }
2058         } else if (qctl->qc_cmd == LUSTRE_Q_GETINFO ||
2059                    qctl->qc_cmd == Q_GETOINFO) {
2060                 char bgtimebuf[40];
2061                 char igtimebuf[40];
2062
2063                 sec2str(qctl->qc_dqinfo.dqi_bgrace, bgtimebuf);
2064                 sec2str(qctl->qc_dqinfo.dqi_igrace, igtimebuf);
2065                 printf("Block grace time: %s; Inode grace time: %s\n",
2066                        bgtimebuf, igtimebuf);
2067         }
2068 }
2069
2070 static int print_obd_quota(char *mnt, struct if_quotactl *qctl, int is_mdt)
2071 {
2072         int rc = 0, rc1 = 0, count = 0;
2073         __u32 valid = qctl->qc_valid;
2074
2075         rc = llapi_get_obd_count(mnt, &count, is_mdt);
2076         if (rc) {
2077                 fprintf(stderr, "can not get %s count: %s\n",
2078                         is_mdt ? "mdt": "ost", strerror(errno));
2079                 return rc;
2080         }
2081
2082         for (qctl->qc_idx = 0; qctl->qc_idx < count; qctl->qc_idx++) {
2083                 qctl->qc_valid = is_mdt ? QC_MDTIDX : QC_OSTIDX;
2084                 rc = llapi_quotactl(mnt, qctl);
2085                 if (rc) {
2086                         /* It is remote client case. */
2087                         if (errno == EOPNOTSUPP) {
2088                                 rc = 0;
2089                                 goto out;
2090                         }
2091
2092                         if (!rc1)
2093                                 rc1 = rc;
2094                         fprintf(stderr, "quotactl %s%d failed.\n",
2095                                 is_mdt ? "mdt": "ost", qctl->qc_idx);
2096                         continue;
2097                 }
2098
2099                 print_quota(obd_uuid2str(&qctl->obd_uuid), qctl, qctl->qc_valid, 0);
2100         }
2101
2102 out:
2103         qctl->qc_valid = valid;
2104         return rc ? : rc1;
2105 }
2106
2107 static int lfs_quota(int argc, char **argv)
2108 {
2109         int c;
2110         char *mnt, *name = NULL;
2111         struct if_quotactl qctl = { .qc_cmd = LUSTRE_Q_GETQUOTA,
2112                                     .qc_type = UGQUOTA };
2113         char *obd_type = (char *)qctl.obd_type;
2114         char *obd_uuid = (char *)qctl.obd_uuid.uuid;
2115         int rc, rc1 = 0, rc2 = 0, rc3 = 0, verbose = 0, pass = 0;
2116         __u32 valid = QC_GENERAL, idx = 0;
2117
2118         optind = 0;
2119         while ((c = getopt(argc, argv, "ugto:i:I:v")) != -1) {
2120                 switch (c) {
2121                 case 'u':
2122                         if (qctl.qc_type != UGQUOTA) {
2123                                 fprintf(stderr, "error: use either -u or -g\n");
2124                                 return CMD_HELP;
2125                         }
2126                         qctl.qc_type = USRQUOTA;
2127                         break;
2128                 case 'g':
2129                         if (qctl.qc_type != UGQUOTA) {
2130                                 fprintf(stderr, "error: use either -u or -g\n");
2131                                 return CMD_HELP;
2132                         }
2133                         qctl.qc_type = GRPQUOTA;
2134                         break;
2135                 case 't':
2136                         qctl.qc_cmd = LUSTRE_Q_GETINFO;
2137                         break;
2138                 case 'o':
2139                         valid = qctl.qc_valid = QC_UUID;
2140                         strncpy(obd_uuid, optarg, sizeof(qctl.obd_uuid));
2141                         break;
2142                 case 'i':
2143                         valid = qctl.qc_valid = QC_MDTIDX;
2144                         idx = qctl.qc_idx = atoi(optarg);
2145                         break;
2146                 case 'I':
2147                         valid = qctl.qc_valid = QC_OSTIDX;
2148                         idx = qctl.qc_idx = atoi(optarg);
2149                         break;
2150                 case 'v':
2151                         verbose = 1;
2152                         break;
2153                 default:
2154                         fprintf(stderr, "error: %s: option '-%c' "
2155                                         "unrecognized\n", argv[0], c);
2156                         return CMD_HELP;
2157                 }
2158         }
2159
2160         /* current uid/gid info for "lfs quota /path/to/lustre/mount" */
2161         if (qctl.qc_cmd == LUSTRE_Q_GETQUOTA && qctl.qc_type == UGQUOTA &&
2162             optind == argc - 1) {
2163 ug_output:
2164                 memset(&qctl, 0, sizeof(qctl)); /* spoiled by print_*_quota */
2165                 qctl.qc_cmd = LUSTRE_Q_GETQUOTA;
2166                 qctl.qc_valid = valid;
2167                 qctl.qc_idx = idx;
2168                 if (pass++ == 0) {
2169                         qctl.qc_type = USRQUOTA;
2170                         qctl.qc_id = geteuid();
2171                 } else {
2172                         qctl.qc_type = GRPQUOTA;
2173                         qctl.qc_id = getegid();
2174                 }
2175                 rc = id2name(&name, qctl.qc_id,
2176                              (qctl.qc_type == USRQUOTA) ? USER : GROUP);
2177                 if (rc)
2178                         name = "<unknown>";
2179         /* lfs quota -u username /path/to/lustre/mount */
2180         } else if (qctl.qc_cmd == LUSTRE_Q_GETQUOTA) {
2181                 /* options should be followed by u/g-name and mntpoint */
2182                 if (optind + 2 != argc || qctl.qc_type == UGQUOTA) {
2183                         fprintf(stderr, "error: missing quota argument(s)\n");
2184                         return CMD_HELP;
2185                 }
2186
2187                 name = argv[optind++];
2188                 rc = name2id(&qctl.qc_id, name,
2189                              (qctl.qc_type == USRQUOTA) ? USER : GROUP);
2190                 if (rc) {
2191                         fprintf(stderr,"error: can't find id for name %s: %s\n",
2192                                 name, strerror(errno));
2193                         return CMD_HELP;
2194                 }
2195         } else if (optind + 1 != argc || qctl.qc_type == UGQUOTA) {
2196                 fprintf(stderr, "error: missing quota info argument(s)\n");
2197                 return CMD_HELP;
2198         }
2199
2200         if (qctl.qc_cmd == LUSTRE_Q_GETQUOTA)
2201                 print_quota_title(name, &qctl);
2202
2203         mnt = argv[optind];
2204
2205         rc1 = llapi_quotactl(mnt, &qctl);
2206         if (rc1 == -1 && errno == EALREADY) {
2207                 fprintf(stderr, "\n%s quotas are not enabled.\n",
2208                         qctl.qc_type == USRQUOTA ? "user" : "group");
2209                 goto out;
2210         }
2211         if (rc1 && *obd_type)
2212                 fprintf(stderr, "%s %s ", obd_type, obd_uuid);
2213
2214         if (qctl.qc_valid != QC_GENERAL)
2215                 mnt = "";
2216
2217         print_quota(mnt, &qctl, QC_GENERAL, rc1);
2218
2219         if (qctl.qc_valid == QC_GENERAL && qctl.qc_cmd != LUSTRE_Q_GETINFO && verbose) {
2220                 rc2 = print_obd_quota(mnt, &qctl, 1);
2221                 rc3 = print_obd_quota(mnt, &qctl, 0);
2222         }
2223
2224         if (rc1 || rc2 || rc3)
2225                 printf("Some errors happened when getting quota info. "
2226                        "Some devices may be not working or deactivated. "
2227                        "The data in \"[]\" is inaccurate.\n");
2228
2229 out:
2230         if (pass == 1)
2231                 goto ug_output;
2232
2233         return 0;
2234 }
2235 #endif /* HAVE_SYS_QUOTA_H! */
2236
2237 static int flushctx_ioctl(char *mp)
2238 {
2239         int fd, rc;
2240
2241         fd = open(mp, O_RDONLY);
2242         if (fd == -1) {
2243                 fprintf(stderr, "flushctx: error open %s: %s\n",
2244                         mp, strerror(errno));
2245                 return -1;
2246         }
2247
2248         rc = ioctl(fd, LL_IOC_FLUSHCTX);
2249         if (rc == -1)
2250                 fprintf(stderr, "flushctx: error ioctl %s: %s\n",
2251                         mp, strerror(errno));
2252
2253         close(fd);
2254         return rc;
2255 }
2256
2257 static int lfs_flushctx(int argc, char **argv)
2258 {
2259         int     kdestroy = 0, c;
2260         FILE   *proc;
2261         char    procline[PATH_MAX], *line;
2262         int     rc = 0;
2263
2264         optind = 0;
2265         while ((c = getopt(argc, argv, "k")) != -1) {
2266                 switch (c) {
2267                 case 'k':
2268                         kdestroy = 1;
2269                         break;
2270                 default:
2271                         fprintf(stderr, "error: %s: option '-%c' "
2272                                         "unrecognized\n", argv[0], c);
2273                         return CMD_HELP;
2274                 }
2275         }
2276
2277         if (kdestroy)
2278                 system("kdestroy > /dev/null");
2279
2280         if (optind >= argc) {
2281                 /* flush for all mounted lustre fs. */
2282                 proc = fopen("/proc/mounts", "r");
2283                 if (!proc) {
2284                         fprintf(stderr, "error: %s: can't open /proc/mounts\n",
2285                                 argv[0]);
2286                         return -1;
2287                 }
2288
2289                 while ((line = fgets(procline, PATH_MAX, proc)) != NULL) {
2290                         char dev[PATH_MAX];
2291                         char mp[PATH_MAX];
2292                         char fs[PATH_MAX];
2293
2294                         if (sscanf(line, "%s %s %s", dev, mp, fs) != 3) {
2295                                 fprintf(stderr, "%s: unexpected format in "
2296                                                 "/proc/mounts\n",
2297                                         argv[0]);
2298                                 return -1;
2299                         }
2300
2301                         if (strcmp(fs, "lustre") != 0)
2302                                 continue;
2303                         /* we use '@' to determine it's a client. are there
2304                          * any other better way?
2305                          */
2306                         if (strchr(dev, '@') == NULL)
2307                                 continue;
2308
2309                         if (flushctx_ioctl(mp))
2310                                 rc = -1;
2311                 }
2312         } else {
2313                 /* flush fs as specified */
2314                 while (optind < argc) {
2315                         if (flushctx_ioctl(argv[optind++]))
2316                                 rc = -1;
2317                 }
2318         }
2319
2320         return rc;
2321 }
2322
2323 static int lfs_lsetfacl(int argc, char **argv)
2324 {
2325         argv[0]++;
2326         return(llapi_lsetfacl(argc, argv));
2327 }
2328
2329 static int lfs_lgetfacl(int argc, char **argv)
2330 {
2331         argv[0]++;
2332         return(llapi_lgetfacl(argc, argv));
2333 }
2334
2335 static int lfs_rsetfacl(int argc, char **argv)
2336 {
2337         argv[0]++;
2338         return(llapi_rsetfacl(argc, argv));
2339 }
2340
2341 static int lfs_rgetfacl(int argc, char **argv)
2342 {
2343         argv[0]++;
2344         return(llapi_rgetfacl(argc, argv));
2345 }
2346
2347 static int lfs_cp(int argc, char **argv)
2348 {
2349         return(llapi_cp(argc, argv));
2350 }
2351
2352 static int lfs_ls(int argc, char **argv)
2353 {
2354         return(llapi_ls(argc, argv));
2355 }
2356
2357 /* A helper function to return single, whole lines delimited by newline.
2358    Returns length of line.  Not reentrant! */
2359 static int get_next_full_line(int fd, char **ptr)
2360 {
2361         static char buf[8192]; /* bigger than MAX_PATH_LENGTH */
2362         static char *sptr = buf, *eptr = buf;
2363         static int len, rem;
2364
2365         if ((*ptr == NULL) /* first time */
2366             || (eptr >= buf + len) /* buffer empty */) {
2367                 sptr = eptr = buf;
2368                 len = read(fd, buf, sizeof(buf));
2369                 if (len <= 0)
2370                         return len;
2371         } else {
2372                 sptr = eptr + 1;
2373         }
2374
2375 full_line:
2376         while (eptr < buf + len) {
2377                 eptr++;
2378                 /* parse full lines */
2379                 if (*eptr == '\n') {
2380                         *eptr = '\0';
2381                         *ptr = sptr;
2382                         return (eptr - sptr);
2383                 }
2384         }
2385
2386         /* partial line; move to front of buf */
2387         rem = buf + len - sptr;
2388         memcpy(buf, sptr, rem);
2389         sptr = buf;
2390         eptr = buf + rem;
2391         len = read(fd, eptr, sizeof(buf) - rem);
2392         if (len <= 0)
2393                 return len;
2394         len += rem;
2395         goto full_line;
2396 }
2397
2398 static int lfs_changelog(int argc, char **argv)
2399 {
2400         long long startrec = 0, endrec = 0, recnum;
2401         int fd, len;
2402         char c, *mdd, *ptr = NULL;
2403         struct option long_opts[] = {
2404                 {"follow", no_argument, 0, 'f'},
2405                 {0, 0, 0, 0}
2406         };
2407         char short_opts[] = "f";
2408         int follow = 0;
2409
2410         optind = 0;
2411         while ((c = getopt_long(argc, argv, short_opts,
2412                                 long_opts, NULL)) != -1) {
2413                 switch (c) {
2414                 case 'f':
2415                         follow++;
2416                         break;
2417                 case '?':
2418                         return CMD_HELP;
2419                 default:
2420                         fprintf(stderr, "error: %s: option '%s' unrecognized\n",
2421                                 argv[0], argv[optind - 1]);
2422                         return CMD_HELP;
2423                 }
2424         }
2425         if (optind >= argc)
2426                 return CMD_HELP;
2427
2428         mdd = argv[optind++];
2429         if (argc > optind)
2430                 startrec = strtoll(argv[optind++], NULL, 10);
2431         if (argc > optind)
2432                 endrec = strtoll(argv[optind++], NULL, 10);
2433
2434         fd = llapi_changelog_open(mdd, startrec);
2435         if (fd < 0) {
2436                 fprintf(stderr, "%s Can't open changelog: %s\n", argv[0],
2437                         strerror(errno = -fd));
2438                 return fd;
2439         }
2440
2441         while ((len = get_next_full_line(fd, &ptr)) >= 0) {
2442                 if (len == 0) {
2443                         struct pollfd pfds[1];
2444                         int rc;
2445
2446                         if (!follow)
2447                                 break;
2448                         pfds[0].fd = fd;
2449                         pfds[0].events = POLLIN;
2450                         rc = poll(pfds, 1, -1);
2451                         if (rc < 0)
2452                                 break;
2453                         continue;
2454                 }
2455      /* eg. 2 02MKDIR 4405821890 t=[0x100000400/0x5] p=[0x100000400/0x4] pics */
2456                 sscanf(ptr, "%lld *", &recnum);
2457                 if (endrec && recnum > endrec)
2458                         break;
2459                 if (recnum < startrec)
2460                         continue;
2461                 printf("%.*s\n", len, ptr);
2462         }
2463
2464         close(fd);
2465
2466         if (len < 0) {
2467                 fprintf(stderr, "read err %d\n", errno);
2468                 return -errno;
2469         }
2470
2471         return 0;
2472 }
2473
2474 static int lfs_changelog_clear(int argc, char **argv)
2475 {
2476         long long endrec;
2477         int rc;
2478
2479         if (argc != 4)
2480                 return CMD_HELP;
2481
2482         endrec = strtoll(argv[3], NULL, 10);
2483
2484         rc = llapi_changelog_clear(argv[1], argv[2], endrec);
2485         if (rc)
2486                 fprintf(stderr, "%s error: %s\n", argv[0],
2487                         strerror(errno = -rc));
2488         return rc;
2489 }
2490
2491 static int lfs_fid2path(int argc, char **argv)
2492 {
2493         struct option long_opts[] = {
2494                 {"cur", no_argument, 0, 'c'},
2495                 {"link", required_argument, 0, 'l'},
2496                 {"rec", required_argument, 0, 'r'},
2497                 {0, 0, 0, 0}
2498         };
2499         char c, short_opts[] = "cl:r:";
2500         char *device, *fid, *path;
2501         long long recno = -1;
2502         int linkno = -1;
2503         int lnktmp;
2504         int printcur = 0;
2505         int rc;
2506
2507         optind = 0;
2508         while ((c = getopt_long(argc, argv, short_opts,
2509                                 long_opts, NULL)) != -1) {
2510                 switch (c) {
2511                 case 'c':
2512                         printcur++;
2513                         break;
2514                 case 'l':
2515                         linkno = strtol(optarg, NULL, 10);
2516                         break;
2517                 case 'r':
2518                         recno = strtoll(optarg, NULL, 10);
2519                         break;
2520                 case '?':
2521                         return CMD_HELP;
2522                 default:
2523                         fprintf(stderr, "error: %s: option '%s' unrecognized\n",
2524                                 argv[0], argv[optind - 1]);
2525                         return CMD_HELP;
2526                 }
2527         }
2528
2529         device = argv[optind++];
2530         fid = argv[optind++];
2531         if (optind != argc)
2532                 return CMD_HELP;
2533
2534         path = calloc(1, PATH_MAX);
2535
2536         lnktmp = (linkno >= 0) ? linkno : 0;
2537         while (1) {
2538                 int oldtmp = lnktmp;
2539                 long long rectmp = recno;
2540                 rc = llapi_fid2path(device, fid, path, PATH_MAX, &rectmp,
2541                                     &lnktmp);
2542                 if (rc < 0) {
2543                         fprintf(stderr, "%s error: %s\n", argv[0],
2544                                 strerror(errno = -rc));
2545                         break;
2546                 }
2547
2548                 if (printcur)
2549                         fprintf(stdout, "%lld %s\n", rectmp, path);
2550                 else
2551                         fprintf(stdout, "%s\n", path);
2552
2553                 if (linkno >= 0)
2554                         /* specified linkno */
2555                         break;
2556                 if (oldtmp == lnktmp)
2557                         /* no more links */
2558                         break;
2559         }
2560
2561         free(path);
2562         return rc;
2563 }
2564
2565 static int lfs_path2fid(int argc, char **argv)
2566 {
2567         char *path;
2568         lustre_fid fid;
2569         int rc;
2570
2571         if (argc != 2)
2572                 return CMD_HELP;
2573
2574         path = argv[1];
2575         rc = llapi_path2fid(path, &fid);
2576         if (rc) {
2577                 fprintf(stderr, "can't get fid for %s: %s\n", path,
2578                         strerror(errno = -rc));
2579                 return rc;
2580         }
2581
2582         printf(DFID"\n", PFID(&fid));
2583
2584         return 0;
2585 }
2586
2587 int main(int argc, char **argv)
2588 {
2589         int rc;
2590
2591         setlinebuf(stdout);
2592
2593         ptl_initialize(argc, argv);
2594         if (obd_initialize(argc, argv) < 0)
2595                 exit(2);
2596         if (dbg_initialize(argc, argv) < 0)
2597                 exit(3);
2598
2599         Parser_init("lfs > ", cmdlist);
2600
2601         if (argc > 1) {
2602                 rc = Parser_execarg(argc - 1, argv + 1, cmdlist);
2603         } else {
2604                 rc = Parser_commands();
2605         }
2606
2607         obd_finalize(argc, argv);
2608         return rc < 0 ? -rc : rc;
2609 }
2610