-SUBDIRS = src pybuild pylipe .
+SUBDIRS = src src/lipe_scan3 pybuild pylipe .
build_dir = `pwd`/build
rpmbuild_opt =
AC_MSG_RESULT([$enable_server])
AM_CONDITIONAL([BUILD_SERVER], [test x$enable_server = xyes])
-AS_IF([test "x$enable_server" = xyes ],
- [AC_DEFINE(HAVE_LDISKFS, 1, [enable ldiskfs support])])
+AM_COND_IF([BUILD_SERVER],
+ [AC_DEFINE(HAVE_LDISKFS, 1, [enable ldiskfs support])
+ PKG_CHECK_MODULES([GUILE], [guile-2.0])])
# -------- check whether enable zfs support --------
AC_MSG_CHECKING([whether enable zfs support])
AC_CONFIG_FILES([Makefile
lipe.spec
src/Makefile
+ src/lipe_scan3/Makefile
pybuild/Makefile
pylipe/Makefile])
AC_OUTPUT
%description client
Provides lipe tools run on lustre client.
-%endif # end server
+%endif # server
%prep
src/lfill \
src/lipe_scan \
src/lipe_scan2 \
+ src/lipe_scan3/lipe_scan3 \
$RPM_BUILD_ROOT%{_bindir}
%if %{with laudit}
cp src/laudit \
src/laudit-report \
$RPM_BUILD_ROOT%{_bindir}
-%endif
+%endif # laudit
%if %{with hotpool}
cp src/lamigo \
$RPM_BUILD_ROOT%{_sbindir}
mkdir -p $RPM_BUILD_ROOT%{ddntoolsdir}/
install -m 0755 scripts/*.sh $RPM_BUILD_ROOT%{ddntoolsdir}/
-%endif
+%endif # hotpool
cp -a pylipe $RPM_BUILD_ROOT%{python_sitelib}
cp -a pyloris $RPM_BUILD_ROOT%{python_sitelib}
%if %{with laudit}
mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/laudit
cp -a laudit.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/laudit
-%endif
+%endif # laudit
%if %{with hotpool}
cp -a example_configs/hotpool/* $RPM_BUILD_ROOT%{_sysconfdir}/
-%endif
-%endif # end server
+%endif # hotpool
+%endif # server
%if %{with systemd}
mkdir -p $RPM_BUILD_ROOT%{_unitdir}/
$RPM_BUILD_ROOT%{_unitdir}/lpurge@.service
install -m 0644 -D systemd/lamigo@.service \
$RPM_BUILD_ROOT%{_unitdir}/lamigo@.service
-%endif
-%endif # end server
-%else
+%endif # hotpool
+%endif # server
+%else # !systemd
mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d
install -m 0744 -D init.d/lpcc \
$RPM_BUILD_ROOT%{_sysconfdir}/rc.d/init.d/lpcc
-%endif
+%endif # systemd
install -m 0644 man/lpcc.8 $RPM_BUILD_ROOT%{_mandir}/man8/
install -m 0644 man/lpcc-start.8 $RPM_BUILD_ROOT%{_mandir}/man8/
install -m 0644 man/lpcc-stop.8 $RPM_BUILD_ROOT%{_mandir}/man8/
install -m 0644 man/laudit.1 $RPM_BUILD_ROOT%{_mandir}/man1/
install -m 0644 man/laudit-report.1 $RPM_BUILD_ROOT%{_mandir}/man1/
install -m 0644 man/laudit.conf.5 $RPM_BUILD_ROOT%{_mandir}/man5/
-%endif
-%endif #end server
+%endif # laudit
+%endif # server
%clean
%config(noreplace) %{_sysconfdir}/lpcc.conf
%if %{with systemd}
%{_unitdir}/lpcc.service
-%else
+%else # !systemd
%{_sysconfdir}/rc.d/init.d/lpcc
-%endif
+%endif # systemd
%{_mandir}/man8/lpcc.8*
%{_mandir}/man8/lpcc-start.8*
%{_mandir}/man8/lpcc-stop.8*
%{_unitdir}/lpurge@.service
%{_unitdir}/lamigo@.service
%{ddntoolsdir}/*.sh
-%endif
+%endif # hotpool
%files client
%defattr(-,root,root)
%{_mandir}/man1/laudit.1*
%{_mandir}/man1/laudit-report.1*
%{_mandir}/man5/laudit.conf.5*
-%endif
+%endif # laudit
%files
%defattr(-,root,root)
%{_bindir}/ldsync
-%{_bindir}/lipe_convert_expr
-%{_bindir}/lipe_launch
%{_bindir}/lfill
-%{_bindir}/lipe_scan
-%{_bindir}/lipe_scan2
+%{_bindir}/lipe-func.sh
+%{_bindir}/lipe_convert_expr
+%{_bindir}/lipe_delete
%{_bindir}/lipe_find
%{_bindir}/lipe_find2
-%{_bindir}/lipe_delete
+%{_bindir}/lipe_launch
%{_bindir}/lipe_purge
-%{_bindir}/lipe-func.sh
+%{_bindir}/lipe_scan
+%{_bindir}/lipe_scan2
+%{_bindir}/lipe_scan3
%{python2_sitelib}/pylipe
%config(noreplace) %{_sysconfdir}/lipe_launch.json
%{_mandir}/man1/lipe_scan.1*
%{_mandir}/man1/lipe_find.1*
%{_mandir}/man1/lfill.1*
-%endif # end server
-
+%endif # server
%changelog
* Tue Jun 30 2020 Gu Zheng <gzheng@ddn.com> [1.5]
--- /dev/null
+lipe_scan3
+ls3_main.x
+*.gcda
+*.gcno
+*.gcov
--- /dev/null
+AUTOMAKE_OPTIONS = -Wall foreign
+ACLOCAL_AMFLAGS = ${ALOCAL_FLAGS}
+
+if BUILD_SERVER
+
+BUILT_SOURCES = ls3_main.x
+bin_PROGRAMS = lipe_scan3
+
+# Define LUSTRE_UTILS to prevent #error when including lustre_ioctl.h.
+lipe_scan3_CPPFLAGS = -D_GNU_SOURCE -DLUSTRE_UTILS $(GUILE_CFLAGS) -I .. -I ../.. -include config.h
+lipe_scan3_CFLAGS = -Wall -Werror -g $(json_c_CFLAGS) $(GUILE_CFLAGS) -coverage
+lipe_scan3_LDFLAGS = -pthread $(json_c_LIBS) -llnetconfig -llustreapi $(GUILE_LIBS)
+
+lipe_scan3_SOURCES = \
+ ../lipe_version.c \
+ ../lipe_version.h \
+ ../list.h \
+ ls3_debug.h \
+ ls3_main.c \
+ ls3_object_attrs.c \
+ ls3_object_attrs.h \
+ ls3_scan.c \
+ ls3_scan.h
+
+ls3_main.x: ls3_main.c
+ guile-snarf -o $@ $< $(CPPFLAGS) $(lipe_scan3_CPPFLAGS)
+
+endif # BUILD_SERVER
+
+all: all-am
--- /dev/null
+;; lipe_scan3 common Scheme definitions
+(use-modules (rnrs bytevectors))
+(use-modules (ice-9 iconv)) ; bytevector->string
+;; (use-modules (ice-9 threads)) ; with-mutex
+
+(define S_IFMT #o0170000)
+(define S_IFDIR #o0040000)
+(define S_IFCHR #o0020000)
+(define S_IFBLK #o0060000)
+(define S_IFREG #o0100000)
+(define S_IFIFO #o0010000)
+(define S_IFLNK #o0120000)
+(define S_IFSOCK #o0140000)
+
+(define (perm)
+ (logand (mode) #o07777))
+
+(define (type)
+ (logand (mode) S_IFMT))
+
+(define (char->type c)
+ (cond
+ ((eq? c #\d) S_IFDIR)
+ ((eq? c #\c) S_IFCHR)
+ ((eq? c #\b) S_IFBLK)
+ ((eq? c #\f) S_IFREG)
+ ((eq? c #\p) S_IFIFO)
+ ((eq? c #\l) S_IFLNK)
+ ((eq? c #\s) S_IFSOCK)))
+
+(define (type->char t)
+ (cond
+ ((= t S_IFDIR) #\d)
+ ((= t S_IFCHR) #\c)
+ ((= t S_IFBLK) #\b)
+ ((= t S_IFREG) #\f)
+ ((= t S_IFIFO) #\p)
+ ((= t S_IFLNK) #\l)
+ ((= t S_IFSOCK) #\s)))
+
+;; fid
+;;
+;; Note: The struct type <fid>, constructor (make-fid), and slot accessors
+;; accessors (fid-{seq,oid,ver}), are defined in ls3_main.c.
+
+(set-struct-vtable-name! <fid> 'fid)
+
+;; (define-inlinable (%fid-printer x port)
+;; (format port "[0x~x:0x~x:0x~x]" (fid-seq x) (fid-oid x) (fid-ver x)))
+;;
+;; (struct-set! <fid> vtable-index-printer %fid-printer)
+
+(define-inlinable (%c-style-string->number s)
+ (cond
+ ((string-prefix? "0x" s)
+ (string->number (substring s 2) 16))
+ ((string-prefix? "0" s)
+ (string->number s 8))
+ (else
+ (string->number s 10))))
+
+(define (fid? x)
+ (and (struct? x)
+ (eq? (struct-vtable x) <fid>)))
+
+(define (fid=? x1 x2)
+ (and (= (fid-seq x1) (fid-seq x2))
+ (= (fid-oid x1) (fid-oid x2))
+ (= (fid-ver x1) (fid-ver x2))))
+
+(define (fid->string x)
+ (%fid-printer x #f))
+
+(define (string->fid s)
+ (apply make-fid
+ (map %c-style-string->number
+ (string-split (string-trim-right (string-trim s #\[)
+ #\])
+ #\:))))
+
+;; xattrs
+
+(define (xattr-ref-string name)
+ (utf8->string (or (xattr-ref name) #vu8())))
--- /dev/null
+#ifndef _LS3_DEBUG_H_
+#define _LS3_DEBUG_H_
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+#include <com_err.h>
+#include <ext2fs/ext2_err.h>
+
+enum {
+ LS3_EXIT_BREAK_MAX = 126,
+ /* system(), sh -c, ... 127 */
+ LS3_EXIT_USAGE_ERROR = 128,
+ LS3_EXIT_FATAL_ERROR,
+ LS3_EXIT_CLIENT_ERROR,
+ LS3_EXIT_SCAN_ERROR,
+ LS3_EXIT_EXT2_ERROR,
+ LS3_EXIT_READ_ATTR_ERROR,
+ LS3_EXIT_POLICY_ERROR,
+};
+
+extern bool ls3_debug;
+extern const char *ls3_progname_;
+extern __thread int ls3_tid;
+
+static inline const char *ls3_progname(void)
+{
+ return ls3_progname_ != NULL ? ls3_progname_ : program_invocation_short_name;
+}
+
+#define LS3_DEBUG(fmt, args...) \
+ do { \
+ if (ls3_debug) \
+ fprintf(stderr, "%s[%d]: DEBUG %s:%d: "fmt, \
+ ls3_progname(), ls3_tid, __func__, __LINE__, ##args); \
+ } while (0)
+
+#define LS3_DEBUG_B(x) LS3_DEBUG("%s = %s\n", #x, (x) ? "true" : "false")
+#define LS3_DEBUG_D(x) LS3_DEBUG("%s = %"PRIdMAX"\n", #x, (intmax_t)(x))
+#define LS3_DEBUG_P(x) LS3_DEBUG("%s = %p\n", #x, x)
+#define LS3_DEBUG_S(x) LS3_DEBUG("%s = '%s'\n", #x, x)
+#define LS3_DEBUG_U(x) LS3_DEBUG("%s = %"PRIuMAX"\n", #x, (uintmax_t)(x))
+#define LS3_DEBUG_X(x) LS3_DEBUG("%s = %#"PRIxMAX"\n", #x, (uintmax_t)(x))
+
+#define LS3_ERROR(fmt, args...) \
+ fprintf(stderr, "%s[%d]: ERROR: "fmt, ls3_progname(), ls3_tid, ##args)
+
+#define LS3_WARNING(fmt, args...) \
+ fprintf(stderr, "%s[%d]: WARNING: "fmt, ls3_progname(), ls3_tid, ##args)
+
+#define LS3_ERROR_ONCE(fmt, args...) \
+ do { \
+ static int ls3_error_once; \
+ if (ls3_error_once) \
+ break; \
+ ls3_error_once = 1; \
+ LS3_ERROR(fmt, ##args); \
+ } while (0)
+
+#define LS3_FATAL(fmt, args...) \
+ do { \
+ fprintf(stderr, "%s[%d]: FATAL: "fmt, ls3_progname(), ls3_tid, ##args); \
+ exit(LS3_EXIT_FATAL_ERROR); \
+ } while (0)
+
+#define LS3_OOM_AT(file, line, func, size) \
+ LS3_FATAL("out of memory at (%s:%d:%s), size = %zd\n", \
+ (file), (line), (func), (ssize_t)(size))
+
+#define LS3_OOM(size) \
+ LS3_OOM_AT(__FILE__, __LINE__, __func__, (size))
+
+static inline void *xmalloc1(const char *file, int line, const char *func, size_t size)
+{
+ void *ptr = malloc(size);
+
+ if (ptr == NULL && size != 0)
+ LS3_OOM_AT(file, line, func, size);
+
+ return ptr;
+}
+
+static inline void *xcalloc1(const char *file, int line, const char *func, size_t nmemb, size_t size)
+{
+ void *ptr = calloc(nmemb, size);
+
+ if (ptr == NULL && (nmemb * size) != 0)
+ LS3_OOM_AT(file, line, func, (nmemb * size));
+
+ return ptr;
+}
+
+static inline void *xstrdup1(const char *file, int line, const char *func, const char *s)
+{
+ void *ptr;
+
+ if (s == NULL)
+ LS3_FATAL("NULL pointer at (%s:%d:%s)\n", file, line, func);
+
+ ptr = strdup(s);
+
+ if (ptr == NULL)
+ LS3_OOM_AT(file, line, func, strlen(s) + 1);
+
+ return ptr;
+}
+
+#define xmalloc(size) (xmalloc1(__FILE__, __LINE__, __func__, (size)))
+#define xcalloc(nmemb, size) (xcalloc1(__FILE__, __LINE__, __func__, (nmemb), (size)))
+#define xstrdup(s) (xstrdup1(__FILE__, __LINE__, __func__, (s)))
+
+static inline const char *xstrerror(intmax_t err)
+{
+ return imaxabs(err) < EXT2_ET_BASE ? strerror(imaxabs(err)) : error_message(imaxabs(err));
+}
+
+#endif
--- /dev/null
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <fnmatch.h>
+#include <getopt.h>
+#include <limits.h>
+#include <malloc.h>
+#include <pthread.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <linux/lustre/lustre_user.h>
+#include <lustre/lustreapi.h>
+#include <json-c/json.h>
+#include <libguile.h>
+#include "lipe_version.h"
+#include "list.h"
+#include "ls3_debug.h"
+#include "ls3_object_attrs.h"
+#include "ls3_scan.h"
+
+#define LS3_SCAN "lipe-scan"
+#define LS3_PRINT_FILE_FID "print-file-fid"
+#define LS3_PRINT_SELF_FID "print-self-fid"
+#define LS3_PRINT_JSON "print-json"
+#define LS3_PRINT_ABSOLUTE_PATH "print-absolute-path"
+#define LS3_PRINT_RELATIVE_PATH "print-relative-path"
+
+#define LS3_GETOPT_DEVICE_PATH "lipe-getopt-device-path"
+#define LS3_GETOPT_CLIENT_MOUNT_PATH "lipe-getopt-client-mount-path"
+#define LS3_GETOPT_REQUIRED_ATTRS "lipe-getopt-required-attrs"
+#define LS3_GETOPT_THREAD_COUNT "lipe-getopt-thread-count"
+
+static const char *ls3_device_path;
+static const char *ls3_client_mount_path = LS3_CSTR_AUTO;
+static int ls3_thread_count;
+static enum ls3_object_attr_bit ls3_required_attrs = LS3_OBJECT_ATTR_DEFAULT;
+
+static SCM ls3_scm_fid_vtable = SCM_UNDEFINED;
+SCM ls3_scm_policy_exception_handler;
+
+bool ls3_debug;
+const char *ls3_progname_;
+__thread int ls3_tid; /* gettid() thread id for debug messages. */
+
+/* LS3_CURRENT is a thread local evaluation context which contains
+ * pointers to the lipe object and attributes for the current inode of
+ * this scanning thread.
+ *
+ * Calling the uid scheme procedure returns the uid of the current
+ * inode. So for example we can write
+ *
+ * (= (uid) 42) ; file owned by uid 42
+ * (not (= (uid) 0)) ; file not owned by root
+ * (> (nlink) 1) ; file with multiple hardlinks
+ * (member "ralph" (names)) ; file has a link named "ralph"
+ */
+__thread struct ls3_context LS3_CURRENT;
+
+#define TRY_HELP_1() \
+ do { \
+ fprintf(stderr, \
+ "Try '%s --help' for more information.\n", \
+ program_invocation_short_name); \
+ exit(LS3_EXIT_USAGE_ERROR); \
+ } while (0)
+
+
+#define TRY_HELP(fmt, args...) \
+ do { \
+ fprintf(stderr, "%s: "fmt, \
+ program_invocation_short_name, \
+ ##args); \
+ TRY_HELP_1(); \
+ } while (0)
+
+struct attr_bit_name {
+ enum ls3_object_attr_bit abn_bits;
+ const char *abn_name;
+};
+
+/* Names are chosen to match JSON field names. See
+ * lipe_object_attrs_to_json(). */
+
+static struct attr_bit_name attr_bit_names[] = {
+#define X(bits, name) { .abn_bits = (bits), .abn_name = (name), }
+ X(LS3_OBJECT_ATTR_ALL, "all"),
+ X(LS3_OBJECT_ATTR_BASE, "base"), /* ino, uid, gid, nlink, mode, [amc]time */
+ X(LS3_OBJECT_ATTR_BASE, "ino"),
+ X(LS3_OBJECT_ATTR_BASE, "uid"),
+ X(LS3_OBJECT_ATTR_BASE, "gid"),
+ X(LS3_OBJECT_ATTR_BASE, "flags"),
+ X(LS3_OBJECT_ATTR_BASE, "nlink"),
+ X(LS3_OBJECT_ATTR_BASE, "mode"),
+ X(LS3_OBJECT_ATTR_BASE, "atime"),
+ X(LS3_OBJECT_ATTR_BASE, "mtime"),
+ X(LS3_OBJECT_ATTR_BASE, "ctime"),
+ X(LS3_OBJECT_ATTR_SELF_FID, "self_fid"),
+ X(LS3_OBJECT_ATTR_FILE_FID, "file_fid"),
+ X(LS3_OBJECT_ATTR_LINKS, "links"),
+ X(LS3_OBJECT_ATTR_HSM, "hsm"),
+ X(LS3_OBJECT_ATTR_LOV, "lov"),
+ X(LS3_OBJECT_ATTR_LMV, "lmv"),
+ X(LS3_OBJECT_ATTR_SIZE, "size"),
+ X(LS3_OBJECT_ATTR_SIZE, "blocks"),
+ X(LS3_OBJECT_ATTR_SOM, "som"),
+ X(LS3_OBJECT_ATTR_XATTRS, "xattrs"),
+ X(LS3_OBJECT_ATTR_PROJID, "projid"),
+ X(LS3_OBJECT_ATTR_PATHS, "paths"),
+#undef X
+};
+
+static int attr_bits_parse1(const char *name, enum ls3_object_attr_bit *val)
+{
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(attr_bit_names); i++) {
+ if (strcmp(attr_bit_names[i].abn_name, name) == 0) {
+ *val |= attr_bit_names[i].abn_bits;
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+/* str should be a comma separated list of bit names as in
+ * attr_bit_names[]. The bits named in str will be or-ed into
+ * *val. Callers must initialize *val to 0 for the simple usage. */
+static int attr_bits_parse(const char *str, enum ls3_object_attr_bit *pval)
+{
+ enum ls3_object_attr_bit val = 0;
+ char *dup = NULL;
+ char *pos;
+ char *name;
+ int rc = 0;
+
+ /* Secret backdoor hex parsing. */
+ if (strncmp(str, "#x", 2) == 0) {
+ errno = 0;
+ val = strtoul(str + 2, NULL, 16);
+ if (errno != 0)
+ rc = -EINVAL;
+ } else {
+ dup = strdup(str);
+ if (dup == NULL) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ pos = dup;
+ while ((name = strsep(&pos, ",")) != NULL) {
+ char *n;
+
+ /* convert name to lower and s/-/_/. */
+ for (n = name; *n != '\0'; n++) {
+ *n = tolower(*n);
+ *n = (*n == '-') ? '_' : *n;
+ }
+
+ rc = attr_bits_parse1(name, &val);
+ if (rc < 0)
+ goto out;
+ }
+ }
+
+ *pval = val;
+out:
+ free(dup);
+
+ return rc;
+}
+
+static void ls3_list_attrs(void)
+{
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(attr_bit_names); i++)
+ printf("%s\n", attr_bit_names[i].abn_name);
+}
+
+static enum ls3_json_attr print_json_attrs = LS3_JSON_ATTR_DEFAULT;
+static const char *print_delim = "\n";
+static bool print_null_delim;
+static bool print_all_paths;
+
+#define X(s) SCM_GLOBAL_VARIABLE_INIT(ls3_ ## s, #s, scm_from_ulong(s))
+X(LS3_OBJECT_ATTR_BASE);
+X(LS3_OBJECT_ATTR_FILE_FID);
+X(LS3_OBJECT_ATTR_FILTER_FID);
+X(LS3_OBJECT_ATTR_HSM);
+X(LS3_OBJECT_ATTR_LINKS);
+X(LS3_OBJECT_ATTR_LMV);
+X(LS3_OBJECT_ATTR_LOV);
+X(LS3_OBJECT_ATTR_PATHS);
+X(LS3_OBJECT_ATTR_PROJID);
+X(LS3_OBJECT_ATTR_SELF_FID);
+X(LS3_OBJECT_ATTR_SIZE);
+X(LS3_OBJECT_ATTR_SOM);
+X(LS3_OBJECT_ATTR_XATTRS);
+#undef X
+
+SCM_SYMBOL(ls3_sym_no_context, "*lipe-no-context*");
+SCM_GLOBAL_SYMBOL(ls3_sym_read_attr_error, "*lipe-read-attr-error*");
+SCM_GLOBAL_SYMBOL(ls3_sym_scan_break, "*lipe-scan-break*");
+SCM_GLOBAL_SYMBOL(ls3_sym_scan_continue, "*lipe-scan-continue*");
+
+SCM_GLOBAL_VARIABLE_INIT(ls3_scm_fid_vtable_var, "<fid>", ls3_scm_fid_vtable);
+
+#define LS3_SCM_FID(fid) \
+ ((struct lu_fid *)SCM_STRUCT_DATA(fid))
+
+static struct lu_fid *ls3_scm_unpack_fid(const char *func_name, int pos, SCM fid)
+#define FUNC_NAME func_name
+{
+ SCM_VALIDATE_STRUCT(pos, fid);
+
+ if (!scm_is_eq(ls3_scm_fid_vtable, SCM_STRUCT_VTABLE(fid)))
+ scm_wrong_type_arg_msg(func_name, pos, fid, "fid");
+
+ return LS3_SCM_FID(fid);
+}
+#undef FUNC_NAME
+
+#define LS3_SCM_UNPACK_FID(pos, fid) \
+ ls3_scm_unpack_fid(FUNC_NAME, pos, fid)
+
+static SCM ls3_scm_from_fid(const struct lu_fid *c_fid)
+{
+ SCM fid;
+
+ fid = scm_make_struct(ls3_scm_fid_vtable, SCM_INUM0, SCM_EOL);
+ *LS3_SCM_FID(fid) = *c_fid;
+
+ return fid;
+}
+
+#if 0
+static void ls3_scm_to_fid(SCM fid, struct lu_fid *c_fid)
+#define FUNC_NAME "ls3_scm_to_fid"
+{
+ *c_fid = *LS3_SCM_UNPACK_FID(1, fid);
+}
+#undef FUNC_NAME
+#endif
+
+static SCM ls3_scm_fid_printer(SCM x, SCM port)
+#define FUNC_NAME "ls3_fid_printer"
+{
+ struct lu_fid *fid = LS3_SCM_UNPACK_FID(1, x);
+ char buf[FID_LEN];
+
+ snprintf(buf, sizeof(buf), DFID, PFID(fid));
+ if (scm_is_eq(port, SCM_BOOL_F))
+ return scm_from_locale_string(buf);
+
+ if (scm_is_eq(port, SCM_BOOL_T))
+ port = scm_current_output_port();
+ else
+ port = SCM_COERCE_OUTPORT(port);
+
+ SCM_VALIDATE_OPOUTPORT(2, port);
+ scm_puts(buf, port);
+
+ return SCM_UNSPECIFIED;
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_fid_seq, "fid-seq", 1, 0, 0, (SCM fid), "return FID sequence")
+#define FUNC_NAME s_ls3_scm_fid_seq
+{
+ return scm_from_uint64(LS3_SCM_UNPACK_FID(1, fid)->f_seq);
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_fid_oid, "fid-oid", 1, 0, 0, (SCM fid), "return FID OID")
+#define FUNC_NAME s_ls3_scm_fid_oid
+{
+ return scm_from_uint32(LS3_SCM_UNPACK_FID(1, fid)->f_oid);
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_fid_ver, "fid-ver", 1, 0, 0, (SCM fid), "return FID version")
+#define FUNC_NAME s_ls3_scm_fid_ver
+{
+ return scm_from_uint32(LS3_SCM_UNPACK_FID(1, fid)->f_ver);
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_make_fid, "make-fid", 3, 0, 0, (SCM seq, SCM oid, SCM ver), "return a newly allocated FID")
+#define FUNC_NAME s_ls3_scm_make_fid
+{
+ struct lu_fid c_fid;
+
+ SCM_VALIDATE_ULONG_COPY(1, seq, c_fid.f_seq);
+ SCM_VALIDATE_UINT_COPY(2, oid, c_fid.f_oid);
+ SCM_VALIDATE_UINT_COPY(3, ver, c_fid.f_ver);
+
+ return ls3_scm_from_fid(&c_fid);
+}
+#undef FUNC_NAME
+
+SCM_GLOBAL_VARIABLE_INIT(ls3_FNM_CASEFOLD, "FNM_CASEFOLD", scm_from_int(FNM_CASEFOLD));
+
+SCM_DEFINE(ls3_scm_fnmatch_p, "fnmatch?", 2, 1, 0,
+ (SCM pattern, SCM string, SCM flags), "match filename or pathname")
+#define FUNC_NAME s_ls3_scm_fnmatch_p
+{
+ char *c_pattern = NULL;
+ char *c_string = NULL;
+ int c_flags;
+
+ SCM_VALIDATE_STRING(1, pattern);
+ c_pattern = scm_to_locale_string(pattern);
+ SCM_VALIDATE_STRING(2, string);
+ c_string = scm_to_locale_string(string);
+ if (SCM_UNBNDP(flags))
+ c_flags = 0;
+ else
+ SCM_VALIDATE_INT_COPY(3, flags, c_flags);
+
+ return fnmatch(c_pattern, c_string, c_flags) == 0 ? SCM_BOOL_T : SCM_BOOL_F;
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_debug_enable, "lipe-debug-enable", 0, 1, 0,
+ (SCM enable), "get or set debugging")
+{
+ bool old_debug = ls3_debug;
+
+ if (!SCM_UNBNDP(enable))
+ ls3_debug = scm_is_true(enable);
+
+ return scm_from_bool(old_debug);
+}
+
+SCM_DEFINE(ls3_scm_gettid, "lipe-gettid", 0, 0, 0, (), "print caller's thread ID (not pthread id)")
+{
+ return scm_from_ulong(syscall(SYS_gettid));
+}
+
+
+SCM_DEFINE(ls3_scm_getopt_device_path, LS3_GETOPT_DEVICE_PATH, 0, 0, 0, (), "return device path from options or #f")
+{
+ if (ls3_device_path != NULL)
+ return scm_from_locale_string(ls3_device_path);
+
+ return SCM_BOOL_F;
+}
+
+/* Global command line options accessors. */
+SCM_DEFINE(ls3_scm_getopt_client_mount_path, LS3_GETOPT_CLIENT_MOUNT_PATH, 0, 0, 0, (), "return client mount path from options")
+{
+ /* Normally this should just return #t to tell lipe-scan to
+ * try infer the client mount from the device.
+ * #t => auto detect.
+ * "" or #f => no auto detect. */
+
+ if (ls3_client_mount_path == LS3_CSTR_AUTO)
+ return SCM_BOOL_T;
+ else if (ls3_client_mount_path == LS3_CSTR_NONE)
+ return SCM_BOOL_F;
+ else
+ return scm_from_locale_string(ls3_client_mount_path);
+}
+
+SCM_DEFINE(ls3_scm_getopt_required_attrs, LS3_GETOPT_REQUIRED_ATTRS, 0, 0, 0, (), "return required object attrs from options")
+{
+ return scm_from_uint(ls3_required_attrs);
+}
+
+SCM_DEFINE(ls3_scm_getopt_thread_count, LS3_GETOPT_THREAD_COUNT, 0, 0, 0, (), "return thread count from options")
+{
+ return scm_from_int(ls3_thread_count);
+}
+
+/* Scanning context (instance and tread info) accessors. */
+
+/* To be called when scheme procedures that depend on the scanning
+ * context are called outside of scanning. Throws an exception. */
+__attribute__((noreturn))
+static void ls3_no_context(void)
+{
+ scm_throw(ls3_sym_no_context,
+ scm_list_1(scm_from_utf8_string("not in scanning context")));
+
+ __builtin_unreachable();
+}
+
+/* Return context if we are a scanning thread with a current object,
+ * attrs, .... Otherwise throw a no_context exception. */
+static struct ls3_context *ls3_current(void)
+{
+ if (LS3_CURRENT.lc_object == NULL)
+ ls3_no_context();
+
+ return &LS3_CURRENT;
+}
+
+static struct ls3_instance *ls3_instance(void)
+{
+ return ls3_current()->lc_instance;
+}
+
+SCM_DEFINE(ls3_scm_thread_index_ref, "lipe-scan-thread-index", 0, 0, 0,
+ (), "return scan thread index")
+{
+ return ls3_current()->lc_scm_thread_index;
+}
+
+#define LS3_INSTANCE_REF(name, member) \
+ SCM_DEFINE(ls3_scm_ ## member, name, 0, 0, 0, (), "return current instance member") \
+ { \
+ return ls3_instance()->member; \
+ }
+
+LS3_INSTANCE_REF("lipe-scan-fsname", li_scm_fsname);
+LS3_INSTANCE_REF("lipe-scan-client-mount-path", li_scm_client_mount_path);
+LS3_INSTANCE_REF("lipe-scan-client-mount-fd", li_scm_client_mount_fd);
+LS3_INSTANCE_REF("lipe-scan-device-name", li_scm_device_name);
+LS3_INSTANCE_REF("lipe-scan-device-path", li_scm_device_path);
+LS3_INSTANCE_REF("lipe-scan-thread-count", li_scm_thread_count);
+
+/* Current object attribute accessors. */
+
+__attribute__((noreturn))
+static void ls3_throw_read_attr_error(enum ls3_object_attr_bit bits,
+ int error)
+{
+ struct ls3_object_attrs *loa = ls3_current()->lc_attrs;
+
+ scm_throw(ls3_sym_read_attr_error,
+ scm_list_3(scm_from_ulong(loa->loa_ino),
+ scm_from_ulong(bits),
+ scm_from_int(error)));
+
+ __builtin_unreachable();
+}
+
+static struct ls3_object_attrs *ls3_attrs_try(enum ls3_object_attr_bit bits)
+{
+ struct ls3_object_attrs *loa = ls3_current()->lc_attrs;
+
+ bits &= LS3_OBJECT_ATTR_ALL;
+
+ if ((loa->loa_attr_bits & bits) == bits)
+ return loa;
+
+ ls3_read_attrs(ls3_current()->lc_instance, ls3_current()->lc_object,
+ loa,
+ bits,
+ false /* quit_on_error */);
+
+ return loa;
+}
+
+static struct ls3_object_attrs *ls3_attrs(enum ls3_object_attr_bit bits)
+{
+ struct ls3_context *lc = ls3_current();
+ struct ls3_object_attrs *loa = lc->lc_attrs;
+ int rc;
+
+ bits &= LS3_OBJECT_ATTR_ALL;
+
+ if ((loa->loa_attr_bits & bits) == bits)
+ return loa;
+
+ errno = 0;
+ rc = ls3_read_attrs(lc->lc_instance,
+ lc->lc_object,
+ loa,
+ bits,
+ true /* quit_on_error */);
+ if (rc < 0)
+ ls3_throw_read_attr_error(bits, rc);
+
+ assert(rc == 0);
+ assert((loa->loa_attr_bits & bits) == bits);
+
+ return loa;
+}
+
+SCM_DEFINE(ls3_scm_current_attrs, "lipe-scan-current-attrs", 0, 0, 0,
+ (), "return current object attribute bits")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(0);
+
+ return scm_from_ulong(loa->loa_attr_bits);
+}
+
+#define LS3_DEF_ATTR_U(name, member) \
+ SCM_DEFINE(ls3_scm_attr_ ## member ## _ref, \
+ name, \
+ 0, \
+ 0, \
+ 0, \
+ (), \
+ "return " member " attribute of current file") \
+ { \
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_BASE); \
+ \
+ return scm_from_uintmax(loa->loa_ ## member); \
+ }
+
+LS3_DEF_ATTR_U("atime", atime);
+LS3_DEF_ATTR_U("blocks", blocks); /* 512B blocks */
+LS3_DEF_ATTR_U("ctime", ctime);
+LS3_DEF_ATTR_U("flags", flags);
+LS3_DEF_ATTR_U("gid", gid);
+LS3_DEF_ATTR_U("ino", ino);
+LS3_DEF_ATTR_U("mode", mode);
+LS3_DEF_ATTR_U("mtime", mtime);
+LS3_DEF_ATTR_U("nlink", nlink);
+LS3_DEF_ATTR_U("projid", projid);
+LS3_DEF_ATTR_U("size", size);
+LS3_DEF_ATTR_U("uid", uid);
+
+SCM_DEFINE(ls3_scm_file_fid_ref, "file-fid", 0, 0, 0, (), "return FID of current file")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_FILE_FID);
+
+ return ls3_scm_from_fid(&loa->loa_file_fid);
+}
+
+SCM_DEFINE(ls3_scm_self_fid_ref, "self-fid", 0, 0, 0,
+ (), "return FID of current file or OST object")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_SELF_FID);
+
+ return ls3_scm_from_fid(&loa->loa_self_fid);
+}
+
+SCM_DEFINE(ls3_scm_links_list, "links", 0, 0, 0,
+ (), "return links of current file as a list of (parent-fid, name) pairs")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_LINKS);
+ struct lipe_link_entry *lle;
+ SCM lis = SCM_EOL;
+
+ lipe_list_for_each_entry_reverse(lle, &loa->loa_links, lle_linkage)
+ lis = scm_cons(scm_cons(ls3_scm_from_fid(&lle->lle_parent_fid),
+ scm_from_locale_string(lle->lle_name)),
+ lis);
+
+ return lis;
+}
+
+static char *path_append_2(const char *base, const char *rest)
+{
+ size_t base_len = strlen(base);
+ size_t rest_len = strlen(rest);
+ size_t path_size = base_len + 1 + rest_len + 1;
+ char *path = xcalloc(path_size, 1);
+
+ /* Assume that base is already a non-empty canonicalized
+ * absolute path and that rest is relative. */
+
+ if (base_len == 1 && base[0] == '/') {
+ path[0] = '/';
+ strcpy(path + 1, rest);
+ return path;
+ }
+
+ if (rest_len == 0 || (rest_len == 1 && rest[0] == '.'))
+ return xstrdup(base);
+
+ strcpy(path, base);
+ path[base_len] = '/';
+ strcpy(path + base_len + 1, rest);
+
+ return path;
+}
+
+SCM_DEFINE(ls3_scm_absolute_paths, "absolute-paths", 0, 0, 0, (),
+ "return absolute paths of current file")
+{
+ struct ls3_instance *li = ls3_instance();
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_PATHS);
+ struct lipe_path_entry *lpe;
+ SCM lis = SCM_EOL;
+ char *path;
+
+ lipe_list_for_each_entry_reverse(lpe, &loa->loa_paths, lpe_linkage) {
+ path = path_append_2(li->li_client_mount_path, lpe->lpe_path);
+ lis = scm_cons(scm_from_locale_string(path), lis);
+ free(path);
+ }
+
+ return lis;
+}
+
+SCM_DEFINE(ls3_scm_relative_paths, "relative-paths", 0, 0, 0, (),
+ "return paths of current file relative to the client mount")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_PATHS);
+ struct lipe_path_entry *lpe;
+ SCM lis = SCM_EOL;
+
+ lipe_list_for_each_entry_reverse(lpe, &loa->loa_paths, lpe_linkage)
+ lis = scm_cons(scm_from_locale_string(lpe->lpe_path), lis);
+
+ return lis;
+}
+
+static SCM ls3_scm_c_xattr_value(const void *value, unsigned int value_len)
+{
+ SCM bv;
+ void *bv_buf;
+
+ bv = scm_c_make_bytevector(value_len);
+ bv_buf = SCM_BYTEVECTOR_CONTENTS(bv);
+ memcpy(bv_buf, value, value_len);
+
+ return bv;
+}
+
+SCM_DEFINE(ls3_scm_xattrs_list, "xattrs", 0, 0, 0,
+ (), "return xattrs of current file as a list of (name, value) pairs")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_XATTRS);
+ struct lipe_xattr *lx;
+ SCM lis = SCM_EOL;
+
+ lipe_list_for_each_entry_reverse(lx, &loa->loa_xattrs, lx_linkage)
+ lis = scm_cons(scm_cons(scm_from_locale_string(lx->lx_name),
+ ls3_scm_c_xattr_value(lx->lx_value, lx->lx_value_len)),
+ lis);
+
+ return lis;
+}
+
+SCM_DEFINE(ls3_scm_xattr_ref, "xattr-ref", 1, 0, 0,
+ (SCM s_name), "get xattr value for name of current file as a bytevector or #f")
+#define FUNC_NAME s_ls3_scm_xattr_ref
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_XATTRS);
+ const struct lipe_xattr *lx;
+ char *c_name = NULL;
+ SCM bv_or_false = SCM_BOOL_F;
+
+ SCM_VALIDATE_STRING(1, s_name);
+ c_name = scm_to_locale_string(s_name);
+
+ lipe_list_for_each_entry(lx, &loa->loa_xattrs, lx_linkage) {
+ if (strcmp(c_name, lx->lx_name) != 0)
+ continue;
+
+ bv_or_false = ls3_scm_c_xattr_value(lx->lx_value, lx->lx_value_len);
+ goto out;
+ }
+out:
+ free(c_name);
+
+ return bv_or_false;
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_pools_list, "pools", 0, 0, 0,
+ (), "return pools of current file")
+{
+ struct ls3_object_attrs *loa = ls3_attrs_try(LS3_OBJECT_ATTR_LOV);
+ struct llapi_layout *ll = NULL;
+ char buf[LOV_MAXPOOLNAME + 1];
+ SCM lis = SCM_EOL;
+ int rc;
+
+ /* TODO Distinguish between missing xattr and other error. */
+ if (!(loa->loa_attr_bits & LS3_OBJECT_ATTR_LOV))
+ return SCM_EOL;
+
+ /* XXX llapi_layout_*() functions return -1 on error and set errno. */
+ errno = 0;
+ ll = llapi_layout_get_by_xattr(loa->loa_lum, loa->loa_lum_size, 0);
+ if (ll == NULL) {
+ rc = -1;
+ goto out;
+ }
+
+ rc = llapi_layout_comp_use(ll, LLAPI_LAYOUT_COMP_USE_LAST);
+ if (rc < 0)
+ goto out;
+
+ while (rc == 0) {
+ buf[0] = '\0';
+
+ rc = llapi_layout_pool_name_get(ll, buf, sizeof(buf));
+ if (rc < 0)
+ goto out;
+
+ if (buf[0] != '\0')
+ lis = scm_cons(scm_from_locale_string(buf), lis);
+
+ rc = llapi_layout_comp_use(ll, LLAPI_LAYOUT_COMP_USE_PREV);
+ if (rc < 0)
+ goto out;
+ }
+out:
+ llapi_layout_free(ll);
+
+ if (rc < 0)
+ ls3_throw_read_attr_error(LS3_OBJECT_ATTR_LOV,
+ errno != 0 ? errno : EINVAL);
+
+ return lis;
+}
+
+/* We provide some canned printing policies --print-fid, --print-json,
+ * --print-absolute-path, --print-relative-path. */
+
+#define LS3_PRINT_DELIM_C(fmt, c, args...) \
+ printf(fmt "%c", ##args, c)
+
+#define LS3_PRINT_DELIM_S(fmt, s, args...) \
+ printf(fmt "%s", ##args, s)
+
+#define LS3_PRINT_DELIM(fmt, args...) \
+ do { \
+ if (print_null_delim) \
+ LS3_PRINT_DELIM_C(fmt, '\0', ##args); \
+ else \
+ LS3_PRINT_DELIM_S(fmt, print_delim, ##args); \
+ } while (0)
+
+SCM_DEFINE(ls3_scm_print_file_fid, LS3_PRINT_FILE_FID, 0, 0, 0,
+ (), "print FID of current file")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_FILE_FID);
+ const struct lu_fid *fid = &loa->loa_file_fid;
+
+ LS3_PRINT_DELIM(DFID, PFID(fid));
+
+ return SCM_UNSPECIFIED;
+}
+
+SCM_DEFINE(ls3_scm_print_self_fid, LS3_PRINT_SELF_FID, 0, 0, 0,
+ (), "print self FID of current file or OST object")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_SELF_FID);
+ const struct lu_fid *fid = &loa->loa_self_fid;
+
+ LS3_PRINT_DELIM(DFID, PFID(fid));
+
+ return SCM_UNSPECIFIED;
+}
+
+SCM_DEFINE(ls3_scm_print_json, LS3_PRINT_JSON, 0, 1, 0,
+ (SCM bits), "print JSON representation of current file")
+{
+ struct ls3_context *lc = ls3_current();
+ unsigned long c_bits;
+ struct json_object *obj;
+ const char *str;
+
+ if (SCM_UNBNDP(bits))
+ c_bits = print_json_attrs;
+ else
+ SCM_VALIDATE_ULONG_COPY(1, bits, c_bits);
+
+ obj = ls3_object_attrs_to_json(lc->lc_instance, lc->lc_object, lc->lc_attrs, c_bits);
+ str = json_object_to_json_string_ext(obj, JSON_C_TO_STRING_PLAIN);
+
+ LS3_PRINT_DELIM("%s", str);
+
+ json_object_put(obj);
+
+ return SCM_UNSPECIFIED;
+}
+
+SCM_DEFINE(ls3_scm_print_absolute_path, LS3_PRINT_ABSOLUTE_PATH, 0, 0, 0,
+ (), "print absolute path of current file")
+{
+ struct ls3_instance *li = ls3_instance();
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_PATHS);
+ const struct lipe_path_entry *lpe;
+
+ lipe_list_for_each_entry(lpe, &loa->loa_paths, lpe_linkage) {
+ LS3_PRINT_DELIM("%s/%s", li->li_client_mount_path, lpe->lpe_path);
+
+ if (!print_all_paths)
+ break;
+ }
+
+ return SCM_UNSPECIFIED;
+}
+
+SCM_DEFINE(ls3_scm_print_relative_path, LS3_PRINT_RELATIVE_PATH, 0, 0, 0,
+ (), "print relative path of current file")
+{
+ struct ls3_object_attrs *loa = ls3_attrs(LS3_OBJECT_ATTR_PATHS);
+ const struct lipe_path_entry *lpe;
+
+ lipe_list_for_each_entry(lpe, &loa->loa_paths, lpe_linkage) {
+ LS3_PRINT_DELIM("%s", lpe->lpe_path);
+
+ if (!print_all_paths)
+ break;
+ }
+
+ return SCM_UNSPECIFIED;
+}
+
+#if 0
+SCM_GLOBAL_VARIABLE_INIT(ls3_scm_rmfid_max_fids, "%RMFID-MAX-FIDS",
+ scm_from_ulong(OBD_MAX_FIDS_IN_ARRAY));
+
+SCM_DEFINE(ls3_scm_rmfid, "%rmfid", 1, 2, 0,
+ (SCM vec, SCM start, SCM end), "remove a vector of FIDs")
+#define FUNC_NAME s_ls3_scm_rmfid
+{
+ SCM res[2] = { SCM_UNDEFINED, SCM_UNDEFINED };
+ SCM bv = SCM_UNDEFINED;
+ struct fid_array *fa = NULL;
+ size_t c_len;
+ size_t c_start;
+ size_t c_end;
+ size_t c_count;
+ size_t fa_size;
+ size_t i;
+ int rc;
+
+ // Validate ls3_client_mount_fd
+ SCM_VALIDATE_VECTOR(1, vec);
+
+ c_len = scm_c_vector_length(vec);
+
+ if (SCM_UNBNDP(start))
+ c_start = 0;
+ else
+ c_start = scm_to_unsigned_integer(start, 0, c_len);
+
+ if (SCM_UNBNDP(end))
+ c_end = c_len;
+ else
+ c_end = scm_to_unsigned_integer(end, c_start, c_len);
+
+ c_count = c_end - c_start;
+
+ SCM_ASSERT_RANGE(1, scm_from_size_t(c_count), c_count <= OBD_MAX_FIDS_IN_ARRAY);
+
+ fa_size = offsetof(struct fid_array, fa_fids[c_count]);
+ LS3_DEBUG_U(c_len);
+ LS3_DEBUG_U(c_start);
+ LS3_DEBUG_U(c_end);
+ LS3_DEBUG_U(c_count);
+ LS3_DEBUG_U(fa_size);
+
+ bv = scm_c_make_bytevector(fa_size);
+ fa = (struct fid_array *)SCM_BYTEVECTOR_CONTENTS(bv);
+ memset(fa, 0, fa_size);
+
+ for (i = c_start; i < c_end; i++) {
+ ls3_scm_to_fid(scm_c_vector_ref(vec, i),
+ &fa->fa_fids[fa->fa_nr]);
+ fa->fa_nr++;
+ }
+
+ assert(fa->fa_nr == c_count);
+
+ rc = ioctl(ls3_client_mount_fd, LL_IOC_RMFID, fa);
+ if (rc < 0) {
+ res[0] = SCM_UNSPECIFIED;
+ res[1] = scm_from_int(errno);
+ return scm_c_values(res, ARRAY_SIZE(res));
+ }
+ LS3_DEBUG_D(rc);
+
+ res[0] = scm_c_make_vector(fa->fa_nr, SCM_UNDEFINED);
+ res[1] = scm_from_int(0);
+
+ for (i = 0; i < fa->fa_nr; i++) {
+ struct lu_fid *fid = &fa->fa_fids[i];
+ __s32 rc2 = fid->f_ver;
+
+ fid->f_ver = 0;
+ SCM_SIMPLE_VECTOR_SET(res[0], i,
+ scm_cons(ls3_scm_from_fid(fid),
+ scm_from_int(-rc2)));
+ }
+
+ return scm_c_values(res, ARRAY_SIZE(res));
+}
+#undef FUNC_NAME
+#endif
+
+SCM_DEFINE(ls3_scm_scan, LS3_SCAN, 5, 0, 0,
+ (SCM s_device_path,
+ SCM s_client_mount_path,
+ SCM s_policy_thunk,
+ SCM s_required_attrs,
+ SCM s_thread_count),
+ "scan a device")
+#define FUNC_NAME s_ls3_scm_scan
+{
+ const char *c_device_path;
+ const char *c_client_mount_path;
+ unsigned int c_required_attrs;
+ int c_thread_count;
+ int rc;
+
+ SCM_VALIDATE_STRING(1, s_device_path);
+ c_device_path = scm_to_locale_string(s_device_path);
+
+ if (scm_is_eq(s_client_mount_path, SCM_BOOL_F)) {
+ c_client_mount_path = LS3_CSTR_NONE;
+ } else if (scm_is_eq(s_client_mount_path, SCM_BOOL_T)) {
+ c_client_mount_path = LS3_CSTR_AUTO;
+ } else {
+ SCM_VALIDATE_STRING(2, s_client_mount_path);
+ c_client_mount_path = scm_to_locale_string(s_client_mount_path);
+ }
+
+ SCM_VALIDATE_THUNK(3, s_policy_thunk);
+ SCM_VALIDATE_UINT_COPY(4, s_required_attrs, c_required_attrs);
+ SCM_VALIDATE_INT_COPY(5, s_thread_count, c_thread_count);
+
+ LS3_DEBUG_S(c_device_path);
+ LS3_DEBUG_S(c_client_mount_path);
+ LS3_DEBUG_X(c_required_attrs);
+ LS3_DEBUG_D(c_thread_count);
+
+ rc = ls3_scan(c_device_path,
+ c_client_mount_path,
+ s_policy_thunk,
+ c_required_attrs,
+ c_thread_count);
+ LS3_DEBUG_D(rc);
+
+ return scm_from_int(rc);
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_scan_break, "lipe-scan-break", 1, 0, 0, (SCM rc),
+ "stop all scanning threads and return rc from lipe-scan")
+#define FUNC_NAME s_ls3_scm_scan_break
+{
+ int c_rc;
+
+ SCM_VALIDATE_INT_COPY(1, rc, c_rc);
+ LS3_DEBUG_D(c_rc);
+
+ if (!(0 <= c_rc && c_rc <= LS3_EXIT_BREAK_MAX))
+ LS3_FATAL("%s exit status %d is not in allowed range [0, %d]\n",
+ FUNC_NAME, c_rc, LS3_EXIT_BREAK_MAX);
+
+ scm_throw(ls3_sym_scan_break, scm_list_1(rc));
+
+ __builtin_unreachable();
+}
+#undef FUNC_NAME
+
+SCM_DEFINE(ls3_scm_scan_continue, "lipe-scan-continue", 0, 0, 0, (),
+ "stop evaluating policy on the current object and go to the next")
+#define FUNC_NAME s_ls3_scm_scan_continue
+{
+ scm_throw(ls3_sym_scan_continue, SCM_EOL);
+
+ __builtin_unreachable();
+}
+#undef FUNC_NAME
+
+#define USAGE() \
+ TRY_HELP("Usage: %s [OPTION]... [--print-*|--script=SCRIPT] DEVICE...\n", \
+ program_invocation_short_name)
+
+static void help(void)
+{
+ printf(
+"Usage: %s [OPTION]... [--print-*|--script=SCRIPT] DEVICE...\n"
+"Scan DEVICE... ... and something, something ... scheme ... to stdout.\n"
+"Exactly one of the following options must be used:\n"
+/* -i, --interactive is intentionally undocumented */
+" --print-file-fid print the FID of the file for each object\n"
+" --print-self-fid print the FID of the file for each object\n"
+" --print-json[=ATTRS] print a JSON object for each file with atributes specified by ATTRS\n"
+" ATTRS must be a comma separated list of attributes\n"
+" (use '--list-json-attrs' to see available attributes)\n"
+" --print-relative-path print a mount relative path for each object\n"
+" --print-absolute-path print an absolute path for each object\n"
+" -s, --script=SCRIPT support for #! executable scripts\n"
+"\n"
+"Mandatory arguments to long options are mandatory for short options too.\n"
+" --all-paths print all file paths when files have multiple hardlinks\n"
+" for --print-relative-path or --print-absolute-path\n"
+" --null use a zero byte (the ASCII NUL character) to delimit output of --print-*\n"
+" --client-mount=MOUNT use the Lustre client at MOUNT for FID to path\n"
+" --no-client-mount disable automatic client mount detection\n"
+" --required-attrs=ATTR... only apply policy to inodes with ATTR...\n"
+" (use '--list-attrs' to see available attributes)\n"
+" --thread-count=COUNT use COUNT scanning threads\n"
+" --list-attrs list available attribute names and exit\n"
+" --list-json-attrs list available JSON attribute names and exit\n"
+
+" --debug remove insects from device\n"
+" -l, --load=FILE read and evaluate FILE before scanning\n"
+" multiple --load options may be used\n"
+" -h, --help display this help text and exit\n"
+" -v, --version output version information and exit\n"
+/* TODO --delim=DELIM */
+,
+program_invocation_short_name);
+}
+
+enum {
+ LS3_OPT_ALL_PATHS = 1,
+ LS3_OPT_CLIENT_MOUNT,
+ LS3_OPT_DEBUG,
+ LS3_OPT_LIST_ATTRS,
+ LS3_OPT_LIST_JSON_ATTRS,
+ LS3_OPT_NO_CLIENT_MOUNT,
+ LS3_OPT_NULL,
+ LS3_OPT_PRINT_FILE_FID,
+ LS3_OPT_PRINT_SELF_FID,
+ LS3_OPT_PRINT_JSON,
+ LS3_OPT_PRINT_ABSOLUTE_PATH,
+ LS3_OPT_PRINT_RELATIVE_PATH,
+ LS3_OPT_REQUIRED_ATTRS,
+ LS3_OPT_THREAD_COUNT,
+
+ LS3_OPT_HELP = 'h',
+ LS3_OPT_INTERACTIVE = 'i',
+ LS3_OPT_LOAD = 'l',
+ LS3_OPT_SCRIPT = 's',
+ LS3_OPT_VERSION = 'v',
+};
+
+static struct option options[] = {
+ { "client-mount", required_argument, NULL, LS3_OPT_CLIENT_MOUNT },
+ { "debug", no_argument, NULL, LS3_OPT_DEBUG },
+ { "list-attrs", no_argument, NULL, LS3_OPT_LIST_ATTRS },
+ { "list-json-attrs", no_argument, NULL, LS3_OPT_LIST_JSON_ATTRS },
+ { "no-client-mount", no_argument, NULL, LS3_OPT_NO_CLIENT_MOUNT },
+ { "null", no_argument, NULL, LS3_OPT_NULL },
+ { LS3_PRINT_FILE_FID, no_argument, NULL, LS3_OPT_PRINT_FILE_FID },
+ { LS3_PRINT_SELF_FID, no_argument, NULL, LS3_OPT_PRINT_SELF_FID },
+ { "print-json", optional_argument, NULL, LS3_OPT_PRINT_JSON },
+ { "print-absolute-path", no_argument, NULL, LS3_OPT_PRINT_ABSOLUTE_PATH },
+ { "print-relative-path", no_argument, NULL, LS3_OPT_PRINT_RELATIVE_PATH },
+ { "all-paths", no_argument, NULL, LS3_OPT_ALL_PATHS },
+ { "required-attrs", required_argument, NULL, LS3_OPT_REQUIRED_ATTRS },
+ { "thread-count", required_argument, NULL, LS3_OPT_THREAD_COUNT },
+
+ { "help", no_argument, NULL, LS3_OPT_HELP },
+ { "interactive", no_argument, NULL, LS3_OPT_INTERACTIVE },
+ { "load", required_argument, NULL, LS3_OPT_LOAD },
+ { "script", required_argument, NULL, LS3_OPT_SCRIPT },
+ { "version", no_argument, NULL, LS3_OPT_VERSION },
+
+ { NULL },
+};
+
+static int ls3_scm_to_exit_status(SCM rc)
+{
+ if (scm_is_integer(rc)) {
+ intmax_t status = scm_to_intmax(rc);
+
+ /* FIXME [0, 255) remap map libext2fs errors... */
+ /* FIXME errnos vs user defined errors. */
+ return imaxabs(status);
+ }
+
+ return scm_is_true(rc) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+static void ls3_main_scm(void *data, int argc, char *argv[])
+{
+ bool interactive = false;
+ const char *load_file;
+ const char *script_file = NULL;
+ const char *policy = NULL;
+ char *end;
+ int rc;
+ int c;
+
+ ls3_tid = syscall(SYS_gettid);
+
+ /* Need scm_c_eval_string("#t") here to prevent.
+ * Backtrace:
+ * In ice-9/boot-9.scm:
+ * 157: 3 [catch #t #<catch-closure 2774e80> ...]
+ * In unknown file:
+ * ?: 2 [apply-smob/1 #<catch-closure 2774e80>]
+ * In ice-9/eval-string.scm:
+ * 70: 1 [eval-string "(begin (display #t) (newline))" #:module ...]
+ * In unknown file:
+ * ?: 0 [scm-error misc-error #f "~A ~S" ("no such language" scheme) #f]
+ *
+ * ERROR: In#t
+ * procedure scm-error:
+ * ERROR: no such language scheme
+ */
+ scm_c_eval_string("#t");
+
+ ls3_scm_fid_vtable = scm_make_vtable(
+ scm_from_utf8_string("uwuw"),
+ scm_c_define_gsubr("%fid-printer", 2, 0, 0, &ls3_scm_fid_printer));
+
+ ls3_scm_policy_exception_handler = scm_c_make_gsubr(
+ "%lipe-policy-exception-handler",
+ 1, 0, 1,
+ &ls3_policy_exception_handler);
+
+ /* Define our primitives before loading any files. */
+#ifndef SCM_MAGIC_SNARFER
+# include "ls3_main.x"
+#endif
+
+ while ((c = getopt_long(argc, argv, "hil:s:v", options, NULL)) != EOF) {
+ switch (c) {
+ case LS3_OPT_CLIENT_MOUNT:
+ ls3_client_mount_path = optarg;
+ break;
+ case LS3_OPT_DEBUG:
+ ls3_debug = true;
+ break;
+ case LS3_OPT_LIST_ATTRS:
+ ls3_list_attrs();
+ exit(EXIT_SUCCESS);
+ case LS3_OPT_LIST_JSON_ATTRS:
+ ls3_list_json_attrs();
+ exit(EXIT_SUCCESS);
+ case LS3_OPT_NO_CLIENT_MOUNT:
+ ls3_client_mount_path = LS3_CSTR_NONE;
+ break;
+ case LS3_OPT_NULL:
+ print_null_delim = true;
+ break;
+ case LS3_OPT_PRINT_FILE_FID:
+ policy = LS3_PRINT_FILE_FID;
+ break;
+ case LS3_OPT_PRINT_SELF_FID:
+ policy = LS3_PRINT_SELF_FID;
+ break;
+ case LS3_OPT_PRINT_JSON:
+ policy = LS3_PRINT_JSON;
+ if (optarg != NULL) {
+ rc = ls3_json_attrs_parse(optarg, &print_json_attrs);
+ if (rc < 0)
+ TRY_HELP("invalid JSON attrs '%s'\n", optarg);
+ }
+ break;
+ case LS3_OPT_PRINT_ABSOLUTE_PATH:
+ policy = LS3_PRINT_ABSOLUTE_PATH;
+ break;
+ case LS3_OPT_PRINT_RELATIVE_PATH:
+ policy = LS3_PRINT_RELATIVE_PATH;
+ break;
+ case LS3_OPT_REQUIRED_ATTRS:
+ ls3_required_attrs = 0;
+ rc = attr_bits_parse(optarg, &ls3_required_attrs);
+ if (rc < 0)
+ TRY_HELP("invalid required attrs '%s'\n", optarg);
+ break;
+ case LS3_OPT_THREAD_COUNT:
+ errno = 0;
+ ls3_thread_count = strtol(optarg, &end, 0);
+ if (ls3_thread_count == 0 || errno != 0 || *end != '\0')
+ TRY_HELP("invalid thread count '%s'\n", optarg);
+ break;
+ case LS3_OPT_HELP:
+ help();
+ exit(EXIT_SUCCESS);
+ case LS3_OPT_INTERACTIVE:
+ interactive = true;
+ break;
+ case LS3_OPT_LOAD:
+ load_file = optarg;
+ LS3_DEBUG_S(load_file);
+ scm_c_primitive_load(load_file);
+ break;
+ case LS3_OPT_SCRIPT:
+ script_file = optarg;
+ break;
+ case LS3_OPT_VERSION:
+ lipe_version();
+ exit(EXIT_SUCCESS);
+ case '?':
+ TRY_HELP_1();
+ }
+ }
+
+ LS3_DEBUG_S(PACKAGE_VERSION);
+ LS3_DEBUG_S(LIPE_REVISION);
+ LS3_DEBUG_S(ls3_client_mount_path);
+ LS3_DEBUG_D(ls3_thread_count);
+ LS3_DEBUG_B(interactive);
+ LS3_DEBUG_S(policy);
+ LS3_DEBUG_S(script_file);
+
+ if (interactive) {
+ scm_shell(1, argv);
+ __builtin_unreachable();
+ }
+
+ /* Just like Kurt Wallander. There can only be one. */
+ if ((policy != NULL) + (script_file != NULL) != 1)
+ TRY_HELP("exactly one of the --print-* or --script options must be used\n");
+
+ int exit_status = EXIT_SUCCESS;
+
+ if (policy != NULL) {
+ int i;
+
+ /* Each positional parameter is a device to scan. */
+ if (optind == argc)
+ USAGE();
+
+ for (i = optind; i < argc; i++) {
+ SCM scan_rc;
+ int scan_status;
+
+ ls3_device_path = argv[i];
+ LS3_DEBUG_S(ls3_device_path);
+
+#define X(s) scm_variable_ref(scm_c_lookup(s))
+ scan_rc = ls3_scm_scan(scm_call_0(X(LS3_GETOPT_DEVICE_PATH)),
+ scm_call_0(X(LS3_GETOPT_CLIENT_MOUNT_PATH)),
+ X(policy),
+ scm_call_0(X(LS3_GETOPT_REQUIRED_ATTRS)),
+ scm_call_0(X(LS3_GETOPT_THREAD_COUNT)));
+#undef X
+ scan_status = ls3_scm_to_exit_status(scan_rc);
+ LS3_DEBUG_D(scan_status);
+ if (exit_status == EXIT_SUCCESS && scan_status != EXIT_SUCCESS)
+ exit_status = scan_status;
+ }
+ } else {
+ SCM script_rc;
+
+ /* Forward positional parameters to script. */
+ scm_set_program_arguments(-1, argv + optind, (char *)script_file);
+ script_rc = scm_c_primitive_load(script_file);
+ exit_status = ls3_scm_to_exit_status(script_rc);
+ }
+
+ LS3_DEBUG_D(exit_status);
+ exit(exit_status);
+}
+
+int main(int argc, char *argv[])
+{
+ lipe_version_init();
+
+ /* scm_boot_guile() does not return. Instead it calls exit(0)
+ * when ls3_main_scm() returns. To exit with another
+ * status we need to call exit() from ls3_main_scm()
+ * or a callee. */
+ scm_boot_guile(argc, argv, &ls3_main_scm, NULL /* data */);
+ __builtin_unreachable();
+}
--- /dev/null
+#include "ls3_object_attrs.h"
+#include <assert.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <ctype.h>
+#include <errno.h>
+#include <string.h>
+#include <malloc.h>
+#include <sys/ioctl.h>
+#include <json-c/json.h>
+#include <linux/lustre/lustre_fid.h>
+#include <linux/lustre/lustre_idl.h>
+#include <linux/lustre/lustre_ioctl.h>
+#include "list.h"
+#include "ls3_debug.h"
+#include "ls3_scan.h"
+
+char *bytes_to_base64(const unsigned char *b, size_t b_size)
+{
+ char *s = NULL;
+ size_t s_size;
+ size_t i;
+ size_t j;
+
+ static const char E[64] =
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ "abcdefghijklmnopqrstuvwxyz"
+ "0123456789"
+ "+/";
+
+ s_size = 4 * ((b_size + 2) / 3) + 1;
+ s = xmalloc(s_size);
+
+ i = 0;
+ j = 0;
+ while (i < b_size) {
+ unsigned int o1 = (i < b_size) ? b[i++] : 0;
+ unsigned int o2 = (i < b_size) ? b[i++] : 0;
+ unsigned int o3 = (i < b_size) ? b[i++] : 0;
+ unsigned int q = (o1 << 16) | (o2 << 8) | (o3 << 0);
+
+ s[j++] = E[(q >> 18) & 63];
+ s[j++] = E[(q >> 12) & 63];
+ s[j++] = E[(q >> 6) & 63];
+ s[j++] = E[(q >> 0) & 63];
+ }
+
+ switch (b_size % 3) {
+ case 1:
+ s[j - 2] = '=';
+ case 2:
+ s[j - 1] = '=';
+ case 0:
+ s[j - 0] = '\0';
+ }
+
+ return s;
+}
+
+static struct json_object *
+bytes_to_json_string_base64(const void *b, size_t b_size)
+{
+ struct json_object *obj = NULL;
+ char *s = NULL;
+
+ s = bytes_to_base64(b, b_size);
+ if (s == NULL)
+ goto out;
+
+ obj = json_object_new_string(s);
+out:
+ free(s);
+
+ return obj;
+}
+
+static struct json_object *fid_to_json(const struct lu_fid *fid)
+{
+ char fid_buf[FID_LEN + 1];
+
+ snprintf(fid_buf, sizeof(fid_buf), DFID, PFID(fid));
+
+ return json_object_new_string(fid_buf);
+}
+
+static struct json_object *lipe_link_entry_list_to_json(const struct lipe_list_head *links)
+{
+ struct json_object *arr;
+ const struct lipe_link_entry *lle;
+
+ arr = json_object_new_array();
+
+ lipe_list_for_each_entry(lle, links, lle_linkage) {
+ struct json_object *obj = json_object_new_object();
+
+ json_object_object_add(obj, "parent_fid",
+ fid_to_json(&lle->lle_parent_fid));
+
+ json_object_object_add(obj, "name",
+ json_object_new_string(lle->lle_name));
+
+ json_object_array_add(arr, obj);
+ }
+
+ return arr;
+}
+
+static struct json_object *lipe_path_entry_list_to_json(const struct lipe_list_head *paths)
+{
+ struct json_object *arr;
+ const struct lipe_path_entry *lpe;
+
+ arr = json_object_new_array();
+
+ lipe_list_for_each_entry(lpe, paths, lpe_linkage)
+ json_object_array_add(arr,
+ json_object_new_string(lpe->lpe_path));
+
+ return arr;
+}
+
+static struct json_object *lipe_xattr_list_to_json(const struct lipe_list_head *xattrs)
+{
+ struct json_object *arr;
+ const struct lipe_xattr *lx;
+
+ arr = json_object_new_array();
+
+ lipe_list_for_each_entry(lx, xattrs, lx_linkage) {
+ struct json_object *obj = json_object_new_object();
+
+ json_object_object_add(obj, "name",
+ json_object_new_string(lx->lx_name));
+
+ json_object_object_add(obj, "value",
+ bytes_to_json_string_base64(
+ (const unsigned char *)lx->lx_value,
+ lx->lx_value_len));
+
+ json_object_array_add(arr, obj);
+ }
+
+ return arr;
+}
+
+static struct json_object *lipe_som_attrs_to_json(const struct lustre_som_attrs *lsa)
+{
+ struct json_object *obj;
+ struct json_object *flags;
+
+ /* The SOM_FL_* flags currently used in lsa->lsa_valid are all
+ * mutually exclusive. But we return an array of flag names in
+ * case we have non-mutually exclusive flags in the
+ * future. And we ignore SOM_FL_UNKNOWN which is 0.
+ *
+ * "som": {
+ * "flags": [
+ * "lazy"
+ * ],
+ * "_flags": 4,
+ * "size": 16777216,
+ * "blocks": 32768
+ * }
+ */
+
+ flags = json_object_new_array();
+
+ if (lsa->lsa_valid & SOM_FL_STRICT)
+ json_object_array_add(flags, json_object_new_string("strict"));
+
+ if (lsa->lsa_valid & SOM_FL_STALE)
+ json_object_array_add(flags, json_object_new_string("stale"));
+
+ if (lsa->lsa_valid & SOM_FL_LAZY)
+ json_object_array_add(flags, json_object_new_string("lazy"));
+
+ obj = json_object_new_object();
+ json_object_object_add(obj, "flags", flags);
+ json_object_object_add(obj, "_flags", json_object_new_int64(lsa->lsa_valid));
+ json_object_object_add(obj, "size", json_object_new_int64(lsa->lsa_size));
+ json_object_object_add(obj, "blocks", json_object_new_int64(lsa->lsa_blocks));
+
+ return obj;
+}
+
+struct ls3_json_attr_desc {
+ const char *jad_name;
+ struct json_object *(*jad_encode)(struct ls3_instance *, struct ls3_object_attrs *);
+ enum ls3_object_attr_bit jad_attr_bits;
+};
+
+#define LS3_JSON_ENCODE_J(m, expr) \
+ static struct json_object *ls3_json_encode_ ## m(struct ls3_instance *li, struct ls3_object_attrs *loa) \
+ { \
+ return (expr); \
+ }
+
+#define LS3_JSON_ENCODE_I(m) \
+ LS3_JSON_ENCODE_J(m, json_object_new_int64(loa->loa_ ## m))
+
+#define LS3_JSON_ENCODE_S(m) \
+ LS3_JSON_ENCODE_J(m, json_object_new_string(li->li_ ## m))
+
+LS3_JSON_ENCODE_I(ino);
+LS3_JSON_ENCODE_I(mode);
+LS3_JSON_ENCODE_I(nlink);
+LS3_JSON_ENCODE_I(uid);
+LS3_JSON_ENCODE_I(gid);
+LS3_JSON_ENCODE_I(size);
+LS3_JSON_ENCODE_I(atime);
+LS3_JSON_ENCODE_I(mtime);
+LS3_JSON_ENCODE_I(ctime);
+LS3_JSON_ENCODE_I(blocks);
+LS3_JSON_ENCODE_I(projid);
+LS3_JSON_ENCODE_I(flags);
+
+/* TODO HSM, LOV, LMV FILTER_FID */
+
+LS3_JSON_ENCODE_J(file_fid, fid_to_json(&loa->loa_file_fid));
+LS3_JSON_ENCODE_J(self_fid, fid_to_json(&loa->loa_self_fid));
+LS3_JSON_ENCODE_J(links, lipe_link_entry_list_to_json(&loa->loa_links));
+LS3_JSON_ENCODE_J(paths, lipe_path_entry_list_to_json(&loa->loa_paths));
+LS3_JSON_ENCODE_J(som, lipe_som_attrs_to_json(&loa->loa_som));
+LS3_JSON_ENCODE_J(xattrs, lipe_xattr_list_to_json(&loa->loa_xattrs));
+
+LS3_JSON_ENCODE_S(device_name);
+LS3_JSON_ENCODE_S(device_path);
+
+static const struct ls3_json_attr_desc ls3_json_attr_descs[] = {
+ [LS3_JSON_BIT_INO] = { "ino", &ls3_json_encode_ino, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_MODE] = { "mode", &ls3_json_encode_mode, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_NLINK] = { "nlink", &ls3_json_encode_nlink, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_UID] = { "uid", &ls3_json_encode_uid, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_GID] = { "gid", &ls3_json_encode_gid, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_PROJID] = { "projid", &ls3_json_encode_projid, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_FLAGS] = { "flags", &ls3_json_encode_flags, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_ATIME] = { "atime", &ls3_json_encode_atime, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_MTIME] = { "mtime", &ls3_json_encode_mtime, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_CTIME] = { "ctime", &ls3_json_encode_ctime, LS3_OBJECT_ATTR_BASE },
+ [LS3_JSON_BIT_SIZE] = { "size", &ls3_json_encode_size, LS3_OBJECT_ATTR_SIZE },
+ [LS3_JSON_BIT_BLOCKS] = { "blocks", &ls3_json_encode_blocks, LS3_OBJECT_ATTR_SIZE },
+ [LS3_JSON_BIT_FILE_FID] = { "file_fid", &ls3_json_encode_file_fid, LS3_OBJECT_ATTR_FILE_FID },
+ [LS3_JSON_BIT_SELF_FID] = { "self_fid",&ls3_json_encode_self_fid, LS3_OBJECT_ATTR_SELF_FID },
+/* [LS3_JSON_BIT_LOV] = { "lov", &ls3_json_encode_lov, LS3_OBJECT_ATTR_LOV }, */
+/* [LS3_JSON_BIT_LMV] = { "lmv", &ls3_json_encode_lmv, LS3_OBJECT_ATTR_LMV }, */
+ [LS3_JSON_BIT_LINKS] = { "links", &ls3_json_encode_links, LS3_OBJECT_ATTR_LINKS },
+ [LS3_JSON_BIT_PATHS] = { "paths", &ls3_json_encode_paths, LS3_OBJECT_ATTR_PATHS },
+ [LS3_JSON_BIT_SOM] = { "som", &ls3_json_encode_som, LS3_OBJECT_ATTR_SOM },
+ [LS3_JSON_BIT_XATTRS] = { "xattrs", &ls3_json_encode_xattrs, LS3_OBJECT_ATTR_XATTRS },
+ [LS3_JSON_BIT_DEVICE_NAME] = { "device_name", &ls3_json_encode_device_name, 0 },
+ [LS3_JSON_BIT_DEVICE_PATH] = { "device_path", &ls3_json_encode_device_path, 0 },
+};
+
+void ls3_list_json_attrs(void)
+{
+ size_t i;
+
+ printf("all\n");
+
+ for (i = 0; i < ARRAY_SIZE(ls3_json_attr_descs); i++) {
+ if (ls3_json_attr_descs[i].jad_name != NULL)
+ printf("%s\n", ls3_json_attr_descs[i].jad_name);
+ }
+}
+
+static int ls3_json_attrs_parse1(const char *name, enum ls3_json_attr *pattrs)
+{
+ size_t i;
+
+ if (strcmp(name, "all") == 0) {
+ *pattrs = LS3_JSON_ATTR_ALL;
+ return 0;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(ls3_json_attr_descs); i++) {
+ if (ls3_json_attr_descs[i].jad_name != NULL &&
+ strcmp(ls3_json_attr_descs[i].jad_name, name) == 0) {
+ *pattrs |= (1UL << i);
+ return 0;
+ }
+ }
+
+ return -EINVAL;
+}
+
+int ls3_json_attrs_parse(const char *str, enum ls3_json_attr *pattrs)
+{
+ enum ls3_json_attr attrs = 0;
+ char *dup = NULL;
+ char *pos;
+ char *name;
+ int rc = 0;
+
+ /* Secret backdoor hex parsing. */
+ if (strncmp(str, "#x", 2) == 0) {
+ errno = 0;
+ attrs = strtoul(str + 2, NULL, 16);
+ if (errno != 0)
+ rc = -EINVAL;
+ } else {
+ dup = strdup(str);
+ if (dup == NULL) {
+ rc = -ENOMEM;
+ goto out;
+ }
+
+ pos = dup;
+ while ((name = strsep(&pos, ",")) != NULL) {
+ char *n;
+
+ /* convert name to lower and s/-/_/. */
+ for (n = name; *n != '\0'; n++) {
+ *n = tolower(*n);
+ *n = (*n == '-') ? '_' : *n;
+ }
+
+ rc = ls3_json_attrs_parse1(name, &attrs);
+ if (rc < 0)
+ goto out;
+ }
+ }
+
+ *pattrs = attrs;
+out:
+ free(dup);
+
+ return rc;
+}
+
+struct json_object *
+ls3_object_attrs_to_json(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa,
+ enum ls3_json_attr json_attrs)
+{
+ struct json_object *obj = json_object_new_object();
+ enum ls3_json_attr bit;
+
+ for (bit = 0; bit < ARRAY_SIZE(ls3_json_attr_descs); bit++) {
+ const struct ls3_json_attr_desc *jad;
+
+ jad = &ls3_json_attr_descs[bit];
+ if (!((1UL << bit) & json_attrs) ||
+ jad->jad_name == NULL ||
+ jad->jad_encode == NULL)
+ continue;
+
+ ls3_read_attrs(li, lo, loa, jad->jad_attr_bits, true /* quit_on_error */);
+
+ if (jad->jad_attr_bits == (jad->jad_attr_bits & loa->loa_attr_bits))
+ json_object_object_add(obj, jad->jad_name, (*jad->jad_encode)(li, loa));
+ }
+
+ return obj;
+}
+
+static void lipe_object_attrs_links_fini(struct ls3_object_attrs *attrs)
+{
+ struct lipe_link_entry *lle, *n;
+
+ lipe_list_for_each_entry_safe(lle, n, &attrs->loa_links, lle_linkage) {
+ lipe_list_del_init(&lle->lle_linkage);
+ free(lle->lle_name);
+ free(lle);
+ }
+
+ assert(lipe_list_empty(&attrs->loa_links));
+}
+
+static void lipe_object_attrs_paths_fini(struct ls3_object_attrs *attrs)
+{
+ struct lipe_path_entry *lpe, *n;
+
+ lipe_list_for_each_entry_safe(lpe, n, &attrs->loa_paths, lpe_linkage) {
+ lipe_list_del_init(&lpe->lpe_linkage);
+ free(lpe->lpe_path);
+ free(lpe);
+ }
+
+ assert(lipe_list_empty(&attrs->loa_paths));
+}
+
+static void lipe_object_attrs_xattrs_fini(struct ls3_object_attrs *attrs)
+{
+ struct lipe_xattr *lx, *n;
+
+ lipe_list_for_each_entry_safe(lx, n, &attrs->loa_xattrs, lx_linkage) {
+ lipe_list_del_init(&lx->lx_linkage);
+ free(lx->lx_name);
+ free(lx->lx_value);
+ free(lx);
+ }
+
+ assert(lipe_list_empty(&attrs->loa_xattrs));
+}
+
+void lipe_object_attrs_reset(struct ls3_object_attrs *attrs)
+{
+ struct lov_user_md *lum = attrs->loa_lum;
+ int lum_size = attrs->loa_lum_size;
+
+ lipe_object_attrs_links_fini(attrs);
+ lipe_object_attrs_paths_fini(attrs);
+ lipe_object_attrs_xattrs_fini(attrs);
+ memset(lum, 0, lum_size);
+ memset(attrs, 0, sizeof(*attrs));
+ attrs->loa_lum = lum;
+ attrs->loa_lum_size = lum_size;
+ LIPE_INIT_LIST_HEAD(&attrs->loa_links);
+ LIPE_INIT_LIST_HEAD(&attrs->loa_paths);
+ LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs);
+}
+
+int lipe_object_attrs_init(struct ls3_object_attrs *attrs)
+{
+ memset(attrs, 0, sizeof(*attrs));
+ attrs->loa_lum_size = lov_user_md_size(LOV_MAX_STRIPE_COUNT,
+ LOV_USER_MAGIC_V3);
+
+ attrs->loa_lum = xcalloc(1, attrs->loa_lum_size);
+
+ LIPE_INIT_LIST_HEAD(&attrs->loa_links);
+ LIPE_INIT_LIST_HEAD(&attrs->loa_paths);
+ LIPE_INIT_LIST_HEAD(&attrs->loa_xattrs);
+
+ return 0;
+}
+
+void lipe_object_attrs_fini(struct ls3_object_attrs *attrs)
+{
+ lipe_object_attrs_links_fini(attrs);
+ lipe_object_attrs_paths_fini(attrs);
+ lipe_object_attrs_xattrs_fini(attrs);
+ free(attrs->loa_lum);
+}
+
+int lipe_object_attrs_add_link(struct ls3_object_attrs *attrs,
+ const struct lu_fid *parent_fid,
+ const char *name)
+{
+ struct lipe_link_entry *lle = NULL;
+
+ lle = xcalloc(1, sizeof(*lle));
+ lle->lle_parent_fid = *parent_fid;
+ lle->lle_name = xstrdup(name);
+
+ lipe_list_add_tail(&lle->lle_linkage, &attrs->loa_links);
+
+ return 0;
+}
+
+int lipe_object_attrs_set_links(struct ls3_object_attrs *attrs,
+ const void *link_xattr,
+ size_t link_xattr_size)
+{
+ const struct link_ea_header *leh;
+ const struct link_ea_entry *lee;
+ unsigned long len;
+ unsigned long pos;
+ unsigned int reccount;
+ unsigned int i;
+ int rc;
+
+ leh = link_xattr;
+ if (leh->leh_magic == LINK_EA_MAGIC) {
+ reccount = leh->leh_reccount;
+ len = leh->leh_len;
+ } else if (leh->leh_magic == __swab32(LINK_EA_MAGIC)) {
+ reccount = __swab32(leh->leh_reccount);
+ len = __swab64(leh->leh_len);
+ } else {
+ LS3_ERROR("link xattr magic mismatch, expected 0x%lx, got 0x%x\n",
+ LINK_EA_MAGIC, leh->leh_magic);
+ return -EINVAL;
+ }
+
+ if (len > link_xattr_size) {
+ LS3_ERROR("link xattr invalid length %lu, should smaller than %zu\n",
+ len, link_xattr_size);
+ return -ERANGE;
+ }
+
+ pos = sizeof(*leh);
+ lee = (const struct link_ea_entry *)(leh + 1);
+ for (i = 0; i < reccount; i++) {
+ unsigned int reclen = (lee->lee_reclen[0] << 8) | lee->lee_reclen[1];
+ struct lu_fid parent_fid;
+
+ pos += reclen;
+ if (pos > len) {
+ LS3_ERROR("link xattr length exceeded, expected %lu, got %lu\n",
+ len, pos);
+ return -EOVERFLOW;
+ }
+
+ fid_be_to_cpu(&parent_fid, (const struct lu_fid *)&lee->lee_parent_fid);
+
+ rc = lipe_object_attrs_add_link(attrs, &parent_fid, lee->lee_name);
+ if (rc < 0)
+ return rc;
+
+ lee = (const struct link_ea_entry *)((const char *)lee + reclen);
+ }
+
+ if (pos != len) {
+ LS3_ERROR("link xattr length mismatch, expected %lu, got %lu\n",
+ len, pos);
+ return -EOVERFLOW;
+ }
+
+ attrs->loa_attr_bits |= LS3_OBJECT_ATTR_LINKS;
+
+ return 0;
+}
+
+int lipe_object_attrs_add_path(struct ls3_object_attrs *attrs,
+ const char *path)
+{
+ struct lipe_path_entry *lpe = NULL;
+
+ lpe = xcalloc(1, sizeof(*lpe));
+ lpe->lpe_path = xstrdup(path);
+
+ lipe_list_add_tail(&lpe->lpe_linkage, &attrs->loa_paths);
+
+ return 0;
+}
+
+/* Fixup DNE striped directory path with '//'. Root => "". Does not
+ * return "/" for root. See also copy_strip_dne_path(). */
+static void lipe_fid2path_fixup(char *path)
+{
+ char *d, *s;
+
+ for (d = path, s = path; *s != '\0'; s++) {
+ if (*s == '/' && *(s + 1) == '/')
+ continue;
+
+ *(d++) = *s;
+ }
+
+ *d = '\0';
+}
+
+int lipe_object_attrs_set_paths(struct ls3_object_attrs *loa,
+ int client_mount_fd)
+{
+ struct getinfo_fid2path *gf = NULL;
+ unsigned int pathlen = PATH_MAX;
+ unsigned int linkno;
+ int rc;
+
+ if (client_mount_fd < 0) {
+ LS3_ERROR_ONCE("must supply a client mount to get paths\n");
+ rc = -ENOTTY;
+ goto out;
+ }
+
+ if (!(loa->loa_attr_bits & LS3_OBJECT_ATTR_FILE_FID)) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ gf = xcalloc(1, sizeof(*gf) + pathlen);
+
+ linkno = 0;
+ while (1) {
+ memset(gf, 0, sizeof(*gf));
+ gf->gf_fid = loa->loa_file_fid;
+ gf->gf_linkno = linkno;
+ gf->gf_pathlen = pathlen;
+ gf->gf_u.gf_path[0] = '\0';
+
+ rc = ioctl(client_mount_fd, OBD_IOC_FID2PATH, gf);
+ if (rc < 0) {
+ if (errno == ENODATA) {
+ rc = 0;
+ break;
+ }
+
+ rc = -errno;
+ goto out;
+ }
+
+ lipe_fid2path_fixup(gf->gf_u.gf_path);
+
+ rc = lipe_object_attrs_add_path(loa, gf->gf_u.gf_path);
+ if (rc < 0)
+ goto out;
+
+ if (gf->gf_linkno == linkno)
+ break;
+
+ linkno = gf->gf_linkno;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_PATHS;
+out:
+ free(gf);
+
+ return rc;
+}
+
+int lipe_object_attrs_add_xattr(struct ls3_object_attrs *attrs,
+ const char *name,
+ const void *value, size_t value_len)
+{
+ struct lipe_xattr *xattr;
+
+ xattr = xcalloc(1, sizeof(*xattr));
+ xattr->lx_name = xstrdup(name);
+ xattr->lx_value = xcalloc(value_len + 1, 1);
+ memcpy(xattr->lx_value, value, value_len);
+ xattr->lx_value[value_len] = '\0';
+ xattr->lx_value_len = value_len;
+ lipe_list_add_tail(&xattr->lx_linkage, &attrs->loa_xattrs);
+
+ return 0;
+}
--- /dev/null
+/*
+ * Copyright (c) 2016, 2020, DDN Storage Corporation.
+ */
+#ifndef _LS3_OBJECT_ATTRS_H_
+#define _LS3_OBJECT_ATTRS_H_
+#include <stdbool.h>
+#include <stdint.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <linux/types.h>
+#include <linux/lustre/lustre_user.h>
+#include "list.h"
+
+struct ls3_instance;
+struct lipe_object;
+struct json_object;
+
+enum ls3_object_attr_bit {
+ /* Includes ino, mode, nlink, uid, gid, atime, ctime, mtime
+ * and flags. Does not include size/blocks. */
+ LS3_OBJECT_ATTR_BASE = 0x0001,
+ LS3_OBJECT_ATTR_PROJID = 0x0002,
+
+ /* OST objects XATTR_NAME_FID which saves the prarent FID
+ * (i.e. the FID of the file on MDT). Files on MDT don't have
+ * this xattr. */
+ LS3_OBJECT_ATTR_FILTER_FID = 0x0004, /* XATTR_NAME_FID */
+ LS3_OBJECT_ATTR_HSM = 0x0008, /* XATTR_NAME_HSM */
+ LS3_OBJECT_ATTR_LINKS = 0x0010, /* XATTR_NAME_LINK */
+ LS3_OBJECT_ATTR_LMV = 0x0020, /* XATTR_NAME_LMV */
+ LS3_OBJECT_ATTR_LOV = 0x0040, /* XATTR_NAME_LOV */
+ LS3_OBJECT_ATTR_SELF_FID = 0x0080, /* XATTR_NAME_LMA */
+ LS3_OBJECT_ATTR_SOM = 0x0100, /* XATTR_NAME_SOM */
+
+ LS3_OBJECT_ATTR_XATTRS = 0x0200,
+
+ /* self_fid on MDT, FID of parent file on OST */
+ LS3_OBJECT_ATTR_FILE_FID = 0x0400,
+
+ /* The file size on MDT is always 0 (without Data on MDT). But
+ * Lazy Size on MDT can be used as an estimated size since the
+ * inaccuracy doesn't bother the policy engine. So, if DoM,
+ * use real size; if LSoM exists, use it as file size. */
+ LS3_OBJECT_ATTR_SIZE = 0x0800,
+
+ /* Paths from fid2path ioctl for LS3_OBJECT_ATTR_FID on client. */
+ LS3_OBJECT_ATTR_PATHS = 0x1000,
+
+/* LS3_OBJECT_ATTR_EMPTY TODO Make work for striped directories. */
+/* LS3_OBJECT_ATTR_ENTRIES TODO Make work for striped directories. */
+
+ LS3_OBJECT_ATTR_ALL = 0x1fff,
+ LS3_OBJECT_ATTR_DEFAULT = -1U,
+};
+
+enum {
+ LS3_JSON_BIT_INO,
+ LS3_JSON_BIT_MODE,
+ LS3_JSON_BIT_NLINK,
+ LS3_JSON_BIT_UID,
+ LS3_JSON_BIT_GID,
+ LS3_JSON_BIT_PROJID,
+ LS3_JSON_BIT_FLAGS,
+ LS3_JSON_BIT_ATIME,
+ LS3_JSON_BIT_MTIME,
+ LS3_JSON_BIT_CTIME,
+ LS3_JSON_BIT_SIZE,
+ LS3_JSON_BIT_BLOCKS,
+ LS3_JSON_BIT_FILE_FID,
+ LS3_JSON_BIT_SELF_FID,
+ LS3_JSON_BIT_LOV,
+ LS3_JSON_BIT_LMV,
+ LS3_JSON_BIT_POOLS,
+ LS3_JSON_BIT_LINKS,
+ LS3_JSON_BIT_PATHS,
+ LS3_JSON_BIT_SOM,
+ LS3_JSON_BIT_XATTRS,
+ LS3_JSON_BIT_DEVICE_NAME,
+ LS3_JSON_BIT_DEVICE_PATH,
+};
+
+enum ls3_json_attr {
+ LS3_JSON_ATTR_INO = 1U << LS3_JSON_BIT_INO,
+ LS3_JSON_ATTR_MODE = 1U << LS3_JSON_BIT_MODE,
+ LS3_JSON_ATTR_NLINK = 1U << LS3_JSON_BIT_NLINK,
+ LS3_JSON_ATTR_UID = 1U << LS3_JSON_BIT_UID,
+ LS3_JSON_ATTR_GID = 1U << LS3_JSON_BIT_GID,
+ LS3_JSON_ATTR_PROJID = 1U << LS3_JSON_BIT_PROJID,
+ LS3_JSON_ATTR_FLAGS = 1U << LS3_JSON_BIT_FLAGS,
+ LS3_JSON_ATTR_ATIME = 1U << LS3_JSON_BIT_ATIME,
+ LS3_JSON_ATTR_MTIME = 1U << LS3_JSON_BIT_MTIME,
+ LS3_JSON_ATTR_CTIME = 1U << LS3_JSON_BIT_CTIME,
+ LS3_JSON_ATTR_SIZE = 1U << LS3_JSON_BIT_SIZE,
+ LS3_JSON_ATTR_BLOCKS = 1U << LS3_JSON_BIT_BLOCKS,
+ LS3_JSON_ATTR_FILE_FID = 1U << LS3_JSON_BIT_FILE_FID,
+ LS3_JSON_ATTR_SELF_FID = 1U << LS3_JSON_BIT_SELF_FID,
+ LS3_JSON_ATTR_LOV = 1U << LS3_JSON_BIT_LOV,
+ LS3_JSON_ATTR_LMV = 1U << LS3_JSON_BIT_LMV,
+ LS3_JSON_ATTR_POOLS = 1U << LS3_JSON_BIT_POOLS,
+ LS3_JSON_ATTR_LINKS = 1U << LS3_JSON_BIT_LINKS,
+ LS3_JSON_ATTR_PATHS = 1U << LS3_JSON_BIT_PATHS,
+ LS3_JSON_ATTR_SOM = 1U << LS3_JSON_BIT_SOM,
+ LS3_JSON_ATTR_XATTRS = 1U << LS3_JSON_BIT_XATTRS,
+ LS3_JSON_ATTR_DEVICE_NAME = 1U << LS3_JSON_BIT_DEVICE_NAME,
+ LS3_JSON_ATTR_DEVICE_PATH = 1U << LS3_JSON_BIT_DEVICE_PATH,
+
+ /* Print the fast and mostly correct attributes by default. */
+ LS3_JSON_ATTR_DEFAULT =
+ LS3_JSON_ATTR_INO |
+ LS3_JSON_ATTR_MODE |
+ LS3_JSON_ATTR_NLINK |
+ LS3_JSON_ATTR_UID |
+ LS3_JSON_ATTR_GID |
+ LS3_JSON_ATTR_PROJID |
+ LS3_JSON_ATTR_FLAGS |
+ LS3_JSON_ATTR_ATIME |
+ LS3_JSON_ATTR_MTIME |
+ LS3_JSON_ATTR_CTIME |
+ LS3_JSON_ATTR_FILE_FID |
+ LS3_JSON_ATTR_SOM |
+ LS3_JSON_ATTR_DEVICE_NAME,
+
+ LS3_JSON_ATTR_ALL = -1U,
+};
+
+struct lipe_link_entry {
+ struct lipe_list_head lle_linkage;
+ struct lu_fid lle_parent_fid;
+ char *lle_name;
+};
+
+struct lipe_path_entry {
+ struct lipe_list_head lpe_linkage;
+ char *lpe_path;
+};
+
+struct lipe_xattr {
+ /* Linked to loa_xattrs */
+ struct lipe_list_head lx_linkage;
+ /* Name of the xattr */
+ char *lx_name;
+ /* Value of the xattr */
+ char *lx_value;
+ /* Length of value, value is not necessarily determined by \0 */
+ unsigned int lx_value_len;
+};
+
+struct ls3_object_attrs {
+ enum ls3_object_attr_bit loa_attr_bits;
+ int64_t loa_ino;
+ int64_t loa_atime;
+ int64_t loa_mtime;
+ int64_t loa_ctime;
+ int64_t loa_size;
+ int64_t loa_mode;
+ int64_t loa_blocks; /* number of 512B blocks. */
+ int64_t loa_flags;
+ int64_t loa_nlink;
+ uid_t loa_uid;
+ gid_t loa_gid;
+ int64_t loa_projid;
+ struct lu_fid loa_file_fid;
+ struct lu_fid loa_self_fid;
+
+ char loa_leh_buf[XATTR_SIZE_MAX];
+ char loa_lmv_buf[XATTR_SIZE_MAX];
+ struct lov_user_md *loa_lum;
+ int loa_lum_size;
+ struct hsm_user_state loa_hsm_state;
+ struct lustre_som_attrs loa_som;
+ /* The entry number. If inode is not directory, value is zero */
+ int64_t loa_entries;
+ /*
+ * Whether the inode is empty. An inode is empty iff:
+ * 1) Inode is regular file and file size is zero;
+ * 2) Inode is directory and only has "." and ".." as child.
+ * A inode that is neither regular file nor directory is always
+ * considered to be not empty.
+ */
+ bool loa_is_empty;
+ /*
+ * The object on OST has EA of XATTR_NAME_FID which saves the
+ * prarent FID (i.e. the FID of the file on MDT). Files on MDT don't
+ * have that EA.
+ */
+ struct filter_fid loa_filter_fid;
+ int loa_filter_fid_size;
+ /* Link xattr entries. */
+ struct lipe_list_head loa_links;
+ /* Lipe path entries */
+ struct lipe_list_head loa_paths;
+ /*
+ * All of names and values of the EAs.
+ */
+ struct lipe_list_head loa_xattrs;
+};
+
+void ls3_list_json_attrs(void);
+int ls3_json_attrs_parse(const char *str, enum ls3_json_attr *attrs);
+
+struct json_object *
+ls3_object_attrs_to_json(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa,
+ enum ls3_json_attr attrs);
+
+int lipe_object_attrs_init(struct ls3_object_attrs *attrs);
+void lipe_object_attrs_fini(struct ls3_object_attrs *attrs);
+void lipe_object_attrs_reset(struct ls3_object_attrs *attrs);
+int lipe_object_attrs_add_link(struct ls3_object_attrs *attrs,
+ const struct lu_fid *parent_fid,
+ const char *name);
+int lipe_object_attrs_set_links(struct ls3_object_attrs *attrs,
+ const void *link_xattr,
+ size_t link_xattr_size);
+int lipe_object_attrs_add_path(struct ls3_object_attrs *attrs,
+ const char *path);
+int lipe_object_attrs_set_paths(struct ls3_object_attrs *attrs,
+ int mnt_dir_fd);
+int lipe_object_attrs_add_xattr(struct ls3_object_attrs *attrs,
+ const char *name,
+ const void *value, size_t value_len);
+
+#endif /* _LS3_OBJECT_ATTRS_H_ */
--- /dev/null
+/*
+ * Copyright (c) 2017, DDN Storage Corporation.
+ */
+/*
+ *
+ * Tool for scanning the MDT and print list of matched files.
+ *
+ * Author: Li Xi <lixi@ddn.com>
+ */
+#include "ls3_scan.h"
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <linux/lustre/lustre_disk.h>
+#include <linux/lustre/lustre_ostid.h>
+#include <linux/lustre/lustre_user.h>
+#include <lustre/lustreapi.h>
+#include <pthread.h>
+#include <libguile.h>
+#include <com_err.h>
+#include <ext2fs/ext2fs.h>
+#include "ls3_debug.h"
+#include "ls3_object_attrs.h"
+
+/* XXX We are mixing libext2fs errcode_t (long), pthread positive rcs,
+ * pthread_join (void **) rcs, and errnos. This is why we have all the
+ * intptr_t and intmax_t business. */
+
+struct ls3_scan_control {
+ pthread_mutex_t sc_group_mutex;
+ unsigned long sc_group_current;
+ unsigned long sc_group_end;
+ unsigned long sc_group_batch;
+
+ struct ls3_thread_info *sc_thread_infos;
+ size_t sc_thread_count;
+ intmax_t sc_thread_rc_max;
+ intmax_t sc_break_rc;
+ uintmax_t sc_read_attr_error_count;
+ uintmax_t sc_other_error_count;
+};
+
+struct ls3_thread_info {
+ struct ls3_scan_control *ti_scan_control;
+ /* ID returned by pthread_create() */
+ pthread_t ti_thread_id;
+ /* Application-defined thread index */
+ int ti_thread_index;
+ struct ls3_instance *ti_instance;
+ unsigned long ti_group_end;
+ unsigned long ti_group_start;
+ uintmax_t ti_read_attr_error_count;
+ uintmax_t ti_other_error_count;
+ sig_atomic_t ti_started;
+ sig_atomic_t ti_should_stop;
+};
+
+const char LS3_CSTR_AUTO[] = "AUTO";
+const char LS3_CSTR_NONE[] = "NONE";
+
+#ifndef EXT4_SNAPFILE_FL
+# define EXT4_SNAPFILE_FL 0x01000000 /* Inode is a snapshot */
+#endif
+
+/* Due to a bug in e2fsprogs-1.45.6.wc3, and potential ongoing changes in
+ * upstream releases, the definition of EXT2_ET_EA_NAME_NOT_FOUND may be
+ * different for different e2fsprogs build. The actual error code returned
+ * here is not super critical, since it is mostly "does xattr exist or not",
+ * but this should be flexible in what is accepted now and in the future.
+ */
+#define EXT2_ET_EA_NAME_NOT_FOUND_1445WC1 (2133571512L)
+#define EXT2_ET_EA_NAME_NOT_FOUND_1456WC1 (2133571513L)
+#define EXT2_ET_EA_NAME_NOT_FOUND_1456WC3 (2133571514L)
+#define EXT2_ET_EA_NAME_NOT_FOUND_1462WC3 (2133571515L)
+
+static bool xattr_name_not_found(long errcode)
+{
+ return errcode == EXT2_ET_EA_NAME_NOT_FOUND ||
+ errcode == EXT2_ET_EA_NAME_NOT_FOUND_1445WC1 ||
+ errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC1 ||
+ errcode == EXT2_ET_EA_NAME_NOT_FOUND_1456WC3 ||
+ errcode == EXT2_ET_EA_NAME_NOT_FOUND_1462WC3;
+}
+
+/* FIXME Wrap ext2fs_attr_get() and check xattr_name_not_found(). */
+
+static int
+ldiskfs_read_attr_self_fid(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ char buf[XATTR_SIZE_MAX];
+ struct lustre_mdt_attrs *lma;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_SELF_FID)
+ return 0;
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_LMA + strlen("trusted."),
+ buf, sizeof(buf), &size);
+ if (rc) {
+ LS3_DEBUG_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_LMA, rc);
+ return -ENODATA;
+ }
+
+ lma = (struct lustre_mdt_attrs *)buf;
+ fid_le_to_cpu(&loa->loa_self_fid, &lma->lma_self_fid);
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_SELF_FID;
+
+ return 0;
+}
+
+static int
+ldiskfs_read_attr_filter_fid(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ struct filter_fid *ff = &loa->loa_filter_fid;
+ struct lu_fid pfid;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_FILTER_FID)
+ return 0;
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_FID + strlen("trusted."),
+ (char *)ff, sizeof(*ff), &size);
+ if (rc) {
+ LS3_DEBUG_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_FID, rc);
+ return -ENODATA;
+ }
+
+ fid_le_to_cpu(&pfid, &ff->ff_parent);
+ ff->ff_parent = pfid;
+ loa->loa_filter_fid_size = size;
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_FILTER_FID;
+
+ return 0;
+}
+
+static int
+ldiskfs_read_attr_file_fid(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_FILE_FID)
+ return 0;
+
+ if (li->li_device_is_mdt) {
+ rc = ldiskfs_read_attr_self_fid(li, lo, loa);
+ if (rc < 0)
+ return rc;
+
+ loa->loa_file_fid = loa->loa_self_fid;
+ } else if (li->li_device_is_ost) {
+ rc = ldiskfs_read_attr_filter_fid(li, lo, loa);
+ if (rc < 0)
+ return rc;
+
+ loa->loa_file_fid = loa->loa_filter_fid.ff_parent;
+ loa->loa_file_fid.f_ver = 0;
+ } else {
+ LS3_ERROR_ONCE("cannot get file-fid on device '%s' with unrecognized type\n",
+ li->li_device_path);
+ return -EINVAL;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_FILE_FID;
+
+ return 0;
+}
+
+static int
+ldiskfs_read_attr_links(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_LINKS)
+ return 0;
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_LINK + strlen("trusted."),
+ loa->loa_leh_buf, sizeof(loa->loa_leh_buf),
+ &size);
+ if (rc) {
+ LS3_DEBUG_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_LINK, (long)rc);
+ return -ENODATA;
+ }
+
+ rc = lipe_object_attrs_set_links(loa, loa->loa_leh_buf, size);
+ if (rc) {
+ LS3_ERROR_OBJ(lo, "cannot decode link xattr: rc = %ld\n", rc);
+ return rc;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_LINKS;
+
+ return 0;
+}
+
+static int
+ldiskfs_read_attr_paths(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ int rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_PATHS)
+ return 0;
+
+ rc = ldiskfs_read_attr_file_fid(li, lo, loa);
+ if (rc < 0)
+ return rc;
+
+ if (li->li_device_is_mdt) {
+ /* We cannot use links to make paths faster because
+ * the linkea is not updated properly after
+ * unlink. But requiring a link xattr before fid2path
+ * prevents MDT crashes when we pass fids of OI_scrub
+ * or other internal files. */
+ rc = ldiskfs_read_attr_links(li, lo, loa);
+ if (rc < 0)
+ return rc;
+ }
+
+ return lipe_object_attrs_set_paths(loa, li->li_client_mount_fd);
+}
+
+static int ldiskfs_copy_xattr(char *name, char *value, size_t value_len,
+ ext2_ino_t ino, void *data)
+{
+ struct ls3_object_attrs *loa = (struct ls3_object_attrs *)data;
+
+ return lipe_object_attrs_add_xattr(loa, name, value, value_len);
+}
+
+static int ldiskfs_read_attr_xattrs(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ ext2_ino_t ino = lo->u.lo_ldiskfs.lol_ino;
+ struct ext2_xattr_handle *handle = NULL;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_XATTRS)
+ return 0;
+
+ rc = ext2fs_xattrs_open(fs, ino, &handle);
+ if (rc) {
+ LS3_ERROR_OBJ(lo, "cannot open xattrs: rc = %ld\n", rc);
+ return rc;
+ }
+
+ rc = ext2fs_xattrs_read(handle);
+ if (rc) {
+ LS3_ERROR_OBJ(lo, "cannot read xattrs: rc = %ld\n", rc);
+ goto out_close;
+ }
+
+ rc = ext2fs_xattrs_iterate(handle, ldiskfs_copy_xattr, loa);
+ if (rc) {
+ LS3_ERROR_OBJ(lo, "cannot iterate xattrs: rc = %ld\n", rc);
+ goto out_close;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_XATTRS;
+out_close:
+ ext2fs_xattrs_close(&handle);
+
+ return rc;
+}
+
+static int ldiskfs_read_attr_lov(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ struct lov_user_md *lum = loa->loa_lum;
+ struct llapi_layout *layout = NULL;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_LOV)
+ return 0;
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_LOV + strlen("trusted."),
+ (char *)lum, loa->loa_lum_size, &size);
+ if (rc) {
+ LS3_DEBUG_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_LOV, rc);
+ rc = -ENODATA;
+ goto out;
+ }
+
+ layout = llapi_layout_get_by_xattr(lum, size, 0);
+ if (layout == NULL) {
+ LS3_ERROR_OBJ(lo, "cannot decode '%s' xattr: rc = %ld\n",
+ XATTR_NAME_LOV, rc);
+ rc = -errno;
+ goto out;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_LOV;
+out:
+ llapi_layout_free(layout);
+
+ return rc;
+}
+
+static int ldiskfs_read_attr_lmv(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_LMV)
+ return 0;
+
+ memset(loa->loa_lmv_buf, 0, sizeof(loa->loa_lmv_buf));
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_LMV + strlen("trusted."),
+ loa->loa_lmv_buf, sizeof(loa->loa_lmv_buf),
+ &size);
+ if (xattr_name_not_found(rc)) {
+ LS3_DEBUG_OBJ(lo, "has no '%s' xattr ignoring rc = %ld\n",
+ XATTR_NAME_LMV, rc);
+ return 1; /* XXX */
+ } else if (rc) {
+ LS3_ERROR_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_LMV, rc);
+ return -ENODATA;
+ }
+
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_LMV;
+
+ return 0;
+}
+
+static bool lo_is_dir_stripe(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ union lmv_mds_md *lmm;
+ int rc;
+
+ rc = ldiskfs_read_attr_lmv(li, lo, loa);
+ if (rc != 0 && rc != -ENOTSUP)
+ return false;
+
+ lmm = (union lmv_mds_md *)loa->loa_lmv_buf;
+ if (lmm->lmv_magic == LMV_MAGIC_STRIPE)
+ return true;
+
+ return false;
+}
+
+static int ldiskfs_read_attr_hsm(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ struct hsm_user_state *hus = &loa->loa_hsm_state;
+ char buf[XATTR_SIZE_MAX];
+ int size;
+ struct hsm_attrs *hsm;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_HSM)
+ return 0;
+
+ assert(sizeof(*hsm) < XATTR_SIZE_MAX);
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_HSM + strlen("trusted."),
+ buf, sizeof(buf), &size);
+ if (xattr_name_not_found(rc)) {
+ LS3_DEBUG_OBJ(lo, "has no '%s' xattr\n", XATTR_NAME_HSM);
+ hus->hus_states = 0;
+ hus->hus_archive_id = 0;
+ return -ENODATA;
+ }
+
+ if (rc != 0) {
+ LS3_ERROR_OBJ(lo, "cannot get '%s' xattr: rc = %s\n",
+ XATTR_NAME_HSM, xstrerror(rc));
+ return rc;
+ }
+
+ hsm = (struct hsm_attrs *)buf;
+ hus->hus_states = ext2fs_le32_to_cpu(hsm->hsm_flags);
+ hus->hus_archive_id = ext2fs_le64_to_cpu(hsm->hsm_arch_id);
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_HSM;
+
+ return 0;
+}
+
+static int ldiskfs_read_attr_som(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ ext2_filsys fs = lo->u.lo_ldiskfs.lol_fs;
+ struct ext2_inode_large *inode = lo->u.lo_ldiskfs.lol_inode;
+ int size;
+ struct lustre_som_attrs *som = &loa->loa_som;
+ errcode_t rc;
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_SOM)
+ return 0;
+
+ rc = ext2fs_attr_get(fs, (struct ext2_inode *)inode,
+ EXT2_ATTR_INDEX_TRUSTED,
+ XATTR_NAME_SOM + strlen("trusted."),
+ (char *)som, sizeof(*som), &size);
+ if (rc) {
+ LS3_DEBUG_OBJ(lo, "cannot get '%s' xattr: rc = %ld\n",
+ XATTR_NAME_SOM, rc);
+ return -ENODATA;
+ }
+
+ if (size != sizeof(*som)) {
+ LS3_ERROR_OBJ(lo, "unexpected size of '%s' xattr, expected %zu, got %d\n",
+ XATTR_NAME_SOM, sizeof(*som), size);
+ return -ENODATA;
+ }
+
+ som->lsa_valid = ext2fs_le16_to_cpu(som->lsa_valid);
+ som->lsa_size = ext2fs_le64_to_cpu(som->lsa_size);
+ som->lsa_blocks = ext2fs_le64_to_cpu(som->lsa_blocks);
+
+ switch (loa->loa_som.lsa_valid) {
+ case SOM_FL_STRICT:
+ loa->loa_size = som->lsa_size;
+ loa->loa_blocks = som->lsa_blocks;
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_SIZE;
+ /* fallthrough */
+ case SOM_FL_STALE:
+ case SOM_FL_LAZY:
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_SOM;
+ rc = 0;
+ break;
+ default:
+ rc = -ENOTSUP;
+ break;
+ }
+
+ return 0;
+}
+
+static int ldiskfs_read_attr_size(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa)
+{
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_SIZE)
+ return 0;
+
+ assert(loa->loa_attr_bits & LS3_OBJECT_ATTR_BASE);
+
+ if (!S_ISREG(loa->loa_mode)) {
+ /* FIXME This is wrong for striped directories. */
+ loa->loa_attr_bits |= LS3_OBJECT_ATTR_SIZE;
+ return 0;
+ }
+
+ ldiskfs_read_attr_som(li, lo, loa);
+
+ if (loa->loa_attr_bits & LS3_OBJECT_ATTR_SIZE)
+ return 0;
+
+ /* Try stat? */
+
+ return -ENODATA;
+}
+
+static bool ls3_need_read_attr(const struct ls3_object_attrs *loa,
+ enum ls3_object_attr_bit need_bits,
+ enum ls3_object_attr_bit bit)
+{
+ return ~loa->loa_attr_bits & need_bits & bit;
+}
+
+static const struct {
+ unsigned int r_bit;
+ int (*r_read)(struct ls3_instance *,
+ struct lipe_object *,
+ struct ls3_object_attrs *);
+} ls3_ldiskfs_attr_readers[] = {
+ { LS3_OBJECT_ATTR_FILE_FID, &ldiskfs_read_attr_file_fid },
+ { LS3_OBJECT_ATTR_FILTER_FID, &ldiskfs_read_attr_filter_fid },
+ { LS3_OBJECT_ATTR_HSM, &ldiskfs_read_attr_hsm },
+ { LS3_OBJECT_ATTR_LINKS, &ldiskfs_read_attr_links },
+ { LS3_OBJECT_ATTR_LMV, &ldiskfs_read_attr_lmv },
+ { LS3_OBJECT_ATTR_LOV, &ldiskfs_read_attr_lov },
+ { LS3_OBJECT_ATTR_PATHS, &ldiskfs_read_attr_paths },
+ { LS3_OBJECT_ATTR_SELF_FID, &ldiskfs_read_attr_self_fid },
+ { LS3_OBJECT_ATTR_SIZE, &ldiskfs_read_attr_size },
+ { LS3_OBJECT_ATTR_SOM, &ldiskfs_read_attr_som },
+ { LS3_OBJECT_ATTR_XATTRS, &ldiskfs_read_attr_xattrs },
+};
+
+int ls3_read_attrs(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa,
+ enum ls3_object_attr_bit need_bits,
+ bool quit_on_error)
+{
+ int rc = 0;
+ size_t i;
+
+ for (i = 0; i < ARRAY_SIZE(ls3_ldiskfs_attr_readers); i++) {
+ if (ls3_need_read_attr(loa, need_bits, ls3_ldiskfs_attr_readers[i].r_bit)) {
+ int rc2 = (*ls3_ldiskfs_attr_readers[i].r_read)(li, lo, loa);
+ if (rc2 != 0) {
+ if (quit_on_error)
+ return rc2;
+
+ if (rc == 0)
+ rc = rc2;
+ }
+ }
+ }
+
+ assert((need_bits & loa->loa_attr_bits) == need_bits || rc < 0);
+
+ return rc;
+}
+
+static bool ldiskfs_scan_get_next_group_batch(struct ls3_scan_control *sc,
+ unsigned long *next_start,
+ unsigned long *next_end)
+{
+ bool found_batch;
+
+ assert(sc->sc_group_current <= sc->sc_group_end);
+
+ pthread_mutex_lock(&sc->sc_group_mutex);
+
+ found_batch = sc->sc_group_current < sc->sc_group_end;
+ if (found_batch) {
+ *next_start = sc->sc_group_current;
+ sc->sc_group_current += sc->sc_group_batch;
+ if (sc->sc_group_current >= sc->sc_group_end)
+ sc->sc_group_current = sc->sc_group_end;
+ *next_end = sc->sc_group_current;
+ }
+
+ pthread_mutex_unlock(&sc->sc_group_mutex);
+
+ return found_batch;
+}
+
+/*
+ * Return allocated blocks in 512 byte
+ *
+ * The similar function in E2fsprogs has a bug LU-13103
+ */
+static blkcnt_t blocks_from_inode(ext2_filsys fs, struct ext2_inode_large *inode)
+{
+ blkcnt_t b;
+
+ b = inode->i_blocks;
+ if (ext2fs_has_feature_huge_file(fs->super)) {
+ b += ((long long) inode->osd2.linux2.l_i_blocks_hi) << 32;
+ if (inode->i_flags & EXT4_HUGE_FILE_FL)
+ b *= fs->blocksize / 512;
+ }
+ return b;
+}
+
+/* XXX There are some pitfalls in debugfs/ext4 {a.m,c}time extra field encoding.
+ * Search for "[BUG] ext4 timestamps corruption". */
+__s64 inode_time(__u32 xtime, __u32 xtime_extra)
+{
+ __u64 time = (__s32)xtime;
+
+ time += (__u64)(xtime_extra & EXT4_EPOCH_MASK) << 32;
+
+ return time;
+}
+
+static void ldiskfs_copy_inode_attrs(ext2_filsys fs,
+ struct ls3_object_attrs *loa,
+ ext2_ino_t ino,
+ struct ext2_inode_large *inode)
+{
+ loa->loa_ino = ino;
+ loa->loa_atime = inode_time(inode->i_atime, inode->i_atime_extra);
+ loa->loa_mtime = inode_time(inode->i_mtime, inode->i_mtime_extra);
+ loa->loa_ctime = inode_time(inode->i_ctime, inode->i_ctime_extra);
+ /* crtime? */
+ loa->loa_size = EXT2_I_SIZE(inode);
+ loa->loa_mode = inode->i_mode; /* __u16 */
+ loa->loa_uid = inode_uid(*inode);
+ loa->loa_gid = inode_gid(*inode);
+ loa->loa_flags = inode->i_flags; /* __u32 */
+ loa->loa_nlink = inode->i_links_count; /* __u16 */
+ loa->loa_blocks = blocks_from_inode(fs, inode);
+ loa->loa_projid = inode_projid(*inode);
+
+ loa->loa_attr_bits = LS3_OBJECT_ATTR_BASE | LS3_OBJECT_ATTR_PROJID;
+}
+
+static int ldiskfs_scan_groups(ext2_inode_scan scan,
+ ext2_filsys fs,
+ struct ls3_object_attrs *loa,
+ struct lipe_object *lo,
+ struct ext2_inode_large *inode, int inode_size,
+ struct ls3_thread_info *ti,
+ unsigned long start_group)
+{
+ struct ls3_instance *li;
+ int rc;
+
+ LS3_DEBUG_U(start_group);
+
+ li = ti->ti_instance;
+
+ rc = ext2fs_inode_scan_goto_blockgroup(scan, start_group);
+ if (rc) {
+ LS3_ERROR("cannot goto block group %lu: %s\n",
+ start_group, error_message(rc));
+ goto out;
+ }
+
+ while (!ext2fs_get_next_inode_full(scan, &lo->u.lo_ldiskfs.lol_ino,
+ (struct ext2_inode *)inode,
+ inode_size)) {
+ ext2_ino_t ino;
+ int rc2;
+
+ ino = lo->u.lo_ldiskfs.lol_ino;
+ if (ino == 0)
+ break;
+
+ if (ti->ti_should_stop) {
+ LS3_DEBUG("thread %d stopping before scan complete\n", ti->ti_thread_index);
+ break;
+ }
+
+ if (ext2fs_fast_test_inode_bitmap2(fs->inode_map, ino) == 0)
+ /* deleted - always skip for now */
+ continue;
+
+ if (inode->i_flags & EXT4_SNAPFILE_FL) {
+ /* skip snapshot inodes */
+ continue;
+ }
+
+ if (!LINUX_S_ISDIR(inode->i_mode) &&
+ (inode->i_flags & EXT2_NODUMP_FL)) {
+ /* skip files which are not to be backuped */
+ ext2fs_fast_unmark_inode_bitmap2(fs->inode_map, ino);
+ continue;
+ }
+
+ lipe_object_attrs_reset(loa);
+ ldiskfs_copy_inode_attrs(fs, loa, ino, inode);
+
+ rc2 = ls3_read_attrs(li, lo, loa, li->li_required_attrs,
+ true /* quit on error. */);
+ if (rc2 != 0) {
+ LS3_DEBUG_OBJ(lo, "missing required attrs %#"PRIxMAX"\n",
+ (uintmax_t)(li->li_required_attrs & ~loa->loa_attr_bits));
+ continue;
+ }
+
+ assert((loa->loa_attr_bits & li->li_required_attrs) ==
+ li->li_required_attrs);
+
+ /* Requiring fid_is_norm() prevents us from returning
+ * ROOT, .lustre, .lustre/fid, etc. It also prevents
+ * crashes when we try to access objects by or using
+ * fid2path. */
+ if ((li->li_required_attrs & LS3_OBJECT_ATTR_SELF_FID) &&
+ !fid_is_norm(&loa->loa_self_fid) && !fid_is_idif(&loa->loa_self_fid))
+ continue;
+
+ if (lo_is_dir_stripe(li, lo, loa))
+ continue;
+
+ LS3_CURRENT.lc_object = lo;
+ LS3_CURRENT.lc_attrs = loa;
+
+ scm_catch(SCM_BOOL_T, li->li_scm_policy_thunk, ls3_scm_policy_exception_handler);
+
+ LS3_CURRENT.lc_object = NULL;
+ LS3_CURRENT.lc_attrs = NULL;
+ }
+
+ rc = 0;
+out:
+ return rc;
+}
+
+static errcode_t ls3_done_group_callback(ext2_filsys fs, ext2_inode_scan scan,
+ dgrp_t group, void *data)
+{
+ struct ls3_thread_info *ti = data;
+
+ ti->ti_group_start++;
+
+ return ti->ti_group_start >= ti->ti_group_end;
+}
+
+/* Open and scan a slice of an ext2 FS. We have already entered scheme context. */
+static void *ls3_scan_thread_start_scm(void *arg)
+{
+ struct ls3_thread_info *ti = arg;
+ struct ls3_instance *li = ti->ti_instance;
+ const char *dev = li->li_device_path;
+ ext2_filsys fs = NULL;
+ ext2_inode_scan scan = NULL;
+ struct ext2_inode_large *inode = NULL;
+ struct lipe_object lo = {
+ };
+ struct ls3_object_attrs loa;
+ int inode_size;
+ intptr_t rc;
+
+ LS3_DEBUG_D(ti->ti_thread_index);
+
+ /* Note LS3_CURRENT is a thread local context. */
+ LS3_CURRENT.lc_instance = li;
+ LS3_CURRENT.lc_thread_info = ti;
+ LS3_CURRENT.lc_scm_thread_index = scm_from_int(ti->ti_thread_index);
+
+ rc = lipe_object_attrs_init(&loa);
+ if (rc != 0) {
+ LS3_ERROR("cannot initialize object attrs: %s\n", xstrerror(rc));
+ goto out;
+ }
+
+ rc = ext2fs_open(dev, EXT2_FLAG_SOFTSUPP_FEATURES,
+ 0, 0, unix_io_manager, &fs);
+ if (rc != 0) {
+ LS3_ERROR("cannot open backing filesystem in '%s': %s\n",
+ dev, error_message(rc));
+ goto out_free_attrs;
+ }
+
+ lo.u.lo_ldiskfs.lol_fs = fs;
+
+ rc = ext2fs_read_inode_bitmap(fs);
+ if (rc) {
+ LS3_ERROR("cannot read inode bitmap from '%s': %s\n", dev,
+ error_message(rc));
+ goto out_close;
+ }
+
+ rc = ext2fs_open_inode_scan(fs, fs->inode_blocks_per_group, &scan);
+ if (rc) {
+ LS3_ERROR("cannot open inode scan on '%s': %s\n", dev,
+ error_message(rc));
+ goto out_close;
+ }
+
+ inode_size = EXT2_INODE_SIZE(fs->super);
+ if (inode_size < sizeof(*inode))
+ inode_size = sizeof(*inode);
+
+ inode = xcalloc(1, inode_size);
+ lo.u.lo_ldiskfs.lol_inode = inode;
+
+ while (1) {
+ unsigned long start_group;
+ unsigned long end_group;
+ bool found_batch;
+
+ found_batch = ldiskfs_scan_get_next_group_batch(ti->ti_scan_control,
+ &start_group, &end_group);
+ if (!found_batch)
+ break;
+
+ ti->ti_group_start = start_group;
+ ti->ti_group_end = end_group;
+
+ LS3_DEBUG("begin scan of groups [%lu, %lu)\n",
+ ti->ti_group_start, ti->ti_group_end);
+
+ ext2fs_set_inode_callback(scan, &ls3_done_group_callback, ti);
+
+ rc = ldiskfs_scan_groups(scan, fs, &loa, &lo, inode, inode_size,
+ ti, start_group);
+ if (rc != 0)
+ goto out_free;
+
+ LS3_DEBUG("end scan of groups [%lu, %lu)\n",
+ ti->ti_group_start, ti->ti_group_end);
+ }
+
+ LS3_CURRENT.lc_instance = NULL;
+
+out_free:
+ free(inode);
+ ext2fs_close_inode_scan(scan);
+out_close:
+ ext2fs_close(fs);
+out_free_attrs:
+ lipe_object_attrs_fini(&loa);
+out:
+ LS3_DEBUG_D(rc);
+
+ return (void *)rc;
+}
+
+/* pthread_create() thread start routine. */
+static void *ls3_scan_thread_start(void *arg)
+{
+ ls3_tid = syscall(SYS_gettid);
+ return scm_with_guile(&ls3_scan_thread_start_scm, arg);
+}
+
+static int ls3_scan_control_start(struct ls3_scan_control *sc, struct ls3_instance *li,
+ size_t thread_count)
+{
+ pthread_attr_t attr, *pattr = NULL;
+ int rc;
+ int rc2;
+ int i;
+
+ sc->sc_thread_infos = xcalloc(thread_count, sizeof(sc->sc_thread_infos[0]));
+ sc->sc_thread_count = thread_count;
+
+ rc = pthread_attr_init(&attr);
+ if (rc) {
+ LS3_ERROR("cannot initialize pthread attributes: %s\n", xstrerror(rc));
+ goto out;
+ }
+
+ pattr = &attr;
+
+ for (i = 0; i < sc->sc_thread_count; i++) {
+ struct ls3_thread_info *ti = &sc->sc_thread_infos[i];
+ ti->ti_instance = li;
+ ti->ti_scan_control = sc;
+ ti->ti_thread_index = i;
+
+ rc = pthread_create(&ti->ti_thread_id, pattr, &ls3_scan_thread_start, ti);
+ if (rc != 0) {
+ LS3_ERROR("cannot create scanning thread %d: %s\n",
+ ti->ti_thread_index, xstrerror(rc));
+ rc = LS3_EXIT_SCAN_ERROR;
+ break;
+ }
+
+ ti->ti_started = true;
+ }
+out:
+ if (pattr != NULL) {
+ rc2 = pthread_attr_destroy(pattr);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot destroy thread attribute: %s\n", xstrerror(rc2));
+ rc = LS3_EXIT_SCAN_ERROR;
+ }
+ }
+
+ return rc;
+}
+
+static void ls3_scan_control_stop(struct ls3_scan_control *sc)
+{
+ int i;
+
+ for (i = 0; i < sc->sc_thread_count; i++)
+ sc->sc_thread_infos[i].ti_should_stop = 1;
+}
+
+static int ls3_scan_control_join(struct ls3_scan_control *sc)
+{
+ int rc = 0;
+ int i;
+
+ for (i = 0; i < sc->sc_thread_count; i++) {
+ struct ls3_thread_info *ti = &sc->sc_thread_infos[i];
+ void *thread_rcp = NULL;
+ intptr_t thread_rc;
+ int rc2;
+
+ if (!ti->ti_started)
+ continue;
+
+ rc2 = pthread_join(ti->ti_thread_id, &thread_rcp);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot join thread %d: %s\n", ti->ti_thread_index, xstrerror(rc2));
+ rc = LS3_EXIT_SCAN_ERROR;
+ continue;
+ }
+
+ thread_rc = (intptr_t)thread_rcp;
+
+ LS3_DEBUG("scanning thread %d completed with rc = %"PRIdPTR"\n",
+ ti->ti_thread_index, thread_rc);
+
+ if (sc->sc_thread_rc_max < imaxabs(thread_rc))
+ sc->sc_thread_rc_max = imaxabs(thread_rc);
+
+ sc->sc_read_attr_error_count += ti->ti_read_attr_error_count;
+ sc->sc_other_error_count += ti->ti_other_error_count;
+ }
+
+ return rc;
+}
+
+#define DEFAULT_GROUP_BATCH 10
+
+static intmax_t ls3_scan_fs(struct ls3_instance *li, ext2_filsys fs, size_t thread_count)
+{
+ unsigned long group_total, group_batch;
+ struct ls3_scan_control sc = {
+ .sc_thread_count = 0,
+ .sc_break_rc = INTMAX_MIN,
+ };
+ intmax_t rc, rc2;
+
+ pthread_mutex_init(&sc.sc_group_mutex, NULL);
+
+ group_total = fs->group_desc_count;
+ LS3_DEBUG_U(group_total);
+ sc.sc_group_end = group_total;
+ sc.sc_group_current = 0;
+
+ /* keep the scan batch in a reasonable small one */
+ group_batch = group_total / thread_count;
+ if (!group_batch)
+ group_batch = 1;
+ else if (group_batch > DEFAULT_GROUP_BATCH)
+ group_batch = DEFAULT_GROUP_BATCH;
+ LS3_DEBUG_U(group_batch);
+ sc.sc_group_batch = group_batch;
+
+ rc = ls3_scan_control_start(&sc, li, thread_count);
+ if (rc != 0) {
+ LS3_ERROR("cannot start scanning threads: %s\n", xstrerror(rc));
+ ls3_scan_control_stop(&sc);
+ }
+
+ rc2 = ls3_scan_control_join(&sc);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot join scanning threads: %s\n", xstrerror(rc2));
+ if (rc == 0)
+ rc = rc2;
+ }
+
+ free(sc.sc_thread_infos);
+
+ LS3_DEBUG_D(rc);
+ LS3_DEBUG_D(sc.sc_thread_rc_max);
+ LS3_DEBUG_U(sc.sc_read_attr_error_count);
+ LS3_DEBUG_U(sc.sc_other_error_count);
+
+ if (sc.sc_read_attr_error_count > 0)
+ LS3_WARNING("device '%s': read attr error count %"PRIuMAX"\n",
+ li->li_device_path, sc.sc_read_attr_error_count);
+
+ if (sc.sc_other_error_count > 0)
+ LS3_WARNING("device '%s': other error count %"PRIuMAX"\n",
+ li->li_device_path, sc.sc_other_error_count);
+
+ /* If (lipe-scan-break rc) was called then return rc
+ * regardless of other errors. */
+ if (sc.sc_break_rc != INTMAX_MIN)
+ return sc.sc_break_rc;
+
+ return rc != 0 ? rc : sc.sc_thread_rc_max;
+}
+
+static int ls3_read_ldd(const char *device, ext2_filsys fs,
+ struct lustre_disk_data *ldd)
+{
+ ext2_file_t file = NULL;
+ ext2_ino_t ino;
+ unsigned int got;
+ int rc = LS3_EXIT_EXT2_ERROR;
+ errcode_t rc2;
+
+ rc2 = ext2fs_namei(fs, EXT2_ROOT_INO, EXT2_ROOT_INO, MOUNT_DATA_FILE, &ino);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot find '%s' in '%s': %s\n",
+ MOUNT_DATA_FILE, device, error_message(rc2));
+ goto out;
+ }
+
+ rc2 = ext2fs_file_open(fs, ino, 0, &file);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot open '%s' (ino %llu) from '%s': %s\n",
+ MOUNT_DATA_FILE, (unsigned long long)ino, device,
+ error_message(rc2));
+ goto out;
+ }
+
+ rc2 = ext2fs_file_read(file, ldd, sizeof(*ldd), &got);
+ if (rc2 != 0) {
+ LS3_ERROR("cannot read '%s' (ino %llu) from '%s': %s\n",
+ MOUNT_DATA_FILE, (unsigned long long)ino, device,
+ error_message(rc2));
+ goto out_file;
+ }
+
+ if (got == 0) {
+ LS3_ERROR("mount data file '%s' (ino %llu) in '%s' is empty\n",
+ MOUNT_DATA_FILE, (unsigned long long)ino, device);
+ goto out_file;
+ }
+
+ rc = 0;
+out_file:
+ ext2fs_file_close(file);
+out:
+ return rc;
+}
+
+static int ls3_scan_client_mount_init(struct ls3_instance *li, const char *path, const char *ldd_fsname)
+{
+ char *path_canon = NULL;
+ char path_auto[PATH_MAX] = "";
+ char fsname[PATH_MAX] = "";
+ int fd = -1;
+ int rc;
+
+ LS3_DEBUG_S(path);
+ LS3_DEBUG_S(ldd_fsname);
+
+ if (path == LS3_CSTR_NONE) {
+ rc = 0;
+ goto out;
+ } else if (path == LS3_CSTR_AUTO) {
+ rc = llapi_search_rootpath(path_auto, ldd_fsname);
+ if (rc < 0) {
+ LS3_ERROR("cannot find client mount for fsname '%s': %s\n", ldd_fsname, xstrerror(rc));
+ goto out;
+ }
+
+ LS3_DEBUG_S(path_auto);
+ path_canon = canonicalize_file_name(path_auto);
+ LS3_DEBUG_S(path_canon);
+ } else {
+ path_canon = canonicalize_file_name(path);
+ if (path_canon == NULL) {
+ rc = -errno;
+ LS3_ERROR("cannot canonicalize client mount path '%s': %s\n",
+ path, xstrerror(rc));
+ goto out;
+ }
+
+ if (strcmp(path_canon, path) != 0)
+ LS3_WARNING("using canonicalized client mount path '%s' instead of '%s'\n",
+ path_canon, path);
+ }
+
+ fd = open(path_canon, O_RDONLY|O_DIRECTORY);
+ if (fd < 0) {
+ rc = -errno;
+ LS3_ERROR("cannot open client mount '%s': %s\n", path_canon, xstrerror(rc));
+ goto out;
+ }
+
+ rc = llapi_get_fsname_instance(path_canon, fsname, sizeof(fsname), NULL, 0);
+ if (rc < 0) {
+ LS3_ERROR("cannot get Lustre client fsname of '%s': %s\n",
+ path_canon, xstrerror(rc));
+ goto out;
+ }
+
+ if (strcmp(fsname, ldd_fsname) != 0) {
+ LS3_ERROR("Lustre client '%s' (fsname '%s') does not match device '%s' (fsname '%s')\n",
+ path_canon, fsname, li->li_device_path, ldd_fsname);
+ rc = LS3_EXIT_USAGE_ERROR;
+ goto out;
+ }
+
+ li->li_client_mount_path = path_canon;
+ li->li_scm_client_mount_path = scm_from_locale_string(li->li_client_mount_path);
+ li->li_client_mount_fd = fd;
+ li->li_scm_client_mount_fd = scm_from_int(fd);
+ path_canon = NULL;
+ fd = -1;
+ rc = 0;
+out:
+ if (path == LS3_CSTR_AUTO && rc < 0)
+ rc = 0;
+
+ if (rc < 0)
+ rc = LS3_EXIT_CLIENT_ERROR;
+
+ free(path_canon);
+
+ if (!(fd < 0))
+ close(fd);
+
+ LS3_DEBUG_D(li->li_client_mount_fd);
+
+ return rc;
+}
+
+static int ls3_scan_required_attrs_init(struct ls3_instance *li,
+ enum ls3_object_attr_bit required_attrs)
+{
+ enum ls3_object_attr_bit attrs = LS3_OBJECT_ATTR_BASE | LS3_OBJECT_ATTR_SELF_FID;
+
+ if (li->li_device_is_mdt)
+ attrs |= LS3_OBJECT_ATTR_LINKS;
+
+ if (li->li_device_is_ost)
+ attrs |= LS3_OBJECT_ATTR_FILTER_FID;
+
+ if (required_attrs == LS3_OBJECT_ATTR_DEFAULT)
+ li->li_required_attrs = attrs;
+ else
+ li->li_required_attrs = required_attrs;
+
+ if (li->li_required_attrs & LS3_OBJECT_ATTR_PATHS)
+ li->li_required_attrs |= LS3_OBJECT_ATTR_FILE_FID;
+
+ return 0;
+}
+
+static void ls3_scan_fini(struct ls3_instance *li)
+{
+ free(li->li_device_path);
+ free(li->li_device_name);
+ free(li->li_client_mount_path);
+ if (!(li->li_client_mount_fd < 0))
+ close(li->li_client_mount_fd);
+}
+
+int ls3_scan(const char *device_path,
+ const char *client_mount_path,
+ SCM policy_thunk,
+ enum ls3_object_attr_bit required_attrs,
+ int thread_count)
+{
+ struct ls3_instance li = {
+ .li_device_path = xstrdup(device_path),
+ .li_client_mount_fd = -1,
+ .li_scm_device_path = scm_from_locale_string(device_path),
+ .li_scm_device_name = scm_from_locale_string(""),
+ .li_scm_fsname = scm_from_locale_string(""),
+ .li_scm_client_mount_path = scm_from_locale_string(""),
+ .li_scm_client_mount_fd = scm_from_int(-1),
+ .li_scm_policy_thunk = policy_thunk,
+ .li_scm_thread_count = scm_from_int(0),
+ };
+ struct lustre_disk_data ldd;
+ ext2_filsys fs = NULL;
+ long rc;
+
+ memset(&ldd, 0, sizeof(ldd));
+
+ LS3_DEBUG_S(device_path);
+ LS3_DEBUG_S(client_mount_path);
+ LS3_DEBUG_X(required_attrs);
+ LS3_DEBUG_D(thread_count);
+
+ rc = ext2fs_open(device_path,
+ EXT2_FLAG_64BITS |
+ EXT2_FLAG_SKIP_MMP |
+ EXT2_FLAG_IGNORE_SB_ERRORS |
+ EXT2_FLAG_SUPER_ONLY,
+ 0 /* superblock */,
+ 0 /* block_size */,
+ unix_io_manager,
+ &fs);
+ if (rc != 0) {
+ LS3_ERROR("cannot open backing filesystem '%s': %s\n",
+ device_path, error_message(rc));
+ rc = LS3_EXIT_SCAN_ERROR;
+ goto out;
+ }
+
+ li.li_device_name = xstrdup((const char *)fs->super->s_volume_name);
+ LS3_DEBUG_S(li.li_device_name);
+ li.li_scm_device_name = scm_from_locale_string(li.li_device_name);
+
+ rc = ls3_read_ldd(device_path, fs, &ldd);
+ if (rc != 0)
+ goto out_fs;
+
+ LS3_DEBUG_X(ldd.ldd_flags);
+ LS3_DEBUG_U(ldd.ldd_svindex);
+ LS3_DEBUG_S(ldd.ldd_fsname);
+ LS3_DEBUG_S(ldd.ldd_svname);
+
+ li.li_scm_fsname = scm_from_locale_string(ldd.ldd_fsname);
+
+ switch (ldd.ldd_flags & (LDD_F_SV_TYPE_MDT | LDD_F_SV_TYPE_OST)) {
+ case LDD_F_SV_TYPE_MDT:
+ li.li_device_is_mdt = true;
+ break;
+ case LDD_F_SV_TYPE_OST:
+ li.li_device_is_ost = true;
+ break;
+ default:
+ LS3_WARNING("device '%s' has unrecognized type %#x\n",
+ li.li_device_path, ldd.ldd_flags & LDD_F_SV_TYPE_MASK);
+ break;
+ }
+
+ rc = ls3_scan_client_mount_init(&li, client_mount_path, ldd.ldd_fsname);
+ if (rc != 0)
+ goto out;
+
+ rc = ls3_scan_required_attrs_init(&li, required_attrs);
+ if (rc != 0)
+ goto out;
+
+ if ((li.li_required_attrs & LS3_OBJECT_ATTR_PATHS) && li.li_client_mount_fd < 0) {
+ LS3_ERROR("'paths' attr requires a client mount\n");
+ rc = LS3_EXIT_USAGE_ERROR;
+ goto out;
+ }
+
+ if (thread_count < 0)
+ thread_count = sysconf(_SC_NPROCESSORS_ONLN);
+ else if (thread_count == 0)
+ thread_count = sysconf(_SC_NPROCESSORS_ONLN) / 2;
+
+ if (thread_count <= 0)
+ thread_count = 1;
+
+ li.li_scm_thread_count = scm_from_int(thread_count);
+
+ rc = ls3_scan_fs(&li, fs, thread_count);
+ LS3_DEBUG_D(rc);
+out_fs:
+ ext2fs_close(fs);
+out:
+ ls3_scan_fini(&li);
+
+ return rc;
+}
+
+SCM ls3_policy_exception_handler(SCM key, SCM args)
+{
+ struct ls3_thread_info *ti = LS3_CURRENT.lc_thread_info;
+
+ if (ti == NULL || ti->ti_scan_control == NULL)
+ return SCM_BOOL_T;
+
+ if (scm_is_eq(key, ls3_sym_read_attr_error)) {
+ ti->ti_read_attr_error_count++;
+ } else if (scm_is_eq(key, ls3_sym_scan_break)) {
+ struct ls3_scan_control *sc = ti->ti_scan_control;
+ SCM rc;
+
+ LS3_DEBUG("break\n");
+ rc = SCM_CAR(args);
+ sc->sc_break_rc = scm_to_int(rc);
+ LS3_DEBUG_D(sc->sc_break_rc);
+ ls3_scan_control_stop(sc);
+ } else if (scm_is_eq(key, ls3_sym_scan_continue)) {
+ /* OK */
+ } else {
+ SCM fmt = scm_from_locale_string("ERROR: caught ~a ~s\n");
+
+ scm_simple_format(scm_current_error_port(), fmt,
+ scm_list_2(key, args));
+
+ ti->ti_other_error_count++;
+ }
+
+ return SCM_BOOL_T;
+}
--- /dev/null
+/*
+ * Copyright (c) 2016, 2021, DDN Storage Corporation.
+ *
+ * Author: Li Xi <lixi@ddn.com>
+ * Author: John L. Hammond <jhammond@whamcloud.com>
+ */
+#ifndef _LIPE_SCAN3_H_
+#define _LIPE_SCAN3_H_
+#include <stdbool.h>
+#include <stddef.h>
+#include <inttypes.h>
+#include <ext2fs/ext2fs.h>
+#include <libguile.h>
+#include "ls3_object_attrs.h"
+
+#ifndef ARRAY_SIZE
+# define ARRAY_SIZE(a) ((sizeof(a)) / (sizeof((a)[0])))
+#endif
+
+struct lipe_object_ldiskfs {
+ ext2_filsys lol_fs;
+ ext2_ino_t lol_ino;
+ struct ext2_inode_large *lol_inode;
+};
+
+struct lipe_object {
+ union {
+ struct lipe_object_ldiskfs lo_ldiskfs;
+ } u;
+};
+
+struct ls3_instance {
+ char *li_device_path;
+ char *li_device_name;
+ bool li_device_is_mdt;
+ bool li_device_is_ost;
+ char *li_client_mount_path;
+ int li_client_mount_fd;
+ enum ls3_object_attr_bit li_required_attrs;
+ SCM li_scm_policy_thunk;
+ SCM li_scm_device_path;
+ SCM li_scm_device_name;
+ SCM li_scm_fsname;
+ SCM li_scm_client_mount_path;
+ SCM li_scm_client_mount_fd;
+ SCM li_scm_thread_count;
+};
+
+struct ls3_context {
+ struct ls3_instance *lc_instance;
+ struct ls3_thread_info *lc_thread_info;
+ SCM lc_scm_thread_index; // 0,1,2... < thread_count.
+ struct lipe_object *lc_object;
+ struct ls3_object_attrs *lc_attrs;
+};
+
+extern const char LS3_CSTR_AUTO[];
+extern const char LS3_CSTR_NONE[];
+
+extern __thread struct ls3_context LS3_CURRENT;
+extern SCM ls3_sym_read_attr_error;
+extern SCM ls3_sym_scan_break;
+extern SCM ls3_sym_scan_continue;
+extern SCM ls3_scm_policy_exception_handler;
+SCM ls3_policy_exception_handler(SCM key, SCM args);
+
+/* typedef __u32 __bitwise ext2_ino_t; */
+
+static inline uintmax_t lo_ino(const struct lipe_object *lo)
+{
+ return lo->u.lo_ldiskfs.lol_ino;
+}
+
+#define LS3_DEBUG_OBJ(lo, fmt, args...) \
+ LS3_DEBUG("ino = %"PRIuMAX": " fmt, lo_ino(lo), ##args)
+
+#define LS3_ERROR_OBJ(lo, fmt, args...) \
+ LS3_ERROR("ino = %"PRIuMAX": " fmt, lo_ino(lo), ##args)
+
+int ls3_read_attrs(struct ls3_instance *li,
+ struct lipe_object *lo,
+ struct ls3_object_attrs *loa,
+ enum ls3_object_attr_bit need_bits,
+ bool quit_on_error);
+
+int ls3_scan(const char *device_path,
+ const char *client_mount_path,
+ SCM policy_thunk,
+ enum ls3_object_attr_bit required_attrs,
+ int thread_count);
+
+#endif /* _LIPE_SCAN3_H_ */
--- /dev/null
+#!/usr/bin/env lipe_scan3_script
+!#
+
+(for-each (lambda (device-path)
+ (lipe-scan device-path
+ (lipe-getopt-client-mount-path)
+ (lambda ()
+ (if (>= (ino) 1000)
+ (lipe-scan-break 7))
+ (format #t "~a ~a\n" (lipe-scan-thread-index) (ino)))
+ (lipe-getopt-required-attrs)
+ (lipe-getopt-thread-count)))
+ (cdr (program-arguments)))
--- /dev/null
+#!/usr/bin/env lipe_scan3_script
+!#
+
+(for-each (lambda (device-path)
+ (lipe-scan device-path
+ (lipe-getopt-client-mount-path)
+ (lambda ()
+ (format #t "~a ~a\n" (lipe-scan-thread-index) (ino))
+ (lipe-scan-continue)
+ (format #t "REACHED\n"))
+ (lipe-getopt-required-attrs)
+ (lipe-getopt-thread-count)))
+ (cdr (program-arguments)))
--- /dev/null
+#!/usr/bin/env lipe_scan3_script
+!#
+
+(for-each (lambda (device-path)
+ (lipe-scan device-path
+ (lipe-getopt-client-mount-path)
+ (lambda () (format #t "~a ~a\n" (lipe-scan-device-name) (ino)))
+ (lipe-getopt-required-attrs)
+ (lipe-getopt-thread-count)))
+ (cdr (program-arguments)))
--- /dev/null
+#!/usr/bin/env lipe_scan3_script
+!#
+
+(for-each (lambda (device-path)
+ (lipe-scan device-path
+ (lipe-getopt-client-mount-path)
+ print-json
+ (lipe-getopt-required-attrs)
+ (lipe-getopt-thread-count)))
+ (cdr (program-arguments)))
--- /dev/null
+#!/usr/bin/env lipe_scan3_script
+!#
+
+(use-modules (ice-9 readline))
+(use-modules (ice-9 threads)) ; with-mutex
+(use-modules (system repl common))
+(use-modules (system repl repl))
+
+(define read-eval-print-mutex (make-mutex))
+
+(define (lipe-repl-stack-depth)
+ (length (cond
+ ((fluid-ref *repl-stack*) => cdr)
+ (else '()))))
+
+(define (lipe-repl-prompt)
+ (let ((depth (lipe-repl-stack-depth)))
+ (if (zero? depth)
+ (format #f "[tid=~s device=~s ino=~s]> " (lipe-gettid) (lipe-scan-device-name) (ino))
+ (format #f "[tid=~s device=~s ino=~s depth=~a]> " (lipe-gettid) (lipe-scan-device-name) (ino) depth))))
+
+(define (lipe-repl-policy)
+ (dynamic-wind
+ (lambda ()
+ (set! repl-welcome (lambda (repl) #t))
+ (repl-default-prompt-set! lipe-repl-prompt))
+ (lambda ()
+ (with-mutex read-eval-print-mutex
+ (activate-readline)
+ (start-repl)))
+ (lambda ()
+ (repl-default-prompt-set! "> "))))
+
+
+(for-each (lambda (device-path)
+ (lipe-scan device-path
+ (lipe-getopt-client-mount-path)
+ lipe-repl-policy
+ (lipe-getopt-required-attrs)
+ (lipe-getopt-thread-count)))
+ (cdr (program-arguments)))
--- /dev/null
+#!/bin/bash
+#
+# This exists just to make '#!/usr/bin/env lipe_scan3_script'
+# work. With coreutils 8.30 we can use the -S, --split-string option
+# to /usr/bin/env.
+#
+# FIXME load lipe_scan_functions.scm here (or in ls3_main.c).
+
+exec lipe_scan3 --script "$@"
--- /dev/null
+(import (rnrs base (6))) ;; assert
+
+(assert #t)
+(assert #f)
+(assert #t)
--- /dev/null
+(import (rnrs base (6))) ;; assert
+
+(define root-fid-string "[0x200000007:0x1:0x0]")
+(define root-fid-seq #x200000007)
+(define root-fid-oid #x1)
+(define root-fid-ver #x0)
+(define root-fid (make-fid root-fid-seq root-fid-oid root-fid-ver))
+
+(assert (fid? root-fid))
+(assert (not (fid? #t)))
+(assert (fid=? root-fid (make-fid root-fid-seq root-fid-oid root-fid-ver)))
+(assert (not (fid=? root-fid (make-fid 42 root-fid-oid root-fid-ver))))
+(assert (not (fid=? root-fid (make-fid root-fid-seq 42 root-fid-ver))))
+(assert (not (fid=? root-fid (make-fid root-fid-seq root-fid-oid 42))))
+
+(assert (= (fid-seq root-fid) root-fid-seq))
+(assert (= (fid-oid root-fid) root-fid-oid))
+(assert (= (fid-ver root-fid) root-fid-ver))
+
+(assert (fid=? root-fid (string->fid root-fid-string)))
+(assert (string=? root-fid-string (fid->string root-fid)))
+
+(write root-fid)
+(newline)
+(display root-fid)
+(newline)
+(%fid-printer root-fid #t)
+(newline)
+(format #t "~a\n" root-fid)
+(format #f "~a\n" root-fid)
+(format (current-error-port) "~a\n" root-fid)
+
+(format #t "~s\n" root-fid)
+(format #f "~s\n" root-fid)
+(format (current-error-port) "~s\n" root-fid)
+
+(assert (catch
+ 'wrong-type-arg
+ (lambda () (fid=? root-fid #t) #f)
+ (lambda x #t)))
+
+(assert (catch
+ 'wrong-type-arg
+ (lambda () (fid-seq #t) #f)
+ (lambda x #t)))
+
+(assert (catch
+ 'wrong-type-arg
+ (lambda () (fid-oid #t) #f)
+ (lambda x #t)))
+
+(assert (catch
+ 'wrong-type-arg
+ (lambda () (fid-ver #t) #f)
+ (lambda x #t)))
+
+(let* ((v (make-vtable "pwpwpw"))
+ (s (make-struct/no-tail v 4 8 15)))
+ (assert (catch
+ 'wrong-type-arg
+ (lambda () (fid-seq s) #f)
+ (lambda x #t))))
--- /dev/null
+(import (rnrs base (6))) ;; assert
+
+(assert (procedure? fnmatch?))
+(assert (integer? FNM_CASEFOLD))
+(assert (positive? FNM_CASEFOLD))
+
+(assert (eq? #t (fnmatch? "f0" "f0")))
+(assert (eq? #t (fnmatch? "f*" "f0")))
+(assert (eq? #t (fnmatch? "f*" "f0" 0)))
+(assert (eq? #t (fnmatch? "f*" "f0" FNM_CASEFOLD)))
+(assert (eq? #t (fnmatch? "F*" "f0" FNM_CASEFOLD)))
+(assert (eq? #t (fnmatch? "f*" "F0" FNM_CASEFOLD)))
+
+(assert (eq? #f (fnmatch? "f0" "g0")))
+(assert (eq? #f (fnmatch? "f*" "g0")))
+(assert (eq? #f (fnmatch? "f*" "g0" 0)))
+(assert (eq? #f (fnmatch? "f*" "g0" FNM_CASEFOLD)))
+(assert (eq? #f (fnmatch? "F*" "g0" FNM_CASEFOLD)))
+(assert (eq? #f (fnmatch? "f*" "G0" FNM_CASEFOLD)))
+
+(assert (catch 'wrong-type-arg (lambda () (fnmatch? #t "f0") #f) list))
+(assert (catch 'wrong-type-arg (lambda () (fnmatch? "f*" #t) #f) list))
+(assert (catch 'wrong-type-arg (lambda () (fnmatch? "f*" "f0" #t) #f) list))
+
+(assert (catch 'wrong-number-of-args (lambda () (fnmatch?) #f) list))
+(assert (catch 'wrong-number-of-args (lambda () (fnmatch? "f*") #f) list))
+(assert (catch 'wrong-number-of-args (lambda () (fnmatch? "f*" "f0" 0 0) #f) list))
--- /dev/null
+(import (rnrs base (6))) ;; assert
+
+(assert (procedure? lipe-debug-enable))
+(assert (eq? #f (lipe-debug-enable)))
+(assert (eq? #f (lipe-debug-enable #t)))
+(assert (eq? #t (lipe-debug-enable)))
+(assert (eq? #t (lipe-debug-enable #f)))
+(assert (eq? #f (lipe-debug-enable)))
+
+(assert (procedure? lipe-gettid))
+(assert (integer? (lipe-gettid)))
+(assert (positive? (lipe-gettid)))
+
+(assert (procedure? lipe-getopt-device-path))
+(assert (eq? #f (lipe-getopt-device-path)))
+
+(assert (procedure? lipe-getopt-client-mount-path))
+(assert (eq? #t (lipe-getopt-client-mount-path)))
+
+(assert (procedure? lipe-getopt-thread-count))
+(assert (eq? 0 (lipe-getopt-thread-count)))
+
+(assert (procedure? lipe-getopt-required-attrs))
+(assert (= #xffffffff (lipe-getopt-required-attrs)))
+
+(for-each (lambda (proc)
+ (format #t "~a\n" proc)
+ (assert (procedure? proc))
+ (assert (catch '*lipe-no-context*
+ (lambda () (proc) #f)
+ list)))
+ (list
+ ino
+ mode
+ nlink
+ flags
+ uid
+ gid
+ projid
+ atime
+ mtime
+ ctime
+ size
+ blocks
+
+ file-fid
+ self-fid
+ links
+ pools
+ xattrs
+
+ absolute-paths
+ relative-paths
+
+ lipe-scan-client-mount-fd
+ lipe-scan-client-mount-path
+ lipe-scan-current-attrs
+ lipe-scan-device-name
+ lipe-scan-device-path
+ lipe-scan-fsname
+ lipe-scan-thread-count
+ lipe-scan-thread-index
+
+ print-file-fid
+ print-self-fid
+ print-json
+ print-absolute-path
+ print-relative-path
+))
+
+(assert (catch '*lipe-no-context*
+ (lambda () (xattr-ref "user.foo") #f)
+ list))
+
+(assert (catch '*lipe-scan-break*
+ (lambda () (lipe-scan-break 0) #f)
+ list))
+
+(assert (catch '*lipe-scan-continue*
+ (lambda () (lipe-scan-continue) #f)
+ list))
+
+(assert (procedure? lipe-scan))
+
+;; k:lipe_scan3# echo '(format #t "~a\n" (lipe-getopt-client-mount-path))' | lipe_scan3 --client-mount=/mnt/lustre -s /dev/stdin
* \param head the head for your list.
* \param member the name of the list_struct within the struct.
*/
-#define list_for_each_entry_reverse(pos, head, member) \
+#define lipe_list_for_each_entry_reverse(pos, head, member) \
for (pos = lipe_list_entry((head)->prev, typeof(*pos), member); \
prefetch(pos->member.prev), &pos->member != (head); \
pos = lipe_list_entry(pos->member.prev, typeof(*pos), member))
noinst_SCRIPTS += setup-cifs.sh parallel-scale-cifs.sh
noinst_SCRIPTS += posix.sh sanity-scrub.sh scrub-performance.sh ha.sh pjdfstest.sh
noinst_SCRIPTS += sanity-lfsck.sh lfsck-performance.sh sanity-lipe.sh
+noinst_SCRIPTS += sanity-lipe-scan3.sh
noinst_SCRIPTS += resolveip
noinst_SCRIPTS += sanity-hsm.sh sanity-lsnapshot.sh sanity-pfl.sh sanity-flr.sh
noinst_SCRIPTS += sanity-dom.sh sanity-pcc.sh dom-performance.sh sanity-lnet.sh
--- /dev/null
+#!/bin/bash
+#
+# Tests for lipe_find and lipe_scan.
+#
+# lipe_find - search for files on Lustre devices or directories
+# lipe_scan - fast scan tool based on LiPE (Lustre integrated Policy Engine)
+
+ONLY=${ONLY:-"$*"}
+
+LUSTRE=${LUSTRE:-$(dirname $0)/..}
+. $LUSTRE/tests/test-framework.sh
+init_test_env $@
+. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
+init_logging
+FAIL_ON_ERROR=false
+
+declare -r ROOT_FID="[0x200000007:0x1:0x0]"
+
+# bug number for skipped test:
+ALWAYS_EXCEPT="$SANITY_LIPE_SCAN3_EXCEPT"
+# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
+
+(( OSTCOUNT >= 2 )) || skip_env "need at least 2 OSTs"
+
+[[ $(facet_fstype mds1) = ldiskfs ]] || skip_env "need ldiskfs on MDS"
+
+! remote_mds_nodsh || skip_env "remote MDS with nodsh"
+! remote_ost_nodsh || skip_env "remote OSS with nodsh"
+
+# check if lipe_scan3 is installed on MDS(s)
+for t in lipe_scan3; do
+ do_nodes $(comma_list $(all_mdts_nodes)) "which $t" &>/dev/null ||
+ skip_env "$t is not installed on MDS"
+done
+
+which jq || skip_env "jq is not installed"
+
+build_test_filter
+check_and_setup_lustre
+
+# mount Lustre clients on MDS node(s)
+mount_client_on_mds() {
+ local mds_nodes
+
+ mds_nodes=$(exclude_items_from_list $(comma_list $(all_mdts_nodes)) \
+ $HOSTNAME)
+ if [[ -n $mds_nodes ]]; then
+ zconf_mount_clients $mds_nodes $MOUNT ||
+ error "failed to mount client on MDS node $mds_nodes"
+ stack_trap "zconf_umount_clients $mds_nodes $MOUNT"
+ fi
+}
+
+# if the test suite was run on an MDS, then return the MDS facet
+lipe_get_local_mds() {
+ local facet
+ local facets
+
+ facets=$(get_facets MDS)
+ for facet in ${facets//,/ }; do
+ if local_node $(facet_active_host $facet); then
+ echo -n $facet
+ return
+ fi
+ done
+}
+
+function lipe_scan3_body() {
+ local device="$1"
+ local body="$2"
+ shift 2
+
+ sync
+ printf '(lipe-scan "%q" (lipe-getopt-client-mount-path) (lambda () %s) (lipe-getopt-required-attrs) (lipe-getopt-thread-count))' "${device}" "${body}" |
+ lipe_scan3 --script=/dev/stdin "$@"
+}
+
+function lipe_scan3_format() {
+ local device="$1"
+ local expr="$2"
+ shift 2
+
+ sync
+ printf '(lipe-scan "%q" (lipe-getopt-client-mount-path) (lambda () (format #t "~a\n" %s)) (lipe-getopt-required-attrs) (lipe-getopt-thread-count))' "${device}" "${expr}" |
+ lipe_scan3 --script=/dev/stdin "$@"
+}
+
+function init_lipe_scan3_env() {
+ # Check "$MOUNT" is a Lustre client mount point.
+ local fid=$($LFS path2fid "$MOUNT")
+ [[ "$fid" == "$ROOT_FID" ]] || error "'$MOUNT' is not a lustre client mount"
+
+ find "$MOUNT" -mindepth 1 -delete || error "cannot clean '$MOUNT'"
+ find "$MOUNT" -mindepth 1 | grep . && error "find -delete did not delete all files from '$MOUNT'"
+ sync
+}
+
+function init_lipe_scan3_env_file() {
+ local file="$1"
+ local index
+
+ init_lipe_scan3_env
+
+ mcreate $file || error "cannot create $file"
+ index=$($LFS getstripe --mdt-index $file)
+ ((index == 0)) || error "file '$file' is not on MDT0000"
+ sync
+}
+
+function expect_stdout() {
+ "$@" | grep --quiet . || error "command '$*' should write to stdout"
+}
+
+function expect_no_stdout() {
+ "$@" | grep . && error "command '$*' should not write to stdout"
+ true
+}
+
+function expect_stderr() {
+ "$@" 2>&1 > /dev/null | grep --quiet . || error "command '$*' should write to stderr"
+}
+
+function expect_no_stderr() {
+ "$@" 2>&1 > /dev/null | grep . && error "command '$*' should not write to stderr"
+ true
+}
+
+function expect_success() {
+ "$@" > /dev/null || error "command '$*' failed"
+}
+
+function expect_failure() {
+ "$@" 2> /dev/null && error "command '$*' should fail"
+ true
+}
+
+function expect_print() {
+ expect_success "$@"
+ expect_stdout "$@"
+ expect_no_stderr "$@"
+}
+
+function expect_empty() {
+ expect_success "$@"
+ expect_no_stdout "$@"
+ expect_no_stderr "$@"
+}
+
+function expect_error() {
+ expect_failure "$@"
+ expect_no_stdout "$@"
+ expect_stderr "$@"
+}
+
+function expect_expr() {
+ local device="$1"
+ local expr="$2"
+ local a0="$3"
+ local a1
+ shift 3
+
+ a1=$(lipe_scan3_format "$device" "$expr" "$@")
+ [[ "$a0" == "$a1" ]] || error "$expr expect '$a0', got '$a1'"
+}
+
+function expect_attr() {
+ local device="$1"
+ local proc="$2"
+ local a0="$3"
+ local a1
+ shift 3
+
+ a1=$(lipe_scan3_format "$device" "($proc)" "$@")
+ [[ "$a0" == "$a1" ]] || error "$proc expect '$a0', got '$a1'"
+}
+
+test_0() {
+ expect_success true
+ expect_failure false
+ expect_no_stdout true
+ expect_no_stderr true
+ expect_stdout echo something
+ expect_stderr grep --barf
+ expect_print echo output
+ expect_empty true
+ expect_error grep --barf
+}
+run_test 0 "expect functions meet our expectations"
+
+test_10() {
+ expect_print lipe_scan3 -h
+ expect_print lipe_scan3 --help
+
+ expect_print lipe_scan3 -v
+ expect_print lipe_scan3 --version
+
+ expect_error lipe_scan3
+ expect_error lipe_scan3 --barf
+
+ expect_print lipe_scan3 --interactive < /dev/null
+ expect_print lipe_scan3 -i < /dev/null
+
+ expect_print lipe_scan3 --list-attrs
+ expect_print lipe_scan3 --list-json-attrs
+
+ expect_error lipe_scan3 -s /dev/null --thread-count=zzz
+ expect_error lipe_scan3 -s /dev/null --thread-count=42q
+ expect_error lipe_scan3 -s /dev/null --thread-count=''
+}
+run_test 10 "lipe_scan3 option handling"
+
+test_11() {
+ expect_error lipe_scan3 --print-self-fid /dev/null
+ expect_error lipe_scan3 --print-self-fid /dev/zero
+ expect_error lipe_scan3 --print-self-fid /dev/zapper/mds1_flakey
+ expect_error lipe_scan3 --print-self-fid /dev/
+ expect_error lipe_scan3 --print-self-fid ''
+ expect_error lipe_scan3 --print-self-fid
+}
+run_test 11 "lipe_scan3 bad device handling"
+
+test_12() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local tmp=$(mktemp -d)
+ local fid
+
+ init_lipe_scan3_env_file "$file"
+ fid=$($LFS path2fid "$file")
+ touch $tmp/$tfile
+
+ expect_error lipe_scan3 "$device" --print-absolute-path --client-mount=''
+ expect_error lipe_scan3 "$device" --print-absolute-path --client-mount=/dev/null
+ expect_error lipe_scan3 "$device" --print-absolute-path --client-mount=$tmp
+ expect_error lipe_scan3 "$device" --print-absolute-path --client-mount=$tmp/noent
+ expect_error lipe_scan3 "$device" --print-absolute-path --client-mount=$tmp/$tfile
+
+ # We print a warning for non canonicalized client mounts.
+ expect_stderr lipe_scan3 "$device" --print-absolute-path --client-mount=$MOUNT/
+ expect_stderr lipe_scan3 "$device" --print-absolute-path --client-mount=$MOUNT//
+ expect_stderr lipe_scan3 "$device" --print-absolute-path --client-mount=$MOUNT/.
+ expect_stderr lipe_scan3 "$device" --print-absolute-path --client-mount=$MOUNT/./.
+ (cd $MOUNT && expect_stderr lipe_scan3 "$device" --print-absolute-path --client-mount=.)
+
+ expect_attr "$device" self-fid "$fid" --client-mount=$MOUNT/
+ expect_attr "$device" self-fid "$fid" --no-client-mount
+ expect_attr "$device" absolute-paths "($file)" --client-mount=$MOUNT/
+
+ # FIXME
+ # expect_error lipe_scan3 "$device" --print-absolute-path --no-client-mount
+ # expect_error lipe_scan3 "$device" --print-relative-path --no-client-mount
+
+ umount_client $MOUNT
+ expect_attr "$device" self-fid "$fid"
+ mount_client $MOUNT
+}
+run_test 12 "--client-mount is handled correctly"
+
+test_13() {
+ expect_success lipe_scan3 -s /dev/null --required-attrs=#x0
+ expect_success lipe_scan3 -s /dev/null --required-attrs=all
+ expect_success lipe_scan3 -s /dev/null --required-attrs=ino
+ expect_success lipe_scan3 -s /dev/null --required-attrs=mode
+
+ lipe_scan3 --list-attrs | while read attr; do
+ expect_success lipe_scan3 -s /dev/null --required-attrs=$attr
+ expect_success lipe_scan3 -s /dev/null --required-attrs=ino,$attr
+ done
+
+ expect_error lipe_scan3 -s /dev/null --required-attrs=barf
+ expect_error lipe_scan3 -s /dev/null --required-attrs=ino,barf
+ expect_error lipe_scan3 -s /dev/null --required-attrs=barf,ino
+}
+run_test 13 "--required-attrs is handled correctly"
+
+# TODO missing script
+# TODO invalid script
+
+test_90() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+
+ init_lipe_scan3_env
+
+ # Create some files to be deleted.
+ touch $MOUNT/f0
+ mkdir $MOUNT/d0
+ mkfifo $MOUNT/p0
+ # lfs mkdir -c ... $MOUNT/d1 ....
+ ln -s $MOUNT/f0 $MOUNT/l0
+ mknod $MOUNT/c0 c 1 3
+ # ...
+
+ # Now delete those files.
+ init_lipe_scan3_env
+
+ expect_empty lipe_scan3 "$device" --print-file-fid
+ expect_empty lipe_scan3 "$device" --print-self-fid
+ expect_empty lipe_scan3 "$device" --print-json
+ expect_empty lipe_scan3 "$device" --print-absolute-path
+ expect_empty lipe_scan3 "$device" --print-relative-path
+ expect_empty lipe_scan3_format "$device" '(ino)'
+}
+run_test 90 "lipe_scan3 on an empty FS"
+
+test_100() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local proc
+
+ init_lipe_scan3_env_file "$file"
+
+ lipe_scan3 "$device" --print-self-fid
+ lipe_scan3_format "$device" '(self-fid)'
+ lipe_scan3_format "$device" '(ino)'
+
+ for proc in ino atime blocks ctime self-fid file-fid flags gid mode mtime nlink projid size uid; do
+ expect_print lipe_scan3_format "$device" "($proc)"
+ done
+
+ for proc in absolute-paths relative-paths links pools xattrs; do
+ expect_print lipe_scan3_format "$device" "($proc)"
+ done
+}
+run_test 100 "lipe_scan3 attrs and scan functions do something"
+
+declare -r S_IFREG=0100000
+declare -a MODES=(0 1 0111 0222 0444 0555 0666 0777)
+
+test_101() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local mode
+
+ init_lipe_scan3_env_file "$file"
+
+ mode=0$(stat --format=%a $file) # octal access rights
+ expect_attr "$device" mode $((S_IFREG | mode)) # $((...)) converts to decimal
+
+ for mode in "${MODES[@]}"; do
+ chmod "$mode" $file || error "cannot set mode to '$mode'"
+ expect_attr "$device" mode $((S_IFREG | mode))
+ done
+
+ # dir ...
+}
+run_test 101 "lipe_scan3 mode does the right thing"
+
+declare -a IDS=(
+ 0
+ 42
+ 500
+ 65534
+ 65535
+ 65536
+ 2147483646 # (INT_MAX - 1)
+ 2147483647 # (INT_MAX)
+ 2147483648 # (INT_MAX + 1)
+ 2177452800
+ 4294967293 # (UINT_MAX - 2)
+)
+
+test_102() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local id
+
+ init_lipe_scan3_env_file "$file"
+
+ id=$(stat --format=%u $file) # uid
+ expect_attr "$device" uid "$id"
+
+ for id in "${IDS[@]}"; do
+ chown $id $file || error "cannot set UID to '$id'"
+ expect_attr "$device" uid "$id"
+ done
+
+ id=$(stat --format=%g $file) # gid
+ expect_attr "$device" gid "$id"
+
+ for id in "${IDS[@]}"; do
+ chown :$id $file || error "cannot set GID to '$id'"
+ expect_attr "$device" gid "$id"
+ done
+}
+run_test 102 "lipe_scan3 uid/gid do the right thing"
+
+# XXX There are some pitfalls in debugfs/ext4 timestamp extra field encoding.
+# Search for "[BUG] ext4 timestamps corruption"
+
+declare -a TIMES=(
+ 0 # 1970
+ 481516 # 1970
+ 48151623 # 1971
+ 631152000 # 1990
+ 946684800 # 2000
+ 1640995200 # 2022
+ 2145916800 # 2038
+ 2147483646 # 2038 (INT_MAX - 1)
+ 2147483647 # 2038 (INT_MAX)
+ 2147483648 # 2038 (INT_MAX + 1)
+ 2177452800 # 2039
+ 4294967294 # 2106 (UINT_MAX - 1)
+ 4294967295 # 2106 (UINT_MAX)
+ 4294967296 # 2106 (UINT_MAX + 1)
+ 4815162342 # 2122
+)
+
+test_103() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local time
+
+ init_lipe_scan3_env_file "$file"
+
+ time=$(stat --format=%X $file) # atime
+ expect_attr "$device" atime "$time"
+
+ for time in "${TIMES[@]}"; do
+ touch --date=@$time -a $file || error "cannot set atime to '$time'"
+ expect_attr "$device" atime "$time"
+ done
+
+ time=$(stat --format=%Y $file) # mtime
+ expect_attr "$device" mtime "$time"
+
+ for time in "${TIMES[@]}"; do
+ touch --date=@$time -m $file || error "cannot set mtime to '$time'"
+ expect_attr "$device" mtime "$time"
+ done
+
+ time=$(stat --format=%Z $file) # ctime
+ expect_attr "$device" ctime "$time"
+
+ # We could use LL_IOC_FUTIMES_3 here.
+}
+run_test 103 "lipe_scan3 a/m/c-time do the right thing"
+
+declare -a SIZES=(
+ 0
+ 42
+ 2147483646 # (INT_MAX - 1)
+ 2147483647 # (INT_MAX)
+ 2147483648 # (INT_MAX + 1)
+ 2177452800
+ 4294967294 # (UINT_MAX - 1)
+ 4294967295 # (UINT_MAX)
+ 4294967296 # (UINT_MAX + 1)
+ 4815162342
+ 8589934591 # 0x1ffffffff
+ 48151623420
+ 481516234200
+ 9223372036854775807
+)
+
+test_104() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local size
+
+ init_lipe_scan3_env_file "$file"
+
+ expect_attr "$device" size 0
+
+ for size in "${SIZES[@]}"; do
+ $TRUNCATE $file "$size"
+ expect_attr "$device" size "$size"
+ done
+
+ expect_attr "$device" blocks 0
+}
+run_test 104 "lipe_scan3 size/blocks do the right thing"
+
+test_105() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+
+ init_lipe_scan3_env_file "$file"
+
+ expect_attr "$device" nlink 1
+ ln $file $file-2
+ ln $file $file-3
+ expect_attr "$device" nlink 3
+
+ # If we hold $file open and remove all three links then
+ # lipe_scan3 will still return a link count of 1 because of
+ # the PENDING/$fid link.
+}
+run_test 105 "lipe_scan3 nlink does the right thing"
+
+test_106() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local id
+
+ init_lipe_scan3_env_file "$file"
+
+ id=$($LFS project "$file" | awk '{ print $1 }')
+ expect_attr "$device" projid "$id"
+
+ for id in "${IDS[@]}"; do
+ $LFS project -p "$id" "$file" || error "cannot set projid to '$id'"
+ expect_attr "$device" projid "$id"
+ done
+}
+run_test 106 "lipe_scan3 projid does the right thing"
+
+test_107() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fid
+
+ init_lipe_scan3_env_file "$file"
+ fid=$($LFS path2fid $file)
+ expect_attr "$device" file-fid "$fid"
+ expect_attr "$device" self-fid "$fid"
+}
+run_test 107 "lipe_scan3 file-fid and self-fid do the right thing"
+
+test_108() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fd
+
+ init_lipe_scan3_env_file "$file"
+
+ # (links) returns a list of parent-fid, name pairs
+ # (([0x200000007:0x1:0x0] . "f0") ...)
+
+ expect_expr "$device" '(caar (links))' "$ROOT_FID"
+ expect_expr "$device" '(cdar (links))' "$tfile"
+
+ ln $file $file-2
+ expect_expr "$device" '(length (links))' 2
+
+ ln $file $file-3
+ expect_expr "$device" '(length (links))' 3
+
+ # FIXME link xattr is not updated when removing last link.
+ # exec {fd}>$file
+ # rm -- "$file" "$file-2" "$file-3"
+ # expect_expr "$device" '(length (links))' 0
+ # exec {fd}>&-
+}
+run_test 108 "lipe_scan3 links does the right thing"
+
+test_109() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+
+ init_lipe_scan3_env_file "$file"
+ expect_expr "$device" '(absolute-paths)' "($file)"
+ expect_expr "$device" '(relative-paths)' "($tfile)"
+
+ ln $file $file-2
+ expect_expr "$device" '(absolute-paths)' "($file $file-2)"
+ expect_expr "$device" '(relative-paths)' "($tfile $tfile-2)"
+
+ ln $file $file-3
+ expect_expr "$device" '(absolute-paths)' "($file $file-2 $file-3)"
+ expect_expr "$device" '(relative-paths)' "($tfile $tfile-2 $tfile-3)"
+}
+run_test 109 "lipe_scan3 paths thunks do the right thing"
+
+test_110() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local params
+ local fd
+ local a1
+ local a2
+
+ params=$($LCTL get_param 'llite.*.xattr_cache')
+ $LCTL set_param 'llite.*.xattr_cache=0'
+ stack_trap "$LCTL set_param $params"
+
+ init_lipe_scan3_env
+ echo XXX > "$file"
+ sync
+
+ # Try to get the SoM synced.
+ exec {fd}<"$file"
+ $LFS data_version -r "$file"
+ stat "$file"
+ exec {fd}<&-
+
+ sync
+ $LFS getsom "$file"
+ lipe_scan3 "$device" --print-json=som
+
+ # file: /mnt/lustre/f110.sanity-lipe-scan3 size: 4 blocks: 8 flags: 4
+ a1=$($LFS getsom "$file" | awk '{ print $4 }')
+ a2=$(lipe_scan3 "$device" --print-json=som | jq .som.size)
+ ((a1 == a2)) || error "som.size expected '$a1', got '$a2'"
+
+ a1=$($LFS getsom "$file" | awk '{ print $6 }')
+ a2=$(lipe_scan3 "$device" --print-json=som | jq .som.blocks)
+ ((a1 == a2)) || error "som.blocks expected '$a1', got '$a2'"
+
+ a1=$($LFS getsom "$file" | awk '{ print $8 }')
+ a2=$(lipe_scan3 "$device" --print-json=som | jq .som._flags)
+ ((a1 == a2)) || error "som.flags expected '$a1', got '$a2'"
+
+ # .som.flags should be ["lazy"] here
+}
+run_test 110 "lipe_scan3 som attr works"
+
+test_120() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fd
+
+ init_lipe_scan3_env_file "$file"
+ echo XXX > "$file"
+
+ expect_expr "$device" '(car (assoc "trusted.lov" (xattrs)))' trusted.lov
+ expect_print lipe_scan3_format "$device" '(xattr-ref "trusted.lov")'
+
+ setfattr -n user.NAME -v VALUE "$file"
+ expect_expr "$device" '(car (assoc "user.NAME" (xattrs)))' user.NAME
+ expect_print lipe_scan3_format "$device" '(xattr-ref "user.NAME")'
+
+ # expect_expr "$device" '(xattr-ref-string "user.NAME")' VALUE
+}
+run_test 120 "lipe_scan3 xattrs does the right thing"
+
+test_130() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fd
+
+ init_lipe_scan3_env_file "$file"
+ echo XXX > "$file"
+
+ expect_expr "$device" '(pools)' '()'
+}
+run_test 130 "lipe_scan3 pools does the right thing"
+
+test_200() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local proc
+
+ init_lipe_scan3_env_file "$file"
+
+ lipe_scan3_format "$device" '(lipe-scan-device-name)'
+ expect_attr "$device" lipe-scan-fsname "$FSNAME"
+ expect_attr "$device" lipe-scan-client-mount-path "$MOUNT"
+ expect_attr "$device" lipe-scan-device-name "$FSNAME-MDT0000"
+ expect_attr "$device" lipe-scan-device-path "$device"
+ expect_attr "$device" lipe-scan-thread-count 1 --thread-count=1
+ expect_attr "$device" lipe-scan-thread-count 2 --thread-count=2
+ expect_attr "$device" lipe-scan-thread-index 0 --thread-count=1
+
+ for proc in \
+ lipe-debug-enable \
+ lipe-gettid \
+ lipe-scan-client-mount-fd \
+ lipe-scan-current-attrs \
+ lipe-scan-thread-index; do
+ expect_print lipe_scan3_format "$device" "($proc)"
+ done
+}
+run_test 200 "lipe-scan-* procedures do the right thing"
+
+test_201() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local rc
+
+ init_lipe_scan3_env_file "$file"
+
+ lipe_scan3_body "$device" '(lipe-scan-break 7)'
+ rc=$?
+ ((rc == 7)) || error "(lipe-scan-break 7) should return 7"
+}
+run_test 201 "lipe-scan-break works"
+
+# lipe-scan-continue
+
+test_300() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fid
+ local out
+
+ init_lipe_scan3_env_file "$file"
+ fid=$($LFS path2fid "$file")
+
+lipe_scan3 "$device" --print-file-fid
+
+ out=$(lipe_scan3 "$device" --print-file-fid)
+ [[ "$out" == "$fid" ]] || error "--print-file-fid should print '$fid'"
+
+ out=$(lipe_scan3 "$device" --print-self-fid)
+ [[ "$out" == "$fid" ]] || error "--print-self-fid should print '$fid'"
+}
+run_test 300 "--print-*-fid options work"
+
+test_301() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local fid
+ local fd
+
+ init_lipe_scan3_env
+ echo XXX > "$file"
+ sync
+
+ # Try to get the SoM synced.
+ exec {fd}<"$file"
+ $LFS data_version -r "$file"
+ stat "$file"
+ exec {fd}<&-
+
+ fid=$($LFS path2fid "$file")
+
+ lipe_scan3 "$device" --print-json | jq . || error "--print-json should print JSON"
+ lipe_scan3 "$device" --print-json=#x0 | jq . || error "--print-json should print JSON"
+ lipe_scan3 "$device" --print-json=#xffffffff | jq . || error "--print-json should print JSON"
+ lipe_scan3 "$device" --print-json=all | jq . || error "--print-json should print JSON"
+ lipe_scan3 "$device" --print-json=ino | jq . || error "--print-json should print JSON"
+}
+run_test 301 "--print-json options work"
+
+test_302() {
+ local facet=mds1
+ local device="$(facet_device $facet)"
+ local file=$MOUNT/$tfile
+ local out
+
+ init_lipe_scan3_env_file "$file"
+
+ out=$(lipe_scan3 "$device" --print-absolute-path)
+ [[ "$out" == "$file" ]] || error "--print-absolute-path should print absolute path"
+
+ out=$(lipe_scan3 "$device" --print-relative-path)
+ [[ "$out" == "$tfile" ]] || error "--print-relative-path should print relative path"
+
+ # TODO --all-paths
+ # TODO --null
+ # TODO --delim
+}
+run_test 302 "--print-*-path options work"
+
+# loading and scripts
+
+test_400() {
+ expect_empty lipe_scan3 --script=/dev/null
+ expect_empty lipe_scan3 -s /dev/null
+ expect_empty lipe_scan3 --load=/dev/null --script=/dev/null
+ expect_empty lipe_scan3 -l /dev/null -s /dev/null
+}
+run_test 400 "empty scripts and loaded files"
+
+test_401() {
+ local rc
+
+ echo '#t' | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 0)) || error "rc expect 0, got '$rc'"
+
+ echo '#f' | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 1)) || error "rc expect 1, got '$rc'"
+
+ echo '0' | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 0)) || error "rc expect 0, got '$rc'"
+
+ echo '7' | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 7)) || error "rc expect 7, got '$rc'"
+
+ echo 'cons' | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 0)) || error "rc expect 0, got '$rc'"
+
+ echo "'()" | lipe_scan3 --script=/dev/stdin
+ rc=$?
+ ((rc == 0)) || error "rc expect 0, got '$rc'"
+}
+run_test 401 "lipe_scan3 script exit statuses work"
+
+# TODO Run on OST.
+# --print-file-fid works on OST
+# --print-self-fid works on OST
+# --print-absolute-paths works on OST
+# --print-json on OST
+# Automagic required attrs work on OST
+
+complete $SECONDS
+check_and_cleanup_lustre
+exit_status
git://git.whamcloud.com / fs / lustre-release.git / commitdiff