+#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <getopt.h>
+#include <linux/lustre/lustre_idl.h>
+#include <lustre/lustreapi.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <sys/types.h>
-#include <lustre/lustreapi.h>
+#include <sys/vfs.h>
#include <unistd.h>
+#include "debug.h"
+
-#define LPCC_PURGE_DEFAULT_CONFIG_FILE "/etc/lpcc_purge.conf"
-#define LPCC_PURGE_PIDFILE "/var/run/lpcc_purge.pid"
+#define DEF_HIGH_USAGE 90
+#define DEF_LOW_USAGE 75
+#define DEF_INTERVAL 5
+#define DEF_SCAN_THREADS 1
+
+#define OPT_LOG_LEVEL 4
struct lpcc_purge_options {
- char *o_cfg_file;
- bool o_exit;
+ char *o_cache;
+ char *o_mount;
+ unsigned int o_rwid;
+
+ double o_high_usage;
+ double o_low_usage;
+
+ int o_interval;
+ int o_scan_threads;
};
static struct lpcc_purge_options opt = {
- .o_cfg_file = LPCC_PURGE_DEFAULT_CONFIG_FILE,
- .o_exit = false,
+ .o_rwid = -1,
+ .o_high_usage = DEF_HIGH_USAGE,
+ .o_low_usage = DEF_LOW_USAGE,
+ .o_interval = DEF_INTERVAL,
+ .o_scan_threads = DEF_SCAN_THREADS,
+};
+
+bool exit_flag = false;
+struct lpcc_purge_stats {
+ double s_start_usage;
+};
+static struct lpcc_purge_stats stats = {
+ .s_start_usage = -1,
};
static void lpcc_purge_null_handler(int signal)
{
psignal(signal, "exiting");
- opt.o_exit = true;
+ exit_flag = true;
}
static void usage(void)
{
printf("Usage: %s [options]\n"
- "\t-f, --config-file FILE\n"
+ "\t-b, --debug, enable debugging output\n"
+ "\t --log_level={debug|info|normal|warn|error|fatal|off}, set log level (default: info)\n"
+ "\t-f, --config-file=FILE\n"
+ "\t-C, --cache=DIR root directory of PCC\n"
+ "\t-M, --mount=DIR local Lustre client's mountpoint\n"
+ "\t-A, --rwid=NUM the rwid of PCC\n"
+ "\t-H, --high_usage=NUM %% of space or inode to start purging (default: %u)\n"
+ "\t-L, --low_usage=NUM %% of space or inode to stop purging (default: %u)\n"
+ "\t-i, --interval=NUM, seconds to next check (default: %u)\n"
+ "\t-t, --scan_threads=NUM scanning threads (default: %u)\n"
"\t-h, --help, print this help message\n",
- program_invocation_short_name
- );
- exit(0);
+ program_invocation_short_name,
+ DEF_HIGH_USAGE,
+ DEF_LOW_USAGE,
+ DEF_INTERVAL,
+ DEF_SCAN_THREADS
+ );
}
static struct option long_options[] = {
+ { "debug", no_argument, NULL, 'b'},
{ "config-file", required_argument, NULL, 'f'},
+ { "log_level", required_argument, NULL, OPT_LOG_LEVEL},
+ { "cache", required_argument, NULL, 'C'},
+ { "mount", required_argument, NULL, 'M'},
+ { "rwid", required_argument, NULL, 'A'},
+ { "high_usage", required_argument, NULL, 'H'},
+ { "low_usage", required_argument, NULL, 'L'},
+ { "interval", required_argument, NULL, 'i'},
+ { "scan_threads", required_argument, NULL, 't'},
{ "help", no_argument, NULL, 'h' },
{ NULL }
};
+static struct option *lpcc_purge_keyword_lookup(const char *keyword)
+{
+ int i = 0;
+
+ while (long_options[i].name) {
+ if (strcmp(keyword, long_options[i].name) == 0)
+ return long_options + i;
+ i++;
+ }
+ return NULL;
+}
+
+static void lpcc_purge_process_opt(int c, char *optarg);
+
+static void load_config(const char *path)
+{
+ size_t len = 0;
+ char *buf = NULL;
+
+ FILE *f = fopen(path, "r");
+ if (!f) {
+ llapi_error(LLAPI_MSG_FATAL, errno,
+ "cannot open config file '%s'", path);
+ exit(1);
+ }
+
+ while (!feof(f)) {
+ struct option *opt;
+ char *s, *t;
+ len = PATH_MAX;
+
+ if (getline(&buf, &len, f) <= 0)
+ break;
+ s = buf;
+ while (isspace(*s))
+ s++;
+ t = strchr(s, '#');
+ if (t)
+ *t = '\0';
+ if (*s == '#')
+ continue;
+ t = strsep(&s, "=\n");
+ if (!t || *t == 0)
+ continue;
+ opt = lpcc_purge_keyword_lookup(t);
+ if (!opt) {
+ llapi_error(LLAPI_MSG_ERROR, EINVAL,
+ "unknown tunable '%s'", t);
+ continue;
+ }
+ if (opt->val == 'f') {
+ /* you shall not pass! */
+ continue;
+ }
+
+ if (opt->has_arg == required_argument ||
+ opt->has_arg == optional_argument) {
+ optarg = strsep(&s, "\n ");
+ if (!optarg &&
+ opt->has_arg == required_argument) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "no argument for '%s'", t);
+ exit(1);
+ }
+ }
+ else {
+ optarg = NULL;
+ }
+ llapi_printf(LLAPI_MSG_DEBUG, "conf: %s %s\n", t,
+ optarg ? optarg : "");
+ lpcc_purge_process_opt(opt->val, optarg);
+ }
+
+ free(buf);
+ fclose(f);
+}
+
+static const char *log_level_strs[] = {
+ [LLAPI_MSG_OFF] = "off",
+ [LLAPI_MSG_FATAL] = "fatal",
+ [LLAPI_MSG_ERROR] = "error",
+ [LLAPI_MSG_WARN] = "warn",
+ [LLAPI_MSG_NORMAL] = "normal",
+ [LLAPI_MSG_INFO] = "info",
+ [LLAPI_MSG_DEBUG] = "debug",
+};
+
+static enum llapi_message_level parse_log_level(const char *level_str)
+{
+ enum llapi_message_level level = LLAPI_MSG_INFO;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(log_level_strs); i++) {
+ if (strcasecmp(log_level_strs[i], level_str) == 0) {
+ level = i;
+ break;
+ }
+ }
+
+ if (i == sizeof(log_level_strs) / sizeof(char *)) {
+ llapi_error(LLAPI_MSG_WARN, -EINVAL,
+ "Unrecognized log level string '%s'",
+ optarg);
+ }
+
+ return level;
+}
+
+static void parse_mountpoint(const char *name)
+{
+ int rc;
+ struct statfs statbuf;
+
+ rc = statfs(name, &statbuf);
+ if (rc != 0) {
+ llapi_error(LLAPI_MSG_FATAL, -rc,
+ "cannot statfs '%s'", name);
+ exit(1);
+ }
+ if (statbuf.f_type != LL_SUPER_MAGIC) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "'%s' is not a lustre mount point", name);
+ exit(1);
+ }
+}
+
+static void parse_cache_dir(const char *name)
+{
+ struct stat stat1;
+ int rc;
+
+ rc = stat(name, &stat1);
+ if (rc) {
+ llapi_error(LLAPI_MSG_FATAL, errno,
+ "cannot stat cache dir '%s'", name);
+ exit(1);
+ }
+
+ if (!S_ISDIR(stat1.st_mode)) {
+ llapi_error(LLAPI_MSG_FATAL, errno,
+ "cache path '%s' is not a directory", name);
+ exit(1);
+ }
+}
+
static void lpcc_purge_process_opt(int c, char *optarg)
{
+ long value;
+ char *endptr = NULL;
+ enum llapi_message_level log_level;
+
switch(c) {
case 'f':
- opt.o_cfg_file = optarg;
+ load_config(optarg);
break;
- }
+ case 'b':
+ llapi_msg_set_level(LLAPI_MSG_MAX);
+ break;
+ case OPT_LOG_LEVEL:
+ log_level = parse_log_level(optarg);
+ llapi_printf(LLAPI_MSG_INFO, "Set log level: %s\n",
+ log_level_strs[log_level]);
+ llapi_msg_set_level(log_level);
- return;
+ break;
+ case 'C':
+ parse_cache_dir(optarg);
+ opt.o_cache = strdup(optarg);
+ break;
+ case 'M':
+ parse_mountpoint(optarg);
+ opt.o_mount = strdup(optarg);
+ break;
+ case 'A':
+ value = strtol(optarg, &endptr, 10);
+ if (*endptr != '\0' || value <= 0) {
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid rwid: '%s'",
+ optarg);
+ exit(1);
+ }
+ opt.o_rwid = value;
+ break;
+ case 'H':
+ value = strtol(optarg, &endptr, 10);
+ if (*endptr != '\0' || value < 1 || value > 99) {
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid high watermark: '%s'",
+ optarg);
+ exit(1);
+ }
+ opt.o_high_usage = value;
+ break;
+ case 'L':
+ value = strtol(optarg, &endptr, 10);
+ if (*endptr != '\0' || value < 1 || value > 99) {
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid low watermark: '%s'",
+ optarg);
+ exit(1);
+ }
+ opt.o_low_usage = value;
+ break;
+ case 'i':
+ value = strtol(optarg, &endptr, 10);
+ if (*endptr != '\0' || value <= 0) {
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid check interval: '%s'",
+ optarg);
+ exit(1);
+ }
+ opt.o_interval = value;
+ break;
+ case 't':
+ value = strtol(optarg, &endptr, 10);
+ if (*endptr != '\0' || value <= 0) {
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid scan threads count: '%s'",
+ optarg);
+ exit(1);
+ }
+ opt.o_scan_threads = value;
+ break;
+ default:
+ llapi_error(LLAPI_MSG_FATAL, -EINVAL,
+ "invalid argument: '%s'",
+ optarg);
+ exit(1);
+ break;
+ }
}
static void lpcc_purge_parse_opts(int argc, char **argv)
int c;
while ((c = getopt_long(argc, argv,
- "hf:",
+ "bf:C:M:A:H:L:i:t:h",
long_options, NULL))
!= EOF) {
switch(c) {
}
}
-static int load_config()
+void lpcc_purge_verify_opts(void)
{
- return 0;
+ if (opt.o_mount == NULL) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "lustre mount point must be specified");
+ exit(1);
+ }
+ if (opt.o_cache == NULL) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "pcc cache dir must be specified");
+ exit(1);
+ }
+ if (opt.o_rwid == -1) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "rwid mount point must be specified");
+ exit(1);
+ }
+ // check freehi > freelo
+ if (opt.o_high_usage <= opt.o_low_usage) {
+ llapi_error(LLAPI_MSG_FATAL, EINVAL,
+ "high usage (%.1f) must be larger than low usage (%.1f)",
+ opt.o_high_usage, opt.o_low_usage);
+ exit(1);
+ }
}
-static void lpcc_purge_lock_pidfile(void)
+/* It seems that lustre/utils/pid_file.c does not exist in RHEL7.x,
+ * so copy the function here
+ */
+static int create_pid_file(const char *path)
{
- char buf[1024];
- int fd, rc, sz;
+ char buf[3 * sizeof(long long) + 2];
+ size_t buf_len;
+ int fd = -1;
+ int rc2;
- snprintf(buf, sizeof(buf), LPCC_PURGE_PIDFILE);
- fd = open(buf, O_RDWR | O_CREAT, 0600);
+ fd = open(path, O_RDWR|O_CREAT|O_CLOEXEC, 0600);
if (fd < 0) {
- llapi_error(LLAPI_MSG_FATAL, errno,
- "cannot create pidfile");
- exit(1);
+ fprintf(stderr, "%s: cannot open '%s': %s\n",
+ program_invocation_short_name, path, strerror(errno));
+ return -1;
}
- rc = flock(fd, LOCK_EX | LOCK_NB);
- if (rc < 0) {
- sz = read(fd, buf, sizeof(buf));
- /* protect against garbage */
- if (sz > 10)
- sz = 0;
- if (sz > 0)
- buf[sz] = 0;
- llapi_error(LLAPI_MSG_FATAL, EBUSY,
- "another lpcc_purge is running, locked by %s",
- sz > 0 ? buf : "[unknown]");
- exit(1);
+
+ struct flock fl = {
+ .l_type = F_WRLCK,
+ .l_whence = SEEK_SET,
+ };
+
+ rc2 = fcntl(fd, F_SETLK, &fl);
+ if (rc2 < 0) {
+ fprintf(stderr, "%s: cannot lock '%s': %s\n",
+ program_invocation_short_name, path, strerror(errno));
+ goto out;
}
- rc = ftruncate(fd, 0);
- if (rc < 0) {
- llapi_error(LLAPI_MSG_FATAL, errno,
- "cannot truncate pidfile");
- exit(1);
+ rc2 = ftruncate(fd, 0);
+ if (rc2 < 0) {
+ fprintf(stderr, "%s: cannot truncate '%s': %s\n",
+ program_invocation_short_name, path, strerror(errno));
+ goto out;
}
- sz = snprintf(buf, sizeof(buf), "%d\n", getpid());
- rc = write(fd, buf, sz);
- if (rc < 0 || rc != sz) {
+ buf_len = snprintf(buf, sizeof(buf), "%lld\n", (long long)getpid());
+ rc2 = write(fd, buf, buf_len);
+ if (rc2 < 0) {
+ fprintf(stderr, "%s: cannot write '%s': %s\n",
+ program_invocation_short_name, path, strerror(errno));
+ goto out;
+ }
+out:
+ if (rc2 < 0 && !(fd < 0)) {
+ close(fd);
+ fd = -1;
+ }
+
+ return fd;
+}
+
+static void lpcc_purge_lock_pidfile(void)
+{
+ char buf[PATH_MAX];
+ int fd;
+
+ snprintf(buf, sizeof(buf), "/var/run/lpcc_purge-%d.pid", opt.o_rwid);
+ fd = create_pid_file(buf);
+ if (fd < 0) {
llapi_error(LLAPI_MSG_FATAL, errno,
- "cannot write pidfile");
+ "cannot create pidfile '%s'", buf);
exit(1);
}
-
+ /* we keep the fd open to hold the flock,
+ it will be closed automatically when the process exits */
}
-static void lpcc_purge_unlock_pidfile(void)
+static double lpcc_purge_get_fs_usage(const char *fs)
{
- char buf[1024];
int rc;
+ struct statfs statfs_buf;
- snprintf(buf, sizeof(buf), LPCC_PURGE_PIDFILE);
+ rc = statfs(opt.o_cache, &statfs_buf);
+ if (rc) {
+ llapi_error(LLAPI_MSG_FATAL, errno, "cannot statfs '%s'", fs);
+ exit(1);
+ }
- rc = unlink(buf);
- if (rc < 0) {
- llapi_error(LLAPI_MSG_ERROR, errno,
- "cannot unlink pidfile");
+ return 100.0 * (statfs_buf.f_blocks - statfs_buf.f_bfree) / statfs_buf.f_blocks;
+}
+
+static void lpcc_purge_wait_for_scan(void)
+{
+ while (!exit_flag) {
+ double usage = lpcc_purge_get_fs_usage(opt.o_cache);
+ if (usage >= opt.o_high_usage) {
+ stats.s_start_usage = usage;
+ break;
+ } else {
+ sleep(opt.o_interval);
+ }
}
+}
- return;
+static void lpcc_purge_scan(void)
+{
+}
+
+static void lpcc_purge_free_space(void)
+{
}
int main(int argc, char *argv[])
{
- int rc = -1;
+ int rc = 0;
llapi_msg_set_level(LLAPI_MSG_INFO);
signal(SIGUSR1, &lpcc_purge_null_handler);
lpcc_purge_parse_opts(argc, argv);
- rc = load_config();
- if (rc) {
- goto out;
- }
+ lpcc_purge_verify_opts();
lpcc_purge_lock_pidfile();
- while(!opt.o_exit) {
- sleep(10);
+ while(!exit_flag) {
+
+ lpcc_purge_wait_for_scan();
+
+ lpcc_purge_scan();
+
+ lpcc_purge_free_space();
}
-out:
- lpcc_purge_unlock_pidfile();
return rc == 0 ? 0 : 255;
}
git://git.whamcloud.com / fs / lustre-release.git / commitdiff