EX-3054 lipe: unlink pcc cache file if it's needed

Cache file is not guaranteed to be deleted after detach
operation returns success. So we unlink the cache file
if it's needed.

Change --force_clear option to --clear_hashdir option.
It tries to remove empty hash dir recursively up to the cache
dir root. The option is off by default.

Change-Id: I49911c688faaf6c7baa814b260a4ff492de077fc
Signed-off-by: Lei Feng <flei@whamcloud.com>
Test-Parameters: trivial
Reviewed-on: https://review.whamcloud.com/43403
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: John L. Hammond <jhammond@whamcloud.com>
Reviewed-by: Andreas Dilger <adilger@whamcloud.com>

diff --git a/lipe/src/lpcc_purge.c b/lipe/src/lpcc_purge.c
index e3946fe..42d9f68 100644 (file)
--- a/lipe/src/lpcc_purge.c
+++ b/lipe/src/lpcc_purge.c
@@ -13,6 +13,7 @@
 #include <stdlib.h>
 #include <signal.h>
 #include <sys/file.h>
+#include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/vfs.h>
@@ -33,7 +34,7 @@
 
 #define OPT_DRY_RUN                    1
 #define OPT_CANDIDATE_NUM              2
-#define OPT_FORCE_CLEAR                        3
+#define OPT_CLEAR_HASHDIR              3
 #define OPT_LOG_LEVEL                  4
 
 struct lpcc_purge_options {
@@ -50,7 +51,7 @@ struct lpcc_purge_options {
        char *o_dumpfile;
 
        bool o_dry_run;
-       bool o_force_clear;
+       bool o_clear_hashdir;
 };
 static struct lpcc_purge_options opt = {
        .o_rwid                 = -1,
@@ -60,7 +61,7 @@ static struct lpcc_purge_options opt = {
        .o_scan_threads         = DEF_SCAN_THREADS,
        .o_candidate_num        = DEF_CANDIDATE_NUM,
        .o_dry_run              = false,
-       .o_force_clear          = false,
+       .o_clear_hashdir        = false,
 };
 
 bool exit_flag = false;
@@ -219,7 +220,7 @@ static void lpcc_purge_usr1_handler(int sig)
        json_object_object_add(j_config, "interval", json_object_new_int64(opt.o_interval));
        json_object_object_add(j_config, "scan_threads", json_object_new_int64(opt.o_scan_threads));
        json_object_object_add(j_config, "candidate_num", json_object_new_int64(opt.o_candidate_num));
-       json_object_object_add(j_config, "force_clear", json_object_new_boolean(opt.o_force_clear));
+       json_object_object_add(j_config, "clear_hashdir", json_object_new_boolean(opt.o_clear_hashdir));
        json_object_object_add(j_all, "config", j_config);
 
        json_object *j_stats = json_object_new_object();
@@ -266,7 +267,8 @@ static void usage(void)
                "\t-t, --scan-threads=NUM scanning threads (default: %u)\n"
                "\t    --candidate-num=NUM, candidate number of approximate LRU (default: %d)\n"
                "\t-w, --dump=FILE, dump stats to FILE when signal USR1 is recieved (default: /var/run/lpcc_purge-PID.stats)\n"
-               "\t    --dry-run, scan once but don't detach file really\n"
+               "\t    --clear-hashdir, clear empty hash dir after detaching file\n"
+               "\t    --dry-run, scan once but do not detach file really\n"
                "\t-h, --help, print this help message\n",
 
                program_invocation_short_name,
@@ -292,7 +294,7 @@ static struct option long_options[] = {
        { "dry-run", no_argument, NULL, OPT_DRY_RUN},
        { "candidate-num", required_argument, NULL, OPT_CANDIDATE_NUM},
        { "dump", required_argument, NULL, 'w'},
-       { "force-clear", no_argument, NULL, OPT_FORCE_CLEAR},
+       { "clear-hashdir", no_argument, NULL, OPT_CLEAR_HASHDIR},
        { "help", no_argument, NULL, 'h' },
        { NULL }
 };
@@ -545,8 +547,8 @@ static void lpcc_purge_process_opt(int c, char *optarg)
                }
                opt.o_candidate_num = value;
                break;
-       case OPT_FORCE_CLEAR:
-               opt.o_force_clear = true;
+       case OPT_CLEAR_HASHDIR:
+               opt.o_clear_hashdir = true;
                break;
        default:
                llapi_error(LLAPI_MSG_FATAL, -EINVAL,
@@ -715,19 +717,11 @@ static void lpcc_purge_wait_for_scan(void)
        }
 }
 
-/**
- * This is a temporary function to remove the cache file and empty hash dir
- * recursively up to the cache dir.
- *
- * The function is only for testing, it's not safe in production enviroment.
- */
-static int remove_hash_dir(const char *path)
+static int clear_hash_dir(const char *path)
 {
        int rc;
        char dir[PATH_MAX], tmp[PATH_MAX];
 
-       unlink(path);
-
        strcpy(tmp, path);
        strcpy(dir, dirname(tmp));
 
@@ -739,9 +733,8 @@ static int remove_hash_dir(const char *path)
                        }
                        else {
                                rc = -errno;
-                               llapi_printf(LLAPI_MSG_WARN,
-                                       "cannot remove hash dir: %s. errno = %d\n",
-                                       dir, -rc);
+                               llapi_error(LLAPI_MSG_WARN, -rc,
+                                       "cannot remove hash dir: %s", dir);
                        }
                        break;
                }
@@ -754,6 +747,58 @@ static int remove_hash_dir(const char *path)
        return rc;
 }
 
+static int lpcc_purge_detach_by_fid(const char *mntpath,
+       const struct lu_fid *fid, const char *cache_file)
+{
+       int rc;
+       int fd;
+       struct lu_pcc_detach_fid detach;
+
+       fd = open(mntpath, O_DIRECTORY | O_RDONLY);
+       if (fd < 0) {
+               rc = -errno;
+               llapi_error(LLAPI_MSG_ERROR, rc, "cannot open root path: %s",
+                           mntpath);
+               return rc;
+       }
+
+       detach.pccd_fid = *fid;
+       detach.pccd_flags = PCC_DETACH_FL_UNCACHE;
+       rc = ioctl(fd, LL_IOC_PCC_DETACH_BY_FID, &detach);
+       close(fd);
+
+       if (rc) {
+               /* skip */
+               llapi_error(LLAPI_MSG_DEBUG, rc,
+                            "cannot detach fid: "DFID"", PFID(fid));
+               return rc;
+       }
+
+       if (detach.pccd_flags & PCC_DETACH_FL_CACHE_REMOVED) {
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "Detach and remove the PCC cached fid: "DFID"\n",
+                            PFID(fid));
+       }
+       else if (detach.pccd_flags & PCC_DETACH_FL_ATTACHING) {
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "fid "DFID" is being attached, skip it", PFID(fid));
+       }
+       else {
+               llapi_printf(LLAPI_MSG_DEBUG,
+                            "Remove non-cached file: %s flags: %X\n",
+                            cache_file, detach.pccd_flags);
+               rc = unlink(cache_file);
+               if (rc < 0 && errno != ENOENT) {
+                       rc = -errno;
+                       llapi_error(LLAPI_MSG_ERROR, rc,
+                                   "Unlink %s failed", cache_file);
+               }
+
+       }
+
+       return rc;
+}
+
 static int lpcc_purge_detach_candidate(const char *mnt,
        const struct lpcc_purge_candidate *candidate)
 {
@@ -780,21 +825,26 @@ static int lpcc_purge_detach_candidate(const char *mnt,
                /* skip entirely */
                return 0;
 
-       if (opt.o_dry_run) {
-               return -1;
-       }
+       pthread_mutex_lock(&stats.s_lock);
+       stats.s_purged_objs ++;
+       stats.s_total_purged_objs ++;
+       pthread_mutex_unlock(&stats.s_lock);
+
+       if (opt.o_dry_run)
+               return rc;
 
-       rc = llapi_pcc_detach_fid(mnt, &candidate->c_fid, PCC_DETACH_FL_UNCACHE);
+       rc = lpcc_purge_detach_by_fid(mnt, &candidate->c_fid, candidate->c_path);
        if (rc) {
                llapi_error(LLAPI_MSG_WARN, -rc,  "cannot detach fid: "DFID,
                        PFID(&candidate->c_fid));
                pthread_mutex_lock(&stats.s_lock);
                stats.s_total_failed_objs++;
                pthread_mutex_unlock(&stats.s_lock);
+               return rc;
        }
 
-       if (opt.o_force_clear)
-               remove_hash_dir(candidate->c_path);
+       if (opt.o_clear_hashdir)
+               rc = clear_hash_dir(candidate->c_path);
 
        return 0;
 }