Whamcloud - gitweb
LU-9679 modules: Use LIST_HEAD for declaring list_heads
[fs/lustre-release.git] / lustre / osc / osc_cache.c
index 5394a75..891d560 100644 (file)
@@ -57,10 +57,10 @@ static int osc_io_unplug_async(const struct lu_env *env,
 static void osc_free_grant(struct client_obd *cli, unsigned int nr_pages,
                           unsigned int lost_grant, unsigned int dirty_grant);
 
-static void osc_extent_tree_dump0(int level, struct osc_object *obj,
+static void osc_extent_tree_dump0(int mask, struct osc_object *obj,
                                  const char *func, int line);
-#define osc_extent_tree_dump(lvl, obj) \
-       osc_extent_tree_dump0(lvl, obj, __func__, __LINE__)
+#define osc_extent_tree_dump(mask, obj) \
+       osc_extent_tree_dump0(mask, obj, __func__, __LINE__)
 
 static void osc_unreserve_grant(struct client_obd *cli, unsigned int reserved,
                                unsigned int unused);
@@ -104,18 +104,19 @@ static inline char list_empty_marker(struct list_head *list)
 static const char *oes_strings[] = {
        "inv", "active", "cache", "locking", "lockdone", "rpc", "trunc", NULL };
 
-#define OSC_EXTENT_DUMP(lvl, extent, fmt, ...) do {                          \
+#define OSC_EXTENT_DUMP_WITH_LOC(file, func, line, mask, extent, fmt, ...) do {\
+       static struct cfs_debug_limit_state cdls;                             \
        struct osc_extent *__ext = (extent);                                  \
        char __buf[16];                                                       \
                                                                              \
-       CDEBUG(lvl,                                                           \
+       __CDEBUG_WITH_LOC(file, func, line, mask, &cdls,                      \
                "extent %p@{" EXTSTR ", "                                     \
                "[%d|%d|%c|%s|%s|%p], [%d|%d|%c|%c|%p|%u|%p]} " fmt,          \
                /* ----- extent part 0 ----- */                               \
                __ext, EXTPARA(__ext),                                        \
                /* ----- part 1 ----- */                                      \
-               atomic_read(&__ext->oe_refc),                         \
-               atomic_read(&__ext->oe_users),                        \
+               atomic_read(&__ext->oe_refc),                                 \
+               atomic_read(&__ext->oe_users),                                \
                list_empty_marker(&__ext->oe_link),                           \
                oes_strings[__ext->oe_state], ext_flags(__ext, __buf),        \
                __ext->oe_obj,                                                \
@@ -126,12 +127,16 @@ static const char *oes_strings[] = {
                __ext->oe_dlmlock, __ext->oe_mppr, __ext->oe_owner,           \
                /* ----- part 4 ----- */                                      \
                ## __VA_ARGS__);                                              \
-       if (lvl == D_ERROR && __ext->oe_dlmlock != NULL)                      \
+       if (mask == D_ERROR && __ext->oe_dlmlock != NULL)                     \
                LDLM_ERROR(__ext->oe_dlmlock, "extent: %p", __ext);           \
        else                                                                  \
                LDLM_DEBUG(__ext->oe_dlmlock, "extent: %p", __ext);           \
 } while (0)
 
+#define OSC_EXTENT_DUMP(mask, ext, fmt, ...)                           \
+       OSC_EXTENT_DUMP_WITH_LOC(__FILE__, __func__, __LINE__,          \
+                                mask, ext, fmt, ## __VA_ARGS__)
+
 #undef EASSERTF
 #define EASSERTF(expr, ext, fmt, args...) do {                         \
        if (!(expr)) {                                                  \
@@ -215,6 +220,7 @@ static int osc_extent_sanity_check0(struct osc_extent *ext,
                        GOTO(out, rc = 60);
                if (ext->oe_fsync_wait && !ext->oe_urgent && !ext->oe_hp)
                        GOTO(out, rc = 65);
+               /* fallthrough */
        default:
                if (atomic_read(&ext->oe_users) > 0)
                        GOTO(out, rc = 70);
@@ -263,9 +269,9 @@ static int osc_extent_sanity_check0(struct osc_extent *ext,
 
 out:
        if (rc != 0)
-               OSC_EXTENT_DUMP(D_ERROR, ext,
-                               "%s:%d sanity check %p failed with rc = %d\n",
-                               func, line, ext, rc);
+               OSC_EXTENT_DUMP_WITH_LOC(__FILE__, func, line, D_ERROR, ext,
+                                        "sanity check %p failed: rc = %d\n",
+                                        ext, rc);
        return rc;
 }
 
@@ -579,6 +585,7 @@ int osc_extent_release(const struct lu_env *env, struct osc_extent *ext)
                         * osc_cache_truncate_start(). */
                        osc_extent_state_set(ext, OES_TRUNC);
                        ext->oe_trunc_pending = 0;
+                       osc_object_unlock(obj);
                } else {
                        int grant = 0;
 
@@ -591,8 +598,6 @@ int osc_extent_release(const struct lu_env *env, struct osc_extent *ext)
                                grant += cli->cl_grant_extent_tax;
                        if (osc_extent_merge(env, ext, next_extent(ext)) == 0)
                                grant += cli->cl_grant_extent_tax;
-                       if (grant > 0)
-                               osc_unreserve_grant(cli, 0, grant);
 
                        if (ext->oe_urgent)
                                list_move_tail(&ext->oe_link,
@@ -601,8 +606,10 @@ int osc_extent_release(const struct lu_env *env, struct osc_extent *ext)
                                list_move_tail(&ext->oe_link,
                                               &obj->oo_full_exts);
                        }
+                       osc_object_unlock(obj);
+                       if (grant > 0)
+                               osc_unreserve_grant(cli, 0, grant);
                }
-               osc_object_unlock(obj);
 
                osc_io_unplug_async(env, cli, obj);
        }
@@ -954,9 +961,8 @@ static int osc_extent_wait(const struct lu_env *env, struct osc_extent *ext,
                        "%s: wait ext to %u timedout, recovery in progress?\n",
                        cli_name(osc_cli(obj)), state);
 
-               lwi = LWI_INTR(NULL, NULL);
-               rc = l_wait_event(ext->oe_waitq, extent_wait_cb(ext, state),
-                                 &lwi);
+               wait_event_idle(ext->oe_waitq, extent_wait_cb(ext, state));
+               rc = 0;
        }
        if (rc == 0 && ext->oe_rc < 0)
                rc = ext->oe_rc;
@@ -1228,34 +1234,34 @@ out:
        RETURN(rc);
 }
 
-static void osc_extent_tree_dump0(int level, struct osc_object *obj,
+static void osc_extent_tree_dump0(int mask, struct osc_object *obj,
                                  const char *func, int line)
 {
        struct osc_extent *ext;
        int cnt;
 
-       if (!cfs_cdebug_show(level, DEBUG_SUBSYSTEM))
+       if (!cfs_cdebug_show(mask, DEBUG_SUBSYSTEM))
                return;
 
-       CDEBUG(level, "Dump object %p extents at %s:%d, mppr: %u.\n",
+       CDEBUG(mask, "Dump object %p extents at %s:%d, mppr: %u.\n",
               obj, func, line, osc_cli(obj)->cl_max_pages_per_rpc);
 
        /* osc_object_lock(obj); */
        cnt = 1;
        for (ext = first_extent(obj); ext != NULL; ext = next_extent(ext))
-               OSC_EXTENT_DUMP(level, ext, "in tree %d.\n", cnt++);
+               OSC_EXTENT_DUMP(mask, ext, "in tree %d.\n", cnt++);
 
        cnt = 1;
        list_for_each_entry(ext, &obj->oo_hp_exts, oe_link)
-               OSC_EXTENT_DUMP(level, ext, "hp %d.\n", cnt++);
+               OSC_EXTENT_DUMP(mask, ext, "hp %d.\n", cnt++);
 
        cnt = 1;
        list_for_each_entry(ext, &obj->oo_urgent_exts, oe_link)
-               OSC_EXTENT_DUMP(level, ext, "urgent %d.\n", cnt++);
+               OSC_EXTENT_DUMP(mask, ext, "urgent %d.\n", cnt++);
 
        cnt = 1;
        list_for_each_entry(ext, &obj->oo_reading_exts, oe_link)
-               OSC_EXTENT_DUMP(level, ext, "reading %d.\n", cnt++);
+               OSC_EXTENT_DUMP(mask, ext, "reading %d.\n", cnt++);
        /* osc_object_unlock(obj); */
 }
 
@@ -1375,9 +1381,9 @@ static int osc_completion(const struct lu_env *env, struct osc_async_page *oap,
        RETURN(0);
 }
 
-#define OSC_DUMP_GRANT(lvl, cli, fmt, args...) do {                    \
+#define OSC_DUMP_GRANT(mask, cli, fmt, args...) do {                   \
        struct client_obd *__tmp = (cli);                               \
-       CDEBUG(lvl, "%s: grant { dirty: %ld/%ld dirty_pages: %ld/%lu "  \
+       CDEBUG(mask, "%s: grant { dirty: %ld/%ld dirty_pages: %ld/%lu " \
               "dropped: %ld avail: %ld, dirty_grant: %ld, "            \
               "reserved: %ld, flight: %d } lru {in list: %ld, "        \
               "left: %ld, waiters: %d }" fmt "\n",                     \
@@ -1398,7 +1404,6 @@ static void osc_consume_write_grant(struct client_obd *cli,
 {
        assert_spin_locked(&cli->cl_loi_list_lock);
        LASSERT(!(pga->flag & OBD_BRW_FROM_GRANT));
-       atomic_long_inc(&obd_dirty_pages);
        cli->cl_dirty_pages++;
        pga->flag |= OBD_BRW_FROM_GRANT;
        CDEBUG(D_CACHE, "using %lu grant credits for brw %p page %p\n",
@@ -1466,13 +1471,20 @@ static void __osc_unreserve_grant(struct client_obd *cli,
        }
 }
 
-static void osc_unreserve_grant(struct client_obd *cli,
-                               unsigned int reserved, unsigned int unused)
+static void osc_unreserve_grant_nolock(struct client_obd *cli,
+                                      unsigned int reserved,
+                                      unsigned int unused)
 {
-       spin_lock(&cli->cl_loi_list_lock);
        __osc_unreserve_grant(cli, reserved, unused);
        if (unused > 0)
                osc_wake_cache_waiters(cli);
+}
+
+static void osc_unreserve_grant(struct client_obd *cli,
+                               unsigned int reserved, unsigned int unused)
+{
+       spin_lock(&cli->cl_loi_list_lock);
+       osc_unreserve_grant_nolock(cli, reserved, unused);
        spin_unlock(&cli->cl_loi_list_lock);
 }
 
@@ -1542,19 +1554,23 @@ static int osc_enter_cache_try(struct client_obd *cli,
        if (rc < 0)
                return 0;
 
-       if (cli->cl_dirty_pages < cli->cl_dirty_max_pages &&
-           1 + atomic_long_read(&obd_dirty_pages) <= obd_max_dirty_pages) {
-               osc_consume_write_grant(cli, &oap->oap_brw_page);
-               if (transient) {
-                       cli->cl_dirty_transit++;
-                       atomic_long_inc(&obd_dirty_transit_pages);
-                       oap->oap_brw_flags |= OBD_BRW_NOCACHE;
-               }
-               rc = 1;
-       } else {
-               __osc_unreserve_grant(cli, bytes, bytes);
-               rc = 0;
+       if (cli->cl_dirty_pages < cli->cl_dirty_max_pages) {
+               if (atomic_long_add_return(1, &obd_dirty_pages) <=
+                   obd_max_dirty_pages) {
+                       osc_consume_write_grant(cli, &oap->oap_brw_page);
+                       if (transient) {
+                               cli->cl_dirty_transit++;
+                               atomic_long_inc(&obd_dirty_transit_pages);
+                               oap->oap_brw_flags |= OBD_BRW_NOCACHE;
+                       }
+                       rc = 1;
+                       goto out;
+               } else
+                       atomic_long_dec(&obd_dirty_pages);
        }
+       __osc_unreserve_grant(cli, bytes, bytes);
+
+out:
        return rc;
 }
 
@@ -1601,7 +1617,8 @@ static int osc_enter_cache(const struct lu_env *env, struct client_obd *cli,
        }
 
        /* Hopefully normal case - cache space and write credits available */
-       if (osc_enter_cache_try(cli, oap, bytes, 0)) {
+       if (list_empty(&cli->cl_cache_waiters) &&
+           osc_enter_cache_try(cli, oap, bytes, 0)) {
                OSC_DUMP_GRANT(D_CACHE, cli, "granted from cache\n");
                GOTO(out, rc = 0);
        }
@@ -1686,27 +1703,21 @@ void osc_wake_cache_waiters(struct client_obd *cli)
        ENTRY;
        list_for_each_safe(l, tmp, &cli->cl_cache_waiters) {
                ocw = list_entry(l, struct osc_cache_waiter, ocw_entry);
-               list_del_init(&ocw->ocw_entry);
 
                ocw->ocw_rc = -EDQUOT;
-               /* we can't dirty more */
-               if ((cli->cl_dirty_pages  >= cli->cl_dirty_max_pages) ||
-                   (1 + atomic_long_read(&obd_dirty_pages) >
-                    obd_max_dirty_pages)) {
-                       CDEBUG(D_CACHE, "no dirty room: dirty: %ld "
-                              "osc max %ld, sys max %ld\n",
-                              cli->cl_dirty_pages, cli->cl_dirty_max_pages,
-                              obd_max_dirty_pages);
-                       goto wakeup;
-               }
 
                if (osc_enter_cache_try(cli, ocw->ocw_oap, ocw->ocw_grant, 0))
                        ocw->ocw_rc = 0;
-wakeup:
-               CDEBUG(D_CACHE, "wake up %p for oap %p, avail grant %ld, %d\n",
-                      ocw, ocw->ocw_oap, cli->cl_avail_grant, ocw->ocw_rc);
 
-               wake_up(&ocw->ocw_waitq);
+               if (ocw->ocw_rc == 0 ||
+                   !(cli->cl_dirty_pages > 0 || cli->cl_w_in_flight > 0)) {
+                       list_del_init(&ocw->ocw_entry);
+                       CDEBUG(D_CACHE, "wake up %p for oap %p, avail grant "
+                              "%ld, %d\n", ocw, ocw->ocw_oap,
+                              cli->cl_avail_grant, ocw->ocw_rc);
+
+                       wake_up(&ocw->ocw_waitq);
+               }
        }
 
        EXIT;
@@ -1991,36 +2002,6 @@ static int try_to_add_extent_for_io(struct client_obd *cli,
        RETURN(1);
 }
 
-static inline unsigned osc_max_write_chunks(const struct client_obd *cli)
-{
-       /*
-        * LU-8135:
-        *
-        * The maximum size of a single transaction is about 64MB in ZFS.
-        * #define DMU_MAX_ACCESS (64 * 1024 * 1024)
-        *
-        * Since ZFS is a copy-on-write file system, a single dirty page in
-        * a chunk will result in the rewrite of the whole chunk, therefore
-        * an RPC shouldn't be allowed to contain too many chunks otherwise
-        * it will make transaction size much bigger than 64MB, especially
-        * with big block size for ZFS.
-        *
-        * This piece of code is to make sure that OSC won't send write RPCs
-        * with too many chunks. The maximum chunk size that an RPC can cover
-        * is set to PTLRPC_MAX_BRW_SIZE, which is defined to 16MB. Ideally
-        * OST should tell the client what the biggest transaction size is,
-        * but it's good enough for now.
-        *
-        * This limitation doesn't apply to ldiskfs, which allows as many
-        * chunks in one RPC as we want. However, it won't have any benefits
-        * to have too many discontiguous pages in one RPC.
-        *
-        * An osc_extent won't cover over a RPC size, so the chunks in an
-        * osc_extent won't bigger than PTLRPC_MAX_BRW_SIZE >> chunkbits.
-        */
-       return PTLRPC_MAX_BRW_SIZE >> cli->cl_chunkbits;
-}
-
 /**
  * In order to prevent multiple ptlrpcd from breaking contiguous extents,
  * get_write_extent() takes all appropriate extents in atomic.
@@ -2104,7 +2085,7 @@ osc_send_write_rpc(const struct lu_env *env, struct client_obd *cli,
                   struct osc_object *osc)
 __must_hold(osc)
 {
-       struct list_head   rpclist = LIST_HEAD_INIT(rpclist);
+       LIST_HEAD(rpclist);
        struct osc_extent *ext;
        struct osc_extent *tmp;
        struct osc_extent *first = NULL;
@@ -2180,7 +2161,7 @@ __must_hold(osc)
 {
        struct osc_extent *ext;
        struct osc_extent *next;
-       struct list_head rpclist = LIST_HEAD_INIT(rpclist);
+       LIST_HEAD(rpclist);
        struct extent_rpc_data data = {
                .erd_rpc_list   = &rpclist,
                .erd_page_count = 0,
@@ -2363,9 +2344,6 @@ int osc_prep_async_page(struct osc_object *osc, struct osc_page *ops,
        oap->oap_obj_off = offset;
        LASSERT(!(offset & ~PAGE_MASK));
 
-       if (cfs_capable(CFS_CAP_SYS_RESOURCE))
-               oap->oap_brw_flags = OBD_BRW_NOQUOTA;
-
        INIT_LIST_HEAD(&oap->oap_pending_item);
        INIT_LIST_HEAD(&oap->oap_rpc_item);
 
@@ -2377,13 +2355,14 @@ int osc_prep_async_page(struct osc_object *osc, struct osc_page *ops,
 EXPORT_SYMBOL(osc_prep_async_page);
 
 int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
-                      struct osc_page *ops)
+                      struct osc_page *ops, cl_commit_cbt cb)
 {
        struct osc_io *oio = osc_env_io(env);
        struct osc_extent     *ext = NULL;
        struct osc_async_page *oap = &ops->ops_oap;
        struct client_obd     *cli = oap->oap_cli;
        struct osc_object     *osc = oap->oap_obj;
+       struct pagevec        *pvec = &osc_env_info(env)->oti_pagevec;
        pgoff_t index;
        unsigned int tmp;
        unsigned int grants = 0;
@@ -2405,7 +2384,7 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 
        /* Set the OBD_BRW_SRVLOCK before the page is queued. */
        brw_flags |= ops->ops_srvlock ? OBD_BRW_SRVLOCK : 0;
-       if (cfs_capable(CFS_CAP_SYS_RESOURCE)) {
+       if (oio->oi_cap_sys_resource) {
                brw_flags |= OBD_BRW_NOQUOTA;
                cmd |= OBD_BRW_NOQUOTA;
        }
@@ -2462,7 +2441,6 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
                /* it doesn't need any grant to dirty this page */
                spin_lock(&cli->cl_loi_list_lock);
                rc = osc_enter_cache_try(cli, oap, grants, 0);
-               spin_unlock(&cli->cl_loi_list_lock);
                if (rc == 0) { /* try failed */
                        grants = 0;
                        need_release = 1;
@@ -2476,10 +2454,11 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
                        } else {
                                OSC_EXTENT_DUMP(D_CACHE, ext,
                                                "expanded for %lu.\n", index);
-                               osc_unreserve_grant(cli, grants, tmp);
+                               osc_unreserve_grant_nolock(cli, grants, tmp);
                                grants = 0;
                        }
                }
+               spin_unlock(&cli->cl_loi_list_lock);
                rc = 0;
        } else if (ext != NULL) {
                /* index is located outside of active extent */
@@ -2502,7 +2481,14 @@ int osc_queue_async_io(const struct lu_env *env, struct cl_io *io,
 
                rc = 0;
                if (grants == 0) {
-                       /* we haven't allocated grant for this page. */
+                       /* We haven't allocated grant for this page, and we
+                        * must not hold a page lock while we do enter_cache,
+                        * so we must mark dirty & unlock any pages in the
+                        * write commit pagevec. */
+                       if (pagevec_count(pvec)) {
+                               cb(env, io, pvec);
+                               pagevec_reinit(pvec);
+                       }
                        rc = osc_enter_cache(env, cli, oap, tmp);
                        if (rc == 0)
                                grants = tmp;
@@ -2808,7 +2794,7 @@ int osc_cache_truncate_start(const struct lu_env *env, struct osc_object *obj,
        struct osc_extent *ext;
        struct osc_extent *waiting = NULL;
        pgoff_t index;
-       struct list_head list = LIST_HEAD_INIT(list);
+       LIST_HEAD(list);
        int result = 0;
        bool partial;
        ENTRY;
@@ -3026,7 +3012,7 @@ int osc_cache_writeback_range(const struct lu_env *env, struct osc_object *obj,
                              pgoff_t start, pgoff_t end, int hp, int discard)
 {
        struct osc_extent *ext;
-       struct list_head discard_list = LIST_HEAD_INIT(discard_list);
+       LIST_HEAD(discard_list);
        bool unplug = false;
        int result = 0;
        ENTRY;
@@ -3059,10 +3045,25 @@ int osc_cache_writeback_range(const struct lu_env *env, struct osc_object *obj,
                                        list_move_tail(&ext->oe_link, list);
                                unplug = true;
                        } else {
+                               struct client_obd *cli = osc_cli(obj);
+                               int pcc_bits = cli->cl_chunkbits - PAGE_SHIFT;
+                               pgoff_t align_by = (1 << pcc_bits);
+                               pgoff_t a_start = round_down(start, align_by);
+                               pgoff_t a_end = round_up(end, align_by);
+
+                               /* overflow case */
+                               if (end && !a_end)
+                                       a_end = CL_PAGE_EOF;
                                /* the only discarder is lock cancelling, so
-                                * [start, end] must contain this extent */
-                               EASSERT(ext->oe_start >= start &&
-                                       ext->oe_max_end <= end, ext);
+                                * [start, end], aligned by chunk size, must
+                                * contain this extent */
+                               LASSERTF(ext->oe_start >= a_start &&
+                                        ext->oe_end <= a_end,
+                                        "ext [%lu, %lu] reg [%lu, %lu] "
+                                        "orig [%lu %lu] align %lu bits "
+                                        "%d\n", ext->oe_start, ext->oe_end,
+                                        a_start, a_end, start, end,
+                                        align_by, pcc_bits);
                                osc_extent_state_set(ext, OES_LOCKING);
                                ext->oe_owner = current;
                                list_move_tail(&ext->oe_link,