Whamcloud - gitweb
land clio.
[fs/lustre-release.git] / lustre / llite / llite_capa.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * GPL HEADER START
5  *
6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License version 2 only,
10  * as published by the Free Software Foundation.
11  *
12  * This program is distributed in the hope that it will be useful, but
13  * WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * General Public License version 2 for more details (a copy is included
16  * in the LICENSE file that accompanied this code).
17  *
18  * You should have received a copy of the GNU General Public License
19  * version 2 along with this program; If not, see
20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
21  *
22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
23  * CA 95054 USA or visit www.sun.com if you need additional information or
24  * have any questions.
25  *
26  * GPL HEADER END
27  */
28 /*
29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
30  * Use is subject to license terms.
31  */
32 /*
33  * This file is part of Lustre, http://www.lustre.org/
34  * Lustre is a trademark of Sun Microsystems, Inc.
35  *
36  * lustre/llite/llite_capa.c
37  *
38  * Author: Lai Siyao <lsy@clusterfs.com>
39  */
40
41 #define DEBUG_SUBSYSTEM S_LLITE
42
43 #include <linux/fs.h>
44 #include <linux/version.h>
45 #include <asm/uaccess.h>
46 #include <linux/file.h>
47 #include <linux/kmod.h>
48
49 #include <lustre_lite.h>
50 #include "llite_internal.h"
51
52 /* for obd_capa.c_list, client capa might stay in three places:
53  * 1. ll_capa_list.
54  * 2. ll_idle_capas.
55  * 3. stand alone: just allocated.
56  */
57
58 /* capas for oss writeback and those failed to renew */
59 static LIST_HEAD(ll_idle_capas);
60 static struct ptlrpc_thread ll_capa_thread;
61 static struct list_head *ll_capa_list = &capa_list[CAPA_SITE_CLIENT];
62
63 /* llite capa renewal timer */
64 struct timer_list ll_capa_timer;
65 /* for debug: indicate whether capa on llite is enabled or not */
66 static atomic_t ll_capa_debug = ATOMIC_INIT(0);
67 static unsigned long long ll_capa_renewed = 0;
68 static unsigned long long ll_capa_renewal_noent = 0;
69 static unsigned long long ll_capa_renewal_failed = 0;
70 static unsigned long long ll_capa_renewal_retries = 0;
71
72 static inline void update_capa_timer(struct obd_capa *ocapa, cfs_time_t expiry)
73 {
74         if (time_before(expiry, ll_capa_timer.expires) ||
75             !timer_pending(&ll_capa_timer)) {
76                 mod_timer(&ll_capa_timer, expiry);
77                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
78                            "ll_capa_timer update: %lu/%lu by", expiry, jiffies);
79         }
80 }
81
82 static inline cfs_time_t capa_renewal_time(struct obd_capa *ocapa)
83 {
84         return cfs_time_sub(ocapa->c_expiry,
85                             cfs_time_seconds(ocapa->c_capa.lc_timeout) / 2);
86 }
87
88 static inline int capa_is_to_expire(struct obd_capa *ocapa)
89 {
90         return cfs_time_beforeq(capa_renewal_time(ocapa), cfs_time_current());
91 }
92
93 static inline int have_expired_capa(void)
94 {
95         struct obd_capa *ocapa = NULL;
96         int expired = 0;
97
98         /* if ll_capa_list has client capa to expire or ll_idle_capas has
99          * expired capa, return 1.
100          */
101         spin_lock(&capa_lock);
102         if (!list_empty(ll_capa_list)) {
103                 ocapa = list_entry(ll_capa_list->next, struct obd_capa, c_list);
104                 expired = capa_is_to_expire(ocapa);
105                 if (!expired)
106                         update_capa_timer(ocapa, capa_renewal_time(ocapa));
107         } else if (!list_empty(&ll_idle_capas)) {
108                 ocapa = list_entry(ll_idle_capas.next, struct obd_capa, c_list);
109                 expired = capa_is_expired(ocapa);
110                 if (!expired)
111                         update_capa_timer(ocapa, ocapa->c_expiry);
112         }
113         spin_unlock(&capa_lock);
114
115         if (expired)
116                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "expired");
117         return expired;
118 }
119
120 static inline int ll_capa_check_stop(void)
121 {
122         return (ll_capa_thread.t_flags & SVC_STOPPING) ? 1: 0;
123 }
124
125 static void sort_add_capa(struct obd_capa *ocapa, struct list_head *head)
126 {
127         struct obd_capa *tmp;
128         struct list_head *before = NULL;
129
130         /* TODO: client capa is sorted by expiry, this could be optimized */
131         list_for_each_entry_reverse(tmp, head, c_list) {
132                 if (cfs_time_aftereq(ocapa->c_expiry, tmp->c_expiry)) {
133                         before = &tmp->c_list;
134                         break;
135                 }
136         }
137
138         LASSERT(&ocapa->c_list != before);
139         list_add(&ocapa->c_list, before ?: head);
140 }
141
142 static inline int obd_capa_open_count(struct obd_capa *oc)
143 {
144         struct ll_inode_info *lli = ll_i2info(oc->u.cli.inode);
145         return atomic_read(&lli->lli_open_count);
146 }
147
148 static void ll_delete_capa(struct obd_capa *ocapa)
149 {
150         struct ll_inode_info *lli = ll_i2info(ocapa->u.cli.inode);
151
152         if (capa_for_mds(&ocapa->c_capa)) {
153                 LASSERT(lli->lli_mds_capa == ocapa);
154                 lli->lli_mds_capa = NULL;
155         } else if (capa_for_oss(&ocapa->c_capa)) {
156                 list_del_init(&ocapa->u.cli.lli_list);
157         }
158
159         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "free client");
160         list_del(&ocapa->c_list);
161         capa_count[CAPA_SITE_CLIENT]--;
162         free_capa(ocapa);
163 }
164
165 /* three places where client capa is deleted:
166  * 1. capa_thread_main(), main place to delete expired capa.
167  * 2. ll_clear_inode_capas() in ll_clear_inode().
168  * 3. ll_truncate_free_capa() delete truncate capa explicitly in ll_truncate().
169  */
170 static int capa_thread_main(void *unused)
171 {
172         struct obd_capa *ocapa, *tmp, *next;
173         struct inode *inode = NULL;
174         struct l_wait_info lwi = { 0 };
175         int rc;
176         ENTRY;
177
178         cfs_daemonize("ll_capa");
179
180         ll_capa_thread.t_flags = SVC_RUNNING;
181         wake_up(&ll_capa_thread.t_ctl_waitq);
182
183         while (1) {
184                 l_wait_event(ll_capa_thread.t_ctl_waitq,
185                              (ll_capa_check_stop() || have_expired_capa()),
186                              &lwi);
187
188                 if (ll_capa_check_stop())
189                         break;
190
191                 next = NULL;
192
193                 spin_lock(&capa_lock);
194                 list_for_each_entry_safe(ocapa, tmp, ll_capa_list, c_list) {
195                         LASSERT(ocapa->c_capa.lc_opc != CAPA_OPC_OSS_TRUNC);
196
197                         if (!capa_is_to_expire(ocapa)) {
198                                 next = ocapa;
199                                 break;
200                         }
201
202                         list_del_init(&ocapa->c_list);
203
204                         /* for MDS capability, only renew those which belong to
205                          * dir, or its inode is opened, or client holds LOOKUP
206                          * lock.
207                          */
208                         if (capa_for_mds(&ocapa->c_capa) &&
209                             !S_ISDIR(ocapa->u.cli.inode->i_mode) &&
210                             obd_capa_open_count(ocapa) == 0 &&
211                             !ll_have_md_lock(ocapa->u.cli.inode,
212                                              MDS_INODELOCK_LOOKUP)) {
213                                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
214                                            "skip renewal for");
215                                 sort_add_capa(ocapa, &ll_idle_capas);
216                                 continue;
217                         }
218
219                         /* for OSS capability, only renew those whose inode is
220                          * opened.
221                          */
222                         if (capa_for_oss(&ocapa->c_capa) &&
223                             obd_capa_open_count(ocapa) == 0) {
224                                 /* oss capa with open count == 0 won't renew,
225                                  * move to idle list */
226                                 sort_add_capa(ocapa, &ll_idle_capas);
227                                 continue;
228                         }
229
230                         /* NB iput() is in ll_update_capa() */
231                         inode = igrab(ocapa->u.cli.inode);
232                         if (inode == NULL) {
233                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
234                                            "igrab failed for");
235                                 continue;
236                         }
237
238                         capa_get(ocapa);
239                         ll_capa_renewed++;
240                         spin_unlock(&capa_lock);
241
242                         rc = md_renew_capa(ll_i2mdexp(inode), ocapa,
243                                            ll_update_capa);
244                         spin_lock(&capa_lock);
245                         if (rc) {
246                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
247                                            "renew failed: %d", rc);
248                                 ll_capa_renewal_failed++;
249                         }
250                 }
251
252                 if (next)
253                         update_capa_timer(next, capa_renewal_time(next));
254
255                 list_for_each_entry_safe(ocapa, tmp, &ll_idle_capas, c_list) {
256                         if (!capa_is_expired(ocapa)) {
257                                 if (!next)
258                                         update_capa_timer(ocapa, ocapa->c_expiry);
259                                 break;
260                         }
261
262                         if (atomic_read(&ocapa->c_refc)) {
263                                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
264                                            "expired(c_refc %d), don't release",
265                                            atomic_read(&ocapa->c_refc));
266                                 /* don't try to renew any more */
267                                 list_del_init(&ocapa->c_list);
268                                 continue;
269                         }
270
271                         /* expired capa is released. */
272                         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "release expired");
273                         ll_delete_capa(ocapa);
274                 }
275
276                 spin_unlock(&capa_lock);
277         }
278
279         ll_capa_thread.t_flags = SVC_STOPPED;
280         wake_up(&ll_capa_thread.t_ctl_waitq);
281         RETURN(0);
282 }
283
284 void ll_capa_timer_callback(unsigned long unused)
285 {
286         wake_up(&ll_capa_thread.t_ctl_waitq);
287 }
288
289 int ll_capa_thread_start(void)
290 {
291         int rc;
292         ENTRY;
293
294         init_waitqueue_head(&ll_capa_thread.t_ctl_waitq);
295
296         rc = kernel_thread(capa_thread_main, NULL, 0);
297         if (rc < 0) {
298                 CERROR("cannot start expired capa thread: rc %d\n", rc);
299                 RETURN(rc);
300         }
301         wait_event(ll_capa_thread.t_ctl_waitq,
302                    ll_capa_thread.t_flags & SVC_RUNNING);
303
304         RETURN(0);
305 }
306
307 void ll_capa_thread_stop(void)
308 {
309         ll_capa_thread.t_flags = SVC_STOPPING;
310         wake_up(&ll_capa_thread.t_ctl_waitq);
311         wait_event(ll_capa_thread.t_ctl_waitq,
312                    ll_capa_thread.t_flags & SVC_STOPPED);
313 }
314
315 static struct obd_capa *do_lookup_oss_capa(struct inode *inode, int opc)
316 {
317         struct ll_inode_info *lli = ll_i2info(inode);
318         struct obd_capa *ocapa;
319
320         /* inside capa_lock */
321         list_for_each_entry(ocapa, &lli->lli_oss_capas, u.cli.lli_list) {
322                 if ((capa_opc(&ocapa->c_capa) & opc) != opc)
323                         continue;
324
325                 LASSERT(lu_fid_eq(capa_fid(&ocapa->c_capa),
326                                   ll_inode2fid(inode)));
327                 LASSERT(ocapa->c_site == CAPA_SITE_CLIENT);
328
329                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "found client");
330                 return ocapa;
331         }
332
333         return NULL;
334 }
335
336 struct obd_capa *ll_osscapa_get(struct inode *inode, __u64 opc)
337 {
338         struct ll_inode_info *lli = ll_i2info(inode);
339         struct obd_capa *ocapa;
340         int found = 0;
341
342         ENTRY;
343
344         if ((ll_i2sbi(inode)->ll_flags & LL_SBI_OSS_CAPA) == 0)
345                 RETURN(NULL);
346
347         LASSERT(opc == CAPA_OPC_OSS_WRITE || opc == CAPA_OPC_OSS_RW ||
348                 opc == CAPA_OPC_OSS_TRUNC);
349
350         spin_lock(&capa_lock);
351         list_for_each_entry(ocapa, &lli->lli_oss_capas, u.cli.lli_list) {
352                 if (capa_is_expired(ocapa))
353                         continue;
354                 if ((opc & CAPA_OPC_OSS_WRITE) &&
355                     capa_opc_supported(&ocapa->c_capa, CAPA_OPC_OSS_WRITE)) {
356                         found = 1; break;
357                 } else if ((opc & CAPA_OPC_OSS_READ) &&
358                            capa_opc_supported(&ocapa->c_capa,
359                                               CAPA_OPC_OSS_READ)) {
360                         found = 1; break;
361                 } else if ((opc & CAPA_OPC_OSS_TRUNC) &&
362                            capa_opc_supported(&ocapa->c_capa, opc)) {
363                         found = 1; break;
364                 }
365         }
366
367         if (found) {
368                 LASSERT(lu_fid_eq(capa_fid(&ocapa->c_capa),
369                                   ll_inode2fid(inode)));
370                 LASSERT(ocapa->c_site == CAPA_SITE_CLIENT);
371
372                 capa_get(ocapa);
373
374                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "found client");
375         } else {
376                 ocapa = NULL;
377
378                 if (atomic_read(&ll_capa_debug)) {
379                         CERROR("no capability for "DFID" opc "LPX64"\n",
380                                PFID(&lli->lli_fid), opc);
381                         atomic_set(&ll_capa_debug, 0);
382                 }
383         }
384         spin_unlock(&capa_lock);
385
386         RETURN(ocapa);
387 }
388 EXPORT_SYMBOL(ll_osscapa_get);
389
390 struct obd_capa *ll_mdscapa_get(struct inode *inode)
391 {
392         struct ll_inode_info *lli = ll_i2info(inode);
393         struct obd_capa *ocapa;
394         ENTRY;
395
396         LASSERT(inode != NULL);
397
398         if ((ll_i2sbi(inode)->ll_flags & LL_SBI_MDS_CAPA) == 0)
399                 RETURN(NULL);
400
401         spin_lock(&capa_lock);
402         ocapa = capa_get(lli->lli_mds_capa);
403         spin_unlock(&capa_lock);
404         if (!ocapa && atomic_read(&ll_capa_debug)) {
405                 CERROR("no mds capability for "DFID"\n", PFID(&lli->lli_fid));
406                 atomic_set(&ll_capa_debug, 0);
407         }
408
409         RETURN(ocapa);
410 }
411
412 static struct obd_capa *do_add_mds_capa(struct inode *inode,
413                                         struct obd_capa *ocapa)
414 {
415         struct ll_inode_info *lli = ll_i2info(inode);
416         struct obd_capa *old = lli->lli_mds_capa;
417         struct lustre_capa *capa = &ocapa->c_capa;
418
419         if (!old) {
420                 ocapa->u.cli.inode = inode;
421                 lli->lli_mds_capa = ocapa;
422                 capa_count[CAPA_SITE_CLIENT]++;
423
424                 DEBUG_CAPA(D_SEC, capa, "add MDS");
425         } else {
426                 spin_lock(&old->c_lock);
427                 old->c_capa = *capa;
428                 spin_unlock(&old->c_lock);
429
430                 DEBUG_CAPA(D_SEC, capa, "update MDS");
431
432                 free_capa(ocapa);
433                 ocapa = old;
434         }
435         return ocapa;
436 }
437
438 static inline void inode_add_oss_capa(struct inode *inode,
439                                       struct obd_capa *ocapa)
440 {
441         struct ll_inode_info *lli = ll_i2info(inode);
442         struct obd_capa *tmp;
443         struct list_head *next = NULL;
444
445         /* capa is sorted in lli_oss_capas so lookup can always find the
446          * latest one */
447         list_for_each_entry(tmp, &lli->lli_oss_capas, u.cli.lli_list) {
448                 if (cfs_time_after(ocapa->c_expiry, tmp->c_expiry)) {
449                         next = &tmp->u.cli.lli_list;
450                         break;
451                 }
452         }
453         LASSERT(&ocapa->u.cli.lli_list != next);
454         list_move_tail(&ocapa->u.cli.lli_list, next ?: &lli->lli_oss_capas);
455 }
456
457 static struct obd_capa *do_add_oss_capa(struct inode *inode,
458                                         struct obd_capa *ocapa)
459 {
460         struct obd_capa *old;
461         struct lustre_capa *capa = &ocapa->c_capa;
462
463         LASSERTF(S_ISREG(inode->i_mode),
464                  "inode has oss capa, but not regular file, mode: %d\n",
465                  inode->i_mode);
466
467         /* FIXME: can't replace it so easily with fine-grained opc */
468         old = do_lookup_oss_capa(inode, capa_opc(capa) & CAPA_OPC_OSS_ONLY);
469         if (!old) {
470                 ocapa->u.cli.inode = inode;
471                 INIT_LIST_HEAD(&ocapa->u.cli.lli_list);
472                 capa_count[CAPA_SITE_CLIENT]++;
473
474                 DEBUG_CAPA(D_SEC, capa, "add OSS");
475         } else {
476                 spin_lock(&old->c_lock);
477                 old->c_capa = *capa;
478                 spin_unlock(&old->c_lock);
479
480                 DEBUG_CAPA(D_SEC, capa, "update OSS");
481
482                 free_capa(ocapa);
483                 ocapa = old;
484         }
485
486         inode_add_oss_capa(inode, ocapa);
487         return ocapa;
488 }
489
490 struct obd_capa *ll_add_capa(struct inode *inode, struct obd_capa *ocapa)
491 {
492         spin_lock(&capa_lock);
493         ocapa = capa_for_mds(&ocapa->c_capa) ? do_add_mds_capa(inode, ocapa) :
494                                                do_add_oss_capa(inode, ocapa);
495
496         /* truncate capa won't renew */
497         if (ocapa->c_capa.lc_opc != CAPA_OPC_OSS_TRUNC) {
498                 set_capa_expiry(ocapa);
499                 list_del(&ocapa->c_list);
500                 sort_add_capa(ocapa, ll_capa_list);
501
502                 update_capa_timer(ocapa, capa_renewal_time(ocapa));
503         }
504
505         spin_unlock(&capa_lock);
506
507         atomic_set(&ll_capa_debug, 1);
508         return ocapa;
509 }
510
511 static inline void delay_capa_renew(struct obd_capa *oc, cfs_time_t delay)
512 {
513         /* NB: set a fake expiry for this capa to prevent it renew too soon */
514         oc->c_expiry = cfs_time_add(oc->c_expiry, cfs_time_seconds(delay));
515 }
516
517 int ll_update_capa(struct obd_capa *ocapa, struct lustre_capa *capa)
518 {
519         struct inode *inode = ocapa->u.cli.inode;
520         int rc = 0;
521         ENTRY;
522
523         LASSERT(ocapa);
524
525         if (IS_ERR(capa)) {
526                 /* set error code */
527                 rc = PTR_ERR(capa);
528                 spin_lock(&capa_lock);
529                 if (rc == -ENOENT) {
530                         DEBUG_CAPA(D_SEC, &ocapa->c_capa,
531                                    "renewal canceled because object removed");
532                         ll_capa_renewal_noent++;
533                 } else {
534                         ll_capa_renewal_failed++;
535
536                         /* failed capa won't be renewed any longer, but if -EIO,
537                          * client might be doing recovery, retry in 2 min. */
538                         if (rc == -EIO && !capa_is_expired(ocapa)) {
539                                 delay_capa_renew(ocapa, 120);
540                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
541                                            "renewal failed: -EIO, retry in 2 mins");
542                                 ll_capa_renewal_retries++;
543                                 GOTO(retry, rc);
544                         } else {
545                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
546                                            "renewal failed(rc: %d) for", rc);
547                         }
548                 }
549
550                 list_del(&ocapa->c_list);
551                 sort_add_capa(ocapa, &ll_idle_capas);
552                 spin_unlock(&capa_lock);
553
554                 capa_put(ocapa);
555                 iput(inode);
556                 return rc;
557         }
558
559         spin_lock(&ocapa->c_lock);
560         LASSERT(!memcmp(&ocapa->c_capa, capa,
561                         offsetof(struct lustre_capa, lc_flags)));
562         ocapa->c_capa = *capa;
563         set_capa_expiry(ocapa);
564         spin_unlock(&ocapa->c_lock);
565
566         spin_lock(&capa_lock);
567         if (capa_for_oss(capa))
568                 inode_add_oss_capa(inode, ocapa);
569         DEBUG_CAPA(D_SEC, capa, "renew");
570         EXIT;
571 retry:
572         list_del_init(&ocapa->c_list);
573         sort_add_capa(ocapa, ll_capa_list);
574         update_capa_timer(ocapa, capa_renewal_time(ocapa));
575         spin_unlock(&capa_lock);
576
577         capa_put(ocapa);
578         iput(inode);
579         return rc;
580 }
581
582 void ll_capa_open(struct inode *inode)
583 {
584         struct ll_inode_info *lli = ll_i2info(inode);
585
586         if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
587             == 0)
588                 return;
589
590         if (!S_ISREG(inode->i_mode))
591                 return;
592
593         atomic_inc(&lli->lli_open_count);
594 }
595
596 void ll_capa_close(struct inode *inode)
597 {
598         struct ll_inode_info *lli = ll_i2info(inode);
599
600         if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
601             == 0)
602                 return;
603
604         if (!S_ISREG(inode->i_mode))
605                 return;
606
607         atomic_dec(&lli->lli_open_count);
608 }
609
610 /* delete CAPA_OPC_OSS_TRUNC only */
611 void ll_truncate_free_capa(struct obd_capa *ocapa)
612 {
613         if (!ocapa)
614                 return;
615
616         LASSERT(ocapa->c_capa.lc_opc & CAPA_OPC_OSS_TRUNC);
617         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "free truncate");
618
619         capa_put(ocapa);
620         spin_lock(&capa_lock);
621         ll_delete_capa(ocapa);
622         spin_unlock(&capa_lock);
623 }
624
625 void ll_clear_inode_capas(struct inode *inode)
626 {
627         struct ll_inode_info *lli = ll_i2info(inode);
628         struct obd_capa *ocapa, *tmp;
629
630         spin_lock(&capa_lock);
631         ocapa = lli->lli_mds_capa;
632         if (ocapa)
633                 ll_delete_capa(ocapa);
634
635         list_for_each_entry_safe(ocapa, tmp, &lli->lli_oss_capas,
636                                  u.cli.lli_list)
637                 ll_delete_capa(ocapa);
638         spin_unlock(&capa_lock);
639 }
640
641 void ll_print_capa_stat(struct ll_sb_info *sbi)
642 {
643         if (sbi->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
644                 LCONSOLE_INFO("Fid capabilities renewed: %llu\n"
645                               "Fid capabilities renewal ENOENT: %llu\n"
646                               "Fid capabilities failed to renew: %llu\n"
647                               "Fid capabilities renewal retries: %llu\n",
648                               ll_capa_renewed, ll_capa_renewal_noent,
649                               ll_capa_renewal_failed, ll_capa_renewal_retries);
650 }