Whamcloud - gitweb
536644d94705291cdcdb32e2a5a7d2a315ccaf84
[fs/lustre-release.git] / lustre / llite / llite_capa.c
1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2  * vim:expandtab:shiftwidth=8:tabstop=8:
3  *
4  * Copyright (C) 2005 Cluster File Systems, Inc.
5  *
6  * Author: Lai Siyao <lsy@clusterfs.com>
7  *
8  *   This file is part of Lustre, http://www.lustre.org.
9  *
10  *   Lustre is free software; you can redistribute it and/or
11  *   modify it under the terms of version 2 of the GNU General Public
12  *   License as published by the Free Software Foundation.
13  *
14  *   Lustre is distributed in the hope that it will be useful,
15  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  *   GNU General Public License for more details.
18  *
19  *   You should have received a copy of the GNU General Public License
20  *   along with Lustre; if not, write to the Free Software
21  *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22  */
23
24 #define DEBUG_SUBSYSTEM S_LLITE
25
26 #include <linux/fs.h>
27 #include <linux/version.h>
28 #include <asm/uaccess.h>
29 #include <linux/file.h>
30 #include <linux/kmod.h>
31
32 #include <lustre_lite.h>
33 #include "llite_internal.h"
34
35 /* for obd_capa.c_list, client capa might stay in three places:
36  * 1. ll_capa_list.
37  * 2. ll_idle_capas.
38  * 3. stand alone: just allocated.
39  */
40
41 /* capas for oss writeback and those failed to renew */
42 static LIST_HEAD(ll_idle_capas);
43 static struct ptlrpc_thread ll_capa_thread;
44 static struct list_head *ll_capa_list = &capa_list[CAPA_SITE_CLIENT];
45
46 /* llite capa renewal timer */
47 struct timer_list ll_capa_timer;
48 /* for debug: indicate whether capa on llite is enabled or not */
49 static atomic_t ll_capa_debug = ATOMIC_INIT(0);
50 static unsigned long long ll_capa_renewed = 0;
51 static unsigned long long ll_capa_renewal_noent = 0;
52 static unsigned long long ll_capa_renewal_failed = 0;
53 static unsigned long long ll_capa_renewal_retries = 0;
54
55 static inline void update_capa_timer(struct obd_capa *ocapa, cfs_time_t expiry)
56 {
57         if (time_before(expiry, ll_capa_timer.expires) ||
58             !timer_pending(&ll_capa_timer)) {
59                 mod_timer(&ll_capa_timer, expiry);
60                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
61                            "ll_capa_timer update: %lu/%lu by", expiry, jiffies);
62         }
63 }
64
65 static inline cfs_time_t capa_renewal_time(struct obd_capa *ocapa)
66 {
67         return cfs_time_sub(ocapa->c_expiry,
68                             cfs_time_seconds(ocapa->c_capa.lc_timeout) / 2);
69 }
70
71 static inline int capa_is_to_expire(struct obd_capa *ocapa)
72 {
73         return cfs_time_beforeq(capa_renewal_time(ocapa), cfs_time_current());
74 }
75
76 static inline int have_expired_capa(void)
77 {
78         struct obd_capa *ocapa = NULL;
79         int expired = 0;
80
81         /* if ll_capa_list has client capa to expire or ll_idle_capas has
82          * expired capa, return 1.
83          */
84         spin_lock(&capa_lock);
85         if (!list_empty(ll_capa_list)) {
86                 ocapa = list_entry(ll_capa_list->next, struct obd_capa, c_list);
87                 expired = capa_is_to_expire(ocapa);
88                 if (!expired)
89                         update_capa_timer(ocapa, capa_renewal_time(ocapa));
90         } else if (!list_empty(&ll_idle_capas)) {
91                 ocapa = list_entry(ll_idle_capas.next, struct obd_capa, c_list);
92                 expired = capa_is_expired(ocapa);
93                 if (!expired)
94                         update_capa_timer(ocapa, ocapa->c_expiry);
95         }
96         spin_unlock(&capa_lock);
97
98         if (expired)
99                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "expired");
100         return expired;
101 }
102
103 static inline int ll_capa_check_stop(void)
104 {
105         return (ll_capa_thread.t_flags & SVC_STOPPING) ? 1: 0;
106 }
107
108 static void sort_add_capa(struct obd_capa *ocapa, struct list_head *head)
109 {
110         struct obd_capa *tmp;
111         struct list_head *before = NULL;
112
113         /* TODO: client capa is sorted by expiry, this could be optimized */
114         list_for_each_entry_reverse(tmp, head, c_list) {
115                 if (cfs_time_aftereq(ocapa->c_expiry, tmp->c_expiry)) {
116                         before = &tmp->c_list;
117                         break;
118                 }
119         }
120
121         LASSERT(&ocapa->c_list != before);
122         list_add(&ocapa->c_list, before ?: head);
123 }
124
125 static inline int obd_capa_open_count(struct obd_capa *oc)
126 {
127         struct ll_inode_info *lli = ll_i2info(oc->u.cli.inode);
128         return atomic_read(&lli->lli_open_count);
129 }
130
131 static void ll_delete_capa(struct obd_capa *ocapa)
132 {
133         struct ll_inode_info *lli = ll_i2info(ocapa->u.cli.inode);
134
135         if (capa_for_mds(&ocapa->c_capa)) {
136                 LASSERT(lli->lli_mds_capa == ocapa);
137                 lli->lli_mds_capa = NULL;
138         } else if (capa_for_oss(&ocapa->c_capa)) {
139                 list_del_init(&ocapa->u.cli.lli_list);
140         }
141
142         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "free client");
143         list_del(&ocapa->c_list);
144         capa_count[CAPA_SITE_CLIENT]--;
145         free_capa(ocapa);
146 }
147
148 /* three places where client capa is deleted:
149  * 1. capa_thread_main(), main place to delete expired capa.
150  * 2. ll_clear_inode_capas() in ll_clear_inode().
151  * 3. ll_truncate_free_capa() delete truncate capa explicitly in ll_truncate().
152  */
153 static int capa_thread_main(void *unused)
154 {
155         struct obd_capa *ocapa, *tmp, *next;
156         struct inode *inode = NULL;
157         struct l_wait_info lwi = { 0 };
158         int rc;
159         ENTRY;
160
161         cfs_daemonize("ll_capa");
162
163         ll_capa_thread.t_flags = SVC_RUNNING;
164         wake_up(&ll_capa_thread.t_ctl_waitq);
165
166         while (1) {
167                 l_wait_event(ll_capa_thread.t_ctl_waitq,
168                              (ll_capa_check_stop() || have_expired_capa()),
169                              &lwi);
170
171                 if (ll_capa_check_stop())
172                         break;
173
174                 next = NULL;
175
176                 spin_lock(&capa_lock);
177                 list_for_each_entry_safe(ocapa, tmp, ll_capa_list, c_list) {
178                         LASSERT(ocapa->c_capa.lc_opc != CAPA_OPC_OSS_TRUNC);
179
180                         if (!capa_is_to_expire(ocapa)) {
181                                 next = ocapa;
182                                 break;
183                         }
184
185                         list_del_init(&ocapa->c_list);
186
187                         /* for MDS capability, only renew those which belong to
188                          * dir, or its inode is opened, or client holds LOOKUP
189                          * lock.
190                          */
191                         if (capa_for_mds(&ocapa->c_capa) &&
192                             !S_ISDIR(ocapa->u.cli.inode->i_mode) &&
193                             obd_capa_open_count(ocapa) == 0 &&
194                             !ll_have_md_lock(ocapa->u.cli.inode,
195                                              MDS_INODELOCK_LOOKUP)) {
196                                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
197                                            "skip renewal for");
198                                 sort_add_capa(ocapa, &ll_idle_capas);
199                                 continue;
200                         }
201
202                         /* for OSS capability, only renew those whose inode is
203                          * opened.
204                          */
205                         if (capa_for_oss(&ocapa->c_capa) &&
206                             obd_capa_open_count(ocapa) == 0) {
207                                 /* oss capa with open count == 0 won't renew,
208                                  * move to idle list */
209                                 sort_add_capa(ocapa, &ll_idle_capas);
210                                 continue;
211                         }
212
213                         /* NB iput() is in ll_update_capa() */
214                         inode = igrab(ocapa->u.cli.inode);
215                         if (inode == NULL) {
216                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
217                                            "igrab failed for");
218                                 continue;
219                         }
220
221                         capa_get(ocapa);
222                         ll_capa_renewed++;
223                         spin_unlock(&capa_lock);
224
225                         rc = md_renew_capa(ll_i2mdexp(inode), ocapa,
226                                            ll_update_capa);
227                         spin_lock(&capa_lock);
228                         if (rc) {
229                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
230                                            "renew failed: %d", rc);
231                                 ll_capa_renewal_failed++;
232                         }
233                 }
234
235                 if (next)
236                         update_capa_timer(next, capa_renewal_time(next));
237
238                 list_for_each_entry_safe(ocapa, tmp, &ll_idle_capas, c_list) {
239                         if (!capa_is_expired(ocapa)) {
240                                 if (!next)
241                                         update_capa_timer(ocapa, ocapa->c_expiry);
242                                 break;
243                         }
244
245                         if (atomic_read(&ocapa->c_refc)) {
246                                 DEBUG_CAPA(D_SEC, &ocapa->c_capa,
247                                            "expired(c_refc %d), don't release",
248                                            atomic_read(&ocapa->c_refc));
249                                 /* don't try to renew any more */
250                                 list_del_init(&ocapa->c_list);
251                                 continue;
252                         }
253
254                         /* expired capa is released. */
255                         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "release expired");
256                         ll_delete_capa(ocapa);
257                 }
258
259                 spin_unlock(&capa_lock);
260         }
261
262         ll_capa_thread.t_flags = SVC_STOPPED;
263         wake_up(&ll_capa_thread.t_ctl_waitq);
264         RETURN(0);
265 }
266
267 void ll_capa_timer_callback(unsigned long unused)
268 {
269         wake_up(&ll_capa_thread.t_ctl_waitq);
270 }
271
272 int ll_capa_thread_start(void)
273 {
274         int rc;
275         ENTRY;
276
277         init_waitqueue_head(&ll_capa_thread.t_ctl_waitq);
278
279         rc = kernel_thread(capa_thread_main, NULL, 0);
280         if (rc < 0) {
281                 CERROR("cannot start expired capa thread: rc %d\n", rc);
282                 RETURN(rc);
283         }
284         wait_event(ll_capa_thread.t_ctl_waitq,
285                    ll_capa_thread.t_flags & SVC_RUNNING);
286
287         RETURN(0);
288 }
289
290 void ll_capa_thread_stop(void)
291 {
292         ll_capa_thread.t_flags = SVC_STOPPING;
293         wake_up(&ll_capa_thread.t_ctl_waitq);
294         wait_event(ll_capa_thread.t_ctl_waitq,
295                    ll_capa_thread.t_flags & SVC_STOPPED);
296 }
297
298 static struct obd_capa *do_lookup_oss_capa(struct inode *inode, uid_t uid,
299                                            int opc)
300 {
301         struct ll_inode_info *lli = ll_i2info(inode);
302         struct obd_capa *ocapa;
303
304         /* inside capa_lock */
305         list_for_each_entry(ocapa, &lli->lli_oss_capas, u.cli.lli_list) {
306                 if (uid != capa_uid(&ocapa->c_capa))
307                         continue;
308                 if ((capa_opc(&ocapa->c_capa) & opc) != opc)
309                         continue;
310
311                 LASSERT(lu_fid_eq(capa_fid(&ocapa->c_capa),
312                                   ll_inode2fid(inode)));
313                 LASSERT(ocapa->c_site == CAPA_SITE_CLIENT);
314
315                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "found client");
316                 return ocapa;
317         }
318
319         return NULL;
320 }
321
322 /* FIXME: once uid is 0, this is mmaped IO, or fsync, truncate. */
323 struct obd_capa *ll_osscapa_get(struct inode *inode, uid_t uid, __u64 opc)
324 {
325         struct ll_inode_info *lli = ll_i2info(inode);
326         struct obd_capa *ocapa;
327         int found = 0;
328
329         if ((ll_i2sbi(inode)->ll_flags & LL_SBI_OSS_CAPA) == 0)
330                 return NULL;
331         ENTRY;
332
333         LASSERT(opc == CAPA_OPC_OSS_WRITE || opc == CAPA_OPC_OSS_RW ||
334                 opc == CAPA_OPC_OSS_TRUNC);
335
336         spin_lock(&capa_lock);
337         list_for_each_entry(ocapa, &lli->lli_oss_capas, u.cli.lli_list) {
338                 if (capa_is_expired(ocapa))
339                         continue;
340                 if (uid != 0 && uid != capa_uid(&ocapa->c_capa))
341                         continue;
342                 if ((opc & CAPA_OPC_OSS_WRITE) &&
343                     capa_opc_supported(&ocapa->c_capa, CAPA_OPC_OSS_WRITE)) {
344                         found = 1; break;
345                 } else if ((opc & CAPA_OPC_OSS_READ) &&
346                            capa_opc_supported(&ocapa->c_capa,
347                                               CAPA_OPC_OSS_READ)) {
348                         found = 1; break;
349                 } else if ((opc & CAPA_OPC_OSS_TRUNC) &&
350                            capa_opc_supported(&ocapa->c_capa, opc)) {
351                         found = 1; break;
352                 }
353         }
354
355         if (found) {
356                 LASSERT(lu_fid_eq(capa_fid(&ocapa->c_capa),
357                                   ll_inode2fid(inode)));
358                 LASSERT(ocapa->c_site == CAPA_SITE_CLIENT);
359
360                 capa_get(ocapa);
361
362                 DEBUG_CAPA(D_SEC, &ocapa->c_capa, "found client");
363         } else {
364                 ocapa = NULL;
365
366                 if (atomic_read(&ll_capa_debug)) {
367                         CERROR("no capability for "DFID" opc "LPX64"\n",
368                                PFID(&lli->lli_fid), opc);
369                         atomic_set(&ll_capa_debug, 0);
370                 }
371         }
372         spin_unlock(&capa_lock);
373
374         RETURN(ocapa);
375 }
376
377 struct obd_capa *ll_mdscapa_get(struct inode *inode)
378 {
379         struct ll_inode_info *lli = ll_i2info(inode);
380         struct obd_capa *ocapa;
381         ENTRY;
382
383         LASSERT(inode != NULL);
384         
385         if ((ll_i2sbi(inode)->ll_flags & LL_SBI_MDS_CAPA) == 0)
386                 RETURN(NULL);
387
388         spin_lock(&capa_lock);
389         ocapa = capa_get(lli->lli_mds_capa);
390         spin_unlock(&capa_lock);
391         if (!ocapa && atomic_read(&ll_capa_debug)) {
392                 CERROR("no mds capability for "DFID"\n", PFID(&lli->lli_fid));
393                 atomic_set(&ll_capa_debug, 0);
394         }
395
396         RETURN(ocapa);
397 }
398
399 static struct obd_capa *do_add_mds_capa(struct inode *inode,
400                                         struct obd_capa *ocapa)
401 {
402         struct ll_inode_info *lli = ll_i2info(inode);
403         struct obd_capa *old = lli->lli_mds_capa;
404         struct lustre_capa *capa = &ocapa->c_capa;
405
406         if (!old) {
407                 ocapa->u.cli.inode = inode;
408                 lli->lli_mds_capa = ocapa;
409                 capa_count[CAPA_SITE_CLIENT]++;
410
411                 DEBUG_CAPA(D_SEC, capa, "add MDS");
412         } else {
413                 spin_lock(&old->c_lock);
414                 old->c_capa = *capa;
415                 spin_unlock(&old->c_lock);
416
417                 DEBUG_CAPA(D_SEC, capa, "update MDS");
418
419                 free_capa(ocapa);
420                 ocapa = old;
421         }
422         return ocapa;
423 }
424
425 static inline void inode_add_oss_capa(struct inode *inode,
426                                       struct obd_capa *ocapa)
427 {
428         struct ll_inode_info *lli = ll_i2info(inode);
429         struct obd_capa *tmp;
430         struct list_head *next = NULL;
431
432         /* capa is sorted in lli_oss_capas so lookup can always find the
433          * latest one */
434         list_for_each_entry(tmp, &lli->lli_oss_capas, u.cli.lli_list) {
435                 if (cfs_time_after(ocapa->c_expiry, tmp->c_expiry)) {
436                         next = &tmp->u.cli.lli_list;
437                         break;
438                 }
439         }
440         LASSERT(&ocapa->u.cli.lli_list != next);
441         list_move_tail(&ocapa->u.cli.lli_list, next ?: &lli->lli_oss_capas);
442 }
443
444 static struct obd_capa *do_add_oss_capa(struct inode *inode,
445                                         struct obd_capa *ocapa)
446 {
447         struct obd_capa *old;
448         struct lustre_capa *capa = &ocapa->c_capa;
449
450         LASSERTF(S_ISREG(inode->i_mode),
451                  "inode has oss capa, but not regular file, mode: %d\n",
452                  inode->i_mode);
453
454         /* FIXME: can't replace it so easily with fine-grained opc */
455         old = do_lookup_oss_capa(inode, capa_uid(capa),
456                                  capa_opc(capa) & CAPA_OPC_OSS_ONLY);
457         if (!old) {
458                 ocapa->u.cli.inode = inode;
459                 INIT_LIST_HEAD(&ocapa->u.cli.lli_list);
460                 capa_count[CAPA_SITE_CLIENT]++;
461
462                 DEBUG_CAPA(D_SEC, capa, "add OSS");
463         } else {
464                 spin_lock(&old->c_lock);
465                 old->c_capa = *capa;
466                 spin_unlock(&old->c_lock);
467
468                 DEBUG_CAPA(D_SEC, capa, "update OSS");
469
470                 free_capa(ocapa);
471                 ocapa = old;
472         }
473
474         inode_add_oss_capa(inode, ocapa);
475         return ocapa;
476 }
477
478 struct obd_capa *ll_add_capa(struct inode *inode, struct obd_capa *ocapa)
479 {
480         spin_lock(&capa_lock);
481         ocapa = capa_for_mds(&ocapa->c_capa) ? do_add_mds_capa(inode, ocapa) :
482                                                do_add_oss_capa(inode, ocapa);
483
484         /* truncate capa won't renew */
485         if (ocapa->c_capa.lc_opc != CAPA_OPC_OSS_TRUNC) {
486                 set_capa_expiry(ocapa);
487                 list_del(&ocapa->c_list);
488                 sort_add_capa(ocapa, ll_capa_list);
489
490                 update_capa_timer(ocapa, capa_renewal_time(ocapa));
491         }
492
493         spin_unlock(&capa_lock);
494
495         atomic_set(&ll_capa_debug, 1);
496         return ocapa;
497 }
498
499 static inline void delay_capa_renew(struct obd_capa *oc, cfs_time_t delay)
500 {
501         /* NB: set a fake expiry for this capa to prevent it renew too soon */
502         oc->c_expiry = cfs_time_add(oc->c_expiry, cfs_time_seconds(delay));
503 }
504
505 int ll_update_capa(struct obd_capa *ocapa, struct lustre_capa *capa)
506 {
507         struct inode *inode = ocapa->u.cli.inode;
508         int rc = 0;
509         ENTRY;
510
511         LASSERT(ocapa);
512
513         if (IS_ERR(capa)) {
514                 /* set error code */
515                 rc = PTR_ERR(capa);
516                 spin_lock(&capa_lock);
517                 if (rc == -ENOENT) {
518                         DEBUG_CAPA(D_SEC, &ocapa->c_capa,
519                                    "renewal canceled because object removed");
520                         ll_capa_renewal_noent++;
521                 } else {
522                         ll_capa_renewal_failed++;
523
524                         /* failed capa won't be renewed any longer, but if -EIO,
525                          * client might be doing recovery, retry in 2 min. */
526                         if (rc == -EIO && !capa_is_expired(ocapa)) {
527                                 delay_capa_renew(ocapa, 120);
528                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
529                                            "renewal failed: -EIO, retry in 2 mins");
530                                 ll_capa_renewal_retries++;
531                                 GOTO(retry, rc);
532                         } else {
533                                 DEBUG_CAPA(D_ERROR, &ocapa->c_capa,
534                                            "renewal failed(rc: %d) for", rc);
535                         }
536                 }
537
538                 list_del(&ocapa->c_list);
539                 sort_add_capa(ocapa, &ll_idle_capas);
540                 spin_unlock(&capa_lock);
541
542                 capa_put(ocapa);
543                 iput(inode);
544                 return rc;
545         }
546
547         spin_lock(&ocapa->c_lock);
548         LASSERT(!memcmp(&ocapa->c_capa, capa,
549                         offsetof(struct lustre_capa, lc_flags)));
550         ocapa->c_capa = *capa;
551         set_capa_expiry(ocapa);
552         spin_unlock(&ocapa->c_lock);
553
554         spin_lock(&capa_lock);
555         if (capa_for_oss(capa))
556                 inode_add_oss_capa(inode, ocapa);
557         DEBUG_CAPA(D_SEC, capa, "renew");
558         EXIT;
559 retry:
560         list_del_init(&ocapa->c_list);
561         sort_add_capa(ocapa, ll_capa_list);
562         update_capa_timer(ocapa, capa_renewal_time(ocapa));
563         spin_unlock(&capa_lock);
564
565         capa_put(ocapa);
566         iput(inode);
567         return rc;
568 }
569
570 void ll_capa_open(struct inode *inode)
571 {
572         struct ll_inode_info *lli = ll_i2info(inode);
573
574         if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
575             == 0)
576                 return;
577
578         if (!S_ISREG(inode->i_mode))
579                 return;
580
581         atomic_inc(&lli->lli_open_count);
582 }
583
584 void ll_capa_close(struct inode *inode)
585 {
586         struct ll_inode_info *lli = ll_i2info(inode);
587
588         if ((ll_i2sbi(inode)->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
589             == 0)
590                 return;
591
592         if (!S_ISREG(inode->i_mode))
593                 return;
594
595         atomic_dec(&lli->lli_open_count);
596 }
597
598 /* delete CAPA_OPC_OSS_TRUNC only */
599 void ll_truncate_free_capa(struct obd_capa *ocapa)
600 {
601         if (!ocapa)
602                 return;
603
604         LASSERT(ocapa->c_capa.lc_opc & CAPA_OPC_OSS_TRUNC);
605         DEBUG_CAPA(D_SEC, &ocapa->c_capa, "free truncate");
606
607         capa_put(ocapa);
608         spin_lock(&capa_lock);
609         ll_delete_capa(ocapa);
610         spin_unlock(&capa_lock);
611 }
612
613 void ll_clear_inode_capas(struct inode *inode)
614 {
615         struct ll_inode_info *lli = ll_i2info(inode);
616         struct obd_capa *ocapa, *tmp;
617
618         spin_lock(&capa_lock);
619         ocapa = lli->lli_mds_capa;
620         if (ocapa)
621                 ll_delete_capa(ocapa);
622                 
623         list_for_each_entry_safe(ocapa, tmp, &lli->lli_oss_capas,
624                                  u.cli.lli_list)
625                 ll_delete_capa(ocapa);
626         spin_unlock(&capa_lock);
627 }
628
629 void ll_print_capa_stat(struct ll_sb_info *sbi)
630 {
631         if (sbi->ll_flags & (LL_SBI_MDS_CAPA | LL_SBI_OSS_CAPA))
632                 LCONSOLE_INFO("Fid capabilities renewed: %llu\n"
633                               "Fid capabilities renewal ENOENT: %llu\n"
634                               "Fid capabilities failed to renew: %llu\n"
635                               "Fid capabilities renewal retries: %llu\n",
636                               ll_capa_renewed, ll_capa_renewal_noent,
637                               ll_capa_renewal_failed, ll_capa_renewal_retries);
638 }