4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; If not, see
18 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
20 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
21 * CA 95054 USA or visit www.sun.com if you need additional information or
27 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
28 * Use is subject to license terms.
31 * This file is part of Lustre, http://www.lustre.org/
32 * Lustre is a trademark of Sun Microsystems, Inc.
34 * lustre/obdclass/llog_cat.c
36 * OST<->MDS recovery logging infrastructure.
38 * Invariants in implementation:
39 * - we do not share logs among different OST<->MDS connections, so that
40 * if an OST or MDS fails it need only look at log(s) relevant to itself
42 * Author: Andreas Dilger <adilger@clusterfs.com>
45 #define DEBUG_SUBSYSTEM S_LOG
48 #include <liblustre.h>
51 #include <obd_class.h>
52 #include <lustre_log.h>
54 #include "llog_internal.h"
56 /* Create a new log handle and add it to the open list.
57 * This log handle will be closed when all of the records in it are removed.
59 * Assumes caller has already pushed us into the kernel context and is locking.
61 static struct llog_handle *llog_cat_new_log(const struct lu_env *env,
62 struct llog_handle *cathandle)
64 struct llog_handle *loghandle;
65 struct llog_log_hdr *llh;
66 struct llog_logid_rec rec = { { 0 }, };
67 int rc, index, bitmap_size;
70 llh = cathandle->lgh_hdr;
71 bitmap_size = LLOG_BITMAP_SIZE(llh);
73 index = (cathandle->lgh_last_idx + 1) % bitmap_size;
75 /* maximum number of available slots in catlog is bitmap_size - 2 */
76 if (llh->llh_cat_idx == index) {
77 CERROR("no free catalog slots for log...\n");
78 RETURN(ERR_PTR(-ENOSPC));
81 if (OBD_FAIL_CHECK(OBD_FAIL_MDS_LLOG_CREATE_FAILED))
82 RETURN(ERR_PTR(-ENOSPC));
84 rc = llog_create(env, cathandle->lgh_ctxt, &loghandle, NULL, NULL);
88 rc = llog_init_handle(env, loghandle,
89 LLOG_F_IS_PLAIN | LLOG_F_ZAP_WHEN_EMPTY,
90 &cathandle->lgh_hdr->llh_tgtuuid);
92 GOTO(out_destroy, rc);
97 cfs_spin_lock(&loghandle->lgh_hdr_lock);
99 if (ext2_set_bit(index, llh->llh_bitmap)) {
100 CERROR("argh, index %u already set in log bitmap?\n",
102 cfs_spin_unlock(&loghandle->lgh_hdr_lock);
103 LBUG(); /* should never happen */
105 cfs_spin_unlock(&loghandle->lgh_hdr_lock);
107 cathandle->lgh_last_idx = index;
108 llh->llh_tail.lrt_index = index;
110 CDEBUG(D_RPCTRACE,"new recovery log "LPX64":%x for index %u of catalog "
111 LPX64"\n", loghandle->lgh_id.lgl_oid, loghandle->lgh_id.lgl_ogen,
112 index, cathandle->lgh_id.lgl_oid);
113 /* build the record for this log in the catalog */
114 rec.lid_hdr.lrh_len = sizeof(rec);
115 rec.lid_hdr.lrh_index = index;
116 rec.lid_hdr.lrh_type = LLOG_LOGID_MAGIC;
117 rec.lid_id = loghandle->lgh_id;
118 rec.lid_tail.lrt_len = sizeof(rec);
119 rec.lid_tail.lrt_index = index;
121 /* update the catalog: header and record */
122 rc = llog_write_rec(env, cathandle, &rec.lid_hdr,
123 &loghandle->u.phd.phd_cookie, 1, NULL, index);
125 GOTO(out_destroy, rc);
128 loghandle->lgh_hdr->llh_cat_idx = index;
129 cathandle->u.chd.chd_current_log = loghandle;
130 LASSERT(cfs_list_empty(&loghandle->u.phd.phd_entry));
131 cfs_list_add_tail(&loghandle->u.phd.phd_entry,
132 &cathandle->u.chd.chd_head);
136 llog_destroy(env, loghandle);
141 /* Open an existent log handle and add it to the open list.
142 * This log handle will be closed when all of the records in it are removed.
144 * Assumes caller has already pushed us into the kernel context and is locking.
145 * We return a lock on the handle to ensure nobody yanks it from us.
147 int llog_cat_id2handle(const struct lu_env *env, struct llog_handle *cathandle,
148 struct llog_handle **res, struct llog_logid *logid)
150 struct llog_handle *loghandle;
154 if (cathandle == NULL)
157 cfs_list_for_each_entry(loghandle, &cathandle->u.chd.chd_head,
159 struct llog_logid *cgl = &loghandle->lgh_id;
161 if (cgl->lgl_oid == logid->lgl_oid) {
162 if (cgl->lgl_ogen != logid->lgl_ogen) {
163 CERROR("log "LPX64" generation %x != %x\n",
164 logid->lgl_oid, cgl->lgl_ogen,
168 loghandle->u.phd.phd_cat_handle = cathandle;
173 rc = llog_create(env, cathandle->lgh_ctxt, &loghandle, logid, NULL);
175 CERROR("error opening log id "LPX64":%x: rc %d\n",
176 logid->lgl_oid, logid->lgl_ogen, rc);
178 rc = llog_init_handle(env, loghandle, LLOG_F_IS_PLAIN, NULL);
180 cfs_list_add(&loghandle->u.phd.phd_entry,
181 &cathandle->u.chd.chd_head);
185 loghandle->u.phd.phd_cat_handle = cathandle;
186 loghandle->u.phd.phd_cookie.lgc_lgl = cathandle->lgh_id;
187 loghandle->u.phd.phd_cookie.lgc_index =
188 loghandle->lgh_hdr->llh_cat_idx;
196 int llog_cat_put(const struct lu_env *env, struct llog_handle *cathandle)
198 struct llog_handle *loghandle, *n;
203 cfs_list_for_each_entry_safe(loghandle, n, &cathandle->u.chd.chd_head,
205 int err = llog_close(env, loghandle);
207 CERROR("error closing loghandle\n");
209 rc = llog_close(env, cathandle);
212 EXPORT_SYMBOL(llog_cat_put);
215 * lockdep markers for nested struct llog_handle::lgh_lock locking.
222 /** Return the currently active log handle. If the current log handle doesn't
223 * have enough space left for the current record, start a new one.
225 * If reclen is 0, we only want to know what the currently active log is,
226 * otherwise we get a lock on this log so nobody can steal our space.
228 * Assumes caller has already pushed us into the kernel context and is locking.
230 * NOTE: loghandle is write-locked upon successful return
232 static struct llog_handle *llog_cat_current_log(const struct lu_env *env,
233 struct llog_handle *cathandle,
236 struct llog_handle *loghandle = NULL;
239 cfs_down_read_nested(&cathandle->lgh_lock, LLOGH_CAT);
240 loghandle = cathandle->u.chd.chd_current_log;
242 struct llog_log_hdr *llh = loghandle->lgh_hdr;
244 cfs_down_write_nested(&loghandle->lgh_lock, LLOGH_LOG);
245 if (loghandle->lgh_last_idx < LLOG_BITMAP_SIZE(llh) - 1) {
246 cfs_up_read(&cathandle->lgh_lock);
249 cfs_up_write(&loghandle->lgh_lock);
254 cfs_down_write(&loghandle->lgh_lock);
255 cfs_up_read(&cathandle->lgh_lock);
258 cfs_up_read(&cathandle->lgh_lock);
260 /* time to create new log */
262 /* first, we have to make sure the state hasn't changed */
263 cfs_down_write_nested(&cathandle->lgh_lock, LLOGH_CAT);
264 loghandle = cathandle->u.chd.chd_current_log;
266 struct llog_log_hdr *llh = loghandle->lgh_hdr;
268 cfs_down_write_nested(&loghandle->lgh_lock, LLOGH_LOG);
269 if (loghandle->lgh_last_idx < LLOG_BITMAP_SIZE(llh) - 1) {
270 cfs_up_write(&cathandle->lgh_lock);
273 cfs_up_write(&loghandle->lgh_lock);
277 CDEBUG(D_INODE, "creating new log\n");
278 loghandle = llog_cat_new_log(env, cathandle);
279 if (!IS_ERR(loghandle))
280 cfs_down_write_nested(&loghandle->lgh_lock, LLOGH_LOG);
281 cfs_up_write(&cathandle->lgh_lock);
285 /* Add a single record to the recovery log(s) using a catalog
286 * Returns as llog_write_record
288 * Assumes caller has already pushed us into the kernel context.
290 int llog_cat_add_rec(const struct lu_env *env, struct llog_handle *cathandle,
291 struct llog_rec_hdr *rec, struct llog_cookie *reccookie,
294 struct llog_handle *loghandle;
298 LASSERT(rec->lrh_len <= LLOG_CHUNK_SIZE);
299 loghandle = llog_cat_current_log(env, cathandle, 1);
300 if (IS_ERR(loghandle))
301 RETURN(PTR_ERR(loghandle));
302 /* loghandle is already locked by llog_cat_current_log() for us */
303 rc = llog_write_rec(env, loghandle, rec, reccookie, 1, buf, -1);
305 CERROR("llog_write_rec %d: lh=%p\n", rc, loghandle);
306 cfs_up_write(&loghandle->lgh_lock);
308 /* to create a new plain log */
309 loghandle = llog_cat_current_log(env, cathandle, 1);
310 if (IS_ERR(loghandle))
311 RETURN(PTR_ERR(loghandle));
312 rc = llog_write_rec(env, loghandle, rec, reccookie, 1, buf,
314 cfs_up_write(&loghandle->lgh_lock);
319 EXPORT_SYMBOL(llog_cat_add_rec);
321 /* For each cookie in the cookie array, we clear the log in-use bit and either:
322 * - the log is empty, so mark it free in the catalog header and delete it
323 * - the log is not empty, just write out the log header
325 * The cookies may be in different log files, so we need to get new logs
328 * Assumes caller has already pushed us into the kernel context.
330 int llog_cat_cancel_records(const struct lu_env *env,
331 struct llog_handle *cathandle, int count,
332 struct llog_cookie *cookies)
334 int i, index, rc = 0;
337 cfs_down_write_nested(&cathandle->lgh_lock, LLOGH_CAT);
338 for (i = 0; i < count; i++, cookies++) {
339 struct llog_handle *loghandle;
340 struct llog_logid *lgl = &cookies->lgc_lgl;
342 rc = llog_cat_id2handle(env, cathandle, &loghandle, lgl);
344 CERROR("Cannot find log "LPX64"\n", lgl->lgl_oid);
348 cfs_down_write_nested(&loghandle->lgh_lock, LLOGH_LOG);
349 rc = llog_cancel_rec(env, loghandle, cookies->lgc_index);
350 cfs_up_write(&loghandle->lgh_lock);
352 if (rc == 1) { /* log has been destroyed */
353 index = loghandle->u.phd.phd_cookie.lgc_index;
354 if (cathandle->u.chd.chd_current_log == loghandle)
355 cathandle->u.chd.chd_current_log = NULL;
356 llog_free_handle(loghandle);
359 llog_cat_set_first_idx(cathandle, index);
360 rc = llog_cancel_rec(env, cathandle, index);
362 CDEBUG(D_RPCTRACE,"cancel plain log at index %u"
363 " of catalog "LPX64"\n",
364 index, cathandle->lgh_id.lgl_oid);
367 cfs_up_write(&cathandle->lgh_lock);
371 EXPORT_SYMBOL(llog_cat_cancel_records);
373 int llog_cat_process_cb(const struct lu_env *env, struct llog_handle *cat_llh,
374 struct llog_rec_hdr *rec, void *data)
376 struct llog_process_data *d = data;
377 struct llog_logid_rec *lir = (struct llog_logid_rec *)rec;
378 struct llog_handle *llh;
382 if (rec->lrh_type != LLOG_LOGID_MAGIC) {
383 CERROR("invalid record in catalog\n");
386 CDEBUG(D_HA, "processing log "LPX64":%x at index %u of catalog "
387 LPX64"\n", lir->lid_id.lgl_oid, lir->lid_id.lgl_ogen,
388 rec->lrh_index, cat_llh->lgh_id.lgl_oid);
390 rc = llog_cat_id2handle(env, cat_llh, &llh, &lir->lid_id);
392 CERROR("Cannot find handle for log "LPX64"\n",
393 lir->lid_id.lgl_oid);
397 if (rec->lrh_index < d->lpd_startcat)
398 /* Skip processing of the logs until startcat */
401 if (d->lpd_startidx > 0) {
402 struct llog_process_cat_data cd;
404 cd.lpcd_first_idx = d->lpd_startidx;
405 cd.lpcd_last_idx = 0;
406 rc = llog_process(env, llh, d->lpd_cb, d->lpd_data, &cd);
407 /* Continue processing the next log from idx 0 */
410 rc = llog_process(env, llh, d->lpd_cb, d->lpd_data, NULL);
416 int llog_cat_process_or_fork(const struct lu_env *env,
417 struct llog_handle *cat_llh,
418 llog_cb_t cb, void *data, int startcat,
419 int startidx, bool fork)
421 struct llog_process_data d;
422 struct llog_log_hdr *llh = cat_llh->lgh_hdr;
426 LASSERT(llh->llh_flags & LLOG_F_IS_CAT);
429 d.lpd_startcat = startcat;
430 d.lpd_startidx = startidx;
432 if (llh->llh_cat_idx > cat_llh->lgh_last_idx) {
433 struct llog_process_cat_data cd;
435 CWARN("catlog "LPX64" crosses index zero\n",
436 cat_llh->lgh_id.lgl_oid);
438 cd.lpcd_first_idx = llh->llh_cat_idx;
439 cd.lpcd_last_idx = 0;
440 rc = llog_process_or_fork(env, cat_llh, llog_cat_process_cb,
445 cd.lpcd_first_idx = 0;
446 cd.lpcd_last_idx = cat_llh->lgh_last_idx;
447 rc = llog_process_or_fork(env, cat_llh, llog_cat_process_cb,
450 rc = llog_process_or_fork(env, cat_llh, llog_cat_process_cb,
456 EXPORT_SYMBOL(llog_cat_process_or_fork);
458 int llog_cat_process(const struct lu_env *env, struct llog_handle *cat_llh,
459 llog_cb_t cb, void *data, int startcat, int startidx)
461 return llog_cat_process_or_fork(env, cat_llh, cb, data, startcat,
464 EXPORT_SYMBOL(llog_cat_process);
467 int llog_cat_process_thread(void *data)
469 struct llog_process_cat_args *args = data;
470 struct llog_ctxt *ctxt = args->lpca_ctxt;
471 struct llog_handle *llh = NULL;
472 llog_cb_t cb = args->lpca_cb;
473 struct llog_thread_info *lgi;
478 cfs_daemonize_ctxt("ll_log_process");
480 rc = lu_env_init(&env, LCT_LOCAL);
483 lgi = llog_info(&env);
486 lgi->lgi_logid = *(struct llog_logid *)(args->lpca_arg);
487 rc = llog_create(&env, ctxt, &llh, &lgi->lgi_logid, NULL);
489 CERROR("llog_create() failed %d\n", rc);
492 rc = llog_init_handle(&env, llh, LLOG_F_IS_CAT, NULL);
494 CERROR("llog_init_handle failed %d\n", rc);
495 GOTO(release_llh, rc);
499 rc = llog_cat_process(&env, llh, cb, NULL, 0, 0);
500 if (rc != LLOG_PROC_BREAK && rc != 0)
501 CERROR("llog_cat_process() failed %d\n", rc);
502 cb(&env, llh, NULL, NULL);
504 CWARN("No callback function for recovery\n");
508 * Make sure that all cached data is sent.
510 llog_sync(ctxt, NULL, 0);
511 GOTO(release_llh, rc);
513 rc = llog_cat_put(&env, llh);
515 CERROR("llog_cat_put() failed %d\n", rc);
523 EXPORT_SYMBOL(llog_cat_process_thread);
526 static int llog_cat_reverse_process_cb(const struct lu_env *env,
527 struct llog_handle *cat_llh,
528 struct llog_rec_hdr *rec, void *data)
530 struct llog_process_data *d = data;
531 struct llog_logid_rec *lir = (struct llog_logid_rec *)rec;
532 struct llog_handle *llh;
535 if (le32_to_cpu(rec->lrh_type) != LLOG_LOGID_MAGIC) {
536 CERROR("invalid record in catalog\n");
539 CDEBUG(D_HA, "processing log "LPX64":%x at index %u of catalog "
540 LPX64"\n", lir->lid_id.lgl_oid, lir->lid_id.lgl_ogen,
541 le32_to_cpu(rec->lrh_index), cat_llh->lgh_id.lgl_oid);
543 rc = llog_cat_id2handle(env, cat_llh, &llh, &lir->lid_id);
545 CERROR("Cannot find handle for log "LPX64"\n",
546 lir->lid_id.lgl_oid);
550 rc = llog_reverse_process(env, llh, d->lpd_cb, d->lpd_data, NULL);
554 int llog_cat_reverse_process(const struct lu_env *env,
555 struct llog_handle *cat_llh,
556 llog_cb_t cb, void *data)
558 struct llog_process_data d;
559 struct llog_process_cat_data cd;
560 struct llog_log_hdr *llh = cat_llh->lgh_hdr;
564 LASSERT(llh->llh_flags & LLOG_F_IS_CAT);
568 if (llh->llh_cat_idx > cat_llh->lgh_last_idx) {
569 CWARN("catalog "LPX64" crosses index zero\n",
570 cat_llh->lgh_id.lgl_oid);
572 cd.lpcd_first_idx = 0;
573 cd.lpcd_last_idx = cat_llh->lgh_last_idx;
574 rc = llog_reverse_process(env, cat_llh,
575 llog_cat_reverse_process_cb,
580 cd.lpcd_first_idx = le32_to_cpu(llh->llh_cat_idx);
581 cd.lpcd_last_idx = 0;
582 rc = llog_reverse_process(env, cat_llh,
583 llog_cat_reverse_process_cb,
586 rc = llog_reverse_process(env, cat_llh,
587 llog_cat_reverse_process_cb,
593 EXPORT_SYMBOL(llog_cat_reverse_process);
595 int llog_cat_set_first_idx(struct llog_handle *cathandle, int index)
597 struct llog_log_hdr *llh = cathandle->lgh_hdr;
598 int i, bitmap_size, idx;
601 bitmap_size = LLOG_BITMAP_SIZE(llh);
602 if (llh->llh_cat_idx == (index - 1)) {
603 idx = llh->llh_cat_idx + 1;
604 llh->llh_cat_idx = idx;
605 if (idx == cathandle->lgh_last_idx)
607 for (i = (index + 1) % bitmap_size;
608 i != cathandle->lgh_last_idx;
609 i = (i + 1) % bitmap_size) {
610 if (!ext2_test_bit(i, llh->llh_bitmap)) {
611 idx = llh->llh_cat_idx + 1;
612 llh->llh_cat_idx = idx;
614 llh->llh_cat_idx = 0;
620 CDEBUG(D_RPCTRACE, "set catlog "LPX64" first idx %u\n",
621 cathandle->lgh_id.lgl_oid, llh->llh_cat_idx);
627 /* callback func for llog_process in llog_obd_origin_setup */
628 int cat_cancel_cb(const struct lu_env *env, struct llog_handle *cathandle,
629 struct llog_rec_hdr *rec, void *data)
631 struct llog_logid_rec *lir = (struct llog_logid_rec *)rec;
632 struct llog_handle *loghandle;
633 struct llog_log_hdr *llh;
638 if (rec->lrh_type != LLOG_LOGID_MAGIC) {
639 CERROR("%s: invalid record in catalog\n",
640 loghandle->lgh_ctxt->loc_obd->obd_name);
643 CDEBUG(D_HA, "processing log "LPX64":%x at index %u of catalog "
644 LPX64"\n", lir->lid_id.lgl_oid, lir->lid_id.lgl_ogen,
645 rec->lrh_index, cathandle->lgh_id.lgl_oid);
647 rc = llog_cat_id2handle(env, cathandle, &loghandle, &lir->lid_id);
649 CERROR("%s: cannot find handle for llog "LPX64"\n",
650 loghandle->lgh_ctxt->loc_obd->obd_name,
651 lir->lid_id.lgl_oid);
653 index = rec->lrh_index;
659 llh = loghandle->lgh_hdr;
660 if ((llh->llh_flags & LLOG_F_ZAP_WHEN_EMPTY) &&
661 (llh->llh_count == 1)) {
662 rc = llog_destroy(env, loghandle);
664 CERROR("%s: fail to destroy empty log: rc = %d\n",
665 loghandle->lgh_ctxt->loc_obd->obd_name, rc);
667 index = loghandle->u.phd.phd_cookie.lgc_index;
668 llog_free_handle(loghandle);
672 llog_cat_set_first_idx(cathandle, index);
673 rc = llog_cancel_rec(env, cathandle, index);
676 "cancel log "LPX64":%x at index %u of catalog "
677 LPX64"\n", lir->lid_id.lgl_oid,
678 lir->lid_id.lgl_ogen, rec->lrh_index,
679 cathandle->lgh_id.lgl_oid);