1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2001, 2002 Cluster File Systems, Inc.
5 * Author: Peter J. Braam <braam@clusterfs.com>
6 * Author: Phil Schwan <phil@clusterfs.com>
8 * This file is part of Lustre, http://www.lustre.org.
10 * Lustre is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Lustre is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with Lustre; if not, write to the Free Software
21 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 * Storage Target Handling functions
24 * Lustre Object Server Module (OST)
26 * This server is single threaded at present (but can easily be multi
27 * threaded). For testing and management it is treated as an
28 * obd_device, although it does not export a full OBD method table
29 * (the requests are coming in over the wire, so object target
30 * modules do not have a full method table.)
35 #include <linux/version.h>
36 #include <linux/module.h>
38 #include <linux/stat.h>
39 #include <linux/locks.h>
40 #include <linux/ext2_fs.h>
41 #include <linux/quotaops.h>
42 #include <asm/unistd.h>
44 #define DEBUG_SUBSYSTEM S_OST
46 #include <linux/obd_support.h>
47 #include <linux/obd.h>
48 #include <linux/obd_class.h>
49 #include <linux/lustre_lib.h>
50 #include <linux/lustre_idl.h>
51 #include <linux/lustre_mds.h>
52 #include <linux/obd_class.h>
56 static int ost_destroy(struct ost_obd *ost, struct ptlrpc_request *req)
63 conn.oc_id = req->rq_req.ost->connid;
64 conn.oc_dev = ost->ost_tgt;
66 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
67 &req->rq_replen, &req->rq_repbuf);
69 CERROR("cannot pack reply\n");
73 req->rq_rep.ost->result = obd_destroy(&conn, &req->rq_req.ost->oa);
79 static int ost_getattr(struct ost_obd *ost, struct ptlrpc_request *req)
86 conn.oc_id = req->rq_req.ost->connid;
87 conn.oc_dev = ost->ost_tgt;
89 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
90 &req->rq_replen, &req->rq_repbuf);
92 CERROR("cannot pack reply\n");
95 req->rq_rep.ost->oa.o_id = req->rq_req.ost->oa.o_id;
96 req->rq_rep.ost->oa.o_valid = req->rq_req.ost->oa.o_valid;
98 req->rq_rep.ost->result = obd_getattr(&conn, &req->rq_rep.ost->oa);
104 static int ost_create(struct ost_obd *ost, struct ptlrpc_request *req)
106 struct obd_conn conn;
111 conn.oc_id = req->rq_req.ost->connid;
112 conn.oc_dev = ost->ost_tgt;
114 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
115 &req->rq_replen, &req->rq_repbuf);
117 CERROR("cannot pack reply\n");
121 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
122 sizeof(req->rq_req.ost->oa));
124 req->rq_rep.ost->result =obd_create(&conn, &req->rq_rep.ost->oa);
130 static int ost_punch(struct ost_obd *ost, struct ptlrpc_request *req)
132 struct obd_conn conn;
137 conn.oc_id = req->rq_req.ost->connid;
138 conn.oc_dev = ost->ost_tgt;
140 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
141 &req->rq_replen, &req->rq_repbuf);
143 CERROR("cannot pack reply\n");
147 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
148 sizeof(req->rq_req.ost->oa));
150 req->rq_rep.ost->result = obd_punch(&conn, &req->rq_rep.ost->oa,
151 req->rq_rep.ost->oa.o_size,
152 req->rq_rep.ost->oa.o_blocks);
159 static int ost_setattr(struct ost_obd *ost, struct ptlrpc_request *req)
161 struct obd_conn conn;
166 conn.oc_id = req->rq_req.ost->connid;
167 conn.oc_dev = ost->ost_tgt;
169 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
170 &req->rq_replen, &req->rq_repbuf);
172 CERROR("cannot pack reply\n");
176 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
177 sizeof(req->rq_req.ost->oa));
179 req->rq_rep.ost->result = obd_setattr(&conn, &req->rq_rep.ost->oa);
185 static int ost_connect(struct ost_obd *ost, struct ptlrpc_request *req)
187 struct obd_conn conn;
192 conn.oc_dev = ost->ost_tgt;
194 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
195 &req->rq_replen, &req->rq_repbuf);
197 CERROR("cannot pack reply\n");
201 req->rq_rep.ost->result = obd_connect(&conn);
203 CDEBUG(D_IOCTL, "rep buffer %p, id %d\n", req->rq_repbuf, conn.oc_id);
204 req->rq_rep.ost->connid = conn.oc_id;
209 static int ost_disconnect(struct ost_obd *ost, struct ptlrpc_request *req)
211 struct obd_conn conn;
216 conn.oc_dev = ost->ost_tgt;
217 conn.oc_id = req->rq_req.ost->connid;
219 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
220 &req->rq_replen, &req->rq_repbuf);
222 CERROR("cannot pack reply\n");
225 CDEBUG(D_IOCTL, "Disconnecting %d\n", conn.oc_id);
226 req->rq_rep.ost->result = obd_disconnect(&conn);
232 static int ost_get_info(struct ost_obd *ost, struct ptlrpc_request *req)
234 struct obd_conn conn;
242 conn.oc_id = req->rq_req.ost->connid;
243 conn.oc_dev = ost->ost_tgt;
245 ptr = ost_req_buf1(req->rq_req.ost);
246 req->rq_rep.ost->result = obd_get_info(&conn,
247 req->rq_req.ost->buflen1, ptr,
250 rc = ost_pack_rep(val, vallen, NULL, 0, &req->rq_rephdr,
251 &req->rq_rep, &req->rq_replen, &req->rq_repbuf);
253 CERROR("cannot pack reply\n");
261 static int ost_brw_read(struct ost_obd *obddev, struct ptlrpc_request *req)
263 struct ptlrpc_bulk_desc **bulk_vec = NULL;
264 struct ptlrpc_bulk_desc *bulk = NULL;
265 struct obd_conn conn;
268 int objcount, niocount;
269 char *tmp1, *tmp2, *end2;
272 struct niobuf *nb, *src;
273 struct obd_ioobj *ioo;
274 struct ost_req *r = req->rq_req.ost;
278 tmp1 = ost_req_buf1(r);
279 tmp2 = ost_req_buf2(r);
280 end2 = tmp2 + req->rq_req.ost->buflen2;
281 objcount = r->buflen1 / sizeof(*ioo);
282 niocount = r->buflen2 / sizeof(*nb);
285 conn.oc_id = req->rq_req.ost->connid;
286 conn.oc_dev = req->rq_obd->u.ost.ost_tgt;
288 for (i = 0; i < objcount; i++) {
289 ost_unpack_ioo((void *)&tmp1, &ioo);
290 if (tmp2 + ioo->ioo_bufcnt > end2) {
294 for (j = 0; j < ioo->ioo_bufcnt; j++) {
295 ost_unpack_niobuf((void *)&tmp2, &nb);
299 rc = ost_pack_rep(NULL, 0, NULL, 0,
300 &req->rq_rephdr, &req->rq_rep,
301 &req->rq_replen, &req->rq_repbuf);
303 CERROR("cannot pack reply\n");
306 OBD_ALLOC(res, sizeof(struct niobuf) * niocount);
312 /* The unpackers move tmp1 and tmp2, so reset them before using */
313 tmp1 = ost_req_buf1(r);
314 tmp2 = ost_req_buf2(r);
315 req->rq_rep.ost->result = obd_preprw
316 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
317 niocount, (struct niobuf *)tmp2, (struct niobuf *)res);
319 if (req->rq_rep.ost->result) {
324 for (i = 0; i < niocount; i++) {
325 bulk = ptlrpc_prep_bulk(&req->rq_peer);
327 CERROR("cannot alloc bulk desc\n");
332 src = &((struct niobuf *)tmp2)[i];
334 bulk->b_xid = src->xid;
335 bulk->b_buf = (void *)(unsigned long)src->addr;
336 bulk->b_buflen = PAGE_SIZE;
337 rc = ptlrpc_send_bulk(bulk, OST_BULK_PORTAL);
342 wait_event_interruptible(bulk->b_waitq,
343 ptlrpc_check_bulk_sent(bulk));
345 if (bulk->b_flags == PTL_RPC_INTR) {
350 OBD_FREE(bulk, sizeof(*bulk));
356 dst = &((struct niobuf *)tmp2)[i];
357 memcpy((void *)(unsigned long)dst->addr,
358 (void *)(unsigned long)src->addr, PAGE_SIZE);
364 OBD_FREE(res, sizeof(struct niobuf) * niocount);
366 OBD_FREE(bulk, sizeof(*bulk));
367 if (bulk_vec != NULL) {
368 for (i = 0; i < niocount; i++) {
369 if (bulk_vec[i] != NULL)
370 OBD_FREE(bulk_vec[i], sizeof(*bulk));
373 niocount * sizeof(struct ptlrpc_bulk_desc *));
380 int ost_brw_write(struct ost_obd *obddev, struct ptlrpc_request *req)
382 struct ptlrpc_bulk_desc **bulk_vec = NULL;
383 struct ptlrpc_bulk_desc *bulk = NULL;
384 struct obd_conn conn;
387 int objcount, niocount;
388 char *tmp1, *tmp2, *end2;
391 struct niobuf *nb, *dst;
392 struct obd_ioobj *ioo;
393 struct ost_req *r = req->rq_req.ost;
397 tmp1 = ost_req_buf1(r);
398 tmp2 = ost_req_buf2(r);
399 end2 = tmp2 + req->rq_req.ost->buflen2;
400 objcount = r->buflen1 / sizeof(*ioo);
401 niocount = r->buflen2 / sizeof(*nb);
404 conn.oc_id = req->rq_req.ost->connid;
405 conn.oc_dev = req->rq_obd->u.ost.ost_tgt;
407 for (i = 0; i < objcount; i++) {
408 ost_unpack_ioo((void *)&tmp1, &ioo);
409 if (tmp2 + ioo->ioo_bufcnt > end2) {
413 for (j = 0; j < ioo->ioo_bufcnt; j++) {
414 ost_unpack_niobuf((void *)&tmp2, &nb);
418 rc = ost_pack_rep(NULL, 0, NULL, niocount * sizeof(*nb),
419 &req->rq_rephdr, &req->rq_rep,
420 &req->rq_replen, &req->rq_repbuf);
422 CERROR("cannot pack reply\n");
425 res = ost_rep_buf2(req->rq_rep.ost);
427 /* The unpackers move tmp1 and tmp2, so reset them before using */
428 tmp1 = ost_req_buf1(r);
429 tmp2 = ost_req_buf2(r);
430 req->rq_rep.ost->result = obd_preprw
431 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
432 niocount, (struct niobuf *)tmp2, (struct niobuf *)res);
434 if (req->rq_rep.ost->result) {
439 /* Setup buffers for the incoming pages, then send the niobufs
440 * describing those buffers to the OSC. */
441 OBD_ALLOC(bulk_vec, niocount * sizeof(struct ptlrpc_bulk_desc *));
442 if (bulk_vec == NULL) {
443 CERROR("cannot alloc bulk desc vector\n");
446 memset(bulk_vec, 0, niocount * sizeof(struct ptlrpc_bulk_desc *));
448 for (i = 0; i < niocount; i++) {
449 struct ptlrpc_service *srv = req->rq_obd->u.ost.ost_service;
451 bulk_vec[i] = ptlrpc_prep_bulk(&req->rq_peer);
452 if (bulk_vec[i] == NULL) {
453 CERROR("cannot alloc bulk desc\n");
458 spin_lock(&srv->srv_lock);
459 bulk_vec[i]->b_xid = srv->srv_xid++;
460 spin_unlock(&srv->srv_lock);
462 dst = &((struct niobuf *)res)[i];
463 dst->xid = HTON__u32(bulk_vec[i]->b_xid);
465 bulk_vec[i]->b_buf = (void *)(unsigned long)dst->addr;
466 bulk_vec[i]->b_buflen = PAGE_SIZE;
467 bulk_vec[i]->b_portal = OSC_BULK_PORTAL;
468 rc = ptlrpc_register_bulk(bulk_vec[i]);
474 src = &((struct niobuf *)tmp2)[i];
475 memcpy((void *)(unsigned long)dst->addr,
476 (void *)(unsigned long)src->addr, src->len);
483 OBD_FREE(bulk, sizeof(*bulk));
484 if (bulk_vec != NULL) {
485 for (i = 0; i < niocount; i++) {
486 if (bulk_vec[i] != NULL)
487 OBD_FREE(bulk_vec[i], sizeof(*bulk));
490 niocount * sizeof(struct ptlrpc_bulk_desc *));
497 int ost_commit_page(struct obd_conn *conn, struct page *page)
499 struct obd_ioobj obj;
504 memset(&buf, 0, sizeof(buf));
505 memset(&obj, 0, sizeof(obj));
510 rc = obd_commitrw(OBD_BRW_WRITE, conn, 1, &obj, 1, &buf);
516 int ost_brw(struct ost_obd *obddev, struct ptlrpc_request *req)
518 struct ost_req *r = req->rq_req.ost;
521 if (cmd == OBD_BRW_READ)
522 return ost_brw_read(obddev, req);
524 return ost_brw_write(obddev, req);
527 int ost_brw_complete(struct ost_obd *obddev, struct ptlrpc_request *req)
529 struct obd_conn conn;
531 int objcount, niocount;
532 char *tmp1, *tmp2, *end2;
534 struct obd_ioobj *ioo;
535 struct ost_req *r = req->rq_req.ost;
539 tmp1 = ost_req_buf1(r);
540 tmp2 = ost_req_buf2(r);
541 end2 = tmp2 + req->rq_req.ost->buflen2;
542 objcount = r->buflen1 / sizeof(*ioo);
543 niocount = r->buflen2 / sizeof(*nb);
546 conn.oc_id = req->rq_req.ost->connid;
547 conn.oc_dev = req->rq_obd->u.ost.ost_tgt;
549 for (i = 0; i < objcount; i++) {
550 ost_unpack_ioo((void *)&tmp1, &ioo);
551 if (tmp2 + ioo->ioo_bufcnt > end2) {
555 for (j = 0; j < ioo->ioo_bufcnt; j++) {
556 ost_unpack_niobuf((void *)&tmp2, &nb);
560 rc = ost_pack_rep(NULL, 0, NULL, 0,
561 &req->rq_rephdr, &req->rq_rep,
562 &req->rq_replen, &req->rq_repbuf);
564 CERROR("cannot pack reply\n");
568 /* The unpackers move tmp1 and tmp2, so reset them before using */
569 tmp1 = ost_req_buf1(r);
570 tmp2 = ost_req_buf2(r);
571 req->rq_rep.ost->result = obd_commitrw
572 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
573 niocount, (struct niobuf *)tmp2);
578 static int ost_handle(struct obd_device *obddev,
579 struct ptlrpc_service *svc,
580 struct ptlrpc_request *req)
583 struct ost_obd *ost = &obddev->u.ost;
584 struct ptlreq_hdr *hdr;
588 hdr = (struct ptlreq_hdr *)req->rq_reqbuf;
589 if (NTOH__u32(hdr->type) != OST_TYPE_REQ) {
590 CERROR("lustre_ost: wrong packet type sent %d\n",
591 NTOH__u32(hdr->type));
596 rc = ost_unpack_req(req->rq_reqbuf, req->rq_reqlen,
597 &req->rq_reqhdr, &req->rq_req);
599 CERROR("lustre_ost: Invalid request\n");
604 switch (req->rq_reqhdr->opc) {
607 CDEBUG(D_INODE, "connect\n");
608 rc = ost_connect(ost, req);
611 CDEBUG(D_INODE, "disconnect\n");
612 rc = ost_disconnect(ost, req);
615 CDEBUG(D_INODE, "get_info\n");
616 rc = ost_get_info(ost, req);
619 CDEBUG(D_INODE, "create\n");
620 rc = ost_create(ost, req);
623 CDEBUG(D_INODE, "destroy\n");
624 rc = ost_destroy(ost, req);
627 CDEBUG(D_INODE, "getattr\n");
628 rc = ost_getattr(ost, req);
631 CDEBUG(D_INODE, "setattr\n");
632 rc = ost_setattr(ost, req);
635 CDEBUG(D_INODE, "brw\n");
636 rc = ost_brw(ost, req);
638 case OST_BRW_COMPLETE:
639 CDEBUG(D_INODE, "brw_complete\n");
640 rc = ost_brw_complete(ost, req);
643 CDEBUG(D_INODE, "punch\n");
644 rc = ost_punch(ost, req);
647 req->rq_status = -ENOTSUPP;
648 return ptlrpc_error(obddev, svc, req);
654 CERROR("ost: processing error %d\n", rc);
655 ptlrpc_error(obddev, svc, req);
657 CDEBUG(D_INODE, "sending reply\n");
658 ptlrpc_reply(obddev, svc, req);
665 /* mount the file system (secretly) */
666 static int ost_setup(struct obd_device *obddev, obd_count len,
670 struct obd_ioctl_data* data = buf;
671 struct ost_obd *ost = &obddev->u.ost;
672 struct obd_device *tgt;
676 if (data->ioc_dev < 0 || data->ioc_dev > MAX_OBD_DEVICES) {
681 tgt = &obd_dev[data->ioc_dev];
683 if ( ! (tgt->obd_flags & OBD_ATTACHED) ||
684 ! (tgt->obd_flags & OBD_SET_UP) ){
685 CERROR("device not attached or not set up (%d)\n",
691 ost->ost_conn.oc_dev = tgt;
692 err = obd_connect(&ost->ost_conn);
694 CERROR("fail to connect to device %d\n", data->ioc_dev);
698 ost->ost_service = ptlrpc_init_svc( 64 * 1024,
705 if (!ost->ost_service) {
706 obd_disconnect(&ost->ost_conn);
710 rpc_register_service(ost->ost_service, "self");
712 err = ptlrpc_start_thread(obddev, ost->ost_service, "lustre_ost");
714 obd_disconnect(&ost->ost_conn);
723 static int ost_cleanup(struct obd_device * obddev)
725 struct ost_obd *ost = &obddev->u.ost;
730 if ( !list_empty(&obddev->obd_gen_clients) ) {
731 CERROR("still has clients!\n");
736 ptlrpc_stop_thread(ost->ost_service);
737 rpc_unregister_service(ost->ost_service);
739 if (!list_empty(&ost->ost_service->srv_reqs)) {
740 // XXX reply with errors and clean up
741 CERROR("Request list not empty!\n");
743 OBD_FREE(ost->ost_service, sizeof(*ost->ost_service));
745 err = obd_disconnect(&ost->ost_conn);
747 CERROR("lustre ost: fail to disconnect device\n");
756 /* use obd ops to offer management infrastructure */
757 static struct obd_ops ost_obd_ops = {
759 o_cleanup: ost_cleanup,
762 static int __init ost_init(void)
764 obd_register_type(&ost_obd_ops, LUSTRE_OST_NAME);
768 static void __exit ost_exit(void)
770 obd_unregister_type(LUSTRE_OST_NAME);
773 MODULE_AUTHOR("Peter J. Braam <braam@clusterfs.com>");
774 MODULE_DESCRIPTION("Lustre Object Storage Target (OST) v0.01");
775 MODULE_LICENSE("GPL");
777 module_init(ost_init);
778 module_exit(ost_exit);