1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2001, 2002 Cluster File Systems, Inc.
5 * Author: Peter J. Braam <braam@clusterfs.com>
6 * Author: Phil Schwan <phil@clusterfs.com>
8 * This file is part of Lustre, http://www.lustre.org.
10 * Lustre is free software; you can redistribute it and/or
11 * modify it under the terms of version 2 of the GNU General Public
12 * License as published by the Free Software Foundation.
14 * Lustre is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with Lustre; if not, write to the Free Software
21 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
23 * Storage Target Handling functions
24 * Lustre Object Server Module (OST)
26 * This server is single threaded at present (but can easily be multi
27 * threaded). For testing and management it is treated as an
28 * obd_device, although it does not export a full OBD method table
29 * (the requests are coming in over the wire, so object target
30 * modules do not have a full method table.)
35 #include <linux/version.h>
36 #include <linux/module.h>
38 #include <linux/stat.h>
39 #include <linux/locks.h>
40 #include <linux/ext2_fs.h>
41 #include <linux/quotaops.h>
42 #include <asm/unistd.h>
44 #define DEBUG_SUBSYSTEM S_OST
46 #include <linux/obd_ost.h>
47 #include <linux/lustre_net.h>
49 static int ost_destroy(struct ost_obd *ost, struct ptlrpc_request *req)
56 conn.oc_id = req->rq_req.ost->connid;
57 conn.oc_dev = ost->ost_tgt;
59 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
60 &req->rq_replen, &req->rq_repbuf);
62 CERROR("cannot pack reply\n");
66 req->rq_rep.ost->result = obd_destroy(&conn, &req->rq_req.ost->oa);
72 static int ost_getattr(struct ost_obd *ost, struct ptlrpc_request *req)
79 conn.oc_id = req->rq_req.ost->connid;
80 conn.oc_dev = ost->ost_tgt;
82 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
83 &req->rq_replen, &req->rq_repbuf);
85 CERROR("cannot pack reply\n");
88 req->rq_rep.ost->oa.o_id = req->rq_req.ost->oa.o_id;
89 req->rq_rep.ost->oa.o_valid = req->rq_req.ost->oa.o_valid;
91 req->rq_rep.ost->result = obd_getattr(&conn, &req->rq_rep.ost->oa);
97 static int ost_open(struct ost_obd *ost, struct ptlrpc_request *req)
104 conn.oc_id = req->rq_req.ost->connid;
105 conn.oc_dev = ost->ost_tgt;
107 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
108 &req->rq_replen, &req->rq_repbuf);
110 CERROR("cannot pack reply\n");
113 req->rq_rep.ost->oa.o_id = req->rq_req.ost->oa.o_id;
114 req->rq_rep.ost->oa.o_valid = req->rq_req.ost->oa.o_valid;
116 req->rq_rep.ost->result = obd_open(&conn, &req->rq_rep.ost->oa);
122 static int ost_close(struct ost_obd *ost, struct ptlrpc_request *req)
124 struct obd_conn conn;
129 conn.oc_id = req->rq_req.ost->connid;
130 conn.oc_dev = ost->ost_tgt;
132 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
133 &req->rq_replen, &req->rq_repbuf);
135 CERROR("cannot pack reply\n");
138 req->rq_rep.ost->oa.o_id = req->rq_req.ost->oa.o_id;
139 req->rq_rep.ost->oa.o_valid = req->rq_req.ost->oa.o_valid;
141 req->rq_rep.ost->result = obd_close(&conn, &req->rq_rep.ost->oa);
148 static int ost_create(struct ost_obd *ost, struct ptlrpc_request *req)
150 struct obd_conn conn;
155 conn.oc_id = req->rq_req.ost->connid;
156 conn.oc_dev = ost->ost_tgt;
158 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
159 &req->rq_replen, &req->rq_repbuf);
161 CERROR("cannot pack reply\n");
165 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
166 sizeof(req->rq_req.ost->oa));
168 req->rq_rep.ost->result =obd_create(&conn, &req->rq_rep.ost->oa);
174 static int ost_punch(struct ost_obd *ost, struct ptlrpc_request *req)
176 struct obd_conn conn;
181 conn.oc_id = req->rq_req.ost->connid;
182 conn.oc_dev = ost->ost_tgt;
184 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
185 &req->rq_replen, &req->rq_repbuf);
187 CERROR("cannot pack reply\n");
191 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
192 sizeof(req->rq_req.ost->oa));
194 req->rq_rep.ost->result = obd_punch(&conn, &req->rq_rep.ost->oa,
195 req->rq_rep.ost->oa.o_size,
196 req->rq_rep.ost->oa.o_blocks);
203 static int ost_setattr(struct ost_obd *ost, struct ptlrpc_request *req)
205 struct obd_conn conn;
210 conn.oc_id = req->rq_req.ost->connid;
211 conn.oc_dev = ost->ost_tgt;
213 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
214 &req->rq_replen, &req->rq_repbuf);
216 CERROR("cannot pack reply\n");
220 memcpy(&req->rq_rep.ost->oa, &req->rq_req.ost->oa,
221 sizeof(req->rq_req.ost->oa));
223 req->rq_rep.ost->result = obd_setattr(&conn, &req->rq_rep.ost->oa);
229 static int ost_connect(struct ost_obd *ost, struct ptlrpc_request *req)
231 struct obd_conn conn;
236 conn.oc_dev = ost->ost_tgt;
238 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
239 &req->rq_replen, &req->rq_repbuf);
241 CERROR("cannot pack reply\n");
245 req->rq_rep.ost->result = obd_connect(&conn);
247 CDEBUG(D_IOCTL, "rep buffer %p, id %d\n", req->rq_repbuf, conn.oc_id);
248 req->rq_rep.ost->connid = conn.oc_id;
253 static int ost_disconnect(struct ost_obd *ost, struct ptlrpc_request *req)
255 struct obd_conn conn;
260 conn.oc_dev = ost->ost_tgt;
261 conn.oc_id = req->rq_req.ost->connid;
263 rc = ost_pack_rep(NULL, 0, NULL, 0, &req->rq_rephdr, &req->rq_rep,
264 &req->rq_replen, &req->rq_repbuf);
266 CERROR("cannot pack reply\n");
269 CDEBUG(D_IOCTL, "Disconnecting %d\n", conn.oc_id);
270 req->rq_rep.ost->result = obd_disconnect(&conn);
276 static int ost_get_info(struct ost_obd *ost, struct ptlrpc_request *req)
278 struct obd_conn conn;
286 conn.oc_id = req->rq_req.ost->connid;
287 conn.oc_dev = ost->ost_tgt;
289 ptr = ost_req_buf1(req->rq_req.ost);
290 req->rq_rep.ost->result = obd_get_info(&conn,
291 req->rq_req.ost->buflen1, ptr,
294 rc = ost_pack_rep(val, vallen, NULL, 0, &req->rq_rephdr,
295 &req->rq_rep, &req->rq_replen, &req->rq_repbuf);
297 CERROR("cannot pack reply\n");
305 static int ost_brw_read(struct ost_obd *obddev, struct ptlrpc_request *req)
307 struct ptlrpc_bulk_desc **bulk_vec = NULL;
308 struct ptlrpc_bulk_desc *bulk = NULL;
309 struct obd_conn conn;
312 int objcount, niocount;
313 char *tmp1, *tmp2, *end2;
316 struct niobuf *nb, *src, *dst;
317 struct obd_ioobj *ioo;
318 struct ost_req *r = req->rq_req.ost;
322 tmp1 = ost_req_buf1(r);
323 tmp2 = ost_req_buf2(r);
324 end2 = tmp2 + req->rq_req.ost->buflen2;
325 objcount = r->buflen1 / sizeof(*ioo);
326 niocount = r->buflen2 / sizeof(*nb);
329 conn.oc_id = req->rq_req.ost->connid;
330 conn.oc_dev = req->rq_obd->u.ost.ost_tgt;
332 for (i = 0; i < objcount; i++) {
333 ost_unpack_ioo((void *)&tmp1, &ioo);
334 if (tmp2 + ioo->ioo_bufcnt > end2) {
338 for (j = 0; j < ioo->ioo_bufcnt; j++) {
339 ost_unpack_niobuf((void *)&tmp2, &nb);
343 rc = ost_pack_rep(NULL, 0, NULL, 0,
344 &req->rq_rephdr, &req->rq_rep,
345 &req->rq_replen, &req->rq_repbuf);
347 CERROR("cannot pack reply\n");
350 OBD_ALLOC(res, sizeof(struct niobuf) * niocount);
356 /* The unpackers move tmp1 and tmp2, so reset them before using */
357 tmp1 = ost_req_buf1(r);
358 tmp2 = ost_req_buf2(r);
359 req->rq_rep.ost->result = obd_preprw
360 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
361 niocount, (struct niobuf *)tmp2, (struct niobuf *)res);
363 if (req->rq_rep.ost->result) {
368 for (i = 0; i < niocount; i++) {
369 bulk = ptlrpc_prep_bulk(&req->rq_peer);
371 CERROR("cannot alloc bulk desc\n");
376 src = &((struct niobuf *)res)[i];
377 dst = &((struct niobuf *)tmp2)[i];
378 bulk->b_xid = dst->xid;
379 bulk->b_buf = (void *)(unsigned long)src->addr;
380 bulk->b_buflen = PAGE_SIZE;
381 rc = ptlrpc_send_bulk(bulk, OST_BULK_PORTAL);
386 wait_event_interruptible(bulk->b_waitq,
387 ptlrpc_check_bulk_sent(bulk));
389 if (bulk->b_flags == PTL_RPC_INTR) {
394 OBD_FREE(bulk, sizeof(*bulk));
400 dst = &((struct niobuf *)tmp2)[i];
401 memcpy((void *)(unsigned long)dst->addr,
402 (void *)(unsigned long)src->addr, PAGE_SIZE);
406 /* The unpackers move tmp1 and tmp2, so reset them before using */
407 tmp1 = ost_req_buf1(r);
408 tmp2 = ost_req_buf2(r);
409 req->rq_rep.ost->result = obd_commitrw
410 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
411 niocount, (struct niobuf *)res);
415 OBD_FREE(res, sizeof(struct niobuf) * niocount);
417 OBD_FREE(bulk, sizeof(*bulk));
418 if (bulk_vec != NULL) {
419 for (i = 0; i < niocount; i++) {
420 if (bulk_vec[i] != NULL)
421 OBD_FREE(bulk_vec[i], sizeof(*bulk));
424 niocount * sizeof(struct ptlrpc_bulk_desc *));
431 static int ost_commit_page(struct obd_conn *conn, struct page *page)
433 struct obd_ioobj obj;
438 memset(&buf, 0, sizeof(buf));
439 memset(&obj, 0, sizeof(obj));
444 rc = obd_commitrw(OBD_BRW_WRITE, conn, 1, &obj, 1, &buf);
449 static int ost_brw_write_cb(struct ptlrpc_bulk_desc *bulk, void *data)
455 rc = ost_commit_page(&bulk->b_conn, bulk->b_page);
457 CERROR("ost_commit_page failed: %d\n", rc);
463 int ost_brw_write(struct ost_obd *obddev, struct ptlrpc_request *req)
465 struct obd_conn conn;
468 int objcount, niocount;
469 char *tmp1, *tmp2, *end2;
472 struct niobuf *nb, *dst;
473 struct obd_ioobj *ioo;
474 struct ost_req *r = req->rq_req.ost;
478 tmp1 = ost_req_buf1(r);
479 tmp2 = ost_req_buf2(r);
480 end2 = tmp2 + req->rq_req.ost->buflen2;
481 objcount = r->buflen1 / sizeof(*ioo);
482 niocount = r->buflen2 / sizeof(*nb);
485 conn.oc_id = req->rq_req.ost->connid;
486 conn.oc_dev = req->rq_obd->u.ost.ost_tgt;
488 for (i = 0; i < objcount; i++) {
489 ost_unpack_ioo((void *)&tmp1, &ioo);
490 if (tmp2 + ioo->ioo_bufcnt > end2) {
494 for (j = 0; j < ioo->ioo_bufcnt; j++) {
495 ost_unpack_niobuf((void *)&tmp2, &nb);
499 rc = ost_pack_rep(NULL, 0, NULL, niocount * sizeof(*nb),
500 &req->rq_rephdr, &req->rq_rep,
501 &req->rq_replen, &req->rq_repbuf);
503 CERROR("cannot pack reply\n");
506 res = ost_rep_buf2(req->rq_rep.ost);
508 /* The unpackers move tmp1 and tmp2, so reset them before using */
509 tmp1 = ost_req_buf1(r);
510 tmp2 = ost_req_buf2(r);
511 req->rq_rep.ost->result = obd_preprw
512 (cmd, &conn, objcount, (struct obd_ioobj *)tmp1,
513 niocount, (struct niobuf *)tmp2, (struct niobuf *)res);
515 if (req->rq_rep.ost->result) {
520 for (i = 0; i < niocount; i++) {
521 struct ptlrpc_bulk_desc *bulk;
522 struct ptlrpc_service *srv = req->rq_obd->u.ost.ost_service;
524 bulk = ptlrpc_prep_bulk(&req->rq_peer);
526 CERROR("cannot alloc bulk desc\n");
531 spin_lock(&srv->srv_lock);
532 bulk->b_xid = srv->srv_xid++;
533 spin_unlock(&srv->srv_lock);
535 dst = &((struct niobuf *)res)[i];
536 dst->xid = HTON__u32(bulk->b_xid);
538 bulk->b_buf = (void *)(unsigned long)dst->addr;
539 bulk->b_cb = ost_brw_write_cb;
540 bulk->b_page = dst->page;
541 memcpy(&(bulk->b_conn), &conn, sizeof(conn));
542 bulk->b_buflen = PAGE_SIZE;
543 bulk->b_portal = OSC_BULK_PORTAL;
544 rc = ptlrpc_register_bulk(bulk);
550 src = &((struct niobuf *)tmp2)[i];
551 memcpy((void *)(unsigned long)dst->addr,
552 (void *)(unsigned long)src->addr, src->len);
562 int ost_brw(struct ost_obd *obddev, struct ptlrpc_request *req)
564 struct ost_req *r = req->rq_req.ost;
567 if (cmd == OBD_BRW_READ)
568 return ost_brw_read(obddev, req);
570 return ost_brw_write(obddev, req);
573 static int ost_handle(struct obd_device *obddev,
574 struct ptlrpc_service *svc,
575 struct ptlrpc_request *req)
578 struct ost_obd *ost = &obddev->u.ost;
579 struct ptlreq_hdr *hdr;
583 hdr = (struct ptlreq_hdr *)req->rq_reqbuf;
584 if (NTOH__u32(hdr->type) != OST_TYPE_REQ) {
585 CERROR("lustre_ost: wrong packet type sent %d\n",
586 NTOH__u32(hdr->type));
591 rc = ost_unpack_req(req->rq_reqbuf, req->rq_reqlen,
592 &req->rq_reqhdr, &req->rq_req);
594 CERROR("lustre_ost: Invalid request\n");
599 switch (req->rq_reqhdr->opc) {
602 CDEBUG(D_INODE, "connect\n");
603 rc = ost_connect(ost, req);
606 CDEBUG(D_INODE, "disconnect\n");
607 rc = ost_disconnect(ost, req);
610 CDEBUG(D_INODE, "get_info\n");
611 rc = ost_get_info(ost, req);
614 CDEBUG(D_INODE, "create\n");
615 rc = ost_create(ost, req);
618 CDEBUG(D_INODE, "destroy\n");
619 rc = ost_destroy(ost, req);
622 CDEBUG(D_INODE, "getattr\n");
623 rc = ost_getattr(ost, req);
626 CDEBUG(D_INODE, "setattr\n");
627 rc = ost_setattr(ost, req);
630 CDEBUG(D_INODE, "setattr\n");
631 rc = ost_open(ost, req);
634 CDEBUG(D_INODE, "setattr\n");
635 rc = ost_close(ost, req);
638 CDEBUG(D_INODE, "brw\n");
639 rc = ost_brw(ost, req);
642 CDEBUG(D_INODE, "punch\n");
643 rc = ost_punch(ost, req);
646 req->rq_status = -ENOTSUPP;
647 return ptlrpc_error(obddev, svc, req);
653 CERROR("ost: processing error %d\n", rc);
654 ptlrpc_error(obddev, svc, req);
656 CDEBUG(D_INODE, "sending reply\n");
657 ptlrpc_reply(obddev, svc, req);
664 /* mount the file system (secretly) */
665 static int ost_setup(struct obd_device *obddev, obd_count len,
669 struct obd_ioctl_data* data = buf;
670 struct ost_obd *ost = &obddev->u.ost;
671 struct obd_device *tgt;
675 if (data->ioc_dev < 0 || data->ioc_dev > MAX_OBD_DEVICES) {
680 tgt = &obd_dev[data->ioc_dev];
682 if ( ! (tgt->obd_flags & OBD_ATTACHED) ||
683 ! (tgt->obd_flags & OBD_SET_UP) ){
684 CERROR("device not attached or not set up (%d)\n",
690 ost->ost_conn.oc_dev = tgt;
691 err = obd_connect(&ost->ost_conn);
693 CERROR("fail to connect to device %d\n", data->ioc_dev);
697 ost->ost_service = ptlrpc_init_svc( 64 * 1024,
704 if (!ost->ost_service) {
705 obd_disconnect(&ost->ost_conn);
709 rpc_register_service(ost->ost_service, "self");
711 err = ptlrpc_start_thread(obddev, ost->ost_service, "lustre_ost");
713 obd_disconnect(&ost->ost_conn);
722 static int ost_cleanup(struct obd_device * obddev)
724 struct ost_obd *ost = &obddev->u.ost;
729 if ( !list_empty(&obddev->obd_gen_clients) ) {
730 CERROR("still has clients!\n");
735 ptlrpc_stop_thread(ost->ost_service);
736 rpc_unregister_service(ost->ost_service);
738 if (!list_empty(&ost->ost_service->srv_reqs)) {
739 // XXX reply with errors and clean up
740 CERROR("Request list not empty!\n");
742 OBD_FREE(ost->ost_service, sizeof(*ost->ost_service));
744 err = obd_disconnect(&ost->ost_conn);
746 CERROR("lustre ost: fail to disconnect device\n");
755 /* use obd ops to offer management infrastructure */
756 static struct obd_ops ost_obd_ops = {
758 o_cleanup: ost_cleanup,
761 static int __init ost_init(void)
763 obd_register_type(&ost_obd_ops, LUSTRE_OST_NAME);
767 static void __exit ost_exit(void)
769 obd_unregister_type(LUSTRE_OST_NAME);
772 MODULE_AUTHOR("Peter J. Braam <braam@clusterfs.com>");
773 MODULE_DESCRIPTION("Lustre Object Storage Target (OST) v0.01");
774 MODULE_LICENSE("GPL");
776 module_init(ost_init);
777 module_exit(ost_exit);