1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
2 * vim:expandtab:shiftwidth=8:tabstop=8:
4 * Copyright (C) 2003 Cluster File Systems, Inc.
5 * Author: Eric Barton <eeb@clusterfs.com>
7 * This file is part of Lustre, http://www.lustre.org.
9 * Lustre is free software; you can redistribute it and/or
10 * modify it under the terms of version 2 of the GNU General Public
11 * License as published by the Free Software Foundation.
13 * Lustre is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with Lustre; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
29 #include <sys/ioctl.h>
30 #include <sys/types.h>
33 #include <liblustre.h>
37 obdio_iocinit (struct obdio_conn *conn)
39 memset (&conn->oc_data, 0, sizeof (conn->oc_data));
40 conn->oc_data.ioc_version = OBD_IOCTL_VERSION;
41 conn->oc_data.ioc_addr = conn->oc_conn_addr;
42 conn->oc_data.ioc_cookie = conn->oc_conn_cookie;
43 conn->oc_data.ioc_len = sizeof (conn->oc_data);
47 obdio_ioctl (struct obdio_conn *conn, int cmd)
49 char *buf = conn->oc_buffer;
53 rc = obd_ioctl_pack (&conn->oc_data, &buf, sizeof (conn->oc_buffer));
55 fprintf (stderr, "obdio_ioctl: obd_ioctl_pack: %d (%s)\n",
56 rc, strerror (errno));
60 rc = ioctl (conn->oc_fd, cmd, buf);
64 rc2 = obd_ioctl_unpack (&conn->oc_data, buf, sizeof (conn->oc_buffer));
66 fprintf (stderr, "obdio_ioctl: obd_ioctl_unpack: %d (%s)\n",
67 rc2, strerror (errno));
75 obdio_connect (int device)
77 struct obdio_conn *conn;
80 conn = malloc (sizeof (*conn));
82 fprintf (stderr, "obdio_connect: no memory\n");
85 memset (conn, 0, sizeof (*conn));
87 conn->oc_fd = open ("/dev/obd", O_RDWR);
88 if (conn->oc_fd < 0) {
89 fprintf (stderr, "obdio_connect: Can't open /dev/obd: %s\n",
95 conn->oc_data.ioc_dev = device;
96 rc = obdio_ioctl (conn, OBD_IOC_DEVICE);
98 fprintf (stderr, "obdio_connect: Can't set device %d: %s\n",
99 device, strerror (errno));
103 obdio_iocinit (conn);
104 rc = obdio_ioctl (conn, OBD_IOC_CONNECT);
106 fprintf (stderr, "obdio_connect: Can't connect to device %d: %s\n",
107 device, strerror (errno));
111 conn->oc_conn_addr = conn->oc_data.ioc_addr;
112 conn->oc_conn_cookie = conn->oc_data.ioc_cookie;
121 obdio_disconnect (struct obdio_conn *conn)
124 /* obdclass will automatically close on last ref */
129 obdio_open (struct obdio_conn *conn, uint64_t oid, struct lustre_handle *fh)
133 obdio_iocinit (conn);
135 conn->oc_data.ioc_obdo1.o_id = oid;
136 conn->oc_data.ioc_obdo1.o_mode = S_IFREG;
137 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
139 rc = obdio_ioctl (conn, OBD_IOC_OPEN);
142 memcpy (fh, obdo_handle(&conn->oc_data.ioc_obdo1), sizeof (*fh));
148 obdio_close (struct obdio_conn *conn, uint64_t oid, struct lustre_handle *fh)
150 obdio_iocinit (conn);
153 conn->oc_data.ioc_obdo1.o_id = oid;
154 conn->oc_data.ioc_obdo1.o_mode = S_IFREG;
155 memcpy (obdo_handle (&conn->oc_data.ioc_obdo1), fh, sizeof (*fh));
156 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE |
157 OBD_MD_FLMODE | OBD_MD_FLHANDLE;
159 return (obdio_ioctl (conn, OBD_IOC_CLOSE));
163 obdio_pread (struct obdio_conn *conn, uint64_t oid,
164 char *buffer, uint32_t count, uint64_t offset)
166 obdio_iocinit (conn);
168 conn->oc_data.ioc_obdo1.o_id = oid;
169 conn->oc_data.ioc_obdo1.o_mode = S_IFREG;
170 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
172 conn->oc_data.ioc_pbuf2 = buffer;
173 conn->oc_data.ioc_plen2 = count;
174 conn->oc_data.ioc_count = count;
175 conn->oc_data.ioc_offset = offset;
177 return (obdio_ioctl (conn, OBD_IOC_BRW_READ));
181 obdio_pwrite (struct obdio_conn *conn, uint64_t oid,
182 char *buffer, uint32_t count, uint64_t offset)
184 obdio_iocinit (conn);
186 conn->oc_data.ioc_obdo1.o_id = oid;
187 conn->oc_data.ioc_obdo1.o_mode = S_IFREG;
188 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
190 conn->oc_data.ioc_pbuf2 = buffer;
191 conn->oc_data.ioc_plen2 = count;
192 conn->oc_data.ioc_count = count;
193 conn->oc_data.ioc_offset = offset;
195 return (obdio_ioctl (conn, OBD_IOC_BRW_WRITE));
199 obdio_enqueue (struct obdio_conn *conn, uint64_t oid,
200 int mode, uint64_t offset, uint32_t count,
201 struct lustre_handle *lh)
205 obdio_iocinit (conn);
207 conn->oc_data.ioc_obdo1.o_id = oid;
208 conn->oc_data.ioc_obdo1.o_mode = S_IFREG;
209 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLID | OBD_MD_FLTYPE | OBD_MD_FLMODE;
211 conn->oc_data.ioc_conn1 = mode;
212 conn->oc_data.ioc_count = count;
213 conn->oc_data.ioc_offset = offset;
215 rc = obdio_ioctl (conn, ECHO_IOC_ENQUEUE);
218 memcpy (lh, obdo_handle (&conn->oc_data.ioc_obdo1), sizeof (*lh));
224 obdio_cancel (struct obdio_conn *conn, struct lustre_handle *lh)
226 obdio_iocinit (conn);
228 memcpy (obdo_handle (&conn->oc_data.ioc_obdo1), lh, sizeof (*lh));
229 conn->oc_data.ioc_obdo1.o_valid = OBD_MD_FLHANDLE;
231 return (obdio_ioctl (conn, ECHO_IOC_CANCEL));
235 obdio_alloc_aligned_buffer (void **spacep, int size)
237 int pagesize = getpagesize();
238 void *space = malloc (size + pagesize - 1);
244 return ((void *)(((unsigned long)space + pagesize - 1) & ~(pagesize - 1)));
247 struct obdio_barrier *
248 obdio_new_barrier (uint64_t oid, uint64_t id, int npeers)
250 struct obdio_barrier *b;
252 b = (struct obdio_barrier *)malloc (sizeof (*b));
254 fprintf (stderr, "obdio_new_barrier "LPX64": Can't allocate\n", oid);
260 b->ob_npeers = npeers;
267 obdio_setup_barrier (struct obdio_conn *conn, struct obdio_barrier *b)
269 struct lustre_handle fh;
270 struct lustre_handle lh;
274 struct obdio_barrier *fileb;
276 if (b->ob_ordinal != 0 ||
278 fprintf (stderr, "obdio_setup_barrier: invalid parameter\n");
282 rc = obdio_open (conn, b->ob_oid, &fh);
284 fprintf (stderr, "obdio_setup_barrier "LPX64": Failed to open object: %s\n",
285 b->ob_oid, strerror (errno));
289 fileb = (struct obdio_barrier *) obdio_alloc_aligned_buffer (&space, getpagesize ());
291 fprintf (stderr, "obdio_setup_barrier "LPX64": Can't allocate page buffer\n",
297 memset (fileb, 0, getpagesize ());
300 rc = obdio_enqueue (conn, b->ob_oid, LCK_PW, 0, getpagesize (), &lh);
302 fprintf (stderr, "obdio_setup_barrier "LPX64": Error on enqueue: %s\n",
303 b->ob_oid, strerror (errno));
307 rc = obdio_pwrite (conn, b->ob_oid, (void *)fileb, getpagesize (), 0);
309 fprintf (stderr, "obdio_setup_barrier "LPX64": Error on write: %s\n",
310 b->ob_oid, strerror (errno));
312 rc2 = obdio_cancel (conn, &lh);
313 if (rc == 0 && rc2 != 0) {
314 fprintf (stderr, "obdio_setup_barrier "LPX64": Error on cancel: %s\n",
315 b->ob_oid, strerror (errno));
321 rc2 = obdio_close (conn, b->ob_oid, &fh);
322 if (rc == 0 && rc2 != 0) {
323 fprintf (stderr, "obdio_setup_barrier "LPX64": Error on close: %s\n",
324 b->ob_oid, strerror (errno));
332 obdio_barrier (struct obdio_conn *conn, struct obdio_barrier *b)
334 struct lustre_handle fh;
335 struct lustre_handle lh;
339 struct obdio_barrier *fileb;
342 rc = obdio_open (conn, b->ob_oid, &fh);
344 fprintf (stderr, "obdio_barrier "LPX64": Error on open: %s\n",
345 b->ob_oid, strerror (errno));
349 fileb = (struct obdio_barrier *) obdio_alloc_aligned_buffer (&space, getpagesize ());
351 fprintf (stderr, "obdio_barrier "LPX64": Can't allocate page buffer\n",
357 rc = obdio_enqueue (conn, b->ob_oid, LCK_PW, 0, getpagesize (), &lh);
359 fprintf (stderr, "obdio_barrier "LPX64": Error on PW enqueue: %s\n",
360 b->ob_oid, strerror (errno));
364 memset (fileb, 0xeb, getpagesize ());
365 rc = obdio_pread (conn, b->ob_oid, (void *)fileb, getpagesize (), 0);
367 fprintf (stderr, "obdio_barrier "LPX64": Error on initial read: %s\n",
368 b->ob_oid, strerror (errno));
372 if (fileb->ob_id != b->ob_id ||
373 fileb->ob_oid != b->ob_oid ||
374 fileb->ob_npeers != b->ob_npeers ||
375 fileb->ob_count >= b->ob_npeers ||
376 fileb->ob_ordinal != b->ob_ordinal) {
377 fprintf (stderr, "obdio_barrier "LPX64": corrupt on initial read\n", b->ob_id);
378 fprintf (stderr, " got ["LPX64","LPX64","LPX64","LPX64","LPX64"]\n",
379 fileb->ob_id, fileb->ob_oid, fileb->ob_npeers,
380 fileb->ob_ordinal, fileb->ob_count);
381 fprintf (stderr, " expected ["LPX64","LPX64","LPX64","LPX64","LPX64"]\n",
382 b->ob_id, b->ob_oid, b->ob_npeers,
383 b->ob_ordinal, b->ob_count);
389 if (fileb->ob_count == fileb->ob_npeers) { /* I'm the last joiner */
390 fileb->ob_count = 0; /* join count for next barrier */
391 fileb->ob_ordinal++; /* signal all joined */
394 rc = obdio_pwrite (conn, b->ob_oid, (void *)fileb, getpagesize (), 0);
396 fprintf (stderr, "obdio_barrier "LPX64": Error on initial write: %s\n",
397 b->ob_oid, strerror (errno));
402 b->ob_ordinal++; /* now I wait... */
403 while (fileb->ob_ordinal != b->ob_ordinal) {
405 rc = obdio_cancel (conn, &lh);
407 fprintf (stderr, "obdio_barrier "LPX64": Error on %s cancel: %s\n",
408 b->ob_oid, mode, strerror (errno));
413 rc = obdio_enqueue (conn, b->ob_oid, LCK_PR, 0, getpagesize (), &lh);
415 fprintf (stderr, "obdio_barrier "LPX64": Error on PR enqueue: %s\n",
416 b->ob_oid, strerror (errno));
420 memset (fileb, 0xeb, getpagesize ());
421 rc = obdio_pread (conn, b->ob_oid, (void *)fileb, getpagesize (), 0);
423 fprintf (stderr, "obdio_barrier "LPX64": Error on read: %s\n",
424 b->ob_oid, strerror (errno));
428 if (fileb->ob_id != b->ob_id ||
429 fileb->ob_oid != b->ob_oid ||
430 fileb->ob_npeers != b->ob_npeers ||
431 fileb->ob_count >= b->ob_npeers ||
432 (fileb->ob_ordinal != b->ob_ordinal - 1 &&
433 fileb->ob_ordinal != b->ob_ordinal)) {
434 fprintf (stderr, "obdio_barrier "LPX64": corrupt\n", b->ob_id);
435 fprintf (stderr, " got ["LPX64","LPX64","LPX64","LPX64","LPX64"]\n",
436 fileb->ob_id, fileb->ob_oid, fileb->ob_npeers,
437 fileb->ob_ordinal, fileb->ob_count);
438 fprintf (stderr, " expected ["LPX64","LPX64","LPX64","LPX64","LPX64"]\n",
439 b->ob_id, b->ob_oid, b->ob_npeers,
440 b->ob_ordinal, b->ob_count);
447 rc2 = obdio_cancel (conn, &lh);
448 if (rc == 0 && rc2 != 0) {
449 fprintf (stderr, "obdio_barrier "LPX64": Error on cancel: %s\n",
450 b->ob_oid, strerror (errno));
456 rc2 = obdio_close (conn, b->ob_oid, &fh);
457 if (rc == 0 && rc2 != 0) {
458 fprintf (stderr, "obdio_barrier "LPX64": Error on close: %s\n",
459 b->ob_oid, strerror (errno));