X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Fportals%2Funals%2Fprocapi.c;h=e40c4b9c9b1413a7f0c7cbf994317376aa0e12a9;hb=944a01f5d1adb1ab5f7721d5c2d1cafcd3b2c915;hp=f4820b26ca38251b71d6f25fb7b0962aaca970c4;hpb=c39da0c8b593d91ccf458a520ec2d0fd704a4bd9;p=fs%2Flustre-release.git diff --git a/lustre/portals/unals/procapi.c b/lustre/portals/unals/procapi.c index f4820b2..e40c4b9 100644 --- a/lustre/portals/unals/procapi.c +++ b/lustre/portals/unals/procapi.c @@ -2,6 +2,7 @@ * vim:expandtab:shiftwidth=8:tabstop=8: * * Copyright (c) 2002 Cray Inc. + * Copyright (c) 2003 Cluster File Systems, Inc. * * This file is part of Lustre, http://www.lustre.org. * @@ -31,13 +32,34 @@ #include #include #include +#ifndef __CYGWIN__ #include +#endif +#include #include #include #include #include +/* XXX CFS workaround, to give a chance to let nal thread wake up + * from waiting in select + */ +static int procbridge_notifier_handler(void *arg) +{ + static char buf[8]; + procbridge p = (procbridge) arg; + + syscall(SYS_read, p->notifier[1], buf, sizeof(buf)); + return 1; +} + +void procbridge_wakeup_nal(procbridge p) +{ + static char buf[8]; + syscall(SYS_write, p->notifier[0], buf, sizeof(buf)); +} + /* Function: forward * Arguments: nal_t *nal: pointer to my top-side nal structure * id: the command to pass to the lower layer @@ -48,35 +70,22 @@ * forwards a packaged api call from the 'api' side to the 'library' * side, and collects the result */ -#define forward_failure(operand,fd,buffer,length)\ - if(syscall(SYS_##operand,fd,buffer,length)!=length){\ - lib_fini(b->nal_cb);\ - return(PTL_SEGV);\ - } -static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len, - void *ret, ptl_size_t ret_len) +static int procbridge_forward(nal_t *n, int id, void *args, size_t args_len, + void *ret, size_t ret_len) { - bridge b=(bridge)n->nal_data; - procbridge p=(procbridge)b->local; - int lib=p->to_lib[1]; - int k; + bridge b = (bridge) n->nal_data; - forward_failure(write,lib, &id, sizeof(id)); - forward_failure(write,lib,&args_len, sizeof(args_len)); - forward_failure(write,lib,&ret_len, sizeof(ret_len)); - forward_failure(write,lib,args, args_len); - - do { - k=syscall(SYS_read, p->from_lib[0], ret, ret_len); - } while ((k!=ret_len) && (errno += EINTR)); + if (id == PTL_FINI) { + lib_fini(b->nal_cb); - if(k!=ret_len){ - perror("nal: read return block"); - return PTL_SEGV; + if (b->shutdown) + (*b->shutdown)(b); } + + lib_dispatch(b->nal_cb, NULL, id, args, ret); + return (PTL_OK); } -#undef forward_failure /* Function: shutdown @@ -86,33 +95,43 @@ static int procbridge_forward(nal_t *n, int id, void *args, ptl_size_t args_len, * cleanup nal state, reclaim the lower side thread and * its state using PTL_FINI codepoint */ -static int procbridge_shutdown(nal_t *n, int ni) +static void procbridge_shutdown(nal_t *n) { bridge b=(bridge)n->nal_data; procbridge p=(procbridge)b->local; - int code=PTL_FINI; - syscall(SYS_write, p->to_lib[1],&code,sizeof(code)); - syscall(SYS_read, p->from_lib[0],&code,sizeof(code)); + p->nal_flags |= NAL_FLAG_STOPPING; + procbridge_wakeup_nal(p); - syscall(SYS_close, p->to_lib[0]); - syscall(SYS_close, p->to_lib[1]); - syscall(SYS_close, p->from_lib[0]); - syscall(SYS_close, p->from_lib[1]); + do { + pthread_mutex_lock(&p->mutex); + if (p->nal_flags & NAL_FLAG_STOPPED) { + pthread_mutex_unlock(&p->mutex); + break; + } + pthread_cond_wait(&p->cond, &p->mutex); + pthread_mutex_unlock(&p->mutex); + } while (1); free(p); - return(0); } -/* Function: validate - * useless stub - */ -static int procbridge_validate(nal_t *nal, void *base, ptl_size_t extent) +static void procbridge_lock(nal_t * n, unsigned long *flags) { - return(0); + bridge b=(bridge)n->nal_data; + procbridge p=(procbridge)b->local; + + pthread_mutex_lock(&p->mutex); } +static void procbridge_unlock(nal_t * n, unsigned long *flags) +{ + bridge b=(bridge)n->nal_data; + procbridge p=(procbridge)b->local; + + pthread_mutex_unlock(&p->mutex); +} /* Function: yield * Arguments: pid: @@ -122,36 +141,64 @@ static int procbridge_validate(nal_t *nal, void *base, ptl_size_t extent) * overload it to explicitly block until signalled by the * lower half. */ -static void procbridge_yield(nal_t *n) +static int procbridge_yield(nal_t *n, unsigned long *flags, int milliseconds) { bridge b=(bridge)n->nal_data; procbridge p=(procbridge)b->local; - pthread_mutex_lock(&p->mutex); - pthread_cond_wait(&p->cond,&p->mutex); - pthread_mutex_unlock(&p->mutex); + if (milliseconds == 0) + return 0; + + if (milliseconds < 0) { + pthread_cond_wait(&p->cond,&p->mutex); + } else { + struct timeval then; + struct timeval now; + struct timespec timeout; + + gettimeofday(&then, NULL); + timeout.tv_sec = then.tv_sec + milliseconds/1000; + timeout.tv_nsec = then.tv_usec * 1000 + milliseconds % 1000 * 1000000; + if (timeout.tv_nsec >= 1000000000) { + timeout.tv_sec++; + timeout.tv_nsec -= 1000000000; + } + + pthread_cond_timedwait(&p->cond, &p->mutex, &timeout); + + gettimeofday(&now, NULL); + milliseconds -= (now.tv_sec - then.tv_sec) * 1000 + + (now.tv_usec - then.tv_usec) / 1000; + + if (milliseconds < 0) + milliseconds = 0; + } + + return (milliseconds); } +/* forward decl */ +extern int procbridge_startup (nal_t *, ptl_pid_t, + ptl_ni_limits_t *, ptl_ni_limits_t *); -static void procbridge_lock(nal_t * nal, unsigned long *flags){} -static void procbridge_unlock(nal_t * nal, unsigned long *flags){} /* api_nal * the interface vector to allow the generic code to access * this nal. this is seperate from the library side nal_cb. * TODO: should be dyanmically allocated */ -static nal_t api_nal = { - ni: {0}, +nal_t procapi_nal = { nal_data: NULL, - forward: procbridge_forward, + startup: procbridge_startup, shutdown: procbridge_shutdown, - validate: procbridge_validate, + forward: procbridge_forward, yield: procbridge_yield, lock: procbridge_lock, unlock: procbridge_unlock }; -/* Function: bridge_init +ptl_nid_t tcpnal_mynid; + +/* Function: procbridge_startup * * Arguments: pid: requested process id (port offset) * PTL_ID_ANY not supported. @@ -159,125 +206,74 @@ static nal_t api_nal = { * and effectively ignored * actual: limits actually allocated and returned * - * Returns: a pointer to my statically allocated top side NAL - * structure + * Returns: portals rc * * initializes the tcp nal. we define unix_failure as an * error wrapper to cut down clutter. */ -#define unix_failure(operand,fd,buffer,length,text)\ - if(syscall(SYS_##operand,fd,buffer,length)!=length){\ - perror(text);\ - return(NULL);\ - } -#if 0 -static nal_t *bridge_init(ptl_interface_t nal, - ptl_pid_t pid_request, - ptl_ni_limits_t *desired, - ptl_ni_limits_t *actual, - int *rc) +int procbridge_startup (nal_t *nal, ptl_pid_t requested_pid, + ptl_ni_limits_t *requested_limits, + ptl_ni_limits_t *actual_limits) { + nal_init_args_t args; + procbridge p; bridge b; - static int initialized=0; - ptl_ni_limits_t limits = {-1,-1,-1,-1,-1}; + /* XXX nal_type is purely private to tcpnal here */ + int nal_type = PTL_IFACE_TCP;/* PTL_IFACE_DEFAULT FIXME hack */ - if(initialized) return (&api_nal); + LASSERT(nal == &procapi_nal); init_unix_timer(); b=(bridge)malloc(sizeof(struct bridge)); p=(procbridge)malloc(sizeof(struct procbridge)); - api_nal.nal_data=b; + nal->nal_data=b; b->local=p; - if(pipe(p->to_lib) || pipe(p->from_lib)) { - perror("nal_init: pipe"); - return(NULL); - } - - if (desired) limits = *desired; - unix_failure(write,p->to_lib[1], &pid_request, sizeof(pid_request), - "nal_init: write"); - unix_failure(write,p->to_lib[1], &limits, sizeof(ptl_ni_limits_t), - "nal_init: write"); - unix_failure(write,p->to_lib[1], &nal, sizeof(ptl_interface_t), - "nal_init: write"); - - if(pthread_create(&p->t, NULL, nal_thread, b)) { - perror("nal_init: pthread_create"); - return(NULL); - } - - unix_failure(read,p->from_lib[0], actual, sizeof(ptl_ni_limits_t), - "tcp_init: read"); - unix_failure(read,p->from_lib[0], rc, sizeof(rc), - "nal_init: read"); - - if(*rc) return(NULL); + args.nia_requested_pid = requested_pid; + args.nia_requested_limits = requested_limits; + args.nia_actual_limits = actual_limits; + args.nia_nal_type = nal_type; + args.nia_bridge = b; - initialized = 1; + /* init procbridge */ pthread_mutex_init(&p->mutex,0); pthread_cond_init(&p->cond, 0); + p->nal_flags = 0; - return (&api_nal); -} -#endif - -ptl_nid_t tcpnal_mynid; - -nal_t *procbridge_interface(int num_interface, - ptl_pt_index_t ptl_size, - ptl_ac_index_t acl_size, - ptl_pid_t requested_pid) -{ - procbridge p; - bridge b; - static int initialized=0; - ptl_ni_limits_t limits = {-1,-1,-1,-1,-1}; - int rc, nal_type = PTL_IFACE_TCP;/* PTL_IFACE_DEFAULT FIXME hack */ - - if(initialized) return (&api_nal); - - init_unix_timer(); - - b=(bridge)malloc(sizeof(struct bridge)); - p=(procbridge)malloc(sizeof(struct procbridge)); - api_nal.nal_data=b; - b->local=p; - - if(pipe(p->to_lib) || pipe(p->from_lib)) { - perror("nal_init: pipe"); - return(NULL); + /* initialize notifier */ + if (socketpair(AF_UNIX, SOCK_STREAM, 0, p->notifier)) { + perror("socketpair failed"); + return PTL_FAIL; } - if (ptl_size) - limits.max_ptable_index = ptl_size; - if (acl_size) - limits.max_atable_index = acl_size; - - unix_failure(write,p->to_lib[1], &requested_pid, sizeof(requested_pid), - "nal_init: write"); - unix_failure(write,p->to_lib[1], &limits, sizeof(ptl_ni_limits_t), - "nal_init: write"); - unix_failure(write,p->to_lib[1], &nal_type, sizeof(nal_type), - "nal_init: write"); + if (!register_io_handler(p->notifier[1], READ_HANDLER, + procbridge_notifier_handler, p)) { + perror("fail to register notifier handler"); + return PTL_FAIL; + } - if(pthread_create(&p->t, NULL, nal_thread, b)) { + /* create nal thread */ + if (pthread_create(&p->t, NULL, nal_thread, &args)) { perror("nal_init: pthread_create"); - return(NULL); + return PTL_FAIL; } - unix_failure(read,p->from_lib[0], &rc, sizeof(rc), - "nal_init: read"); - - if(rc) return(NULL); + do { + pthread_mutex_lock(&p->mutex); + if (p->nal_flags & (NAL_FLAG_RUNNING | NAL_FLAG_STOPPED)) { + pthread_mutex_unlock(&p->mutex); + break; + } + pthread_cond_wait(&p->cond, &p->mutex); + pthread_mutex_unlock(&p->mutex); + } while (1); + + if (p->nal_flags & NAL_FLAG_STOPPED) + return PTL_FAIL; b->nal_cb->ni.nid = tcpnal_mynid; - initialized = 1; - pthread_mutex_init(&p->mutex,0); - pthread_cond_init(&p->cond, 0); - return (&api_nal); + return PTL_OK; } -#undef unix_failure