* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
* Use is subject to license terms.
*
- * Copyright (c) 2011, Whamcloud, Inc.
+ * Copyright (c) 2011, 2012, Intel Corporation.
*/
/*
* This file is part of Lustre, http://www.lustre.org/
#define CFS_WS_NAME_LEN 16
typedef struct cfs_wi_sched {
- cfs_list_t ws_list; /* chain on global list */
+ struct list_head ws_list; /* chain on global list */
#ifdef __KERNEL__
/** serialised workitems */
- cfs_spinlock_t ws_lock;
+ spinlock_t ws_lock;
/** where schedulers sleep */
- cfs_waitq_t ws_waitq;
+ wait_queue_head_t ws_waitq;
#endif
/** concurrent workitems */
- cfs_list_t ws_runq;
+ struct list_head ws_runq;
/** rescheduled running-workitems, a workitem can be rescheduled
* while running in wi_action(), but we don't to execute it again
* unless it returns from wi_action(), so we put it on ws_rerunq
* while rescheduling, and move it to runq after it returns
* from wi_action() */
- cfs_list_t ws_rerunq;
+ struct list_head ws_rerunq;
/** CPT-table for this scheduler */
struct cfs_cpt_table *ws_cptab;
/** CPT id for affinity */
struct cfs_workitem_data {
/** serialize */
- cfs_spinlock_t wi_glock;
+ spinlock_t wi_glock;
/** list of all schedulers */
- cfs_list_t wi_scheds;
+ struct list_head wi_scheds;
/** WI module is initialized */
int wi_init;
/** shutting down the whole WI module */
static inline void
cfs_wi_sched_lock(cfs_wi_sched_t *sched)
{
- cfs_spin_lock(&sched->ws_lock);
+ spin_lock(&sched->ws_lock);
}
static inline void
cfs_wi_sched_unlock(cfs_wi_sched_t *sched)
{
- cfs_spin_unlock(&sched->ws_lock);
+ spin_unlock(&sched->ws_lock);
}
static inline int
{
cfs_wi_sched_lock(sched);
if (sched->ws_stopping) {
- cfs_wi_sched_unlock(sched);
- return 0;
- }
+ cfs_wi_sched_unlock(sched);
+ return 0;
+ }
- if (!cfs_list_empty(&sched->ws_runq)) {
- cfs_wi_sched_unlock(sched);
- return 0;
- }
- cfs_wi_sched_unlock(sched);
- return 1;
+ if (!list_empty(&sched->ws_runq)) {
+ cfs_wi_sched_unlock(sched);
+ return 0;
+ }
+ cfs_wi_sched_unlock(sched);
+ return 1;
}
#else /* !__KERNEL__ */
static inline void
cfs_wi_sched_lock(cfs_wi_sched_t *sched)
{
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
}
static inline void
cfs_wi_sched_unlock(cfs_wi_sched_t *sched)
{
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
}
#endif /* __KERNEL__ */
void
cfs_wi_exit(struct cfs_wi_sched *sched, cfs_workitem_t *wi)
{
- LASSERT(!cfs_in_interrupt()); /* because we use plain spinlock */
+ LASSERT(!in_interrupt()); /* because we use plain spinlock */
LASSERT(!sched->ws_stopping);
cfs_wi_sched_lock(sched);
LASSERT(wi->wi_running);
#endif
if (wi->wi_scheduled) { /* cancel pending schedules */
- LASSERT(!cfs_list_empty(&wi->wi_list));
- cfs_list_del_init(&wi->wi_list);
+ LASSERT(!list_empty(&wi->wi_list));
+ list_del_init(&wi->wi_list);
LASSERT(sched->ws_nscheduled > 0);
sched->ws_nscheduled--;
}
- LASSERT(cfs_list_empty(&wi->wi_list));
+ LASSERT(list_empty(&wi->wi_list));
wi->wi_scheduled = 1; /* LBUG future schedule attempts */
cfs_wi_sched_unlock(sched);
{
int rc;
- LASSERT(!cfs_in_interrupt()); /* because we use plain spinlock */
+ LASSERT(!in_interrupt()); /* because we use plain spinlock */
LASSERT(!sched->ws_stopping);
/*
rc = !(wi->wi_running);
if (wi->wi_scheduled) { /* cancel pending schedules */
- LASSERT(!cfs_list_empty(&wi->wi_list));
- cfs_list_del_init(&wi->wi_list);
+ LASSERT(!list_empty(&wi->wi_list));
+ list_del_init(&wi->wi_list);
LASSERT(sched->ws_nscheduled > 0);
sched->ws_nscheduled--;
- wi->wi_scheduled = 0;
- }
+ wi->wi_scheduled = 0;
+ }
- LASSERT (cfs_list_empty(&wi->wi_list));
+ LASSERT (list_empty(&wi->wi_list));
- cfs_wi_sched_unlock(sched);
- return rc;
+ cfs_wi_sched_unlock(sched);
+ return rc;
}
EXPORT_SYMBOL(cfs_wi_deschedule);
void
cfs_wi_schedule(struct cfs_wi_sched *sched, cfs_workitem_t *wi)
{
- LASSERT(!cfs_in_interrupt()); /* because we use plain spinlock */
+ LASSERT(!in_interrupt()); /* because we use plain spinlock */
LASSERT(!sched->ws_stopping);
- cfs_wi_sched_lock(sched);
+ cfs_wi_sched_lock(sched);
- if (!wi->wi_scheduled) {
- LASSERT (cfs_list_empty(&wi->wi_list));
+ if (!wi->wi_scheduled) {
+ LASSERT (list_empty(&wi->wi_list));
- wi->wi_scheduled = 1;
+ wi->wi_scheduled = 1;
sched->ws_nscheduled++;
- if (!wi->wi_running) {
- cfs_list_add_tail(&wi->wi_list, &sched->ws_runq);
+ if (!wi->wi_running) {
+ list_add_tail(&wi->wi_list, &sched->ws_runq);
#ifdef __KERNEL__
- cfs_waitq_signal(&sched->ws_waitq);
+ wake_up(&sched->ws_waitq);
#endif
- } else {
- cfs_list_add(&wi->wi_list, &sched->ws_rerunq);
- }
- }
+ } else {
+ list_add(&wi->wi_list, &sched->ws_rerunq);
+ }
+ }
- LASSERT (!cfs_list_empty(&wi->wi_list));
- cfs_wi_sched_unlock(sched);
- return;
+ LASSERT (!list_empty(&wi->wi_list));
+ cfs_wi_sched_unlock(sched);
+ return;
}
EXPORT_SYMBOL(cfs_wi_schedule);
cfs_wi_scheduler (void *arg)
{
struct cfs_wi_sched *sched = (cfs_wi_sched_t *)arg;
- char name[16];
-
- if (sched->ws_cptab != NULL && sched->ws_cpt >= 0) {
- snprintf(name, sizeof(name), "%s_%02d_%02d",
- sched->ws_name, sched->ws_cpt, sched->ws_nthreads);
- } else {
- snprintf(name, sizeof(name), "%s_%02d",
- sched->ws_name, sched->ws_nthreads);
- }
- cfs_daemonize(name);
cfs_block_allsigs();
/* CPT affinity scheduler? */
if (sched->ws_cptab != NULL)
- cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt);
+ if (cfs_cpt_bind(sched->ws_cptab, sched->ws_cpt) != 0)
+ CWARN("Failed to bind %s on CPT %d\n",
+ sched->ws_name, sched->ws_cpt);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
LASSERT(sched->ws_starting == 1);
sched->ws_starting--;
sched->ws_nthreads++;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
cfs_wi_sched_lock(sched);
while (!sched->ws_stopping) {
- int nloops = 0;
- int rc;
- cfs_workitem_t *wi;
-
- while (!cfs_list_empty(&sched->ws_runq) &&
- nloops < CFS_WI_RESCHED) {
- wi = cfs_list_entry(sched->ws_runq.next,
- cfs_workitem_t, wi_list);
+ int nloops = 0;
+ int rc;
+ cfs_workitem_t *wi;
+
+ while (!list_empty(&sched->ws_runq) &&
+ nloops < CFS_WI_RESCHED) {
+ wi = list_entry(sched->ws_runq.next,
+ cfs_workitem_t, wi_list);
LASSERT(wi->wi_scheduled && !wi->wi_running);
- cfs_list_del_init(&wi->wi_list);
+ list_del_init(&wi->wi_list);
LASSERT(sched->ws_nscheduled > 0);
sched->ws_nscheduled--;
if (rc != 0) /* WI should be dead, even be freed! */
continue;
- wi->wi_running = 0;
- if (cfs_list_empty(&wi->wi_list))
+ wi->wi_running = 0;
+ if (list_empty(&wi->wi_list))
continue;
LASSERT(wi->wi_scheduled);
- /* wi is rescheduled, should be on rerunq now, we
- * move it to runq so it can run action now */
- cfs_list_move_tail(&wi->wi_list, &sched->ws_runq);
+ /* wi is rescheduled, should be on rerunq now, we
+ * move it to runq so it can run action now */
+ list_move_tail(&wi->wi_list, &sched->ws_runq);
}
- if (!cfs_list_empty(&sched->ws_runq)) {
- cfs_wi_sched_unlock(sched);
- /* don't sleep because some workitems still
- * expect me to come back soon */
- cfs_cond_resched();
- cfs_wi_sched_lock(sched);
- continue;
- }
+ if (!list_empty(&sched->ws_runq)) {
+ cfs_wi_sched_unlock(sched);
+ /* don't sleep because some workitems still
+ * expect me to come back soon */
+ cond_resched();
+ cfs_wi_sched_lock(sched);
+ continue;
+ }
- cfs_wi_sched_unlock(sched);
- cfs_wait_event_interruptible_exclusive(sched->ws_waitq,
- !cfs_wi_sched_cansleep(sched), rc);
- cfs_wi_sched_lock(sched);
+ cfs_wi_sched_unlock(sched);
+ rc = wait_event_interruptible_exclusive(sched->ws_waitq,
+ !cfs_wi_sched_cansleep(sched));
+ cfs_wi_sched_lock(sched);
}
cfs_wi_sched_unlock(sched);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
sched->ws_nthreads--;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
- return 0;
+ return 0;
}
#else /* __KERNEL__ */
int
cfs_wi_check_events (void)
{
- int n = 0;
- cfs_workitem_t *wi;
+ int n = 0;
+ cfs_workitem_t *wi;
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
- for (;;) {
+ for (;;) {
struct cfs_wi_sched *sched = NULL;
struct cfs_wi_sched *tmp;
/** rerunq is always empty for userspace */
- cfs_list_for_each_entry(tmp,
- &cfs_wi_data.wi_scheds, ws_list) {
- if (!cfs_list_empty(&tmp->ws_runq)) {
+ list_for_each_entry(tmp, &cfs_wi_data.wi_scheds, ws_list) {
+ if (!list_empty(&tmp->ws_runq)) {
sched = tmp;
break;
}
if (sched == NULL)
break;
- wi = cfs_list_entry(sched->ws_runq.next,
+ wi = list_entry(sched->ws_runq.next,
cfs_workitem_t, wi_list);
- cfs_list_del_init(&wi->wi_list);
+ list_del_init(&wi->wi_list);
LASSERT(sched->ws_nscheduled > 0);
sched->ws_nscheduled--;
- LASSERT (wi->wi_scheduled);
- wi->wi_scheduled = 0;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ LASSERT(wi->wi_scheduled);
+ wi->wi_scheduled = 0;
+ spin_unlock(&cfs_wi_data.wi_glock);
- n++;
- (*wi->wi_action) (wi);
+ n++;
+ (*wi->wi_action) (wi);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
- }
+ spin_lock(&cfs_wi_data.wi_glock);
+ }
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
- return n;
+ spin_unlock(&cfs_wi_data.wi_glock);
+ return n;
}
#endif
void
cfs_wi_sched_destroy(struct cfs_wi_sched *sched)
{
- int i;
-
LASSERT(cfs_wi_data.wi_init);
LASSERT(!cfs_wi_data.wi_stopping);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
if (sched->ws_stopping) {
CDEBUG(D_INFO, "%s is in progress of stopping\n",
sched->ws_name);
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
return;
}
- LASSERT(!cfs_list_empty(&sched->ws_list));
+ LASSERT(!list_empty(&sched->ws_list));
sched->ws_stopping = 1;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
- i = 2;
#ifdef __KERNEL__
- cfs_waitq_broadcast(&sched->ws_waitq);
+ wake_up_all(&sched->ws_waitq);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
- while (sched->ws_nthreads > 0) {
- CDEBUG(IS_PO2(++i) ? D_WARNING : D_NET,
- "waiting for %d threads of WI sched[%s] to terminate\n",
- sched->ws_nthreads, sched->ws_name);
+ spin_lock(&cfs_wi_data.wi_glock);
+ {
+ int i = 2;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
- cfs_pause(cfs_time_seconds(1) / 20);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ while (sched->ws_nthreads > 0) {
+ CDEBUG(IS_PO2(++i) ? D_WARNING : D_NET,
+ "waiting for %d threads of WI sched[%s] to "
+ "terminate\n", sched->ws_nthreads,
+ sched->ws_name);
+
+ spin_unlock(&cfs_wi_data.wi_glock);
+ cfs_pause(cfs_time_seconds(1) / 20);
+ spin_lock(&cfs_wi_data.wi_glock);
+ }
}
- cfs_list_del(&sched->ws_list);
+ list_del(&sched->ws_list);
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
-#else
- SET_BUT_UNUSED(i);
+ spin_unlock(&cfs_wi_data.wi_glock);
#endif
LASSERT(sched->ws_nscheduled == 0);
int cpt, int nthrs, struct cfs_wi_sched **sched_pp)
{
struct cfs_wi_sched *sched;
- int rc;
LASSERT(cfs_wi_data.wi_init);
LASSERT(!cfs_wi_data.wi_stopping);
if (sched == NULL)
return -ENOMEM;
- strncpy(sched->ws_name, name, CFS_WS_NAME_LEN);
+ if (strlen(name) > sizeof(sched->ws_name)-1) {
+ LIBCFS_FREE(sched, sizeof(*sched));
+ return -E2BIG;
+ }
+ strlcpy(sched->ws_name, name, sizeof(sched->ws_name));
+
sched->ws_cptab = cptab;
sched->ws_cpt = cpt;
#ifdef __KERNEL__
- cfs_spin_lock_init(&sched->ws_lock);
- cfs_waitq_init(&sched->ws_waitq);
+ spin_lock_init(&sched->ws_lock);
+ init_waitqueue_head(&sched->ws_waitq);
#endif
- CFS_INIT_LIST_HEAD(&sched->ws_runq);
- CFS_INIT_LIST_HEAD(&sched->ws_rerunq);
- CFS_INIT_LIST_HEAD(&sched->ws_list);
+ INIT_LIST_HEAD(&sched->ws_runq);
+ INIT_LIST_HEAD(&sched->ws_rerunq);
+ INIT_LIST_HEAD(&sched->ws_list);
- rc = 0;
#ifdef __KERNEL__
- while (nthrs > 0) {
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ for (; nthrs > 0; nthrs--) {
+ char name[16];
+ struct task_struct *task;
+
+ spin_lock(&cfs_wi_data.wi_glock);
while (sched->ws_starting > 0) {
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
- cfs_schedule();
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
+ schedule();
+ spin_lock(&cfs_wi_data.wi_glock);
}
sched->ws_starting++;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
-
- rc = cfs_create_thread(cfs_wi_scheduler, sched, 0);
- if (rc >= 0) {
- nthrs--;
- continue;
+ spin_unlock(&cfs_wi_data.wi_glock);
+
+ if (sched->ws_cptab != NULL && sched->ws_cpt >= 0) {
+ snprintf(name, sizeof(name), "%s_%02d_%02d",
+ sched->ws_name, sched->ws_cpt,
+ sched->ws_nthreads);
+ } else {
+ snprintf(name, sizeof(name), "%s_%02d",
+ sched->ws_name, sched->ws_nthreads);
}
- CERROR("Failed to create thread for WI scheduler %s: %d\n",
- name, rc);
+ task = kthread_run(cfs_wi_scheduler, sched, name);
+ if (IS_ERR(task)) {
+ int rc = PTR_ERR(task);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ CERROR("Failed to create thread for "
+ "WI scheduler %s: %d\n", name, rc);
- /* make up for cfs_wi_sched_destroy */
- cfs_list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
- sched->ws_starting--;
+ spin_lock(&cfs_wi_data.wi_glock);
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ /* make up for cfs_wi_sched_destroy */
+ list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
+ sched->ws_starting--;
- cfs_wi_sched_destroy(sched);
- return rc;
+ spin_unlock(&cfs_wi_data.wi_glock);
+
+ cfs_wi_sched_destroy(sched);
+ return rc;
+ }
}
-#else
- SET_BUT_UNUSED(rc);
#endif
- cfs_spin_lock(&cfs_wi_data.wi_glock);
- cfs_list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
+ list_add(&sched->ws_list, &cfs_wi_data.wi_scheds);
+ spin_unlock(&cfs_wi_data.wi_glock);
*sched_pp = sched;
return 0;
{
memset(&cfs_wi_data, 0, sizeof(cfs_wi_data));
- cfs_spin_lock_init(&cfs_wi_data.wi_glock);
- CFS_INIT_LIST_HEAD(&cfs_wi_data.wi_scheds);
+ spin_lock_init(&cfs_wi_data.wi_glock);
+ INIT_LIST_HEAD(&cfs_wi_data.wi_scheds);
cfs_wi_data.wi_init = 1;
return 0;
{
struct cfs_wi_sched *sched;
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
cfs_wi_data.wi_stopping = 1;
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
#ifdef __KERNEL__
/* nobody should contend on this list */
- cfs_list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
+ list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
sched->ws_stopping = 1;
- cfs_waitq_broadcast(&sched->ws_waitq);
+ wake_up_all(&sched->ws_waitq);
}
- cfs_list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ list_for_each_entry(sched, &cfs_wi_data.wi_scheds, ws_list) {
+ spin_lock(&cfs_wi_data.wi_glock);
while (sched->ws_nthreads != 0) {
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
cfs_pause(cfs_time_seconds(1) / 20);
- cfs_spin_lock(&cfs_wi_data.wi_glock);
+ spin_lock(&cfs_wi_data.wi_glock);
}
- cfs_spin_unlock(&cfs_wi_data.wi_glock);
+ spin_unlock(&cfs_wi_data.wi_glock);
}
#endif
- while (!cfs_list_empty(&cfs_wi_data.wi_scheds)) {
- sched = cfs_list_entry(cfs_wi_data.wi_scheds.next,
+ while (!list_empty(&cfs_wi_data.wi_scheds)) {
+ sched = list_entry(cfs_wi_data.wi_scheds.next,
struct cfs_wi_sched, ws_list);
- cfs_list_del(&sched->ws_list);
+ list_del(&sched->ws_list);
LIBCFS_FREE(sched, sizeof(*sched));
}