X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=blobdiff_plain;f=lnet%2Flnet%2Fapi-ni.c;h=7a3c8a39a6c8fb8a2b8b29e4b4efc907aa134bc5;hp=5761d288d6af92857fa85110f990b47214058b58;hb=2bcfce77a2712960fadac2f034c1df4826780608;hpb=8fab48a8ef0bad6961c2ca1e2959726252ba43ac diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 5761d28..7a3c8a3 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -27,7 +27,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. * Use is subject to license terms. * - * Copyright (c) 2011, 2013, Intel Corporation. + * Copyright (c) 2011, 2014, Intel Corporation. */ /* * This file is part of Lustre, http://www.lustre.org/ @@ -756,6 +756,8 @@ lnet_prepare(lnet_pid_t requested_pid) INIT_LIST_HEAD(&the_lnet.ln_nis_cpt); INIT_LIST_HEAD(&the_lnet.ln_nis_zombie); INIT_LIST_HEAD(&the_lnet.ln_routers); + INIT_LIST_HEAD(&the_lnet.ln_drop_rules); + INIT_LIST_HEAD(&the_lnet.ln_delay_rules); rc = lnet_create_remote_nets_table(); if (rc != 0) @@ -1268,6 +1270,20 @@ lnet_ni_tq_credits(lnet_ni_t *ni) } static void +lnet_ni_unlink_locked(lnet_ni_t *ni) +{ + if (!list_empty(&ni->ni_cptlist)) { + list_del_init(&ni->ni_cptlist); + lnet_ni_decref_locked(ni, 0); + } + + /* move it to zombie list and nobody can find it anymore */ + LASSERT(!list_empty(&ni->ni_list)); + list_move(&ni->ni_list, &the_lnet.ln_nis_zombie); + lnet_ni_decref_locked(ni, 0); /* drop ln_nis' ref */ +} + +static void lnet_clear_zombies_nis_locked(void) { int i; @@ -1346,14 +1362,7 @@ lnet_shutdown_lndnis(void) while (!list_empty(&the_lnet.ln_nis)) { ni = list_entry(the_lnet.ln_nis.next, lnet_ni_t, ni_list); - /* move it to zombie list and nobody can find it anymore */ - list_move(&ni->ni_list, &the_lnet.ln_nis_zombie); - lnet_ni_decref_locked(ni, 0); /* drop ln_nis' ref */ - - if (!list_empty(&ni->ni_cptlist)) { - list_del_init(&ni->ni_cptlist); - lnet_ni_decref_locked(ni, 0); - } + lnet_ni_unlink_locked(ni); } /* Drop the cached eqwait NI. */ @@ -1386,68 +1395,25 @@ lnet_shutdown_lndnis(void) lnet_net_unlock(LNET_LOCK_EX); } -int -lnet_shutdown_lndni(__u32 net) +/* shutdown down the NI and release refcount */ +static void +lnet_shutdown_lndni(struct lnet_ni *ni) { - lnet_ping_info_t *pinfo; - lnet_handle_md_t md_handle; - lnet_ni_t *found_ni = NULL; - int ni_count; - int rc; - - if (LNET_NETTYP(net) == LOLND) - return -EINVAL; - - ni_count = lnet_get_ni_count(); - - /* create and link a new ping info, before removing the old one */ - rc = lnet_ping_info_setup(&pinfo, &md_handle, ni_count - 1, false); - if (rc != 0) - return rc; - - /* proceed with shutting down the NI */ lnet_net_lock(LNET_LOCK_EX); - - found_ni = lnet_net2ni_locked(net, 0); - if (found_ni == NULL) { - lnet_net_unlock(LNET_LOCK_EX); - lnet_ping_md_unlink(pinfo, &md_handle); - lnet_ping_info_free(pinfo); - return -EINVAL; - } - - /* decrement the reference counter on found_ni which was - * incremented when we called lnet_net2ni_locked() */ - lnet_ni_decref_locked(found_ni, 0); - - /* Move ni to zombie list so nobody can find it anymore */ - list_move(&found_ni->ni_list, &the_lnet.ln_nis_zombie); - - /* Drop the lock reference for the ln_nis ref. */ - lnet_ni_decref_locked(found_ni, 0); - - if (!list_empty(&found_ni->ni_cptlist)) { - list_del_init(&found_ni->ni_cptlist); - lnet_ni_decref_locked(found_ni, 0); - } - + lnet_ni_unlink_locked(ni); lnet_net_unlock(LNET_LOCK_EX); /* Do peer table cleanup for this ni */ - lnet_peer_tables_cleanup(found_ni); + lnet_peer_tables_cleanup(ni); lnet_net_lock(LNET_LOCK_EX); lnet_clear_zombies_nis_locked(); lnet_net_unlock(LNET_LOCK_EX); - - lnet_ping_target_update(pinfo, md_handle); - - return 0; } static int lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, - __s32 peer_cr, __s32 peer_buf_cr, __s32 credits) + __s32 peer_cr, __s32 peer_buf_cr, __s32 credits) { int rc = 0; int lnd_type; @@ -1456,29 +1422,28 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, int i; lnd_type = LNET_NETTYP(LNET_NIDNET(ni->ni_nid)); - if (!libcfs_isknown_lnd(lnd_type)) - return -EINVAL; + + LASSERT(libcfs_isknown_lnd(lnd_type)); if (lnd_type == CIBLND || lnd_type == OPENIBLND || lnd_type == IIBLND || lnd_type == VIBLND) { CERROR("LND %s obsoleted\n", libcfs_lnd2str(lnd_type)); - return -EINVAL; + goto failed0; } /* Make sure this new NI is unique. */ lnet_net_lock(LNET_LOCK_EX); - if (!lnet_net_unique(LNET_NIDNET(ni->ni_nid), - &the_lnet.ln_nis)) { + if (!lnet_net_unique(LNET_NIDNET(ni->ni_nid), &the_lnet.ln_nis)) { if (lnd_type == LOLND) { lnet_net_unlock(LNET_LOCK_EX); lnet_ni_free(ni); return 0; } + lnet_net_unlock(LNET_LOCK_EX); CERROR("Net %s is not unique\n", libcfs_net2str(LNET_NIDNET(ni->ni_nid))); - lnet_net_unlock(LNET_LOCK_EX); - return -EINVAL; + goto failed0; } lnet_net_unlock(LNET_LOCK_EX); @@ -1489,7 +1454,7 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, if (lnd == NULL) { LNET_MUTEX_UNLOCK(&the_lnet.ln_lnd_mutex); rc = request_module("%s", - libcfs_lnd2modname(lnd_type)); + libcfs_lnd2modname(lnd_type)); LNET_MUTEX_LOCK(&the_lnet.ln_lnd_mutex); lnd = lnet_find_lnd_by_type(lnd_type); @@ -1500,18 +1465,18 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, libcfs_lnd2modname(lnd_type), rc); #ifndef HAVE_MODULE_LOADING_SUPPORT LCONSOLE_ERROR_MSG(0x104, "Your kernel must be " - "compiled with kernel module " - "loading support."); + "compiled with kernel module " + "loading support."); #endif - return -EINVAL; + goto failed0; } } #else if (lnd == NULL) { LNET_MUTEX_UNLOCK(&the_lnet.ln_lnd_mutex); CERROR("LND %s not supported\n", - libcfs_lnd2str(lnd_type)); - return -EINVAL; + libcfs_lnd2str(lnd_type)); + goto failed0; } #endif @@ -1526,14 +1491,12 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, LNET_MUTEX_UNLOCK(&the_lnet.ln_lnd_mutex); if (rc != 0) { - LCONSOLE_ERROR_MSG(0x105, "Error %d starting up LNI %s" - "\n", + LCONSOLE_ERROR_MSG(0x105, "Error %d starting up LNI %s\n", rc, libcfs_lnd2str(lnd->lnd_type)); lnet_net_lock(LNET_LOCK_EX); lnd->lnd_refcount--; lnet_net_unlock(LNET_LOCK_EX); - lnet_ni_free(ni); - return -EINVAL; + goto failed0; } /* If given some LND tunable parameters, parse those now to @@ -1560,17 +1523,17 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, lnet_ni_addref_locked(ni, 0); list_add_tail(&ni->ni_list, &the_lnet.ln_nis); if (ni->ni_cpts != NULL) { - list_add_tail(&ni->ni_cptlist, - &the_lnet.ln_nis_cpt); lnet_ni_addref_locked(ni, 0); + list_add_tail(&ni->ni_cptlist, &the_lnet.ln_nis_cpt); } lnet_net_unlock(LNET_LOCK_EX); + if (lnd->lnd_type == LOLND) { lnet_ni_addref(ni); LASSERT(the_lnet.ln_loni == NULL); the_lnet.ln_loni = ni; - return 1; + return 0; } #ifndef __KERNEL__ @@ -1582,19 +1545,26 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, } else { # ifndef HAVE_LIBPTHREAD LCONSOLE_ERROR_MSG(0x106, "LND %s not supported in a " - "single-threaded runtime\n", - libcfs_lnd2str(lnd_type)); - goto failed; + "single-threaded runtime\n", + libcfs_lnd2str(lnd_type)); + /* shutdown the NI since if we get here then it must've already + * been started + */ + lnet_shutdown_lndni(ni); + return -EINVAL; # endif } #endif - if (ni->ni_peertxcredits == 0 || - ni->ni_maxtxcredits == 0) { + if (ni->ni_peertxcredits == 0 || ni->ni_maxtxcredits == 0) { LCONSOLE_ERROR_MSG(0x107, "LNI %s has no %scredits\n", libcfs_lnd2str(lnd->lnd_type), - ni->ni_peertxcredits == 0 ? - "" : "per-peer "); - goto failed; + ni->ni_peertxcredits == 0 ? + "" : "per-peer "); + /* shutdown the NI since if we get here then it must've already + * been started + */ + lnet_shutdown_lndni(ni); + return -EINVAL; } cfs_percpt_for_each(tq, i, ni->ni_tx_queues) { @@ -1604,45 +1574,48 @@ lnet_startup_lndni(struct lnet_ni *ni, __s32 peer_timeout, } CDEBUG(D_LNI, "Added LNI %s [%d/%d/%d/%d]\n", - libcfs_nid2str(ni->ni_nid), ni->ni_peertxcredits, - lnet_ni_tq_credits(ni) * LNET_CPT_NUMBER, - ni->ni_peerrtrcredits, ni->ni_peertimeout); + libcfs_nid2str(ni->ni_nid), ni->ni_peertxcredits, + lnet_ni_tq_credits(ni) * LNET_CPT_NUMBER, + ni->ni_peerrtrcredits, ni->ni_peertimeout); - return 1; -failed: - lnet_shutdown_lndni(LNET_NIDNET(ni->ni_nid)); + return 0; +failed0: lnet_ni_free(ni); return -EINVAL; } static int -lnet_startup_lndnis(struct list_head *nilist, __s32 peer_timeout, - __s32 peer_cr, __s32 peer_buf_cr, __s32 credits, - int *ni_count) +lnet_startup_lndnis(struct list_head *nilist) { - int rc = 0; - struct lnet_ni *ni; + struct lnet_ni *ni; + int rc; + int lnd_type; + int ni_count = 0; while (!list_empty(nilist)) { ni = list_entry(nilist->next, lnet_ni_t, ni_list); list_del(&ni->ni_list); - rc = lnet_startup_lndni(ni, peer_timeout, peer_cr, - peer_buf_cr, credits); + rc = lnet_startup_lndni(ni, -1, -1, -1, -1); + if (rc < 0) goto failed; - if (ni_count != NULL && rc == 1) - (*ni_count)++; + ni_count++; } - return 0; -failed: - while (!list_empty(nilist)) { - ni = list_entry(nilist->next, lnet_ni_t, ni_list); - list_del(&ni->ni_list); - lnet_ni_free(ni); + if (the_lnet.ln_eq_waitni != NULL && ni_count > 1) { + lnd_type = the_lnet.ln_eq_waitni->ni_lnd->lnd_type; + LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network" + "\n", + libcfs_lnd2str(lnd_type)); + rc = -EINVAL; + goto failed; } + + return ni_count; +failed: lnet_shutdown_lndnis(); + return rc; } @@ -1767,10 +1740,8 @@ int LNetNIInit(lnet_pid_t requested_pid) { int im_a_router = 0; - int rc; - int ni_count = 0; - int lnd_type; - struct lnet_ni *ni; + int rc, rc2; + int ni_count; lnet_ping_info_t *pinfo; lnet_handle_md_t md_handle; struct list_head net_head; @@ -1789,38 +1760,49 @@ LNetNIInit(lnet_pid_t requested_pid) } rc = lnet_prepare(requested_pid); - if (rc != 0) - goto failed0; + if (rc != 0) { + LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); + return rc; + } - rc = lnet_parse_networks(&net_head, - !the_lnet.ln_nis_from_mod_params ? - lnet_get_networks() : ""); - if (rc < 0) - goto failed1; + /* Add in the loopback network */ + if (lnet_ni_alloc(LNET_MKNET(LOLND, 0), NULL, &net_head) == NULL) { + rc = -ENOMEM; + goto failed0; + } - rc = lnet_startup_lndnis(&net_head, -1, -1, -1, -1, &ni_count); - if (rc != 0) - goto failed1; + /* If LNet is being initialized via DLC it is possible + * that the user requests not to load module parameters (ones which + * are supported by DLC) on initialization. Therefore, make sure not + * to load networks, routes and forwarding from module parameters + * in this case. On cleanup in case of failure only clean up + * routes if it has been loaded */ + if (!the_lnet.ln_nis_from_mod_params) { + rc = lnet_parse_networks(&net_head, + lnet_get_networks()); + if (rc < 0) + goto failed0; + } - if (the_lnet.ln_eq_waitni != NULL && ni_count > 1) { - lnd_type = the_lnet.ln_eq_waitni->ni_lnd->lnd_type; - LCONSOLE_ERROR_MSG(0x109, "LND %s can only run single-network" - "\n", - libcfs_lnd2str(lnd_type)); - goto failed1; + ni_count = lnet_startup_lndnis(&net_head); + if (ni_count < 0) { + rc = ni_count; + goto failed0; } - rc = lnet_parse_routes(lnet_get_routes(), &im_a_router); - if (rc != 0) - goto failed2; + if (!the_lnet.ln_nis_from_mod_params) { + rc = lnet_parse_routes(lnet_get_routes(), &im_a_router); + if (rc != 0) + goto failed1; - rc = lnet_check_routes(); - if (rc != 0) - goto failed2; + rc = lnet_check_routes(); + if (rc != 0) + goto failed2; - rc = lnet_rtrpools_alloc(im_a_router); - if (rc != 0) - goto failed2; + rc = lnet_rtrpools_alloc(im_a_router); + if (rc != 0) + goto failed2; + } rc = lnet_acceptor_start(); if (rc != 0) @@ -1838,6 +1820,7 @@ LNetNIInit(lnet_pid_t requested_pid) if (rc != 0) goto failed4; + lnet_fault_init(); lnet_proc_init(); LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); @@ -1845,22 +1828,24 @@ LNetNIInit(lnet_pid_t requested_pid) return 0; failed4: - the_lnet.ln_refcount = 0; lnet_ping_md_unlink(pinfo, &md_handle); lnet_ping_info_free(pinfo); + rc2 = LNetEQFree(the_lnet.ln_ping_target_eq); + LASSERT(rc2 == 0); failed3: + the_lnet.ln_refcount = 0; lnet_acceptor_stop(); - rc = LNetEQFree(the_lnet.ln_ping_target_eq); - LASSERT(rc == 0); failed2: - lnet_destroy_routes(); - lnet_shutdown_lndnis(); + if (!the_lnet.ln_nis_from_mod_params) + lnet_destroy_routes(); failed1: - lnet_unprepare(); + lnet_shutdown_lndnis(); failed0: + lnet_unprepare(); LASSERT(rc < 0); LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); while (!list_empty(&net_head)) { + struct lnet_ni *ni; ni = list_entry(net_head.next, struct lnet_ni, ni_list); list_del_init(&ni->ni_list); lnet_ni_free(ni); @@ -1889,7 +1874,9 @@ LNetNIFini() if (the_lnet.ln_refcount != 1) { the_lnet.ln_refcount--; } else { - LASSERT (!the_lnet.ln_niinit_self); + LASSERT(!the_lnet.ln_niinit_self); + + lnet_fault_fini(); lnet_proc_fini(); lnet_router_checker_stop(); @@ -2011,12 +1998,12 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets, /* Create a ni structure for the network string */ rc = lnet_parse_networks(&net_head, nets); - if (rc < 0) - return rc; + if (rc <= 0) + return rc == 0 ? -EINVAL : rc; LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); - if (rc != 1) { + if (rc > 1) { rc = -EINVAL; /* only add one interface per call */ goto failed0; } @@ -2026,19 +2013,13 @@ lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets, if (rc != 0) goto failed0; - /* pick up first one that is not LOLND */ - while (!list_empty(&net_head)) { - ni = list_entry(net_head.next, struct lnet_ni, ni_list); - list_del_init(&ni->ni_list); - if (LNET_NIDNET(ni->ni_nid) == LOLND) { - lnet_ni_free(ni); - continue; - } - rc = lnet_startup_lndni(ni, peer_timeout, peer_cr, - peer_buf_cr, credits); - if (rc < 0) - goto failed1; - } + ni = list_entry(net_head.next, struct lnet_ni, ni_list); + list_del_init(&ni->ni_list); + + rc = lnet_startup_lndni(ni, peer_timeout, peer_cr, + peer_buf_cr, credits); + if (rc != 0) + goto failed1; lnet_ping_target_update(pinfo, md_handle); LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); @@ -2061,10 +2042,38 @@ failed0: int lnet_dyn_del_ni(__u32 net) { - int rc; + lnet_ni_t *ni; + lnet_ping_info_t *pinfo; + lnet_handle_md_t md_handle; + int rc; + + /* don't allow userspace to shutdown the LOLND */ + if (LNET_NETTYP(net) == LOLND) + return -EINVAL; LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); - rc = lnet_shutdown_lndni(net); + /* create and link a new ping info, before removing the old one */ + rc = lnet_ping_info_setup(&pinfo, &md_handle, + lnet_get_ni_count() - 1, false); + if (rc != 0) + goto out; + + ni = lnet_net2ni(net); + if (ni == NULL) { + rc = -EINVAL; + goto failed; + } + + /* decrement the reference counter taken by lnet_net2ni() */ + lnet_ni_decref_locked(ni, 0); + + lnet_shutdown_lndni(ni); + lnet_ping_target_update(pinfo, md_handle); + goto out; +failed: + lnet_ping_md_unlink(pinfo, &md_handle); + lnet_ping_info_free(pinfo); +out: LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); return rc; @@ -2092,6 +2101,8 @@ LNetCtl(unsigned int cmd, void *arg) lnet_ni_t *ni; int rc; + CLASSERT(LIBCFS_IOC_DATA_MAX >= sizeof(struct lnet_ioctl_net_config) + + sizeof(struct lnet_ioctl_config_data)); LASSERT(the_lnet.ln_init); switch (cmd) { @@ -2105,6 +2116,10 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_ADD_ROUTE: config = arg; + + if (config->cfg_hdr.ioc_len < sizeof(*config)) + return -EINVAL; + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); rc = lnet_add_route(config->cfg_net, config->cfg_config_u.cfg_route.rtr_hop, @@ -2116,6 +2131,10 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_DEL_ROUTE: config = arg; + + if (config->cfg_hdr.ioc_len < sizeof(*config)) + return -EINVAL; + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); rc = lnet_del_route(config->cfg_net, config->cfg_nid); LNET_MUTEX_UNLOCK(&the_lnet.ln_api_mutex); @@ -2123,6 +2142,10 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_GET_ROUTE: config = arg; + + if (config->cfg_hdr.ioc_len < sizeof(*config)) + return -EINVAL; + return lnet_get_route(config->cfg_count, &config->cfg_net, &config->cfg_config_u.cfg_route.rtr_hop, @@ -2133,7 +2156,13 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_GET_NET: { struct lnet_ioctl_net_config *net_config; + size_t total = sizeof(*config) + sizeof(*net_config); + config = arg; + + if (config->cfg_hdr.ioc_len < total) + return -EINVAL; + net_config = (struct lnet_ioctl_net_config *) config->cfg_bulk; if (config == NULL || net_config == NULL) @@ -2157,6 +2186,9 @@ LNetCtl(unsigned int cmd, void *arg) { struct lnet_ioctl_lnet_stats *lnet_stats = arg; + if (lnet_stats->st_hdr.ioc_len < sizeof(*lnet_stats)) + return -EINVAL; + lnet_counters_get(&lnet_stats->st_cntrs); return 0; } @@ -2164,6 +2196,10 @@ LNetCtl(unsigned int cmd, void *arg) #if defined(__KERNEL__) && defined(LNET_ROUTER) case IOC_LIBCFS_CONFIG_RTR: config = arg; + + if (config->cfg_hdr.ioc_len < sizeof(*config)) + return -EINVAL; + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); if (config->cfg_config_u.cfg_buffers.buf_enable) { rc = lnet_rtrpools_enable(); @@ -2176,6 +2212,10 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_ADD_BUF: config = arg; + + if (config->cfg_hdr.ioc_len < sizeof(*config)) + return -EINVAL; + LNET_MUTEX_LOCK(&the_lnet.ln_api_mutex); rc = lnet_rtrpools_adjust(config->cfg_config_u.cfg_buffers. buf_tiny, @@ -2189,13 +2229,23 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_GET_BUF: { struct lnet_ioctl_pool_cfg *pool_cfg; + size_t total = sizeof(*config) + sizeof(*pool_cfg); + config = arg; + + if (config->cfg_hdr.ioc_len < total) + return -EINVAL; + pool_cfg = (struct lnet_ioctl_pool_cfg *)config->cfg_bulk; return lnet_get_rtr_pool_cfg(config->cfg_count, pool_cfg); } case IOC_LIBCFS_GET_PEER_INFO: { struct lnet_ioctl_peer *peer_info = arg; + + if (peer_info->pr_hdr.ioc_len < sizeof(*peer_info)) + return -EINVAL; + return lnet_get_peer_info( peer_info->pr_count, &peer_info->pr_nid, @@ -2233,6 +2283,9 @@ LNetCtl(unsigned int cmd, void *arg) lnet_net_unlock(LNET_LOCK_EX); return 0; + case IOC_LIBCFS_LNET_FAULT: + return lnet_fault_ctl(data->ioc_flags, data); + case IOC_LIBCFS_PING: id.nid = data->ioc_nid; id.pid = data->ioc_u32[0];