lustre/contrib/adio_driver_mpich2-1.0.7.patch

   1 diff -ruN ad_lustre_orig/ad_lustre_aggregate.c ad_lustre/ad_lustre_aggregate.c
   2 --- ad_lustre_orig/ad_lustre_aggregate.c        1970-01-01 08:00:00.000000000 +0800
   3 +++ ad_lustre/ad_lustre_aggregate.c     2008-10-17 17:30:00.000000000 +0800
   4 @@ -0,0 +1,502 @@
   5 +/* -*- Mode: C; c-basic-offset:4 ; -*- */
   6 +/*
   7 + *   Copyright (C) 1997 University of Chicago.
   8 + *   See COPYRIGHT notice in top-level directory.
   9 + *
  10 + *   Copyright (C) 2007 Oak Ridge National Laboratory
  11 + *
  12 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
  13 + */
  14 +
  15 +#include "ad_lustre.h"
  16 +#include "adio_extern.h"
  17 +
  18 +void ADIOI_LUSTRE_Get_striping_info(ADIO_File fd, int ** striping_info_ptr,
  19 +                                   int mode)
  20 +{
  21 +    int *striping_info = NULL;
  22 +    /* get striping information:
  23 +     *  striping_info[0]: stripe_size
  24 +     *  striping_info[1]: stripe_count
  25 +     *  striping_info[2]: avail_cb_nodes
  26 +     */
  27 +    int stripe_size, stripe_count, CO = 1, CO_max = 1, CO_nodes, lflag;
  28 +    int avail_cb_nodes, divisor, nprocs_for_coll = fd->hints->cb_nodes;
  29 +    char *value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
  30 +
  31 +    /* Get hints value */
  32 +    /* stripe size */
  33 +    MPI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL, value, &lflag);
  34 +    if (lflag)
  35 +       stripe_size = atoi(value);
  36 +    /* stripe count */
  37 +    /* stripe_size and stripe_count have been validated in ADIOI_LUSTRE_Open() */
  38 +    MPI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL, value, &lflag);
  39 +    if (lflag)
  40 +       stripe_count = atoi(value);
  41 +
  42 +    /* Calculate the available number of I/O clients, that is
  43 +     *  avail_cb_nodes=min(cb_nodes, stripe_count*CO), where
  44 +     *  CO=1 by default
  45 +     */
  46 +    if (!mode) {
  47 +        /* for collective read,
  48 +        * if "CO" clients access the same OST simultaneously,
  49 +        * the OST disk seek time would be much. So, to avoid this,
  50 +        * it might be better if 1 client only accesses 1 OST.
  51 +        * So, we set CO = 1 to meet the above requirement.
  52 +        */
  53 +       CO = 1;
  54 +       /*XXX: maybe there are other better way for collective read */
  55 +    } else {
  56 +        /* CO_max: the largest number of IO clients for each ost group */
  57 +        CO_max = (nprocs_for_coll - 1)/ stripe_count + 1;
  58 +        /* CO also has been validated in ADIOI_LUSTRE_Open(), >0 */
  59 +       MPI_Info_get(fd->info, "CO", MPI_MAX_INFO_VAL, value, &lflag);
  60 +       if (lflag)
  61 +           CO = atoi(value);
  62 +       CO = ADIOI_MIN(CO_max, CO);
  63 +    }
  64 +    /* Calculate how many IO clients we need */
  65 +    /* To avoid extent lock conflicts,
  66 +     * avail_cb_nodes should divide (stripe_count*CO) exactly,
  67 +     * so that each OST is accessed by only one or more constant clients. */
  68 +    avail_cb_nodes = ADIOI_MIN(nprocs_for_coll, stripe_count * CO);
  69 +    if (avail_cb_nodes == nprocs_for_coll) {
  70 +        CO_nodes = stripe_count * CO;
  71 +        do {
  72 +            /* find the divisor of CO_nodes */
  73 +            divisor = 1;
  74 +            do {
  75 +                divisor ++;
  76 +            } while (CO_nodes % divisor);
  77 +            CO_nodes = CO_nodes / divisor;
  78 +            /* if stripe_count*CO is a prime number, change nothing */
  79 +            if ((CO_nodes <= avail_cb_nodes) && (CO_nodes != 1)) {
  80 +                avail_cb_nodes = CO_nodes;
  81 +                break;
  82 +            }
  83 +        } while (CO_nodes != 1);
  84 +    }
  85 +
  86 +    *striping_info_ptr = (int *) ADIOI_Malloc(3 * sizeof(int));
  87 +    striping_info = *striping_info_ptr;
  88 +    striping_info[0] = stripe_size;
  89 +    striping_info[1] = stripe_count;
  90 +    striping_info[2] = avail_cb_nodes;
  91 +
  92 +    ADIOI_Free(value);
  93 +}
  94 +
  95 +int ADIOI_LUSTRE_Calc_aggregator(ADIO_File fd, ADIO_Offset off,
  96 +                                 ADIO_Offset *len, int *striping_info)
  97 +{
  98 +    int rank_index, rank;
  99 +    ADIO_Offset avail_bytes;
 100 +    int stripe_size = striping_info[0];
 101 +    int avail_cb_nodes = striping_info[2];
 102 +
 103 +    /* Produce the stripe-contiguous pattern for Lustre */
 104 +    rank_index = (int)((off / stripe_size) % avail_cb_nodes);
 105 +
 106 +    avail_bytes = (off / (ADIO_Offset)stripe_size + 1) *
 107 +                  (ADIO_Offset)stripe_size - off;
 108 +    if (avail_bytes < *len) {
 109 +       /* this proc only has part of the requested contig. region */
 110 +       *len = avail_bytes;
 111 +    }
 112 +    /* map our index to a rank */
 113 +    /* NOTE: FOR NOW WE DON'T HAVE A MAPPING...JUST DO 0..NPROCS_FOR_COLL */
 114 +    rank = fd->hints->ranklist[rank_index];
 115 +
 116 +    return rank;
 117 +}
 118 +
 119 +void ADIOI_LUSTRE_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list,
 120 +                             int *len_list, int contig_access_count,
 121 +                             int *striping_info, int nprocs,
 122 +                              int *count_my_req_procs_ptr,
 123 +                             int **count_my_req_per_proc_ptr,
 124 +                             ADIOI_Access ** my_req_ptr,
 125 +                             int **buf_idx_ptr)
 126 +{
 127 +    /* Nothing different from ADIOI_Calc_my_req(), except calling
 128 +     * ADIOI_Lustre_Calc_aggregator() instead of the old one */
 129 +    int *count_my_req_per_proc, count_my_req_procs, *buf_idx;
 130 +    int i, l, proc;
 131 +    ADIO_Offset avail_len, rem_len, curr_idx, off;
 132 +    ADIOI_Access *my_req;
 133 +
 134 +    *count_my_req_per_proc_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
 135 +    count_my_req_per_proc = *count_my_req_per_proc_ptr;
 136 +
 137 +    /* buf_idx is relevant only if buftype_is_contig.
 138 +     * buf_idx[i] gives the index into user_buf where data received
 139 +     * from proc. i should be placed. This allows receives to be done
 140 +     * without extra buffer. This can't be done if buftype is not contig.
 141 +     */
 142 +    buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int));
 143 +    /* initialize buf_idx to -1 */
 144 +    for (i = 0; i < nprocs; i++)
 145 +       buf_idx[i] = -1;
 146 +
 147 +    /* one pass just to calculate how much space to allocate for my_req;
 148 +     * contig_access_count was calculated way back in ADIOI_Calc_my_off_len()
 149 +     */
 150 +    for (i = 0; i < contig_access_count; i++) {
 151 +       /* short circuit offset/len processing if len == 0
 152 +        * (zero-byte  read/write
 153 +        */
 154 +       if (len_list[i] == 0)
 155 +           continue;
 156 +       off = offset_list[i];
 157 +       avail_len = len_list[i];
 158 +       /* we set avail_len to be the total size of the access.
 159 +        * then ADIOI_LUSTRE_Calc_aggregator() will modify the value to return
 160 +        * the amount that was available.
 161 +        */
 162 +       proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
 163 +       count_my_req_per_proc[proc]++;
 164 +       /* figure out how many data is remaining in the access
 165 +        * we'll take care of this data (if there is any)
 166 +        * in the while loop below.
 167 +        */
 168 +       rem_len = len_list[i] - avail_len;
 169 +
 170 +       while (rem_len != 0) {
 171 +           off += avail_len;   /* point to first remaining byte */
 172 +           avail_len = rem_len;        /* save remaining size, pass to calc */
 173 +           proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
 174 +           count_my_req_per_proc[proc]++;
 175 +           rem_len -= avail_len;       /* reduce remaining length by amount from fd */
 176 +       }
 177 +    }
 178 +
 179 +    *my_req_ptr = (ADIOI_Access *) ADIOI_Malloc(nprocs * sizeof(ADIOI_Access));
 180 +    my_req = *my_req_ptr;
 181 +
 182 +    count_my_req_procs = 0;
 183 +    for (i = 0; i < nprocs; i++) {
 184 +       if (count_my_req_per_proc[i]) {
 185 +           my_req[i].offsets = (ADIO_Offset *)
 186 +                               ADIOI_Malloc(count_my_req_per_proc[i] *
 187 +                                             sizeof(ADIO_Offset));
 188 +           my_req[i].lens = (int *) ADIOI_Malloc(count_my_req_per_proc[i] *
 189 +                                                 sizeof(int));
 190 +           count_my_req_procs++;
 191 +       }
 192 +       my_req[i].count = 0;    /* will be incremented where needed later */
 193 +    }
 194 +
 195 +    /* now fill in my_req */
 196 +    curr_idx = 0;
 197 +    for (i = 0; i < contig_access_count; i++) {
 198 +       if (len_list[i] == 0)
 199 +           continue;
 200 +       off = offset_list[i];
 201 +       avail_len = len_list[i];
 202 +       proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info);
 203 +
 204 +       /* for each separate contiguous access from this process */
 205 +       if (buf_idx[proc] == -1)
 206 +           buf_idx[proc] = (int) curr_idx;
 207 +
 208 +       l = my_req[proc].count;
 209 +       curr_idx += (int) avail_len;    /* NOTE: Why is curr_idx an int?  Fix? */
 210 +
 211 +       rem_len = len_list[i] - avail_len;
 212 +
 213 +       /* store the proc, offset, and len information in an array
 214 +        * of structures, my_req. Each structure contains the
 215 +        * offsets and lengths located in that process's FD,
 216 +        * and the associated count.
 217 +        */
 218 +       my_req[proc].offsets[l] = off;
 219 +       my_req[proc].lens[l] = (int) avail_len;
 220 +       my_req[proc].count++;
 221 +
 222 +       while (rem_len != 0) {
 223 +           off += avail_len;
 224 +           avail_len = rem_len;
 225 +           proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len,
 226 +                                                striping_info);
 227 +           if (buf_idx[proc] == -1)
 228 +               buf_idx[proc] = (int) curr_idx;
 229 +
 230 +           l = my_req[proc].count;
 231 +           curr_idx += avail_len;
 232 +           rem_len -= avail_len;
 233 +
 234 +           my_req[proc].offsets[l] = off;
 235 +           my_req[proc].lens[l] = (int) avail_len;
 236 +           my_req[proc].count++;
 237 +       }
 238 +    }
 239 +
 240 +#ifdef AGG_DEBUG
 241 +    for (i = 0; i < nprocs; i++) {
 242 +       if (count_my_req_per_proc[i] > 0) {
 243 +           FPRINTF(stdout, "data needed from %d (count = %d):\n",
 244 +                           i, my_req[i].count);
 245 +           for (l = 0; l < my_req[i].count; l++) {
 246 +               FPRINTF(stdout, "   off[%d] = %lld, len[%d] = %d\n",
 247 +                               l, my_req[i].offsets[l], l, my_req[i].lens[l]);
 248 +           }
 249 +       }
 250 +    }
 251 +#endif
 252 +#if 0
 253 +    for (i = 0; i < nprocs; i++) {
 254 +       FPRINTF(stdout, "buf_idx[%d] = 0x%x\n", i, buf_idx[i]);
 255 +    }
 256 +#endif
 257 +
 258 +    *count_my_req_procs_ptr = count_my_req_procs;
 259 +    *buf_idx_ptr = buf_idx;
 260 +}
 261 +
 262 +int ADIOI_LUSTRE_Docollect(ADIO_File fd, int contig_access_count,
 263 +                          int *len_list, int nprocs)
 264 +{
 265 +    /* If the processes are non-interleaved, we will check the req_size.
 266 +     *   if (avg_req_size > big_req_size) {
 267 +     *       docollect = 0;
 268 +     *   }
 269 +     */
 270 +
 271 +    int i, docollect = 1, lflag, big_req_size = 0;
 272 +    ADIO_Offset req_size = 0, total_req_size;
 273 +    int avg_req_size, total_access_count;
 274 +    char *value = NULL;
 275 +
 276 +    /* calculate total_req_size and total_access_count */
 277 +    for (i = 0; i < contig_access_count; i++)
 278 +        req_size += len_list[i];
 279 +    MPI_Allreduce(&req_size, &total_req_size, 1, MPI_LONG_LONG_INT, MPI_SUM,
 280 +               fd->comm);
 281 +    MPI_Allreduce(&contig_access_count, &total_access_count, 1, MPI_INT, MPI_SUM,
 282 +               fd->comm);
 283 +    /* estimate average req_size */
 284 +    avg_req_size = (int)(total_req_size / total_access_count);
 285 +
 286 +    /* get hint of big_req_size */
 287 +    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
 288 +    MPI_Info_get(fd->info, "big_req_size", MPI_MAX_INFO_VAL, value, &lflag);
 289 +    if (lflag)
 290 +        big_req_size = atoi(value);
 291 +    /* Don't perform collective I/O if there are big requests */
 292 +    if ((big_req_size > 0) && (avg_req_size > big_req_size))
 293 +        docollect = 0;
 294 +
 295 +    ADIOI_Free(value);
 296 +
 297 +    return docollect;
 298 +}
 299 +
 300 +void ADIOI_LUSTRE_Calc_others_req(ADIO_File fd, int count_my_req_procs,
 301 +                                 int *count_my_req_per_proc,
 302 +                                 ADIOI_Access * my_req,
 303 +                                 int nprocs, int myrank,
 304 +                                  ADIO_Offset start_offset,
 305 +                                  ADIO_Offset end_offset,
 306 +                                  int *striping_info,
 307 +                                 int *count_others_req_procs_ptr,
 308 +                                 ADIOI_Access ** others_req_ptr)
 309 +{
 310 +    /* what requests of other processes will be written by this process */
 311 +
 312 +    int *count_others_req_per_proc, count_others_req_procs, proc;
 313 +    int i, j, lflag, samesize = 0, contiguous = 0;
 314 +    int avail_cb_nodes = striping_info[2];
 315 +    MPI_Request *send_requests, *recv_requests;
 316 +    MPI_Status *statuses;
 317 +    ADIOI_Access *others_req;
 318 +    char *value = NULL;
 319 +    ADIO_Offset min_st_offset, off, req_len, avail_len, rem_len, *all_lens;
 320 +
 321 +    /* There are two hints, which could reduce some MPI communication overhead,
 322 +     * if the users knows the I/O pattern and set them correctly. */
 323 +    /* They are
 324 +     * contiguous_data: if the data are contiguous,
 325 +     *                  we don't need to do MPI_Alltoall().
 326 +     * same_io_size: And if the data req size is same,
 327 +     *               we can calculate the offset directly
 328 +     */
 329 +    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
 330 +    /* hint of contiguous data */
 331 +    MPI_Info_get(fd->info, "contiguous_data", MPI_MAX_INFO_VAL, value, &lflag);
 332 +    if (lflag && !strcmp(value, "yes"))
 333 +        contiguous = 1;
 334 +    /* hint of same io size */
 335 +    MPI_Info_get(fd->info, "same_io_size", MPI_MAX_INFO_VAL, value, &lflag);
 336 +    if (lflag && !strcmp(value, "yes"))
 337 +        samesize = 1;
 338 +    ADIOI_Free(value);
 339 +
 340 +    *others_req_ptr = (ADIOI_Access *) ADIOI_Malloc(nprocs *
 341 +                                                    sizeof(ADIOI_Access));
 342 +    others_req = *others_req_ptr;
 343 +
 344 +    /* if the data are contiguous, we can calulate the offset and length
 345 +     * of the other requests simply, instead of MPI_Alltoall() */
 346 +    if (contiguous) {
 347 +        for (i = 0; i < nprocs; i++) {
 348 +            others_req[i].count = 0;
 349 +        }
 350 +        req_len = end_offset - start_offset + 1;
 351 +        all_lens = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
 352 +
 353 +        /* same req size ? */
 354 +        if (samesize == 0) {
 355 +            /* calculate the min_st_offset */
 356 +            MPI_Allreduce(&start_offset, &min_st_offset, 1, MPI_LONG_LONG,
 357 +                          MPI_MIN, fd->comm);
 358 +            /* exchange request length */
 359 +            MPI_Allgather(&req_len, 1, ADIO_OFFSET, all_lens, 1, ADIO_OFFSET,
 360 +                          fd->comm);
 361 +        } else { /* same request size */
 362 +            /* calculate the 1st request's offset */
 363 +            min_st_offset = start_offset - myrank * req_len;
 364 +            /* assign request length to all_lens[] */
 365 +            for (i = 0; i < nprocs; i ++)
 366 +               all_lens[i] = req_len;
 367 +        }
 368 +        if (myrank < avail_cb_nodes) {
 369 +            /* This is a IO client and it will receive data from others */
 370 +            off = min_st_offset;
 371 +            /* calcaulte other_req[i].count */
 372 +            for (i = 0; i < nprocs; i++) {
 373 +                avail_len = all_lens[i];
 374 +                rem_len = avail_len;
 375 +                while (rem_len > 0) {
 376 +                   proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len,
 377 +                                                        striping_info);
 378 +                    if (proc == myrank) {
 379 +                        others_req[i].count ++;
 380 +                    }
 381 +                    off += avail_len;
 382 +                    rem_len -= avail_len;
 383 +                    avail_len = rem_len;
 384 +                }
 385 +            }
 386 +            /* calculate offset and len for each request */
 387 +            off = min_st_offset;
 388 +            for (i = 0; i < nprocs; i++) {
 389 +                if (others_req[i].count) {
 390 +                   others_req[i].offsets = (ADIO_Offset *)
 391 +                                            ADIOI_Malloc(others_req[i].count *
 392 +                                                        sizeof(ADIO_Offset));
 393 +                   others_req[i].lens = (int *)
 394 +                                         ADIOI_Malloc(others_req[i].count *
 395 +                                                      sizeof(int));
 396 +                    others_req[i].mem_ptrs = (MPI_Aint *)
 397 +                                             ADIOI_Malloc(others_req[i].count *
 398 +                                                         sizeof(MPI_Aint));
 399 +                }
 400 +                j = 0;
 401 +                avail_len = all_lens[i];
 402 +                rem_len = avail_len;
 403 +                while (rem_len > 0) {
 404 +                   proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len,
 405 +                                                        striping_info);
 406 +                    if (proc == myrank) {
 407 +                        others_req[i].offsets[j] = off;
 408 +                        others_req[i].lens[j] = (int)avail_len;
 409 +                        j ++;
 410 +                    }
 411 +                    off += avail_len;
 412 +                    rem_len -= avail_len;
 413 +                    avail_len = rem_len;
 414 +                }
 415 +            }
 416 +        }
 417 +        ADIOI_Free(all_lens);
 418 +    } else {
 419 +        /* multiple non-contiguous requests */
 420 +        /* first find out how much to send/recv and from/to whom */
 421 +
 422 +        /*
 423 +         * count_others_req_procs:
 424 +         *    number of processes whose requests will be written by
 425 +         *    this process (including this process itself)
 426 +         * count_others_req_per_proc[i]:
 427 +         *    how many separate contiguous requests of proc[i] will be
 428 +         *    written by this process.
 429 +         */
 430 +
 431 +        count_others_req_per_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int));
 432 +
 433 +        MPI_Alltoall(count_my_req_per_proc, 1, MPI_INT,
 434 +                    count_others_req_per_proc, 1, MPI_INT, fd->comm);
 435 +
 436 +        count_others_req_procs = 0;
 437 +        for (i = 0; i < nprocs; i++) {
 438 +           if (count_others_req_per_proc[i]) {
 439 +               others_req[i].count = count_others_req_per_proc[i];
 440 +               others_req[i].offsets = (ADIO_Offset *)
 441 +                                        ADIOI_Malloc(others_req[i].count *
 442 +                                                sizeof(ADIO_Offset));
 443 +               others_req[i].lens = (int *)
 444 +                                    ADIOI_Malloc(others_req[i].count *
 445 +                                                  sizeof(int));
 446 +               others_req[i].mem_ptrs = (MPI_Aint *)
 447 +                                        ADIOI_Malloc(others_req[i].count *
 448 +                                                     sizeof(MPI_Aint));
 449 +               count_others_req_procs++;
 450 +           } else
 451 +               others_req[i].count = 0;
 452 +        }
 453 +
 454 +        /* now send the calculated offsets and lengths to respective processes */
 455 +
 456 +        send_requests = (MPI_Request *) ADIOI_Malloc(2 * (count_my_req_procs + 1) *
 457 +                                                     sizeof(MPI_Request));
 458 +        recv_requests = (MPI_Request *) ADIOI_Malloc(2 * (count_others_req_procs+1)*
 459 +                                                    sizeof(MPI_Request));
 460 +        /* +1 to avoid a 0-size malloc */
 461 +
 462 +        j = 0;
 463 +        for (i = 0; i < nprocs; i++) {
 464 +           if (others_req[i].count) {
 465 +               MPI_Irecv(others_req[i].offsets, others_req[i].count,
 466 +                         ADIO_OFFSET, i, i + myrank, fd->comm,
 467 +                         &recv_requests[j]);
 468 +               j++;
 469 +               MPI_Irecv(others_req[i].lens, others_req[i].count,
 470 +                         MPI_INT, i, i + myrank + 1, fd->comm,
 471 +                         &recv_requests[j]);
 472 +               j++;
 473 +           }
 474 +        }
 475 +
 476 +        j = 0;
 477 +        for (i = 0; i < nprocs; i++) {
 478 +           if (my_req[i].count) {
 479 +               MPI_Isend(my_req[i].offsets, my_req[i].count,
 480 +                         ADIO_OFFSET, i, i + myrank, fd->comm,
 481 +                         &send_requests[j]);
 482 +               j++;
 483 +               MPI_Isend(my_req[i].lens, my_req[i].count,
 484 +                         MPI_INT, i, i + myrank + 1, fd->comm,
 485 +                         &send_requests[j]);
 486 +               j++;
 487 +           }
 488 +        }
 489 +
 490 +        statuses = (MPI_Status *)
 491 +                   ADIOI_Malloc((1 + 2 * ADIOI_MAX(count_my_req_procs,
 492 +                                                  count_others_req_procs)) *
 493 +                                         sizeof(MPI_Status));
 494 +        /* +1 to avoid a 0-size malloc */
 495 +
 496 +        MPI_Waitall(2 * count_my_req_procs, send_requests, statuses);
 497 +        MPI_Waitall(2 * count_others_req_procs, recv_requests, statuses);
 498 +
 499 +        ADIOI_Free(send_requests);
 500 +        ADIOI_Free(recv_requests);
 501 +        ADIOI_Free(statuses);
 502 +        ADIOI_Free(count_others_req_per_proc);
 503 +
 504 +        *count_others_req_procs_ptr = count_others_req_procs;
 505 +    }
 506 +}
 507 diff -ruN ad_lustre_orig/ad_lustre.c ad_lustre/ad_lustre.c
 508 --- ad_lustre_orig/ad_lustre.c  2008-09-17 14:36:57.000000000 +0800
 509 +++ ad_lustre/ad_lustre.c       2008-10-17 17:03:42.000000000 +0800
 510 @@ -1,9 +1,11 @@
 511  /* -*- Mode: C; c-basic-offset:4 ; -*- */
 512 -/*
 513 - *   Copyright (C) 2001 University of Chicago.
 514 +/*
 515 + *   Copyright (C) 2001 University of Chicago.
 516   *   See COPYRIGHT notice in top-level directory.
 517   *
 518   *   Copyright (C) 2007 Oak Ridge National Laboratory
 519 + *
 520 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
 521   */
 522
 523  #include "ad_lustre.h"
 524 @@ -13,12 +15,12 @@
 525      ADIOI_LUSTRE_ReadContig, /* ReadContig */
 526      ADIOI_LUSTRE_WriteContig, /* WriteContig */
 527      ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
 528 -    ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
 529 +    ADIOI_LUSTRE_WriteStridedColl, /* WriteStridedColl */
 530      ADIOI_GEN_SeekIndividual, /* SeekIndividual */
 531      ADIOI_GEN_Fcntl, /* Fcntl */
 532      ADIOI_LUSTRE_SetInfo, /* SetInfo */
 533      ADIOI_GEN_ReadStrided, /* ReadStrided */
 534 -    ADIOI_GEN_WriteStrided, /* WriteStrided */
 535 +    ADIOI_LUSTRE_WriteStrided, /* WriteStrided */
 536      ADIOI_GEN_Close, /* Close */
 537  #if defined(ROMIO_HAVE_WORKING_AIO) && !defined(CRAY_XT_LUSTRE)
 538      ADIOI_GEN_IreadContig, /* IreadContig */
 539 diff -ruN ad_lustre_orig/ad_lustre.h ad_lustre/ad_lustre.h
 540 --- ad_lustre_orig/ad_lustre.h  2008-09-17 14:36:57.000000000 +0800
 541 +++ ad_lustre/ad_lustre.h       2008-10-17 17:11:11.000000000 +0800
 542 @@ -1,9 +1,11 @@
 543  /* -*- Mode: C; c-basic-offset:4 ; -*- */
 544 -/*
 545 - *   Copyright (C) 1997 University of Chicago.
 546 +/*
 547 + *   Copyright (C) 1997 University of Chicago.
 548   *   See COPYRIGHT notice in top-level directory.
 549   *
 550   *   Copyright (C) 2007 Oak Ridge National Laboratory
 551 + *
 552 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
 553   */
 554
 555  #ifndef AD_UNIX_INCLUDE
 556 @@ -24,7 +26,32 @@
 557
 558  /*#include <fcntl.h>*/
 559  #include <sys/ioctl.h>
 560 +#ifdef WITH_LUSTRE
 561  #include "lustre/lustre_user.h"
 562 +#else
 563 +/* copy something from lustre_user.h here */
 564 +#  define LOV_USER_MAGIC 0x0BD10BD0
 565 +#  define LL_IOC_LOV_SETSTRIPE  _IOW ('f', 154, long)
 566 +#  define LL_IOC_LOV_GETSTRIPE  _IOW ('f', 155, long)
 567 +#  define lov_user_ost_data lov_user_ost_data_v1
 568 +struct lov_user_ost_data_v1 {     /* per-stripe data structure */
 569 +        __u64 l_object_id;        /* OST object ID */
 570 +        __u64 l_object_gr;        /* OST object group (creating MDS number) */
 571 +        __u32 l_ost_gen;          /* generation of this OST index */
 572 +        __u32 l_ost_idx;          /* OST index in LOV */
 573 +} __attribute__((packed));
 574 +#define lov_user_md lov_user_md_v1
 575 +struct lov_user_md_v1 {           /* LOV EA user data (host-endian) */
 576 +        __u32 lmm_magic;          /* magic number = LOV_USER_MAGIC_V1 */
 577 +        __u32 lmm_pattern;        /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */
 578 +        __u64 lmm_object_id;      /* LOV object ID */
 579 +        __u64 lmm_object_gr;      /* LOV object group */
 580 +        __u32 lmm_stripe_size;    /* size of stripe in bytes */
 581 +        __u16 lmm_stripe_count;   /* num stripes in use for this object */
 582 +        __u16 lmm_stripe_offset;  /* starting stripe offset in lmm_objects */
 583 +        struct lov_user_ost_data_v1 lmm_objects[0]; /* per-stripe data */
 584 +} __attribute__((packed));
 585 +#endif
 586  #include "adio.h"
 587  /*#include "adioi.h"*/
 588
 589 @@ -41,24 +68,31 @@
 590
 591  void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code);
 592  void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code);
 593 -void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count,
 594 -                      MPI_Datatype datatype, int file_ptr_type,
 595 -                     ADIO_Offset offset, ADIO_Status *status, int
 596 -                    *error_code);
 597 -void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count,
 598 -                      MPI_Datatype datatype, int file_ptr_type,
 599 -                      ADIO_Offset offset, ADIO_Status *status, int
 600 -                     *error_code);
 601 +void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count,
 602 +                             MPI_Datatype datatype, int file_ptr_type,
 603 +                             ADIO_Offset offset, ADIO_Status *status,
 604 +                             int *error_code);
 605 +void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count,
 606 +                              MPI_Datatype datatype, int file_ptr_type,
 607 +                              ADIO_Offset offset, ADIO_Status *status,
 608 +                              int *error_code);
 609 +void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count,
 610 +                              MPI_Datatype datatype, int file_ptr_type,
 611 +                              ADIO_Offset offset, ADIO_Status *status,
 612 +                              int *error_code);
 613  void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count,
 614 -                      MPI_Datatype datatype, int file_ptr_type,
 615 -                      ADIO_Offset offset, ADIO_Status *status, int
 616 -                      *error_code);
 617 +                                  MPI_Datatype datatype, int file_ptr_type,
 618 +                                  ADIO_Offset offset, ADIO_Status *status,
 619 +                                   int *error_code);
 620  void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count,
 621 -                      MPI_Datatype datatype, int file_ptr_type,
 622 -                      ADIO_Offset offset, ADIO_Status *status, int
 623 -                      *error_code);
 624 +                                 MPI_Datatype datatype, int file_ptr_type,
 625 +                                 ADIO_Offset offset, ADIO_Status *status,
 626 +                                  int *error_code);
 627 +void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count,
 628 +                             MPI_Datatype datatype, int file_ptr_type,
 629 +                             ADIO_Offset offset, ADIO_Status *status,
 630 +                              int *error_code);
 631  void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct,
 632                        int *error_code);
 633  void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
 634 -
 635  #endif /* End of AD_UNIX_INCLUDE */
 636 diff -ruN ad_lustre_orig/ad_lustre_hints.c ad_lustre/ad_lustre_hints.c
 637 --- ad_lustre_orig/ad_lustre_hints.c    2008-09-17 14:36:57.000000000 +0800
 638 +++ ad_lustre/ad_lustre_hints.c 2008-10-20 14:36:48.000000000 +0800
 639 @@ -1,9 +1,11 @@
 640  /* -*- Mode: C; c-basic-offset:4 ; -*- */
 641 -/*
 642 - *   Copyright (C) 1997 University of Chicago.
 643 +/*
 644 + *   Copyright (C) 1997 University of Chicago.
 645   *   See COPYRIGHT notice in top-level directory.
 646   *
 647   *   Copyright (C) 2007 Oak Ridge National Laboratory
 648 + *
 649 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
 650   */
 651
 652  #include "ad_lustre.h"
 653 @@ -11,130 +13,173 @@
 654
 655  void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
 656  {
 657 -    char *value, *value_in_fd;
 658 -    int flag, tmp_val[3], str_factor=-1, str_unit=0, start_iodev=-1;
 659 -    struct lov_user_md lum = { 0 };
 660 -    int err, myrank, fd_sys, perm, amode, old_mask;
 661 +    char *value = NULL;
 662 +    int flag, tmp_val, int_val, str_factor, str_unit, start_iodev;
 663 +    static char myname[] = "ADIOI_LUSTRE_SETINFO";
 664
 665 +    *error_code = MPI_SUCCESS;
 666      value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
 667 +
 668      if ( (fd->info) == MPI_INFO_NULL) {
 669 -       /* This must be part of the open call. can set striping parameters
 670 -           if necessary. */
 671 +       /* This must be part of the open call. can set striping parameters
 672 +           if necessary. */
 673         MPI_Info_create(&(fd->info));
 674
 675         MPI_Info_set(fd->info, "direct_read", "false");
 676         MPI_Info_set(fd->info, "direct_write", "false");
 677         fd->direct_read = fd->direct_write = 0;
 678 -
 679 -       /* has user specified striping or server buffering parameters
 680 +
 681 +       /* has user specified striping or server buffering parameters
 682             and do they have the same value on all processes? */
 683         if (users_info != MPI_INFO_NULL) {
 684 -           MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
 685 -                        value, &flag);
 686 -           if (flag)
 687 -               str_unit=atoi(value);
 688 -
 689 -           MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
 690 -                        value, &flag);
 691 -           if (flag)
 692 -               str_factor=atoi(value);
 693 -
 694 -           MPI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL,
 695 +            /* direct read and write */
 696 +           MPI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL,
 697                          value, &flag);
 698 -           if (flag)
 699 -               start_iodev=atoi(value);
 700 -
 701 -           MPI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL,
 702 -                            value, &flag);
 703             if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
 704                 MPI_Info_set(fd->info, "direct_read", "true");
 705                 fd->direct_read = 1;
 706             }
 707 -
 708 -           MPI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL,
 709 +           MPI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL,
 710                              value, &flag);
 711             if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
 712                 MPI_Info_set(fd->info, "direct_write", "true");
 713                 fd->direct_write = 1;
 714             }
 715 +            /*  stripe size */
 716 +           MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
 717 +                        value, &flag);
 718 +           if (flag && (str_unit = atoi(value))) {
 719 +               tmp_val = str_unit;
 720 +               MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 721 +               if (tmp_val != str_unit) {
 722 +                   MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 723 +                                                      "striping_unit",
 724 +                                                      error_code);
 725 +                    ADIOI_Free(value);
 726 +                   return;
 727 +               }
 728 +               MPI_Info_set(fd->info, "striping_unit", value);
 729 +           }
 730 +            /* stripe count */
 731 +           MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
 732 +                        value, &flag);
 733 +           if (flag && (str_factor = atoi(value))) {
 734 +               tmp_val = str_factor;
 735 +               MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 736 +               if (tmp_val != str_factor) {
 737 +                   MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 738 +                                                      "striping_factor",
 739 +                                                      error_code);
 740 +                    ADIOI_Free(value);
 741 +                   return;
 742 +               }
 743 +               MPI_Info_set(fd->info, "striping_factor", value);
 744 +           }
 745 +            /* stripe offset */
 746 +            MPI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL,
 747 +                        value, &flag);
 748 +           if (flag && ((start_iodev = atoi(value)) >= 0)) {
 749 +               tmp_val = start_iodev;
 750 +               MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 751 +               if (tmp_val != start_iodev) {
 752 +                   MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 753 +                                                      "start_iodevice",
 754 +                                                      error_code);
 755 +                    ADIOI_Free(value);
 756 +                   return;
 757 +               }
 758 +               MPI_Info_set(fd->info, "start_iodevice", value);
 759 +           }
 760         }
 761 -
 762 -       MPI_Comm_rank(fd->comm, &myrank);
 763 -       if (myrank == 0) {
 764 -           tmp_val[0] = str_factor;
 765 -           tmp_val[1] = str_unit;
 766 -           tmp_val[2] = start_iodev;
 767 +    }
 768 +    if (users_info != MPI_INFO_NULL) {
 769 +        /* CO: IO Clients/OST,
 770 +         * to keep the load balancing between clients and OSTs */
 771 +        MPI_Info_get(users_info, "CO", MPI_MAX_INFO_VAL, value,
 772 +                     &flag);
 773 +       if (flag && (int_val = atoi(value)) > 0) {
 774 +            tmp_val = int_val;
 775 +           MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 776 +           if (tmp_val != int_val) {
 777 +                MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 778 +                                                   "CO",
 779 +                                                   error_code);
 780 +                ADIOI_Free(value);
 781 +               return;
 782 +           }
 783 +           MPI_Info_set(fd->info, "CO", value);
 784         }
 785 -       MPI_Bcast(tmp_val, 3, MPI_INT, 0, fd->comm);
 786 -
 787 -       if (tmp_val[0] != str_factor
 788 -               || tmp_val[1] != str_unit
 789 -               || tmp_val[2] != start_iodev) {
 790 -           FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: All keys"
 791 -                   "-striping_factor:striping_unit:start_iodevice "
 792 -                   "need to be identical across all processes\n");
 793 -           MPI_Abort(MPI_COMM_WORLD, 1);
 794 -               } else if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
 795 -            /* if user has specified striping info, process 0 tries to set it */
 796 -           if (!myrank) {
 797 -               if (fd->perm == ADIO_PERM_NULL) {
 798 -                   old_mask = umask(022);
 799 -                   umask(old_mask);
 800 -                   perm = old_mask ^ 0666;
 801 -               }
 802 -               else perm = fd->perm;
 803 -
 804 -               amode = 0;
 805 -               if (fd->access_mode & ADIO_CREATE)
 806 -                   amode = amode | O_CREAT;
 807 -               if (fd->access_mode & ADIO_RDONLY)
 808 -                   amode = amode | O_RDONLY;
 809 -               if (fd->access_mode & ADIO_WRONLY)
 810 -                   amode = amode | O_WRONLY;
 811 -               if (fd->access_mode & ADIO_RDWR)
 812 -                   amode = amode | O_RDWR;
 813 -               if (fd->access_mode & ADIO_EXCL)
 814 -                   amode = amode | O_EXCL;
 815 -
 816 -               /* we need to create file so ensure this is set */
 817 -               amode = amode | O_LOV_DELAY_CREATE | O_CREAT;
 818 -
 819 -               fd_sys = open(fd->filename, amode, perm);
 820 -               if (fd_sys == -1) {
 821 -                   if (errno != EEXIST)
 822 -                       fprintf(stderr,
 823 -                               "Failure to open file %s %d %d\n",strerror(errno), amode, perm);
 824 -               } else {
 825 -                   lum.lmm_magic = LOV_USER_MAGIC;
 826 -                   lum.lmm_pattern = 0;
 827 -                   lum.lmm_stripe_size = str_unit;
 828 -                   lum.lmm_stripe_count = str_factor;
 829 -                   lum.lmm_stripe_offset = start_iodev;
 830 -
 831 -                   err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum);
 832 -                   if (err == -1 && errno != EEXIST) {
 833 -                       fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
 834 -                   }
 835 -                   close(fd_sys);
 836 -              }
 837 -           } /* End of striping parameters validation */
 838 +        /* big_req_size:
 839 +         * if the req size is bigger than this,
 840 +         * collective IO may not be performed.
 841 +         */
 842 +       MPI_Info_get(users_info, "big_req_size", MPI_MAX_INFO_VAL, value,
 843 +                     &flag);
 844 +       if (flag && (int_val = atoi(value)) > 0) {
 845 +            tmp_val = int_val;
 846 +           MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 847 +           if (tmp_val != int_val) {
 848 +               MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 849 +                                                  "big_req_size",
 850 +                                                  error_code);
 851 +                ADIOI_Free(value);
 852 +               return;
 853 +           }
 854 +           MPI_Info_set(fd->info, "big_req_size", value);
 855 +        }
 856 +        /* ds_in_coll: disable data sieving in collective IO */
 857 +       MPI_Info_get(users_info, "ds_in_coll", MPI_MAX_INFO_VAL,
 858 +                    value, &flag);
 859 +       if (flag && (!strcmp(value, "enable") ||
 860 +                     !strcmp(value, "ENABLE"))) {
 861 +            tmp_val = int_val = 1;
 862 +           MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 863 +           if (tmp_val != int_val) {
 864 +               MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 865 +                                                  "ds_in_coll",
 866 +                                                  error_code);
 867 +                ADIOI_Free(value);
 868 +                return;
 869 +           }
 870 +           MPI_Info_set(fd->info, "ds_in_coll", "enable");
 871 +       }
 872 +        /* contiguous_data: whether the data are contiguous */
 873 +       MPI_Info_get(users_info, "contiguous_data", MPI_MAX_INFO_VAL,
 874 +                    value, &flag);
 875 +        if (flag && (!strcmp(value, "yes") ||
 876 +                     !strcmp(value, "YES"))) {
 877 +            tmp_val = int_val = 1;
 878 +           MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 879 +           if (tmp_val != int_val) {
 880 +               MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 881 +                                                  "contiguous_data",
 882 +                                                  error_code);
 883 +                ADIOI_Free(value);
 884 +                return;
 885 +           }
 886 +           MPI_Info_set(fd->info, "contiguous_data", "yes");
 887 +       }
 888 +        /* same_io_size: whether the req size is same */
 889 +       MPI_Info_get(users_info, "same_io_size", MPI_MAX_INFO_VAL,
 890 +                    value, &flag);
 891 +        if (flag && (!strcmp(value, "yes") ||
 892 +                     !strcmp(value, "YES"))) {
 893 +            tmp_val = int_val = 1;
 894 +           MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
 895 +           if (tmp_val != int_val) {
 896 +               MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
 897 +                                                  "same_io_size",
 898 +                                                  error_code);
 899 +                ADIOI_Free(value);
 900 +                return;
 901 +           }
 902 +           MPI_Info_set(fd->info, "same_io_size", "yes");
 903         }
 904 -
 905 -       MPI_Barrier(fd->comm);
 906 -       /* set the values for collective I/O and data sieving parameters */
 907 -       ADIOI_GEN_SetInfo(fd, users_info, error_code);
 908 -    } else {
 909 -       /* The file has been opened previously and fd->fd_sys is a valid
 910 -           file descriptor. cannot set striping parameters now. */
 911 -
 912 -       /* set the values for collective I/O and data sieving parameters */
 913 -       ADIOI_GEN_SetInfo(fd, users_info, error_code);
 914      }
 915 -
 916 -    if (ADIOI_Direct_read) fd->direct_read = 1;
 917 -    if (ADIOI_Direct_write) fd->direct_write = 1;
 918 -
 919      ADIOI_Free(value);
 920 +    /* set the values for collective I/O and data sieving parameters */
 921 +    ADIOI_GEN_SetInfo(fd, users_info, error_code);
 922
 923 -    *error_code = MPI_SUCCESS;
 924 +    if (ADIOI_Direct_read) fd->direct_read = 1;
 925 +    if (ADIOI_Direct_write) fd->direct_write = 1;
 926  }
 927 diff -ruN ad_lustre_orig/ad_lustre_open.c ad_lustre/ad_lustre_open.c
 928 --- ad_lustre_orig/ad_lustre_open.c     2008-09-17 14:36:57.000000000 +0800
 929 +++ ad_lustre/ad_lustre_open.c  2008-09-17 18:55:50.000000000 +0800
 930 @@ -1,18 +1,21 @@
 931  /* -*- Mode: C; c-basic-offset:4 ; -*- */
 932 -/*
 933 - *   Copyright (C) 1997 University of Chicago.
 934 +/*
 935 + *   Copyright (C) 1997 University of Chicago.
 936   *   See COPYRIGHT notice in top-level directory.
 937   *
 938   *   Copyright (C) 2007 Oak Ridge National Laboratory
 939 + *
 940 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
 941   */
 942
 943  #include "ad_lustre.h"
 944
 945  void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code)
 946  {
 947 -    int perm, old_mask, amode, amode_direct;
 948 +    int perm, old_mask, amode = 0, amode_direct = 0, flag = 0, err, myrank;
 949 +    int stripe_size = 0, stripe_count = 0, stripe_offset = -1;
 950      struct lov_user_md lum = { 0 };
 951 -    char *value;
 952 +    char *value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
 953
 954  #if defined(MPICH2) || !defined(PRINT_ERR_MSG)
 955      static char myname[] = "ADIOI_LUSTRE_OPEN";
 956 @@ -22,12 +25,57 @@
 957         old_mask = umask(022);
 958         umask(old_mask);
 959         perm = old_mask ^ 0666;
 960 -    }
 961 -    else perm = fd->perm;
 962 +    } else
 963 +       perm = fd->perm;
 964
 965 -    amode = 0;
 966 -    if (fd->access_mode & ADIO_CREATE)
 967 +    if (fd->access_mode & ADIO_CREATE) {
 968         amode = amode | O_CREAT;
 969 +        /* Check striping info
 970 +         * if already set by SetInfo(), set them to lum; otherwise, set by lum
 971 +         */
 972 +        MPI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL, value,
 973 +                    &flag);
 974 +        if (flag)
 975 +           stripe_size = atoi(value);
 976 +
 977 +        MPI_Info_get(fd->info, "striping_factor", MPI_MAX_INFO_VAL, value,
 978 +                    &flag);
 979 +        if (flag)
 980 +           stripe_count = atoi(value);
 981 +
 982 +        MPI_Info_get(fd->info, "start_iodevice", MPI_MAX_INFO_VAL, value,
 983 +                    &flag);
 984 +        if (flag)
 985 +           stripe_offset = atoi(value);
 986 +
 987 +        /* if user has specified striping info,
 988 +         * process 0 will try to check and set it.
 989 +         */
 990 +        if ((stripe_size > 0) || (stripe_count > 0) || (stripe_offset >= 0)) {
 991 +           MPI_Comm_rank(fd->comm, &myrank);
 992 +           if (myrank == 0) {
 993 +               int fd_sys = open(fd->filename, amode, perm);
 994 +               if (fd_sys == -1) {
 995 +                   if (errno != EEXIST)
 996 +                       FPRINTF(stderr, "Failure to open file %s %d %d\n",
 997 +                               strerror(errno), amode, perm);
 998 +               } else {
 999 +                   lum.lmm_magic = LOV_USER_MAGIC;
1000 +                   lum.lmm_pattern = 1;
1001 +                   lum.lmm_stripe_size = stripe_size;
1002 +                   lum.lmm_stripe_count = stripe_count;
1003 +                   lum.lmm_stripe_offset = stripe_offset;
1004 +
1005 +                   if (ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum))
1006 +                       FPRINTF(stderr,
1007 +                               "Failure to set striping info to Lustre!\n");
1008 +                   close(fd_sys);
1009 +               }
1010 +           }
1011 +           MPI_Barrier(fd->comm);
1012 +        }
1013 +    }
1014 +
1015      if (fd->access_mode & ADIO_RDONLY)
1016         amode = amode | O_RDONLY;
1017      if (fd->access_mode & ADIO_WRONLY)
1018 @@ -42,32 +90,36 @@
1019      fd->fd_sys = open(fd->filename, amode|O_CREAT, perm);
1020
1021      if (fd->fd_sys != -1) {
1022 -        int err;
1023 -
1024 -        value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
1025 -
1026          /* get file striping information and set it in info */
1027 -        lum.lmm_magic = LOV_USER_MAGIC;
1028 -        err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *) &lum);
1029 -
1030 -        if (!err) {
1031 -            sprintf(value, "%d", lum.lmm_stripe_size);
1032 -            MPI_Info_set(fd->info, "striping_unit", value);
1033 -
1034 -            sprintf(value, "%d", lum.lmm_stripe_count);
1035 -            MPI_Info_set(fd->info, "striping_factor", value);
1036 -
1037 -            sprintf(value, "%d", lum.lmm_stripe_offset);
1038 -            MPI_Info_set(fd->info, "start_iodevice", value);
1039 -        }
1040 -        ADIOI_Free(value);
1041 +       lum.lmm_magic = LOV_USER_MAGIC;
1042 +       err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *) &lum);
1043
1044 +       if (!err) {
1045 +           if (lum.lmm_stripe_size && lum.lmm_stripe_count &&
1046 +                (lum.lmm_stripe_offset >= 0)) {
1047 +               sprintf(value, "%d", lum.lmm_stripe_size);
1048 +               MPI_Info_set(fd->info, "striping_unit", value);
1049 +
1050 +               sprintf(value, "%d", lum.lmm_stripe_count);
1051 +               MPI_Info_set(fd->info, "striping_factor", value);
1052 +
1053 +               sprintf(value, "%d", lum.lmm_stripe_offset);
1054 +               MPI_Info_set(fd->info, "start_iodevice", value);
1055 +           } else {
1056 +               FPRINTF(stderr, "Striping info is invalid!\n");
1057 +               ADIOI_Free(value);
1058 +               MPI_Abort(MPI_COMM_WORLD, 1);
1059 +           }
1060 +       } else {
1061 +           FPRINTF(stderr, "Failed to get striping info from Lustre!\n");
1062 +            ADIOI_Free(value);
1063 +           MPI_Abort(MPI_COMM_WORLD, 1);
1064 +       }
1065          if (fd->access_mode & ADIO_APPEND)
1066              fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
1067 -    }
1068 -
1069 +    }
1070      if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND))
1071 -       fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
1072 +        fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END);
1073
1074      fd->fd_direct = -1;
1075      if (fd->direct_write || fd->direct_read) {
1076 @@ -81,20 +133,22 @@
1077      }
1078
1079      /* --BEGIN ERROR HANDLING-- */
1080 -    if (fd->fd_sys == -1 || ((fd->fd_direct == -1) &&
1081 -               (fd->direct_write || fd->direct_read))) {
1082 +    if (fd->fd_sys == -1 || ((fd->fd_direct == -1) &&
1083 +       (fd->direct_write || fd->direct_read))) {
1084         if (errno == ENAMETOOLONG)
1085             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
1086 -                                              MPIR_ERR_RECOVERABLE, myname,
1087 -                                              __LINE__, MPI_ERR_BAD_FILE,
1088 +                                              MPIR_ERR_RECOVERABLE,
1089 +                                              myname, __LINE__,
1090 +                                              MPI_ERR_BAD_FILE,
1091                                                "**filenamelong",
1092                                                "**filenamelong %s %d",
1093                                                fd->filename,
1094                                                strlen(fd->filename));
1095         else if (errno == ENOENT)
1096             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
1097 -                                              MPIR_ERR_RECOVERABLE, myname,
1098 -                                              __LINE__, MPI_ERR_NO_SUCH_FILE,
1099 +                                              MPIR_ERR_RECOVERABLE,
1100 +                                              myname, __LINE__,
1101 +                                              MPI_ERR_NO_SUCH_FILE,
1102                                                "**filenoexist",
1103                                                "**filenoexist %s",
1104                                                fd->filename);
1105 @@ -108,27 +162,30 @@
1106                                                fd->filename);
1107         else if (errno == EACCES) {
1108             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
1109 -                                              MPIR_ERR_RECOVERABLE, myname,
1110 -                                              __LINE__, MPI_ERR_ACCESS,
1111 +                                              MPIR_ERR_RECOVERABLE,
1112 +                                              myname, __LINE__,
1113 +                                              MPI_ERR_ACCESS,
1114                                                "**fileaccess",
1115 -                                              "**fileaccess %s",
1116 -                                              fd->filename );
1117 -       }
1118 -       else if (errno == EROFS) {
1119 +                                              "**fileaccess %s",
1120 +                                              fd->filename);
1121 +       } else if (errno == EROFS) {
1122             /* Read only file or file system and write access requested */
1123             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
1124 -                                              MPIR_ERR_RECOVERABLE, myname,
1125 -                                              __LINE__, MPI_ERR_READ_ONLY,
1126 -                                              "**ioneedrd", 0 );
1127 -       }
1128 -       else {
1129 +                                              MPIR_ERR_RECOVERABLE,
1130 +                                              myname, __LINE__,
1131 +                                              MPI_ERR_READ_ONLY,
1132 +                                              "**ioneedrd", 0);
1133 +       } else {
1134             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
1135 -                                              MPIR_ERR_RECOVERABLE, myname,
1136 -                                              __LINE__, MPI_ERR_IO, "**io",
1137 +                                              MPIR_ERR_RECOVERABLE,
1138 +                                              myname, __LINE__,
1139 +                                              MPI_ERR_IO, "**io",
1140                                                "**io %s", strerror(errno));
1141         }
1142 -    }
1143 +    } else {
1144      /* --END ERROR HANDLING-- */
1145 -    else *error_code = MPI_SUCCESS;
1146 +        *error_code = MPI_SUCCESS;
1147 +    }
1148
1149 +    ADIOI_Free(value);
1150  }
1151 diff -ruN ad_lustre_orig/ad_lustre_rwcontig.c ad_lustre/ad_lustre_rwcontig.c
1152 --- ad_lustre_orig/ad_lustre_rwcontig.c 2008-09-17 14:36:57.000000000 +0800
1153 +++ ad_lustre/ad_lustre_rwcontig.c      2008-10-15 22:44:35.000000000 +0800
1154 @@ -1,9 +1,11 @@
1155  /* -*- Mode: C; c-basic-offset:4 ; -*- */
1156 -/*
1157 - *   Copyright (C) 1997 University of Chicago.
1158 +/*
1159 + *   Copyright (C) 1997 University of Chicago.
1160   *   See COPYRIGHT notice in top-level directory.
1161   *
1162   *   Copyright (C) 2007 Oak Ridge National Laboratory
1163 + *
1164 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
1165   */
1166
1167  #define _XOPEN_SOURCE 600
1168 diff -ruN ad_lustre_orig/ad_lustre_wrcoll.c ad_lustre/ad_lustre_wrcoll.c
1169 --- ad_lustre_orig/ad_lustre_wrcoll.c   1970-01-01 08:00:00.000000000 +0800
1170 +++ ad_lustre/ad_lustre_wrcoll.c        2008-10-17 16:34:36.000000000 +0800
1171 @@ -0,0 +1,880 @@
1172 +/* -*- Mode: C; c-basic-offset:4 ; -*- */
1173 +/*
1174 + *   Copyright (C) 1997 University of Chicago.
1175 + *   See COPYRIGHT notice in top-level directory.
1176 + *
1177 + *   Copyright (C) 2007 Oak Ridge National Laboratory
1178 + *
1179 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
1180 + */
1181 +
1182 +#include "ad_lustre.h"
1183 +#include "adio_extern.h"
1184 +
1185 +/* prototypes of functions used for collective writes only. */
1186 +static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, void *buf,
1187 +                                       MPI_Datatype datatype, int nprocs,
1188 +                                       int myrank,
1189 +                                       ADIOI_Access *others_req,
1190 +                                       ADIOI_Access *my_req,
1191 +                                       ADIO_Offset *offset_list,
1192 +                                       int *len_list,
1193 +                                       int contig_access_count,
1194 +                                       int * striping_info,
1195 +                                       int *buf_idx, int *error_code);
1196 +static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, void *buf,
1197 +                                         ADIOI_Flatlist_node * flat_buf,
1198 +                                         char **send_buf,
1199 +                                         ADIO_Offset * offset_list,
1200 +                                         int *len_list, int *send_size,
1201 +                                         MPI_Request * requests,
1202 +                                         int *sent_to_proc, int nprocs,
1203 +                                         int myrank, int contig_access_count,
1204 +                                         int * striping_info,
1205 +                                         int *send_buf_idx,
1206 +                                          int *curr_to_proc,
1207 +                                         int *done_to_proc, int iter,
1208 +                                         MPI_Aint buftype_extent);
1209 +static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, void *buf,
1210 +                                        char *write_buf,
1211 +                                        ADIOI_Flatlist_node * flat_buf,
1212 +                                        ADIO_Offset * offset_list,
1213 +                                        int *len_list, int *send_size,
1214 +                                        int *recv_size, ADIO_Offset off,
1215 +                                        int size, int *count,
1216 +                                        int *start_pos, int *partial_recv,
1217 +                                        int *sent_to_proc, int nprocs,
1218 +                                        int myrank, int buftype_is_contig,
1219 +                                        int contig_access_count,
1220 +                                        int * striping_info,
1221 +                                        ADIOI_Access * others_req,
1222 +                                        int *send_buf_idx,
1223 +                                        int *curr_to_proc,
1224 +                                        int *done_to_proc, int *hole,
1225 +                                        int iter, MPI_Aint buftype_extent,
1226 +                                        int *buf_idx, int *error_code);
1227 +void ADIOI_Heap_merge(ADIOI_Access * others_req, int *count,
1228 +                      ADIO_Offset * srt_off, int *srt_len, int *start_pos,
1229 +                      int nprocs, int nprocs_recv, int total_elements);
1230 +
1231 +void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count,
1232 +                                  MPI_Datatype datatype,
1233 +                                  int file_ptr_type, ADIO_Offset offset,
1234 +                                  ADIO_Status * status, int *error_code)
1235 +{
1236 +    ADIOI_Access *my_req;
1237 +    /* array of nprocs access structures, one for each other process has
1238 +       this process's request */
1239 +
1240 +    ADIOI_Access *others_req;
1241 +    /* array of nprocs access structures, one for each other process
1242 +       whose request is written by this process. */
1243 +
1244 +    int i, filetype_is_contig, nprocs, myrank, do_collect = 0;
1245 +    int contig_access_count = 0, buftype_is_contig, interleave_count = 0;
1246 +    int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs;
1247 +    ADIO_Offset orig_fp, start_offset, end_offset, off;
1248 +    ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *end_offsets = NULL;
1249 +    int *buf_idx = NULL, *len_list = NULL, *striping_info = NULL;
1250 +    int old_error, tmp_error;
1251 +
1252 +    MPI_Comm_size(fd->comm, &nprocs);
1253 +    MPI_Comm_rank(fd->comm, &myrank);
1254 +
1255 +    orig_fp = fd->fp_ind;
1256 +
1257 +    /* IO patten identification if cb_write isn't disabled */
1258 +    if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
1259 +       /* For this process's request, calculate the list of offsets and
1260 +          lengths in the file and determine the start and end offsets. */
1261 +       ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset,
1262 +                             &offset_list, &len_list, &start_offset,
1263 +                             &end_offset, &contig_access_count);
1264 +
1265 +       /* each process communicates its start and end offsets to other
1266 +          processes. The result is an array each of start and end offsets stored
1267 +          in order of process rank. */
1268 +       st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
1269 +       end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset));
1270 +       MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1,
1271 +                     ADIO_OFFSET, fd->comm);
1272 +       MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1,
1273 +                     ADIO_OFFSET, fd->comm);
1274 +       /* are the accesses of different processes interleaved? */
1275 +       for (i = 1; i < nprocs; i++)
1276 +           if ((st_offsets[i] < end_offsets[i-1]) &&
1277 +                (st_offsets[i] <= end_offsets[i]))
1278 +                interleave_count++;
1279 +       /* This is a rudimentary check for interleaving, but should suffice
1280 +          for the moment. */
1281 +
1282 +       /* Two typical access patterns can benefit from collective write.
1283 +         *   1) the processes are interleaved, and
1284 +         *   2) the req size is small.
1285 +         */
1286 +        if (interleave_count > 0) {
1287 +           do_collect = 1;
1288 +        } else {
1289 +            do_collect = ADIOI_LUSTRE_Docollect(fd, contig_access_count,
1290 +                                               len_list, nprocs);
1291 +        }
1292 +    }
1293 +    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
1294 +
1295 +    /* Decide if collective I/O should be done */
1296 +    if ((!do_collect && fd->hints->cb_write == ADIOI_HINT_AUTO) ||
1297 +        fd->hints->cb_write == ADIOI_HINT_DISABLE) {
1298 +
1299 +       int filerange_is_contig = 0;
1300 +
1301 +       /* use independent accesses */
1302 +       if (fd->hints->cb_write != ADIOI_HINT_DISABLE) {
1303 +           ADIOI_Free(offset_list);
1304 +           ADIOI_Free(len_list);
1305 +            ADIOI_Free(st_offsets);
1306 +            ADIOI_Free(end_offsets);
1307 +       }
1308 +
1309 +       fd->fp_ind = orig_fp;
1310 +       ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
1311 +       if (buftype_is_contig && filetype_is_contig) {
1312 +           if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
1313 +               off = fd->disp + (fd->etype_size) * offset;
1314 +               ADIO_WriteContig(fd, buf, count, datatype,
1315 +                                ADIO_EXPLICIT_OFFSET,
1316 +                                off, status, error_code);
1317 +           } else
1318 +               ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL,
1319 +                                0, status, error_code);
1320 +       } else {
1321 +           ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type,
1322 +                             offset, status, error_code);
1323 +       }
1324 +       return;
1325 +    }
1326 +
1327 +    /* Get Lustre hints information */
1328 +    ADIOI_LUSTRE_Get_striping_info(fd, &striping_info, 1);
1329 +    /* calculate what portions of the access requests of this process are
1330 +     * located in which process
1331 +     */
1332 +    ADIOI_LUSTRE_Calc_my_req(fd, offset_list, len_list, contig_access_count,
1333 +                             striping_info, nprocs, &count_my_req_procs,
1334 +                             &count_my_req_per_proc, &my_req, &buf_idx);
1335 +    /* calculate what process's requests will be written by this process */
1336 +    ADIOI_LUSTRE_Calc_others_req(fd, count_my_req_procs,
1337 +                                 count_my_req_per_proc,
1338 +                                my_req, nprocs, myrank,
1339 +                                 start_offset, end_offset, striping_info,
1340 +                                 &count_others_req_procs, &others_req);
1341 +    ADIOI_Free(count_my_req_per_proc);
1342 +
1343 +    /* exchange data and write in sizes of no more than stripe_size. */
1344 +    ADIOI_LUSTRE_Exch_and_write(fd, buf, datatype, nprocs, myrank,
1345 +                                others_req, my_req,
1346 +                                offset_list, len_list, contig_access_count,
1347 +                               striping_info, buf_idx, error_code);
1348 +
1349 +    old_error = *error_code;
1350 +    if (*error_code != MPI_SUCCESS)
1351 +       *error_code = MPI_ERR_IO;
1352 +
1353 +    /* optimization: if only one process performing i/o, we can perform
1354 +     * a less-expensive Bcast  */
1355 +#ifdef ADIOI_MPE_LOGGING
1356 +    MPE_Log_event(ADIOI_MPE_postwrite_a, 0, NULL);
1357 +#endif
1358 +    if (fd->hints->cb_nodes == 1)
1359 +       MPI_Bcast(error_code, 1, MPI_INT,
1360 +                 fd->hints->ranklist[0], fd->comm);
1361 +    else {
1362 +       tmp_error = *error_code;
1363 +       MPI_Allreduce(&tmp_error, error_code, 1, MPI_INT,
1364 +                     MPI_MAX, fd->comm);
1365 +    }
1366 +#ifdef ADIOI_MPE_LOGGING
1367 +    MPE_Log_event(ADIOI_MPE_postwrite_b, 0, NULL);
1368 +#endif
1369 +
1370 +    if ((old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO))
1371 +       *error_code = old_error;
1372 +
1373 +
1374 +    if (!buftype_is_contig)
1375 +       ADIOI_Delete_flattened(datatype);
1376 +
1377 +    /* free all memory allocated for collective I/O */
1378 +    /* free others_req */
1379 +    for (i = 0; i < nprocs; i++) {
1380 +       if (others_req[i].count) {
1381 +           ADIOI_Free(others_req[i].offsets);
1382 +           ADIOI_Free(others_req[i].lens);
1383 +           ADIOI_Free(others_req[i].mem_ptrs);
1384 +       }
1385 +    }
1386 +    ADIOI_Free(others_req);
1387 +    /* free my_req here */
1388 +    for (i = 0; i < nprocs; i++) {
1389 +       if (my_req[i].count) {
1390 +           ADIOI_Free(my_req[i].offsets);
1391 +           ADIOI_Free(my_req[i].lens);
1392 +       }
1393 +    }
1394 +    ADIOI_Free(my_req);
1395 +    ADIOI_Free(buf_idx);
1396 +    ADIOI_Free(offset_list);
1397 +    ADIOI_Free(len_list);
1398 +    ADIOI_Free(st_offsets);
1399 +    ADIOI_Free(end_offsets);
1400 +    ADIOI_Free(striping_info);
1401 +
1402 +#ifdef HAVE_STATUS_SET_BYTES
1403 +    if (status) {
1404 +       int bufsize, size;
1405 +       /* Don't set status if it isn't needed */
1406 +       MPI_Type_size(datatype, &size);
1407 +       bufsize = size * count;
1408 +       MPIR_Status_set_bytes(status, datatype, bufsize);
1409 +    }
1410 +    /* This is a temporary way of filling in status. The right way is to
1411 +     * keep track of how much data was actually written during collective I/O.
1412 +     */
1413 +#endif
1414 +
1415 +    fd->fp_sys_posn = -1;      /* set it to null. */
1416 +}
1417 +
1418 +static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, void *buf,
1419 +                                       MPI_Datatype datatype, int nprocs,
1420 +                                       int myrank, ADIOI_Access *others_req,
1421 +                                        ADIOI_Access *my_req,
1422 +                                       ADIO_Offset *offset_list,
1423 +                                        int *len_list, int contig_access_count,
1424 +                                       int *striping_info, int *buf_idx,
1425 +                                        int *error_code)
1426 +{
1427 +    int hole, i, j, m, flag, ntimes = 1 , max_ntimes, buftype_is_contig;
1428 +    ADIO_Offset st_loc = -1, end_loc = -1, min_st_loc, max_end_loc;
1429 +    ADIO_Offset off, req_off, send_off, iter_st_off, *off_list;
1430 +    ADIO_Offset max_size, step_size = 0;
1431 +    int real_size, req_len, send_len;
1432 +    int *recv_curr_offlen_ptr, *recv_count, *recv_size;
1433 +    int *send_curr_offlen_ptr, *send_size;
1434 +    int *partial_recv, *sent_to_proc, *recv_start_pos;
1435 +    int *send_buf_idx, *curr_to_proc, *done_to_proc;
1436 +    char *write_buf = NULL, *value;
1437 +    MPI_Status status;
1438 +    ADIOI_Flatlist_node *flat_buf = NULL;
1439 +    MPI_Aint buftype_extent;
1440 +    int stripe_size = striping_info[0], avail_cb_nodes = striping_info[2];
1441 +    int lflag, data_sieving = 0;
1442 +
1443 +    *error_code = MPI_SUCCESS; /* changed below if error */
1444 +
1445 +    /* calculate the number of writes of stripe size to be done.
1446 +     * That gives the no. of communication phases as well.
1447 +     * Note:
1448 +     *   Because we redistribute data in stripe-contiguous pattern for Lustre,
1449 +     *   each process has the same no. of communication phases.
1450 +     */
1451 +
1452 +    for (i = 0; i < nprocs; i++) {
1453 +       if (others_req[i].count) {
1454 +           st_loc = others_req[i].offsets[0];
1455 +           end_loc = others_req[i].offsets[0];
1456 +           break;
1457 +       }
1458 +    }
1459 +    for (i = 0; i < nprocs; i++) {
1460 +       for (j = 0; j < others_req[i].count; j++) {
1461 +           st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]);
1462 +           end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j] +
1463 +                                          others_req[i].lens[j] - 1));
1464 +       }
1465 +    }
1466 +    /* this process does no writing. */
1467 +    if ((st_loc == -1) && (end_loc == -1))
1468 +       ntimes = 0;
1469 +    MPI_Allreduce(&end_loc, &max_end_loc, 1, MPI_LONG_LONG_INT, MPI_MAX, fd->comm);
1470 +    /* avoid min_st_loc be -1 */
1471 +    if (st_loc == -1)
1472 +        st_loc = max_end_loc;
1473 +    MPI_Allreduce(&st_loc, &min_st_loc, 1, MPI_LONG_LONG_INT, MPI_MIN, fd->comm);
1474 +    /* align downward */
1475 +    min_st_loc -= min_st_loc % (ADIO_Offset)stripe_size;
1476 +
1477 +    /* Each time, only avail_cb_nodes number of IO clients perform IO,
1478 +     * so, step_size=avail_cb_nodes*stripe_size IO will be performed at most,
1479 +     * and ntimes=whole_file_portion/step_size
1480 +     */
1481 +    step_size = (ADIO_Offset) avail_cb_nodes * stripe_size;
1482 +    max_ntimes = (int)((max_end_loc - min_st_loc) / step_size + 1);
1483 +    if (ntimes)
1484 +       write_buf = (char *) ADIOI_Malloc(stripe_size);
1485 +
1486 +    /* calculate the start offset for each iteration */
1487 +    off_list = (ADIO_Offset *) ADIOI_Malloc(max_ntimes * sizeof(ADIO_Offset));
1488 +    for (m = 0; m < max_ntimes; m ++)
1489 +        off_list[m] = max_end_loc;
1490 +    for (i = 0; i < nprocs; i++) {
1491 +        for (j = 0; j < others_req[i].count; j ++) {
1492 +            req_off = others_req[i].offsets[j];
1493 +            m = (int)((req_off - min_st_loc) / step_size);
1494 +            off_list[m] = ADIOI_MIN(off_list[m], req_off);
1495 +        }
1496 +    }
1497 +
1498 +    recv_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
1499 +    send_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int));
1500 +    /* their use is explained below. calloc initializes to 0. */
1501 +
1502 +    recv_count = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1503 +    /* to store count of how many off-len pairs per proc are satisfied
1504 +       in an iteration. */
1505 +
1506 +    send_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1507 +    /* total size of data to be sent to each proc. in an iteration.
1508 +       Of size nprocs so that I can use MPI_Alltoall later. */
1509 +
1510 +    recv_size = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1511 +    /* total size of data to be recd. from each proc. in an iteration. */
1512 +
1513 +    sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int));
1514 +    /* amount of data sent to each proc so far. Used in
1515 +       ADIOI_Fill_send_buffer. initialized to 0 here. */
1516 +
1517 +    send_buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1518 +    curr_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1519 +    done_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1520 +    /* Above three are used in ADIOI_Fill_send_buffer */
1521 +
1522 +    recv_start_pos = (int *) ADIOI_Malloc(nprocs * sizeof(int));
1523 +    /* used to store the starting value of recv_curr_offlen_ptr[i] in
1524 +       this iteration */
1525 +
1526 +    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
1527 +    if (!buftype_is_contig) {
1528 +       ADIOI_Flatten_datatype(datatype);
1529 +       flat_buf = ADIOI_Flatlist;
1530 +       while (flat_buf->type != datatype)
1531 +           flat_buf = flat_buf->next;
1532 +    }
1533 +    MPI_Type_extent(datatype, &buftype_extent);
1534 +
1535 +    iter_st_off = min_st_loc;
1536 +
1537 +    /* Although we have recognized the data according to OST index,
1538 +     * a read-modify-write will be done if there is a hole between the data.
1539 +     * For example: if blocksize=60, xfersize=30 and stripe_size=100,
1540 +     * then rank0 will collect data [0, 30] and [60, 90] then write. There
1541 +     * is a hole in [30, 60], which will cause a read-modify-write in [0, 90].
1542 +     *
1543 +     * To reduce its impact on the performance, we disable data sieving
1544 +     * by default, unless the hint "ds_in_coll" is enabled.
1545 +     */
1546 +    /* check the hint for data sieving */
1547 +    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
1548 +    MPI_Info_get(fd->info, "ds_in_coll", MPI_MAX_INFO_VAL, value, &lflag);
1549 +    if (lflag && !strcmp(value, "enable"))
1550 +        data_sieving = 1;
1551 +    ADIOI_Free(value);
1552 +
1553 +    for (m = 0; m < max_ntimes; m++) {
1554 +       /* go through all others_req and my_req to check which will be received
1555 +         * and sent in this iteration.
1556 +         */
1557 +
1558 +       /* Note that MPI guarantees that displacements in filetypes are in
1559 +          monotonically nondecreasing order and that, for writes, the
1560 +          filetypes cannot specify overlapping regions in the file. This
1561 +          simplifies implementation a bit compared to reads. */
1562 +
1563 +       /*
1564 +           off         = start offset in the file for the data to be written in
1565 +                         this iteration
1566 +           iter_st_off = start offset of this iteration
1567 +           real_size   = size of data written (bytes) corresponding to off
1568 +           max_size    = possible maximum size of data written in this iteration
1569 +           req_off     = offset in the file for a particular contiguous request minus
1570 +                         what was satisfied in previous iteration
1571 +           send_off    = offset the request needed by other processes in this iteration
1572 +           req_len     = size corresponding to req_off
1573 +           send_len    = size corresponding to send_off
1574 +         */
1575 +
1576 +       /* first calculate what should be communicated */
1577 +       for (i = 0; i < nprocs; i++)
1578 +           recv_count[i] = recv_size[i] = send_size[i] = 0;
1579 +
1580 +        off = off_list[m];
1581 +        max_size = ADIOI_MIN(step_size, max_end_loc - iter_st_off + 1);
1582 +        real_size = (int) ADIOI_MIN((off / stripe_size + 1) * stripe_size - off,
1583 +                                    end_loc - off + 1);
1584 +
1585 +       for (i = 0; i < nprocs; i++) {
1586 +            if (my_req[i].count) {
1587 +                for (j = send_curr_offlen_ptr[i]; j < my_req[i].count; j++) {
1588 +                    send_off = my_req[i].offsets[j];
1589 +                    send_len = my_req[i].lens[j];
1590 +                    if (send_off < iter_st_off + max_size) {
1591 +                        send_size[i] += send_len;
1592 +                    } else {
1593 +                        break;
1594 +                    }
1595 +                }
1596 +                send_curr_offlen_ptr[i] = j;
1597 +            }
1598 +           if (others_req[i].count) {
1599 +               recv_start_pos[i] = recv_curr_offlen_ptr[i];
1600 +               for (j = recv_curr_offlen_ptr[i]; j < others_req[i].count; j++) {
1601 +                    req_off = others_req[i].offsets[j];
1602 +                    req_len = others_req[i].lens[j];
1603 +                   if (req_off < iter_st_off + max_size) {
1604 +                       recv_count[i]++;
1605 +                       MPI_Address(write_buf + req_off - off,
1606 +                                   &(others_req[i].mem_ptrs[j]));
1607 +                        recv_size[i] += req_len;
1608 +                   } else {
1609 +                       break;
1610 +                    }
1611 +               }
1612 +               recv_curr_offlen_ptr[i] = j;
1613 +           }
1614 +       }
1615 +        /* use variable "hole" to pass data_sieving flag into W_Exchange_data */
1616 +        hole = data_sieving;
1617 +       ADIOI_LUSTRE_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list,
1618 +                                     len_list, send_size, recv_size, off, real_size,
1619 +                                     recv_count, recv_start_pos, partial_recv,
1620 +                                     sent_to_proc, nprocs, myrank,
1621 +                                     buftype_is_contig, contig_access_count,
1622 +                                     striping_info, others_req, send_buf_idx,
1623 +                                     curr_to_proc, done_to_proc, &hole, m,
1624 +                                     buftype_extent, buf_idx, error_code);
1625 +       if (*error_code != MPI_SUCCESS)
1626 +            goto over;
1627 +
1628 +       flag = 0;
1629 +       for (i = 0; i < nprocs; i++)
1630 +           if (recv_count[i]) {
1631 +               flag = 1;
1632 +               break;
1633 +           }
1634 +       if (flag) {
1635 +            /* check whether to do data sieving */
1636 +            if(data_sieving) {
1637 +               ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE,
1638 +                                ADIO_EXPLICIT_OFFSET, off, &status,
1639 +                                error_code);
1640 +            } else {
1641 +                /* if there is no hole, write data in one time;
1642 +                 * otherwise, write data in several times */
1643 +                if (!hole) {
1644 +                    ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE,
1645 +                                     ADIO_EXPLICIT_OFFSET, off, &status,
1646 +                                     error_code);
1647 +                } else {
1648 +                    for (i = 0; i < nprocs; i++) {
1649 +                        if (others_req[i].count) {
1650 +                            for (j = 0; j < others_req[i].count; j++) {
1651 +                                if (others_req[i].offsets[j] < off + real_size &&
1652 +                                    others_req[i].offsets[j] >= off) {
1653 +                                    ADIO_WriteContig(fd,
1654 +                                                     write_buf + others_req[i].offsets[j] - off,
1655 +                                                     others_req[i].lens[j],
1656 +                                                     MPI_BYTE, ADIO_EXPLICIT_OFFSET,
1657 +                                                     others_req[i].offsets[j], &status,
1658 +                                                     error_code);
1659 +                                   if (*error_code != MPI_SUCCESS)
1660 +                                       goto over;
1661 +                                }
1662 +                            }
1663 +                        }
1664 +                    }
1665 +                }
1666 +            }
1667 +           if (*error_code != MPI_SUCCESS)
1668 +               goto over;
1669 +       }
1670 +        iter_st_off += max_size;
1671 +    }
1672 +over:
1673 +    if (ntimes)
1674 +       ADIOI_Free(write_buf);
1675 +    ADIOI_Free(recv_curr_offlen_ptr);
1676 +    ADIOI_Free(send_curr_offlen_ptr);
1677 +    ADIOI_Free(recv_count);
1678 +    ADIOI_Free(send_size);
1679 +    ADIOI_Free(recv_size);
1680 +    ADIOI_Free(sent_to_proc);
1681 +    ADIOI_Free(recv_start_pos);
1682 +    ADIOI_Free(send_buf_idx);
1683 +    ADIOI_Free(curr_to_proc);
1684 +    ADIOI_Free(done_to_proc);
1685 +    ADIOI_Free(off_list);
1686 +}
1687 +
1688 +static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, void *buf,
1689 +                                        char *write_buf,
1690 +                                        ADIOI_Flatlist_node * flat_buf,
1691 +                                        ADIO_Offset * offset_list,
1692 +                                        int *len_list, int *send_size,
1693 +                                        int *recv_size, ADIO_Offset off,
1694 +                                        int size, int *count,
1695 +                                        int *start_pos, int *partial_recv,
1696 +                                        int *sent_to_proc, int nprocs,
1697 +                                        int myrank, int buftype_is_contig,
1698 +                                        int contig_access_count,
1699 +                                        int * striping_info,
1700 +                                        ADIOI_Access * others_req,
1701 +                                        int *send_buf_idx,
1702 +                                        int *curr_to_proc, int *done_to_proc,
1703 +                                         int *hole, int iter,
1704 +                                         MPI_Aint buftype_extent,
1705 +                                        int *buf_idx, int *error_code)
1706 +{
1707 +    int i, j, nprocs_recv, nprocs_send, err;
1708 +    char **send_buf = NULL;
1709 +    MPI_Request *requests, *send_req;
1710 +    MPI_Datatype *recv_types;
1711 +    MPI_Status *statuses, status;
1712 +    int *srt_len, sum, sum_recv;
1713 +    ADIO_Offset *srt_off;
1714 +    int data_sieving = *hole;
1715 +    static char myname[] = "ADIOI_W_EXCHANGE_DATA";
1716 +
1717 +    /* create derived datatypes for recv */
1718 +    nprocs_recv = 0;
1719 +    for (i = 0; i < nprocs; i++)
1720 +       if (recv_size[i])
1721 +           nprocs_recv++;
1722 +
1723 +    recv_types = (MPI_Datatype *) ADIOI_Malloc((nprocs_recv + 1) *
1724 +                                              sizeof(MPI_Datatype));
1725 +    /* +1 to avoid a 0-size malloc */
1726 +
1727 +    j = 0;
1728 +    for (i = 0; i < nprocs; i++) {
1729 +       if (recv_size[i]) {
1730 +           MPI_Type_hindexed(count[i],
1731 +                             &(others_req[i].lens[start_pos[i]]),
1732 +                             &(others_req[i].mem_ptrs[start_pos[i]]),
1733 +                             MPI_BYTE, recv_types + j);
1734 +           /* absolute displacements; use MPI_BOTTOM in recv */
1735 +           MPI_Type_commit(recv_types + j);
1736 +           j++;
1737 +       }
1738 +    }
1739 +
1740 +    /* To avoid a read-modify-write,
1741 +     * check if there are holes in the data to be written.
1742 +     * For this, merge the (sorted) offset lists others_req using a heap-merge.
1743 +     */
1744 +
1745 +    sum = 0;
1746 +    for (i = 0; i < nprocs; i++)
1747 +       sum += count[i];
1748 +    srt_off = (ADIO_Offset *) ADIOI_Malloc((sum + 1) * sizeof(ADIO_Offset));
1749 +    srt_len = (int *) ADIOI_Malloc((sum + 1) * sizeof(int));
1750 +    /* +1 to avoid a 0-size malloc */
1751 +
1752 +    ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos,
1753 +                    nprocs, nprocs_recv, sum);
1754 +
1755 +    /* check if there are any holes */
1756 +    *hole = 0;
1757 +    for (i = 0; i < sum - 1; i++) {
1758 +        if (srt_off[i] + srt_len[i] < srt_off[i + 1]) {
1759 +            *hole = 1;
1760 +           break;
1761 +       }
1762 +    }
1763 +    /* In some cases (see John Bent ROMIO REQ # 835), an odd interaction
1764 +     * between aggregation, nominally contiguous regions, and cb_buffer_size
1765 +     * should be handled with a read-modify-write (otherwise we will write out
1766 +     * more data than we receive from everyone else (inclusive), so override
1767 +     * hole detection
1768 +     */
1769 +    if (*hole == 0) {
1770 +        sum_recv = 0;
1771 +        for (i = 0; i < nprocs; i++)
1772 +            sum_recv += recv_size[i];
1773 +       if (size > sum_recv)
1774 +           *hole = 1;
1775 +    }
1776 +    /* check the hint for data sieving */
1777 +    if (data_sieving && nprocs_recv && *hole) {
1778 +        ADIO_ReadContig(fd, write_buf, size, MPI_BYTE,
1779 +                        ADIO_EXPLICIT_OFFSET, off, &status, &err);
1780 +        // --BEGIN ERROR HANDLING--
1781 +        if (err != MPI_SUCCESS) {
1782 +            *error_code = MPIO_Err_create_code(err,
1783 +                                               MPIR_ERR_RECOVERABLE,
1784 +                                               myname, __LINE__,
1785 +                                               MPI_ERR_IO,
1786 +                                               "**ioRMWrdwr", 0);
1787 +            ADIOI_Free(recv_types);
1788 +            ADIOI_Free(srt_off);
1789 +            ADIOI_Free(srt_len);
1790 +            return;
1791 +        }
1792 +        // --END ERROR HANDLING--
1793 +    }
1794 +    ADIOI_Free(srt_off);
1795 +    ADIOI_Free(srt_len);
1796 +
1797 +    nprocs_send = 0;
1798 +    for (i = 0; i < nprocs; i++)
1799 +       if (send_size[i])
1800 +           nprocs_send++;
1801 +
1802 +    if (fd->atomicity) {
1803 +       /* bug fix from Wei-keng Liao and Kenin Coloma */
1804 +       requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + 1) *
1805 +                                                sizeof(MPI_Request));
1806 +       send_req = requests;
1807 +    } else {
1808 +       requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1)*
1809 +                                                sizeof(MPI_Request));
1810 +       /* +1 to avoid a 0-size malloc */
1811 +
1812 +       /* post receives */
1813 +       j = 0;
1814 +       for (i = 0; i < nprocs; i++) {
1815 +           if (recv_size[i]) {
1816 +               MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i,
1817 +                         myrank + i + 100 * iter, fd->comm, requests + j);
1818 +               j++;
1819 +           }
1820 +       }
1821 +       send_req = requests + nprocs_recv;
1822 +    }
1823 +
1824 +    /* post sends.
1825 +     * if buftype_is_contig, data can be directly sent from
1826 +     * user buf at location given by buf_idx. else use send_buf.
1827 +     */
1828 +    if (buftype_is_contig) {
1829 +       j = 0;
1830 +       for (i = 0; i < nprocs; i++)
1831 +           if (send_size[i]) {
1832 +               MPI_Isend(((char *) buf) + buf_idx[i], send_size[i],
1833 +                         MPI_BYTE, i, myrank + i + 100 * iter, fd->comm,
1834 +                         send_req + j);
1835 +               j++;
1836 +               buf_idx[i] += send_size[i];
1837 +           }
1838 +    } else if (nprocs_send) {
1839 +       /* buftype is not contig */
1840 +       send_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char *));
1841 +       for (i = 0; i < nprocs; i++)
1842 +           if (send_size[i])
1843 +               send_buf[i] = (char *) ADIOI_Malloc(send_size[i]);
1844 +
1845 +       ADIOI_LUSTRE_Fill_send_buffer(fd, buf, flat_buf, send_buf, offset_list,
1846 +                                      len_list, send_size, send_req,
1847 +                                      sent_to_proc, nprocs, myrank,
1848 +                                      contig_access_count, striping_info,
1849 +                                      send_buf_idx, curr_to_proc, done_to_proc,
1850 +                                      iter, buftype_extent);
1851 +       /* the send is done in ADIOI_Fill_send_buffer */
1852 +    }
1853 +
1854 +       /* bug fix from Wei-keng Liao and Kenin Coloma */
1855 +    if (fd->atomicity) {
1856 +       j = 0;
1857 +       for (i = 0; i < nprocs; i++) {
1858 +           MPI_Status wkl_status;
1859 +           if (recv_size[i]) {
1860 +               MPI_Recv(MPI_BOTTOM, 1, recv_types[j], i,
1861 +                        myrank + i + 100 * iter, fd->comm, &wkl_status);
1862 +               j++;
1863 +           }
1864 +       }
1865 +    }
1866 +
1867 +    for (i = 0; i < nprocs_recv; i++)
1868 +       MPI_Type_free(recv_types + i);
1869 +    ADIOI_Free(recv_types);
1870 +
1871 +       /* bug fix from Wei-keng Liao and Kenin Coloma */
1872 +       /* +1 to avoid a 0-size malloc */
1873 +    if (fd->atomicity) {
1874 +       statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + 1) *
1875 +                                              sizeof(MPI_Status));
1876 +    } else {
1877 +       statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1) *
1878 +                                              sizeof(MPI_Status));
1879 +    }
1880 +
1881 +#ifdef NEEDS_MPI_TEST
1882 +    i = 0;
1883 +    if (fd->atomicity) {
1884 +       /* bug fix from Wei-keng Liao and Kenin Coloma */
1885 +       while (!i)
1886 +           MPI_Testall(nprocs_send, send_req, &i, statuses);
1887 +    } else {
1888 +       while (!i)
1889 +           MPI_Testall(nprocs_send + nprocs_recv, requests, &i, statuses);
1890 +    }
1891 +#else
1892 +       /* bug fix from Wei-keng Liao and Kenin Coloma */
1893 +    if (fd->atomicity)
1894 +       MPI_Waitall(nprocs_send, send_req, statuses);
1895 +    else
1896 +       MPI_Waitall(nprocs_send + nprocs_recv, requests, statuses);
1897 +#endif
1898 +    ADIOI_Free(statuses);
1899 +    ADIOI_Free(requests);
1900 +    if (!buftype_is_contig && nprocs_send) {
1901 +       for (i = 0; i < nprocs; i++)
1902 +           if (send_size[i])
1903 +               ADIOI_Free(send_buf[i]);
1904 +       ADIOI_Free(send_buf);
1905 +    }
1906 +}
1907 +
1908 +#define ADIOI_BUF_INCR \
1909 +{ \
1910 +    while (buf_incr) { \
1911 +        size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \
1912 +        user_buf_idx += size_in_buf; \
1913 +        flat_buf_sz -= size_in_buf; \
1914 +        if (!flat_buf_sz) { \
1915 +            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
1916 +            else { \
1917 +                flat_buf_idx = 0; \
1918 +                n_buftypes++; \
1919 +            } \
1920 +            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
1921 +                              n_buftypes*buftype_extent; \
1922 +            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
1923 +        } \
1924 +        buf_incr -= size_in_buf; \
1925 +    } \
1926 +}
1927 +
1928 +
1929 +#define ADIOI_BUF_COPY \
1930 +{ \
1931 +    while (size) { \
1932 +        size_in_buf = ADIOI_MIN(size, flat_buf_sz); \
1933 +        memcpy(&(send_buf[p][send_buf_idx[p]]), \
1934 +               ((char *) buf) + user_buf_idx, size_in_buf); \
1935 +        send_buf_idx[p] += size_in_buf; \
1936 +        user_buf_idx += size_in_buf; \
1937 +        flat_buf_sz -= size_in_buf; \
1938 +        if (!flat_buf_sz) { \
1939 +            if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \
1940 +            else { \
1941 +                flat_buf_idx = 0; \
1942 +                n_buftypes++; \
1943 +            } \
1944 +            user_buf_idx = flat_buf->indices[flat_buf_idx] + \
1945 +                              n_buftypes*buftype_extent; \
1946 +            flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \
1947 +        } \
1948 +        size -= size_in_buf; \
1949 +        buf_incr -= size_in_buf; \
1950 +    } \
1951 +    ADIOI_BUF_INCR \
1952 +}
1953 +
1954 +static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, void *buf,
1955 +                                         ADIOI_Flatlist_node * flat_buf,
1956 +                                         char **send_buf,
1957 +                                         ADIO_Offset * offset_list,
1958 +                                         int *len_list, int *send_size,
1959 +                                         MPI_Request * requests,
1960 +                                         int *sent_to_proc, int nprocs,
1961 +                                         int myrank,
1962 +                                         int contig_access_count,
1963 +                                         int * striping_info,
1964 +                                         int *send_buf_idx,
1965 +                                         int *curr_to_proc,
1966 +                                         int *done_to_proc, int iter,
1967 +                                         MPI_Aint buftype_extent)
1968 +{
1969 +    /* this function is only called if buftype is not contig */
1970 +    int i, p, flat_buf_idx, size;
1971 +    int flat_buf_sz, buf_incr, size_in_buf, jj, n_buftypes;
1972 +    ADIO_Offset off, len, rem_len, user_buf_idx;
1973 +
1974 +    /* curr_to_proc[p] = amount of data sent to proc. p that has already
1975 +     * been accounted for so far
1976 +     * done_to_proc[p] = amount of data already sent to proc. p in
1977 +     * previous iterations
1978 +     * user_buf_idx = current location in user buffer
1979 +     * send_buf_idx[p] = current location in send_buf of proc. p
1980 +     */
1981 +
1982 +    for (i = 0; i < nprocs; i++) {
1983 +       send_buf_idx[i] = curr_to_proc[i] = 0;
1984 +       done_to_proc[i] = sent_to_proc[i];
1985 +    }
1986 +    jj = 0;
1987 +
1988 +    user_buf_idx = flat_buf->indices[0];
1989 +    flat_buf_idx = 0;
1990 +    n_buftypes = 0;
1991 +    flat_buf_sz = flat_buf->blocklens[0];
1992 +
1993 +    /* flat_buf_idx = current index into flattened buftype
1994 +     * flat_buf_sz = size of current contiguous component in flattened buf
1995 +     */
1996 +    for (i = 0; i < contig_access_count; i++) {
1997 +       off = offset_list[i];
1998 +       rem_len = (ADIO_Offset) len_list[i];
1999 +
2000 +       /*this request may span to more than one process */
2001 +       while (rem_len != 0) {
2002 +           len = rem_len;
2003 +           /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no
2004 +            * longer than the single region that processor "p" is responsible
2005 +            * for.
2006 +            */
2007 +           p = ADIOI_LUSTRE_Calc_aggregator(fd, off, &len, striping_info);
2008 +
2009 +           if (send_buf_idx[p] < send_size[p]) {
2010 +               if (curr_to_proc[p] + len > done_to_proc[p]) {
2011 +                   if (done_to_proc[p] > curr_to_proc[p]) {
2012 +                       size = (int) ADIOI_MIN(curr_to_proc[p] + len -
2013 +                                              done_to_proc[p],
2014 +                                              send_size[p] -
2015 +                                              send_buf_idx[p]);
2016 +                       buf_incr = done_to_proc[p] - curr_to_proc[p];
2017 +                       ADIOI_BUF_INCR
2018 +                           buf_incr = (int) (curr_to_proc[p] + len -
2019 +                                             done_to_proc[p]);
2020 +                       curr_to_proc[p] = done_to_proc[p] + size;
2021 +                       ADIOI_BUF_COPY
2022 +                    } else {
2023 +                       size = (int) ADIOI_MIN(len, send_size[p] -
2024 +                                              send_buf_idx[p]);
2025 +                       buf_incr = (int) len;
2026 +                       curr_to_proc[p] += size;
2027 +                       ADIOI_BUF_COPY
2028 +                    }
2029 +                   if (send_buf_idx[p] == send_size[p]) {
2030 +                       MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p,
2031 +                                 myrank + p + 100 * iter, fd->comm,
2032 +                                 requests + jj);
2033 +                       jj++;
2034 +                   }
2035 +               } else {
2036 +                   curr_to_proc[p] += (int) len;
2037 +                   buf_incr = (int) len;
2038 +                   ADIOI_BUF_INCR
2039 +                }
2040 +           } else {
2041 +               buf_incr = (int) len;
2042 +               ADIOI_BUF_INCR
2043 +            }
2044 +           off += len;
2045 +           rem_len -= len;
2046 +       }
2047 +    }
2048 +    for (i = 0; i < nprocs; i++)
2049 +       if (send_size[i])
2050 +           sent_to_proc[i] = curr_to_proc[i];
2051 +}
2052 diff -ruN ad_lustre_orig/ad_lustre_wrstr.c ad_lustre/ad_lustre_wrstr.c
2053 --- ad_lustre_orig/ad_lustre_wrstr.c    1970-01-01 08:00:00.000000000 +0800
2054 +++ ad_lustre/ad_lustre_wrstr.c 2008-10-13 15:34:53.000000000 +0800
2055 @@ -0,0 +1,472 @@
2056 +/* -*- Mode: C; c-basic-offset:4 ; -*- */
2057 +/*
2058 + *   Copyright (C) 1997 University of Chicago.
2059 + *   See COPYRIGHT notice in top-level directory.
2060 + *
2061 + *   Copyright (C) 2007 Oak Ridge National Laboratory
2062 + *
2063 + *   Copyright (C) 2008 Sun Microsystems, Lustre group
2064 + */
2065 +
2066 +#include "ad_lustre.h"
2067 +#include "adio_extern.h"
2068 +
2069 +#define ADIOI_BUFFERED_WRITE \
2070 +{ \
2071 +    if (req_off >= writebuf_off + writebuf_len) { \
2072 +        if (writebuf_len) { \
2073 +           ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
2074 +                  ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
2075 +           if (!(fd->atomicity)) \
2076 +                ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2077 +           if (*error_code != MPI_SUCCESS) { \
2078 +               *error_code = MPIO_Err_create_code(*error_code, \
2079 +                                                  MPIR_ERR_RECOVERABLE, myname, \
2080 +                                                  __LINE__, MPI_ERR_IO, \
2081 +                                                  "**iowswc", 0); \
2082 +               ADIOI_Free(writebuf); \
2083 +               return; \
2084 +           } \
2085 +        } \
2086 +       writebuf_off = req_off; \
2087 +        /* stripe_size alignment */ \
2088 +        writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \
2089 +                                       (writebuf_off / stripe_size + 1) * \
2090 +                                       stripe_size - writebuf_off);\
2091 +       if (!(fd->atomicity)) \
2092 +            ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2093 +       ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,\
2094 +                        writebuf_off, &status1, error_code); \
2095 +       if (*error_code != MPI_SUCCESS) { \
2096 +           *error_code = MPIO_Err_create_code(*error_code, \
2097 +                                              MPIR_ERR_RECOVERABLE, myname, \
2098 +                                              __LINE__, MPI_ERR_IO, \
2099 +                                              "**iowsrc", 0); \
2100 +            ADIOI_Free(writebuf); \
2101 +           return; \
2102 +       } \
2103 +    } \
2104 +    write_sz = (int) ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off); \
2105 +    memcpy(writebuf + req_off - writebuf_off, (char *)buf + userbuf_off, write_sz);\
2106 +    while (write_sz != req_len) {\
2107 +        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
2108 +                         ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
2109 +        if (!(fd->atomicity)) \
2110 +            ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2111 +        if (*error_code != MPI_SUCCESS) { \
2112 +            *error_code = MPIO_Err_create_code(*error_code, \
2113 +                                               MPIR_ERR_RECOVERABLE, myname, \
2114 +                                               __LINE__, MPI_ERR_IO, \
2115 +                                               "**iowswc", 0); \
2116 +            ADIOI_Free(writebuf); \
2117 +            return; \
2118 +        } \
2119 +        req_len -= write_sz; \
2120 +        userbuf_off += write_sz; \
2121 +        writebuf_off += writebuf_len; \
2122 +        /* stripe_size alignment */ \
2123 +        writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \
2124 +                                       (writebuf_off / stripe_size + 1) * \
2125 +                                       stripe_size - writebuf_off);\
2126 +       if (!(fd->atomicity)) \
2127 +            ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
2128 +        ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,\
2129 +                        writebuf_off, &status1, error_code); \
2130 +       if (*error_code != MPI_SUCCESS) { \
2131 +           *error_code = MPIO_Err_create_code(*error_code, \
2132 +                                              MPIR_ERR_RECOVERABLE, myname, \
2133 +                                              __LINE__, MPI_ERR_IO, \
2134 +                                              "**iowsrc", 0); \
2135 +            ADIOI_Free(writebuf); \
2136 +           return; \
2137 +       } \
2138 +        write_sz = ADIOI_MIN(req_len, writebuf_len); \
2139 +        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
2140 +    } \
2141 +}
2142 +
2143 +
2144 +/* this macro is used when filetype is contig and buftype is not contig.
2145 +   it does not do a read-modify-write and does not lock*/
2146 +#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
2147 +{ \
2148 +    if (req_off >= writebuf_off + writebuf_len) { \
2149 +       writebuf_off = req_off; \
2150 +        /* stripe_size alignment */ \
2151 +        writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \
2152 +                                       (writebuf_off / stripe_size + 1) * \
2153 +                                       stripe_size - writebuf_off);\
2154 +    } \
2155 +    write_sz = (int) ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off); \
2156 +    memcpy(writebuf + req_off - writebuf_off, (char *)buf + userbuf_off, write_sz);\
2157 +    while (req_len) { \
2158 +        ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \
2159 +                         ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \
2160 +        if (*error_code != MPI_SUCCESS) { \
2161 +            *error_code = MPIO_Err_create_code(*error_code, \
2162 +                                               MPIR_ERR_RECOVERABLE, myname, \
2163 +                                               __LINE__, MPI_ERR_IO, \
2164 +                                               "**iowswc", 0); \
2165 +            ADIOI_Free(writebuf); \
2166 +            return; \
2167 +        } \
2168 +        req_len -= write_sz; \
2169 +        userbuf_off += write_sz; \
2170 +        writebuf_off += writebuf_len; \
2171 +        /* stripe_size alignment */ \
2172 +        writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \
2173 +                                       (writebuf_off / stripe_size + 1) * \
2174 +                                       stripe_size - writebuf_off);\
2175 +        write_sz = ADIOI_MIN(req_len, writebuf_len); \
2176 +        memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
2177 +    } \
2178 +}
2179 +
2180 +void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count,
2181 +                              MPI_Datatype datatype, int file_ptr_type,
2182 +                              ADIO_Offset offset, ADIO_Status * status,
2183 +                              int *error_code)
2184 +{
2185 +    /* offset is in units of etype relative to the filetype. */
2186 +    ADIOI_Flatlist_node *flat_buf, *flat_file;
2187 +    int i, j, k, bwr_size, fwr_size = 0, st_index = 0;
2188 +    int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;
2189 +    int n_filetypes, etype_in_filetype;
2190 +    ADIO_Offset abs_off_in_filetype = 0;
2191 +    int filetype_size, etype_size, buftype_size, req_len;
2192 +    MPI_Aint filetype_extent, buftype_extent;
2193 +    int buf_count, buftype_is_contig, filetype_is_contig;
2194 +    ADIO_Offset userbuf_off;
2195 +    ADIO_Offset off, req_off, disp, end_offset = 0, writebuf_off, start_off;
2196 +    char *writebuf;
2197 +    int flag, st_fwr_size, st_n_filetypes, writebuf_len, write_sz;
2198 +    ADIO_Status status1;
2199 +    int new_bwr_size, new_fwr_size;
2200 +    char * value;
2201 +    int stripe_size, lflag = 0;
2202 +    static char myname[] = "ADIOI_LUSTRE_WriteStrided";
2203 +    int myrank;
2204 +    MPI_Comm_rank(fd->comm, &myrank);
2205 +
2206 +    if (fd->hints->ds_write == ADIOI_HINT_DISABLE) {
2207 +       /* if user has disabled data sieving on writes, use naive
2208 +        * approach instead.
2209 +        */
2210 +       ADIOI_GEN_WriteStrided_naive(fd,
2211 +                                    buf,
2212 +                                    count,
2213 +                                    datatype,
2214 +                                    file_ptr_type,
2215 +                                    offset, status, error_code);
2216 +       return;
2217 +    }
2218 +
2219 +    *error_code = MPI_SUCCESS; /* changed below if error */
2220 +
2221 +    ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
2222 +    ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
2223 +
2224 +    MPI_Type_size(fd->filetype, &filetype_size);
2225 +    if (!filetype_size) {
2226 +       *error_code = MPI_SUCCESS;
2227 +       return;
2228 +    }
2229 +
2230 +    MPI_Type_extent(fd->filetype, &filetype_extent);
2231 +    MPI_Type_size(datatype, &buftype_size);
2232 +    MPI_Type_extent(datatype, &buftype_extent);
2233 +    etype_size = fd->etype_size;
2234 +
2235 +    bufsize = buftype_size * count;
2236 +
2237 +    /* get striping info */
2238 +    value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char));
2239 +    MPI_Info_get(fd->info, "striping_unit", MPI_MAX_INFO_VAL, value, &lflag);
2240 +    if (lflag)
2241 +       stripe_size = atoi(value);
2242 +    ADIOI_Free(value);
2243 +
2244 +    /* Different buftype to different filetype */
2245 +    if (!buftype_is_contig && filetype_is_contig) {
2246 +        /* noncontiguous in memory, contiguous in file. */
2247 +       ADIOI_Flatten_datatype(datatype);
2248 +       flat_buf = ADIOI_Flatlist;
2249 +       while (flat_buf->type != datatype)
2250 +           flat_buf = flat_buf->next;
2251 +
2252 +       off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :
2253 +           fd->disp + etype_size * offset;
2254 +
2255 +       start_off = off;
2256 +       end_offset = start_off + bufsize - 1;
2257 +       writebuf_off = start_off;
2258 +        /* write stripe size buffer each time */
2259 +       writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size));
2260 +        writebuf_len = (int) ADIOI_MIN(bufsize,
2261 +                                       (writebuf_off / stripe_size + 1) *
2262 +                                       stripe_size - writebuf_off);
2263 +
2264 +        /* if atomicity is true, lock the region to be accessed */
2265 +       if (fd->atomicity)
2266 +           ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize);
2267 +
2268 +       for (j = 0; j < count; j++) {
2269 +           for (i = 0; i < flat_buf->count; i++) {
2270 +               userbuf_off = j * buftype_extent + flat_buf->indices[i];
2271 +               req_off = off;
2272 +               req_len = flat_buf->blocklens[i];
2273 +               ADIOI_BUFFERED_WRITE_WITHOUT_READ
2274 +               off += flat_buf->blocklens[i];
2275 +           }
2276 +        }
2277 +
2278 +       /* write the buffer out finally */
2279 +       ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE,
2280 +                        ADIO_EXPLICIT_OFFSET, writebuf_off, &status1,
2281 +                        error_code);
2282 +
2283 +       if (fd->atomicity)
2284 +           ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize);
2285 +       if (*error_code != MPI_SUCCESS) {
2286 +            ADIOI_Free(writebuf);
2287 +           return;
2288 +        }
2289 +       ADIOI_Free(writebuf);
2290 +       if (file_ptr_type == ADIO_INDIVIDUAL)
2291 +           fd->fp_ind = off;
2292 +    } else {
2293 +        /* noncontiguous in file */
2294 +        /* filetype already flattened in ADIO_Open */
2295 +       flat_file = ADIOI_Flatlist;
2296 +       while (flat_file->type != fd->filetype)
2297 +           flat_file = flat_file->next;
2298 +       disp = fd->disp;
2299 +
2300 +       if (file_ptr_type == ADIO_INDIVIDUAL) {
2301 +           offset = fd->fp_ind;        /* in bytes */
2302 +           n_filetypes = -1;
2303 +           flag = 0;
2304 +           while (!flag) {
2305 +               n_filetypes++;
2306 +               for (i = 0; i < flat_file->count; i++) {
2307 +                   if (disp + flat_file->indices[i] +
2308 +                       (ADIO_Offset) n_filetypes * filetype_extent +
2309 +                       flat_file->blocklens[i] >= offset) {
2310 +                       st_index = i;
2311 +                       fwr_size = (int) (disp + flat_file->indices[i] +
2312 +                                         (ADIO_Offset) n_filetypes *
2313 +                                         filetype_extent +
2314 +                                         flat_file->blocklens[i] -
2315 +                                         offset);
2316 +                       flag = 1;
2317 +                       break;
2318 +                   }
2319 +               }
2320 +           }
2321 +       } else {
2322 +           n_etypes_in_filetype = filetype_size / etype_size;
2323 +           n_filetypes = (int) (offset / n_etypes_in_filetype);
2324 +           etype_in_filetype = (int) (offset % n_etypes_in_filetype);
2325 +           size_in_filetype = etype_in_filetype * etype_size;
2326 +
2327 +           sum = 0;
2328 +           for (i = 0; i < flat_file->count; i++) {
2329 +               sum += flat_file->blocklens[i];
2330 +               if (sum > size_in_filetype) {
2331 +                   st_index = i;
2332 +                   fwr_size = sum - size_in_filetype;
2333 +                   abs_off_in_filetype = flat_file->indices[i] +
2334 +                       size_in_filetype - (sum - flat_file->blocklens[i]);
2335 +                   break;
2336 +               }
2337 +           }
2338 +
2339 +           /* abs. offset in bytes in the file */
2340 +           offset = disp + (ADIO_Offset) n_filetypes *filetype_extent +
2341 +                    abs_off_in_filetype;
2342 +       }
2343 +
2344 +       start_off = offset;
2345 +
2346 +       /* If the file bytes is actually contiguous, we do not need data sieve at all */
2347 +       if (bufsize <= fwr_size) {
2348 +            req_off = start_off;
2349 +            req_len = bufsize;
2350 +            end_offset = start_off + bufsize - 1;
2351 +           writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size));
2352 +           memset(writebuf, -1, ADIOI_MIN(bufsize, stripe_size));
2353 +            writebuf_off = 0;
2354 +            writebuf_len = 0;
2355 +            userbuf_off = 0;
2356 +            ADIOI_BUFFERED_WRITE_WITHOUT_READ
2357 +       } else {
2358 +           /* Calculate end_offset, the last byte-offset that will be accessed.
2359 +              e.g., if start_offset=0 and 100 bytes to be write, end_offset=99 */
2360 +           st_fwr_size = fwr_size;
2361 +           st_n_filetypes = n_filetypes;
2362 +           i = 0;
2363 +           j = st_index;
2364 +           off = offset;
2365 +           fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
2366 +           while (i < bufsize) {
2367 +               i += fwr_size;
2368 +               end_offset = off + fwr_size - 1;
2369 +
2370 +               if (j < (flat_file->count - 1))
2371 +                   j++;
2372 +               else {
2373 +                   j = 0;
2374 +                   n_filetypes++;
2375 +               }
2376 +
2377 +               off = disp + flat_file->indices[j] +
2378 +                     (ADIO_Offset) n_filetypes * filetype_extent;
2379 +               fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize - i);
2380 +           }
2381 +
2382 +           writebuf_off = 0;
2383 +           writebuf_len = 0;
2384 +           writebuf = (char *) ADIOI_Malloc(stripe_size);
2385 +           memset(writebuf, -1, stripe_size);
2386 +           /* if atomicity is true, lock the region to be accessed */
2387 +           if (fd->atomicity)
2388 +               ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize);
2389 +
2390 +           if (buftype_is_contig && !filetype_is_contig) {
2391 +               /* contiguous in memory, noncontiguous in file. should be the most
2392 +                  common case. */
2393 +               i = 0;
2394 +               j = st_index;
2395 +               off = offset;
2396 +               n_filetypes = st_n_filetypes;
2397 +               fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
2398 +               while (i < bufsize) {
2399 +                   if (fwr_size) {
2400 +                       /* TYPE_UB and TYPE_LB can result in
2401 +                          fwr_size = 0. save system call in such cases */
2402 +                       /*
2403 +                        lseek(fd->fd_sys, off, SEEK_SET);
2404 +                       err = write(fd->fd_sys, ((char *) buf) + i, fwr_size);
2405 +                        */
2406 +                       req_off = off;
2407 +                       req_len = fwr_size;
2408 +                       userbuf_off = i;
2409 +                       ADIOI_BUFFERED_WRITE
2410 +                    }
2411 +                   i += fwr_size;
2412 +
2413 +                   if (off + fwr_size < disp + flat_file->indices[j] +
2414 +                                        flat_file->blocklens[j] +
2415 +                           (ADIO_Offset) n_filetypes * filetype_extent)
2416 +                       off += fwr_size;
2417 +                   /* did not reach end of contiguous block in filetype.
2418 +                   no more I/O needed. off is incremented by fwr_size. */
2419 +                   else {
2420 +                       if (j < (flat_file->count - 1))
2421 +                           j++;
2422 +                       else {
2423 +                           j = 0;
2424 +                           n_filetypes++;
2425 +                       }
2426 +                       off = disp + flat_file->indices[j] +
2427 +                             (ADIO_Offset) n_filetypes * filetype_extent;
2428 +                       fwr_size = ADIOI_MIN(flat_file->blocklens[j],
2429 +                                             bufsize - i);
2430 +                   }
2431 +               }
2432 +           } else {
2433 +                   /* noncontiguous in memory as well as in file */
2434 +               ADIOI_Flatten_datatype(datatype);
2435 +               flat_buf = ADIOI_Flatlist;
2436 +               while (flat_buf->type != datatype)
2437 +                   flat_buf = flat_buf->next;
2438 +
2439 +               k = num = buf_count = 0;
2440 +               i = (int) (flat_buf->indices[0]);
2441 +               j = st_index;
2442 +               off = offset;
2443 +               n_filetypes = st_n_filetypes;
2444 +               fwr_size = st_fwr_size;
2445 +               bwr_size = flat_buf->blocklens[0];
2446 +
2447 +               while (num < bufsize) {
2448 +                   size = ADIOI_MIN(fwr_size, bwr_size);
2449 +                   if (size) {
2450 +                       /*
2451 +                        lseek(fd->fd_sys, off, SEEK_SET);
2452 +                        err = write(fd->fd_sys, ((char *) buf) + i, size);
2453 +                        */
2454 +                       req_off = off;
2455 +                       req_len = size;
2456 +                       userbuf_off = i;
2457 +                       ADIOI_BUFFERED_WRITE
2458 +                    }
2459 +
2460 +                   new_fwr_size = fwr_size;
2461 +                   new_bwr_size = bwr_size;
2462 +
2463 +                   if (size == fwr_size) {
2464 +                       /* reached end of contiguous block in file */
2465 +                       if (j < (flat_file->count - 1)) {
2466 +                           j++;
2467 +                        } else {
2468 +                           j = 0;
2469 +                           n_filetypes++;
2470 +                       }
2471 +                       off = disp + flat_file->indices[j] +
2472 +                             (ADIO_Offset) n_filetypes * filetype_extent;
2473 +                        new_fwr_size = flat_file->blocklens[j];
2474 +                       if (size != bwr_size) {
2475 +                           i += size;
2476 +                           new_bwr_size -= size;
2477 +                       }
2478 +                   }
2479 +                   if (size == bwr_size) {
2480 +                       /* reached end of contiguous block in memory */
2481 +                       k = (k + 1) % flat_buf->count;
2482 +                       buf_count++;
2483 +                       i = (int) (buftype_extent *
2484 +                                  (buf_count / flat_buf->count) +
2485 +                                 flat_buf->indices[k]);
2486 +                       new_bwr_size = flat_buf->blocklens[k];
2487 +                       if (size != fwr_size) {
2488 +                           off += size;
2489 +                           new_fwr_size -= size;
2490 +                       }
2491 +                   }
2492 +                   num += size;
2493 +                   fwr_size = new_fwr_size;
2494 +                   bwr_size = new_bwr_size;
2495 +               }
2496 +            }
2497 +
2498 +           /* write the buffer out finally */
2499 +           if (writebuf_len) {
2500 +               ADIO_WriteContig(fd, writebuf, writebuf_len,
2501 +                                MPI_BYTE, ADIO_EXPLICIT_OFFSET,
2502 +                                writebuf_off, &status1, error_code);
2503 +               if (!(fd->atomicity))
2504 +                   ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
2505 +               if (*error_code != MPI_SUCCESS) {
2506 +                    ADIOI_Free(writebuf);
2507 +                   return;
2508 +                }
2509 +           }
2510 +           if (fd->atomicity)
2511 +               ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize);
2512 +       }
2513 +        ADIOI_Free(writebuf);
2514 +       if (file_ptr_type == ADIO_INDIVIDUAL)
2515 +           fd->fp_ind = off;
2516 +    }
2517 +    fd->fp_sys_posn = -1;      /* set it to null. */
2518 +
2519 +#ifdef HAVE_STATUS_SET_BYTES
2520 +    MPIR_Status_set_bytes(status, datatype, bufsize);
2521 +    /* This is a temporary way of filling in status. The right way is to
2522 +    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
2523 +#endif
2524 +
2525 +    if (!buftype_is_contig)
2526 +        ADIOI_Delete_flattened(datatype);
2527 +}
2528 diff -ruN ad_lustre_orig/Makefile.in ad_lustre/Makefile.in
2529 --- ad_lustre_orig/Makefile.in  2008-09-17 14:36:57.000000000 +0800
2530 +++ ad_lustre/Makefile.in       2008-10-17 17:03:06.000000000 +0800
2531 @@ -16,7 +16,9 @@
2532  @VPATH@
2533
2534  AD_LUSTRE_OBJECTS = ad_lustre.o ad_lustre_open.o \
2535 -      ad_lustre_rwcontig.o ad_lustre_hints.o
2536 +      ad_lustre_rwcontig.o ad_lustre_wrcoll.o ad_lustre_wrstr.o  \
2537 +      ad_lustre_hints.o ad_lustre_aggregate.o
2538 +
2539
2540  default: $(LIBNAME)
2541         @if [ "@ENABLE_SHLIB@" != "none" ] ; then \
2542 diff -ruN ad_lustre_orig/README ad_lustre/README
2543 --- ad_lustre_orig/README       2008-09-17 14:36:57.000000000 +0800
2544 +++ ad_lustre/README    2008-10-17 16:50:15.000000000 +0800
2545 @@ -5,6 +5,23 @@
2546    o To post the code for ParColl (Partitioned collective IO)
2547
2548  -----------------------------------------------------
2549 +V05:
2550 +-----------------------------------------------------
2551 +Improved data redistribution
2552 +  o Improve I/O pattern identification. Besides checking interleaving,
2553 +    if request I/O size is small, collective I/O will be performed.
2554 +    The hint big_req_size can be used to define the req size value.
2555 +  o Provide hint CO for load balancing to control the number of
2556 +    IO clients for each OST
2557 +  o Produce stripe-contiguous I/O pattern that Lustre prefers
2558 +  o Reduce the collective overhead by hints contiguous_data and
2559 +    same_io_size to remove unnecessary MPI_Alltoall()
2560 +  o Control read-modify-write in data sieving in collective IO
2561 +    by hint ds_in_coll.
2562 +  o Reduce extent lock conflicts by make each OST accessed by one or
2563 +    more constant clients.
2564 +
2565 +-----------------------------------------------------
2566  V04:
2567  -----------------------------------------------------
2568    o Direct IO and Lockless IO support
2569 --- common/ad_write_coll_orig.c 2008-10-15 11:24:31.000000000 +0800
2570 +++ common/ad_write_coll.c      2008-10-15 11:25:39.000000000 +0800
2571 @@ -42,7 +42,7 @@
2572                             int *send_buf_idx, int *curr_to_proc,
2573                             int *done_to_proc, int iter,
2574                             MPI_Aint buftype_extent);
2575 -static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
2576 +void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
2577                        ADIO_Offset *srt_off, int *srt_len, int *start_pos,
2578                        int nprocs, int nprocs_recv, int total_elements);
2579
2580 @@ -921,7 +921,7 @@
2581
2582
2583
2584 -static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
2585 +void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count,
2586                       ADIO_Offset *srt_off, int *srt_len, int *start_pos,
2587                       int nprocs, int nprocs_recv, int total_elements)
2588  {