From: Emoly Liu Date: Wed, 24 May 2017 13:37:02 +0000 (+0800) Subject: LU-9550 adio: delete the old Lustre ADIO driver patches X-Git-Tag: 2.9.59~37 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=02e0adddf474091a182e0721febb969564b6398b LU-9550 adio: delete the old Lustre ADIO driver patches Since Lustre ADIO driver was released by MPICH2 about 8 years ago, those old Lustre ADIO driver patches (lustre/contrib/*mpich*.patch) can be deleted. Test-Parameters: trivial Signed-off-by: Emoly Liu Change-Id: Ibbd4c67b36065d913c6e3b71895f2413481c923c Reviewed-on: https://review.whamcloud.com/27264 Tested-by: Jenkins Reviewed-by: Niu Yawei Reviewed-by: Yang Sheng Tested-by: Maloo Reviewed-by: Andreas Dilger --- diff --git a/debian/lustre-dev.install b/debian/lustre-dev.install index 65ab682..851d6af 100644 --- a/debian/lustre-dev.install +++ b/debian/lustre-dev.install @@ -1,5 +1,3 @@ -lustre/contrib/README usr/share/doc/lustre-dev/contrib -lustre/contrib/mpich-1.2.6-lustre.patch usr/share/doc/lustre-dev/contrib debian/tmp/usr/include/lustre/* usr/include/lustre debian/tmp/usr/lib/*.so usr/lib debian/tmp/usr/lib/*.a usr/lib diff --git a/debian/rules b/debian/rules index 21fe689..70cc1d8 100755 --- a/debian/rules +++ b/debian/rules @@ -113,7 +113,6 @@ configure-stamp: autogen-stamp debian/control.main debian/control.modules.in if [ "$(BUILDDIR)" != "." ]; then \ mkdir -p $(BUILDDIR)/build $(BUILDDIR)/lustre/contrib; \ cp build/Makefile $(BUILDDIR)/build/; \ - cp lustre/contrib/mpich-*.patch $(BUILDDIR)/lustre/contrib/; \ fi ( cd $(BUILDDIR) && \ $(SRCDIR)/configure --disable-dependency-tracking \ diff --git a/lustre.spec.in b/lustre.spec.in index 64264f9..6ed1996 100644 --- a/lustre.spec.in +++ b/lustre.spec.in @@ -392,6 +392,7 @@ if [ -d $RPM_BUILD_ROOT%{_libdir}/lustre/snmp ] ; then fi %if %{with lustre_utils} +mkdir -p $RPM_BUILD_ROOT/%{_datadir}/lustre find $RPM_BUILD_ROOT%{_libdir}/@PACKAGE@ -name \*.la -type f -exec rm -f {} \; %endif @@ -438,9 +439,9 @@ echo '%{_sbindir}/wiretest' >>lustre-tests.files %if %{with manpages} %{_mandir}/man?/* %endif +%{_datadir}/lustre %{_includedir}/lustre %endif -%{_datadir}/lustre %{_sysconfdir}/udev/rules.d/99-lustre.rules %config(noreplace) %{_sysconfdir}/ldev.conf %if %{with lnet_dlc} diff --git a/lustre/contrib/Makefile.am b/lustre/contrib/Makefile.am index 5a8e66c..f5c1e23 100644 --- a/lustre/contrib/Makefile.am +++ b/lustre/contrib/Makefile.am @@ -1,5 +1,4 @@ # Contributions Makefile -EXTRA_DIST = mpich-*.patch +EXTRA_DIST = pkgdata_DATA = $(EXTRA_DIST) - diff --git a/lustre/contrib/README b/lustre/contrib/README deleted file mode 100644 index 77b5de2..0000000 --- a/lustre/contrib/README +++ /dev/null @@ -1,2 +0,0 @@ -The files in this directory are user-contributed and are not supported by -Sun Microsystems, Inc. in any way. diff --git a/lustre/contrib/adio_driver_mpich2-1.0.7.patch b/lustre/contrib/adio_driver_mpich2-1.0.7.patch deleted file mode 100644 index 9bb0126..0000000 --- a/lustre/contrib/adio_driver_mpich2-1.0.7.patch +++ /dev/null @@ -1,2213 +0,0 @@ ---- configure_orig.in 2009-03-01 13:50:30.000000000 +0800 -+++ configure.in 2009-02-27 13:35:42.000000000 +0800 -@@ -1123,8 +1123,14 @@ - if test -n "$file_system_testfs"; then - AC_DEFINE(ROMIO_TESTFS,1,[Define for ROMIO with TESTFS]) - fi -+# -+# Verify presence of lustre/lustre_user.h -+# - if test -n "$file_system_lustre"; then -- AC_DEFINE(ROMIO_LUSTRE,1,[Define for ROMIO with LUSTRE]) -+ AC_CHECK_HEADERS(lustre/lustre_user.h, -+ AC_DEFINE(ROMIO_LUSTRE,1,[Define for ROMIO with LUSTRE]), -+ AC_MSG_ERROR([LUSTRE support requested but cannot find lustre/lustre_user.h header file]) -+ ) - fi - - if test -n "$file_system_xfs"; then ---- adio/include/adioi_orig.h 2009-03-01 14:00:48.000000000 +0800 -+++ adio/include/adioi.h 2009-04-24 15:26:44.000000000 +0800 -@@ -52,6 +52,12 @@ - struct { - int debugmask; - } pvfs2; -+ struct { -+ int start_iodevice; -+ int co_ratio; -+ int coll_threshold; -+ int ds_in_coll; -+ } lustre; - } fs_hints; - - }; -diff -ruN adio/ad_lustre_orig/ad_lustre_aggregate.c adio/ad_lustre/ad_lustre_aggregate.c ---- adio/ad_lustre_orig/ad_lustre_aggregate.c 1970-01-01 08:00:00.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_aggregate.c 2009-05-05 15:22:40.000000000 +0800 -@@ -0,0 +1,304 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ * -+ * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. -+ */ -+ -+#include "ad_lustre.h" -+#include "adio_extern.h" -+ -+#undef AGG_DEBUG -+ -+void ADIOI_LUSTRE_Get_striping_info(ADIO_File fd, int **striping_info_ptr, -+ int mode) -+{ -+ int *striping_info = NULL; -+ /* get striping information: -+ * striping_info[0]: stripe_size -+ * striping_info[1]: stripe_count -+ * striping_info[2]: avail_cb_nodes -+ */ -+ int stripe_size, stripe_count, CO = 1, CO_max = 1, CO_nodes, lflag; -+ int avail_cb_nodes, divisor, nprocs_for_coll = fd->hints->cb_nodes; -+ char *value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL + 1) * sizeof(char)); -+ -+ /* Get hints value */ -+ /* stripe size */ -+ stripe_size = fd->hints->striping_unit; -+ /* stripe count */ -+ /* stripe_size and stripe_count have been validated in ADIOI_LUSTRE_Open() */ -+ stripe_count = fd->hints->striping_factor; -+ -+ /* Calculate the available number of I/O clients, that is -+ * avail_cb_nodes=min(cb_nodes, stripe_count*CO), where -+ * CO=1 by default -+ */ -+ if (!mode) { -+ /* for collective read, -+ * if "CO" clients access the same OST simultaneously, -+ * the OST disk seek time would be much. So, to avoid this, -+ * it might be better if 1 client only accesses 1 OST. -+ * So, we set CO = 1 to meet the above requirement. -+ */ -+ CO = 1; -+ /*XXX: maybe there are other better way for collective read */ -+ } else { -+ /* CO_max: the largest number of IO clients for each ost group */ -+ CO_max = (nprocs_for_coll - 1)/ stripe_count + 1; -+ /* CO also has been validated in ADIOI_LUSTRE_Open(), >0 */ -+ CO = fd->hints->fs_hints.lustre.co_ratio; -+ CO = ADIOI_MIN(CO_max, CO); -+ } -+ /* Calculate how many IO clients we need */ -+ /* To avoid extent lock conflicts, -+ * avail_cb_nodes should divide (stripe_count*CO) exactly, -+ * so that each OST is accessed by only one or more constant clients. */ -+ CO_nodes = stripe_count * CO; -+ avail_cb_nodes = ADIOI_MIN(nprocs_for_coll, CO_nodes); -+ if (avail_cb_nodes < CO_nodes) { -+ do { -+ /* find the divisor of CO_nodes */ -+ divisor = 1; -+ do { -+ divisor ++; -+ } while (CO_nodes % divisor); -+ CO_nodes = CO_nodes / divisor; -+ /* if stripe_count*CO is a prime number, change nothing */ -+ if ((CO_nodes <= avail_cb_nodes) && (CO_nodes != 1)) { -+ avail_cb_nodes = CO_nodes; -+ break; -+ } -+ } while (CO_nodes != 1); -+ } -+ -+ *striping_info_ptr = (int *) ADIOI_Malloc(3 * sizeof(int)); -+ striping_info = *striping_info_ptr; -+ striping_info[0] = stripe_size; -+ striping_info[1] = stripe_count; -+ striping_info[2] = avail_cb_nodes; -+ -+ ADIOI_Free(value); -+} -+ -+int ADIOI_LUSTRE_Calc_aggregator(ADIO_File fd, ADIO_Offset off, -+ ADIO_Offset *len, int *striping_info) -+{ -+ int rank_index, rank; -+ ADIO_Offset avail_bytes; -+ int stripe_size = striping_info[0]; -+ int avail_cb_nodes = striping_info[2]; -+ -+ /* Produce the stripe-contiguous pattern for Lustre */ -+ rank_index = (int)((off / stripe_size) % avail_cb_nodes); -+ -+ /* we index into fd_end with rank_index, and fd_end was allocated to be no -+ * bigger than fd->hins->cb_nodes. If we ever violate that, we're -+ * overrunning arrays. Obviously, we should never ever hit this abort -+ */ -+ if (rank_index >= fd->hints->cb_nodes) -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ -+ avail_bytes = (off / (ADIO_Offset)stripe_size + 1) * -+ (ADIO_Offset)stripe_size - off; -+ if (avail_bytes < *len) { -+ /* this proc only has part of the requested contig. region */ -+ *len = avail_bytes; -+ } -+ /* map our index to a rank */ -+ /* NOTE: FOR NOW WE DON'T HAVE A MAPPING...JUST DO 0..NPROCS_FOR_COLL */ -+ rank = fd->hints->ranklist[rank_index]; -+ -+ return rank; -+} -+ -+/* ADIOI_LUSTRE_Calc_my_req() - calculate what portions of the access requests -+ * of this process are located in the file domains of various processes -+ * (including this one) -+ */ -+void ADIOI_LUSTRE_Calc_my_req(ADIO_File fd, ADIO_Offset *offset_list, -+ int *len_list, int contig_access_count, -+ int *striping_info, int nprocs, -+ int *count_my_req_procs_ptr, -+ int **count_my_req_per_proc_ptr, -+ ADIOI_Access **my_req_ptr, -+ int **buf_idx_ptr) -+{ -+ /* Nothing different from ADIOI_Calc_my_req(), except calling -+ * ADIOI_Lustre_Calc_aggregator() instead of the old one */ -+ int *count_my_req_per_proc, count_my_req_procs, *buf_idx; -+ int i, l, proc; -+ ADIO_Offset avail_len, rem_len, curr_idx, off; -+ ADIOI_Access *my_req; -+ -+ *count_my_req_per_proc_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); -+ count_my_req_per_proc = *count_my_req_per_proc_ptr; -+ /* count_my_req_per_proc[i] gives the no. of contig. requests of this -+ * process in process i's file domain. calloc initializes to zero. -+ * I'm allocating memory of size nprocs, so that I can do an -+ * MPI_Alltoall later on. -+ */ -+ -+ buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* buf_idx is relevant only if buftype_is_contig. -+ * buf_idx[i] gives the index into user_buf where data received -+ * from proc. i should be placed. This allows receives to be done -+ * without extra buffer. This can't be done if buftype is not contig. -+ */ -+ -+ /* initialize buf_idx to -1 */ -+ for (i = 0; i < nprocs; i++) -+ buf_idx[i] = -1; -+ -+ /* one pass just to calculate how much space to allocate for my_req; -+ * contig_access_count was calculated way back in ADIOI_Calc_my_off_len() -+ */ -+ for (i = 0; i < contig_access_count; i++) { -+ /* short circuit offset/len processing if len == 0 -+ * (zero-byte read/write -+ */ -+ if (len_list[i] == 0) -+ continue; -+ off = offset_list[i]; -+ avail_len = len_list[i]; -+ /* note: we set avail_len to be the total size of the access. -+ * then ADIOI_LUSTRE_Calc_aggregator() will modify the value to return -+ * the amount that was available. -+ */ -+ proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info); -+ count_my_req_per_proc[proc]++; -+ -+ /* figure out how many data is remaining in the access -+ * we'll take care of this data (if there is any) -+ * in the while loop below. -+ */ -+ rem_len = len_list[i] - avail_len; -+ -+ while (rem_len != 0) { -+ off += avail_len; /* point to first remaining byte */ -+ avail_len = rem_len; /* save remaining size, pass to calc */ -+ proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info); -+ count_my_req_per_proc[proc]++; -+ rem_len -= avail_len; /* reduce remaining length by amount from fd */ -+ } -+ } -+ -+ /* now allocate space for my_req, offset, and len */ -+ *my_req_ptr = (ADIOI_Access *) ADIOI_Malloc(nprocs * sizeof(ADIOI_Access)); -+ my_req = *my_req_ptr; -+ -+ count_my_req_procs = 0; -+ for (i = 0; i < nprocs; i++) { -+ if (count_my_req_per_proc[i]) { -+ my_req[i].offsets = (ADIO_Offset *) -+ ADIOI_Malloc(count_my_req_per_proc[i] * -+ sizeof(ADIO_Offset)); -+ my_req[i].lens = (int *) ADIOI_Malloc(count_my_req_per_proc[i] * -+ sizeof(int)); -+ count_my_req_procs++; -+ } -+ my_req[i].count = 0; /* will be incremented where needed later */ -+ } -+ -+ /* now fill in my_req */ -+ curr_idx = 0; -+ for (i = 0; i < contig_access_count; i++) { -+ /* short circuit offset/len processing if len == 0 -+ * (zero-byte read/write */ -+ if (len_list[i] == 0) -+ continue; -+ off = offset_list[i]; -+ avail_len = len_list[i]; -+ proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, striping_info); -+ -+ /* for each separate contiguous access from this process */ -+ if (buf_idx[proc] == -1) -+ buf_idx[proc] = (int) curr_idx; -+ -+ l = my_req[proc].count; -+ curr_idx += (int) avail_len; /* NOTE: Why is curr_idx an int? Fix? */ -+ -+ rem_len = len_list[i] - avail_len; -+ -+ /* store the proc, offset, and len information in an array -+ * of structures, my_req. Each structure contains the -+ * offsets and lengths located in that process's FD, -+ * and the associated count. -+ */ -+ my_req[proc].offsets[l] = off; -+ my_req[proc].lens[l] = (int) avail_len; -+ my_req[proc].count++; -+ -+ while (rem_len != 0) { -+ off += avail_len; -+ avail_len = rem_len; -+ proc = ADIOI_LUSTRE_Calc_aggregator(fd, off, &avail_len, -+ striping_info); -+ if (buf_idx[proc] == -1) -+ buf_idx[proc] = (int) curr_idx; -+ -+ l = my_req[proc].count; -+ curr_idx += avail_len; -+ rem_len -= avail_len; -+ -+ my_req[proc].offsets[l] = off; -+ my_req[proc].lens[l] = (int) avail_len; -+ my_req[proc].count++; -+ } -+ } -+ -+#ifdef AGG_DEBUG -+ for (i = 0; i < nprocs; i++) { -+ if (count_my_req_per_proc[i] > 0) { -+ FPRINTF(stdout, "data needed from %d (count = %d):\n", -+ i, my_req[i].count); -+ for (l = 0; l < my_req[i].count; l++) { -+ FPRINTF(stdout, " off[%d] = %lld, len[%d] = %d\n", -+ l, my_req[i].offsets[l], l, my_req[i].lens[l]); -+ } -+ } -+ } -+#if 0 -+ for (i = 0; i < nprocs; i++) { -+ FPRINTF(stdout, "buf_idx[%d] = 0x%x\n", i, buf_idx[i]); -+ } -+#endif -+#endif -+ -+ *count_my_req_procs_ptr = count_my_req_procs; -+ *buf_idx_ptr = buf_idx; -+} -+ -+int ADIOI_LUSTRE_Docollect(ADIO_File fd, int contig_access_count, -+ int *len_list, int nprocs) -+{ -+ /* If the processes are non-interleaved, we will check the req_size. -+ * if (avg_req_size > big_req_size) { -+ * docollect = 0; -+ * } -+ */ -+ -+ int i, docollect = 1, lflag, big_req_size = 0; -+ ADIO_Offset req_size = 0, total_req_size; -+ int avg_req_size, total_access_count; -+ -+ /* calculate total_req_size and total_access_count */ -+ for (i = 0; i < contig_access_count; i++) -+ req_size += len_list[i]; -+ MPI_Allreduce(&req_size, &total_req_size, 1, MPI_LONG_LONG_INT, MPI_SUM, -+ fd->comm); -+ MPI_Allreduce(&contig_access_count, &total_access_count, 1, MPI_INT, MPI_SUM, -+ fd->comm); -+ /* estimate average req_size */ -+ avg_req_size = (int)(total_req_size / total_access_count); -+ /* get hint of big_req_size */ -+ big_req_size = fd->hints->fs_hints.lustre.coll_threshold; -+ /* Don't perform collective I/O if there are big requests */ -+ if ((big_req_size > 0) && (avg_req_size > big_req_size)) -+ docollect = 0; -+ -+ return docollect; -+} -diff -ruN adio/ad_lustre_orig/ad_lustre.c adio/ad_lustre/ad_lustre.c ---- adio/ad_lustre_orig/ad_lustre.c 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/ad_lustre.c 2008-10-17 17:03:42.000000000 +0800 -@@ -1,9 +1,11 @@ - /* -*- Mode: C; c-basic-offset:4 ; -*- */ --/* -- * Copyright (C) 2001 University of Chicago. -+/* -+ * Copyright (C) 2001 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - */ - - #include "ad_lustre.h" -@@ -13,12 +15,12 @@ - ADIOI_LUSTRE_ReadContig, /* ReadContig */ - ADIOI_LUSTRE_WriteContig, /* WriteContig */ - ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */ -- ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */ -+ ADIOI_LUSTRE_WriteStridedColl, /* WriteStridedColl */ - ADIOI_GEN_SeekIndividual, /* SeekIndividual */ - ADIOI_GEN_Fcntl, /* Fcntl */ - ADIOI_LUSTRE_SetInfo, /* SetInfo */ - ADIOI_GEN_ReadStrided, /* ReadStrided */ -- ADIOI_GEN_WriteStrided, /* WriteStrided */ -+ ADIOI_LUSTRE_WriteStrided, /* WriteStrided */ - ADIOI_GEN_Close, /* Close */ - #if defined(ROMIO_HAVE_WORKING_AIO) && !defined(CRAY_XT_LUSTRE) - ADIOI_GEN_IreadContig, /* IreadContig */ -diff -ruN adio/ad_lustre_orig/ad_lustre.h adio/ad_lustre/ad_lustre.h ---- adio/ad_lustre_orig/ad_lustre.h 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/ad_lustre.h 2009-05-05 15:34:58.000000000 +0800 -@@ -1,9 +1,11 @@ - /* -*- Mode: C; c-basic-offset:4 ; -*- */ --/* -- * Copyright (C) 1997 University of Chicago. -+/* -+ * Copyright (C) 1997 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - */ - - #ifndef AD_UNIX_INCLUDE -@@ -24,7 +26,7 @@ - - /*#include */ - #include --#include "lustre/lustre_user.h" -+#include - #include "adio.h" - /*#include "adioi.h"*/ - -@@ -41,24 +43,31 @@ - - void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code); - void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code); --void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -- MPI_Datatype datatype, int file_ptr_type, -- ADIO_Offset offset, ADIO_Status *status, int -- *error_code); --void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -- MPI_Datatype datatype, int file_ptr_type, -- ADIO_Offset offset, ADIO_Status *status, int -- *error_code); -+void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); -+void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); - void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -- MPI_Datatype datatype, int file_ptr_type, -- ADIO_Offset offset, ADIO_Status *status, int -- *error_code); -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); - void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count, -- MPI_Datatype datatype, int file_ptr_type, -- ADIO_Offset offset, ADIO_Status *status, int -- *error_code); -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); -+void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, -+ int *error_code); - void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, - int *error_code); - void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code); -- - #endif /* End of AD_UNIX_INCLUDE */ -diff -ruN adio/ad_lustre_orig/ad_lustre_hints.c adio/ad_lustre/ad_lustre_hints.c ---- adio/ad_lustre_orig/ad_lustre_hints.c 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_hints.c 2009-04-24 15:35:05.000000000 +0800 -@@ -1,9 +1,11 @@ - /* -*- Mode: C; c-basic-offset:4 ; -*- */ --/* -- * Copyright (C) 1997 University of Chicago. -+/* -+ * Copyright (C) 1997 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - */ - - #include "ad_lustre.h" -@@ -12,46 +14,56 @@ - void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code) - { - char *value, *value_in_fd; -- int flag, tmp_val[3], str_factor=-1, str_unit=0, start_iodev=-1; -+ int flag, stripe_val[3], str_factor = -1, str_unit=0, start_iodev=-1; - struct lov_user_md lum = { 0 }; - int err, myrank, fd_sys, perm, amode, old_mask; -+ int int_val, tmp_val; -+ static char myname[] = "ADIOI_LUSTRE_SETINFO"; - - value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char)); - if ( (fd->info) == MPI_INFO_NULL) { -- /* This must be part of the open call. can set striping parameters -- if necessary. */ -+ /* This must be part of the open call. can set striping parameters -+ if necessary. */ - MPI_Info_create(&(fd->info)); - - MPI_Info_set(fd->info, "direct_read", "false"); - MPI_Info_set(fd->info, "direct_write", "false"); - fd->direct_read = fd->direct_write = 0; -- -- /* has user specified striping or server buffering parameters -+ /* initialize lustre hints */ -+ MPI_Info_set(fd->info, "romio_lustre_co_ratio", "1"); -+ fd->hints->fs_hints.lustre.co_ratio = 1; -+ MPI_Info_set(fd->info, "romio_lustre_coll_threshold", "0"); -+ fd->hints->fs_hints.lustre.coll_threshold = 0; -+ MPI_Info_set(fd->info, "romio_lustre_ds_in_coll", "enable"); -+ fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_ENABLE; -+ -+ /* has user specified striping or server buffering parameters - and do they have the same value on all processes? */ - if (users_info != MPI_INFO_NULL) { -- MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, -+ /* striping information */ -+ MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, - value, &flag); -- if (flag) -+ if (flag) - str_unit=atoi(value); - -- MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, -+ MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, - value, &flag); -- if (flag) -+ if (flag) - str_factor=atoi(value); - -- MPI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, -- value, &flag); -- if (flag) -+ MPI_Info_get(users_info, "romio_lustre_start_iodevice", -+ MPI_MAX_INFO_VAL, value, &flag); -+ if (flag) - start_iodev=atoi(value); - -- MPI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL, -- value, &flag); -+ /* direct read and write */ -+ MPI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL, -+ value, &flag); - if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) { - MPI_Info_set(fd->info, "direct_read", "true"); - fd->direct_read = 1; - } -- -- MPI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL, -+ MPI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL, - value, &flag); - if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) { - MPI_Info_set(fd->info, "direct_write", "true"); -@@ -59,22 +71,23 @@ - } - } - -+ /* set striping information with ioctl */ - MPI_Comm_rank(fd->comm, &myrank); - if (myrank == 0) { -- tmp_val[0] = str_factor; -- tmp_val[1] = str_unit; -- tmp_val[2] = start_iodev; -+ stripe_val[0] = str_factor; -+ stripe_val[1] = str_unit; -+ stripe_val[2] = start_iodev; - } -- MPI_Bcast(tmp_val, 3, MPI_INT, 0, fd->comm); -+ MPI_Bcast(stripe_val, 3, MPI_INT, 0, fd->comm); - -- if (tmp_val[0] != str_factor -- || tmp_val[1] != str_unit -- || tmp_val[2] != start_iodev) { -+ if (stripe_val[0] != str_factor -+ || stripe_val[1] != str_unit -+ || stripe_val[2] != start_iodev) { - FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: All keys" - "-striping_factor:striping_unit:start_iodevice " - "need to be identical across all processes\n"); - MPI_Abort(MPI_COMM_WORLD, 1); -- } else if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) { -+ } else if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) { - /* if user has specified striping info, process 0 tries to set it */ - if (!myrank) { - if (fd->perm == ADIO_PERM_NULL) { -@@ -100,9 +113,9 @@ - amode = amode | O_LOV_DELAY_CREATE | O_CREAT; - - fd_sys = open(fd->filename, amode, perm); -- if (fd_sys == -1) { -- if (errno != EEXIST) -- fprintf(stderr, -+ if (fd_sys == -1) { -+ if (errno != EEXIST) -+ fprintf(stderr, - "Failure to open file %s %d %d\n",strerror(errno), amode, perm); - } else { - lum.lmm_magic = LOV_USER_MAGIC; -@@ -112,25 +125,73 @@ - lum.lmm_stripe_offset = start_iodev; - - err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum); -- if (err == -1 && errno != EEXIST) { -+ if (err == -1 && errno != EEXIST) { - fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno)); - } - close(fd_sys); - } - } /* End of striping parameters validation */ - } -- - MPI_Barrier(fd->comm); -- /* set the values for collective I/O and data sieving parameters */ -- ADIOI_GEN_SetInfo(fd, users_info, error_code); -- } else { -- /* The file has been opened previously and fd->fd_sys is a valid -- file descriptor. cannot set striping parameters now. */ -- -- /* set the values for collective I/O and data sieving parameters */ -- ADIOI_GEN_SetInfo(fd, users_info, error_code); - } -- -+ /* get other hint */ -+ if (users_info != MPI_INFO_NULL) { -+ /* CO: IO Clients/OST, -+ * to keep the load balancing between clients and OSTs */ -+ MPI_Info_get(users_info, "romio_lustre_co_ratio", MPI_MAX_INFO_VAL, value, -+ &flag); -+ if (flag && (int_val = atoi(value)) > 0) { -+ tmp_val = int_val; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != int_val) { -+ MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname, -+ "romio_lustre_co_ratio", -+ error_code); -+ ADIOI_Free(value); -+ return; -+ } -+ MPI_Info_set(fd->info, "romio_lustre_co_ratio", value); -+ fd->hints->fs_hints.lustre.co_ratio = atoi(value); -+ } -+ /* coll_threshold: -+ * if the req size is bigger than this, collective IO may not be performed. -+ */ -+ MPI_Info_get(users_info, "romio_lustre_coll_threshold", MPI_MAX_INFO_VAL, value, -+ &flag); -+ if (flag && (int_val = atoi(value)) > 0) { -+ tmp_val = int_val; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != int_val) { -+ MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname, -+ "romio_lustre_coll_threshold", -+ error_code); -+ ADIOI_Free(value); -+ return; -+ } -+ MPI_Info_set(fd->info, "romio_lustre_coll_threshold", value); -+ fd->hints->fs_hints.lustre.coll_threshold = atoi(value); -+ } -+ /* ds_in_coll: disable data sieving in collective IO */ -+ MPI_Info_get(users_info, "romio_lustre_ds_in_coll", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag && (!strcmp(value, "disable") || -+ !strcmp(value, "DISABLE"))) { -+ tmp_val = int_val = 2; -+ MPI_Bcast(&tmp_val, 2, MPI_INT, 0, fd->comm); -+ if (tmp_val != int_val) { -+ MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname, -+ "romio_lustre_ds_in_coll", -+ error_code); -+ ADIOI_Free(value); -+ return; -+ } -+ MPI_Info_set(fd->info, "romio_lustre_ds_in_coll", "disable"); -+ fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_DISABLE; -+ } -+ } -+ /* set the values for collective I/O and data sieving parameters */ -+ ADIOI_GEN_SetInfo(fd, users_info, error_code); -+ - if (ADIOI_Direct_read) fd->direct_read = 1; - if (ADIOI_Direct_write) fd->direct_write = 1; - -diff -ruN adio/ad_lustre_orig/ad_lustre_open.c adio/ad_lustre/ad_lustre_open.c ---- adio/ad_lustre_orig/ad_lustre_open.c 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_open.c 2009-03-01 11:32:32.000000000 +0800 -@@ -1,9 +1,11 @@ - /* -*- Mode: C; c-basic-offset:4 ; -*- */ --/* -- * Copyright (C) 1997 University of Chicago. -+/* -+ * Copyright (C) 1997 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - */ - - #include "ad_lustre.h" -@@ -51,14 +53,17 @@ - err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *) &lum); - - if (!err) { -+ fd->hints->striping_unit = lum.lmm_stripe_size; - sprintf(value, "%d", lum.lmm_stripe_size); - MPI_Info_set(fd->info, "striping_unit", value); - -+ fd->hints->striping_factor = lum.lmm_stripe_count; - sprintf(value, "%d", lum.lmm_stripe_count); - MPI_Info_set(fd->info, "striping_factor", value); - -+ fd->hints->fs_hints.lustre.start_iodevice = lum.lmm_stripe_offset; - sprintf(value, "%d", lum.lmm_stripe_offset); -- MPI_Info_set(fd->info, "start_iodevice", value); -+ MPI_Info_set(fd->info, "romio_lustre_start_iodevice", value); - } - ADIOI_Free(value); - -diff -ruN adio/ad_lustre_orig/ad_lustre_rwcontig.c adio/ad_lustre/ad_lustre_rwcontig.c ---- adio/ad_lustre_orig/ad_lustre_rwcontig.c 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_rwcontig.c 2009-05-05 15:34:29.000000000 +0800 -@@ -1,9 +1,11 @@ - /* -*- Mode: C; c-basic-offset:4 ; -*- */ --/* -- * Copyright (C) 1997 University of Chicago. -+/* -+ * Copyright (C) 1997 University of Chicago. - * See COPYRIGHT notice in top-level directory. - * - * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. - */ - - #define _XOPEN_SOURCE 600 -@@ -136,10 +138,23 @@ - if (err == -1) goto ioerr; - } - -- if (io_mode) -+ if (io_mode) { -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_write_a, 0, NULL); -+#endif - err = write(fd->fd_sys, buf, len); -- else -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_write_b, 0, NULL); -+#endif -+ } else { -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_read_a, 0, NULL); -+#endif - err = read(fd->fd_sys, buf, len); -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_read_b, 0, NULL); -+#endif -+ } - } else { - err = ADIOI_LUSTRE_Directio(fd, buf, len, offset, io_mode); - } -diff -ruN adio/ad_lustre_orig/ad_lustre_wrcoll.c adio/ad_lustre/ad_lustre_wrcoll.c ---- adio/ad_lustre_orig/ad_lustre_wrcoll.c 1970-01-01 08:00:00.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_wrcoll.c 2009-04-24 14:48:34.000000000 +0800 -@@ -0,0 +1,934 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ * -+ * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. -+ */ -+ -+#include "ad_lustre.h" -+#include "adio_extern.h" -+ -+/* prototypes of functions used for collective writes only. */ -+static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, void *buf, -+ MPI_Datatype datatype, int nprocs, -+ int myrank, -+ ADIOI_Access *others_req, -+ ADIOI_Access *my_req, -+ ADIO_Offset *offset_list, -+ int *len_list, -+ int contig_access_count, -+ int *striping_info, -+ int *buf_idx, int *error_code); -+static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, void *buf, -+ ADIOI_Flatlist_node *flat_buf, -+ char **send_buf, -+ ADIO_Offset *offset_list, -+ int *len_list, int *send_size, -+ MPI_Request *requests, -+ int *sent_to_proc, int nprocs, -+ int myrank, int contig_access_count, -+ int *striping_info, -+ int *send_buf_idx, -+ int *curr_to_proc, -+ int *done_to_proc, int iter, -+ MPI_Aint buftype_extent); -+static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, void *buf, -+ char *write_buf, -+ ADIOI_Flatlist_node *flat_buf, -+ ADIO_Offset *offset_list, -+ int *len_list, int *send_size, -+ int *recv_size, ADIO_Offset off, -+ int size, int *count, -+ int *start_pos, int *partial_recv, -+ int *sent_to_proc, int nprocs, -+ int myrank, int buftype_is_contig, -+ int contig_access_count, -+ int *striping_info, -+ ADIOI_Access *others_req, -+ int *send_buf_idx, -+ int *curr_to_proc, -+ int *done_to_proc, int *hole, -+ int iter, MPI_Aint buftype_extent, -+ int *buf_idx, int *error_code); -+void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, -+ ADIO_Offset *srt_off, int *srt_len, int *start_pos, -+ int nprocs, int nprocs_recv, int total_elements); -+ -+void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, -+ int file_ptr_type, ADIO_Offset offset, -+ ADIO_Status *status, int *error_code) -+{ -+ /* Uses a generalized version of the extended two-phase method described -+ * in "An Extended Two-Phase Method for Accessing Sections of -+ * Out-of-Core Arrays", Rajeev Thakur and Alok Choudhary, -+ * Scientific Programming, (5)4:301--317, Winter 1996. -+ * http://www.mcs.anl.gov/home/thakur/ext2ph.ps -+ */ -+ -+ ADIOI_Access *my_req; -+ /* array of nprocs access structures, one for each other process has -+ this process's request */ -+ -+ ADIOI_Access *others_req; -+ /* array of nprocs access structures, one for each other process -+ whose request is written by this process. */ -+ -+ int i, filetype_is_contig, nprocs, myrank, do_collect = 0; -+ int contig_access_count = 0, buftype_is_contig, interleave_count = 0; -+ int *count_my_req_per_proc, count_my_req_procs, count_others_req_procs; -+ ADIO_Offset orig_fp, start_offset, end_offset, off; -+ ADIO_Offset *offset_list = NULL, *st_offsets = NULL, *end_offsets = NULL; -+ int *buf_idx = NULL, *len_list = NULL, *striping_info = NULL; -+ int old_error, tmp_error; -+ -+ MPI_Comm_size(fd->comm, &nprocs); -+ MPI_Comm_rank(fd->comm, &myrank); -+ -+ orig_fp = fd->fp_ind; -+ -+ /* IO patten identification if cb_write isn't disabled */ -+ if (fd->hints->cb_write != ADIOI_HINT_DISABLE) { -+ /* For this process's request, calculate the list of offsets and -+ lengths in the file and determine the start and end offsets. */ -+ -+ /* Note: end_offset points to the last byte-offset that will be accessed. -+ * e.g., if start_offset=0 and 100 bytes to be read, end_offset=99 -+ */ -+ -+ ADIOI_Calc_my_off_len(fd, count, datatype, file_ptr_type, offset, -+ &offset_list, &len_list, &start_offset, -+ &end_offset, &contig_access_count); -+ -+ /* each process communicates its start and end offsets to other -+ * processes. The result is an array each of start and end offsets -+ * stored in order of process rank. -+ */ -+ st_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset)); -+ end_offsets = (ADIO_Offset *) ADIOI_Malloc(nprocs * sizeof(ADIO_Offset)); -+ MPI_Allgather(&start_offset, 1, ADIO_OFFSET, st_offsets, 1, -+ ADIO_OFFSET, fd->comm); -+ MPI_Allgather(&end_offset, 1, ADIO_OFFSET, end_offsets, 1, -+ ADIO_OFFSET, fd->comm); -+ /* are the accesses of different processes interleaved? */ -+ for (i = 1; i < nprocs; i++) -+ if ((st_offsets[i] < end_offsets[i-1]) && -+ (st_offsets[i] <= end_offsets[i])) -+ interleave_count++; -+ /* This is a rudimentary check for interleaving, but should suffice -+ for the moment. */ -+ -+ /* Two typical access patterns can benefit from collective write. -+ * 1) the processes are interleaved, and -+ * 2) the req size is small. -+ */ -+ if (interleave_count > 0) { -+ do_collect = 1; -+ } else { -+ do_collect = ADIOI_LUSTRE_Docollect(fd, contig_access_count, -+ len_list, nprocs); -+ } -+ } -+ ADIOI_Datatype_iscontig(datatype, &buftype_is_contig); -+ -+ /* Decide if collective I/O should be done */ -+ if ((!do_collect && fd->hints->cb_write == ADIOI_HINT_AUTO) || -+ fd->hints->cb_write == ADIOI_HINT_DISABLE) { -+ -+ int filerange_is_contig = 0; -+ -+ /* use independent accesses */ -+ if (fd->hints->cb_write != ADIOI_HINT_DISABLE) { -+ ADIOI_Free(offset_list); -+ ADIOI_Free(len_list); -+ ADIOI_Free(st_offsets); -+ ADIOI_Free(end_offsets); -+ } -+ -+ fd->fp_ind = orig_fp; -+ ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig); -+ if (buftype_is_contig && filetype_is_contig) { -+ if (file_ptr_type == ADIO_EXPLICIT_OFFSET) { -+ off = fd->disp + (fd->etype_size) * offset; -+ ADIO_WriteContig(fd, buf, count, datatype, -+ ADIO_EXPLICIT_OFFSET, -+ off, status, error_code); -+ } else -+ ADIO_WriteContig(fd, buf, count, datatype, ADIO_INDIVIDUAL, -+ 0, status, error_code); -+ } else { -+ ADIO_WriteStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+ } -+ return; -+ } -+ -+ /* Get Lustre hints information */ -+ ADIOI_LUSTRE_Get_striping_info(fd, &striping_info, 1); -+ -+ /* calculate what portions of the access requests of this process are -+ * located in which process -+ */ -+ ADIOI_LUSTRE_Calc_my_req(fd, offset_list, len_list, contig_access_count, -+ striping_info, nprocs, &count_my_req_procs, -+ &count_my_req_per_proc, &my_req, &buf_idx); -+ -+ /* based on everyone's my_req, calculate what requests of other processes -+ * will be accessed by this process. -+ * count_others_req_procs = number of processes whose requests (including -+ * this process itself) will be accessed by this process -+ * count_others_req_per_proc[i] indicates how many separate contiguous -+ * requests of proc. i will be accessed by this process. -+ */ -+ -+ ADIOI_Calc_others_req(fd, count_my_req_procs, count_my_req_per_proc, -+ my_req, nprocs, myrank, &count_others_req_procs, -+ &others_req); -+ ADIOI_Free(count_my_req_per_proc); -+ -+ /* exchange data and write in sizes of no more than stripe_size. */ -+ ADIOI_LUSTRE_Exch_and_write(fd, buf, datatype, nprocs, myrank, -+ others_req, my_req, -+ offset_list, len_list, contig_access_count, -+ striping_info, buf_idx, error_code); -+ -+ /* If this collective write is followed by an independent write, -+ * it's possible to have those subsequent writes on other processes -+ * race ahead and sneak in before the read-modify-write completes. -+ * We carry out a collective communication at the end here so no one -+ * can start independent i/o before collective I/O completes. -+ * -+ * need to do some gymnastics with the error codes so that if something -+ * went wrong, all processes report error, but if a process has a more -+ * specific error code, we can still have that process report the -+ * additional information */ -+ -+ old_error = *error_code; -+ if (*error_code != MPI_SUCCESS) -+ *error_code = MPI_ERR_IO; -+ -+ /* optimization: if only one process performing i/o, we can perform -+ * a less-expensive Bcast */ -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_postwrite_a, 0, NULL); -+#endif -+ if (fd->hints->cb_nodes == 1) -+ MPI_Bcast(error_code, 1, MPI_INT, -+ fd->hints->ranklist[0], fd->comm); -+ else { -+ tmp_error = *error_code; -+ MPI_Allreduce(&tmp_error, error_code, 1, MPI_INT, -+ MPI_MAX, fd->comm); -+ } -+#ifdef ADIOI_MPE_LOGGING -+ MPE_Log_event(ADIOI_MPE_postwrite_b, 0, NULL); -+#endif -+ -+ if ((old_error != MPI_SUCCESS) && (old_error != MPI_ERR_IO)) -+ *error_code = old_error; -+ -+ -+ if (!buftype_is_contig) -+ ADIOI_Delete_flattened(datatype); -+ -+ /* free all memory allocated for collective I/O */ -+ /* free others_req */ -+ for (i = 0; i < nprocs; i++) { -+ if (others_req[i].count) { -+ ADIOI_Free(others_req[i].offsets); -+ ADIOI_Free(others_req[i].lens); -+ ADIOI_Free(others_req[i].mem_ptrs); -+ } -+ } -+ ADIOI_Free(others_req); -+ /* free my_req here */ -+ for (i = 0; i < nprocs; i++) { -+ if (my_req[i].count) { -+ ADIOI_Free(my_req[i].offsets); -+ ADIOI_Free(my_req[i].lens); -+ } -+ } -+ ADIOI_Free(my_req); -+ ADIOI_Free(buf_idx); -+ ADIOI_Free(offset_list); -+ ADIOI_Free(len_list); -+ ADIOI_Free(st_offsets); -+ ADIOI_Free(end_offsets); -+ ADIOI_Free(striping_info); -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (status) { -+ int bufsize, size; -+ /* Don't set status if it isn't needed */ -+ MPI_Type_size(datatype, &size); -+ bufsize = size * count; -+ MPIR_Status_set_bytes(status, datatype, bufsize); -+ } -+ /* This is a temporary way of filling in status. The right way is to -+ * keep track of how much data was actually written during collective I/O. -+ */ -+#endif -+ -+ fd->fp_sys_posn = -1; /* set it to null. */ -+} -+ -+/* If successful, error_code is set to MPI_SUCCESS. Otherwise an error -+ * code is created and returned in error_code. -+ */ -+static void ADIOI_LUSTRE_Exch_and_write(ADIO_File fd, void *buf, -+ MPI_Datatype datatype, int nprocs, -+ int myrank, ADIOI_Access *others_req, -+ ADIOI_Access *my_req, -+ ADIO_Offset *offset_list, -+ int *len_list, int contig_access_count, -+ int *striping_info, int *buf_idx, -+ int *error_code) -+{ -+ /* Send data to appropriate processes and write in sizes of no more -+ * than lustre stripe_size. -+ * The idea is to reduce the amount of extra memory required for -+ * collective I/O. If all data were written all at once, which is much -+ * easier, it would require temp space more than the size of user_buf, -+ * which is often unacceptable. For example, to write a distributed -+ * array to a file, where each local array is 8Mbytes, requiring -+ * at least another 8Mbytes of temp space is unacceptable. -+ */ -+ -+ int hole, i, j, m, flag, ntimes = 1 , max_ntimes, buftype_is_contig; -+ ADIO_Offset st_loc = -1, end_loc = -1, min_st_loc, max_end_loc; -+ ADIO_Offset off, req_off, send_off, iter_st_off, *off_list; -+ ADIO_Offset max_size, step_size = 0; -+ int real_size, req_len, send_len; -+ int *recv_curr_offlen_ptr, *recv_count, *recv_size; -+ int *send_curr_offlen_ptr, *send_size; -+ int *partial_recv, *sent_to_proc, *recv_start_pos; -+ int *send_buf_idx, *curr_to_proc, *done_to_proc; -+ char *write_buf = NULL; -+ MPI_Status status; -+ ADIOI_Flatlist_node *flat_buf = NULL; -+ MPI_Aint buftype_extent; -+ int stripe_size = striping_info[0], avail_cb_nodes = striping_info[2]; -+ int data_sieving = 0; -+ -+ *error_code = MPI_SUCCESS; /* changed below if error */ -+ /* only I/O errors are currently reported */ -+ -+ /* calculate the number of writes of stripe size to be done. -+ * That gives the no. of communication phases as well. -+ * Note: -+ * Because we redistribute data in stripe-contiguous pattern for Lustre, -+ * each process has the same no. of communication phases. -+ */ -+ -+ for (i = 0; i < nprocs; i++) { -+ if (others_req[i].count) { -+ st_loc = others_req[i].offsets[0]; -+ end_loc = others_req[i].offsets[0]; -+ break; -+ } -+ } -+ for (i = 0; i < nprocs; i++) { -+ for (j = 0; j < others_req[i].count; j++) { -+ st_loc = ADIOI_MIN(st_loc, others_req[i].offsets[j]); -+ end_loc = ADIOI_MAX(end_loc, (others_req[i].offsets[j] + -+ others_req[i].lens[j] - 1)); -+ } -+ } -+ /* this process does no writing. */ -+ if ((st_loc == -1) && (end_loc == -1)) -+ ntimes = 0; -+ MPI_Allreduce(&end_loc, &max_end_loc, 1, MPI_LONG_LONG_INT, MPI_MAX, fd->comm); -+ /* avoid min_st_loc be -1 */ -+ if (st_loc == -1) -+ st_loc = max_end_loc; -+ MPI_Allreduce(&st_loc, &min_st_loc, 1, MPI_LONG_LONG_INT, MPI_MIN, fd->comm); -+ /* align downward */ -+ min_st_loc -= min_st_loc % (ADIO_Offset)stripe_size; -+ -+ /* Each time, only avail_cb_nodes number of IO clients perform IO, -+ * so, step_size=avail_cb_nodes*stripe_size IO will be performed at most, -+ * and ntimes=whole_file_portion/step_size -+ */ -+ step_size = (ADIO_Offset) avail_cb_nodes * stripe_size; -+ max_ntimes = (int)((max_end_loc - min_st_loc) / step_size + 1); -+ if (ntimes) -+ write_buf = (char *) ADIOI_Malloc(stripe_size); -+ -+ /* calculate the start offset for each iteration */ -+ off_list = (ADIO_Offset *) ADIOI_Malloc(max_ntimes * sizeof(ADIO_Offset)); -+ for (m = 0; m < max_ntimes; m ++) -+ off_list[m] = max_end_loc; -+ for (i = 0; i < nprocs; i++) { -+ for (j = 0; j < others_req[i].count; j ++) { -+ req_off = others_req[i].offsets[j]; -+ m = (int)((req_off - min_st_loc) / step_size); -+ off_list[m] = ADIOI_MIN(off_list[m], req_off); -+ } -+ } -+ -+ recv_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); -+ send_curr_offlen_ptr = (int *) ADIOI_Calloc(nprocs, sizeof(int)); -+ /* their use is explained below. calloc initializes to 0. */ -+ -+ recv_count = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* to store count of how many off-len pairs per proc are satisfied -+ in an iteration. */ -+ -+ send_size = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* total size of data to be sent to each proc. in an iteration. -+ Of size nprocs so that I can use MPI_Alltoall later. */ -+ -+ recv_size = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* total size of data to be recd. from each proc. in an iteration. */ -+ -+ sent_to_proc = (int *) ADIOI_Calloc(nprocs, sizeof(int)); -+ /* amount of data sent to each proc so far. Used in -+ ADIOI_Fill_send_buffer. initialized to 0 here. */ -+ -+ send_buf_idx = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ curr_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ done_to_proc = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* Above three are used in ADIOI_Fill_send_buffer */ -+ -+ recv_start_pos = (int *) ADIOI_Malloc(nprocs * sizeof(int)); -+ /* used to store the starting value of recv_curr_offlen_ptr[i] in -+ this iteration */ -+ -+ ADIOI_Datatype_iscontig(datatype, &buftype_is_contig); -+ if (!buftype_is_contig) { -+ ADIOI_Flatten_datatype(datatype); -+ flat_buf = ADIOI_Flatlist; -+ while (flat_buf->type != datatype) -+ flat_buf = flat_buf->next; -+ } -+ MPI_Type_extent(datatype, &buftype_extent); -+ /* I need to check if there are any outstanding nonblocking writes to -+ * the file, which could potentially interfere with the writes taking -+ * place in this collective write call. Since this is not likely to be -+ * common, let me do the simplest thing possible here: Each process -+ * completes all pending nonblocking operations before completing. -+ */ -+ /*ADIOI_Complete_async(error_code); -+ if (*error_code != MPI_SUCCESS) return; -+ MPI_Barrier(fd->comm); -+ */ -+ -+ iter_st_off = min_st_loc; -+ -+ /* Although we have recognized the data according to OST index, -+ * a read-modify-write will be done if there is a hole between the data. -+ * For example: if blocksize=60, xfersize=30 and stripe_size=100, -+ * then rank0 will collect data [0, 30] and [60, 90] then write. There -+ * is a hole in [30, 60], which will cause a read-modify-write in [0, 90]. -+ * -+ * To reduce its impact on the performance, we can disable data sieving -+ * by hint "ds_in_coll". -+ */ -+ /* check the hint for data sieving */ -+ data_sieving = fd->hints->fs_hints.lustre.ds_in_coll; -+ -+ for (m = 0; m < max_ntimes; m++) { -+ /* go through all others_req and my_req to check which will be received -+ * and sent in this iteration. -+ */ -+ -+ /* Note that MPI guarantees that displacements in filetypes are in -+ monotonically nondecreasing order and that, for writes, the -+ filetypes cannot specify overlapping regions in the file. This -+ simplifies implementation a bit compared to reads. */ -+ -+ /* -+ off = start offset in the file for the data to be written in -+ this iteration -+ iter_st_off = start offset of this iteration -+ real_size = size of data written (bytes) corresponding to off -+ max_size = possible maximum size of data written in this iteration -+ req_off = offset in the file for a particular contiguous request minus -+ what was satisfied in previous iteration -+ send_off = offset the request needed by other processes in this iteration -+ req_len = size corresponding to req_off -+ send_len = size corresponding to send_off -+ */ -+ -+ /* first calculate what should be communicated */ -+ for (i = 0; i < nprocs; i++) -+ recv_count[i] = recv_size[i] = send_size[i] = 0; -+ -+ off = off_list[m]; -+ max_size = ADIOI_MIN(step_size, max_end_loc - iter_st_off + 1); -+ real_size = (int) ADIOI_MIN((off / stripe_size + 1) * stripe_size - off, -+ end_loc - off + 1); -+ -+ for (i = 0; i < nprocs; i++) { -+ if (my_req[i].count) { -+ for (j = send_curr_offlen_ptr[i]; j < my_req[i].count; j++) { -+ send_off = my_req[i].offsets[j]; -+ send_len = my_req[i].lens[j]; -+ if (send_off < iter_st_off + max_size) { -+ send_size[i] += send_len; -+ } else { -+ break; -+ } -+ } -+ send_curr_offlen_ptr[i] = j; -+ } -+ if (others_req[i].count) { -+ recv_start_pos[i] = recv_curr_offlen_ptr[i]; -+ for (j = recv_curr_offlen_ptr[i]; j < others_req[i].count; j++) { -+ req_off = others_req[i].offsets[j]; -+ req_len = others_req[i].lens[j]; -+ if (req_off < iter_st_off + max_size) { -+ recv_count[i]++; -+ MPI_Address(write_buf + req_off - off, -+ &(others_req[i].mem_ptrs[j])); -+ recv_size[i] += req_len; -+ } else { -+ break; -+ } -+ } -+ recv_curr_offlen_ptr[i] = j; -+ } -+ } -+ /* use variable "hole" to pass data_sieving flag into W_Exchange_data */ -+ hole = data_sieving; -+ ADIOI_LUSTRE_W_Exchange_data(fd, buf, write_buf, flat_buf, offset_list, -+ len_list, send_size, recv_size, off, real_size, -+ recv_count, recv_start_pos, partial_recv, -+ sent_to_proc, nprocs, myrank, -+ buftype_is_contig, contig_access_count, -+ striping_info, others_req, send_buf_idx, -+ curr_to_proc, done_to_proc, &hole, m, -+ buftype_extent, buf_idx, error_code); -+ if (*error_code != MPI_SUCCESS) -+ goto over; -+ -+ flag = 0; -+ for (i = 0; i < nprocs; i++) -+ if (recv_count[i]) { -+ flag = 1; -+ break; -+ } -+ if (flag) { -+ /* check whether to do data sieving */ -+ if(data_sieving == ADIOI_HINT_ENABLE) { -+ ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE, -+ ADIO_EXPLICIT_OFFSET, off, &status, -+ error_code); -+ } else { -+ /* if there is no hole, write data in one time; -+ * otherwise, write data in several times */ -+ if (!hole) { -+ ADIO_WriteContig(fd, write_buf, real_size, MPI_BYTE, -+ ADIO_EXPLICIT_OFFSET, off, &status, -+ error_code); -+ } else { -+ for (i = 0; i < nprocs; i++) { -+ if (others_req[i].count) { -+ for (j = 0; j < others_req[i].count; j++) { -+ if (others_req[i].offsets[j] < off + real_size && -+ others_req[i].offsets[j] >= off) { -+ ADIO_WriteContig(fd, -+ write_buf + others_req[i].offsets[j] - off, -+ others_req[i].lens[j], -+ MPI_BYTE, ADIO_EXPLICIT_OFFSET, -+ others_req[i].offsets[j], &status, -+ error_code); -+ if (*error_code != MPI_SUCCESS) -+ goto over; -+ } -+ } -+ } -+ } -+ } -+ } -+ if (*error_code != MPI_SUCCESS) -+ goto over; -+ } -+ iter_st_off += max_size; -+ } -+over: -+ if (ntimes) -+ ADIOI_Free(write_buf); -+ ADIOI_Free(recv_curr_offlen_ptr); -+ ADIOI_Free(send_curr_offlen_ptr); -+ ADIOI_Free(recv_count); -+ ADIOI_Free(send_size); -+ ADIOI_Free(recv_size); -+ ADIOI_Free(sent_to_proc); -+ ADIOI_Free(recv_start_pos); -+ ADIOI_Free(send_buf_idx); -+ ADIOI_Free(curr_to_proc); -+ ADIOI_Free(done_to_proc); -+ ADIOI_Free(off_list); -+} -+ -+/* Sets error_code to MPI_SUCCESS if successful, or creates an error code -+ * in the case of error. -+ */ -+static void ADIOI_LUSTRE_W_Exchange_data(ADIO_File fd, void *buf, -+ char *write_buf, -+ ADIOI_Flatlist_node *flat_buf, -+ ADIO_Offset *offset_list, -+ int *len_list, int *send_size, -+ int *recv_size, ADIO_Offset off, -+ int size, int *count, -+ int *start_pos, int *partial_recv, -+ int *sent_to_proc, int nprocs, -+ int myrank, int buftype_is_contig, -+ int contig_access_count, -+ int *striping_info, -+ ADIOI_Access *others_req, -+ int *send_buf_idx, -+ int *curr_to_proc, int *done_to_proc, -+ int *hole, int iter, -+ MPI_Aint buftype_extent, -+ int *buf_idx, int *error_code) -+{ -+ int i, j, nprocs_recv, nprocs_send, err; -+ char **send_buf = NULL; -+ MPI_Request *requests, *send_req; -+ MPI_Datatype *recv_types; -+ MPI_Status *statuses, status; -+ int *srt_len, sum, sum_recv; -+ ADIO_Offset *srt_off; -+ int data_sieving = *hole; -+ static char myname[] = "ADIOI_W_EXCHANGE_DATA"; -+ -+ /* create derived datatypes for recv */ -+ nprocs_recv = 0; -+ for (i = 0; i < nprocs; i++) -+ if (recv_size[i]) -+ nprocs_recv++; -+ -+ recv_types = (MPI_Datatype *) ADIOI_Malloc((nprocs_recv + 1) * -+ sizeof(MPI_Datatype)); -+ /* +1 to avoid a 0-size malloc */ -+ -+ j = 0; -+ for (i = 0; i < nprocs; i++) { -+ if (recv_size[i]) { -+ MPI_Type_hindexed(count[i], -+ &(others_req[i].lens[start_pos[i]]), -+ &(others_req[i].mem_ptrs[start_pos[i]]), -+ MPI_BYTE, recv_types + j); -+ /* absolute displacements; use MPI_BOTTOM in recv */ -+ MPI_Type_commit(recv_types + j); -+ j++; -+ } -+ } -+ -+ /* To avoid a read-modify-write, -+ * check if there are holes in the data to be written. -+ * For this, merge the (sorted) offset lists others_req using a heap-merge. -+ */ -+ -+ sum = 0; -+ for (i = 0; i < nprocs; i++) -+ sum += count[i]; -+ srt_off = (ADIO_Offset *) ADIOI_Malloc((sum + 1) * sizeof(ADIO_Offset)); -+ srt_len = (int *) ADIOI_Malloc((sum + 1) * sizeof(int)); -+ /* +1 to avoid a 0-size malloc */ -+ -+ ADIOI_Heap_merge(others_req, count, srt_off, srt_len, start_pos, -+ nprocs, nprocs_recv, sum); -+ -+ /* check if there are any holes */ -+ *hole = 0; -+ for (i = 0; i < sum - 1; i++) { -+ if (srt_off[i] + srt_len[i] < srt_off[i + 1]) { -+ *hole = 1; -+ break; -+ } -+ } -+ /* In some cases (see John Bent ROMIO REQ # 835), an odd interaction -+ * between aggregation, nominally contiguous regions, and cb_buffer_size -+ * should be handled with a read-modify-write (otherwise we will write out -+ * more data than we receive from everyone else (inclusive), so override -+ * hole detection -+ */ -+ if (*hole == 0) { -+ sum_recv = 0; -+ for (i = 0; i < nprocs; i++) -+ sum_recv += recv_size[i]; -+ if (size > sum_recv) -+ *hole = 1; -+ } -+ /* check the hint for data sieving */ -+ if (data_sieving == ADIOI_HINT_ENABLE && nprocs_recv && *hole) { -+ ADIO_ReadContig(fd, write_buf, size, MPI_BYTE, -+ ADIO_EXPLICIT_OFFSET, off, &status, &err); -+ // --BEGIN ERROR HANDLING-- -+ if (err != MPI_SUCCESS) { -+ *error_code = MPIO_Err_create_code(err, -+ MPIR_ERR_RECOVERABLE, -+ myname, __LINE__, -+ MPI_ERR_IO, -+ "**ioRMWrdwr", 0); -+ ADIOI_Free(recv_types); -+ ADIOI_Free(srt_off); -+ ADIOI_Free(srt_len); -+ return; -+ } -+ // --END ERROR HANDLING-- -+ } -+ ADIOI_Free(srt_off); -+ ADIOI_Free(srt_len); -+ -+ nprocs_send = 0; -+ for (i = 0; i < nprocs; i++) -+ if (send_size[i]) -+ nprocs_send++; -+ -+ if (fd->atomicity) { -+ /* bug fix from Wei-keng Liao and Kenin Coloma */ -+ requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + 1) * -+ sizeof(MPI_Request)); -+ send_req = requests; -+ } else { -+ requests = (MPI_Request *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1)* -+ sizeof(MPI_Request)); -+ /* +1 to avoid a 0-size malloc */ -+ -+ /* post receives */ -+ j = 0; -+ for (i = 0; i < nprocs; i++) { -+ if (recv_size[i]) { -+ MPI_Irecv(MPI_BOTTOM, 1, recv_types[j], i, -+ myrank + i + 100 * iter, fd->comm, requests + j); -+ j++; -+ } -+ } -+ send_req = requests + nprocs_recv; -+ } -+ -+ /* post sends. -+ * if buftype_is_contig, data can be directly sent from -+ * user buf at location given by buf_idx. else use send_buf. -+ */ -+ if (buftype_is_contig) { -+ j = 0; -+ for (i = 0; i < nprocs; i++) -+ if (send_size[i]) { -+ MPI_Isend(((char *) buf) + buf_idx[i], send_size[i], -+ MPI_BYTE, i, myrank + i + 100 * iter, fd->comm, -+ send_req + j); -+ j++; -+ buf_idx[i] += send_size[i]; -+ } -+ } else if (nprocs_send) { -+ /* buftype is not contig */ -+ send_buf = (char **) ADIOI_Malloc(nprocs * sizeof(char *)); -+ for (i = 0; i < nprocs; i++) -+ if (send_size[i]) -+ send_buf[i] = (char *) ADIOI_Malloc(send_size[i]); -+ -+ ADIOI_LUSTRE_Fill_send_buffer(fd, buf, flat_buf, send_buf, offset_list, -+ len_list, send_size, send_req, -+ sent_to_proc, nprocs, myrank, -+ contig_access_count, striping_info, -+ send_buf_idx, curr_to_proc, done_to_proc, -+ iter, buftype_extent); -+ /* the send is done in ADIOI_Fill_send_buffer */ -+ } -+ -+ /* bug fix from Wei-keng Liao and Kenin Coloma */ -+ if (fd->atomicity) { -+ j = 0; -+ for (i = 0; i < nprocs; i++) { -+ MPI_Status wkl_status; -+ if (recv_size[i]) { -+ MPI_Recv(MPI_BOTTOM, 1, recv_types[j], i, -+ myrank + i + 100 * iter, fd->comm, &wkl_status); -+ j++; -+ } -+ } -+ } -+ -+ for (i = 0; i < nprocs_recv; i++) -+ MPI_Type_free(recv_types + i); -+ ADIOI_Free(recv_types); -+ -+ /* bug fix from Wei-keng Liao and Kenin Coloma */ -+ /* +1 to avoid a 0-size malloc */ -+ if (fd->atomicity) { -+ statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + 1) * -+ sizeof(MPI_Status)); -+ } else { -+ statuses = (MPI_Status *) ADIOI_Malloc((nprocs_send + nprocs_recv + 1) * -+ sizeof(MPI_Status)); -+ } -+ -+#ifdef NEEDS_MPI_TEST -+ i = 0; -+ if (fd->atomicity) { -+ /* bug fix from Wei-keng Liao and Kenin Coloma */ -+ while (!i) -+ MPI_Testall(nprocs_send, send_req, &i, statuses); -+ } else { -+ while (!i) -+ MPI_Testall(nprocs_send + nprocs_recv, requests, &i, statuses); -+ } -+#else -+ /* bug fix from Wei-keng Liao and Kenin Coloma */ -+ if (fd->atomicity) -+ MPI_Waitall(nprocs_send, send_req, statuses); -+ else -+ MPI_Waitall(nprocs_send + nprocs_recv, requests, statuses); -+#endif -+ ADIOI_Free(statuses); -+ ADIOI_Free(requests); -+ if (!buftype_is_contig && nprocs_send) { -+ for (i = 0; i < nprocs; i++) -+ if (send_size[i]) -+ ADIOI_Free(send_buf[i]); -+ ADIOI_Free(send_buf); -+ } -+} -+ -+#define ADIOI_BUF_INCR \ -+{ \ -+ while (buf_incr) { \ -+ size_in_buf = ADIOI_MIN(buf_incr, flat_buf_sz); \ -+ user_buf_idx += size_in_buf; \ -+ flat_buf_sz -= size_in_buf; \ -+ if (!flat_buf_sz) { \ -+ if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \ -+ else { \ -+ flat_buf_idx = 0; \ -+ n_buftypes++; \ -+ } \ -+ user_buf_idx = flat_buf->indices[flat_buf_idx] + \ -+ n_buftypes*buftype_extent; \ -+ flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \ -+ } \ -+ buf_incr -= size_in_buf; \ -+ } \ -+} -+ -+ -+#define ADIOI_BUF_COPY \ -+{ \ -+ while (size) { \ -+ size_in_buf = ADIOI_MIN(size, flat_buf_sz); \ -+ memcpy(&(send_buf[p][send_buf_idx[p]]), \ -+ ((char *) buf) + user_buf_idx, size_in_buf); \ -+ send_buf_idx[p] += size_in_buf; \ -+ user_buf_idx += size_in_buf; \ -+ flat_buf_sz -= size_in_buf; \ -+ if (!flat_buf_sz) { \ -+ if (flat_buf_idx < (flat_buf->count - 1)) flat_buf_idx++; \ -+ else { \ -+ flat_buf_idx = 0; \ -+ n_buftypes++; \ -+ } \ -+ user_buf_idx = flat_buf->indices[flat_buf_idx] + \ -+ n_buftypes*buftype_extent; \ -+ flat_buf_sz = flat_buf->blocklens[flat_buf_idx]; \ -+ } \ -+ size -= size_in_buf; \ -+ buf_incr -= size_in_buf; \ -+ } \ -+ ADIOI_BUF_INCR \ -+} -+ -+static void ADIOI_LUSTRE_Fill_send_buffer(ADIO_File fd, void *buf, -+ ADIOI_Flatlist_node *flat_buf, -+ char **send_buf, -+ ADIO_Offset *offset_list, -+ int *len_list, int *send_size, -+ MPI_Request *requests, -+ int *sent_to_proc, int nprocs, -+ int myrank, -+ int contig_access_count, -+ int *striping_info, -+ int *send_buf_idx, -+ int *curr_to_proc, -+ int *done_to_proc, int iter, -+ MPI_Aint buftype_extent) -+{ -+ /* this function is only called if buftype is not contig */ -+ int i, p, flat_buf_idx, size; -+ int flat_buf_sz, buf_incr, size_in_buf, jj, n_buftypes; -+ ADIO_Offset off, len, rem_len, user_buf_idx; -+ -+ /* curr_to_proc[p] = amount of data sent to proc. p that has already -+ * been accounted for so far -+ * done_to_proc[p] = amount of data already sent to proc. p in -+ * previous iterations -+ * user_buf_idx = current location in user buffer -+ * send_buf_idx[p] = current location in send_buf of proc. p -+ */ -+ -+ for (i = 0; i < nprocs; i++) { -+ send_buf_idx[i] = curr_to_proc[i] = 0; -+ done_to_proc[i] = sent_to_proc[i]; -+ } -+ jj = 0; -+ -+ user_buf_idx = flat_buf->indices[0]; -+ flat_buf_idx = 0; -+ n_buftypes = 0; -+ flat_buf_sz = flat_buf->blocklens[0]; -+ -+ /* flat_buf_idx = current index into flattened buftype -+ * flat_buf_sz = size of current contiguous component in flattened buf -+ */ -+ for (i = 0; i < contig_access_count; i++) { -+ off = offset_list[i]; -+ rem_len = (ADIO_Offset) len_list[i]; -+ -+ /*this request may span to more than one process */ -+ while (rem_len != 0) { -+ len = rem_len; -+ /* NOTE: len value is modified by ADIOI_Calc_aggregator() to be no -+ * longer than the single region that processor "p" is responsible -+ * for. -+ */ -+ p = ADIOI_LUSTRE_Calc_aggregator(fd, off, &len, striping_info); -+ -+ if (send_buf_idx[p] < send_size[p]) { -+ if (curr_to_proc[p] + len > done_to_proc[p]) { -+ if (done_to_proc[p] > curr_to_proc[p]) { -+ size = (int) ADIOI_MIN(curr_to_proc[p] + len - -+ done_to_proc[p], -+ send_size[p] - -+ send_buf_idx[p]); -+ buf_incr = done_to_proc[p] - curr_to_proc[p]; -+ ADIOI_BUF_INCR -+ buf_incr = (int) (curr_to_proc[p] + len - -+ done_to_proc[p]); -+ curr_to_proc[p] = done_to_proc[p] + size; -+ ADIOI_BUF_COPY -+ } else { -+ size = (int) ADIOI_MIN(len, send_size[p] - -+ send_buf_idx[p]); -+ buf_incr = (int) len; -+ curr_to_proc[p] += size; -+ ADIOI_BUF_COPY -+ } -+ if (send_buf_idx[p] == send_size[p]) { -+ MPI_Isend(send_buf[p], send_size[p], MPI_BYTE, p, -+ myrank + p + 100 * iter, fd->comm, -+ requests + jj); -+ jj++; -+ } -+ } else { -+ curr_to_proc[p] += (int) len; -+ buf_incr = (int) len; -+ ADIOI_BUF_INCR -+ } -+ } else { -+ buf_incr = (int) len; -+ ADIOI_BUF_INCR -+ } -+ off += len; -+ rem_len -= len; -+ } -+ } -+ for (i = 0; i < nprocs; i++) -+ if (send_size[i]) -+ sent_to_proc[i] = curr_to_proc[i]; -+} -diff -ruN adio/ad_lustre_orig/ad_lustre_wrstr.c adio/ad_lustre/ad_lustre_wrstr.c ---- adio/ad_lustre_orig/ad_lustre_wrstr.c 1970-01-01 08:00:00.000000000 +0800 -+++ adio/ad_lustre/ad_lustre_wrstr.c 2009-02-27 10:35:18.000000000 +0800 -@@ -0,0 +1,467 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ * -+ * Copyright (C) 2007 Oak Ridge National Laboratory -+ * -+ * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. -+ */ -+ -+#include "ad_lustre.h" -+#include "adio_extern.h" -+ -+#define ADIOI_BUFFERED_WRITE \ -+{ \ -+ if (req_off >= writebuf_off + writebuf_len) { \ -+ if (writebuf_len) { \ -+ ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \ -+ ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \ -+ if (!(fd->atomicity)) \ -+ ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ -+ if (*error_code != MPI_SUCCESS) { \ -+ *error_code = MPIO_Err_create_code(*error_code, \ -+ MPIR_ERR_RECOVERABLE, myname, \ -+ __LINE__, MPI_ERR_IO, \ -+ "**iowswc", 0); \ -+ ADIOI_Free(writebuf); \ -+ return; \ -+ } \ -+ } \ -+ writebuf_off = req_off; \ -+ /* stripe_size alignment */ \ -+ writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \ -+ (writebuf_off / stripe_size + 1) * \ -+ stripe_size - writebuf_off);\ -+ if (!(fd->atomicity)) \ -+ ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ -+ ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,\ -+ writebuf_off, &status1, error_code); \ -+ if (*error_code != MPI_SUCCESS) { \ -+ *error_code = MPIO_Err_create_code(*error_code, \ -+ MPIR_ERR_RECOVERABLE, myname, \ -+ __LINE__, MPI_ERR_IO, \ -+ "**iowsrc", 0); \ -+ ADIOI_Free(writebuf); \ -+ return; \ -+ } \ -+ } \ -+ write_sz = (int) ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off); \ -+ memcpy(writebuf + req_off - writebuf_off, (char *)buf + userbuf_off, write_sz);\ -+ while (write_sz != req_len) {\ -+ ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \ -+ ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \ -+ if (!(fd->atomicity)) \ -+ ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ -+ if (*error_code != MPI_SUCCESS) { \ -+ *error_code = MPIO_Err_create_code(*error_code, \ -+ MPIR_ERR_RECOVERABLE, myname, \ -+ __LINE__, MPI_ERR_IO, \ -+ "**iowswc", 0); \ -+ ADIOI_Free(writebuf); \ -+ return; \ -+ } \ -+ req_len -= write_sz; \ -+ userbuf_off += write_sz; \ -+ writebuf_off += writebuf_len; \ -+ /* stripe_size alignment */ \ -+ writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \ -+ (writebuf_off / stripe_size + 1) * \ -+ stripe_size - writebuf_off);\ -+ if (!(fd->atomicity)) \ -+ ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \ -+ ADIO_ReadContig(fd, writebuf, writebuf_len, MPI_BYTE, ADIO_EXPLICIT_OFFSET,\ -+ writebuf_off, &status1, error_code); \ -+ if (*error_code != MPI_SUCCESS) { \ -+ *error_code = MPIO_Err_create_code(*error_code, \ -+ MPIR_ERR_RECOVERABLE, myname, \ -+ __LINE__, MPI_ERR_IO, \ -+ "**iowsrc", 0); \ -+ ADIOI_Free(writebuf); \ -+ return; \ -+ } \ -+ write_sz = ADIOI_MIN(req_len, writebuf_len); \ -+ memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\ -+ } \ -+} -+ -+ -+/* this macro is used when filetype is contig and buftype is not contig. -+ it does not do a read-modify-write and does not lock*/ -+#define ADIOI_BUFFERED_WRITE_WITHOUT_READ \ -+{ \ -+ if (req_off >= writebuf_off + writebuf_len) { \ -+ writebuf_off = req_off; \ -+ /* stripe_size alignment */ \ -+ writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \ -+ (writebuf_off / stripe_size + 1) * \ -+ stripe_size - writebuf_off);\ -+ } \ -+ write_sz = (int) ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off); \ -+ memcpy(writebuf + req_off - writebuf_off, (char *)buf + userbuf_off, write_sz);\ -+ while (req_len) { \ -+ ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, \ -+ ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, error_code); \ -+ if (*error_code != MPI_SUCCESS) { \ -+ *error_code = MPIO_Err_create_code(*error_code, \ -+ MPIR_ERR_RECOVERABLE, myname, \ -+ __LINE__, MPI_ERR_IO, \ -+ "**iowswc", 0); \ -+ ADIOI_Free(writebuf); \ -+ return; \ -+ } \ -+ req_len -= write_sz; \ -+ userbuf_off += write_sz; \ -+ writebuf_off += writebuf_len; \ -+ /* stripe_size alignment */ \ -+ writebuf_len = (int) ADIOI_MIN(end_offset - writebuf_off + 1, \ -+ (writebuf_off / stripe_size + 1) * \ -+ stripe_size - writebuf_off);\ -+ write_sz = ADIOI_MIN(req_len, writebuf_len); \ -+ memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\ -+ } \ -+} -+ -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status * status, -+ int *error_code) -+{ -+ /* offset is in units of etype relative to the filetype. */ -+ ADIOI_Flatlist_node *flat_buf, *flat_file; -+ int i, j, k, bwr_size, fwr_size = 0, st_index = 0; -+ int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype; -+ int n_filetypes, etype_in_filetype; -+ ADIO_Offset abs_off_in_filetype = 0; -+ int filetype_size, etype_size, buftype_size, req_len; -+ MPI_Aint filetype_extent, buftype_extent; -+ int buf_count, buftype_is_contig, filetype_is_contig; -+ ADIO_Offset userbuf_off; -+ ADIO_Offset off, req_off, disp, end_offset = 0, writebuf_off, start_off; -+ char *writebuf; -+ int flag, st_fwr_size, st_n_filetypes, writebuf_len, write_sz; -+ ADIO_Status status1; -+ int new_bwr_size, new_fwr_size; -+ int stripe_size; -+ static char myname[] = "ADIOI_LUSTRE_WriteStrided"; -+ int myrank; -+ MPI_Comm_rank(fd->comm, &myrank); -+ -+ if (fd->hints->ds_write == ADIOI_HINT_DISABLE) { -+ /* if user has disabled data sieving on writes, use naive -+ * approach instead. -+ */ -+ ADIOI_GEN_WriteStrided_naive(fd, -+ buf, -+ count, -+ datatype, -+ file_ptr_type, -+ offset, status, error_code); -+ return; -+ } -+ -+ *error_code = MPI_SUCCESS; /* changed below if error */ -+ -+ ADIOI_Datatype_iscontig(datatype, &buftype_is_contig); -+ ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig); -+ -+ MPI_Type_size(fd->filetype, &filetype_size); -+ if (!filetype_size) { -+ *error_code = MPI_SUCCESS; -+ return; -+ } -+ -+ MPI_Type_extent(fd->filetype, &filetype_extent); -+ MPI_Type_size(datatype, &buftype_size); -+ MPI_Type_extent(datatype, &buftype_extent); -+ etype_size = fd->etype_size; -+ -+ bufsize = buftype_size * count; -+ -+ /* get striping info */ -+ stripe_size = fd->hints->striping_unit; -+ -+ /* Different buftype to different filetype */ -+ if (!buftype_is_contig && filetype_is_contig) { -+ /* noncontiguous in memory, contiguous in file. */ -+ ADIOI_Flatten_datatype(datatype); -+ flat_buf = ADIOI_Flatlist; -+ while (flat_buf->type != datatype) -+ flat_buf = flat_buf->next; -+ -+ off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : -+ fd->disp + etype_size * offset; -+ -+ start_off = off; -+ end_offset = start_off + bufsize - 1; -+ writebuf_off = start_off; -+ /* write stripe size buffer each time */ -+ writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size)); -+ writebuf_len = (int) ADIOI_MIN(bufsize, -+ (writebuf_off / stripe_size + 1) * -+ stripe_size - writebuf_off); -+ -+ /* if atomicity is true, lock the region to be accessed */ -+ if (fd->atomicity) -+ ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize); -+ -+ for (j = 0; j < count; j++) { -+ for (i = 0; i < flat_buf->count; i++) { -+ userbuf_off = j * buftype_extent + flat_buf->indices[i]; -+ req_off = off; -+ req_len = flat_buf->blocklens[i]; -+ ADIOI_BUFFERED_WRITE_WITHOUT_READ -+ off += flat_buf->blocklens[i]; -+ } -+ } -+ -+ /* write the buffer out finally */ -+ ADIO_WriteContig(fd, writebuf, writebuf_len, MPI_BYTE, -+ ADIO_EXPLICIT_OFFSET, writebuf_off, &status1, -+ error_code); -+ -+ if (fd->atomicity) -+ ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize); -+ if (*error_code != MPI_SUCCESS) { -+ ADIOI_Free(writebuf); -+ return; -+ } -+ ADIOI_Free(writebuf); -+ if (file_ptr_type == ADIO_INDIVIDUAL) -+ fd->fp_ind = off; -+ } else { -+ /* noncontiguous in file */ -+ /* filetype already flattened in ADIO_Open */ -+ flat_file = ADIOI_Flatlist; -+ while (flat_file->type != fd->filetype) -+ flat_file = flat_file->next; -+ disp = fd->disp; -+ -+ if (file_ptr_type == ADIO_INDIVIDUAL) { -+ offset = fd->fp_ind; /* in bytes */ -+ n_filetypes = -1; -+ flag = 0; -+ while (!flag) { -+ n_filetypes++; -+ for (i = 0; i < flat_file->count; i++) { -+ if (disp + flat_file->indices[i] + -+ (ADIO_Offset) n_filetypes * filetype_extent + -+ flat_file->blocklens[i] >= offset) { -+ st_index = i; -+ fwr_size = (int) (disp + flat_file->indices[i] + -+ (ADIO_Offset) n_filetypes * -+ filetype_extent + -+ flat_file->blocklens[i] - -+ offset); -+ flag = 1; -+ break; -+ } -+ } -+ } -+ } else { -+ n_etypes_in_filetype = filetype_size / etype_size; -+ n_filetypes = (int) (offset / n_etypes_in_filetype); -+ etype_in_filetype = (int) (offset % n_etypes_in_filetype); -+ size_in_filetype = etype_in_filetype * etype_size; -+ -+ sum = 0; -+ for (i = 0; i < flat_file->count; i++) { -+ sum += flat_file->blocklens[i]; -+ if (sum > size_in_filetype) { -+ st_index = i; -+ fwr_size = sum - size_in_filetype; -+ abs_off_in_filetype = flat_file->indices[i] + -+ size_in_filetype - (sum - flat_file->blocklens[i]); -+ break; -+ } -+ } -+ -+ /* abs. offset in bytes in the file */ -+ offset = disp + (ADIO_Offset) n_filetypes *filetype_extent + -+ abs_off_in_filetype; -+ } -+ -+ start_off = offset; -+ -+ /* If the file bytes is actually contiguous, we do not need data sieve at all */ -+ if (bufsize <= fwr_size) { -+ req_off = start_off; -+ req_len = bufsize; -+ end_offset = start_off + bufsize - 1; -+ writebuf = (char *) ADIOI_Malloc(ADIOI_MIN(bufsize, stripe_size)); -+ memset(writebuf, -1, ADIOI_MIN(bufsize, stripe_size)); -+ writebuf_off = 0; -+ writebuf_len = 0; -+ userbuf_off = 0; -+ ADIOI_BUFFERED_WRITE_WITHOUT_READ -+ } else { -+ /* Calculate end_offset, the last byte-offset that will be accessed. -+ e.g., if start_offset=0 and 100 bytes to be write, end_offset=99 */ -+ st_fwr_size = fwr_size; -+ st_n_filetypes = n_filetypes; -+ i = 0; -+ j = st_index; -+ off = offset; -+ fwr_size = ADIOI_MIN(st_fwr_size, bufsize); -+ while (i < bufsize) { -+ i += fwr_size; -+ end_offset = off + fwr_size - 1; -+ -+ if (j < (flat_file->count - 1)) -+ j++; -+ else { -+ j = 0; -+ n_filetypes++; -+ } -+ -+ off = disp + flat_file->indices[j] + -+ (ADIO_Offset) n_filetypes * filetype_extent; -+ fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize - i); -+ } -+ -+ writebuf_off = 0; -+ writebuf_len = 0; -+ writebuf = (char *) ADIOI_Malloc(stripe_size); -+ memset(writebuf, -1, stripe_size); -+ /* if atomicity is true, lock the region to be accessed */ -+ if (fd->atomicity) -+ ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, bufsize); -+ -+ if (buftype_is_contig && !filetype_is_contig) { -+ /* contiguous in memory, noncontiguous in file. should be the most -+ common case. */ -+ i = 0; -+ j = st_index; -+ off = offset; -+ n_filetypes = st_n_filetypes; -+ fwr_size = ADIOI_MIN(st_fwr_size, bufsize); -+ while (i < bufsize) { -+ if (fwr_size) { -+ /* TYPE_UB and TYPE_LB can result in -+ fwr_size = 0. save system call in such cases */ -+ /* -+ lseek(fd->fd_sys, off, SEEK_SET); -+ err = write(fd->fd_sys, ((char *) buf) + i, fwr_size); -+ */ -+ req_off = off; -+ req_len = fwr_size; -+ userbuf_off = i; -+ ADIOI_BUFFERED_WRITE -+ } -+ i += fwr_size; -+ -+ if (off + fwr_size < disp + flat_file->indices[j] + -+ flat_file->blocklens[j] + -+ (ADIO_Offset) n_filetypes * filetype_extent) -+ off += fwr_size; -+ /* did not reach end of contiguous block in filetype. -+ no more I/O needed. off is incremented by fwr_size. */ -+ else { -+ if (j < (flat_file->count - 1)) -+ j++; -+ else { -+ j = 0; -+ n_filetypes++; -+ } -+ off = disp + flat_file->indices[j] + -+ (ADIO_Offset) n_filetypes * filetype_extent; -+ fwr_size = ADIOI_MIN(flat_file->blocklens[j], -+ bufsize - i); -+ } -+ } -+ } else { -+ /* noncontiguous in memory as well as in file */ -+ ADIOI_Flatten_datatype(datatype); -+ flat_buf = ADIOI_Flatlist; -+ while (flat_buf->type != datatype) -+ flat_buf = flat_buf->next; -+ -+ k = num = buf_count = 0; -+ i = (int) (flat_buf->indices[0]); -+ j = st_index; -+ off = offset; -+ n_filetypes = st_n_filetypes; -+ fwr_size = st_fwr_size; -+ bwr_size = flat_buf->blocklens[0]; -+ -+ while (num < bufsize) { -+ size = ADIOI_MIN(fwr_size, bwr_size); -+ if (size) { -+ /* -+ lseek(fd->fd_sys, off, SEEK_SET); -+ err = write(fd->fd_sys, ((char *) buf) + i, size); -+ */ -+ req_off = off; -+ req_len = size; -+ userbuf_off = i; -+ ADIOI_BUFFERED_WRITE -+ } -+ -+ new_fwr_size = fwr_size; -+ new_bwr_size = bwr_size; -+ -+ if (size == fwr_size) { -+ /* reached end of contiguous block in file */ -+ if (j < (flat_file->count - 1)) { -+ j++; -+ } else { -+ j = 0; -+ n_filetypes++; -+ } -+ off = disp + flat_file->indices[j] + -+ (ADIO_Offset) n_filetypes * filetype_extent; -+ new_fwr_size = flat_file->blocklens[j]; -+ if (size != bwr_size) { -+ i += size; -+ new_bwr_size -= size; -+ } -+ } -+ if (size == bwr_size) { -+ /* reached end of contiguous block in memory */ -+ k = (k + 1) % flat_buf->count; -+ buf_count++; -+ i = (int) (buftype_extent * -+ (buf_count / flat_buf->count) + -+ flat_buf->indices[k]); -+ new_bwr_size = flat_buf->blocklens[k]; -+ if (size != fwr_size) { -+ off += size; -+ new_fwr_size -= size; -+ } -+ } -+ num += size; -+ fwr_size = new_fwr_size; -+ bwr_size = new_bwr_size; -+ } -+ } -+ -+ /* write the buffer out finally */ -+ if (writebuf_len) { -+ ADIO_WriteContig(fd, writebuf, writebuf_len, -+ MPI_BYTE, ADIO_EXPLICIT_OFFSET, -+ writebuf_off, &status1, error_code); -+ if (!(fd->atomicity)) -+ ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); -+ if (*error_code != MPI_SUCCESS) { -+ ADIOI_Free(writebuf); -+ return; -+ } -+ } -+ if (fd->atomicity) -+ ADIOI_UNLOCK(fd, start_off, SEEK_SET, bufsize); -+ } -+ ADIOI_Free(writebuf); -+ if (file_ptr_type == ADIO_INDIVIDUAL) -+ fd->fp_ind = off; -+ } -+ fd->fp_sys_posn = -1; /* set it to null. */ -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ MPIR_Status_set_bytes(status, datatype, bufsize); -+ /* This is a temporary way of filling in status. The right way is to -+ keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */ -+#endif -+ -+ if (!buftype_is_contig) -+ ADIOI_Delete_flattened(datatype); -+} -diff -ruN adio/ad_lustre_orig/Makefile.in adio/ad_lustre/Makefile.in ---- adio/ad_lustre_orig/Makefile.in 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/Makefile.in 2008-10-17 17:03:06.000000000 +0800 -@@ -16,7 +16,9 @@ - @VPATH@ - - AD_LUSTRE_OBJECTS = ad_lustre.o ad_lustre_open.o \ -- ad_lustre_rwcontig.o ad_lustre_hints.o -+ ad_lustre_rwcontig.o ad_lustre_wrcoll.o ad_lustre_wrstr.o \ -+ ad_lustre_hints.o ad_lustre_aggregate.o -+ - - default: $(LIBNAME) - @if [ "@ENABLE_SHLIB@" != "none" ] ; then \ -diff -ruN adio/ad_lustre_orig/README adio/ad_lustre/README ---- adio/ad_lustre_orig/README 2008-09-17 14:36:56.000000000 +0800 -+++ adio/ad_lustre/README 2009-04-24 09:46:20.000000000 +0800 -@@ -5,6 +5,21 @@ - o To post the code for ParColl (Partitioned collective IO) - - ----------------------------------------------------- -+V05: -+----------------------------------------------------- -+Improved data redistribution -+ o Improve I/O pattern identification. Besides checking interleaving, -+ if request I/O size is small, collective I/O will be performed. -+ The hint bigsize can be used to define the req size value. -+ o Provide hint CO for load balancing to control the number of -+ IO clients for each OST -+ o Produce stripe-contiguous I/O pattern that Lustre prefers -+ o Control read-modify-write in data sieving in collective IO -+ by hint ds_in_coll. -+ o Reduce extent lock conflicts by make each OST accessed by one or -+ more constant clients. -+ -+----------------------------------------------------- - V04: - ----------------------------------------------------- - o Direct IO and Lockless IO support ---- adio/common/ad_write_coll_orig.c 2009-02-27 22:06:46.000000000 +0800 -+++ adio/common/ad_write_coll.c 2008-10-15 11:25:38.000000000 +0800 -@@ -42,7 +42,7 @@ - int *send_buf_idx, int *curr_to_proc, - int *done_to_proc, int iter, - MPI_Aint buftype_extent); --static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, -+void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, - ADIO_Offset *srt_off, int *srt_len, int *start_pos, - int nprocs, int nprocs_recv, int total_elements); - -@@ -921,7 +921,7 @@ - - - --static void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, -+void ADIOI_Heap_merge(ADIOI_Access *others_req, int *count, - ADIO_Offset *srt_off, int *srt_len, int *start_pos, - int nprocs, int nprocs_recv, int total_elements) - { diff --git a/lustre/contrib/mpich-1.2.6-lustre.patch b/lustre/contrib/mpich-1.2.6-lustre.patch deleted file mode 100644 index d32fab9..0000000 --- a/lustre/contrib/mpich-1.2.6-lustre.patch +++ /dev/null @@ -1,1829 +0,0 @@ -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.c 2005-12-06 11:54:37.883130927 -0500 -@@ -0,0 +1,37 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 2001 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+/* adioi.h has the ADIOI_Fns_struct define */ -+#include "adioi.h" -+ -+struct ADIOI_Fns_struct ADIO_LUSTRE_operations = { -+ ADIOI_LUSTRE_Open, /* Open */ -+ ADIOI_LUSTRE_ReadContig, /* ReadContig */ -+ ADIOI_LUSTRE_WriteContig, /* WriteContig */ -+ ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */ -+ ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */ -+ ADIOI_GEN_SeekIndividual, /* SeekIndividual */ -+ ADIOI_LUSTRE_Fcntl, /* Fcntl */ -+ ADIOI_LUSTRE_SetInfo, /* SetInfo */ -+ ADIOI_GEN_ReadStrided, /* ReadStrided */ -+ ADIOI_GEN_WriteStrided, /* WriteStrided */ -+ ADIOI_LUSTRE_Close, /* Close */ -+ ADIOI_LUSTRE_IreadContig, /* IreadContig */ -+ ADIOI_LUSTRE_IwriteContig, /* IwriteContig */ -+ ADIOI_LUSTRE_ReadDone, /* ReadDone */ -+ ADIOI_LUSTRE_WriteDone, /* WriteDone */ -+ ADIOI_LUSTRE_ReadComplete, /* ReadComplete */ -+ ADIOI_LUSTRE_WriteComplete, /* WriteComplete */ -+ ADIOI_LUSTRE_IreadStrided, /* IreadStrided */ -+ ADIOI_LUSTRE_IwriteStrided, /* IwriteStrided */ -+ ADIOI_GEN_Flush, /* Flush */ -+ ADIOI_LUSTRE_Resize, /* Resize */ -+ ADIOI_GEN_Delete, /* Delete */ -+}; -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_close.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_close.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_close.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_close.c 2005-12-06 11:54:37.895129327 -0500 -@@ -0,0 +1,32 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_close.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code) -+{ -+ int err; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_CLOSE"; -+#endif -+ -+ err = close(fd->fd_sys); -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_done.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_done.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_done.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_done.c 2005-12-06 11:54:37.898128927 -0500 -@@ -0,0 +1,188 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_done.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+int ADIOI_LUSTRE_ReadDone(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+#ifndef NO_AIO -+ int done=0; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READDONE"; -+#endif -+#ifdef AIO_SUN -+ aio_result_t *result=0, *tmp; -+#else -+ int err; -+#endif -+#ifdef AIO_HANDLE_IN_AIOCB -+ struct aiocb *tmp1; -+#endif -+#endif -+ -+ if (*request == ADIO_REQUEST_NULL) { -+ *error_code = MPI_SUCCESS; -+ return 1; -+ } -+ -+#ifdef NO_AIO -+/* HP, FreeBSD, Linux */ -+#ifdef HAVE_STATUS_SET_BYTES -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ (*request)->fd->async_count--; -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ *error_code = MPI_SUCCESS; -+ return 1; -+#endif -+ -+#ifdef AIO_SUN -+ if ((*request)->queued) { -+ tmp = (aio_result_t *) (*request)->handle; -+ if (tmp->aio_return == AIO_INPROGRESS) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else if (tmp->aio_return != -1) { -+ result = (aio_result_t *) aiowait(0); /* dequeue any one request */ -+ done = 1; -+ (*request)->nbytes = tmp->aio_return; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(tmp->aio_errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(tmp->aio_errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ } /* if ((*request)->queued) ... */ -+ else { -+ /* ADIOI_Complete_Async completed this request, but request object -+ was not freed. */ -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifdef AIO_HANDLE_IN_AIOCB -+/* IBM */ -+ if ((*request)->queued) { -+ tmp1 = (struct aiocb *) (*request)->handle; -+ errno = aio_error(tmp1->aio_handle); -+ if (errno == EINPROG) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+ err = aio_return(tmp1->aio_handle); -+ (*request)->nbytes = err; -+ errno = aio_error(tmp1->aio_handle); -+ -+ done = 1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } -+ } /* if ((*request)->queued) */ -+ else { -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ if ((*request)->queued) { -+ errno = aio_error((const struct aiocb *) (*request)->handle); -+ if (errno == EINPROGRESS) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+ err = aio_return((struct aiocb *) (*request)->handle); -+ (*request)->nbytes = err; -+ errno = aio_error((struct aiocb *) (*request)->handle); -+ -+ done = 1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } -+ } /* if ((*request)->queued) */ -+ else { -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifndef NO_AIO -+ if (done) { -+ /* if request is still queued in the system, it is also there -+ on ADIOI_Async_list. Delete it from there. */ -+ if ((*request)->queued) ADIOI_Del_req_from_list(request); -+ -+ (*request)->fd->async_count--; -+ if ((*request)->handle) ADIOI_Free((*request)->handle); -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ } -+ return done; -+#endif -+ -+} -+ -+ -+int ADIOI_LUSTRE_WriteDone(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+ return ADIOI_LUSTRE_ReadDone(request, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_fcntl.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_fcntl.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_fcntl.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_fcntl.c 2005-12-06 11:54:37.901128527 -0500 -@@ -0,0 +1,126 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_fcntl.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+#include "adio_extern.h" -+/* #ifdef MPISGI -+#include "mpisgi2.h" -+#endif */ -+ -+void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code) -+{ -+ int i, ntimes; -+ ADIO_Offset curr_fsize, alloc_size, size, len, done; -+ ADIO_Status status; -+ char *buf; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_FCNTL"; -+#endif -+ -+ switch(flag) { -+ case ADIO_FCNTL_GET_FSIZE: -+ fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END); -+ if (fd->fp_sys_posn != -1) -+ lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET); -+ if (fcntl_struct->fsize == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ break; -+ -+ case ADIO_FCNTL_SET_DISKSPACE: -+ /* will be called by one process only */ -+ /* On file systems with no preallocation function, I have to -+ explicitly write -+ to allocate space. Since there could be holes in the file, -+ I need to read up to the current file size, write it back, -+ and then write beyond that depending on how much -+ preallocation is needed. -+ read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */ -+ -+ curr_fsize = lseek(fd->fd_sys, 0, SEEK_END); -+ alloc_size = fcntl_struct->diskspace; -+ -+ size = ADIOI_MIN(curr_fsize, alloc_size); -+ -+ ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ; -+ buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ); -+ done = 0; -+ -+ for (i=0; i curr_fsize) { -+ memset(buf, 0, ADIOI_PREALLOC_BUFSZ); -+ size = alloc_size - curr_fsize; -+ ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ; -+ for (i=0; ifp_sys_posn != -1) -+ lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET); -+ *error_code = MPI_SUCCESS; -+ break; -+ -+ case ADIO_FCNTL_SET_IOMODE: -+ /* for implementing PFS I/O modes. will not occur in MPI-IO -+ implementation.*/ -+ if (fd->iomode != fcntl_struct->iomode) { -+ fd->iomode = fcntl_struct->iomode; -+ MPI_Barrier(MPI_COMM_WORLD); -+ } -+ *error_code = MPI_SUCCESS; -+ break; -+ -+ case ADIO_FCNTL_SET_ATOMICITY: -+ fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1; -+ *error_code = MPI_SUCCESS; -+ break; -+ -+ default: -+ FPRINTF(stderr, "Unknown flag passed to ADIOI_LUSTRE_Fcntl\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_flush.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_flush.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_flush.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_flush.c 2005-12-06 11:54:37.903128261 -0500 -@@ -0,0 +1,14 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_flush.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Flush(ADIO_File fd, int *error_code) -+{ -+ ADIOI_GEN_Flush(fd, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.h mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.h ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.h 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre.h 2005-12-06 11:54:37.891129861 -0500 -@@ -0,0 +1,36 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre.h,v 1.2 2005/07/07 14:38:17 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#ifndef AD_UNIX_INCLUDE -+#define AD_UNIX_INCLUDE -+ -+/* temp*/ -+#define HAVE_ASM_TYPES_H 1 -+ -+#include -+#include -+#include -+#include -+#include "lustre/lustre_user.h" -+#include "adio.h" -+ -+#ifndef NO_AIO -+#ifdef AIO_SUN -+#include -+#else -+#include -+#ifdef NEEDS_ADIOCB_T -+typedef struct adiocb adiocb_t; -+#endif -+#endif -+#endif -+ -+int ADIOI_LUSTRE_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset, -+ int wr, void *handle); -+ -+#endif -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_hints.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_hints.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_hints.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_hints.c 2005-12-06 11:54:37.904128127 -0500 -@@ -0,0 +1,130 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_hints.c,v 1.2 2005/07/07 14:38:17 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code) -+{ -+ char *value, *value_in_fd; -+ int flag, tmp_val, str_factor=-1, str_unit=0, start_iodev=-1; -+ struct lov_user_md lum = { 0 }; -+ int err, myrank, fd_sys, perm, amode, old_mask; -+ -+ if ( (fd->info) == MPI_INFO_NULL) { -+ /* This must be part of the open call. can set striping parameters -+ if necessary. */ -+ MPI_Info_create(&(fd->info)); -+ -+ /* has user specified striping or server buffering parameters -+ and do they have the same value on all processes? */ -+ if (users_info != MPI_INFO_NULL) { -+ value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char)); -+ -+ MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ str_factor=atoi(value); -+ tmp_val = str_factor; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != str_factor) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"striping_factor\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ str_unit=atoi(value); -+ tmp_val = str_unit; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != str_unit) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"striping_unit\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ MPI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ start_iodev=atoi(value); -+ tmp_val = start_iodev; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != start_iodev) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"start_iodevice\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ /* if user has specified striping info, process 0 tries to set it */ -+ if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) { -+ MPI_Comm_rank(fd->comm, &myrank); -+ if (!myrank) { -+ if (fd->perm == ADIO_PERM_NULL) { -+ old_mask = umask(022); -+ umask(old_mask); -+ perm = old_mask ^ 0666; -+ } -+ else perm = fd->perm; -+ -+ amode = 0; -+ if (fd->access_mode & ADIO_CREATE) -+ amode = amode | O_CREAT; -+ if (fd->access_mode & ADIO_RDWR || -+ (fd->access_mode & ADIO_RDONLY && -+ fd->access_mode & ADIO_WRONLY)) -+ amode = amode | O_RDWR; -+ else if (fd->access_mode & ADIO_WRONLY) -+ amode = amode | O_WRONLY; -+ else if (fd->access_mode & ADIO_RDONLY) -+ amode = amode | O_RDONLY; -+ if (fd->access_mode & ADIO_EXCL) -+ amode = amode | O_EXCL; -+ -+ /* we need to create file so ensure this is set */ -+ amode = amode | O_LOV_DELAY_CREATE | O_CREAT; -+ -+ fd_sys = open(fd->filename, amode, perm); -+ if (fd_sys == -1) { -+ if (errno != EEXIST) -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: Failure to open file %s %d %d\n",strerror(errno), amode, perm); -+ } else { -+ lum.lmm_magic = LOV_USER_MAGIC; -+ lum.lmm_pattern = 0; -+ lum.lmm_stripe_size = str_unit; -+ lum.lmm_stripe_count = str_factor; -+ lum.lmm_stripe_offset = start_iodev; -+ -+ err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum); -+ if (err == -1 && errno != EEXIST) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: Failure to set stripe info %s \n",strerror(errno)); -+ } -+ -+ close(fd_sys); -+ } -+ -+ } -+ MPI_Barrier(fd->comm); -+ } -+ -+ ADIOI_Free(value); -+ } -+ -+ /* set the values for collective I/O and data sieving parameters */ -+ ADIOI_GEN_SetInfo(fd, users_info, error_code); -+ } -+ -+ else { -+ /* The file has been opened previously and fd->fd_sys is a valid -+ file descriptor. cannot set striping parameters now. */ -+ -+ /* set the values for collective I/O and data sieving parameters */ -+ ADIOI_GEN_SetInfo(fd, users_info, error_code); -+ -+ } -+ -+ *error_code = MPI_SUCCESS; -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iread.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iread.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iread.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iread.c 2005-12-06 11:54:37.904128127 -0500 -@@ -0,0 +1,106 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_iread.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_IreadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int *error_code) -+{ -+ int len, typesize; -+#ifdef NO_AIO -+ ADIO_Status status; -+#else -+ int err=-1; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_IREADCONTIG"; -+#endif -+#endif -+ -+ (*request) = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_READ; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ -+ MPI_Type_size(datatype, &typesize); -+ len = count * typesize; -+ -+#ifdef NO_AIO -+ /* HP, FreeBSD, Linux */ -+ /* no support for nonblocking I/O. Use blocking I/O. */ -+ -+ ADIOI_LUSTRE_ReadContig(fd, buf, len, MPI_BYTE, file_ptr_type, offset, -+ &status, error_code); -+ (*request)->queued = 0; -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Get_elements(&status, MPI_BYTE, &len); -+ (*request)->nbytes = len; -+ } -+#endif -+ -+#else -+ if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind; -+ err = ADIOI_LUSTRE_aio(fd, buf, len, offset, 0, &((*request)->handle)); -+ if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; -+ -+ (*request)->queued = 1; -+ ADIOI_Add_req_to_list(request); -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+#endif /* NO_AIO */ -+ -+ fd->fp_sys_posn = -1; /* set it to null. */ -+ fd->async_count++; -+} -+ -+ -+ -+void ADIOI_LUSTRE_IreadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code) -+{ -+ ADIO_Status status; -+#ifdef HAVE_STATUS_SET_BYTES -+ int typesize; -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_READ; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ (*request)->queued = 0; -+ (*request)->handle = 0; -+ -+/* call the blocking version. It is faster because it does data sieving. */ -+ ADIOI_LUSTRE_ReadStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, &status, error_code); -+ -+ fd->async_count++; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Type_size(datatype, &typesize); -+ (*request)->nbytes = count * typesize; -+ } -+#endif -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iwrite.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iwrite.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iwrite.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_iwrite.c 2005-12-06 11:54:37.906127861 -0500 -@@ -0,0 +1,268 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_iwrite.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_IwriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int *error_code) -+{ -+ int len, typesize; -+#ifdef NO_AIO -+ ADIO_Status status; -+#else -+ int err=-1; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_IWRITECONTIG"; -+#endif -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_WRITE; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ -+ MPI_Type_size(datatype, &typesize); -+ len = count * typesize; -+ -+#ifdef NO_AIO -+ /* HP, FreeBSD, Linux */ -+ /* no support for nonblocking I/O. Use blocking I/O. */ -+ -+ ADIOI_LUSTRE_WriteContig(fd, buf, len, MPI_BYTE, file_ptr_type, offset, -+ &status, error_code); -+ (*request)->queued = 0; -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Get_elements(&status, MPI_BYTE, &len); -+ (*request)->nbytes = len; -+ } -+#endif -+ -+#else -+ if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind; -+ err = ADIOI_LUSTRE_aio(fd, buf, len, offset, 1, &((*request)->handle)); -+ if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; -+ -+ (*request)->queued = 1; -+ ADIOI_Add_req_to_list(request); -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+#endif /* NO_AIO */ -+ -+ fd->fp_sys_posn = -1; /* set it to null. */ -+ fd->async_count++; -+} -+ -+ -+ -+ -+void ADIOI_LUSTRE_IwriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code) -+{ -+ ADIO_Status status; -+#ifdef HAVE_STATUS_SET_BYTES -+ int typesize; -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_WRITE; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ (*request)->queued = 0; -+ (*request)->handle = 0; -+ -+/* call the blocking version. It is faster because it does data sieving. */ -+ ADIOI_LUSTRE_WriteStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, &status, error_code); -+ -+ fd->async_count++; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Type_size(datatype, &typesize); -+ (*request)->nbytes = count * typesize; -+ } -+#endif -+} -+ -+ -+/* This function is for implementation convenience. It is not user-visible. -+ It takes care of the differences in the interface for nonblocking I/O -+ on various Unix machines! If wr==1 write, wr==0 read. */ -+ -+int ADIOI_LUSTRE_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset, -+ int wr, void *handle) -+{ -+ int err=-1, fd_sys; -+ -+#ifndef NO_AIO -+ int error_code; -+#ifdef AIO_SUN -+ aio_result_t *result; -+#else -+ struct aiocb *aiocbp; -+#endif -+#endif -+ -+ fd_sys = fd->fd_sys; -+ -+#ifdef AIO_SUN -+ result = (aio_result_t *) ADIOI_Malloc(sizeof(aio_result_t)); -+ result->aio_return = AIO_INPROGRESS; -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* the man pages say EPROCLIM, but in reality errno is set to EAGAIN! */ -+ -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again.*/ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((aio_result_t **) handle) = result; -+#endif -+ -+#ifdef NO_FD_IN_AIOCB -+/* IBM */ -+ aiocbp = (struct aiocb *) ADIOI_Malloc(sizeof(struct aiocb)); -+ aiocbp->aio_whence = SEEK_SET; -+ aiocbp->aio_offset = offset; -+ aiocbp->aio_buf = buf; -+ aiocbp->aio_nbytes = len; -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again. */ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((struct aiocb **) handle) = aiocbp; -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ -+ aiocbp = (struct aiocb *) ADIOI_Calloc(sizeof(struct aiocb), 1); -+ aiocbp->aio_fildes = fd_sys; -+ aiocbp->aio_offset = offset; -+ aiocbp->aio_buf = buf; -+ aiocbp->aio_nbytes = len; -+ -+#ifdef AIO_PRIORITY_DEFAULT -+/* DEC */ -+ aiocbp->aio_reqprio = AIO_PRIO_DFL; /* not needed in DEC Unix 4.0 */ -+ aiocbp->aio_sigevent.sigev_signo = 0; -+#else -+ aiocbp->aio_reqprio = 0; -+#endif -+ -+#ifdef AIO_SIGNOTIFY_NONE -+/* SGI IRIX 6 */ -+ aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE; -+#else -+ aiocbp->aio_sigevent.sigev_signo = 0; -+#endif -+ -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again. */ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((struct aiocb **) handle) = aiocbp; -+#endif -+ -+ return err; -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_open.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_open.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_open.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_open.c 2005-12-06 11:54:37.906127861 -0500 -@@ -0,0 +1,100 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_open.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code) -+{ -+ int perm, old_mask, amode; -+ struct lov_user_md lum = { 0 }; -+ char *value; -+ -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_OPEN"; -+#endif -+ -+ if (fd->perm == ADIO_PERM_NULL) { -+ old_mask = umask(022); -+ umask(old_mask); -+ perm = old_mask ^ 0666; -+ } -+ else perm = fd->perm; -+ -+ amode = 0; -+ if (fd->access_mode & ADIO_CREATE) -+ amode = amode | O_CREAT; -+ if (fd->access_mode & ADIO_RDONLY) -+ amode = amode | O_RDONLY; -+ if (fd->access_mode & ADIO_WRONLY) -+ amode = amode | O_WRONLY; -+ if (fd->access_mode & ADIO_RDWR) -+ amode = amode | O_RDWR; -+ if (fd->access_mode & ADIO_EXCL) -+ amode = amode | O_EXCL; -+ -+ fd->fd_sys = open(fd->filename, amode, perm); -+ -+ if (fd->fd_sys != -1) { -+ int err; -+ -+ value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char)); -+ -+ /* get file striping information and set it in info */ -+ lum.lmm_magic = LOV_USER_MAGIC; -+ err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *) &lum); -+ -+ if (!err) { -+ sprintf(value, "%d", lum.lmm_stripe_size); -+ MPI_Info_set(fd->info, "striping_unit", value); -+ -+ sprintf(value, "%d", lum.lmm_stripe_count); -+ MPI_Info_set(fd->info, "striping_factor", value); -+ -+ sprintf(value, "%d", lum.lmm_stripe_offset); -+ MPI_Info_set(fd->info, "start_iodevice", value); -+ } -+ ADIOI_Free(value); -+ -+ if (fd->access_mode & ADIO_APPEND) -+ fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END); -+ } -+ -+ -+ if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) -+ fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END); -+ -+ if (fd->fd_sys == -1) { -+#ifdef MPICH2 -+ if (errno == ENAMETOOLONG) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_BAD_FILE, "**filenamelong", "**filenamelong %s %d", fd->filename, strlen(fd->filename) ); -+ else if (errno == ENOENT) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_NO_SUCH_FILE, "**filenoexist", "**filenoexist %s", fd->filename ); -+ else if (errno == ENOTDIR || errno == ELOOP) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_BAD_FILE, "**filenamedir", "**filenamedir %s", fd->filename ); -+ else if (errno == EACCES) { -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_ACCESS, "**fileaccess", "**fileaccess %s", -+ fd->filename ); -+ } -+ else if (errno == EROFS) { -+ /* Read only file or file system and write access requested */ -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_READ_ONLY, "**ioneedrd", 0 ); -+ } -+ else { -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ } -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(ADIO_FILE_NULL, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_rdcoll.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_rdcoll.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_rdcoll.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_rdcoll.c 2005-12-06 11:54:37.907127727 -0500 -@@ -0,0 +1,18 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_rdcoll.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_ReadStridedColl(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_read.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_read.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_read.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_read.c 2005-12-06 11:54:37.907127727 -0500 -@@ -0,0 +1,67 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_read.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int *error_code) -+{ -+ int err=-1, datatype_size, len; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READCONTIG"; -+#endif -+ -+ MPI_Type_size(datatype, &datatype_size); -+ len = datatype_size * count; -+ -+ if (file_ptr_type == ADIO_EXPLICIT_OFFSET) { -+ if (fd->fp_sys_posn != offset) -+ lseek(fd->fd_sys, offset, SEEK_SET); -+ err = read(fd->fd_sys, buf, len); -+ fd->fp_sys_posn = offset + len; -+ /* individual file pointer not updated */ -+ } -+ else { /* read from curr. location of ind. file pointer */ -+ if (fd->fp_sys_posn != fd->fp_ind) -+ lseek(fd->fd_sys, fd->fp_ind, SEEK_SET); -+ err = read(fd->fd_sys, buf, len); -+ fd->fp_ind += err; -+ fd->fp_sys_posn = fd->fp_ind; -+ } -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (err != -1) MPIR_Status_set_bytes(status, datatype, err); -+#endif -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -+ -+ -+ -+ -+void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_ReadStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_resize.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_resize.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_resize.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_resize.c 2005-12-06 11:54:37.909127460 -0500 -@@ -0,0 +1,32 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_resize.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Resize(ADIO_File fd, ADIO_Offset size, int *error_code) -+{ -+ int err; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_RESIZE"; -+#endif -+ -+ err = ftruncate(fd->fd_sys, size); -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_seek.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_seek.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_seek.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_seek.c 2005-12-06 11:54:37.911127194 -0500 -@@ -0,0 +1,15 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_seek.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+ADIO_Offset ADIOI_LUSTRE_SeekIndividual(ADIO_File fd, ADIO_Offset offset, -+ int whence, int *error_code) -+{ -+ return ADIOI_GEN_SeekIndividual(fd, offset, whence, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wait.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wait.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wait.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wait.c 2005-12-06 11:54:37.914126794 -0500 -@@ -0,0 +1,188 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_wait.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadComplete(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+#ifndef NO_AIO -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READCOMPLETE"; -+#endif -+#ifdef AIO_SUN -+ aio_result_t *result=0, *tmp; -+#else -+ int err; -+#endif -+#ifdef AIO_HANDLE_IN_AIOCB -+ struct aiocb *tmp1; -+#endif -+#endif -+ -+ if (*request == ADIO_REQUEST_NULL) { -+ *error_code = MPI_SUCCESS; -+ return; -+ } -+ -+#ifdef AIO_SUN -+ if ((*request)->queued) { /* dequeue it */ -+ tmp = (aio_result_t *) (*request)->handle; -+ while (tmp->aio_return == AIO_INPROGRESS) usleep(1000); -+ /* sleep for 1 ms., until done. Is 1 ms. a good number? */ -+ /* when done, dequeue any one request */ -+ result = (aio_result_t *) aiowait(0); -+ -+ (*request)->nbytes = tmp->aio_return; -+ -+ if (tmp->aio_return == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(tmp->aio_errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(tmp->aio_errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ -+/* aiowait only dequeues a request. The completion of a request can be -+ checked by just checking the aio_return flag in the handle passed -+ to the original aioread()/aiowrite(). Therefore, I need to ensure -+ that aiowait() is called exactly once for each previous -+ aioread()/aiowrite(). This is also taken care of in ADIOI_xxxDone */ -+ } -+ else *error_code = MPI_SUCCESS; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifdef AIO_HANDLE_IN_AIOCB -+/* IBM */ -+ if ((*request)->queued) { -+ do { -+ err = aio_suspend(1, (struct aiocb **) &((*request)->handle)); -+ } while ((err == -1) && (errno == EINTR)); -+ -+ tmp1 = (struct aiocb *) (*request)->handle; -+ if (err != -1) { -+ err = aio_return(tmp1->aio_handle); -+ (*request)->nbytes = err; -+ errno = aio_error(tmp1->aio_handle); -+ } -+ else (*request)->nbytes = -1; -+ -+/* on DEC, it is required to call aio_return to dequeue the request. -+ IBM man pages don't indicate what function to use for dequeue. -+ I'm assuming it is aio_return! POSIX says aio_return may be called -+ only once on a given handle. */ -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } /* if ((*request)->queued) */ -+ else *error_code = MPI_SUCCESS; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ if ((*request)->queued) { -+ do { -+ err = aio_suspend((const aiocb_t **) &((*request)->handle), 1, 0); -+ } while ((err == -1) && (errno == EINTR)); -+ -+ if (err != -1) { -+ err = aio_return((struct aiocb *) (*request)->handle); -+ (*request)->nbytes = err; -+ errno = aio_error((struct aiocb *) (*request)->handle); -+ } -+ else (*request)->nbytes = -1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } /* if ((*request)->queued) */ -+ else *error_code = MPI_SUCCESS; -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+#endif -+ -+#ifndef NO_AIO -+ if ((*request)->queued != -1) { -+ -+ /* queued = -1 is an internal hack used when the request must -+ be completed, but the request object should not be -+ freed. This is used in ADIOI_Complete_async, because the user -+ will call MPI_Wait later, which would require status to -+ be filled. Ugly but works. queued = -1 should be used only -+ in ADIOI_Complete_async. -+ This should not affect the user in any way. */ -+ -+ /* if request is still queued in the system, it is also there -+ on ADIOI_Async_list. Delete it from there. */ -+ if ((*request)->queued) ADIOI_Del_req_from_list(request); -+ -+ (*request)->fd->async_count--; -+ if ((*request)->handle) ADIOI_Free((*request)->handle); -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ } -+ -+#else -+/* HP, FreeBSD, Linux */ -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ (*request)->fd->async_count--; -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ *error_code = MPI_SUCCESS; -+#endif -+} -+ -+ -+void ADIOI_LUSTRE_WriteComplete(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+ ADIOI_LUSTRE_ReadComplete(request, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wrcoll.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wrcoll.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wrcoll.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_wrcoll.c 2005-12-06 11:54:37.914126794 -0500 -@@ -0,0 +1,18 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_wrcoll.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_WriteStridedColl(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_write.c mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_write.c ---- mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_write.c 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/ad_lustre_write.c 2005-12-06 11:54:37.914126794 -0500 -@@ -0,0 +1,66 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_write.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int *error_code) -+{ -+ int err=-1, datatype_size, len; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_WRITECONTIG"; -+#endif -+ -+ MPI_Type_size(datatype, &datatype_size); -+ len = datatype_size * count; -+ -+ if (file_ptr_type == ADIO_EXPLICIT_OFFSET) { -+ if (fd->fp_sys_posn != offset) -+ lseek(fd->fd_sys, offset, SEEK_SET); -+ err = write(fd->fd_sys, buf, len); -+ fd->fp_sys_posn = offset + err; -+ /* individual file pointer not updated */ -+ } -+ else { /* write from curr. location of ind. file pointer */ -+ if (fd->fp_sys_posn != fd->fp_ind) -+ lseek(fd->fd_sys, fd->fp_ind, SEEK_SET); -+ err = write(fd->fd_sys, buf, len); -+ fd->fp_ind += err; -+ fd->fp_sys_posn = fd->fp_ind; -+ } -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, err); -+#endif -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -+ -+ -+ -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_WriteStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -r -u --new-file mpich-1.2.6/romio/adio/ad_lustre/Makefile.in mpich-1.2.6/romio/adio/ad_lustre/Makefile.in ---- mpich-1.2.6/romio/adio/ad_lustre/Makefile.in 1969-12-31 19:00:00.000000000 -0500 -+++ mpich-1.2.6/romio/adio/ad_lustre/Makefile.in 2005-12-06 11:54:37.883130927 -0500 -@@ -0,0 +1,47 @@ -+CC = @CC@ -+AR = @AR@ -+LIBNAME = @LIBNAME@ -+srcdir = @srcdir@ -+CC_SHL = @CC_SHL@ -+SHLIBNAME = @SHLIBNAME@ -+ -+INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I${srcdir}/../include -I../include -+CFLAGS = @CFLAGS@ $(INCLUDE_DIR) -+ -+C_COMPILE_SHL = $(CC_SHL) @CFLAGS@ $(INCLUDE_DIR) -+ -+@VPATH@ -+ -+AD_LUSTRE_OBJECTS = ad_lustre_close.o ad_lustre_read.o \ -+ ad_lustre_open.o ad_lustre_write.o ad_lustre_done.o \ -+ ad_lustre_fcntl.o ad_lustre_iread.o ad_lustre_iwrite.o ad_lustre_wait.o \ -+ ad_lustre_resize.o ad_lustre_hints.o \ -+ ad_lustre.o -+ -+ -+default: $(LIBNAME) -+ @if [ "@ENABLE_SHLIB@" != "none" ] ; then \ -+ $(MAKE) $(SHLIBNAME).la ;\ -+ fi -+ -+.SUFFIXES: $(SUFFIXES) .p .lo -+ -+.c.o: -+ $(CC) $(CFLAGS) -c $< -+.c.lo: -+ $(C_COMPILE_SHL) -c $< -+ @mv -f $*.o $*.lo -+ -+$(LIBNAME): $(AD_LUSTRE_OBJECTS) -+ $(AR) $(LIBNAME) $(AD_LUSTRE_OBJECTS) -+ -+AD_LUSTRE_LOOBJECTS=$(AD_LUSTRE_OBJECTS:.o=.lo) -+$(SHLIBNAME).la: $(AD_LUSTRE_LOOBJECTS) -+ $(AR) $(SHLIBNAME).la $(AD_LUSTRE_LOOBJECTS) -+ -+coverage: -+ -@for file in ${AD_LUSTRE_OBJECTS:.o=.c} ; do \ -+ gcov -b -f $$file ; done -+ -+clean: -+ @rm -f *.o *.lo ---- mpich-1.2.6/romio/Makefile.in 2004-01-27 18:27:35.000000000 -0500 -+++ mpich-1.2.6/romio/Makefile.in 2005-12-06 11:54:38.000000000 -0500 -@@ -14,7 +14,7 @@ DIRS = mpi-io adio/common - MPIO_DIRS = mpi-io - EXTRA_SRC_DIRS = @EXTRA_SRC_DIRS@ - FILE_SYS_DIRS = @FILE_SYS_DIRS@ --ALL_DIRS = mpi-io mpi-io/fortran mpi2-other/info mpi2-other/info/fortran mpi2-other/array mpi2-other/array/fortran adio/common adio/ad_pfs adio/ad_piofs adio/ad_nfs adio/ad_ufs adio/ad_xfs adio/ad_hfs adio/ad_sfs adio/ad_testfs adio/ad_pvfs adio/ad_pvfs2 test -+ALL_DIRS = mpi-io mpi-io/fortran mpi2-other/info mpi2-other/info/fortran mpi2-other/array mpi2-other/array/fortran adio/common adio/ad_pfs adio/ad_piofs adio/ad_nfs adio/ad_ufs adio/ad_xfs adio/ad_hfs adio/ad_sfs adio/ad_testfs adio/ad_pvfs adio/ad_pvfs2 adio/ad_lustre test - SHELL = /bin/sh - - @VPATH@ ---- mpich-1.2.6/romio/configure.in 2004-08-02 09:37:31.000000000 -0400 -+++ mpich-1.2.6/romio/configure.in 2005-12-06 11:54:38.000000000 -0500 -@@ -90,7 +90,7 @@ MPIO_REQ_REAL_POBJECTS="_iotest.o _iowai - # - have_aio=no - # --known_filesystems="nfs ufs pfs piofs pvfs pvfs2 testfs xfs hfs sfs" -+known_filesystems="nfs ufs pfs piofs pvfs pvfs2 testfs xfs hfs sfs lustre" - known_mpi_impls="mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi" - # - # Defaults -@@ -1270,6 +1270,9 @@ fi - if test -n "$file_system_testfs"; then - AC_DEFINE(ROMIO_TESTFS,1,[Define for TESTFS]) - fi -+if test -n "$file_system_lustre"; then -+ AC_DEFINE(ROMIO_LUSTRE,1,[Define for LUSTRE]) -+fi - if test -n "$file_system_piofs"; then - AC_DEFINE(PIOFS,1,[Define for PIOFS]) - USER_CFLAGS="$USER_CFLAGS -bI:/usr/include/piofs/piofs.exp" -@@ -1634,7 +1637,7 @@ AC_OUTPUT(Makefile localdefs mpi-io/Make - adio/ad_nfs/Makefile adio/ad_ufs/Makefile \ - adio/ad_xfs/Makefile adio/ad_hfs/Makefile \ - adio/ad_sfs/Makefile adio/ad_pfs/Makefile \ -- adio/ad_testfs/Makefile adio/ad_pvfs/Makefile \ -+ adio/ad_testfs/Makefile adio/ad_lustre/Makefile adio/ad_pvfs/Makefile \ - adio/ad_pvfs2/Makefile adio/ad_piofs/Makefile \ - mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile \ - mpi2-other/array/fortran/Makefile test/fmisc.f \ ---- mpich-1.2.6/romio/configure 2004-08-04 12:08:28.000000000 -0400 -+++ mpich-1.2.6/romio/configure 2005-12-06 11:54:38.000000000 -0500 -@@ -623,7 +623,7 @@ MPIO_REQ_REAL_POBJECTS="_iotest.o _iowai - # - have_aio=no - # --known_filesystems="nfs ufs pfs piofs pvfs pvfs2 testfs xfs hfs sfs" -+known_filesystems="nfs ufs pfs piofs pvfs pvfs2 testfs lustre xfs hfs sfs" - known_mpi_impls="mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi" - # - # Defaults -@@ -4022,6 +4022,13 @@ if test -n "$file_system_testfs"; then - EOF - - fi -+if test -n "$file_system_lustre"; then -+ cat >> confdefs.h <<\EOF -+#define LUSTRE 1 -+EOF -+ -+fi -+ - if test -n "$file_system_piofs"; then - cat >> confdefs.h <<\EOF - #define PIOFS 1 -@@ -4746,7 +4753,7 @@ trap 'rm -fr `echo "Makefile localdefs m - adio/ad_xfs/Makefile adio/ad_hfs/Makefile \ - adio/ad_sfs/Makefile adio/ad_pfs/Makefile \ - adio/ad_testfs/Makefile adio/ad_pvfs/Makefile \ -- adio/ad_pvfs2/Makefile adio/ad_piofs/Makefile \ -+ adio/ad_pvfs2/Makefile adio/ad_piofs/Makefile adio/ad_lustre/Makefile\ - mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile \ - mpi2-other/array/fortran/Makefile test/fmisc.f \ - test/fcoll_test.f test/pfcoll_test.f test/fperf.f adio/include/romioconf.h" | sed "s/:[^ ]*//g"` conftest*; exit 1' 1 2 15 -@@ -4912,7 +4919,7 @@ CONFIG_FILES=\${CONFIG_FILES-"Makefile l - adio/ad_nfs/Makefile adio/ad_ufs/Makefile \ - adio/ad_xfs/Makefile adio/ad_hfs/Makefile \ - adio/ad_sfs/Makefile adio/ad_pfs/Makefile \ -- adio/ad_testfs/Makefile adio/ad_pvfs/Makefile \ -+ adio/ad_testfs/Makefile adio/ad_lustre/Makefile adio/ad_pvfs/Makefile \ - adio/ad_pvfs2/Makefile adio/ad_piofs/Makefile \ - mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile \ - mpi2-other/array/fortran/Makefile test/fmisc.f \ ---- mpich-1.2.6/romio/adio/include/romioconf.h.in 2004-08-04 12:08:28.000000000 -0400 -+++ mpich-1.2.6/romio/adio/include/romioconf.h.in 2005-12-06 11:54:38.000000000 -0500 -@@ -192,6 +192,9 @@ - /* Define for TESTFS */ - #undef ROMIO_TESTFS - -+/* Define for LUSTRE */ -+#undef LUSTRE -+ - /* Define for PIOFS */ - #undef PIOFS - ---- mpich-1.2.6/romio/adio/include/mpio_error.h 2002-11-15 11:26:23.000000000 -0500 -+++ mpich-1.2.6/romio/adio/include/mpio_error.h 2005-12-06 11:54:38.000000000 -0500 -@@ -62,6 +62,7 @@ - #define MPIR_ERR_FILETYPE 33 - #define MPIR_ERR_NO_NTFS 35 - #define MPIR_ERR_NO_TESTFS 36 -+#define MPIR_ERR_NO_LUSTRE 37 - - /* MPI_ERR_COMM */ - #ifndef MPIR_ERR_COMM_NULL ---- mpich-1.2.6/romio/adio/include/adioi_fs_proto.h 2003-06-24 18:48:23.000000000 -0400 -+++ mpich-1.2.6/romio/adio/include/adioi_fs_proto.h 2005-12-06 11:54:38.000000000 -0500 -@@ -261,6 +261,68 @@ ADIO_Offset ADIOI_UFS_SeekIndividual(ADI - void ADIOI_UFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code); - #endif - -+#ifdef LUSTRE -+extern struct ADIOI_Fns_struct ADIO_LUSTRE_operations; -+ -+void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_IwriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_IreadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+int ADIOI_LUSTRE_ReadDone(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+int ADIOI_LUSTRE_WriteDone(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadComplete(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteComplete(ADIO_Request *request, ADIO_Status *status, -+ int *error_code); -+void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int -+ *error_code); -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_IreadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_IwriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_Flush(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_Resize(ADIO_File fd, ADIO_Offset size, int *error_code); -+ADIO_Offset ADIOI_LUSTRE_SeekIndividual(ADIO_File fd, ADIO_Offset offset, -+ int whence, int *error_code); -+void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code); -+#endif -+ - #ifdef ROMIO_NTFS - extern struct ADIOI_Fns_struct ADIO_NTFS_operations; - ---- mpich-1.2.6/romio/adio/include/adio.h 2004-06-07 13:59:57.000000000 -0400 -+++ mpich-1.2.6/romio/adio/include/adio.h 2005-12-06 11:54:38.000000000 -0500 -@@ -276,6 +276,7 @@ typedef struct { - #define ADIO_NTFS 158 /* NTFS for Windows NT */ - #define ADIO_TESTFS 159 /* fake file system for testing */ - #define ADIO_PVFS2 160 /* PVFS2: 2nd generation PVFS */ -+#define ADIO_LUSTRE 161 /* Lustre */ - - #define ADIO_SEEK_SET SEEK_SET - #define ADIO_SEEK_CUR SEEK_CUR ---- mpich-1.2.6/romio/adio/common/setfn.c 2003-06-24 18:48:18.000000000 -0400 -+++ mpich-1.2.6/romio/adio/common/setfn.c 2005-12-06 11:54:38.000000000 -0500 -@@ -114,6 +114,16 @@ void ADIOI_SetFunctions(ADIO_File fd) - #endif - break; - -+ case ADIO_LUSTRE: -+#ifdef LUSTRE -+ *(fd->fns) = ADIO_LUSTRE_operations; -+#else -+ FPRINTF(stderr, "ADIOI_SetFunctions: ROMIO has not been configured to use the LUSTRE file system\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+#endif -+ break; -+ -+ - default: - FPRINTF(stderr, "ADIOI_SetFunctions: Unsupported file system type\n"); - MPI_Abort(MPI_COMM_WORLD, 1); ---- mpich-1.2.6/romio/adio/common/ad_fstype.c 2003-09-04 16:24:44.000000000 -0400 -+++ mpich-1.2.6/romio/adio/common/ad_fstype.c 2005-12-06 11:54:38.000000000 -0500 -@@ -204,6 +204,11 @@ static void ADIO_FileSysType_fncall(char - } - } - #elif defined(LINUX) -+#warning use correct include -+# if defined (LUSTRE) -+#define LL_SUPER_MAGIC 0x0BD00BD0 -+# endif -+ - do { - err = statfs(filename, &fsbuf); - } while (err && (errno == ESTALE)); -@@ -218,6 +223,9 @@ static void ADIO_FileSysType_fncall(char - else { - /* FPRINTF(stderr, "%d\n", fsbuf.f_type);*/ - if (fsbuf.f_type == NFS_SUPER_MAGIC) *fstype = ADIO_NFS; -+# if defined (LUSTRE) -+ else if (fsbuf.f_type == LL_SUPER_MAGIC) *fstype = ADIO_LUSTRE; -+#endif - # if defined(ROMIO_PVFS) - else if (fsbuf.f_type == PVFS_SUPER_MAGIC) *fstype = ADIO_PVFS; - # endif -@@ -359,6 +367,11 @@ static void ADIO_FileSysType_prefix(char - { - *fstype = ADIO_TESTFS; - } -+ else if (!strncmp(filename, "lustre:", 7) -+ || !strncmp(filename, "LUSTRE:", 7)) -+ { -+ *fstype = ADIO_LUSTRE; -+ } - else { - #ifdef ROMIO_NTFS - *fstype = ADIO_NTFS; -@@ -644,6 +657,24 @@ void ADIO_ResolveFileType(MPI_Comm comm, - *ops = &ADIO_TESTFS_operations; - #endif - } -+ if (file_system == ADIO_LUSTRE) { -+#ifndef LUSTRE -+# ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**iofstypeunsupported", 0); -+ return; -+# elif defined(PRINT_ERR_MSG) -+ FPRINTF(stderr, "ADIO_ResolveFileType: ROMIO has not been configured to use the LUSTRE file system\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+# else /* MPICH-1 */ -+ myerrcode = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ERR_NO_LUSTRE, -+ myname, (char *) 0, (char *) 0); -+ *error_code = ADIOI_Error(MPI_FILE_NULL, myerrcode, myname); -+# endif -+ return; -+#else -+ *ops = &ADIO_LUSTRE_operations; -+#endif -+ } - *error_code = MPI_SUCCESS; - *fstype = file_system; - return; diff --git a/lustre/contrib/mpich2-1.0.3.patch b/lustre/contrib/mpich2-1.0.3.patch deleted file mode 100644 index 78dda9b..0000000 --- a/lustre/contrib/mpich2-1.0.3.patch +++ /dev/null @@ -1,1831 +0,0 @@ -Date: Fri, 08 Jun 2007 14:04:34 -0400 -From: Weikuan Yu -To: Weikuan Yu -Subject: Re: [Lustre-discuss] MPI-IO for Lustre -Cc: lustre-discuss@clusterfs.com - - -This is the MPICH2 patch I originally started as a base for some ROMIO -optimizations over Lustre. It should work fine for MPICH2-1.0.3 on -experimental systems. However, use it as your risk :) - -Given time, I will try to push out my optimizations after some cleanup. I -would very happy to hear feedbacks on what features people would need most -at the time. - --- -Weikuan - - -diff -ruN romio-orig/adio/ad_lustre/ad_lustre.c romio/adio/ad_lustre/ad_lustre.c ---- romio-orig/adio/ad_lustre/ad_lustre.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre.c 2006-09-06 18:40:56.000844619 -0400 -@@ -0,0 +1,37 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 2001 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+/* adioi.h has the ADIOI_Fns_struct define */ -+#include "adioi.h" -+ -+struct ADIOI_Fns_struct ADIO_LUSTRE_operations = { -+ ADIOI_LUSTRE_Open, /* Open */ -+ ADIOI_LUSTRE_ReadContig, /* ReadContig */ -+ ADIOI_LUSTRE_WriteContig, /* WriteContig */ -+ ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */ -+ ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */ -+ ADIOI_GEN_SeekIndividual, /* SeekIndividual */ -+ ADIOI_LUSTRE_Fcntl, /* Fcntl */ -+ ADIOI_LUSTRE_SetInfo, /* SetInfo */ -+ ADIOI_GEN_ReadStrided, /* ReadStrided */ -+ ADIOI_GEN_WriteStrided, /* WriteStrided */ -+ ADIOI_LUSTRE_Close, /* Close */ -+ ADIOI_LUSTRE_IreadContig, /* IreadContig */ -+ ADIOI_LUSTRE_IwriteContig, /* IwriteContig */ -+ ADIOI_LUSTRE_ReadDone, /* ReadDone */ -+ ADIOI_LUSTRE_WriteDone, /* WriteDone */ -+ ADIOI_LUSTRE_ReadComplete, /* ReadComplete */ -+ ADIOI_LUSTRE_WriteComplete, /* WriteComplete */ -+ ADIOI_LUSTRE_IreadStrided, /* IreadStrided */ -+ ADIOI_LUSTRE_IwriteStrided, /* IwriteStrided */ -+ ADIOI_GEN_Flush, /* Flush */ -+ ADIOI_LUSTRE_Resize, /* Resize */ -+ ADIOI_GEN_Delete, /* Delete */ -+}; -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_close.c romio/adio/ad_lustre/ad_lustre_close.c ---- romio-orig/adio/ad_lustre/ad_lustre_close.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_close.c 2006-09-06 17:10:35.000683211 -0400 -@@ -0,0 +1,32 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_close.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code) -+{ -+ int err; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_CLOSE"; -+#endif -+ -+ err = close(fd->fd_sys); -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_done.c romio/adio/ad_lustre/ad_lustre_done.c ---- romio-orig/adio/ad_lustre/ad_lustre_done.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_done.c 2006-09-06 17:10:35.000692922 -0400 -@@ -0,0 +1,188 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_done.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+int ADIOI_LUSTRE_ReadDone(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+#ifndef NO_AIO -+ int done=0; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READDONE"; -+#endif -+#ifdef AIO_SUN -+ aio_result_t *result=0, *tmp; -+#else -+ int err; -+#endif -+#ifdef AIO_HANDLE_IN_AIOCB -+ struct aiocb *tmp1; -+#endif -+#endif -+ -+ if (*request == ADIO_REQUEST_NULL) { -+ *error_code = MPI_SUCCESS; -+ return 1; -+ } -+ -+#ifdef NO_AIO -+/* HP, FreeBSD, Linux */ -+#ifdef HAVE_STATUS_SET_BYTES -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ (*request)->fd->async_count--; -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ *error_code = MPI_SUCCESS; -+ return 1; -+#endif -+ -+#ifdef AIO_SUN -+ if ((*request)->queued) { -+ tmp = (aio_result_t *) (*request)->handle; -+ if (tmp->aio_return == AIO_INPROGRESS) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else if (tmp->aio_return != -1) { -+ result = (aio_result_t *) aiowait(0); /* dequeue any one request */ -+ done = 1; -+ (*request)->nbytes = tmp->aio_return; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(tmp->aio_errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(tmp->aio_errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ } /* if ((*request)->queued) ... */ -+ else { -+ /* ADIOI_Complete_Async completed this request, but request object -+ was not freed. */ -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifdef AIO_HANDLE_IN_AIOCB -+/* IBM */ -+ if ((*request)->queued) { -+ tmp1 = (struct aiocb *) (*request)->handle; -+ errno = aio_error(tmp1->aio_handle); -+ if (errno == EINPROG) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+ err = aio_return(tmp1->aio_handle); -+ (*request)->nbytes = err; -+ errno = aio_error(tmp1->aio_handle); -+ -+ done = 1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } -+ } /* if ((*request)->queued) */ -+ else { -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ if ((*request)->queued) { -+ errno = aio_error((const struct aiocb *) (*request)->handle); -+ if (errno == EINPROGRESS) { -+ done = 0; -+ *error_code = MPI_SUCCESS; -+ } -+ else { -+ err = aio_return((struct aiocb *) (*request)->handle); -+ (*request)->nbytes = err; -+ errno = aio_error((struct aiocb *) (*request)->handle); -+ -+ done = 1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } -+ } /* if ((*request)->queued) */ -+ else { -+ done = 1; -+ *error_code = MPI_SUCCESS; -+ } -+#ifdef HAVE_STATUS_SET_BYTES -+ if (done && ((*request)->nbytes != -1)) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifndef NO_AIO -+ if (done) { -+ /* if request is still queued in the system, it is also there -+ on ADIOI_Async_list. Delete it from there. */ -+ if ((*request)->queued) ADIOI_Del_req_from_list(request); -+ -+ (*request)->fd->async_count--; -+ if ((*request)->handle) ADIOI_Free((*request)->handle); -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ } -+ return done; -+#endif -+ -+} -+ -+ -+int ADIOI_LUSTRE_WriteDone(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+ return ADIOI_LUSTRE_ReadDone(request, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_fcntl.c romio/adio/ad_lustre/ad_lustre_fcntl.c ---- romio-orig/adio/ad_lustre/ad_lustre_fcntl.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_fcntl.c 2006-09-06 18:43:11.000365177 -0400 -@@ -0,0 +1,127 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_fcntl.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+#include "adio_extern.h" -+/* #ifdef MPISGI -+#include "mpisgi2.h" -+#endif */ -+ -+void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int *error_code) -+{ -+ int i, ntimes; -+ ADIO_Offset curr_fsize, alloc_size, size, len, done; -+ ADIO_Status status; -+ char *buf; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_FCNTL"; -+#endif -+ -+ switch(flag) { -+ case ADIO_FCNTL_GET_FSIZE: -+ fcntl_struct->fsize = lseek(fd->fd_sys, 0, SEEK_END); -+ if (fd->fp_sys_posn != -1) -+ lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET); -+ if (fcntl_struct->fsize == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ break; -+ -+ case ADIO_FCNTL_SET_DISKSPACE: -+ /* will be called by one process only */ -+ /* On file systems with no preallocation function, I have to -+ explicitly write -+ to allocate space. Since there could be holes in the file, -+ I need to read up to the current file size, write it back, -+ and then write beyond that depending on how much -+ preallocation is needed. -+ read/write in sizes of no more than ADIOI_PREALLOC_BUFSZ */ -+ -+ curr_fsize = lseek(fd->fd_sys, 0, SEEK_END); -+ alloc_size = fcntl_struct->diskspace; -+ -+ size = ADIOI_MIN(curr_fsize, alloc_size); -+ -+ ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ; -+ buf = (char *) ADIOI_Malloc(ADIOI_PREALLOC_BUFSZ); -+ done = 0; -+ -+ for (i=0; i curr_fsize) { -+ memset(buf, 0, ADIOI_PREALLOC_BUFSZ); -+ size = alloc_size - curr_fsize; -+ ntimes = (size + ADIOI_PREALLOC_BUFSZ - 1)/ADIOI_PREALLOC_BUFSZ; -+ for (i=0; ifp_sys_posn != -1) -+ lseek(fd->fd_sys, fd->fp_sys_posn, SEEK_SET); -+ *error_code = MPI_SUCCESS; -+ break; -+ -+#if 0 -+ case ADIO_FCNTL_SET_IOMODE: -+ /* for implementing PFS I/O modes. will not occur in MPI-IO -+ implementation.*/ -+ if (fd->iomode != fcntl_struct->iomode) { -+ fd->iomode = fcntl_struct->iomode; -+ MPI_Barrier(MPI_COMM_WORLD); -+ } -+ *error_code = MPI_SUCCESS; -+ break; -+#endif -+ -+ case ADIO_FCNTL_SET_ATOMICITY: -+ fd->atomicity = (fcntl_struct->atomicity == 0) ? 0 : 1; -+ *error_code = MPI_SUCCESS; -+ break; -+ -+ default: -+ FPRINTF(stderr, "Unknown flag passed to ADIOI_LUSTRE_Fcntl\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_flush.c romio/adio/ad_lustre/ad_lustre_flush.c ---- romio-orig/adio/ad_lustre/ad_lustre_flush.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_flush.c 2006-09-06 17:10:35.000711888 -0400 -@@ -0,0 +1,14 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_flush.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Flush(ADIO_File fd, int *error_code) -+{ -+ ADIOI_GEN_Flush(fd, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre.h romio/adio/ad_lustre/ad_lustre.h ---- romio-orig/adio/ad_lustre/ad_lustre.h 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre.h 2006-09-06 17:10:35.000722616 -0400 -@@ -0,0 +1,36 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre.h,v 1.2 2005/07/07 14:38:17 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#ifndef AD_UNIX_INCLUDE -+#define AD_UNIX_INCLUDE -+ -+/* temp*/ -+#define HAVE_ASM_TYPES_H 1 -+ -+#include -+#include -+#include -+#include -+#include "lustre/lustre_user.h" -+#include "adio.h" -+ -+#ifndef NO_AIO -+#ifdef AIO_SUN -+#include -+#else -+#include -+#ifdef NEEDS_ADIOCB_T -+typedef struct adiocb adiocb_t; -+#endif -+#endif -+#endif -+ -+int ADIOI_LUSTRE_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset, -+ int wr, void *handle); -+ -+#endif -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_hints.c romio/adio/ad_lustre/ad_lustre_hints.c ---- romio-orig/adio/ad_lustre/ad_lustre_hints.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_hints.c 2006-09-06 17:10:35.000741994 -0400 -@@ -0,0 +1,130 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_hints.c,v 1.2 2005/07/07 14:38:17 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code) -+{ -+ char *value, *value_in_fd; -+ int flag, tmp_val, str_factor=-1, str_unit=0, start_iodev=-1; -+ struct lov_user_md lum = { 0 }; -+ int err, myrank, fd_sys, perm, amode, old_mask; -+ -+ if ( (fd->info) == MPI_INFO_NULL) { -+ /* This must be part of the open call. can set striping parameters -+ if necessary. */ -+ MPI_Info_create(&(fd->info)); -+ -+ /* has user specified striping or server buffering parameters -+ and do they have the same value on all processes? */ -+ if (users_info != MPI_INFO_NULL) { -+ value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char)); -+ -+ MPI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ str_factor=atoi(value); -+ tmp_val = str_factor; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != str_factor) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"striping_factor\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ MPI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ str_unit=atoi(value); -+ tmp_val = str_unit; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != str_unit) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"striping_unit\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ MPI_Info_get(users_info, "start_iodevice", MPI_MAX_INFO_VAL, -+ value, &flag); -+ if (flag) { -+ start_iodev=atoi(value); -+ tmp_val = start_iodev; -+ MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm); -+ if (tmp_val != start_iodev) { -+ FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: the value for key \"start_iodevice\" must be the same on all processes\n"); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ /* if user has specified striping info, process 0 tries to set it */ -+ if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) { -+ MPI_Comm_rank(fd->comm, &myrank); -+ if (!myrank) { -+ if (fd->perm == ADIO_PERM_NULL) { -+ old_mask = umask(022); -+ umask(old_mask); -+ perm = old_mask ^ 0666; -+ } -+ else perm = fd->perm; -+ -+ amode = 0; -+ if (fd->access_mode & ADIO_CREATE) -+ amode = amode | O_CREAT; -+ if (fd->access_mode & ADIO_RDONLY) -+ amode = amode | O_RDONLY; -+ if (fd->access_mode & ADIO_WRONLY) -+ amode = amode | O_WRONLY; -+ if (fd->access_mode & ADIO_RDWR) -+ amode = amode | O_RDWR; -+ if (fd->access_mode & ADIO_EXCL) -+ amode = amode | O_EXCL; -+ -+ /* we need to create file so ensure this is set */ -+ amode = amode | O_LOV_DELAY_CREATE | O_CREAT; -+ -+ fd_sys = open(fd->filename, amode, perm); -+ if (fd_sys == -1) { -+ if (errno != EEXIST) -+ printf("Failure to open file %s %d %d\n",strerror(errno), amode, perm); -+ } else { -+ lum.lmm_magic = LOV_USER_MAGIC; -+ lum.lmm_pattern = 0; -+ lum.lmm_stripe_size = str_unit; -+ lum.lmm_stripe_count = str_factor; -+ lum.lmm_stripe_offset = start_iodev; -+ -+ err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum); -+ if (err == -1 && errno != EEXIST) { -+ printf("Failure to set stripe info %s \n",strerror(errno)); -+ } -+ -+ close(fd_sys); -+ } -+ -+ } -+ MPI_Barrier(fd->comm); -+ } -+ -+ ADIOI_Free(value); -+ } -+ -+ /* set the values for collective I/O and data sieving parameters */ -+ ADIOI_GEN_SetInfo(fd, users_info, error_code); -+ } -+ -+ else { -+ /* The file has been opened previously and fd->fd_sys is a valid -+ file descriptor. cannot set striping parameters now. */ -+ -+ /* set the values for collective I/O and data sieving parameters */ -+ ADIOI_GEN_SetInfo(fd, users_info, error_code); -+ -+ } -+ -+ *error_code = MPI_SUCCESS; -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_iread.c romio/adio/ad_lustre/ad_lustre_iread.c ---- romio-orig/adio/ad_lustre/ad_lustre_iread.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_iread.c 2006-09-06 17:10:35.000751765 -0400 -@@ -0,0 +1,106 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_iread.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_IreadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int *error_code) -+{ -+ int len, typesize; -+#ifdef NO_AIO -+ ADIO_Status status; -+#else -+ int err=-1; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_IREADCONTIG"; -+#endif -+#endif -+ -+ (*request) = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_READ; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ -+ MPI_Type_size(datatype, &typesize); -+ len = count * typesize; -+ -+#ifdef NO_AIO -+ /* HP, FreeBSD, Linux */ -+ /* no support for nonblocking I/O. Use blocking I/O. */ -+ -+ ADIOI_LUSTRE_ReadContig(fd, buf, len, MPI_BYTE, file_ptr_type, offset, -+ &status, error_code); -+ (*request)->queued = 0; -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Get_elements(&status, MPI_BYTE, &len); -+ (*request)->nbytes = len; -+ } -+#endif -+ -+#else -+ if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind; -+ err = ADIOI_LUSTRE_aio(fd, buf, len, offset, 0, &((*request)->handle)); -+ if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; -+ -+ (*request)->queued = 1; -+ ADIOI_Add_req_to_list(request); -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+#endif /* NO_AIO */ -+ -+ fd->fp_sys_posn = -1; /* set it to null. */ -+ fd->async_count++; -+} -+ -+ -+ -+void ADIOI_LUSTRE_IreadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code) -+{ -+ ADIO_Status status; -+#ifdef HAVE_STATUS_SET_BYTES -+ int typesize; -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_READ; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ (*request)->queued = 0; -+ (*request)->handle = 0; -+ -+/* call the blocking version. It is faster because it does data sieving. */ -+ ADIOI_LUSTRE_ReadStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, &status, error_code); -+ -+ fd->async_count++; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Type_size(datatype, &typesize); -+ (*request)->nbytes = count * typesize; -+ } -+#endif -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_iwrite.c romio/adio/ad_lustre/ad_lustre_iwrite.c ---- romio-orig/adio/ad_lustre/ad_lustre_iwrite.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_iwrite.c 2006-09-06 17:10:35.000761678 -0400 -@@ -0,0 +1,268 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_iwrite.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_IwriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int *error_code) -+{ -+ int len, typesize; -+#ifdef NO_AIO -+ ADIO_Status status; -+#else -+ int err=-1; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_IWRITECONTIG"; -+#endif -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_WRITE; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ -+ MPI_Type_size(datatype, &typesize); -+ len = count * typesize; -+ -+#ifdef NO_AIO -+ /* HP, FreeBSD, Linux */ -+ /* no support for nonblocking I/O. Use blocking I/O. */ -+ -+ ADIOI_LUSTRE_WriteContig(fd, buf, len, MPI_BYTE, file_ptr_type, offset, -+ &status, error_code); -+ (*request)->queued = 0; -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Get_elements(&status, MPI_BYTE, &len); -+ (*request)->nbytes = len; -+ } -+#endif -+ -+#else -+ if (file_ptr_type == ADIO_INDIVIDUAL) offset = fd->fp_ind; -+ err = ADIOI_LUSTRE_aio(fd, buf, len, offset, 1, &((*request)->handle)); -+ if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind += len; -+ -+ (*request)->queued = 1; -+ ADIOI_Add_req_to_list(request); -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+#endif /* NO_AIO */ -+ -+ fd->fp_sys_posn = -1; /* set it to null. */ -+ fd->async_count++; -+} -+ -+ -+ -+ -+void ADIOI_LUSTRE_IwriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code) -+{ -+ ADIO_Status status; -+#ifdef HAVE_STATUS_SET_BYTES -+ int typesize; -+#endif -+ -+ *request = ADIOI_Malloc_request(); -+ (*request)->optype = ADIOI_WRITE; -+ (*request)->fd = fd; -+ (*request)->datatype = datatype; -+ (*request)->queued = 0; -+ (*request)->handle = 0; -+ -+/* call the blocking version. It is faster because it does data sieving. */ -+ ADIOI_LUSTRE_WriteStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, &status, error_code); -+ -+ fd->async_count++; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (*error_code == MPI_SUCCESS) { -+ MPI_Type_size(datatype, &typesize); -+ (*request)->nbytes = count * typesize; -+ } -+#endif -+} -+ -+ -+/* This function is for implementation convenience. It is not user-visible. -+ It takes care of the differences in the interface for nonblocking I/O -+ on various Unix machines! If wr==1 write, wr==0 read. */ -+ -+int ADIOI_LUSTRE_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset, -+ int wr, void *handle) -+{ -+ int err=-1, fd_sys; -+ -+#ifndef NO_AIO -+ int error_code; -+#ifdef AIO_SUN -+ aio_result_t *result; -+#else -+ struct aiocb *aiocbp; -+#endif -+#endif -+ -+ fd_sys = fd->fd_sys; -+ -+#ifdef AIO_SUN -+ result = (aio_result_t *) ADIOI_Malloc(sizeof(aio_result_t)); -+ result->aio_return = AIO_INPROGRESS; -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* the man pages say EPROCLIM, but in reality errno is set to EAGAIN! */ -+ -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again.*/ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aiowrite(fd_sys, buf, len, offset, SEEK_SET, result); -+ else err = aioread(fd_sys, buf, len, offset, SEEK_SET, result); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((aio_result_t **) handle) = result; -+#endif -+ -+#ifdef NO_FD_IN_AIOCB -+/* IBM */ -+ aiocbp = (struct aiocb *) ADIOI_Malloc(sizeof(struct aiocb)); -+ aiocbp->aio_whence = SEEK_SET; -+ aiocbp->aio_offset = offset; -+ aiocbp->aio_buf = buf; -+ aiocbp->aio_nbytes = len; -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again. */ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aio_write(fd_sys, aiocbp); -+ else err = aio_read(fd_sys, aiocbp); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((struct aiocb **) handle) = aiocbp; -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ -+ aiocbp = (struct aiocb *) ADIOI_Calloc(sizeof(struct aiocb), 1); -+ aiocbp->aio_fildes = fd_sys; -+ aiocbp->aio_offset = offset; -+ aiocbp->aio_buf = buf; -+ aiocbp->aio_nbytes = len; -+ -+#ifdef AIO_PRIORITY_DEFAULT -+/* DEC */ -+ aiocbp->aio_reqprio = AIO_PRIO_DFL; /* not needed in DEC Unix 4.0 */ -+ aiocbp->aio_sigevent.sigev_signo = 0; -+#else -+ aiocbp->aio_reqprio = 0; -+#endif -+ -+#ifdef AIO_SIGNOTIFY_NONE -+/* SGI IRIX 6 */ -+ aiocbp->aio_sigevent.sigev_notify = SIGEV_NONE; -+#else -+ aiocbp->aio_sigevent.sigev_signo = 0; -+#endif -+ -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ -+ if (err == -1) { -+ if (errno == EAGAIN) { -+ /* exceeded the max. no. of outstanding requests. -+ complete all previous async. requests and try again. */ -+ -+ ADIOI_Complete_async(&error_code); -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ -+ while (err == -1) { -+ if (errno == EAGAIN) { -+ /* sleep and try again */ -+ sleep(1); -+ if (wr) err = aio_write(aiocbp); -+ else err = aio_read(aiocbp); -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ } -+ else { -+ FPRINTF(stderr, "Unknown errno %d in ADIOI_LUSTRE_aio\n", errno); -+ MPI_Abort(MPI_COMM_WORLD, 1); -+ } -+ } -+ -+ *((struct aiocb **) handle) = aiocbp; -+#endif -+ -+ return err; -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_open.c romio/adio/ad_lustre/ad_lustre_open.c ---- romio-orig/adio/ad_lustre/ad_lustre_open.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_open.c 2006-09-06 17:10:35.000771351 -0400 -@@ -0,0 +1,100 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_open.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code) -+{ -+ int perm, old_mask, amode; -+ struct lov_user_md lum = { 0 }; -+ char *value; -+ -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_OPEN"; -+#endif -+ -+ if (fd->perm == ADIO_PERM_NULL) { -+ old_mask = umask(022); -+ umask(old_mask); -+ perm = old_mask ^ 0666; -+ } -+ else perm = fd->perm; -+ -+ amode = 0; -+ if (fd->access_mode & ADIO_CREATE) -+ amode = amode | O_CREAT; -+ if (fd->access_mode & ADIO_RDONLY) -+ amode = amode | O_RDONLY; -+ if (fd->access_mode & ADIO_WRONLY) -+ amode = amode | O_WRONLY; -+ if (fd->access_mode & ADIO_RDWR) -+ amode = amode | O_RDWR; -+ if (fd->access_mode & ADIO_EXCL) -+ amode = amode | O_EXCL; -+ -+ fd->fd_sys = open(fd->filename, amode, perm); -+ -+ if (fd->fd_sys != -1) { -+ int err; -+ -+ value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char)); -+ -+ /* get file striping information and set it in info */ -+ lum.lmm_magic = LOV_USER_MAGIC; -+ err = ioctl(fd->fd_sys, LL_IOC_LOV_GETSTRIPE, (void *) &lum); -+ -+ if (!err) { -+ sprintf(value, "%d", lum.lmm_stripe_size); -+ MPI_Info_set(fd->info, "striping_unit", value); -+ -+ sprintf(value, "%d", lum.lmm_stripe_count); -+ MPI_Info_set(fd->info, "striping_factor", value); -+ -+ sprintf(value, "%d", lum.lmm_stripe_offset); -+ MPI_Info_set(fd->info, "start_iodevice", value); -+ } -+ ADIOI_Free(value); -+ -+ if (fd->access_mode & ADIO_APPEND) -+ fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END); -+ } -+ -+ -+ if ((fd->fd_sys != -1) && (fd->access_mode & ADIO_APPEND)) -+ fd->fp_ind = fd->fp_sys_posn = lseek(fd->fd_sys, 0, SEEK_END); -+ -+ if (fd->fd_sys == -1) { -+#ifdef MPICH2 -+ if (errno == ENAMETOOLONG) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_BAD_FILE, "**filenamelong", "**filenamelong %s %d", fd->filename, strlen(fd->filename)); -+ else if (errno == ENOENT) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_NO_SUCH_FILE, "**filenoexist", "**filenoexist %s", fd->filename); -+ else if (errno == ENOTDIR || errno == ELOOP) -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_BAD_FILE, "**filenamedir", "**filenamedir %s", fd->filename); -+ else if (errno == EACCES) { -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_ACCESS, "**fileaccess", "**fileaccess %s", -+ fd->filename); -+ } -+ else if (errno == EROFS) { -+ /* Read only file or file system and write access requested */ -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_READ_ONLY, "**ioneedrd", 0); -+ } -+ else { -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ } -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(ADIO_FILE_NULL, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_rdcoll.c romio/adio/ad_lustre/ad_lustre_rdcoll.c ---- romio-orig/adio/ad_lustre/ad_lustre_rdcoll.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_rdcoll.c 2006-09-06 17:10:35.000780880 -0400 -@@ -0,0 +1,18 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_rdcoll.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_ReadStridedColl(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_read.c romio/adio/ad_lustre/ad_lustre_read.c ---- romio-orig/adio/ad_lustre/ad_lustre_read.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_read.c 2006-09-06 17:10:35.000790846 -0400 -@@ -0,0 +1,67 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_read.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int *error_code) -+{ -+ int err=-1, datatype_size, len; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READCONTIG"; -+#endif -+ -+ MPI_Type_size(datatype, &datatype_size); -+ len = datatype_size * count; -+ -+ if (file_ptr_type == ADIO_EXPLICIT_OFFSET) { -+ if (fd->fp_sys_posn != offset) -+ lseek(fd->fd_sys, offset, SEEK_SET); -+ err = read(fd->fd_sys, buf, len); -+ fd->fp_sys_posn = offset + len; -+ /* individual file pointer not updated */ -+ } -+ else { /* read from curr. location of ind. file pointer */ -+ if (fd->fp_sys_posn != fd->fp_ind) -+ lseek(fd->fd_sys, fd->fp_ind, SEEK_SET); -+ err = read(fd->fd_sys, buf, len); -+ fd->fp_ind += err; -+ fd->fp_sys_posn = fd->fp_ind; -+ } -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (err != -1) MPIR_Status_set_bytes(status, datatype, err); -+#endif -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -+ -+ -+ -+ -+void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_ReadStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_resize.c romio/adio/ad_lustre/ad_lustre_resize.c ---- romio-orig/adio/ad_lustre/ad_lustre_resize.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_resize.c 2006-09-06 17:10:35.000807397 -0400 -@@ -0,0 +1,32 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_resize.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_Resize(ADIO_File fd, ADIO_Offset size, int *error_code) -+{ -+ int err; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_RESIZE"; -+#endif -+ -+ err = ftruncate(fd->fd_sys, size); -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_seek.c romio/adio/ad_lustre/ad_lustre_seek.c ---- romio-orig/adio/ad_lustre/ad_lustre_seek.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_seek.c 2006-09-06 17:10:35.000816583 -0400 -@@ -0,0 +1,15 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_seek.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+ADIO_Offset ADIOI_LUSTRE_SeekIndividual(ADIO_File fd, ADIO_Offset offset, -+ int whence, int *error_code) -+{ -+ return ADIOI_GEN_SeekIndividual(fd, offset, whence, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_wait.c romio/adio/ad_lustre/ad_lustre_wait.c ---- romio-orig/adio/ad_lustre/ad_lustre_wait.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_wait.c 2006-09-06 18:45:39.000190529 -0400 -@@ -0,0 +1,188 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_wait.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_ReadComplete(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+#ifndef NO_AIO -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_READCOMPLETE"; -+#endif -+#ifdef AIO_SUN -+ aio_result_t *result=0, *tmp; -+#else -+ int err; -+#endif -+#ifdef AIO_HANDLE_IN_AIOCB -+ struct aiocb *tmp1; -+#endif -+#endif -+ -+ if (*request == ADIO_REQUEST_NULL) { -+ *error_code = MPI_SUCCESS; -+ return; -+ } -+ -+#ifdef AIO_SUN -+ if ((*request)->queued) { /* dequeue it */ -+ tmp = (aio_result_t *) (*request)->handle; -+ while (tmp->aio_return == AIO_INPROGRESS) usleep(1000); -+ /* sleep for 1 ms., until done. Is 1 ms. a good number? */ -+ /* when done, dequeue any one request */ -+ result = (aio_result_t *) aiowait(0); -+ -+ (*request)->nbytes = tmp->aio_return; -+ -+ if (tmp->aio_return == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(tmp->aio_errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(tmp->aio_errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ -+/* aiowait only dequeues a request. The completion of a request can be -+ checked by just checking the aio_return flag in the handle passed -+ to the original aioread()/aiowrite(). Therefore, I need to ensure -+ that aiowait() is called exactly once for each previous -+ aioread()/aiowrite(). This is also taken care of in ADIOI_xxxDone */ -+ } -+ else *error_code = MPI_SUCCESS; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#endif -+ -+#ifdef AIO_HANDLE_IN_AIOCB -+/* IBM */ -+ if ((*request)->queued) { -+ do { -+ err = aio_suspend(1, (struct aiocb **) &((*request)->handle)); -+ } while ((err == -1) && (errno == EINTR)); -+ -+ tmp1 = (struct aiocb *) (*request)->handle; -+ if (err != -1) { -+ err = aio_return(tmp1->aio_handle); -+ (*request)->nbytes = err; -+ errno = aio_error(tmp1->aio_handle); -+ } -+ else (*request)->nbytes = -1; -+ -+/* on DEC, it is required to call aio_return to dequeue the request. -+ IBM man pages don't indicate what function to use for dequeue. -+ I'm assuming it is aio_return! POSIX says aio_return may be called -+ only once on a given handle. */ -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } /* if ((*request)->queued) */ -+ else *error_code = MPI_SUCCESS; -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ -+#elif (!defined(NO_AIO) && !defined(AIO_SUN)) -+/* DEC, SGI IRIX 5 and 6 */ -+ if ((*request)->queued) { -+ do { -+ err = aio_suspend((const struct aiocb_t **) &((*request)->handle), 1, 0); -+ } while ((err == -1) && (errno == EINTR)); -+ -+ if (err != -1) { -+ err = aio_return((struct aiocb *) (*request)->handle); -+ (*request)->nbytes = err; -+ errno = aio_error((struct aiocb *) (*request)->handle); -+ } -+ else (*request)->nbytes = -1; -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+ return; -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else /* MPICH-1 */ -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error((*request)->fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+ } /* if ((*request)->queued) */ -+ else *error_code = MPI_SUCCESS; -+#ifdef HAVE_STATUS_SET_BYTES -+ if ((*request)->nbytes != -1) -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+#endif -+ -+#ifndef NO_AIO -+ if ((*request)->queued != -1) { -+ -+ /* queued = -1 is an internal hack used when the request must -+ be completed, but the request object should not be -+ freed. This is used in ADIOI_Complete_async, because the user -+ will call MPI_Wait later, which would require status to -+ be filled. Ugly but works. queued = -1 should be used only -+ in ADIOI_Complete_async. -+ This should not affect the user in any way. */ -+ -+ /* if request is still queued in the system, it is also there -+ on ADIOI_Async_list. Delete it from there. */ -+ if ((*request)->queued) ADIOI_Del_req_from_list(request); -+ -+ (*request)->fd->async_count--; -+ if ((*request)->handle) ADIOI_Free((*request)->handle); -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ } -+ -+#else -+/* HP, FreeBSD, Linux */ -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ MPIR_Status_set_bytes(status, (*request)->datatype, (*request)->nbytes); -+#endif -+ (*request)->fd->async_count--; -+ ADIOI_Free_request((ADIOI_Req_node *) (*request)); -+ *request = ADIO_REQUEST_NULL; -+ *error_code = MPI_SUCCESS; -+#endif -+} -+ -+ -+void ADIOI_LUSTRE_WriteComplete(ADIO_Request *request, ADIO_Status *status, int *error_code) -+{ -+ ADIOI_LUSTRE_ReadComplete(request, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_wrcoll.c romio/adio/ad_lustre/ad_lustre_wrcoll.c ---- romio-orig/adio/ad_lustre/ad_lustre_wrcoll.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_wrcoll.c 2006-09-06 17:10:35.000835460 -0400 -@@ -0,0 +1,18 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_wrcoll.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_WriteStridedColl(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/ad_lustre_write.c romio/adio/ad_lustre/ad_lustre_write.c ---- romio-orig/adio/ad_lustre/ad_lustre_write.c 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/ad_lustre_write.c 2006-09-06 17:10:35.000844658 -0400 -@@ -0,0 +1,66 @@ -+/* -*- Mode: C; c-basic-offset:4 ; -*- */ -+/* -+ * $Id: ad_lustre_write.c,v 1.1.1.1 2004/11/04 11:03:38 liam Exp $ -+ * -+ * Copyright (C) 1997 University of Chicago. -+ * See COPYRIGHT notice in top-level directory. -+ */ -+ -+#include "ad_lustre.h" -+ -+void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int *error_code) -+{ -+ int err=-1, datatype_size, len; -+#if defined(MPICH2) || !defined(PRINT_ERR_MSG) -+ static char myname[] = "ADIOI_LUSTRE_WRITECONTIG"; -+#endif -+ -+ MPI_Type_size(datatype, &datatype_size); -+ len = datatype_size * count; -+ -+ if (file_ptr_type == ADIO_EXPLICIT_OFFSET) { -+ if (fd->fp_sys_posn != offset) -+ lseek(fd->fd_sys, offset, SEEK_SET); -+ err = write(fd->fd_sys, buf, len); -+ fd->fp_sys_posn = offset + err; -+ /* individual file pointer not updated */ -+ } -+ else { /* write from curr. location of ind. file pointer */ -+ if (fd->fp_sys_posn != fd->fp_ind) -+ lseek(fd->fd_sys, fd->fp_ind, SEEK_SET); -+ err = write(fd->fd_sys, buf, len); -+ fd->fp_ind += err; -+ fd->fp_sys_posn = fd->fp_ind; -+ } -+ -+#ifdef HAVE_STATUS_SET_BYTES -+ if (err != -1 && status) MPIR_Status_set_bytes(status, datatype, err); -+#endif -+ -+ if (err == -1) { -+#ifdef MPICH2 -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**io", -+ "**io %s", strerror(errno)); -+#elif defined(PRINT_ERR_MSG) -+ *error_code = MPI_ERR_UNKNOWN; -+#else -+ *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR, -+ myname, "I/O Error", "%s", strerror(errno)); -+ ADIOI_Error(fd, *error_code, myname); -+#endif -+ } -+ else *error_code = MPI_SUCCESS; -+} -+ -+ -+ -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code) -+{ -+ ADIOI_GEN_WriteStrided(fd, buf, count, datatype, file_ptr_type, -+ offset, status, error_code); -+} -diff -ruN romio-orig/adio/ad_lustre/Makefile.in romio/adio/ad_lustre/Makefile.in ---- romio-orig/adio/ad_lustre/Makefile.in 1969-12-31 19:00:00.000000000 -0500 -+++ romio/adio/ad_lustre/Makefile.in 2006-09-06 18:48:56.000800829 -0400 -@@ -0,0 +1,51 @@ -+CC = @CC@ -+AR = @AR@ -+RANLIB = @RANLIB@ -+LIBNAME = @LIBNAME@ -+srcdir = @srcdir@ -+CC_SHL = @CC_SHL@ -+SHLIBNAME = @SHLIBNAME@ -+ -+INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I${srcdir}/../include -I../include -I../../include -I${srcdir}/../../../../include -I../../../../include -+CFLAGS = @CPPFLAGS@ @CFLAGS@ $(INCLUDE_DIR) -+ -+top_builddir = @master_topbuild_dir@ -+LIBTOOL = @LIBTOOL@ -+C_COMPILE_SHL = $(CC_SHL) @CFLAGS@ $(INCLUDE_DIR) -+ -+@VPATH@ -+ -+AD_LUSTRE_OBJECTS = ad_lustre.o ad_lustre_close.o ad_lustre_read.o \ -+ ad_lustre_open.o ad_lustre_write.o ad_lustre_done.o \ -+ ad_lustre_fcntl.o ad_lustre_iread.o ad_lustre_iwrite.o ad_lustre_wait.o \ -+ ad_lustre_resize.o ad_lustre_hints.o -+ -+default: $(LIBNAME) -+ @if [ "@ENABLE_SHLIB@" != "none" ] ; then \ -+ $(MAKE) $(SHLIBNAME).la ;\ -+ fi -+ -+.SUFFIXES: $(SUFFIXES) .p .lo -+ -+.c.o: -+ $(CC) $(CFLAGS) -c $< -+.c.lo: -+ $(C_COMPILE_SHL) -c $< -o _s$*.o -+ @mv -f _s$*.o $*.lo -+# $(C_COMPILE_SHL) -c $< -+# @mv -f $*.o $*.lo -+ -+$(LIBNAME): $(AD_LUSTRE_OBJECTS) -+ $(AR) $(LIBNAME) $(AD_LUSTRE_OBJECTS) -+ $(RANLIB) $(LIBNAME) -+ -+AD_LUSTRE_LOOBJECTS=$(AD_LUSTRE_OBJECTS:.o=.lo) -+$(SHLIBNAME).la: $(AD_LUSTRE_LOOBJECTS) -+ $(AR) $(SHLIBNAME).la $(AD_LUSTRE_LOOBJECTS) -+ -+coverage: -+ -@for file in ${AD_LUSTRE_OBJECTS:.o=.c} ; do \ -+ gcov -b -f $$file ; done -+ -+clean: -+ @rm -f *.o *.lo -diff -ruN romio-orig/adio/common/ad_fstype.c romio/adio/common/ad_fstype.c ---- romio-orig/adio/common/ad_fstype.c 2005-08-11 19:33:46.000000000 -0400 -+++ romio/adio/common/ad_fstype.c 2006-09-06 17:41:20.000830936 -0400 -@@ -265,6 +265,9 @@ - /* if UFS support is enabled, default to that */ - *fstype = ADIO_UFS; - return; -+# elif defined(LINUX) && defined(ROMIO_LUSTRE) -+# warning use correct include -+# define LL_SUPER_MAGIC 0x0BD00BD0 - # endif - - /* --BEGIN ERROR HANDLING-- */ -@@ -308,6 +311,13 @@ - } - # endif - -+# ifdef LL_SUPER_MAGIC -+ if (fsbuf.f_type == LL_SUPER_MAGIC) { -+ *fstype = ADIO_LUSTRE; -+ return; -+ } -+# endif -+ - # ifdef PAN_KERNEL_FS_CLIENT_SUPER_MAGIC - if (fsbuf.f_type == PAN_KERNEL_FS_CLIENT_SUPER_MAGIC) { - *fstype = ADIO_PANFS; -@@ -458,6 +468,11 @@ - { - *fstype = ADIO_GRIDFTP; - } -+ else if (!strncmp(filename, "lustre:", 7) -+ || !strncmp(filename, "LUSTRE:", 7)) -+ { -+ *fstype = ADIO_LUSTRE; -+ } - else { - #ifdef ROMIO_NTFS - *fstype = ADIO_NTFS; -@@ -657,6 +672,14 @@ - *ops = &ADIO_GRIDFTP_operations; - #endif - } -+ if (file_system == ADIO_LUSTRE) { -+#ifndef ROMIO_LUSTRE -+ *error_code = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE, myname, __LINE__, MPI_ERR_IO, "**iofstypeunsupported", 0); -+ return; -+#else -+ *ops = &ADIO_LUSTRE_operations; -+#endif -+ } - *error_code = MPI_SUCCESS; - *fstype = file_system; - return; -diff -ruN romio-orig/adio/include/adio.h romio/adio/include/adio.h ---- romio-orig/adio/include/adio.h 2006-06-09 17:45:04.000000000 -0400 -+++ romio/adio/include/adio.h 2006-09-06 17:44:16.000614058 -0400 -@@ -302,6 +302,7 @@ - #define ADIO_PVFS2 160 /* PVFS2: 2nd generation PVFS */ - #define ADIO_PANFS 161 /* Panasas FS */ - #define ADIO_GRIDFTP 162 /* Globus GridFTP */ -+#define ADIO_LUSTRE 163 /* Lustre */ - - #define ADIO_SEEK_SET SEEK_SET - #define ADIO_SEEK_CUR SEEK_CUR -diff -ruN romio-orig/adio/include/adioi_fs_proto.h romio/adio/include/adioi_fs_proto.h ---- romio-orig/adio/include/adioi_fs_proto.h 2005-06-08 17:16:39.000000000 -0400 -+++ romio/adio/include/adioi_fs_proto.h 2006-09-06 17:48:11.000523566 -0400 -@@ -49,6 +49,68 @@ - /* prototypes are in adio/ad_sfs/ad_sfs.h */ - #endif - -+#ifdef ROMIO_LUSTRE -+extern struct ADIOI_Fns_struct ADIO_LUSTRE_operations; -+ -+void ADIOI_LUSTRE_Open(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_Close(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_IwriteContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_IreadContig(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+int ADIOI_LUSTRE_ReadDone(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+int ADIOI_LUSTRE_WriteDone(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadComplete(ADIO_Request *request, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteComplete(ADIO_Request *request, ADIO_Status *status, -+ int *error_code); -+void ADIOI_LUSTRE_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int -+ *error_code); -+void ADIOI_LUSTRE_WriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_WriteStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_ReadStridedColl(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Status *status, int -+ *error_code); -+void ADIOI_LUSTRE_IreadStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_IwriteStrided(ADIO_File fd, void *buf, int count, -+ MPI_Datatype datatype, int file_ptr_type, -+ ADIO_Offset offset, ADIO_Request *request, int -+ *error_code); -+void ADIOI_LUSTRE_Flush(ADIO_File fd, int *error_code); -+void ADIOI_LUSTRE_Resize(ADIO_File fd, ADIO_Offset size, int *error_code); -+ADIO_Offset ADIOI_LUSTRE_SeekIndividual(ADIO_File fd, ADIO_Offset offset, -+ int whence, int *error_code); -+void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code); -+#endif -+ - #ifdef ROMIO_NTFS - extern struct ADIOI_Fns_struct ADIO_NTFS_operations; - /* prototypes are in adio/ad_ntfs/ad_ntfs.h */ -diff -ruN romio-orig/adio/include/mpio_error.h romio/adio/include/mpio_error.h ---- romio-orig/adio/include/mpio_error.h 2005-05-23 19:27:50.000000000 -0400 -+++ romio/adio/include/mpio_error.h 2006-09-06 17:10:35.000984078 -0400 -@@ -63,6 +63,7 @@ - #define MPIR_ERR_FILETYPE 33 - #define MPIR_ERR_NO_NTFS 35 - #define MPIR_ERR_NO_TESTFS 36 -+#define MPIR_ERR_NO_LUSTRE 37 - - /* MPI_ERR_COMM */ - #ifndef MPIR_ERR_COMM_NULL -diff -ruN romio-orig/adio/include/romioconf.h.in romio/adio/include/romioconf.h.in ---- romio-orig/adio/include/romioconf.h.in 2006-08-11 09:48:44.000000000 -0400 -+++ romio/adio/include/romioconf.h.in 2006-09-06 17:43:08.000599274 -0400 -@@ -276,6 +276,9 @@ - /* Define for ROMIO with PVFS2 */ - #undef ROMIO_PVFS2 - -+/* Define for ROMIO with LUSTRE */ -+#undef ROMIO_LUSTRE -+ - /* Define if int64_t must be defined for PVFS */ - #undef ROMIO_PVFS_NEEDS_INT64_DEFINITION - -diff -ruN romio-orig/configure romio/configure ---- romio-orig/configure 2006-08-11 09:48:45.000000000 -0400 -+++ romio/configure 2006-09-06 17:20:57.000555513 -0400 -@@ -1400,7 +1400,7 @@ - # - have_aio=no - # --known_filesystems="nfs ufs pfs pvfs pvfs2 testfs xfs panfs gridftp" -+known_filesystems="nfs ufs pfs pvfs pvfs2 testfs xfs panfs gridftp lustre" - known_mpi_impls="mpich2_mpi mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi" - # - # Defaults -@@ -7490,6 +7490,14 @@ - - fi - -+if test -n "$file_system_lustre"; then -+ -+cat >>confdefs.h <<\_ACEOF -+#define ROMIO_LUSTRE 1 -+_ACEOF -+ -+fi -+ - # - # Check for presence and characteristics of async. I/O calls if - # not disabled. -@@ -11977,7 +11985,7 @@ - # are active will be called by the top level ROMIO make - ac_config_commands="$ac_config_commands default-1" - -- ac_config_files="$ac_config_files Makefile localdefs mpi-io/Makefile mpi2-other/info/Makefile mpi2-other/array/Makefile adio/common/Makefile test/Makefile test/misc.c test/large_file.c test/runtests util/romioinstall include/mpio.h include/mpiof.h adio/ad_nfs/Makefile adio/ad_ufs/Makefile adio/ad_panfs/Makefile adio/ad_xfs/Makefile adio/ad_sfs/Makefile adio/ad_pfs/Makefile adio/ad_testfs/Makefile adio/ad_pvfs/Makefile adio/ad_pvfs2/Makefile adio/ad_gridftp/Makefile mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile mpi2-other/array/fortran/Makefile test/fmisc.f test/fcoll_test.f test/pfcoll_test.f test/fperf.f mpi-io/glue/mpich2/Makefile mpi-io/glue/mpich1/Makefile mpi-io/glue/default/Makefile" -+ ac_config_files="$ac_config_files Makefile localdefs mpi-io/Makefile mpi2-other/info/Makefile mpi2-other/array/Makefile adio/common/Makefile test/Makefile test/misc.c test/large_file.c test/runtests util/romioinstall include/mpio.h include/mpiof.h adio/ad_nfs/Makefile adio/ad_ufs/Makefile adio/ad_panfs/Makefile adio/ad_xfs/Makefile adio/ad_sfs/Makefile adio/ad_pfs/Makefile adio/ad_testfs/Makefile adio/ad_pvfs/Makefile adio/ad_pvfs2/Makefile adio/ad_gridftp/Makefile adio/ad_lustre/Makefile mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile mpi2-other/array/fortran/Makefile test/fmisc.f test/fcoll_test.f test/pfcoll_test.f test/fperf.f mpi-io/glue/mpich2/Makefile mpi-io/glue/mpich1/Makefile mpi-io/glue/default/Makefile" - cat >confcache <<\_ACEOF - # This file is a shell script that caches the results of configure - # tests run on this system so they can be shared between configure -@@ -12535,6 +12543,7 @@ - "adio/ad_pvfs/Makefile" ) CONFIG_FILES="$CONFIG_FILES adio/ad_pvfs/Makefile" ;; - "adio/ad_pvfs2/Makefile" ) CONFIG_FILES="$CONFIG_FILES adio/ad_pvfs2/Makefile" ;; - "adio/ad_gridftp/Makefile" ) CONFIG_FILES="$CONFIG_FILES adio/ad_gridftp/Makefile" ;; -+ "adio/ad_lustre/Makefile" ) CONFIG_FILES="$CONFIG_FILES adio/ad_lustre/Makefile" ;; - "mpi-io/fortran/Makefile" ) CONFIG_FILES="$CONFIG_FILES mpi-io/fortran/Makefile" ;; - "mpi2-other/info/fortran/Makefile" ) CONFIG_FILES="$CONFIG_FILES mpi2-other/info/fortran/Makefile" ;; - "mpi2-other/array/fortran/Makefile" ) CONFIG_FILES="$CONFIG_FILES mpi2-other/array/fortran/Makefile" ;; -diff -ruN romio-orig/configure.in romio/configure.in ---- romio-orig/configure.in 2006-07-24 17:55:57.000000000 -0400 -+++ romio/configure.in 2006-09-06 17:16:13.000525117 -0400 -@@ -93,7 +93,7 @@ - # - have_aio=no - # --known_filesystems="nfs ufs pfs pvfs pvfs2 testfs xfs panfs gridftp" -+known_filesystems="nfs ufs pfs pvfs pvfs2 testfs xfs panfs gridftp lustre" - known_mpi_impls="mpich2_mpi mpich_mpi sgi_mpi hp_mpi cray_mpi lam_mpi" - # - # Defaults -@@ -1062,6 +1062,9 @@ - if test -n "$file_system_testfs"; then - AC_DEFINE(ROMIO_TESTFS,1,[Define for ROMIO with TESTFS]) - fi -+if test -n "$file_system_lustre"; then -+ AC_DEFINE(ROMIO_LUSTRE,1,[Define for ROMIO with LUSTRE]) -+fi - - if test -n "$file_system_xfs"; then - AC_DEFINE(ROMIO_XFS,1,[Define for ROMIO with XFS]) -@@ -2024,6 +2027,7 @@ - adio/ad_testfs/Makefile adio/ad_pvfs/Makefile \ - adio/ad_pvfs2/Makefile \ - adio/ad_gridftp/Makefile \ -+ adio/ad_lustre/Makefile \ - mpi-io/fortran/Makefile mpi2-other/info/fortran/Makefile \ - mpi2-other/array/fortran/Makefile test/fmisc.f \ - test/fcoll_test.f test/pfcoll_test.f test/fperf.f \ -diff -ruN romio-orig/Makefile.in romio/Makefile.in ---- romio-orig/Makefile.in 2005-05-24 18:53:11.000000000 -0400 -+++ romio/Makefile.in 2006-09-06 17:13:25.000393429 -0400 -@@ -14,7 +14,7 @@ - MPIO_DIRS = mpi-io - EXTRA_SRC_DIRS = @EXTRA_SRC_DIRS@ - FILE_SYS_DIRS = @FILE_SYS_DIRS@ --ALL_DIRS = mpi-io mpi-io/fortran mpi2-other/info mpi2-other/info/fortran mpi2-other/array mpi2-other/array/fortran adio/common adio/ad_pfs adio/ad_piofs adio/ad_nfs adio/ad_ufs adio/ad_xfs adio/ad_hfs adio/ad_sfs adio/ad_testfs adio/ad_pvfs adio/ad_pvfs2 adio/ad_panfs adio/ad_gridftp test -+ALL_DIRS = mpi-io mpi-io/fortran mpi2-other/info mpi2-other/info/fortran mpi2-other/array mpi2-other/array/fortran adio/common adio/ad_pfs adio/ad_piofs adio/ad_nfs adio/ad_ufs adio/ad_xfs adio/ad_hfs adio/ad_sfs adio/ad_testfs adio/ad_pvfs adio/ad_pvfs2 adio/ad_panfs adio/ad_gridftp adio/ad_lustre test - SHELL = /bin/sh - - @VPATH@