o=Adilger,grev
i=grev
limit unlink tests patch
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
log "===== $0 ### 1 NODE CREATE ###"
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
log "===== $0 ### 1 NODE CREATE ###"
- echo "Running creates on 1 node(s)."
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate creates for a single client failed, aborting"
fi
log "===== $0 ### 1 NODE UNLINK ###"
fi
log "===== $0 ### 1 NODE UNLINK ###"
- echo "Running unlinks on 1 node(s)."
- COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD}
+ if [ -f "$LOG" ]; then
+ CREATED=$(awk '/total:/ { print $7 }' $LOG)
+ [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
+ fi
+
+ COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate unlink on a single client failed, aborting"
+[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
echo "NO test for create on multiple nodes."
else
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
log "===== $0 ### $NUM_CLIENTS NODES CREATE ###"
if [ -n "$NOMULTI" ]; then
echo "NO test for create on multiple nodes."
else
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
log "===== $0 ### $NUM_CLIENTS NODES CREATE ###"
- echo "Running creates on ${NUM_CLIENTS} node(s)."
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate create on multiple nodes failed, aborting"
- echo "Running unlinks on ${NUM_CLIENTS} node(s)."
+ log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###"
- COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD}
+ if [ -f "$LOG" ]; then
+ CREATED=$(awk '/total:/ { print $7 }' $LOG)
+ [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
+ fi
+
+ COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate unlink on multiple nodes failed, aborting"
-equals_msg `basename $0`: test complete, cleaning up
-mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
-mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
log "===== $0 ### 1 NODE CREATE ###"
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
log "===== $0 ### 1 NODE CREATE ###"
- echo "Running creates on 1 node(s)."
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate create for a single client failed, aborting"
echo "NO Test for unlinks for a single client."
else
log "===== $0 ### 1 NODE UNLINK ###"
echo "NO Test for unlinks for a single client."
else
log "===== $0 ### 1 NODE UNLINK ###"
- echo "Running unlinks on 1 node(s)."
- COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD}
+ if [ -f "$LOG" ]; then
+ CREATED=$(awk '/total:/ { print $7 }' $LOG)
+ [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
+ fi
+
+ COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate unlinks for a single client failed, aborting"
else
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
else
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
- log "===== $0 ### $NUM_CLIENTS NODES CREATE ###"
- echo "Running creates on ${NUM_CLIENTS} node(s) with $THREADS_PER_CLIENT threads per client."
+ log "===== $0 ### $NUM_CLIENTS NODES CREATE with $THREADS_PER_CLIENT threads per client ###"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate create on multiple nodes failed, aborting"
fi
fi
if [ -n "$NOUNLINK" ]; then
echo "NO Test for unlinks multiple nodes."
else
fi
fi
if [ -n "$NOUNLINK" ]; then
echo "NO Test for unlinks multiple nodes."
else
- log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###"
- echo "Running unlinks on ${NUM_CLIENTS} node(s) with $THREADS_PER_CLIENT threads per client."
+ log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###"
+
+ if [ -f "$LOG" ]; then
+ CREATED=$(awk '/total:/ { print $7 }' $LOG)
+ [ $CREATED -gt 0 ] && NUM_FILES=$CREATED
+ fi
- COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink --time ${TIME_PERIOD}
+ COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
echo "+ ${COMMAND}"
mpi_run -np $((NUM_CLIENTS * THREADS_PER_CLIENT)) -machinefile ${MACHINEFILE} \
${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate unlinks multiple nodes failed, aborting"
fi
fi
fi
equals_msg `basename $0`: test complete, cleaning up
fi
fi
fi
equals_msg `basename $0`: test complete, cleaning up
-mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore
-mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
- echo "Test preparation: creating ${NUM_FILES} files."
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT))
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT))
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
# No lockup if error occurs on file creation, abort.
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
# No lockup if error occurs on file creation, abort.
- [ ${PIPESTATUS[0]} != 0 ] && error "mpirun ... mdsrate ... file creation failed, aborting"
+ [ ${PIPESTATUS[0]} != 0 ] && error "mdsrate file creation failed, aborting"
fi
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION}
fi
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION}
echo "NO Test for lookups on a single client."
else
log "===== $0 ### 1 NODE LOOKUPS ###"
echo "NO Test for lookups on a single client."
else
log "===== $0 ### 1 NODE LOOKUPS ###"
- echo "Running lookups on 1 node(s)."
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate lookups on a single client failed, aborting"
+[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
echo "NO test for lookups on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
if [ -n "$NOMULTI" ]; then
echo "NO test for lookups on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
- echo "Running lookups on ${NUM_CLIENTS} node(s)."
echo "+" ${COMMAND}
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
echo "+" ${COMMAND}
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate lookups on multiple nodes failed, aborting"
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
- echo "Test preparation: creating ${NUM_FILES} files."
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --dir ${TESTDIR}
--nfiles ${NUM_FILES} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --dir ${TESTDIR}
--nfiles ${NUM_FILES} --filefmt 'f%%d'"
fi
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
fi
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
- [ ${PIPESTATUS[0]} != 0 ] && error "mpirun ... mdsrate ... file creation failed, aborting"
+ [ ${PIPESTATUS[0]} != 0 ] && error "mdsrate file creation failed, aborting"
echo "NO Test for stats on a single client."
else
log "===== $0 ### 1 NODE STAT ###"
echo "NO Test for stats on a single client."
else
log "===== $0 ### 1 NODE STAT ###"
- echo "Running stats on 1 node(s)."
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate stats on a single client failed, aborting"
+[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
echo "NO test for stats on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
if [ -n "$NOMULTI" ]; then
echo "NO test for stats on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
- echo "Running stats on ${NUM_CLIENTS} node(s)."
- NUM_THREADS=$(get_node_count ${NODES_TO_USE//,/ })
- mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
+ mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate stats on multiple nodes failed, aborting"
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore
log "===== $0 Test preparation: creating ${NUM_FILES} files."
- echo "Test preparation: creating ${NUM_FILES} files."
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR}
--nfiles ${NUM_FILES} --filefmt 'f%%d'"
COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR}
--nfiles ${NUM_FILES} --filefmt 'f%%d'"
fi
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
fi
mpi_run -np ${NUM_THREADS} -machinefile ${MACHINEFILE} ${COMMAND} 2>&1
- [ ${PIPESTATUS[0]} != 0 ] && error "Error running mdsrate, aborting..."
+ [ ${PIPESTATUS[0]} != 0 ] && error "mdsrate file creation failed, aborting"
echo "NO Test for stats on a single client."
else
log "===== $0 ### 1 NODE STAT ###"
echo "NO Test for stats on a single client."
else
log "===== $0 ### 1 NODE STAT ###"
- echo "Running stats on 1 node(s)."
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
echo "+" ${COMMAND}
mpi_run -np 1 -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate on a single client failed, aborting"
+[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
echo "NO test for stats on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
if [ -n "$NOMULTI" ]; then
echo "NO test for stats on multiple nodes."
else
log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
- echo "Running stats on ${NUM_CLIENTS} node(s)."
echo "+" ${COMMAND}
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
echo "+" ${COMMAND}
mpi_run -np ${NUM_CLIENTS} -machinefile ${MACHINEFILE} ${COMMAND} | tee ${LOG}
if [ ${PIPESTATUS[0]} != 0 ]; then
- [ -f $LOG ] && cat $LOG
- error "mpirun ... mdsrate ... failed, aborting"
+ [ -f $LOG ] && sed -e "s/^/log: /" $LOG
+ error "mdsrate stats on multiple nodes failed, aborting"