3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin
14 ALWAYS_EXCEPT="$SANITY_PCC_EXCEPT "
15 # bug number for skipped test:
17 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
19 ENABLE_PROJECT_QUOTAS=${ENABLE_PROJECT_QUOTAS:-true}
20 HSMTOOL_ARCHIVE_FORMAT=v1
22 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
24 . $LUSTRE/tests/test-framework.sh
26 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
29 MULTIOP=${MULTIOP:-multiop}
30 OPENFILE=${OPENFILE:-openfile}
31 MOUNT_2=${MOUNT_2:-"yes"}
34 # script only handles up to 10 MDTs (because of MDT_PREFIX)
35 [ $MDSCOUNT -gt 9 ] &&
36 error "script cannot handle more than 9 MDTs, please fix" && exit
38 check_and_setup_lustre
40 if [[ "$MDS1_VERSION" -lt $(version_code 2.12.52) ]]; then
41 skip "Need MDS version at least 2.12.52"
44 # $RUNAS_ID may get set incorrectly somewhere else
45 if [[ $UID -eq 0 && $RUNAS_ID -eq 0 ]]; then
46 skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" && exit
48 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
49 if getent group nobody; then
51 elif getent group nogroup; then
54 error "No generic nobody group"
59 # if there is no CLIENT1 defined, some tests can be ran on localhost
60 CLIENT1=${CLIENT1:-$HOSTNAME}
61 # if CLIENT2 doesn't exist then use CLIENT1 instead
62 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
64 # Exception is the test which need two separate nodes
65 CLIENT2=${CLIENT2:-$CLIENT1}
71 local expected_size="$3"
73 size=$(do_facet $client stat "--printf=%s" $fpath)
74 [[ $size == "$expected_size" ]] || error \
75 "expected $fpath size: $expected_size got: $size"
82 local lustre_fpath="$3"
83 local expected_size="$4"
85 check_file_size $client $lpcc_fpath $expected_size
86 check_file_size $client $lustre_fpath $expected_size
93 local expected_data="$3"
96 # if $pid is set, then run command within namespace for that process
97 path_data=$(do_facet $client ${pid:+nsenter -t $pid -U -m} cat $path)
98 [[ "x$path_data" == "x$expected_data" ]] ||
99 error "expected $path: $expected_data, got: $path_data"
105 local lpcc_fpath="$2"
106 local lustre_fpath="$3"
107 local expected_data="$4"
109 check_file_data "$client" "$lpcc_fpath" "$expected_data"
110 check_file_data "$client" "$lustre_fpath" "$expected_data"
116 local lustre_path="$2"
117 local fid=$(path2fid $lustre_path)
119 local seq=$(echo $fid | awk -F ':' '{print $1}')
120 local oid=$(echo $fid | awk -F ':' '{print $2}')
121 local ver=$(echo $fid | awk -F ':' '{print $3}')
123 case "$HSMTOOL_ARCHIVE_FORMAT" in
125 printf "%s/%04x/%04x/%04x/%04x/%04x/%04x/%s" \
126 $hsm_root $((oid & 0xFFFF)) \
127 $((oid >> 16 & 0xFFFF)) \
129 $((seq >> 16 & 0xFFFF)) \
130 $((seq >> 32 & 0xFFFF)) \
131 $((seq >> 48 & 0xFFFF)) $fid
134 printf "%s/%04x/%s" $hsm_root $(((oid ^ seq) & 0xFFFF)) $fid
141 local lustre_path="$1"
142 local expected_state="$2"
143 local facet=${3:-$SINGLEAGT}
145 local state=$(do_facet $facet $myRUNAS $LFS pcc state $lustre_path |
146 awk -F 'type: ' '{print $2}' | awk -F ',' '{print $1}')
148 [[ "x$state" == "x$expected_state" ]] || error \
149 "$lustre_path expected pcc state: $expected_state, but got: $state"
155 # populate MDT device array
158 # cleanup from previous bad setup
161 # for recovery tests, coordinator needs to be started at mount
163 # the lustre conf must be without hsm on (like for sanity.sh)
164 echo "Set HSM on and start"
165 cdt_set_mount_state enabled
166 cdt_check_state enabled
168 echo "Set sanity-hsm HSM policy"
169 cdt_set_sanity_policy
171 # finished requests are quickly removed from list
172 set_hsm_param grace_delay 10
174 cleanup_pcc_mapping() {
175 local facet=${1:-$SINGLEAGT}
177 do_facet $facet $LCTL pcc clear $MOUNT
180 setup_pcc_mapping() {
181 local facet=${1:-$SINGLEAGT}
182 local hsm_root=${hsm_root:-$(hsm_root "$facet")}
185 [ -z "$param" ] && param="projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
186 stack_trap "cleanup_pcc_mapping $facet" EXIT
187 do_facet $facet $LCTL pcc add $MOUNT $hsm_root -p $param
195 do_facet $facet lsof $mntpt || true
196 do_facet $facet $UMOUNT $mntpt
207 do_facet $facet mkdir -p $mntpt || error "mkdir -p $hsm_root failed"
208 stack_trap "do_facet $facet rm -rf $mntpt" EXIT
209 do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
210 stack_trap "do_facet $facet rm -f $file" EXIT
211 do_facet $facet mkfs.ext4 $file ||
212 error "mkfs.ext4 $file failed"
213 do_facet $facet file $file
214 do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt ||
215 error "mount -o loop,usrquota,grpquota $file $mntpt failed"
216 stack_trap "umount_loopdev $facet $mntpt" EXIT
223 local agt_facet=$SINGLEAGT
224 local loopfile="$TMP/$tfile"
225 local mntpt="/mnt/pcc.$tdir"
226 local hsm_root="$mntpt/$tdir"
227 local file=$DIR/$tdir/$tfile
232 $project && enable_project_quota
234 do_facet $SINGLEAGT rm -rf $hsm_root
235 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
236 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
238 is_project_quota_supported || project=false
240 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
242 $project && lfs project -sp $project_id $DIR2/$tdir
244 do_facet $SINGLEAGT "echo -n attach_origin > $file"
246 check_lpcc_state $file "none"
247 do_facet $SINGLEAGT $LFS pcc attach -i \
248 $HSM_ARCHIVE_NUMBER $file ||
249 error "pcc attach $file failed"
252 check_lpcc_state $file "readwrite"
253 # HSM released exists archived status
254 check_hsm_flags $file "0x0000000d"
255 lpcc_path=$(lpcc_fid2path $hsm_root $file)
256 check_lpcc_data $SINGLEAGT $lpcc_path $file "attach_origin"
258 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=7654321 count=1
259 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 7654321
261 do_facet $SINGLEAGT $TRUNCATE $file 1234567 ||
262 error "truncate failed"
263 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1234567
264 check_lpcc_state $file "readwrite"
266 do_facet $SINGLEAGT "echo -n file_data > $file"
267 check_lpcc_state $file "readwrite"
268 # HSM released exists archived status
269 check_hsm_flags $file "0x0000000d"
270 check_lpcc_data $SINGLEAGT $lpcc_path $file "file_data"
272 echo "Restore testing..."
273 if [ $CLIENTCOUNT -lt 2 -o $restore ]; then
274 $LFS hsm_restore $file || error \
275 "failed to restore $file"
276 wait_request_state $(path2fid $file) RESTORE SUCCEED
278 path_data=$(do_node $CLIENT2 cat $file)
279 [[ "x$path_data" == "xfile_data" ]] || error \
280 "expected file_data, got: $path_data"
283 check_lpcc_state $file "none"
284 # HSM exists archived status
285 check_hsm_flags $file "0x00000009"
287 echo -n "new_data" > $file
288 check_lpcc_state $file "none"
289 # HSM exists dirty archived status
290 check_hsm_flags $file "0x0000000b"
291 check_file_data $SINGLEAGT $file "new_data"
293 echo "Attach and detach testing"
295 do_facet $SINGLEAGT "echo -n new_data2 > $file"
297 check_lpcc_state $file "none"
298 do_facet $SINGLEAGT $LFS pcc attach -i \
299 $HSM_ARCHIVE_NUMBER $file ||
300 error "PCC attach $file failed"
302 check_lpcc_state $file "readwrite"
303 # HSM released exists archived status
304 check_hsm_flags $file "0x0000000d"
305 do_facet $SINGLEAGT "echo -n attach_detach > $file"
306 echo "Start to detach the $file"
307 do_facet $SINGLEAGT $LFS pcc detach $file ||
308 error "PCC detach $file failed"
309 wait_request_state $(path2fid $file) REMOVE SUCCEED
311 check_lpcc_state $file "none"
312 # The file is removed from PCC
313 check_hsm_flags $file "0x00000000"
314 check_file_data $SINGLEAGT $file "attach_detach"
318 lpcc_rw_test true false
320 run_test 1a "Test manual lfs pcc attach with manual HSM restore"
323 lpcc_rw_test false false
325 run_test 1b "Test manual lfs pcc attach with restore on remote access"
328 lpcc_rw_test true true
330 run_test 1c "Test automated attach using Project ID with manual HSM restore"
333 lpcc_rw_test false true
335 run_test 1d "Test Project ID with remote access"
338 local file=$DIR/$tdir/$tfile
339 local hsm_root=$(hsm_root)
342 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
343 setup_pcc_mapping $SINGLEAGT \
344 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
345 $LCTL pcc list $MOUNT
346 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
347 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
349 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
350 error "failed to dd write to $file"
351 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
352 $file || error "failed to attach file $file"
353 check_lpcc_state $file "readwrite"
354 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
355 error "failed to dd read from $file"
356 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
357 error "failed to truncate $file"
358 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
359 error "failed to truncate $file"
360 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
361 error "failed to dd write to $file"
362 check_lpcc_state $file "readwrite"
364 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
365 error "failed to detach file $file"
366 check_lpcc_state $file "none"
368 # non-root user is forbidden to access PCC file directly
369 lpcc_path=$(lpcc_fid2path $hsm_root $file)
370 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
371 error "non-root user can touch access PCC file $lpcc_path"
372 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
373 count=1 && error "non-root user can read PCC file $lpcc_path"
374 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
375 count=1 && error "non-root user can write PCC file $lpcc_path"
377 local perm=$(do_facet $SINGLEAGT stat -c %a $lpcc_path)
379 [[ $perm == "0" ]] || error "PCC file permission ($perm) is not zero"
381 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
382 $file || error "failed to attach file $file"
383 check_lpcc_state $file "readwrite"
385 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
386 error "failed to detach file $file"
387 wait_request_state $(path2fid $file) REMOVE SUCCEED
388 check_lpcc_state $file "none"
390 run_test 1e "Test RW-PCC with non-root user"
394 local agt_facet=$SINGLEAGT
395 local loopfile="$TMP/$tfile"
396 local mntpt="/mnt/pcc.$tdir"
397 local hsm_root="$mntpt/$tdir"
398 local file=$DIR/$tdir/$tfile
400 ! is_project_quota_supported &&
401 skip "project quota is not supported"
404 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
405 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
406 setup_pcc_mapping $SINGLEAGT \
407 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0"
409 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
410 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
411 $LFS project -sp $project_id $DIR/$tdir ||
412 error "failed to set project for $DIR/$tdir"
414 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
415 error "failed to dd write to $file"
417 check_lpcc_state $file "readwrite"
418 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
419 error "failed to dd read from $file"
420 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
421 error "failed to truncate $file"
422 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
423 error "failed to truncate $file"
424 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=256 count=1 ||
425 error "failed to dd write from $file"
426 check_lpcc_state $file "readwrite"
428 # non-root user is forbidden to access PCC file directly
429 lpcc_path=$(lpcc_fid2path $hsm_root $file)
430 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
431 error "non-root user can touch access PCC file $lpcc_path"
432 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
433 count=1 && error "non-root user can read PCC file $lpcc_path"
434 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
435 count=1 && error "non-root user can write PCC file $lpcc_path"
437 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
438 error "failed to detach file $file"
439 wait_request_state $(path2fid $file) REMOVE SUCCEED
440 check_lpcc_state $file "none"
442 run_test 1f "Test auto RW-PCC cache with non-root user"
445 local loopfile="$TMP/$tfile"
446 local mntpt="/mnt/pcc.$tdir"
447 local hsm_root="$mntpt/$tdir"
448 local file=$DIR/$tfile
450 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
451 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
454 dd if=/dev/zero of=$file bs=1024 count=1 ||
455 error "failed to dd write to $file"
456 chmod 600 $file || error "chmod 600 $file failed"
457 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
458 error "non-root user can dd write $file"
459 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
460 error "non-root user can dd read $file"
461 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
462 error "failed to attach file $file"
463 check_lpcc_state $file "readwrite"
464 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
465 error "non-root user can dd write to $file"
466 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
467 error "non-root user can dd read $file"
468 chmod 777 $DIR2/$tfile || error "chmod 777 $DIR2/$tfile failed"
469 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
470 error "non-root user cannot write $file with permission (777)"
472 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
473 error "non-root user or non owner can detach $file"
474 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
475 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
476 error "failed to detach file $file"
477 wait_request_state $(path2fid $file) REMOVE SUCCEED
478 check_lpcc_state $file "none"
479 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
480 error "non-root user cannot read to $file with permisson (777)"
482 run_test 1g "General permission test for RW-PCC"
485 # When a process created a LPCC file and holding the open,
486 # another process on the same client should be able to open the file.
490 local agt_facet=$SINGLEAGT
491 local loopfile="$TMP/$tfile"
492 local mntpt="/mnt/pcc.$tdir"
493 local hsm_root="$mntpt/$tdir"
494 local agt_host=$(facet_active_host $SINGLEAGT)
496 ! is_project_quota_supported &&
497 skip "project quota is not supported" && return
500 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
501 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
503 file=$DIR/$tdir/multiop
504 $LFS mkdir -i -1 -c $MDSCOUNT $DIR/$tdir
507 do_facet $SINGLEAGT $LFS project -sp $project_id $DIR/$tdir ||
508 error "failed to set project quota"
509 rmultiop_start $agt_host $file O_c || error "open $file failed"
510 # HSM released exists archived status
511 check_hsm_flags $file "0x0000000d"
512 do_facet $SINGLEAGT "echo -n multiopen_data > $file" ||
513 error "failed to echo multiopen_data to $file"
515 lpcc_path=$(lpcc_fid2path $hsm_root $file)
516 do_facet $SINGLEAGT ls -l $lpcc_path ||
517 error "failed to ls $lpcc_path"
518 check_lpcc_data $SINGLEAGT $lpcc_path $file "multiopen_data"
519 # HSM released exists archived status
520 check_hsm_flags $file "0x0000000d"
522 do_facet $SINGLEAGT $LFS pcc detach $file ||
523 error "failed to detach $file"
524 rmultiop_stop $agt_host || error "close $file failed"
526 run_test 2a "Test multi open when creating"
528 get_remote_client() {
529 current_id=$(do_facet $SINGLEAGT hostname)
530 for client in ${CLIENTS//,/ }
532 r_id=$(do_node $client hostname)
533 if [ $r_id != $current_id ]; then
541 # When a process created a LPCC file and holding the open, another
542 # process on the different client should be able to open the file
543 # and perform IO on the file.
546 local agt_facet=$SINGLEAGT
547 local agt_host=$(facet_active_host $SINGLEAGT)
548 local loopfile="$TMP/$tfile"
549 local mntpt="/mnt/pcc.$tdir"
550 local hsm_root="$mntpt/$tdir"
552 needclients 2 || return 0
554 remote_client=$(get_remote_client)
557 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
558 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
560 file=$DIR/$tdir/multiop
564 do_facet $SINGLEAGT "echo -n file_data > $file"
565 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
566 $file || error "PCC attach $file failed"
567 check_lpcc_state $file "readwrite"
569 rmultiop_start $agt_host $file O_c || error "open $file failed"
571 do_node $remote_client "echo -n multiopen_data > $file"
573 # PCC cached file should be automatically detached
574 check_lpcc_state $file "none"
576 check_file_data $SINGLEAGT $file "multiopen_data"
577 rmultiop_stop $agt_host || error "close $file failed"
578 check_file_data $SINGLEAGT $file "multiopen_data"
580 do_node $remote_client cat $file || error \
581 "cat $file on remote client failed"
582 do_node $remote_client echo -n "multiopen_data" > $file \
583 || error "write $file on remote client failed"
585 run_test 2b "Test multi remote open when creating"
588 local agt_host=$(facet_active_host $SINGLEAGT)
589 local loopfile="$TMP/$tfile"
590 local mntpt="/mnt/pcc.$tdir"
591 local hsm_root="$mntpt/$tdir"
592 local file=$DIR/$tdir/$tfile
593 local file2=$DIR2/$tdir/$tfile
596 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
597 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
602 do_facet $SINGLEAGT "echo -n file_data > $file"
603 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
604 $file || error "PCC attach $file failed"
605 check_lpcc_state $file "readwrite"
607 rmultiop_start $agt_host $file O_c || error "open $file failed"
609 echo -n multiopen_data > $file2
611 # PCC cached file should be automatically detached
612 check_lpcc_state $file "none"
614 check_file_data $SINGLEAGT $file "multiopen_data"
615 rmultiop_stop $agt_host || error "close $file failed"
616 check_file_data $SINGLEAGT $file "multiopen_data"
618 cat $file2 || error "cat $file on mount $MOUNT2 failed"
619 echo -n "multiopen_data" > $file2 ||
620 error "write $file on mount $MOUNT2 failed"
622 run_test 2c "Test multi open on different mount points when creating"
625 local file=$DIR/$tdir/$tfile
626 local file2=$DIR2/$tdir/$tfile
628 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
629 setup_pcc_mapping $SINGLEAGT \
630 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
632 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
633 dd if=/dev/zero of=$file2 bs=1024 count=1 ||
634 error "failed to dd write to $file"
636 echo "Start to RW-PCC attach/detach the file: $file"
637 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
638 error "failed to attach file $file"
639 check_lpcc_state $file "readwrite"
640 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
641 error "failed to detach file $file"
642 check_lpcc_state $file "none"
644 echo "Repeat to RW-PCC attach/detach the same file: $file"
645 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
646 error "failed to attach file $file"
647 check_lpcc_state $file "readwrite"
648 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
649 error "failed to detach file $file"
650 check_lpcc_state $file "none"
652 rm -f $file || error "failed to remove $file"
653 echo "ropcc_data" > $file
655 echo "Start to RO-PCC attach/detach the file: $file"
656 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
657 error "failed to attach file $file"
658 check_lpcc_state $file "readonly"
659 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
660 error "failed to detach file $file"
661 check_lpcc_state $file "none"
663 echo "Repeat to RO-PCC attach/detach the same file: $file"
664 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
665 error "failed to attach file $file"
666 check_lpcc_state $file "readonly"
667 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
668 error "failed to detach file $file"
669 check_lpcc_state $file "none"
671 run_test 3a "Repeat attach/detach operations"
675 local file=$DIR/$tdir/$tfile
677 needclients 3 || return 0
679 # Start all of the copytools and setup PCC
680 for n in $(seq $AGTCOUNT); do
681 copytool setup -f agt$n -a $n -m $MOUNT -h $(hsm_root agt$n)
682 setup_pcc_mapping agt$n "projid={100}\ rwid=$n\ auto_attach=0"
685 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
686 dd if=/dev/zero of=$file bs=1024 count=1 ||
687 error "failed to dd write to $file"
689 echo "Start to RW-PCC attach/detach $file on $agt1_HOST"
690 do_facet agt1 $LFS pcc attach -i 1 $file ||
691 error "failed to attach file $file"
692 check_lpcc_state $file "readwrite" agt1
693 do_facet agt1 $LFS pcc detach -k $file ||
694 error "failed to detach file $file"
695 check_lpcc_state $file "none" agt1
697 echo "Repeat to RW-PCC attach/detach $file on $agt2_HOST"
698 do_facet agt2 $LFS pcc attach -i 2 $file ||
699 error "failed to attach file $file"
700 check_lpcc_state $file "readwrite" agt2
701 do_facet agt2 $LFS pcc detach -k $file ||
702 error "failed to detach file $file"
703 check_lpcc_state $file "none" agt2
705 echo "Try RW-PCC attach on two agents"
706 do_facet agt1 $LFS pcc attach -i 1 $file ||
707 error "failed to attach file $file"
708 check_lpcc_state $file "readwrite" agt1
709 do_facet agt2 $LFS pcc attach -i 2 $file ||
710 error "failed to attach file $file"
711 check_lpcc_state $file "readwrite" agt2
712 # The later attach PCC agent should succeed,
713 # the former agent should be detached automatically.
714 check_lpcc_state $file "none" agt1
715 do_facet agt2 $LFS pcc detach -k $file ||
716 error "failed to detach file $file"
717 check_lpcc_state $file "none" agt2
719 echo "Start to RO-PCC attach/detach $file on $agt1_HOST"
720 do_facet agt1 $LFS pcc attach -r -i 1 $file ||
721 error "failed to attach file $file"
722 check_lpcc_state $file "readonly" agt1
723 do_facet agt1 $LFS pcc detach -k $file ||
724 error "failed to detach file $file"
725 check_lpcc_state $file "none" agt1
727 echo "Repeat to RO-PCC attach/detach $file on $agt2_HOST"
728 do_facet agt2 $LFS pcc attach -r -i 2 $file ||
729 error "failed to attach file $file"
730 check_lpcc_state $file "readonly" agt2
731 do_facet agt2 $LFS pcc detach -k $file ||
732 error "failed to detach file $file"
733 check_lpcc_state $file "none" agt2
735 echo "Try RO-PCC attach on two agents"
736 do_facet agt1 $LFS pcc attach -r -i 1 $file ||
737 error "failed to attach file $file"
738 check_lpcc_state $file "readonly" agt1
739 do_facet agt2 $LFS pcc attach -r -i 2 $file ||
740 error "failed to attach file $file"
741 check_lpcc_state $file "readonly" agt2
742 check_lpcc_state $file "readonly" agt1
743 do_facet agt2 $LFS pcc detach -k $file ||
744 error "failed to detach file $file"
745 check_lpcc_state $file "none" agt2
746 do_facet agt1 $LFS pcc detach -k $file ||
747 error "failed to detach file $file"
748 check_lpcc_state $file "none" agt1
750 run_test 3b "Repeat attach/detach operations on multiple clients"
754 local loopfile="$TMP/$tfile"
755 local mntpt="/mnt/pcc.$tdir"
756 local hsm_root="$mntpt/$tdir"
757 local excepts="-e 6 -e 7 -e 8 -e 9"
759 ! is_project_quota_supported &&
760 skip "project quota is not supported" && return
763 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
764 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
767 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
768 lfs project -sp $project_id $DIR/$tdir ||
769 error "lfs project -sp $project_id $DIR/$tdir failed"
771 # 1. mmap_sanity tst7 failed on the local ext4 filesystem.
772 # It seems that Lustre filesystem does special process for tst 7.
773 # 2. There is a mmap problem for PCC when multiple clients read/write
774 # on a shared mmapped file for mmap_sanity tst 6.
775 # 3. Current CentOS8 kernel does not strictly obey POSIX syntax for
776 # mmap() within the maping but beyond current end of the underlying
777 # files: It does not send SIGBUS signals to the process.
778 # 4. For negative file offset, sanity_mmap also failed on 48 bits
779 # ldiksfs backend due to too large offset: "Value too large for
780 # defined data type".
781 # mmap_sanity tst7/tst8/tst9 all failed on Lustre and local ext4.
782 # Thus, we exclude sanity tst6/tst7/tst8/tst9 from the PCC testing.
783 $LUSTRE/tests/mmap_sanity -d $DIR/$tdir -m $DIR2/$tdir $excepts ||
784 error "mmap_sanity test failed"
787 # Revoke the layout lock, the PCC-cached file will be
788 # detached automatically.
789 do_facet $SINGLEAGT $LCTL \
790 set_param ldlm.namespaces.*mdc*.lru_size=clear
791 rm -rf $DIR/$tdir || error "failed to remove $DIR/$tdir"
793 run_test 4 "Auto cache test for mmap"
796 local file=$DIR/$tfile
797 local loopfile="$TMP/$tfile"
798 local mntpt="/mnt/pcc.$tdir"
799 local hsm_root="$mntpt/$tdir"
801 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
802 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
805 do_facet $SINGLEAGT "echo -n attach_mmap_data > $file" ||
806 error "echo $file failed"
808 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
809 error "failed to attach file $file"
810 check_lpcc_state $file "readwrite"
812 local content=$($MMAP_CAT $file)
814 [[ $content == "attach_mmap_data" ]] ||
815 error "mmap cat data mismatch: $content"
817 $LFS hsm_restore $file || error "failed to restore $file"
818 wait_request_state $(path2fid $file) RESTORE SUCCEED
819 check_lpcc_state $file "none"
821 content=$($MMAP_CAT $file)
822 [[ $content == "attach_mmap_data" ]] ||
823 error "mmap cat data mismatch: $content"
825 run_test 5 "Mmap & cat a RW-PCC cached file"
828 local loopfile="$TMP/$tfile"
829 local mntpt="/mnt/pcc.$tdir"
830 local hsm_root="$mntpt/$tdir"
831 local file=$DIR/$tfile
834 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
835 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
838 echo -n mmap_write_data > $file || error "echo write $file failed"
839 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
840 error "failed to attach file $file"
841 check_lpcc_state $file "readwrite"
843 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
844 error "could not mmap $file"
845 check_lpcc_state $file "readwrite"
846 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
847 # After mmap write via multiop, the first character of each page
849 [[ $content == "nmap_write_data" ]] ||
850 error "mmap write data mismatch: $content"
851 check_lpcc_state $file "readwrite"
853 do_facet $SINGLEAGT $LFS pcc detach $file ||
854 error "failed to detach file $file"
855 wait_request_state $(path2fid $file) REMOVE SUCCEED
857 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
858 [[ $content == "nmap_write_data" ]] ||
859 error "mmap write data mismatch: $content"
861 run_test 6 "Test mmap write on RW-PCC "
864 local loopfile="$TMP/$tfile"
865 local mntpt="/mnt/pcc.$tdir"
866 local hsm_root="$mntpt/$tdir"
867 local file=$DIR/$tfile
870 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
871 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
875 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
876 error "failed to attach file $file"
877 check_lpcc_state $file "readwrite"
878 check_file_data $SINGLEAGT $file "QQQQQ"
879 # define OBD_FAIL_LLITE_PCC_DETACH_MKWRITE 0x1412
880 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1412
881 # HSM released exists archived status
882 check_hsm_flags $file "0x0000000d"
884 # multiop mmap write increase the first character of each page with 1
885 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
886 error "mmap write $file failed"
887 check_lpcc_state $file "none"
888 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
889 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
891 run_test 7a "Fake file detached between fault() and page_mkwrite() for RW-PCC"
894 local loopfile="$TMP/$tfile"
895 local mntpt="/mnt/pcc.$tdir"
896 local hsm_root="$mntpt/$tdir"
897 local file=$DIR/$tfile
901 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
902 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
903 setup_pcc_mapping $SINGLEAGT \
904 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
907 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
908 error "failed to attach file $file"
909 check_lpcc_state $file "readwrite"
910 check_file_data $SINGLEAGT $file "QQQQQ"
911 # define OBD_FAIL_LLITE_PCC_MKWRITE_PAUSE 0x1413
912 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1413 fail_val=20
913 # HSM released exists archived status
914 check_hsm_flags $file "0x0000000d"
916 # multiop mmap write increases the first character of each page with 1
917 do_facet $SINGLEAGT $MULTIOP $file OSMWUc &
921 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
922 error "failed to detach file $file"
924 wait $pid || error "multiop mmap write failed"
925 check_lpcc_state $file "none"
926 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
927 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
929 run_test 7b "Test the race with concurrent mkwrite and detach"
932 local loopfile="$TMP/$tfile"
933 local mntpt="/mnt/pcc.$tdir"
934 local hsm_root="$mntpt/$tdir"
935 local file=$DIR/$tfile
937 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
938 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
942 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
943 error "failed to attach file $file"
944 check_lpcc_state $file "readwrite"
945 check_file_data $SINGLEAGT $file "QQQQQ"
947 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
948 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
949 do_facet $SINGLEAGT "echo -n ENOSPC_write > $file"
950 # Above write will return -ENOSPC failure and retry the IO on normal
951 # IO path. It will restore the HSM released file.
952 check_lpcc_state $file "none"
953 check_file_data $SINGLEAGT $file "ENOSPC_write"
955 run_test 8 "Test fake -ENOSPC tolerance for RW-PCC"
958 local loopfile="$TMP/$tfile"
959 local mntpt="/mnt/pcc.9a"
960 local hsm_root="$mntpt/$tdir"
961 local file=$DIR/$tfile
963 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
965 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
967 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
969 touch $file || error "touch $file failed"
970 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
971 error "fail to attach $file"
972 check_lpcc_state $file "readwrite"
973 # write 60M data, it is larger than the capacity of PCC backend
974 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=60 ||
975 error "fail to dd write $file"
976 check_lpcc_state $file "none"
977 check_file_size $SINGLEAGT $file 62914560
979 run_test 9 "Test -ENOSPC tolerance on loop PCC device for RW-PCC"
981 test_usrgrp_quota() {
982 local loopfile="$TMP/$tfile"
983 local mntpt="/mnt/pcc.$tdir"
984 local hsm_root="$mntpt/$tdir"
985 local state="readwrite"
990 [[ $ug == "g" ]] && id=$RUNAS_GID
991 [[ -z $ro ]] || state="readonly"
992 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
993 do_facet $SINGLEAGT quotacheck -c$ug $mntpt ||
994 error "quotacheck -c$ug $mntpt failed"
995 do_facet $SINGLEAGT quotaon -$ug $mntpt ||
996 error "quotaon -$ug $mntpt failed"
997 do_facet $SINGLEAGT setquota -$ug $id 0 20480 0 0 $mntpt ||
998 error "setquota -$ug $id on $mntpt failed"
999 do_facet $SINGLEAGT repquota -${ug}vs $mntpt
1001 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
1003 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1005 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1007 local file1=$DIR/$tdir/${ug}quotaA
1008 local file2=$DIR/$tdir/${ug}quotaB
1010 dd if=/dev/zero of=$file1 bs=1M count=15 ||
1011 error "dd write $file1 failed"
1012 dd if=/dev/zero of=$file2 bs=1M count=15 ||
1013 error "dd write $file2 failed"
1014 chown $RUNAS_ID:$RUNAS_GID $file1 ||
1015 error "chown $RUNAS_ID:$RUNAS_GID $file1 failed"
1016 chown $RUNAS_ID:$RUNAS_GID $file2 ||
1017 error "chown $RUNAS_ID:$RUNAS_GID $file2 failed"
1018 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $ro \
1019 $file1 || error "attach $file1 failed"
1020 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $ro \
1021 $file2 && error "attach $file2 should fail due to quota limit"
1022 check_lpcc_state $file1 $state
1023 check_lpcc_state $file2 "none"
1025 if [[ -n $ro ]]; then
1026 do_facet $SINGLEAGT $LFS pcc detach $file1 ||
1027 error "detach $file1 failed"
1031 echo "Test -EDQUOT error tolerance for RW-PCC"
1032 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file1 bs=1M count=30 ||
1033 error "dd write $file1 failed"
1034 # -EDQUOT error should be tolerated via fallback to normal Lustre path.
1035 check_lpcc_state $file1 "none"
1039 test_usrgrp_quota "u"
1041 run_test 10a "Test RW-PCC with user quota on loop PCC device"
1044 test_usrgrp_quota "g"
1046 run_test 10b "Test RW-PCC with group quota on loop PCC device"
1049 test_usrgrp_quota "u" "-r"
1051 run_test 10c "Test RO-PCC with user quota on loop PCC device"
1054 test_usrgrp_quota "g" "-r"
1056 run_test 10d "Test RO-PCC with group quota on loop PCC device"
1059 local loopfile="$TMP/$tfile"
1060 local mntpt="/mnt/pcc.$tdir"
1061 local hsm_root="$mntpt/$tdir"
1062 local file=$DIR/$tfile
1066 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1067 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1070 do_facet $SINGLEAGT "echo -n QQQQQ > $file"
1071 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1072 lpcc_dir=$(dirname $lpcc_path)
1073 echo "Lustre file: $file LPCC dir: $lpcc_dir"
1074 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
1075 error "mkdir -p $lpcc_dir failed"
1076 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1077 error "failed to attach $file"
1078 check_lpcc_state $file "readwrite"
1079 check_file_data $SINGLEAGT $file "QQQQQ"
1080 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1081 error "failed to detach $file"
1082 rm $file || error "rm $file failed"
1084 # The parent directory of the PCC file is immutable
1085 do_facet $SINGLEAGT "echo -n immutable_dir > $file"
1086 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1087 lpcc_dir=$(dirname $lpcc_path)
1088 echo "Lustre file: $file LPCC dir: $lpcc_dir"
1089 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
1090 error "mkdir -p $lpcc_dir failed"
1091 do_facet $SINGLEAGT chattr +i $lpcc_dir ||
1092 error "chattr +i $lpcc_dir failed"
1093 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
1094 error "attach $file with immutable directory should be failed"
1095 do_facet $SINGLEAGT chattr -i $lpcc_dir ||
1096 error "chattr -i $lpcc_dir failed"
1097 rm $file || error "rm $file failed"
1099 # The PCC file path is set to a directory
1100 do_facet $SINGLEAGT "echo -n pcc_file_path_is_dir > $file"
1101 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1102 do_facet $SINGLEAGT mkdir -p $lpcc_path ||
1103 error "mkdir -p $lpcc_path failed"
1104 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
1105 error "attach $file should fail as PCC path is a directory"
1106 rm $file || error "rm $file failed"
1108 run_test 11 "Test attach fault injection with simulated PCC file path"
1111 local file=$DIR/$tfile
1112 local hsm_root=$(hsm_root)
1116 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1117 setup_pcc_mapping $SINGLEAGT \
1118 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1120 echo -n race_rw_attach_hsmremove > $file
1121 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1122 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1123 error "attach $file failed"
1124 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1125 error "detach $file failed"
1126 # HSM released exists archived status
1127 check_hsm_flags $file "0x0000000d"
1128 # define OBD_FAIL_LLITE_PCC_ATTACH_PAUSE 0x1414
1129 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1414 fail_val=20
1130 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &
1132 $LFS hsm_state $file
1134 wait_request_state $(path2fid $file) RESTORE SUCCEED
1135 $LFS hsm_remove $file || error "hsm remove $file failed"
1137 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1138 error "RW-PCC cached file '$lpcc_path' should be removed"
1142 run_test 12 "RW-PCC attach races with concurrent HSM remove"
1145 local idstr="${1}id"
1146 local rule="${idstr}={$2}"
1148 local file=$DIR/$tdir/$tfile
1150 setup_pcc_mapping $SINGLEAGT \
1151 "$rule\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1152 $LCTL pcc list $MOUNT
1154 do_facet $SINGLEAGT $LFS mkdir -i 0 $DIR/$tdir
1155 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1157 rm -f $file || error "rm $file failed"
1158 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1159 error "failed to dd write to $file"
1160 check_lpcc_state $file "readwrite"
1161 do_facet $SINGLEAGT $myRUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
1162 error "failed to dd read from $file"
1163 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 256 ||
1164 error "failed to truncate $file"
1165 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 2048 ||
1166 error "failed to truncate $file"
1167 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1168 error "failed to dd write from $file"
1169 check_lpcc_state $file "readwrite"
1171 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1172 error "failed to detach file $file"
1173 wait_request_state $(path2fid $file) REMOVE SUCCEED
1174 check_lpcc_state $file "none"
1180 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1181 test_rule_id "u" "$RUNAS_ID" "runas -u $RUNAS_ID"
1182 test_rule_id "g" "$RUNAS_GID" "runas -u $RUNAS_ID -g $RUNAS_GID"
1184 run_test 13a "Test auto RW-PCC create caching for UID/GID rule"
1189 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1190 setup_pcc_mapping $SINGLEAGT \
1191 "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1192 $LCTL pcc list $MOUNT
1194 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1195 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1197 file=$DIR/$tdir/prefix.h5
1198 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1199 error "failed to dd write to $file"
1200 check_lpcc_state $file "readwrite"
1201 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1202 error "failed to detach file $file"
1203 check_lpcc_state $file "none"
1204 rm $file || error "rm $file failed"
1206 file=$DIR/$tdir/suffix.doc
1207 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1208 error "failed to dd write to $file"
1209 check_lpcc_state $file "readwrite"
1210 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1211 error "failed to detach file $file"
1212 check_lpcc_state $file "none"
1213 rm $file || error "rm $file failed"
1215 file=$DIR/$tdir/MidPADdle
1216 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1217 error "failed to dd write to $file"
1218 check_lpcc_state $file "readwrite"
1219 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1220 error "failed to detach file $file"
1221 check_lpcc_state $file "none"
1222 rm $file || error "rm $file failed"
1224 file=$DIR/$tdir/Midpad
1225 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1226 error "failed to dd write to $file"
1227 check_lpcc_state $file "none"
1228 rm $file || error "rm $file failed"
1230 run_test 13b "Test auto RW-PCC create caching for file name with wildcard"
1235 local loopfile="$TMP/$tfile"
1236 local mntpt="/mnt/pcc.$tdir"
1237 local hsm_root="$mntpt/$tdir"
1239 ! is_project_quota_supported &&
1240 echo "Skip project quota is not supported" && return 0
1242 enable_project_quota
1243 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1244 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1245 setup_pcc_mapping $SINGLEAGT \
1246 "projid={100\ 200}\&fname={*.h5},uid={$RUNAS_ID}\&gid={$RUNAS_GID}\ rwid=$HSM_ARCHIVE_NUMBER"
1247 $LCTL pcc list $MOUNT
1248 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1249 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1251 mkdir -p $DIR/$tdir/proj || error "mkdir $DIR/$tdir/proj failed"
1252 mkdir -p $DIR/$tdir/proj2 || error "mkdir $DIR/$tdir/proj2 failed"
1253 $LFS project -sp 100 $DIR/$tdir/proj ||
1254 error "failed to set project for $DIR/$tdir/proj"
1255 $LFS project -sp 200 $DIR/$tdir/proj2 ||
1256 error "failed to set project for $DIR/$tdir/proj2"
1258 file=$DIR/$tdir/proj/notcache
1259 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1260 error "failed to dd write to $file"
1261 check_lpcc_state $file "none"
1262 rm $file || error "rm $file failed"
1264 file=$DIR/$tdir/proj/autocache.h5
1265 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1266 error "failed to dd write to $file"
1267 check_lpcc_state $file "readwrite"
1268 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1269 error "failed to detach $file"
1270 rm $file || error "rm $file failed"
1272 file=$DIR/$tdir/proj2/notcache
1273 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1274 error "failed to dd write to $file"
1275 check_lpcc_state $file "none"
1276 rm $file || error "rm $file failed"
1278 file=$DIR/$tdir/proj2/autocache.h5
1279 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1280 error "failed to dd write to $file"
1281 check_lpcc_state $file "readwrite"
1282 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1283 error "failed to detach $file"
1284 rm $file || error "rm $file failed"
1286 file=$DIR/$tdir/ugidcache
1287 myRUNAS="runas -u $RUNAS_ID -g $RUNAS_GID"
1288 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1289 error "failed to dd write to $file"
1290 check_lpcc_state $file "readwrite"
1291 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1292 error "failed to detach $file"
1293 rm $file || error "rm $file failed"
1295 run_test 13c "Check auto RW-PCC create caching for UID/GID/ProjID/fname rule"
1298 local file=$DIR/$tdir/$tfile
1300 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1301 setup_pcc_mapping $SINGLEAGT \
1302 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1304 mkdir -p $DIR/$tdir || error "mkdir -p $DIR/$tdir failed"
1305 do_facet $SINGLEAGT "echo -n autodetach_data > $file"
1306 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1307 $file || error "PCC attach $file failed"
1308 check_lpcc_state $file "readwrite"
1310 # Revoke the layout lock, the PCC-cached file will be
1311 # detached automatically.
1312 do_facet $SINGLEAGT $LCTL \
1313 set_param ldlm.namespaces.*mdc*.lru_size=clear
1314 check_file_data $SINGLEAGT $file "autodetach_data"
1315 check_lpcc_state $file "none"
1317 rm $file || error "rm $file failed"
1318 do_facet $SINGLEAGT "echo -n ro_autodetach_data > $file"
1319 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1320 error "PCC attach $file failed"
1321 check_lpcc_state $file "readonly"
1323 # Revoke the layout lock, the PCC-cached file will be
1324 # detached automatically.
1325 do_facet $SINGLEAGT $LCTL \
1326 set_param ldlm.namespaces.*mdc*.lru_size=clear
1327 check_file_data $SINGLEAGT $file "ro_autodetach_data"
1328 check_lpcc_state $file "none"
1330 run_test 14 "Revocation of the layout lock should detach the file automatically"
1333 local loopfile="$TMP/$tfile"
1334 local mntpt="/mnt/pcc.$tdir"
1335 local hsm_root="$mntpt/$tdir"
1336 local file=$DIR/$tdir/$tfile
1338 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1339 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1342 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1343 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
1345 echo "Verify open attach for non-root user"
1346 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1347 error "failed to dd write to $file"
1348 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1349 $file || error "failed to attach file $file"
1350 do_facet $SINGLEAGT $RUNAS $LFS pcc state $file
1351 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1352 # Revoke the layout lock, the PCC-cached file will be
1353 # detached automatically.
1354 do_facet $SINGLEAGT $LCTL \
1355 set_param ldlm.namespaces.*mdc*.lru_size=clear
1356 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1357 # Detach the file but keep the cache , as the file layout generation
1358 # is not changed, so the file is still valid cached in PCC, and can
1359 # be reused from PCC cache directly.
1360 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
1361 error "PCC detach $file failed"
1362 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1363 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
1364 error "PCC detach $file failed"
1365 rm $file || error "rm $file failed"
1367 echo "Verify auto attach at open for RW-PCC"
1368 do_facet $SINGLEAGT "echo -n autoattach_data > $file"
1369 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1370 $file || error "RW-PCC attach $file failed"
1371 check_lpcc_state $file "readwrite"
1373 # Revoke the layout lock, the PCC-cached file will be
1374 # detached automatically.
1375 do_facet $SINGLEAGT $LCTL \
1376 set_param ldlm.namespaces.*mdc*.lru_size=clear
1377 check_file_data $SINGLEAGT $file "autoattach_data"
1378 check_lpcc_state $file "readwrite"
1380 # Detach the file with -k option, as the file layout generation
1381 # is not changed, so the file is still valid cached in PCC,
1382 # and can be reused from PCC cache directly.
1383 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1384 error "RW-PCC detach $file failed"
1385 check_lpcc_state $file "readwrite"
1386 # HSM released exists archived status
1387 check_hsm_flags $file "0x0000000d"
1388 check_file_data $SINGLEAGT $file "autoattach_data"
1390 # HSM restore the PCC cached file, the layout generation
1391 # was changed, so the file can not be auto attached.
1392 $LFS hsm_restore $file || error "failed to restore $file"
1393 wait_request_state $(path2fid $file) RESTORE SUCCEED
1394 check_lpcc_state $file "none"
1395 # HSM exists archived status
1396 check_hsm_flags $file "0x00000009"
1398 echo "Verify auto attach at open for RO-PCC"
1399 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1400 error "RO-PCC attach $file failed"
1401 check_lpcc_state $file "readonly"
1403 # Revoke the layout lock, the PCC-cached file will be
1404 # detached automatically.
1405 do_facet $SINGLEAGT $LCTL \
1406 set_param ldlm.namespaces.*mdc*.lru_size=clear
1407 check_file_data $SINGLEAGT $file "autoattach_data"
1408 check_lpcc_state $file "readonly"
1410 # Detach the file with "-k" option, as the file layout generation
1411 # is not changed, so the file is still valid cached in PCC,
1412 # and can be reused from PCC cache directly.
1413 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1414 error "RO-PCC detach $file failed"
1415 check_lpcc_state $file "readonly"
1416 check_file_data $SINGLEAGT $file "autoattach_data"
1417 do_facet $SINGLEAGT $LFS pcc detach $file ||
1418 error "RO-PCC detach $file failed"
1420 run_test 15 "Test auto attach at open when file is still valid cached"
1423 local loopfile="$TMP/$tfile"
1424 local mntpt="/mnt/pcc.$tdir"
1425 local hsm_root="$mntpt/$tdir"
1426 local file=$DIR/$tfile
1429 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1430 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1433 echo "Test detach for RW-PCC"
1434 do_facet $SINGLEAGT "echo -n detach_data > $file"
1435 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1436 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1437 $file || error "RW-PCC attach $file failed"
1438 check_lpcc_state $file "readwrite"
1439 # HSM released exists archived status
1440 check_hsm_flags $file "0x0000000d"
1442 echo "Test for reusing valid PCC cache"
1443 # Valid PCC cache can be reused
1444 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1445 error "PCC detach $file failed"
1446 check_lpcc_state $file "readwrite"
1447 # HSM released exists archived status
1448 check_hsm_flags $file "0x0000000d"
1450 echo "Test for the default detach"
1451 # Permanent detach by default, it will remove the PCC copy
1452 do_facet $SINGLEAGT $LFS pcc detach $file ||
1453 error "RW-PCC detach $file failed"
1454 wait_request_state $(path2fid $file) REMOVE SUCCEED
1455 check_lpcc_state $file "none"
1456 # File is removed from PCC backend
1457 check_hsm_flags $file "0x00000000"
1458 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1459 error "RW-PCC cached file '$lpcc_path' should be removed"
1461 echo "Test detach for RO-PCC"
1462 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1463 error "RO-PCC attach $file failed"
1464 check_lpcc_state $file "readonly"
1466 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1467 error "RO-PCC detach $file failed"
1468 check_lpcc_state $file "readonly"
1470 do_facet $SINGLEAGT $LFS pcc detach $file ||
1471 error "RO-PCC detach $file failed"
1472 check_lpcc_state $file "none"
1473 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1474 error "RO-PCC cached file '$lpcc_path' should be removed"
1478 run_test 16 "Test detach with different options"
1481 local agt_host=$(facet_active_host $SINGLEAGT)
1482 local loopfile="$TMP/$tfile"
1483 local mntpt="/mnt/pcc.$tdir"
1484 local hsm_root="$mntpt/$tdir"
1485 local file=$DIR/$tfile
1487 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1488 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1489 setup_pcc_mapping $SINGLEAGT \
1490 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0"
1492 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1494 do_facet $SINGLEAGT "echo -n layout_refresh_data > $file"
1495 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1496 error "PCC attach $file failed"
1497 check_lpcc_state $file "readwrite"
1499 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1500 error "PCC detach $file failed"
1501 check_lpcc_state $file "none"
1503 # Truncate should attach the file into PCC automatically
1504 # as the PCC copy is still valid.
1505 echo "Verify auto attach during IO for truncate"
1506 do_facet $SINGLEAGT $TRUNCATE $file 4 || error "truncate $file failed"
1507 check_lpcc_state $file "readwrite"
1509 echo "Verify auto attach during IO for read/write"
1510 rmultiop_start $agt_host $file O_r || error "open $file failed"
1513 # Revoke the layout lock, the PCC-cached file will be
1514 # detached automatically.
1515 do_facet $SINGLEAGT $LCTL \
1516 set_param ldlm.namespaces.*mdc*.lru_size=clear
1518 check_lpcc_state $file "none"
1519 rmultiop_stop $agt_host || error "close $file failed"
1521 check_lpcc_state $file "readwrite"
1523 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1524 error "PCC detach $file failed"
1525 check_lpcc_state $file "none"
1527 run_test 17 "Test auto attach for layout refresh"
1530 local agt_host=$(facet_active_host $SINGLEAGT)
1531 local loopfile="$TMP/$tfile"
1532 local mntpt="/mnt/pcc.$tdir"
1533 local hsm_root="$mntpt/$tdir"
1534 local file=$DIR/$tfile
1538 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1539 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1540 setup_pcc_mapping $SINGLEAGT \
1541 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
1543 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1544 do_facet $SINGLEAGT dd if=/dev/urandom of=$file bs=1M count=4 ||
1545 error "failed to write $file"
1546 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1547 error "failed to attach $file"
1548 do_facet $SINGLEAGT $LFS pcc state $file
1549 check_lpcc_state $file "readwrite"
1550 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1551 error "failed to detach $file"
1552 do_facet $SINGLEAGT $LFS pcc state $file
1553 $CHECKSTAT -s 4194304 $file
1554 dd if=/dev/zero of=$DIR2/$tfile seek=1k bs=1k count=1 ||
1555 error "failed to write $DIR2/$tfile"
1556 oldmd5=$(md5sum $DIR2/$tfile | awk '{print $1}')
1557 $CHECKSTAT -s 1049600 $DIR2/$tfile || error "$DIR2/$tfile size wrong"
1559 local lpcc_path=$(lpcc_fid2path $hsm_root $file)
1561 do_facet $SINGLEAGT $LFS pcc state $file
1562 check_file_size $SINGLEAGT $lpcc_path 4194304
1563 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1564 error "failed to attach $file"
1565 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1049600
1566 newmd5=$(do_facet $SINGLEAGT md5sum $file | awk '{print $1}')
1567 [ "$oldmd5" == "$newmd5" ] || error "md5sum differ: $oldmd5 != $newmd5"
1568 do_facet $SINGLEAGT $LFS pcc detach $file ||
1569 error "failed to detach $file"
1571 run_test 18 "Verify size correctness after re-attach the file"
1574 local agt_host=$(facet_active_host $SINGLEAGT)
1575 local loopfile="$TMP/$tfile"
1576 local mntpt="/mnt/pcc.$tdir"
1577 local hsm_root="$mntpt/$tdir"
1578 local file=$DIR/$tfile
1580 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1581 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1582 setup_pcc_mapping $SINGLEAGT \
1583 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1585 do_facet $SINGLEAGT "echo -n QQQQQ > $file" || error "echo $file failed"
1586 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1587 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1588 error "Failed to attach $file"
1589 check_lpcc_state $file "readwrite"
1590 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1591 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1592 error "Failed to detach $file"
1593 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1594 error "Failed to attach $file"
1595 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1596 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1597 error "Failed to detach $file"
1599 run_test 19 "Verify the file re-attach works as expected"
1602 local agt_host=$(facet_active_host $SINGLEAGT)
1603 local loopfile="$TMP/$tfile"
1604 local mntpt="/mnt/pcc.$tdir"
1605 local hsm_root="$mntpt/$tdir"
1606 local file=$DIR/$tfile
1608 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1609 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1610 setup_pcc_mapping $SINGLEAGT \
1611 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
1613 do_facet $SINGLEAGT "echo -n QQQQQ > $file" ||
1614 error "echo $file failed"
1615 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1616 error "Failed to attach $file"
1617 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1618 check_lpcc_state $file "readwrite"
1619 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1620 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1621 check_lpcc_state $file "readwrite"
1622 do_facet $SINGLEAGT $LFS pcc detach $file ||
1623 error "Failed to detach $file"
1625 run_test 20 "Auto attach works after the inode was once evicted from cache"
1628 local loopfile="$TMP/$tfile"
1629 local mntpt="/mnt/pcc.$tdir"
1630 local hsm_root="$mntpt/$tdir"
1631 local file=$DIR/$tfile
1633 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1634 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1635 setup_pcc_mapping $SINGLEAGT \
1636 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1638 do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1639 echo "Plain layout info before PCC-RO attach '$file':"
1640 $LFS getstripe -v $file
1641 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1642 error "RW-PCC attach $file failed"
1643 check_lpcc_state $file "readonly"
1644 echo -e "\nFLR layout info after PCC-RO attach '$file':"
1645 $LFS getstripe -v $file
1646 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1647 error "failed to detach file $file"
1648 check_lpcc_state $file "none"
1649 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1650 $LFS getstripe -v $file
1652 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1653 error "failed to attach file $file"
1654 check_lpcc_state $file "readonly"
1655 echo -e "\nFLR layout info after RO-PCC attach $file again:"
1656 $LFS getstripe -v $file
1657 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1658 error "failed to detach file $file"
1659 check_lpcc_state $file "none"
1660 echo -e "\nFLR layout info after RO-PCC detach '$file' again:"
1661 $LFS getstripe -v $file
1663 run_test 21a "PCC-RO storing as a plain HSM mirror component for plain layout"
1666 local loopfile="$TMP/$tfile"
1667 local mntpt="/mnt/pcc.$tdir"
1668 local hsm_root="$mntpt/$tdir"
1669 local file=$DIR/$tfile
1671 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1672 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1673 setup_pcc_mapping $SINGLEAGT \
1674 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1676 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
1677 error "create mirrored file $file failed"
1678 #do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1679 echo "FLR layout before PCC-RO attach '$file':"
1680 $LFS getstripe -v $file
1681 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1682 error "failed to attach file $file"
1683 check_lpcc_state $file "readonly"
1684 echo -e "\nFLR layout after PCC-RO attach '$file':"
1685 $LFS getstripe -v $file
1686 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1687 error "failed to detach file $file"
1688 check_lpcc_state $file "none"
1689 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1690 $LFS getstripe -v $file
1692 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1693 error "failed to attach file $file"
1694 check_lpcc_state $file "readonly"
1695 echo -e "\nFLR layout after PCC-RO attach '$file' again:"
1696 $LFS getstripe -v $file
1697 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1698 error "failed to detach file $file"
1699 check_lpcc_state $file "none"
1700 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1701 $LFS getstripe -v $file
1703 run_test 21b "PCC-RO stroing as a plain HSM mirror component for FLR layouts"
1706 local loopfile="$TMP/$tfile"
1707 local mntpt="/mnt/pcc.$tdir"
1708 local hsm_root="$mntpt/$tdir"
1709 local file=$DIR/$tfile
1710 local file2=$DIR2/$tfile
1713 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1714 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1715 setup_pcc_mapping $SINGLEAGT \
1716 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1718 do_facet $SINGLEAGT "echo -n pccro_hsm_release > $file"
1719 fid=$(path2fid $file)
1720 $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file ||
1721 error "Archive $file failed"
1722 wait_request_state $fid ARCHIVE SUCCEED
1723 $LFS hsm_state $file
1725 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1726 error "RO-PCC attach $file failed"
1727 # HSM exists archived status
1728 check_hsm_flags $file "0x00000009"
1729 check_lpcc_state $file "readonly"
1730 check_file_data $SINGLEAGT $file "pccro_hsm_release"
1732 $LFS hsm_release $file || error "HSM released $file failed"
1733 $LFS getstripe $file
1734 $LFS hsm_state $file
1735 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1736 error "failed to detach $file"
1737 check_lpcc_state $file "none"
1738 unlink $file || error "unlink $file failed"
1740 run_test 21c "Verify HSM release works storing PCC-RO as HSM mirror component"
1743 local loopfile="$TMP/$tfile"
1744 local mntpt="/mnt/pcc.$tdir"
1745 local hsm_root="$mntpt/$tdir"
1746 local file=$DIR/$tfile
1748 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1749 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1752 echo "pccro_init_data" > $file
1753 $LFS getstripe $file
1754 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1755 error "failed to PCC-RO attach file $file"
1756 check_lpcc_state $file "readonly"
1757 echo "PCC-RO attach '$file':"
1758 $LFS getstripe -v $file
1760 echo "Write invalidated PCC-RO cache:"
1761 echo -n "write_mod_data" > $file
1762 check_lpcc_state $file "none"
1763 $LFS getstripe -v $file
1764 check_file_data $SINGLEAGT $file "write_mod_data"
1766 run_test 21d "Write should invalidate PCC-RO caching"
1769 local loopfile="$TMP/$tfile"
1770 local mntpt="/mnt/pcc.$tdir"
1771 local hsm_root="$mntpt/$tdir"
1772 local file=$DIR/$tfile
1774 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1775 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1778 echo "pccro_init_data" > $file
1779 $LFS getstripe $file
1780 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1781 error "failed to PCC-RO attach file $file"
1782 check_lpcc_state $file "readonly"
1783 echo "PCC-RO attach '$file':"
1784 $LFS getstripe -v $file
1786 echo "Trucate invalidate PCC-RO file '$file':"
1787 $TRUNCATE $file 256 || error "failed to truncate $file"
1788 $LFS getstripe -v $file
1789 check_lpcc_state $file "none"
1790 check_file_size $SINGLEAGT $file 256
1792 run_test 21e "Truncate should invalidate PCC-RO caching"
1795 local loopfile="$TMP/$tfile"
1796 local mntpt="/mnt/pcc.$tdir"
1797 local hsm_root="$mntpt/$tdir"
1798 local file=$DIR/$tfile
1800 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1801 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1804 echo "pccro_mmap_data" > $file
1805 $LFS getstripe $file
1806 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1807 error "failed to PCC-RO attach file $file"
1808 check_lpcc_state $file "readonly"
1809 echo "PCC-RO attach '$file':"
1810 $LFS getstripe -v $file
1812 echo "Mmap write invalidate PCC-RO caching:"
1813 # Mmap write will invalidate the RO-PCC cache
1814 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
1815 error "mmap write $file failed"
1816 check_lpcc_state $file "none"
1817 $LFS getstripe -v $file
1818 # After mmap-write by MULTIOP, the first character of the content
1819 # will be increased with 1.
1820 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
1821 [[ $content == "qccro_mmap_data" ]] ||
1822 error "mmap_cat data mismatch: $content"
1824 run_test 21f "mmap write should invalidate PCC-RO caching"
1827 local loopfile="$TMP/$tfile"
1828 local mntpt="/mnt/pcc.$tdir"
1829 local hsm_root="$mntpt/$tdir"
1830 local file=$DIR/$tfile
1832 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1833 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1834 setup_pcc_mapping $SINGLEAGT \
1835 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1837 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
1838 error "create mirrored file '$file' failed"
1839 do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1840 echo "FLR layout before PCC-RO attach '$file':"
1841 $LFS getstripe -v $file
1842 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1843 error "failed to PCC-RO attach '$file'"
1844 echo "FLR layout after PCC-RO attach '$file':"
1845 $LFS getstripe -v $file
1846 echo "Layout after Write invalidate '$file':"
1847 echo -n pccro_write_invalidate_mirror > $file
1848 $LFS getstripe -v $file
1850 run_test 21g "PCC-RO for file under FLR write pending state"
1853 local loopfile="$TMP/$tfile"
1854 local mntpt="/mnt/pcc.$tdir"
1855 local hsm_root="$mntpt/$tdir"
1856 local file=$DIR/$tfile
1858 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1859 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1860 setup_pcc_mapping $SINGLEAGT \
1861 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1863 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
1864 error "create mirrored file $file failed"
1865 #do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1866 echo "FLR layout before PCC-RO attach '$file':"
1867 $LFS getstripe -v $file
1868 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1869 error "failed to attach file $file"
1870 check_lpcc_state $file "readonly"
1871 echo -e "\nFLR layout after PCC-RO attach '$file':"
1872 $LFS getstripe -v $file
1874 $LFS mirror extend -N -S 8M -c -1 $file ||
1875 error "mirror extend $file failed"
1876 echo -e "\nFLR layout after extend a mirror:"
1877 $LFS getstripe -v $file
1878 $LFS pcc state $file
1879 check_lpcc_state $file "none"
1881 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1882 error "failed to attach file $file"
1883 check_lpcc_state $file "readonly"
1884 echo -e "\nFLR layout after PCC-RO attach '$file' again:"
1885 $LFS getstripe -v $file
1886 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1887 error "failed to detach file $file"
1888 check_lpcc_state $file "none"
1890 run_test 21h "Extend mirror once file was PCC-RO cached"
1893 local loopfile="$TMP/$tfile"
1894 local mntpt="/mnt/pcc.$tdir"
1895 local hsm_root="$mntpt/$tdir"
1896 local file=$DIR/$tfile
1897 local file2=$DIR2/$tfile
1900 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1901 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1902 setup_pcc_mapping $SINGLEAGT \
1903 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1905 do_facet $SINGLEAGT "echo -n hsm_release_pcc_file > $file"
1906 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1907 error "RW-PCC attach $file failed"
1908 check_lpcc_state $file "readwrite"
1909 # HSM released exists archived status
1910 check_hsm_flags $file "0x0000000d"
1912 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1913 error "RW-PCC detach $file failed"
1914 check_lpcc_state $file "none"
1915 # HSM released exists archived status
1916 check_hsm_flags $file "0x0000000d"
1918 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1919 error "failed to PCC-RO attach $file"
1921 $LFS hsm_state $file
1922 $LFS hsm_release $file || error "HSM released $file failed"
1923 echo "Layout after HSM release $file:"
1924 $LFS getstripe -v $file
1925 echo "PCC state $file:"
1926 $LFS pcc state $file
1927 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1928 error "RO-PCC attach $file failed"
1929 echo "Layout after PCC-RO attach $file again:"
1930 $LFS getstripe -v $file
1932 $LFS pcc state $file
1934 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1935 error "RW-PCC detach $file failed"
1936 check_lpcc_state $file "none"
1938 run_test 21i "HSM release increase layout gen, should invalidate PCC-RO cache"
1941 local loopfile="$TMP/$tfile"
1942 local mntpt="/mnt/pcc.$tdir"
1943 local hsm_root="$mntpt/$tdir"
1944 local file=$DIR/$tfile
1945 local file2=$DIR2/$tfile
1948 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1949 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1950 setup_pcc_mapping $SINGLEAGT \
1951 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1953 do_facet $SINGLEAGT "echo -n roattach_data > $file"
1955 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1956 error "RW-PCC attach $file failed"
1957 check_lpcc_state $file "readwrite"
1958 # HSM released exists archived status
1959 check_hsm_flags $file "0x0000000d"
1961 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1962 error "RW-PCC detach $file failed"
1963 check_lpcc_state $file "none"
1964 # HSM released exists archived status
1965 check_hsm_flags $file "0x0000000d"
1967 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1968 error "RO-PCC attach $file failed"
1969 echo "Layout after PCC-RO attach $file:"
1970 $LFS getstripe -v $file
1971 # HSM exists archived status
1972 check_hsm_flags $file "0x00000009"
1973 check_lpcc_state $file "readonly"
1974 check_file_data $SINGLEAGT $file "roattach_data"
1976 $LFS hsm_release $file || error "HSM released $file failed"
1977 echo "Layout after HSM release $file:"
1978 $LFS getstripe -v $file
1979 # HSM released exists archived status
1980 check_hsm_flags $file "0x0000000d"
1981 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1982 error "RO-PCC attach $file failed"
1983 echo "Layout after PCC-RO attach $file again:"
1984 $LFS getstripe -v $file
1985 check_lpcc_state $file "readonly"
1986 check_file_data $SINGLEAGT $file "roattach_data"
1987 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1988 error "failed to detach $file"
1989 echo "Layout after PCC-RO detach $file:"
1990 $LFS getstripe -v $file
1991 rm -f $file2 || error "rm -f $file failed"
1992 do_facet $SINGLEAGT "echo -n roattach_data2 > $file"
1993 fid=$(path2fid $file)
1994 $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file ||
1995 error "Archive $file failed"
1996 wait_request_state $fid ARCHIVE SUCCEED
1997 $LFS hsm_release $file || error "HSM released $file failed"
1998 # HSM released exists archived status
1999 check_hsm_flags $file "0x0000000d"
2000 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
2001 error "RO-PCC attach $file failed"
2002 check_lpcc_state $file "readonly"
2003 check_file_data $SINGLEAGT $file "roattach_data2"
2004 do_facet $SINGLEAGT $LFS pcc detach $file ||
2005 error "RO-PCC detach $file failed"
2007 run_test 22 "Test RO-PCC attach for the HSM released file"
2010 local loopfile="$TMP/$tfile"
2011 local mntpt="/mnt/pcc.$tdir"
2012 local hsm_root="$mntpt/$tdir"
2013 local file=$DIR/$tfile
2016 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2017 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2020 echo "ropcc_data" > $file
2021 lpcc_path=$(lpcc_fid2path $hsm_root $file)
2023 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2024 error "failed to RO-PCC attach file $file"
2025 check_lpcc_state $file "readonly"
2026 check_lpcc_data $SINGLEAGT $lpcc_path $file "ropcc_data"
2028 local content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2030 [[ $content == "ropcc_data" ]] ||
2031 error "mmap_cat data mismatch: $content"
2032 check_lpcc_state $file "readonly"
2034 echo -n "write_mod_data" > $file
2035 echo "Write should invalidate the RO-PCC cache:"
2036 $LFS getstripe -v $file
2037 check_lpcc_state $file "none"
2038 check_file_data $SINGLEAGT $file "write_mod_data"
2040 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2041 error "failed to RO-PCC attach file $file"
2042 check_lpcc_state $file "readonly"
2043 echo "PCC-RO attach '$file' again:"
2044 $LFS getstripe -v $file
2046 echo "Truncate invalidate the RO-PCC cache:"
2047 $TRUNCATE $file 256 || error "failed to truncate $file"
2048 $LFS getstripe -v $file
2049 echo "Finish trucate operation"
2050 check_lpcc_state $file "none"
2051 check_file_size $SINGLEAGT $file 256
2053 echo "Mmap write invalidates RO-PCC caching"
2054 echo -n mmap_write_data > $file || error "echo write $file failed"
2055 $LFS getstripe -v $file
2056 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2057 error "failed to RO-PCC attach file $file"
2058 check_lpcc_state $file "readonly"
2059 echo "PCC-RO attach '$file' again:"
2060 $LFS getstripe -v $file
2061 echo "Mmap write $file via multiop"
2062 # Mmap write will invalidate the RO-PCC cache
2063 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
2064 error "mmap write $file failed"
2065 check_lpcc_state $file "none"
2066 $LFS getstripe -v $file
2067 # After mmap-write by MULTIOP, the first character of the content
2069 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2070 [[ $content == "nmap_write_data" ]] ||
2071 error "mmap_cat data mismatch: $content"
2073 run_test 23 "Test write/truncate/mmap-write invalidating RO-PCC caching"
2076 local loopfile="$TMP/$tfile"
2077 local mntpt="/mnt/pcc.$tdir"
2078 local hsm_root="$mntpt/$tdir"
2079 local file=$DIR/$tdir/$tfile
2082 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2083 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2084 setup_pcc_mapping $SINGLEAGT \
2085 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2086 $LCTL pcc list $MOUNT
2088 chmod 777 $DIR/$tdir
2090 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
2091 error "failed to dd write to $file"
2092 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER \
2093 $file || error "failed to attach file $file"
2094 check_lpcc_state $file "readonly"
2095 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2096 error "failed to dd read from $file"
2097 check_lpcc_state $file "readonly"
2099 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
2100 error "failed to detach file $file"
2101 check_lpcc_state $file "none"
2103 # non-root user is forbidden to access PCC file directly
2104 lpcc_path=$(lpcc_fid2path $hsm_root $file)
2105 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
2106 error "non-root user can touch access PCC file $lpcc_path"
2107 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
2108 count=1 && error "non-root user can read PCC file $lpcc_path"
2109 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
2110 count=1 && error "non-root user can write PCC file $lpcc_path"
2112 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER \
2113 $file || error "failed to attach file $file"
2114 check_lpcc_state $file "readonly"
2116 # Test RO-PCC detach as non-root user
2117 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
2118 error "failed to detach file $file"
2119 check_lpcc_state $file "none"
2120 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
2121 error "RO-PCC cached file '$lpcc_path' should be removed"
2125 run_test 24a "Test RO-PCC with non-root user"
2128 local loopfile="$TMP/$tfile"
2129 local mntpt="/mnt/pcc.$tdir"
2130 local hsm_root="$mntpt/$tdir"
2131 local file=$DIR/$tdir/$tfile
2133 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2134 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2137 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2138 dd if=/dev/zero of=$file bs=1024 count=1 ||
2139 error "failed to dd write $file"
2140 chmod 600 $file || error "chmod 600 $file failed"
2141 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
2142 error "non-root user can dd write $file"
2143 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
2144 error "non-root user can dd read $file"
2145 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2146 error "failed to attach file $file"
2147 check_lpcc_state $file "readonly"
2148 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
2149 error "non-root user can dd write $file"
2150 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
2151 error "non-root user can dd read $file"
2152 chmod 777 $file || error "chmod 777 $file failed"
2153 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2154 error "non-root user cannot read $file with permission (777)"
2155 check_lpcc_state $file "readonly"
2157 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
2158 error "non-root user or non owner can detach $file"
2159 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
2160 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
2161 error "failed to detach file $file"
2162 check_lpcc_state $file "none"
2163 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2164 error "non-root user cannot read $file with permission (777)"
2166 run_test 24b "General permission test for RO-PCC"
2169 local loopfile="$TMP/$tfile"
2170 local mntpt="/mnt/pcc.$tdir"
2171 local hsm_root="$mntpt/$tdir"
2172 local file=$DIR/$tdir/$tfile
2175 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2176 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2179 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2181 echo "ro_fake_mmap_cat_err" > $file
2182 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2183 error "failed to attach RO-PCC file $file"
2184 check_lpcc_state $file "readonly"
2185 check_file_data $SINGLEAGT $file "ro_fake_mmap_cat_err"
2187 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
2188 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
2189 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2190 [[ $content == "ro_fake_mmap_cat_err" ]] ||
2191 error "failed to fall back to Lustre I/O path for mmap-read"
2192 # Above mmap read will return VM_FAULT_SIGBUS failure and
2193 # retry the IO on normal IO path.
2194 check_lpcc_state $file "readonly"
2195 check_file_data $SINGLEAGT $file "ro_fake_mmap_cat_err"
2197 do_facet $SINGLEAGT $LFS pcc detach $file ||
2198 error "failed to detach RO-PCC file $file"
2199 check_lpcc_state $file "none"
2201 do_facet $SINGLEAGT $LCTL set_param fail_loc=0
2202 echo "ro_fake_cat_err" > $file
2203 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2204 error "failed to attach RO-PCC file $file"
2205 check_lpcc_state $file "readonly"
2206 check_file_data $SINGLEAGT $file "ro_fake_cat_err"
2208 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
2209 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
2210 # Fake read I/O will return -EIO failure and
2211 # retry the IO on normal IO path.
2212 check_file_data $SINGLEAGT $file "ro_fake_cat_err"
2213 check_lpcc_state $file "readonly"
2215 do_facet $SINGLEAGT $LFS pcc detach $file ||
2216 error "failed to detach RO-PCC file $file"
2217 check_lpcc_state $file "none"
2219 run_test 25 "Tolerate fake read failure for RO-PCC"
2222 local agt_host=$(facet_active_host $SINGLEAGT)
2223 local loopfile="$TMP/$tfile"
2224 local mntpt="/mnt/pcc.$tdir"
2225 local hsm_root="$mntpt/$tdir"
2226 local file=$DIR/$tfile
2228 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2229 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2230 setup_pcc_mapping $SINGLEAGT \
2231 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2233 echo -n attach_keep_open > $file
2234 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2235 error "attach $file failed"
2236 check_lpcc_state $file "readonly"
2237 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2238 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2239 error "detach $file failed"
2240 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2241 error "attach $file failed"
2242 check_lpcc_state $file "readonly"
2243 check_file_data $SINGLEAGT $file "attach_keep_open"
2244 check_lpcc_state $file "readonly"
2245 do_facet $SINGLEAGT $LFS pcc detach $file ||
2246 error "detach $file failed"
2247 rmultiop_stop $agt_host || error "multiop $file close failed"
2249 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2250 error "attach $file failed"
2251 check_lpcc_state $file "readonly"
2252 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2253 do_facet $SINGLEAGT $LFS pcc detach $file ||
2254 error "detach $file failed"
2255 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2256 error "attach $file failed"
2257 check_lpcc_state $file "readonly"
2258 check_file_data $SINGLEAGT $file "attach_keep_open"
2259 check_lpcc_state $file "readonly"
2260 do_facet $SINGLEAGT $LFS pcc detach $file ||
2261 error "detach $file failed"
2262 rmultiop_stop $agt_host || error "multiop $file close failed"
2264 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2265 error "attach $file failed"
2266 check_lpcc_state $file "readonly"
2267 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2268 do_facet $SINGLEAGT $LFS pcc detach $file ||
2269 error "detach $file failed"
2270 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2271 error "attach $file failed"
2272 check_lpcc_state $file "readonly"
2273 check_file_data $SINGLEAGT $file "attach_keep_open"
2274 check_lpcc_state $file "readonly"
2275 rmultiop_stop $agt_host || error "multiop $file close failed"
2276 do_facet $SINGLEAGT $LFS pcc detach $file ||
2277 error "detach $file failed"
2279 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2280 error "attach $file failed"
2281 check_lpcc_state $file "readwrite"
2282 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2283 do_facet $SINGLEAGT $LFS pcc detach $file ||
2284 error "detach $file failed"
2285 wait_request_state $(path2fid $file) REMOVE SUCCEED
2286 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2287 error "attach $file failed"
2288 check_lpcc_state $file "readonly"
2289 check_file_data $SINGLEAGT $file "attach_keep_open"
2290 check_lpcc_state $file "readonly"
2291 rmultiop_stop $agt_host || error "multiop $file close failed"
2292 check_lpcc_state $file "readonly"
2293 do_facet $SINGLEAGT $LFS pcc detach $file ||
2294 error "detach $file failed"
2296 rm $file || error "rm $file failed"
2297 echo -n attach_keep_open > $file
2298 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2299 error "attach $file failed"
2300 check_lpcc_state $file "readwrite"
2301 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2302 do_facet $SINGLEAGT $LFS pcc detach $file ||
2303 error "detach $file failed"
2304 wait_request_state $(path2fid $file) REMOVE SUCCEED
2305 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2306 error "attach $file failed"
2307 check_lpcc_state $file "readonly"
2308 check_file_data $SINGLEAGT $file "attach_keep_open"
2309 check_lpcc_state $file "readonly"
2310 do_facet $SINGLEAGT $LFS pcc detach $file ||
2311 error "detach $file failed"
2312 rmultiop_stop $agt_host || error "multiop $file close failed"
2313 check_lpcc_state $file "none"
2315 run_test 26 "Repeat the attach/detach when the file has multiple openers"
2318 local agt_host=$(facet_active_host $SINGLEAGT)
2319 local loopfile="$TMP/$tfile"
2320 local mntpt="/mnt/pcc.$tdir"
2321 local hsm_root="$mntpt/$tdir"
2322 local file=$DIR/$tfile
2324 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2325 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2326 setup_pcc_mapping $SINGLEAGT \
2327 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=1"
2329 echo -n auto_attach_multi_open > $file
2330 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2331 error "attach $file failed"
2332 check_lpcc_state $file "readwrite"
2333 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2334 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2335 error "detach $file failed"
2336 check_lpcc_state $file "readwrite"
2337 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2338 check_lpcc_state $file "readwrite"
2339 do_facet $SINGLEAGT $LFS pcc detach $file ||
2340 error "detach $file failed"
2341 wait_request_state $(path2fid $file) REMOVE SUCCEED
2342 check_lpcc_state $file "none"
2343 rmultiop_stop $agt_host || error "multiop $file close failed"
2345 rm $file || error "rm $file failed"
2346 echo -n auto_attach_multi_open > $file
2347 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2348 error "attach $file failed"
2349 check_lpcc_state $file "readwrite"
2350 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2351 do_facet $SINGLEAGT $LCTL \
2352 set_param ldlm.namespaces.*mdc*.lru_size=clear
2353 check_lpcc_state $file "readwrite"
2354 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2355 check_lpcc_state $file "readwrite"
2356 do_facet $SINGLEAGT $LFS pcc detach $file ||
2357 error "detach $file failed"
2358 wait_request_state $(path2fid $file) REMOVE SUCCEED
2359 check_lpcc_state $file "none"
2360 rmultiop_stop $agt_host || error "multiop $file close failed"
2362 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2363 error "attach $file failed"
2364 check_lpcc_state $file "readonly"
2365 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2366 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2367 error "detach $file failed"
2368 check_lpcc_state $file "readonly"
2369 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2370 check_lpcc_state $file "readonly"
2371 do_facet $SINGLEAGT $LFS pcc detach $file ||
2372 error "detach $file failed"
2373 check_lpcc_state $file "none"
2374 rmultiop_stop $agt_host || error "multiop $file close failed"
2376 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2377 error "attach $file failed"
2378 check_lpcc_state $file "readonly"
2379 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2380 do_facet $SINGLEAGT $LCTL \
2381 set_param ldlm.namespaces.*mdc*.lru_size=clear
2382 check_lpcc_state $file "readonly"
2383 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2384 check_lpcc_state $file "readonly"
2385 do_facet $SINGLEAGT $LFS pcc detach $file ||
2386 error "detach $file failed"
2387 check_lpcc_state $file "none"
2388 rmultiop_stop $agt_host || error "multiop $file close failed"
2390 run_test 27 "Auto attach at open when the file has multiple openers"
2393 local agt_host=$(facet_active_host $SINGLEAGT)
2394 local loopfile="$TMP/$tfile"
2395 local mntpt="/mnt/pcc.$tdir"
2396 local hsm_root="$mntpt/$tdir"
2397 local file=$DIR/$tfile
2398 local file2=$DIR2/$tfile
2401 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2402 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2403 setup_pcc_mapping $SINGLEAGT \
2404 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2406 echo -n rw_attach_hasopen_fail > $file
2407 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2408 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
2409 error "attach $file should fail"
2410 rmultiop_stop $agt_host || error "multiop $file close failed"
2411 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2412 error "attach $file should fail"
2413 check_lpcc_state $file "readwrite"
2414 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2415 error "detach $file failed"
2416 check_lpcc_state $file "none"
2418 multiop_bg_pause $file2 O_c || error "multiop $file2 failed"
2420 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
2421 error "attach $file should fail"
2422 kill -USR1 $multipid
2423 wait $multipid || error "multiop $file2 close failed"
2424 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
2425 error "attach $file should fail"
2426 check_lpcc_state $file "readwrite"
2427 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2428 error "detach $file failed"
2429 check_lpcc_state $file "none"
2431 run_test 28 "RW-PCC attach should fail when the file has cluster-wide openers"
2433 #test 101: containers and PCC
2434 #LU-15170: Test mount namespaces with PCC
2435 #This tests the cases where the PCC mount is not present in the container by
2436 #creating a mount namespace without the PCC mount in it (this is probably the
2437 #standard config for most containers)
2439 local loopfile="$TMP/$tfile"
2440 local mntpt="/mnt/pcc.$tdir"
2441 local hsm_root="$mntpt/$tdir"
2442 local file=$DIR/$tdir/$tfile
2444 # Some kernels such as RHEL7 default to 0 user namespaces
2445 local maxuserns=$(do_facet $SINGLEAGT cat /proc/sys/user/max_user_namespaces)
2446 do_facet $SINGLEAGT "echo 10 > /proc/sys/user/max_user_namespaces"
2447 stack_trap "do_facet $SINGLEAGT 'echo $maxuserns > /proc/sys/user/max_user_namespaces'"
2449 echo "creating user namespace for $RUNAS_ID"
2450 # Create a mount and user namespace with this command, and leave the
2451 # process running so we can do the rest of our steps
2452 local start=$SECONDS
2453 local PID=$(do_facet $SINGLEAGT \
2454 "$RUNAS unshare -Um sleep 600 &>/dev/null & echo \\\$!")
2455 local elapsed=$((SECONDS - start))
2458 do_facet $SINGLEAGT ps auxww | grep sleep
2459 echo "Created NS: child (sleep) pid=$PID in $elapsed seconds"
2460 [[ -n "$PID" ]] || error "remote sleep start failed"
2461 stack_trap "do_facet $SINGLEAGT kill -9 $PID" EXIT
2462 (( elapsed < 300 )) || error "remote sleep took $elapsed sec to start"
2464 # Map 'RUNAS' to root in the namespace, so it has rights to do whatever
2465 # This is handled by '-r' in unshare in newer versions
2466 do_facet $SINGLEAGT $RUNAS newuidmap $PID 0 $RUNAS_ID 1 ||
2467 error "could not map uid $RUNAS_ID to root in namespace"
2468 do_facet $SINGLEAGT $RUNAS newgidmap $PID 0 $RUNAS_GID 1 ||
2469 error "could not map gid $RUNAS_GID to root in namespace"
2471 # Create PCC after creating namespace; namespace will not have PCC
2473 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2475 # Create a temp file inside the PCC mount to verify mount namespace
2476 do_facet $SINGLEAGT touch $mntpt/$tfile.tmp
2477 stack_trap "do_facet $SINGLEAGT rm -f $mntpt/$tfile.tmp" EXIT
2478 echo "Check for temp file in PCC mount"
2479 do_facet $SINGLEAGT test -f $mntpt/$tfile.tmp ||
2480 error "Should see $mntpt/$tfile.tmp"
2481 echo "Check for temp file in PCC mount from inside namespace"
2482 do_facet $SINGLEAGT nsenter -t $PID -U -m test -f $mntpt/$tfile.tmp &&
2483 error "Should not see $mntpt/$tfile.tmp from namespace"
2484 rm -f $mntpt/$tfile.tmp
2487 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2488 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
2490 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2491 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
2493 echo "Verify open attach from inside mount namespace"
2494 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
2495 error "failed to dd write to $file"
2496 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach \
2497 -i $HSM_ARCHIVE_NUMBER $file || error "cannot attach $file"
2498 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc state $file
2500 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
2501 # Revoke the layout lock, the PCC-cached file will be
2502 # detached automatically.
2503 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
2504 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
2505 # Detach the file but keep the cache, as the file layout generation
2506 # is not changed, so the file is still valid cached in PCC, and can
2507 # be reused from PCC cache directly.
2508 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach -k $file ||
2509 error "PCC detach $file failed"
2510 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
2511 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach $file ||
2512 error "PCC detach $file failed"
2513 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
2514 error "failed to dd write to $file"
2515 rm -f $file || error "rm $file failed"
2517 echo "Verify auto attach at open from inside NS for RW-PCC"
2518 # nsenter has strange behavior with echo, which means we have to place
2519 # this in a script so we can use sh, otherwise it doesn't execute echo
2521 # NB: using /bin/echo instead of the shell built in does not help
2522 echo "echo -n autoattach_data > $file" > $DIR/$tdir/$tfile.shell
2523 # File is owned by root, make it accessible to RUNAS user
2524 chmod a+rw $DIR/$tdir/$tfile.shell
2525 stack_trap 'rm -f $DIR/$tdir/$tfile.shell' EXIT
2526 do_facet $SINGLEAGT nsenter -t $PID -U -m "bash $DIR/$tdir/$tfile.shell"
2527 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
2528 $file || error "RW-PCC attach $file failed"
2529 check_lpcc_state $file "readwrite"
2531 # Revoke the layout lock, the PCC-cached file will be
2532 # detached automatically.
2533 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
2534 check_file_data $SINGLEAGT $file "autoattach_data" $PID
2535 check_lpcc_state $file "readwrite"
2537 # Detach the file with -k option, as the file layout generation
2538 # is not changed, so the file is still valid cached in PCC,
2539 # and can be reused from PCC cache directly.
2540 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2541 error "RW-PCC detach $file failed"
2542 check_lpcc_state $file "readwrite"
2543 # HSM released exists archived status
2544 check_hsm_flags $file "0x0000000d"
2545 check_file_data $SINGLEAGT $file "autoattach_data" $PID
2547 # HSM restore the PCC cached file, the layout generation
2548 # was changed, so the file can not be auto attached.
2549 $LFS hsm_restore $file || error "failed to restore $file"
2550 wait_request_state $(path2fid $file) RESTORE SUCCEED
2551 check_lpcc_state $file "none"
2552 # HSM exists archived status
2553 check_hsm_flags $file "0x00000009"
2555 run_test 101a "Test auto attach in mount namespace (simulated container)"
2558 grep -q io_uring_setup /proc/kallsyms ||
2559 skip "Client OS does not support io_uring I/O engine"
2560 io_uring_probe || skip "kernel does not support io_uring fully"
2562 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2563 skip "Server does not support PCC-RO"
2565 which fio || skip_env "no fio installed"
2566 fio --enghelp | grep -q io_uring ||
2567 skip_env "fio does not support io_uring I/O engine"
2569 local loopfile="$TMP/$tfile"
2570 local mntpt="/mnt/pcc.$tdir"
2571 local hsm_root="$mntpt/$tdir"
2572 local file=$DIR/$tfile
2574 setup_loopdev client $loopfile $mntpt 60
2575 mkdir $hsm_root || error "mkdir $hsm_root failed"
2576 setup_pcc_mapping client \
2577 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
2578 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0
2580 local ioengine="io_uring"
2584 do_facet $SINGLEAGT fio --name=seqwrite --ioengine=$ioengine \
2585 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
2586 --iodepth=64 --size=$size --filename=$file --rw=write ||
2587 error "fio seqwrite $file failed"
2589 # Read the file will trigger the buffered read from Lustre OSTs and
2590 # write to PCC copy as @pcc_async_threshold is set with 0.
2591 do_facet $SINGLEAGT fio --name=seqread --ioengine=$ioengine \
2592 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
2593 --iodepth=64 --size=$size --filename=$file --rw=read ||
2594 error "fio seqread $file failed"
2596 run_test 102 "PCC-RO should not hange for io_uring I/O engine"
2598 complete_test $SECONDS
2599 check_and_cleanup_lustre