3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin
14 ALWAYS_EXCEPT="$SANITY_PCC_EXCEPT "
15 # bug number for skipped test:
17 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
19 ENABLE_PROJECT_QUOTAS=${ENABLE_PROJECT_QUOTAS:-true}
20 HSMTOOL_ARCHIVE_FORMAT=v1
22 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
24 . $LUSTRE/tests/test-framework.sh
26 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
29 MULTIOP=${MULTIOP:-multiop}
30 OPENFILE=${OPENFILE:-openfile}
31 MOUNT_2=${MOUNT_2:-"yes"}
34 # script only handles up to 10 MDTs (because of MDT_PREFIX)
35 [ $MDSCOUNT -gt 9 ] &&
36 error "script cannot handle more than 9 MDTs, please fix" && exit
38 check_and_setup_lustre
40 if [[ "$MDS1_VERSION" -lt $(version_code 2.12.52) ]]; then
41 skip "Need MDS version at least 2.12.52"
44 # $RUNAS_ID may get set incorrectly somewhere else
45 if [[ $UID -eq 0 && $RUNAS_ID -eq 0 ]]; then
46 skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" && exit
48 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
49 if getent group nobody; then
51 elif getent group nogroup; then
54 error "No generic nobody group"
59 # if there is no CLIENT1 defined, some tests can be ran on localhost
60 CLIENT1=${CLIENT1:-$HOSTNAME}
61 # if CLIENT2 doesn't exist then use CLIENT1 instead
62 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
64 # Exception is the test which need two separate nodes
65 CLIENT2=${CLIENT2:-$CLIENT1}
71 local expected_size="$3"
73 size=$(do_facet $client stat "--printf=%s" $fpath)
74 [[ $size == "$expected_size" ]] || error \
75 "expected $fpath size: $expected_size got: $size"
82 local lustre_fpath="$3"
83 local expected_size="$4"
85 check_file_size $client $lpcc_fpath $expected_size
86 check_file_size $client $lustre_fpath $expected_size
93 local expected_data="$3"
96 # if $pid is set, then run command within namespace for that process
97 path_data=$(do_facet $client ${pid:+nsenter -t $pid -U -m} cat $path)
98 [[ "x$path_data" == "x$expected_data" ]] ||
99 error "expected $path: $expected_data, got: $path_data"
105 local lpcc_fpath="$2"
106 local lustre_fpath="$3"
107 local expected_data="$4"
109 check_file_data "$client" "$lpcc_fpath" "$expected_data"
110 check_file_data "$client" "$lustre_fpath" "$expected_data"
116 local lustre_path="$2"
117 local fid=$(path2fid $lustre_path)
119 local seq=$(echo $fid | awk -F ':' '{print $1}')
120 local oid=$(echo $fid | awk -F ':' '{print $2}')
121 local ver=$(echo $fid | awk -F ':' '{print $3}')
123 case "$HSMTOOL_ARCHIVE_FORMAT" in
125 printf "%s/%04x/%04x/%04x/%04x/%04x/%04x/%s" \
126 $hsm_root $((oid & 0xFFFF)) \
127 $((oid >> 16 & 0xFFFF)) \
129 $((seq >> 16 & 0xFFFF)) \
130 $((seq >> 32 & 0xFFFF)) \
131 $((seq >> 48 & 0xFFFF)) $fid
134 printf "%s/%04x/%s" $hsm_root $(((oid ^ seq) & 0xFFFF)) $fid
141 local lustre_path="$1"
142 local expected_state="$2"
143 local facet=${3:-$SINGLEAGT}
145 local state=$(do_facet $facet $myRUNAS $LFS pcc state $lustre_path |
146 awk -F 'type: ' '{print $2}' | awk -F ',' '{print $1}')
148 [[ "x$state" == "x$expected_state" ]] || error \
149 "$lustre_path expected pcc state: $expected_state, but got: $state"
155 # populate MDT device array
158 # cleanup from previous bad setup
161 # for recovery tests, coordinator needs to be started at mount
163 # the lustre conf must be without hsm on (like for sanity.sh)
164 echo "Set HSM on and start"
165 cdt_set_mount_state enabled
166 cdt_check_state enabled
168 echo "Set sanity-hsm HSM policy"
169 cdt_set_sanity_policy
171 # finished requests are quickly removed from list
172 set_hsm_param grace_delay 10
174 cleanup_pcc_mapping() {
175 local facet=${1:-$SINGLEAGT}
177 do_facet $facet $LCTL pcc clear $MOUNT
180 setup_pcc_mapping() {
181 local facet=${1:-$SINGLEAGT}
182 local hsm_root=${hsm_root:-$(hsm_root "$facet")}
185 [ -z "$param" ] && param="projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
186 stack_trap "cleanup_pcc_mapping $facet" EXIT
187 do_facet $facet $LCTL pcc add $MOUNT $hsm_root -p $param
196 do_facet $facet mkdir -p $mntpt || error "mkdir -p $hsm_root failed"
197 stack_trap "do_facet $facet rm -rf $mntpt" EXIT
198 do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
199 stack_trap "do_facet $facet rm -f $file" EXIT
200 do_facet $facet mkfs.ext4 $file ||
201 error "mkfs.ext4 $file failed"
202 do_facet $facet file $file
203 do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt ||
204 error "mount -o loop,usrquota,grpquota $file $mntpt failed"
205 stack_trap "do_facet $facet $UMOUNT $mntpt" EXIT
212 local agt_facet=$SINGLEAGT
213 local loopfile="$TMP/$tfile"
214 local mntpt="/mnt/pcc.$tdir"
215 local hsm_root="$mntpt/$tdir"
216 local file=$DIR/$tdir/$tfile
221 $project && enable_project_quota
223 do_facet $SINGLEAGT rm -rf $hsm_root
224 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
225 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
227 is_project_quota_supported || project=false
229 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
231 $project && lfs project -sp $project_id $DIR/$tdir
233 do_facet $SINGLEAGT "echo -n attach_origin > $file"
235 check_lpcc_state $file "none"
236 do_facet $SINGLEAGT $LFS pcc attach -i \
237 $HSM_ARCHIVE_NUMBER $file ||
238 error "pcc attach $file failed"
241 check_lpcc_state $file "readwrite"
242 # HSM released exists archived status
243 check_hsm_flags $file "0x0000000d"
244 lpcc_path=$(lpcc_fid2path $hsm_root $file)
245 check_lpcc_data $SINGLEAGT $lpcc_path $file "attach_origin"
247 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=7654321 count=1
248 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 7654321
250 do_facet $SINGLEAGT $TRUNCATE $file 1234567 ||
251 error "truncate failed"
252 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1234567
253 check_lpcc_state $file "readwrite"
255 do_facet $SINGLEAGT "echo -n file_data > $file"
256 check_lpcc_state $file "readwrite"
257 # HSM released exists archived status
258 check_hsm_flags $file "0x0000000d"
259 check_lpcc_data $SINGLEAGT $lpcc_path $file "file_data"
261 echo "Restore testing..."
262 if [ $CLIENTCOUNT -lt 2 -o $restore ]; then
263 $LFS hsm_restore $file || error \
264 "failed to restore $file"
265 wait_request_state $(path2fid $file) RESTORE SUCCEED
267 path_data=$(do_node $CLIENT2 cat $file)
268 [[ "x$path_data" == "xfile_data" ]] || error \
269 "expected file_data, got: $path_data"
272 check_lpcc_state $file "none"
273 # HSM exists archived status
274 check_hsm_flags $file "0x00000009"
276 echo -n "new_data" > $file
277 check_lpcc_state $file "none"
278 # HSM exists dirty archived status
279 check_hsm_flags $file "0x0000000b"
280 check_file_data $SINGLEAGT $file "new_data"
282 echo "Attach and detach testing"
284 do_facet $SINGLEAGT "echo -n new_data2 > $file"
286 check_lpcc_state $file "none"
287 do_facet $SINGLEAGT $LFS pcc attach -i \
288 $HSM_ARCHIVE_NUMBER $file ||
289 error "PCC attach $file failed"
291 check_lpcc_state $file "readwrite"
292 # HSM released exists archived status
293 check_hsm_flags $file "0x0000000d"
294 do_facet $SINGLEAGT "echo -n attach_detach > $file"
295 echo "Start to detach the $file"
296 do_facet $SINGLEAGT $LFS pcc detach $file ||
297 error "PCC detach $file failed"
298 wait_request_state $(path2fid $file) REMOVE SUCCEED
300 check_lpcc_state $file "none"
301 # The file is removed from PCC
302 check_hsm_flags $file "0x00000000"
303 check_file_data $SINGLEAGT $file "attach_detach"
307 lpcc_rw_test true false
309 run_test 1a "Test manual lfs pcc attach with manual HSM restore"
312 lpcc_rw_test false false
314 run_test 1b "Test manual lfs pcc attach with restore on remote access"
317 lpcc_rw_test true true
319 run_test 1c "Test automated attach using Project ID with manual HSM restore"
322 lpcc_rw_test false true
324 run_test 1d "Test Project ID with remote access"
327 local file=$DIR/$tdir/$tfile
328 local hsm_root=$(hsm_root)
331 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
332 setup_pcc_mapping $SINGLEAGT \
333 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
334 $LCTL pcc list $MOUNT
335 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
336 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
338 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
339 error "failed to dd write to $file"
340 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
341 $file || error "failed to attach file $file"
342 check_lpcc_state $file "readwrite"
343 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
344 error "failed to dd read from $file"
345 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
346 error "failed to truncate $file"
347 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
348 error "failed to truncate $file"
349 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
350 error "failed to dd write to $file"
351 check_lpcc_state $file "readwrite"
353 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
354 error "failed to detach file $file"
355 check_lpcc_state $file "none"
357 # non-root user is forbidden to access PCC file directly
358 lpcc_path=$(lpcc_fid2path $hsm_root $file)
359 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
360 error "non-root user can touch access PCC file $lpcc_path"
361 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
362 count=1 && error "non-root user can read PCC file $lpcc_path"
363 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
364 count=1 && error "non-root user can write PCC file $lpcc_path"
366 local perm=$(do_facet $SINGLEAGT stat -c %a $lpcc_path)
368 [[ $perm == "0" ]] || error "PCC file permission ($perm) is not zero"
370 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
371 $file || error "failed to attach file $file"
372 check_lpcc_state $file "readwrite"
374 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
375 error "failed to detach file $file"
376 check_lpcc_state $file "none"
377 wait_request_state $(path2fid $file) REMOVE SUCCEED
379 run_test 1e "Test RW-PCC with non-root user"
383 local agt_facet=$SINGLEAGT
384 local loopfile="$TMP/$tfile"
385 local mntpt="/mnt/pcc.$tdir"
386 local hsm_root="$mntpt/$tdir"
387 local file=$DIR/$tdir/$tfile
389 ! is_project_quota_supported &&
390 skip "project quota is not supported"
393 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
394 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
395 setup_pcc_mapping $SINGLEAGT \
396 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0"
398 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
399 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
400 $LFS project -sp $project_id $DIR/$tdir ||
401 error "failed to set project for $DIR/$tdir"
403 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
404 error "failed to dd write to $file"
406 check_lpcc_state $file "readwrite"
407 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
408 error "failed to dd read from $file"
409 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
410 error "failed to truncate $file"
411 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
412 error "failed to truncate $file"
413 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=256 count=1 ||
414 error "failed to dd write from $file"
415 check_lpcc_state $file "readwrite"
417 # non-root user is forbidden to access PCC file directly
418 lpcc_path=$(lpcc_fid2path $hsm_root $file)
419 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
420 error "non-root user can touch access PCC file $lpcc_path"
421 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
422 count=1 && error "non-root user can read PCC file $lpcc_path"
423 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
424 count=1 && error "non-root user can write PCC file $lpcc_path"
426 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
427 error "failed to detach file $file"
428 wait_request_state $(path2fid $file) REMOVE SUCCEED
429 check_lpcc_state $file "none"
431 run_test 1f "Test auto RW-PCC cache with non-root user"
434 local loopfile="$TMP/$tfile"
435 local mntpt="/mnt/pcc.$tdir"
436 local hsm_root="$mntpt/$tdir"
437 local file=$DIR/$tfile
439 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
440 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
443 dd if=/dev/zero of=$file bs=1024 count=1 ||
444 error "failed to dd write to $file"
445 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
446 error "non-root user can dd write to $file"
447 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
448 error "failed to attach file $file"
449 check_lpcc_state $file "readwrite"
450 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
451 error "non-root user can dd write to $file"
452 chmod 777 $file || error "chmod 777 $file failed"
453 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
454 error "non-root user cannot write $file with permission (777)"
456 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
457 error "non-root user or non owner can detach $file"
458 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
459 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
460 error "failed to detach file $file"
461 check_lpcc_state $file "none"
462 wait_request_state $(path2fid $file) REMOVE SUCCEED
463 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
464 error "non-root user cannot read to $file with permisson (777)"
466 run_test 1g "General permission test for RW-PCC"
469 # When a process created a LPCC file and holding the open,
470 # another process on the same client should be able to open the file.
474 local agt_facet=$SINGLEAGT
475 local loopfile="$TMP/$tfile"
476 local mntpt="/mnt/pcc.$tdir"
477 local hsm_root="$mntpt/$tdir"
478 local agt_host=$(facet_active_host $SINGLEAGT)
480 ! is_project_quota_supported &&
481 skip "project quota is not supported" && return
484 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
485 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
487 file=$DIR/$tdir/multiop
488 $LFS mkdir -i -1 -c $MDSCOUNT $DIR/$tdir
491 do_facet $SINGLEAGT $LFS project -sp $project_id $DIR/$tdir ||
492 error "failed to set project quota"
493 rmultiop_start $agt_host $file O_c || error "open $file failed"
494 # HSM released exists archived status
495 check_hsm_flags $file "0x0000000d"
496 do_facet $SINGLEAGT "echo -n multiopen_data > $file" ||
497 error "failed to echo multiopen_data to $file"
499 lpcc_path=$(lpcc_fid2path $hsm_root $file)
500 do_facet $SINGLEAGT ls -l $lpcc_path ||
501 error "failed to ls $lpcc_path"
502 check_lpcc_data $SINGLEAGT $lpcc_path $file "multiopen_data"
503 # HSM released exists archived status
504 check_hsm_flags $file "0x0000000d"
506 do_facet $SINGLEAGT $LFS pcc detach $file ||
507 error "failed to detach $file"
508 rmultiop_stop $agt_host || error "close $file failed"
510 run_test 2a "Test multi open when creating"
512 get_remote_client() {
513 current_id=$(do_facet $SINGLEAGT hostname)
514 for client in ${CLIENTS//,/ }
516 r_id=$(do_node $client hostname)
517 if [ $r_id != $current_id ]; then
525 # When a process created a LPCC file and holding the open, another
526 # process on the different client should be able to open the file
527 # and perform IO on the file.
530 local agt_facet=$SINGLEAGT
531 local agt_host=$(facet_active_host $SINGLEAGT)
532 local loopfile="$TMP/$tfile"
533 local mntpt="/mnt/pcc.$tdir"
534 local hsm_root="$mntpt/$tdir"
536 needclients 2 || return 0
538 remote_client=$(get_remote_client)
541 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
542 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
544 file=$DIR/$tdir/multiop
548 do_facet $SINGLEAGT "echo -n file_data > $file"
549 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
550 $file || error "PCC attach $file failed"
551 check_lpcc_state $file "readwrite"
553 rmultiop_start $agt_host $file O_c || error "open $file failed"
555 do_node $remote_client "echo -n multiopen_data > $file"
557 # PCC cached file should be automatically detached
558 check_lpcc_state $file "none"
560 check_file_data $SINGLEAGT $file "multiopen_data"
561 rmultiop_stop $agt_host || error "close $file failed"
562 check_file_data $SINGLEAGT $file "multiopen_data"
564 do_node $remote_client cat $file || error \
565 "cat $file on remote client failed"
566 do_node $remote_client echo -n "multiopen_data" > $file \
567 || error "write $file on remote client failed"
569 run_test 2b "Test multi remote open when creating"
572 local agt_host=$(facet_active_host $SINGLEAGT)
573 local loopfile="$TMP/$tfile"
574 local mntpt="/mnt/pcc.$tdir"
575 local hsm_root="$mntpt/$tdir"
576 local file=$DIR/$tdir/$tfile
577 local file2=$DIR2/$tdir/$tfile
580 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
581 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
586 do_facet $SINGLEAGT "echo -n file_data > $file"
587 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
588 $file || error "PCC attach $file failed"
589 check_lpcc_state $file "readwrite"
591 rmultiop_start $agt_host $file O_c || error "open $file failed"
593 echo -n multiopen_data > $file2
595 # PCC cached file should be automatically detached
596 check_lpcc_state $file "none"
598 check_file_data $SINGLEAGT $file "multiopen_data"
599 rmultiop_stop $agt_host || error "close $file failed"
600 check_file_data $SINGLEAGT $file "multiopen_data"
602 cat $file2 || error "cat $file on mount $MOUNT2 failed"
603 echo -n "multiopen_data" > $file2 ||
604 error "write $file on mount $MOUNT2 failed"
606 run_test 2c "Test multi open on different mount points when creating"
609 local file=$DIR/$tdir/$tfile
611 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
612 setup_pcc_mapping $SINGLEAGT \
613 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
615 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
616 dd if=/dev/zero of=$file bs=1024 count=1 ||
617 error "failed to dd write to $file"
619 echo "Start to attach/detach the file: $file"
620 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
621 error "failed to attach file $file"
622 check_lpcc_state $file "readwrite"
623 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
624 error "failed to detach file $file"
625 check_lpcc_state $file "none"
627 echo "Repeat to attach/detach the same file: $file"
628 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
629 error "failed to attach file $file"
630 check_lpcc_state $file "readwrite"
631 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
632 error "failed to detach file $file"
633 check_lpcc_state $file "none"
635 run_test 3a "Repeat attach/detach operations"
639 local file=$DIR/$tdir/$tfile
641 needclients 3 || return 0
643 # Start all of the copytools and setup PCC
644 for n in $(seq $AGTCOUNT); do
645 copytool setup -f agt$n -a $n -m $MOUNT
646 setup_pcc_mapping agt$n "projid={100}\ rwid=$n\ auto_attach=0"
649 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
650 dd if=/dev/zero of=$file bs=1024 count=1 ||
651 error "failed to dd write to $file"
653 echo "Start to attach/detach $file on $agt1_HOST"
654 do_facet agt1 $LFS pcc attach -i 1 $file ||
655 error "failed to attach file $file"
656 check_lpcc_state $file "readwrite" agt1
657 do_facet agt1 $LFS pcc detach -k $file ||
658 error "failed to detach file $file"
659 check_lpcc_state $file "none" agt1
661 echo "Repeat to attach/detach $file on $agt2_HOST"
662 do_facet agt2 $LFS pcc attach -i 2 $file ||
663 error "failed to attach file $file"
664 check_lpcc_state $file "readwrite" agt2
665 do_facet agt2 $LFS pcc detach -k $file ||
666 error "failed to detach file $file"
667 check_lpcc_state $file "none" agt2
669 echo "Try attach on two agents"
670 do_facet agt1 $LFS pcc attach -i 1 $file ||
671 error "failed to attach file $file"
672 check_lpcc_state $file "readwrite" agt1
673 do_facet agt2 $LFS pcc attach -i 2 $file ||
674 error "failed to attach file $file"
675 check_lpcc_state $file "readwrite" agt2
676 # The later attach PCC agent should succeed,
677 # the former agent should be detached automatically.
678 check_lpcc_state $file "none" agt1
679 do_facet agt2 $LFS pcc detach -k $file ||
680 error "failed to detach file $file"
681 check_lpcc_state $file "none" agt2
683 run_test 3b "Repeat attach/detach operations on multiple clients"
687 local loopfile="$TMP/$tfile"
688 local mntpt="/mnt/pcc.$tdir"
689 local hsm_root="$mntpt/$tdir"
690 local excepts="-e 6 -e 7 -e 8 -e 9"
692 ! is_project_quota_supported &&
693 skip "project quota is not supported" && return
696 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
697 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
700 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
701 lfs project -sp $project_id $DIR/$tdir ||
702 error "lfs project -sp $project_id $DIR/$tdir failed"
704 # 1. mmap_sanity tst7 failed on the local ext4 filesystem.
705 # It seems that Lustre filesystem does special process for tst 7.
706 # 2. There is a mmap problem for PCC when multiple clients read/write
707 # on a shared mmapped file for mmap_sanity tst 6.
708 # 3. Current CentOS8 kernel does not strictly obey POSIX syntax for
709 # mmap() within the maping but beyond current end of the underlying
710 # files: It does not send SIGBUS signals to the process.
711 # 4. For negative file offset, sanity_mmap also failed on 48 bits
712 # ldiksfs backend due to too large offset: "Value too large for
713 # defined data type".
714 # mmap_sanity tst7/tst8/tst9 all failed on Lustre and local ext4.
715 # Thus, we exclude sanity tst6/tst7/tst8/tst9 from the PCC testing.
716 $LUSTRE/tests/mmap_sanity -d $DIR/$tdir -m $DIR2/$tdir $excepts ||
717 error "mmap_sanity test failed"
720 # Revoke the layout lock, the PCC-cached file will be
721 # detached automatically.
722 do_facet $SINGLEAGT $LCTL \
723 set_param ldlm.namespaces.*mdc*.lru_size=clear
724 rm -rf $DIR/$tdir || error "failed to remove $DIR/$tdir"
726 run_test 4 "Auto cache test for mmap"
729 local file=$DIR/$tfile
730 local loopfile="$TMP/$tfile"
731 local mntpt="/mnt/pcc.$tdir"
732 local hsm_root="$mntpt/$tdir"
734 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
735 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
738 do_facet $SINGLEAGT "echo -n attach_mmap_data > $file" ||
739 error "echo $file failed"
741 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
742 error "failed to attach file $file"
743 check_lpcc_state $file "readwrite"
745 local content=$($MMAP_CAT $file)
747 [[ $content == "attach_mmap_data" ]] ||
748 error "mmap cat data mismatch: $content"
750 $LFS hsm_restore $file || error "failed to restore $file"
751 wait_request_state $(path2fid $file) RESTORE SUCCEED
752 check_lpcc_state $file "none"
754 content=$($MMAP_CAT $file)
755 [[ $content == "attach_mmap_data" ]] ||
756 error "mmap cat data mismatch: $content"
758 run_test 5 "Mmap & cat a RW-PCC cached file"
761 local loopfile="$TMP/$tfile"
762 local mntpt="/mnt/pcc.$tdir"
763 local hsm_root="$mntpt/$tdir"
764 local file=$DIR/$tfile
767 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
768 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
771 echo -n mmap_write_data > $file || error "echo write $file failed"
772 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
773 error "failed to attach file $file"
774 check_lpcc_state $file "readwrite"
776 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
777 error "could not mmap $file"
778 check_lpcc_state $file "readwrite"
779 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
780 # After mmap write via multiop, the first character of each page
782 [[ $content == "nmap_write_data" ]] ||
783 error "mmap write data mismatch: $content"
784 check_lpcc_state $file "readwrite"
786 do_facet $SINGLEAGT $LFS pcc detach $file ||
787 error "failed to detach file $file"
788 wait_request_state $(path2fid $file) REMOVE SUCCEED
790 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
791 [[ $content == "nmap_write_data" ]] ||
792 error "mmap write data mismatch: $content"
794 run_test 6 "Test mmap write on RW-PCC "
797 local loopfile="$TMP/$tfile"
798 local mntpt="/mnt/pcc.$tdir"
799 local hsm_root="$mntpt/$tdir"
800 local file=$DIR/$tfile
803 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
804 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
808 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
809 error "failed to attach file $file"
810 check_lpcc_state $file "readwrite"
811 check_file_data $SINGLEAGT $file "QQQQQ"
812 # define OBD_FAIL_LLITE_PCC_DETACH_MKWRITE 0x1412
813 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1412
814 # HSM released exists archived status
815 check_hsm_flags $file "0x0000000d"
817 # multiop mmap write increase the first character of each page with 1
818 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
819 error "mmap write $file failed"
820 check_lpcc_state $file "none"
821 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
822 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
824 run_test 7a "Fake file detached between fault() and page_mkwrite() for RW-PCC"
827 local loopfile="$TMP/$tfile"
828 local mntpt="/mnt/pcc.$tdir"
829 local hsm_root="$mntpt/$tdir"
830 local file=$DIR/$tfile
834 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
835 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
836 setup_pcc_mapping $SINGLEAGT \
837 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
840 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
841 error "failed to attach file $file"
842 check_lpcc_state $file "readwrite"
843 check_file_data $SINGLEAGT $file "QQQQQ"
844 # define OBD_FAIL_LLITE_PCC_MKWRITE_PAUSE 0x1413
845 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1413 fail_val=20
846 # HSM released exists archived status
847 check_hsm_flags $file "0x0000000d"
849 # multiop mmap write increases the first character of each page with 1
850 do_facet $SINGLEAGT $MULTIOP $file OSMWUc &
854 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
855 error "failed to detach file $file"
857 wait $pid || error "multiop mmap write failed"
858 check_lpcc_state $file "none"
859 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
860 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
862 run_test 7b "Test the race with concurrent mkwrite and detach"
865 local loopfile="$TMP/$tfile"
866 local mntpt="/mnt/pcc.$tdir"
867 local hsm_root="$mntpt/$tdir"
868 local file=$DIR/$tfile
870 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
871 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
875 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
876 error "failed to attach file $file"
877 check_lpcc_state $file "readwrite"
878 check_file_data $SINGLEAGT $file "QQQQQ"
880 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
881 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
882 do_facet $SINGLEAGT "echo -n ENOSPC_write > $file"
883 # Above write will return -ENOSPC failure and retry the IO on normal
884 # IO path. It will restore the HSM released file.
885 check_lpcc_state $file "none"
886 check_file_data $SINGLEAGT $file "ENOSPC_write"
888 run_test 8 "Test fake -ENOSPC tolerance for RW-PCC"
891 local loopfile="$TMP/$tfile"
892 local mntpt="/mnt/pcc.9a"
893 local hsm_root="$mntpt/$tdir"
894 local file=$DIR/$tfile
896 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
898 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
900 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
902 touch $file || error "touch $file failed"
903 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
904 error "fail to attach $file"
905 check_lpcc_state $file "readwrite"
906 # write 60M data, it is larger than the capacity of PCC backend
907 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=60 ||
908 error "fail to dd write $file"
909 check_lpcc_state $file "none"
910 check_file_size $SINGLEAGT $file 62914560
912 run_test 9 "Test -ENOSPC tolerance on loop PCC device for RW-PCC"
914 test_usrgrp_quota() {
915 local loopfile="$TMP/$tfile"
916 local mntpt="/mnt/pcc.$tdir"
917 local hsm_root="$mntpt/$tdir"
921 [[ $ug == "g" ]] && id=$RUNAS_GID
923 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
924 do_facet $SINGLEAGT quotacheck -c$ug $mntpt ||
925 error "quotacheck -c$ug $mntpt failed"
926 do_facet $SINGLEAGT quotaon -$ug $mntpt ||
927 error "quotaon -$ug $mntpt failed"
928 do_facet $SINGLEAGT setquota -$ug $id 0 20480 0 0 $mntpt ||
929 error "setquota -$ug $id on $mntpt failed"
930 do_facet $SINGLEAGT repquota -${ug}vs $mntpt
932 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
934 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
936 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
938 local file1=$DIR/$tdir/${ug}quotaA
939 local file2=$DIR/$tdir/${ug}quotaB
941 dd if=/dev/zero of=$file1 bs=1M count=15 ||
942 error "dd write $file1 failed"
943 dd if=/dev/zero of=$file2 bs=1M count=15 ||
944 error "dd write $file2 failed"
945 chown $RUNAS_ID:$RUNAS_GID $file1 ||
946 error "chown $RUNAS_ID:$RUNAS_GID $file1 failed"
947 chown $RUNAS_ID:$RUNAS_GID $file2 ||
948 error "chown $RUNAS_ID:$RUNAS_GID $file2 failed"
949 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
950 $file1 || error "attach $file1 failed"
951 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
952 $file2 && error "attach $file2 should fail due to quota limit"
953 check_lpcc_state $file1 "readwrite"
954 check_lpcc_state $file2 "none"
956 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file1 bs=1M count=30 ||
957 error "dd write $file1 failed"
958 # -EDQUOT error should be tolerated via fallback to normal Lustre path.
959 check_lpcc_state $file1 "none"
960 do_facet $SINGLEAGT $LFS pcc detach -k $file1 ||
961 error "failed to detach file $file"
966 test_usrgrp_quota "u"
968 run_test 10a "Test RW-PCC with user quota on loop PCC device"
971 test_usrgrp_quota "g"
973 run_test 10b "Test RW-PCC with group quota on loop PCC device"
976 local loopfile="$TMP/$tfile"
977 local mntpt="/mnt/pcc.$tdir"
978 local hsm_root="$mntpt/$tdir"
979 local file=$DIR/$tfile
983 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
984 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
987 do_facet $SINGLEAGT "echo -n QQQQQ > $file"
988 lpcc_path=$(lpcc_fid2path $hsm_root $file)
989 lpcc_dir=$(dirname $lpcc_path)
990 echo "Lustre file: $file LPCC dir: $lpcc_dir"
991 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
992 error "mkdir -p $lpcc_dir failed"
993 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
994 error "failed to attach $file"
995 check_lpcc_state $file "readwrite"
996 check_file_data $SINGLEAGT $file "QQQQQ"
997 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
998 error "failed to detach $file"
999 rm $file || error "rm $file failed"
1001 # The parent directory of the PCC file is immutable
1002 do_facet $SINGLEAGT "echo -n immutable_dir > $file"
1003 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1004 lpcc_dir=$(dirname $lpcc_path)
1005 echo "Lustre file: $file LPCC dir: $lpcc_dir"
1006 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
1007 error "mkdir -p $lpcc_dir failed"
1008 do_facet $SINGLEAGT chattr +i $lpcc_dir ||
1009 error "chattr +i $lpcc_dir failed"
1010 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
1011 error "attach $file with immutable directory should be failed"
1012 do_facet $SINGLEAGT chattr -i $lpcc_dir ||
1013 error "chattr -i $lpcc_dir failed"
1014 rm $file || error "rm $file failed"
1016 # The PCC file path is set to a directory
1017 do_facet $SINGLEAGT "echo -n pcc_file_path_is_dir > $file"
1018 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1019 do_facet $SINGLEAGT mkdir -p $lpcc_path ||
1020 error "mkdir -p $lpcc_path failed"
1021 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
1022 error "attach $file should fail as PCC path is a directory"
1023 rm $file || error "rm $file failed"
1025 run_test 11 "Test attach fault injection with simulated PCC file path"
1028 local file=$DIR/$tfile
1029 local hsm_root=$(hsm_root)
1033 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1034 setup_pcc_mapping $SINGLEAGT \
1035 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1037 echo -n race_rw_attach_hsmremove > $file
1038 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1039 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1040 error "attach $file failed"
1041 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1042 error "detach $file failed"
1043 # HSM released exists archived status
1044 check_hsm_flags $file "0x0000000d"
1045 # define OBD_FAIL_LLITE_PCC_ATTACH_PAUSE 0x1414
1046 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1414 fail_val=20
1047 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &
1049 $LFS hsm_state $file
1051 wait_request_state $(path2fid $file) RESTORE SUCCEED
1052 $LFS hsm_remove $file || error "hsm remove $file failed"
1054 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1055 error "RW-PCC cached file '$lpcc_path' should be removed"
1059 run_test 12 "RW-PCC attach races with concurrent HSM remove"
1062 local idstr="${1}id"
1063 local rule="${idstr}={$2}"
1065 local file=$DIR/$tdir/$tfile
1067 setup_pcc_mapping $SINGLEAGT \
1068 "$rule\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1069 $LCTL pcc list $MOUNT
1071 do_facet $SINGLEAGT $LFS mkdir -i 0 $DIR/$tdir
1072 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1074 rm -f $file || error "rm $file failed"
1075 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1076 error "failed to dd write to $file"
1077 check_lpcc_state $file "readwrite"
1078 do_facet $SINGLEAGT $myRUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
1079 error "failed to dd read from $file"
1080 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 256 ||
1081 error "failed to truncate $file"
1082 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 2048 ||
1083 error "failed to truncate $file"
1084 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1085 error "failed to dd write from $file"
1086 check_lpcc_state $file "readwrite"
1088 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1089 error "failed to detach file $file"
1090 wait_request_state $(path2fid $file) REMOVE SUCCEED
1091 check_lpcc_state $file "none"
1097 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1098 test_rule_id "u" "500" "runas -u 500"
1099 test_rule_id "g" "500" "runas -u 500 -g 500"
1101 run_test 13a "Test auto RW-PCC create caching for UID/GID rule"
1106 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1107 setup_pcc_mapping $SINGLEAGT \
1108 "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1109 $LCTL pcc list $MOUNT
1111 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1112 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1114 file=$DIR/$tdir/prefix.h5
1115 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1116 error "failed to dd write to $file"
1117 check_lpcc_state $file "readwrite"
1118 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1119 error "failed to detach file $file"
1120 check_lpcc_state $file "none"
1121 rm $file || error "rm $file failed"
1123 file=$DIR/$tdir/suffix.doc
1124 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1125 error "failed to dd write to $file"
1126 check_lpcc_state $file "readwrite"
1127 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1128 error "failed to detach file $file"
1129 check_lpcc_state $file "none"
1130 rm $file || error "rm $file failed"
1132 file=$DIR/$tdir/MidPADdle
1133 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1134 error "failed to dd write to $file"
1135 check_lpcc_state $file "readwrite"
1136 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1137 error "failed to detach file $file"
1138 check_lpcc_state $file "none"
1139 rm $file || error "rm $file failed"
1141 file=$DIR/$tdir/Midpad
1142 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1143 error "failed to dd write to $file"
1144 check_lpcc_state $file "none"
1145 rm $file || error "rm $file failed"
1147 run_test 13b "Test auto RW-PCC create caching for file name with wildcard"
1152 local loopfile="$TMP/$tfile"
1153 local mntpt="/mnt/pcc.$tdir"
1154 local hsm_root="$mntpt/$tdir"
1156 ! is_project_quota_supported &&
1157 echo "Skip project quota is not supported" && return 0
1159 enable_project_quota
1160 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1161 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1162 setup_pcc_mapping $SINGLEAGT \
1163 "projid={100\ 200}\&fname={*.h5},uid={500}\&gid={1000}\ rwid=$HSM_ARCHIVE_NUMBER"
1164 $LCTL pcc list $MOUNT
1165 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1166 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1168 mkdir -p $DIR/$tdir/proj || error "mkdir $DIR/$tdir/proj failed"
1169 mkdir -p $DIR/$tdir/proj2 || error "mkdir $DIR/$tdir/proj2 failed"
1170 $LFS project -sp 100 $DIR/$tdir/proj ||
1171 error "failed to set project for $DIR/$tdir/proj"
1172 $LFS project -sp 200 $DIR/$tdir/proj2 ||
1173 error "failed to set project for $DIR/$tdir/proj2"
1175 file=$DIR/$tdir/proj/notcache
1176 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1177 error "failed to dd write to $file"
1178 check_lpcc_state $file "none"
1179 rm $file || error "rm $file failed"
1181 file=$DIR/$tdir/proj/autocache.h5
1182 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1183 error "failed to dd write to $file"
1184 check_lpcc_state $file "readwrite"
1185 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1186 error "failed to detach $file"
1187 rm $file || error "rm $file failed"
1189 file=$DIR/$tdir/proj2/notcache
1190 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1191 error "failed to dd write to $file"
1192 check_lpcc_state $file "none"
1193 rm $file || error "rm $file failed"
1195 file=$DIR/$tdir/proj2/autocache.h5
1196 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1197 error "failed to dd write to $file"
1198 check_lpcc_state $file "readwrite"
1199 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1200 error "failed to detach $file"
1201 rm $file || error "rm $file failed"
1203 file=$DIR/$tdir/ugidcache
1204 myRUNAS="runas -u 500 -g 1000"
1205 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1206 error "failed to dd write to $file"
1207 check_lpcc_state $file "readwrite"
1208 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1209 error "failed to detach $file"
1210 rm $file || error "rm $file failed"
1212 run_test 13c "Check auto RW-PCC create caching for UID/GID/ProjID/fname rule"
1215 local file=$DIR/$tdir/$tfile
1217 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1218 setup_pcc_mapping $SINGLEAGT \
1219 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1221 mkdir -p $DIR/$tdir || error "mkdir -p $DIR/$tdir failed"
1222 do_facet $SINGLEAGT "echo -n autodetach_data > $file"
1223 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1224 $file || error "PCC attach $file failed"
1225 check_lpcc_state $file "readwrite"
1227 # Revoke the layout lock, the PCC-cached file will be
1228 # detached automatically.
1229 do_facet $SINGLEAGT $LCTL \
1230 set_param ldlm.namespaces.*mdc*.lru_size=clear
1231 check_file_data $SINGLEAGT $file "autodetach_data"
1232 check_lpcc_state $file "none"
1234 run_test 14 "Revocation of the layout lock should detach the file automatically"
1237 local loopfile="$TMP/$tfile"
1238 local mntpt="/mnt/pcc.$tdir"
1239 local hsm_root="$mntpt/$tdir"
1240 local file=$DIR/$tdir/$tfile
1242 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1243 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1246 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1247 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
1249 echo "Check open attach for non-root user"
1250 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1251 error "failed to dd write to $file"
1252 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1253 $file || error "failed to attach file $file"
1254 do_facet $SINGLEAGT $RUNAS $LFS pcc state $file
1255 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1256 # Revoke the layout lock, the PCC-cached file will be
1257 # detached automatically.
1258 do_facet $SINGLEAGT $LCTL \
1259 set_param ldlm.namespaces.*mdc*.lru_size=clear
1260 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1261 # Detach the file but keep the cache , as the file layout generation
1262 # is not changed, so the file is still valid cached in PCC, and can
1263 # be reused from PCC cache directly.
1264 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
1265 error "PCC detach $file failed"
1266 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1267 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
1268 error "PCC detach $file failed"
1269 rm $file || error "rm $file failed"
1271 echo "check open attach for root user"
1272 do_facet $SINGLEAGT "echo -n autoattach_data > $file"
1273 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1274 $file || error "PCC attach $file failed"
1275 check_lpcc_state $file "readwrite"
1277 # Revoke the layout lock, the PCC-cached file will be
1278 # detached automatically.
1279 do_facet $SINGLEAGT $LCTL \
1280 set_param ldlm.namespaces.*mdc*.lru_size=clear
1281 check_file_data $SINGLEAGT $file "autoattach_data"
1282 check_lpcc_state $file "readwrite"
1284 # Detach the file with -k option, as the file layout generation
1285 # is not changed, so the file is still valid cached in PCC,
1286 # and can be reused from PCC cache directly.
1287 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1288 error "PCC detach $file failed"
1289 check_lpcc_state $file "readwrite"
1290 # HSM released exists archived status
1291 check_hsm_flags $file "0x0000000d"
1292 check_file_data $SINGLEAGT $file "autoattach_data"
1294 # HSM restore the PCC cached file, the layout generation
1295 # was changed, so the file can not be auto attached.
1296 $LFS hsm_restore $file || error "failed to restore $file"
1297 wait_request_state $(path2fid $file) RESTORE SUCCEED
1298 check_lpcc_state $file "none"
1299 # HSM exists archived status
1300 check_hsm_flags $file "0x00000009"
1303 run_test 15 "Test auto attach at open when file is still valid cached"
1306 local loopfile="$TMP/$tfile"
1307 local mntpt="/mnt/pcc.$tdir"
1308 local hsm_root="$mntpt/$tdir"
1309 local file=$DIR/$tfile
1312 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1313 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1316 do_facet $SINGLEAGT "echo -n detach_data > $file"
1317 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1318 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1319 $file || error "PCC attach $file failed"
1320 check_lpcc_state $file "readwrite"
1321 # HSM released exists archived status
1322 check_hsm_flags $file "0x0000000d"
1324 echo "Test for reusing valid PCC cache"
1325 # Valid PCC cache can be reused
1326 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1327 error "PCC detach $file failed"
1328 check_lpcc_state $file "readwrite"
1329 # HSM released exists archived status
1330 check_hsm_flags $file "0x0000000d"
1332 echo "Test for the default detach"
1333 # Permanent detach by default, it will remove the PCC copy
1334 do_facet $SINGLEAGT $LFS pcc detach $file ||
1335 error "PCC detach $file failed"
1336 wait_request_state $(path2fid $file) REMOVE SUCCEED
1337 check_lpcc_state $file "none"
1338 # File is removed from PCC backend
1339 check_hsm_flags $file "0x00000000"
1340 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1341 error "RW-PCC cached file '$lpcc_path' should be removed"
1345 run_test 16 "Test detach with different options"
1348 local agt_host=$(facet_active_host $SINGLEAGT)
1349 local loopfile="$TMP/$tfile"
1350 local mntpt="/mnt/pcc.$tdir"
1351 local hsm_root="$mntpt/$tdir"
1352 local file=$DIR/$tfile
1354 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1355 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1356 setup_pcc_mapping $SINGLEAGT \
1357 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0"
1359 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1361 do_facet $SINGLEAGT "echo -n layout_refresh_data > $file"
1362 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1363 error "PCC attach $file failed"
1364 check_lpcc_state $file "readwrite"
1366 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1367 error "PCC detach $file failed"
1368 check_lpcc_state $file "none"
1370 # Truncate should attach the file into PCC automatically
1371 # as the PCC copy is still valid.
1372 echo "Verify auto attach during IO for truncate"
1373 do_facet $SINGLEAGT $TRUNCATE $file 4 || error "truncate $file failed"
1374 check_lpcc_state $file "readwrite"
1376 echo "Verify auto attach during IO for read/write"
1377 rmultiop_start $agt_host $file O_r || error "open $file failed"
1380 # Revoke the layout lock, the PCC-cached file will be
1381 # detached automatically.
1382 do_facet $SINGLEAGT $LCTL \
1383 set_param ldlm.namespaces.*mdc*.lru_size=clear
1385 check_lpcc_state $file "none"
1386 rmultiop_stop $agt_host || error "close $file failed"
1388 check_lpcc_state $file "readwrite"
1390 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1391 error "PCC detach $file failed"
1392 check_lpcc_state $file "none"
1394 run_test 17 "Test auto attach for layout refresh"
1397 local agt_host=$(facet_active_host $SINGLEAGT)
1398 local loopfile="$TMP/$tfile"
1399 local mntpt="/mnt/pcc.$tdir"
1400 local hsm_root="$mntpt/$tdir"
1401 local file=$DIR/$tfile
1405 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1406 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1407 setup_pcc_mapping $SINGLEAGT \
1408 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
1410 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1411 do_facet $SINGLEAGT dd if=/dev/urandom of=$file bs=1M count=4 ||
1412 error "failed to write $file"
1413 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1414 error "failed to attach $file"
1415 do_facet $SINGLEAGT $LFS pcc state $file
1416 check_lpcc_state $file "readwrite"
1417 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1418 error "failed to detach $file"
1419 do_facet $SINGLEAGT $LFS pcc state $file
1420 $CHECKSTAT -s 4194304 $file
1421 dd if=/dev/zero of=$DIR2/$tfile seek=1k bs=1k count=1 ||
1422 error "failed to write $DIR2/$tfile"
1423 oldmd5=$(md5sum $DIR2/$tfile | awk '{print $1}')
1424 $CHECKSTAT -s 1049600 $DIR2/$tfile || error "$DIR2/$tfile size wrong"
1426 local lpcc_path=$(lpcc_fid2path $hsm_root $file)
1428 do_facet $SINGLEAGT $LFS pcc state $file
1429 check_file_size $SINGLEAGT $lpcc_path 4194304
1430 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1431 error "failed to attach $file"
1432 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1049600
1433 newmd5=$(do_facet $SINGLEAGT md5sum $file | awk '{print $1}')
1434 [ "$oldmd5" == "$newmd5" ] || error "md5sum differ: $oldmd5 != $newmd5"
1435 do_facet $SINGLEAGT $LFS pcc detach $file ||
1436 error "failed to detach $file"
1438 run_test 18 "Verify size correctness after re-attach the file"
1441 local agt_host=$(facet_active_host $SINGLEAGT)
1442 local loopfile="$TMP/$tfile"
1443 local mntpt="/mnt/pcc.$tdir"
1444 local hsm_root="$mntpt/$tdir"
1445 local file=$DIR/$tfile
1447 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1448 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1449 setup_pcc_mapping $SINGLEAGT \
1450 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1452 do_facet $SINGLEAGT "echo -n QQQQQ > $file" || error "echo $file failed"
1453 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1454 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1455 error "Failed to attach $file"
1456 check_lpcc_state $file "readwrite"
1457 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1458 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1459 error "Failed to detach $file"
1460 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1461 error "Failed to attach $file"
1462 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1463 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1464 error "Failed to detach $file"
1466 run_test 19 "Verify the file re-attach works as expected"
1469 local agt_host=$(facet_active_host $SINGLEAGT)
1470 local loopfile="$TMP/$tfile"
1471 local mntpt="/mnt/pcc.$tdir"
1472 local hsm_root="$mntpt/$tdir"
1473 local file=$DIR/$tfile
1475 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1476 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1477 setup_pcc_mapping $SINGLEAGT \
1478 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
1480 do_facet $SINGLEAGT "echo -n QQQQQ > $file" ||
1481 error "echo $file failed"
1482 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
1483 error "Failed to attach $file"
1484 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1485 check_lpcc_state $file "readwrite"
1486 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1487 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1488 check_lpcc_state $file "readwrite"
1489 do_facet $SINGLEAGT $LFS pcc detach $file ||
1490 error "Failed to detach $file"
1492 run_test 20 "Auto attach works after the inode was once evicted from cache"
1494 #test 101: containers and PCC
1495 #LU-15170: Test mount namespaces with PCC
1496 #This tests the cases where the PCC mount is not present in the container by
1497 #creating a mount namespace without the PCC mount in it (this is probably the
1498 #standard config for most containers)
1500 local loopfile="$TMP/$tfile"
1501 local mntpt="/mnt/pcc.$tdir"
1502 local hsm_root="$mntpt/$tdir"
1503 local file=$DIR/$tdir/$tfile
1505 # Some kernels such as RHEL7 default to 0 user namespaces
1506 local maxuserns=$(do_facet $SINGLEAGT cat /proc/sys/user/max_user_namespaces)
1507 do_facet $SINGLEAGT "echo 10 > /proc/sys/user/max_user_namespaces"
1508 stack_trap "do_facet $SINGLEAGT 'echo $maxuserns > /proc/sys/user/max_user_namespaces'"
1510 echo "creating user namespace for $RUNAS_ID"
1511 # Create a mount and user namespace with this command, and leave the
1512 # process running so we can do the rest of our steps
1513 local start=$SECONDS
1514 local PID=$(do_facet $SINGLEAGT \
1515 "$RUNAS unshare -Um sleep 600 &>/dev/null & echo \\\$!")
1516 local elapsed=$((SECONDS - start))
1519 do_facet $SINGLEAGT ps auxww | grep sleep
1520 echo "Created NS: child (sleep) pid=$PID in $elapsed seconds"
1521 [[ -n "$PID" ]] || error "remote sleep start failed"
1522 stack_trap "do_facet $SINGLEAGT kill -9 $PID" EXIT
1523 (( elapsed < 300 )) || error "remote sleep took $elapsed sec to start"
1525 # Map 'RUNAS' to root in the namespace, so it has rights to do whatever
1526 # This is handled by '-r' in unshare in newer versions
1527 do_facet $SINGLEAGT $RUNAS newuidmap $PID 0 $RUNAS_ID 1 ||
1528 error "could not map uid $RUNAS_ID to root in namespace"
1529 do_facet $SINGLEAGT $RUNAS newgidmap $PID 0 $RUNAS_GID 1 ||
1530 error "could not map gid $RUNAS_GID to root in namespace"
1532 # Create PCC after creating namespace; namespace will not have PCC
1534 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1536 # Create a temp file inside the PCC mount to verify mount namespace
1537 do_facet $SINGLEAGT touch $mntpt/$tfile.tmp
1538 stack_trap "do_facet $SINGLEAGT rm -f $mntpt/$tfile.tmp" EXIT
1539 echo "Check for temp file in PCC mount"
1540 do_facet $SINGLEAGT test -f $mntpt/$tfile.tmp ||
1541 error "Should see $mntpt/$tfile.tmp"
1542 echo "Check for temp file in PCC mount from inside namespace"
1543 do_facet $SINGLEAGT nsenter -t $PID -U -m test -f $mntpt/$tfile.tmp &&
1544 error "Should not see $mntpt/$tfile.tmp from namespace"
1545 rm -f $mntpt/$tfile.tmp
1548 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1549 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
1551 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1552 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
1554 echo "Verify open attach from inside mount namespace"
1555 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
1556 error "failed to dd write to $file"
1557 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach \
1558 -i $HSM_ARCHIVE_NUMBER $file || error "cannot attach $file"
1559 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc state $file
1561 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1562 # Revoke the layout lock, the PCC-cached file will be
1563 # detached automatically.
1564 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
1565 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1566 # Detach the file but keep the cache, as the file layout generation
1567 # is not changed, so the file is still valid cached in PCC, and can
1568 # be reused from PCC cache directly.
1569 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach -k $file ||
1570 error "PCC detach $file failed"
1571 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1572 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach $file ||
1573 error "PCC detach $file failed"
1574 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
1575 error "failed to dd write to $file"
1576 rm -f $file || error "rm $file failed"
1578 echo "Verify auto attach at open from inside NS for RW-PCC"
1579 # nsenter has strange behavior with echo, which means we have to place
1580 # this in a script so we can use sh, otherwise it doesn't execute echo
1582 # NB: using /bin/echo instead of the shell built in does not help
1583 echo "echo -n autoattach_data > $file" > $DIR/$tdir/$tfile.shell
1584 # File is owned by root, make it accessible to RUNAS user
1585 chmod a+rw $DIR/$tdir/$tfile.shell
1586 stack_trap 'rm -f $DIR/$tdir/$tfile.shell' EXIT
1587 do_facet $SINGLEAGT nsenter -t $PID -U -m "sh $DIR/$tdir/$tfile.shell"
1588 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
1589 $file || error "RW-PCC attach $file failed"
1590 check_lpcc_state $file "readwrite"
1592 # Revoke the layout lock, the PCC-cached file will be
1593 # detached automatically.
1594 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
1595 check_file_data $SINGLEAGT $file "autoattach_data" $PID
1596 check_lpcc_state $file "readwrite"
1598 # Detach the file with -k option, as the file layout generation
1599 # is not changed, so the file is still valid cached in PCC,
1600 # and can be reused from PCC cache directly.
1601 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1602 error "RW-PCC detach $file failed"
1603 check_lpcc_state $file "readwrite"
1604 # HSM released exists archived status
1605 check_hsm_flags $file "0x0000000d"
1606 check_file_data $SINGLEAGT $file "autoattach_data" $PID
1608 # HSM restore the PCC cached file, the layout generation
1609 # was changed, so the file can not be auto attached.
1610 $LFS hsm_restore $file || error "failed to restore $file"
1611 wait_request_state $(path2fid $file) RESTORE SUCCEED
1612 check_lpcc_state $file "none"
1613 # HSM exists archived status
1614 check_hsm_flags $file "0x00000009"
1616 run_test 101a "Test auto attach in mount namespace (simulated container)"
1619 grep -q io_uring_setup /proc/kallsyms ||
1620 skip "Client OS does not support io_uring I/O engine"
1621 io_uring_probe || skip "kernel does not support io_uring fully"
1623 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1624 skip "Server does not support PCC-RO"
1626 which fio || skip_env "no fio installed"
1627 fio --enghelp | grep -q io_uring ||
1628 skip_env "fio does not support io_uring I/O engine"
1630 local loopfile="$TMP/$tfile"
1631 local mntpt="/mnt/pcc.$tdir"
1632 local hsm_root="$mntpt/$tdir"
1633 local file=$DIR/$tfile
1635 setup_loopdev client $loopfile $mntpt 60
1636 mkdir $hsm_root || error "mkdir $hsm_root failed"
1637 setup_pcc_mapping client \
1638 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
1639 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0
1641 local ioengine="io_uring"
1645 do_facet $SINGLEAGT fio --name=seqwrite --ioengine=$ioengine \
1646 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
1647 --iodepth=64 --size=$size --filename=$file --rw=write ||
1648 error "fio seqwrite $file failed"
1650 # Read the file will trigger the buffered read from Lustre OSTs and
1651 # write to PCC copy as @pcc_async_threshold is set with 0.
1652 do_facet $SINGLEAGT fio --name=seqread --ioengine=$ioengine \
1653 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
1654 --iodepth=64 --size=$size --filename=$file --rw=read ||
1655 error "fio seqread $file failed"
1657 run_test 102 "PCC-RO should not hange for io_uring I/O engine"
1660 check_and_cleanup_lustre