3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin
14 ALWAYS_EXCEPT="$SANITY_PCC_EXCEPT "
15 # bug number for skipped test:
17 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
19 ENABLE_PROJECT_QUOTAS=${ENABLE_PROJECT_QUOTAS:-true}
20 HSMTOOL_ARCHIVE_FORMAT=v2
22 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
24 . $LUSTRE/tests/test-framework.sh
26 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
29 MULTIOP=${MULTIOP:-multiop}
30 OPENFILE=${OPENFILE:-openfile}
31 MOUNT_2=${MOUNT_2:-"yes"}
34 # script only handles up to 10 MDTs (because of MDT_PREFIX)
35 [ $MDSCOUNT -gt 9 ] &&
36 error "script cannot handle more than 9 MDTs, please fix" && exit
38 check_and_setup_lustre
40 if [[ "$MDS1_VERSION" -lt $(version_code 2.12.52) ]]; then
41 skip "Need MDS version at least 2.12.52"
44 # $RUNAS_ID may get set incorrectly somewhere else
45 if [[ $UID -eq 0 && $RUNAS_ID -eq 0 ]]; then
46 skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" && exit
48 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
49 if getent group nobody; then
51 elif getent group nogroup; then
54 error "No generic nobody group"
57 if [[ -r /etc/redhat-release ]]; then
58 rhel_version=$(sed -e 's/[^0-9.]*//g' /etc/redhat-release)
59 if (( $(version_code $rhel_version) >= $(version_code 9.3.0) )); then
60 always_except LU-17289 102 # fio io_uring
61 always_except LU-17781 33 # inconsistent LSOM
62 elif (( $(version_code $rhel_version) >= $(version_code 8.9.0) )); then
63 always_except LU-17781 33 # inconsistent LSOM
67 if [[ "$CLIENT_OS_ID_LIKE" =~ "suse" ]]; then
68 if (( CLIENT_OS_VERSION_CODE >= $(version_code 15.3) &&
69 CLIENT_OS_VERSION_CODE <= $(version_code 15.5) )); then
70 always_except LU-18298 1c 1d # passed on SLES15 SP6
76 # if there is no CLIENT1 defined, some tests can be ran on localhost
77 CLIENT1=${CLIENT1:-$HOSTNAME}
78 # if CLIENT2 doesn't exist then use CLIENT1 instead
79 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
81 # Exception is the test which need two separate nodes
82 CLIENT2=${CLIENT2:-$CLIENT1}
88 local expected_size="$3"
90 size=$(do_facet $client stat "--printf=%s" $fpath)
91 [[ $size == "$expected_size" ]] || error \
92 "expected $fpath size: $expected_size got: $size"
99 local lustre_fpath="$3"
100 local expected_size="$4"
102 check_file_size $client $lpcc_fpath $expected_size
103 check_file_size $client $lustre_fpath $expected_size
110 local expected_data="$3"
113 # if $pid is set, then run command within namespace for that process
114 path_data=$(do_facet $client ${pid:+nsenter -t $pid -U -m} cat $path)
115 [[ "x$path_data" == "x$expected_data" ]] ||
116 error "expected $path: $expected_data, got: $path_data"
122 local lpcc_fpath="$2"
123 local lustre_fpath="$3"
124 local expected_data="$4"
126 check_file_data "$client" "$lpcc_fpath" "$expected_data"
127 check_file_data "$client" "$lustre_fpath" "$expected_data"
133 local lustre_path="$2"
134 local fid=${3:-$(path2fid $lustre_path)}
136 local seq=$(echo $fid | awk -F ':' '{print $1}')
137 local oid=$(echo $fid | awk -F ':' '{print $2}')
138 local ver=$(echo $fid | awk -F ':' '{print $3}')
140 case "$HSMTOOL_ARCHIVE_FORMAT" in
142 printf "%s/%04x/%04x/%04x/%04x/%04x/%04x/%s" \
143 $hsm_root $((oid & 0xFFFF)) \
144 $((oid >> 16 & 0xFFFF)) \
146 $((seq >> 16 & 0xFFFF)) \
147 $((seq >> 32 & 0xFFFF)) \
148 $((seq >> 48 & 0xFFFF)) $fid
151 printf "%s/%04x/%s" $hsm_root $(((oid ^ seq) & 0xFFFF)) $fid
158 local lustre_path="$1"
159 local expected_state="$2"
160 local facet=${3:-$SINGLEAGT}
162 local state=$(do_facet $facet $myRUNAS $LFS pcc state $lustre_path |
163 awk -F 'type: ' '{print $2}' | awk -F ',' '{print $1}')
165 [[ "x$state" == "x$expected_state" ]] || error \
166 "$lustre_path expected pcc state: $expected_state, but got: $state"
172 # populate MDT device array
175 # cleanup from previous bad setup
178 # for recovery tests, coordinator needs to be started at mount
180 # the lustre conf must be without hsm on (like for sanity.sh)
181 echo "Set HSM on and start"
182 cdt_set_mount_state enabled
183 cdt_check_state enabled
185 echo "Set sanity-hsm HSM policy"
186 cdt_set_sanity_policy
188 # finished requests are quickly removed from list
189 set_hsm_param grace_delay 10
191 cleanup_pcc_mapping() {
192 local facet=${1:-$SINGLEAGT}
194 echo "Cleanup PCC backend on $MOUNT"
195 do_facet $facet $LCTL pcc clear $MOUNT
198 setup_pcc_mapping() {
199 local facet=${1:-$SINGLEAGT}
200 local hsm_root=${hsm_root:-$(hsm_root "$facet")}
203 [ -z "$param" ] && param="projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
204 stack_trap "cleanup_pcc_mapping $facet" EXIT
205 do_facet $facet $LCTL pcc add $MOUNT $hsm_root -p "$param" ||
206 error "Setup PCC backend $hsm_root on $MOUNT failed"
215 for ((i = 0; i < 10; i++)); do
216 if do_facet $facet lsof $mntpt; then
217 echo "$mntpt is busy, wait 1 second..."
220 echo "$mntpt is idle now"
225 do_facet $facet $UMOUNT $mntpt
236 do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
237 stack_trap "do_facet $facet rmdir $mntpt" EXIT
238 do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
239 stack_trap "do_facet $facet rm -f $file" EXIT
240 do_facet $facet mount
241 do_facet $facet $UMOUNT $mntpt
242 do_facet $facet mount
243 do_facet $facet mkfs.ext4 $file || error "mkfs.ext4 $file failed"
244 local mcs=$(do_facet $facet tune2fs -l $file |& grep metadata_csum_seed)
246 do_facet $facet "tune2fs -O ^metadata_csum_seed $file" ||
247 error "failed to turn off metadata_csum_seed feature"
248 do_facet $facet file $file
249 do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt ||
250 error "mount -o loop,usrquota,grpquota $file $mntpt failed"
251 stack_trap "umount_loopdev $facet $mntpt" EXIT
254 setup_loopdev_project() {
260 do_facet $facet mkdir -p $mntpt || error "mkdir -p $mntpt failed"
261 stack_trap "do_facet $facet rmdir $mntpt" EXIT
262 do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
263 stack_trap "do_facet $facet rm -f $file" EXIT
264 do_facet $facet $UMOUNT $mntpt
265 do_facet $facet mkfs.ext4 -O project,quota $file ||
266 error "mkfs.ext4 -O project,quota $file failed"
267 local mcs=$(do_facet $facet tune2fs -l $file |& grep metadata_csum_seed)
269 do_facet $facet "tune2fs -O ^metadata_csum_seed $file" ||
270 error "failed to turn off metadata_csum_seed feature"
271 do_facet $facet file $file
272 do_facet $facet mount -t ext4 -o loop,prjquota $file $mntpt ||
273 error "mount -o loop,prjquota $file $mntpt failed"
274 stack_trap "umount_loopdev $facet $mntpt" EXIT
275 do_facet $facet mount | grep $mntpt
279 local mode='\x00\x00\x00\x00'
280 local raw="$(printf ""\\\\x%02x"" {0..63})"
284 [[ $(lscpu) =~ Byte\ Order.*Little ]] && size='\x40\x00\x00\x00' ||
285 size='\x00\x00\x00\x40'
286 key="${mode}${raw}${size}"
287 do_facet $SINGLEAGT "echo -en '${key}' |
288 keyctl padd logon fscrypt:4242424242424242 @u"
291 setup_for_enc_tests() {
292 local agthost=${SINGLEAGT}_HOST
294 # remount client with test_dummy_encryption option
295 zconf_umount ${!agthost} $MOUNT ||
296 error "umount $SINGLEAGT $MOUNT failed"
297 zconf_mount ${!agthost} $MOUNT ${MOUNT_OPTS},test_dummy_encryption ||
298 error "mount $SINGLEAGT with '-o test_dummy_encryption' failed"
302 # this directory will be encrypted, because of dummy mode
303 do_facet $SINGLEAGT mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
306 cleanup_for_enc_tests() {
307 local agthost=${SINGLEAGT}_HOST
309 do_facet $SINGLEAGT rm -rf $DIR/$tdir
310 # remount client normally
311 zconf_umount ${!agthost} $MOUNT ||
312 error "umount $SINGLEAGT $MOUNT failed"
313 zconf_mount ${!agthost} $MOUNT ||
314 error "remount $SINGLEAGT failed"
321 local agt_facet=$SINGLEAGT
322 local loopfile="$TMP/$tfile"
323 local mntpt="/mnt/pcc.$tdir"
324 local hsm_root="$mntpt/$tdir"
325 local file=$DIR/$tdir/$tfile
330 $project && enable_project_quota
332 do_facet $SINGLEAGT rm -rf $hsm_root
333 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
334 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
336 is_project_quota_supported || project=false
338 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
339 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
340 $project && lfs project -sp $project_id $DIR2/$tdir
342 do_facet $SINGLEAGT "echo -n attach_origin > $file"
344 check_lpcc_state $file "none"
345 do_facet $SINGLEAGT $LFS pcc attach -w \
346 -i $HSM_ARCHIVE_NUMBER $file ||
347 error "pcc attach $file failed"
350 check_lpcc_state $file "readwrite"
351 # HSM released exists archived status
352 check_hsm_flags $file "0x0000000d"
353 lpcc_path=$(lpcc_fid2path $hsm_root $file)
354 check_lpcc_data $SINGLEAGT $lpcc_path $file "attach_origin"
356 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=7654321 count=1
357 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 7654321
359 do_facet $SINGLEAGT $TRUNCATE $file 1234567 ||
360 error "truncate failed"
361 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1234567
362 check_lpcc_state $file "readwrite"
364 do_facet $SINGLEAGT "echo -n file_data > $file"
365 check_lpcc_state $file "readwrite"
366 # HSM released exists archived status
367 check_hsm_flags $file "0x0000000d"
368 check_lpcc_data $SINGLEAGT $lpcc_path $file "file_data"
370 echo "Restore testing..."
371 if [ $CLIENTCOUNT -lt 2 -o $restore ]; then
372 $LFS hsm_restore $file || error \
373 "failed to restore $file"
374 wait_request_state $(path2fid $file) RESTORE SUCCEED
376 path_data=$(do_node $CLIENT2 cat $file)
377 [[ "x$path_data" == "xfile_data" ]] || error \
378 "expected file_data, got: $path_data"
381 check_lpcc_state $file "none"
382 # HSM exists archived status
383 check_hsm_flags $file "0x00000009"
385 echo -n "new_data" > $file
386 check_lpcc_state $file "none"
387 # HSM exists dirty archived status
388 check_hsm_flags $file "0x0000000b"
389 check_file_data $SINGLEAGT $file "new_data"
391 echo "Attach and detach testing"
393 do_facet $SINGLEAGT "echo -n new_data2 > $file"
395 check_lpcc_state $file "none"
396 do_facet $SINGLEAGT $LFS pcc attach -w \
397 -i $HSM_ARCHIVE_NUMBER $file ||
398 error "PCC attach $file failed"
400 check_lpcc_state $file "readwrite"
401 # HSM released exists archived status
402 check_hsm_flags $file "0x0000000d"
403 do_facet $SINGLEAGT "echo -n attach_detach > $file"
404 echo "Start to detach the $file"
405 do_facet $SINGLEAGT $LFS pcc detach $file ||
406 error "PCC detach $file failed"
407 wait_request_state $(path2fid $file) REMOVE SUCCEED
409 check_lpcc_state $file "none"
410 # The file is removed from PCC
411 check_hsm_flags $file "0x00000000"
412 check_file_data $SINGLEAGT $file "attach_detach"
416 lpcc_rw_test true false
418 run_test 1a "Test manual lfs pcc attach with manual HSM restore"
421 lpcc_rw_test false false
423 run_test 1b "Test manual lfs pcc attach with restore on remote access"
426 lpcc_rw_test true true
428 run_test 1c "Test automated attach using Project ID with manual HSM restore"
431 lpcc_rw_test false true
433 run_test 1d "Test Project ID with remote access"
436 local file=$DIR/$tdir/$tfile
437 local hsm_root=$(hsm_root)
440 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
441 setup_pcc_mapping $SINGLEAGT \
442 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
443 $LCTL pcc list $MOUNT
444 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
445 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
447 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
448 error "failed to dd write to $file"
449 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
450 $file || error "failed to attach file $file"
451 check_lpcc_state $file "readwrite"
452 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
453 error "failed to dd read from $file"
454 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
455 error "failed to truncate $file"
456 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
457 error "failed to truncate $file"
458 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
459 error "failed to dd write to $file"
460 check_lpcc_state $file "readwrite"
462 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
463 error "failed to detach file $file"
464 check_lpcc_state $file "none"
466 # non-root user is forbidden to access PCC file directly
467 lpcc_path=$(lpcc_fid2path $hsm_root $file)
468 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
469 error "non-root user can touch access PCC file $lpcc_path"
470 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
471 count=1 && error "non-root user can read PCC file $lpcc_path"
472 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
473 count=1 && error "non-root user can write PCC file $lpcc_path"
475 local perm=$(do_facet $SINGLEAGT stat -c %a $lpcc_path)
477 [[ $perm == "0" ]] || error "PCC file permission ($perm) is not zero"
479 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
480 $file || error "failed to attach file $file"
481 check_lpcc_state $file "readwrite"
483 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
484 error "failed to detach file $file"
485 wait_request_state $(path2fid $file) REMOVE SUCCEED
486 check_lpcc_state $file "none"
488 run_test 1e "Test RW-PCC with non-root user"
492 local agt_facet=$SINGLEAGT
493 local loopfile="$TMP/$tfile"
494 local mntpt="/mnt/pcc.$tdir"
495 local hsm_root="$mntpt/$tdir"
496 local file=$DIR/$tdir/$tfile
498 is_project_quota_supported || skip "project quota is not supported"
501 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
502 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
503 setup_pcc_mapping $SINGLEAGT \
504 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0\ pccrw=1"
506 do_facet $SINGLEAGT $LFS mkdir -i0 -c1 $DIR/$tdir
507 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
508 $LFS project -sp $project_id $DIR/$tdir ||
509 error "failed to set project for $DIR/$tdir"
511 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
512 error "failed to dd write to $file"
514 check_lpcc_state $file "readwrite"
515 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
516 error "failed to dd read from $file"
517 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
518 error "failed to truncate $file"
519 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
520 error "failed to truncate $file"
521 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=256 count=1 ||
522 error "failed to dd write from $file"
523 check_lpcc_state $file "readwrite"
525 # non-root user is forbidden to access PCC file directly
526 lpcc_path=$(lpcc_fid2path $hsm_root $file)
527 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
528 error "non-root user can touch access PCC file $lpcc_path"
529 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
530 count=1 && error "non-root user can read PCC file $lpcc_path"
531 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
532 count=1 && error "non-root user can write PCC file $lpcc_path"
534 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
535 error "failed to detach file $file"
536 wait_request_state $(path2fid $file) REMOVE SUCCEED
537 check_lpcc_state $file "none"
539 run_test 1f "Test auto RW-PCC cache with non-root user"
542 local loopfile="$TMP/$tfile"
543 local mntpt="/mnt/pcc.$tdir"
544 local hsm_root="$mntpt/$tdir"
545 local file=$DIR/$tfile
547 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
548 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
549 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
551 dd if=/dev/zero of=$file bs=1024 count=1 ||
552 error "failed to dd write to $file"
553 chmod 600 $file || error "chmod 600 $file failed"
554 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
555 error "non-root user can dd write $file"
556 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
557 error "non-root user can dd read $file"
558 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
559 error "failed to attach file $file"
560 check_lpcc_state $file "readwrite"
561 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
562 error "non-root user can dd write to $file"
563 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
564 error "non-root user can dd read $file"
565 chmod 777 $DIR2/$tfile || error "chmod 777 $DIR2/$tfile failed"
566 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
567 error "non-root user cannot write $file with permission (777)"
569 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
570 error "non-root user or non owner can detach $file"
571 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
572 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
573 error "failed to detach file $file"
574 wait_request_state $(path2fid $file) REMOVE SUCCEED
575 check_lpcc_state $file "none"
576 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
577 error "non-root user cannot read to $file with permisson (777)"
579 run_test 1g "General permission test for RW-PCC"
582 # When a process created a LPCC file and holding the open,
583 # another process on the same client should be able to open the file.
587 local agt_facet=$SINGLEAGT
588 local loopfile="$TMP/$tfile"
589 local mntpt="/mnt/pcc.$tdir"
590 local hsm_root="$mntpt/$tdir"
591 local agt_host=$(facet_active_host $SINGLEAGT)
593 is_project_quota_supported || skip "project quota is not supported"
596 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
597 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
598 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
599 file=$DIR/$tdir/multiop
600 $LFS mkdir -i -1 -c $MDSCOUNT $DIR/$tdir
603 do_facet $SINGLEAGT $LFS project -sp $project_id $DIR/$tdir ||
604 error "failed to set project quota"
605 rmultiop_start $agt_host $file O_c || error "open $file failed"
606 # HSM released exists archived status
607 check_hsm_flags $file "0x0000000d"
608 do_facet $SINGLEAGT "echo -n multiopen_data > $file" ||
609 error "failed to echo multiopen_data to $file"
611 lpcc_path=$(lpcc_fid2path $hsm_root $file)
612 do_facet $SINGLEAGT ls -l $lpcc_path ||
613 error "failed to ls $lpcc_path"
614 check_lpcc_data $SINGLEAGT $lpcc_path $file "multiopen_data"
615 # HSM released exists archived status
616 check_hsm_flags $file "0x0000000d"
618 do_facet $SINGLEAGT $LFS pcc detach $file ||
619 error "failed to detach $file"
620 rmultiop_stop $agt_host || error "close $file failed"
622 run_test 2a "Test multi open when creating"
624 get_remote_client() {
625 current_id=$(do_facet $SINGLEAGT hostname)
626 for client in ${CLIENTS//,/ }
628 r_id=$(do_node $client hostname)
629 if [ $r_id != $current_id ]; then
637 # When a process created a LPCC file and holding the open, another
638 # process on the different client should be able to open the file
639 # and perform IO on the file.
642 local agt_facet=$SINGLEAGT
643 local agt_host=$(facet_active_host $SINGLEAGT)
644 local loopfile="$TMP/$tfile"
645 local mntpt="/mnt/pcc.$tdir"
646 local hsm_root="$mntpt/$tdir"
648 needclients 2 || return 0
650 remote_client=$(get_remote_client)
653 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
654 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
655 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
656 file=$DIR/$tdir/multiop
660 do_facet $SINGLEAGT "echo -n file_data > $file"
661 do_facet $SINGLEAGT lfs pcc attach -w -i $HSM_ARCHIVE_NUMBER \
662 $file || error "PCC attach $file failed"
663 check_lpcc_state $file "readwrite"
665 rmultiop_start $agt_host $file O_c || error "open $file failed"
667 do_node $remote_client "echo -n multiopen_data > $file"
669 # PCC cached file should be automatically detached
670 check_lpcc_state $file "none"
672 check_file_data $SINGLEAGT $file "multiopen_data"
673 rmultiop_stop $agt_host || error "close $file failed"
674 check_file_data $SINGLEAGT $file "multiopen_data"
676 do_node $remote_client cat $file || error \
677 "cat $file on remote client failed"
678 do_node $remote_client echo -n "multiopen_data" > $file \
679 || error "write $file on remote client failed"
681 run_test 2b "Test multi remote open when creating"
684 local agt_host=$(facet_active_host $SINGLEAGT)
685 local loopfile="$TMP/$tfile"
686 local mntpt="/mnt/pcc.$tdir"
687 local hsm_root="$mntpt/$tdir"
688 local file=$DIR/$tdir/$tfile
689 local file2=$DIR2/$tdir/$tfile
692 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
693 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
694 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
698 do_facet $SINGLEAGT "echo -n file_data > $file"
699 do_facet $SINGLEAGT lfs pcc attach -w -i $HSM_ARCHIVE_NUMBER \
700 $file || error "PCC attach $file failed"
701 check_lpcc_state $file "readwrite"
703 rmultiop_start $agt_host $file O_c || error "open $file failed"
705 echo -n multiopen_data > $file2
707 # PCC cached file should be automatically detached
708 check_lpcc_state $file "none"
710 check_file_data $SINGLEAGT $file "multiopen_data"
711 rmultiop_stop $agt_host || error "close $file failed"
712 check_file_data $SINGLEAGT $file "multiopen_data"
714 cat $file2 || error "cat $file on mount $MOUNT2 failed"
715 echo -n "multiopen_data" > $file2 ||
716 error "write $file on mount $MOUNT2 failed"
718 run_test 2c "Test multi open on different mount points when creating"
721 local file=$DIR/$tdir/$tfile
722 local file2=$DIR2/$tdir/$tfile
724 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
725 skip "Server does not support PCC-RO"
727 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
728 setup_pcc_mapping $SINGLEAGT \
729 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1\ pccro=1"
731 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
732 dd if=/dev/zero of=$file2 bs=1024 count=1 ||
733 error "failed to dd write to $file"
735 echo "Start to RW-PCC attach/detach the file: $file"
736 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
737 error "failed to attach file $file"
738 check_lpcc_state $file "readwrite"
739 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
740 error "failed to detach file $file"
741 check_lpcc_state $file "none"
743 echo "Repeat to RW-PCC attach/detach the same file: $file"
744 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
745 error "failed to attach file $file"
746 check_lpcc_state $file "readwrite"
747 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
748 error "failed to detach file $file"
749 check_lpcc_state $file "none"
751 rm -f $file || error "failed to remove $file"
752 echo "pccro_data" > $file
754 echo "Start to RO-PCC attach/detach the file: $file"
755 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
756 error "failed to attach file $file"
757 check_lpcc_state $file "readonly"
758 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
759 error "failed to detach file $file"
760 check_lpcc_state $file "none"
762 echo "Repeat to RO-PCC attach/detach the same file: $file"
763 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
764 error "failed to attach file $file"
765 check_lpcc_state $file "readonly"
766 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
767 error "failed to detach file $file"
768 check_lpcc_state $file "none"
770 run_test 3a "Repeat attach/detach operations"
774 local file=$DIR/$tdir/$tfile
776 needclients 3 || return 0
778 # Start all of the copytools and setup PCC
779 for n in $(seq $AGTCOUNT); do
780 copytool setup -f agt$n -a $n -m $MOUNT -h $(hsm_root agt$n)
781 setup_pcc_mapping agt$n "projid={100}\ rwid=$n\ auto_attach=0\ pccrw=1\ pccro=1"
784 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
785 dd if=/dev/zero of=$file bs=1024 count=1 ||
786 error "failed to dd write to $file"
788 echo "Start to RW-PCC attach/detach $file on $agt1_HOST"
789 do_facet agt1 $LFS pcc attach -w -i 1 $file ||
790 error "failed to attach file $file"
791 check_lpcc_state $file "readwrite" agt1
792 do_facet agt1 $LFS pcc detach -k $file ||
793 error "failed to detach file $file"
794 check_lpcc_state $file "none" agt1
796 echo "Repeat to RW-PCC attach/detach $file on $agt2_HOST"
797 do_facet agt2 $LFS pcc attach -w -i 2 $file ||
798 error "failed to attach file $file"
799 check_lpcc_state $file "readwrite" agt2
800 do_facet agt2 $LFS pcc detach -k $file ||
801 error "failed to detach file $file"
802 check_lpcc_state $file "none" agt2
804 echo "Try RW-PCC attach on two agents"
805 do_facet agt1 $LFS pcc attach -w -i 1 $file ||
806 error "failed to attach file $file"
807 check_lpcc_state $file "readwrite" agt1
808 do_facet agt2 $LFS pcc attach -w -i 2 $file ||
809 error "failed to attach file $file"
810 check_lpcc_state $file "readwrite" agt2
811 # The later attach PCC agent should succeed,
812 # the former agent should be detached automatically.
813 check_lpcc_state $file "none" agt1
814 do_facet agt2 $LFS pcc detach -k $file ||
815 error "failed to detach file $file"
816 check_lpcc_state $file "none" agt2
818 echo "Start to RO-PCC attach/detach $file on $agt1_HOST"
819 do_facet agt1 $LFS pcc attach -r -i 1 $file ||
820 error "failed to attach file $file"
821 check_lpcc_state $file "readonly" agt1
822 do_facet agt1 $LFS pcc detach -k $file ||
823 error "failed to detach file $file"
824 check_lpcc_state $file "none" agt1
826 echo "Repeat to RO-PCC attach/detach $file on $agt2_HOST"
827 do_facet agt2 $LFS pcc attach -r -i 2 $file ||
828 error "failed to attach file $file"
829 check_lpcc_state $file "readonly" agt2
830 do_facet agt2 $LFS pcc detach -k $file ||
831 error "failed to detach file $file"
832 check_lpcc_state $file "none" agt2
834 echo "Try RO-PCC attach on two agents"
835 do_facet agt1 $LFS pcc attach -r -i 1 $file ||
836 error "failed to attach file $file"
837 check_lpcc_state $file "readonly" agt1
838 do_facet agt2 $LFS pcc attach -r -i 2 $file ||
839 error "failed to attach file $file"
840 check_lpcc_state $file "readonly" agt2
841 check_lpcc_state $file "readonly" agt1
842 do_facet agt2 $LFS pcc detach -k $file ||
843 error "failed to detach file $file"
844 check_lpcc_state $file "none" agt2
845 do_facet agt1 $LFS pcc detach -k $file ||
846 error "failed to detach file $file"
847 check_lpcc_state $file "none" agt1
849 run_test 3b "Repeat attach/detach operations on multiple clients"
853 local loopfile="$TMP/$tfile"
854 local mntpt="/mnt/pcc.$tdir"
855 local hsm_root="$mntpt/$tdir"
856 local excepts="-e 7 -e 8 -e 9"
858 is_project_quota_supported || skip "project quota is not supported"
861 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
862 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
863 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
865 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
866 lfs project -sp $project_id $DIR/$tdir ||
867 error "lfs project -sp $project_id $DIR/$tdir failed"
869 # 1. mmap_sanity tst7 failed on the local ext4 filesystem.
870 # It seems that Lustre filesystem does special process for tst 7.
871 # 2. Current CentOS8 kernel does not strictly obey POSIX syntax for
872 # mmap() within the maping but beyond current end of the underlying
873 # files: It does not send SIGBUS signals to the process.
874 # 3. For negative file offset, sanity_mmap also failed on 48 bits
875 # ldiksfs backend due to too large offset: "Value too large for
876 # defined data type".
877 # mmap_sanity tst7/tst8/tst9 all failed on Lustre and local ext4.
878 # Thus, we exclude sanity tst7/tst8/tst9 from the PCC testing.
879 $LUSTRE/tests/mmap_sanity -d $DIR/$tdir -m $DIR2/$tdir $excepts ||
880 error "mmap_sanity test failed"
883 # Revoke the layout lock, the PCC-cached file will be
884 # detached automatically.
885 do_facet $SINGLEAGT $LCTL \
886 set_param ldlm.namespaces.*mdc*.lru_size=clear
887 rm -rf $DIR/$tdir || error "failed to remove $DIR/$tdir"
889 run_test 4 "Auto cache test for mmap"
892 local file=$DIR/$tfile
893 local loopfile="$TMP/$tfile"
894 local mntpt="/mnt/pcc.$tdir"
895 local hsm_root="$mntpt/$tdir"
897 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
898 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
899 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
901 do_facet $SINGLEAGT "echo -n attach_mmap_data > $file" ||
902 error "echo $file failed"
904 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
905 error "failed to attach file $file"
906 check_lpcc_state $file "readwrite"
908 local content=$($MMAP_CAT $file)
910 [[ $content == "attach_mmap_data" ]] ||
911 error "mmap cat data mismatch: $content"
913 $LFS hsm_restore $file || error "failed to restore $file"
914 wait_request_state $(path2fid $file) RESTORE SUCCEED
915 check_lpcc_state $file "none"
917 content=$($MMAP_CAT $file)
918 [[ $content == "attach_mmap_data" ]] ||
919 error "mmap cat data mismatch: $content"
921 run_test 5 "Mmap & cat a RW-PCC cached file"
924 local loopfile="$TMP/$tfile"
925 local mntpt="/mnt/pcc.$tdir"
926 local hsm_root="$mntpt/$tdir"
927 local file=$DIR/$tfile
930 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
931 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
932 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
934 echo -n mmap_write_data > $file || error "echo write $file failed"
935 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
936 error "failed to attach file $file"
937 check_lpcc_state $file "readwrite"
939 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
940 error "could not mmap $file"
941 check_lpcc_state $file "readwrite"
942 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
943 # After mmap write via multiop, the first character of each page
945 [[ $content == "nmap_write_data" ]] ||
946 error "mmap write data mismatch: $content"
947 check_lpcc_state $file "readwrite"
949 do_facet $SINGLEAGT $LFS pcc detach $file ||
950 error "failed to detach file $file"
951 wait_request_state $(path2fid $file) REMOVE SUCCEED
953 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
954 [[ $content == "nmap_write_data" ]] ||
955 error "mmap write data mismatch: $content"
957 run_test 6 "Test mmap write on RW-PCC "
960 local loopfile="$TMP/$tfile"
961 local mntpt="/mnt/pcc.$tdir"
962 local hsm_root="$mntpt/$tdir"
963 local file=$DIR/$tfile
966 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
967 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
968 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
971 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
972 error "failed to attach file $file"
973 check_lpcc_state $file "readwrite"
974 check_file_data $SINGLEAGT $file "QQQQQ"
975 # define OBD_FAIL_LLITE_PCC_DETACH_MKWRITE 0x1412
976 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1412
977 # HSM released exists archived status
978 check_hsm_flags $file "0x0000000d"
980 # multiop mmap write increase the first character of each page with 1
981 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
982 error "mmap write $file failed"
983 check_lpcc_state $file "none"
984 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
985 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
987 run_test 7a "Fake file detached between fault() and page_mkwrite() for RW-PCC"
990 local loopfile="$TMP/$tfile"
991 local mntpt="/mnt/pcc.$tdir"
992 local hsm_root="$mntpt/$tdir"
993 local file=$DIR/$tfile
997 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
998 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
999 setup_pcc_mapping $SINGLEAGT \
1000 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
1002 echo "QQQQQ" > $file
1003 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1004 error "failed to attach file $file"
1005 check_lpcc_state $file "readwrite"
1006 check_file_data $SINGLEAGT $file "QQQQQ"
1007 # define OBD_FAIL_LLITE_PCC_MKWRITE_PAUSE 0x1413
1008 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1413 fail_val=20
1009 # HSM released exists archived status
1010 check_hsm_flags $file "0x0000000d"
1012 # multiop mmap write increases the first character of each page with 1
1013 do_facet $SINGLEAGT $MULTIOP $file OSMWUc &
1017 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1018 error "failed to detach file $file"
1020 wait $pid || error "multiop mmap write failed"
1021 check_lpcc_state $file "none"
1022 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
1023 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
1025 run_test 7b "Test the race with concurrent mkwrite and detach"
1028 local loopfile="$TMP/$tfile"
1029 local mntpt="/mnt/pcc.$tdir"
1030 local hsm_root="$mntpt/$tdir"
1031 local file=$DIR/$tfile
1033 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1034 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1035 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1037 echo "QQQQQ" > $file
1038 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1039 error "failed to attach file $file"
1040 check_lpcc_state $file "readwrite"
1041 check_file_data $SINGLEAGT $file "QQQQQ"
1043 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
1044 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
1045 do_facet $SINGLEAGT "echo -n ENOSPC_write > $file"
1046 # Above write will return -ENOSPC failure and retry the IO on normal
1047 # IO path. It will restore the HSM released file.
1048 check_lpcc_state $file "none"
1049 check_file_data $SINGLEAGT $file "ENOSPC_write"
1051 run_test 8 "Test fake -ENOSPC tolerance for RW-PCC"
1054 local loopfile="$TMP/$tfile"
1055 local mntpt="/mnt/pcc.9a"
1056 local hsm_root="$mntpt/$tdir"
1057 local file=$DIR/$tfile
1059 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1061 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
1062 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1063 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1065 touch $file || error "touch $file failed"
1066 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1067 error "fail to attach $file"
1068 check_lpcc_state $file "readwrite"
1069 # write 60M data, it is larger than the capacity of PCC backend
1070 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=60 ||
1071 error "fail to dd write $file"
1072 check_lpcc_state $file "none"
1073 check_file_size $SINGLEAGT $file 62914560
1075 run_test 9 "Test -ENOSPC tolerance on loop PCC device for RW-PCC"
1077 test_usrgrp_quota() {
1078 local loopfile="$TMP/$tfile"
1079 local mntpt="/mnt/pcc.$tdir"
1080 local hsm_root="$mntpt/$tdir"
1081 local state="readonly"
1087 [[ $ug == "g" ]] && id=$RUNAS_GID
1093 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1094 do_facet $SINGLEAGT quotacheck -c$ug $mntpt ||
1095 error "quotacheck -c$ug $mntpt failed"
1096 do_facet $SINGLEAGT quotaon -$ug $mntpt ||
1097 error "quotaon -$ug $mntpt failed"
1098 do_facet $SINGLEAGT setquota -$ug $id 0 20480 0 0 $mntpt ||
1099 error "setquota -$ug $id on $mntpt failed"
1100 do_facet $SINGLEAGT repquota -${ug}vs $mntpt
1102 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
1103 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ $mode=1"
1104 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1106 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1108 local file1=$DIR/$tdir/${ug}quotaA
1109 local file2=$DIR/$tdir/${ug}quotaB
1111 dd if=/dev/zero of=$file1 bs=1M count=15 ||
1112 error "dd write $file1 failed"
1113 dd if=/dev/zero of=$file2 bs=1M count=15 ||
1114 error "dd write $file2 failed"
1115 chown $RUNAS_ID:$RUNAS_GID $file1 ||
1116 error "chown $RUNAS_ID:$RUNAS_GID $file1 failed"
1117 chown $RUNAS_ID:$RUNAS_GID $file2 ||
1118 error "chown $RUNAS_ID:$RUNAS_GID $file2 failed"
1119 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $rw \
1120 $file1 || error "attach $file1 failed"
1121 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $rw \
1122 $file2 && error "attach $file2 should fail due to quota limit"
1123 check_lpcc_state $file1 $state
1124 check_lpcc_state $file2 "none"
1126 if [[ -z $rw ]]; then
1127 do_facet $SINGLEAGT $LFS pcc detach $file1 ||
1128 error "detach $file1 failed"
1132 echo "Test -EDQUOT error tolerance for RW-PCC"
1133 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file1 bs=1M count=30 ||
1134 error "dd write $file1 failed"
1135 # -EDQUOT error should be tolerated via fallback to normal Lustre path.
1136 check_lpcc_state $file1 "none"
1140 test_usrgrp_quota "u" "-w"
1142 run_test 10a "Test RW-PCC with user quota on loop PCC device"
1145 test_usrgrp_quota "g" "-w"
1147 run_test 10b "Test RW-PCC with group quota on loop PCC device"
1150 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1151 skip "Server does not support PCC-RO"
1153 test_usrgrp_quota "u"
1155 run_test 10c "Test RO-PCC with user quota on loop PCC device"
1158 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1159 skip "Server does not support PCC-RO"
1161 test_usrgrp_quota "g"
1163 run_test 10d "Test RO-PCC with group quota on loop PCC device"
1165 test_usrgrp_edquot() {
1166 local loopfile="$TMP/$tfile"
1167 local mntpt="/mnt/pcc.$tdir"
1168 local hsm_root="$mntpt/$tdir"
1169 local file=$DIR/$tfile
1173 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1174 skip "Server does not support PCC-RO"
1176 [[ $ug == "g" ]] && id=$RUNAS_GID
1177 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1178 do_facet $SINGLEAGT quotacheck -c$ug $mntpt ||
1179 error "quotacheck -c$ug $mntpt failed"
1180 do_facet $SINGLEAGT quotaon -$ug $mntpt ||
1181 error "quotaon -$ug $mntpt failed"
1182 do_facet $SINGLEAGT setquota -$ug $id 0 4096 0 0 $mntpt ||
1183 error "setquota -$ug $id on $mntpt failed"
1184 do_facet $SINGLEAGT repquota -${ug}vs $mntpt
1185 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
1186 setup_pcc_mapping $SINGLEAGT \
1187 "${ug}id={$id}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
1188 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1190 dd if=/dev/zero of=$file bs=1M count=2 ||
1191 error "dd write $file failed"
1192 chown $RUNAS_ID:$RUNAS_GID $file ||
1193 error "chown $RUNAS_ID:$RUNAS_GID $file failed"
1194 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1M count=2 ||
1195 error "dd read $file failed"
1196 check_lpcc_state $file "readonly"
1197 $LFS getstripe -v $file
1198 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=5 ||
1199 error "dd write $file failed"
1200 check_lpcc_state $file "none"
1201 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1M count=5 ||
1202 error "dd read $file failed"
1203 do_facet $SINGLEAGT $LFS pcc state $file
1204 $LFS getstripe -v $file
1205 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1206 error "PCC-RO attach $file failed"
1208 do_facet $SINGLEAGT $LFS pcc detach $file || error "detach $file failed"
1212 test_usrgrp_edquot "u"
1214 run_test 10e "Tolerate -EDQUOT failure when auto PCC-RO attach with user quota"
1217 test_usrgrp_edquot "g"
1219 run_test 10f "Tolerate -EDQUOT failure when auto PCC-RO attach with group quota"
1222 local loopfile="$TMP/$tfile"
1223 local mntpt="/mnt/pcc.$tdir"
1224 local hsm_root="$mntpt/$tdir"
1225 local file=$DIR/$tfile
1229 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1230 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1231 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1233 do_facet $SINGLEAGT "echo -n QQQQQ > $file"
1234 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1235 lpcc_dir=$(dirname $lpcc_path)
1236 echo "Lustre file: $file LPCC dir: $lpcc_dir"
1237 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
1238 error "mkdir -p $lpcc_dir failed"
1239 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1240 error "failed to attach $file"
1241 check_lpcc_state $file "readwrite"
1242 check_file_data $SINGLEAGT $file "QQQQQ"
1243 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1244 error "failed to detach $file"
1245 rm $file || error "rm $file failed"
1247 # The parent directory of the PCC file is immutable
1248 do_facet $SINGLEAGT "echo -n immutable_dir > $file"
1249 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1250 lpcc_dir=$(dirname $lpcc_path)
1251 echo "Lustre file: $file LPCC dir: $lpcc_dir"
1252 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
1253 error "mkdir -p $lpcc_dir failed"
1254 do_facet $SINGLEAGT chattr +i $lpcc_dir ||
1255 error "chattr +i $lpcc_dir failed"
1256 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file &&
1257 error "attach $file with immutable directory should be failed"
1258 do_facet $SINGLEAGT chattr -i $lpcc_dir ||
1259 error "chattr -i $lpcc_dir failed"
1260 rm $file || error "rm $file failed"
1262 # The PCC file path is set to a directory
1263 do_facet $SINGLEAGT "echo -n pcc_file_path_is_dir > $file"
1264 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1265 do_facet $SINGLEAGT mkdir -p $lpcc_path ||
1266 error "mkdir -p $lpcc_path failed"
1267 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file &&
1268 error "attach $file should fail as PCC path is a directory"
1269 rm $file || error "rm $file failed"
1271 run_test 11 "Test attach fault injection with simulated PCC file path"
1274 local file=$DIR/$tfile
1275 local hsm_root=$(hsm_root)
1279 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1280 setup_pcc_mapping $SINGLEAGT \
1281 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
1283 echo -n race_rw_attach_hsmremove > $file
1284 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1285 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1286 error "attach $file failed"
1287 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1288 error "detach $file failed"
1289 # HSM released exists archived status
1290 check_hsm_flags $file "0x0000000d"
1291 # define OBD_FAIL_LLITE_PCC_ATTACH_PAUSE 0x1414
1292 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1414 fail_val=20
1293 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file &
1295 $LFS hsm_state $file
1297 wait_request_state $(path2fid $file) RESTORE SUCCEED
1298 $LFS hsm_remove $file || error "hsm remove $file failed"
1300 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1301 error "RW-PCC cached file '$lpcc_path' should be removed"
1305 run_test 12 "RW-PCC attach races with concurrent HSM remove"
1308 local idstr="${1}id"
1309 local rule="${idstr}={$2}"
1311 local file=$DIR/$tdir/$tfile
1313 setup_pcc_mapping $SINGLEAGT \
1314 "$rule\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
1315 $LCTL pcc list $MOUNT
1317 do_facet $SINGLEAGT $LFS mkdir -i 0 $DIR/$tdir
1318 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1320 rm -f $file || error "rm $file failed"
1321 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1322 error "failed to dd write to $file"
1323 check_lpcc_state $file "readwrite"
1324 do_facet $SINGLEAGT $myRUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
1325 error "failed to dd read from $file"
1326 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 256 ||
1327 error "failed to truncate $file"
1328 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 2048 ||
1329 error "failed to truncate $file"
1330 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1331 error "failed to dd write from $file"
1332 check_lpcc_state $file "readwrite"
1334 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1335 error "failed to detach file $file"
1336 wait_request_state $(path2fid $file) REMOVE SUCCEED
1337 check_lpcc_state $file "none"
1343 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1344 test_rule_id "u" "$RUNAS_ID" "runas -u $RUNAS_ID"
1345 test_rule_id "g" "$RUNAS_GID" "runas -u $RUNAS_ID -g $RUNAS_GID"
1347 run_test 13a "Test auto RW-PCC create caching for UID/GID rule"
1352 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1353 setup_pcc_mapping $SINGLEAGT \
1354 "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
1355 $LCTL pcc list $MOUNT
1357 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1358 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1360 file=$DIR/$tdir/prefix.h5
1361 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1362 error "failed to dd write to $file"
1363 check_lpcc_state $file "readwrite"
1364 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1365 error "failed to detach file $file"
1366 check_lpcc_state $file "none"
1367 rm $file || error "rm $file failed"
1369 file=$DIR/$tdir/suffix.doc
1370 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1371 error "failed to dd write to $file"
1372 check_lpcc_state $file "readwrite"
1373 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1374 error "failed to detach file $file"
1375 check_lpcc_state $file "none"
1376 rm $file || error "rm $file failed"
1378 file=$DIR/$tdir/MidPADdle
1379 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1380 error "failed to dd write to $file"
1381 check_lpcc_state $file "readwrite"
1382 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach -k $file ||
1383 error "failed to detach file $file"
1384 check_lpcc_state $file "none"
1385 rm $file || error "rm $file failed"
1387 file=$DIR/$tdir/Midpad
1388 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1389 error "failed to dd write to $file"
1390 check_lpcc_state $file "none"
1391 rm $file || error "rm $file failed"
1393 run_test 13b "Test auto RW-PCC create caching for file name with wildcard"
1398 local loopfile="$TMP/$tfile"
1399 local mntpt="/mnt/pcc.$tdir"
1400 local hsm_root="$mntpt/$tdir"
1402 is_project_quota_supported || skip "project quota is not supported"
1404 enable_project_quota
1405 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1406 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1407 setup_pcc_mapping $SINGLEAGT \
1408 "projid={100\ 200}\&fname={*.h5},uid={$RUNAS_ID}\&gid={$RUNAS_GID}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1409 $LCTL pcc list $MOUNT
1410 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1411 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1413 mkdir -p $DIR/$tdir/proj || error "mkdir $DIR/$tdir/proj failed"
1414 mkdir -p $DIR/$tdir/proj2 || error "mkdir $DIR/$tdir/proj2 failed"
1415 $LFS project -sp 100 $DIR/$tdir/proj ||
1416 error "failed to set project for $DIR/$tdir/proj"
1417 $LFS project -sp 200 $DIR/$tdir/proj2 ||
1418 error "failed to set project for $DIR/$tdir/proj2"
1420 file=$DIR/$tdir/proj/notcache
1421 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1422 error "failed to dd write to $file"
1423 check_lpcc_state $file "none"
1424 rm $file || error "rm $file failed"
1426 file=$DIR/$tdir/proj/autocache.h5
1427 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1428 error "failed to dd write to $file"
1429 check_lpcc_state $file "readwrite"
1430 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1431 error "failed to detach $file"
1432 rm $file || error "rm $file failed"
1434 file=$DIR/$tdir/proj2/notcache
1435 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1436 error "failed to dd write to $file"
1437 check_lpcc_state $file "none"
1438 rm $file || error "rm $file failed"
1440 file=$DIR/$tdir/proj2/autocache.h5
1441 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1442 error "failed to dd write to $file"
1443 check_lpcc_state $file "readwrite"
1444 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1445 error "failed to detach $file"
1446 rm $file || error "rm $file failed"
1448 file=$DIR/$tdir/ugidcache
1449 myRUNAS="runas -u $RUNAS_ID -g $RUNAS_GID"
1450 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1451 error "failed to dd write to $file"
1452 check_lpcc_state $file "readwrite"
1453 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1454 error "failed to detach $file"
1455 rm $file || error "rm $file failed"
1457 run_test 13c "Check auto RW-PCC create caching for UID/GID/ProjID/fname rule"
1460 local file=$DIR/$tdir/$tfile
1462 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1463 skip "Server does not support PCC-RO"
1465 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1466 setup_pcc_mapping $SINGLEAGT \
1467 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1\ pccro=1"
1469 mkdir -p $DIR/$tdir || error "mkdir -p $DIR/$tdir failed"
1470 do_facet $SINGLEAGT "echo -n autodetach_data > $file"
1471 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
1472 $file || error "PCC attach $file failed"
1473 check_lpcc_state $file "readwrite"
1475 # Revoke the layout lock, the PCC-cached file will be
1476 # detached automatically.
1477 do_facet $SINGLEAGT $LCTL \
1478 set_param ldlm.namespaces.*mdc*.lru_size=clear
1479 check_file_data $SINGLEAGT $file "autodetach_data"
1480 check_lpcc_state $file "none"
1482 rm $file || error "rm $file failed"
1483 do_facet $SINGLEAGT "echo -n ro_autodetach_data > $file"
1484 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1485 error "PCC attach $file failed"
1486 check_lpcc_state $file "readonly"
1488 # Revoke the layout lock, the PCC-cached file will be
1489 # detached automatically.
1490 do_facet $SINGLEAGT $LCTL \
1491 set_param ldlm.namespaces.*mdc*.lru_size=clear
1492 check_file_data $SINGLEAGT $file "ro_autodetach_data"
1493 check_lpcc_state $file "none"
1495 run_test 14 "Revocation of the layout lock should detach the file automatically"
1498 local loopfile="$TMP/$tfile"
1499 local mntpt="/mnt/pcc.$tdir"
1500 local hsm_root="$mntpt/$tdir"
1501 local file=$DIR/$tdir/$tfile
1503 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1504 skip "Server does not support PCC-RO"
1506 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1507 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1508 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1\ pccro=1"
1510 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
1511 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
1513 echo "Verify open attach for non-root user"
1514 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1515 error "failed to dd write to $file"
1516 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
1517 $file || error "failed to attach file $file"
1518 do_facet $SINGLEAGT $RUNAS $LFS pcc state $file
1519 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1520 # Revoke the layout lock, the PCC-cached file will be
1521 # detached automatically.
1522 do_facet $SINGLEAGT $LCTL \
1523 set_param ldlm.namespaces.*mdc*.lru_size=clear
1524 check_lpcc_state $file "none" $SINGLEAGT "$RUNAS"
1525 do_facet $SINGLEAGT $RUNAS $MULTIOP $file oc ||
1526 error "failed to open $file"
1527 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1528 # Detach the file but keep the cache , as the file layout generation
1529 # is not changed, so the file is still valid cached in PCC, and can
1530 # be reused from PCC cache directly.
1531 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
1532 error "PCC detach $file failed"
1533 check_lpcc_state $file "none" $SINGLEAGT "$RUNAS"
1534 do_facet $SINGLEAGT $RUNAS $MULTIOP $file oc ||
1535 error "failed to open $file"
1536 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
1537 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
1538 error "PCC detach $file failed"
1539 rm $file || error "rm $file failed"
1541 echo "Verify auto attach at open for RW-PCC"
1542 do_facet $SINGLEAGT "echo -n autoattach_data > $file"
1543 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
1544 $file || error "RW-PCC attach $file failed"
1545 check_lpcc_state $file "readwrite"
1547 # Revoke the layout lock, the PCC-cached file will be
1548 # detached automatically.
1549 do_facet $SINGLEAGT $LCTL \
1550 set_param ldlm.namespaces.*mdc*.lru_size=clear
1551 check_file_data $SINGLEAGT $file "autoattach_data"
1552 check_lpcc_state $file "readwrite"
1554 # Detach the file with -k option, as the file layout generation
1555 # is not changed, so the file is still valid cached in PCC,
1556 # and can be reused from PCC cache directly.
1557 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1558 error "RW-PCC detach $file failed"
1559 check_lpcc_state $file "none"
1560 do_facet $SINGLEAGT $MULTIOP $file oc || error "failed to open $file"
1561 check_lpcc_state $file "readwrite"
1562 # HSM released exists archived status
1563 check_hsm_flags $file "0x0000000d"
1564 check_file_data $SINGLEAGT $file "autoattach_data"
1566 # HSM restore the PCC cached file, the layout generation
1567 # was changed, so the file can not be auto attached.
1568 $LFS hsm_restore $file || error "failed to restore $file"
1569 wait_request_state $(path2fid $file) RESTORE SUCCEED
1570 check_lpcc_state $file "none"
1571 # HSM exists archived status
1572 check_hsm_flags $file "0x00000009"
1574 echo "Verify auto attach at open for RO-PCC"
1575 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1576 error "RO-PCC attach $file failed"
1577 check_lpcc_state $file "readonly"
1579 # Revoke the layout lock, the PCC-cached file will be
1580 # detached automatically.
1581 do_facet $SINGLEAGT $LCTL \
1582 set_param ldlm.namespaces.*mdc*.lru_size=clear
1583 check_file_data $SINGLEAGT $file "autoattach_data"
1584 check_lpcc_state $file "readonly"
1586 # Detach the file with "-k" option, as the file layout generation
1587 # is not changed, so the file is still valid cached in PCC,
1588 # and can be reused from PCC cache directly.
1589 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1590 error "RO-PCC detach $file failed"
1591 check_lpcc_state $file "none"
1592 do_facet $SINGLEAGT $MULTIOP $file oc || error "failed to open $file"
1593 check_lpcc_state $file "readonly"
1594 check_file_data $SINGLEAGT $file "autoattach_data"
1595 do_facet $SINGLEAGT $LFS pcc detach $file ||
1596 error "RO-PCC detach $file failed"
1598 run_test 15 "Test auto attach at open when file is still valid cached"
1601 local loopfile="$TMP/$tfile"
1602 local mntpt="/mnt/pcc.$tdir"
1603 local hsm_root="$mntpt/$tdir"
1604 local file=$DIR/$tfile
1607 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1608 skip "Server does not support PCC-RO"
1610 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1611 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1612 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1\ pccro=1"
1614 echo "Test detach for RW-PCC"
1615 do_facet $SINGLEAGT "echo -n detach_data > $file"
1616 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1617 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
1618 $file || error "RW-PCC attach $file failed"
1619 check_lpcc_state $file "readwrite"
1620 # HSM released exists archived status
1621 check_hsm_flags $file "0x0000000d"
1623 echo "Test for reusing valid PCC cache"
1624 # Valid PCC cache can be reused
1625 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1626 error "PCC detach $file failed"
1627 check_lpcc_state $file "none"
1628 # HSM released exists archived status
1629 check_hsm_flags $file "0x0000000d"
1631 echo "Test for the default detach"
1632 # Permanent detach by default, it will remove the PCC copy
1633 do_facet $SINGLEAGT $LFS pcc detach $file ||
1634 error "RW-PCC detach $file failed"
1635 wait_request_state $(path2fid $file) REMOVE SUCCEED
1636 check_lpcc_state $file "none"
1637 # File is removed from PCC backend
1638 check_hsm_flags $file "0x00000000"
1639 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1640 error "RW-PCC cached file '$lpcc_path' should be removed"
1642 echo "Test detach for RO-PCC"
1643 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1644 error "RO-PCC attach $file failed"
1645 check_lpcc_state $file "readonly"
1647 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1648 error "RO-PCC detach $file failed"
1649 check_lpcc_state $file "none"
1650 # Reading the file will re-attach the file in readonly mode
1651 do_facet $SINGLEAGT cat $file || error "cat $file failed"
1652 check_lpcc_state $file "readonly"
1654 do_facet $SINGLEAGT $LFS pcc detach $file ||
1655 error "RO-PCC detach $file failed"
1656 check_lpcc_state $file "none"
1657 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
1658 error "RO-PCC cached file '$lpcc_path' should be removed"
1662 run_test 16 "Test detach with different options"
1665 local agt_host=$(facet_active_host $SINGLEAGT)
1666 local loopfile="$TMP/$tfile"
1667 local mntpt="/mnt/pcc.$tdir"
1668 local hsm_root="$mntpt/$tdir"
1669 local file=$DIR/$tfile
1671 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1672 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1673 setup_pcc_mapping $SINGLEAGT \
1674 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=0\ stat_attach=0\ pccrw=1"
1676 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1678 do_facet $SINGLEAGT "echo -n layout_refresh_data > $file"
1679 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1680 error "PCC attach $file failed"
1681 check_lpcc_state $file "readwrite"
1683 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1684 error "PCC detach $file failed"
1685 check_lpcc_state $file "none"
1687 # Truncate should attach the file into PCC automatically
1688 # as the PCC copy is still valid.
1689 echo "Verify auto attach during IO for truncate"
1690 do_facet $SINGLEAGT $TRUNCATE $file 4 || error "truncate $file failed"
1691 check_lpcc_state $file "readwrite"
1693 echo "Verify auto attach during IO for read/write"
1694 rmultiop_start $agt_host $file O_r || error "open $file failed"
1697 # Revoke the layout lock, the PCC-cached file will be
1698 # detached automatically.
1699 do_facet $SINGLEAGT $LCTL \
1700 set_param ldlm.namespaces.*mdc*.lru_size=clear
1702 check_lpcc_state $file "none"
1703 rmultiop_stop $agt_host || error "close $file failed"
1705 check_lpcc_state $file "readwrite"
1707 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1708 error "PCC detach $file failed"
1709 check_lpcc_state $file "none"
1711 run_test 17 "Test auto attach for layout refresh"
1714 local agt_host=$(facet_active_host $SINGLEAGT)
1715 local loopfile="$TMP/$tfile"
1716 local mntpt="/mnt/pcc.$tdir"
1717 local hsm_root="$mntpt/$tdir"
1718 local file=$DIR/$tfile
1722 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1723 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1724 setup_pcc_mapping $SINGLEAGT \
1725 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1727 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
1728 do_facet $SINGLEAGT dd if=/dev/urandom of=$file bs=1M count=4 ||
1729 error "failed to write $file"
1730 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1731 error "failed to attach $file"
1732 do_facet $SINGLEAGT $LFS pcc state $file
1733 check_lpcc_state $file "readwrite"
1734 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1735 error "failed to detach $file"
1736 do_facet $SINGLEAGT $LFS pcc state $file
1737 $CHECKSTAT -s 4194304 $file
1738 dd if=/dev/zero of=$DIR2/$tfile seek=1k bs=1k count=1 ||
1739 error "failed to write $DIR2/$tfile"
1740 oldmd5=$(md5sum $DIR2/$tfile | awk '{print $1}')
1741 $CHECKSTAT -s 1049600 $DIR2/$tfile || error "$DIR2/$tfile size wrong"
1743 local lpcc_path=$(lpcc_fid2path $hsm_root $file)
1745 do_facet $SINGLEAGT $LFS pcc state $file
1746 check_file_size $SINGLEAGT $lpcc_path 4194304
1747 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1748 error "failed to attach $file"
1749 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1049600
1750 newmd5=$(do_facet $SINGLEAGT md5sum $file | awk '{print $1}')
1751 [ "$oldmd5" == "$newmd5" ] || error "md5sum differ: $oldmd5 != $newmd5"
1752 do_facet $SINGLEAGT $LFS pcc detach $file ||
1753 error "failed to detach $file"
1755 run_test 18 "Verify size correctness after re-attach the file"
1758 local agt_host=$(facet_active_host $SINGLEAGT)
1759 local loopfile="$TMP/$tfile"
1760 local mntpt="/mnt/pcc.$tdir"
1761 local hsm_root="$mntpt/$tdir"
1762 local file=$DIR/$tfile
1764 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1765 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1766 setup_pcc_mapping $SINGLEAGT \
1767 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
1769 do_facet $SINGLEAGT "echo -n QQQQQ > $file" || error "echo $file failed"
1770 lpcc_path=$(lpcc_fid2path $hsm_root $file)
1771 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1772 error "Failed to attach $file"
1773 check_lpcc_state $file "readwrite"
1774 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1775 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1776 error "Failed to detach $file"
1777 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1778 error "Failed to attach $file"
1779 check_lpcc_sizes $SINGLEAGT $file $lpcc_path 5
1780 do_facet $SINGLEAGT $LFS pcc detach --keep $file ||
1781 error "Failed to detach $file"
1783 run_test 19 "Verify the file re-attach works as expected"
1786 local agt_host=$(facet_active_host $SINGLEAGT)
1787 local loopfile="$TMP/$tfile"
1788 local mntpt="/mnt/pcc.$tdir"
1789 local hsm_root="$mntpt/$tdir"
1790 local file=$DIR/$tfile
1792 setup_loopdev $SINGLEAGT $loopfile $mntpt 120
1793 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1794 setup_pcc_mapping $SINGLEAGT \
1795 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
1797 do_facet $SINGLEAGT "echo -n QQQQQ > $file" ||
1798 error "echo $file failed"
1799 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
1800 error "Failed to attach $file"
1801 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1802 check_lpcc_state $file "readwrite"
1803 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1804 do_facet $SINGLEAGT "echo 3 > /proc/sys/vm/drop_caches"
1805 check_lpcc_state $file "readwrite"
1806 do_facet $SINGLEAGT $LFS pcc detach $file ||
1807 error "Failed to detach $file"
1809 run_test 20 "Auto attach works after the inode was once evicted from cache"
1812 local loopfile="$TMP/$tfile"
1813 local mntpt="/mnt/pcc.$tdir"
1814 local hsm_root="$mntpt/$tdir"
1815 local file=$DIR/$tfile
1817 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1818 skip "Server does not support PCC-RO"
1820 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1821 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1822 setup_pcc_mapping $SINGLEAGT \
1823 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1825 do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1826 echo "Plain layout info before PCC-RO attach '$file':"
1827 $LFS getstripe -v $file
1828 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1829 error "RW-PCC attach $file failed"
1830 check_lpcc_state $file "readonly"
1831 echo -e "\nFLR layout info after PCC-RO attach '$file':"
1832 $LFS getstripe -v $file
1833 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1834 error "failed to detach file $file"
1835 check_lpcc_state $file "none"
1836 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1837 $LFS getstripe -v $file
1839 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1840 error "failed to attach file $file"
1841 check_lpcc_state $file "readonly"
1842 echo -e "\nFLR layout info after RO-PCC attach $file again:"
1843 $LFS getstripe -v $file
1844 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1845 error "failed to detach file $file"
1846 check_lpcc_state $file "none"
1847 echo -e "\nFLR layout info after RO-PCC detach '$file' again:"
1848 $LFS getstripe -v $file
1850 run_test 21a "PCC-RO storing as a plain HSM mirror component for plain layout"
1853 local loopfile="$TMP/$tfile"
1854 local mntpt="/mnt/pcc.$tdir"
1855 local hsm_root="$mntpt/$tdir"
1856 local file=$DIR/$tfile
1858 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1859 skip "Server does not support PCC-RO"
1861 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1862 do_facet $SINGLEAGT mkdir -p $hsm_root ||
1863 error "failed to mkdir $hsm_root"
1864 setup_pcc_mapping $SINGLEAGT \
1865 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1867 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
1868 error "create mirrored file $file failed"
1869 #do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
1870 echo "FLR layout before PCC-RO attach '$file':"
1871 $LFS getstripe -v $file
1872 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1873 error "failed to attach file $file"
1874 check_lpcc_state $file "readonly"
1875 echo -e "\nFLR layout after PCC-RO attach '$file':"
1876 $LFS getstripe -v $file
1877 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1878 error "failed to detach file $file"
1879 check_lpcc_state $file "none"
1880 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1881 $LFS getstripe -v $file
1883 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1884 error "failed to attach file $file"
1885 check_lpcc_state $file "readonly"
1886 echo -e "\nFLR layout after PCC-RO attach '$file' again:"
1887 $LFS getstripe -v $file
1888 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1889 error "failed to detach file $file"
1890 check_lpcc_state $file "none"
1891 echo -e "\nFLR layout info after PCC-RO detach '$file':"
1892 $LFS getstripe -v $file
1894 run_test 21b "PCC-RO stroing as a plain HSM mirror component for FLR layouts"
1897 local loopfile="$TMP/$tfile"
1898 local mntpt="/mnt/pcc.$tdir"
1899 local hsm_root="$mntpt/$tdir"
1900 local file=$DIR/$tfile
1903 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1904 skip "Server does not support PCC-RO"
1906 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1907 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1908 setup_pcc_mapping $SINGLEAGT \
1909 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
1911 do_facet $SINGLEAGT "echo -n pccro_hsm_release > $file"
1912 fid=$(path2fid $file)
1913 $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file ||
1914 error "Archive $file failed"
1915 wait_request_state $fid ARCHIVE SUCCEED
1916 $LFS hsm_state $file
1918 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
1919 error "RO-PCC attach $file failed"
1920 # HSM exists archived status
1921 check_hsm_flags $file "0x00000009"
1922 check_lpcc_state $file "readonly"
1923 check_file_data $SINGLEAGT $file "pccro_hsm_release"
1925 $LFS hsm_release $file || error "HSM released $file failed"
1926 $LFS getstripe $file
1927 $LFS hsm_state $file
1928 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
1929 error "failed to detach $file"
1930 check_lpcc_state $file "none"
1932 unlink $file || error "unlink $file failed"
1934 run_test 21c "Verify HSM release works storing PCC-RO as HSM mirror component"
1937 local loopfile="$TMP/$tfile"
1938 local mntpt="/mnt/pcc.$tdir"
1939 local hsm_root="$mntpt/$tdir"
1940 local file=$DIR/$tfile
1942 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1943 skip "Server does not support PCC-RO"
1945 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1946 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1949 echo "pccro_init_data" > $file
1950 $LFS getstripe $file
1951 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1952 error "failed to PCC-RO attach file $file"
1953 check_lpcc_state $file "readonly"
1954 echo "PCC-RO attach '$file':"
1955 $LFS getstripe -v $file
1957 echo "Write invalidated PCC-RO cache:"
1958 echo -n "write_mod_data" > $file
1959 check_lpcc_state $file "none"
1960 $LFS getstripe -v $file
1961 check_file_data $SINGLEAGT $file "write_mod_data"
1963 run_test 21d "Write should invalidate PCC-RO caching"
1966 local loopfile="$TMP/$tfile"
1967 local mntpt="/mnt/pcc.$tdir"
1968 local hsm_root="$mntpt/$tdir"
1969 local file=$DIR/$tfile
1971 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
1972 skip "Server does not support PCC-RO"
1974 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
1975 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1978 echo "pccro_init_data" > $file
1979 $LFS getstripe $file
1980 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
1981 error "failed to PCC-RO attach file $file"
1982 check_lpcc_state $file "readonly"
1983 echo "PCC-RO attach '$file':"
1984 $LFS getstripe -v $file
1986 echo "Trucate invalidate PCC-RO file '$file':"
1987 $TRUNCATE $file 256 || error "failed to truncate $file"
1988 $LFS getstripe -v $file
1989 check_lpcc_state $file "none"
1990 check_file_size $SINGLEAGT $file 256
1992 run_test 21e "Truncate should invalidate PCC-RO caching"
1995 local loopfile="$TMP/$tfile"
1996 local mntpt="/mnt/pcc.$tdir"
1997 local hsm_root="$mntpt/$tdir"
1998 local file=$DIR/$tfile
2000 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2001 skip "Server does not support PCC-RO"
2003 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2004 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2007 echo "pccro_mmap_data" > $file
2008 $LFS getstripe $file
2009 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2010 error "failed to PCC-RO attach file $file"
2011 check_lpcc_state $file "readonly"
2012 echo "PCC-RO attach '$file':"
2013 $LFS getstripe -v $file
2015 echo "Mmap write invalidate PCC-RO caching:"
2016 # Mmap write will invalidate the RO-PCC cache
2017 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
2018 error "mmap write $file failed"
2019 check_lpcc_state $file "none"
2020 $LFS getstripe -v $file
2021 # After mmap-write by MULTIOP, the first character of the content
2022 # will be increased with 1.
2023 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2024 [[ $content == "qccro_mmap_data" ]] ||
2025 error "mmap_cat data mismatch: $content"
2027 run_test 21f "mmap write should invalidate PCC-RO caching"
2030 local loopfile="$TMP/$tfile"
2031 local mntpt="/mnt/pcc.$tdir"
2032 local hsm_root="$mntpt/$tdir"
2033 local file=$DIR/$tfile
2035 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2036 skip "Server does not support PCC-RO"
2038 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2039 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2040 setup_pcc_mapping $SINGLEAGT \
2041 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2043 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
2044 error "create mirrored file '$file' failed"
2045 do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
2046 echo "FLR layout before PCC-RO attach '$file':"
2047 $LFS getstripe -v $file
2048 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2049 error "failed to PCC-RO attach '$file'"
2050 echo "FLR layout after PCC-RO attach '$file':"
2051 $LFS getstripe -v $file
2052 echo "Layout after Write invalidate '$file':"
2053 echo -n pccro_write_invalidate_mirror > $file
2054 $LFS getstripe -v $file
2056 run_test 21g "PCC-RO for file under FLR write pending state"
2059 local loopfile="$TMP/$tfile"
2060 local mntpt="/mnt/pcc.$tdir"
2061 local hsm_root="$mntpt/$tdir"
2062 local file=$DIR/$tfile
2064 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2065 skip "Server does not support PCC-RO"
2067 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2068 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2069 setup_pcc_mapping $SINGLEAGT \
2070 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2072 $LFS mirror create -N -S 4M -c 2 -N -S 1M -c -1 $file ||
2073 error "create mirrored file $file failed"
2074 #do_facet $SINGLEAGT "echo -n pccro_as_mirror_layout > $file"
2075 echo "FLR layout before PCC-RO attach '$file':"
2076 $LFS getstripe -v $file
2077 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2078 error "failed to attach file $file"
2079 check_lpcc_state $file "readonly"
2080 echo -e "\nFLR layout after PCC-RO attach '$file':"
2081 $LFS getstripe -v $file
2083 $LFS mirror extend -N -S 8M -c -1 $file ||
2084 error "mirror extend $file failed"
2085 echo -e "\nFLR layout after extend a mirror:"
2086 $LFS getstripe -v $file
2087 $LFS pcc state $file
2088 check_lpcc_state $file "none"
2090 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2091 error "failed to attach file $file"
2092 check_lpcc_state $file "readonly"
2093 echo -e "\nFLR layout after PCC-RO attach '$file' again:"
2094 $LFS getstripe -v $file
2095 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2096 error "failed to detach file $file"
2097 check_lpcc_state $file "none"
2099 run_test 21h "Extend mirror once file was PCC-RO cached"
2102 local loopfile="$TMP/$tfile"
2103 local mntpt="/mnt/pcc.$tdir"
2104 local hsm_root="$mntpt/$tdir"
2105 local file=$DIR/$tfile
2106 local file2=$DIR2/$tfile
2109 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2110 skip "Server does not support PCC-RO"
2112 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2113 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2114 setup_pcc_mapping $SINGLEAGT \
2115 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccro=1\ pccrw=1"
2117 do_facet $SINGLEAGT "echo -n hsm_release_pcc_file > $file"
2118 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2119 error "RW-PCC attach $file failed"
2120 check_lpcc_state $file "readwrite"
2121 # HSM released exists archived status
2122 check_hsm_flags $file "0x0000000d"
2124 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2125 error "RW-PCC detach $file failed"
2126 check_lpcc_state $file "none"
2127 # HSM released exists archived status
2128 check_hsm_flags $file "0x0000000d"
2130 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2131 error "failed to PCC-RO attach $file"
2133 $LFS hsm_state $file
2134 $LFS hsm_release $file || error "HSM released $file failed"
2135 echo "Layout after HSM release $file:"
2136 $LFS getstripe -v $file
2137 echo "PCC state $file:"
2138 $LFS pcc state $file
2139 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
2140 error "RO-PCC attach $file failed"
2141 echo "Layout after PCC-RO attach $file again:"
2142 $LFS getstripe -v $file
2144 $LFS pcc state $file
2146 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2147 error "RW-PCC detach $file failed"
2148 check_lpcc_state $file "none"
2150 run_test 21i "HSM release increase layout gen, should invalidate PCC-RO cache"
2153 local loopfile="$TMP/$tfile"
2154 local mntpt="/mnt/pcc.$tdir"
2155 local hsm_root="$mntpt/$tdir"
2156 local tmpfile=$TMP/abc
2157 local file=$DIR/$tdir/$tfile
2164 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2165 skip "Server does not support PCC-RO"
2167 $LCTL get_param mdc.*.import | grep -q client_encryption ||
2168 skip "client encryption not supported"
2170 mount.lustre --help |& grep -q "test_dummy_encryption:" ||
2171 skip "need dummy encryption support"
2173 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2174 stack_trap cleanup_for_enc_tests EXIT
2176 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2179 do_facet $SINGLEAGT "yes 1 | dd of=$tmpfile bs=1 count=5000 conv=fsync"
2180 do_facet $SINGLEAGT cp $tmpfile $file
2181 do_facet $SINGLEAGT $LFS getstripe $file
2182 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2183 error "failed to PCC-RO attach file $file"
2184 check_lpcc_state $file "readonly"
2185 echo "PCC-RO attach '$file':"
2186 do_facet $SINGLEAGT $LFS getstripe -v $file
2188 do_facet $SINGLEAGT cmp -bl $tmpfile $file ||
2189 error "file $file is corrupted (1)"
2190 fid=$(do_facet $SINGLEAGT lfs path2fid $file | tr -d '[]')
2191 lpcc_path=$(lpcc_fid2path $hsm_root $file $fid)
2192 do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path ||
2193 error "file $lpcc_path is corrupted (2)"
2195 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2196 error "failed to PCC-RO detach file $file"
2197 do_facet $SINGLEAGT cmp -s -n 4096 $tmpfile $lpcc_path &&
2198 error "file $lpcc_path is corrupted (3)"
2200 size=$(do_facet $SINGLEAGT stat "--printf=%s" $lpcc_path)
2201 [ $size == 8192 ] || error "PCC file $lpcc_path incorrect size $size"
2203 do_facet $SINGLEAGT cmp -bl $tmpfile $file ||
2204 error "file $file is corrupted (4)"
2205 do_facet $SINGLEAGT cmp -bl -n 4096 $tmpfile $lpcc_path ||
2206 error "file $lpcc_path is corrupted (5)"
2208 do_facet $SINGLEAGT cp $tmpfile ${file}_2
2209 do_facet $SINGLEAGT $LFS getstripe ${file}_2
2210 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER ${file}_2 ||
2211 error "failed to PCC-RO attach file ${file}_2"
2212 check_lpcc_state ${file}_2 "readonly"
2213 echo "PCC-RO attach '${file}_2':"
2214 do_facet $SINGLEAGT $LFS getstripe -v ${file}_2
2216 do_facet $SINGLEAGT $LFS pcc detach ${file}_2 ||
2217 error "failed to PCC-RO detach file ${file}_2"
2218 do_facet $SINGLEAGT cmp -bl $tmpfile ${file}_2 ||
2219 error "file ${file}_2 is corrupted (6)"
2222 # remove fscrypt key from keyring
2223 key=$(do_facet $SINGLEAGT keyctl show |
2224 awk '$7 ~ "^fscrypt:" {print $1}')
2225 [ -n "$key" ] || error "fscrypt key empty on $SINGLEAGT"
2226 do_facet $SINGLEAGT keyctl revoke $key
2227 do_facet $SINGLEAGT keyctl reap
2228 do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*.lru_size=clear
2230 scrambledfile=$(do_facet $SINGLEAGT find $DIR/$tdir/ \
2231 -maxdepth 1 -mindepth 1 -type f)
2232 do_facet $SINGLEAGT $LFS pcc detach -k $scrambledfile ||
2233 error "failed to PCC-RO detach file $scrambledfile (2)"
2235 do_facet $SINGLEAGT rm -f $tmpfile
2237 run_test 21j "PCC-RO for encrypted file"
2240 local loopfile="$TMP/$tfile"
2241 local mntpt="/mnt/pcc.$tdir"
2242 local hsm_root="$mntpt/$tdir"
2243 local file=$DIR/$tfile
2244 local file2=$DIR2/$tfile
2247 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2248 skip "Server does not support PCC-RO"
2250 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2251 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2252 setup_pcc_mapping $SINGLEAGT \
2253 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1\ pccro=1"
2255 do_facet $SINGLEAGT "echo -n roattach_data > $file"
2257 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2258 error "RW-PCC attach $file failed"
2259 check_lpcc_state $file "readwrite"
2260 # HSM released exists archived status
2261 check_hsm_flags $file "0x0000000d"
2263 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2264 error "RW-PCC detach $file failed"
2265 check_lpcc_state $file "none"
2266 # HSM released exists archived status
2267 check_hsm_flags $file "0x0000000d"
2269 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
2270 error "RO-PCC attach $file failed"
2271 echo "Layout after PCC-RO attach $file:"
2272 $LFS getstripe -v $file
2273 # HSM exists archived status
2274 check_hsm_flags $file "0x00000009"
2275 check_lpcc_state $file "readonly"
2276 check_file_data $SINGLEAGT $file "roattach_data"
2278 $LFS hsm_release $file || error "HSM released $file failed"
2279 echo "Layout after HSM release $file:"
2280 $LFS getstripe -v $file
2281 # HSM released exists archived status
2282 check_hsm_flags $file "0x0000000d"
2283 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
2284 error "RO-PCC attach $file failed"
2285 echo "Layout after PCC-RO attach $file again:"
2286 $LFS getstripe -v $file
2287 check_lpcc_state $file "readonly"
2288 check_file_data $SINGLEAGT $file "roattach_data"
2289 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2290 error "failed to detach $file"
2291 echo "Layout after PCC-RO detach $file:"
2292 $LFS getstripe -v $file
2293 rm -f $file2 || error "rm -f $file failed"
2294 do_facet $SINGLEAGT "echo -n roattach_data2 > $file"
2295 fid=$(path2fid $file)
2296 $LFS hsm_archive --archive $HSM_ARCHIVE_NUMBER $file ||
2297 error "Archive $file failed"
2298 wait_request_state $fid ARCHIVE SUCCEED
2299 $LFS hsm_release $file || error "HSM released $file failed"
2300 # HSM released exists archived status
2301 check_hsm_flags $file "0x0000000d"
2302 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER -r $file ||
2303 error "RO-PCC attach $file failed"
2304 check_lpcc_state $file "readonly"
2305 check_file_data $SINGLEAGT $file "roattach_data2"
2306 do_facet $SINGLEAGT $LFS pcc detach $file ||
2307 error "RO-PCC detach $file failed"
2309 run_test 22 "Test RO-PCC attach for the HSM released file"
2312 local loopfile="$TMP/$tfile"
2313 local mntpt="/mnt/pcc.$tdir"
2314 local hsm_root="$mntpt/$tdir"
2315 local file=$DIR/$tfile
2318 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2319 skip "Server does not support PCC-RO"
2321 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2322 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2325 echo "pccro_data" > $file
2326 lpcc_path=$(lpcc_fid2path $hsm_root $file)
2328 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2329 error "failed to RO-PCC attach file $file"
2330 check_lpcc_state $file "readonly"
2331 check_lpcc_data $SINGLEAGT $lpcc_path $file "pccro_data"
2333 local content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2335 [[ $content == "pccro_data" ]] ||
2336 error "mmap_cat data mismatch: $content"
2337 check_lpcc_state $file "readonly"
2339 echo -n "write_mod_data" > $file
2340 echo "Write should invalidate the RO-PCC cache:"
2341 $LFS getstripe -v $file
2342 check_lpcc_state $file "none"
2343 check_file_data $SINGLEAGT $file "write_mod_data"
2345 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2346 error "failed to RO-PCC attach file $file"
2347 check_lpcc_state $file "readonly"
2348 echo "PCC-RO attach '$file' again:"
2349 $LFS getstripe -v $file
2351 echo "Truncate invalidate the RO-PCC cache:"
2352 $TRUNCATE $file 256 || error "failed to truncate $file"
2353 $LFS getstripe -v $file
2354 echo "Finish trucate operation"
2355 check_lpcc_state $file "none"
2356 check_file_size $SINGLEAGT $file 256
2358 echo "Mmap write invalidates RO-PCC caching"
2359 echo -n mmap_write_data > $file || error "echo write $file failed"
2360 $LFS getstripe -v $file
2361 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2362 error "failed to RO-PCC attach file $file"
2363 check_lpcc_state $file "readonly"
2364 echo "PCC-RO attach '$file' again:"
2365 $LFS getstripe -v $file
2366 echo "Mmap write $file via multiop"
2367 # Mmap write will invalidate the RO-PCC cache
2368 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
2369 error "mmap write $file failed"
2370 check_lpcc_state $file "none"
2371 $LFS getstripe -v $file
2372 # After mmap-write by MULTIOP, the first character of the content
2374 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2375 [[ $content == "nmap_write_data" ]] ||
2376 error "mmap_cat data mismatch: $content"
2378 run_test 23 "Test write/truncate/mmap-write invalidating RO-PCC caching"
2381 local loopfile="$TMP/$tfile"
2382 local mntpt="/mnt/pcc.$tdir"
2383 local hsm_root="$mntpt/$tdir"
2384 local file=$DIR/$tdir/$tfile
2387 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2388 skip "Server does not support PCC-RO"
2390 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2391 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2392 setup_pcc_mapping $SINGLEAGT \
2393 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2394 $LCTL pcc list $MOUNT
2396 chmod 777 $DIR/$tdir
2398 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
2399 error "failed to dd write to $file"
2400 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER \
2401 $file || error "failed to attach file $file"
2402 check_lpcc_state $file "readonly"
2403 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2404 error "failed to dd read from $file"
2405 check_lpcc_state $file "readonly"
2407 do_facet $SINGLEAGT $RUNAS $LFS pcc detach -k $file ||
2408 error "failed to detach file $file"
2409 check_lpcc_state $file "none"
2411 # non-root user is forbidden to access PCC file directly
2412 lpcc_path=$(lpcc_fid2path $hsm_root $file)
2413 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
2414 error "non-root user can touch access PCC file $lpcc_path"
2415 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
2416 count=1 && error "non-root user can read PCC file $lpcc_path"
2417 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
2418 count=1 && error "non-root user can write PCC file $lpcc_path"
2420 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER \
2421 $file || error "failed to attach file $file"
2422 check_lpcc_state $file "readonly"
2424 # Test RO-PCC detach as non-root user
2425 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
2426 error "failed to detach file $file"
2427 check_lpcc_state $file "none"
2428 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
2429 error "RO-PCC cached file '$lpcc_path' should be removed"
2433 run_test 24a "Test RO-PCC with non-root user"
2436 local loopfile="$TMP/$tfile"
2437 local mntpt="/mnt/pcc.$tdir"
2438 local hsm_root="$mntpt/$tdir"
2439 local file=$DIR/$tdir/$tfile
2441 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2442 skip "Server does not support PCC-RO"
2444 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2445 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2448 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2449 dd if=/dev/zero of=$file bs=1024 count=1 ||
2450 error "failed to dd write $file"
2451 chmod 600 $file || error "chmod 600 $file failed"
2452 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
2453 error "non-root user can dd write $file"
2454 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
2455 error "non-root user can dd read $file"
2456 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2457 error "failed to attach file $file"
2458 check_lpcc_state $file "readonly"
2459 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
2460 error "non-root user can dd write $file"
2461 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 &&
2462 error "non-root user can dd read $file"
2463 chmod 777 $file || error "chmod 777 $file failed"
2464 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2465 error "non-root user cannot read $file with permission (777)"
2466 check_lpcc_state $file "readonly"
2468 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
2469 error "non-root user or non owner can detach $file"
2470 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
2471 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
2472 error "failed to detach file $file"
2473 check_lpcc_state $file "none"
2474 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
2475 error "non-root user cannot read $file with permission (777)"
2477 run_test 24b "General permission test for RO-PCC"
2480 local loopfile="$TMP/$tfile"
2481 local mntpt="/mnt/pcc.$tdir"
2482 local hsm_root="$mntpt/$tdir"
2483 local file=$DIR/$tdir/$tfile
2486 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2487 skip "Server does not support PCC-RO"
2489 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2490 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2493 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2495 echo "ro_fake_mmap_cat_err" > $file
2496 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2497 error "failed to attach RO-PCC file $file"
2498 check_lpcc_state $file "readonly"
2499 check_file_data $SINGLEAGT $file "ro_fake_mmap_cat_err"
2501 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
2502 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
2503 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
2504 [[ $content == "ro_fake_mmap_cat_err" ]] ||
2505 error "failed to fall back to Lustre I/O path for mmap-read"
2506 # Above mmap read will return VM_FAULT_SIGBUS failure and
2507 # retry the IO on normal IO path.
2508 check_lpcc_state $file "none"
2509 check_file_data $SINGLEAGT $file "ro_fake_mmap_cat_err"
2511 do_facet $SINGLEAGT $LFS pcc detach $file ||
2512 error "failed to detach RO-PCC file $file"
2513 check_lpcc_state $file "none"
2515 do_facet $SINGLEAGT $LCTL set_param fail_loc=0
2516 echo "ro_fake_cat_err" > $file
2517 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2518 error "failed to attach RO-PCC file $file"
2519 check_lpcc_state $file "readonly"
2520 check_file_data $SINGLEAGT $file "ro_fake_cat_err"
2522 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
2523 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
2524 # Fake read I/O will return -EIO failure and
2525 # retry the IO on normal IO path.
2526 check_file_data $SINGLEAGT $file "ro_fake_cat_err"
2527 check_lpcc_state $file "readonly"
2529 do_facet $SINGLEAGT $LFS pcc detach $file ||
2530 error "failed to detach RO-PCC file $file"
2531 check_lpcc_state $file "none"
2533 run_test 25 "Tolerate fake read failure for RO-PCC"
2536 local agt_host=$(facet_active_host $SINGLEAGT)
2537 local loopfile="$TMP/$tfile"
2538 local mntpt="/mnt/pcc.$tdir"
2539 local hsm_root="$mntpt/$tdir"
2540 local file=$DIR/$tfile
2542 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2543 skip "Server does not support PCC-RO"
2545 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2546 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2547 setup_pcc_mapping $SINGLEAGT \
2548 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1\ pccro=1"
2550 echo -n attach_keep_open > $file
2551 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2552 error "attach $file failed"
2553 check_lpcc_state $file "readonly"
2554 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2555 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2556 error "detach $file failed"
2557 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2558 error "attach $file failed"
2559 check_lpcc_state $file "readonly"
2560 check_file_data $SINGLEAGT $file "attach_keep_open"
2561 check_lpcc_state $file "readonly"
2562 do_facet $SINGLEAGT $LFS pcc detach $file ||
2563 error "detach $file failed"
2564 rmultiop_stop $agt_host || error "multiop $file close failed"
2566 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2567 error "attach $file failed"
2568 check_lpcc_state $file "readonly"
2569 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2570 do_facet $SINGLEAGT $LFS pcc detach $file ||
2571 error "detach $file failed"
2572 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2573 error "attach $file failed"
2574 check_lpcc_state $file "readonly"
2575 check_file_data $SINGLEAGT $file "attach_keep_open"
2576 check_lpcc_state $file "readonly"
2577 do_facet $SINGLEAGT $LFS pcc detach $file ||
2578 error "detach $file failed"
2579 rmultiop_stop $agt_host || error "multiop $file close failed"
2581 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2582 error "attach $file failed"
2583 check_lpcc_state $file "readonly"
2584 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2585 do_facet $SINGLEAGT $LFS pcc detach $file ||
2586 error "detach $file failed"
2587 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2588 error "attach $file failed"
2589 check_lpcc_state $file "readonly"
2590 check_file_data $SINGLEAGT $file "attach_keep_open"
2591 check_lpcc_state $file "readonly"
2592 rmultiop_stop $agt_host || error "multiop $file close failed"
2593 do_facet $SINGLEAGT $LFS pcc detach $file ||
2594 error "detach $file failed"
2596 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2597 error "attach $file failed"
2598 check_lpcc_state $file "readwrite"
2599 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2600 do_facet $SINGLEAGT $LFS pcc detach $file ||
2601 error "detach $file failed"
2602 wait_request_state $(path2fid $file) REMOVE SUCCEED
2603 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2604 error "attach $file failed"
2605 check_lpcc_state $file "readonly"
2606 check_file_data $SINGLEAGT $file "attach_keep_open"
2607 check_lpcc_state $file "readonly"
2608 rmultiop_stop $agt_host || error "multiop $file close failed"
2609 check_lpcc_state $file "readonly"
2610 do_facet $SINGLEAGT $LFS pcc detach $file ||
2611 error "detach $file failed"
2613 rm $file || error "rm $file failed"
2614 echo -n attach_keep_open > $file
2615 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2616 error "attach $file failed"
2617 check_lpcc_state $file "readwrite"
2618 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2619 do_facet $SINGLEAGT $LFS pcc detach $file ||
2620 error "detach $file failed"
2621 wait_request_state $(path2fid $file) REMOVE SUCCEED
2622 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2623 error "attach $file failed"
2624 check_lpcc_state $file "readonly"
2625 check_file_data $SINGLEAGT $file "attach_keep_open"
2626 check_lpcc_state $file "readonly"
2627 do_facet $SINGLEAGT $LFS pcc detach $file ||
2628 error "detach $file failed"
2629 rmultiop_stop $agt_host || error "multiop $file close failed"
2630 check_lpcc_state $file "none"
2632 run_test 26 "Repeat the attach/detach when the file has multiple openers"
2635 local agt_host=$(facet_active_host $SINGLEAGT)
2636 local loopfile="$TMP/$tfile"
2637 local mntpt="/mnt/pcc.$tdir"
2638 local hsm_root="$mntpt/$tdir"
2639 local file=$DIR/$tfile
2641 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2642 skip "Server does not support PCC-RO"
2644 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2645 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2646 setup_pcc_mapping $SINGLEAGT \
2647 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ open_attach=1\ pccrw=1\ pccro=1"
2649 echo -n auto_attach_multi_open > $file
2650 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2651 error "attach $file failed"
2652 check_lpcc_state $file "readwrite"
2653 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2654 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2655 error "detach $file failed"
2656 check_lpcc_state $file "none"
2657 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2658 check_lpcc_state $file "readwrite"
2659 do_facet $SINGLEAGT $LFS pcc detach $file ||
2660 error "detach $file failed"
2661 wait_request_state $(path2fid $file) REMOVE SUCCEED
2662 check_lpcc_state $file "none"
2663 rmultiop_stop $agt_host || error "multiop $file close failed"
2665 rm $file || error "rm $file failed"
2666 echo -n auto_attach_multi_open > $file
2667 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2668 error "attach $file failed"
2669 check_lpcc_state $file "readwrite"
2670 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2671 do_facet $SINGLEAGT $LCTL \
2672 set_param ldlm.namespaces.*mdc*.lru_size=clear
2673 check_lpcc_state $file "none"
2674 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2675 check_lpcc_state $file "readwrite"
2676 do_facet $SINGLEAGT $LFS pcc detach $file ||
2677 error "detach $file failed"
2678 wait_request_state $(path2fid $file) REMOVE SUCCEED
2679 check_lpcc_state $file "none"
2680 rmultiop_stop $agt_host || error "multiop $file close failed"
2682 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2683 error "attach $file failed"
2684 check_lpcc_state $file "readonly"
2685 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2686 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2687 error "detach $file failed"
2688 check_lpcc_state $file "none"
2689 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2690 check_lpcc_state $file "readonly"
2691 do_facet $SINGLEAGT $LFS pcc detach $file ||
2692 error "detach $file failed"
2693 check_lpcc_state $file "none"
2694 rmultiop_stop $agt_host || error "multiop $file close failed"
2696 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2697 error "attach $file failed"
2698 check_lpcc_state $file "readonly"
2699 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2700 do_facet $SINGLEAGT $LCTL \
2701 set_param ldlm.namespaces.*mdc*.lru_size=clear
2702 check_lpcc_state $file "none"
2703 check_file_data $SINGLEAGT $file "auto_attach_multi_open"
2704 check_lpcc_state $file "readonly"
2705 do_facet $SINGLEAGT $LFS pcc detach $file ||
2706 error "detach $file failed"
2707 check_lpcc_state $file "none"
2708 rmultiop_stop $agt_host || error "multiop $file close failed"
2710 run_test 27 "Auto attach at open when the file has multiple openers"
2713 local agt_host=$(facet_active_host $SINGLEAGT)
2714 local loopfile="$TMP/$tfile"
2715 local mntpt="/mnt/pcc.$tdir"
2716 local hsm_root="$mntpt/$tdir"
2717 local file=$DIR/$tfile
2718 local file2=$DIR2/$tfile
2721 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2722 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER" -h "$hsm_root"
2723 setup_pcc_mapping $SINGLEAGT \
2724 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1"
2726 echo -n rw_attach_hasopen_fail > $file
2727 rmultiop_start $agt_host $file O_c || error "multiop $file failed"
2728 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file &&
2729 error "attach $file should fail"
2730 rmultiop_stop $agt_host || error "multiop $file close failed"
2731 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2732 error "attach $file should fail"
2733 check_lpcc_state $file "readwrite"
2734 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2735 error "detach $file failed"
2736 check_lpcc_state $file "none"
2738 multiop_bg_pause $file2 O_c || error "multiop $file2 failed"
2740 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file &&
2741 error "attach $file should fail"
2742 kill -USR1 $multipid
2743 wait $multipid || error "multiop $file2 close failed"
2744 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2745 error "failed to attach $file"
2746 check_lpcc_state $file "readwrite"
2747 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2748 error "detach $file failed"
2749 check_lpcc_state $file "none"
2751 run_test 28 "RW-PCC attach should fail when the file has cluster-wide openers"
2754 local project_id=100
2755 local agt_facet=$SINGLEAGT
2756 local loopfile="$TMP/$tfile"
2757 local mntpt="/mnt/pcc.$tdir"
2758 local hsm_root="$mntpt/$tdir"
2759 local file=$DIR/$tdir/$tfile
2760 local file2=$DIR2/$tdir/$tfile
2762 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2763 skip "Server does not support PCC-RO"
2765 is_project_quota_supported || skip "project quota is not supported"
2767 enable_project_quota
2768 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2769 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2770 setup_pcc_mapping $SINGLEAGT \
2771 "projid={$project_id}\ rwid=$HSM_ARCHIVE_NUMBER\ pccro=1"
2772 $LCTL pcc list $MOUNT
2774 do_facet $SINGLEAGT mkdir -p $DIR/$tdir ||
2775 error "mkdir $DIR/$tdir failed"
2776 do_facet $SINGLEAGT "echo -n ro_uptodate > $file" ||
2777 error "failed to write $file"
2778 check_lpcc_state $file "none"
2779 $LFS project -sp $project_id $file ||
2780 error "failed to set project for $file"
2781 $LFS project -d $file
2782 check_file_data $SINGLEAGT $file "ro_uptodate"
2783 check_lpcc_state $file "readonly"
2784 check_file_data $SINGLEAGT $file "ro_uptodate"
2786 echo -n Update_ro_data > $file2
2787 check_lpcc_state $file "none"
2788 check_file_data $SINGLEAGT $file "Update_ro_data"
2789 check_lpcc_state $file "readonly"
2791 do_facet $SINGLEAGT $LFS pcc detach $file ||
2792 error "failed to detach $file"
2794 run_test 29a "Auto readonly caching on RO-PCC backend for O_RDONLY open"
2797 local loopfile="$TMP/$tfile"
2798 local mntpt="/mnt/pcc.$tdir"
2799 local hsm_root="$mntpt/$tdir"
2800 local file=$DIR/myfile.dat
2802 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2803 skip "Server does not support PCC-RO"
2805 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2806 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
2807 setup_pcc_mapping $SINGLEAGT \
2808 "fname={*.dat}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
2809 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
2811 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=4k count=1 ||
2812 error "Write $file failed"
2813 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=4k count=1 ||
2814 error "Read $file failed"
2815 do_facet $SINGLEAGT $LFS pcc state $file
2816 check_lpcc_state $file "readonly"
2817 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=4k count=1 ||
2818 error "Write $file failed"
2819 sysctl vm.drop_caches=3
2820 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=4k count=1 ||
2821 error "Read $file failed"
2822 do_facet $SINGLEAGT $LFS pcc state $file
2823 check_lpcc_state $file "readonly"
2825 do_facet $SINGLEAGT $LFS pcc detach $file || error "detach $file failed"
2827 run_test 29b "Auto PCC-RO attach in atomic_open"
2830 local loopfile="$TMP/$tfile"
2831 local mntpt="/mnt/pcc.$tdir"
2832 local hsm_root="$mntpt/$tdir"
2835 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2836 skip "Server does not support PCC-RO"
2838 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2839 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2840 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1\ pccro=1"
2842 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2844 file=$DIR/$tdir/rwattach
2845 echo -n backend_del_attach > $file
2846 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2847 error "RW-PCC attach $file failed"
2849 file=$DIR/$tdir/rwattachrm
2850 echo -n backend_del_attach_rm > $file
2851 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2852 error "RW-PCC attach $file failed"
2853 rm $file || error "rm $file failed"
2855 file=$DIR/$tdir/roattach
2856 echo -n backend_del_roattach_rm > $file
2857 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2858 error "RO-PCC attach $file failed"
2860 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
2861 do_facet $SINGLEAGT $LCTL pcc del -v -v -v -v $MOUNT $hsm_root ||
2862 error "lctl pcc del $MOUNT $hsm_root failed"
2864 run_test 30 "Test lctl pcc del command"
2867 local loopfile="$TMP/$tfile"
2868 local mntpt="/mnt/pcc.$tdir"
2869 local hsm_root="$mntpt/$tdir"
2875 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2876 skip "Server does not support PCC-RO"
2878 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2879 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2880 setup_pcc_mapping $SINGLEAGT \
2881 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0\ pccrw=1\ pccro=1"
2883 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
2885 file=$DIR/$tdir/rwattach
2886 echo -n backend_del_attach > $file
2887 lpcc_path1=$(lpcc_fid2path $hsm_root $file)
2888 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2889 error "RW-PCC attach $file failed"
2890 check_lpcc_state $file "readwrite"
2891 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2892 error "RW-PCC detach $file failed"
2893 check_lpcc_state $file "none"
2895 file=$DIR/$tdir/rwattachrm
2896 echo -n backend_del_attach_rm > $file
2897 lpcc_path2=$(lpcc_fid2path $hsm_root $file)
2898 do_facet $SINGLEAGT $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER $file ||
2899 error "RW-PCC attach $file failed"
2900 check_lpcc_state $file "readwrite"
2901 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2902 error "RW-PCC detach $file failed"
2903 check_lpcc_state $file "none"
2904 rm $file || error "rm $file failed"
2906 file=$DIR/$tdir/roattach
2907 echo -n backend_del_roattach_rm > $file
2908 lpcc_path3=$(lpcc_fid2path $hsm_root $file)
2909 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2910 error "RO-PCC attach $file failed"
2911 check_lpcc_state $file "readonly"
2912 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2913 error "RO-PCC detach $file failed"
2914 check_lpcc_state $file "none"
2916 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
2917 do_facet $SINGLEAGT $LCTL pcc del -v -v -v -v -k $MOUNT $hsm_root ||
2918 error "lctl pcc del -k $MOUNT $hsm_root failed"
2920 do_facet $SINGLEAGT "[ -f $lpcc_path1 ]" ||
2921 error "PCC copy $lpcc_path1 should retain"
2922 do_facet $SINGLEAGT "[ -f $lpcc_path2 ]" ||
2923 error "PCC copy $lpcc_path1 should retain"
2924 do_facet $SINGLEAGT "[ -f $lpcc_path3 ]" ||
2925 error "PCC copy $lpcc_path1 should retain"
2927 run_test 31 "Test lctl pcc del command with --keep option"
2930 local agt_host=$(facet_active_host $SINGLEAGT)
2931 local loopfile="$TMP/$tfile"
2932 local mntpt="/mnt/pcc.$tdir"
2933 local hsm_root="$mntpt/$tdir"
2934 local file=$DIR/$tfile
2937 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2938 skip "Server does not support PCC-RO"
2940 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2941 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2942 setup_pcc_mapping $SINGLEAGT \
2943 "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ auto_attach=0"
2945 do_facet $SINGLEAGT echo -n roattach_removed > $file
2946 lpcc_path=$(lpcc_fid2path $hsm_root $file)
2947 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2948 error "RO-PCC attach $file failed"
2949 rmultiop_start $agt_host $file o_rc || error "multiop $file failed"
2951 do_facet $SINGLEAGT rm $lpcc_path || error "rm $lpcc_path failed"
2952 rmultiop_stop $agt_host || error "multiop $file read failed"
2954 # file will be detached in @pcc_ioctl_state()
2955 check_lpcc_state $file "readonly"
2957 local content=$(do_facet $SINGLEAGT cat $file)
2958 [[ $content == "roattach_removed" ]] || error "data mismatch: $content"
2959 check_lpcc_state $file "none"
2960 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2961 error "RO-PCC detach $file failed"
2962 check_lpcc_state $file "none"
2964 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
2965 error "RO-PCC attach $file failed"
2966 do_facet $SINGLEAGT rm $lpcc_path || error "rm $lpcc_path failed"
2967 # file will be detached in @pcc_ioctl_state()
2968 check_lpcc_state $file "readonly"
2969 content=$(do_facet $SINGLEAGT cat $file)
2970 [[ $content == "roattach_removed" ]] || error "data mismatch: $content"
2971 check_lpcc_state $file "none"
2972 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
2973 error "RO-PCC detach $file failed"
2974 check_lpcc_state $file "none"
2976 run_test 32 "Test for RO-PCC when PCC copy is deleted"
2979 local loopfile="$TMP/$tfile"
2980 local mntpt="/mnt/pcc.$tdir"
2981 local hsm_root="$mntpt/$tdir"
2982 local file=$DIR/myfile.doc
2983 local file2=$DIR2/myfile.doc
2985 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
2986 skip "Server does not support PCC-RO"
2988 stack_trap "restore_opencache" EXIT
2991 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
2992 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
2994 setup_pcc_mapping $SINGLEAGT \
2995 "fname={*.doc}\&size\<{1M}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
2996 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
2997 touch $file || error "touch $file failed"
2998 $TRUNCATE $file $((1048576 * 2)) || error "Truncate $file failed"
2999 do_facet $SINGLEAGT $MULTIOP $file oc ||
3000 error "failed to readonly open $file"
3001 check_lpcc_state $file "none"
3002 do_facet $SINGLEAGT $LFS pcc state $file
3003 $TRUNCATE $file $((1048576 / 2)) || error "Truncate $file failed"
3004 do_facet $SINGLEAGT $LFS pcc state $file
3005 do_facet $SINGLEAGT $MULTIOP $file oc ||
3006 error "failed to readonly open $file"
3007 check_lpcc_state $file "readonly"
3010 setup_pcc_mapping $SINGLEAGT \
3011 "fname={*.doc}\&size\<{5M}\&size\>{3M}\ roid=5\ pccro=1"
3012 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3013 do_facet $SINGLEAGT $MULTIOP $file oc ||
3014 error "failed to readonly open $file"
3015 check_lpcc_state $file "none"
3016 $TRUNCATE $file2 $((1048576 * 6)) || error "Truncate $file2 failed"
3017 do_facet $SINGLEAGT $MULTIOP $file oc ||
3018 error "failed to readonly open $file"
3019 check_lpcc_state $file "none"
3020 $TRUNCATE $file2 $((1048576 * 4)) || error "Truncate $file2 failed"
3021 do_facet $SINGLEAGT $MULTIOP $file oc ||
3022 error "failed to readonly open $file"
3023 check_lpcc_state $file "readonly"
3026 setup_pcc_mapping $SINGLEAGT \
3027 "fname={*.doc}\&size={5M\ 3M}\ roid=5\ pccro=1"
3028 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3029 do_facet $SINGLEAGT $MULTIOP $file oc ||
3030 error "failed to readonly open $file"
3031 check_lpcc_state $file "none"
3032 $TRUNCATE $file $((1048576 * 5)) || error "Truncate $file failed"
3033 do_facet $SINGLEAGT $MULTIOP $file oc ||
3034 error "failed to readonly open $file"
3035 check_lpcc_state $file "readonly"
3036 do_facet $SINGLEAGT $LFS pcc detach $file ||
3037 error "failed to detach $file"
3038 $TRUNCATE $file $((1048576 * 4)) || error "Truncate $file failed"
3039 do_facet $SINGLEAGT $MULTIOP $file oc ||
3040 error "failed to readonly open $file"
3041 check_lpcc_state $file "none"
3042 $TRUNCATE $file $((1048576 * 3)) || error "Truncate $file failed"
3043 do_facet $SINGLEAGT $MULTIOP $file oc ||
3044 error "failed to readonly open $file"
3045 check_lpcc_state $file "readonly"
3048 run_test 33 "Cache rule with comparator (>, =, <) for file size"
3051 local loopfile="$TMP/$tfile"
3052 local mntpt="/mnt/pcc.$tdir"
3053 local hsm_root="$mntpt/$tdir"
3054 local file=$DIR/$tfile
3056 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3057 skip "Server does not support PCC-RO"
3059 is_project_quota_supported || skip "project quota is not supported"
3061 enable_project_quota
3062 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
3063 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
3065 setup_pcc_mapping $SINGLEAGT \
3066 "projid\>{100}\ roid=5\ pccro=1"
3067 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3068 do_facet $SINGLEAGT "echo -n QQQQQ > $file" ||
3069 error "failed to write $file"
3070 do_facet $SINGLEAGT $MULTIOP $file oc ||
3071 error "failed to readonly open $file"
3072 check_lpcc_state $file "none"
3073 $LFS project -p 99 $file || error "failed to set project for $file"
3074 $LFS project -d $file
3075 do_facet $SINGLEAGT $MULTIOP $file oc ||
3076 error "failed to readonly open $file"
3077 check_lpcc_state $file "none"
3078 $LFS project -p 101 $file || error "failed to set project for $file"
3079 $LFS project -d $file
3080 do_facet $SINGLEAGT $MULTIOP $file oc ||
3081 error "failed to readonly open $file"
3082 check_lpcc_state $file "readonly"
3085 setup_pcc_mapping $SINGLEAGT \
3086 "projid\<{100}\ roid=5\ pccro=1"
3087 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3088 do_facet $SINGLEAGT $MULTIOP $file oc ||
3089 error "failed to readonly open $file"
3090 check_lpcc_state $file "none"
3091 $LFS project -p 102 $file || error "failed to set project for $file"
3092 $LFS project -d $file
3093 do_facet $SINGLEAGT $MULTIOP $file oc ||
3094 error "failed to readonly open $file"
3095 check_lpcc_state $file "none"
3096 $LFS project -p 99 $file || error "failed to set project for $file"
3097 $LFS project -d $file
3098 do_facet $SINGLEAGT $MULTIOP $file oc ||
3099 error "failed to readonly open $file"
3100 check_lpcc_state $file "readonly"
3103 setup_pcc_mapping $SINGLEAGT \
3104 "projid\<{120}\&projid\>{110}\ roid=5\ pccro=1"
3105 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3106 do_facet $SINGLEAGT $MULTIOP $file oc ||
3107 error "failed to readonly open $file"
3108 check_lpcc_state $file "none"
3109 $LFS project -p 105 $file || error "failed to set project for $file"
3110 $LFS project -d $file
3111 do_facet $SINGLEAGT $MULTIOP $file oc ||
3112 error "failed to readonly open $file"
3113 check_lpcc_state $file "none"
3114 $LFS project -p 121 $file || error "failed to set project for $file"
3115 $LFS project -d $file
3116 do_facet $SINGLEAGT $MULTIOP $file oc ||
3117 error "failed to readonly open $file"
3118 check_lpcc_state $file "none"
3119 $LFS project -p 115 $file || error "failed to set project for $file"
3120 $LFS project -d $file
3121 do_facet $SINGLEAGT $MULTIOP $file oc ||
3122 error "failed to readonly open $file"
3123 check_lpcc_state $file "readonly"
3126 run_test 34 "Cache rule with comparator (>, <) for Project ID range"
3129 local loopfile="$TMP/$tfile"
3130 local mntpt="/mnt/pcc.$tdir"
3131 local hsm_root="$mntpt/$tdir"
3132 local file=$DIR/$tfile
3135 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3136 skip "Server does not support PCC-RO"
3138 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
3139 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
3142 echo "pccro_mmap_data" > $file
3143 lpcc_path=$(lpcc_fid2path $hsm_root $file)
3144 do_facet $SINGLEAGT $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file ||
3145 error "failed to PCC-RO attach file $file"
3146 check_lpcc_state $file "readonly"
3147 check_lpcc_data $SINGLEAGT $lpcc_path $file "pccro_mmap_data"
3149 local content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
3151 [[ $content == "pccro_mmap_data" ]] ||
3152 error "mmap_cat data mismatch: $content"
3153 check_lpcc_state $file "readonly"
3155 do_facet $SINGLEAGT $LFS pcc detach $file ||
3156 error "failed to PCC-RO detach $file"
3157 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
3158 [[ $content == "pccro_mmap_data" ]] ||
3159 error "mmap_cat data mismatch: $content"
3160 check_lpcc_state $file "none"
3162 run_test 35 "mmap fault test"
3165 local loopfile="$TMP/$tfile"
3166 local mntpt="/mnt/pcc.$tdir"
3167 local hsm_root="$mntpt/$tdir"
3168 local file=$DIR/$tfile
3170 local state="readonly"
3173 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3174 skip "Server does not support PCC-RO"
3176 [[ -z $rw ]] || state="readwrite"
3177 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
3178 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
3179 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1\ pccro=1"
3181 echo -n backend_clear_verify > $file
3182 lpcc_path=$(lpcc_fid2path $hsm_root $file)
3183 do_facet $SINGLEAGT $LFS pcc attach $rw -i $HSM_ARCHIVE_NUMBER $file ||
3184 error "PCC attach $ro $file failed"
3185 check_lpcc_state $file "$state"
3186 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
3187 error "PCC detach -k $file failed"
3188 do_facet $SINGLEAGT "[ -f $lpcc_path1 ]" ||
3189 error "PCC copy $lpcc_path should retain"
3190 do_facet $SINGLEAGT $LCTL pcc clear -v $MOUNT ||
3191 error "lctl pcc clear -v $MOUNT failed"
3192 do_facet $SINGLEAGT "[ -f $lpcc_path ]" &&
3193 error "PCC copy $lpcc_path should be removed"
3194 rm $file || error "rm $file failed"
3200 run_test 36a "Stale RW-PCC copy should be deleted after remove the PCC backend"
3205 run_test 36b "Stale RO-PCC copy should be deleted after remove the PCC backend"
3208 local loopfile="$TMP/$tfile"
3209 local loopfile2="$TMP/$tfile.2"
3210 local mntpt="/mnt/pcc.$tdir"
3211 local mntpt2="/mnt/pcc.$tdir.2"
3212 local file=$DIR/$tdir/$tfile
3213 local file2=$DIR2/$tdir/$tfile
3215 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3216 skip "Server does not support PCC-RO"
3218 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
3221 setup_loopdev client $loopfile $mntpt 50
3222 setup_loopdev client $loopfile2 $mntpt2 50
3223 $LCTL pcc add $MOUNT $mntpt -p \
3224 "projid={2} roid=$HSM_ARCHIVE_NUMBER auto_attach=0 pccro=1" ||
3225 error "failed to config PCC for $MOUNT $mntpt"
3226 $LCTL pcc add $MOUNT2 $mntpt2 -p \
3227 "projid={2} roid=$HSM_ARCHIVE_NUMBER auto_attach=0 pccro=1" ||
3228 error "failed to config PCC for $MOUNT2 $mntpt2"
3229 $LCTL pcc list $MOUNT
3230 $LCTL pcc list $MOUNT2
3232 cancel_lru_locks mdc
3233 #define CFS_FAIL_ONCE | OBD_FAIL_MDS_LL_PCCRO
3234 $LCTL set_param -n fail_loc=0x80000176 fail_val=10
3235 $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file &
3237 $LFS pcc attach -r -i $HSM_ARCHIVE_NUMBER $file2
3239 $LFS pcc state $file
3240 $LFS pcc state $file2
3242 check_lpcc_state $file "readonly" client
3243 check_lpcc_state $file2 "readonly" client
3245 $LCTL pcc clear $MOUNT
3246 $LCTL pcc clear $MOUNT2
3248 run_test 37 "Multiple readers on a shared file with PCC-RO mode"
3251 local loopfile="$TMP/$tfile"
3252 local mntpt="/mnt/pcc.$tdir"
3253 local hsm_root="$mntpt/$tdir"
3254 local dir=$DIR/$tdir
3255 local file=$dir/$tfile
3257 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3258 skip "Server does not support PCC-RO"
3260 is_project_quota_supported || skip "project quota is not supported"
3262 enable_project_quota
3263 mkdir $dir || error "mkdir $dir failed"
3264 $LFS project -sp 100 $dir ||
3265 error "failed to set project for $dir"
3266 echo "QQQQQ" > $file
3268 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
3269 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3270 setup_pcc_mapping $SINGLEAGT \
3271 "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
3273 do_facet $SINGLEAGT $LFS pcc state $file ||
3274 error "failed to get PCC state for $file"
3275 check_lpcc_state $file "none"
3276 do_facet $SINGLEAGT cat $file || error "cat $file failed"
3277 check_lpcc_state $file "readonly"
3278 do_facet $SINGLEAGT $LFS pcc detach $file ||
3279 error "failed to detach $file"
3280 check_lpcc_state $file "none"
3282 run_test 38 "Verify LFS pcc state does not trigger prefetch for auto PCC-RO"
3285 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3286 skip "Server does not support PCC-RO"
3288 quotaon --help |& grep -q 'project quotas' ||
3289 skip "Not support project quota on local filesystem"
3291 is_project_quota_supported || skip "project quota is not supported"
3293 enable_project_quota
3295 local loopfile="$TMP/$tfile"
3296 local mntpt="/mnt/pcc.$tdir"
3297 local hsm_root="$mntpt/$tdir"
3298 local dir=$DIR/$tdir
3299 local file=$dir/$tfile
3302 setup_loopdev_project $SINGLEAGT $loopfile $mntpt 50
3303 do_facet $SINGLEAGT quotaon -Ppv $mntpt
3304 do_facet $SINGLEAGT setquota -P $id 0 4096 0 0 $mntpt ||
3305 error "setquota -P $id on $mntpt failed"
3306 do_facet $SINGLEAGT repquota -Pvs $mntpt
3308 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3309 setup_pcc_mapping $SINGLEAGT \
3310 "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ proj_quota=1\ pccro=1"
3311 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3313 do_facet $SINGLEAGT mkdir -p $dir || error "mkdir $dir failed"
3314 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=2 ||
3315 error "Write $file failed"
3316 $LFS project -p $id $file || error "failed to set project for $file"
3317 $LFS project -d $file
3318 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=2 ||
3319 error "Read $file failed"
3320 do_facet $SINGLEAGT $LFS pcc state $file
3321 check_lpcc_state $file "readonly"
3322 do_facet $SINGLEAGT repquota -Pvs $mntpt
3323 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=5 ||
3324 error "Write $file failed"
3325 check_lpcc_state $file "none"
3326 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=5 ||
3327 error "Read $file failed"
3328 do_facet $SINGLEAGT repquota -Pvs $mntpt
3329 do_facet $SINGLEAGT $LFS pcc state $file
3330 check_lpcc_state $file "none"
3332 run_test 39 "Test Project quota on loop PCC device"
3334 wait_readonly_attach_fini() {
3336 local facet=${2:-$SINGLEAGT}
3337 local cmd="$LFS pcc state $file | grep -E -c 'type: readonly'"
3340 wait_update_facet $facet "$cmd" "1" 50 ||
3341 error "Async attach $file timed out"
3344 calc_stats_facet() {
3345 local paramfile="$1"
3347 local facet=${3:-$SINGLEAGT}
3349 do_facet $facet $LCTL get_param -n $paramfile |
3350 awk '/^'$stat'/ { sum += $2 } END { printf("%0.0f", sum) }'
3354 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3355 skip "Server does not support PCC-RO"
3357 is_project_quota_supported || skip "project quota is not supported"
3359 enable_project_quota
3361 local loopfile="$TMP/$tfile"
3362 local mntpt="/mnt/pcc.$tdir"
3363 local hsm_root="$mntpt/$tdir"
3364 local dir=$DIR/$tdir
3365 local file=$dir/$tfile
3368 setup_loopdev $SINGLEAGT $loopfile $mntpt 200
3369 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3370 setup_pcc_mapping $SINGLEAGT \
3371 "projid={$id}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
3372 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3374 mkdir -p $dir || error "mkdir $dir failed"
3375 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 ||
3376 error "Write $file failed"
3378 $LFS project -p $id $file || error "failed to set project for $file"
3379 $LFS project -d $file
3380 do_facet $SINGLEAGT $LFS pcc detach $file
3381 do_facet $SINGLEAGT $LFS pcc state $file
3383 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear
3384 do_facet $SINGLEAGT $LCTL set_param osc.*.stats=clear
3385 #define OBD_FAIL_OST_BRW_PAUSE_BULK
3386 set_nodes_failloc "$(osts_nodes)" 0x214 1
3387 echo 3 > /proc/sys/vm/drop_caches
3395 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=5MB
3397 echo "Test open attach with pcc_async_threshold=5MB"
3399 # Open with O_RDONLY flag will trigger auto attach
3400 do_facet $SINGLEAGT $MULTIOP $file oc ||
3401 error "failed to readonly open $file"
3403 rpcs_before=$(calc_stats_facet osc.*.stats ost_read)
3404 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct
3405 rpcs_after=$(calc_stats_facet osc.*.stats ost_read)
3406 echo "Before: $rpcs_before After: $rpcs_after"
3407 [ $rpcs_after -gt $rpcs_before ] ||
3408 error "should send read RPCs to OSTs $rpcs_before: $rpcs_after"
3409 time1=$((SECONDS - stime))
3410 do_facet $SINGLEAGT $LFS pcc state $file
3411 wait_readonly_attach_fini $file
3413 do_facet $SINGLEAGT $LFS pcc detach $file
3414 do_facet $SINGLEAGT $LFS pcc state $file
3415 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G
3416 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear
3417 do_facet $SINGLEAGT $LCTL set_param osc.*.stats=clear
3419 echo "Test open attach with async_threshold=1G"
3421 # Open with O_RDONLY flag will trigger auto attach
3422 do_facet $SINGLEAGT $MULTIOP $file oc ||
3423 error "failed to readonly open $file"
3424 do_facet $SINGLEAGT $LFS pcc state $file
3425 rpcs_before=$(calc_stats_facet osc.*.stats ost_read)
3426 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct
3427 rpcs_after=$(calc_stats_facet osc.*.stats ost_read)
3428 time2=$((SECONDS - stime))
3429 echo "Before: $rpcs_before After: $rpcs_after"
3430 [ $rpcs_after -eq $rpcs_before ] ||
3431 error "should not send OST_READ RPCs to OSTs"
3433 echo "Time1: $time1 Time2: $time2"
3434 # Occasionally async can take a tiny bit longer due to races, that's OK
3435 [ $time1 -le $((time2 + 1)) ] ||
3436 error "Total time for async open attach should be smaller"
3438 do_facet $SINGLEAGT $LFS pcc detach $file
3439 do_facet $SINGLEAGT $LFS pcc state $file
3440 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=5MB
3441 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear
3443 echo "Read 1MB data with async_threshold=5MB"
3445 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct
3446 time1=$((SECONDS - stime))
3447 wait_readonly_attach_fini $file
3449 do_facet $SINGLEAGT $LFS pcc detach $file
3450 do_facet $SINGLEAGT $LFS pcc state $file
3451 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G
3452 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*osc*.lru_size=clear
3454 echo "Read 1MB data with async_threshold=1G"
3456 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=1 iflag=direct
3457 time2=$((SECONDS - stime))
3459 echo "Time1: $time1 Time2: $time2"
3460 # Occasionally async can take a tiny bit longer due to races, that's OK
3461 [ $time1 -le $((time2 + 1)) ] ||
3462 error "Total time for async open attach should be smaller"
3464 run_test 40 "Test async open attach in the background for PCC-RO file"
3467 local loopfile="$TMP/$tfile"
3468 local mntpt="/mnt/pcc.$tdir"
3469 local hsm_root="$mntpt/$tdir"
3470 local file=$DIR/$tfile
3472 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3473 skip "Server does not support PCC-RO"
3475 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
3476 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3477 setup_pcc_mapping $SINGLEAGT \
3478 "mtime\>{1m}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
3479 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3481 echo "pcc_ro_data" > $file || error "echo $file failed"
3482 do_facet $SINGLEAGT cat $file || error "cat $file failed"
3483 check_lpcc_state $file "none"
3485 local mtime=$(date -d "2min ago" +%s)
3487 do_facet $SINGLEAGT touch -m -d @$mtime $file ||
3488 error "failed to change mtime for $file $mtime"
3489 do_facet $SINGLEAGT cat $file || error "cat $file failed"
3490 check_lpcc_state $file "readonly"
3492 run_test 41 "Test mtime rule for PCC-RO open attach with O_RDONLY mode"
3495 local loopfile="$TMP/$tfile"
3496 local mntpt="/mnt/pcc.$tdir"
3497 local hsm_root="$mntpt/$tdir"
3498 local file=$DIR/$tfile
3500 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3501 skip "Server does not support PCC-RO"
3503 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
3504 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3505 setup_pcc_mapping $SINGLEAGT \
3506 "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1"
3507 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3509 do_facet $SINGLEAGT echo -n attach_id_not_specified > $file ||
3510 error "Write $file failed"
3511 do_facet $SINGLEAGT $LFS pcc attach -r $file ||
3512 error "PCC attach -r $file failed"
3513 do_facet $SINGLEAGT $LFS pcc state $file
3514 check_lpcc_state $file "readonly"
3516 run_test 42 "PCC attach without attach ID specified"
3519 local loopfile="$TMP/$tfile"
3520 local mntpt="/mnt/pcc.$tdir"
3521 local hsm_root="$mntpt/$tdir"
3522 local file=$DIR/$tfile
3524 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3525 skip "Server does not support PCC-RO"
3527 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
3528 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3529 setup_pcc_mapping $SINGLEAGT \
3530 "size\<{100M}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1"
3531 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3533 echo "attach_root_user_data" > $file || error "echo $file failed"
3535 do_facet $SINGLEAGT $LFS pcc state $file
3536 # Attach by non-root user should fail.
3537 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -r $file &&
3538 error "PCC attach -r $file should fail for non-root user"
3539 do_facet $SINGLEAGT $RUNAS $LFS pcc state $file
3540 check_lpcc_state $file "none"
3542 run_test 43 "Auto attach at open() should add capacity owner check"
3545 local loopfile="$TMP/$tfile"
3546 local mntpt="/mnt/pcc.$tdir"
3547 local hsm_root="$mntpt/$tdir"
3548 local file=$DIR/$tfile
3552 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3553 skip "Server does not support PCC-RO"
3555 setup_loopdev client $loopfile $mntpt 60
3556 mkdir $hsm_root || error "mkdir $hsm_root failed"
3557 setup_pcc_mapping client \
3558 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1\ mmap_conv=0"
3559 $LCTL pcc list $MOUNT
3561 local thresh=$($LCTL get_param -n llite.*.pcc_async_threshold |
3564 stack_trap "$LCTL set_param llite.*.pcc_async_threshold=$thresh"
3565 $LCTL set_param llite.*.pcc_async_threshold=0
3567 dd if=/dev/zero of=$file bs=$bs count=$count ||
3568 error "Write $file failed"
3574 $LFS getstripe -v $file
3575 clear_stats llite.*.stats
3577 for ((i = 0; i < $n; i++)); do
3579 while [ ! -e $DIR/$tfile.lck ]; do
3580 dd if=$file of=/dev/null bs=$bs count=$count ||
3581 error "Read $file failed"
3582 sleep 0.$((RANDOM % 4 + 1))
3589 while [ ! -e $DIR/$tfile.lck ]; do
3590 $LCTL set_param -n ldlm.namespaces.*mdc*.lru_size=clear ||
3591 error "cancel_lru_locks mdc failed"
3598 touch $DIR/$tfile.lck
3600 for ((i = 0; i < $n; i++)); do
3601 wait ${rpids[$i]} || error "$?: read failed"
3603 wait $lpid || error "$?: lock cancel failed"
3605 echo "Finish ========"
3606 $LFS getstripe -v $file
3607 $LCTL get_param llite.*.stats
3609 local attach_num=$(calc_stats llite.*.stats pcc_attach)
3610 local detach_num=$(calc_stats llite.*.stats pcc_detach)
3611 local autoat_num=$(calc_stats llite.*.stats pcc_auto_attach)
3613 echo "attach $attach_num detach $detach_num auto_attach $autoat_num"
3614 (( $attach_num <= 1 )) || error "attach more than 1 time: $attach_num"
3615 rm -f $DIR/$tfile.lck
3617 run_test 44 "Verify valid auto attach without re-fetching the whole files"
3620 local loopfile="$TMP/$tfile"
3621 local loopfile2="$TMP/$tfile.2"
3622 local mntpt="/mnt/pcc.$tdir"
3623 local mntpt2="/mnt/pcc.$tdir.2"
3624 local file1=$DIR/$tfile
3625 local file2=$DIR2/$tfile
3629 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3630 skip "Server does not support PCC-RO"
3632 setup_loopdev client $loopfile $mntpt 100
3633 setup_loopdev client $loopfile2 $mntpt2 100
3634 stack_trap "$LCTL pcc clear $MOUNT" EXIT
3635 $LCTL pcc add $MOUNT $mntpt -p \
3636 "projid={0} roid=$HSM_ARCHIVE_NUMBER ropcc=1" ||
3637 error "failed to config PCC for $MOUNT $mntpt"
3638 stack_trap "$LCTL pcc clear $MOUNT2" EXIT
3639 $LCTL pcc add $MOUNT2 $mntpt2 -p \
3640 "projid={0} roid=$HSM_ARCHIVE_NUMBER ropcc=1" ||
3641 error "failed to config PCC for $MOUNT2 $mntpt2"
3642 $LCTL pcc list $MOUNT
3643 $LCTL pcc list $MOUNT2
3645 local thresh=$(do_facet $SINGLEAGT $LCTL get_param -n \
3646 llite.*.pcc_async_threshold | head -n 1)
3648 stack_trap "do_facet $SINGLEAGT $LCTL \
3649 set_param llite.*.pcc_async_threshold=$thresh"
3650 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0
3652 dd if=/dev/zero of=$file1 bs=$bs count=$count ||
3653 error "Write $file1 failed"
3660 $LFS getstripe -v $file1
3661 clear_stats llite.*.stats
3663 for ((i = 0; i < $n; i++)); do
3665 while [ ! -e $DIR/$tfile.lck ]; do
3666 dd if=$file1 of=/dev/null bs=$bs count=$count ||
3667 error "Read $file failed"
3668 sleep 0.$((RANDOM % 4 + 1))
3674 for ((i = 0; i < $n; i++)); do
3676 while [ ! -e $DIR/$tfile.lck ]; do
3677 dd if=$file2 of=/dev/null bs=$bs count=$count ||
3678 error "Read $file failed"
3679 sleep 0.$((RANDOM % 4 + 1))
3686 touch $DIR/$tfile.lck
3687 for ((i = 0; i < $n; i++)); do
3688 wait ${pids1[$i]} || error "$?: read failed"
3689 wait ${pids2[$i]} || error "$?: read failed"
3692 $LFS getstripe -v $file1
3693 $LCTL get_param llite.*.stats
3695 local attach_num=$(calc_stats llite.*.stats pcc_attach)
3696 local detach_num=$(calc_stats llite.*.stats pcc_detach)
3697 local autoat_num=$(calc_stats llite.*.stats pcc_auto_attach)
3699 echo "attach $attach_num detach $detach_num auto_attach $autoat_num"
3700 (( attach_num <= 2 )) || error "attach more than 2 time: $attach_num"
3701 rm -f $DIR/$tfile.lck
3703 run_test 45 "Concurrent read access from two mount points"
3706 local loopfile="$TMP/$tfile"
3707 local mntpt="/mnt/pcc.$tdir"
3708 local hsm_root="$mntpt/$tdir"
3709 local file=$DIR/$tfile
3710 local fsuuid=$($LFS getname $MOUNT | awk '{print $1}')
3711 local runascmd="$RUNAS -G0"
3713 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3714 skip "Server does not support PCC-RO"
3716 setup_loopdev client $loopfile $mntpt 60
3717 mkdir $hsm_root || error "mkdir $hsm_root failed"
3718 setup_pcc_mapping client \
3719 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1\ mmap_conv=0"
3720 $LCTL pcc list $MOUNT
3722 local mode=$($LCTL get_param -n llite.$fsuuid.pcc_mode)
3726 echo "QQQQQ" > $file || error "write $file failed"
3727 chmod 664 $file || error "chomd $file failed"
3729 $LCTL set_param llite.$fsuuid.pcc_mode="0" ||
3730 error "Set PCC mode failed"
3731 stack_trap "$LCTL set_param llite.$fsuuid.pcc_mode=$mode" EXIT
3732 $runascmd $LFS pcc attach -r $file &&
3733 error "User should not attach $file"
3734 $runascmd cat $file || error "cat $file failed"
3735 check_lpcc_state $file "none" client
3737 $LCTL set_param llite.$fsuuid.pcc_mode="0400" ||
3738 error "Set PCC mode failed"
3739 stack_trap "$LCTL set_param llite.$fsuuid.pcc_mode=$mode" EXIT
3740 $runascmd $LFS pcc attach -r $file &&
3741 error "User should not attach $file"
3742 $runascmd cat $file || error "cat $file failed"
3743 check_lpcc_state $file "none" client
3745 $LCTL set_param llite.$fsuuid.pcc_mode="0004" ||
3746 error "Set PCC mode failed"
3747 $runascmd cat $file || error "cat $file failed"
3748 $LFS pcc state $file
3749 check_lpcc_state $file "readonly" client
3750 $runascmd $LFS pcc detach $file || error "Detach $file failed"
3752 $runascmd stat $file || error "stat $file failed"
3753 $LFS pcc attach -r $file || error "failed to attach $file"
3754 check_lpcc_state $file "readonly" client
3755 $runascmd $LFS pcc detach $file || error "failed to detach $file"
3757 $LCTL set_param llite.$fsuuid.pcc_mode="0040" ||
3758 error "Set PCC mode failed"
3759 chmod 660 $file || error "chmod $file failed"
3760 $runascmd cat $file || error "cat $file failed"
3761 $LFS pcc state $file
3762 check_lpcc_state $file "readonly" client
3763 $runascmd $LFS pcc detach $file || error "failed to detach $file"
3765 $runascmd $LFS pcc attach -r $file || error "attach $file failed"
3766 stat $file || error "stat $file failed"
3767 $LFS pcc state $file
3768 check_lpcc_state $file "readonly" client
3769 $runascmd $LFS pcc detach $file || error "Detach $file failed"
3771 run_test 46 "Verify PCC mode setting works correctly"
3774 local loopfile="$TMP/$tfile"
3775 local mntpt="/mnt/pcc.$tdir"
3776 local hsm_root="$mntpt/$tdir"
3777 local file=$DIR/$tfile
3779 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3780 skip "Server does not support PCC-RO"
3782 setup_loopdev client $loopfile $mntpt 60
3783 mkdir $hsm_root || error "mkdir $hsm_root failed"
3784 setup_pcc_mapping client \
3785 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1\ mmap_conv=0"
3786 $LCTL pcc list $MOUNT
3791 echo "QQQQQ" > $file || error "echo $file failed"
3792 mtime0=$(stat -c "%Y" $file);
3795 cat $file || error "cat $file failed"
3796 wait_readonly_attach_fini $file client
3797 mtime1=$(stat -c "%Y" $file)
3799 (( mtime0 == mtime1 )) || error "mtime changed from $mtime0 to $mtime1"
3801 run_test 47 "mtime should be kept once file attached into PCC"
3804 local loopfile="$TMP/$tfile"
3805 local mntpt="/mnt/pcc.$tdir"
3806 local hsm_root="$mntpt/$tdir"
3807 local file=$DIR/$tfile
3810 setup_loopdev client $loopfile $mntpt 60
3811 mkdir $hsm_root || error "mkdir $hsm_root failed"
3812 setup_pcc_mapping client \
3813 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1"
3815 echo "QQQQQ" > $file || error "echo $file failed"
3816 lpcc_path=$(lpcc_fid2path $hsm_root $file)
3817 cat $file || error "cat $file failed"
3818 check_lpcc_state $file "readonly" client
3820 rm $lpcc_path || error "rm $lpcc_path failed"
3821 $LFS pcc state $file | grep "(unlinked)" || error "$file not unlinked"
3822 [[ "$(cat $file)" =~ "QQQQQ" ]] || error "read $file content failed"
3823 check_lpcc_state $file "readonly" client
3824 $LFS pcc detach $file || error "detach '$file' failed"
3826 run_test 48 "PCC state should check whether the file in local PCC cache"
3829 local loopfile="$TMP/$tfile"
3830 local mntpt="/mnt/pcc.$tdir"
3831 local hsm_root="$mntpt/$tdir"
3832 local file1=$DIR/$tfile
3833 local file2=$DIR2/$tfile
3835 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3836 skip "Server does not support PCC-RO"
3838 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
3839 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3840 setup_pcc_mapping $SINGLEAGT \
3841 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1\ mmap_conv=0"
3842 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3844 local thresh=$(do_facet $SINGLEAGT $LCTL get_param -n \
3845 llite.*.pcc_async_threshold | head -n 1)
3847 stack_trap "do_facet $SINGLEAGT $LCTL set_param \
3848 llite.*.pcc_async_threshold=$thresh"
3849 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G
3862 do_facet $SINGLEAGT dd if=/dev/zero of=$file1 bs=$bs count=$count ||
3863 error "Write $file failed"
3866 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3867 do_facet $SINGLEAGT dd if=$file1 of=/dev/null bs=$bs count=$count ||
3868 error "Read $file failed"
3869 sleep 0.$((RANDOM % 4 + 1))
3875 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3876 do_facet $SINGLEAGT dd if=$file1 of=/dev/null bs=$bs count=$count ||
3877 error "Read $file failed"
3878 sleep 0.$((RANDOM % 4 + 1))
3884 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3885 do_facet $SINGLEAGT dd if=$file1 of=/dev/null bs=$bs count=$count ||
3886 error "Read $file failed"
3887 sleep 0.$((RANDOM % 4 + 1))
3893 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3894 do_facet $SINGLEAGT dd if=$file2 of=/dev/null bs=$bs count=$count ||
3895 error "Read $file failed"
3896 sleep 0.$((RANDOM % 4 + 1))
3902 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3903 do_facet $SINGLEAGT dd if=$file2 of=/dev/null bs=$bs count=$count ||
3904 error "Read $file failed"
3905 sleep 0.$((RANDOM % 4 + 1))
3911 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3912 do_facet $SINGLEAGT dd if=$file2 of=/dev/null bs=$bs count=$count ||
3913 error "Read $file failed"
3914 sleep 0.$((RANDOM % 4 + 1))
3920 while [ ! -e $DIR/sanity-pcc.96.lck ]; do
3921 do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*mdc*.lru_size=clear ||
3922 error "cancel_lru_locks mdc failed"
3929 touch $DIR/sanity-pcc.96.lck
3931 echo "Finish ========"
3932 wait $rpid11 || error "$?: read failed"
3933 wait $rpid12 || error "$?: read failed"
3934 wait $rpid13 || error "$?: read failed"
3935 wait $rpid21 || error "$?: read failed"
3936 wait $rpid22 || error "$?: read failed"
3937 wait $rpid23 || error "$?: read failed"
3938 wait $lpid || error "$?: lock cancel failed"
3940 do_facet $SINGLEAGT $LFS pcc detach $file
3941 rm -f $DIR/sanity-pcc.96.lck
3943 run_test 96 "Auto attach from multiple read process on a node"
3946 local loopfile="$TMP/$tfile"
3947 local mntpt="/mnt/pcc.$tdir"
3948 local hsm_root="$mntpt/$tdir"
3949 local file=$DIR/$tfile
3951 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
3952 skip "Server does not support PCC-RO"
3954 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
3955 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
3956 setup_pcc_mapping $SINGLEAGT \
3957 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1\ mmap_conv=0"
3958 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
3960 local thresh=$(do_facet $SINGLEAGT $LCTL get_param -n \
3961 llite.*.pcc_async_threshold | head -n 1)
3963 stack_trap "do_facet $SINGLEAGT $LCTL set_param \
3964 llite.*.pcc_async_threshold=$thresh"
3965 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=1G
3971 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 ||
3972 error "Write $file failed"
3975 while [ ! -e $DIR/sanity-pcc.97.lck ]; do
3976 echo "T1. $MMAP_CAT $file ..."
3977 do_facet $SINGLEAGT $MMAP_CAT $file > /dev/null ||
3978 error "$MMAP_CAT $file failed"
3979 sleep 0.$((RANDOM % 4 + 1))
3985 while [ ! -e $DIR/sanity-pcc.97.lck ]; do
3986 echo "T2. $MMAP_CAT $file ..."
3987 do_facet $SINGLEAGT $MMAP_CAT $file > /dev/null ||
3988 error "$MMAP_CAT $file failed"
3989 sleep 0.$((RANDOM % 4 + 1))
3995 while [ ! -e $DIR/sanity-pcc.97.lck ]; do
3996 do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*mdc*.lru_size=clear ||
3997 error "cancel_lru_locks mdc failed"
4004 stack_trap "rm -f $DIR/sanity-pcc.97.lck"
4005 touch $DIR/sanity-pcc.97.lck
4006 wait $mpid1 || error "$?: mmap1 failed"
4007 wait $mpid2 || error "$?: mmap2 failed"
4008 wait $lpid || error "$?: cancel locks failed"
4010 do_facet $SINGLEAGT $LFS pcc detach $file
4011 rm -f $DIR/sanity-pcc.97.lck
4013 run_test 97 "two mmap I/O and layout lock cancel"
4016 local loopfile="$TMP/$tfile"
4017 local mntpt="/mnt/pcc.$tdir"
4018 local hsm_root="$mntpt/$tdir"
4019 local file=$DIR/$tfile
4021 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4022 skip "Server does not support PCC-RO"
4024 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
4025 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4026 setup_pcc_mapping $SINGLEAGT \
4027 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1\ mmap_conv=0"
4028 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
4030 local thresh=$(do_facet $SINGLEAGT $LCTL get_param -n \
4031 llite.*.pcc_async_threshold | head -n 1)
4033 stack_trap "do_facet $SINGLEAGT $LCTL set_param \
4034 llite.*.pcc_async_threshold=$thresh"
4035 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0
4046 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 ||
4047 error "Write $file failed"
4050 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4051 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=50 ||
4052 error "Read $file failed"
4053 sleep 0.$((RANDOM % 4 + 1))
4059 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4060 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=50 ||
4061 error "Read $file failed"
4062 sleep 0.$((RANDOM % 4 + 1))
4068 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4069 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=50 ||
4070 error "Read $file failed"
4071 sleep 0.$((RANDOM % 4 + 1))
4077 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4078 do_facet $SINGLEAGT $MMAP_CAT $file > /dev/null ||
4079 error "$MMAP_CAT $file failed"
4080 sleep 0.$((RANDOM % 2 + 1))
4086 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4087 do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*mdc*.lru_size=clear ||
4088 error "cancel_lru_locks mdc failed"
4095 while [ ! -e $DIR/sanity-pcc.98.lck ]; do
4096 do_facet $SINGLEAGT $LCTL set_param -n ldlm.namespaces.*osc*.lru_size=clear ||
4097 error "cancel_lru_locks mdc failed"
4104 stack_trap "rm -f $DIR/sanity-pcc.98.lck"
4105 touch $DIR/sanity-pcc.98.lck
4106 wait $rpid1 || error "$?: read failed"
4107 wait $rpid2 || error "$?: read failed"
4108 wait $rpid3 || error "$?: read failed"
4109 wait $mpid1 || error "$?: mmap failed"
4110 wait $lpid1 || error "$?: cancel locks failed"
4111 wait $lpid2 || error "$?: cancel locks failed"
4113 do_facet $SINGLEAGT $LFS pcc detach $file
4114 rm -f $DIR/sanity-pcc.98.lck
4116 run_test 98 "racer between auto attach and mmap I/O"
4119 local loopfile="$TMP/$tfile"
4120 local mntpt="/mnt/pcc.$tdir"
4121 local hsm_root="$mntpt/$tdir"
4122 local file=$DIR/$tfile
4125 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4126 skip "Server does not support PCC-RO"
4128 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
4129 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4130 setup_pcc_mapping $SINGLEAGT \
4131 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4132 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
4134 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=$cnt ||
4135 error "Write $file failed"
4144 local lckf=$DIR/$tfile.lck
4147 lpcc_path=$(lpcc_fid2path $hsm_root $file)
4149 while [ ! -e $lckf ]; do
4150 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=$cnt conv=notrunc || {
4152 error "failed to write $file"
4154 sleep 0.$((RANDOM % 4 + 1))
4160 while [ ! -e $lckf ]; do
4161 echo "Read $file ..."
4162 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=$cnt ||
4163 error "failed to read $file"
4164 sleep 0.$((RANDOM % 4 + 1))
4170 while [ ! -e $lckf ]; do
4171 do_facet $SINGLEAGT $MMAP_CAT $file > /dev/null ||
4172 error "failed to mmap_cat $file"
4173 sleep 0.$((RANDOM % 4 + 1))
4179 while [ ! -e $lckf ]; do
4180 echo "Unlink $lpcc_path"
4181 do_facet $SINGLEAGT unlink $lpcc_path
4189 while [ ! -e $lckf ]; do
4190 echo "Detach $file ..."
4191 do_facet $SINGLEAGT $LFS pcc detach $file
4192 sleep 0.$((RANDOM % 8 + 1))
4198 echo "==== DONE ===="
4199 stack_trap "rm -f $lckf"
4201 wait $wpid || error "$?: write failed"
4202 wait $rpid || error "$?: read failed"
4203 wait $rpid2 || error "$?: mmap read2 failed"
4204 wait $upid || error "$?: unlink failed"
4205 wait $dpid || error "$?: detach failed"
4207 echo "==== DONE WIAT ===="
4208 lctl get_param osc.*.rpc_stats
4209 do_facet $SINGLEAGT $LFS pcc detach $file
4212 run_test 99 "race among unlink | mmap read | write | detach for PCC-RO file"
4215 local loopfile="$TMP/$tfile"
4216 local mntpt="/mnt/pcc.$tdir"
4217 local hsm_root="$mntpt/$tdir"
4218 local file=$DIR/$tfile
4221 setup_loopdev $SINGLEAGT $loopfile $mntpt 200
4222 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4223 setup_pcc_mapping $SINGLEAGT \
4224 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4225 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
4227 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=$cnt ||
4228 error "Write $file failed"
4237 local lckf=$DIR/$tfile.lck
4240 lpcc_path=$(lpcc_fid2path $hsm_root $file)
4242 while [ ! -e $lckf ]; do
4243 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=$cnt conv=notrunc || {
4245 error "failed to write $file"
4247 sleep 0.$((RANDOM % 4 + 1))
4253 while [ ! -e $lckf ]; do
4254 echo "Read $file ..."
4255 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=$cnt ||
4256 error "failed to read $file (1)"
4257 sleep 0.$((RANDOM % 4 + 1))
4263 while [ ! -e $lckf ]; do
4264 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=$cnt ||
4265 error "failed to read $file (2)"
4266 sleep 0.$((RANDOM % 4 + 1))
4272 while [ ! -e $lckf ]; do
4273 echo "Unlink $lpcc_path"
4274 do_facet $SINGLEAGT unlink $lpcc_path
4282 while [ ! -e $lckf ]; do
4283 echo "Detach $file ..."
4284 do_facet $SINGLEAGT $LFS pcc detach $file
4285 sleep 0.$((RANDOM % 8 + 1))
4291 echo "==== DONE ===="
4293 wait $wpid || error "$?: write failed"
4294 wait $rpid || error "$?: read failed"
4295 wait $rpid2 || error "$?: read2 failed"
4296 wait $upid || error "$?: unlink failed"
4297 wait $dpid || error "$?: detach failed"
4299 echo "==== DONE WIAT ===="
4300 lctl get_param osc.*.rpc_stats
4301 do_facet $SINGLEAGT $LFS pcc detach $file
4304 run_test 99b "race among unlink | two readers | write | detach for PCC-RO file"
4307 local loopfile="$TMP/$tfile"
4308 local mntpt="/mnt/pcc.$tdir"
4309 local hsm_root="$mntpt/$tdir"
4310 local file=$DIR/$tfile
4312 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4313 skip "Server does not support PCC-RO"
4315 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
4316 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4317 setup_pcc_mapping $SINGLEAGT \
4318 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4319 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
4321 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 ||
4322 error "Write $file failed"
4331 lpcc_path=$(lpcc_fid2path $hsm_root $file)
4333 while [ ! -e $DIR/sanity-pcc.100.lck ]; do
4334 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=50 ||
4335 error "failed to write $file"
4336 sleep 0.$((RANDOM % 4 + 1))
4342 while [ ! -e $DIR/sanity-pcc.100.lck ]; do
4343 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=50 ||
4344 error "failed to write $file"
4345 sleep 0.$((RANDOM % 4 + 1))
4351 while [ ! -e $DIR/sanity-pcc.100.lck ]; do
4352 do_facet $SINGLEAGT dd if=$file of=/dev/null bs=1M count=50 ||
4353 error "failed to write $file"
4354 sleep 0.$((RANDOM % 4 + 1))
4360 while [ ! -e $DIR/sanity-pcc.100.lck ]; do
4361 echo "Unlink $lpcc_path"
4362 do_facet $SINGLEAGT unlink $lpcc_path
4370 while [ ! -e $DIR/sanity-pcc.100.lck ]; do
4371 echo "Detach $file ..."
4372 do_facet $SINGLEAGT $LFS pcc detach $file
4373 sleep 0.$((RANDOM % 8 + 1))
4379 stack_trap "rm -f $DIR/sanity-pcc.100.lck"
4380 touch $DIR/sanity-pcc.100.lck
4381 wait $wpid || error "$?: write failed"
4382 wait $rpid || error "$?: read failed"
4383 wait $rpid2 || error "$?: read2 failed"
4384 wait $upid || error "$?: unlink failed"
4385 wait $dpid || error "$?: detach failed"
4387 do_facet $SINGLEAGT $LFS pcc detach $file
4388 rm -f $DIR/sanity-pcc.100.lck
4390 run_test 100 "race among PCC unlink | read | write | detach for PCC-RO file"
4392 #test 101: containers and PCC
4393 #LU-15170: Test mount namespaces with PCC
4394 #This tests the cases where the PCC mount is not present in the container by
4395 #creating a mount namespace without the PCC mount in it (this is probably the
4396 #standard config for most containers)
4398 local loopfile="$TMP/$tfile"
4399 local mntpt="/mnt/pcc.$tdir"
4400 local hsm_root="$mntpt/$tdir"
4401 local file=$DIR/$tdir/$tfile
4403 # Some kernels such as RHEL7 default to 0 user namespaces
4404 local maxuserns=$(do_facet $SINGLEAGT cat /proc/sys/user/max_user_namespaces)
4405 do_facet $SINGLEAGT "echo 10 > /proc/sys/user/max_user_namespaces"
4406 stack_trap "do_facet $SINGLEAGT 'echo $maxuserns > /proc/sys/user/max_user_namespaces'"
4408 # disable apparmor checking of userns temporarily
4409 if [[ "$CLIENT_OS_ID" == "ubuntu" ]] &&
4410 (( $CLIENT_OS_VERSION_CODE >= $(version_code 24) )); then
4413 userns_val=$(do_facet $SINGLEAGT \
4414 sysctl -n kernel.apparmor_restrict_unprivileged_userns)
4415 if (( "$userns_val" != 0 )); then
4416 do_facet $SINGLEAGT \
4417 sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
4418 stack_trap "do_facet $SINGLEAGT sysctl -w kernel.apparmor_restrict_unprivileged_userns=$userns_val"
4422 echo "creating user namespace for $RUNAS_ID"
4423 # Create a mount and user namespace with this command, and leave the
4424 # process running so we can do the rest of our steps
4425 local start=$SECONDS
4426 local PID=$(do_facet $SINGLEAGT \
4427 "$RUNAS unshare -Um sleep 600 &>/dev/null & echo \\\$!")
4428 local elapsed=$((SECONDS - start))
4431 do_facet $SINGLEAGT ps auxww | grep sleep
4432 echo "Created NS: child (sleep) pid=$PID in $elapsed seconds"
4433 [[ -n "$PID" ]] || error "remote sleep start failed"
4434 stack_trap "do_facet $SINGLEAGT kill -9 $PID" EXIT
4435 (( elapsed < 300 )) || error "remote sleep took $elapsed sec to start"
4437 # Map 'RUNAS' to root in the namespace, so it has rights to do whatever
4438 # This is handled by '-r' in unshare in newer versions
4439 do_facet $SINGLEAGT $RUNAS newuidmap $PID 0 $RUNAS_ID 1 ||
4440 error "could not map uid $RUNAS_ID to root in namespace"
4441 do_facet $SINGLEAGT $RUNAS newgidmap $PID 0 $RUNAS_GID 1 ||
4442 error "could not map gid $RUNAS_GID to root in namespace"
4444 # Create PCC after creating namespace; namespace will not have PCC
4446 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
4448 # Create a temp file inside the PCC mount to verify mount namespace
4449 do_facet $SINGLEAGT touch $mntpt/$tfile.tmp
4450 stack_trap "do_facet $SINGLEAGT rm -f $mntpt/$tfile.tmp" EXIT
4451 echo "Check for temp file in PCC mount"
4452 do_facet $SINGLEAGT test -f $mntpt/$tfile.tmp ||
4453 error "Should see $mntpt/$tfile.tmp"
4454 echo "Check for temp file in PCC mount from inside namespace"
4455 do_facet $SINGLEAGT nsenter -t $PID -U -m test -f $mntpt/$tfile.tmp &&
4456 error "Should not see $mntpt/$tfile.tmp from namespace"
4457 rm -f $mntpt/$tfile.tmp
4460 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
4461 setup_pcc_mapping $SINGLEAGT "projid={100}\ rwid=$HSM_ARCHIVE_NUMBER\ pccrw=1"
4463 mkdir_on_mdt0 $DIR/$tdir || error "mkdir $DIR/$tdir failed"
4464 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
4466 echo "Verify open attach from inside mount namespace"
4467 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
4468 error "failed to dd write to $file"
4469 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -w \
4470 -i $HSM_ARCHIVE_NUMBER $file || error "cannot attach $file"
4471 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc state $file
4473 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
4474 # Revoke the layout lock, the PCC-cached file will be
4475 # detached automatically.
4476 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
4477 check_lpcc_state $file "none" $SINGLEAGT "$RUNAS"
4478 do_facet $SINGLEAGT $RUNAS $MULTIOP $file oc ||
4479 error "failed to open $file"
4480 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
4481 # Detach the file but keep the cache, as the file layout generation
4482 # is not changed, so the file is still valid cached in PCC, and can
4483 # be reused from PCC cache directly.
4484 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach -k $file ||
4485 error "PCC detach $file failed"
4486 check_lpcc_state $file "none" $SINGLEAGT "$RUNAS"
4487 do_facet $SINGLEAGT nsenter -t $PID -U -m $MULTIOP $file oc ||
4488 error "failed to open $file"
4489 check_lpcc_state $file "readwrite" $SINGLEAGT "$RUNAS"
4490 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc detach $file ||
4491 error "PCC detach $file failed"
4492 do_facet $SINGLEAGT nsenter -t $PID -U -m dd if=/dev/zero of=$file bs=1024 count=1 ||
4493 error "failed to dd write to $file"
4494 rm -f $file || error "rm $file failed"
4496 echo "Verify auto attach at open from inside NS for RW-PCC"
4497 # nsenter has strange behavior with echo, which means we have to place
4498 # this in a script so we can use sh, otherwise it doesn't execute echo
4500 # NB: using /bin/echo instead of the shell built in does not help
4501 echo "echo -n autoattach_data > $file" > $DIR/$tdir/$tfile.shell
4502 # File is owned by root, make it accessible to RUNAS user
4503 chmod a+rw $DIR/$tdir/$tfile.shell
4504 stack_trap 'rm -f $DIR/$tdir/$tfile.shell' EXIT
4505 do_facet $SINGLEAGT nsenter -t $PID -U -m "bash $DIR/$tdir/$tfile.shell"
4506 do_facet $SINGLEAGT nsenter -t $PID -U -m $LFS pcc attach -w -i $HSM_ARCHIVE_NUMBER \
4507 $file || error "RW-PCC attach $file failed"
4508 check_lpcc_state $file "readwrite"
4510 # Revoke the layout lock, the PCC-cached file will be
4511 # detached automatically.
4512 do_facet $SINGLEAGT $LCTL set_param ldlm.namespaces.*mdc*.lru_size=clear
4513 check_file_data $SINGLEAGT $file "autoattach_data" $PID
4514 check_lpcc_state $file "readwrite"
4516 # Detach the file with -k option, as the file layout generation
4517 # is not changed, so the file is still valid cached in PCC,
4518 # and can be reused from PCC cache directly.
4519 do_facet $SINGLEAGT $LFS pcc detach -k $file ||
4520 error "RW-PCC detach $file failed"
4521 # HSM released exists archived status
4522 check_hsm_flags $file "0x0000000d"
4523 check_file_data $SINGLEAGT $file "autoattach_data" $PID
4524 check_lpcc_state $file "readwrite"
4526 # HSM restore the PCC cached file, the layout generation
4527 # was changed, so the file can not be auto attached.
4528 $LFS hsm_restore $file || error "failed to restore $file"
4529 wait_request_state $(path2fid $file) RESTORE SUCCEED
4530 check_lpcc_state $file "none"
4531 # HSM exists archived status
4532 check_hsm_flags $file "0x00000009"
4534 run_test 101a "Test auto attach in mount namespace (simulated container)"
4537 grep -q io_uring_setup /proc/kallsyms ||
4538 skip "Client OS does not support io_uring I/O engine"
4539 io_uring_probe || skip "kernel does not support io_uring fully"
4541 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4542 skip "Server does not support PCC-RO"
4544 which fio || skip_env "no fio installed"
4545 fio --enghelp | grep -q io_uring ||
4546 skip_env "fio does not support io_uring I/O engine"
4548 local loopfile="$TMP/$tfile"
4549 local mntpt="/mnt/pcc.$tdir"
4550 local hsm_root="$mntpt/$tdir"
4551 local file=$DIR/$tfile
4553 setup_loopdev client $loopfile $mntpt 60
4554 mkdir $hsm_root || error "mkdir $hsm_root failed"
4555 setup_pcc_mapping client \
4556 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4558 local thresh=$($LCTL get_param -n llite.*.pcc_async_threshold |
4561 stack_trap "do_facet $SINGLEAGT $LCTL set_param \
4562 llite.*.pcc_async_threshold=$thresh"
4563 do_facet $SINGLEAGT $LCTL set_param llite.*.pcc_async_threshold=0
4565 local ioengine="io_uring"
4569 do_facet $SINGLEAGT fio --name=seqwrite --ioengine=$ioengine \
4570 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
4571 --iodepth=64 --size=$size --filename=$file --rw=write ||
4572 error "fio seqwrite $file failed"
4574 # Read the file will trigger the buffered read from Lustre OSTs and
4575 # write to PCC copy as @pcc_async_threshold is set with 0.
4576 do_facet $SINGLEAGT fio --name=seqread --ioengine=$ioengine \
4577 --bs=$PAGE_SIZE --direct=1 --numjobs=$numjobs \
4578 --iodepth=64 --size=$size --filename=$file --rw=read ||
4579 error "fio seqread $file failed"
4581 run_test 102 "PCC-RO should not hange for io_uring I/O engine"
4584 local loopfile="$TMP/$tfile"
4585 local mntpt="/mnt/pcc.$tdir"
4586 local hsm_root="$mntpt/$tdir"
4587 local file=$DIR/$tfile
4590 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4591 skip "Server does not support PCC-RO"
4593 setup_loopdev client $loopfile $mntpt 10
4594 mkdir $hsm_root || error "mkdir $hsm_root failed"
4595 setup_pcc_mapping client \
4596 "projid={0}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4597 $LCTL pcc list $MOUNT
4598 clear_stats llite.*.stats
4600 dd if=/dev/zero of=$file bs=$bs count=1 ||
4601 error "Write $file failed"
4603 cat $file > /dev/null
4604 cat $file > /dev/null
4606 echo "==== stats ===="
4607 $LCTL get_param llite.*.stats
4609 local attach_num=$(calc_stats llite.*.stats pcc_attach_bytes)
4610 local attach_bytes=$(calc_stats_sum llite.*.stats pcc_attach_bytes)
4611 local hit_num=$(calc_stats llite.*.stats pcc_hit_bytes)
4612 local hit_bytes=$(calc_stats_sum llite.*.stats pcc_hit_bytes)
4613 echo "attach_num: $attach_num, attach_bytes: $attach_bytes, hit_num: $hit_num, hit_bytes: $hit_bytes"
4615 (( $attach_num == 1 )) || error "wrong attach number: $attach_num"
4616 (( $attach_bytes == $bs )) || error "wrong attach bytes: $attach_bytes"
4617 (( $hit_num == 2 )) || error "wrong hit number: $hit_num"
4618 (( $hit_bytes == $((2 * bs)) )) || error "wrong hit bytes: $hit_bytes"
4620 run_test 203 "Verify attach/hit bytes statistics data"
4623 local loopfile="$TMP/$tfile"
4624 local mntpt="/mnt/pcc.$tdir"
4625 local hsm_root="$mntpt/$tdir"
4626 local file=$DIR/$tfile.dat
4627 local xattrname=trusted.pin
4630 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4631 skip "Server does not support PCC-RO"
4633 (( $MDS1_VERSION >= $(version_code 2.15.61) )) ||
4634 skip "Need server version at least 2.15.61"
4636 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
4637 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4638 setup_pcc_mapping $SINGLEAGT \
4639 "fname={*.dat}\ roid=$HSM_ARCHIVE_NUMBER\ pccro=1"
4641 do_facet $SINGLEAGT touch $file
4644 do_facet $SINGLEAGT $LFS pcc pin -i $HSM_ARCHIVE_NUMBER $file ||
4645 error "failed to pcc pin $file"
4646 do_facet $SINGLEAGT getfattr $file -d -m $xattrname
4647 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \
4649 [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" ]] ||
4650 error "incorrect xattr $xattrname=$xattrvalue"
4653 do_facet $SINGLEAGT $LFS pcc pin -i 100 $file ||
4654 error "failed to pcc pin $file"
4655 do_facet $SINGLEAGT getfattr $file -d -m $xattrname
4656 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \
4658 [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" &&
4659 "$xattrvalue" =~ "hsm: 100" ]] ||
4660 error "incorrect xattr $xattrname=$xattrvalue"
4662 # pin with id=2 again
4663 do_facet $SINGLEAGT $LFS pcc pin -i $HSM_ARCHIVE_NUMBER $file ||
4664 error "failed to pcc pin $file"
4665 do_facet $SINGLEAGT getfattr $file -d -m $xattrname
4668 do_facet $SINGLEAGT $LFS pcc unpin -i 100 $file ||
4669 error "failed to pcc unpin $file"
4670 do_facet $SINGLEAGT getfattr $file -d -m $xattrname
4671 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \
4673 [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" ]] ||
4674 error "incorrect xattr $xattrname=$xattrvalue"
4677 do_facet $SINGLEAGT $LFS pcc unpin -i $HSM_ARCHIVE_NUMBER $file ||
4678 error "failed to pcc unpin $file"
4679 do_facet $SINGLEAGT getfattr $file -d -m $xattrname
4680 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -d -m $xattrname \
4682 [[ -z "$xattrvalue" ]] ||
4683 error "incorrect xattr $xattrname=$xattrvalue"
4685 # pin/unpin operation should NOT trigger autocache
4686 check_lpcc_state $file "none"
4688 # pin/unpin operation should NOT block autocache triggered by read
4689 do_facet $SINGLEAGT cat $file
4690 check_lpcc_state $file "readonly"
4692 run_test 204a "pin/unpin pcc flag"
4696 local dir=$DIR/$tdir
4697 local file=$dir/$tfile
4700 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4701 skip "Server does not support PCC-RO"
4703 (( $MDS1_VERSION >= $(version_code 2.15.61) )) ||
4704 skip "Need server version at least 2.15.61"
4706 old=$(do_facet mds1 $LCTL get_param -n mdt.*.enable_pin_gid | head -1)
4707 do_facet mds1 $LCTL set_param mdt.*.enable_pin_gid=0
4708 stack_trap "do_facet mds1 $LCTL set_param mdt.*.enable_pin_gid=$old"
4710 mkdir_on_mdt0 -p $dir || error "mkdir $dir failed"
4711 chmod 777 $dir || error "chown $dir failed"
4712 $RUNAS touch $file || error "touch $file failed"
4713 $RUNAS $LFS pcc pin -i $id $file &&
4714 error "normal user should not able to pin any file"
4715 $LFS pcc pin -i $id $file || error "root should be able to pin any file"
4716 getfattr -d -n trusted.pin $file
4717 $RUNAS $LFS pcc unpin -i $id $file &&
4718 error "nonroot user should fail to unpin file"
4719 $LFS pcc unpin -i $id $file ||
4720 error "root should be able to unpin any file"
4721 $RUNAS $LFS pcc pin -i $id $file &&
4722 error "nonroot user should fail to pin file"
4724 do_facet mds1 $LCTL set_param mdt.*.enable_pin_gid=-1
4725 $RUNAS $LFS pcc pin -i $id $file ||
4726 error "failed to pin $file when enable_pin_gid=-1"
4727 $RUNAS $LFS pcc unpin -i $id $file ||
4728 error "failed to unpin $file when enable_pin_gid=-1"
4730 do_facet mds1 $LCTL set_param mdt.*.enable_pin_gid=$RUNAS_GID
4731 $RUNAS $LFS pcc pin -i $id $file ||
4732 error "failed to pin $file when enable_pin_gid=$RUNAS_GID"
4733 $RUNAS $LFS pcc unpin -i $id $file ||
4734 error "failed to unpin $file when enable_pin_gid=$RUNAS_GID"
4735 $LFS pcc pin -i $id $file ||
4736 error "failed to pin $file when enable_pin_gid=$RUNAS_GID"
4737 $LFS pcc unpin -i $id $file ||
4738 error "failed to unpin $file when enable_pin_gid=$RUNAS_GID"
4740 run_test 204b "Permission check for the pin/unpin operation"
4743 local loopfile="$TMP/$tfile"
4744 local mntpt="/mnt/pcc.$tdir"
4745 local hsm_root="$mntpt/$tdir"
4746 local file=$DIR/$tfile
4747 local xattrname="trusted.pin"
4751 $LCTL get_param -n mdc.*.connect_flags | grep -q pcc_ro ||
4752 skip "Server does not support PCC-RO"
4754 (( $MDS1_VERSION >= $(version_code 2.15.61) )) ||
4755 skip "Need server version at least 2.15.61"
4757 do_facet $SINGLEAGT echo -n attach_id_not_specified > $file ||
4758 error "dd write $file failed"
4759 do_facet $SINGLEAGT $LFS pcc pin $file &&
4760 error "Pin should fail for a client without any PCC backend"
4762 setup_loopdev $SINGLEAGT $loopfile $mntpt 60
4763 do_facet $SINGLEAGT mkdir $hsm_root || error "mkdir $hsm_root failed"
4764 setup_pcc_mapping $SINGLEAGT \
4765 "projid={100}\ roid=$HSM_ARCHIVE_NUMBER\ ropcc=1"
4767 do_facet $SINGLEAGT $LFS pcc pin $file || error "failed to pin $file"
4768 do_facet $SINGLEAGT getfattr $file -n $xattrname
4769 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -n $xattrname --only-values)
4770 [[ "$xattrvalue" =~ "hsm: $HSM_ARCHIVE_NUMBER" ]] ||
4771 error "incorrect xattr $xattrname=$xattrvalue"
4773 do_facet $SINGLEAGT $LFS pcc unpin $file || error "failed to PCC unpin $file"
4774 do_facet $SINGLEAGT getfattr $file -n $xattrname
4775 xattrvalue=$(do_facet $SINGLEAGT getfattr $file -n $xattrname --only-values)
4776 [[ -z "$xattrvalue" ]] || error "incorrect xattr $xattrname=$xattrvalue"
4778 run_test 204c "PCC pin/unpin without attach ID specified"
4780 complete_test $SECONDS
4781 check_and_cleanup_lustre