3 # Run select tests by setting ONLY, or as arguments to the script.
4 # Skip specific tests by setting EXCEPT.
11 export PATH=$PWD/$SRCDIR:$SRCDIR:$PWD/$SRCDIR/utils:$PATH:/sbin:/usr/sbin
14 # bug number for skipped test:
16 # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!
18 ENABLE_PROJECT_QUOTAS=${ENABLE_PROJECT_QUOTAS:-true}
20 LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
22 . $LUSTRE/tests/test-framework.sh
24 . ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
27 MULTIOP=${MULTIOP:-multiop}
28 OPENFILE=${OPENFILE:-openfile}
29 MMAP_CAT=${MMAP_CAT:-mmap_cat}
30 MOUNT_2=${MOUNT_2:-"yes"}
33 # script only handles up to 10 MDTs (because of MDT_PREFIX)
34 [ $MDSCOUNT -gt 9 ] &&
35 error "script cannot handle more than 9 MDTs, please fix" && exit
37 check_and_setup_lustre
39 if [[ $(lustre_version_code $SINGLEMDS) -lt $(version_code 2.12.52) ]]; then
40 skip_env "Need MDS version at least 2.12.52" && exit
43 # $RUNAS_ID may get set incorrectly somewhere else
44 if [[ $UID -eq 0 && $RUNAS_ID -eq 0 ]]; then
45 skip_env "\$RUNAS_ID set to 0, but \$UID is also 0!" && exit
47 check_runas_id $RUNAS_ID $RUNAS_GID $RUNAS
48 if getent group nobody; then
50 elif getent group nogroup; then
53 error "No generic nobody group"
58 # if there is no CLIENT1 defined, some tests can be ran on localhost
59 CLIENT1=${CLIENT1:-$HOSTNAME}
60 # if CLIENT2 doesn't exist then use CLIENT1 instead
61 # All tests should use CLIENT2 with MOUNT2 only therefore it will work if
63 # Exception is the test which need two separate nodes
64 CLIENT2=${CLIENT2:-$CLIENT1}
70 local expected_size="$3"
72 size=$(do_facet $client stat "--printf=%s" $fpath)
73 [[ $size == "$expected_size" ]] || error \
74 "expected $fpath size: $expected_size got: $size"
81 local lustre_fpath="$3"
82 local expected_size="$4"
84 check_file_size $client $lpcc_fpath $expected_size
85 check_file_size $client $lustre_fpath $expected_size
92 local expected_data="$3"
94 path_data=$(do_facet $client cat $path)
95 [[ "x$path_data" == "x$expected_data" ]] || error \
96 "expected $path: $expected_data, got: $path_data"
102 local lpcc_fpath="$2"
103 local lustre_fpath="$3"
104 local expected_data="$4"
106 check_file_data "$client" "$lpcc_fpath" "$expected_data"
107 check_file_data "$client" "$lustre_fpath" "$expected_data"
113 local lustre_path="$2"
114 local fid=$(path2fid $lustre_path)
120 f_seq=$(echo $fid | awk -F ':' '{print $1}')
121 f_oid=$(echo $fid | awk -F ':' '{print $2}')
122 f_ver=$(echo $fid | awk -F ':' '{print $3}')
124 printf "%s/%04x/%04x/%04x/%04x/%04x/%04x/%s" \
125 $hsm_root $(($f_oid & 0xFFFF)) \
126 $(($f_oid >> 16 & 0xFFFF)) \
127 $(($f_seq & 0xFFFF)) \
128 $(($f_seq >> 16 & 0xFFFF)) \
129 $(($f_seq >> 32 & 0xFFFF)) \
130 $(($f_seq >> 48 & 0xFFFF)) $fid
135 local lustre_path="$1"
136 local expected_state="$2"
137 local facet=${3:-$SINGLEAGT}
138 local state=$(do_facet $facet $LFS pcc state $lustre_path |
139 awk -F 'type: ' '{print $2}' | awk -F ',' '{print $1}')
141 [[ "x$state" == "x$expected_state" ]] || error \
142 "$lustre_path expected pcc state: $expected_state, but got: $state"
148 # populate MDT device array
151 # cleanup from previous bad setup
154 # for recovery tests, coordinator needs to be started at mount
156 # the lustre conf must be without hsm on (like for sanity.sh)
157 echo "Set HSM on and start"
158 cdt_set_mount_state enabled
159 cdt_check_state enabled
161 echo "Set sanity-hsm HSM policy"
162 cdt_set_sanity_policy
164 # finished requests are quickly removed from list
165 set_hsm_param grace_delay 10
167 cleanup_pcc_mapping() {
168 local facet=${1:-$SINGLEAGT}
170 do_facet $facet $LCTL pcc clear $MOUNT
173 setup_pcc_mapping() {
174 local facet=${1:-$SINGLEAGT}
175 local hsm_root=${hsm_root:-$(hsm_root "$facet")}
178 [ -z "$param" ] && param="projid={100}\ rwid=$HSM_ARCHIVE_NUMBER"
179 stack_trap "cleanup_pcc_mapping $facet" EXIT
180 do_facet $facet $LCTL pcc add $MOUNT $hsm_root -p $param
187 local agt_facet=$SINGLEAGT
188 local hsm_root=$(hsm_root)
189 local file=$DIR/$tdir/$tfile
194 $project && enable_project_quota
196 do_facet $SINGLEAGT rm -rf $hsm_root
197 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
199 is_project_quota_supported || project=false
201 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
203 $project && lfs project -sp $project_id $DIR/$tdir
205 do_facet $SINGLEAGT "echo -n attach_origin > $file"
207 check_lpcc_state $file "none"
208 do_facet $SINGLEAGT $LFS pcc attach -i \
209 $HSM_ARCHIVE_NUMBER $file ||
210 error "pcc attach $file failed"
213 check_lpcc_state $file "readwrite"
214 # HSM released exists archived status
215 check_hsm_flags $file "0x0000000d"
216 lpcc_path=$(lpcc_fid2path $hsm_root $file)
217 check_lpcc_data $SINGLEAGT $lpcc_path $file "attach_origin"
219 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=7654321 count=1
220 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 7654321
222 do_facet $SINGLEAGT $TRUNCATE $file 1234567 ||
223 error "truncate failed"
224 check_lpcc_sizes $SINGLEAGT $lpcc_path $file 1234567
225 check_lpcc_state $file "readwrite"
227 do_facet $SINGLEAGT "echo -n file_data > $file"
228 check_lpcc_state $file "readwrite"
229 # HSM released exists archived status
230 check_hsm_flags $file "0x0000000d"
231 check_lpcc_data $SINGLEAGT $lpcc_path $file "file_data"
233 if [ $CLIENTCOUNT -lt 2 -o $restore ]; then
234 $LFS hsm_restore $file || error \
235 "failed to restore $file"
236 wait_request_state $(path2fid $file) RESTORE SUCCEED
238 path_data=$(do_node $CLIENT2 cat $file)
239 [[ "x$path_data" == "xfile_data" ]] || error \
240 "expected file_data, got: $path_data"
243 check_lpcc_state $file "none"
244 # HSM exists archived status
245 check_hsm_flags $file "0x00000009"
247 echo -n "new_data" > $file
248 check_lpcc_state $file "none"
249 # HSM exists dirty archived status
250 check_hsm_flags $file "0x0000000b"
251 check_file_data $SINGLEAGT $file "new_data"
253 echo "Attach and detach testing"
255 do_facet $SINGLEAGT "echo -n new_data2 > $file"
257 check_lpcc_state $file "none"
258 do_facet $SINGLEAGT $LFS pcc attach -i \
259 $HSM_ARCHIVE_NUMBER $file ||
260 error "PCC attach $file failed"
262 check_lpcc_state $file "readwrite"
263 # HSM released exists archived status
264 check_hsm_flags $file "0x0000000d"
265 do_facet $SINGLEAGT "echo -n attach_detach > $file"
266 echo "Start to detach the $file"
267 do_facet $SINGLEAGT $LFS pcc detach $file ||
268 error "PCC detach $file failed"
269 check_lpcc_state $file "none"
270 # HSM released exists archived status
271 check_hsm_flags $file "0x0000000d"
272 check_file_data $SINGLEAGT $file "attach_detach"
276 lpcc_rw_test true false
278 run_test 1a "Test manual lfs pcc attach with manual HSM restore"
281 lpcc_rw_test false false
283 run_test 1b "Test manual lfs pcc attach with restore on remote access"
286 lpcc_rw_test true true
288 run_test 1c "Test automated attach using Project ID with manual HSM restore"
291 lpcc_rw_test false true
293 run_test 1d "Test Project ID with remote access"
296 local file=$DIR/$tdir/$tfile
297 local hsm_root=$(hsm_root)
300 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
302 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
303 chmod 777 $DIR/$tdir || error "chmod 777 $DIR/$tdir failed"
305 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
306 error "failed to dd write to $file"
307 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
308 $file || error "failed to attach file $file"
309 check_lpcc_state $file "readwrite"
310 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
311 error "failed to dd read from $file"
312 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
313 error "failed to truncate $file"
314 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
315 error "failed to truncate $file"
316 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
317 error "failed to dd write to $file"
318 check_lpcc_state $file "readwrite"
320 # non-root user is forbidden to access PCC file directly
321 lpcc_path=$(lpcc_fid2path $hsm_root $file)
322 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
323 error "non-root user can touch access PCC file $lpcc_path"
324 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
325 count=1 && error "non-root user can read PCC file $lpcc_path"
326 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
327 count=1 && error "non-root user can write PCC file $lpcc_path"
329 local perm=$(do_facet $SINGLEAGT stat -c %a $lpcc_path)
331 [[ $perm == "0" ]] || error "PCC file permission ($perm) is not zero"
333 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
334 error "failed to detach file $file"
335 check_lpcc_state $file "none"
337 run_test 1e "Test RW-PCC with non-root user"
341 local agt_facet=$SINGLEAGT
342 local hsm_root=$(hsm_root)
343 local file=$DIR/$tdir/$tfile
345 ! is_project_quota_supported &&
346 skip "project quota is not supported"
349 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
351 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
352 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
353 $LFS project -sp $project_id $DIR/$tdir ||
354 error "failed to set project for $DIR/$tdir"
356 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
357 error "failed to dd write to $file"
359 check_lpcc_state $file "readwrite"
360 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
361 error "failed to dd read from $file"
362 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 256 ||
363 error "failed to truncate $file"
364 do_facet $SINGLEAGT $RUNAS $TRUNCATE $file 2048 ||
365 error "failed to truncate $file"
366 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
367 error "failed to dd write from $file"
368 check_lpcc_state $file "readwrite"
370 # non-root user is forbidden to access PCC file directly
371 lpcc_path=$(lpcc_fid2path $hsm_root $file)
372 do_facet $SINGLEAGT $RUNAS touch $lpcc_path &&
373 error "non-root user can touch access PCC file $lpcc_path"
374 do_facet $SINGLEAGT $RUNAS dd if=$lpcc_path of=/dev/null bs=1024 \
375 count=1 && error "non-root user can read PCC file $lpcc_path"
376 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$lpcc_path bs=1024 \
377 count=1 && error "non-root user can write PCC file $lpcc_path"
379 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
380 error "failed to detach file $file"
381 check_lpcc_state $file "none"
383 run_test 1f "Test auto RW-PCC cache with non-root user"
386 local file=$DIR/$tfile
388 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
391 dd if=/dev/zero of=$file bs=1024 count=1 ||
392 error "failed to dd write to $file"
393 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
394 error "non-root user can dd write to $file"
395 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
396 error "failed to attach file $file"
397 check_lpcc_state $file "readwrite"
398 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 &&
399 error "non-root user can dd write to $file"
400 chmod 777 $file || error "chmod 777 $file failed"
401 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
402 error "non-root user cannot write $file with permission (777)"
404 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file &&
405 error "non-root user or non owner can detach $file"
406 chown $RUNAS_ID $file || error "chown $RUNAS_ID $file failed"
407 do_facet $SINGLEAGT $RUNAS $LFS pcc detach $file ||
408 error "failed to detach file $file"
409 check_lpcc_state $file "none"
410 do_facet $SINGLEAGT $RUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
411 error "non-root user cannot read to $file with permisson (777)"
413 run_test 1g "General permission test for RW-PCC"
416 # When a process created a LPCC file and holding the open,
417 # another process on the same client should be able to open the file.
421 local agt_facet=$SINGLEAGT
422 local hsm_root=$(hsm_root)
423 local agt_host=$(facet_active_host $SINGLEAGT)
425 ! is_project_quota_supported &&
426 skip "project quota is not supported" && return
429 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
431 file=$DIR/$tdir/multiop
435 do_facet $SINGLEAGT $LFS project -sp $project_id $DIR/$tdir ||
436 error "failed to set project quota"
437 rmultiop_start $agt_host $file O_c || error "open $file failed"
438 # HSM released exists archived status
439 check_hsm_flags $file "0x0000000d"
440 do_facet $SINGLEAGT "echo -n multiopen_data > $file" ||
441 error "failed to echo multiopen_data to $file"
443 lpcc_path=$(lpcc_fid2path $hsm_root $file)
444 do_facet $SINGLEAGT ls -l $lpcc_path ||
445 error "failed to ls $lpcc_path"
446 check_lpcc_data $SINGLEAGT $lpcc_path $file "multiopen_data"
447 # HSM released exists archived status
448 check_hsm_flags $file "0x0000000d"
450 rmultiop_stop $agt_host || error "close $file failed"
452 run_test 2a "Test multi open when creating"
454 get_remote_client() {
455 current_id=$(do_facet $SINGLEAGT hostname)
456 for client in ${CLIENTS//,/ }
458 r_id=$(do_node $client hostname)
459 if [ $r_id != $current_id ]; then
467 # When a process created a LPCC file and holding the open, another
468 # process on the different client should be able to open the file
469 # and perform IO on the file.
472 local agt_facet=$SINGLEAGT
473 local hsm_root=$(hsm_root)
474 local agt_host=$(facet_active_host $SINGLEAGT)
476 needclients 2 || return 0
478 remote_client=$(get_remote_client)
481 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
483 file=$DIR/$tdir/multiop
487 do_facet $SINGLEAGT "echo -n file_data > $file"
488 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
489 $file || error "PCC attach $file failed"
490 check_lpcc_state $file "readwrite"
492 rmultiop_start $agt_host $file O_c || error "open $file failed"
494 do_node $remote_client "echo -n multiopen_data > $file"
496 # PCC cached file should be automatically detached
497 check_lpcc_state $file "none"
499 check_file_data $SINGLEAGT $file "multiopen_data"
500 rmultiop_stop $agt_host || error "close $file failed"
501 check_file_data $SINGLEAGT $file "multiopen_data"
503 do_node $remote_client cat $file || error \
504 "cat $file on remote client failed"
505 do_node $remote_client echo -n "multiopen_data" > $file \
506 || error "write $file on remote client failed"
508 run_test 2b "Test multi remote open when creating"
511 local agt_host=$(facet_active_host $SINGLEAGT)
512 local file=$DIR/$tdir/$tfile
513 local file2=$DIR2/$tdir/$tfile
516 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
521 do_facet $SINGLEAGT "echo -n file_data > $file"
522 do_facet $SINGLEAGT lfs pcc attach -i $HSM_ARCHIVE_NUMBER \
523 $file || error "PCC attach $file failed"
524 check_lpcc_state $file "readwrite"
526 rmultiop_start $agt_host $file O_c || error "open $file failed"
528 echo -n multiopen_data > $file2
530 # PCC cached file should be automatically detached
531 check_lpcc_state $file "none"
533 check_file_data $SINGLEAGT $file "multiopen_data"
534 rmultiop_stop $agt_host || error "close $file failed"
535 check_file_data $SINGLEAGT $file "multiopen_data"
537 cat $file2 || error "cat $file on mount $MOUNT2 failed"
538 echo -n "multiopen_data" > $file2 ||
539 error "write $file on mount $MOUNT2 failed"
541 run_test 2c "Test multi open on different mount points when creating"
544 local file=$DIR/$tdir/$tfile
546 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
549 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
550 dd if=/dev/zero of=$file bs=1024 count=1 ||
551 error "failed to dd write to $file"
553 echo "Start to attach/detach the file: $file"
554 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
555 error "failed to attach file $file"
556 check_lpcc_state $file "readwrite"
557 do_facet $SINGLEAGT $LFS pcc detach $file ||
558 error "failed to detach file $file"
559 check_lpcc_state $file "none"
561 echo "Repeat to attach/detach the same file: $file"
562 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
563 error "failed to attach file $file"
564 check_lpcc_state $file "readwrite"
565 do_facet $SINGLEAGT $LFS pcc detach $file ||
566 error "failed to detach file $file"
567 check_lpcc_state $file "none"
569 run_test 3a "Repeat attach/detach operations"
573 local file=$DIR/$tdir/$tfile
575 needclients 3 || return 0
577 # Start all of the copytools and setup PCC
578 for n in $(seq $AGTCOUNT); do
579 copytool setup -f agt$n -a $n -m $MOUNT
580 setup_pcc_mapping agt$n "projid={100}\ rwid=$n"
583 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
584 dd if=/dev/zero of=$file bs=1024 count=1 ||
585 error "failed to dd write to $file"
587 echo "Start to attach/detach $file on $agt1_HOST"
588 do_facet agt1 $LFS pcc attach -i 1 $file ||
589 error "failed to attach file $file"
590 check_lpcc_state $file "readwrite" agt1
591 do_facet agt1 $LFS pcc detach $file ||
592 error "failed to detach file $file"
593 check_lpcc_state $file "none" agt1
595 echo "Repeat to attach/detach $file on $agt2_HOST"
596 do_facet agt2 $LFS pcc attach -i 2 $file ||
597 error "failed to attach file $file"
598 check_lpcc_state $file "readwrite" agt2
599 do_facet agt2 $LFS pcc detach $file ||
600 error "failed to detach file $file"
601 check_lpcc_state $file "none" agt2
603 echo "Try attach on two agents"
604 do_facet agt1 $LFS pcc attach -i 1 $file ||
605 error "failed to attach file $file"
606 check_lpcc_state $file "readwrite" agt1
607 do_facet agt2 $LFS pcc attach -i 2 $file ||
608 error "failed to attach file $file"
609 check_lpcc_state $file "readwrite" agt2
610 # The later attach PCC agent should succeed,
611 # the former agent should be detached automatically.
612 check_lpcc_state $file "none" agt1
613 do_facet agt2 $LFS pcc detach $file ||
614 error "failed to detach file $file"
615 check_lpcc_state $file "none" agt2
617 run_test 3b "Repeat attach/detach operations on multiple clients"
622 ! is_project_quota_supported &&
623 skip "project quota is not supported" && return
626 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
629 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
630 lfs project -sp $project_id $DIR/$tdir ||
631 error "lfs project -sp $project_id $DIR/$tdir failed"
633 # mmap_sanity tst7 failed on the local ext4 filesystem.
634 # It seems that Lustre filesystem does special process for tst 7.
635 # Thus, we exclude tst7 from the PCC testing.
636 $LUSTRE/tests/mmap_sanity -d $DIR/$tdir -m $DIR2/$tdir -e 7 ||
637 error "mmap_sanity test failed"
640 run_test 4 "Auto cache test for mmap"
643 local file=$DIR/$tdir/$tfile
645 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
648 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
649 do_facet $SINGLEAGT "echo -n attach_mmap_data > $file" ||
650 error "echo $file failed"
652 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
653 error "failed to attach file $file"
654 check_lpcc_state $file "readwrite"
656 local content=$($MMAP_CAT $file)
658 [[ $content == "attach_mmap_data" ]] ||
659 error "mmap cat data mismatch: $content"
661 $LFS hsm_restore $file || error "failed to restore $file"
662 wait_request_state $(path2fid $file) RESTORE SUCCEED
663 check_lpcc_state $file "none"
665 content=$($MMAP_CAT $file)
666 [[ $content == "attach_mmap_data" ]] ||
667 error "mmap cat data mismatch: $content"
669 run_test 5 "Mmap & cat a RW-PCC cached file"
672 local file=$DIR/$tdir/$tfile
675 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
678 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
680 echo -n mmap_write_data > $file || error "echo write $file failed"
681 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
682 error "failed to attach file $file"
683 check_lpcc_state $file "readwrite"
685 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
686 error "could not mmap $file"
687 check_lpcc_state $file "readwrite"
688 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
689 # After mmap write via multiop, the first character of each page
691 [[ $content == "nmap_write_data" ]] ||
692 error "mmap write data mismatch: $content"
693 check_lpcc_state $file "readwrite"
695 do_facet $SINGLEAGT $LFS pcc detach $file ||
696 error "failed to detach file $file"
698 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
699 [[ $content == "nmap_write_data" ]] ||
700 error "mmap write data mismatch: $content"
702 run_test 6 "Test mmap write on RW-PCC "
705 local file=$DIR/$tdir/$tfile
708 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
711 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
713 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
714 error "failed to attach file $file"
715 check_lpcc_state $file "readwrite"
716 check_file_data $SINGLEAGT $file "QQQQQ"
717 # define OBD_FAIL_LLITE_PCC_DETACH_MKWRITE 0x1412
718 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1412
719 # HSM released exists archived status
720 check_hsm_flags $file "0x0000000d"
722 # multiop mmap write increase the first character of each page with 1
723 do_facet $SINGLEAGT $MULTIOP $file OSMWUc ||
724 error "mmap write $file failed"
725 check_lpcc_state $file "none"
726 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
727 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
729 run_test 7a "Fake file detached between fault() and page_mkwrite() for RW-PCC"
732 local file=$DIR/$tdir/$tfile
736 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
739 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
741 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
742 error "failed to attach file $file"
743 check_lpcc_state $file "readwrite"
744 check_file_data $SINGLEAGT $file "QQQQQ"
745 # define OBD_FAIL_LLITE_PCC_MKWRITE_PAUSE 0x1413
746 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1413 fail_val=20
747 # HSM released exists archived status
748 check_hsm_flags $file "0x0000000d"
750 # multiop mmap write increase the first character of each page with 1
751 do_facet $SINGLEAGT $MULTIOP $file OSMWUc &
754 do_facet $SINGLEAGT $LFS pcc detach $file ||
755 error "failed to detach file $file"
757 wait $pid || error "multiop mmap write failed"
758 check_lpcc_state $file "none"
759 content=$(do_facet $SINGLEAGT $MMAP_CAT $file)
760 [[ $content == "RQQQQ" ]] || error "data mismatch: $content"
762 run_test 7b "Test the race with concurrent mkwrite and detach"
765 local file=$DIR/$tdir/$tfile
767 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
770 mkdir -p $DIR/$tdir || error "mkdir $DIR/$tdir failed"
773 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
774 error "failed to attach file $file"
775 check_lpcc_state $file "readwrite"
776 check_file_data $SINGLEAGT $file "QQQQQ"
778 # define OBD_FAIL_LLITE_PCC_FAKE_ERROR 0x1411
779 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1411
780 do_facet $SINGLEAGT "echo -n ENOSPC_write > $file"
781 # Above write will return -ENOSPC failure and retry the IO on normal
782 # IO path. It will restore the HSM released file.
783 check_lpcc_state $file "none"
784 check_file_data $SINGLEAGT $file "ENOSPC_write"
786 run_test 8 "Test fake -ENOSPC tolerance for RW-PCC"
794 do_facet $facet mkdir -p $mntpt || error "mkdir -p $hsm_root failed"
795 stack_trap "do_facet $facet rm -rf $mntpt" EXIT
796 do_facet $facet dd if=/dev/zero of=$file bs=1M count=$size
797 stack_trap "do_facet $facet rm -f $file" EXIT
798 do_facet $facet mkfs.ext4 $file ||
799 error "mkfs.ext4 $file failed"
800 do_facet $facet file $file
801 do_facet $facet mount -t ext4 -o loop,usrquota,grpquota $file $mntpt ||
802 error "mount -o loop,usrquota,grpquota $file $mntpt failed"
803 stack_trap "do_facet $facet $UMOUNT $mntpt" EXIT
807 local loopfile="$TMP/$tfile"
808 local mntpt="/mnt/pcc.9a"
809 local hsm_root="$mntpt/$tdir"
810 local file=$DIR/$tfile
812 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
814 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
816 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
818 touch $file || error "touch $file failed"
819 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
820 error "fail to attach $file"
821 check_lpcc_state $file "readwrite"
822 # write 60M data, it is larger than the capacity of PCC backend
823 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1M count=60 ||
824 error "fail to dd write $file"
825 check_lpcc_state $file "none"
826 check_file_size $SINGLEAGT $file 62914560
828 run_test 9 "Test -ENOSPC tolerance on loop PCC device for RW-PCC"
830 test_usrgrp_quota() {
831 local loopfile="$TMP/$tfile"
832 local mntpt="/mnt/pcc.$tdir"
833 local hsm_root="$mntpt/$tdir"
837 [[ $ug == "g" ]] && id=$RUNAS_GID
839 setup_loopdev $SINGLEAGT $loopfile $mntpt 50
840 do_facet $SINGLEAGT quotacheck -c$ug $mntpt ||
841 error "quotacheck -c$ug $mntpt failed"
842 do_facet $SINGLEAGT quotaon -$ug $mntpt ||
843 error "quotaon -$ug $mntpt failed"
844 do_facet $SINGLEAGT setquota -$ug $id 0 20480 0 0 $mntpt ||
845 error "setquota -$ug $id on $mntpt failed"
846 do_facet $SINGLEAGT repquota -${ug}vs $mntpt
848 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMVER" -h "$hsm_root"
850 do_facet $SINGLEAGT $LCTL pcc list $MOUNT
852 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
854 local file1=$DIR/$tdir/${ug}quotaA
855 local file2=$DIR/$tdir/${ug}quotaB
857 dd if=/dev/zero of=$file1 bs=1M count=15 ||
858 error "dd write $file1 failed"
859 dd if=/dev/zero of=$file2 bs=1M count=15 ||
860 error "dd write $file2 failed"
861 chown $RUNAS_ID:$RUNAS_GID $file1 ||
862 error "chown $RUNAS_ID:$RUNAS_GID $file1 failed"
863 chown $RUNAS_ID:$RUNAS_GID $file2 ||
864 error "chown $RUNAS_ID:$RUNAS_GID $file2 failed"
865 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
866 $file1 || error "attach $file1 failed"
867 do_facet $SINGLEAGT $RUNAS $LFS pcc attach -i $HSM_ARCHIVE_NUMBER \
868 $file2 && error "attach $file2 should fail due to quota limit"
869 check_lpcc_state $file1 "readwrite"
870 check_lpcc_state $file2 "none"
872 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file1 bs=1M count=30 ||
873 error "dd write $file1 failed"
874 # -EDQUOT error should be tolerated via fallback to normal Lustre path.
875 check_lpcc_state $file1 "none"
876 do_facet $SINGLEAGT $LFS pcc detach $file1 ||
877 error "failed to detach file $file"
882 test_usrgrp_quota "u"
884 run_test 10a "Test RW-PCC with user quota on loop PCC device"
887 test_usrgrp_quota "g"
889 run_test 10b "Test RW-PCC with group quota on loop PCC device"
892 local file=$DIR/$tdir/$tfile
893 local hsm_root=$(hsm_root)
894 local file=$DIR/$tdir/$tfile
898 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
901 mkdir $DIR/$tdir || error "mkdir $DIR/$tdir failed"
902 do_facet $SINGLEAGT "echo -n QQQQQ > $file"
903 lpcc_path=$(lpcc_fid2path $hsm_root $file)
904 lpcc_dir=$(dirname $lpcc_path)
905 echo "Lustre file: $file LPCC dir: $lpcc_dir"
906 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
907 error "mkdir -p $lpcc_dir failed"
908 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
909 error "failed to attach $file"
910 check_lpcc_state $file "readwrite"
911 check_file_data $SINGLEAGT $file "QQQQQ"
912 do_facet $SINGLEAGT $LFS pcc detach $file ||
913 error "failed to detach $file"
914 rm $file || error "rm $file failed"
916 # The parent directory of the PCC file is immutable
917 do_facet $SINGLEAGT "echo -n immutable_dir > $file"
918 lpcc_path=$(lpcc_fid2path $hsm_root $file)
919 lpcc_dir=$(dirname $lpcc_path)
920 echo "Lustre file: $file LPCC dir: $lpcc_dir"
921 do_facet $SINGLEAGT mkdir -p $lpcc_dir ||
922 error "mkdir -p $lpcc_dir failed"
923 do_facet $SINGLEAGT chattr +i $lpcc_dir ||
924 error "chattr +i $lpcc_dir failed"
925 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
926 error "attach $file with immutable directory should be failed"
927 do_facet $SINGLEAGT chattr -i $lpcc_dir ||
928 error "chattr -i $lpcc_dir failed"
929 rm $file || error "rm $file failed"
931 # The PCC file path is set to a directory
932 do_facet $SINGLEAGT "echo -n pcc_file_path_is_dir > $file"
933 lpcc_path=$(lpcc_fid2path $hsm_root $file)
934 do_facet $SINGLEAGT mkdir -p $lpcc_path ||
935 error "mkdir -p $lpcc_path failed"
936 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &&
937 error "attach $file should fail as PCC path is a directory"
938 rm $file || error "rm $file failed"
940 run_test 11 "Test attach fault injection with simulated PCC file path"
943 local file=$DIR/$tfile
946 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
949 echo -n race_rw_attach_hsmremove > $file
950 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file ||
951 error "attach $file failed"
952 do_facet $SINGLEAGT $LFS pcc detach $file || error "detach $file failed"
953 # HSM released exists archived status
954 check_hsm_flags $file "0x0000000d"
955 # define OBD_FAIL_LLITE_PCC_ATTACH_PAUSE 0x1414
956 do_facet $SINGLEAGT $LCTL set_param fail_loc=0x1414 fail_val=20
957 do_facet $SINGLEAGT $LFS pcc attach -i $HSM_ARCHIVE_NUMBER $file &
961 wait_request_state $(path2fid $file) RESTORE SUCCEED
962 $LFS hsm_remove $file || error "hsm remove $file failed"
963 wait $pid && error "RW-PCC attach $file should fail"
967 run_test 12 "RW-PCC attach races with concurrent HSM remove"
971 local rule="${idstr}={$2}"
973 local file=$DIR/$tdir/$tfile
975 setup_pcc_mapping $SINGLEAGT "$rule\ rwid=$HSM_ARCHIVE_NUMBER"
976 $LCTL pcc list $MOUNT
978 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
979 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
981 rm -f $file || error "rm $file failed"
982 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
983 error "failed to dd write to $file"
984 check_lpcc_state $file "readwrite"
985 do_facet $SINGLEAGT $myRUNAS dd if=$file of=/dev/null bs=1024 count=1 ||
986 error "failed to dd read from $file"
987 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 256 ||
988 error "failed to truncate $file"
989 do_facet $SINGLEAGT $myRUNAS $TRUNCATE $file 2048 ||
990 error "failed to truncate $file"
991 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
992 error "failed to dd write from $file"
993 check_lpcc_state $file "readwrite"
995 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
996 error "failed to detach file $file"
997 check_lpcc_state $file "none"
1003 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1004 test_rule_id "u" "500" "runas -u 500"
1005 test_rule_id "g" "500" "runas -u 500 -g 500"
1007 run_test 13a "Test auto RW-PCC create caching for UID/GID rule"
1012 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1013 setup_pcc_mapping $SINGLEAGT \
1014 "fname={*.h5\ suffix.*\ Mid*dle}\ rwid=$HSM_ARCHIVE_NUMBER"
1015 $LCTL pcc list $MOUNT
1017 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1018 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1020 file=$DIR/$tdir/prefix.h5
1021 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1022 error "failed to dd write to $file"
1023 check_lpcc_state $file "readwrite"
1024 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1025 error "failed to detach file $file"
1026 check_lpcc_state $file "none"
1027 rm $file || error "rm $file failed"
1029 file=$DIR/$tdir/suffix.doc
1030 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1031 error "failed to dd write to $file"
1032 check_lpcc_state $file "readwrite"
1033 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1034 error "failed to detach file $file"
1035 check_lpcc_state $file "none"
1036 rm $file || error "rm $file failed"
1038 file=$DIR/$tdir/MidPADdle
1039 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1040 error "failed to dd write to $file"
1041 check_lpcc_state $file "readwrite"
1042 do_facet $SINGLEAGT $myRUNAS $LFS pcc detach $file ||
1043 error "failed to detach file $file"
1044 check_lpcc_state $file "none"
1045 rm $file || error "rm $file failed"
1047 file=$DIR/$tdir/Midpad
1048 do_facet $SINGLEAGT $RUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1049 error "failed to dd write to $file"
1050 check_lpcc_state $file "none"
1051 rm $file || error "rm $file failed"
1055 run_test 13b "Test auto RW-PCC create caching for file name with wildcard"
1061 ! is_project_quota_supported &&
1062 echo "Skip project quota is not supported" && return 0
1064 enable_project_quota
1065 copytool setup -m "$MOUNT" -a "$HSM_ARCHIVE_NUMBER"
1066 setup_pcc_mapping $SINGLEAGT \
1067 "projid={100\ 200}\&fname={*.h5},uid={500}\&gid={1000}\ rwid=$HSM_ARCHIVE_NUMBER"
1068 $LCTL pcc list $MOUNT
1069 do_facet $SINGLEAGT mkdir -p $DIR/$tdir
1070 chmod 777 $DIR/$tdir || error "chmod 0777 $DIR/$tdir failed"
1072 mkdir -p $DIR/$tdir/proj || error "mkdir $DIR/$tdir/proj failed"
1073 mkdir -p $DIR/$tdir/proj2 || error "mkdir $DIR/$tdir/proj2 failed"
1074 $LFS project -sp 100 $DIR/$tdir/proj ||
1075 error "failed to set project for $DIR/$tdir/proj"
1076 $LFS project -sp 200 $DIR/$tdir/proj2 ||
1077 error "failed to set project for $DIR/$tdir/proj2"
1079 file=$DIR/$tdir/proj/notcache
1080 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1081 error "failed to dd write to $file"
1082 check_lpcc_state $file "none"
1083 rm $file || error "rm $file failed"
1085 file=$DIR/$tdir/proj/autocache.h5
1086 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1087 error "failed to dd write to $file"
1088 check_lpcc_state $file "readwrite"
1089 do_facet $SINGLEAGT $LFS pcc detach $file ||
1090 error "failed to detach $file"
1091 rm $file || error "rm $file failed"
1093 file=$DIR/$tdir/proj2/notcache
1094 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1095 error "failed to dd write to $file"
1096 check_lpcc_state $file "none"
1097 rm $file || error "rm $file failed"
1099 file=$DIR/$tdir/proj2/autocache.h5
1100 do_facet $SINGLEAGT dd if=/dev/zero of=$file bs=1024 count=1 ||
1101 error "failed to dd write to $file"
1102 check_lpcc_state $file "readwrite"
1103 do_facet $SINGLEAGT $LFS pcc detach $file ||
1104 error "failed to detach $file"
1105 rm $file || error "rm $file failed"
1107 file=$DIR/$tdir/ugidcache
1108 myRUNAS="runas -u 500 -g 1000"
1109 do_facet $SINGLEAGT $myRUNAS dd if=/dev/zero of=$file bs=1024 count=1 ||
1110 error "failed to dd write to $file"
1111 check_lpcc_state $file "readwrite"
1112 do_facet $SINGLEAGT $LFS pcc detach $file ||
1113 error "failed to detach $file"
1114 rm $file || error "rm $file failed"
1118 run_test 13c "Check auto RW-PCC create caching for UID/GID/ProjID/fname rule"
1121 check_and_cleanup_lustre