From be36f62196439346aada3008072120a54d59e4a3 Mon Sep 17 00:00:00 2001 From: Chris Horn Date: Tue, 6 Jun 2017 11:09:21 -0500 Subject: [PATCH] LUDOC-366 nrs: Add delay policy document Delay is a new policy for NRS. This patch adds documentation on how to use it. Signed-off-by: Chris Horn Change-Id: Ibf1d74a9aea0893e07dc4cbe834d54804d214b3c Reviewed-on: https://review.whamcloud.com/27501 Tested-by: Jenkins Reviewed-by: Andreas Dilger --- LustreTuning.xml | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 170 insertions(+), 1 deletion(-) diff --git a/LustreTuning.xml b/LustreTuning.xml index 0b0a2c5..d7f2e04 100644 --- a/LustreTuning.xml +++ b/LustreTuning.xml @@ -1099,6 +1099,18 @@ regular_requests: queued: 2420 active: 268 + - name: tbf + state: stopped + fallback: no + queued: 0 + active: 0 + + - name: delay + state: stopped + fallback: no + queued: 0 + active: 0 + high_priority_requests: - name: fifo state: started @@ -1123,7 +1135,19 @@ high_priority_requests: fallback: no queued: 0 active: 0 - + + - name: tbf + state: stopped + fallback: no + queued: 0 + active: 0 + + - name: delay + state: stopped + fallback: no + queued: 0 + active: 0 + NRS policy state is shown in either one or two sections, depending on the PTLRPC service being queried. The first section is named @@ -1718,6 +1742,151 @@ $ lctl set_param ost.OSS.ost_io.nrs_tbf_rule="hp stop loginnode" +
+ + <indexterm> + <primary>tuning</primary> + <secondary>Network Request Scheduler (NRS) Tuning</secondary> + <tertiary>Delay policy</tertiary> + </indexterm>Delay policy + The NRS Delay policy seeks to perturb the timing of request + processing at the PtlRPC layer, with the goal of simulating high server + load, and finding and exposing timing related problems. When this policy + is active, upon arrival of a request the policy will calculate an offset, + within a defined, user-configurable range, from the request arrival + time, to determine a time after which the request should be handled. + The request is then stored using the cfs_binheap implementation, + which sorts the request according to the assigned start time. + Requests are removed from the binheap for handling once their start + time has been passed. + The Delay policy can be enabled on all types of PtlRPC services, + and has the following tunables that can be used to adjust its behavior: + + + + + {service}.nrs_delay_min + + The + {service}.nrs_delay_min tunable controls the + minimum amount of time, in seconds, that a request will be delayed by + this policy. The default is 5 seconds. To read this value run: + +lctl get_param {service}.nrs_delay_min + For example, to read the minimum delay set on the ost_io + service, run: + +$ lctl get_param ost.OSS.ost_io.nrs_delay_min +ost.OSS.ost_io.nrs_delay_min=reg_delay_min:5 +hp_delay_min:5 + To set the minimum delay in RPC processing, run: + +lctl set_param {service}.nrs_delay_min=0-65535 + This will set the minimum delay time on a given service, for both + regular and high-priority RPCs (if the PtlRPC service supports + high-priority RPCs), to the indicated value. + For example, to set the minimum delay time on the ost_io service + to 10, run: + +$ lctl set_param ost.OSS.ost_io.nrs_delay_min=10 +ost.OSS.ost_io.nrs_delay_min=10 + For PtlRPC services that support high-priority RPCs, to set a + different minimum delay time for regular and high-priority RPCs, run: + + +lctl set_param {service}.nrs_delay_min=reg_delay_min|hp_delay_min:0-65535 + + For example, to set the minimum delay time on the ost_io service + for high-priority RPCs to 3, run: + +$ lctl set_param ost.OSS.ost_io.nrs_delay_min=hp_delay_min:3 +ost.OSS.ost_io.nrs_delay_min=hp_delay_min:3 + Note, in all cases the minimum delay time cannot exceed the + maximum delay time. + + + + {service}.nrs_delay_max + + The + {service}.nrs_delay_max tunable controls the + maximum amount of time, in seconds, that a request will be delayed by + this policy. The default is 300 seconds. To read this value run: + + lctl get_param {service}.nrs_delay_max + For example, to read the maximum delay set on the ost_io + service, run: + +$ lctl get_param ost.OSS.ost_io.nrs_delay_max +ost.OSS.ost_io.nrs_delay_max=reg_delay_max:300 +hp_delay_max:300 + To set the maximum delay in RPC processing, run: + lctl set_param {service}.nrs_delay_max=0-65535 + + This will set the maximum delay time on a given service, for both + regular and high-priority RPCs (if the PtlRPC service supports + high-priority RPCs), to the indicated value. + For example, to set the maximum delay time on the ost_io service + to 60, run: + +$ lctl set_param ost.OSS.ost_io.nrs_delay_max=60 +ost.OSS.ost_io.nrs_delay_max=60 + For PtlRPC services that support high-priority RPCs, to set a + different maximum delay time for regular and high-priority RPCs, run: + + lctl set_param {service}.nrs_delay_max=reg_delay_max|hp_delay_max:0-65535 + For example, to set the maximum delay time on the ost_io service + for high-priority RPCs to 30, run: + +$ lctl set_param ost.OSS.ost_io.nrs_delay_max=hp_delay_max:30 +ost.OSS.ost_io.nrs_delay_max=hp_delay_max:30 + Note, in all cases the maximum delay time cannot be less than the + minimum delay time. + + + + {service}.nrs_delay_pct + + The + {service}.nrs_delay_pct tunable controls the + percentage of requests that will be delayed by this policy. The + default is 100. Note, when a request is not selected for handling by + the delay policy due to this variable then the request will be handled + by whatever fallback policy is defined for that service. If no other + fallback policy is defined then the request will be handled by the + FIFO policy. To read this value run: + lctl get_param {service}.nrs_delay_pct + For example, to read the percentage of requests being delayed on + the ost_io service, run: + +$ lctl get_param ost.OSS.ost_io.nrs_delay_pct +ost.OSS.ost_io.nrs_delay_pct=reg_delay_pct:100 +hp_delay_pct:100 + To set the percentage of delayed requests, run: + +lctl set_param {service}.nrs_delay_pct=0-100 + This will set the percentage of requests delayed on a given + service, for both regular and high-priority RPCs (if the PtlRPC service + supports high-priority RPCs), to the indicated value. + For example, to set the percentage of delayed requests on the + ost_io service to 50, run: + +$ lctl set_param ost.OSS.ost_io.nrs_delay_pct=50 +ost.OSS.ost_io.nrs_delay_pct=50 + + For PtlRPC services that support high-priority RPCs, to set a + different delay percentage for regular and high-priority RPCs, run: + + lctl set_param {service}.nrs_delay_pct=reg_delay_pct|hp_delay_pct:0-100 + + For example, to set the percentage of delayed requests on the + ost_io service for high-priority RPCs to 5, run: + $ lctl set_param ost.OSS.ost_io.nrs_delay_pct=hp_delay_pct:5 +ost.OSS.ost_io.nrs_delay_pct=hp_delay_pct:5 + + + +
-- 1.8.3.1