From 32cfaf9c16679ab7b21e8fbdecc0804eb4edb63f Mon Sep 17 00:00:00 2001 From: Timothy Day Date: Fri, 24 Nov 2023 17:38:10 +0000 Subject: [PATCH] LU-16502 lutf: remove lutf_start.sh wrapper script Remove bash wrapper script lutf_start.sh. Set the environment natively in Python. LUTF currently involves a number of nested wrapper scripts. Hence, this patch aims to simplify LUTF. It also makes it simplier to import this script into another Python script, by providing a reusable function to set the environment natively. Test-Parameters: @lnet Signed-off-by: Timothy Day Change-Id: I56d80c12f9e50f3f8de1668ffa04c855a9829601 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/53237 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Frank Sehr Reviewed-by: Cyril Bordage Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lustre/tests/lutf.sh | 3 +- lustre/tests/lutf/python/config/lutf_start.py | 111 ++++++++++++++++++++------ lustre/tests/lutf/python/config/lutf_start.sh | 30 ------- 3 files changed, 88 insertions(+), 56 deletions(-) mode change 100644 => 100755 lustre/tests/lutf/python/config/lutf_start.py delete mode 100644 lustre/tests/lutf/python/config/lutf_start.sh diff --git a/lustre/tests/lutf.sh b/lustre/tests/lutf.sh index bdc55b3..ef8031d 100755 --- a/lustre/tests/lutf.sh +++ b/lustre/tests/lutf.sh @@ -75,7 +75,8 @@ rm -f /tmp/tf.skip set +e echo "+++++++++++STARTING LUTF" -. "$LUSTRE/tests/lutf/python/config/lutf_start.sh" "$CONFIG" +export LUTF_ENV_VARS="$CONFIG" +"$LUSTRE/tests/lutf/python/config/lutf_start.py" rc=$? echo "-----------STOPPING LUTF: $rc" diff --git a/lustre/tests/lutf/python/config/lutf_start.py b/lustre/tests/lutf/python/config/lutf_start.py old mode 100644 new mode 100755 index d92cd1b..3a7e79b --- a/lustre/tests/lutf/python/config/lutf_start.py +++ b/lustre/tests/lutf/python/config/lutf_start.py @@ -1,28 +1,31 @@ -""" -lutf_start.py is a script intended to be run from the lutf.sh -It relies on a set of environment variables to be set. If they -are not set the script will exit: - -*_HOST: nodes to run the LUTF on. They must be unique (optional) -ONLY: A script to run (optional) -SUITE: A suite to run (optional) -LUTF_SHELL: If specified it'll run the python interpreter (optional) -MASTER_PORT: The port on which the master will listen -TELNET_PORT: The port on which a telnet session can be established to the agent -LUSTRE: The path to the lustre tests directory -PYTHONPATH: The path where the python scripts and libraries are located -LUTFPATH: Path to the lutf directory - -Purpose: --------- -start an instance of the LUTF on the master and the agents -""" - -import os, re, yaml, paramiko, copy +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +# +# This file is part of Lustre, http://www.lustre.org/ +# +# lustre/tests/lutf/python/config/lutf_start.py +# +# Start an instance of the LUTF on the master and the agents. +# lutf_start.py is a script intended to be run from the lutf.sh +# It relies on a set of environment variables to be set. If they +# are not set the script will exit: +# +# *_HOST: nodes to run the LUTF on. They must be unique (optional) +# ONLY: A script to run (optional) +# SUITE: A suite to run (optional) +# LUTF_SHELL: If specified it'll run the python interpreter (optional) +# MASTER_PORT: The port on which the master will listen +# TELNET_PORT: The port on which a telnet session can be established to the agent +# LUSTRE: The path to the lustre tests directory +# LUTFPATH: Path to the lutf directory +# +# Author: Amir Shehata +# + +import os, re, yaml, paramiko, copy, sys import shlex, subprocess, time from pathlib import Path -from lutf_exception import LutfDumper -from lutf_paramiko import lutf_exec_remote_cmd, lutf_put_file, lutf_get_file cfg_yaml = {'lutf': {'shell': True, 'agent': False, 'telnet-port': -1, 'master-address': None, 'master-port': -1, 'node-name': None, @@ -100,7 +103,7 @@ class LUTF: self.__cfg_yaml['lutf']['node-name'] = key self.__cfg_yaml['lutf']['master-name'] = mname self.__cfg_yaml['lutf']['lutf-path'] = os.environ['LUTFPATH'] - self.__cfg_yaml['lutf']['py-path'] = os.environ['PYTHONPATH'] + self.__cfg_yaml['lutf']['py-path'] = ':'.join(sys.path) try: sl = re.split(',| ', os.environ['SUITE']) @@ -212,7 +215,7 @@ class LUTF: lutf_get_file(host, rfpath, os.path.join(tmp_dir, rfname)) def check_environment(self): - needed_vars = ['LUTFPATH','PYTHONPATH','TELNET_PORT', 'MASTER_PORT', + needed_vars = ['LUTFPATH', 'TELNET_PORT', 'MASTER_PORT', 'HOSTNAME', 'LD_LIBRARY_PATH', 'PATH', 'LUSTRE'] for var in needed_vars: @@ -263,7 +266,65 @@ class LUTF: return rc +def configure_lutf_environment(): + # Grab old LD_LIBRARY_PATH + restart = 0 + old = os.environ.get("LD_LIBRARY_PATH") + + # Find or set LUSTRE location + lustre = os.environ.get("LUSTRE") + + if not lustre: + lustre = "./lustre/" + + # Set LUTFPATH, used internally by LUTF + lutf = lustre + "/tests/lutf/" + os.environ["LUTFPATH"] = lutf + + # Set new LD_LIBRARY_PATH + new_path = lutf + ":" + lutf + "src" + + # Resolve LD_LIBRARY_PATH and possibly restart + if old: + if not new_path in os.environ['LD_LIBRARY_PATH']: + os.environ["LD_LIBRARY_PATH"] = old + ":" + new_path + restart = 1 + else: + os.environ["LD_LIBRARY_PATH"] = new_path + restart = 1 + + if restart: + try: + os.execv(sys.argv[0], sys.argv) + except Exception as e: + sys.exit('EXCEPTION: Failed to Execute under modified environment, '+e) + + # Set PYTHONPATH + sys.path.append(lutf) + sys.path.append(lutf + "src/") + sys.path.append(lutf + "python/") + sys.path.append(lutf + "python/tests/") + sys.path.append(lutf + "python/config/") + sys.path.append(lutf + "python/deploy/") + sys.path.append(lutf + "python/infra/") + + # Update PATH + os.environ["PATH"] = os.environ.get("PATH") + ":" + lutf + "src/" + ":" + lutf + + # Set remaining variables + if os.environ.get("TELNET_PORT") is None: + os.environ["TELNET_PORT"] = "8181" + if os.environ.get("MASTER_PORT") is None: + os.environ["MASTER_PORT"] = "8282" + if os.environ.get("LUTF_SHELL") is None: + os.environ["LUTF_SHELL"] = "batch" + if __name__ == '__main__': + configure_lutf_environment() + + from lutf_exception import LutfDumper + from lutf_paramiko import lutf_exec_remote_cmd, lutf_put_file, lutf_get_file + lutf = LUTF() rc = lutf.check_environment() diff --git a/lustre/tests/lutf/python/config/lutf_start.sh b/lustre/tests/lutf/python/config/lutf_start.sh deleted file mode 100644 index 5ca7f33..0000000 --- a/lustre/tests/lutf/python/config/lutf_start.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# note the cfg/.sh should export all environment variables -# required. EX: export ost4_HOST=lustre01 - -export PYTHONPATH=$PYTHONPATH:$LUSTRE/tests/lutf/:$LUSTRE/tests/lutf/src/:$LUSTRE/tests/lutf/python:$LUSTRE/tests/lutf/python/tests/:$LUSTRE/tests/lutf/python/config/:$LUSTRE/tests/lutf/python/deploy:$LUSTRE/tests/lutf/python/infra - -export LUTFPATH=$LUSTRE/tests/lutf/ - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LUTFPATH:$LUTFPATH/src:$LUSTRE/../lnet/utils/lnetconfig/.libs/ - -export PATH=$PATH:$LUSTRE/tests/lutf/src:$LUSTRE/tests/lutf - -if [[ -z "${TELNET_PORT}" ]]; then - export TELNET_PORT=8181 -fi - -if [[ -z "${MASTER_PORT}" ]]; then - export MASTER_PORT=8282 -fi - -if [[ -z "${LUTF_SHELL}" ]]; then - export LUTF_SHELL=batch -fi - -if [[ -z "${PYTHONBIN}" ]]; then - export PYTHONBIN=python3 -fi - -export LUTF_ENV_VARS=$1 -$PYTHONBIN "$LUSTRE/tests/lutf/python/config/lutf_start.py" -- 1.8.3.1