[lldb][test] Support remote run of Shell tests (#95986)

1. This commit adds LLDB_TEST_PLATFORM_URL, LLDB_TEST_SYSROOT,
LLDB_TEST_PLATFORM_WORKING_DIR, LLDB_SHELL_TESTS_DISABLE_REMOTE cmake
flags to pass arguments for cross-compilation and remote running of both Shell&API tests.
2. To run Shell tests remotely, it adds 'platform select' and 'platform connect' commands to %lldb
substitution.
3. 'remote-linux' feature added to lit to disable tests failing with
remote execution.
4. A separate working directory is assigned to each test to avoid
conflicts during parallel test execution.
5. Remote Shell testing is run only when LLDB_TEST_SYSROOT is set for
building test sources. The recommended compiler for that is Clang.

---------

Co-authored-by: Vladimir Vereschaka <vvereschaka@accesssoftek.com>

lldb/docs/resources/test.rst

@@ -60,7 +60,8 @@ something like ``target.BreakpointCreateByName`` [#]_.
 A good rule of thumb is to prefer shell tests when what is being tested is
 relatively simple. Expressivity is limited compared to the API tests, which
 means that you have to have a well-defined test scenario that you can easily
-match with ``FileCheck``.
+match with ``FileCheck``. Though Shell tests can be run remotely, behavior
+specific to remote debugging must be tested with API tests instead.
 
 Another thing to consider are the binaries being debugged, which we call
 inferiors. For shell tests, they have to be relatively simple. The
@@ -592,15 +593,17 @@ test suite, but there are two things to have in mind:
    multiple connections. For more information on how to setup remote debugging
    see the Remote debugging page.
 2. You must tell the test-suite how to connect to the remote system. This is
-   achieved using the ``--platform-name``, ``--platform-url`` and
-   ``--platform-working-dir`` parameters to ``dotest.py``. These parameters
-   correspond to the platform select and platform connect LLDB commands. You
-   will usually also need to specify the compiler and architecture for the
-   remote system.
+   achieved using the ``LLDB_TEST_PLATFORM_URL``, ``LLDB_TEST_PLATFORM_WORKING_DIR``
+   flags to cmake, and ``--platform-name`` parameter to ``dotest.py``.
+   These parameters correspond to the platform select and platform connect
+   LLDB commands. You will usually also need to specify the compiler and
+   architecture for the remote system.
+3. Remote Shell tests execution is currently supported only for Linux target
+   platform. It's triggered when ``LLDB_TEST_SYSROOT`` is provided for building
+   test sources. It can be disabled by setting ``LLDB_TEST_SHELL_DISABLE_REMOTE=On``.
+   Shell tests are not guaranteed to pass against remote target if the compiler
+   being used is other than Clang.
 
-Currently, running the remote test suite is supported only with ``dotest.py`` (or
-dosep.py with a single thread), but we expect this issue to be addressed in the
-near future.
 
 Running tests in QEMU System Emulation Environment
 ``````````````````````````````````````````````````

lldb/test/API/lit.cfg.py

@@ -303,6 +303,13 @@ if is_configured("enabled_plugins"):
 # In particular, (1) is visited at the top of the file, since the script
 # derives other information from it.
 
+if is_configured("lldb_platform_url"):
+    dotest_cmd += ["--platform-url", config.lldb_platform_url]
+if is_configured("lldb_platform_working_dir"):
+    dotest_cmd += ["--platform-working-dir", config.lldb_platform_working_dir]
+if is_configured("cmake_sysroot"):
+    dotest_cmd += ["--sysroot", config.cmake_sysroot]
+
 if is_configured("dotest_user_args_str"):
     dotest_cmd.extend(config.dotest_user_args_str.split(";"))
 

lldb/test/API/lit.site.cfg.py.in

@@ -24,6 +24,9 @@ config.lua_executable = "@Lua_EXECUTABLE@"
 config.lua_test_entry = "TestLuaAPI.py"
 config.dotest_common_args_str = lit_config.substitute("@LLDB_TEST_COMMON_ARGS@")
 config.dotest_user_args_str = lit_config.substitute("@LLDB_TEST_USER_ARGS@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
 config.lldb_enable_python = @LLDB_ENABLE_PYTHON@
 config.dotest_lit_args_str = None
 config.enabled_plugins = []

lldb/test/CMakeLists.txt

@@ -235,6 +235,8 @@ if (CMAKE_SIZEOF_VOID_P EQUAL 8)
   set(LLDB_IS_64_BITS 1)
 endif()
 
+set(LLDB_TEST_SHELL_DISABLE_REMOTE OFF CACHE BOOL "Disable remote Shell tests execution")
+
 # These values are not canonicalized within LLVM.
 llvm_canonicalize_cmake_booleans(
   LLDB_BUILD_INTEL_PT
@@ -244,6 +246,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_SHARED_LIBS
   LLDB_HAS_LIBCXX
+  LLDB_TEST_SHELL_DISABLE_REMOTE
   LLDB_TOOL_LLDB_SERVER_BUILD
   LLDB_USE_SYSTEM_DEBUGSERVER
   LLDB_IS_64_BITS)

lldb/test/Shell/Expr/TestIRMemoryMap.test

@@ -1,4 +1,4 @@
-# UNSUPPORTED: system-windows
+# UNSUPPORTED: system-windows, remote-linux
 
 # RUN: %clangxx_host %p/Inputs/call-function.cpp -g -o %t
 

lldb/test/Shell/Settings/TestEchoCommands.test

@@ -1,8 +1,3 @@
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands true'  -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsAll.out
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands false' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNoComments.out
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-commands false'         -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNone.out
-
-RUN: echo start >%t.file
-RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
-RUN: echo done >>%t.file
-RUN: FileCheck  %S/Inputs/EchoCommandsQuiet.out <%t.file

lldb/test/Shell/Settings/TestEchoCommandsQuiet.test

@@ -0,0 +1,6 @@
+Can't mute platform select command.
+UNSUPPORTED: remote-linux
+RUN: echo start >%t.file
+RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
+RUN: echo done >>%t.file
+RUN: FileCheck  %S/Inputs/EchoCommandsQuiet.out <%t.file

lldb/test/Shell/Target/target-label.test

@@ -4,16 +4,16 @@
 
 target create -l "ls" /bin/ls
 target list
-# CHECK: * target #0 (ls): /bin/ls
+# CHECK: * target #0 (ls): [[LS_PATH:.*]]
 
 script lldb.target.SetLabel("")
 target list
-# CHECK: * target #0: /bin/ls
+# CHECK: * target #0: [[LS_PATH]]
 
 target create -l "cat" /bin/cat
 target list
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH:.*]]
 
 target create -l "cat" /bin/cat
 # CHECK: Cannot use label 'cat' since it's set in target #1.
@@ -22,12 +22,12 @@ target create -l 42 /bin/cat
 # CHECK: error: Cannot use integer as target label.
 
 target select 0
-# CHECK: * target #0: /bin/ls
-# CHECK-NEXT: target #1 (cat): /bin/cat
+# CHECK: * target #0: [[LS_PATH]]
+# CHECK-NEXT: target #1 (cat): [[CAT_PATH]]
 
 target select cat
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]
 
 script lldb.target.GetLabel()
 # CHECK: 'cat'
@@ -36,5 +36,5 @@ script lldb.debugger.GetTargetAtIndex(0).SetLabel('Not cat')
 # CHECK: success
 
 target list
-# CHECK: target #0 (Not cat): /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0 (Not cat): [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]

lldb/test/Shell/helper/toolchain.py

@@ -1,14 +1,17 @@
 import os
 import itertools
 import platform
+import re
 import subprocess
 import sys
 
 import lit.util
+from lit.formats import ShTest
 from lit.llvm import llvm_config
 from lit.llvm.subst import FindTool
 from lit.llvm.subst import ToolSubst
 
+import posixpath
 
 def _get_lldb_init_path(config):
     return os.path.join(config.test_exec_root, "lit-lldb-init-quiet")
@@ -22,6 +25,68 @@ def _disallow(config, execName):
     config.substitutions.append((" {0} ".format(execName), warning.format(execName)))
 
 
+def get_lldb_args(config, suffix=""):
+    lldb_args = []
+    if "remote-linux" in config.available_features:
+        lldb_args += [
+            "-O",
+            '"platform select remote-linux"',
+            "-O",
+            f'"platform connect {config.lldb_platform_url}"',
+        ]
+        if config.lldb_platform_working_dir:
+            dir = posixpath.join(f"{config.lldb_platform_working_dir}", "shell")
+            if suffix:
+                dir += posixpath.join(dir, f"{suffix}")
+            lldb_args += [
+                "-O",
+                f'"platform shell mkdir -p {dir}"',
+                "-O",
+                f'"platform settings -w {dir}"',
+            ]
+    lldb_args += ["--no-lldbinit", "-S", _get_lldb_init_path(config)]
+    return lldb_args
+
+
+class ShTestLldb(ShTest):
+    def __init__(
+        self, execute_external=False, extra_substitutions=[], preamble_commands=[]
+    ):
+        super().__init__(execute_external, extra_substitutions, preamble_commands)
+
+    def execute(self, test, litConfig):
+        # Run each Shell test in a separate directory (on remote).
+
+        # Find directory change command in %lldb substitution.
+        for i, t in enumerate(test.config.substitutions):
+            if re.match(t[0], "%lldb"):
+                cmd = t[1]
+                if '-O "platform settings -w ' in cmd:
+                    # If command is present, it is added by get_lldb_args.
+                    # Replace the path with the tests' path in suite.
+                    # Example:
+                    # bin/lldb
+                    #   -O "platform shell mkdir -p /home/user/shell"
+                    #   -O "platform settings -w /home/user/shell" ...
+                    # =>
+                    # bin/lldb
+                    #   -O "platform shell mkdir -p /home/user/shell/SymbolFile/Breakpad/inline-record.test"
+                    #   -O "platform settings -w /home/user/shell/SymbolFile/Breakpad/inline-record.test" ...
+                    args_def = " ".join(get_lldb_args(test.config))
+                    args_unique = " ".join(
+                        get_lldb_args(
+                            test.config,
+                            posixpath.join(*test.path_in_suite),
+                        )
+                    )
+                    test.config.substitutions[i] = (
+                        t[0],
+                        cmd.replace(args_def, args_unique),
+                    )
+                break
+        return super().execute(test, litConfig)
+
+
 def use_lldb_substitutions(config):
     # Set up substitutions for primary tools.  These tools must come from config.lldb_tools_dir
     # which is basically the build output directory.  We do not want to find these in path or
@@ -34,7 +99,9 @@ def use_lldb_substitutions(config):
     build_script = os.path.join(build_script, "build.py")
     build_script_args = [
         build_script,
-        "--compiler=any",  # Default to best compiler
+        (
+            "--compiler=clang" if config.enable_remote else "--compiler=any"
+        ),  # Default to best compiler
         "--arch=" + str(config.lldb_bitness),
     ]
     if config.lldb_lit_tools_dir:
@@ -56,7 +123,7 @@ def use_lldb_substitutions(config):
         ToolSubst(
             "%lldb",
             command=FindTool("lldb"),
-            extra_args=["--no-lldbinit", "-S", lldb_init],
+            extra_args=get_lldb_args(config),
             unresolved="fatal",
         ),
         ToolSubst(
@@ -138,7 +205,10 @@ def use_support_substitutions(config):
     # Set up substitutions for support tools.  These tools can be overridden at the CMake
     # level (by specifying -DLLDB_LIT_TOOLS_DIR), installed, or as a last resort, we can use
     # the just-built version.
-    host_flags = ["--target=" + config.host_triple]
+    if config.enable_remote:
+        host_flags = ["--target=" + config.target_triple]
+    else:
+        host_flags = ["--target=" + config.host_triple]
     if platform.system() in ["Darwin"]:
         try:
             out = subprocess.check_output(["xcrun", "--show-sdk-path"]).strip()
@@ -165,6 +235,12 @@ def use_support_substitutions(config):
     if config.cmake_sysroot:
         host_flags += ["--sysroot={}".format(config.cmake_sysroot)]
 
+    if config.enable_remote and config.has_libcxx:
+        host_flags += [
+            "-L{}".format(config.libcxx_libs_dir),
+            "-lc++",
+        ]
+
     host_flags = " ".join(host_flags)
     config.substitutions.append(("%clang_host", "%clang " + host_flags))
     config.substitutions.append(("%clangxx_host", "%clangxx " + host_flags))

lldb/test/Shell/lit.cfg.py

@@ -21,7 +21,7 @@ from helper import toolchain
 config.name = "lldb-shell"
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.test_format = toolchain.ShTestLldb(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files. This is overriden
 # by individual lit.local.cfg files in the test subdirectories.
@@ -68,6 +68,13 @@ if lldb_repro_mode:
     lit_config.note("Running Shell tests in {} mode.".format(lldb_repro_mode))
     toolchain.use_lldb_repro_substitutions(config, lldb_repro_mode)
 
+if config.lldb_platform_url and config.cmake_sysroot and config.enable_remote:
+    if re.match(r".*-linux.*", config.target_triple):
+        config.available_features.add("remote-linux")
+else:
+    # After this, enable_remote == True iff remote testing is going to be used.
+    config.enable_remote = False
+
 llvm_config.use_default_substitutions()
 toolchain.use_lldb_substitutions(config)
 toolchain.use_support_substitutions(config)

lldb/test/Shell/lit.site.cfg.py.in

@@ -10,10 +10,15 @@ config.lldb_src_root = "@LLDB_SOURCE_DIR@"
 config.lldb_obj_root = "@LLDB_BINARY_DIR@"
 config.lldb_libs_dir = lit_config.substitute("@LLDB_LIBS_DIR@")
 config.lldb_tools_dir = lit_config.substitute("@LLDB_TOOLS_DIR@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
 # Since it comes from the command line, it may have backslashes which
 # should not need to be escaped.
 config.lldb_lit_tools_dir = lit_config.substitute(r"@LLDB_LIT_TOOLS_DIR@")
-config.cmake_sysroot = lit_config.substitute("@CMAKE_SYSROOT@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
+config.has_libcxx = @LLDB_HAS_LIBCXX@
+config.enable_remote = not @LLDB_TEST_SHELL_DISABLE_REMOTE@
+config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@