""" The LLVM Compiler Infrastructure This file is distributed under the University of Illinois Open Source License. See LICENSE.TXT for details. Provides a class to build Python test event data structures. """ from __future__ import print_function from __future__ import absolute_import # System modules import inspect import time import traceback # Third-party modules # LLDB modules from . import build_exception class EventBuilder(object): """Helper class to build test result event dictionaries.""" BASE_DICTIONARY = None # Test Event Types TYPE_JOB_RESULT = "job_result" TYPE_TEST_RESULT = "test_result" TYPE_TEST_START = "test_start" TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun" TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure" TYPE_SESSION_TERMINATE = "terminate" RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT} # Test/Job Status Tags STATUS_EXCEPTIONAL_EXIT = "exceptional_exit" STATUS_SUCCESS = "success" STATUS_FAILURE = "failure" STATUS_EXPECTED_FAILURE = "expected_failure" STATUS_EXPECTED_TIMEOUT = "expected_timeout" STATUS_UNEXPECTED_SUCCESS = "unexpected_success" STATUS_SKIP = "skip" STATUS_ERROR = "error" STATUS_TIMEOUT = "timeout" """Test methods or jobs with a status matching any of these status values will cause a testrun failure, unless the test methods rerun and do not trigger an issue when rerun.""" TESTRUN_ERROR_STATUS_VALUES = { STATUS_ERROR, STATUS_EXCEPTIONAL_EXIT, STATUS_FAILURE, STATUS_TIMEOUT} @staticmethod def _get_test_name_info(test): """Returns (test-class-name, test-method-name) from a test case instance. @param test a unittest.TestCase instance. @return tuple containing (test class name, test method name) """ test_class_components = test.id().split(".") test_class_name = ".".join(test_class_components[:-1]) test_name = test_class_components[-1] return test_class_name, test_name @staticmethod def bare_event(event_type): """Creates an event with default additions, event type and timestamp. @param event_type the value set for the "event" key, used to distinguish events. @returns an event dictionary with all default additions, the "event" key set to the passed in event_type, and the event_time value set to time.time(). """ if EventBuilder.BASE_DICTIONARY is not None: # Start with a copy of the "always include" entries. event = dict(EventBuilder.BASE_DICTIONARY) else: event = {} event.update({ "event": event_type, "event_time": time.time() }) return event @staticmethod def _assert_is_python_sourcefile(test_filename): if test_filename is not None: if not test_filename.endswith(".py"): raise Exception( "source python filename has unexpected extension: {}".format(test_filename)) return test_filename @staticmethod def _event_dictionary_common(test, event_type): """Returns an event dictionary setup with values for the given event type. @param test the unittest.TestCase instance @param event_type the name of the event type (string). @return event dictionary with common event fields set. """ test_class_name, test_name = EventBuilder._get_test_name_info(test) # Determine the filename for the test case. If there is an attribute # for it, use it. Otherwise, determine from the TestCase class path. if hasattr(test, "test_filename"): test_filename = EventBuilder._assert_is_python_sourcefile( test.test_filename) else: test_filename = EventBuilder._assert_is_python_sourcefile( inspect.getsourcefile(test.__class__)) event = EventBuilder.bare_event(event_type) event.update({ "test_class": test_class_name, "test_name": test_name, "test_filename": test_filename }) return event @staticmethod def _error_tuple_class(error_tuple): """Returns the unittest error tuple's error class as a string. @param error_tuple the error tuple provided by the test framework. @return the error type (typically an exception) raised by the test framework. """ type_var = error_tuple[0] module = inspect.getmodule(type_var) if module: return "{}.{}".format(module.__name__, type_var.__name__) else: return type_var.__name__ @staticmethod def _error_tuple_message(error_tuple): """Returns the unittest error tuple's error message. @param error_tuple the error tuple provided by the test framework. @return the error message provided by the test framework. """ return str(error_tuple[1]) @staticmethod def _error_tuple_traceback(error_tuple): """Returns the unittest error tuple's error message. @param error_tuple the error tuple provided by the test framework. @return the error message provided by the test framework. """ return error_tuple[2] @staticmethod def _event_dictionary_test_result(test, status): """Returns an event dictionary with common test result fields set. @param test a unittest.TestCase instance. @param status the status/result of the test (e.g. "success", "failure", etc.) @return the event dictionary """ event = EventBuilder._event_dictionary_common( test, EventBuilder.TYPE_TEST_RESULT) event["status"] = status return event @staticmethod def _event_dictionary_issue(test, status, error_tuple): """Returns an event dictionary with common issue-containing test result fields set. @param test a unittest.TestCase instance. @param status the status/result of the test (e.g. "success", "failure", etc.) @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @return the event dictionary """ event = EventBuilder._event_dictionary_test_result(test, status) event["issue_class"] = EventBuilder._error_tuple_class(error_tuple) event["issue_message"] = EventBuilder._error_tuple_message(error_tuple) backtrace = EventBuilder._error_tuple_traceback(error_tuple) if backtrace is not None: event["issue_backtrace"] = traceback.format_tb(backtrace) return event @staticmethod def event_for_start(test): """Returns an event dictionary for the test start event. @param test a unittest.TestCase instance. @return the event dictionary """ return EventBuilder._event_dictionary_common( test, EventBuilder.TYPE_TEST_START) @staticmethod def event_for_success(test): """Returns an event dictionary for a successful test. @param test a unittest.TestCase instance. @return the event dictionary """ return EventBuilder._event_dictionary_test_result( test, EventBuilder.STATUS_SUCCESS) @staticmethod def event_for_unexpected_success(test, bugnumber): """Returns an event dictionary for a test that succeeded but was expected to fail. @param test a unittest.TestCase instance. @param bugnumber the issue identifier for the bug tracking the fix request for the test expected to fail (but is in fact passing here). @return the event dictionary """ event = EventBuilder._event_dictionary_test_result( test, EventBuilder.STATUS_UNEXPECTED_SUCCESS) if bugnumber: event["bugnumber"] = str(bugnumber) return event @staticmethod def event_for_failure(test, error_tuple): """Returns an event dictionary for a test that failed. @param test a unittest.TestCase instance. @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @return the event dictionary """ return EventBuilder._event_dictionary_issue( test, EventBuilder.STATUS_FAILURE, error_tuple) @staticmethod def event_for_expected_failure(test, error_tuple, bugnumber): """Returns an event dictionary for a test that failed as expected. @param test a unittest.TestCase instance. @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @param bugnumber the issue identifier for the bug tracking the fix request for the test expected to fail. @return the event dictionary """ event = EventBuilder._event_dictionary_issue( test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple) if bugnumber: event["bugnumber"] = str(bugnumber) return event @staticmethod def event_for_skip(test, reason): """Returns an event dictionary for a test that was skipped. @param test a unittest.TestCase instance. @param reason the reason why the test is being skipped. @return the event dictionary """ event = EventBuilder._event_dictionary_test_result( test, EventBuilder.STATUS_SKIP) event["skip_reason"] = reason return event @staticmethod def event_for_error(test, error_tuple): """Returns an event dictionary for a test that hit a test execution error. @param test a unittest.TestCase instance. @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @return the event dictionary """ event = EventBuilder._event_dictionary_issue( test, EventBuilder.STATUS_ERROR, error_tuple) event["issue_phase"] = "test" return event @staticmethod def event_for_build_error(test, error_tuple): """Returns an event dictionary for a test that hit a test execution error during the test cleanup phase. @param test a unittest.TestCase instance. @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @return the event dictionary """ event = EventBuilder._event_dictionary_issue( test, EventBuilder.STATUS_ERROR, error_tuple) event["issue_phase"] = "build" build_error = error_tuple[1] event["build_command"] = build_error.command event["build_error"] = build_error.build_error return event @staticmethod def event_for_cleanup_error(test, error_tuple): """Returns an event dictionary for a test that hit a test execution error during the test cleanup phase. @param test a unittest.TestCase instance. @param error_tuple the error tuple as reported by the test runner. This is of the form (type, error). @return the event dictionary """ event = EventBuilder._event_dictionary_issue( test, EventBuilder.STATUS_ERROR, error_tuple) event["issue_phase"] = "cleanup" return event @staticmethod def event_for_job_test_add_error(test_filename, exception, backtrace): event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) event["status"] = EventBuilder.STATUS_ERROR if test_filename is not None: event["test_filename"] = EventBuilder._assert_is_python_sourcefile( test_filename) if exception is not None and "__class__" in dir(exception): event["issue_class"] = exception.__class__ event["issue_message"] = exception if backtrace is not None: event["issue_backtrace"] = backtrace return event @staticmethod def event_for_job_exceptional_exit( pid, worker_index, exception_code, exception_description, test_filename, command_line): """Creates an event for a job (i.e. process) exit due to signal. @param pid the process id for the job that failed @param worker_index optional id for the job queue running the process @param exception_code optional code (e.g. SIGTERM integer signal number) @param exception_description optional string containing symbolic representation of the issue (e.g. "SIGTERM") @param test_filename the path to the test filename that exited in some exceptional way. @param command_line the Popen()-style list provided as the command line for the process that timed out. @return an event dictionary coding the job completion description. """ event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT if pid is not None: event["pid"] = pid if worker_index is not None: event["worker_index"] = int(worker_index) if exception_code is not None: event["exception_code"] = exception_code if exception_description is not None: event["exception_description"] = exception_description if test_filename is not None: event["test_filename"] = EventBuilder._assert_is_python_sourcefile( test_filename) if command_line is not None: event["command_line"] = command_line return event @staticmethod def event_for_job_timeout(pid, worker_index, test_filename, command_line): """Creates an event for a job (i.e. process) timeout. @param pid the process id for the job that timed out @param worker_index optional id for the job queue running the process @param test_filename the path to the test filename that timed out. @param command_line the Popen-style list provided as the command line for the process that timed out. @return an event dictionary coding the job completion description. """ event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) event["status"] = "timeout" if pid is not None: event["pid"] = pid if worker_index is not None: event["worker_index"] = int(worker_index) if test_filename is not None: event["test_filename"] = EventBuilder._assert_is_python_sourcefile( test_filename) if command_line is not None: event["command_line"] = command_line return event @staticmethod def event_for_mark_test_rerun_eligible(test): """Creates an event that indicates the specified test is explicitly eligible for rerun. Note there is a mode that will enable test rerun eligibility at the global level. These markings for explicit rerun eligibility are intended for the mode of running where only explicitly re-runnable tests are rerun upon hitting an issue. @param test the TestCase instance to which this pertains. @return an event that specifies the given test as being eligible to be rerun. """ event = EventBuilder._event_dictionary_common( test, EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE) return event @staticmethod def event_for_mark_test_expected_failure(test): """Creates an event that indicates the specified test is expected to fail. @param test the TestCase instance to which this pertains. @return an event that specifies the given test is expected to fail. """ event = EventBuilder._event_dictionary_common( test, EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE) return event @staticmethod def add_entries_to_all_events(entries_dict): """Specifies a dictionary of entries to add to all test events. This provides a mechanism for, say, a parallel test runner to indicate to each inferior dotest.py that it should add a worker index to each. Calling this method replaces all previous entries added by a prior call to this. Event build methods will overwrite any entries that collide. Thus, the passed in dictionary is the base, which gets merged over by event building when keys collide. @param entries_dict a dictionary containing key and value pairs that should be merged into all events created by the event generator. May be None to clear out any extra entries. """ EventBuilder.BASE_DICTIONARY = dict(entries_dict)