diff options
Diffstat (limited to 'packages/Python/lldbsuite/test_event/event_builder.py')
| -rw-r--r-- | packages/Python/lldbsuite/test_event/event_builder.py | 475 |
1 files changed, 475 insertions, 0 deletions
diff --git a/packages/Python/lldbsuite/test_event/event_builder.py b/packages/Python/lldbsuite/test_event/event_builder.py new file mode 100644 index 000000000000..aabbd986bd70 --- /dev/null +++ b/packages/Python/lldbsuite/test_event/event_builder.py @@ -0,0 +1,475 @@ +""" + The LLVM Compiler Infrastructure + +This file is distributed under the University of Illinois Open Source +License. See LICENSE.TXT for details. + +Provides a class to build Python test event data structures. +""" + +from __future__ import print_function +from __future__ import absolute_import + +# System modules +import inspect +import time +import traceback + +# Third-party modules + +# LLDB modules +from . import build_exception + +class EventBuilder(object): + """Helper class to build test result event dictionaries.""" + + BASE_DICTIONARY = None + + # Test Event Types + TYPE_JOB_RESULT = "job_result" + TYPE_TEST_RESULT = "test_result" + TYPE_TEST_START = "test_start" + TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun" + TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure" + TYPE_SESSION_TERMINATE = "terminate" + + RESULT_TYPES = {TYPE_JOB_RESULT, TYPE_TEST_RESULT} + + # Test/Job Status Tags + STATUS_EXCEPTIONAL_EXIT = "exceptional_exit" + STATUS_SUCCESS = "success" + STATUS_FAILURE = "failure" + STATUS_EXPECTED_FAILURE = "expected_failure" + STATUS_EXPECTED_TIMEOUT = "expected_timeout" + STATUS_UNEXPECTED_SUCCESS = "unexpected_success" + STATUS_SKIP = "skip" + STATUS_ERROR = "error" + STATUS_TIMEOUT = "timeout" + + """Test methods or jobs with a status matching any of these + status values will cause a testrun failure, unless + the test methods rerun and do not trigger an issue when rerun.""" + TESTRUN_ERROR_STATUS_VALUES = { + STATUS_ERROR, + STATUS_EXCEPTIONAL_EXIT, + STATUS_FAILURE, + STATUS_TIMEOUT} + + @staticmethod + def _get_test_name_info(test): + """Returns (test-class-name, test-method-name) from a test case instance. + + @param test a unittest.TestCase instance. + + @return tuple containing (test class name, test method name) + """ + test_class_components = test.id().split(".") + test_class_name = ".".join(test_class_components[:-1]) + test_name = test_class_components[-1] + return test_class_name, test_name + + @staticmethod + def bare_event(event_type): + """Creates an event with default additions, event type and timestamp. + + @param event_type the value set for the "event" key, used + to distinguish events. + + @returns an event dictionary with all default additions, the "event" + key set to the passed in event_type, and the event_time value set to + time.time(). + """ + if EventBuilder.BASE_DICTIONARY is not None: + # Start with a copy of the "always include" entries. + event = dict(EventBuilder.BASE_DICTIONARY) + else: + event = {} + + event.update({ + "event": event_type, + "event_time": time.time() + }) + return event + + @staticmethod + def _assert_is_python_sourcefile(test_filename): + if test_filename is not None: + if not test_filename.endswith(".py"): + raise Exception("source python filename has unexpected extension: {}".format(test_filename)) + return test_filename + + @staticmethod + def _event_dictionary_common(test, event_type): + """Returns an event dictionary setup with values for the given event type. + + @param test the unittest.TestCase instance + + @param event_type the name of the event type (string). + + @return event dictionary with common event fields set. + """ + test_class_name, test_name = EventBuilder._get_test_name_info(test) + + # Determine the filename for the test case. If there is an attribute + # for it, use it. Otherwise, determine from the TestCase class path. + if hasattr(test, "test_filename"): + test_filename = EventBuilder._assert_is_python_sourcefile(test.test_filename) + else: + test_filename = EventBuilder._assert_is_python_sourcefile(inspect.getsourcefile(test.__class__)) + + event = EventBuilder.bare_event(event_type) + event.update({ + "test_class": test_class_name, + "test_name": test_name, + "test_filename": test_filename + }) + + return event + + @staticmethod + def _error_tuple_class(error_tuple): + """Returns the unittest error tuple's error class as a string. + + @param error_tuple the error tuple provided by the test framework. + + @return the error type (typically an exception) raised by the + test framework. + """ + type_var = error_tuple[0] + module = inspect.getmodule(type_var) + if module: + return "{}.{}".format(module.__name__, type_var.__name__) + else: + return type_var.__name__ + + @staticmethod + def _error_tuple_message(error_tuple): + """Returns the unittest error tuple's error message. + + @param error_tuple the error tuple provided by the test framework. + + @return the error message provided by the test framework. + """ + return str(error_tuple[1]) + + @staticmethod + def _error_tuple_traceback(error_tuple): + """Returns the unittest error tuple's error message. + + @param error_tuple the error tuple provided by the test framework. + + @return the error message provided by the test framework. + """ + return error_tuple[2] + + @staticmethod + def _event_dictionary_test_result(test, status): + """Returns an event dictionary with common test result fields set. + + @param test a unittest.TestCase instance. + + @param status the status/result of the test + (e.g. "success", "failure", etc.) + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_common( + test, EventBuilder.TYPE_TEST_RESULT) + event["status"] = status + return event + + @staticmethod + def _event_dictionary_issue(test, status, error_tuple): + """Returns an event dictionary with common issue-containing test result + fields set. + + @param test a unittest.TestCase instance. + + @param status the status/result of the test + (e.g. "success", "failure", etc.) + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_test_result(test, status) + event["issue_class"] = EventBuilder._error_tuple_class(error_tuple) + event["issue_message"] = EventBuilder._error_tuple_message(error_tuple) + backtrace = EventBuilder._error_tuple_traceback(error_tuple) + if backtrace is not None: + event["issue_backtrace"] = traceback.format_tb(backtrace) + return event + + @staticmethod + def event_for_start(test): + """Returns an event dictionary for the test start event. + + @param test a unittest.TestCase instance. + + @return the event dictionary + """ + return EventBuilder._event_dictionary_common( + test, EventBuilder.TYPE_TEST_START) + + @staticmethod + def event_for_success(test): + """Returns an event dictionary for a successful test. + + @param test a unittest.TestCase instance. + + @return the event dictionary + """ + return EventBuilder._event_dictionary_test_result( + test, EventBuilder.STATUS_SUCCESS) + + @staticmethod + def event_for_unexpected_success(test, bugnumber): + """Returns an event dictionary for a test that succeeded but was + expected to fail. + + @param test a unittest.TestCase instance. + + @param bugnumber the issue identifier for the bug tracking the + fix request for the test expected to fail (but is in fact + passing here). + + @return the event dictionary + + """ + event = EventBuilder._event_dictionary_test_result( + test, EventBuilder.STATUS_UNEXPECTED_SUCCESS) + if bugnumber: + event["bugnumber"] = str(bugnumber) + return event + + @staticmethod + def event_for_failure(test, error_tuple): + """Returns an event dictionary for a test that failed. + + @param test a unittest.TestCase instance. + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @return the event dictionary + """ + return EventBuilder._event_dictionary_issue( + test, EventBuilder.STATUS_FAILURE, error_tuple) + + @staticmethod + def event_for_expected_failure(test, error_tuple, bugnumber): + """Returns an event dictionary for a test that failed as expected. + + @param test a unittest.TestCase instance. + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @param bugnumber the issue identifier for the bug tracking the + fix request for the test expected to fail. + + @return the event dictionary + + """ + event = EventBuilder._event_dictionary_issue( + test, EventBuilder.STATUS_EXPECTED_FAILURE, error_tuple) + if bugnumber: + event["bugnumber"] = str(bugnumber) + return event + + @staticmethod + def event_for_skip(test, reason): + """Returns an event dictionary for a test that was skipped. + + @param test a unittest.TestCase instance. + + @param reason the reason why the test is being skipped. + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_test_result( + test, EventBuilder.STATUS_SKIP) + event["skip_reason"] = reason + return event + + @staticmethod + def event_for_error(test, error_tuple): + """Returns an event dictionary for a test that hit a test execution error. + + @param test a unittest.TestCase instance. + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_issue( + test, EventBuilder.STATUS_ERROR, error_tuple) + event["issue_phase"] = "test" + return event + + @staticmethod + def event_for_build_error(test, error_tuple): + """Returns an event dictionary for a test that hit a test execution error + during the test cleanup phase. + + @param test a unittest.TestCase instance. + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_issue( + test, EventBuilder.STATUS_ERROR, error_tuple) + event["issue_phase"] = "build" + + build_error = error_tuple[1] + event["build_command"] = build_error.command + event["build_error"] = build_error.build_error + return event + + @staticmethod + def event_for_cleanup_error(test, error_tuple): + """Returns an event dictionary for a test that hit a test execution error + during the test cleanup phase. + + @param test a unittest.TestCase instance. + + @param error_tuple the error tuple as reported by the test runner. + This is of the form (type<error>, error). + + @return the event dictionary + """ + event = EventBuilder._event_dictionary_issue( + test, EventBuilder.STATUS_ERROR, error_tuple) + event["issue_phase"] = "cleanup" + return event + + @staticmethod + def event_for_job_test_add_error(test_filename, exception, backtrace): + event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) + event["status"] = EventBuilder.STATUS_ERROR + if test_filename is not None: + event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename) + if exception is not None and "__class__" in dir(exception): + event["issue_class"] = exception.__class__ + event["issue_message"] = exception + if backtrace is not None: + event["issue_backtrace"] = backtrace + return event + + @staticmethod + def event_for_job_exceptional_exit( + pid, worker_index, exception_code, exception_description, + test_filename, command_line): + """Creates an event for a job (i.e. process) exit due to signal. + + @param pid the process id for the job that failed + @param worker_index optional id for the job queue running the process + @param exception_code optional code + (e.g. SIGTERM integer signal number) + @param exception_description optional string containing symbolic + representation of the issue (e.g. "SIGTERM") + @param test_filename the path to the test filename that exited + in some exceptional way. + @param command_line the Popen()-style list provided as the command line + for the process that timed out. + + @return an event dictionary coding the job completion description. + """ + event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) + event["status"] = EventBuilder.STATUS_EXCEPTIONAL_EXIT + if pid is not None: + event["pid"] = pid + if worker_index is not None: + event["worker_index"] = int(worker_index) + if exception_code is not None: + event["exception_code"] = exception_code + if exception_description is not None: + event["exception_description"] = exception_description + if test_filename is not None: + event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename) + if command_line is not None: + event["command_line"] = command_line + return event + + @staticmethod + def event_for_job_timeout(pid, worker_index, test_filename, command_line): + """Creates an event for a job (i.e. process) timeout. + + @param pid the process id for the job that timed out + @param worker_index optional id for the job queue running the process + @param test_filename the path to the test filename that timed out. + @param command_line the Popen-style list provided as the command line + for the process that timed out. + + @return an event dictionary coding the job completion description. + """ + event = EventBuilder.bare_event(EventBuilder.TYPE_JOB_RESULT) + event["status"] = "timeout" + if pid is not None: + event["pid"] = pid + if worker_index is not None: + event["worker_index"] = int(worker_index) + if test_filename is not None: + event["test_filename"] = EventBuilder._assert_is_python_sourcefile(test_filename) + if command_line is not None: + event["command_line"] = command_line + return event + + @staticmethod + def event_for_mark_test_rerun_eligible(test): + """Creates an event that indicates the specified test is explicitly + eligible for rerun. + + Note there is a mode that will enable test rerun eligibility at the + global level. These markings for explicit rerun eligibility are + intended for the mode of running where only explicitly re-runnable + tests are rerun upon hitting an issue. + + @param test the TestCase instance to which this pertains. + + @return an event that specifies the given test as being eligible to + be rerun. + """ + event = EventBuilder._event_dictionary_common( + test, + EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE) + return event + + @staticmethod + def event_for_mark_test_expected_failure(test): + """Creates an event that indicates the specified test is expected + to fail. + + @param test the TestCase instance to which this pertains. + + @return an event that specifies the given test is expected to fail. + """ + event = EventBuilder._event_dictionary_common( + test, + EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE) + return event + + @staticmethod + def add_entries_to_all_events(entries_dict): + """Specifies a dictionary of entries to add to all test events. + + This provides a mechanism for, say, a parallel test runner to + indicate to each inferior dotest.py that it should add a + worker index to each. + + Calling this method replaces all previous entries added + by a prior call to this. + + Event build methods will overwrite any entries that collide. + Thus, the passed in dictionary is the base, which gets merged + over by event building when keys collide. + + @param entries_dict a dictionary containing key and value + pairs that should be merged into all events created by the + event generator. May be None to clear out any extra entries. + """ + EventBuilder.BASE_DICTIONARY = dict(entries_dict) |