diff options
Diffstat (limited to 'contrib/kyua/engine')
31 files changed, 2583 insertions, 239 deletions
diff --git a/contrib/kyua/engine/Kyuafile b/contrib/kyua/engine/Kyuafile index 1baa63bc9118..5337dc19817a 100644 --- a/contrib/kyua/engine/Kyuafile +++ b/contrib/kyua/engine/Kyuafile @@ -8,6 +8,9 @@ atf_test_program{name="atf_result_test"} atf_test_program{name="config_test"} atf_test_program{name="exceptions_test"} atf_test_program{name="filters_test"} +atf_test_program{name="googletest_test"} +atf_test_program{name="googletest_list_test"} +atf_test_program{name="googletest_result_test"} atf_test_program{name="kyuafile_test"} atf_test_program{name="plain_test"} atf_test_program{name="requirements_test"} diff --git a/contrib/kyua/engine/Makefile.am.inc b/contrib/kyua/engine/Makefile.am.inc deleted file mode 100644 index baa7fe0bb8a0..000000000000 --- a/contrib/kyua/engine/Makefile.am.inc +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright 2010 The Kyua Authors. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of Google Inc. nor the names of its contributors -# may be used to endorse or promote products derived from this software -# without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -ENGINE_CFLAGS = $(STORE_CFLAGS) $(MODEL_CFLAGS) $(UTILS_CFLAGS) -ENGINE_LIBS = libengine.a $(STORE_LIBS) $(MODEL_LIBS) $(UTILS_LIBS) - -noinst_LIBRARIES += libengine.a -libengine_a_CPPFLAGS = $(STORE_CFLAGS) $(UTILS_CFLAGS) -libengine_a_SOURCES = engine/atf.cpp -libengine_a_SOURCES += engine/atf.hpp -libengine_a_SOURCES += engine/atf_list.cpp -libengine_a_SOURCES += engine/atf_list.hpp -libengine_a_SOURCES += engine/atf_result.cpp -libengine_a_SOURCES += engine/atf_result.hpp -libengine_a_SOURCES += engine/atf_result_fwd.hpp -libengine_a_SOURCES += engine/config.cpp -libengine_a_SOURCES += engine/config.hpp -libengine_a_SOURCES += engine/config_fwd.hpp -libengine_a_SOURCES += engine/exceptions.cpp -libengine_a_SOURCES += engine/exceptions.hpp -libengine_a_SOURCES += engine/filters.cpp -libengine_a_SOURCES += engine/filters.hpp -libengine_a_SOURCES += engine/filters_fwd.hpp -libengine_a_SOURCES += engine/kyuafile.cpp -libengine_a_SOURCES += engine/kyuafile.hpp -libengine_a_SOURCES += engine/kyuafile_fwd.hpp -libengine_a_SOURCES += engine/plain.cpp -libengine_a_SOURCES += engine/plain.hpp -libengine_a_SOURCES += engine/requirements.cpp -libengine_a_SOURCES += engine/requirements.hpp -libengine_a_SOURCES += engine/scanner.cpp -libengine_a_SOURCES += engine/scanner.hpp -libengine_a_SOURCES += engine/scanner_fwd.hpp -libengine_a_SOURCES += engine/tap.cpp -libengine_a_SOURCES += engine/tap.hpp -libengine_a_SOURCES += engine/tap_parser.cpp -libengine_a_SOURCES += engine/tap_parser.hpp -libengine_a_SOURCES += engine/tap_parser_fwd.hpp -libengine_a_SOURCES += engine/scheduler.cpp -libengine_a_SOURCES += engine/scheduler.hpp -libengine_a_SOURCES += engine/scheduler_fwd.hpp - -if WITH_ATF -tests_enginedir = $(pkgtestsdir)/engine - -tests_engine_DATA = engine/Kyuafile -EXTRA_DIST += $(tests_engine_DATA) - -tests_engine_PROGRAMS = engine/atf_helpers -engine_atf_helpers_SOURCES = engine/atf_helpers.cpp -engine_atf_helpers_CXXFLAGS = $(UTILS_CFLAGS) $(ATF_CXX_CFLAGS) -engine_atf_helpers_LDADD = $(UTILS_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/atf_test -engine_atf_test_SOURCES = engine/atf_test.cpp -engine_atf_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_atf_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/atf_list_test -engine_atf_list_test_SOURCES = engine/atf_list_test.cpp -engine_atf_list_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_atf_list_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/atf_result_test -engine_atf_result_test_SOURCES = engine/atf_result_test.cpp -engine_atf_result_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_atf_result_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/config_test -engine_config_test_SOURCES = engine/config_test.cpp -engine_config_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_config_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/exceptions_test -engine_exceptions_test_SOURCES = engine/exceptions_test.cpp -engine_exceptions_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_exceptions_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/filters_test -engine_filters_test_SOURCES = engine/filters_test.cpp -engine_filters_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_filters_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/kyuafile_test -engine_kyuafile_test_SOURCES = engine/kyuafile_test.cpp -engine_kyuafile_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_kyuafile_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/plain_helpers -engine_plain_helpers_SOURCES = engine/plain_helpers.cpp -engine_plain_helpers_CXXFLAGS = $(UTILS_CFLAGS) -engine_plain_helpers_LDADD = $(UTILS_LIBS) - -tests_engine_PROGRAMS += engine/plain_test -engine_plain_test_SOURCES = engine/plain_test.cpp -engine_plain_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_plain_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/requirements_test -engine_requirements_test_SOURCES = engine/requirements_test.cpp -engine_requirements_test_CXXFLAGS = $(ENGINE_CFLAGS) $(UTILS_TEST_CFLAGS) \ - $(ATF_CXX_CFLAGS) -engine_requirements_test_LDADD = $(ENGINE_LIBS) $(UTILS_TEST_LIBS) \ - $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/scanner_test -engine_scanner_test_SOURCES = engine/scanner_test.cpp -engine_scanner_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_scanner_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/tap_helpers -engine_tap_helpers_SOURCES = engine/tap_helpers.cpp -engine_tap_helpers_CXXFLAGS = $(UTILS_CFLAGS) -engine_tap_helpers_LDADD = $(UTILS_LIBS) - -tests_engine_PROGRAMS += engine/tap_test -engine_tap_test_SOURCES = engine/tap_test.cpp -engine_tap_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_tap_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/tap_parser_test -engine_tap_parser_test_SOURCES = engine/tap_parser_test.cpp -engine_tap_parser_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_tap_parser_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) - -tests_engine_PROGRAMS += engine/scheduler_test -engine_scheduler_test_SOURCES = engine/scheduler_test.cpp -engine_scheduler_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS) -engine_scheduler_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS) -endif diff --git a/contrib/kyua/engine/atf.hpp b/contrib/kyua/engine/atf.hpp index 34ddc2413235..b9821e5fb7a7 100644 --- a/contrib/kyua/engine/atf.hpp +++ b/contrib/kyua/engine/atf.hpp @@ -40,23 +40,24 @@ namespace engine { /// Implementation of the scheduler interface for atf test programs. class atf_interface : public engine::scheduler::interface { public: - void exec_list(const model::test_program&, - const utils::config::properties_map&) const UTILS_NORETURN; + void exec_list [[noreturn]] ( + const model::test_program&, + const utils::config::properties_map&) const; model::test_cases_map parse_list( const utils::optional< utils::process::status >&, const utils::fs::path&, const utils::fs::path&) const; - void exec_test(const model::test_program&, const std::string&, - const utils::config::properties_map&, - const utils::fs::path&) const - UTILS_NORETURN; + void exec_test [[noreturn]] ( + const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const; - void exec_cleanup(const model::test_program&, const std::string&, - const utils::config::properties_map&, - const utils::fs::path&) const - UTILS_NORETURN; + void exec_cleanup [[noreturn]]( + const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const; model::test_result compute_result( const utils::optional< utils::process::status >&, diff --git a/contrib/kyua/engine/atf_result.cpp b/contrib/kyua/engine/atf_result.cpp index f99b28f9e96e..e8a20dafd558 100644 --- a/contrib/kyua/engine/atf_result.cpp +++ b/contrib/kyua/engine/atf_result.cpp @@ -625,13 +625,21 @@ engine::calculate_atf_result(const optional< process::status >& body_status, try { result = atf_result::load(results_file); } catch (const engine::format_error& error) { - result = atf_result(atf_result::broken, error.what()); + if (body_status) { + result = atf_result(atf_result::broken, + F("Error: %s. Test case %s") % + error.what() % + format_status(body_status.get())); + } else { + // The test case timed out. apply() handles this case later. + } } catch (const std::runtime_error& error) { - if (body_status) + if (body_status) { result = atf_result( - atf_result::broken, F("Premature exit; test case %s") % + atf_result::broken, + F("Error: Premature exit. Test case %s") % format_status(body_status.get())); - else { + } else { // The test case timed out. apply() handles this case later. } } diff --git a/contrib/kyua/engine/atf_result_test.cpp b/contrib/kyua/engine/atf_result_test.cpp index 8ec61dc3c07e..91938e02ebb2 100644 --- a/contrib/kyua/engine/atf_result_test.cpp +++ b/contrib/kyua/engine/atf_result_test.cpp @@ -664,7 +664,7 @@ ATF_TEST_CASE_BODY(calculate_atf_result__missing_file) const status body_status = status::fake_exited(EXIT_SUCCESS); const model::test_result expected( model::test_result_broken, - "Premature exit; test case exited with code 0"); + "Error: Premature exit. Test case exited with code 0"); ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( utils::make_optional(body_status), fs::path("foo"))); } @@ -677,8 +677,9 @@ ATF_TEST_CASE_BODY(calculate_atf_result__bad_file) const status body_status = status::fake_exited(EXIT_SUCCESS); atf::utils::create_file("foo", "invalid\n"); - const model::test_result expected(model::test_result_broken, - "Unknown test result 'invalid'"); + const model::test_result expected( + model::test_result_broken, + "Error: Unknown test result 'invalid'. Test case exited with code 0"); ATF_REQUIRE_EQ(expected, engine::calculate_atf_result( utils::make_optional(body_status), fs::path("foo"))); } diff --git a/contrib/kyua/engine/atf_test.cpp b/contrib/kyua/engine/atf_test.cpp index 9fe7797f4362..5f9b8c59663e 100644 --- a/contrib/kyua/engine/atf_test.cpp +++ b/contrib/kyua/engine/atf_test.cpp @@ -310,7 +310,7 @@ ATF_TEST_CASE_BODY(test__body_only__crashes) const model::test_result exp_result( model::test_result_broken, - F("Premature exit; test case received signal %s (core dumped)") % + F("Error: Empty test result or no new line. Test case received signal %s (core dumped)") % SIGABRT); run_one(this, "crash", exp_result); } diff --git a/contrib/kyua/engine/config.cpp b/contrib/kyua/engine/config.cpp index d53e9936ba91..bf027ce88306 100644 --- a/contrib/kyua/engine/config.cpp +++ b/contrib/kyua/engine/config.cpp @@ -82,7 +82,7 @@ set_defaults(config::tree& tree) for (auto em : execenv::execenvs()) if (em->is_supported()) supported.insert(em->name()); - supported.insert(execenv::default_execenv_name); + supported.insert(DEFAULT_EXECENV_NAME); tree.set< config::strings_set_node >("execenvs", supported); // TODO(jmmv): Automatically derive this from the number of CPUs in the @@ -244,7 +244,7 @@ engine::empty_config(void) // Tests of Kyua itself tend to use an empty config, i.e. default // execution environment is used. Let's allow it. std::set< std::string > supported; - supported.insert(engine::execenv::default_execenv_name); + supported.insert(DEFAULT_EXECENV_NAME); tree.set< config::strings_set_node >("execenvs", supported); return tree; diff --git a/contrib/kyua/engine/execenv/execenv.cpp b/contrib/kyua/engine/execenv/execenv.cpp index b043bcda52cb..663e293a33a7 100644 --- a/contrib/kyua/engine/execenv/execenv.cpp +++ b/contrib/kyua/engine/execenv/execenv.cpp @@ -35,9 +35,6 @@ namespace execenv = engine::execenv; using utils::none; -const char* execenv::default_execenv_name = "host"; - - /// List of registered execution environments, except default host one. /// /// Use register_execenv() to add an entry to this global list. diff --git a/contrib/kyua/engine/execenv/execenv.hpp b/contrib/kyua/engine/execenv/execenv.hpp index e667ff205d85..d167196c7e98 100644 --- a/contrib/kyua/engine/execenv/execenv.hpp +++ b/contrib/kyua/engine/execenv/execenv.hpp @@ -32,6 +32,7 @@ #if !defined(ENGINE_EXECENV_EXECENV_HPP) #define ENGINE_EXECENV_EXECENV_HPP +#include "engine/execenv/execenv_fwd.hpp" #include "model/test_program.hpp" #include "utils/optional.ipp" #include "utils/process/operations_fwd.hpp" @@ -43,9 +44,6 @@ namespace engine { namespace execenv { -extern const char* default_execenv_name; - - /// Abstract interface of an execution environment. class interface { protected: @@ -85,7 +83,7 @@ public: /// scheduler::interface::exec_test() or exec_cleanup(). /// /// \param args The arguments to pass to the binary. - virtual void exec(const args_vector& args) const UTILS_NORETURN = 0; + virtual void exec [[noreturn]] (const args_vector& args) const = 0; }; diff --git a/contrib/kyua/engine/execenv/execenv_fwd.hpp b/contrib/kyua/engine/execenv/execenv_fwd.hpp new file mode 100644 index 000000000000..ddbb2c3bb40f --- /dev/null +++ b/contrib/kyua/engine/execenv/execenv_fwd.hpp @@ -0,0 +1,44 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/execenv/execenv_fwd.hpp +/// Forward declarations for engine/execenv/execenv.hpp + +#if !defined(ENGINE_EXECENV_EXECENV_FWD_HPP) +#define ENGINE_EXECENV_EXECENV_FWD_HPP + +#define DEFAULT_EXECENV_NAME "host" + +namespace engine { +namespace execenv { + + +} // namespace execenv +} // namespace engine + +#endif // !defined(ENGINE_EXECENV_EXECENV_FWD_HPP) diff --git a/contrib/kyua/engine/execenv/execenv_host.hpp b/contrib/kyua/engine/execenv/execenv_host.hpp index 2742366cfd6f..3276ad210574 100644 --- a/contrib/kyua/engine/execenv/execenv_host.hpp +++ b/contrib/kyua/engine/execenv/execenv_host.hpp @@ -53,7 +53,7 @@ public: void init() const; void cleanup() const; - void exec(const args_vector& args) const UTILS_NORETURN; + void exec [[noreturn]] (const args_vector& args) const; }; diff --git a/contrib/kyua/engine/googletest.cpp b/contrib/kyua/engine/googletest.cpp new file mode 100644 index 000000000000..e050410ba1f5 --- /dev/null +++ b/contrib/kyua/engine/googletest.cpp @@ -0,0 +1,219 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest.hpp" + +extern "C" { +#include <unistd.h> +} + +#include <cerrno> +#include <cstdlib> +#include <fstream> + +#include "engine/googletest_list.hpp" +#include "engine/googletest_result.hpp" +#include "engine/exceptions.hpp" +#include "model/test_case.hpp" +#include "model/test_program.hpp" +#include "model/test_result.hpp" +#include "utils/defs.hpp" +#include "utils/env.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/logging/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/exceptions.hpp" +#include "utils/process/operations.hpp" +#include "utils/process/status.hpp" +#include "utils/stream.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::optional; + + +namespace { + + +/// Basename of the file containing the result written by the googletest +/// testcase. +/// +/// TODO: Use more structured output format someday, like the googletest's JSON +/// or XML format to avoid dealing with upstream formatting changes, as there's +/// no guarantee that the output format (which is more or less custom/freeform) +/// won't change in the future, making compatibility with all versions +/// potentially difficult to scrape for. +/// +/// Using either format will require pulling in a third party library and +/// understanding the schema of the format. As it stands in 1.9.0, +/// googletest doesn't document this expectation very well and instead seems +/// to rely on third-party solutions for doing structured output via the +/// listener interfaces. + + +/// Magic numbers returned by exec_list when exec(2) fails. +enum list_exit_code { + exit_eacces = 90, + exit_enoent, + exit_enoexec, +}; + + +} // anonymous namespace + + +/// Executes a test program's list operation. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +void +engine::googletest_interface::exec_list( + const model::test_program& test_program, + const config::properties_map& /*vars*/) const +{ + const process::args_vector args{ + "--gtest_color=no", + "--gtest_list_tests", + }; + + try { + process::exec_unsafe(test_program.absolute_path(), args); + } catch (const process::system_error& e) { + if (e.original_errno() == EACCES) + ::_exit(exit_eacces); + else if (e.original_errno() == ENOENT) + ::_exit(exit_enoent); + else if (e.original_errno() == ENOEXEC) + ::_exit(exit_enoexec); + throw; + } +} + + +/// Computes the test cases list of a test program. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the test. +/// +/// \return A list of test cases. +/// +/// \throw error If there is a problem parsing the test case list. +model::test_cases_map +engine::googletest_interface::parse_list( + const optional< process::status >& status, + const fs::path& stdout_path, + const fs::path& /* stderr_path */) const +{ + if (!status) + throw engine::error("Test case list timed out"); + if (status.get().exited()) { + const int exitstatus = status.get().exitstatus(); + if (exitstatus == EXIT_SUCCESS) { + // Nothing to do; fall through. + } else if (exitstatus == exit_eacces) { + throw engine::error("Permission denied to run test program"); + } else if (exitstatus == exit_enoent) { + throw engine::error("Cannot find test program"); + } else if (exitstatus == exit_enoexec) { + throw engine::error("Invalid test program format"); + } else { + throw engine::error("Test program did not exit cleanly; exited " + "with status: " + std::to_string(exitstatus)); + } + } else { + throw engine::error("Test program received signal"); + } + + std::ifstream input(stdout_path.c_str()); + if (!input) + throw engine::load_error(stdout_path, "Cannot open file for read"); + const model::test_cases_map test_cases = parse_googletest_list(input); + + return test_cases; +} + + +/// Executes a test case of the test program. +/// +/// This method is intended to be called within a subprocess and is expected +/// to terminate execution either by exec(2)ing the test program or by +/// exiting with a failure. +/// +/// \param test_program The test program to execute. +/// \param test_case_name Name of the test case to invoke. +/// \param vars User-provided variables to pass to the test program. +void +engine::googletest_interface::exec_test( + const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars, + const fs::path& /* control_directory */) const +{ + for (config::properties_map::const_iterator iter = vars.begin(); + iter != vars.end(); ++iter) { + utils::setenv(F("TEST_ENV_%s") % (*iter).first, (*iter).second); + } + + process::args_vector args{ + "--gtest_color=no", + F("--gtest_filter=%s") % (test_case_name) + }; + process::exec(test_program.absolute_path(), args); + __builtin_unreachable(); +} + + +/// Computes the result of a test case based on its termination status. +/// +/// \param status The termination status of the subprocess used to execute +/// the exec_test() method or none if the test timed out. +/// \param stdout_path Path to the file containing the stdout of the +/// test. +/// +/// \return A test result. +model::test_result +engine::googletest_interface::compute_result( + const optional< process::status >& status, + const fs::path& /* control_directory */, + const fs::path& stdout_path, + const fs::path& /* stderr_path */) const +{ + if (!status) { + return model::test_result(model::test_result_broken, + "Test case body timed out"); + } + + return calculate_googletest_result(status, stdout_path); +} diff --git a/contrib/kyua/engine/googletest.hpp b/contrib/kyua/engine/googletest.hpp new file mode 100644 index 000000000000..9628f994a066 --- /dev/null +++ b/contrib/kyua/engine/googletest.hpp @@ -0,0 +1,68 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest.hpp +/// Execution engine for test programs that implement the googletest interface. + +#if !defined(ENGINE_GOOGLETEST_HPP) +#define ENGINE_GOOGLETEST_HPP + +#include "engine/scheduler.hpp" + +namespace engine { + + +/// Implementation of the scheduler interface for googletest test programs. +class googletest_interface : public engine::scheduler::interface { +public: + void exec_list [[noreturn]] ( + const model::test_program&, + const utils::config::properties_map&) const; + + model::test_cases_map parse_list( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&) const; + + void exec_test [[noreturn]] ( + const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const; + + model::test_result compute_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&, + const utils::fs::path&, + const utils::fs::path&) const; +}; + + +} // namespace engine + + +#endif // !defined(ENGINE_GOOGLETEST_HPP) diff --git a/contrib/kyua/engine/googletest_helpers.cpp b/contrib/kyua/engine/googletest_helpers.cpp new file mode 100644 index 000000000000..129a08a89b7c --- /dev/null +++ b/contrib/kyua/engine/googletest_helpers.cpp @@ -0,0 +1,330 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +extern "C" { +#include <sys/stat.h> + +#include <getopt.h> +#include <signal.h> +#include <unistd.h> + +extern char** environ; +} + +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> + +#include "utils/env.hpp" +#include "utils/cmdline/options.hpp" +#include "utils/cmdline/parser.ipp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/test_utils.ipp" + +namespace cmdline = utils::cmdline; +namespace fs = utils::fs; + +using cmdline::base_option; +using cmdline::bool_option; +using cmdline::parse; +using cmdline::parsed_cmdline; +using cmdline::string_option; + +namespace fs = utils::fs; + +namespace { + + +/// Prefix for all testcases. +const char *test_suite = "Suite."; + + +/// Logs an error message and exits the test with an error code. +/// +/// \param str The error message to log. +static void +fail(const std::string& str) +{ + std::cerr << str << '\n'; + std::exit(EXIT_FAILURE); +} + + +/// A test scenario that validates the TEST_ENV_* variables. +static void +test_check_configuration_variables(void) +{ + std::set< std::string > vars; + char** iter; + for (iter = environ; *iter != NULL; ++iter) { + if (std::strstr(*iter, "TEST_ENV_") == *iter) { + vars.insert(*iter); + } + } + + std::set< std::string > exp_vars{ + "TEST_ENV_first=some value", + "TEST_ENV_second=some other value" + }; + + if (vars == exp_vars) { + std::cout << +"Note: Google Test filter = Suite.check_configuration_variables\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.check_configuration_variables\n" +"[ OK ] Suite.check_configuration_variables (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (1 ms total)\n" +"[ PASSED ] 1 test.\n" + ; + } else { + std::cout << +"Note: Google Test filter = Suite.Fails\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] Suite.check_configuration_variables\n" +"pass_fail_demo.cc:12: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +"[ FAILED ] Suite.check_configuration_variables (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] Suite.check_configuration_variables\n" +"\n" +" 1 FAILED TEST\n" + << F(" Expected: %s\nFound: %s\n") % exp_vars % vars; + std::exit(EXIT_FAILURE); + } +} + + +/// A test scenario that triggers a crash via abort in order to generate a +/// core dump. +static void +test_crash(void) +{ + std::abort(); +} + + +/// A test scenario that reports some tests as failed. +static void +test_fail(void) +{ + std::cout << ( +"Note: Google Test filter = Suite.fail\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.fail\n" +"gtest_macros_demo.cc:4: Failure\n" +"Failed\n" +"with a reason\n" +"[ FAILED ] Suite.fail (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] Suite.fail\n" +"\n" +" 1 FAILED TEST\n" + ); + std::exit(EXIT_FAILURE); +} + + +/// A test scenario that passes. +static void +test_pass(void) +{ + std::cout << ( +"Note: Google Test filter = Suite.pass\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.pass\n" +"[ OK ] Suite.pass (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" + ); +} + + +/// A test scenario that passes but then exits with non-zero. +static void +test_pass_but_exit_failure(void) +{ + std::cout << ( +"Note: Google Test filter = Suite.pass_but_exit_failure\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.pass_but_exit_failure\n" +"[ OK ] Suite.pass_but_exit_failure (0 ms)\n" +"[----------] 1 test from Suite (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" + ); + + std::exit(70); +} + + +/// A test scenario that times out. +/// +/// Note that the timeout is defined in the Kyuafile, as the TAP interface has +/// no means for test programs to specify this by themselves. +static void +test_timeout(void) +{ + std::cout << ( +"Note: Google Test filter = Suite.incomplete\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from Suite\n" +"[ RUN ] Suite.incomplete\n" + ); + + ::sleep(10); + const fs::path control_dir = fs::path(utils::getenv("CONTROL_DIR").get()); + std::ofstream file((control_dir / "cookie").c_str()); + if (!file) + fail("Failed to create the control cookie"); + file.close(); +} + + +/// Prints out program usage and exits with a non-zero exit code. +static void +usage(const char* argv0) { + std::cout << "usage: " << argv0 << " " + << "[--gtest_color=(auto|yes|no)] " + << "[--gtest_filter=POSITIVE_PATTERNS] " + << "[--gtest_list_tests]" + << "\n\n" + << "This program mocks a googletest test program." + << "\n"; + std::exit(EXIT_FAILURE); +} + + +} // anonymous namespace + + +/// Entry point to the test program. +/// +/// The caller can select which test scenario to run by modifying the program's +/// basename on disk (either by a copy or by a hard link). +/// +/// \todo It may be worth to split this binary into separate, smaller binaries, +/// one for every "test scenario". We use this program as a dispatcher for +/// different "main"s, the only reason being to keep the amount of helper test +/// programs to a minimum. However, putting this each function in its own +/// binary could simplify many other things. +/// +/// \param argc The number of CLI arguments. +/// \param argv The CLI arguments themselves. These are not used because +/// Kyua will not pass any arguments to the plain test program. +int +main(int argc, char** argv) +{ + using scenario_fn_t = void (*)(void); + + char *argv0 = argv[0]; + + std::map<std::string, scenario_fn_t> scenarios{ + { "check_configuration_variables", test_check_configuration_variables }, + { "crash", test_crash }, + { "fail", test_fail }, + { "pass", test_pass }, + { "pass_but_exit_failure", test_pass_but_exit_failure }, + { "timeout", test_timeout } + }; + + const bool_option gtest_list_tests_opt( + "gtest_list_tests", "List tests"); + const string_option gtest_color_opt( + "gtest_color", "Enable/disable color support", "auto"); + const string_option gtest_filter_opt( + "gtest_filter", "", "POSITIVE_PATTERNS"); + + std::vector<const base_option*> options; + /// Ignore `--gtest_color=*`. + options.push_back(>est_color_opt); + options.push_back(>est_filter_opt); + options.push_back(>est_list_tests_opt); + + const parsed_cmdline cmdline = parse(argc, argv, options); + INV(cmdline.arguments().empty()); + + if (cmdline.has_option("gtest_list_tests")) { + std::cout << test_suite << "\n"; + for (auto it = scenarios.begin(); it != scenarios.end(); it++) { + std::cout << " " << it->first << "\n"; + } + return EXIT_SUCCESS; + } + + INV(cmdline.has_option("gtest_filter")); + + auto gtest_filter_arg = cmdline.get_option<string_option>("gtest_filter"); + INV(gtest_filter_arg.find_first_of(test_suite) == 0); + auto testcase = gtest_filter_arg.erase(0, strlen(test_suite)); + + auto scenario = scenarios.find(testcase); + if (scenario == scenarios.end()) { + /// Mimic googletest test programs by printing out a usage message when + /// a test cannot be found. + usage(argv0); + } + + scenario->second(); + + return EXIT_SUCCESS; +} diff --git a/contrib/kyua/engine/googletest_list.cpp b/contrib/kyua/engine/googletest_list.cpp new file mode 100644 index 000000000000..f231cb9e7021 --- /dev/null +++ b/contrib/kyua/engine/googletest_list.cpp @@ -0,0 +1,111 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_list.hpp" + +#include <fstream> +#include <regex> +#include <string> +#include <utility> + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "utils/config/exceptions.hpp" +#include "utils/format/macros.hpp" + +namespace config = utils::config; +namespace fs = utils::fs; + + +namespace { + +/// A regular expression that should match either a test suite or a test case. +const std::string name_expr = "([[:alpha:][:digit:]_]+[[:alpha:][:digit:]_/]*)"; + +/// The separator between a test suite and a test case. +const std::string testsuite_testcase_separator = "."; + +} // anonymous namespace + + +/// Parses the googletest list of test cases from an open stream. +/// +/// \param input The stream to read from. +/// +/// \return The collection of parsed test cases. +/// +/// \throw format_error If there is any problem in the input data. +model::test_cases_map +engine::parse_googletest_list(std::istream& input) +{ + std::string line, test_suite; + + // TODO: These were moved here from the anonymous namespace because of a + // Doxygen bug that's reproducible with the version used by Travis CI. + // + /// A complete regular expression representing a line with a test case + /// definition, e,g., " TestCase", " TestCase/0", or + /// " TestCase/0 # GetParam() = 4". + const std::regex testcase_re( + " " + name_expr + "([[:space:]]+# GetParam\\(\\) = .+)?"); + + /// A complete regular expression representing a line with a test suite + /// definition, e,g., * "TestSuite.", "TestSuite/Prefix.", or + /// "TestSuite/Prefix. # TypeParam = .+". + const std::regex testsuite_re( + name_expr + "\\.([[:space:]]+# TypeParam = .+)?"); + // END TODO + + model::test_cases_map_builder test_cases_builder; + while (std::getline(input, line).good()) { + std::smatch match; + if (std::regex_match(line, match, testcase_re)) { + if (test_suite.empty()) { + throw format_error("Invalid testcase definition: not preceded " + "by a test suite definition"); + } + std::string test_case(match[1]); + test_cases_builder.add(test_suite + test_case); + } else if (std::regex_match(line, match, testsuite_re)) { + test_suite = std::string(match[1]) + + testsuite_testcase_separator; + } else { + // Ignore the line; something might have used output a diagnostic + // message to stdout, e.g., gtest_main. + } + } + const model::test_cases_map test_cases = test_cases_builder.build(); + if (test_cases.empty()) { + // The scheduler interface also checks for the presence of at least one + // test case. However, because the atf format itself requires one test + // case to be always present, we check for this condition here as well. + throw format_error("No test cases"); + } + return test_cases; +} diff --git a/contrib/kyua/engine/googletest_list.hpp b/contrib/kyua/engine/googletest_list.hpp new file mode 100644 index 000000000000..2c9d9e0faf4a --- /dev/null +++ b/contrib/kyua/engine/googletest_list.hpp @@ -0,0 +1,50 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/atf_list.hpp +/// Parser of googletest test case lists. + +#if !defined(ENGINE_GOOGLETEST_LIST_HPP) +#define ENGINE_GOOGLETEST_LIST_HPP + +#include <istream> + +#include "model/metadata_fwd.hpp" +#include "model/test_case_fwd.hpp" +#include "model/types.hpp" +#include "utils/fs/path_fwd.hpp" + +namespace engine { + + +model::test_cases_map parse_googletest_list(std::istream&); + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_LIST_HPP) diff --git a/contrib/kyua/engine/googletest_list_test.cpp b/contrib/kyua/engine/googletest_list_test.cpp new file mode 100644 index 000000000000..5052c2f133ad --- /dev/null +++ b/contrib/kyua/engine/googletest_list_test.cpp @@ -0,0 +1,208 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_list.hpp" + +#include <sstream> +#include <string> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/types.hpp" +#include "utils/datetime.hpp" +#include "utils/format/containers.ipp" +#include "utils/fs/path.hpp" +#include "utils/units.hpp" + +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace units = utils::units; + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__invalid_testcase_definition); +ATF_TEST_CASE_BODY(parse_googletest_list__invalid_testcase_definition) +{ + std::istringstream input1(" \n"); + + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input1)); + + std::istringstream input2(" TestcaseWithoutSuite\n"); + ATF_REQUIRE_THROW_RE(engine::format_error, + "Invalid testcase definition: not preceded by a test suite definition", + engine::parse_googletest_list(input2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__invalid_testsuite_definition); +ATF_TEST_CASE_BODY(parse_googletest_list__invalid_testsuite_definition) +{ + std::istringstream input1("\n"); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input1)); + + std::istringstream input2( +"TestSuiteWithoutSeparator\n" + ); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input2)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__no_test_cases); +ATF_TEST_CASE_BODY(parse_googletest_list__no_test_cases) +{ + std::istringstream input(""); + ATF_REQUIRE_THROW_RE(engine::format_error, "No test cases", + engine::parse_googletest_list(input)); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_test_case); +ATF_TEST_CASE_BODY(parse_googletest_list__one_test_case) +{ + std::istringstream input( +"TestSuite.\n" +" TestCase\n" + ); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite.TestCase").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_case); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_case) +{ + std::istringstream input( +"TestSuite.\n" +" TestCase/0 # GetParam() = 'c'\n" + ); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite.TestCase/0").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_suite); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_suite) +{ + std::string text = +"TestSuite/0. # TypeParam = int\n" +" TestCase\n" + ; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite/0.TestCase").build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__one_parameterized_test_case_and_test_suite); +ATF_TEST_CASE_BODY(parse_googletest_list__one_parameterized_test_case_and_test_suite) +{ + std::string text = +"TestSuite/0. # TypeParam = int\n" +" TestCase/0 # GetParam() = \"herp\"\n" +" TestCase/1 # GetParam() = \"derp\"\n" + ; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("TestSuite/0.TestCase/0") + .add("TestSuite/0.TestCase/1") + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(parse_googletest_list__many_test_cases); +ATF_TEST_CASE_BODY(parse_googletest_list__many_test_cases) +{ + std::string text = +"FirstTestSuite.\n" +" ATestCase\n" +"SecondTestSuite.\n" +" AnotherTestCase\n" +"ThirdTestSuite.\n" +" _\n" +"FourthTestSuite/0. # TypeParam = std::list<int>\n" +" TestCase\n" +"FourthTestSuite/1. # TypeParam = std::list<int>\n" +" TestCase\n" +"FifthTestSuite.\n" +" TestCase/0 # GetParam() = 0\n" +" TestCase/1 # GetParam() = (1, 2, 3)\n" +" TestCase/2 # GetParam() = \"developers. developers\"\n" +"SixthTestSuite/0. # TypeParam = std::map<std::basic_string, int>\n" +" TestCase/0 # GetParam() = 0\n" +" TestCase/1 # GetParam() = (1, 2, 3)\n" + ; + std::istringstream input(text); + const model::test_cases_map tests = engine::parse_googletest_list(input); + + const model::test_cases_map exp_tests = model::test_cases_map_builder() + .add("FirstTestSuite.ATestCase", model::metadata_builder() + .build()) + .add("SecondTestSuite.AnotherTestCase", model::metadata_builder() + .build()) + .add("ThirdTestSuite._") + .add("FourthTestSuite/0.TestCase") + .add("FourthTestSuite/1.TestCase") + .add("FifthTestSuite.TestCase/0") + .add("FifthTestSuite.TestCase/1") + .add("FifthTestSuite.TestCase/2") + .add("SixthTestSuite/0.TestCase/0") + .add("SixthTestSuite/0.TestCase/1") + .build(); + ATF_REQUIRE_EQ(exp_tests, tests); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__invalid_testcase_definition); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__invalid_testsuite_definition); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__no_test_cases); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_test_case); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_parameterized_test_case); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__one_parameterized_test_suite); + ATF_ADD_TEST_CASE(tcs, + parse_googletest_list__one_parameterized_test_case_and_test_suite); + ATF_ADD_TEST_CASE(tcs, parse_googletest_list__many_test_cases); +} diff --git a/contrib/kyua/engine/googletest_result.cpp b/contrib/kyua/engine/googletest_result.cpp new file mode 100644 index 000000000000..a1fc1f72da4d --- /dev/null +++ b/contrib/kyua/engine/googletest_result.cpp @@ -0,0 +1,528 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_result.hpp" + +#include <cstdlib> +#include <fstream> +#include <regex> +#include <utility> + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/fs/path.hpp" +#include "utils/format/macros.hpp" +#include "utils/optional.ipp" +#include "utils/process/status.hpp" +#include "utils/sanity.hpp" +#include "utils/text/exceptions.hpp" +#include "utils/text/operations.ipp" + +namespace fs = utils::fs; +namespace process = utils::process; +namespace text = utils::text; + +using utils::none; +using utils::optional; + + +namespace { + +/// Internal string for specifying invalid output. +const std::string invalid_output_message = "invalid output"; + +/// Regular expression for disabled tests line. +const std::regex disabled_re( + R"RE((YOU HAVE [[:digit:]]+ DISABLED TESTS?))RE" +); + +/// Regular expression for starting sentinel of results block. +const std::regex starting_sentinel_re( + R"(\[[[:space:]]+RUN[[:space:]]+\][[:space:]]+[A-Za-z0-9]+\.[A-Za-z0-9]+)" +); + +/// Regular expression for ending sentinel of results block. +const std::regex ending_sentinel_re( + R"RE(\[[[:space:]]+(FAILED|OK|SKIPPED)[[:space:]]+\])RE" +); + +/// Parses a test result that does not accept a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be "successful". +static engine::googletest_result +parse_without_reason(const std::string& status, const std::string& rest) +{ + if (!rest.empty()) + throw engine::format_error(F("%s cannot have a reason") % status); + + if (status == "skipped") + return engine::googletest_result(engine::googletest_result::skipped); + else { + INV(status == "successful"); + return engine::googletest_result(engine::googletest_result::successful); + } +} + + +/// Parses a test result that needs a reason. +/// +/// \param status The result status name. +/// \param rest The rest of the line after the status name. +/// +/// \return An object representing the test result. +/// +/// \throw format_error If the result is invalid (i.e. rest is invalid). +/// +/// \pre status must be one of "broken", "disabled", "failed", or "skipped". +static engine::googletest_result +parse_with_reason(const std::string& status, const std::string& rest) +{ + using engine::googletest_result; + + INV(!rest.empty()); + + if (status == "broken") + return googletest_result(googletest_result::broken, rest); + else if (status == "disabled") + return googletest_result(googletest_result::disabled, rest); + else if (status == "failed") + return googletest_result(googletest_result::failed, rest); + else if (status == "skipped") + return googletest_result(googletest_result::skipped, rest); + else + PRE_MSG(false, "Unexpected status"); +} + + +/// Formats the termination status of a process to be used with validate_result. +/// +/// \param status The status to format. +/// +/// \return A string describing the status. +static std::string +format_status(const process::status& status) +{ + if (status.exited()) + return F("exited with code %s") % status.exitstatus(); + else if (status.signaled()) + return F("received signal %s%s") % status.termsig() % + (status.coredump() ? " (core dumped)" : ""); + else + return F("terminated in an unknown manner"); +} + + +} // anonymous namespace + + +/// Constructs a raw result with a type. +/// +/// The reason is left uninitialized. +/// +/// \param type_ The type of the result. +engine::googletest_result::googletest_result(const types type_) : + _type(type_) +{ +} + + +/// Constructs a raw result with a type and a reason. +/// +/// \param type_ The type of the result. +/// \param reason_ The reason for the result. +engine::googletest_result::googletest_result( + const types type_, + const std::string& reason_) : + _type(type_), _reason(reason_) +{ +} + + +/// Parses an input stream to extract a test result. +/// +/// If the parsing fails for any reason, the test result is 'broken' and it +/// contains the reason for the parsing failure. Test cases that report results +/// in an inconsistent state cannot be trusted (e.g. the test program code may +/// have a bug), and thus why they are reported as broken instead of just failed +/// (which is a legitimate result for a test case). +/// +/// \param input The stream to read from. +/// +/// \return A generic representation of the result of the test case. +/// +/// \throw format_error If the input is invalid. +engine::googletest_result +engine::googletest_result::parse(std::istream& input) +{ + std::vector<std::string> lines; + + do { + std::string line; + std::getline(input, line, '\n'); + if (!input.eof()) + line.push_back('\n'); + lines.push_back(line); + } while (input.good()); + + bool capture_context = false; + bool valid_output = false; + std::string context, status; + + for (auto& line: lines) { + std::smatch matches; + + if (regex_search(line, matches, disabled_re)) { + context = matches[1]; + status = "disabled"; + valid_output = true; + break; + } + if (regex_search(line, matches, starting_sentinel_re)) { + capture_context = true; + context = ""; + continue; + } + if (regex_search(line, matches, ending_sentinel_re)) { + std::string googletest_res = matches[1]; + if (googletest_res == "OK") { + context = ""; + status = "successful"; + } else if (googletest_res == "FAILED") { + status = "failed"; + } else { + INV(googletest_res == "SKIPPED"); + status = "skipped"; + } + capture_context = false; + valid_output = true; + } + if (capture_context) { + context += line; + } + } + if (!valid_output) { + context = invalid_output_message; + status = "broken"; + } + if (status == "skipped" && context.empty()) + context = bogus_googletest_skipped_nul_message; + if (context.empty()) + return parse_without_reason(status, context); + else + return parse_with_reason(status, context); +} + + +/// Loads a test case result from a file. +/// +/// \param file The file to parse. +/// +/// \return The parsed test case result if all goes well. +/// +/// \throw std::runtime_error If the file does not exist. +/// \throw engine::format_error If the contents of the file are bogus. +engine::googletest_result +engine::googletest_result::load(const fs::path& file) +{ + std::ifstream input(file.c_str()); + if (!input) + throw std::runtime_error("Cannot open results file"); + else + return parse(input); +} + + +/// Gets the type of the result. +/// +/// \return A result type. +engine::googletest_result::types +engine::googletest_result::type(void) const +{ + return _type; +} + + +/// Gets the optional reason of the result. +/// +/// \return The reason of the result if present; none otherwise. +const optional< std::string >& +engine::googletest_result::reason(void) const +{ + return _reason; +} + + +/// Checks whether the result should be reported as good or not. +/// +/// \return True if the result can be considered "good", false otherwise. +bool +engine::googletest_result::good(void) const +{ + switch (_type) { + case googletest_result::disabled: + case googletest_result::skipped: + case googletest_result::successful: + return true; + + case googletest_result::broken: + case googletest_result::failed: + return false; + + default: + UNREACHABLE; + } +} + + +/// Reinterprets a raw result based on the termination status of the test case. +/// +/// This reinterpretation ensures that the termination conditions of the program +/// match what is expected of the paticular result reported by the test program. +/// If such conditions do not match, the test program is considered bogus and is +/// thus reported as broken. +/// +/// This is just a helper function for calculate_result(); the real result of +/// the test case cannot be inferred from apply() only. +/// +/// \param status The exit status of the test program, or none if the test +/// program timed out. +/// +/// \result The adjusted result. The original result is transformed into broken +/// if the exit status of the program does not match our expectations. +engine::googletest_result +engine::googletest_result::apply(const optional< process::status >& status) + const +{ + if (!status) { + return *this; + } + + auto check_status = [&status](bool expect_pass) -> bool { + if (!status.get().exited()) { + return false; + } + auto exit_status = status.get().exitstatus(); + if (expect_pass) { + return exit_status == EXIT_SUCCESS; + } else { + return exit_status == EXIT_FAILURE; + } + }; + + switch (_type) { + case googletest_result::broken: + return *this; + + case googletest_result::disabled: + if (check_status(true)) { + return *this; + } + return googletest_result(googletest_result::broken, + "Disabled test case should have reported success but " + + format_status(status.get())); + + case googletest_result::failed: + if (check_status(false)) { + return *this; + } + return googletest_result(googletest_result::broken, + "Failed test case should have reported failure but " + + format_status(status.get())); + + case googletest_result::skipped: + if (check_status(true)) { + return *this; + } + return googletest_result(googletest_result::broken, + "Skipped test case should have reported success but " + + format_status(status.get())); + + case googletest_result::successful: + if (check_status(true)) { + return *this; + } + return googletest_result(googletest_result::broken, + "Passed test case should have reported success but " + + format_status(status.get())); + } + + UNREACHABLE; +} + + +/// Converts an internal result to the interface-agnostic representation. +/// +/// \return A generic result instance representing this result. +model::test_result +engine::googletest_result::externalize(void) const +{ + switch (_type) { + case googletest_result::broken: + return model::test_result(model::test_result_broken, _reason.get()); + + case googletest_result::disabled: + return model::test_result(model::test_result_skipped, _reason.get()); + + case googletest_result::failed: + return model::test_result(model::test_result_failed, _reason.get()); + + case googletest_result::skipped: + return model::test_result(model::test_result_skipped, _reason.get()); + + case googletest_result::successful: + return model::test_result(model::test_result_passed); + + default: + UNREACHABLE; + } +} + + +/// Compares two raw results for equality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are equal; false otherwise. +bool +engine::googletest_result::operator==(const googletest_result& other) const +{ + return _type == other._type && + _reason == other._reason; +} + + +/// Compares two raw results for inequality. +/// +/// \param other The result to compare to. +/// +/// \return True if the two raw results are different; false otherwise. +bool +engine::googletest_result::operator!=(const googletest_result& other) const +{ + return !(*this == other); +} + + +/// Injects the object into a stream. +/// +/// \param output The stream into which to inject the object. +/// \param object The object to format. +/// +/// \return The output stream. +std::ostream& +engine::operator<<(std::ostream& output, const googletest_result& object) +{ + std::string result_name; + switch (object.type()) { + case googletest_result::broken: result_name = "broken"; break; + case googletest_result::disabled: result_name = "disabled"; break; + case googletest_result::failed: result_name = "failed"; break; + case googletest_result::skipped: result_name = "skipped"; break; + case googletest_result::successful: result_name = "successful"; break; + } + + const optional< std::string >& reason = object.reason(); + + output << F("model::test_result{type=%s, reason=%s}") + % text::quote(result_name, '\'') + % (reason ? text::quote(reason.get(), '\'') : "none"); + + return output; +} + + +/// Calculates the user-visible result of a test case. +/// +/// This function needs to perform magic to ensure that what the test case +/// reports as its result is what the user should really see: i.e. it adjusts +/// the reported status of the test to the exit conditions of its body and +/// cleanup parts. +/// +/// \param body_status The termination status of the process that executed +/// the body of the test. None if the body timed out. +/// \param results_file The path to the results file that the test case body is +/// supposed to have created. +/// +/// \return The calculated test case result. +model::test_result +engine::calculate_googletest_result( + const optional< process::status >& body_status, + const fs::path& results_file) +{ + using engine::googletest_result; + + googletest_result result(googletest_result::broken, "Unknown result"); + try { + result = googletest_result::load(results_file); + /// atf_result.cpp handles this by side-effect by setting a broken + /// result with "", which results in a std::runtime_error throw from + /// atf_result::parse(..). googletest_result::parse doesn't do that and + /// instead returns a broken result with a reason of + /// `invalid_output_message`, so that case needs to be tested for after + /// the fact to make sure the reason why `invalid_output_message` was + /// because of a program that crashed or failed. + if (result.type() == googletest_result::broken && body_status) { + const optional< std::string >& reason = result.reason(); + if (reason && reason.get() == invalid_output_message) { + result = googletest_result( + googletest_result::broken, + F("Error: Premature exit. Test case %s") % + format_status(body_status.get())); + } + } + } catch (const engine::format_error& error) { + if (body_status) + result = googletest_result(googletest_result::broken, + F("Error: %s. Test case %s") % + error.what() % + format_status(body_status.get())); + else { + // The test case timed out. apply() handles this case later. + } + } catch (const std::runtime_error& error) { + if (body_status) + result = googletest_result( + googletest_result::broken, + F("Error: Premature exit. Test case %s") % + format_status(body_status.get())); + else { + // The test case timed out. apply() handles this case later. + } + } + + result = result.apply(body_status); + + return result.externalize(); +} diff --git a/contrib/kyua/engine/googletest_result.hpp b/contrib/kyua/engine/googletest_result.hpp new file mode 100644 index 000000000000..325995be3221 --- /dev/null +++ b/contrib/kyua/engine/googletest_result.hpp @@ -0,0 +1,111 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest_result.hpp +/// Functions and types to process the results of googletest-based test cases. + +#if !defined(ENGINE_GOOGLETEST_RESULT_HPP) +#define ENGINE_GOOGLETEST_RESULT_HPP + +#include "engine/googletest_result_fwd.hpp" + +#include <istream> +#include <ostream> + +#include "model/test_result_fwd.hpp" +#include "utils/optional.hpp" +#include "utils/fs/path_fwd.hpp" +#include "utils/process/status_fwd.hpp" + +namespace engine { + + +/// Internal representation of the raw result files of googletest-based tests. +/// +/// This class is used exclusively to represent the transient result files read +/// from test cases before generating the "public" version of the result. This +/// class should actually not be exposed in the header files, but it is for +/// testing purposes only. +class googletest_result { +public: + /// List of possible types for the test case result. + enum types { + broken, + disabled, + failed, + skipped, + successful, + }; + +private: + /// The test case result. + types _type; + + /// A description of the test case result. + /// + /// Should always be present except for the passed type and sometimes with + /// the skipped type. + utils::optional< std::string > _reason; + +public: + googletest_result(const types); + googletest_result(const types, const std::string&); + + static googletest_result parse(std::istream&); + static googletest_result load(const utils::fs::path&); + + types type(void) const; + const utils::optional< std::string >& reason(void) const; + + bool good(void) const; + googletest_result apply( + const utils::optional< utils::process::status >&) const; + model::test_result externalize(void) const; + + bool operator==(const googletest_result&) const; + bool operator!=(const googletest_result&) const; +}; + + +std::ostream& operator<<(std::ostream&, const googletest_result&); + + +model::test_result calculate_googletest_result( + const utils::optional< utils::process::status >&, + const utils::fs::path&); + + +/// A bogus identifier for nul reasons provided by the test writer. +/// +/// TODO: Support nul messages with skipped results in the schema, etc. +const std::string bogus_googletest_skipped_nul_message = "\n"; + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_RESULT_HPP) diff --git a/contrib/kyua/engine/googletest_result_fwd.hpp b/contrib/kyua/engine/googletest_result_fwd.hpp new file mode 100644 index 000000000000..6f41c75a0b3f --- /dev/null +++ b/contrib/kyua/engine/googletest_result_fwd.hpp @@ -0,0 +1,43 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/// \file engine/googletest_result_fwd.hpp +/// Forward declarations for engine/googletest_result.hpp + +#if !defined(ENGINE_GOOGLETEST_RESULT_FWD_HPP) +#define ENGINE_GOOGLETEST_RESULT_FWD_HPP + +namespace engine { + + +class googletest_result; + + +} // namespace engine + +#endif // !defined(ENGINE_GOOGLETEST_RESULT_FWD_HPP) diff --git a/contrib/kyua/engine/googletest_result_test.cpp b/contrib/kyua/engine/googletest_result_test.cpp new file mode 100644 index 000000000000..90bc2cc62b7d --- /dev/null +++ b/contrib/kyua/engine/googletest_result_test.cpp @@ -0,0 +1,427 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest_result.hpp" + +extern "C" { +#include <signal.h> +} + +#include <cstdlib> +#include <fstream> +#include <sstream> +#include <stdexcept> + +#include <atf-c++.hpp> + +#include "engine/exceptions.hpp" +#include "model/test_result.hpp" +#include "utils/format/macros.hpp" +#include "utils/fs/path.hpp" +#include "utils/process/status.hpp" + +namespace fs = utils::fs; +namespace process = utils::process; + +using utils::none; +using utils::optional; + + +namespace { + + +/// Performs a test for results::parse() that should succeed. +/// +/// \param exp_type The expected type of the result. +/// \param exp_reason The expected reason describing the result, if any. +/// \param text The literal input to parse; can include multiple lines. +static void +parse_ok_test(const engine::googletest_result::types& exp_type, + const char* exp_reason, const char* text) +{ + std::istringstream input(text); + const engine::googletest_result actual = + engine::googletest_result::parse(input); + ATF_REQUIRE_EQ(exp_type, actual.type()); + if (exp_reason != NULL) { + ATF_REQUIRE(actual.reason()); + ATF_REQUIRE_EQ(exp_reason, actual.reason().get()); + } else { + ATF_REQUIRE(!actual.reason()); + } +} + + +/// Wrapper around parse_ok_test to define a test case. +/// +/// \param name The name of the test case; will be prefixed with +/// "googletest_result__parse__". +/// \param exp_type The expected type of the result. +/// \param exp_reason The expected reason describing the result, if any. +/// \param input The literal input to parse. +#define PARSE(name, exp_type, exp_reason, input) \ + ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__parse__ ## name); \ + ATF_TEST_CASE_BODY(googletest_result__parse__ ## name) \ + { \ + parse_ok_test(exp_type, exp_reason, input); \ + } + + +} // anonymous namespace + + +PARSE(broken, + engine::googletest_result::broken, "invalid output", + "invalid input"); + +const char disabled_context[] = ( +"YOU HAVE 1 DISABLED TEST" +); + +const char disabled_message[] = ( +"[==========] Running 0 tests from 0 test cases.\n" +"[==========] 0 tests from 0 test cases ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"\n" +" YOU HAVE 1 DISABLED TEST\n" +"\n" +"\n" +); + +PARSE(disabled, + engine::googletest_result::disabled, disabled_context, + disabled_message); + +const char failed_context[] = ( +"pass_fail_demo.cc:8: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +); + +const char failed_message[] = ( +"Note: Google Test filter = PassFailTest.Fails\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.Fails\n" +"pass_fail_demo.cc:8: Failure\n" +"Expected equality of these values:\n" +" false\n" +" true\n" +"[ FAILED ] PassFailTest.Fails (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ FAILED ] 1 test, listed below:\n" +"[ FAILED ] PassFailTest.Fails\n" +"\n" +" 1 FAILED TEST\n" +); + +PARSE(failed, + engine::googletest_result::failed, failed_context, + failed_message); + +const char skipped_message[] = ( +"Note: Google Test filter = SkipTest.DoesSkip\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from SkipTest\n" +"[ RUN ] SkipTest.DoesSkip\n" +"[ SKIPPED ] SkipTest.DoesSkip (0 ms)\n" +"[----------] 1 test from SkipTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ SKIPPED ] 1 test, listed below:\n" +"[ SKIPPED ] SkipTest.DoesSkip\n" +); + +PARSE(skipped, + engine::googletest_result::skipped, + engine::bogus_googletest_skipped_nul_message.c_str(), + skipped_message +); + +const char skipped_with_reason_context[] = ( +"This is a reason\n" +); + +const char skipped_with_reason_message[] = ( +"Note: Google Test filter = SkipTest.SkipWithReason\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from SkipTest\n" +"[ RUN ] SkipTest.SkipWithReason\n" +"This is a reason\n" +"[ SKIPPED ] SkipTest.SkipWithReason (0 ms)\n" +"[----------] 1 test from SkipTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 0 tests.\n" +"[ SKIPPED ] 1 test, listed below:\n" +"[ SKIPPED ] SkipTest.SkipWithReason\n" +); + +PARSE(skipped_with_reason, + engine::googletest_result::skipped, skipped_with_reason_context, + skipped_with_reason_message); + +const char successful_message[] = ( +"Note: Google Test filter = PassFailTest.Passes\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.Passes\n" +"[ OK ] PassFailTest.Passes (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +PARSE(successful, + engine::googletest_result::successful, NULL, + successful_message); + +const char successful_message2[] = ( +"Note: Google Test filter = ValuesTest.ValuesWorks\n" +"[==========] Running 1 test from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from ValuesTest\n" +"[ RUN ] ValuesTest.ValuesWorks\n" +"[ OK ] ValuesTest.ValuesWorks (0 ms)\n" +"[----------] 1 test from ValuesTest (0 ms total)\n" +" \n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 test.\n" +); + +PARSE(successful2, + engine::googletest_result::successful, NULL, + successful_message2); + +const char successful_parameterized_message[] = ( +"Note: Google Test filter = RangeZeroToFive/ParamDerivedTest/0\n" +"[==========] Running 5 tests from 1 test case.\n" +"[----------] Global test environment set-up.\n" +"[----------] 5 tests from RangeZeroToFive/ParamDerivedTest\n" +"[ RUN ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0\n" +"[ OK ] RangeZeroToFive/ParamDerivedTest.SeesSequence/0 (0 ms)\n" +"[----------] 1 test from RangeZeroToFive/ParamDerivedTest/0 (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 tests from 1 test case ran. (0 ms total)\n" +"[ PASSED ] 1 tests.\n" +); + +PARSE(successful_parameterized, + engine::googletest_result::successful, NULL, + successful_parameterized_message); + +const char successful_message_with_reason[] = ( +"Note: Google Test filter = PassFailTest.PassesWithReason\n" +"[==========] Running 1 test from 1 test suite.\n" +"[----------] Global test environment set-up.\n" +"[----------] 1 test from PassFailTest\n" +"[ RUN ] PassFailTest.PassesWithReason\n" +"This is a reason\n" +"[ OK ] PassFailTest.PassesWithReason (0 ms)\n" +"[----------] 1 test from PassFailTest (0 ms total)\n" +"\n" +"[----------] Global test environment tear-down\n" +"[==========] 1 test from 1 test suite ran. (0 ms total)\n" +"[ PASSED ] 1 tests.\n" +); + +PARSE(successful_with_reason, + engine::googletest_result::successful, NULL, + successful_message_with_reason); + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__ok); +ATF_TEST_CASE_BODY(googletest_result__load__ok) +{ + std::ofstream output("result.txt"); + ATF_REQUIRE(output); + output << skipped_with_reason_message; + output.close(); + + const engine::googletest_result result = engine::googletest_result::load( + utils::fs::path("result.txt")); + ATF_REQUIRE_EQ(engine::googletest_result::skipped, result.type()); + ATF_REQUIRE(result.reason()); + ATF_REQUIRE_EQ(skipped_with_reason_context, result.reason().get()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__load__missing_file); +ATF_TEST_CASE_BODY(googletest_result__load__missing_file) +{ + ATF_REQUIRE_THROW_RE( + std::runtime_error, "Cannot open", + engine::googletest_result::load(utils::fs::path("result.txt"))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__broken); +ATF_TEST_CASE_BODY(googletest_result__apply__broken) +{ + const process::status status = process::status::fake_exited(EXIT_FAILURE); + const engine::googletest_result broken(engine::googletest_result::broken, + "The reason"); + ATF_REQUIRE_EQ(broken, broken.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__disabled); +ATF_TEST_CASE_BODY(googletest_result__apply__disabled) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result disabled( + engine::googletest_result::disabled, "The reason"); + ATF_REQUIRE_EQ(disabled, disabled.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__failed); +ATF_TEST_CASE_BODY(googletest_result__apply__failed) +{ + const process::status status = process::status::fake_exited(EXIT_FAILURE); + const engine::googletest_result failed(engine::googletest_result::failed, + "The reason"); + ATF_REQUIRE_EQ(failed, failed.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__skipped); +ATF_TEST_CASE_BODY(googletest_result__apply__skipped) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result skipped(engine::googletest_result::skipped, + "The reason"); + ATF_REQUIRE_EQ(skipped, skipped.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__apply__successful); +ATF_TEST_CASE_BODY(googletest_result__apply__successful) +{ + const process::status status = process::status::fake_exited(EXIT_SUCCESS); + const engine::googletest_result + successful(engine::googletest_result::successful); + ATF_REQUIRE_EQ(successful, successful.apply(utils::make_optional(status))); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__broken); +ATF_TEST_CASE_BODY(googletest_result__externalize__broken) +{ + const engine::googletest_result raw(engine::googletest_result::broken, + "The reason"); + const model::test_result expected(model::test_result_broken, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__disabled); +ATF_TEST_CASE_BODY(googletest_result__externalize__disabled) +{ + const engine::googletest_result raw(engine::googletest_result::disabled, + "The reason"); + const model::test_result expected(model::test_result_skipped, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__failed); +ATF_TEST_CASE_BODY(googletest_result__externalize__failed) +{ + const engine::googletest_result raw(engine::googletest_result::failed, + "The reason"); + const model::test_result expected(model::test_result_failed, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__skipped); +ATF_TEST_CASE_BODY(googletest_result__externalize__skipped) +{ + const engine::googletest_result raw(engine::googletest_result::skipped, + "The reason"); + const model::test_result expected(model::test_result_skipped, + "The reason"); + ATF_REQUIRE(expected == raw.externalize()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(googletest_result__externalize__successful); +ATF_TEST_CASE_BODY(googletest_result__externalize__successful) +{ + const engine::googletest_result raw(engine::googletest_result::successful); + const model::test_result expected(model::test_result_passed); + ATF_REQUIRE_EQ(expected, raw.externalize()); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__skipped_with_reason); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful2); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_parameterized); + ATF_ADD_TEST_CASE(tcs, googletest_result__parse__successful_with_reason); + + ATF_ADD_TEST_CASE(tcs, googletest_result__load__ok); + ATF_ADD_TEST_CASE(tcs, googletest_result__load__missing_file); + + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__apply__successful); + + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__broken); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__disabled); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__failed); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__skipped); + ATF_ADD_TEST_CASE(tcs, googletest_result__externalize__successful); +} diff --git a/contrib/kyua/engine/googletest_test.cpp b/contrib/kyua/engine/googletest_test.cpp new file mode 100644 index 000000000000..3af4d6801e70 --- /dev/null +++ b/contrib/kyua/engine/googletest_test.cpp @@ -0,0 +1,361 @@ +// Copyright 2024 The Kyua Authors. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of Google Inc. nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "engine/googletest.hpp" + +extern "C" { +#include <sys/stat.h> + +#include <signal.h> +} + +#include <atf-c++.hpp> + +#include "engine/config.hpp" +#include "engine/scheduler.hpp" +#include "model/metadata.hpp" +#include "model/test_case.hpp" +#include "model/test_program_fwd.hpp" +#include "model/test_result.hpp" +#include "utils/config/tree.ipp" +#include "utils/datetime.hpp" +#include "utils/env.hpp" +#include "utils/format/containers.ipp" +#include "utils/format/macros.hpp" +#include "utils/fs/operations.hpp" +#include "utils/fs/path.hpp" +#include "utils/optional.ipp" +#include "utils/stacktrace.hpp" +#include "utils/test_utils.ipp" + +namespace config = utils::config; +namespace datetime = utils::datetime; +namespace fs = utils::fs; +namespace scheduler = engine::scheduler; + +using utils::none; + + +namespace { + + +/// Prefix for all testcases. +const std::string test_suite = "Suite."; + + +/// Runs one googletest test program and checks its result. +/// +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +/// \param user_config User-provided configuration. +/// +/// \return The list of loaded test cases. +static model::test_cases_map +list(const char* program_name, + const fs::path& root, + config::tree user_config = engine::empty_config()) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const scheduler::lazy_test_program program( + "googletest", fs::path(program_name), root, "the-suite", + model::metadata_builder().build(), user_config, handle); + + const model::test_cases_map test_cases = handle.list_tests( + &program, user_config); + + handle.cleanup(); + + return test_cases; +} + + +/// Runs a bogus test program and checks the error result. +/// +/// \param exp_error Expected error string to find. +/// \param program_name Basename of the test program to run. +/// \param root Path to the base of the test suite. +static void +check_list_fail(const char* exp_error, + const char* program_name, + const fs::path& root) +{ + const model::test_cases_map test_cases = list( + program_name, root); + + ATF_REQUIRE_EQ(1, test_cases.size()); + const model::test_case& test_case = test_cases.begin()->second; + ATF_REQUIRE_EQ("__test_cases_list__", test_case.name()); + ATF_REQUIRE(test_case.fake_result()); + ATF_REQUIRE_MATCH(exp_error, + test_case.fake_result().get().reason()); +} + + +/// Runs one googletest test program and checks its result. +/// +/// \param tc Pointer to the calling test case, to obtain srcdir. +/// \param test_case_name Name of the "test case" to select from the helper +/// program. +/// \param exp_result The expected result. +/// \param metadata The test case metadata. +/// \param user_config User-provided configuration. +static void +run_one(const atf::tests::tc* tc, const char* test_case_name, + const model::test_result& exp_result, + const model::metadata& metadata = model::metadata_builder().build(), + config::tree user_config = engine::empty_config()) +{ + scheduler::scheduler_handle handle = scheduler::setup(); + + const std::string test_name = test_suite + std::string(test_case_name); + + const model::test_program_ptr program(new scheduler::lazy_test_program( + "googletest", fs::path("googletest_helpers"), + fs::path(tc->get_config_var("srcdir")), + "the-suite", metadata, user_config, handle)); + + (void)handle.spawn_test(program, test_name.c_str(), user_config); + + scheduler::result_handle_ptr result_handle = handle.wait_any(); + const scheduler::test_result_handle* test_result_handle = + dynamic_cast< const scheduler::test_result_handle* >( + result_handle.get()); + atf::utils::cat_file(result_handle->stdout_file().str(), "stdout: "); + atf::utils::cat_file(result_handle->stderr_file().str(), "stderr: "); + ATF_REQUIRE_EQ(exp_result, test_result_handle->test_result()); + result_handle->cleanup(); + result_handle.reset(); + + handle.cleanup(); +} + + +} // anonymous namespace + + +ATF_TEST_CASE_WITHOUT_HEAD(list__ok); +ATF_TEST_CASE_BODY(list__ok) +{ + const model::test_cases_map test_cases = list( + "googletest_helpers", fs::path(get_config_var("srcdir"))); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__current_directory); +ATF_TEST_CASE_BODY(list__current_directory) +{ + const fs::path helpers = + fs::path(get_config_var("srcdir")) / "googletest_helpers"; + ATF_REQUIRE(::symlink(helpers.c_str(), "googletest_helpers") != -1); + const model::test_cases_map test_cases = list( + "googletest_helpers", fs::path(".")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__relative_path); +ATF_TEST_CASE_BODY(list__relative_path) +{ + const fs::path helpers = + fs::path(get_config_var("srcdir")) / "googletest_helpers"; + ATF_REQUIRE(::mkdir("dir1", 0755) != -1); + ATF_REQUIRE(::mkdir("dir1/dir2", 0755) != -1); + ATF_REQUIRE_EQ(::symlink(helpers.c_str(), + "dir1/dir2/googletest_helpers"), 0); + const model::test_cases_map test_cases = list( + "dir2/googletest_helpers", fs::path("dir1")); + + const model::test_cases_map exp_test_cases = model::test_cases_map_builder() + .add(test_suite + "check_configuration_variables") + .add(test_suite + "crash") + .add(test_suite + "fail") + .add(test_suite + "pass") + .add(test_suite + "pass_but_exit_failure") + .add(test_suite + "timeout", model::metadata_builder() + .build()) + .build(); + ATF_REQUIRE_EQ(exp_test_cases, test_cases); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__missing_test_program); +ATF_TEST_CASE_BODY(list__missing_test_program) +{ + check_list_fail("Cannot find test program", "non-existent", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__not_a_test_program); +ATF_TEST_CASE_BODY(list__not_a_test_program) +{ + atf::utils::create_file("not-valid", "garbage\n"); + ATF_REQUIRE_EQ(::chmod("not-valid", 0755), 0); + check_list_fail("Invalid test program format", "not-valid", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__no_permissions); +ATF_TEST_CASE_BODY(list__no_permissions) +{ + atf::utils::create_file("not-executable", "garbage\n"); + check_list_fail("Permission denied to run test program", + "not-executable", fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__abort); +ATF_TEST_CASE_BODY(list__abort) +{ + atf::utils::create_file("kills_self", "#!/bin/sh\nkill -2 $$\n"); + ATF_REQUIRE_EQ(::chmod("kills_self", 0755), 0); + check_list_fail("Test program received signal", "kills_self", + fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(list__empty); +ATF_TEST_CASE_BODY(list__empty) +{ + atf::utils::create_file("empty_list", "#!/bin/sh\n"); + ATF_REQUIRE_EQ(::chmod("empty_list", 0755), 0); + check_list_fail("No test cases", "empty_list", fs::current_path()); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__passes); +ATF_TEST_CASE_BODY(test__body_only__passes) +{ + const model::test_result exp_result(model::test_result_passed); + run_one(this, "pass", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__fails); +ATF_TEST_CASE_BODY(test__body_only__fails) +{ + const std::string failure_message = ( +"gtest_macros_demo.cc:4: Failure\n" +"Failed\n" +"with a reason\n" +); + const model::test_result exp_result(model::test_result_failed, + failure_message); + run_one(this, "fail", exp_result); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__crashes); +ATF_TEST_CASE_BODY(test__body_only__crashes) +{ + utils::prepare_coredump_test(this); + + const model::test_result exp_result( + model::test_result_broken, + F("Error: Premature exit. Test case received signal %s (core dumped)") % + SIGABRT); + run_one(this, "crash", exp_result); +} + + +ATF_TEST_CASE(test__body_only__times_out); +ATF_TEST_CASE_HEAD(test__body_only__times_out) +{ + set_md_var("timeout", "60"); +} +ATF_TEST_CASE_BODY(test__body_only__times_out) +{ + config::tree user_config = engine::empty_config(); + utils::setenv("CONTROL_DIR", fs::current_path().str()); + const model::metadata metadata = model::metadata_builder() + .set_timeout(datetime::delta(1, 0)).build(); + const model::test_result exp_result( + model::test_result_broken, "Test case body timed out"); + run_one(this, "timeout", exp_result, metadata, user_config); + + ATF_REQUIRE(!atf::utils::file_exists("cookie")); +} + + +ATF_TEST_CASE_WITHOUT_HEAD(test__body_only__configuration_variables); +ATF_TEST_CASE_BODY(test__body_only__configuration_variables) +{ + config::tree user_config = engine::empty_config(); + user_config.set_string("test_suites.the-suite.first", "some value"); + user_config.set_string("test_suites.the-suite.second", "some other value"); + + const model::test_result exp_result(model::test_result_passed); + run_one(this, "check_configuration_variables", exp_result, + model::metadata_builder().build(), user_config); +} + + +ATF_INIT_TEST_CASES(tcs) +{ + scheduler::register_interface( + "googletest", std::shared_ptr< scheduler::interface >( + new engine::googletest_interface())); + + ATF_ADD_TEST_CASE(tcs, list__ok); + ATF_ADD_TEST_CASE(tcs, list__current_directory); + ATF_ADD_TEST_CASE(tcs, list__relative_path); + ATF_ADD_TEST_CASE(tcs, list__missing_test_program); + ATF_ADD_TEST_CASE(tcs, list__not_a_test_program); + ATF_ADD_TEST_CASE(tcs, list__no_permissions); + ATF_ADD_TEST_CASE(tcs, list__abort); + ATF_ADD_TEST_CASE(tcs, list__empty); + + ATF_ADD_TEST_CASE(tcs, test__body_only__passes); + ATF_ADD_TEST_CASE(tcs, test__body_only__fails); + ATF_ADD_TEST_CASE(tcs, test__body_only__crashes); + ATF_ADD_TEST_CASE(tcs, test__body_only__times_out); + ATF_ADD_TEST_CASE(tcs, test__body_only__configuration_variables); +} diff --git a/contrib/kyua/engine/kyuafile.hpp b/contrib/kyua/engine/kyuafile.hpp index 161f4305f4d1..a785f0462f5b 100644 --- a/contrib/kyua/engine/kyuafile.hpp +++ b/contrib/kyua/engine/kyuafile.hpp @@ -34,11 +34,6 @@ #include "engine/kyuafile_fwd.hpp" -#include <string> -#include <vector> - -#include <lutok/state.hpp> - #include "engine/scheduler_fwd.hpp" #include "model/test_program_fwd.hpp" #include "utils/config/tree_fwd.hpp" diff --git a/contrib/kyua/engine/kyuafile_test.cpp b/contrib/kyua/engine/kyuafile_test.cpp index d95f28c71acb..cd11e4cca491 100644 --- a/contrib/kyua/engine/kyuafile_test.cpp +++ b/contrib/kyua/engine/kyuafile_test.cpp @@ -27,19 +27,11 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "engine/kyuafile.hpp" - +#include <memory> extern "C" { #include <unistd.h> } - -#include <stdexcept> -#include <typeinfo> - #include <atf-c++.hpp> -#include <lutok/operations.hpp> -#include <lutok/state.ipp> -#include <lutok/test_utils.hpp> - #include "engine/atf.hpp" #include "engine/exceptions.hpp" #include "engine/plain.hpp" diff --git a/contrib/kyua/engine/plain.hpp b/contrib/kyua/engine/plain.hpp index ee5f3e746781..60bffec9bf04 100644 --- a/contrib/kyua/engine/plain.hpp +++ b/contrib/kyua/engine/plain.hpp @@ -40,18 +40,19 @@ namespace engine { /// Implementation of the scheduler interface for plain test programs. class plain_interface : public engine::scheduler::interface { public: - void exec_list(const model::test_program&, - const utils::config::properties_map&) const UTILS_NORETURN; + void exec_list [[noreturn]] ( + const model::test_program&, + const utils::config::properties_map&) const; model::test_cases_map parse_list( const utils::optional< utils::process::status >&, const utils::fs::path&, const utils::fs::path&) const; - void exec_test(const model::test_program&, const std::string&, - const utils::config::properties_map&, - const utils::fs::path&) const - UTILS_NORETURN; + void exec_test [[noreturn]] ( + const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const; model::test_result compute_result( const utils::optional< utils::process::status >&, diff --git a/contrib/kyua/engine/requirements.cpp b/contrib/kyua/engine/requirements.cpp index d5838b83f33a..7fc5522cffc2 100644 --- a/contrib/kyua/engine/requirements.cpp +++ b/contrib/kyua/engine/requirements.cpp @@ -112,7 +112,7 @@ check_execenv(const std::string& execenv, const config::tree& user_config) { std::string name = execenv; if (name.empty()) - name = engine::execenv::default_execenv_name; // if test claims nothing + name = DEFAULT_EXECENV_NAME; // if test claims nothing std::set< std::string > execenvs; try { @@ -274,7 +274,7 @@ static std::vector< std::shared_ptr< engine::reqs_checker > > _reqs_checkers; } // anonymous namespace -const std::vector< std::shared_ptr< engine::reqs_checker > > +std::vector< std::shared_ptr< engine::reqs_checker > > engine::reqs_checkers() { return _reqs_checkers; diff --git a/contrib/kyua/engine/requirements.hpp b/contrib/kyua/engine/requirements.hpp index 92e80c5122aa..671123060ce4 100644 --- a/contrib/kyua/engine/requirements.hpp +++ b/contrib/kyua/engine/requirements.hpp @@ -32,7 +32,9 @@ #if !defined(ENGINE_REQUIREMENTS_HPP) #define ENGINE_REQUIREMENTS_HPP +#include <memory> #include <string> +#include <vector> #include "model/metadata_fwd.hpp" #include "utils/config/tree_fwd.hpp" @@ -68,7 +70,7 @@ void register_reqs_checker(const std::shared_ptr< reqs_checker > checker); /// Returns the list of registered extra requirement checkers. /// /// \return A vector of pointers to extra requirement checkers. -const std::vector< std::shared_ptr< reqs_checker > > reqs_checkers(); +std::vector< std::shared_ptr< reqs_checker > > reqs_checkers(); } // namespace engine diff --git a/contrib/kyua/engine/scheduler.cpp b/contrib/kyua/engine/scheduler.cpp index d04fc1dfbd3d..6e19a1598f8c 100644 --- a/contrib/kyua/engine/scheduler.cpp +++ b/contrib/kyua/engine/scheduler.cpp @@ -1653,8 +1653,8 @@ scheduler::generate_config(const config::tree& user_config, if (user_config.is_set("unprivileged_user")) { const passwd::user& user = user_config.lookup< engine::user_node >("unprivileged_user"); - // The property is duplicated using both ATF and Kyua naming styles - // for better UX. + // The property is duplicated using both ATF and Kyua naming styles + // for better UX. props["unprivileged-user"] = user.name; props["unprivileged_user"] = user.name; } diff --git a/contrib/kyua/engine/scheduler.hpp b/contrib/kyua/engine/scheduler.hpp index 508a0c0cbfd9..0e241172f1e7 100644 --- a/contrib/kyua/engine/scheduler.hpp +++ b/contrib/kyua/engine/scheduler.hpp @@ -98,9 +98,9 @@ public: /// /// \param test_program The test program to execute. /// \param vars User-provided variables to pass to the test program. - virtual void exec_list(const model::test_program& test_program, - const utils::config::properties_map& vars) - const UTILS_NORETURN = 0; + virtual void exec_list [[noreturn]] ( + const model::test_program& test_program, + const utils::config::properties_map& vars) const = 0; /// Computes the test cases list of a test program. /// @@ -126,11 +126,11 @@ public: /// \param vars User-provided variables to pass to the test program. /// \param control_directory Directory where the interface may place control /// files. - virtual void exec_test(const model::test_program& test_program, - const std::string& test_case_name, - const utils::config::properties_map& vars, - const utils::fs::path& control_directory) - const UTILS_NORETURN = 0; + virtual void exec_test [[noreturn]] ( + const model::test_program& test_program, + const std::string& test_case_name, + const utils::config::properties_map& vars, + const utils::fs::path& control_directory) const = 0; /// Executes a test cleanup routine of the test program. /// @@ -143,11 +143,11 @@ public: /// \param vars User-provided variables to pass to the test program. /// \param control_directory Directory where the interface may place control /// files. - virtual void exec_cleanup(const model::test_program& test_program, - const std::string& test_case_name, - const utils::config::properties_map& vars, - const utils::fs::path& control_directory) - const UTILS_NORETURN; + virtual void exec_cleanup [[noreturn]]( + const model::test_program& test_program, + const std::string& test_case_name, + const utils::config::properties_map& vars, + const utils::fs::path& control_directory) const; /// Computes the result of a test case based on its termination status. /// diff --git a/contrib/kyua/engine/scheduler_test.cpp b/contrib/kyua/engine/scheduler_test.cpp index d91c448f2e5e..93e7627f291a 100644 --- a/contrib/kyua/engine/scheduler_test.cpp +++ b/contrib/kyua/engine/scheduler_test.cpp @@ -131,7 +131,7 @@ class mock_interface : public scheduler::interface { /// /// \param exit_code Exit code. void - do_exit(const int exit_code) const UTILS_NORETURN + do_exit [[noreturn]] (const int exit_code) const { std::cout.flush(); std::cerr.flush(); @@ -140,7 +140,7 @@ class mock_interface : public scheduler::interface { /// Executes a test case that creates various files and then fails. void - exec_create_files_and_fail(void) const UTILS_NORETURN + exec_create_files_and_fail [[noreturn]] (void) const { std::cerr << "This should not be clobbered\n"; atf::utils::create_file("first file", ""); @@ -155,7 +155,7 @@ class mock_interface : public scheduler::interface { /// This is intended to validate that the test runs in an empty directory, /// separate from any control files that the scheduler may have created. void - exec_delete_all(void) const UTILS_NORETURN + exec_delete_all [[noreturn]] (void) const { const int exit_code = ::system("rm *") == -1 ? EXIT_FAILURE : EXIT_SUCCESS; @@ -170,14 +170,14 @@ class mock_interface : public scheduler::interface { /// /// \param exit_code Exit status to terminate the program with. void - exec_exit(const int exit_code) const UTILS_NORETURN + exec_exit [[noreturn]] (const int exit_code) const { do_exit(exit_code); } /// Executes a test case that just fails. void - exec_fail(void) const UTILS_NORETURN + exec_fail [[noreturn]] (void) const { std::cerr << "This should not be clobbered\n"; ::kill(::getpid(), SIGTERM); @@ -191,10 +191,10 @@ class mock_interface : public scheduler::interface { /// number. /// \param vars User-provided variables to pass to the test program. void - exec_print_params(const model::test_program& test_program, - const std::string& test_case_name, - const config::properties_map& vars) const - UTILS_NORETURN + exec_print_params [[noreturn]] ( + const model::test_program& test_program, + const std::string& test_case_name, + const config::properties_map& vars) const { std::cout << F("Test program: %s\n") % test_program.relative_path(); std::cout << F("Test case: %s\n") % test_case_name; @@ -218,9 +218,9 @@ public: /// \param test_program The test program to execute. /// \param vars User-provided variables to pass to the test program. void - exec_list(const model::test_program& test_program, - const config::properties_map& vars) - const UTILS_NORETURN + exec_list [[noreturn]] ( + const model::test_program& test_program, + const config::properties_map& vars) const { const std::string name = test_program.absolute_path().leaf_name(); diff --git a/contrib/kyua/engine/tap.hpp b/contrib/kyua/engine/tap.hpp index b46bf28f0240..51ad76231eaf 100644 --- a/contrib/kyua/engine/tap.hpp +++ b/contrib/kyua/engine/tap.hpp @@ -40,18 +40,19 @@ namespace engine { /// Implementation of the scheduler interface for tap test programs. class tap_interface : public engine::scheduler::interface { public: - void exec_list(const model::test_program&, - const utils::config::properties_map&) const UTILS_NORETURN; + void exec_list [[noreturn]] ( + const model::test_program&, + const utils::config::properties_map&) const; model::test_cases_map parse_list( const utils::optional< utils::process::status >&, const utils::fs::path&, const utils::fs::path&) const; - void exec_test(const model::test_program&, const std::string&, - const utils::config::properties_map&, - const utils::fs::path&) const - UTILS_NORETURN; + void exec_test [[noreturn]] ( + const model::test_program&, const std::string&, + const utils::config::properties_map&, + const utils::fs::path&) const; model::test_result compute_result( const utils::optional< utils::process::status >&, |