head 1.1; branch 1.1.1; access; symbols netbsd-11-0-RC6:1.1.1.2 netbsd-11-0-RC5:1.1.1.2 netbsd-11-0-RC4:1.1.1.2 netbsd-11-0-RC3:1.1.1.2 netbsd-11-0-RC2:1.1.1.2 netbsd-11-0-RC1:1.1.1.2 perseant-exfatfs-base-20250801:1.1.1.2 netbsd-11:1.1.1.2.0.42 netbsd-11-base:1.1.1.2 netbsd-10-1-RELEASE:1.1.1.2 perseant-exfatfs-base-20240630:1.1.1.2 perseant-exfatfs:1.1.1.2.0.40 perseant-exfatfs-base:1.1.1.2 netbsd-8-3-RELEASE:1.1.1.2 netbsd-9-4-RELEASE:1.1.1.2 netbsd-10-0-RELEASE:1.1.1.2 netbsd-10-0-RC6:1.1.1.2 netbsd-10-0-RC5:1.1.1.2 netbsd-10-0-RC4:1.1.1.2 netbsd-10-0-RC3:1.1.1.2 netbsd-10-0-RC2:1.1.1.2 netbsd-10-0-RC1:1.1.1.2 netbsd-10:1.1.1.2.0.38 netbsd-10-base:1.1.1.2 netbsd-9-3-RELEASE:1.1.1.2 cjep_sun2x-base1:1.1.1.2 cjep_sun2x:1.1.1.2.0.36 cjep_sun2x-base:1.1.1.2 cjep_staticlib_x-base1:1.1.1.2 netbsd-9-2-RELEASE:1.1.1.2 cjep_staticlib_x:1.1.1.2.0.34 cjep_staticlib_x-base:1.1.1.2 netbsd-9-1-RELEASE:1.1.1.2 phil-wifi-20200421:1.1.1.2 phil-wifi-20200411:1.1.1.2 is-mlppp:1.1.1.2.0.32 is-mlppp-base:1.1.1.2 phil-wifi-20200406:1.1.1.2 netbsd-8-2-RELEASE:1.1.1.2 netbsd-9-0-RELEASE:1.1.1.2 netbsd-9-0-RC2:1.1.1.2 netbsd-9-0-RC1:1.1.1.2 phil-wifi-20191119:1.1.1.2 netbsd-9:1.1.1.2.0.30 netbsd-9-base:1.1.1.2 phil-wifi-20190609:1.1.1.2 netbsd-8-1-RELEASE:1.1.1.2 netbsd-8-1-RC1:1.1.1.2 pgoyette-compat-merge-20190127:1.1.1.2 pgoyette-compat-20190127:1.1.1.2 pgoyette-compat-20190118:1.1.1.2 pgoyette-compat-1226:1.1.1.2 pgoyette-compat-1126:1.1.1.2 pgoyette-compat-1020:1.1.1.2 pgoyette-compat-0930:1.1.1.2 pgoyette-compat-0906:1.1.1.2 netbsd-7-2-RELEASE:1.1.1.2 pgoyette-compat-0728:1.1.1.2 netbsd-8-0-RELEASE:1.1.1.2 phil-wifi:1.1.1.2.0.28 phil-wifi-base:1.1.1.2 pgoyette-compat-0625:1.1.1.2 netbsd-8-0-RC2:1.1.1.2 pgoyette-compat-0521:1.1.1.2 pgoyette-compat-0502:1.1.1.2 pgoyette-compat-0422:1.1.1.2 netbsd-8-0-RC1:1.1.1.2 pgoyette-compat-0415:1.1.1.2 pgoyette-compat-0407:1.1.1.2 pgoyette-compat-0330:1.1.1.2 pgoyette-compat-0322:1.1.1.2 pgoyette-compat-0315:1.1.1.2 netbsd-7-1-2-RELEASE:1.1.1.2 pgoyette-compat:1.1.1.2.0.26 pgoyette-compat-base:1.1.1.2 netbsd-7-1-1-RELEASE:1.1.1.2 matt-nb8-mediatek:1.1.1.2.0.24 matt-nb8-mediatek-base:1.1.1.2 perseant-stdc-iso10646:1.1.1.2.0.22 perseant-stdc-iso10646-base:1.1.1.2 netbsd-8:1.1.1.2.0.20 netbsd-8-base:1.1.1.2 prg-localcount2-base3:1.1.1.2 prg-localcount2-base2:1.1.1.2 prg-localcount2-base1:1.1.1.2 prg-localcount2:1.1.1.2.0.18 prg-localcount2-base:1.1.1.2 pgoyette-localcount-20170426:1.1.1.2 bouyer-socketcan-base1:1.1.1.2 pgoyette-localcount-20170320:1.1.1.2 netbsd-7-1:1.1.1.2.0.16 netbsd-7-1-RELEASE:1.1.1.2 netbsd-7-1-RC2:1.1.1.2 netbsd-7-nhusb-base-20170116:1.1.1.2 bouyer-socketcan:1.1.1.2.0.14 bouyer-socketcan-base:1.1.1.2 pgoyette-localcount-20170107:1.1.1.2 netbsd-7-1-RC1:1.1.1.2 pgoyette-localcount-20161104:1.1.1.2 netbsd-7-0-2-RELEASE:1.1.1.2 localcount-20160914:1.1.1.2 netbsd-7-nhusb:1.1.1.2.0.12 netbsd-7-nhusb-base:1.1.1.2 pgoyette-localcount-20160806:1.1.1.2 pgoyette-localcount-20160726:1.1.1.2 pgoyette-localcount:1.1.1.2.0.10 pgoyette-localcount-base:1.1.1.2 netbsd-7-0-1-RELEASE:1.1.1.2 netbsd-7-0:1.1.1.2.0.8 netbsd-7-0-RELEASE:1.1.1.2 netbsd-7-0-RC3:1.1.1.2 netbsd-7-0-RC2:1.1.1.2 netbsd-7-0-RC1:1.1.1.2 netbsd-7:1.1.1.2.0.6 netbsd-7-base:1.1.1.2 yamt-pagecache:1.1.1.2.0.4 yamt-pagecache-base9:1.1.1.2 tls-earlyentropy:1.1.1.2.0.2 tls-earlyentropy-base:1.1.1.2 riastradh-xf86-video-intel-2-7-1-pre-2-21-15:1.1.1.2 riastradh-drm2-base3:1.1.1.2 kyua-cli-0-7:1.1.1.2 riastradh-drm2-base2:1.1.1.1 riastradh-drm2-base1:1.1.1.1 riastradh-drm2:1.1.1.1.0.2 riastradh-drm2-base:1.1.1.1 khorben-n900:1.1.1.1.0.8 agc-symver:1.1.1.1.0.6 agc-symver-base:1.1.1.1 tls-maxphys-base:1.1.1.2 tls-maxphys:1.1.1.1.0.4 kyua-cli-0-6:1.1.1.1 GOOGLE-CODE:1.1.1; locks; strict; comment @// @; 1.1 date 2013.02.23.12.34.04; author jmmv; state Exp; branches 1.1.1.1; next ; 1.1.1.1 date 2013.02.23.12.34.04; author jmmv; state Exp; branches 1.1.1.1.4.1; next 1.1.1.2; 1.1.1.2 date 2013.10.18.23.40.14; author jmmv; state Exp; branches 1.1.1.2.4.1; next ; commitid 2U9LzNXaDSHnGP9x; 1.1.1.1.4.1 date 2013.02.23.12.34.04; author tls; state dead; branches; next 1.1.1.1.4.2; 1.1.1.1.4.2 date 2013.02.25.00.26.11; author tls; state Exp; branches; next 1.1.1.1.4.3; 1.1.1.1.4.3 date 2014.08.19.23.46.48; author tls; state Exp; branches; next ; commitid jTnpym9Qu0o4R1Nx; 1.1.1.2.4.1 date 2013.10.18.23.40.14; author yamt; state dead; branches; next 1.1.1.2.4.2; commitid h3iiOYJJCkPr7yBx; 1.1.1.2.4.2 date 2014.05.22.15.45.28; author yamt; state Exp; branches; next ; commitid h3iiOYJJCkPr7yBx; desc @@ 1.1 log @Initial revision @ text @// Copyright 2010 Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of Google Inc. nor the names of its contributors // may be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "engine/test_case.hpp" extern "C" { #include } #include #include "engine/config.hpp" #include "engine/exceptions.hpp" #include "engine/test_program.hpp" #include "engine/test_result.hpp" #include "engine/testers.hpp" #include "utils/config/tree.ipp" #include "utils/datetime.hpp" #include "utils/defs.hpp" #include "utils/format/macros.hpp" #include "utils/logging/operations.hpp" #include "utils/fs/auto_cleaners.hpp" #include "utils/fs/operations.hpp" #include "utils/fs/path.hpp" #include "utils/optional.ipp" #include "utils/passwd.hpp" #include "utils/text/operations.ipp" namespace config = utils::config; namespace fs = utils::fs; namespace logging = utils::logging; namespace passwd = utils::passwd; namespace text = utils::text; using utils::none; using utils::optional; namespace { /// Generates the set of configuration variables for the tester. /// /// \param metadata The metadata of the test. /// \param user_config The configuration variables provided by the user. /// \param test_suite The name of the test suite. /// /// \return The mapping of configuration variables for the tester. static config::properties_map generate_tester_config(const engine::metadata& metadata, const config::tree& user_config, const std::string& test_suite) { config::properties_map props; try { props = user_config.all_properties(F("test_suites.%s") % test_suite, true); } catch (const config::unknown_key_error& unused_error) { // Ignore: not all test suites have entries in the configuration. } if (user_config.is_set("unprivileged_user")) { const passwd::user& user = user_config.lookup< engine::user_node >("unprivileged_user"); props["unprivileged-user"] = user.name; } // TODO(jmmv): This is an ugly hack to cope with an atf-specific // property. We should not be doing this at all, so just consider this // a temporary optimization... if (metadata.has_cleanup()) props["has.cleanup"] = "true"; else props["has.cleanup"] = "false"; return props; } /// Creates a tester. /// /// \param interface_name The name of the tester interface to use. /// \param metadata Metadata of the test case. /// \param user_config User-provided configuration variables. /// /// \return The created tester, on which the test() method can be executed. static engine::tester create_tester(const std::string& interface_name, const engine::metadata& metadata, const config::tree& user_config) { optional< passwd::user > user; if (user_config.is_set("unprivileged_user") && metadata.required_user() == "unprivileged") user = user_config.lookup< engine::user_node >("unprivileged_user"); return engine::tester(interface_name, user, utils::make_optional(metadata.timeout())); } } // anonymous namespace /// Destructor. engine::test_case_hooks::~test_case_hooks(void) { } /// Called once the test case's stdout is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stdout. void engine::test_case_hooks::got_stdout(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Called once the test case's stderr is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stderr. void engine::test_case_hooks::got_stderr(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Internal implementation for a test_case. struct engine::test_case::impl { /// Name of the interface implemented by the test program. const std::string interface_name; /// Test program this test case belongs to. const test_program& _test_program; /// Name of the test case; must be unique within the test program. std::string name; /// Test case metadata. metadata md; /// Fake result to return instead of running the test case. optional< test_result > fake_result; /// Constructor. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name of the test case within the test program. /// \param md_ Metadata of the test case. /// \param fake_result_ Fake result to return instead of running the test /// case. impl(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_, const optional< test_result >& fake_result_) : interface_name(interface_name_), _test_program(test_program_), name(name_), md(md_), fake_result(fake_result_) { } /// Equality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool operator==(const impl& other) const { return (interface_name == other.interface_name && (_test_program.absolute_path() == other._test_program.absolute_path()) && name == other.name && md == other.md && fake_result == other.fake_result); } }; /// Constructs a new test case. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. This is a /// static reference (instead of a test_program_ptr) because the test /// program must exist in order for the test case to exist. /// \param name_ The name of the test case within the test program. Must be /// unique. /// \param md_ Metadata of the test case. engine::test_case::test_case(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_) : _pimpl(new impl(interface_name_, test_program_, name_, md_, none)) { } /// Constructs a new fake test case. /// /// A fake test case is a test case that is not really defined by the test /// program. Such test cases have a name surrounded by '__' and, when executed, /// they return a fixed, pre-recorded result. /// /// This is necessary for the cases where listing the test cases of a test /// program fails. In this scenario, we generate a single test case within /// the test program that unconditionally returns a failure. /// /// TODO(jmmv): Need to get rid of this. We should be able to report the /// status of test programs independently of test cases, as some interfaces /// don't know about the latter at all. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name to give to this fake test case. This name has to be /// prefixed and suffixed by '__' to clearly denote that this is internal. /// \param description_ The description of the test case, if any. /// \param test_result_ The fake result to return when this test case is run. engine::test_case::test_case( const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const std::string& description_, const engine::test_result& test_result_) : _pimpl(new impl(interface_name_, test_program_, name_, metadata_builder().set_description(description_).build(), utils::make_optional(test_result_))) { PRE_MSG(name_.length() > 4 && name_.substr(0, 2) == "__" && name_.substr(name_.length() - 2) == "__", "Invalid fake name provided to fake test case"); } /// Destroys a test case. engine::test_case::~test_case(void) { } /// Gets the name of the interface implemented by the test program. /// /// \return An interface name. const std::string& engine::test_case::interface_name(void) const { return _pimpl->interface_name; } /// Gets the test program this test case belongs to. /// /// \return A reference to the container test program. const engine::test_program& engine::test_case::container_test_program(void) const { return _pimpl->_test_program; } /// Gets the test case name. /// /// \return The test case name, relative to the test program. const std::string& engine::test_case::name(void) const { return _pimpl->name; } /// Gets the test case metadata. /// /// \return The test case metadata. const engine::metadata& engine::test_case::get_metadata(void) const { return _pimpl->md; } /// Gets the fake result pre-stored for this test case. /// /// \return A fake result, or none if not defined. optional< engine::test_result > engine::test_case::fake_result(void) const { return _pimpl->fake_result; } /// Equality comparator. /// /// \warning Because test cases reference their container test programs, and /// test programs include test cases, we cannot perform a full comparison here: /// otherwise, we'd enter an inifinte loop. Therefore, out of necessity, this /// does NOT compare whether the container test programs of the affected test /// cases are the same. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool engine::test_case::operator==(const test_case& other) const { return _pimpl == other._pimpl || *_pimpl == *other._pimpl; } /// Inequality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are different; false otherwise. bool engine::test_case::operator!=(const test_case& other) const { return !(*this == other); } /// Injects the object into a stream. /// /// \param output The stream into which to inject the object. /// \param object The object to format. /// /// \return The output stream. std::ostream& engine::operator<<(std::ostream& output, const test_case& object) { // We skip injecting container_test_program() on purpose to avoid a loop. output << F("test_case{interface=%s, name=%s, metadata=%s}") % text::quote(object.interface_name(), '\'') % text::quote(object.name(), '\'') % object.get_metadata(); return output; } /// Runs the test case in debug mode. /// /// Debug mode gives the caller more control on the execution of the test. It /// should not be used for normal execution of tests; instead, call run(). /// /// \param test_case The test case to debug. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// \param stdout_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stdout' is probably a reasonable value. /// \param stderr_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stderr' is probably a reasonable value. /// /// \return The result of the execution of the test case. engine::test_result engine::debug_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory, const fs::path& stdout_path, const fs::path& stderr_path) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_path, stderr_path, generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } /// Runs the test case. /// /// \param test_case The test case to run. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// /// \return The result of the execution of the test case. engine::test_result engine::run_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file stdout_file(work_directory / "stdout.txt"); const fs::auto_file stderr_file(work_directory / "stderr.txt"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_file.file(), stderr_file.file(), generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } @ 1.1.1.1 log @Initial import of Kyua CLI, version 0.6: This is the main component of Kyua and its build will be guarded by the MKKYUA knob. core@@ has approved this import. Description: Kyua (pronounced Q.A.) is a testing framework for both developers and users. Kyua is different from most other testing frameworks in that it puts the end user experience before anything else. There are multiple reasons for users to run the tests themselves, and Kyua ensures that they can do so in the most convenient way. This module, kyua-cli, provides the command-line interface to the Kyua runtime system. The major purpose of this tool is to run test cases and generate unified reports for their results. @ text @@ 1.1.1.2 log @Update kyua-cli to 0.7: The major reason for this update is to support the just-imported lutok-0.3 and, in turn, support the eventual update of the in-tree lua to 5.2. Changes: Experimental version released on October 18th, 2013. * Made failures from testers more resilent. If a tester fails, the corresponding test case will be marked as broken instead of causing kyua to exit. * Added the '--results-filter' option to the 'report-html' command and set its default value to skip passed results from HTML reports. This is to keep these reports more succint and to avoid generating tons of detail files that will be, in general, useless. * Switched to use Lutok 0.3 to gain compatibility with Lua 5.2. * Issue 69: Cope with the lack of AM_PROG_AR in configure.ac, which first appeared in Automake 1.11.2. Fixes a problem in Ubuntu 10.04 LTS, which appears stuck in 1.11.1. @ text @d421 12 a432 26 try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_path, stderr_path, generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } d470 12 a481 26 try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_file.file(), stderr_file.file(), generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } @ 1.1.1.2.4.1 log @file test_case.cpp was added on branch yamt-pagecache on 2014-05-22 15:45:28 +0000 @ text @d1 510 @ 1.1.1.2.4.2 log @sync with head. for a reference, the tree before this commit was tagged as yamt-pagecache-tag8. this commit was splitted into small chunks to avoid a limitation of cvs. ("Protocol error: too many arguments") @ text @a0 510 // Copyright 2010 Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of Google Inc. nor the names of its contributors // may be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "engine/test_case.hpp" extern "C" { #include } #include #include "engine/config.hpp" #include "engine/exceptions.hpp" #include "engine/test_program.hpp" #include "engine/test_result.hpp" #include "engine/testers.hpp" #include "utils/config/tree.ipp" #include "utils/datetime.hpp" #include "utils/defs.hpp" #include "utils/format/macros.hpp" #include "utils/logging/operations.hpp" #include "utils/fs/auto_cleaners.hpp" #include "utils/fs/operations.hpp" #include "utils/fs/path.hpp" #include "utils/optional.ipp" #include "utils/passwd.hpp" #include "utils/text/operations.ipp" namespace config = utils::config; namespace fs = utils::fs; namespace logging = utils::logging; namespace passwd = utils::passwd; namespace text = utils::text; using utils::none; using utils::optional; namespace { /// Generates the set of configuration variables for the tester. /// /// \param metadata The metadata of the test. /// \param user_config The configuration variables provided by the user. /// \param test_suite The name of the test suite. /// /// \return The mapping of configuration variables for the tester. static config::properties_map generate_tester_config(const engine::metadata& metadata, const config::tree& user_config, const std::string& test_suite) { config::properties_map props; try { props = user_config.all_properties(F("test_suites.%s") % test_suite, true); } catch (const config::unknown_key_error& unused_error) { // Ignore: not all test suites have entries in the configuration. } if (user_config.is_set("unprivileged_user")) { const passwd::user& user = user_config.lookup< engine::user_node >("unprivileged_user"); props["unprivileged-user"] = user.name; } // TODO(jmmv): This is an ugly hack to cope with an atf-specific // property. We should not be doing this at all, so just consider this // a temporary optimization... if (metadata.has_cleanup()) props["has.cleanup"] = "true"; else props["has.cleanup"] = "false"; return props; } /// Creates a tester. /// /// \param interface_name The name of the tester interface to use. /// \param metadata Metadata of the test case. /// \param user_config User-provided configuration variables. /// /// \return The created tester, on which the test() method can be executed. static engine::tester create_tester(const std::string& interface_name, const engine::metadata& metadata, const config::tree& user_config) { optional< passwd::user > user; if (user_config.is_set("unprivileged_user") && metadata.required_user() == "unprivileged") user = user_config.lookup< engine::user_node >("unprivileged_user"); return engine::tester(interface_name, user, utils::make_optional(metadata.timeout())); } } // anonymous namespace /// Destructor. engine::test_case_hooks::~test_case_hooks(void) { } /// Called once the test case's stdout is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stdout. void engine::test_case_hooks::got_stdout(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Called once the test case's stderr is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stderr. void engine::test_case_hooks::got_stderr(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Internal implementation for a test_case. struct engine::test_case::impl { /// Name of the interface implemented by the test program. const std::string interface_name; /// Test program this test case belongs to. const test_program& _test_program; /// Name of the test case; must be unique within the test program. std::string name; /// Test case metadata. metadata md; /// Fake result to return instead of running the test case. optional< test_result > fake_result; /// Constructor. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name of the test case within the test program. /// \param md_ Metadata of the test case. /// \param fake_result_ Fake result to return instead of running the test /// case. impl(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_, const optional< test_result >& fake_result_) : interface_name(interface_name_), _test_program(test_program_), name(name_), md(md_), fake_result(fake_result_) { } /// Equality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool operator==(const impl& other) const { return (interface_name == other.interface_name && (_test_program.absolute_path() == other._test_program.absolute_path()) && name == other.name && md == other.md && fake_result == other.fake_result); } }; /// Constructs a new test case. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. This is a /// static reference (instead of a test_program_ptr) because the test /// program must exist in order for the test case to exist. /// \param name_ The name of the test case within the test program. Must be /// unique. /// \param md_ Metadata of the test case. engine::test_case::test_case(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_) : _pimpl(new impl(interface_name_, test_program_, name_, md_, none)) { } /// Constructs a new fake test case. /// /// A fake test case is a test case that is not really defined by the test /// program. Such test cases have a name surrounded by '__' and, when executed, /// they return a fixed, pre-recorded result. /// /// This is necessary for the cases where listing the test cases of a test /// program fails. In this scenario, we generate a single test case within /// the test program that unconditionally returns a failure. /// /// TODO(jmmv): Need to get rid of this. We should be able to report the /// status of test programs independently of test cases, as some interfaces /// don't know about the latter at all. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name to give to this fake test case. This name has to be /// prefixed and suffixed by '__' to clearly denote that this is internal. /// \param description_ The description of the test case, if any. /// \param test_result_ The fake result to return when this test case is run. engine::test_case::test_case( const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const std::string& description_, const engine::test_result& test_result_) : _pimpl(new impl(interface_name_, test_program_, name_, metadata_builder().set_description(description_).build(), utils::make_optional(test_result_))) { PRE_MSG(name_.length() > 4 && name_.substr(0, 2) == "__" && name_.substr(name_.length() - 2) == "__", "Invalid fake name provided to fake test case"); } /// Destroys a test case. engine::test_case::~test_case(void) { } /// Gets the name of the interface implemented by the test program. /// /// \return An interface name. const std::string& engine::test_case::interface_name(void) const { return _pimpl->interface_name; } /// Gets the test program this test case belongs to. /// /// \return A reference to the container test program. const engine::test_program& engine::test_case::container_test_program(void) const { return _pimpl->_test_program; } /// Gets the test case name. /// /// \return The test case name, relative to the test program. const std::string& engine::test_case::name(void) const { return _pimpl->name; } /// Gets the test case metadata. /// /// \return The test case metadata. const engine::metadata& engine::test_case::get_metadata(void) const { return _pimpl->md; } /// Gets the fake result pre-stored for this test case. /// /// \return A fake result, or none if not defined. optional< engine::test_result > engine::test_case::fake_result(void) const { return _pimpl->fake_result; } /// Equality comparator. /// /// \warning Because test cases reference their container test programs, and /// test programs include test cases, we cannot perform a full comparison here: /// otherwise, we'd enter an inifinte loop. Therefore, out of necessity, this /// does NOT compare whether the container test programs of the affected test /// cases are the same. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool engine::test_case::operator==(const test_case& other) const { return _pimpl == other._pimpl || *_pimpl == *other._pimpl; } /// Inequality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are different; false otherwise. bool engine::test_case::operator!=(const test_case& other) const { return !(*this == other); } /// Injects the object into a stream. /// /// \param output The stream into which to inject the object. /// \param object The object to format. /// /// \return The output stream. std::ostream& engine::operator<<(std::ostream& output, const test_case& object) { // We skip injecting container_test_program() on purpose to avoid a loop. output << F("test_case{interface=%s, name=%s, metadata=%s}") % text::quote(object.interface_name(), '\'') % text::quote(object.name(), '\'') % object.get_metadata(); return output; } /// Runs the test case in debug mode. /// /// Debug mode gives the caller more control on the execution of the test. It /// should not be used for normal execution of tests; instead, call run(). /// /// \param test_case The test case to debug. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// \param stdout_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stdout' is probably a reasonable value. /// \param stderr_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stderr' is probably a reasonable value. /// /// \return The result of the execution of the test case. engine::test_result engine::debug_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory, const fs::path& stdout_path, const fs::path& stderr_path) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_path, stderr_path, generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } } /// Runs the test case. /// /// \param test_case The test case to run. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// /// \return The result of the execution of the test case. engine::test_result engine::run_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file stdout_file(work_directory / "stdout.txt"); const fs::auto_file stderr_file(work_directory / "stderr.txt"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_file.file(), stderr_file.file(), generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } } @ 1.1.1.1.4.1 log @file test_case.cpp was added on branch tls-maxphys on 2013-02-25 00:26:11 +0000 @ text @d1 482 @ 1.1.1.1.4.2 log @resync with head @ text @a0 482 // Copyright 2010 Google Inc. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // * Neither the name of Google Inc. nor the names of its contributors // may be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "engine/test_case.hpp" extern "C" { #include } #include #include "engine/config.hpp" #include "engine/exceptions.hpp" #include "engine/test_program.hpp" #include "engine/test_result.hpp" #include "engine/testers.hpp" #include "utils/config/tree.ipp" #include "utils/datetime.hpp" #include "utils/defs.hpp" #include "utils/format/macros.hpp" #include "utils/logging/operations.hpp" #include "utils/fs/auto_cleaners.hpp" #include "utils/fs/operations.hpp" #include "utils/fs/path.hpp" #include "utils/optional.ipp" #include "utils/passwd.hpp" #include "utils/text/operations.ipp" namespace config = utils::config; namespace fs = utils::fs; namespace logging = utils::logging; namespace passwd = utils::passwd; namespace text = utils::text; using utils::none; using utils::optional; namespace { /// Generates the set of configuration variables for the tester. /// /// \param metadata The metadata of the test. /// \param user_config The configuration variables provided by the user. /// \param test_suite The name of the test suite. /// /// \return The mapping of configuration variables for the tester. static config::properties_map generate_tester_config(const engine::metadata& metadata, const config::tree& user_config, const std::string& test_suite) { config::properties_map props; try { props = user_config.all_properties(F("test_suites.%s") % test_suite, true); } catch (const config::unknown_key_error& unused_error) { // Ignore: not all test suites have entries in the configuration. } if (user_config.is_set("unprivileged_user")) { const passwd::user& user = user_config.lookup< engine::user_node >("unprivileged_user"); props["unprivileged-user"] = user.name; } // TODO(jmmv): This is an ugly hack to cope with an atf-specific // property. We should not be doing this at all, so just consider this // a temporary optimization... if (metadata.has_cleanup()) props["has.cleanup"] = "true"; else props["has.cleanup"] = "false"; return props; } /// Creates a tester. /// /// \param interface_name The name of the tester interface to use. /// \param metadata Metadata of the test case. /// \param user_config User-provided configuration variables. /// /// \return The created tester, on which the test() method can be executed. static engine::tester create_tester(const std::string& interface_name, const engine::metadata& metadata, const config::tree& user_config) { optional< passwd::user > user; if (user_config.is_set("unprivileged_user") && metadata.required_user() == "unprivileged") user = user_config.lookup< engine::user_node >("unprivileged_user"); return engine::tester(interface_name, user, utils::make_optional(metadata.timeout())); } } // anonymous namespace /// Destructor. engine::test_case_hooks::~test_case_hooks(void) { } /// Called once the test case's stdout is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stdout. void engine::test_case_hooks::got_stdout(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Called once the test case's stderr is ready for processing. /// /// It is important to note that this file is only available within this /// callback. Attempting to read the file once the execute function has /// returned will result in an error because the file might have been deleted. /// /// \param unused_file The path to the file containing the stderr. void engine::test_case_hooks::got_stderr(const fs::path& UTILS_UNUSED_PARAM(file)) { } /// Internal implementation for a test_case. struct engine::test_case::impl { /// Name of the interface implemented by the test program. const std::string interface_name; /// Test program this test case belongs to. const test_program& _test_program; /// Name of the test case; must be unique within the test program. std::string name; /// Test case metadata. metadata md; /// Fake result to return instead of running the test case. optional< test_result > fake_result; /// Constructor. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name of the test case within the test program. /// \param md_ Metadata of the test case. /// \param fake_result_ Fake result to return instead of running the test /// case. impl(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_, const optional< test_result >& fake_result_) : interface_name(interface_name_), _test_program(test_program_), name(name_), md(md_), fake_result(fake_result_) { } /// Equality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool operator==(const impl& other) const { return (interface_name == other.interface_name && (_test_program.absolute_path() == other._test_program.absolute_path()) && name == other.name && md == other.md && fake_result == other.fake_result); } }; /// Constructs a new test case. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. This is a /// static reference (instead of a test_program_ptr) because the test /// program must exist in order for the test case to exist. /// \param name_ The name of the test case within the test program. Must be /// unique. /// \param md_ Metadata of the test case. engine::test_case::test_case(const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const metadata& md_) : _pimpl(new impl(interface_name_, test_program_, name_, md_, none)) { } /// Constructs a new fake test case. /// /// A fake test case is a test case that is not really defined by the test /// program. Such test cases have a name surrounded by '__' and, when executed, /// they return a fixed, pre-recorded result. /// /// This is necessary for the cases where listing the test cases of a test /// program fails. In this scenario, we generate a single test case within /// the test program that unconditionally returns a failure. /// /// TODO(jmmv): Need to get rid of this. We should be able to report the /// status of test programs independently of test cases, as some interfaces /// don't know about the latter at all. /// /// \param interface_name_ Name of the interface implemented by the test /// program. /// \param test_program_ The test program this test case belongs to. /// \param name_ The name to give to this fake test case. This name has to be /// prefixed and suffixed by '__' to clearly denote that this is internal. /// \param description_ The description of the test case, if any. /// \param test_result_ The fake result to return when this test case is run. engine::test_case::test_case( const std::string& interface_name_, const test_program& test_program_, const std::string& name_, const std::string& description_, const engine::test_result& test_result_) : _pimpl(new impl(interface_name_, test_program_, name_, metadata_builder().set_description(description_).build(), utils::make_optional(test_result_))) { PRE_MSG(name_.length() > 4 && name_.substr(0, 2) == "__" && name_.substr(name_.length() - 2) == "__", "Invalid fake name provided to fake test case"); } /// Destroys a test case. engine::test_case::~test_case(void) { } /// Gets the name of the interface implemented by the test program. /// /// \return An interface name. const std::string& engine::test_case::interface_name(void) const { return _pimpl->interface_name; } /// Gets the test program this test case belongs to. /// /// \return A reference to the container test program. const engine::test_program& engine::test_case::container_test_program(void) const { return _pimpl->_test_program; } /// Gets the test case name. /// /// \return The test case name, relative to the test program. const std::string& engine::test_case::name(void) const { return _pimpl->name; } /// Gets the test case metadata. /// /// \return The test case metadata. const engine::metadata& engine::test_case::get_metadata(void) const { return _pimpl->md; } /// Gets the fake result pre-stored for this test case. /// /// \return A fake result, or none if not defined. optional< engine::test_result > engine::test_case::fake_result(void) const { return _pimpl->fake_result; } /// Equality comparator. /// /// \warning Because test cases reference their container test programs, and /// test programs include test cases, we cannot perform a full comparison here: /// otherwise, we'd enter an inifinte loop. Therefore, out of necessity, this /// does NOT compare whether the container test programs of the affected test /// cases are the same. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are equal; false otherwise. bool engine::test_case::operator==(const test_case& other) const { return _pimpl == other._pimpl || *_pimpl == *other._pimpl; } /// Inequality comparator. /// /// \param other The other object to compare this one to. /// /// \return True if this object and other are different; false otherwise. bool engine::test_case::operator!=(const test_case& other) const { return !(*this == other); } /// Injects the object into a stream. /// /// \param output The stream into which to inject the object. /// \param object The object to format. /// /// \return The output stream. std::ostream& engine::operator<<(std::ostream& output, const test_case& object) { // We skip injecting container_test_program() on purpose to avoid a loop. output << F("test_case{interface=%s, name=%s, metadata=%s}") % text::quote(object.interface_name(), '\'') % text::quote(object.name(), '\'') % object.get_metadata(); return output; } /// Runs the test case in debug mode. /// /// Debug mode gives the caller more control on the execution of the test. It /// should not be used for normal execution of tests; instead, call run(). /// /// \param test_case The test case to debug. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// \param stdout_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stdout' is probably a reasonable value. /// \param stderr_path The file to which to redirect the stdout of the test. /// For interactive debugging, '/dev/stderr' is probably a reasonable value. /// /// \return The result of the execution of the test case. engine::test_result engine::debug_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory, const fs::path& stdout_path, const fs::path& stderr_path) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_path, stderr_path, generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } /// Runs the test case. /// /// \param test_case The test case to run. /// \param user_config The user configuration that defines the execution of this /// test case. /// \param hooks Hooks to introspect the execution of the test case. /// \param work_directory A directory that can be used to place temporary files. /// /// \return The result of the execution of the test case. engine::test_result engine::run_test_case(const test_case* test_case, const config::tree& user_config, test_case_hooks& hooks, const fs::path& work_directory) { if (test_case->fake_result()) return test_case->fake_result().get(); const std::string skip_reason = check_reqs( test_case->get_metadata(), user_config, test_case->container_test_program().test_suite_name()); if (!skip_reason.empty()) return test_result(test_result::skipped, skip_reason); if (!fs::exists(test_case->container_test_program().absolute_path())) return test_result(test_result::broken, "Test program does not exist"); const fs::auto_file stdout_file(work_directory / "stdout.txt"); const fs::auto_file stderr_file(work_directory / "stderr.txt"); const fs::auto_file result_file(work_directory / "result.txt"); const engine::test_program& test_program = test_case->container_test_program(); const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_file.file(), stderr_file.file(), generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } @ 1.1.1.1.4.3 log @Rebase to HEAD as of a few days ago. @ text @d421 12 a432 26 try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_path, stderr_path, generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_path); hooks.got_stderr(stderr_path); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } d470 12 a481 26 try { const engine::tester tester = create_tester( test_program.interface_name(), test_case->get_metadata(), user_config); tester.test(test_program.absolute_path(), test_case->name(), result_file.file(), stdout_file.file(), stderr_file.file(), generate_tester_config(test_case->get_metadata(), user_config, test_program.test_suite_name())); hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); std::ifstream result_input(result_file.file().c_str()); return engine::test_result::parse(result_input); } catch (const std::runtime_error& e) { // One of the possible explanation for us getting here is if the tester // crashes or doesn't behave as expected. We must record any output // from the process so that we can debug it further. hooks.got_stdout(stdout_file.file()); hooks.got_stderr(stderr_file.file()); return engine::test_result( engine::test_result::broken, F("Caught unexpected exception: %s") % e.what()); } @