#include "test_manager.h" #include #include #include #include #include #include #include "absl/container/flat_hash_set.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" #include "google/protobuf/stubs/status_macros.h" namespace google { namespace protobuf { namespace conformance { namespace internal { namespace { constexpr int kMaximumWildcardExpansions = 20; constexpr int kFailureMessageLengthLimit = 128; bool InsertUniqueTest(absl::flat_hash_set& set, absl::string_view value) { if (!set.emplace(value).second) { return false; } return true; } void IncrementIfUnique(bool unique, int& counter) { if (unique) { ++counter; } } void Normalize(std::string& input) { input.erase(std::remove(input.begin(), input.end(), '\n'), input.end()); } // Sets up a failure message properly for our failure lists. std::string FormatFailureMessage(absl::string_view input) { std::string result(absl::StripAsciiWhitespace(input)); // Remove newlines Normalize(result); // Truncate failure message if needed if (result.length() > kFailureMessageLengthLimit) { result = result.substr(0, kFailureMessageLengthLimit); } return result; } std::string ReformatLine(size_t alignment, absl::string_view line) { size_t comment_pos = line.find('#'); absl::string_view test_name = absl::StripAsciiWhitespace(line.substr(0, comment_pos)); if (test_name.empty()) { return std::string(absl::StripAsciiWhitespace(line)); } absl::string_view message = absl::StripAsciiWhitespace(line.substr(comment_pos + 1)); ABSL_CHECK_GE(alignment, test_name.length()); std::string whitespace(alignment - test_name.length(), ' '); return absl::StrCat(test_name, whitespace, " # ", message); } } // namespace TestManager::~TestManager() { if (!finalized_) { ABSL_LOG(FATAL) << "TestManager::Finalize() was not called before destruction."; } } absl::Status TestManager::LoadFailureList(absl::string_view filename) { std::ifstream infile(std::string{filename}); if (!infile.is_open()) { return absl::InternalError( absl::StrCat("Couldn't open failure list file: ", filename)); } for (std::string line; std::getline(infile, line);) { failure_list_lines_.push_back(line); // Remove comments. std::string test_name = line.substr(0, line.find('#')); test_name.erase( std::remove_if(test_name.begin(), test_name.end(), ::isspace), test_name.end()); if (test_name.empty()) { // Skip empty lines. continue; } // If we remove whitespace from the beginning of a line, and what we have // left at first is a '#', then we have a comment. if (test_name[0] != '#') { // Find our failure message if it exists. Will be set to an empty string // if no message is found. Empty failure messages also pass our tests. size_t comment_pos = line.find('#'); std::string message; if (comment_pos != std::string::npos) { message = line.substr(comment_pos + 1); // +1 to skip the delimiter // If we had only whitespace after the delimiter, we will have an empty // failure message and the test will still pass. message = std::string(absl::StripAsciiWhitespace(message)); } RETURN_IF_ERROR(expected_failure_list_.Insert(test_name)); ABSL_CHECK(expected_failure_messages_.emplace(test_name, message).second); ABSL_CHECK(unseen_expected_failures_.emplace(test_name).second); } } return absl::OkStatus(); } absl::Status TestManager::SaveFailureList(absl::string_view filename) const { std::ofstream outfile(std::string{filename}); // Calculate alignment. size_t alignment = 0; for (const std::string& line : failure_list_lines_) { absl::string_view test_name = absl::StripAsciiWhitespace( absl::string_view(line).substr(0, line.find('#'))); alignment = std::max(alignment, test_name.length()); } for (const auto& failure : new_failures_) { alignment = std::max(alignment, failure.first.length()); } // Output existing failure list, stripping out unseen tests and inserting // new failures. auto to_add = new_failures_.begin(); for (const std::string& line : failure_list_lines_) { absl::string_view test_name = absl::StripAsciiWhitespace( absl::string_view(line).substr(0, line.find('#'))); if (unseen_expected_failures_.contains(test_name) || seen_unexpected_successes_.contains(test_name)) { continue; } while (to_add != new_failures_.end() && test_name > to_add->first) { outfile << ReformatLine(alignment, absl::StrCat(to_add->first, " # ", to_add->second)) << "\n"; ++to_add; } outfile << ReformatLine(alignment, line) << "\n"; } // Add any remaining new failures. while (to_add != new_failures_.end()) { outfile << ReformatLine(alignment, absl::StrCat(to_add->first, " # ", to_add->second)) << "\n"; ++to_add; } return absl::OkStatus(); } absl::Status TestManager::ReportSuccess(absl::string_view test_name) { bool unique = InsertUniqueTest(seen_tests_, test_name); auto failure_match = expected_failure_list_.WalkDownMatch(test_name); if (failure_match.has_value()) { // This was expected to fail, but it succeeded. IncrementIfUnique(unique, number_of_matches_[*failure_match]); IncrementIfUnique(unique, unexpected_successes_); unseen_expected_failures_.erase(*failure_match); seen_unexpected_successes_.insert(*failure_match); return absl::FailedPreconditionError( absl::StrCat("Unexpected success for test: ", test_name)); } // This wasn't expected to fail. IncrementIfUnique(unique, expected_successes_); return absl::OkStatus(); } absl::Status TestManager::ReportFailure(absl::string_view test_name, absl::string_view failure_message) { bool unique = InsertUniqueTest(seen_tests_, test_name); auto failure_match = expected_failure_list_.WalkDownMatch(test_name); std::string formatted_failure_message = FormatFailureMessage(failure_message); if (!failure_match.has_value()) { // This was not expected to fail. IncrementIfUnique(unique, unexpected_failures_); new_failures_[test_name] = formatted_failure_message; return absl::FailedPreconditionError( absl::StrCat("Unexpected failure for test: ", test_name)); } if (expected_failure_messages_[*failure_match] != formatted_failure_message) { IncrementIfUnique(unique, unexpected_failures_); new_failures_[*failure_match] = formatted_failure_message; return absl::FailedPreconditionError( absl::StrCat("Unexpected failure message for test: ", test_name, " expected: ", expected_failure_messages_[*failure_match], " actual: ", formatted_failure_message)); } unseen_expected_failures_.erase(*failure_match); if (number_of_matches_[*failure_match] > kMaximumWildcardExpansions) { IncrementIfUnique(unique, unexpected_failures_); return absl::FailedPreconditionError( absl::StrCat("The wildcard ", *failure_match, " served as matches to too many test " "names exceeding the max amount of ", kMaximumWildcardExpansions, " for test: ", test_name)); } IncrementIfUnique(unique, number_of_matches_[*failure_match]); IncrementIfUnique(unique, expected_failures_); return absl::OkStatus(); } absl::Status TestManager::ReportSkip(absl::string_view test_name) { bool unique = InsertUniqueTest(seen_tests_, test_name); IncrementIfUnique(unique, skipped_); return absl::OkStatus(); } absl::Status TestManager::Finalize() { finalized_ = true; if (!unseen_expected_failures_.empty()) { return absl::FailedPreconditionError( absl::StrCat("The following expected failures were not seen: ", absl::StrJoin(unseen_expected_failures_, ", "))); } return absl::OkStatus(); } } // namespace internal } // namespace conformance } // namespace protobuf } // namespace google