caixiang
/
PX4-Autopilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2 Commits
1 Branch
0 Tags
502 MiB
 
 
 
 
 
 
Branch: 7-nano-minimal-vtol
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '7-nano-minimal-vtol'
${ noResults }
PX4-Autopilot/test/fuzztest/centipede/stats_test.cc

877 lines
34 KiB
Raw Permalink Blame History

// Copyright 2022 The Centipede Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "./centipede/stats.h"
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <filesystem> // NOLINT
#include <sstream>
#include <string>
#include <string_view>
#include <vector>
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "absl/log/check.h"
#include "absl/log/log_entry.h"
#include "absl/log/log_sink.h"
#include "absl/log/log_sink_registry.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_split.h"
#include "absl/strings/string_view.h"
#include "absl/time/civil_time.h"
#include "absl/time/time.h"
#include "./centipede/environment.h"
#include "./centipede/util.h"
#include "./common/logging.h" // IWYU pragma: keep
#include "./common/test_util.h"
namespace fuzztest::internal {
namespace {
using ::testing::ElementsAreArray;
namespace {
class LogCapture : public absl::LogSink {
public:
LogCapture() { absl::AddLogSink(this); }
~LogCapture() override { absl::RemoveLogSink(this); }
void Send(const absl::LogEntry &entry) override {
captured_log_.emplace_back(entry.text_message());
}
std::vector<std::string> CapturedLogLines() const {
// Join->Split normalizes multi-line messages.
return absl::StrSplit(absl::StrJoin(captured_log_, "\n"), '\n');
}
private:
std::vector<std::string> captured_log_;
};
uint64_t CivilTimeToUnixMicros( //
int64_t y, int64_t m, int64_t d, int64_t hh, int64_t mm, int64_t ss) {
return absl::ToUnixMicros(absl::FromCivil(
absl::CivilSecond{y, m, d, hh, mm, ss}, absl::LocalTimeZone()));
}
Environment CreateEnv(std::string_view workdir,
std::string_view experiment_name = "",
std::string_view experiment_flags = "") {
Environment env;
env.workdir = std::string(workdir);
env.experiment_name = std::string(experiment_name);
env.experiment_flags = std::string(experiment_flags);
return env;
}
} // namespace
TEST(Stats, PrintStatsToLog) {
std::vector<std::atomic<Stats>> stats_vec(4);
for (uint64_t i = 0; i < stats_vec.size(); ++i) {
const uint64_t j = i + 1;
CovStats cov_stats = {/*num_covered_pcs=*/21 * j,
/*num_8bit_counter_features=*/22 * j,
/*num_data_flow_features=*/23 * j,
/*num_cmp_features=*/24 * j,
/*num_call_stack_features=*/25 * j,
/*num_bounded_path_features=*/26 * j,
/*num_pc_pair_features=*/27 * j,
/*num_user_features=*/28 * j};
// NOTE: These two don't add up to `num_user_features`, but that's
// ok for the purposes of this test.
cov_stats.num_user0_features = 28 * j + 1;
cov_stats.num_user10_features = 28 * j + 2;
cov_stats.num_unknown_features = 29 * j;
cov_stats.num_funcs_in_frontier = 31 * j;
stats_vec[i].store(Stats{
StatsMeta{/*timestamp_unix_micros=*/CivilTimeToUnixMicros(1970, 1, 1, 0,
0, i)},
ExecStats{
/*fuzz_time_sec=*/10 * j,
/*num_executions=*/12 * j,
/*num_target_crashes=*/13 * j,
},
cov_stats,
CorpusStats{
/*active_corpus_size=*/101 * j,
/*total_corpus_size=*/102 * j,
/*max_corpus_element_size=*/103 * j,
/*avg_corpus_element_size=*/104 * j,
},
RusageStats{
/*engine_rusage_avg_millicores=*/201 * j,
/*engine_rusage_cpu_percent=*/202 * j,
/*engine_rusage_rss_mb=*/203 * j,
/*engine_rusage_vsize_mb=*/204 * j,
},
});
}
const std::vector<Environment> env_vec = {
CreateEnv(
/*workdir=*/"", /*experiment_name=*/"Experiment A",
/*experiment_flags=*/"AAA"),
CreateEnv(/*workdir=*/"", /*experiment_name=*/"Experiment B",
/*experiment_flags=*/"BBB"),
CreateEnv(
/*workdir=*/"", /*experiment_name=*/"Experiment A",
/*experiment_flags=*/"AAA"),
CreateEnv(/*workdir=*/"", /*experiment_name=*/"Experiment B",
/*experiment_flags=*/"BBB")};
StatsLogger stats_logger{stats_vec, env_vec};
{
const std::vector<std::string_view> kExpectedLogLines = {
"Current stats:",
"Coverage:",
" Experiment A: min:\t21\tmax:\t63\tavg:\t42.0\t--\t21\t63",
" Experiment B: min:\t42\tmax:\t84\tavg:\t63.0\t--\t42\t84",
"Number of executions:",
" Experiment A: min:\t12\tmax:\t36\tavg:\t24.0\t--\t12\t36",
" Experiment B: min:\t24\tmax:\t48\tavg:\t36.0\t--\t24\t48",
"Active corpus size:",
" Experiment A: min:\t101\tmax:\t303\tavg:\t202.0\t--\t101\t303",
" Experiment B: min:\t202\tmax:\t404\tavg:\t303.0\t--\t202\t404",
"Max element size:",
" Experiment A: min:\t103\tmax:\t309\tavg:\t206.0\t--\t103\t309",
" Experiment B: min:\t206\tmax:\t412\tavg:\t309.0\t--\t206\t412",
"Avg element size:",
" Experiment A: min:\t104\tmax:\t312\tavg:\t208.0\t--\t104\t312",
" Experiment B: min:\t208\tmax:\t416\tavg:\t312.0\t--\t208\t416",
"Fuzz time (sec):",
" Experiment A: min:\t10\tmax:\t30\tavg:\t20.0\t--\t10\t30",
" Experiment B: min:\t20\tmax:\t40\tavg:\t30.0\t--\t20\t40",
"Num proxy crashes:",
" Experiment A: min:\t13\tmax:\t39\tsum:\t52\t--\t13\t39",
" Experiment B: min:\t26\tmax:\t52\tsum:\t78\t--\t26\t52",
"Total corpus size:",
" Experiment A: min:\t102\tmax:\t306\tsum:\t408\t--\t102\t306",
" Experiment B: min:\t204\tmax:\t408\tsum:\t612\t--\t204\t408",
"Num 8-bit counter features:",
" Experiment A: min:\t22\tmax:\t66\tavg:\t44.0\t--\t22\t66",
" Experiment B: min:\t44\tmax:\t88\tavg:\t66.0\t--\t44\t88",
"Num data flow features:",
" Experiment A: min:\t23\tmax:\t69\tavg:\t46.0\t--\t23\t69",
" Experiment B: min:\t46\tmax:\t92\tavg:\t69.0\t--\t46\t92",
"Num cmp features:",
" Experiment A: min:\t24\tmax:\t72\tavg:\t48.0\t--\t24\t72",
" Experiment B: min:\t48\tmax:\t96\tavg:\t72.0\t--\t48\t96",
"Num call stack features:",
" Experiment A: min:\t25\tmax:\t75\tavg:\t50.0\t--\t25\t75",
" Experiment B: min:\t50\tmax:\t100\tavg:\t75.0\t--\t50\t100",
"Num bounded path features:",
" Experiment A: min:\t26\tmax:\t78\tavg:\t52.0\t--\t26\t78",
" Experiment B: min:\t52\tmax:\t104\tavg:\t78.0\t--\t52\t104",
"Num PC pair features:",
" Experiment A: min:\t27\tmax:\t81\tavg:\t54.0\t--\t27\t81",
" Experiment B: min:\t54\tmax:\t108\tavg:\t81.0\t--\t54\t108",
"Num user features:",
" Experiment A: min:\t28\tmax:\t84\tavg:\t56.0\t--\t28\t84",
" Experiment B: min:\t56\tmax:\t112\tavg:\t84.0\t--\t56\t112",
"Num unknown features:",
" Experiment A: min:\t29\tmax:\t87\tavg:\t58.0\t--\t29\t87",
" Experiment B: min:\t58\tmax:\t116\tavg:\t87.0\t--\t58\t116",
"Num funcs in frontier:",
" Experiment A: min:\t31\tmax:\t93\tavg:\t62.0\t--\t31\t93",
" Experiment B: min:\t62\tmax:\t124\tavg:\t93.0\t--\t62\t124",
"Num user0 features:",
" Experiment A: min:\t29\tmax:\t85\tavg:\t57.0\t--\t29\t85",
" Experiment B: min:\t57\tmax:\t113\tavg:\t85.0\t--\t57\t113",
"Num user1 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user2 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user3 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user4 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user5 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user6 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user7 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user8 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user9 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user10 features:",
" Experiment A: min:\t30\tmax:\t86\tavg:\t58.0\t--\t30\t86",
" Experiment B: min:\t58\tmax:\t114\tavg:\t86.0\t--\t58\t114",
"Num user11 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user12 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user13 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user14 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user15 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Flags:",
" Experiment A: AAA",
" Experiment B: BBB",
};
LogCapture log_capture;
stats_logger.ReportCurrStats();
const auto log_lines = log_capture.CapturedLogLines();
EXPECT_THAT(log_lines, ElementsAreArray(kExpectedLogLines))
<< "\nActual:" << absl::StrJoin(log_lines, "\n") << "\nExpected:\n"
<< absl::StrJoin(kExpectedLogLines, "\n");
}
{
for (auto &stats : stats_vec) {
auto new_stats = stats.load();
new_stats.timestamp_unix_micros += 1000000;
new_stats.fuzz_time_sec += 1;
new_stats.num_executions += 1;
new_stats.num_target_crashes += 1;
new_stats.num_covered_pcs += 1;
new_stats.num_8bit_counter_features += 1;
new_stats.num_data_flow_features += 1;
new_stats.num_cmp_features += 1;
new_stats.num_call_stack_features += 1;
new_stats.num_bounded_path_features += 1;
new_stats.num_pc_pair_features += 1;
new_stats.num_user_features += 1;
new_stats.num_user0_features += 1;
new_stats.num_user10_features += 1;
new_stats.num_unknown_features += 1;
new_stats.num_funcs_in_frontier += 1;
new_stats.active_corpus_size += 1;
new_stats.total_corpus_size += 1;
new_stats.max_corpus_element_size += 1;
new_stats.avg_corpus_element_size += 1;
new_stats.engine_rusage_avg_millicores += 1;
new_stats.engine_rusage_cpu_percent += 1;
new_stats.engine_rusage_rss_mb += 1;
new_stats.engine_rusage_vsize_mb += 1;
stats.store(new_stats);
}
const std::vector<std::string_view> kExpectedLogLines = {
"Current stats:",
"Coverage:",
" Experiment A: min:\t22\tmax:\t64\tavg:\t43.0\t--\t22\t64",
" Experiment B: min:\t43\tmax:\t85\tavg:\t64.0\t--\t43\t85",
"Number of executions:",
" Experiment A: min:\t13\tmax:\t37\tavg:\t25.0\t--\t13\t37",
" Experiment B: min:\t25\tmax:\t49\tavg:\t37.0\t--\t25\t49",
"Active corpus size:",
" Experiment A: min:\t102\tmax:\t304\tavg:\t203.0\t--\t102\t304",
" Experiment B: min:\t203\tmax:\t405\tavg:\t304.0\t--\t203\t405",
"Max element size:",
" Experiment A: min:\t104\tmax:\t310\tavg:\t207.0\t--\t104\t310",
" Experiment B: min:\t207\tmax:\t413\tavg:\t310.0\t--\t207\t413",
"Avg element size:",
" Experiment A: min:\t105\tmax:\t313\tavg:\t209.0\t--\t105\t313",
" Experiment B: min:\t209\tmax:\t417\tavg:\t313.0\t--\t209\t417",
"Fuzz time (sec):",
" Experiment A: min:\t11\tmax:\t31\tavg:\t21.0\t--\t11\t31",
" Experiment B: min:\t21\tmax:\t41\tavg:\t31.0\t--\t21\t41",
"Num proxy crashes:",
" Experiment A: min:\t14\tmax:\t40\tsum:\t54\t--\t14\t40",
" Experiment B: min:\t27\tmax:\t53\tsum:\t80\t--\t27\t53",
"Total corpus size:",
" Experiment A: min:\t103\tmax:\t307\tsum:\t410\t--\t103\t307",
" Experiment B: min:\t205\tmax:\t409\tsum:\t614\t--\t205\t409",
"Num 8-bit counter features:",
" Experiment A: min:\t23\tmax:\t67\tavg:\t45.0\t--\t23\t67",
" Experiment B: min:\t45\tmax:\t89\tavg:\t67.0\t--\t45\t89",
"Num data flow features:",
" Experiment A: min:\t24\tmax:\t70\tavg:\t47.0\t--\t24\t70",
" Experiment B: min:\t47\tmax:\t93\tavg:\t70.0\t--\t47\t93",
"Num cmp features:",
" Experiment A: min:\t25\tmax:\t73\tavg:\t49.0\t--\t25\t73",
" Experiment B: min:\t49\tmax:\t97\tavg:\t73.0\t--\t49\t97",
"Num call stack features:",
" Experiment A: min:\t26\tmax:\t76\tavg:\t51.0\t--\t26\t76",
" Experiment B: min:\t51\tmax:\t101\tavg:\t76.0\t--\t51\t101",
"Num bounded path features:",
" Experiment A: min:\t27\tmax:\t79\tavg:\t53.0\t--\t27\t79",
" Experiment B: min:\t53\tmax:\t105\tavg:\t79.0\t--\t53\t105",
"Num PC pair features:",
" Experiment A: min:\t28\tmax:\t82\tavg:\t55.0\t--\t28\t82",
" Experiment B: min:\t55\tmax:\t109\tavg:\t82.0\t--\t55\t109",
"Num user features:",
" Experiment A: min:\t29\tmax:\t85\tavg:\t57.0\t--\t29\t85",
" Experiment B: min:\t57\tmax:\t113\tavg:\t85.0\t--\t57\t113",
"Num unknown features:",
" Experiment A: min:\t30\tmax:\t88\tavg:\t59.0\t--\t30\t88",
" Experiment B: min:\t59\tmax:\t117\tavg:\t88.0\t--\t59\t117",
"Num funcs in frontier:",
" Experiment A: min:\t32\tmax:\t94\tavg:\t63.0\t--\t32\t94",
" Experiment B: min:\t63\tmax:\t125\tavg:\t94.0\t--\t63\t125",
"Num user0 features:",
" Experiment A: min:\t30\tmax:\t86\tavg:\t58.0\t--\t30\t86",
" Experiment B: min:\t58\tmax:\t114\tavg:\t86.0\t--\t58\t114",
"Num user1 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user2 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user3 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user4 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user5 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user6 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user7 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user8 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user9 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user10 features:",
" Experiment A: min:\t31\tmax:\t87\tavg:\t59.0\t--\t31\t87",
" Experiment B: min:\t59\tmax:\t115\tavg:\t87.0\t--\t59\t115",
"Num user11 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user12 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user13 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user14 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Num user15 features:",
" Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
" Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0",
"Flags:",
" Experiment A: AAA",
" Experiment B: BBB",
};
LogCapture log_capture;
stats_logger.ReportCurrStats();
const auto log_lines = log_capture.CapturedLogLines();
EXPECT_THAT(log_lines, ElementsAreArray(kExpectedLogLines));
}
}
TEST(Stats, DumpStatsToCsvFile) {
const std::filesystem::path workdir = GetTestTempDir(test_info_->name());
std::vector<std::atomic<Stats>> stats_vec(4);
for (uint64_t i = 0; i < stats_vec.size(); ++i) {
const uint64_t j = i + 1;
CovStats cov_stats{/*num_covered_pcs=*/
21 * j,
/*num_8bit_counter_features=*/22 * j,
/*num_data_flow_features=*/23 * j,
/*num_cmp_features=*/24 * j,
/*num_call_stack_features=*/25 * j,
/*num_bounded_path_features=*/26 * j,
/*num_pc_pair_features=*/27 * j,
/*num_user_features=*/28 * j};
// NOTE: These two don't add up to `num_user_features`, but that's
// ok for the purposes of this test.
cov_stats.num_user0_features = 28 * j + 1;
cov_stats.num_user10_features = 28 * j + 2;
cov_stats.num_unknown_features = 29 * j;
cov_stats.num_funcs_in_frontier = 31 * j;
stats_vec[i].store(Stats{
StatsMeta{/*timestamp_unix_micros=*/1000000 * j},
ExecStats{
/*fuzz_time_sec=*/10 * j,
/*.num_executions=*/12 * j,
/*.num_target_crashes=*/13 * j,
},
cov_stats,
CorpusStats{
/*active_corpus_size=*/101 * j,
/*total_corpus_size=*/102 * j,
/*max_corpus_element_size=*/103 * j,
/*avg_corpus_element_size=*/104 * j,
},
RusageStats{
/*engine_rusage_avg_millicores=*/201 * j,
/*engine_rusage_cpu_percent=*/202 * j,
/*engine_rusage_rss_mb=*/203 * j,
/*engine_rusage_vsize_mb=*/204 * j,
},
});
}
const std::vector<Environment> env_vec = {
CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA",
/*experiment_flags=*/"AAA"),
CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB",
/*experiment_flags=*/"BBB"),
CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA",
/*experiment_flags=*/"AAA"),
CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB",
/*experiment_flags=*/"BBB")};
{
StatsCsvFileAppender stats_csv_appender{stats_vec, env_vec};
stats_csv_appender.ReportCurrStats();
// Emulate progress in shard #2 of each experiment. In the second line of
// each CSV, min's shouldn't change, max's should increase by 1, avg's
// should increase by 0.5.
for (int i = 2; i < stats_vec.size(); ++i) {
auto &stats = stats_vec[i];
auto new_stats = stats.load();
new_stats.timestamp_unix_micros += 1;
new_stats.fuzz_time_sec += 1;
new_stats.num_executions += 1;
new_stats.num_target_crashes += 1;
new_stats.num_covered_pcs += 1;
new_stats.num_8bit_counter_features += 1;
new_stats.num_data_flow_features += 1;
new_stats.num_cmp_features += 1;
new_stats.num_call_stack_features += 1;
new_stats.num_bounded_path_features += 1;
new_stats.num_pc_pair_features += 1;
new_stats.num_user_features += 1;
new_stats.num_user0_features += 1;
new_stats.num_user10_features += 1;
new_stats.num_unknown_features += 1;
new_stats.num_funcs_in_frontier += 1;
new_stats.active_corpus_size += 1;
new_stats.total_corpus_size += 1;
new_stats.max_corpus_element_size += 1;
new_stats.avg_corpus_element_size += 1;
new_stats.engine_rusage_avg_millicores += 1;
new_stats.engine_rusage_cpu_percent += 1;
new_stats.engine_rusage_rss_mb += 1;
new_stats.engine_rusage_vsize_mb += 1;
stats.store(new_stats);
}
stats_csv_appender.ReportCurrStats();
}
const std::vector<std::string> kExpectedCsvs = {
workdir / "fuzzing-stats-.000000.ExperimentA.csv",
workdir / "fuzzing-stats-.000000.ExperimentB.csv",
};
const std::vector<std::vector<std::string>> kExpectedCsvLines = {
// CSV #1.
{
// clang-format off
// Header.
"NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg,"
"NumExecs_Min,NumExecs_Max,NumExecs_Avg,"
"ActiveCorpusSize_Min,ActiveCorpusSize_Max,ActiveCorpusSize_Avg,"
"MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg,"
"AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg,"
"UnixMicros_Min,UnixMicros_Max,"
"FuzzTimeSec_Min,FuzzTimeSec_Max,FuzzTimeSec_Avg,"
"NumProxyCrashes_Min,NumProxyCrashes_Max,NumProxyCrashes_Sum,"
"TotalCorpusSize_Min,TotalCorpusSize_Max,TotalCorpusSize_Sum,"
"Num8BitCounterFts_Min,Num8BitCounterFts_Max,Num8BitCounterFts_Avg,"
"NumDataFlowFts_Min,NumDataFlowFts_Max,NumDataFlowFts_Avg,"
"NumCmpFts_Min,NumCmpFts_Max,NumCmpFts_Avg,"
"NumCallStackFts_Min,NumCallStackFts_Max,NumCallStackFts_Avg,"
"NumBoundedPathFts_Min,NumBoundedPathFts_Max,NumBoundedPathFts_Avg,"
"NumPcPairFts_Min,NumPcPairFts_Max,NumPcPairFts_Avg,"
"NumUserFts_Min,NumUserFts_Max,NumUserFts_Avg,"
"NumUnknownFts_Min,NumUnknownFts_Max,NumUnknownFts_Avg,"
"NumFuncsInFrontier_Min,NumFuncsInFrontier_Max,NumFuncsInFrontier_Avg," // NOLINT
"EngineRusageAvgCores_Max,EngineRusageCpuPct_Max,"
"EngineRusageRssMb_Max,EngineRusageVSizeMb_Max,"
"NumUser0Fts_Min,NumUser0Fts_Max,NumUser0Fts_Avg,"
"NumUser1Fts_Min,NumUser1Fts_Max,NumUser1Fts_Avg,"
"NumUser2Fts_Min,NumUser2Fts_Max,NumUser2Fts_Avg,"
"NumUser3Fts_Min,NumUser3Fts_Max,NumUser3Fts_Avg,"
"NumUser4Fts_Min,NumUser4Fts_Max,NumUser4Fts_Avg,"
"NumUser5Fts_Min,NumUser5Fts_Max,NumUser5Fts_Avg,"
"NumUser6Fts_Min,NumUser6Fts_Max,NumUser6Fts_Avg,"
"NumUser7Fts_Min,NumUser7Fts_Max,NumUser7Fts_Avg,"
"NumUser8Fts_Min,NumUser8Fts_Max,NumUser8Fts_Avg,"
"NumUser9Fts_Min,NumUser9Fts_Max,NumUser9Fts_Avg,"
"NumUser10Fts_Min,NumUser10Fts_Max,NumUser10Fts_Avg,"
"NumUser11Fts_Min,NumUser11Fts_Max,NumUser11Fts_Avg,"
"NumUser12Fts_Min,NumUser12Fts_Max,NumUser12Fts_Avg,"
"NumUser13Fts_Min,NumUser13Fts_Max,NumUser13Fts_Avg,"
"NumUser14Fts_Min,NumUser14Fts_Max,NumUser14Fts_Avg,"
"NumUser15Fts_Min,NumUser15Fts_Max,NumUser15Fts_Avg,",
// Line 1.
"21,63,42.0,"
"12,36,24.0,"
"101,303,202.0,"
"103,309,206.0,"
"104,312,208.0,"
"1000000,3000000,"
"10,30,20.0,"
"13,39,52,"
"102,306,408,"
"22,66,44.0,"
"23,69,46.0,"
"24,72,48.0,"
"25,75,50.0,"
"26,78,52.0,"
"27,81,54.0,"
"28,84,56.0,"
"29,87,58.0,"
"31,93,62.0,"
"603,606,"
"609,612,"
"29,85,57.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"30,86,58.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,",
// Line 2.
"21,64,42.5,"
"12,37,24.5,"
"101,304,202.5,"
"103,310,206.5,"
"104,313,208.5,"
"1000000,3000001,"
"10,31,20.5,"
"13,40,53,"
"102,307,409,"
"22,67,44.5,"
"23,70,46.5,"
"24,73,48.5,"
"25,76,50.5,"
"26,79,52.5,"
"27,82,54.5,"
"28,85,56.5,"
"29,88,58.5,"
"31,94,62.5,"
"604,607,"
"610,613,"
"29,86,57.5,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"30,87,58.5,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,",
"", // empty line at EOF
// clang-format on
},
// CSV #2.
{
// clang-format off
// Header.
"NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg,"
"NumExecs_Min,NumExecs_Max,NumExecs_Avg,"
"ActiveCorpusSize_Min,ActiveCorpusSize_Max,ActiveCorpusSize_Avg,"
"MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg,"
"AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg,"
"UnixMicros_Min,UnixMicros_Max,"
"FuzzTimeSec_Min,FuzzTimeSec_Max,FuzzTimeSec_Avg,"
"NumProxyCrashes_Min,NumProxyCrashes_Max,NumProxyCrashes_Sum,"
"TotalCorpusSize_Min,TotalCorpusSize_Max,TotalCorpusSize_Sum,"
"Num8BitCounterFts_Min,Num8BitCounterFts_Max,Num8BitCounterFts_Avg,"
"NumDataFlowFts_Min,NumDataFlowFts_Max,NumDataFlowFts_Avg,"
"NumCmpFts_Min,NumCmpFts_Max,NumCmpFts_Avg,"
"NumCallStackFts_Min,NumCallStackFts_Max,NumCallStackFts_Avg,"
"NumBoundedPathFts_Min,NumBoundedPathFts_Max,NumBoundedPathFts_Avg,"
"NumPcPairFts_Min,NumPcPairFts_Max,NumPcPairFts_Avg,"
"NumUserFts_Min,NumUserFts_Max,NumUserFts_Avg,"
"NumUnknownFts_Min,NumUnknownFts_Max,NumUnknownFts_Avg,"
"NumFuncsInFrontier_Min,NumFuncsInFrontier_Max,NumFuncsInFrontier_Avg," // NOLINT
"EngineRusageAvgCores_Max,EngineRusageCpuPct_Max,"
"EngineRusageRssMb_Max,EngineRusageVSizeMb_Max,"
"NumUser0Fts_Min,NumUser0Fts_Max,NumUser0Fts_Avg,"
"NumUser1Fts_Min,NumUser1Fts_Max,NumUser1Fts_Avg,"
"NumUser2Fts_Min,NumUser2Fts_Max,NumUser2Fts_Avg,"
"NumUser3Fts_Min,NumUser3Fts_Max,NumUser3Fts_Avg,"
"NumUser4Fts_Min,NumUser4Fts_Max,NumUser4Fts_Avg,"
"NumUser5Fts_Min,NumUser5Fts_Max,NumUser5Fts_Avg,"
"NumUser6Fts_Min,NumUser6Fts_Max,NumUser6Fts_Avg,"
"NumUser7Fts_Min,NumUser7Fts_Max,NumUser7Fts_Avg,"
"NumUser8Fts_Min,NumUser8Fts_Max,NumUser8Fts_Avg,"
"NumUser9Fts_Min,NumUser9Fts_Max,NumUser9Fts_Avg,"
"NumUser10Fts_Min,NumUser10Fts_Max,NumUser10Fts_Avg,"
"NumUser11Fts_Min,NumUser11Fts_Max,NumUser11Fts_Avg,"
"NumUser12Fts_Min,NumUser12Fts_Max,NumUser12Fts_Avg,"
"NumUser13Fts_Min,NumUser13Fts_Max,NumUser13Fts_Avg,"
"NumUser14Fts_Min,NumUser14Fts_Max,NumUser14Fts_Avg,"
"NumUser15Fts_Min,NumUser15Fts_Max,NumUser15Fts_Avg,",
// Line 1.
"42,84,63.0,"
"24,48,36.0,"
"202,404,303.0,"
"206,412,309.0,"
"208,416,312.0,"
"2000000,4000000,"
"20,40,30.0,"
"26,52,78,"
"204,408,612,"
"44,88,66.0,"
"46,92,69.0,"
"48,96,72.0,"
"50,100,75.0,"
"52,104,78.0,"
"54,108,81.0,"
"56,112,84.0,"
"58,116,87.0,"
"62,124,93.0,"
"804,808,"
"812,816,"
"57,113,85.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"58,114,86.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,",
// Line 2.
"42,85,63.5,"
"24,49,36.5,"
"202,405,303.5,"
"206,413,309.5,"
"208,417,312.5,"
"2000000,4000001,"
"20,41,30.5,"
"26,53,79,"
"204,409,613,"
"44,89,66.5,"
"46,93,69.5,"
"48,97,72.5,"
"50,101,75.5,"
"52,105,78.5,"
"54,109,81.5,"
"56,113,84.5,"
"58,117,87.5,"
"62,125,93.5,"
"805,809,"
"813,817,"
"57,114,85.5,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"58,115,86.5,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,"
"0,0,0.0,",
"", // empty line at EOF
// clang-format on
},
};
for (int i = 0; i < 2; ++i) {
ASSERT_TRUE(std::filesystem::exists(kExpectedCsvs[i]));
std::string csv_contents;
ReadFromLocalFile(kExpectedCsvs[i], csv_contents);
const std::vector<std::string_view> csv_lines =
absl::StrSplit(csv_contents, '\n');
ASSERT_EQ(csv_lines.size(), kExpectedCsvLines[i].size());
for (size_t j = 0; j < csv_lines.size(); ++j) {
EXPECT_EQ(csv_lines[j], kExpectedCsvLines[i][j]) << VV(i) << VV(j);
}
}
}
TEST(Stats, DumpStatsToExistingCsvFile) {
class TestStatsCsvFileAppender : public StatsCsvFileAppender {
using StatsCsvFileAppender::StatsCsvFileAppender;
private:
std::string GetBackupFilename(const std::string &filename) const override {
return absl::StrCat(filename, ".bak");
}
};
const std::filesystem::path workdir = GetTestTempDir(test_info_->name());
const std::vector<Environment> env_vec = {CreateEnv(workdir.c_str()),
CreateEnv(workdir.c_str())};
std::vector<std::atomic<Stats>> stats_vec(2);
stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/1000000}});
stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/2000000}});
const std::string kExpectedCsv = workdir / "fuzzing-stats-.000000.csv";
const std::string kExpectedCsvBak = workdir / "fuzzing-stats-.000000.csv.bak";
// `StatsCsvFileAppender` creates a brand-new fuzzing-stats file and writes
// the CSV header and a 1st stats line to it.
{
TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec};
stats_csv_appender.ReportCurrStats();
ASSERT_TRUE(std::filesystem::exists(kExpectedCsv));
ASSERT_FALSE(std::filesystem::exists(kExpectedCsvBak));
std::string contents;
ReadFromLocalFile(kExpectedCsv, contents);
const std::vector<std::string> lines = absl::StrSplit(contents, '\n');
// Header + 1 stats line + empty line at EOF.
EXPECT_EQ(lines.size(), 3);
}
// `StatsCsvFileAppender` finds an existing file with a CSV header matching
// the current version and appends a 2nd stats line to it.
{
stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/3000000}});
stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/4000000}});
TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec};
stats_csv_appender.ReportCurrStats();
ASSERT_TRUE(std::filesystem::exists(kExpectedCsv));
ASSERT_FALSE(std::filesystem::exists(kExpectedCsvBak));
std::string contents;
ReadFromLocalFile(kExpectedCsv, contents);
const std::vector<std::string> lines = absl::StrSplit(contents, '\n');
// Header + 2 stats lines + empty line at EOF.
EXPECT_EQ(lines.size(), 4);
}
// `StatsCsvFileAppender` finds an existing file with a CSV header not
// matching the current version (ostensibly created by a previous version of
// Centipede), creates a backup copy of it, and starts a new file scratch,
// writing the new CSV header and a 1st line to it.
{
// Fake a CSV with an outdated header and contents.
const std::vector<std::string> kFakeOldCsvLines = {
"Old,Csv,Header,", "0,1,2,", "10,11,12,", "20,21,22,", "",
};
WriteToLocalFile(kExpectedCsv, absl::StrJoin(kFakeOldCsvLines, "\n"));
stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/5000000}});
stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/6000000}});
TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec};
stats_csv_appender.ReportCurrStats();
ASSERT_TRUE(std::filesystem::exists(kExpectedCsv));
ASSERT_TRUE(std::filesystem::exists(kExpectedCsvBak));
std::string contents;
ReadFromLocalFile(kExpectedCsv, contents);
const std::vector<std::string> lines = absl::StrSplit(contents, '\n');
// Header + 1 stats line + empty line at EOF.
EXPECT_EQ(lines.size(), 3);
std::string contents_bak;
ReadFromLocalFile(kExpectedCsvBak, contents_bak);
const std::vector<std::string> lines_bak =
absl::StrSplit(contents_bak, '\n');
CHECK_NE(lines.size(), kFakeOldCsvLines.size()); // Prevent false positives
EXPECT_EQ(lines_bak.size(), kFakeOldCsvLines.size());
}
}
TEST(Stats, PrintRewardValues) {
std::stringstream ss;
std::vector<std::atomic<Stats>> stats_vec(4);
stats_vec[0].store(
{StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/15}});
stats_vec[1].store(
{StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/10}});
stats_vec[2].store(
{StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/40}});
stats_vec[3].store(
{StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/25}});
PrintRewardValues(stats_vec, ss);
const char *expected =
"REWARD_MAX 40\n"
"REWARD_SECOND_MAX 25\n"
"REWARD_MIN 10\n"
"REWARD_MEDIAN 25\n"
"REWARD_AVERAGE 22.5\n";
EXPECT_EQ(ss.str(), expected);
}
} // namespace
} // namespace fuzztest::internal