Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

FaultInjection: Fix random number generation #30623

Merged
merged 8 commits into from
Aug 18, 2022
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -3843,7 +3843,9 @@ grpc_cc_library(
"src/core/ext/filters/fault_injection/service_config_parser.h",
],
external_deps = [
"absl/base:core_headers",
"absl/memory",
"absl/random",
"absl/status",
"absl/status:statusor",
"absl/strings",
Expand Down
33 changes: 21 additions & 12 deletions src/core/ext/filters/fault_injection/fault_injection_filter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
#include "src/core/ext/filters/fault_injection/fault_injection_filter.h"

#include <stdint.h>
#include <stdlib.h>

#include <algorithm>
#include <atomic>
Expand Down Expand Up @@ -70,12 +69,14 @@ auto AsInt(absl::string_view s) -> absl::optional<T> {
return absl::nullopt;
}

inline bool UnderFraction(const uint32_t numerator,
inline bool UnderFraction(absl::InsecureBitGen* rand_generator,
const uint32_t numerator,
const uint32_t denominator) {
if (numerator <= 0) return false;
if (numerator >= denominator) return true;
// Generate a random number in [0, denominator).
const uint32_t random_number = rand() % denominator;
const uint32_t random_number =
absl::Uniform(absl::IntervalClosedOpen, *rand_generator, 0u, denominator);
return random_number < numerator;
}

Expand Down Expand Up @@ -139,7 +140,8 @@ FaultInjectionFilter::FaultInjectionFilter(ChannelFilter::Args filter_args)
filter_args.channel_stack(),
filter_args.uninitialized_channel_element())),
service_config_parser_index_(
FaultInjectionServiceConfigParser::ParserIndex()) {}
FaultInjectionServiceConfigParser::ParserIndex()),
mu_(new Mutex) {}

// Construct a promise for one call.
ArenaPromise<ServerMetadataHandle> FaultInjectionFilter::MakeCallPromise(
Expand Down Expand Up @@ -219,14 +221,21 @@ FaultInjectionFilter::MakeInjectionDecision(
}
}
// Roll the dice
const bool delay_request =
delay != Duration::Zero() &&
UnderFraction(delay_percentage_numerator,
fi_policy->delay_percentage_denominator);
const bool abort_request =
abort_code != GRPC_STATUS_OK &&
UnderFraction(abort_percentage_numerator,
fi_policy->abort_percentage_denominator);
bool delay_request = delay != Duration::Zero();
bool abort_request = abort_code != GRPC_STATUS_OK;
if (delay_request || abort_request) {
MutexLock lock(mu_.get());
if (delay_request) {
delay_request =
UnderFraction(&delay_rand_generator_, delay_percentage_numerator,
fi_policy->delay_percentage_denominator);
}
if (abort_request) {
abort_request =
UnderFraction(&abort_rand_generator_, abort_percentage_numerator,
fi_policy->abort_percentage_denominator);
}
}

return InjectionDecision(
fi_policy->max_faults, delay_request ? delay : Duration::Zero(),
Expand Down
8 changes: 8 additions & 0 deletions src/core/ext/filters/fault_injection/fault_injection_filter.h
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,16 @@

#include <stddef.h>

#include <memory>

#include "absl/base/thread_annotations.h"
#include "absl/random/random.h"
#include "absl/status/statusor.h"

#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/channel/channel_fwd.h"
#include "src/core/lib/channel/promise_based_filter.h"
#include "src/core/lib/gprpp/sync.h"
#include "src/core/lib/promise/arena_promise.h"
#include "src/core/lib/transport/transport.h"

Expand Down Expand Up @@ -60,6 +65,9 @@ class FaultInjectionFilter : public ChannelFilter {
// The relative index of instances of the same filter.
size_t index_;
const size_t service_config_parser_index_;
std::unique_ptr<Mutex> mu_;
absl::InsecureBitGen abort_rand_generator_ ABSL_GUARDED_BY(mu_);
absl::InsecureBitGen delay_rand_generator_ ABSL_GUARDED_BY(mu_);
};

} // namespace grpc_core
Expand Down
10 changes: 10 additions & 0 deletions test/cpp/end2end/xds/xds_fault_injection_end2end_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,13 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {

TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
CreateAndStartBackends(1);
const uint32_t kFixedDelayMilliseconds = 100000;
#ifndef NDEBUG
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm really not keen on putting #ifndefs in the middle of individual tests. Can we not just increase the timeout even for opt builds?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
#else
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 6000;
#endif
const uint32_t kDelayPercentageCap = 100;
const uint32_t kFixedDelaySeconds = 100;
const uint32_t kDelayPercentagePerHundred = 50;
const double kDelayRate = kDelayPercentagePerHundred / 100.0;
Expand Down Expand Up @@ -250,7 +256,11 @@ TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
CreateAndStartBackends(1);
const uint32_t kFixedDelayMilliseconds = 100000;
#ifndef NDEBUG
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
#else
const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 6000;
#endif const uint32_t kDelayPercentageCap = 100;
const uint32_t kDelayPercentageCap = 100;
const uint32_t kDelayPercentage = 50;
const double kDelayRate = kDelayPercentage / 100.0;
Expand Down
2 changes: 2 additions & 0 deletions tools/distrib/fix_build_deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@
'absl/meta:type_traits',
'absl/random/random.h':
'absl/random',
'absl/random/distributions.h':
'absl/random:distributions',
'absl/status/status.h':
'absl/status',
'absl/status/statusor.h':
Expand Down