Skip to content

Commit

Permalink
ci: fix ui thread starvation in benchmarks (#2009)
Browse files Browse the repository at this point in the history 
Maxing out CPU with a hot while loop also prevented the test runner app from
being able to query the accessibility system to hit the stop button. This causes
lots of retries and eventual failures or timeouts in other areas, and the tests
to run a long time in general.

Instead of having a button press stop the benchmark and contending for time on
the main run loop, just remove the start/stop buttons and start when the view
controller appears, and set a 15 minute delay to stop the benchmarks. Then the
test runner queries the text field for results as usual.
  • Loading branch information
@armcknight
armcknight committed Jul 28, 2022
1 parent ffe3af9 commit 91cf82a
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 51 deletions.
14 changes: 1 addition & 13 deletions Samples/iOS-Swift/PerformanceBenchmarks/SentrySDKPerformanceBenchmarkTests.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,24 +84,12 @@ + (void)tearDown
[app launch];
[app.buttons[@"Performance scenarios"] tap];

XCUIElement *startButton = app.buttons[@"Start test"];
if (![startButton waitForExistenceWithTimeout:5.0]) {
XCTFail(@"Couldn't find benchmark retrieval button.");
}
[startButton tap];

// after hitting the start button, the test app will do CPU intensive work until hitting the
// after navigating to the test, the test app will do CPU intensive work until hitting the
// stop button. wait 15 seconds so that work can be done while the profiler does its thing,
// and the benchmarking observation in the test app will record how much CPU time is used by
// everything
sleep(15);

XCUIElement *stopButton = app.buttons[@"Stop test"];
if (![stopButton waitForExistenceWithTimeout:5.0]) {
XCTFail(@"Couldn't find benchmark retrieval button.");
}
[stopButton tap];

XCUIElement *textField = app.textFields[@"io.sentry.benchmark.value-marshaling-text-field"];
if (![textField waitForExistenceWithTimeout:5.0]) {
XCTFail(@"Couldn't find benchmark value marshaling text field.");
Expand Down
59 changes: 21 additions & 38 deletions Samples/iOS-Swift/iOS-Swift/ViewControllers/PerformanceViewController.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,49 +2,19 @@ import Sentry
import UIKit

class PerformanceViewController: UIViewController {
private let startTestButton = UIButton(type: .custom)
private let stopTestButton = UIButton(type: .custom)
private let valueTextField = UITextField(frame: .zero)

override init(nibName nibNameOrNil: String?, bundle nibBundleOrNil: Bundle?) {
super.init(nibName: nibNameOrNil, bundle: nibBundleOrNil)

startTestButton.addTarget(self, action: #selector(startTest), for: .touchUpInside)
startTestButton.setTitle("Start test", for: .normal)

stopTestButton.addTarget(self, action: #selector(stopTest), for: .touchUpInside)
stopTestButton.setTitle("Stop test", for: .normal)

let buttons = [
startTestButton,
stopTestButton
]
buttons.forEach {
$0.setTitleColor(.black, for: .normal)
}
valueTextField.accessibilityLabel = "io.sentry.benchmark.value-marshaling-text-field"
let stack = UIStackView(arrangedSubviews: buttons + [valueTextField])
stack.axis = .vertical
stack.alignment = .center
stack.distribution = .fillProportionally
stack.translatesAutoresizingMaskIntoConstraints = false
view.addSubview(stack)
valueTextField.translatesAutoresizingMaskIntoConstraints = false
view.addSubview(valueTextField)
NSLayoutConstraint.activate([
stack.leadingAnchor.constraint(equalTo: view.leadingAnchor),
stack.trailingAnchor.constraint(equalTo: view.trailingAnchor),
stack.bottomAnchor.constraint(lessThanOrEqualTo: view.bottomAnchor),
stack.centerYAnchor.constraint(equalTo: view.centerYAnchor)
valueTextField.centerYAnchor.constraint(equalTo: view.centerYAnchor),
valueTextField.widthAnchor.constraint(equalTo: view.widthAnchor, constant: -20)
])

if #available(iOS 11.0, *) {
NSLayoutConstraint.activate([
stack.topAnchor.constraint(greaterThanOrEqualTo: view.safeAreaLayoutGuide.topAnchor)
])
} else {
NSLayoutConstraint.activate([
stack.topAnchor.constraint(greaterThanOrEqualTo: view.topAnchor)
])
}
valueTextField.isHidden = true

view.backgroundColor = .white
}
Expand All @@ -53,6 +23,11 @@ class PerformanceViewController: UIViewController {
fatalError("init(coder:) has not been implemented")
}

override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
startTest()
}

override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
timer?.invalidate()
Expand All @@ -68,8 +43,10 @@ class PerformanceViewController: UIViewController {
private let iterations = 5_000_000
private let range = 1..<Double.greatestFiniteMagnitude
private var transaction: Span?
}

private func doWork(withNumber a: Double) -> Double {
private extension PerformanceViewController {
func doWork(withNumber a: Double) -> Double {
var b: Double
if arc4random() % 2 == 0 {
b = fmod(a, Double.random(in: range))
Expand All @@ -89,16 +66,20 @@ class PerformanceViewController: UIViewController {
}
}

@objc func startTest() {
func startTest() {
SentrySDK.configureScope {
$0.setTag(value: "performance-benchmark", key: "uitest-type")
}
transaction = SentrySDK.startTransaction(name: "io.sentry.benchmark.transaction", operation: "crunch-numbers")
timer = Timer.scheduledTimer(timeInterval: interval, target: self, selector: #selector(doRandomWork), userInfo: nil, repeats: true)
SentryBenchmarking.startBenchmark()

DispatchQueue.main.asyncAfter(deadline: .now() + 15) {
self.stopTest()
}
}

@objc func stopTest() {
func stopTest() {
defer {
timer?.invalidate()
transaction?.finish()
Expand All @@ -110,6 +91,8 @@ class PerformanceViewController: UIViewController {
valueTextField.text = "nil"
return
}

valueTextField.isHidden = false
valueTextField.text = "\(value)"

SentrySDK.configureScope {
Expand Down

0 comments on commit 91cf82a

Please sign in to comment.