Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a CPU usage check for HAWK/PUMA #12464

Merged
merged 2 commits into from May 5, 2021
Merged

Add a CPU usage check for HAWK/PUMA #12464

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8bb84b0
Add a CPU usage check for HAWK/PUMA
alvarocarvajald May 3, 2021
00350f8
Add CPU usage check when HAWK is idle
alvarocarvajald May 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 5 additions & 3 deletions tests/ha/barrier_init.pm
View file
Open in desktop
Expand Up @@ -108,9 +108,11 @@ sub run {
barrier_create("PACEMAKER_CTS_CHECKED_$cluster_name", $num_nodes + 1);

# HAWK_GUI_ barriers also have to wait in the client
barrier_create("HAWK_GUI_INIT_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_GUI_CHECKED_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_FENCE_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_GUI_INIT_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_GUI_CHECKED_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_GUI_CPU_TEST_START_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_GUI_CPU_TEST_FINISH_$cluster_name", $num_nodes + 1);
barrier_create("HAWK_FENCE_$cluster_name", $num_nodes + 1);

# CTDB barriers
barrier_create("CTDB_INIT_$cluster_name", $num_nodes + 1);
Expand Down
38 changes: 38 additions & 0 deletions tests/ha/check_hawk.pm
View file
Open in desktop
Expand Up @@ -19,6 +19,43 @@ use lockapi;
use hacluster qw(get_cluster_name is_node);
use utils 'systemctl';
use version_utils 'is_sle';
use List::Util qw(sum);

sub check_hawk_cpu {
my $cluster_name = get_cluster_name;
my @cpu_usage = ();

barrier_wait("HAWK_GUI_CPU_TEST_START_$cluster_name");
while (!barrier_try_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name")) {
# Wrapping script_output in eval { } as node can be fenced by hawk test from client.
# In fenced node, script_output will croak and kill the test. This prevents it
my $metric = eval {
script_output q@ps axo pcpu,comm | awk '/hawk|puma/ {total += $1} END {print "cpu_usage["total"]"}'@,
proceed_on_failure => 1;
};
if ($@) {
# When script_output croaks, command may be typed when SUT is on the grub menu
# and either boot the system or get into grub editing. If system has booted,
# force a new fence; if it's still in grub menu, do nothing; otherwise send an
# ESC to return SUT to grub menu and exit the loop
if (check_screen('linux-login')) {
reset_consoles;
select_console('root-console');
enter_cmd 'echo b > /proc/sysrq-trigger';
}
else {
send_key 'esc' unless check_screen('grub2');
}
barrier_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name");
last;
}
push @cpu_usage, $metric =~ /cpu_usage\[([\d\.]+)\]/;
sleep bmwqemu::scale_timeout(1);
}
my $cpu_usage = sum(@cpu_usage) / @cpu_usage;
record_info "CPU usage", "HAWK/PUMA CPU usage was $cpu_usage";
record_soft_failure "bsc#1179609 - HAWK/PUMA consume a considerable amount of CPU" if ($cpu_usage >= 50);
}

sub run {
my $cluster_name = get_cluster_name;
Expand Down Expand Up @@ -55,6 +92,7 @@ sub run {
# If testing HAWK GUI, also wait for those barriers
if (get_var('HAWKGUI_TEST_ROLE')) {
barrier_wait("HAWK_GUI_INIT_$cluster_name");
check_hawk_cpu;
barrier_wait("HAWK_GUI_CHECKED_$cluster_name");
}

Expand Down
2 changes: 2 additions & 0 deletions tests/ha/hawk_gui.pm
View file
Open in desktop
Expand Up @@ -86,6 +86,7 @@ sub run {
add_to_known_hosts($node2);
assert_script_run "mkdir -m 1777 $path";
assert_script_run "xhost +";
barrier_wait("HAWK_GUI_CPU_TEST_START_$cluster_name");
my $docker_cmd = "docker run --rm --name test --ipc=host -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=\$DISPLAY -v \$PWD/$path:/$path ";
$docker_cmd .= "$docker_image -b $browser -H $node1 -S $node2 -s $testapi::password -r /$results --virtual-ip $virtual_ip";
enter_cmd "$docker_cmd | tee $logs; echo $pyscr-\$PIPESTATUS > $retcode";
Expand Down Expand Up @@ -114,6 +115,7 @@ sub run {
save_screenshot;

assert_screen "generic-desktop";
barrier_wait("HAWK_GUI_CPU_TEST_FINISH_$cluster_name");

# Error, log and results handling
select_console 'user-console';
Expand Down