1#@!#!123s

: / lib / raider / Raider / Jobs /
Filename : PercCli.pm
back
use strict;
use warnings;

package Raider::Jobs::PercCli;
use base qw( Raider::Jobs );

use Raider::Notification::API;
use Raider::Notification::Email;

use JSON::Tiny qw(decode_json encode_json);

=head1 NAME

Raider::Jobs::PercCli - PercCli specific tasks for checking raid health

=head1 DESCRIPTION

Checks raid health for cards requiring PercCli.

=head1 USAGE

use Raider::Jobs::PercCli;
my job = Raider::Jobs::PercCli->new();

=head1 DOCUMENTATION

At the time of this writing, the most current documentation can be found at the following URL:

https://dl.dell.com/content/manual24984199-dell-poweredge-raid-controller-12-command-line-interface-reference-guide.pdf?language=en-us

Be aware that inaccuracies have been found in that documentation (specifically surrounding "battery"
vs. "energy pack", but there may be others)

=head1 METHODS

=head2 get_icmd()

Returns the PATH to the icmd.

=head2 run_job()

Takes all needed actions for checking raid health.

=head2 get_controller_list()

Returns a list of controllers.

=head2 get_array_list(int)

Returns a list of arrays on the given controller.

=head2 array_health(int, int)

Given the controller and array number, return array health.

Returns:

{
  healthy   => BOOLEAN,
  output    => STRING, (json)
  check_cmd => STRING,
}

=head2 hasBbu(int)

Given a controller number, return whether or not it has a BBU.

=head2 getBbuStatus(int)

Given a controller number, return details on the BBU. If no BBU,
undef will be returned.

Returns:

{
  status      => STRING,
  output      => STRING,
  temperature => STRING,
  check_cmd   => $STRING,
}

=head2 runPercCliCmdJsonSingleton(string)

Given an argument string to pass to PercCli, automatically use the
JSON interface. This method should only be used when you are running
commands against a single controller. If you are trying to run a
command against multiple controllers, you should look at
runPercCliCmdJson() or runPercCliCmd().

=head2 runPercCliCmdJson(string)

Just like runPercCliCmdJsonSingleton(), except all controller details
will be returned, not just the first element.

=head2 runPercCliCmd(string)

Given an argument string to PercCli, runs the command. The JSON interface
is not enforced here. If you want that, you should consider using the JSON
helpers (runPercCliCmdJsonSingleton() or runPercCliCmdJson()).

=cut

sub get_icmd {
  my ($self) = @_;

  return $self->{_PercCli_icmd} if $self->{_PercCli_icmd};

  my $arch = $self->get_arch();
  my $icmd;
  if ( $arch eq '64bit' ) {
    $icmd = '/opt/MegaRAID/perccli/perccli64';
  }
  else {
    $self->logger({ cat => 'c', msg => "get_arch() returned an invalid value: '$arch'" });
  }

  $self->icmd_in_path({ icmd => $icmd });

  $self->{_PercCli_icmd} = $icmd;
  return $icmd;
}

sub get_name {
  return 'PercCli';
}

sub run_job {
  my ($self, $opts) = @_;

  my $output;
  my ($raid_bad, $bbu_bad) = (0, 0);
  # controller level health checks
  for my $controller ( @{ $self->get_controller_list() } ) {

    # RAID logical disks
    for my $array_num ( @{ $self->get_array_list($controller) } ) {

      my $array_health = $self->array_health($controller, $array_num);
      unless ( $array_health->{healthy} ) {
        $output .= "root ~ # $array_health->{check_cmd}\n";
        $output .= "$array_health->{output}\n";
        $output .= "root ~ #\n";
        $raid_bad = 1;
      }

    };

    # BBU
    if ( $opts->{'bbu_check'} ) {
      my $bbu = $self->getBbuStatus($controller);
      # when true, there is a bbu
      if ($bbu) {
        my $name = $self->get_name;
        $bbu_bad = $bbu->{bbu_bad} // $bbu_bad;
        if ( $bbu->{temperature} && $bbu->{temperature} ne 'unknown' ) {
          $self->logger({ cat => 'i', msg => "$name backed controller [$controller] has a BBU temperature of [$bbu->{temperature}]" });
        }

        if ( $bbu->{status} !~ /Optimal/i ) {
          $bbu_bad = 1;
        }

        if ( $bbu_bad ) {
          $self->logger({ cat => 'w', msg => "Error detected with BBU on $name controller [$controller]" });

          my $bbu_check_cmd_pretty = "root ~ # $bbu->{check_cmd}\n";
          $bbu_check_cmd_pretty .= "$bbu->{output}\n";
          $bbu_check_cmd_pretty .= "root ~ #\n";

          $output .= "Error detected with BBU on Adapter $controller. Details below:\n";
          $output .= $bbu_check_cmd_pretty;

        }
      }
    }

  };

  my $notIfAPI = Raider::Notification::API->new();
  my $notIfEmail = Raider::Notification::Email->new();

  if ( $raid_bad || $bbu_bad ) {
    my $name = $self->get_name;
    if ( $notIfAPI->can_alert({ notify_type => 'PercCli' }) ) {
      my $notify_summary;
      if ( $raid_bad && $bbu_bad ) {
        $notify_summary = "$name Hardware RAID and BBU Alarm Detected";
      }
      elsif ( $raid_bad && ! $bbu_bad  ) {
        $notify_summary = "$name Hardware RAID Alarm Detected";
      }
      elsif ( $bbu_bad && ! $raid_bad ) {
        $notify_summary = "$name BBU Alarm Detected";
      }

      $self->logger({ cat => 'w', msg => $notify_summary });
      my $notify_msg = "$notify_summary on [$Raider::Base::base_conf{'hostname'}] at [$self->{uniq_id}]\nDetails:\n $output";
      if ( $notIfAPI->can_api() ) {
        $notIfAPI->send_notification({
          message => $notify_msg
        });
      }
      else {
        $notIfEmail->send_notification({
            subject => "RAID Controller Alarm on $Raider::Base::base_conf{'hostname'}",
            message => $notify_msg
          });
      }
      $notIfAPI->place_alert({notify_type => 'PercCli'});
    }
    else {
      $self->logger({ cat => 'w', msg => "Notification for $name suppressed, alert file for today already exists." });
    }
  }
  else {
    # health is ok, clear alerts for device
    $notIfAPI->clear_alerts_for_device('PercCli');
  }
}

sub get_controller_list {
  my ($self) = @_;

  # controller 1 is numbered 0
  my $num_cntrls = $self->runPercCliCmdJsonSingleton('show ctrlcount')->{'Response Data'}{'Controller Count'} //= 0;

  return [] if $num_cntrls == 0;
  return [ 0 ] if $num_cntrls == 1;

  my @cntrls;
  my $cnt = 0;
  while ($cnt < $num_cntrls) {
    push @cntrls, $cnt;
    $cnt++;
  }

  return \@cntrls;
}

sub get_array_list {
  my ($self, $controller) = @_;

  my @arrays;
  my $arrays_data = $self->runPercCliCmdJsonSingleton("/c$controller/vall show")->{'Response Data'}{'Virtual Drives'} //= [];

  for my $array_data ( @{ $arrays_data } ) {
    my ($vd) = $array_data->{'DG/VD'} =~ /\d+\/(\d+)/;
    push @arrays, $vd if defined $vd;
  }

  return \@arrays;
}

sub array_health {
  my ($self, $controller, $array) = @_;

  my $is_ok = 0;
  my $PercCli_args = "/c$controller/v$array show";
  my $output = $self->runPercCliCmdJsonSingleton($PercCli_args);
  my $array_state = $output->{'Response Data'}{'Virtual Drives'}[0]->{State};

  if ( $self->read_conf_file()->{ignore_rebuilding_array} ) {
    if ( $array_state =~ /^Rbld/i ) {
      $is_ok = 1;
    }
  }

  # haven't yet deemed healthy? Then more checks..
  unless ($is_ok) {
    if ( $array_state =~ /^Optl/i ) {
      $is_ok = 1;
    }
  }

  return {
    healthy   => $is_ok,
    output    => encode_json($output),
    check_cmd => $self->get_icmd() . ' ' . $PercCli_args . ' J',
  };
}

sub hasBbu {
  my ($self, $controller) = @_;

  my $has_bbu = 1;

  $self->{_bbu_name} = 'bbu';

  my $args = "/c$controller/bbu show status";
  my $test = $self->runPercCliCmd($args);
  if ($test =~ m/unexpected TOKEN_UNKNOWN/m) {
    $args = "/c$controller/ep show";
    my $bbu = $self->runPercCliCmdJsonSingleton($args);
    my $battery_count = 0;
    for my $info (@{ $bbu->{'Response Data'}{'Energy Pack Info'} }) {
      if ($info->{Type} eq 'Battery') {
        $battery_count++;
      }
    }
    $has_bbu = $battery_count ? 1 : 0;
    $self->{_bbu_name} = 'ep';
  }
  else {
    my $bbu = $self->runPercCliCmdJsonSingleton($args);
    for my $detailed_status (@{ $bbu->{'Command Status'}{'Detailed Status'} }) {
      if ($detailed_status->{ErrMsg} =~ /Battery\s+is\s+absent/i) {
        $has_bbu = 0;
      }
    }
  }

  return $has_bbu;
}

sub getBbuStatus {
  my ($self, $controller) = @_;

  # no BBU return undef
  return unless $self->hasBbu($controller);

  if ($self->{_bbu_name} eq 'ep') {
    return $self->getEpStatus($controller);
  }

  my $status = 'Degraded'; # asssume degraded, if we dont find something to say otherwise
  my $temperature = 'unknown';

  # because I don't have test HW with this card and a BBU.. i can't be certain what the json
  # response will look like. I do have however, example non-json outputs online to go by. So
  # for now use the non json interface.
  my $PercCli_args = "/c$controller/bbu show status";
  my $output = $self->runPercCliCmd($PercCli_args);

  my $inferred_bad = 0;
  for my $line ( split /^/, $output ) {

    if ( $line =~ /Replacement\s+required\s+Yes/i ) {
      $inferred_bad = 1;
    }

    if ( $line =~ /Battery\s+State\s+(\S+)/i ) {
      $status = $1;
    }

    if ( $line =~ /^Temperature\s+(.+)/i ) {
      $temperature = $1;
    }

    if ( $line =~ /Battery\s+is\s+not\s+being\s+charged/i ) {
      my $name = $self->get_name;
      $self->logger({cat => 'w', msg => "$name backed controller [$controller] has a BBU, but it's not charging"});
      $inferred_bad = 1;
    }

  }

  $status = 'Degraded' if $inferred_bad;

  return {
    status      => $status,
    output      => $output,
    temperature => $temperature,
    check_cmd   => $self->get_icmd() . ' ' . $PercCli_args,
  };
}

sub getEpStatus {
  my ($self, $controller) = @_;

  my $status      = 'Degraded'; # asssume degraded, if we dont find something to say otherwise
  my $temperature = 'unknown';
  my $bbu_bad     = 0;

  my $PercCli_args = "/c$controller/ep show all";
  my $output = $self->runPercCliCmdJsonSingleton($PercCli_args);

  my $name = $self->get_name;
  if (@{$output->{'Response Data'}{'Energy Pack Info'} || []} > 1) {
    $self->logger({cat => 'w', msg => "$name backed controller [$controller] has more than one BBU. Raider needs to be updated to handle this scenario"});
    $bbu_bad = 1;
  }

  $status = $output->{'Response Data'}{'Energy Pack Info'}[0]{Status} // $status;
  $temperature = $output->{'Response Data'}{'Energy Pack Info'}[0]{'Temperature(C)'} // $temperature;

  my $percentCharge = $output->{'Response Data'}{Status}{'Charge(%)'};
  if (defined $percentCharge && $percentCharge < 50) {
    $self->logger({cat => 'w', msg => "$name backed controller [$controller] has a BBU, but it does not appear to be charging"});
    $status = 'Degraded';
  }

  return {
    status      => $status,
    output      => $output,
    temperature => $temperature,
    check_cmd   => $self->get_icmd() . ' ' . $PercCli_args . ' J',
  };
}

sub runPercCliCmdJsonSingleton {
  my ($self, $args) = @_;

  return $self->runPercCliCmdJson($args)->{Controllers}[0];
}

sub runPercCliCmdJson {
  my ($self, $args) = @_;

  my $name = $self->get_name;

  $args .= ' J';
  my $json = $self->runPercCliCmd($args);
  my $output = eval { decode_json($json) };
  if (my $e = $@) {
    my $icmd = $self->get_icmd;
    $self->logger({cat => 'c', msg => "$name command [$icmd $args] returned invalid json output: $e"});
  }

  for my $controller ( @{ $output->{controllers}} ) {
    if ($controller->{'Command Status'}{Status} eq 'Failure') {
      my $icmd = $self->get_icmd;

      $self->logger({cat => 'w', msg => "$name command [$icmd $args] indicated it failed [$controller->{'Command Status'}{Status}]"});
    }
  }

  return $output;
}

sub runPercCliCmd {
  my ($self, $args) = @_;

  my $icmd = $self->get_icmd;
  my $cmd = "$icmd $args";
  my $output = `$cmd`;

  #my $cmd_error_code = $? >> 8;
  #if ($cmd_error_code != 0) {
  #  my $name = $self->get_name;
  #  $self->logger({cat => 'w', msg => "$name command [$cmd] exited with a non-zero code of [$cmd_error_code]"});
  #}

  return $output;
}


1;