1#@!#!123s

: / usr / lib / raider / Raider / Jobs /
Filename : StorCli.pm
back
use strict;
use warnings;

package Raider::Jobs::StorCli;
use base qw( Raider::Jobs );

use Raider::Notification::API;
use Raider::Notification::Email;

use JSON::Tiny qw(decode_json encode_json);

=head1 NAME

Raider::Jobs::StorCli - StorCli specific tasks for checking raid health

=head1 DESCRIPTION

Checks raid health for cards requiring storcli.

=head1 USAGE

use Raider::Jobs::StorCli;
my job = Raider::Jobs::StorCli->new();

=head1 METHODS

=head2 get_icmd()

Returns the PATH to the icmd.

=head2 run_job()

Takes all needed actions for checking raid health.

=head2 get_controller_list()

Returns a list of controllers.

=head2 get_array_list(int)

Returns a list of arrays on the given controller.

=head2 array_health(int, int)

Given the controller and array number, return array health.

Returns:

{
  healthy   => BOOLEAN,
  output    => STRING, (json)
  check_cmd => STRING,
}

=head2 hasBbu(int)

Given a controller number, return whether or not it has a BBU.

=head2 getBbuStatus(int)

Given a controller number, return details on the BBU. If no BBU,
undef will be returned.

Returns:

{
  status      => STRING,
  output      => STRING,
  temperature => STRING,
  check_cmd   => $STRING,
}

=head2 runStorCliCmdJsonSingleton(string)

Given an argument string to pass to storcli, automatically use the
JSON interface. This method should only be used when you are running
commands against a single controller. If you are trying to run a
command against multiple controllers, you should look at
runStorCliCmdJson() or runStorCliCmd().

=head2 runStorCliCmdJson(string)

Just like runStorCliCmdJsonSingleton(), except all controller details
will be returned, not just the first element.

=head2 runStorCliCmd(string)

Given an argument string to storcli, runs the command. The JSON interface
is not enforced here. If you want that, you should consider using the JSON
helpers (runStorCliCmdJsonSingleton() or runStorCliCmdJson()).

=cut

sub get_icmd {
  my ($self) = @_;

  return $self->{_StorCli_icmd} if $self->{_StorCli_icmd};

  my $arch = $self->get_arch();
  my $icmd;
  if ( $arch eq '64bit' ) {
    $icmd = '/opt/MegaRAID/storcli/storcli64';
  }
  elsif ( $arch eq '32bit' ) {
    $icmd = '/opt/MegaRAID/storcli/storcli';
  }
  else {
    $self->logger({ cat => 'c', msg => "get_arch() returned an invalid value: '$arch'" });
  }

  $self->icmd_in_path({ icmd => $icmd });

  $self->{_StorCli_icmd} = $icmd;
  return $icmd;
}

sub run_job {
  my ($self, $opts) = @_;

  my $output;
  my ($raid_bad, $bbu_bad) = (0, 0);

  # controller level health checks
  for my $controller ( @{ $self->get_controller_list() } ) {

    # RAID logical disks
    for my $array_num ( @{ $self->get_array_list($controller) } ) {

      my $array_health = $self->array_health($controller, $array_num);
      unless ( $array_health->{healthy} ) {
        $output .= "root ~ # $array_health->{check_cmd}\n";
        $output .= "$array_health->{output}\n";
        $output .= "root ~ #\n";
        $raid_bad = 1;
      }

    };

    # BBU
    if ( $opts->{'bbu_check'} ) {
      my $bbu = $self->getBbuStatus($controller);
      # when true, there is a bbu
      if ($bbu) {

        if ( $bbu->{temperature} ) {
          $self->logger({ cat => 'i', msg => "StorCli backed controller [$controller] has a BBU temperature of [$bbu->{temperature}]" });
        }

        if ( $bbu->{status} !~ /Optimal/i ) {
          $bbu_bad = 1;
        }

        if ( $bbu_bad ) {
          $self->logger({ cat => 'w', msg => "Error detected with BBU on StorCli controller [$controller]" });

          my $bbu_check_cmd_pretty = "root ~ # $bbu->{check_cmd}\n";
          $bbu_check_cmd_pretty .= "$bbu->{output}\n";
          $bbu_check_cmd_pretty .= "root ~ #\n";

          $output .= "Error detected with BBU on Adapter $controller. Details below:\n";
          $output .= $bbu_check_cmd_pretty;

        }
      }
    }

  };

  my $notIfAPI = Raider::Notification::API->new();
  my $notIfEmail = Raider::Notification::Email->new();

  if ( $raid_bad || $bbu_bad ) {
    if ( $notIfAPI->can_alert({ notify_type => 'StorCli' }) ) {

      my $notify_summary;
      if ( $raid_bad && $bbu_bad ) {
        $notify_summary = 'StorCli Hardware RAID and BBU Alarm Detected';
      }
      elsif ( $raid_bad && ! $bbu_bad  ) {
        $notify_summary = 'StorCli Hardware RAID Alarm Detected';
      }
      elsif ( $bbu_bad && ! $raid_bad ) {
        $notify_summary = 'StorCli BBU Alarm Detected';
      }

      $self->logger({ cat => 'w', msg => $notify_summary });
      my $notify_msg = "$notify_summary on [$Raider::Base::base_conf{'hostname'}] at [$self->{uniq_id}]\nDetails:\n $output";
      if ( $notIfAPI->can_api() ) {
        $notIfAPI->send_notification({
          message => $notify_msg
        });
      }
      else {
        $notIfEmail->send_notification({
            subject => "RAID Controller Alarm on $Raider::Base::base_conf{'hostname'}",
            message => $notify_msg
          });
      }
      $notIfAPI->place_alert({notify_type => 'StorCli'});
    }
    else {
      $self->logger({ cat => 'w', msg => 'Notification for StorCli suppressed, alert file for today already exists.' });
    }
  }
  else {
    # health is ok, clear alerts for device
    $notIfAPI->clear_alerts_for_device('StorCli');
  }
}

sub get_controller_list {
  my ($self) = @_;

  # controller 1 is numbered 0
  my $num_cntrls = $self->runStorCliCmdJsonSingleton('show ctrlcount')->{'Response Data'}{'Controller Count'} //= 0;

  return [] if $num_cntrls == 0;
  return [ 0 ] if $num_cntrls == 1;

  my @cntrls;
  my $cnt = 0;
  while ($cnt < $num_cntrls) {
    push @cntrls, $cnt;
    $cnt++;
  }

  return \@cntrls;
}

sub get_array_list {
  my ($self, $controller) = @_;

  my @arrays;
  my $arrays_data = $self->runStorCliCmdJsonSingleton("/c$controller/vall show")->{'Response Data'}{'Virtual Drives'} //= [];

  for my $array_data ( @{ $arrays_data } ) {
    my ($vd) = $array_data->{'DG/VD'} =~ /\d+\/(\d+)/;
    push @arrays, $vd if defined $vd;
  }

  return \@arrays;
}

sub array_health {
  my ($self, $controller, $array) = @_;

  my $is_ok = 0;
  my $storcli_args = "/c$controller/v$array show";
  my $output = $self->runStorCliCmdJsonSingleton($storcli_args);
  my $array_state = $output->{'Response Data'}{'Virtual Drives'}[0]->{State};

  if ( $self->read_conf_file()->{ignore_rebuilding_array} ) {
    if ( $array_state =~ /^Rbld/i ) {
      $is_ok = 1;
    }
  }

  # haven't yet deemed healthy? Then more checks..
  unless ($is_ok) {
    if ( $array_state =~ /^Optl/i ) {
      $is_ok = 1;
    }
  }

  return {
    healthy   => $is_ok,
    output    => encode_json($output),
    check_cmd => $self->get_icmd() . ' ' . $storcli_args . ' J',
  };
}

sub hasBbu {
  my ($self, $controller) = @_;

  my $has_bbu = 1;
  my $bbu = $self->runStorCliCmdJsonSingleton("/c$controller/bbu show status");
  for my $detailed_status (@{ $bbu->{'Command Status'}{'Detailed Status'} }) {
    if ($detailed_status->{ErrMsg} =~ /Battery\s+is\s+absent/i) {
      $has_bbu = 0;
    }
  }

  return $has_bbu;
}

sub getBbuStatus {
  my ($self, $controller) = @_;

  # no BBU return undef
  return unless $self->hasBbu($controller);

  my $status = 'Degraded'; # asssume degraded, if we dont find something to say otherwise
  my $temperature = 'unknown';

  # because I don't have test HW with this card and a BBU.. i can't be certain what the json
  # response will look like. I do have however, example non-json outputs online to go by. So
  # for now use the non json interface.
  my $storcli_args = "/c$controller/bbu show status";
  my $output = $self->runStorCliCmd($storcli_args);

  my $inferred_bad = 0;
  for my $line ( split /^/, $output ) {

    if ( $line =~ /Replacement\s+required\s+Yes/i ) {
      $inferred_bad = 1;
    }

    if ( $line =~ /Battery\s+State\s+(\S+)/i ) {
      $status = $1;
    }

    if ( $line =~ /^Temperature\s+(.+)/i ) {
      $temperature = $1;
    }

    if ( $line =~ /Battery\s+is\s+not\s+being\s+charged/i ) {
      $self->logger({cat => 'w', msg => "StorCli backed controller [$controller] has a BBU, but it's not charging"});
      $inferred_bad = 1;
    }

  }

  $status = 'Degraded' if $inferred_bad;

  return {
    status      => $status,
    output      => $output,
    temperature => $temperature,
    check_cmd   => $self->get_icmd() . ' ' . $storcli_args,
  };
}

sub runStorCliCmdJsonSingleton {
  my ($self, $args) = @_;

  return $self->runStorCliCmdJson($args)->{Controllers}[0];
}

sub runStorCliCmdJson {
  my ($self, $args) = @_;

  $args .= ' J';
  my $json = $self->runStorCliCmd($args);
  my $output = eval { decode_json($json) };
  if (my $e = $@) {
    my $icmd = $self->get_icmd;
    $self->logger({cat => 'c', msg => "storcli command [$icmd $args] returned invalid json output: $e"});
  }

  for my $controller ( @{ $output->{controllers}} ) {
    if ($controller->{'Command Status'}{Status} eq 'Failure') {
      my $icmd = $self->get_icmd;
      $self->logger({cat => 'w', msg => "storcli command [$icmd $args] indicated it failed [$controller->{'Command Status'}{Status}]"});
    }
  }

  return $output;
}

sub runStorCliCmd {
  my ($self, $args) = @_;

  my $icmd = $self->get_icmd;
  my $cmd = "$icmd $args";
  my $output = `$cmd`;

  #my $cmd_error_code = $? >> 8;
  #if ($cmd_error_code != 0) {
  #  $self->logger({cat => 'w', msg => "storcli command [$cmd] exited with a non-zero code of [$cmd_error_code]"});
  #}

  return $output;
}


1;