fan-speed-control.pl

#!/usr/bin/perl

use strict;
use warnings;
use List::MoreUtils qw( apply );
use File::Temp qw(tempfile);

my $static_speed_low=0x02;
my $static_speed_high=0x12;   # this is the speed value at 100% demand
                              # ie what we consider the point we don't
                              # really want to get hotter but still
                              # tolerate
my $ipmi_inlet_sensorname="Inlet Temp";

my $default_threshold=32;  # the ambient temperature we use above
                           # which we default back to letting the drac
                           # control the fans
my $base_temp     = 30;    # no fans when below this temp
my $desired_temp1 = 40;    # aim to keep the temperature below this
my $desired_temp2 = 45;    # really ramp up fans above this
my $desired_temp3 = 55;    # really ramp up fans above this
my $demand1       = 5;     # prescaled (not taking into effect static_speed_low/high) demand at temp1
my $demand2       = 40;    # prescaled (not taking into effect static_speed_low/high) demand at temp2
my $demand3       = 200;   # prescaled (not taking into effect static_speed_low/high) demand at temp3

my $hysteresis    = 2;     # don't ramp up velocity unless demand
                           # difference is greater than this.  Ramp
                           # down ASAP however, to bias quietness, and
                           # thus end up removing noise changes for
                           # just small changes in computing

# check inlet temp every minute, hddtemp every minute (but FIXME:
# ensure doesn't spinup spundown disks), and sensors every few seconds

# background information:
# https://www.dell.com/community/PowerEdge-Hardware-General/T130-Fan-Speed-Algorithm/td-p/5052692
# https://serverfault.com/questions/715387/how-do-i-stop-dell-r730xd-fans-from-going-full-speed-when-broadcom-qlogic-netxtr/733064#733064
# could monitor H710 temperature with sudo env /opt/MegaRAID/MegaCli/MegaCli64 -AdpAllInfo -aALL | grep -i temperature

my @ambient_ipmitemps=();
my @hddtemps=();
my @coretemps=();
my @cputemps=();

my $current_mode;
my $lastfan;

my $quiet=0;          # whether to print stats at all
my $print_stats = 1;  # whether to print stats this run

sub is_num {
  my ($val) = @_;
  if ( $val =~ /^[-+]?(\d*\.?\d+|\d+\.?\d*)+$/ ) {
    return 1;
  }
  print "is_num($val)=0\n"; # should probably warn about failures to parse values, but if you don't care about a particular error, perhaps add this clause: if !$quiet;
  return 0;
}

# returns undef if there are no inputs, and ignores inputs that are
# undef
sub average {
  my (@v) = (@_);

  my $div = 0;
  my $tot = 0;
  foreach my $v (@v) {
    if (defined $v && is_num($v)) {
      $tot += $v;
      $div++;
    }
  }
  my $avg=undef;
  if ($div > 0) {
    $avg = sprintf "%.2f", $tot/$div;
  }
  return $avg;
}

# returns undef if there are no inputs, and ignores inputs that are
# undef
sub max {
  my (@v) = (@_);
  my $max=undef;
  foreach my $v (@v) {
    if (defined $v) {
      if (!defined $max or $v > $max) {
        $max = $v;
      }
    }
  }
  return $max;
}

sub set_fans_default {
  if (!defined $current_mode or $current_mode ne "default") {
    $current_mode="default";
    $lastfan=undef;
    print "--> enable dynamic fan control\n";
    foreach my $attempt (1..10) {
      system("ipmitool raw 0x30 0x30 0x01 0x01") == 0 and return 1;
      sleep 1;
      print "  Retrying dynamic control $attempt\n";
    }
    print "Retries of dynamic control all failed\n";
    return 0;
  }
  return 1;
}

sub set_fans_servo {
  my ($ambient_temp, $_cputemps, $_coretemps, $_hddtemps) = (@_);
  my (@cputemps)  = @$_cputemps;
  my (@coretemps) = @$_coretemps;
  my (@hddtemps)  = @$_hddtemps;

  # two thirds weighted CPU temps vs hdd temps, but if the HDD temps
  # creep above this value, use them exclusively (more important to
  # keep them cool than the CPUs)
  my $weighted_temp = max(average(
                                  average(@cputemps), average(@coretemps), average(@hddtemps)),
                          average(@hddtemps));

  if ((!defined $weighted_temp) or ($weighted_temp == 0)) {
    print "Error reading all temperatures! Fallback to idrac control\n";
    set_fans_default();
    return;
  }
  print "weighted_temp = $weighted_temp ; ambient_temp $ambient_temp\n" if $print_stats;

  if ((!defined $current_mode) or ($current_mode ne "set")) {
    print "--> disable dynamic fan control\n";
    system("ipmitool raw 0x30 0x30 0x01 0x00") == 0 or return 0;
    # if this fails, want to return telling caller not to think weve
    # made a change
    $current_mode="set";
  }

  # FIXME: probably want to take into account ambient temperature - if
  # the difference between weighted_temp and ambient_temp is small
  # because ambient_temp is large, then less need to run the fans
  # because there's still low power demands
  my $demand = 0; # sort of starts off with a range roughly 0-255,
                  # which we multiply later to be ranged roughly
                  # between 0-100% of
                  # ($static_speed_low - $static_speed_high)
  if ($weighted_temp > $base_temp and
      $weighted_temp < $desired_temp1) {
    # slope m = (y2-y1)/(x2-x1)
    # y - y1 = (x-x1)(y2-y1)/(x2-x1)
    # y1 = 0 ; x1 = base_temp ; y2 = demand1 ; x2 = desired_temp1
    # x = weighted_temp
    $demand = 0        + ($weighted_temp - $base_temp    ) * ($demand1 - 0       )/($desired_temp1 - $base_temp    );
  } elsif (($weighted_temp >= $desired_temp1) or ($weighted_temp < $desired_temp2)) {
    # y1 = demand1 ; x1 = desired_temp1 ; y2 = demand2 ; x2 = desired_temp2
    $demand = $demand1 + ($weighted_temp - $desired_temp1) * ($demand2 - $demand1)/($desired_temp2 - $desired_temp1);
  } elsif ($weighted_temp >= $desired_temp2) {
    # y1 = demand2 ; x1 = desired_temp2 ; y2 = demand3 ; x2 = desired_temp3
    # demand will increase above $demand3 for temps above $desired_temp3, linearly, until we cap it below
    $demand = $demand2 + ($weighted_temp - $desired_temp2) * ($demand3 - $demand2)/($desired_temp3 - $desired_temp2);
  }
  printf "demand(%0.2f)", $demand if $print_stats;
  $demand = int($static_speed_low + $demand/100*($static_speed_high-$static_speed_low));
  if ($demand>255) {
    $demand=255;
  }
  printf " -> %i\n", $demand if $print_stats;
  # ramp down the fans quickly upon lack of demand, don't ramp them up
  # to tiny spikes of 1 fan unit.  FIXME: But should implement long
  # term smoothing of +/- 1 fan unit
  if (!defined $lastfan or $demand < $lastfan or $demand > $lastfan + $hysteresis) {
    $lastfan = $demand;
    $demand = sprintf("0x%x", $demand);
#    print "demand = $demand\n";
    print "--> ipmitool raw 0x30 0x30 0x02 0xff $demand\n";
    system("ipmitool raw 0x30 0x30 0x02 0xff $demand") == 0 or return 0;
    # if this fails, want to return telling caller not to think weve
    # made a change
  }
  return 1;
}

my ($tempfh, $tempfilename) = tempfile("fan-speed-control.XXXXX", TMPDIR => 1);

$SIG{TERM} = $SIG{HUP} = $SIG{INT} = sub { my $signame = shift ; $SIG{$signame} = 'DEFAULT' ; print "Resetting fans back to default\n"; set_fans_default ; kill $signame, $$ };
END {
  my $exit = $?;
  unlink $tempfilename;
  print "Resetting fans back to default\n";
  set_fans_default;
  $? = $exit;
}

if (defined $ARGV[0] && $ARGV[0] eq "-q") {
  $quiet=1;
  $print_stats=0;
}

my $last_reset_hddtemps=time;
my $last_reset_ambient_ipmitemps=time;
my $ambient_temp=20;
while () {
  if (!@hddtemps) {
    # could just be a simple pipe, but hddtemp has a strong posibility
    # to be stuck in a D state, and hold STDERR open despite a kill
    # -9, so instead just send it to a tempfile, and read from that tempfile
    system("timeout -k 1 20 hddtemp /dev/sd? /dev/nvme?n? | grep -v 'not available' > $tempfilename");
    @hddtemps=`cat < $tempfilename`;
  }
  if (!@ambient_ipmitemps) {
    @ambient_ipmitemps=`timeout -k 1 20 ipmitool sdr type temperature | grep "$ipmi_inlet_sensorname" | grep [0-9] || echo " | $ambient_temp degrees C"` # ipmitool often fails - just keep using the previous result til it succeeds
  }
  @coretemps=`timeout -k 1 20 sensors | grep [0-9]`;
  @cputemps=grep {/^Package id/} @coretemps;
  @coretemps=grep {/^Core/} @coretemps;
  # filter in numbers only and remove all extraneous output, and some
  # devices permanently return a *temperature* of 255, so filter them
  # out too.
  @hddtemps=grep {/[0-9]/ && !/255/} @hddtemps;

  chomp @cputemps;
  chomp @coretemps;
  chomp @ambient_ipmitemps;
  chomp @hddtemps;

  # apply from List::MoreUtils

  # "..?C" refers to single octet ascii degree symbol that old
  # versions used to output, and 2 octet unicode degree symbol
  @cputemps = apply { s/.*:  *([-+0-9.]+)..?C\b.*/$1/ } @cputemps;
  @coretemps = apply { s/.*:  *([-+0-9.]+)..?C\b.*/$1/ } @coretemps;
  @ambient_ipmitemps = apply { s/.*\| ([^ ]*) degrees C.*/$1/ } @ambient_ipmitemps;
  @hddtemps = apply { s/.*:  *([-+0-9.]+)..?C\b.*/$1/ } @hddtemps;
  #FIXME: it is more important to keep hdds cool than CPUs.  We should
  #put differnt offsets on them - possibly as easily as adding "10" to
  #hddtemp (but need to work out how to keep log output sane)

  print "\n" if $print_stats;

  print "cputemps=", join (" ; ", @cputemps), "\n" if $print_stats;
  print "coretemps=", join (" ; ", @coretemps), "\n" if $print_stats;
  print "ambient_ipmitemps=", join (" ; ", @ambient_ipmitemps), "\n" if $print_stats;
  print "hddtemps=", join (" ; ", @hddtemps), "\n" if $print_stats;

  $ambient_temp = average(@ambient_ipmitemps);
  # FIXME: hysteresis
  if ($ambient_temp > $default_threshold) {
    print "fallback because of high ambient temperature $ambient_temp > $default_threshold\n";
    if (!set_fans_default()) {
      # return for next loop without resetting timers and delta change if that fails
      next;
    }
  } else {
    if (!set_fans_servo($ambient_temp, \@cputemps, \@coretemps, \@hddtemps)) {
      # return for next loop without resetting timers and delta change if that fails
      next;
    }
  }

  $print_stats = 0;
  # every 20 minutes (enough to establish spin-down), invalidate the
  # cache of the slowly changing hdd temperatures to allow them to be
  # refreshed
  if (time - $last_reset_hddtemps > 1200) {
    @hddtemps=();
    $last_reset_hddtemps=time;
  }
  # every 60 seconds, invalidate the cache of the slowly changing
  # ambient temperatures to allow them to be refreshed
  if (time - $last_reset_ambient_ipmitemps > 60) {
    @ambient_ipmitemps=();
    $current_mode="reset"; # just in case the RAC has rebooted, it
                           # will go back into default control, so
                           # make sure we set it appropriately once
                           # per minute
    $last_reset_ambient_ipmitemps=time;
    $print_stats = 1 if !$quiet;
  }
  sleep 3;
}