#!/usr/bin/perl

eval 'exec /usr/bin/perl  -S $0 ${1+"$@"}'
    if 0; # not running under some shell

#  Copyright 2002, 2003, 2004, 2005 University of Southern California
#  Copyright 2015 Princeton University
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation or under the same terms as perl itself.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#  Please send comments to cses@princeton.edu
#########################################################

use strict;
use warnings;
use Curses;
use POSIX qw/strerror/;
use File::Basename;
use Switch;

# For slurmtop, we require the SlurmtopAPI module (installed by the 
# slurmtop binary RPM). if we are running out of the source tree, we
# use the copy of SlurmtopAPI in the src directory instead of the
# centrally installed one.
our $scriptName;
BEGIN {
    use lib 'src';
    use ExtUtils::testlib;

    eval "use Slurm qw/:all/"; die $@ if $@;
}
# enable warnings if running under testing.
if ( -d $INC[0] ) {
   $^W=1;
}
eval "use schedtop"; die $@ if $@;

my $pbs_node_state = {
    'ALLOCATED' => 'busy',
    'COMPLETING' => 'busy',
    'DOWN' => 'down',
    'DRAINED' => 'offline',
    'DRAINING' => 'offline,busy',
    'ERROR' => 'down',
    'FAIL' => 'offline',
    'FAILING' => 'offline,busy',
    'FUTURE' => 'unknown',
    'IDLE' => 'free',
    'MAINT' => 'reserve',
    'MIXED' => 'free',
    'PERFCTRS' => 'offline',
    'PERFCTRS (NPC)' => 'offline',
    'POWER_DOWN' => 'down',
    'POWER_UP' => 'down',
    'RESERVED' => 'reserve',
    'UNKNOWN' => 'unknown'
};

my $pbs_job_state = {
    'BOOT_FAIL' => 'C',
    'CANCELLED' => 'C',
    'COMPLETED' => 'C',
    'CONFIGURING' => 'S',
    'COMPLETING' => 'E',
    'FAILED' => 'C',
    'NODE_FAIL' => 'C',
    'PENDING' => 'Q',
    'PREEMPTED' => 'C',
    'RUNNING' => 'R',
    'RESIZING' => 'R',
    'SUSPENDED' => 'S',
    'TIMEOUT' => 'C',
    'SPECIAL_EXIT' => 'Q',
};

our $queue_type = "qos";

chomp(my $host=`hostname -s`);
my $script_name = basename($0);

schedtop::readrc("/etc/${script_name}rc");
schedtop::readrc("$ENV{HOME}/.${script_name}rc");

schedtop::process_args($script_name, $host, @ARGV);

my $slurm = Slurm::new();

my $status = schedtop::main_loop(\&get_info_modSLURM, \&get_version,
        \&get_server_data, \&get_job_data, \&set_node_state, \&set_job_state);

exit($status);

###############################################################
## All subroutines below here
###############################################################

# We gather node and job data by calling the subroutine get_info_modSLURM(). It
# connects directly to the server and requests the desired data.
sub get_info_modSLURM
{
    my ($server, $Nodes, $Jobs, $State_count) = @_;

    my $slurmNodes = $slurm->load_node();
    unless($slurmNodes) {
        die "failed to load node info: " . $slurm->strerror();
    }
    my $jobs = $slurm->load_jobs(0, SHOW_ALL | SHOW_DETAIL);
    unless($jobs) {
        die "failed to load job info: " . $slurm->strerror();
    }
    my $status = "";

    # Loop over node array.
    my $nodeptr;
    my $rank=0;
    $State_count->{_mprocs} = 0;
    foreach (@{$slurmNodes->{node_array}}) {
        # if we are not allowed access to a node reply will be empty, check
        next unless exists $_->{name};
        #print $_->{name}."\n";
        $Nodes->{$server}{$_->{name}}{seen}=1;
        $nodeptr=$Nodes->{$server}{$_->{name}};

        $nodeptr->{rank}=$rank++;
        $nodeptr->{job}=();
        delete $nodeptr->{status}{jobs};
        delete $nodeptr->{status}{message};
        $State_count->{_nodes}++;
        delete $nodeptr->{note};

        my $value=$_->{cpus};
        $nodeptr->{np} = $value;
        $State_count->{_procs} += $value;
        $State_count->{_mprocs} = $State_count->{_mprocs} > $value
                                    ? $State_count->{_mprocs}
                                    : $value;
        # Figure out the SLURM node state.
        # '*' suffix to SLURM state means node not responding: consider it 'down'.
        # '#' suffix to SLURM state means node powering up or being configured: mark 'down'.
        # '+' suffix to ALLOCATED state means some jobs are completing: ignore.
        # '~' suffix to SLURM state means power saving mode: ignore.
        # '$' suffix to SLURM state means node is currently in a reservation with a flag
        #     value of "maintenance" or is scheduled to be rebooted: mark it 'reserve'.
        my $statestr = my $statestr2 = $slurm->node_state_string($_->{node_state});
        $statestr2 =~ s/[@*#+~\$]$//;
        if (exists $pbs_node_state->{$statestr2}) {
            $nodeptr->{state} = $pbs_node_state->{$statestr2};
        } else {
            printf("Unexpected node state code: '%s'\n", $statestr2);
            $nodeptr->{state} = 'unknown';
        }
        if (!($nodeptr->{state} =~ m/down/) && $statestr =~ m/[*#]$/) { 
            $nodeptr->{state} = 'down,' . $nodeptr->{state};
        }
        if (!($nodeptr->{state} =~ m/reserve/) && $statestr =~ m/[\$]$/) {
            $nodeptr->{state} = 'reserve,' . $nodeptr->{state};
        }
        $State_count->{$nodeptr->{state}}++;

        # Ignore PBS 'properties' values; they are optional.

        # Hard-code the ntype value to match PBS.
        $nodeptr->{ntype} = 'cluster';

        if (exists $_->{reason}) {
            $nodeptr->{note} = $_->{reason};
        }
        # Mimic the PBS node status array as best as we can.
        $nodeptr->{status}{loadave} = $_->{cpu_load} / 100.;
        $nodeptr->{status}{physmem} = $_->{real_memory} * 1024;
        $nodeptr->{status}{totmem} = $_->{real_memory} * 1024;
        # TODO: not sure how to get availmem, so doing a hack here.
        $nodeptr->{status}{availmem} = ($_->{alloc_cpus} > 0 ? 0 : $_->{real_memory} * 1024);
        $nodeptr->{status}{nsessions} = '?';
        $nodeptr->{status}{opsys} = $_->{os};
        $nodeptr->{status}{ncpus} = $nodeptr->{np};
        $nodeptr->{status}{state} = $nodeptr->{state};
        # TODO: can add as many SLURM node atributes here. They will be displayed 
        # in the slurmtop node status window.
    }

    # Loop over job array.
    my $jobptr;
    foreach (@{$jobs->{job_array}}) {
        my $jobID;
        # Mimic logic in sacct utility.
        my $arrayJobID = $_->{array_job_id};
        my $arrayTaskStr = $_->{array_task_str};
        my $arrayTaskID = $_->{array_task_id};
        if ($arrayTaskStr) {
            $jobID = sprintf("%s[%s]", $arrayJobID, $arrayTaskStr);
        } elsif ($arrayTaskID != NO_VAL) {
            $jobID = sprintf("%s[%s]", $arrayJobID, $arrayTaskID);
        } else {
            $jobID = $_->{job_id};
        }
        $Jobs->{$jobID}{seen}=1;
        $jobptr=$Jobs->{$jobID};

        #$jobptr->{server} = $_->{alloc_node};
        $jobptr->{server} = $server;
        $jobptr->{user} = getpwuid($_->{user_id});
        if ($queue_type eq "qos") {
            $jobptr->{queue} = $_->{qos};
        } elsif ($queue_type eq "partition") {
            $jobptr->{queue} = $_->{partition};
        } else {
            $jobptr->{queue} = "";
        }
        $jobptr->{jname} = $_->{name};
        $jobptr->{ncount} = $_->{num_nodes};
        $jobptr->{ncpus} = $_->{num_cpus};
        # 'nodes' field is in PBS nodes=X:ppn=Y format. We replace with 'ncpus' field.
        # $jobptr->{nodes} = $_->{value};

        # Convert job state to state string and then translate to PBS job state.
        my $statestr = $slurm->job_state_string($_->{job_state});
        if (exists $pbs_job_state->{$statestr}) {
            $jobptr->{state} = $pbs_job_state->{$statestr};
            # Held pending jobs have a priority of zero.
            if ($jobptr->{state} eq "Q" && $_->{priority} == 0) {
                $jobptr->{state} = "H";
            }
        } else {
            printf("Unexpected job state code: '%s'\n", $statestr);
            $jobptr->{state} = '?';
        }
        if ($jobptr->{state} eq "R") {
            $State_count->{"_rjobs"}++;
        }
        # Convert requested walltime to seconds.
        if (exists $_->{time_limit} && $_->{time_limit} != INFINITE) {
            $jobptr->{reqt} = format_intvl($_->{time_limit} * 60);
        }
        # Calculate elapsed time from job's start time.
        # Ignore if 1 year in the future for PENDING job; it means that 
        # this job or a job it depends on has an INFINITE time request.
        if (exists $_->{start_time} && $_->{start_time} > 0) {
            if ($statestr ne "PENDING") {
                $jobptr->{elpt} = format_intvl(time-$_->{start_time});
            } else {
                my $delay = int((($_->{start_time} - $_->{submit_time}) / (60 * 60 * 24)) + 0.5);
                if ($delay < 365) {
                    $jobptr->{elpt} = format_intvl(time-$_->{start_time});
                }
            }
        }
        # Check if job has resources allocated to it.
        # If so, assign job to node's job status and CPU.
        if ($_->{'node_resrcs'}) {
            if ($jobptr->{state} eq "R") {
                foreach my $nr (@{$_->{'node_resrcs'}}) {
                    my $hl = Slurm::Hostlist::create($nr->{'nodes'});
                    while (my $node = $hl->shift()) {
                        $nodeptr=$Nodes->{$server}{$node};
                        if (! $nodeptr->{status}{jobs}) {
                            $nodeptr->{status}{jobs} = $jobID;
                        } else {
                            $nodeptr->{status}{jobs} .= ", $jobID";
                        }
                        if (scalar keys %{$nodeptr->{job}} == 0) {
                            $State_count->{"_anodes"}++;
                        }
                        my $cl = Slurm::Hostlist::create('[' . $nr->{'cpu_ids'} . ']');
                        while (defined (my $cpu_id = $cl->shift())) {
                            # Uncomment next 3 lines if Slurm suspected of 
                            # mis-allocating jobs to CPUs.
                            #if ($nodeptr->{job}{$cpu_id} || $cpu_id >= $nodeptr->{np}) {
                            #    $status = "Node $node has one or more invalid cpu allocations";
                            #}
                            $nodeptr->{job}{$cpu_id} = $jobID;
                            $State_count->{"_aprocs"}++;
                        }
                    }
                }
            } else {
                #printf("Job %s in state %s (%s) has allocated resources\n", 
                #        $jobID, $jobptr->{state}, $statestr);
            }
        }
        $State_count->{"_njobs"}++;
    }
    undef $slurmNodes;
    undef $jobs;

    return $status;
}

sub get_version {
    (my $major, my $minor, my $micro) = $slurm->api_version();
    my $VERSION = "$major.$minor.$micro";
    return "SLURM v$VERSION";
}

sub get_server_data {
    my ($server, $data_ref) = @_;

    return "This information is not currently available for SLURM";
    #return "";
}

sub get_job_data {
    my ($server, $job, $data_ref, $joberr) = @_;

    my (@array_job, $array_ref);
    # Check for array job spec (i.e., '123456[2]').
    if (@array_job = ( $job =~ /^(\d+)\[(\d+)\]$/ )) {
        # If array job spec, we need to retrieve all the individual array tasks.
        $array_ref = $slurm->load_job($array_job[0], SHOW_ALL | SHOW_DETAIL);
    } else {
        $array_ref = $slurm->load_job($job, SHOW_ALL | SHOW_DETAIL);
    }
    $$joberr = "";
    if (defined $array_ref) {
        my $job_attribs;
        if (scalar @array_job > 0) {
            foreach (@{$array_ref->{job_array}}) {
                if ($_->{array_task_id} == $array_job[1]) {
                    $job_attribs = $_;
                    last;
                }
            }
        } else {
            $job_attribs = $array_ref->{job_array}[0];
        }
        foreach my $key (sort keys %{$job_attribs}) {
            push(@$data_ref, { name => $key, value => $job_attribs->{$key} });
        }
    } else {
        $$joberr = "Unknown job id (did it exit?)";
    }
    return "";
}

sub set_node_state {
    my ($server, $node, $state) = @_;

    my $rc;
    switch ($state) {
        case 'o' {
            $rc = $slurm->update_node({node_names=>$node, node_state=>NODE_STATE_DRAIN});
        }
        case 'r' {
            $rc = $slurm->update_node({node_names=>$node, node_state=>NODE_RESUME});
        }
    }
    my $str = '';
    if ($rc) {
        $str = $slurm->strerror();
    }
    return $str;
}

sub set_job_state {
    my ($server, $jobid, $oldstate, $newstate) = @_;

    my $str = "";
    switch ($newstate) {
        case 'd' {
            # Pass SIGKILL as signal number (like scancel command does).
            my $rc = $slurm->kill_job($jobid, 9);
            if ($rc) {
                $str = $slurm->strerror();
            }
        }
        case 'H' {
            if (!($oldstate =~ /Q|H/)) {
                $str = $slurm->strerror(ESLURM_JOB_NOT_PENDING);
            } else {
                my $rc = $slurm->update_job({job_id=>$jobid, priority=>0, alloc_sid=>0});
                if ($rc) {
                    $str = $slurm->strerror();
                }
            }
        }
        case 'R' {
            if (!($oldstate =~ /Q|H/)) {
                $str = $slurm->strerror(ESLURM_JOB_NOT_PENDING);
            } else {
                my $rc = $slurm->update_job({job_id=>$jobid, priority=>INFINITE, alloc_sid=>0});
                if ($rc) {
                    $str = $slurm->strerror();
                }
            }
        }
        case 'r' {
            my $rc = $slurm->requeue($jobid, 0);
            if ($rc) {
                $str = $slurm->strerror();
            }
        }
    }
    if (length($str) > 0) {
        $str = "$jobid: $str";
    }
    return $str;
}

sub format_intvl {
    my $intvl = shift;
    my $sign = '';
    if ($intvl < 0) {
        $intvl = -$intvl;
        $sign = '-';
    }
    my $secs = $intvl % 60;
    $intvl = $intvl / 60;
    my $mins = $intvl % 60;
    my $hrs = int($intvl / 60);
    return sprintf('%s%.02d:%.02d:%.02d', $sign, $hrs, $mins, $secs);
}    

__END__

=head1 NAME

=over 4

=item slurmtop - monitoring utility for SLURM

=back

=head1 SYNOPSIS

slurmtop [OPTION]... [@hostname]...

=head1 DESCRIPTION

Draws a full-terminal display of your nodes and jobs.  By default, the primary grid
displays every CPU of every node, representing each CPU with a single character.  
The specific character denotes the state of the CPU or identifies the job running 
on that CPU.  

The secondary grids each display a single node whose number of CPUs is larger than 
the CPU threshold specified by maxnodegrid or by the number of CPUs specified with 
show_cpu, or whose CPUs are too numerous to be displayed on a single terminal line 
(e.g., SGI UV).

The job listing shows the job name, queue name, state, etc, and on the far left,
the character used to identify nodes in the upper grid.

By default, the number of nodes per line in the primary grid is adjusted to fit the 
terminal width, but can be shrunk or expanded on the command line with C<-C>, or 
interactively with C<c>.  Setting the number of columns to zero restores the default
behavior.

slurmtop is now largely auto-configurable, and usually needs no special settings
in the configuration files to produce a reasonable display, other than perhaps the
B<nodesort> and/or B<maxrows> variables, plus any personal preferences such as B<colorize>.

For queued jobs (i.e., status Q), the B<Elapsed> column of the job listing will show 
the expected delay before the given job will start running as a negative time interval.
This applies only to jobs that SLURM currently considers eligible to run.

The B<Queue> column in the job listing uses the job's QOS value by default.
This can be changed to using the partition value using the B<main::queue_type> parameter 
in the /etc/slurmtoprc or ~/.slurmtoprc config file:

=item B<main::queue_type>

Define the data source for the Queue column in the job listing.  The current choices 
are B<"qos"> (default) or B<"partition">.

=back

Example:

    main::queue_type="partition"

=head1 COMMAND-LINE OPTIONS

=over 4

=item   B<-h>

print help menu and exit

=item   B<-V>

print version and exit

=item   B<-A>

Toggle array job compression. This defaults to 'on', so that all jobs that
make up an array are shown as a single entry represented by a single letter.
The total number of active cores for the compressed array is appended to the job line.

=item   B<-c> num

the number of node columns to display per line in the primary grid 
(0 sets the number of nodes based on terminal width)

=item   B<-C>

toggle the use of the colors in the display

=item   B<-fillbg>

fill the background with black instead of using the terminal's default.  Use this
if the display looks bad on your colored or transparent background.  
Corresponding interactive command is "B<f>".

=item   B<-G>

toggle grid display

=item   B<-J>

toggle the display of job letters in the node grid.  This is handy because you can
see the node state "hidden" behind the job letter.

=item   B<-m> num

primary grid's maximum per-node CPU count.  Can be reduced from the default 
to force larger CPU nodes into their own secondary grid.

=item   B<-n>

remove spaces between each node in the primary grid display for a more
compact display. In the secondary grids, this removes spaces between
each set of 10 cpus.

=item   B<-q>

queue name for limiting the view of the grid and job list or "B<all>".  Only one 
name is supported at this time.  Corresponding interactive command is "B<L>".

=item   B<-Q>

toggle job queue display

=item   B<-s> num

the number of seconds to wait between display updates

=item   B<-S>

toggle state summary display

=item   B<-t>

toggle the display of jobs with status 'Q' (not running) in the jobs queue display.  
This can reduce the size of the queue display considerably in some environments.

(Mnemonic: I don't know, toggle?  B<-q> and B<-Q> were already used for something more important.)

=item   B<-u>

username(s) for limiting the view of the grid and job list.  Can be a
comma-separated list of usernames, "B<all>" (default), or "B<me>".  "B<me>" is a 
pseudonym for the username running slurmtop(1).

=back

=head1 INTERACTIVE COMMANDS

A number of single-key commands are recognized while slurmtop(1) is running.

The arrow keys, PageUp, and PageDown keys will scroll the display if it 
doesn't fit in your X terminal.

When prompted to type something, ctrl-G or the Enter key can be used to cancel the command.

=over 4

=item B<h>

Display help screen, version, and current settings

=item B<q>

Quit slurmtop(1)

=item   B<space>

Immediately update display

=item B<A>

Toggle array job compression. This defaults to 'on', so that all jobs that
make up an array are shown as a single entry represented by a single letter.
The total number of active cores for the compressed array is appended to the job line.
Type 'A' to expand the array so that each job has its own letter.

=item B<c>

Prompts for the number of node columns to display per line in the primary grid 
(0 sets the number of nodes based on terminal width)

=item B<C>

Toggle the use of the colors in the display

=item B<f>

Toggle filling the background with black instead of using the terminal's default.  
Use this if the display looks bad on your colored or transparent background.  
Corresponding command-line option is "B<-fillbg>".

=item B<G>

Toggle grid display

=item B<J>

Toggle the display of job letters in the node grid.  This is handy because you can
see the node state "hidden" behind the job letter.

=item B<L>

Prompt for a queue name.  The grid and job listing will be limited to the named
queue.  Input "B<all>" to remove all limitations (the default).  
Only one name is supported at this time.  Corresponding  command-line option 
is "B<-q>".

=item B<m>

Prompts for the primary grid's maximum per-node CPU count.  Can be reduced 
from the default to force larger CPU nodes into their own secondary grid.

=item B<N>

Toggle removing spaces between each node in the promary grid display for a more
compact display. In the secondary grids, this toggles removing spaces between
each set of 10 cpus.

=item B<Q>

Toggle job queue display

=item B<s>

Prompts for the number of seconds to wait between display updates

=item B<S>

Toggle the state summary display 

=item B<t>

Toggle the display of jobs with status 'Q' (not running) in the jobs queue display.  
This can reduce the size of the queue display considerably in some environments.

(Mnemonic: I don't know, toggle?  B<q> and B<Q> were already used for something more important.)

=item B<u>

Prompts for a username.  The grid and job listing will be limited to the named
user(s).  Input "B<all>" to remove all limitations (default), and "B<me>" to
limit to the current username running slurmtop(1).  If the username or "B<me>" is
prefixed with a B<+> or B<->, the username will be added or removed from the
list of usernames to be limited.  "B<a>" and "B<m>" are shortcuts for "B<all>" and
"B<me>".

=back

=head1 SUBWINDOW COMMANDS

=over 4

=item B</>

Prompts the user for a search string, for displaying the details of the specified
object.  The search can optionally begin with one of the following pattern specifiers
(think: mutt): B<~s> for a server, B<~n> for a node, or B<~j> for a job number.
If no pattern specifier is found, slurmtop will attempt to find the object that
best matches the search string. The string can be a server name, nodename, or a
job number.  Nodenames can optionally be followed by a space and the server
name.  Job numbers may optionally be followed by a dot and the server name.

If an object is found, a subwindow will be opened displaying details: i.e., 
a B<server status report>, a B<node status report>, or a B<job details report>.  

(Mnemonic: like using / to search for text in vi or less)

=item B<n>

Prompts the user for a node name.  A B<node status report> subwindow will be
displayed for the given node name.  This is the same report as if the user 
typed B</nodename>.

(Mnemonic: node status report)

=item B<l>

Prompts the user for a job id.  A B<job load report> subwindow will be
displayed for the given jobid.  

(Mnemonic: load average)

=back

=head2 Server Status Report

This subwindow shows the scheduling server's characteristics, including its
state, queued jobs statistics, and various scheduling defaults.

=over 4

=item B<q>

Exits the subwindow.

=back

=head2 Node Status Report

This subwindow shows the node's hardware characteristics, as well as its load average, 
and the IDs of the jobs running on that node.

=over 4

=item B<j>

Jumps you directly to the B<job details report> subwindow for the job running on CPU 0
(as if the user typed B</jobid> for that job number).

=item B<o>

If running with appropriate permission (e.g., root), sets the node I<offline>.

=item B<r>

If running with appropriate permission (e.g., root), I<restores> the node's state.

=item B<q>

Exits the subwindow.

=back

=head2 Job Load Report

This subwindow shows the current load average, the physical and available memory, 
and the number of sessions for the node on which the job is running.  Available 
physical memory will be negative in the event of swapping.

=over 4

=item B<j>

Jumps you directly to the B<job details report> subwindow for this job
(as if the user typed B</jobid> for that job number).

=item B<q>

Exits the subwindow.

=back

=head2 Job Details Report

This subwindow shows all the job characteristics maintained by the scheduler for
the specified job.

=over 4

=item B<l>

Jumps directly to the associated B<job load report> subwindow,
as if the user had used the B<l> command for that job number.

=item B<d>

If running with appropriate permission (e.g., job owner), I<deletes> the specified job.

=item B<H>

If running with appropriate permission (e.g., job owner) and if the job is in the
proper state (e.g., not running), places the job in the I<hold> (H) state.

=item B<R>

If running with appropriate permission (e.g., job owner) and if the job is in the
proper state (e.g., held), I<releases> the job from the hold (H) state.

=item B<r>

If running with appropriate permission (e.g., job owner) and if the job is in the
proper state (e.g., running), queues the job to be I<re-run>.

=item B<q>

Exits the subwindow.

=back

=head1 CONFIGURATION VARIABLES

slurmtop(1) has many configuration variables that can set on the command line,
interactively, or from configuration files.  When slurmtop(1) starts, it first
initializes these variables with built-in defaults, then reads in
F</etc/slurmtoprc>, then reads F<~/.slurmtoprc>, and finally parses the command line
arguments.  Note that several of the command line arguments and interactive
commands are toggles, they don't directly set the value of the configuration.
In contrast, the configuration files are never toggles.

slurmtop(1) is now largely auto-configurable, and often needs no special settings
in the configuration files to produce a reasonable display, other than perhaps the 
B<nodesort> and/or B<maxrows> variables, plus any personal preferences such as B<colorize>.

The configuration files may contain the following name=value pairs:

=over 4

=item B<colorize>

Use colors in the display, 1 or 0
(See "B<-C>" option and "B<C>" commmand)

=item B<columns>

The number of node columns to display per line in the primary grid 
(0 sets the number of nodes based on terminal width)
(See "B<-c>" option and "B<c>" commmand)

=item B<compact_summary>

Show node state summary line in compact format, 1 or 0

=item B<compress_arrays>

Compress array jobs, 1 or 0. This defaults to 'on' (1), so that all jobs that
make up an array are shown as a single entry represented by a single letter.
The total number of active cores for the compressed array is appended to the job line.
(See "B<-A>" option and "B<A>" commmand)

=item B<fillbg>

Fill background with black instead of using the terminal's default, 1 or 0. 
Use this if the display looks bad on your colored or transparent background.  
(See "B<-fillbg>" option and "B<f>" commmand)

=item B<host>

Comma-separated list of hostnames running the scheduler

=item B<maxcolums>

Number of columns in the large scrollable panel, positive integer, 300 default.
The terminal width cannot usefully be widened more than B<maxcolumns>
number of characters.

=item B<maxnodegrid>

Primary grid's maximum per-node CPU count, positive integer.  Defaults to the 
number of CPUs in B<show_cpu>.  Can be reduced from the default to force 
larger CPU nodes into their own secondary grid.  If B<show_cpu> not explicitly set,
sets B<show_cpu> to a list of CPU numnbers from 0 to B<maxnodegrid>-1.
(See "B<-m>" option and "B<m>" commmand)

=item B<maxrows>

Number of rows in the large scrollable panel, positive integer, 1500 default.
The number of lines required to display all grids and jobs must not exceed 
B<maxrows>. If it does, a helpful warning will be displayed on each refresh
indicating the value that B<maxrows> must have to view all grids and jobs.

=item B<nodesort>

Define the sorting method for the nodes in the main display grid.  The current
possible methods are (enclosed in quotes):

=over 4

=item B<ordered>

Preserves the order given from the resource manager without sorting; good for nodes
that don't follow a specific pattern or order.

=item B<lexical>

Simple alphabetical sort.  Fastest method for nodes with zero-padded names such
as I<node0023>.

=item B<integer>

The first numbers found for an integer sort.  Useful if you are unfortunate
enough to not have zero-padded nodes, like I<node1> and I<node23>.

=item B<mixed>

Lexical sort followed by an integer sort.  Should give meaningful results in
all cases, especially if you are *really* unfortunate enough to not have
zero-padded nodes and have different leading strings, like I<lin34> and I<win5>.
This is the default.

=item B<mixed2>

Mixed sort followed by another mixed sort.  Useful for admins that
name their nodes after rack positions, like I<rack1node4> and I<rack10node12>.

=back

=item B<nodesort_host>

Defines sorting methods on a per-server basis.  It is a comma-delimited list of
"host=method" pairs surrounded by parentheses, i.e., 
I<nodesort_host=(serv1=ordered,serv2=lexical)>.  The host part is first checked as
an exact match, otherwise is interpreted as a perl regexp (first match wins).

=item B<nospace>

Remove spaces between each node in the primary grid display for a more
compact display, 1 or 0. In the secondary grids, this removes spaces between
each set of 10 cpus.
(See "B<-n>" option and "B<N>" commmand)

=item B<show_cpu>

Comma-separated list of CPU numbers to display, e.g., 
I<show_cpu=0,1,2,3,4,5,6,7,8,9,10,11,12>. If B<maxnodegrid> not explicitly set, 
sets B<maxnodegrid> to the number of CPUs in the list.

=item B<show_grid>

Show the grid display, 1 or 0.
(See "B<-G>" option and "B<G>" commmand)

=item B<show_jobs>

Enable the display of job letters in the node grid, 1 or 0.  This is handy because you can
see the node state "hidden" behind the job letter.
(See "B<-J>" option and "B<J>" commmand)

=item B<show_onlyq>

Queue name to limit the view in the grid and job list.  Only one name is
supported at this time.
(See "B<-q>" option and "B<L>" commmand)

=item B<show_qqueue>

Toggle the display of jobs with status 'Q' (not running) in the jobs queue display,
1 or 0.  This can reduce the size of the queue display considerably in some 
environments.
(See "B<-t>" option and "B<t>" commmand)

=item B<show_queue>

Show the job queue display, 1 or 0.
(See "B<-Q>" option and "B<Q>" commmand)

=item B<show_summary>

Display the state summary at the top of the screen, 1 or 0.
(See "B<-S>" option and "B<S>" commmand)

=item B<show_user>

Username(s) to limit the view in the grid and job list.  Can be a comma-separated
list of users, "B<all>" (default), or "B<me>".  "B<me>" is a pseudonym for the username 
running slurmtop(1).
(See "B<-u>" option and "B<u>" commmand)

It might be reasonable for a site to have B<show_user=me> in F</etc/slurmtoprc>
and for admin users to have B<show_user=all> in their own F<~/.slurmtoprc>.  
Members of a group might want all of their groupmates's usernames in their own
F<~/.slurmtoprc>.

=item B<sleeptime>

Number of seconds to wait between display updates, positive integer.
(See "B<-s>" option and "B<s>" commmand)

=back

A sample configuration file:

    # I'm colorblind and don't see colors well
    colorize=0

    # my 20 CPU machines should get separate grids
    maxnodegrid=19

    # all of my scheduler servers
    host=teraserver,bigbird,testhpc

    # teraserver has strict naming, testhpc should preserve the scheduler's order
    nodesort_host=(.*\.usc.edu=integer,teraserver=lexical,testhpc=ordered)

=head1 SMP ENVIRONMENTS

slurmtop(1) replicated the pbstop functionality for SLURM clusters, and was run
on a number of large clusters, including a 644-node Dell Sandybridge cluster 
with 10,304 cores, and an SGI UV 1000 NUMAlink system with 1,536 cores.
With this kind of pedigree, slurmtop(1) is fairly flexible.

=head1 FILES

=over 4

=item F</etc/slurmtoprc>

The global configuration file  

=item F<~/.slurmtoprc>

The personal configuration file.

=back

=head1 SEE ALSO

=over 4

=item Slurm(3pm)

=back

=head1 BUGS

The curses code is very inefficient and the display gets corrupted at times.
It can't produce plain text output like top's "batch" mode.
grep FIXME from slurmtop for more!

=head1 AUTHOR

slurmtop(1) was developed by Dennis McRitchie E<lt>dmcr@princeton.eduE<gt> at 
Princeton University and is based on modified pbstop code (schedtop.pm) along 
with the SLURM API.  pbstop was originally written by Garrick Staples.  
The node grid and lettering concept was from Dennis Smith.  It was enhanced by 
Gareth Williams at CSIRO to provide the ability to support/expand/collapse array jobs.
