#!/usr/local/bin/perl

# ===========================================================================
#
#                            PUBLIC DOMAIN NOTICE
#                     Center for Information Technology (CIT)
#                        National Institute of Health (NIH)
#
#  This software/database is a "United States Government Work" under the
#  terms of the United States Copyright Act.  It was written as part of
#  the author's official duties as a United States Government employee and
#  thus cannot be copyrighted.  This software is freely available
#  to the public for use.  The Center for Information Technology, The
#  National Institutes of Health, and the U.S. Government have not placed
#  any restriction on its use or reproduction.
#
#  Although all reasonable efforts have been taken to ensure the accuracy
#  and reliability of the software and data, CIT, NIH and the U.S.
#  Government do not and cannot warrant the performance or results that
#  may be obtained by using this software or data. CIT, NIH and the U.S.
#  Government disclaim all warranties, express or implied, including
#  warranties of performance, merchantability or fitness for any particular
#  purpose.
#
#  Please cite the author and the "NIH Biowulf Cluster" in any work or product
#  based on this material.
#
# ===========================================================================

# This script is an adaptation of the swarm script for use with SLURM.
use Getopt::Long qw(GetOptionsFromString :config no_ignore_case);
Getopt::Long::Configure("bundling"); # allows option bundling
use POSIX qw(strftime floor ceil);
use File::Temp qw/tempfile tempdir/;
use File::Basename qw/basename/;
use File::Spec;
require File::Spec::Unix;
use File::Path qw(make_path);
#use lib "/usr/local/slurm/lib/perl5/site_perl/5.24.3/x86_64-linux-thread-multi-ld";
use lib "/usr/local/slurm/lib64/perl5";
use Slurm;
use FileHandle;
use Text::Wrap;
use Sys::Hostname;

use strict;

dieWithError("Don't run on helix!  Run on the biowulf cluster!") if (hostname() =~ /helix/);

#use Data::Dump qw/pp/;

$SIG{HUP}  = \&catch_sig;
$SIG{INT}  = \&catch_sig;
$SIG{KILL} = \&catch_sig;
$SIG{TERM} = \&catch_sig;
$SIG{STOP} = \&catch_sig;

my $PAR;
my $SWARM;
setDefaults();
privilegedUser();

my %OPT;
my %SLURMOPT1;
my %SLURMOPT2;
my $SBATCHOPT;
setOptions();

validatePartition();
parseCommands();
avoidSbatchCommands();
distributeCommands();
adjustTime();
report();
writeCommandFiles() unless $OPT{'no-scripts'};
writeBatchScript() unless $OPT{'no-scripts'};
sleep(2) unless $OPT{debug}; # sleep 2 seconds to prevent weirdness
submitSlurm();
createSymlink() unless $OPT{'no-scripts'};
writeLog() unless $OPT{'no-log'};

#===============================================================================
sub setDefaults
{
  $PAR->{programname} = 'swarm';                                # Program name is swarm, duh
  $PAR->{version} = '23.6.1';                                   # Version
  $PAR->{b} = 1;                                                # Default bundle value of 1
  $PAR->{user} = (getpwuid($>))[0];                             # Who is running swarm?
  chomp($PAR->{host} = `/bin/hostname`);                        # On what host is swarm being run?
  $PAR->{pwd} = $ENV{'PWD'};                                    # what directory is this?
  $PAR->{shell} = "/bin/bash";                                  # default shell of command scripts
  $PAR->{date} = strftime("%b %d %Y %T", (localtime)[0 .. 5]);  # default datestamp formatting
  $PAR->{'comment-char'} = '#';                                 # All content beginning with this character is deleted
  $PAR->{logfile} = "/usr/local/logs/swarm.log";                # The logfile which will be updated
  $PAR->{tempdir_index} = '/usr/local/logs/swarm_tempdir.idx';  # Keep index of tempdirs
  $PAR->{maxarraysize} = 1000;                                  # Limits number of subjobs per swarm
  $PAR->{maxmemsize} = 3000;                                    # Absolute maximum memory of all possible nodes available, in GB
  @{$PAR->{disallowed_partitions}} = ("huygens","interactive"); # Disallow these partitions
  $PAR->{largemem_threshold} = 499;                             # Jobs requiring this many GB or less can't run on largemem
  $PAR->{helpwidth} = 80;                                       # Maximum width of the help page as printed with --help or -h
  $PAR->{default_partition} = "norm";


# Create the basedir where the batch script and command scripts will be written
  $PAR->{basedir} = "/spin1/swarm/$PAR->{user}";
  isDirCreateable($PAR->{basedir}) || dieWithError("can't write to $PAR->{basedir}");
  if (!-d $PAR->{basedir}) {
    mkdir $PAR->{basedir} || dieWithError("Can't create swarm script basedir!");
    chmod 02750,$PAR->{basedir};
  }

  dieWithError("Can't find sbatch.  Is slurm in your path?") unless (`which sbatch 2>/dev/null`); # Make sure sbatch is accessible

# Get configuration for partitions
  my $slurm = Slurm::new();
  my $x = $slurm->load_partitions();
  foreach my $i (@{$x->{partition_array}}) {
    next if (grep(/^$i->{name}$/,@{$PAR->{disallowed_partitions}}));
    $PAR->{slurm_partitions}{$i->{name}}{default_time} = $i->{default_time};
    $PAR->{slurm_partitions}{$i->{name}}{max_time} = $i->{max_time};
  }

# Options data structure
#
# * d = description that is displayed by --help
# * o = array reference of options shown to user; if o doesn't exist, it is assumed the option is the same as entry key
# * t = type, either 's', 'i', 'f', or none
# * e = explanatory type for the --help display
# * E = environment variable associated with option
# * h = hidden from --help display
# * G = group id
# * O = order id
# * D = division id

  $PAR->{options} = [
# Basic
    { 'f'            => {d=>"DEPRECATED: name of file with list of command lines to execute, with a single command line per subjob",o=>['f','file'],t =>'s',e =>'[file]',G=>'Basic',D=>1},},
    { 'g'            => {d=>"gb per process (can be fractions of GB, e.g. 3.5)",o=>['g','gb-per-process'],t=>'f',e=>'[float]',G=>'Basic',D=>1}, },
    { 't'            => {d=>"threads per process (can be an integer or the word auto).  This option is only valid for multi-threaded swarms (-p 1)",o=>['t','threads-per-process'],t =>'s',e=>'[int]/"auto"',G=>'Basic',D=>1},},
    { 'p'            => {d=>"processes per subjob (default = 1), this option is only valid for single-threaded swarms (-t 1)",o=>['p','processes-per-subjob'],t=>'i',e=>'[int]',G=>'Basic',D=>1},},
    { 'b'            => {d=>"bundle more than one command line per subjob and run sequentially (this automatically multiplies the time needed per subjob)",o=>['b','bundle'],t=>'i',e =>'[int]',G=>'Basic',D=>1},},
    { 'noht'         => {d=>"don't use hyperthreading, equivalent to slurm option --threads-per-core=1",G=>'Basic',D=>1},},
    { 'usecsh'       => {d=>"use tcsh as the shell instead of bash",G=>'Basic',D=>1},},
    { 'err-exit'     => {d=>"exit the subjob immediately on first non-zero exit status", G=>'Basic',D=>1},},
    { 'module'       => {d=>"provide a list of environment modules to load prior to execution (comma delimited)",o=>['m','module'],t=>'s',e=>'[str]',G=>'Basic',D=>1},},
    { 'no-comment'   => {d=>"don't ignore text following comment character $PAR->{'comment-char'}",G=>'Basic',D=>1},},
    { 'comment-char' => {d=>"use something other than $PAR->{'comment-char'} as the comment character",o=>['c','comment-char'],t=>'s',e=>'[char]',G=>'Basic',D=>1},},
    { 'maxrunning'   => {d=>"limit the number of simultaenously running subjobs",t=>'i',e=>'[int]',G=>'Basic',D=>1},},
    { 'merge-output' => {d=>"combine STDOUT and STDERR into a single file per subjob (.o)",G=>'Basic',D=>1},},
    { 'logdir'       => {d=>"directory to which .o and .e files are to be written (default is current working directory)",t=>'s',e=>'[dir]',G=>'Basic',D=>1},},
    { 'noout'        => {d=>"completely throw away STDOUT",G=>'Basic',D=>1},},
    { 'noerr'        => {d=>"completely throw away STDERR",G=>'Basic',D=>1},},
    { 'time-per-command' => {d=>"time per command (same as --time)",t=>'s',e=>'[str]',G=>'Basic',D=>1},},
    { 'time-per-subjob'  => {d=>"time per subjob, regardless of -b or -p",t=>'s',e=>'[str]',G=>'Basic',D=>1},},
# Development
    { 'no-scripts'   => {d=>"don't create temporary swarm scripts (with --debug or --devel)",G=>'Development',D=>1},},
    { 'no-run'       => {d=>"don't actually run",G=>'Development',D=>1},},
    { 'debug'        => {d=>"don't actually run",G=>'Development',D=>1},},
    { 'devel'        => {d=>"combine --debug and --no-scripts, and be very chatty",G=>'Development',D=>1},},
    { 'verbose'      => {d=>"can range from 0 to 6, with 6 the most verbose",o =>['v','verbose'],t=>'i',e=>'[int]',G=>'Development',D=>1},},
    { 'silent'       => {d=>"don't give any feedback, just jobid",G=>'Development',D=>1},},
    { 'h'            => {d=>"print this help message",o=>['h','help'],G=>'Development',D=>1},},
    { 'version'      => {d=>"print version and exit",o=>['V','version'],G=>'Development',D=>1},},
# Advanced
#    { 'evaluate'     => {d=>"only keep commands that evaluate as true (EXPERIMENTAL)",G=>'Advanced',D=>1},},
# Hidden
    #{ 'logfile'      => {d=>"use alternate logfile",o=>['logfile'],G=>'Hidden',D=>1},},
# sbatch
    { 'job-name'     => {d=>"set the name of the job", o => ['J','job-name'],t=>'s',e=>'[str]',E=>'SBATCH_JOB_NAME',G=>'sbatch',D=>3},},
    { 'dependency'   => {d=>"set up dependency (i.e. run swarm before or after)",t=>'s',e=>'[str]',G=>'sbatch',D=>3},},
    { 'time'         => {d=>"change the walltime for each subjob (default is 04:00:00, or 4 hours)",t=>'s',e=>'[str]',E=>'SBATCH_TIMELIMIT',G=>'sbatch',D=>3},},
    { 'licenses'     => {d=>"obtain software licenses (e.g. --licenses=matlab)",o=>['L','licenses'],t=>'s',e=>'[str]',G=>'sbatch',D=>3},},
    { 'partition'    => {d=>"change the partition (default is $PAR->{default_partition})",t=>'s',e=>'[str]',E=>'SBATCH_PARTITION',G=>'sbatch',D=>3},},
    { 'gres'         => {d=>"set generic resources for swarm",t=>'s',e=>'[str]',G=>'sbatch',D=>3},},
    { 'qos'          => {d=>"set quality of service for swarm",t=>'s',e=>'[str]',E=>'SBATCH_QOS',G=>'sbatch',D=>3},},
    { 'reservation'  => {d=>"select a slurm reservation",t=>'s',e=>'[str]',E=>'SBATCH_RESERVATION',G=>'sbatch',D=>3},},
    { 'exclusive'    => {d=>"allocate a single node per subjob, same as -t auto",E=>'SBATCH_EXCLUSIVE',G=>'sbatch',D=>2},},
    { 'sbatch'       => {d=>"add sbatch-specific options to swarm; these options will be added last, which means that swarm options for allocation of cpus and memory take precedence",t=>'s',e=>'[str]',G=>'sbatch',D=>4},},
  ];
}
#==============================================================================
sub printOptions
{
  $Text::Wrap::columns = $PAR->{helpwidth};

  print "Usage: swarm [swarm options] [sbatch options] swarmfile\n";

  my $pG;
  foreach my $x (@{$PAR->{options}}) {
    my ($k) = (keys %{$x});
    next if ((defined $x->{$k}{G}) && ($x->{$k}{G} eq 'Hidden') && (!$PAR->{privilegedUser}));
    my ($f,$a);
    my @option_strings = (defined $x->{$k}{o}) ? @{$x->{$k}{o}} : ($k);
    my @n;
    foreach my $s (@option_strings) {
      if (length($s) == 1) { push @n,"-${s}"; }
      else { push @n,"--${s}"; }
    }
    my $opt_list=join(",",@n);
    my $opt_example = (defined $x->{$k}{e}) ? $x->{$k}{e} : "";
    my @text = split(/ /,$x->{$k}{d});

# Print option group title
    if (defined $x->{$k}{G}) {
      if ((not defined $pG) || ($pG ne $x->{$k}{G})) {
        $pG = $x->{$k}{G};
        print "\n$x->{$k}{G} options:\n\n";
      }
    }

# Print option and example
    if (length("$opt_list $opt_example") > 25) {
      my $n = $PAR->{helpwidth}-4;
      $f = sprintf("  %-${n}s ","$opt_list $opt_example");
    }
    else {
      $f = sprintf("  %-25s ","$opt_list $opt_example");
    }
    $a = sprintf("  %-25s ",'');
    print wrap($f,$a,@text);
    print "\n";
  }

# Print input environment variables
  print "\nEnvironment variables:\n\n";

  print "  The following environment variables will affect how sbatch allocates\n";
  print "  resources:\n\n";

  foreach my $x (@{$PAR->{options}}) {
    my ($k) = (keys %{$x});
    next if ((defined $x->{$k}{G}) && ($x->{$k}{G} eq 'Hidden') && (!$PAR->{privilegedUser}));
    if (defined $x->{$k}{E}) {
      printf("  %-24s  Same as --%s\n",$x->{$k}{E},$k);
    }
  }

  print "\n  The following environment variables are set within a swarm:\n\n";
  print "  SWARM_PROC_ID          can be 0 or 1\n";
  print "\nFor more information, type \"man swarm\".\n";
  print "\nLast modification date: 07 Jun 23 (David Hoover)\n";

  exit;
}
#==============================================================================
sub distributeCommands
# The commands have been parsed into an array, and the number of processes per node has been determined.  Now figure
# out how to distribute the commands to the command scripts.
{
  $PAR->{commands} = scalar(@{$SWARM->{COMMANDS}});

  foreach my $i (0 .. $#{$SWARM->{COMMANDS}}) {
    my ($sj,$cpu);
# No bundling, no packing
    if ($PAR->{b} == 1) {
      if (!$OPT{p}) {
        $sj = $i % $PAR->{maxarraysize};
        $cpu = 0;
      }
# No bundling, but yes packing
      else {
        $sj = (floor($i/2)) % $PAR->{maxarraysize};
        $cpu = $i % 2;
      }
    }
# Yes bundling, but no packing
    else {
      if (!$OPT{p}) {
        $sj = (floor($i/$PAR->{b})) % $PAR->{maxarraysize};
        $cpu = 0;
      }
# Yes bundling, and yes packing
      else {
        $sj = (floor($i/(2*$PAR->{b}))) % $PAR->{maxarraysize};
        $cpu = $i % 2;
      }
    }
# Add the command number to the list
    push @{${$SWARM->{CMDLISTS}}[$sj][$cpu]},$i;
  }

# Update bundle factor
  my $max=0;
  foreach my $sj (0 .. $#{$SWARM->{CMDLISTS}}) {
    my $num_cmds_in_subjob=0;
    $num_cmds_in_subjob += scalar(@{${$SWARM->{CMDLISTS}}[$sj][0]});
    $num_cmds_in_subjob += scalar(@{${$SWARM->{CMDLISTS}}[$sj][1]}) if (defined ${$SWARM->{CMDLISTS}}[$sj][1]);
    $max = $num_cmds_in_subjob if ($num_cmds_in_subjob > $max);
  }
  my $b_old = $OPT{b};
  $OPT{b} = $max;
  $PAR->{b} = $max;
  $PAR->{b} /= $OPT{p} if ($OPT{p});
  my $b_new = $PAR->{b}; 

  print "\nBundle factor was changed from $b_old to $b_new" if (($b_old != $b_new) && ($OPT{verbose} > 0));

  $PAR->{subjobs} = scalar(@{$SWARM->{CMDLISTS}});
}
#==============================================================================
sub report
{
  print "\n" if ($OPT{devel});
# Report where the command files are written
  print "Command files written to $PAR->{tempdir}\n" if (($OPT{verbose} > 1) && (!$OPT{'no-scripts'}));
  my $len1 = length($#{$SWARM->{CMDLISTS}});
  my $len2 = length($#{$SWARM->{COMMANDS}});

# Report what modules will be loaded
  print "Loading modules $OPT{module}\n" if (($OPT{verbose} > 2) && ($OPT{module}));

# Report what modules will be loaded
  print "Using comment character $OPT{'comment-char'}\n" if (($OPT{verbose} > 2) && ($OPT{'comment-char'}));

# update cpus-per-task and mem for single-threaded swarms
  if (defined $OPT{p}) {
    $SLURMOPT2{"cpus-per-task"} = $OPT{p};
    $SLURMOPT2{"mem"} = $SLURMOPT2{"mem"}*$OPT{p};
  }

# Calculate total number of cpus allocated
  $PAR->{cpus} = $PAR->{subjobs};
  if (defined $OPT{p}) {
    $PAR->{cpus} *= $OPT{p};
  }
  elsif ($OPT{t}) {
    $PAR->{cpus} *= $OPT{t};
  }

# Graphical representation of the swarm
  my $top = "(0tqq (B";
  my $pre = "(0x   ".$top;
#(0x   mqq (Btext # maybe in the future
#(0mqq (Btext

  my $outputfiles;
  print "-"x60,"\n"."SWARM\n" if ($OPT{verbose} > 3);
  foreach my $sj (0 .. $#{$SWARM->{CMDLISTS}}) {

# How many commands will be run in this subjob?
    my $num_cmds_in_subjob=0;
    $num_cmds_in_subjob += scalar(@{${$SWARM->{CMDLISTS}}[$sj][0]});
    $num_cmds_in_subjob += scalar(@{${$SWARM->{CMDLISTS}}[$sj][1]}) if (defined ${$SWARM->{CMDLISTS}}[$sj][1]);
    if ($OPT{verbose} > 3) {
      printf ("%ssubjob %${len1}d: %${len2}d command%s (%d cpu%s, %.2f gb)\n",
        $top,
        $sj,
        $num_cmds_in_subjob,
        write_s_if_needed($num_cmds_in_subjob),
        $SLURMOPT2{"cpus-per-task"},
        write_s_if_needed($SLURMOPT2{"cpus-per-task"}),
        ($SLURMOPT2{"mem"}/1024),
      );
    }
    if ($OPT{verbose} > 4) {
      foreach my $cpu (0 .. $#{${$SWARM->{CMDLISTS}}[$sj]}) {     
        if ($OPT{verbose} > 5) {
          my @tmp;
          foreach my $cmd (@{${$SWARM->{CMDLISTS}}[$sj][$cpu]}) { push @tmp,${$SWARM->{COMMANDS}}[$cmd]; }
          print $pre . (join ';',@tmp)."\n";
        }
        else {
          print $pre . (join ';',@{${$SWARM->{CMDLISTS}}[$sj][$cpu]})."\n";
        }
        $outputfiles++;
      }
    }
  }

  if ($OPT{verbose} > 3) {
    print "-"x60,"\n";
    printf "%${len1}d subjob%s, %${len2}d commands, %d output file%s\n",$PAR->{subjobs},write_s_if_needed($PAR->{subjobs}),$PAR->{commands},$outputfiles,write_s_if_needed($outputfiles);
  }
  if ($OPT{verbose} > 0) {
    print "$PAR->{commands} commands run in $PAR->{subjobs} subjob".write_s_if_needed($PAR->{subjobs}).", ";
    print "each command requiring $OPT{g} gb and $OPT{t} thread".write_s_if_needed($OPT{t});
    if ($OPT{p}) {
      print ", packing $OPT{p} processes simultaneously per subjob";
    }
    if ($PAR->{b} > 1) {
      print ", running $PAR->{b} processes serially per subjob";
    }
    if ((not defined $OPT{t}) || ($OPT{t} ne 'auto')) {
# allocating by core, not by cpu
      my $cpus = $SLURMOPT2{"cpus-per-task"};
      ($cpus%2) && $cpus++;
      $cpus *= $PAR->{subjobs};
      my $cores = $cpus/2;
      print ", allocating $cores core".write_s_if_needed($cores)." and $cpus cpu".write_s_if_needed($cpus);
    }
    if ($PAR->{multinode}) {
      print " across $PAR->{multinode} nodes each";
    }
    if (defined $OPT{maxrunning}) {
      print ", with $OPT{maxrunning} subjobs running simultaneously";
    }
    print "\n";
  }

  print "\n" if ($OPT{devel});
}
#==============================================================================
sub write_s_if_needed
{
  if (shift > 1) { return "s"; }
  else { return ""; }
}
#==============================================================================
sub insertLmodInit
# Insert two lines of code to load modules.  NOTE: This only works in a bash script.  It is meant to be
# inserted into the batch script, not the command scripts, which may use bash or tcsh to run.
{
  return unless $OPT{module};
  my $str;
  $str .= "source /usr/local/lmod/lmod/lmod/init/bash\n";
  $str .= "module load $OPT{module}\n";
  return $str;
}
#==============================================================================
sub writeCommandFiles
{
# tcsh module
  my $module_extra;
  if (($OPT{module}) && ($OPT{usecsh})) {
    $module_extra = "module load $OPT{module} ; ";
  }

  mkdir $PAR->{tempdir};
  chmod 02750,$PAR->{tempdir};
  append_to_tempdir_index(); # Record tempdir creation in the index
  my $len = length($#{$SWARM->{CMDLISTS}});
# Walk through each subjob
  foreach my $sj (0 .. $#{$SWARM->{CMDLISTS}}) {
    my %fileContents;
    ###my $suffix = sprintf "%0${len}d",$sj;
    ###my $commandFileName = $PAR->{tempdir}."/cmd.$suffix";
    my $commandFileName = $PAR->{tempdir}."/cmd.$sj";
# Walk through each commandline
    foreach my $cpu (0 .. $#{${$SWARM->{CMDLISTS}}[$sj]}) {
      foreach my $comm (@{${$SWARM->{CMDLISTS}}[$sj][$cpu]}) {
# Change command filename if packing
        ###$commandFileName = $PAR->{tempdir}."/cmd.$suffix"."_".($cpu) if $OPT{p};
        $commandFileName = $PAR->{tempdir}."/cmd.$sj"."_".($cpu) if $OPT{p};
        $fileContents{$commandFileName} .= "( $module_extra " . ${$SWARM->{COMMANDS}}[$comm]." )\n";
      }
    }
# Print the contents
    foreach my $file (sort keys %fileContents) {
      printToFile($file,$fileContents{$file},0640);
    }
  }
}
#==============================================================================
sub printToFile
# Open file, write contents, flush and close.  'nuff said.
{
  my ($file,$contents,$mode) = @_;
  my $fh = FileHandle->new;
  if ($fh->open("> $file")) {
    print $fh $contents;
    $fh->flush;
    $fh->close;
  }
  else { dieWithError("Can't write to $file\n"); }
  chmod $mode,$file;
}
#==============================================================================
sub appendToFile
# Open file with append, write contents, flush and close.  'nuff said.
{
  my ($file,$contents) = @_;
  my $fh = FileHandle->new;
  if ($fh->open(">> $file")) {
    print $fh $contents;
    $fh->flush;
    $fh->close;
  }
  else { dieWithError("Can't write to $file\n"); }
}
#==============================================================================
sub writeBatchScript
#
# This is the file that sbatch calls.  Commands are normally run as a simple
# shell call, e.g.
#
#   bash [ commandfile ]
{
  my $fileContents;
  my $len = length($#{$SWARM->{CMDLISTS}});
  my $len3 = length($OPT{p}) if $OPT{p};
  $fileContents .= "#!/bin/bash\n";

# The swarm.batch file loads the modules, and the command scripts will inherit the environment.  Thie
# can be screwed up if the user fiddles with the environment within the commands.
  $fileContents .= insertLmodInit();

# Generate the rest of the batch script.  NOTE: The path to the temporary directory is hard-coded in
# the swarm.batch script.  This allows a swarm to be rerun, albeit with the correct sbatch options.
# These can be found in the swarm logfile.
###  $fileContents .= <<EOF1;
###d=$PAR->{tempdir}
###z=\$(printf '%0${len}d' \${SLURM_ARRAY_TASK_ID}) 
###EOF1
  $fileContents .= "d=$PAR->{tempdir}\n";

# Write the command executed block at the top of the output
  my $ce_top = "---- COMMAND EXECUTED: ---------------------------------------------------------";
  my $ce_bot = "--------------------------------------------------------------------------------";

  (my $stupid_logdir = $OPT{logdir})=~s/"/\\"/g;
  $stupid_logdir=~s/`/\\`/g;
  $stupid_logdir=~s/\$/\\\$/g;
  (my $stupid_job_name = $PAR->{'job-name'})=~s/"/\\"/g;;
  $stupid_job_name=~s/`/\\`/g;
  $stupid_job_name=~s/\$/\\\$/g;

# Run the commands in series on a single node
  if ($OPT{p}) {
    foreach my $i (0 .. $OPT{p}-1) {
      ###my $tag = sprintf "%0${len3}d",$i;
      ###$fileContents .= "[[ -s \${d}/cmd.\${z}_$tag ]] && ( /bin/echo '$ce_top' && /bin/cat \${d}/cmd.\${z}_$tag && /bin/echo '$ce_bot' && $PAR->{shell} \${d}/cmd.\${z}_$tag ";
      $fileContents .= "[[ -s \${d}/cmd.\${SLURM_ARRAY_TASK_ID}_$i ]] && ( /bin/echo '$ce_top' && /bin/cat \${d}/cmd.\${SLURM_ARRAY_TASK_ID}_$i && /bin/echo '$ce_bot' && SWARM_PROC_ID=$i $PAR->{shell} \${d}/cmd.\${SLURM_ARRAY_TASK_ID}_$i ) ";
# Determine output/error redirects
      if ($OPT{'merge-output'}) {
        $fileContents .= "1> \"${stupid_logdir}/${stupid_job_name}_\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}_$i.o\" 2> \"${stupid_logdir}/${stupid_job_name}_\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}_$i.o\" ";
      }
      else {
        if ($OPT{noout}) {
          $fileContents .= "1> /dev/null ";
        }
        else {
          $fileContents .= "1> \"${stupid_logdir}/${stupid_job_name}_\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}_$i.o\" ";
        }
        if ($OPT{noerr}) {
          $fileContents .= "2> /dev/null ";
        }
        else {
          $fileContents .= "2> \"${stupid_logdir}/${stupid_job_name}_\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}_$i.e\" ";
        }
      }
      $fileContents .= " &\n";
    }
    $fileContents .= "wait\n";
    $fileContents .= "exitcode=0\n";
  }
  else {
    ###$fileContents .= "/bin/echo '$ce_top' && /bin/cat \${d}/cmd.\${z} && /bin/echo '$ce_bot' && $PAR->{shell} \${d}/cmd.\${z}\n";
    $fileContents .= "/bin/echo '$ce_top' && /bin/cat \${d}/cmd.\${SLURM_ARRAY_TASK_ID} && /bin/echo '$ce_bot' && SWARM_PROC_ID=0 $PAR->{shell} \${d}/cmd.\${SLURM_ARRAY_TASK_ID}\n";
# Capture final exit code
    $fileContents .= "exitcode=\$?\n";
  }

  $fileContents .= "exit \$exitcode\n"; # Exit with the final exit code from the given command

# Print the batch file
  printToFile($PAR->{batchfile},$fileContents,0640);
}
#==============================================================================
sub submitSlurm
{
  $PAR->{sbatch_options} .= "--array=0-$#{$SWARM->{CMDLISTS}}";
  $PAR->{sbatch_options} .= '%'.$OPT{maxrunning} if (defined $OPT{maxrunning});

  (my $stupid_logdir = $OPT{logdir})=~s/"/\\"/g;
  $stupid_logdir=~s/`/\\`/g;
  $stupid_logdir=~s/\$/\\\$/g;
  (my $stupid_job_name = $PAR->{'job-name'})=~s/"/\\"/g;
  $stupid_job_name=~s/`/\\`/g;
  $stupid_job_name=~s/\$/\\\$/g;

  $PAR->{sbatch_options} .= " --job-name=\"".$stupid_job_name."\"" unless ($SLURMOPT2{'job-name'});

# output and error handled in the batch script
  if ($OPT{p}) {
    $PAR->{sbatch_options} .= " --output=/dev/null";
    $PAR->{sbatch_options} .= " --error=/dev/null";
  }
  else {
    if ($OPT{'merge-output'}) {
      $PAR->{sbatch_options} .= " --output=\"".$stupid_logdir."/".$stupid_job_name."_\%A_\%a.o\"";
    }
    else {
      if ($OPT{noout}) {
        $PAR->{sbatch_options} .= " --output=/dev/null";
      }
      else {
        $PAR->{sbatch_options} .= " --output=\"".$stupid_logdir."/".$stupid_job_name."_\%A_\%a.o\"";
      }
      if ($OPT{noerr}) {
        $PAR->{sbatch_options} .= " --error=/dev/null";
      }
      else {
        $PAR->{sbatch_options} .= " --error=\"".$stupid_logdir."/".$stupid_job_name."_\%A_\%a.e\"";
      }
    }
  }

# slurm options
  foreach my $i (sort keys %SLURMOPT1) {
    $PAR->{sbatch_options} .= " --$i" if ($SLURMOPT1{$i});
  }
  foreach my $i (sort keys %SLURMOPT2) {
    if ($i eq 'job-name') {
      (my $s = $SLURMOPT2{$i})=~s/"/\\"/g;
      $s=~s/`/\\`/g;
      $s=~s/\$/\\\$/g;
      $PAR->{sbatch_options} .= " --${i}=\"$s\"" if ($SLURMOPT2{$i});
    }
    else {
      $PAR->{sbatch_options} .= " --${i}=$SLURMOPT2{$i}" if ($SLURMOPT2{$i});
    }
  }
  if ($SBATCHOPT) {
    $PAR->{sbatch_options} .= " ".$SBATCHOPT;
  }

  print "sbatch $PAR->{sbatch_options} $PAR->{batchfile}\n\n" if ($OPT{verbose} > 1);
  unless ($OPT{'no-run'}) {
    if (!-d $OPT{logdir}) {
      make_path($OPT{logdir}) || dieWithError("can't create logdir $OPT{logdir}");
    }
    elsif (not writeable($OPT{logdir})) {
      dieWithError("can't write to logdir $OPT{logdir}");
    }
    my $slurm_response = `sbatch $PAR->{sbatch_options} $PAR->{batchfile}`;
    print $slurm_response;
    if ($slurm_response=~/^(\d+)$/) {
      $PAR->{jobid} = $1;
    }
    else {
# Clean up!
      system("/usr/bin/rm -rf $PAR->{tempdir}");
      dieWithError("Something went wrong with sbatch!  I don't know the jobid!");
    }
  }
}
#==============================================================================
sub writeable
# Is the directory writeable by the user?
{
  my $dir = shift;
  my (undef, $filename) = tempfile(DIR=>$dir,OPEN => 0, UNLINK=>1);
  if (open FILE, ">$filename") {
    unlink $filename;
    return 1;
  }
  else {
    return;
  }
}
#==============================================================================
sub createSymlink
# Create a symlink to the temporary directory with the jobid
{
  my $num;
  if ($PAR->{jobid}) { 
    symlink ("$PAR->{tempdir}","$PAR->{basedir}/$PAR->{jobid}");
  }
}
#==============================================================================
sub parseCommands
{
  open(COMMANDFILE, "$PAR->{Commandfile}") || dieWithError("Can't read $PAR->{Commandfile}: $!");
  #my @COMMANDS = evaluateCommands(mergeLineContinuations(<COMMANDFILE>));
  my @COMMANDS = mergeLineContinuations(<COMMANDFILE>);
  close COMMANDFILE;
  foreach (@COMMANDS) {
    chomp;
    s/#SWARM.*//g; # remove #SWARM directives
    s/\r//g;         # remove carriage returns
    next if /^\s*$/; # ignore blank lines
    if (!$OPT{'no-comment'}) {
      next if /^\s*$PAR->{'comment-char'}/; # remove comment lines
      s/$PAR->{'comment-char'}.*//g; # remove comment lines
    }
    s/[\s;]+$//;     # don't allow final semicolons or spaces
    push @{$SWARM->{COMMANDS}},$_;
  }

# Make sure that the swarmfile actually has commands in it!
  if ((not defined $SWARM->{COMMANDS}) || (ref($SWARM->{COMMANDS}) != /ARRAY/) || (@{$SWARM->{COMMANDS}} < 1)) {
    dieWithError("No commands in swarmfile $PAR->{Commandfile}");
  }
}
#==============================================================================
#sub evaluateCommands
#{
## evaluate test conditions if present
#  my @tmp = @_;
#  return @tmp unless $OPT{evaluate};
#  my @filtered;
#  foreach (@tmp) {
#    if (m/^(.*?)\s*#EVAL(.*)$/) {
#      my $line = $1;
#      my (undef,$fn) = tempfile('tmpXXXXXXXX',OPEN => 0,DIR=>'/tmp',SUFFIX=>'.sw');
#      my $eval = $2;
#      open FILE, ">$fn";
#      if ($PAR->{shell} eq '/bin/bash') {
#        print FILE "if [ $eval ] ; then echo 1 ; else echo 2 ; fi\n";
#      }
#      elsif ($PAR->{shell} eq '/bin/tcsh') {
#        print FILE "if ( $eval ) then\necho 1\nelse\necho 2\nendif\n";
#      }
#      close FILE;
#      chomp(my $ret = `$PAR->{shell} $fn 2>&1`);
#      unlink $fn;
#      if ($ret == 1) {
#        push @filtered,"$line\n";
#      }
#      elsif ($ret == 2) {
## drop the command
#      }
#      else {
#        dieWithError("commandline evaluation failure:\n$ret");
#      }
#    }
#    else {
#      push @filtered,$_;
#    }
#  }
#  return @filtered;
#}
#==============================================================================
sub mergeLineContinuations
# If a swarmfile has line continuation markers (one or more spaces, followed
# by a single backslash, immediately followed by end-of-line), then the line
# will be continued with the next line.  Suggested by Wolfgang Resch, 11/29/12.
{
  my @lines = @_;
  my @merged;
  my $cmd;
  my $i;
  foreach my $l (@lines) {
    $i++;
    chomp $l;
    if ($l=~/\\\s*$/) { # attempt at line continuation
      if ($l=~/ \\$/) { # only strict, proper format is allowed
        $cmd .= " $`";
      } else {
        dieWithError("bad line continuation format: line $i of $OPT{f}"); 
      }
    }
    else {
      $cmd .= " $l";
      dieWithError("last line of $OPT{f} cannot have line continuation") if ($cmd =~/\\$/); 
      push @merged,$cmd;
      undef $cmd;
    }
  }
  dieWithError("last line of $OPT{f} cannot have line continuation") if (defined $cmd); 
  return @merged;
}
#===============================================================================
sub avoidSbatchCommands
{
# Walk through each command
  COMM: foreach my $c (@{$SWARM->{COMMANDS}}) {
# Split the command up into distinct processes
    PROC: foreach my $e (split /\;/,$c) {
# Get rid of funny bashisms and simplify the words
      $e=~s/\(|\)|\[|\]|\{|\}//g;
      $e=~s/^\s+|\s+$//g;
      $e=~s/\s+/ /g;
# Split the processes into distinct words
      my @a = split / /,$e;
      my $x;
# Find the zeroth argument of the process, eliminating bashisms
      WORD: foreach my $w (@a) {
        next WORD if ($w =~/=/);
        next WORD if ($w eq 'do');
        next WORD if ($w eq 'if');
        next WORD if ($w eq 'in');
        $x = $w;
        last WORD;
      }
# Hopefully there is a zeroth argument
      next PROC unless $x;
# Don't allow the swarm to call sbatch or sinteractive from within the swarm
      if (($x eq 'sbatch') || ($x eq 'sinteractive')) {
        dieWithError("do not call sbatch or sinteractive from a swarm");
      }
    }
  }
}
#===============================================================================
sub setOptions
{   
# capture command line 
    
  $PAR->{commandLine} = $0;
  my $quotenext;
  foreach my $arg (@ARGV) { 
    if ($quotenext) {
      $PAR->{commandLine} .= " '$arg'"; 
      undef $quotenext;
      next;
    } 
    $quotenext=1 if ($arg=~/^--sbatch$/);
    $PAR->{commandLine} .= " $arg"; 
  }

# Create temporary hashref ($D) of options, along with array of options for GetOptions and GetOptionsFromString
  my ($D,@OPTIONS) = makeOptionsArray();
  getCommandLineOptions($D,@OPTIONS); # Commandline options
  getEnvironmentOptions(); # Enviroment options
  getFileDirectives($D,@OPTIONS); # Swarmfile directives

# Determine job-name
  if ($SLURMOPT2{'job-name'}) {
    $PAR->{'job-name'} = $SLURMOPT2{'job-name'};
  }
  else {
    $PAR->{'job-name'} = "swarm";
  }

# Special for exclusivity
  $SLURMOPT2{"ntasks-per-node"} = 1 if ($SLURMOPT1{exclusive});

# Set partition from somewhere
  if (not defined $SLURMOPT2{partition}) {
    if (defined $SBATCHOPT) {
      if ($SBATCHOPT=~/(^\-\-partition\s*=?\s*)(\w+)/) {
        $SLURMOPT2{partition} = $2;
        $SBATCHOPT=~s/${1}${2}//g;
      }
      elsif ($SBATCHOPT=~/(^\-p\s*=?\s*)(\w+)/) {
        $SLURMOPT2{partition} = $2;
        $SBATCHOPT=~s/${1}${2}//g;
      }
      elsif ($SBATCHOPT=~/( \-\-partition\s*=?\s*)(\w+)/) {
        $SLURMOPT2{partition} = $2;
        $SBATCHOPT=~s/${1}${2}//g;
      }
      elsif ($SBATCHOPT=~/( \-p\s*=?\s*)(\w+)/) {
        $SLURMOPT2{partition} = $2;
        $SBATCHOPT=~s/${1}${2}//g;
      }
      else {
        $SLURMOPT2{partition} = $PAR->{default_partition};
      }
    }
    else {
      $SLURMOPT2{partition} = $PAR->{default_partition};
    }
  }

# Look to see if this is a multinode job
  my $nodes;
  if (defined $SBATCHOPT) {
    if ($SBATCHOPT=~/^\-\-nodes\s*=?\s*(\d+)/) { $nodes = $1; }
    elsif ($SBATCHOPT=~/ \-\-nodes\s*=?\s*(\d+)/) { $nodes = $1; }
    elsif ($SBATCHOPT=~/^\-N\s*=?\s*(\d+)/) { $nodes = $1; }
    elsif ($SBATCHOPT=~/ \-N\s*=?\s*(\d+)/) { $nodes = $1; }
  }
  $PAR->{multinode} = $nodes if ($nodes && $nodes > 1);

# Be quiet!
  $OPT{verbose} = 0 if $OPT{silent};

# development run
  if ($OPT{devel}) {
    $OPT{'no-scripts'} = 1;
    $OPT{'no-run'} = 1;
    $OPT{'no-log'} = 1;
    $OPT{debug} = 1;
    $OPT{verbose} = 3 unless (defined $OPT{verbose});
    print "\n" if ($OPT{verbose} > 4);
  }
  elsif ($OPT{debug}) {
    $OPT{'no-scripts'} = 1;
    $OPT{verbose} = 2;
    $OPT{'no-run'} = 1;
    $OPT{'no-log'} = 1;
  }

# Default verbosity
  $OPT{verbose} = 0 if (not defined $OPT{verbose});

# Create a temporary name for the batch script and command script directory.
  (undef,$PAR->{tempname}) = tempfile('XXXXXXXXXX',OPEN => 0);
  $PAR->{tempdir} = "$PAR->{basedir}/$PAR->{tempname}";
  $PAR->{batchfile} = $PAR->{tempdir}."/swarm.batch";

# Be chatty
  print "basedir = $PAR->{basedir}\n" if ($OPT{verbose} > 4);
  print "script dir = $PAR->{tempdir}\n" if ($OPT{verbose} > 4);

# avoid stupidness
  $OPT{b}=1 if ((not defined $OPT{b}) || ($OPT{b} <= 1));
  $PAR->{b} = $OPT{b};

# set cpus-per-task or ntasks-per-node
  if ($OPT{t}) {
    if ($OPT{t} eq "auto") {
      undef $SLURMOPT2{"cpus-per-task"};
      $SLURMOPT1{"exclusive"}=1;
    }
    else {
# Don't be stupid
      dieWithError("-t must be either a number or \"auto\"") if !($OPT{t}=~/^\d+$/ || $OPT{t} eq "auto");
      dieWithError("-t must be either a number >=1") if ($OPT{t} < 1);
      $SLURMOPT2{"cpus-per-task"} = $OPT{t};
    }
  }
  else {
    $OPT{t} = 1;
    $SLURMOPT2{"cpus-per-task"} = 1;
  }

# pack only allowed for single threaded swarms 
  if ($OPT{t} && $OPT{p}) {
    if (($OPT{t} > 1) && ($OPT{p} > 1)) {
      dieWithError("-t is for multi-threaded or multi-node commands, -p is for single-threaded commands.  Use one or the other!");
    }
  }
  undef $OPT{p} if ($OPT{p} <= 1);
# Don't allow p to be more than 2
  dieWithError("-p can't be more than 2.") if ($OPT{p} > 2);

# Don't allow hyperthreading, which implies no packing
  $SLURMOPT2{"threads-per-core"} = 1 if ($OPT{noht});

  if (defined $OPT{g}) { # assign to --mem, converted to MB
    dieWithError("-g must be between 0.1 and ".$PAR->{maxmemsize}) if (($OPT{g} <= 0.1) || ($OPT{g} > $PAR->{maxmemsize}));
    if (($OPT{g} > $PAR->{largemem_threshold}) && ($SLURMOPT2{partition} ne "largemem")) {
      dieWithError("-g $OPT{g} requires --partition largemem");
    }
  }
  else {
    $OPT{g} = 1.5;
  }
  if (($OPT{g} <= $PAR->{largemem_threshold}) && ($SLURMOPT2{partition} eq "largemem")) {
    dieWithError("-g <= $PAR->{largemem_threshold} must not run on the largemem partition");
  }

  $SLURMOPT2{"mem"} = ceil($OPT{g}*1024); 

# After getopts, argv should be -1 for this script.
  dieWithError("excess commandline arguments (@ARGV). See $PAR->{programname} --help") if ($#ARGV > -1 || $#ARGV < -1);

  $PAR->{Commandfile} = $OPT{f};

# Swarmfile must be a file
  dieWithError("where is swarmfile?") if (! -f $OPT{f});

# choose shell to run under
  $PAR->{shell} = ($OPT{usecsh}) ? "/bin/tcsh" : "/bin/bash";

# modify shell if wanted
  $PAR->{shell} .= " -e" if ($OPT{'err-exit'});

# set 'comment-char'
  $PAR->{'comment-char'} = substr($OPT{'comment-char'},0,1) if $OPT{'comment-char'};

# Convert module from comma-delimited list to space-delimited list
  if ($OPT{module}) {
    my @tmp = split /,/,$OPT{module};
    $OPT{module} = join " ",@tmp;
  }

# Make sure that the logdir can be written to
  if (defined $OPT{logdir}) {
    isDirCreateable($OPT{logdir}) || dieWithError("can't write to $OPT{logdir}");
  }
  else {
    $OPT{logdir} = $PAR->{pwd};
  }

# Be reasonable
  if ((defined $OPT{maxrunning}) && ($OPT{maxrunning} < 1)) {
    undef $OPT{maxrunning};
  }

# noout and noerr are incompatible with merge-output
  if ($OPT{'merge-output'}) {
    if ($OPT{noout} || $OPT{noerr}) {
      dieWithError("--noout and --noerr are incompatible with --merge-output");
    }
  }

  if (defined $OPT{"time-per-command"}) {
    $SLURMOPT2{"time"} = $OPT{"time-per-command"};
  }
  elsif (defined $OPT{"time-per-subjob"}) {
    $SLURMOPT2{"time"} = $OPT{"time-per-subjob"};
  }

# Validate time
  if (defined $SLURMOPT2{"time"}) {
    my $x = clock_to_seconds($SLURMOPT2{"time"});
    if ((!$x) || ($x !~/^\d+$/)) {
      dieWithError("--time ".$SLURMOPT2{"time"}." is invalid");
    }
  }
}
#==============================================================================
sub dieWithError
{
  my $message = shift;
  die "ERROR: $message\n";
}
#==============================================================================
sub writeLog
# log some useful information
{
  my $x = $PAR->{sbatch_options};   # Make Susan happy
  $x=~s/%A/$PAR->{jobid}/g;

  my $fileContents = "date=".$PAR->{date}            ."; "
          . "host="       .$PAR->{host}            ."; "
          . "jobid="      .$PAR->{jobid}           ."; "
          . "user="       .$PAR->{user}            ."; "
          . "pwd="        .$PAR->{pwd}             ."; "
          . "ncmds="      .$PAR->{commands}        ."; "
          . "soptions="   .$x                      ."; "
          . "njobs="      .$PAR->{subjobs}         ."; "
          . "tempname="   .$PAR->{tempname}        ."; "
          . "job-name="   .$PAR->{'job-name'}      ."; "
          . "command="    .$PAR->{commandLine}     ."\n";

  appendToFile($PAR->{logfile},$fileContents);
  chmod 0666, $PAR->{logfile};
}
#==============================================================================
sub isDirCreateable
{
  my $path = shift;
  my $str = File::Spec->rel2abs($path); # convert path to absolute
  while ((!-d $str) && ($str)) { 
    $str =~s/\/[^\/]*$//; 
  }
  return writeable($str);
}
#==============================================================================
sub validatePartition
# Don't let the use choose an unknown or invalid partition
{
  my @names = (sort keys %{$PAR->{slurm_partitions}});
  foreach my $part (split /,/,$SLURMOPT2{partition}) {
    dieWithError("Unknown or disallowed partition '$part'!") unless (grep /^$part$/,@names);
    dieWithError("Partition '$part' is only valid for multinode jobs") if (($part eq 'ibddr') && !$PAR->{multinode});
    dieWithError("Partition '$part' is only valid for multinode jobs") if (($part eq 'ibfdr') && !$PAR->{multinode});
    dieWithError("Partition '$part' is only valid for multinode jobs") if (($part eq 'ibqdr') && !$PAR->{multinode});
  }
}
#==============================================================================
sub adjustTime
# The timelimit must be changed to account for bundling and/or folding
{
# Don't bother if partition is unlimited
  if ($SLURMOPT2{partition} eq "unlimited") {
    undef $SLURMOPT2{"time"};
    return;
  }

# Account for multiple partitions
  $PAR->{default_time} = 999999999999; # really long time
  $PAR->{max_time} = 999999999999; # really long time
  foreach my $part (split /,/,$SLURMOPT2{partition}) {
    my $part_found;
    foreach my $name (sort keys %{$PAR->{slurm_partitions}}) {
      if ($name eq $part ) {
        my $i = $PAR->{slurm_partitions}{$name};
        $part_found=1;
        my $t = ($i->{default_time}*60); # time is in minutes
        $PAR->{default_time} = $t if ($t < $PAR->{default_time});
        $t = ($i->{max_time}*60); # time is in minutes
        $PAR->{max_time} = $t if ($t < $PAR->{max_time});
      }
    }
    ($part_found) || dieWithError("undefined time limit for $SLURMOPT2{partition} partition!");
  }

# Find requested_time from user
  if (defined $SLURMOPT2{"time"}) {
    $PAR->{requested_time} = clock_to_seconds($SLURMOPT2{"time"});
  }
  else {
    $PAR->{requested_time} = $PAR->{default_time};
  } 

# Do we need to adjust the time?  With folding, we now have to determine this on a per-subjob basis
  my $max_adjusted_time = 0;
  foreach my $sj (0 .. $#{$SWARM->{CMDLISTS}}) {
    my $max_cmds_in_subjob = scalar(@{${$SWARM->{CMDLISTS}}[$sj][0]});
    if (defined ${$SWARM->{CMDLISTS}}[$sj][1]) {
      my $z = scalar(@{${$SWARM->{CMDLISTS}}[$sj][1]});
      if ($z && ($z > $max_cmds_in_subjob)) {
        $max_cmds_in_subjob = $z;
      }
    }
    my $adjusted_time = undef;
    if ($OPT{"time-per-subjob"}) {
      $adjusted_time = $PAR->{requested_time};
      $max_adjusted_time = $adjusted_time;
    }
    else {
      $adjusted_time = $PAR->{requested_time} * $max_cmds_in_subjob;
      $max_adjusted_time = $adjusted_time if ($max_adjusted_time < $adjusted_time);
    }
    push @{$SWARM->{SUBJOB_TIME}},$adjusted_time;
  }

# Die if the max_adjusted_time is too great
  dieWithError("Total time for bundled commands is greater than partition walltime limit.\nTry lowering the time per command (--time=".seconds_to_clock($PAR->{requested_time})."), lowering the bundle factor\n(if not autobundled), picking another partition, or splitting up the swarmfile.") if ($max_adjusted_time > $PAR->{max_time});

# Set time to the maximum amount
  $SLURMOPT2{"time"} = seconds_to_clock($max_adjusted_time);
}
#==============================================================================
sub seconds_to_clock
# converts seconds to clock time (D*-HH:MM:SS)
{
  my($time) = @_;
  my($sec) = 0;
  my($min) = 0;
  my($hrs) = 0;
  my($day) = 0;
  $sec = $time;
  $min = $sec / 60;
  $sec = $sec % 60;
  if ($min > 60) {
    $hrs = $min / 60;
    $min = $min % 60;
    if ($hrs > 24) {
      $day = $hrs / 24;
      $hrs = $hrs % 24;
      return sprintf ("%d-%02d:%02d:%02d",$day,$hrs,$min,$sec);
    }
    else { return sprintf ("%02d:%02d:%02d",$hrs,$min,$sec); }
  }
  else { return sprintf ("%02d:%02d",$min,$sec); }
}
#==============================================================================
sub clock_to_seconds
# converts clock time (D*-HH:MM:SS) to seconds
{
  my($clock) = @_;
# days-hours:minutes:seconds
  if ($clock=~/^(\d+)-(\d+):(\d+):(\d+)$/) {
    return ($1*86400)+($2*3600)+($3*60)+$4;
  }
# days-hours:minutes
  elsif ($clock=~/^(\d+)-(\d+):(\d+)$/) {
    return ($1*86400)+($2*3600)+($3*60);
  }
# days-hours
  elsif ($clock=~/^(\d+)-(\d+)$/) {
    return ($1*86400)+($2*3600);
  }
# hours:minutes:seconds
  elsif ($clock=~/^(\d+):(\d+):(\d+)$/) {
    return ($1*3600)+($2*60)+$3;
  }
# minutes:seconds
  elsif ($clock=~/^(\d+):(\d+)$/) {
    return ($1*60)+$2;
  }
# minutes
  elsif ($clock=~/^(\d+)$/) {
    return ($1*60);
  }
}
#==============================================================================
sub catch_sig
# If a signal is thrown during the process, attempt to be neat
{
  if ($PAR->{jobid}) {
    warn("WARNING: swarm ended after job was submitted, canceling job $PAR->{jobid}\n");
    system("scancel $PAR->{jobid}");
  }
  else {
    dieWithError("swarm ended before job was submitted");
  }
}
#==============================================================================
sub append_to_tempdir_index
{
  my $p = 1;
  $p = 2 if ($OPT{p} == 2);
  my $fileContents = time().",$PAR->{user},$PAR->{tempname},$PAR->{subjobs},$p\n";
  appendToFile($PAR->{tempdir_index},$fileContents);
  chmod 0666, $PAR->{tempdir_index};
}
#==============================================================================
sub getCommandLineOptions
{
  my ($D,@O) = @_;

# Get options from the command line and update temporary hashref
  GetOptions(@O) || exit 1;

# Update global hashes with values
  foreach my $k (sort keys %{$D->{1}}) { $OPT{$k}       = $D->{1}{$k} if ((defined $D->{1}{$k}) && (not defined $OPT{$k})); }
  foreach my $k (sort keys %{$D->{2}}) { $SLURMOPT1{$k} = $D->{2}{$k} if ((defined $D->{2}{$k}) && (not defined $SLURMOPT1{$k})); } 
  foreach my $k (sort keys %{$D->{3}}) { $SLURMOPT2{$k} = $D->{3}{$k} if ((defined $D->{3}{$k}) && (not defined $SLURMOPT2{$k})); } 
  $SBATCHOPT = $D->{4}{sbatch} if ((defined $D->{4}{sbatch}) && (not defined $SBATCHOPT));

# Wipe out temporary values
  foreach my $n (keys %{$D}) { foreach my $k (keys %{$D->{$n}}) { undef $D->{$n}{$k}; } }

# Take actions immediately
  printOptions() if $OPT{h};
  if ($OPT{version}) { print "swarm $PAR->{version}\n"; exit; }

# Assume first non-option argument is swarmfile -- radical change
  if (($#ARGV==0) && (defined $ARGV[0]) && (not defined $OPT{f})) {
    $OPT{f} = $ARGV[0];
    shift(@ARGV);
  }

  dieWithError("swarmfile not specified, see $PAR->{programname} --help") if (not defined $OPT{f});
  dieWithError("swarmfile '$OPT{f}' doesn't exist, see $PAR->{programname} --help") if (! -f $OPT{f});
  dieWithError("swarmfile '$OPT{f}' can't be read, see $PAR->{programname} --help") if (! -r $OPT{f});
}
#==============================================================================
sub getEnvironmentOptions
{
# Set options based on environment variables
  foreach my $x (@{$PAR->{options}}) {
    foreach my $k (keys %{$x}) {
      if ($x->{$k}{D} == 1) {
        $OPT{$k} = $ENV{$x->{$k}{E}} if ((defined $x->{$k}{E}) && (defined $ENV{$x->{$k}{E}}) && (not defined $OPT{$k}));
      }
      if ($x->{$k}{D} == 2) {
        $SLURMOPT1{$k} = $ENV{$x->{$k}{E}} if ((defined $x->{$k}{E}) && (defined $ENV{$x->{$k}{E}}) && (not defined $SLURMOPT1{$k}));
      }
      if ($x->{$k}{D} == 3) {
        $SLURMOPT2{$k} = $ENV{$x->{$k}{E}} if ((defined $x->{$k}{E}) && (defined $ENV{$x->{$k}{E}}) && (not defined $SLURMOPT2{$k}));
      }
    }
  }
}
#==============================================================================
sub getFileDirectives
{
# Get swarmfile directives ( added 2021-12-06 )
  my ($D,@O) = @_;
  my $Y;
  if (open FILE, "<$OPT{f}") {
    while (<FILE>) {
      if (m/^#SWARM\s+(.*)$/) {
        $Y .= $1." ";
      }
    }
    close FILE;
  }
  else {
    die("Can't read $OPT{f}");
  }

  if ($Y) {
    $Y =~ s/\s+/ /g;  # multiple spaces to single spaces
    $Y =~ s/^\s+//;   # remove preceding spaces
    $Y =~ s/\s+$//;   # remove trailing spaces
    GetOptionsFromString($Y,@O);

# Fold swarmfile directives into original options
    foreach my $k (sort keys %{$D->{1}}) { $OPT{$k}       = $D->{1}{$k} if ((defined $D->{1}{$k}) && (not defined $OPT{$k})); }
    foreach my $k (sort keys %{$D->{2}}) { $SLURMOPT1{$k} = $D->{2}{$k} if ((defined $D->{2}{$k}) && (not defined $SLURMOPT1{$k})); } 
    foreach my $k (sort keys %{$D->{3}}) { $SLURMOPT2{$k} = $D->{3}{$k} if ((defined $D->{3}{$k}) && (not defined $SLURMOPT2{$k})); } 
    $SBATCHOPT = $D->{4}{sbatch} if ((defined $D->{4}{sbatch}) && (not defined $SBATCHOPT));
    foreach my $n (keys %{$D}) { foreach my $k (keys %{$D->{$n}}) { undef $D->{$n}{$k}; } }
  }
}
#==============================================================================
sub makeOptionsArray
{
  my ($D,@Z);
  foreach my $x (@{$PAR->{options}}) {
    my ($k) = (keys %{$x});
    my @option_strings = (defined $x->{$k}{o}) ? @{$x->{$k}{o}} : ($k);
    foreach my $o (@option_strings) {
      if ($x->{$k}{t}) {
        push @Z,("$o=$x->{$k}{t}" => \$D->{$x->{$k}{D}}{$k});
      }
      else {
        push @Z,($o => \$D->{$x->{$k}{D}}{$k});
      }
    }
  }
 
  return ($D,@Z);
}
#==============================================================================
sub privilegedUser
{
# Allow root, webcpu, helixmon, and all staff members to have access to all quotas
  if (($< == 0) || ($< == 8296) || ($< == 30854)) { $PAR->{privilegedUser} = 1; return; }
  my $u = getpwuid($<);
  $PAR->{privilegedUser} = 1 if (grep /^$u$/,(split / /,(getgrnam("staff"))[3]));
  return;
}
#==============================================================================

