#!/usr/bin/perl
#
#     File: pbs_log_analyzer
#
#   Author: Dael Maselli - LNF - INFN
#
#     Date: 23 February 2005
#  Updated: 11 June 2006
#
#  Purpose: Analyze PBS server logs and prints some summarization
#

$pbs_log_dir = '/var/spool/pbs/server_priv/accounting/';

( $mindate = $ARGV[0] ) =~ s/\r|\n|-q//g;
( $maxdate = $ARGV[1] ) =~ s/\r|\n|-q//g;

$wq = 1;
$wq = 0 if join(' ', @ARGV) =~ /-q/;

$sep = '+' . '-' x 35 . "+\n";

$help = '
Usage: pbs_log_analyzer [STARTDATE [ENDDATE]] [-q]

 Dates has to be expressed as AAAAMMDD

 Examples: 

          To search in all log files:
             pbs_log_analyzer

          To search since 01 Jan 2004 to 31 Mar 2004:
             pbs_log_analyzer 20040101 20040331

          To search since 01 Jan 2004 to now:
             pbs_log_analyzer 20040101

          To show only summary for all queues and
          search since 01 Jan 2004 to 31 Mar 2004:
             pbs_log_analyzer 20040101 20040331 -q

';

if ( $ARGV[0] =~ /-h/ ||
     $mindate !~ /^$|^\d{8}$/ ||
     $maxdate !~ /^$|^\d{8}$/    ) {
    print $help;
    exit;
}

@jobs = ();

opendir(DIR, $pbs_log_dir) || die "can't opendir $pbs_log_dir: $!";

@dir = sort { ($a=~/^(.+)(\.gz)?$/)[0] <=> ($b=~/^(.+)(\.gz)?$/)[0] } readdir(DIR);

print "\n";

foreach ( @dir ) {

    $gzipped = 0;
    print STDERR "\rAnalyzing logs". sprintf( "%15s %3d%", " ($_):", ( $done++ / $#{@dir} * 100 ) ) ."";
    next if /^\./;
    $logname = $_;
    if ( $logname =~ s/\.gz$//g ) {
	$gzipped = 1;
    }
    next if ( $mindate && $logname < $mindate ) || ( $maxdate && $logname > $maxdate );

    if ( $gzipped ) {
	open( LF, "zcat $pbs_log_dir$_ | " );
    } else {
	open( LF, "<$pbs_log_dir$_" );
    }


    while ( <LF> ) {
	if ( /^(\d\d)\/\d\d\/(\d\d\d\d) \d\d:\d\d:\d\d;E;(\d+)\.(.+);(.+)$/ ) {
#	if ( /^(\d\d)\/\d\d\/(\d\d\d\d) \d\d:\d\d:\d\d;(.+);(\d+)\.(.+);user=(\S+) group=(\S+) jobname=(\S+) queue=(\S+) ctime=(\d+) qtime=(\d+) etime=(\d+) start=(\d+) exec_host=(\S+) Resource_List.cput=(\d\d:\d\d:\d\d) Resource_List.file=(\S+) Resource_List.neednodes=(\S+) Resource_List.nodect=(\d+) Resource_List.nodes=(\S+) Resource_List.vmem=(\S+) session=(\d+) end=(\d+) Exit_status=(\d+) resources_used.cput=(\d\d:\d\d:\d\d) resources_used.mem=(\S+) resources_used.vmem=(\S+) resources_used.walltime=(\d\d:\d\d:\d\d)/ ) {

	    

	    $ym      = $2.$1;
	    $jobid   = $3;
	    $server  = $4;

	    @jobdata = split /\s+/, $5;

	    for ( $i=0 ; $i<=$#jobdata ; $i++ ) {
		$jobdata[$i] =~ /^(.+)=(.+)$/;
		$jd{$1} = $2;
	    }

	    $requser = $ownuser = $user = $jd{'user'};
	    $jobname = $jd{'jobname'};
	    $queue   = $jd{'queue'};
	    $nodect  = $jd{'Resource_List.nodect'};
	    $nodect  = 1 if $nodect < 1;
	    $exstat  = $jd{'Exit_status'};
	    $cput    = sc2s($jd{'resources_used.cput'});
	    $mem     = h2kb($jd{'resources_used.mem'});
	    $vmem    = h2kb($jd{'resources_used.vmem'});
	    $wtime   = sc2s($jd{'resources_used.walltime'});

	    $jobcount++;

#	    $users{$user}++;   
#	    $s10jobcount++ if $wtime > 10;
#	    $yms{$ym}++;
#	    $exits{$exstat}++;
#	    $tcput += $cput;
#	    $tcputpn += $cput/$nodect;
#	    $twtime += $wtime;
#	    $tmem += $mem;
#	    $tvmem += $vmem;
#
#	    $mulnodect++ if $nodect > 1;
	    
	    $queues{'ALL QUEUES'}++;
	    $qyms{'ALL QUEUES'}{$ym}++;
	    $qexits{'ALL QUEUES'}{$exstat}++;
	    $qusers{'ALL QUEUES'}{$user}++;
	    $qs10jobcount{'ALL QUEUES'}++ if $wtime > 10;
	    $qtcput{'ALL QUEUES'}  += $cput;
	    $qtcputpn{'ALL QUEUES'}+= $cput/$nodect;
	    $qtwtime{'ALL QUEUES'} += $wtime;
	    $qtctntime{'ALL QUEUES'} += $cput/$nodect/$wtime if $wtime;
            #print if $wtime && ($cput/$nodect/$wtime) > 1;
	    $qtcttime{'ALL QUEUES'} += $cput/$wtime if $wtime;
	    $qtmem{'ALL QUEUES'}   += $mem;
	    $qtvmem{'ALL QUEUES'}  += $vmem;
	    $qmulnodect{'ALL QUEUES'}++ if $nodect > 1;

	    $queues{$queue}++;
	    $qyms{$queue}{$ym}++;
	    $qexits{$queue}{$exstat}++;
	    $qusers{$queue}{$user}++;
	    $qs10jobcount{$queue}++ if $wtime > 10;
	    $qtcput{$queue}  += $cput;
	    $qtcputpn{$queue}+= $cput/$nodect;
	    $qtwtime{$queue} += $wtime;
	    $qtctntime{$queue} += $cput/$nodect/$wtime if $wtime;
	    $qtcttime{$queue} += $cput/$wtime if $wtime;
	    $qtmem{$queue}   += $mem;
	    $qtvmem{$queue}  += $vmem;
	    $qmulnodect{$queue}++ if $nodect > 1;

	}
    }
    close(LF);
}
closedir DIR;

print STDERR "\r                                                                 \r";

if ( !$jobcount ) {
    print "No Jobs in that period\n";
    exit 1;
}

#print $sep;
#print "| ALL QUEUES" . " " x 24 ."|\n";
#print $sep;

#print "\n -- Jobs per Users\n";
#print_vsorted ( %users );
#
#print "\n -- Jobs per Exit Status\n";
#print_vsorted ( %exits );
#
#print "\n -- Jobs per YearMonth\n";
#print_ksorted ( %yms );
#
#print "\nTotal  JOB   Count:        ".sprintf ( "%10d",   $jobcount             )."\n";
#print   ">10s   JOB   Count:        ".sprintf ( "%10d",   $s10jobcount          )."\n";
#print   ">1node JOB   Count:        ".sprintf ( "%10d",   $mulnodect            )."\n";
#
#print "\nTotal CPU  Time (hours):   ".sprintf ( "%10.3f", ($tcput / 3600 )      )."\n";
#print   "Total WALL Time (hours):   ".sprintf ( "%10.3f", ($twtime / 3600 )     )."\n";
#
#print "\nCPU Time / WALL Time (%):  ".sprintf ( "%10.3f", ( $tcput / $twtime ) * 100 )."\n";
#print   "CPU / NODE# / WALL   (%):  ".sprintf ( "%10.3f", ( $tcputpn / $twtime ) * 100 )."\n";
#
#print "\nMedium CPU  Time (hours):  ".sprintf ( "%10.3f", ($tcput / $jobcount / 3600 )  )."\n";
#print   "Medium WALL Time (hours):  ".sprintf ( "%10.3f", ($twtime / $jobcount / 3600 ) )."\n";
#
#print "\nMedium MEM  Usage (MB):    ".sprintf ( "%10.3f", ($tmem / $jobcount / 1024 )   )."\n";
#print   "Medium VMEM Usage (MB):    ".sprintf ( "%10.3f", ($tvmem / $jobcount / 1024 )  )."\n";
#

foreach ( sort keys %queues ) {

    $queue = $_;

    if ( $wq || $queue eq 'ALL QUEUES') {

	print  "\n\n";
	print $sep;
	printf "| QUEUE: %-26s |\n", $_;
	print $sep;

	if ( $queue eq 'ALL QUEUES' ) {
	    print "\n -- Jobs per Queues\n";
	    print_vsorted ( %queues );
	}

	print "\n -- Jobs per Users\n";
	print_vsorted ( %{$qusers{$queue}} );

	print "\n -- Jobs per Exit Status\n";
	print_vsorted ( %{$qexits{$queue}} );
	
	print "\n -- Jobs per YearMonth\n";
	print_ksorted ( %{$qyms{$queue}} );
	
	print "\nTotal  JOB   Count:        ".sprintf ( "%10d",   $queues{$queue}             )."\n";
	print   "Exit 0 JOB   Count:        ".sprintf ( "%10d",   $qexits{$queue}{0}          )."\n";
	print   ">1node JOB   Count:        ".sprintf ( "%10d",   $qmulnodect{$queue}         )."\n";
	
	print "\nTotal CPU  Time (hours):   ".sprintf ( "%10.3f", ($qtcput{$queue} / 3600 )      )."\n";
	print   "Total WALL Time (hours):   ".sprintf ( "%10.3f", ($qtwtime{$queue} / 3600 )     )."\n";
	
	print "\nCPU Time / WALL Time (%):  ".sprintf ( "%10.3f", ($qtcput{$queue} / $qtwtime{$queue} ) * 100 )."\n";
	print   "CPU / NODE# / WALL   (%):  ".sprintf ( "%10.3f", ($qtcputpn{$queue} / $qtwtime{$queue} ) * 100 )."\n";

	print "\nJCPU Time / WALL Time (%): ".sprintf ( "%10.3f", ($qtcttime{$queue} / $queues{$queue} ) * 100 )."\n";
	print   "JCPU / NODE# / WALL   (%): ".sprintf ( "%10.3f", ($qtctntime{$queue} / $queues{$queue} ) * 100 )."\n";

	print "\nMedium CPU   Time (hours): ".sprintf ( "%10.4f", ($qtcput{$queue} / $queues{$queue} / 3600 )  )."\n";
	print   "Medium WALL  Time (hours): ".sprintf ( "%10.4f", ($qtwtime{$queue} / $queues{$queue} ) / 3600 )."\n";
	
	print "\nMedium MEM  Usage (MB):    ".sprintf ( "%10.3f", ($qtmem{$queue}  / $queues{$queue}) / 1024 )."\n";
	print   "Medium VMEM Usage (MB):    ".sprintf ( "%10.3f", ($qtvmem{$queue} / $queues{$queue}) / 1024 )."\n";

    }	

    $qtab .= sprintf "| %-10s | %5d | %5d | %5d | %10.3f | %10.3f | %10.3f | %10.3f | %10.4f | %10.4f | %10.3f | %10.3f |\n",
    $queue, $queues{$queue}, $qexits{$queue}{0}, $qmulnodect{$queue},
    ($qtcput{$queue}/3600), ($qtwtime{$queue}/3600),
    ($qtcput{$queue}/$qtwtime{$queue}*100), ($qtcputpn{$queue}/$qtwtime{$queue}*100),
    ($qtcput{$queue}/$queues{$queue}/3600), ($qtwtime{$queue}/$queues{$queue}/3600),
    ($qtmem{$queue}/$queues{$queue}/1024), ($qtvmem{$queue}/$queues{$queue}/1024);

}

print  "\n";

print "+--------------------------------------------------------------------------------------------------------------------------------------------+\n";
print "| Queue Name | Job # | ex0 # | >1n # | Tot CPU H  | Tot Wall H | CPU/WALL % | CPU/N/WAL% | Med CPU H  | Med WALL H | Med MEM MB | Md VMEM MB |\n";
print "+--------------------------------------------------------------------------------------------------------------------------------------------+\n";
print $qtab;
print "+--------------------------------------------------------------------------------------------------------------------------------------------+\n";

print  "\n";



sub sc2s {
    my ( $in ) = @_;
    $in =~ /(\d\d):(\d\d):(\d\d)/;
    return $3 + ( $2 * 60 ) + ( $1 * 3600 );
}

sub h2kb {
    my ( $in ) = @_;
    $in =~ /(\d+)(\D+)/i;

    $in *= 1 if /kb$/;
    $in *= 1024 if /mb$/;
    
    $in =~ s/\D//g;

    return $in;
}

sub print_vsorted {
    my ( %in ) = @_;
    foreach ( sort { $in{$b} <=> $in{$a} } keys %in ) {
	printf "%8d %s\n", $in{$_}, $_;
    }
}

sub print_ksorted {
    my ( %in ) = @_;
    foreach ( sort { $a <=> $b } keys %in ) {
	printf "%8d %s\n", $in{$_}, $_;
    }
}