#!/usr/bin/perl -w
#
# Check what packages are needed to get upgraded on all machines
#
# Depends heavily on having ssh key authentication set up to all
# boxes. That's why I run it on my own workstation.
#
# Regular debian upgrades are detected by running 
#  apt-get upgrade -s
# on every machine, and parsing the output.

# We have decided to maintain some packages manually on some 
# machines, so that system-level upgrades will not disturb 
# applications, which may need more hand-holding. These are
# extracted from our apt repository, and queried on every 
# server with apt-cache policy. This way, as soon as a package
# is released on our repo, it will get listed here.
#
# 11-Mar-2011 Heikki: Started this
# 22-Mar-2011 Heikki: Adding manually maintained packages
# 15-Aug-2011 Heikki: Adding a total in the headline, for nagiosgrapher
# 21-May-2012 Heikki: Added a date since when a package has been pending
# 31-May-2012 Heikki: Pointing to the new wiki
# 01-Jan-2013 Heikki: Get hosts from nagios-us as well.
#
# TODO: Assumes that we release our restricted packages for all versions
# and architectures at the same time. Gets only the highest version from
# all, and reports anything less than this. Good enough for now.
#
# TODO: Get the dates from ls --full-time /var/cache/apt/archives/
# and display next to the packages, so we can see how long they have
# been lingering. Boldface them if older than some limit

#### Init
use strict;
my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much
my $year =`date +%Y`;
#my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/';
my $wikilink = 'https://twiki.indexdata.com/twiki/bin/view/ID/';
my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages";

#### Host comments
my %hostcomments = (
      "ariel"    => "<i>Niels Erik</i> does the manual upgrades",
      "bellone"  => "<i>Niels Erik</i> does the manual upgrades",
      "cfrepous" => "<i>Wolfram</i> does the manual upgrades",
      "leopard"  => "<i>Wolfram</i> does the manual upgrades",
      "lsd"      => "<i>Heikki</i> takes care of all upgrades",
      );
      

#### Get list of hosts
# I could use a hard-coded list, but I would forget to maintain it.
# Nagios knows most of our hosts. It even knows which are worth 
# checking, they have a command to check apts!
print "Getting hostlist from nagios\n" if $debug;
my $hostlist1 = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
  or die "Could not get host list from nagios (dk)";

print "Getting hostlist from nagios-us\n" if $debug;
my $hostlist2 = `ssh nagios-us grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
  or die "Could not get host list from nagios (dk)";

my $hostlist = $hostlist1 . $hostlist2;
print "Got list:\n$hostlist\n" if $debug>2;

###### Get list of packages that can be manually maintained
print "getting restricted package versions\n" if $debug;
my %restrpkgs;
my $restplines = `$restrictedpackages`
  or die "Could not get the list of restricted packages " .
         "from $restrictedpackages: $! ";
print "Got package list: \n$restplines\n" if $debug>2;
my $pname;
my $pver;
for my $pline ( split("\n",$restplines) ) {
    chomp($pline);
    $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/;
    $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/;
    print "$pline: p=$pname v=$pver\n" if $debug>2;
    if ( $pname && $pver ) {
        print "\nPackage $pname version $pver \n" if $debug>2;
        if ( ! $restrpkgs{$pname} ) {
            $restrpkgs{$pname} = $pver;
            print "found $pname, first version $pver\n" if $debug>1;
        } else {
            my $bver = $restrpkgs{$pname};
            `dpkg --compare-versions "$bver" lt "$pver" 2>/dev/null `;
            if ( ! $? ) {
                print "found $pname, better version $pver (better than $bver)\n"
                    if $debug>1;
                $restrpkgs{$pname} = $pver;
            } else {
                print "found $pname, but version $pver is no better than $bver\n"
                    if $debug>2;
            }
        }
        $pname = ""; # clear for the next one.
        $pver = "";
    }
}

print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n" if $debug;
if ( $debug >1 ) {
    for $pname ( sort (keys(%restrpkgs)) ) {
        print "  $pname " . $restrpkgs{$pname} . "\n";
    }
}

# Statistics
my %summary;
my ( %sechosts, %secpkgs );
my ( %ownhosts, %ownpkgs );
my ( %manhosts, %manpkgs );
my ( %normhosts, %normpkgs );
my %okhosts;
my %skiphosts;
my %allhosts;
my $sectot = 0;
my $owntot = 0;
my $mantot = 0;
my $normtot = 0;
my %updlinks;
my %debversions;

# Pending modification dates
my %olddates;  # Read in from the file
my %newdates;  # To be written in the new version of the file
my $datefilename = "aptcheck.data";
my $dateoldfilename = "aptcheck.old";
my $thisdate = "*"; # indicates really old stuff
if ( -f $datefilename ) {
  print "Reading dates from $datefilename\n" if $debug;
  open F, $datefilename or die "Could not open date file $datefilename: $!";
  while (<F>) {
    chop();
    my ($pkg, $date) = split;
    next unless $pkg;  # skip empty lines
    $olddates{$pkg} = $date;
    print "Date for '$pkg' is '$date' \n" if $debug;
  }
  close F;
  $thisdate = `date +%F`;
  chomp($thisdate);
} else {
  print "No datefile $datefilename found, starting from scratch\n";
}


my $table = "<table>\n";

#for my $hline ( split("\n",$hostlist) ) {
for my $hline ( sort( split("\n",$hostlist) ) ) {
    next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ );
    my $H = $1;
    next if ($H =~ /^commands/ );
    next if ($H =~ /^servicegroups/ );
    print "Checking $H\n" if $debug;
    $allhosts{$H}=1;
    my $cmd0 = "cat /etc/debian_version";
    my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs)));
    my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' ";
    # Note, do not append -qq, we want some output even when nothing to do
    print "ssh -q $H \"$cmd0; $cmd1 ; $cmd2 \" 2>/dev/null \n" if ($debug>1);
    my $apt = `ssh -q $H "$cmd0; $cmd1 ; $cmd2 " 2>/dev/null`;
    if ( !$apt ) {
        $table .= "<tr><td colspan='3'>&nbsp;</td></tr>\n";
        $table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)\n";
        $skiphosts{$H}=1;
        next;
    }
    print "Got apts for $H: \n$apt\n" if $debug>2;
    my $det = ""; # detail lines
    my $pkgs = 0;
    my $secs = 0;
    my $own = 0;
    my $man = 0;
    my $restrname = "";
    my $restrinst = "";
    my $restrcand = "";
    my $debver = 0;
    for my $p ( split("\n",$apt) ) {
        if ( !$debver ) {  # first line
          $debver = 1;
          $p =~ s/(5[0-9.]+)/$1 LENNY !!!/;
          $p =~ s/(6[0-9.]+)/$1 squeeze/;
          $p =~ s/(7[0-9.]+)/$1 wheezy/;
          $p = "&nbsp;Debian $p";
          $debversions{$H} = $p;
          print "Deb version for $H is $p\n" if ($debug>1);
          next;
        }
        # parse apt-cache output
        $restrname = $1 if $p =~ /^(\S+):$/;
        $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/;
        $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/;
        if ( $p =~ /^\s+Version table:/ ) { # have all for that package
            my $bver = $restrpkgs{$restrname};
            if ( ( $restrinst eq $restrcand ) &&
                 ( $restrinst ne $bver ) ) { 
                # if different, it is a regular apt upgrade, and will be seen
                # later. AND we want to have a different version in our repo
                `dpkg --compare-versions "$bver" lt "$restrinst"  2>/dev/null`;
                if ( $? ) { # It was not a downgrade 
                            # manual packages may be ahead of the repo!
                    $mantot++;
                    $man++;
                    $pkgs++;
                    $manhosts{$H} = 1;
                    $manpkgs{$restrname} = 1;
                    $det .= "<tr>";
                    $det .= "<td>&nbsp;&nbsp;$restrname <b>(M)</b></td>";
                    $det .= "<td>". strdiff($bver,$restrinst)."</td>";
                    $det .= "<td>". strdiff($restrinst,$bver)."</td>";
                    my $datekey = "$H:$restrname";
                    if ( $olddates{$datekey} ) {
                        $newdates{$datekey} = $olddates{$datekey};
                    } else {
                        $newdates{$datekey} = $thisdate;
                    }
                    $det .= "<td>" . $newdates{$datekey} . "</td>";
                    $det .= "</tr>\n";
                    my $key = "$restrname";
                    if ( !$summary{$key} ) {
                        $summary{$key} = "";
                    }
                    $summary{$key} .= "$H ";
                }
            }
            $restrname = ""; # clear for next round
            $restrinst = "";
            $restrcand = "";
        }
        next unless $p =~
            /^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/;
        my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 );
        print "$H: $pkg: $cur -> $new ($src)\n" if $debug>1;
        $det .= "<tr><td>&nbsp;&nbsp;";
        $pkgs++;
        my $key = $pkg;
        if ( $src =~ /Security/ ) {
            $det .= "<b>$pkg (s)</b> ";
            $sechosts{$H} = 1;
            $secpkgs{$pkg} = 1;
            $secs++;
            $sectot++;
        } elsif ( $src =~ /Indexdata/ ) {
            $det .= "<i><b>$pkg</b> (id) </i>";
            $ownhosts{$H}=1;
            $ownpkgs{$pkg}=1;
            $own++;
            $owntot++;
        } else {
            $det .= "$pkg ";
            $normhosts{$H}=1;
            $normpkgs{$pkg}=1;
            $normtot++;
        }
        if ( !$summary{$key} ) {
            $summary{$key} = "";
        }
        $summary{$key} .= "$H ";
        $new = strdiff($cur,$new);
        $cur = strdiff($new,$cur);
        $det .= "</td> ";
        $det .= "<td>$cur</td> ";
        $det .= "<td>$new</td> ";
        my $datekey = "$H:$pkg";
        if ( $olddates{$datekey} ) {
            $newdates{$datekey} = $olddates{$datekey};
        } else {
            $newdates{$datekey} = $thisdate;
        }
        $det .= "<td>" . $newdates{$datekey} . "</td>";
        $det .= "</tr>\n";

    }
    $table .= "<tr><td colspan='4'>&nbsp;</td></tr>\n";
    $table .= "<tr><td colspan='4'><a name='$H'><b><u>$H</u></b></a> &nbsp;\n";
    if ( $pkgs ) {
        $table .= "<b>$pkgs</b> packages to upgrade. ";
        $table .= "<b>$secs security</b>. " if $secs;
        $table .= " $own from indexdata. " if $own;
        $table .= " $man manual. " if $man;
    } else {
        $table .= "ok";
        $okhosts{$H} = 1;
    }
    my $updlink = $wikilink . ucfirst($H) . "Updates" . $year;
    # Fix some pages that do not follow the convention.
    # Mostly because the host names would not make proper WikiWords
    $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/; 
    $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/; 
    $updlinks{$H} = $updlink;
    $table .= "&nbsp;<a href='$updlink' >Upd</a>";
    $table .= "&nbsp;" . $debversions{$H};
    $table .= "</td></tr>\n";
    $table .= "<tr><td>$hostcomments{$H}</td></tr>\n"
        if ( $hostcomments{$H} );
    $table .= $det if $pkgs;
    print "\n$table\n" if $debug>2;
    last if $H =~/diane/ && $debug;
}
$table .= "</table>\n";

# Save the date file
`mv -f $datefilename $dateoldfilename`;
open F, ">$datefilename" or die "Could not open date file $datefilename for writing";
for  my $k (sort(keys(%newdates)) ) {
    print F "$k " . $newdates{$k}. "\n";
    print "date for '$k' '" . $newdates{$k}. "'\n" if $debug;
}
close F
  or die "Could not close date file $datefilename: $!";

# Page header
my $outfile = "/tmp/aptcheck.html";
open F, ">$outfile"
    or die "Could not open $outfile for writing: $!";
print F "<html>\n";
print F "<head><title>Apt upgrade status</title></head>\n";
print F "<body>\n";
print F "<H1>Apt package status</H1>\n";
print F "<b>" .  ( $sectot + $owntot + $mantot + $normtot ) . 
        "</b> packages pending (<b>$sectot</b> critical) \n";

print F "<H2>Debug run, many hosts missing!</H2>\n"
   if $debug;


# Summary table: one row for per host group
print F "<p/>\n";
print F "<table border='1' >\n";
print F "<tr><td>&nbsp;</td>" ;
print F "<td><b>Hosts</b></td>\n";
print F "<td><b>Packages</b></td></tr>\n";

if ( $sectot ) {
    print F "<tr><td><b>Security</b><br/>" . scalar(keys(%sechosts)) . 
        "&nbsp;/&nbsp;" .  scalar(keys(%secpkgs)) . "&nbsp;/&nbsp;$sectot </td>\n" ;
    print F "<td>";
    for my $HH ( sort(keys(%sechosts)) ) {
        my $upd = $updlinks{$HH} || "#" ;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
    }
    print F "</td>";
    print F "<td>";
    for my $PP ( sort(keys(%secpkgs)) ) {
        print F "<a href='#$PP'>$PP</a> ";
    }
    print F "</td>";
    print F "</tr>\n";
}
if ( $owntot ) {
    print F "<tr><td><b>Indexdata</b><br/>" . scalar(keys(%ownhosts)) . 
        "&nbsp;/&nbsp;" .  scalar(keys(%ownpkgs)) . "&nbsp;/&nbsp;$owntot </td>\n" ;
    print F "<td>";
    for my $HH ( sort(keys(%ownhosts)) ) {
        my $upd = $updlinks{$HH} || "#" ;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
        #print F "<a href='#$HH'><b>$HH</b></a> ";
    }
    print F "</td>";
    print F "<td>";
    for my $PP ( sort(keys(%ownpkgs)) ) {
        print F "<a href='#$PP'>$PP</a> ";
    }
    print F "</td>";
    print F "</tr>\n";
}
if ( $mantot ) {
    print F "<tr><td><b>Manual</b><br/>" . scalar(keys(%manhosts)) . 
        "&nbsp;/&nbsp;" .  scalar(keys(%manpkgs)) . "&nbsp;/&nbsp;$mantot </td>\n" ;
    print F "<td>";
    for my $HH ( sort(keys(%manhosts)) ) {
        my $upd = $updlinks{$HH} || "#" ;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
        #print F "<a href='#$HH'><b>$HH</b></a> ";
    }
    print F "</td>";
    print F "<td>";
    for my $PP ( sort(keys(%manpkgs)) ) {
        print F "<a href='#$PP'>$PP</a> ";
    }
    print F "</td>";
    print F "</tr>\n";
}
if ( $normtot ) {
    print F "<tr><td>Normal<br/>" . scalar(keys(%normhosts)) . 
        "&nbsp;/&nbsp;" .  scalar(keys(%normpkgs)) . "&nbsp;/&nbsp;$normtot </td>\n" ;
    print F "<td>";
    for my $HH ( sort(keys(%normhosts)) ) {
        my $upd = $updlinks{$HH} || "#" ;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
        #print F "<a href='#$HH'><b>$HH</b></a> ";
    }
    print F "</td>";
    print F "<td>";
    for my $PP ( sort(keys(%normpkgs)) ) {
        print F "<a href='#$PP'>$PP</a> ";
    }
    print F "</td>";
    print F "</tr>\n";
}
if ( %skiphosts ) {
    print F "<tr><td>Skipped " . scalar(keys(%skiphosts)) . "</td>\n";
    print F "<td colspan='2'>";
    for my $HH ( sort(keys(%skiphosts)) ) {
        my $upd = $updlinks{$HH} ||
                  $wikilink . ucfirst($HH) . "Updates" . $year;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
        #print F "<a href='#$HH'><b>$HH</b></a> ";
    }
    print F "</td></tr>\n";
}
#if ( %okhosts ) {
if ( 1 ) {
    print F "<tr><td>Ok " . scalar(keys(%okhosts)) . "</td>\n";
    print F "<td colspan='2'>";
    for my $HH ( sort(keys(%okhosts)) ) {
        my $upd = $updlinks{$HH} || "#" ;
        print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
        #print F "<a href='#$HH'><b>$HH</b></a> ";
    }
    if ( !%okhosts ) {
        print F "<b>None at all!</b>";
    }
    print F "</td></tr>\n";
}
print F "</table>\n";

# Graph 
#my $secs = 60*60*24 * 7 * 2; # 2 weeks in secods
#my $secs = "1m"; # one month, let nagios do the math
my $secs = "45d"; 
print F "<p/>" .
        "<a href='http://nagios.indexdata.com/cgi-bin/nagios3/graphs.cgi?" .
        "host=nagios&service=Apt%20Summary'>\n".
        "<img src='http://nagios.indexdata.com/" .
              "cgi-bin/nagios3/rrd2-system.cgi?" .
              "host=nagios&service=Apt%20Summary&" .
              "start=-$secs&" .
              "width=800&height=100&type=AVERAGE' /> ".
        "</a>" .
        "<br/>\n";

# The host table
print F $table;

# Package table
print F "<p/><b><u>Packages</u></b>\n";
print F "<table>\n";
for my $P ( sort(keys(%summary)) ) {
    my $PN = $P;
    $PN = "<b>$P&nbsp;(s)</b>" if ($secpkgs{$P});
    $PN = "<i>$P&nbsp;(id)</i>" if ($ownpkgs{$P});
    $PN = "$P&nbsp;<b>(M)</b>" if ($manpkgs{$P});
    print F "<tr><td><a name='$P'/>$PN</td>\n";
    print F "<td>";
    for my $HH ( split(' ',$summary{$P} )) {
        print F "<a href=#$HH>$HH</a> ";
    }
    print F "</td>\n";

}
print F "</table>\n";

print F "<p/>Produced " . `date`.
        " on " . `hostname` . " by " . `whoami` .
        "<br/>\n";
print F "</body></html>\n";

close(F)
    or die "Could not close $outfile: $!";

system "scp -q $outfile nagios:/var/www/heikki/index.html";

exit(0);

# Helper to take two strings and highligt that part of the second
# that is different from the first. 
sub strdiff {
    my $x = shift;
    my $y = shift;
    print "strdiff: '$x' '$y' \n" if $debug>2;
    if ( $x eq $y ) {
        return "$x <b>??</b>";
    }
    my $a = 0;
    while ( $a < length($y) &&
        substr($x,$a,1) eq substr($y,$a,1) ) {
        $a++;
    }
    if ( $a == length($y) ) {
        return "$y";
    }
    my $b = 1;
    while ( $b < length($y)-$a &&
        substr($x,-$b,1) eq substr($y, -$b,1) ) {
        $b++;
    }
    my $c = length($y) - $b +1;
    print "strdiff:   a=$a " . substr($y,0,$a) ."\n" if $debug>2;
    print "strdiff:   b=$b " . "\n" if $debug>2;
    print "strdiff:   c=$c " . substr($y,$c) ."\n" if $debug>2;
    print "strdiff:        " . substr($y,$a, $c-$a) ."\n" if $debug>2;
    my $z =  substr($y,0,$a) .
             "<b>" . substr($y,$a, $c-$a) . "</b>" .
             substr($y,$c);
    print "strdiff:        " . $z ."\n" if $debug>2;
    print "\n" if $debug>2;
    return $z;
}
