3 # Check what packages are needed to get upgraded on all machines
5 # Depends heavily on having ssh key authentication set up to all
6 # boxes. That's why I run it on my own workstation.
8 # Regular debian upgrades are detected by running
10 # on every machine, and parsing the output.
12 # We have decided to maintain some packages manually on some
13 # machines, so that system-level upgrades will not disturb
14 # applications, which may need more hand-holding. These are
15 # extracted from our apt repository, and queried on every
16 # server with apt-cache policy. This way, as soon as a package
17 # is released on our repo, it will get listed here.
19 # 11-Mar-2011 Heikki: Started this
20 # 22-Mar-2011 Heikki: Adding manually maintained packages
21 # 15-Aug-2011 Heikki: Adding a total in the headline, for nagiosgrapher
22 # 21-May-2012 Heikki: Added a date since when a package has been pending
23 # 31-May-2012 Heikki: Pointing to the new wiki
24 # 01-Jan-2013 Heikki: Get hosts from nagios-us as well.
26 # TODO: Assumes that we release our restricted packages for all versions
27 # and architectures at the same time. Gets only the highest version from
28 # all, and reports anything less than this. Good enough for now.
30 # TODO: Check CentOs systems too
31 # Need a check_rpm plugin, and nagios checks defined on the CentOs machines,
32 # then we can grep for that check in nagios configs, and find rpm machines
33 # that way. Or, check all machines listed in nagios, determine what they
34 # run, and use proper commands to check for packages.
36 # TODO: Summary section for hosts that have too old updates still pending
37 # - those that are marked with (!)
41 my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much
43 #my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/';
44 my $wikilink = 'https://twiki.indexdata.com/twiki/bin/view/ID/';
45 my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages";
46 my $updlink="<i>-u</i>"; # to display after a name, liking to the upd page
50 "ariel" => "<i>Niels Erik</i> does the manual upgrades",
51 "bellone" => "<i>Niels Erik</i> does the manual upgrades",
52 "cfrepous" => "<i>Wolfram</i> does the manual upgrades",
53 "leopard" => "<i>Wolfram</i> does the manual upgrades",
54 "lsd" => "<i>Heikki</i> takes care of all upgrades",
58 #### Get list of hosts
59 # I could use a hard-coded list, but I would forget to maintain it.
60 # Nagios knows most of our hosts. It even knows which are worth
61 # checking, they have a command to check apts!
62 print "Getting hostlist from nagios\n" if $debug;
63 my $hostlist1 = `ssh nagios grep -l \\"define host\\" /etc/nagios3/indexdata-conf.d/*.cfg`
64 or die "Could not get host list from nagios (dk)";
66 print "Getting hostlist from nagios-us\n" if $debug;
67 #my $hostlist2 = `ssh nagios-us grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
68 my $hostlist2 = `ssh nagios-us grep -l \\"define host\\" /etc/nagios3/indexdata-conf.d/*.cfg`
69 or die "Could not get host list from nagios (us)";
71 my $hostlist = $hostlist1 . $hostlist2;
72 print "Got list:\n$hostlist\n" if $debug>2;
74 ###### Get list of packages that can be manually maintained
75 print "getting restricted package versions\n" if $debug;
77 my $restplines = `$restrictedpackages`
78 or die "Could not get the list of restricted packages " .
79 "from $restrictedpackages: $! ";
80 print "Got package list: \n$restplines\n" if $debug>2;
83 for my $pline ( split("\n",$restplines) ) {
85 $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/;
86 $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/;
87 print "$pline: p=$pname v=$pver\n" if $debug>2;
88 if ( $pname && $pver ) {
89 print "\nPackage $pname version $pver \n" if $debug>2;
90 if ( ! $restrpkgs{$pname} ) {
91 $restrpkgs{$pname} = $pver;
92 print "found $pname, first version $pver\n" if $debug>1;
94 my $bver = $restrpkgs{$pname};
95 `dpkg --compare-versions "$bver" lt "$pver" 2>/dev/null `;
97 print "found $pname, better version $pver (better than $bver)\n"
99 $restrpkgs{$pname} = $pver;
101 print "found $pname, but version $pver is no better than $bver\n"
105 $pname = ""; # clear for the next one.
110 print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n" if $debug;
112 for $pname ( sort (keys(%restrpkgs)) ) {
113 print " $pname " . $restrpkgs{$pname} . "\n";
119 my ( %sechosts, %secpkgs );
120 my ( %ownhosts, %ownpkgs );
121 my ( %manhosts, %manpkgs );
122 my ( %normhosts, %normpkgs );
133 # Pending modification dates
134 my %olddates; # Read in from the file
135 my %newdates; # To be written in the new version of the file
136 my $datefilename = "aptcheck.data";
137 my $dateoldfilename = "aptcheck.old";
138 my $thisdate = "*"; # indicates really old stuff
139 my $warndate; # Older than this will be boldfaced
141 if ( -f $datefilename ) {
142 print "Reading dates from $datefilename\n" if $debug;
143 open F, $datefilename or die "Could not open date file $datefilename: $!";
146 my ($pkg, $date) = split;
147 next unless $pkg; # skip empty lines
148 $olddates{$pkg} = $date;
149 print "Date for '$pkg' is '$date' \n" if $debug;
152 $thisdate = `date +%F`;
154 $warndate = `date +%F -d "30 days ago"` ; ;
156 print "Dates: now: '$thisdate' warn: '$warndate'\n" if $debug;
158 print "No datefile $datefilename found, starting from scratch\n";
162 my $table = "<table>\n";
165 for my $hline ( sort( split("\n",$hostlist) ) ) {
166 next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ );
168 print "Checking $H\n" if $debug;
169 next if ($H =~ /^xdummy/ );
170 next if ($H =~ /^commands/ );
171 next if ($H =~ /^contacts/ );
172 next if ($H =~ /^servicegroups/ );
173 next if ($H =~ /^hostgroups/ );
174 next if ($H =~ /^host-/ );
175 last if ($H =~/cookie/) && $debug;
178 "if test -f /etc/debian_version; " .
180 " cat /etc/debian_version; " .
181 " apt-cache -q policy " . join(" ",sort(keys(%restrpkgs))) .";" .
182 " apt-get upgrade -s -o 'Debug::NoLocking=true'; " .
184 " cat /etc/redhat-release; " .
185 # " yum check-update -q; ".
186 " echo done ;". # to be sure we have one "real" line
189 my $cmd0 = "cat /etc/debian_version";
190 my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs)));
191 my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' ";
192 # Note, do not append -qq, we want some output even when nothing to do
193 print "ssh -q $H \"$cmd\" 2>/dev/null \n" if ($debug>1);
194 my $apt = `ssh -q $H "$cmd" 2>/dev/null`;
196 $table .= "<tr><td colspan='3'> </td></tr>\n";
197 $table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)</td></tr>\n";
201 print "Got apts for $H: \n$apt\n" if $debug>2;
202 my $det = ""; # detail lines
212 for my $p ( split("\n",$apt) ) {
213 if ( !$versionseen ) { # first line
215 if ( $p =~ /Centos/i ) {
216 print "CentOs version for $H is $p\n" if ($debug>1);
217 $debversions{$H} = $p;
218 } else { # must be Debian
219 $p =~ s/(5[0-9.]+)/$1 LENNY !!!/;
220 $p =~ s/(6[0-9.]+)/$1 squeeze/;
221 $p =~ s/(7[0-9.]+)/$1 wheezy/;
222 $p = " Debian $p";
223 $debversions{$H} = $p;
225 print "Deb version for $H is $p\n" if ($debug>1);
229 if ( ! $is_debian ) {
230 $table .= "<tr><td colspan='3'> </td></tr>\n";
231 $table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)</td></tr>\n";
232 $table .= "<tr><td colspan='3'>Looks like Centos, " .
233 "not yet implemented </td></tr>\n";
237 # parse apt-cache output
238 $restrname = $1 if $p =~ /^(\S+):$/;
239 $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/;
240 $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/;
241 if ( $p =~ /^\s+Version table:/ ) { # have all for that package
242 my $bver = $restrpkgs{$restrname};
243 if ( ( $restrinst eq $restrcand ) &&
244 ( $restrinst ne $bver ) ) {
245 # if different, it is a regular apt upgrade, and will be seen
246 # later. AND we want to have a different version in our repo
247 `dpkg --compare-versions "$bver" lt "$restrinst" 2>/dev/null`;
248 if ( $? ) { # It was not a downgrade
249 # manual packages may be ahead of the repo!
254 $manpkgs{$restrname} = 1;
256 $det .= "<td> $restrname <b>(M)</b></td>";
257 $det .= "<td>". strdiff($bver,$restrinst)."</td>";
258 $det .= "<td>". strdiff($restrinst,$bver)."</td>";
259 my $datekey = "$H:$restrname";
260 if ( $olddates{$datekey} ) {
261 $newdates{$datekey} = $olddates{$datekey};
263 $newdates{$datekey} = $thisdate;
265 my $dispdate = $newdates{$datekey};
266 # if ( $dispdate lt $warndate ) {
267 if ( 0 ) { # manual packages don't need to be highlighted
268 $dispdate = "<b>$dispdate !</b>";
270 $det .= "<td>" . $dispdate . "</td>";
272 my $key = "$restrname";
273 if ( !$summary{$key} ) {
276 $summary{$key} .= "$H ";
279 $restrname = ""; # clear for next round
284 /^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/;
285 my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 );
286 print "$H: $pkg: $cur -> $new ($src)\n" if $debug>1;
287 $det .= "<tr><td> ";
290 if ( $src =~ /Security/ ) {
291 $det .= "<b>$pkg (s)</b> ";
296 } elsif ( $src =~ /Indexdata/ ) {
297 $det .= "<i><b>$pkg</b> (id) </i>";
308 if ( !$summary{$key} ) {
311 $summary{$key} .= "$H ";
312 $new = strdiff($cur,$new);
313 $cur = strdiff($new,$cur);
315 $det .= "<td>$cur</td> ";
316 $det .= "<td>$new</td> ";
317 my $datekey = "$H:$pkg";
318 if ( $olddates{$datekey} ) {
319 $newdates{$datekey} = $olddates{$datekey};
321 $newdates{$datekey} = $thisdate;
323 my $dispdate = $newdates{$datekey};
324 if ( ( $dispdate lt $warndate ) && ( $src =~ /Security/) ) {
325 $dispdate = "<b>$dispdate !</b>";
327 $det .= "<td>" . $dispdate . "</td>";
331 $table .= "<tr><td colspan='4'> </td></tr>\n";
332 $table .= "<tr><td colspan='4'><a name='$H'><b><u>$H</u></b></a> \n";
334 $table .= "<b>$pkgs</b> packages to upgrade. ";
335 $table .= "<b>$secs security</b>. " if $secs;
336 $table .= " $own from indexdata. " if $own;
337 $table .= " $man manual. " if $man;
342 my $updlink = $wikilink . ucfirst($H) . "Updates" . $year;
343 # Fix some pages that do not follow the convention.
344 # Mostly because the host names would not make proper WikiWords
345 $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/;
346 $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/;
347 $updlinks{$H} = $updlink;
348 $table .= " <a href='$updlink' >Upd</a>";
349 $table .= " " . $debversions{$H};
350 $table .= "</td></tr>\n";
351 $table .= "<tr><td>$hostcomments{$H}</td></tr>\n"
352 if ( $hostcomments{$H} );
353 $table .= $det if $pkgs;
354 print "\n$table\n" if $debug>2;
356 $table .= "</table>\n";
360 `mv -f $datefilename $dateoldfilename`;
361 open F, ">$datefilename" or die "Could not open date file $datefilename for writing";
362 for my $k (sort(keys(%newdates)) ) {
363 print F "$k " . $newdates{$k}. "\n";
364 print "date for '$k' '" . $newdates{$k}. "'\n" if $debug;
367 or die "Could not close date file $datefilename: $!";
369 print "Not updating the date file, this is a debug run\n";
373 my $outfile = "/tmp/aptcheck.html";
375 or die "Could not open $outfile for writing: $!";
377 print F "<head><title>Apt upgrade status</title></head>\n";
379 print F "<H1>Package status</H1>\n";
380 print F "<b>" . ( $sectot + $owntot + $mantot + $normtot ) .
381 "</b> packages pending (<b>$sectot</b> critical) \n";
383 print F "<H2>Debug run, many hosts missing!</H2>\n"
387 # Summary table: one row for per host group
389 print F "<table border='1' >\n";
390 print F "<tr><td> </td>" ;
391 print F "<td><b>Hosts</b></td>\n";
392 print F "<td><b>Packages</b></td></tr>\n";
395 print F "<tr><td><b>Security</b><br/>" . scalar(keys(%sechosts)) .
396 " / " . scalar(keys(%secpkgs)) . " / $sectot </td>\n" ;
398 for my $HH ( sort(keys(%sechosts)) ) {
399 my $upd = $updlinks{$HH} || "#" ;
400 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
404 for my $PP ( sort(keys(%secpkgs)) ) {
405 print F "<a href='#$PP'>$PP</a> ";
411 print F "<tr><td><b>Indexdata</b><br/>" . scalar(keys(%ownhosts)) .
412 " / " . scalar(keys(%ownpkgs)) . " / $owntot </td>\n" ;
414 for my $HH ( sort(keys(%ownhosts)) ) {
415 my $upd = $updlinks{$HH} || "#" ;
416 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
417 #print F "<a href='#$HH'><b>$HH</b></a> ";
421 for my $PP ( sort(keys(%ownpkgs)) ) {
422 print F "<a href='#$PP'>$PP</a> ";
428 print F "<tr><td><b>Manual</b><br/>" . scalar(keys(%manhosts)) .
429 " / " . scalar(keys(%manpkgs)) . " / $mantot </td>\n" ;
431 for my $HH ( sort(keys(%manhosts)) ) {
432 my $upd = $updlinks{$HH} || "#" ;
433 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
434 #print F "<a href='#$HH'><b>$HH</b></a> ";
438 for my $PP ( sort(keys(%manpkgs)) ) {
439 print F "<a href='#$PP'>$PP</a> ";
445 print F "<tr><td>Normal<br/>" . scalar(keys(%normhosts)) .
446 " / " . scalar(keys(%normpkgs)) . " / $normtot </td>\n" ;
448 for my $HH ( sort(keys(%normhosts)) ) {
449 my $upd = $updlinks{$HH} || "#" ;
450 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
451 #print F "<a href='#$HH'><b>$HH</b></a> ";
455 for my $PP ( sort(keys(%normpkgs)) ) {
456 print F "<a href='#$PP'>$PP</a> ";
462 print F "<tr><td>Skipped " . scalar(keys(%skiphosts)) . "</td>\n";
463 print F "<td colspan='2'>";
464 for my $HH ( sort(keys(%skiphosts)) ) {
465 my $upd = $updlinks{$HH} ||
466 $wikilink . ucfirst($HH) . "Updates" . $year;
467 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
468 #print F "<a href='#$HH'><b>$HH</b></a> ";
470 print F "</td></tr>\n";
473 print F "<tr><td>Ok <br/>" . scalar(keys(%okhosts)) .
474 " of " . scalar(keys(%allhosts)) . "</td>\n";
475 print F "<td colspan='2'>";
476 for my $HH ( sort(keys(%okhosts)) ) {
477 my $upd = $updlinks{$HH} || "#" ;
478 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>$updlink</a> ";
479 #print F "<a href='#$HH'><b>$HH</b></a> ";
482 print F "<b>None at all!</b>";
484 print F "</td></tr>\n";
486 print F "</table>\n";
488 print F "<p/><b>" . ( $sectot + $owntot + $mantot + $normtot ) .
489 "</b> packages pending (<b>$sectot</b> critical) \n";
492 #my $secs = 60*60*24 * 7 * 2; # 2 weeks in secods
493 #my $secs = "1m"; # one month, let nagios do the math
496 "<a href='http://nagios.indexdata.com/cgi-bin/nagios3/graphs.cgi?" .
497 "host=nagios&service=Apt%20Summary'>\n".
498 "<img src='http://nagios.indexdata.com/" .
499 "cgi-bin/nagios3/rrd2-system.cgi?" .
500 "host=nagios&service=Apt%20Summary&" .
502 "width=800&height=100&type=AVERAGE' /> ".
510 print F "<p/><b><u>Packages</u></b>\n";
512 for my $P ( sort(keys(%summary)) ) {
514 $PN = "<b>$P (s)</b>" if ($secpkgs{$P});
515 $PN = "<i>$P (id)</i>" if ($ownpkgs{$P});
516 $PN = "$P <b>(M)</b>" if ($manpkgs{$P});
517 print F "<tr><td><a name='$P'/>$PN</td>\n";
519 for my $HH ( split(' ',$summary{$P} )) {
520 print F "<a href=#$HH>$HH</a> ";
525 print F "</table>\n";
527 print F "<p/>Produced " . `date`.
528 " on " . `hostname` . " by " . `whoami` .
530 print F "</body></html>\n";
533 or die "Could not close $outfile: $!";
535 system "scp -q $outfile nagios:/var/www/heikki/index.html";
539 # Helper to take two strings and highligt that part of the second
540 # that is different from the first.
544 print "strdiff: '$x' '$y' \n" if $debug>2;
546 return "$x <b>??</b>";
549 while ( $a < length($y) &&
550 substr($x,$a,1) eq substr($y,$a,1) ) {
553 if ( $a == length($y) ) {
557 while ( $b < length($y)-$a &&
558 substr($x,-$b,1) eq substr($y, -$b,1) ) {
561 my $c = length($y) - $b +1;
562 print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>2;
563 print "strdiff: b=$b " . "\n" if $debug>2;
564 print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>2;
565 print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>2;
566 my $z = substr($y,0,$a) .
567 "<b>" . substr($y,$a, $c-$a) . "</b>" .
569 print "strdiff: " . $z ."\n" if $debug>2;
570 print "\n" if $debug>2;