3 # Check what packages are needed to get upgraded on all machines
5 # Depends heavily on having ssh key authentication set up to all
6 # boxes. That's why I run it on my own workstation.
8 # Regular debian upgrades are detected by running
10 # on every machine, and parsing the output.
12 # We have decided to maintain some packages manually on some
13 # machines, so that system-level upgrades will not disturb
14 # applications, which may need more hand-holding. These are
15 # extracted from our apt repository, and queried on every
16 # server with apt-cache policy. This way, as soon as a package
17 # is released on our repo, it will get listed here.
19 # 11-Mar-2011 Heikki: Started this
20 # 22-Mar-2011 Heikki: Adding manually maintained packages
21 # 15-Aug-2011 Heikki: Adding a total in the headline, for nagiosgrapher
23 # TODO: Assumes that we release our restricted packages for all versions
24 # and architectures at the same time. Gets only the highest version from
25 # all, and reports anything less than this. Good enough for now.
27 # TODO: Get the dates from ls --full-time /var/cache/apt/archives/
28 # and display next to the packages, so we can see how long they have
29 # been lingering. Boldface them if older than some limit
33 my $debug= $ARGV[0] || 0; # 0=none, 1=some, 2=more, 3=much
35 my $wikilink = 'http://twiki.indexdata.dk/cgi-bin/twiki/view/ID/';
36 my $restrictedpackages = "ssh -q kebab cat /home/ftp/pub/debian/dists/*/restricted/*/Packages";
40 "ariel" => "<i>Niels Erik</i> does the manual upgrades",
41 "bellone" => "<i>Niels Erik</i> does the manual upgrades",
42 "cfrepous" => "<i>Wolfram</i> does the manual upgrades",
43 "leopard" => "<i>Wolfram</i> does the manual upgrades",
44 "lsd" => "<i>Heikki</i> takes care of all upgrades",
48 #### Get list of hosts
49 # I could use a hard-coded list, but I would forget to maintain it.
50 # Nagios knows most of our hosts. It even knows which are worth
51 # checking, they have a command to check apts!
53 my $hostlist = `ssh nagios grep -l Apt /etc/nagios3/indexdata-conf.d/*.cfg`
54 or die "Could not get host list";
56 print "Got list:\n$hostlist\n" if $debug>2;
58 ###### Get list of packages that can be manually maintained
59 print "getting restricted package versions\n" if $debug;
61 my $restplines = `$restrictedpackages`
62 or die "Could not get the list of restricted packages " .
63 "from $restrictedpackages: $! ";
64 print "Got package list: \n$restplines\n" if $debug>2;
67 for my $pline ( split("\n",$restplines) ) {
69 $pname = $1 if $pline =~ /^Package:\s+(\S*)\s*$/;
70 $pver = $1 if $pline =~ /^Version:\s+(\S*)\s*$/;
71 print "$pline: p=$pname v=$pver\n" if $debug>2;
72 if ( $pname && $pver ) {
73 print "\nPackage $pname version $pver \n" if $debug>2;
74 if ( ! $restrpkgs{$pname} ) {
75 $restrpkgs{$pname} = $pver;
76 print "found $pname, first version $pver\n" if $debug>1;
78 my $bver = $restrpkgs{$pname};
79 `dpkg --compare-versions "$bver" lt "$pver" `;
81 print "found $pname, better version $pver (better than $bver)\n"
83 $restrpkgs{$pname} = $pver;
85 print "found $pname, but version $pver is no better than $bver\n"
89 $pname = ""; # clear for the next one.
94 print "got " . scalar(keys(%restrpkgs)) . " restricted packages\n" if $debug;
96 for $pname ( sort (keys(%restrpkgs)) ) {
97 print " $pname " . $restrpkgs{$pname} . "\n";
103 my ( %sechosts, %secpkgs );
104 my ( %ownhosts, %ownpkgs );
105 my ( %manhosts, %manpkgs );
106 my ( %normhosts, %normpkgs );
116 # Pending modification dates
117 my %olddates; # Read in from the file
118 my %newdates; # To be written in the new version of the file
119 my $datefilename = "aptcheck.data";
120 my $dateoldfilename = "aptcheck.old";
121 my $thisdate = "*"; # indicates really old stuff
122 if ( -f $datefilename ) {
123 print "Reading dates from $datefilename\n" if $debug;
124 open F, $datefilename or die "Could not open date file $datefilename: $!";
127 my ($pkg, $date) = split;
128 next unless $pkg; # skip empty lines
129 $olddates{$pkg} = $date;
130 print "Date for '$pkg' is '$date' \n" if $debug;
133 $thisdate = `date +%F`;
136 print "No datefile $datefilename found, starting from scratch\n";
140 my $table = "<table>\n";
142 for my $hline ( split("\n",$hostlist) ) {
143 next unless ( $hline =~ /\/([a-z0-9-]+)\.cfg$/ );
145 next if ($H =~ /^commands/ );
146 next if ($H =~ /^servicegroups/ );
147 print "Checking $H\n" if $debug;
149 my $cmd1 = "apt-cache -q policy " . join(" ",sort(keys(%restrpkgs)));
150 my $cmd2 = "apt-get upgrade -s -o 'Debug::NoLocking=true' ";
151 # Note, do not append -qq, we want some output even when nothing to do
152 print "ssh -q $H \"$cmd1 ; $cmd2 \" 2>/dev/null" if ($debug>1);
153 my $apt = `ssh -q $H "$cmd1 ; $cmd2 " 2>/dev/null`;
155 $table .= "<tr><td colspan='3'> </td></tr>\n";
156 $table .= "<tr><td colspan='3'><b><u>$H</u></b> (skipped)\n";
160 print "Got apts for $H: \n$apt\n" if $debug>2;
161 my $det = ""; # detail lines
169 for my $p ( split("\n",$apt) ) {
170 # parse apt-cache output
171 $restrname = $1 if $p =~ /^(\S+):$/;
172 $restrinst = $1 if $p =~ /^\s+Installed:\s+(\S+)$/;
173 $restrcand = $1 if $p =~ /^\s+Candidate:\s+(\S+)$/;
174 if ( $p =~ /^\s+Version table:/ ) { # have all for that package
175 my $bver = $restrpkgs{$restrname};
176 if ( ( $restrinst eq $restrcand ) &&
177 ( $restrinst ne $bver ) ) {
178 # if different, it is a regular apt upgrade, and will be seen
179 # later. AND we want to have a different version in our repo
180 `dpkg --compare-versions "$bver" lt "$restrinst" `;
181 if ( $? ) { # It was not a downgrade
182 # manual packages may be ahead of the repo!
187 $manpkgs{$restrname} = 1;
189 $det .= "<td> $restrname <b>(M)</b></td>";
190 $det .= "<td>". strdiff($bver,$restrinst)."</td>";
191 $det .= "<td>". strdiff($restrinst,$bver)."</td>";
192 my $datekey = "$H:$restrname";
193 if ( $olddates{$datekey} ) {
194 $newdates{$datekey} = $olddates{$datekey};
196 $newdates{$datekey} = $thisdate;
198 $det .= "<td>" . $newdates{$datekey} . "</td>";
200 my $key = "$restrname";
201 if ( !$summary{$key} ) {
204 $summary{$key} .= "$H ";
207 $restrname = ""; # clear for next round
212 /^Inst ([^ ]+) \[([^]]+)\] \(([^ ]+) ([^:]+):/;
213 my ( $pkg,$cur,$new,$src ) = ( $1,$2,$3,$4 );
214 print "$H: $pkg: $cur -> $new ($src)\n" if $debug>1;
215 $det .= "<tr><td> ";
218 if ( $src =~ /Security/ ) {
219 $det .= "<b>$pkg (s)</b> ";
224 } elsif ( $src =~ /Indexdata/ ) {
225 $det .= "<i><b>$pkg</b> (id) </i>";
236 if ( !$summary{$key} ) {
239 $summary{$key} .= "$H ";
240 $new = strdiff($cur,$new);
241 $cur = strdiff($new,$cur);
243 $det .= "<td>$cur</td> ";
244 $det .= "<td>$new</td> ";
245 my $datekey = "$H:$pkg";
246 if ( $olddates{$datekey} ) {
247 $newdates{$datekey} = $olddates{$datekey};
249 $newdates{$datekey} = $thisdate;
251 $det .= "<td>" . $newdates{$datekey} . "</td>";
255 $table .= "<tr><td colspan='3'> </td></tr>\n";
256 $table .= "<tr><td colspan='3'><a name='$H'><b><u>$H</u></b></a> \n";
258 $table .= "<b>$pkgs</b> packages to upgrade. ";
259 $table .= "<b>$secs security</b>. " if $secs;
260 $table .= " $own from indexdata. " if $own;
261 $table .= " $man manual. " if $man;
266 my $updlink = $wikilink . ucfirst($H) . "Updates" . $year;
267 # Fix some pages that do not follow the convention.
268 # Mostly because the host names would not make proper WikiWords
269 $updlink =~ s/Bugzilla3Updates/BugzillaUpdates/;
270 $updlink =~ s/Opencontent-solrUpdates/OpenContentSolrUpdates/;
271 $updlinks{$H} = $updlink;
272 $table .= " <a href='$updlink' >Upd</a>";
273 $table .= "</td></tr>\n";
274 $table .= "<tr><td>$hostcomments{$H}</td></tr>\n"
275 if ( $hostcomments{$H} );
276 $table .= $det if $pkgs;
277 print "\n$table\n" if $debug>2;
278 last if $H =~/diane/ && $debug;
280 $table .= "</table>\n";
283 `mv -f $datefilename $dateoldfilename`;
284 open F, ">$datefilename" or die "Could not open date file $datefilename for writing";
285 for my $k (keys(%newdates) ) {
286 print F "$k " . $newdates{$k}. "\n";
287 print "date for '$k' '" . $newdates{$k}. "'\n" if $debug;
290 or die "Could not close date file $datefilename: $!";
293 my $outfile = "/tmp/aptcheck.html";
295 or die "Could not open $outfile for writing: $!";
297 print F "<head><title>Apt upgrade status</title></head>\n";
299 print F "<H1>Apt package status</H1>\n";
300 print F "<b>" . ( $sectot + $owntot + $mantot + $normtot ) .
301 "</b> packages pending (<b>$sectot</b> critical) \n";
303 print F "<H2>Debug run, many hosts missing!</H2>\n"
307 # Summary table: one row for per host group
309 print F "<table border='1' >\n";
310 print F "<tr><td> </td>" ;
311 print F "<td><b>Hosts</b></td>\n";
312 print F "<td><b>Packages</b></td></tr>\n";
315 print F "<tr><td><b>Security</b><br/>" . scalar(keys(%sechosts)) .
316 " / " . scalar(keys(%secpkgs)) . " / $sectot </td>\n" ;
318 for my $HH ( sort(keys(%sechosts)) ) {
319 my $upd = $updlinks{$HH} || "#" ;
320 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
324 for my $PP ( sort(keys(%secpkgs)) ) {
325 print F "<a href='#$PP'>$PP</a> ";
331 print F "<tr><td><b>Indexdata</b><br/>" . scalar(keys(%ownhosts)) .
332 " / " . scalar(keys(%ownpkgs)) . " / $owntot </td>\n" ;
334 for my $HH ( sort(keys(%ownhosts)) ) {
335 my $upd = $updlinks{$HH} || "#" ;
336 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
337 #print F "<a href='#$HH'><b>$HH</b></a> ";
341 for my $PP ( sort(keys(%ownpkgs)) ) {
342 print F "<a href='#$PP'>$PP</a> ";
348 print F "<tr><td><b>Manual</b><br/>" . scalar(keys(%manhosts)) .
349 " / " . scalar(keys(%manpkgs)) . " / $mantot </td>\n" ;
351 for my $HH ( sort(keys(%manhosts)) ) {
352 my $upd = $updlinks{$HH} || "#" ;
353 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
354 #print F "<a href='#$HH'><b>$HH</b></a> ";
358 for my $PP ( sort(keys(%manpkgs)) ) {
359 print F "<a href='#$PP'>$PP</a> ";
365 print F "<tr><td>Normal<br/>" . scalar(keys(%normhosts)) .
366 " / " . scalar(keys(%normpkgs)) . " / $normtot </td>\n" ;
368 for my $HH ( sort(keys(%normhosts)) ) {
369 my $upd = $updlinks{$HH} || "#" ;
370 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
371 #print F "<a href='#$HH'><b>$HH</b></a> ";
375 for my $PP ( sort(keys(%normpkgs)) ) {
376 print F "<a href='#$PP'>$PP</a> ";
382 print F "<tr><td>Skipped " . scalar(keys(%skiphosts)) . "</td>\n";
383 print F "<td colspan='2'>";
384 for my $HH ( sort(keys(%skiphosts)) ) {
385 my $upd = $updlinks{$HH} ||
386 $wikilink . ucfirst($HH) . "Updates" . $year;
387 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
388 #print F "<a href='#$HH'><b>$HH</b></a> ";
390 print F "</td></tr>\n";
394 print F "<tr><td>Ok " . scalar(keys(%okhosts)) . "</td>\n";
395 print F "<td colspan='2'>";
396 for my $HH ( sort(keys(%okhosts)) ) {
397 my $upd = $updlinks{$HH} || "#" ;
398 print F "<a href='#$HH'><b>$HH</b></a><a href='$upd'>,</a> ";
399 #print F "<a href='#$HH'><b>$HH</b></a> ";
402 print F "<b>None at all!</b>";
404 print F "</td></tr>\n";
406 print F "</table>\n";
409 #my $secs = 60*60*24 * 7 * 2; # 2 weeks in secods
410 #my $secs = "1m"; # one month, let nagios do the math
413 "<a href='http://nagios.indexdata.com/cgi-bin/nagios3/graphs.cgi?" .
414 "host=nagios&service=Apt%20Summary'>\n".
415 "<img src='http://nagios.indexdata.com/" .
416 "cgi-bin/nagios3/rrd2-system.cgi?" .
417 "host=nagios&service=Apt%20Summary&" .
419 "width=800&height=100&type=AVERAGE' /> ".
427 print F "<p/><b><u>Packages</u></b>\n";
429 for my $P ( sort(keys(%summary)) ) {
431 $PN = "<b>$P (s)</b>" if ($secpkgs{$P});
432 $PN = "<i>$P (id)</i>" if ($ownpkgs{$P});
433 $PN = "$P <b>(M)</b>" if ($manpkgs{$P});
434 print F "<tr><td><a name='$P'/>$PN</td>\n";
436 for my $HH ( split(' ',$summary{$P} )) {
437 print F "<a href=#$HH>$HH</a> ";
442 print F "</table>\n";
444 print F "<p/>Packages marked with * are from the time before started to " .
445 "track package dates \n";
446 print F "<p/>Produced " . `date`.
447 " on " . `hostname` . " by " . `whoami` .
449 print F "</body></html>\n";
452 or die "Could not close $outfile: $!";
454 system "scp -q $outfile nagios:/var/www/heikki/index.html";
458 # Helper to take two strings and highligt that part of the second
459 # that is different from the first.
463 print "strdiff: '$x' '$y' \n" if $debug>2;
465 return "$x <b>??</b>";
468 while ( $a < length($y) &&
469 substr($x,$a,1) eq substr($y,$a,1) ) {
472 if ( $a == length($y) ) {
476 while ( $b < length($y)-$a &&
477 substr($x,-$b,1) eq substr($y, -$b,1) ) {
480 my $c = length($y) - $b +1;
481 print "strdiff: a=$a " . substr($y,0,$a) ."\n" if $debug>2;
482 print "strdiff: b=$b " . "\n" if $debug>2;
483 print "strdiff: c=$c " . substr($y,$c) ."\n" if $debug>2;
484 print "strdiff: " . substr($y,$a, $c-$a) ."\n" if $debug>2;
485 my $z = substr($y,0,$a) .
486 "<b>" . substr($y,$a, $c-$a) . "</b>" .
488 print "strdiff: " . $z ."\n" if $debug>2;
489 print "\n" if $debug>2;