From: Mike Taylor Date: Thu, 6 May 2010 09:56:31 +0000 (+0100) Subject: Merge branch 'master' of ssh://git.indexdata.com/home/git/pub/irspy X-Git-Tag: CPAN-v1.02~54^2~17^2~2 X-Git-Url: http://lists.indexdata.dk/cgi-bin?a=commitdiff_plain;h=2a660313d0c3180a692b8dd276feaad1882173b2;hp=e94b9498d9ecba1701f61a43cf582dc94df0f1e7;p=irspy-moved-to-github.git Merge branch 'master' of ssh://git.indexdata.com/home/git/pub/irspy --- diff --git a/Changes b/Changes index c302cd2..e67dfed 100644 --- a/Changes +++ b/Changes @@ -1,4 +1,3 @@ -$Id: Changes,v 1.23 2007-09-18 16:58:01 mike Exp $ Revision history for Perl extension ZOOM::IRSpy. diff --git a/MANIFEST b/MANIFEST index 4074cb1..807760f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -7,7 +7,7 @@ README bin/Makefile bin/irspy-stats.pl bin/irspy.pl -bin/irspy_xsltproc.pl +bin/irspy-xsltproc.pl bin/reindex.pl bin/setrlimit.c lib/ZOOM/IRSpy.pm @@ -46,7 +46,7 @@ web/htdocs/admin/check.html web/htdocs/admin/delete.html web/htdocs/admin/edit.html web/htdocs/admin/index.html -web/htdocs/admin/upload.html +web/htdocs/upload.html web/htdocs/ap.html web/htdocs/beach.jpeg web/htdocs/chrome/layout.mc diff --git a/Makefile.PL b/Makefile.PL index 0566a8c..0568172 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -1,4 +1,3 @@ -# $Id: Makefile.PL,v 1.14 2007-05-09 12:04:36 mike Exp $ use 5.008; use strict; diff --git a/README b/README index e19d4c1..8020b8d 100644 --- a/README +++ b/README @@ -1,4 +1,3 @@ - ZOOM::IRSpy =========== @@ -32,11 +31,12 @@ sudo apt-get install \ you may need one of the following packages for development: sudo apt-get install \ - make perl-doc libxml2-utils tidy + make perl-doc libxml2-utils tidy yaz libyaz4-dev -and may enable mod_apreq +and may enable mod_apreq: sudo a2enmod apreq -sudo a2enmod apreq +Note: mod_perl works only safe without threads. Please use the non-threaded +debian package: sudo apt-get install apache2-mpm-prefork To install this module type the following: diff --git a/archive/Notes b/archive/Notes index d0fdc1f..d2a5311 100644 --- a/archive/Notes +++ b/archive/Notes @@ -1,4 +1,3 @@ -$Id: Notes,v 1.1 2006-05-31 12:04:19 mike Exp $ Some things that it would be nice to check (varying priorities!) diff --git a/archive/architecture-notes b/archive/architecture-notes index 4764775..6f6395a 100644 --- a/archive/architecture-notes +++ b/archive/architecture-notes @@ -1,4 +1,3 @@ -$Id: architecture-notes,v 1.1 2007-02-20 12:10:02 mike Exp $ The basic approach is this: The IRSpy::Connection is an IRSpy-specific subclass of ZOOM::Connection, and its only important extra diff --git a/archive/separate-apache b/archive/separate-apache index 14c9611..fef03c9 100644 --- a/archive/separate-apache +++ b/archive/separate-apache @@ -1,4 +1,3 @@ -$Id: separate-apache,v 1.1 2007-03-22 12:25:36 mike Exp $ /etc/apache-irspy -- Apache configuration /var/log/apache-irspy -- Apache logs diff --git a/bin/.gdbinit b/bin/.gdbinit index 2a49aa4..00091b8 100644 --- a/bin/.gdbinit +++ b/bin/.gdbinit @@ -1,4 +1,3 @@ -# $Id: .gdbinit,v 1.7 2007-03-15 11:34:51 mike Exp $ set env YAZ_LOG=irspy,irspy_test,irspy_task,zoom,zoom_details,irspy_event set args -I../lib irspy.pl -t Quick -f 'net.host=*indexdata*' localhost:8018/IR-Explain---1 set args -I../lib irspy.pl -n 2 -t Quick localhost:8018/IR-Explain---1 z3950.loc.gov:7090/Voyager bagel.indexdata.dk/gils bagel.indexdata.dk:210/marc diff --git a/bin/.perldb b/bin/.perldb index 5d3ba06..cf40584 100644 --- a/bin/.perldb +++ b/bin/.perldb @@ -1,2 +1 @@ -# $Id: .perldb,v 1.1 2007-02-09 10:48:37 mike Exp $ print "=== loading .perldb ===\n" diff --git a/bin/irspy-rewrite-records.pl b/bin/irspy-rewrite-records.pl old mode 100644 new mode 100755 index f994de6..fc3de55 --- a/bin/irspy-rewrite-records.pl +++ b/bin/irspy-rewrite-records.pl @@ -1,22 +1,64 @@ #!/usr/bin/perl - +# # perl -I ../lib irspy-rewrite-records.pl localhost:8018/IR-Explain---1 +use lib '../lib'; +use Data::Dumper; +use Getopt::Long; +use ZOOM::IRSpy; +use ZOOM::IRSpy::Utils qw(render_record validate_record); + use strict; use warnings; -use ZOOM::IRSpy; -use ZOOM::IRSpy::Utils qw(render_record); -my($dbname) = @ARGV; -die "$0 no database name specified" if !defined $dbname; -my $spy = new ZOOM::IRSpy($dbname, "admin", "fruitbat"); -my $rs = $spy->{conn}->search(new ZOOM::Query::CQL("cql.allRecords=1")); -print STDERR "rewriting ", $rs->size(), " target records"; +my $irspy_to_zeerex_xsl = '../xsl/irspy2zeerex.xsl'; +my $debug = 1; +my $cql_query = "cql.allRecords=1"; -foreach my $i (1 .. $rs->size()) { - my $xml = render_record($rs, $i-1, "zeerex"); +sub usage { + my $message = shift; + + warn "$message\n" if defined $message; + + < \$irspy_to_zeerex_xsl, + "debug=i" => \$debug, + "query=s" => \$cql_query, +); + +my $dbname = shift; +die usage("no database name specified\n") if !defined $dbname; + +$ZOOM::IRSpy::irspy_to_zeerex_xsl = $irspy_to_zeerex_xsl + if $irspy_to_zeerex_xsl; + +my $spy = new ZOOM::IRSpy( $dbname, "admin", "fruitbat" ); +my $rs = $spy->{conn}->search( new ZOOM::Query::CQL($cql_query) ); +print STDERR "rewriting ", $rs->size(), " target records\n" if $debug; + +foreach my $i ( 1 .. $rs->size() ) { + my $xml = render_record( $rs, $i - 1, "zeerex" ); my $rec = $spy->{libxml}->parse_string($xml)->documentElement(); - ZOOM::IRSpy::_rewrite_zeerex_record($spy->{conn}, $rec); - print STDERR "."; + + if ( $debug >= 2 ) { + my ( $ok, $errors ) = validate_record($rec); + if ( !$ok ) { + my @e = @$errors; + my $id = shift @e; + print "Id: $id => ", join( " / ", @e ), "\n"; + } + } + ZOOM::IRSpy::_rewrite_zeerex_record( $spy->{conn}, $rec ); + print STDERR "." if $debug == 1; } -print STDERR "\nDone\n"; +print STDERR "Done\n" if $debug; + diff --git a/bin/irspy-update.sh b/bin/irspy-update.sh index 90d5259..5113d50 100755 --- a/bin/irspy-update.sh +++ b/bin/irspy-update.sh @@ -8,11 +8,27 @@ home=/usr/local/src/git cd $home/irspy/bin || exit 2 logdir=../tmp +lockfile=../tmp/irspy-update.lock +: ${irspy_test=Main} +if [ -f $lockfile ]; then + pid=`cat $lockfile` + if kill -0 $pid 2>/dev/null; then + echo "This script is already running with pid: $pid" + exit 1 + fi +fi +echo $$ > $lockfile || exit 2 + +weekday=`date '+%w'` for i in 0 1 2 3 4 5 6 do - logfile=$logdir/irspy-mod-$i.log.`date '+%w'` - YAZ_LOG=irspy,irspy_test nice -10 time perl -I../lib irspy.pl -n 50 -d -M 3500 -a -t Main -m 7,$i localhost:8018/IR-Explain---1 > $logfile 2>&1 + logfile=$logdir/irspy-mod-$i.log.$weekday + YAZ_LOG=irspy,irspy_test,irspy_task nice -10 time perl -I../lib irspy.pl -n 50 -d -M 3500 -f'cql.allRecords=1 not zeerex.disabled = 1' -t $irspy_test -m 7,$i localhost:8018/IR-Explain---1 > $logfile 2>&1 + + sleep 1 # catch ctr-c before compressing the log gzip -f $logfile done +rm -f $lockfile + diff --git a/lib/ZOOM/IRSpy.pm b/lib/ZOOM/IRSpy.pm index 81fa911..4e2040c 100644 --- a/lib/ZOOM/IRSpy.pm +++ b/lib/ZOOM/IRSpy.pm @@ -1,4 +1,3 @@ -# $Id: IRSpy.pm,v 1.90 2008-07-16 11:42:13 mike Exp $ package ZOOM::IRSpy; @@ -26,6 +25,7 @@ our $VERSION = '1.02'; our $irspy_to_zeerex_xsl = dirname(__FILE__) . '/../../xsl/irspy2zeerex.xsl'; our $debug = 0; our $xslt_max_depth = 250; +our $max_timeout_errors = 3; # Enumeration for callback functions to return @@ -443,10 +443,12 @@ sub check { while (1) { my @copy_conn = @conn; # avoid alias problems after splice() my $nconn = scalar(@copy_conn); + foreach my $i0 (0 .. $#copy_conn) { my $conn = $copy_conn[$i0]; #print "connection $i0 of $nconn/", scalar(@conn), " is $conn\n"; next if !defined $conn; + if (!$conn->current_task()) { if (!$conn->next_task()) { # Out of tasks: we need a new test @@ -460,6 +462,12 @@ sub check { "checking for next test after '$address'"); $nextaddr = $this->_next_test($address); } + + if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) { + $conn->log("irspy", "Got to many timeouts, stop testing"); + undef $nextaddr; + } + if (!defined $nextaddr) { $conn->log("irspy", "has no more tests: removing"); $this->_rewrite_irspy_record($conn); @@ -501,6 +509,13 @@ sub check { my $task = $conn->next_task(); die "no next task queued for $conn" if !defined $task; + + # do not run the next task if we got too many timeouts + if (ZOOM::IRSpy::Test::zoom_error_timeout_check($conn)) { + $conn->log("irspy_task", "Got to many timeouts for this target, do not start a new task"); + next; + } + $conn->log("irspy_task", "preparing task $task"); $conn->next_task(0); $conn->current_task($task); diff --git a/lib/ZOOM/IRSpy/Connection.pm b/lib/ZOOM/IRSpy/Connection.pm index 469968e..6157f98 100644 --- a/lib/ZOOM/IRSpy/Connection.pm +++ b/lib/ZOOM/IRSpy/Connection.pm @@ -1,4 +1,3 @@ -# $Id: Connection.pm,v 1.21 2007-12-20 12:35:01 mike Exp $ package ZOOM::IRSpy::Connection; diff --git a/lib/ZOOM/IRSpy/Maintenance.pod b/lib/ZOOM/IRSpy/Maintenance.pod index 4c8e310..ef08f45 100644 --- a/lib/ZOOM/IRSpy/Maintenance.pod +++ b/lib/ZOOM/IRSpy/Maintenance.pod @@ -1,4 +1,3 @@ -# $Id: Maintenance.pod,v 1.4 2006-09-18 10:27:57 mike Exp $ package ZOOM::IRSpy::Maintenance; diff --git a/lib/ZOOM/IRSpy/Node.pm b/lib/ZOOM/IRSpy/Node.pm index 9d2f5c7..7e0b9b1 100644 --- a/lib/ZOOM/IRSpy/Node.pm +++ b/lib/ZOOM/IRSpy/Node.pm @@ -1,4 +1,3 @@ -# $Id: Node.pm,v 1.6 2007-02-28 17:34:54 mike Exp $ package ZOOM::IRSpy::Node; diff --git a/lib/ZOOM/IRSpy/Record.pm b/lib/ZOOM/IRSpy/Record.pm index e80fe12..c5b6c1b 100644 --- a/lib/ZOOM/IRSpy/Record.pm +++ b/lib/ZOOM/IRSpy/Record.pm @@ -1,4 +1,3 @@ -# $Id: Record.pm,v 1.28 2007-12-12 08:49:58 mike Exp $ package ZOOM::IRSpy::Record; ### I don't think there's any reason for this to be separate from @@ -42,6 +41,7 @@ sub new { target => $target, parser => $parser, zeerex => $parser->parse_string($zeerex)->documentElement(), + zoom_error => { TIMEOUT => 0 }, }, $class; #Scalar::Util::weaken($this->{irspy}); @@ -50,6 +50,7 @@ sub new { return $this; } +sub zoom_error { return shift->{'zoom_error'} } sub _empty_zeerex_record { my($target) = @_; diff --git a/lib/ZOOM/IRSpy/Stats.pm b/lib/ZOOM/IRSpy/Stats.pm index f701740..6a59c6b 100644 --- a/lib/ZOOM/IRSpy/Stats.pm +++ b/lib/ZOOM/IRSpy/Stats.pm @@ -1,4 +1,3 @@ -# $Id: Stats.pm,v 1.7 2007-10-31 16:42:13 mike Exp $ package ZOOM::IRSpy::Stats; @@ -114,7 +113,7 @@ sub _gather_stats { # Record syntax support by database foreach my $node ($xc->findnodes('e:recordInfo/e:recordSyntax/@name')) { - $this->{recordSyntaxes}->{$node->findvalue(".")}++; + $this->{recordSyntaxes}->{lc($node->findvalue("."))}++; } # Explain support @@ -136,7 +135,7 @@ sub _gather_stats { # Top Domains my $host = $xc->findvalue('e:serverInfo/e:host'); $host =~ s/.*\.//; - $this->{domains}->{$host}++; + $this->{domains}->{lc($host)}++; # Implementation foreach my $node ($xc->findnodes('i:status/i:serverImplementationName/@value')) { diff --git a/lib/ZOOM/IRSpy/Task.pm b/lib/ZOOM/IRSpy/Task.pm index 4340198..a115649 100644 --- a/lib/ZOOM/IRSpy/Task.pm +++ b/lib/ZOOM/IRSpy/Task.pm @@ -1,4 +1,3 @@ -# $Id: Task.pm,v 1.7 2007-08-01 15:11:03 mike Exp $ package ZOOM::IRSpy::Task; diff --git a/lib/ZOOM/IRSpy/Task/Connect.pm b/lib/ZOOM/IRSpy/Task/Connect.pm index 5080a7a..117f062 100644 --- a/lib/ZOOM/IRSpy/Task/Connect.pm +++ b/lib/ZOOM/IRSpy/Task/Connect.pm @@ -1,4 +1,3 @@ -# $Id: Connect.pm,v 1.6 2006-11-02 16:11:44 mike Exp $ # See ZOOM/IRSpy/Task/Search.pm for documentation diff --git a/lib/ZOOM/IRSpy/Task/Retrieve.pm b/lib/ZOOM/IRSpy/Task/Retrieve.pm index 4d7a8c5..1580e94 100644 --- a/lib/ZOOM/IRSpy/Task/Retrieve.pm +++ b/lib/ZOOM/IRSpy/Task/Retrieve.pm @@ -1,4 +1,3 @@ -# $Id: Retrieve.pm,v 1.6 2007-05-09 11:05:30 mike Exp $ package ZOOM::IRSpy::Task::Retrieve; diff --git a/lib/ZOOM/IRSpy/Task/Search.pm b/lib/ZOOM/IRSpy/Task/Search.pm index 68397c0..8437586 100644 --- a/lib/ZOOM/IRSpy/Task/Search.pm +++ b/lib/ZOOM/IRSpy/Task/Search.pm @@ -1,4 +1,3 @@ -# $Id: Search.pm,v 1.16 2007-12-18 11:59:42 mike Exp $ package ZOOM::IRSpy::Task::Search; diff --git a/lib/ZOOM/IRSpy/Test.pm b/lib/ZOOM/IRSpy/Test.pm index 765cd4f..0bf89bd 100644 --- a/lib/ZOOM/IRSpy/Test.pm +++ b/lib/ZOOM/IRSpy/Test.pm @@ -1,4 +1,3 @@ -# $Id: Test.pm,v 1.8 2007-04-18 15:24:45 mike Exp $ package ZOOM::IRSpy::Test; @@ -8,6 +7,9 @@ use warnings; use Scalar::Util; +use Exporter 'import'; +our @EXPORT = qw(zoom_error_timeout_update zoom_error_timeout_check); + =head1 NAME ZOOM::IRSpy::Test - base class for tests in IRSpy @@ -22,6 +24,7 @@ I<## To follow> =cut + sub subtests { () } sub timeout { undef } @@ -34,6 +37,29 @@ sub start { } +our $max_timeout_errors = $ZOOM::IRSpy::max_timeout_errors; + +sub zoom_error_timeout_update { + my ($conn, $exception) = @_; + + if ($exception =~ /Timeout/i) { + $conn->record->zoom_error->{TIMEOUT}++; + $conn->log("irspy_test", "Increase timeout error counter to: " . + $conn->record->zoom_error->{TIMEOUT}); + } +} + +sub zoom_error_timeout_check { + my $conn = shift; + + if ($conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors) { + $conn->log("irspy_test", "Got $max_timeout_errors or more timeouts, give up..."); + return 1; + } + + return 0; +} + =head1 SEE ALSO ZOOM::IRSpy diff --git a/lib/ZOOM/IRSpy/Test/Main.pm b/lib/ZOOM/IRSpy/Test/Main.pm index 6814217..428afd3 100644 --- a/lib/ZOOM/IRSpy/Test/Main.pm +++ b/lib/ZOOM/IRSpy/Test/Main.pm @@ -1,4 +1,3 @@ -# $Id: Main.pm,v 1.14 2007-04-18 15:24:55 mike Exp $ package ZOOM::IRSpy::Test::Main; diff --git a/lib/ZOOM/IRSpy/Test/Ping.pm b/lib/ZOOM/IRSpy/Test/Ping.pm index bc915db..31b7f4d 100644 --- a/lib/ZOOM/IRSpy/Test/Ping.pm +++ b/lib/ZOOM/IRSpy/Test/Ping.pm @@ -1,4 +1,3 @@ -# $Id: Ping.pm,v 1.28 2008-07-16 11:51:09 mike Exp $ # See the "Main" test package for documentation diff --git a/lib/ZOOM/IRSpy/Test/Quick.pm b/lib/ZOOM/IRSpy/Test/Quick.pm index 8a2f54b..c184ccd 100644 --- a/lib/ZOOM/IRSpy/Test/Quick.pm +++ b/lib/ZOOM/IRSpy/Test/Quick.pm @@ -1,4 +1,3 @@ -# $Id: Quick.pm,v 1.4 2007-04-18 15:25:07 mike Exp $ package ZOOM::IRSpy::Test::Quick; diff --git a/lib/ZOOM/IRSpy/Test/Record/Fetch.pm b/lib/ZOOM/IRSpy/Test/Record/Fetch.pm index ccaaa9f..4d42200 100644 --- a/lib/ZOOM/IRSpy/Test/Record/Fetch.pm +++ b/lib/ZOOM/IRSpy/Test/Record/Fetch.pm @@ -1,4 +1,3 @@ -# $Id: Fetch.pm,v 1.31 2009-04-21 13:42:58 mike Exp $ # See the "Main" test package for documentation @@ -11,6 +10,7 @@ use warnings; use ZOOM::IRSpy::Test; our @ISA = qw(ZOOM::IRSpy::Test); +our $max_timeout_errors = $ZOOM::IRSpy::max_timeout_errors; # These queries my @queries = ( @@ -42,12 +42,22 @@ sub completed_search { $conn->log("irspy_test", "Fetch test search (", $task->render_query(), ") ", ref $event && $event->isa("ZOOM::Exception") ? "failed: $event" : "found $n records (event=$event)"); + + # remember how often a target record hit a timeout + if (ref $event && $event->isa("ZOOM::Exception")) { + if ($event =~ /Timeout/i) { + $conn->record->zoom_error->{TIMEOUT}++; + $conn->log("irspy_test", "Increase timeout error counter to: " . + $conn->record->zoom_error->{TIMEOUT}); + } + } + if ($n == 0) { $task->{rs}->destroy(); my $qindex = $udata->{queryindex}+1; my $q = $queries[$qindex]; return ZOOM::IRSpy::Status::TEST_SKIPPED - if !defined $q; + if !defined $q || $conn->record->zoom_error->{TIMEOUT} >= $max_timeout_errors; $conn->log("irspy_test", "Trying another search ..."); $conn->irspy_search_pqf($queries[$qindex], { queryindex => $qindex }, {}, @@ -66,7 +76,7 @@ sub completed_search { 'librismarc', 'mab', 'normarc', -# 'opac', + 'opac', 'picamarc', 'rusmarc', 'summary', diff --git a/lib/ZOOM/IRSpy/Test/Record/Main.pm b/lib/ZOOM/IRSpy/Test/Record/Main.pm index 67bd259..0dc78c7 100644 --- a/lib/ZOOM/IRSpy/Test/Record/Main.pm +++ b/lib/ZOOM/IRSpy/Test/Record/Main.pm @@ -1,4 +1,3 @@ -# $Id: Main.pm,v 1.1 2006-10-23 13:54:52 sondberg Exp $ package ZOOM::IRSpy::Test::Record::Main; diff --git a/lib/ZOOM/IRSpy/Test/ResultSet/Main.pm b/lib/ZOOM/IRSpy/Test/ResultSet/Main.pm index 7b3e3ab..c562246 100644 --- a/lib/ZOOM/IRSpy/Test/ResultSet/Main.pm +++ b/lib/ZOOM/IRSpy/Test/ResultSet/Main.pm @@ -1,4 +1,3 @@ -# $Id: Main.pm,v 1.1 2006-11-02 11:46:40 sondberg Exp $ package ZOOM::IRSpy::Test::ResultSet::Main; diff --git a/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm b/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm index eb3a342..9acc097 100644 --- a/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm +++ b/lib/ZOOM/IRSpy/Test/ResultSet/Named.pm @@ -1,4 +1,3 @@ -# $Id: Named.pm,v 1.6 2007-08-01 15:11:20 mike Exp $ # See the "Main" test package for documentation @@ -104,6 +103,7 @@ sub error { my($conn, $task, $test_args, $exception) = @_; $conn->log("irspy_test", "Named resultset check failed:", $exception); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TASK_DONE; } diff --git a/lib/ZOOM/IRSpy/Test/Search/Bib1.pm b/lib/ZOOM/IRSpy/Test/Search/Bib1.pm index 7dc644e..fd4d1d6 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Bib1.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Bib1.pm @@ -1,4 +1,3 @@ -# $Id: Bib1.pm,v 1.18 2007-03-15 11:40:27 mike Exp $ # See the "Main" test package for documentation @@ -52,6 +51,7 @@ sub error { $conn->log("irspy_test", "search on access-point $attr had error: ", $exception); update($conn, $attr, 0); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TEST_BAD if ($exception->code() == 1 || # permanent system error @@ -69,5 +69,4 @@ sub update { 'ok' => $ok); } - 1; diff --git a/lib/ZOOM/IRSpy/Test/Search/Boolean.pm b/lib/ZOOM/IRSpy/Test/Search/Boolean.pm index 8d3e5bb..ea8233a 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Boolean.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Boolean.pm @@ -1,4 +1,3 @@ -# $Id: Boolean.pm,v 1.5 2007-03-15 11:40:39 mike Exp $ # See the "Main" test package for documentation @@ -53,6 +52,7 @@ sub error { $conn->log("irspy_test", "search using boolean operator ", $operator, " had error: ", $exception); update($conn, $operator, 0); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TASK_DONE; } diff --git a/lib/ZOOM/IRSpy/Test/Search/CQL.pm b/lib/ZOOM/IRSpy/Test/Search/CQL.pm index 5a5385c..5c21281 100644 --- a/lib/ZOOM/IRSpy/Test/Search/CQL.pm +++ b/lib/ZOOM/IRSpy/Test/Search/CQL.pm @@ -1,4 +1,3 @@ -# $Id: CQL.pm,v 1.1 2007-05-01 15:32:16 mike Exp $ # See the "Main" test package for documentation @@ -53,6 +52,7 @@ sub error { $task->{rs}->destroy(); $conn->log("irspy_test", "CQL search on '$index' had error: $exception"); $conn->record()->store_result("search_cql", index => $index, ok => 0); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TEST_BAD if $exception->code() == 11; # Unsupported query type diff --git a/lib/ZOOM/IRSpy/Test/Search/DBDate.pm b/lib/ZOOM/IRSpy/Test/Search/DBDate.pm index 5359477..b349f1b 100644 --- a/lib/ZOOM/IRSpy/Test/Search/DBDate.pm +++ b/lib/ZOOM/IRSpy/Test/Search/DBDate.pm @@ -1,4 +1,3 @@ -# $Id: DBDate.pm,v 1.4 2007-02-23 15:03:44 mike Exp $ # This plugin tests searching on BIB-1 access-point 1011 (Date/time # added to db), the significance of which is that this search @@ -51,6 +50,7 @@ sub error { my $rec = $conn->record(); $rec->append_entry("irspy:status", "" . isodate(time()) . ""); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TEST_BAD; } diff --git a/lib/ZOOM/IRSpy/Test/Search/Dan1.pm b/lib/ZOOM/IRSpy/Test/Search/Dan1.pm index 51b6607..415c5aa 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Dan1.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Dan1.pm @@ -1,4 +1,3 @@ -# $Id: Dan1.pm,v 1.7 2007-03-15 11:40:52 mike Exp $ # See the "Main" test package for documentation @@ -49,6 +48,7 @@ sub error { $conn->log("irspy_test", "search on access-point $attr had error: ", $exception); update($conn, $attr, 0); + zoom_error_timeout_update($conn, $exception); ### How about TEST_BAD if $exception->code() == 121? return ZOOM::IRSpy::Status::TASK_DONE; diff --git a/lib/ZOOM/IRSpy/Test/Search/Explain.pm b/lib/ZOOM/IRSpy/Test/Search/Explain.pm index 28a18da..087b846 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Explain.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Explain.pm @@ -1,4 +1,3 @@ -# $Id: Explain.pm,v 1.7 2007-03-15 11:41:24 mike Exp $ # See the "Main" test package for documentation @@ -54,6 +53,7 @@ sub error { $task->{rs}->destroy(); $conn->log("irspy_test", "Explain category lookup failed: ", $exception); update($conn, $category, 0); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TEST_BAD if ($exception->code() == 109 || # Database unavailable diff --git a/lib/ZOOM/IRSpy/Test/Search/Main.pm b/lib/ZOOM/IRSpy/Test/Search/Main.pm index f657a9d..9e928ce 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Main.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Main.pm @@ -1,4 +1,3 @@ -# $Id: Main.pm,v 1.8 2007-05-01 15:30:38 mike Exp $ package ZOOM::IRSpy::Test::Search::Main; diff --git a/lib/ZOOM/IRSpy/Test/Search/Title.pm b/lib/ZOOM/IRSpy/Test/Search/Title.pm index 86f5aec..16d3a06 100644 --- a/lib/ZOOM/IRSpy/Test/Search/Title.pm +++ b/lib/ZOOM/IRSpy/Test/Search/Title.pm @@ -1,4 +1,3 @@ -# $Id: Title.pm,v 1.10 2007-02-23 15:03:44 mike Exp $ # See the "Main" test package for documentation @@ -45,6 +44,7 @@ sub error { my $rec = $conn->record(); $rec->append_entry("irspy:status", "" . isodate(time()) . ""); + zoom_error_timeout_update($conn, $exception); return ZOOM::IRSpy::Status::TEST_BAD; } diff --git a/lib/ZOOM/IRSpy/Utils.pm b/lib/ZOOM/IRSpy/Utils.pm index dfe4e99..4fd7677 100644 --- a/lib/ZOOM/IRSpy/Utils.pm +++ b/lib/ZOOM/IRSpy/Utils.pm @@ -1,4 +1,3 @@ -# $Id: Utils.pm,v 1.38 2009-04-15 18:16:45 wosch Exp $ package ZOOM::IRSpy::Utils; @@ -10,6 +9,8 @@ use Scalar::Util; use Exporter 'import'; our @EXPORT_OK = qw(utf8param + trimField + utf8paramTrim isodate xml_encode cql_quote @@ -21,6 +22,7 @@ our @EXPORT_OK = qw(utf8param modify_xml_document bib1_access_point render_record + validate_record calc_reliability_string calc_reliability_stats); @@ -83,7 +85,6 @@ sub utf8param { return $cooked; } - # Utility functions follow, exported for use of web UI sub utf8param_apache1 { my($r, $key, $value) = @_; @@ -105,6 +106,26 @@ sub isodate { $year+1900, $mon+1, $mday, $hour, $min, $sec); } +# strips whitespaces at start and ends of a field +sub trimField { + my $field = shift; + + $field =~ s/^\s+//; + $field =~ s/\s+$//; + + return $field; +} + +# utf8param() with trim +sub utf8paramTrim { + my $result = utf8param(@_); + + if (defined $result) { + $result = trimField($result); + } + + return $result; +} # I can't -- just can't, can't, can't -- believe that this function # isn't provided by one of the core XML modules. But the evidence all @@ -332,7 +353,7 @@ sub modify_xml_document { } } else { - next if !$value; # No need to create a new empty node + next if !defined $value; # No need to create a new empty node my($ppath, $selector) = $xpath =~ /(.*)\/(.*)/; dom_add_node($xc, $ppath, $selector, $value, @addAfter); #print "New $key ($xpath) = '$value'
\n"; @@ -796,9 +817,59 @@ sub calc_reliability_stats { return (0, 0, 0) if $nall == 0; my @okpings = $xc->findnodes('i:status/i:probe[@ok = "1"]'); my $nok = @okpings; - my $percent = int(100*$nok/$nall); + my $percent = int(100*$nok/$nall + 0.5); return ($nok, $nall, $percent); } +# +# validate_record( record, ( "port" => 1, "database" => 1, "country" => 0, ... )) +# +sub validate_record { + my $rec = shift; + my %args = @_; + + my %required = map { $_ => 1 } qw/port host database protocol/; + my %optional = map { $_ => 1 } qw/country type hosturl contact language/; + my %tests = ( %required, %args ); + + my $xc = irspy_xpath_context($rec); + + my $protocol = $xc->findnodes("e:serverInfo/\@protocol") || ""; + my $port = $xc->findnodes("e:serverInfo/e:port") || ""; + my $host = $xc->findnodes("e:serverInfo/e:host") || ""; + my $dbname = $xc->findnodes("e:serverInfo/e:database") || ""; + + my $id = irspy_make_identifier($protocol, $host, $port, $dbname); + + if ($protocol =~ /\s+$/ || $dbname =~ /\s+$/) { + warn "xxx: $protocol:$host:$port:$dbname: whitespaces\n"; + } + + my @errors = $id; + + if ($tests{'protocol'}) { + push(@errors, 'protocol number is not valid') if $protocol !~ /^(z39\.50|sru|srw|tcp)$/i; + } + + if ($tests{'port'}) { + push(@errors, 'port number is not valid') if $port !~ /^\d+$/; + } + + if ($tests{'host'}) { + push(@errors, 'host name is not valid') if $host !~ /^[0-9a-z]+[0-9a-z\.\-]*\.[0-9a-z]+$/i; + } + + if ($tests{'database'}) { + push(@errors, 'database name is not valid') if $dbname =~ m,/,i; + push(@errors, 'database has trailing spaces') if $dbname =~ /^\s+|\s+$/; + } + + if ($tests{'hosturl'}) { + my $hosturl = $xc->findnodes("i:status/i:hostURL") || ""; + push(@errors, 'This hosturl name is not valid') if $hosturl !~ /^\w+$/i; + } + + return ( !$#errors, \@errors ); +} 1; diff --git a/lib/ZOOM/IRSpy/Web.pm b/lib/ZOOM/IRSpy/Web.pm index 7f7dd9d..f99ae67 100644 --- a/lib/ZOOM/IRSpy/Web.pm +++ b/lib/ZOOM/IRSpy/Web.pm @@ -1,4 +1,3 @@ -# $Id: Web.pm,v 1.5 2007-05-08 16:20:04 mike Exp $ package ZOOM::IRSpy::Web; diff --git a/lib/ZOOM/IRSpy/WebService.pod b/lib/ZOOM/IRSpy/WebService.pod index a30bfa6..a30d52a 100644 --- a/lib/ZOOM/IRSpy/WebService.pod +++ b/lib/ZOOM/IRSpy/WebService.pod @@ -1,4 +1,3 @@ -# $Id: WebService.pod,v 1.5 2007-01-24 09:28:02 mike Exp $ package ZOOM::IRSpy::WebService; diff --git a/lib/ZOOM/IRSpy/x.pl b/lib/ZOOM/IRSpy/x.pl index 20b2d97..c2d0a21 100644 --- a/lib/ZOOM/IRSpy/x.pl +++ b/lib/ZOOM/IRSpy/x.pl @@ -1,6 +1,5 @@ #!/usr/bin/perl -w -# $Id: x.pl,v 1.1 2007-02-28 16:52:34 mike Exp $ ### This should be massaged into a test-suite script in ../../../t diff --git a/lib/ZOOM/Pod.pm b/lib/ZOOM/Pod.pm index 5faee4d..15644ed 100644 --- a/lib/ZOOM/Pod.pm +++ b/lib/ZOOM/Pod.pm @@ -1,4 +1,3 @@ -# $Id: Pod.pm,v 1.22 2006-10-06 11:33:07 mike Exp $ package ZOOM::Pod; diff --git a/lib/ZOOM/XML/Simple.pm b/lib/ZOOM/XML/Simple.pm index e666ba0..8c8a62e 100644 --- a/lib/ZOOM/XML/Simple.pm +++ b/lib/ZOOM/XML/Simple.pm @@ -1,4 +1,3 @@ -# $Id: Simple.pm,v 1.2 2006-07-21 11:29:17 mike Exp $ package ZOOM::XML::Simple; diff --git a/web/conf/apache2.0/irspy-test.conf b/web/conf/apache2.0/irspy-test.conf index bc53345..c091986 100644 --- a/web/conf/apache2.0/irspy-test.conf +++ b/web/conf/apache2.0/irspy-test.conf @@ -16,6 +16,9 @@ ErrorLog /var/log/apache2/irspy-test-error.log CustomLog /var/log/apache2/irspy-test-access.log combined DocumentRoot /usr/local/src/git/irspy-test/web/htdocs + + Alias /robots.txt /usr/local/src/git/irspy-test/web/htdocs/robots-test.txt + SetHandler perl-script PerlHandler HTML::Mason::ApacheHandler diff --git a/web/conf/apache2.0/irspy.conf b/web/conf/apache2.0/irspy.conf index 5b0a6e3..53720a3 100644 --- a/web/conf/apache2.0/irspy.conf +++ b/web/conf/apache2.0/irspy.conf @@ -1,5 +1,3 @@ -# $Id: cfspy.conf,v 1.4 2009-04-16 18:31:20 wosch Exp $ -# # Sample configuration for running an IRSpy web-site under Apache 2.x. # # This is the configuration that I use on the development machine diff --git a/web/htdocs/add.html b/web/htdocs/add.html deleted file mode 100644 index 1ff60a9..0000000 --- a/web/htdocs/add.html +++ /dev/null @@ -1,4 +0,0 @@ -%# $Id: add.html,v 1.3 2007-07-16 11:55:31 mike Exp $ -<& /chrome/layout.mc, %ARGS, - title => "Add target record", - component => "/details/edit.mc", op => "new" &> diff --git a/web/htdocs/add_target.html b/web/htdocs/add_target.html new file mode 100644 index 0000000..9df79ee --- /dev/null +++ b/web/htdocs/add_target.html @@ -0,0 +1,3 @@ +<& /chrome/layout.mc, %ARGS, + title => "Add target record", + component => "/details/edit.mc", op => "new" &> diff --git a/web/htdocs/admin/all.html b/web/htdocs/admin/all.html index 2fd1516..c7881b0 100644 --- a/web/htdocs/admin/all.html +++ b/web/htdocs/admin/all.html @@ -1,3 +1,2 @@ -%# $Id: all.html,v 1.1 2007-02-20 19:17:14 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Test all targets", component => "/details/check.mc", id => "" &> diff --git a/web/htdocs/admin/check.html b/web/htdocs/admin/check.html index 8925856..81354e8 100644 --- a/web/htdocs/admin/check.html +++ b/web/htdocs/admin/check.html @@ -1,4 +1,3 @@ -%# $Id: check.html,v 1.1 2007-02-20 19:17:14 mike Exp $ <%args> @id $test => undef diff --git a/web/htdocs/admin/delete.html b/web/htdocs/admin/delete.html index 8206bf0..51182d8 100644 --- a/web/htdocs/admin/delete.html +++ b/web/htdocs/admin/delete.html @@ -1,4 +1,3 @@ -%# $Id: delete.html,v 1.1 2007-02-20 19:17:14 mike Exp $ <%args> $id => undef diff --git a/web/htdocs/admin/edit.html b/web/htdocs/admin/edit.html index 70915a4..bf4d758 100644 --- a/web/htdocs/admin/edit.html +++ b/web/htdocs/admin/edit.html @@ -1,4 +1,3 @@ -%# $Id: edit.html,v 1.1 2007-02-20 19:17:14 mike Exp $ <%args> $op $id => undef diff --git a/web/htdocs/ap.html b/web/htdocs/ap.html index 05401aa..435ced8 100644 --- a/web/htdocs/ap.html +++ b/web/htdocs/ap.html @@ -1,4 +1,3 @@ -%# $Id: ap.html,v 1.1 2007-06-28 12:19:28 mike Exp $ <%args> $set diff --git a/web/htdocs/chrome/layout.mc b/web/htdocs/chrome/layout.mc index d7dc956..4de8b67 100644 --- a/web/htdocs/chrome/layout.mc +++ b/web/htdocs/chrome/layout.mc @@ -1,4 +1,3 @@ -%# $Id: layout.mc,v 1.38 2007-07-17 13:07:57 mike Exp $ <%args> $debug => undef $title @@ -8,7 +7,7 @@ $component use URI::Escape qw(uri_escape uri_escape_utf8); use ZOOM; use ZOOM::IRSpy::Web; -use ZOOM::IRSpy::Utils qw(utf8param isodate xml_encode cql_target cql_quote +use ZOOM::IRSpy::Utils qw(utf8param trimField utf8paramTrim isodate xml_encode cql_target cql_quote irspy_xpath_context irspy_make_identifier irspy_record2identifier irspy_identifier2target modify_xml_document @@ -64,9 +63,9 @@ use ZOOM::IRSpy::Utils qw(utf8param isodate xml_encode cql_target cql_quote

Home
- Test all targets
+ Find a target
- Add a target
+ Add a target
Upload a target
Statistics

diff --git a/web/htdocs/details/ap.mc b/web/htdocs/details/ap.mc index 51ba3ea..4db59c8 100644 --- a/web/htdocs/details/ap.mc +++ b/web/htdocs/details/ap.mc @@ -1,4 +1,3 @@ -%# $Id: ap.mc,v 1.1 2007-06-28 12:19:37 mike Exp $ <%args> $id $set diff --git a/web/htdocs/details/check.mc b/web/htdocs/details/check.mc index 7d22a03..bc99268 100644 --- a/web/htdocs/details/check.mc +++ b/web/htdocs/details/check.mc @@ -1,4 +1,3 @@ -%# $Id: check.mc,v 1.18 2007-05-09 10:45:14 mike Exp $ <%args> @id $test => "Quick" diff --git a/web/htdocs/details/country-list.mc b/web/htdocs/details/country-list.mc index 296946e..f986334 100644 --- a/web/htdocs/details/country-list.mc +++ b/web/htdocs/details/country-list.mc @@ -1,4 +1,3 @@ -%# $Id: country-list.mc,v 1.2 2007-03-30 11:14:56 mike Exp $ <%perl> return [ "", diff --git a/web/htdocs/details/delete.mc b/web/htdocs/details/delete.mc index a1ae5f2..fce37c1 100644 --- a/web/htdocs/details/delete.mc +++ b/web/htdocs/details/delete.mc @@ -1,4 +1,3 @@ -%# $Id: delete.mc,v 1.7 2007-06-27 11:09:32 mike Exp $ <%args> $id $really => 0 diff --git a/web/htdocs/details/doc.mc b/web/htdocs/details/doc.mc index 58c255f..5b93d30 100644 --- a/web/htdocs/details/doc.mc +++ b/web/htdocs/details/doc.mc @@ -1,4 +1,3 @@ -%# $Id: doc.mc,v 1.4 2007-05-03 14:13:19 mike Exp $ <%once> use Pod::Html; use IO::Dir; diff --git a/web/htdocs/details/edit.mc b/web/htdocs/details/edit.mc index de18268..4d926f6 100644 --- a/web/htdocs/details/edit.mc +++ b/web/htdocs/details/edit.mc @@ -1,4 +1,3 @@ -%# $Id: edit.mc,v 1.40 2009-04-15 18:16:46 wosch Exp $ <%args> $op $id => undef ### should be extracted using utf8param() @@ -37,10 +36,11 @@ my $conn = new ZOOM::Connection($db, 0, user => "admin", password => "fruitbat", elementSetName => "zeerex"); -my $protocol = utf8param($r, "protocol"); -my $host = utf8param($r, "host"); -my $port = utf8param($r, "port"); -my $dbname = utf8param($r, "dbname"); +my $protocol = utf8paramTrim($r, "protocol"); +my $host = utf8paramTrim($r, "host"); +my $port = utf8paramTrim($r, "port"); +my $dbname = utf8paramTrim($r, "dbname"); +my $title = utf8paramTrim($r, "title"); if ((!defined $port || $port eq "") && (defined $protocol && $protocol ne "")) { @@ -54,6 +54,7 @@ my $newid; if (defined $protocol && $protocol ne "" && defined $host && $host ne "" && defined $port && $port ne "" && + defined $title && $title ne "" && defined $dbname && $dbname ne "") { $newid = irspy_make_identifier($protocol, $host, $port, $dbname); } @@ -67,7 +68,7 @@ if (!defined $id) { } elsif (!defined $newid) { # Tried to create new record but data is insufficient print qq[

- Please specify protocol, host, port and database name.

\n]; + Please specify title, protocol, host, port and database name.

\n]; undef $update; } elsif ($host !~ /^\w+\.[\w.]*\w$/i) { print qq[

@@ -336,6 +337,8 @@ my @fields = qw(e:title e:description) ], [ subjects => 2, "Subjects", "e:databaseInfo/e:subjects", qw(e:title e:description) ], + [ disabled => [ qw(0 1) ], + "Target Test Disabled", "i:status/i:disabled" ], ); # Update record with submitted data @@ -343,7 +346,7 @@ my %fieldsByKey = map { ( $_->[0], $_) } @fields; my %data; foreach my $key (&utf8param($r)) { next if grep { $key eq $_ } qw(op id update); - $data{$key} = utf8param($r, $key); + $data{$key} = trimField( utf8param($r, $key) ); } my @changedFields = modify_xml_document($xc, \%fieldsByKey, \%data); if ($update && @changedFields) { diff --git a/web/htdocs/details/error.mc b/web/htdocs/details/error.mc index 81a73e4..085b153 100644 --- a/web/htdocs/details/error.mc +++ b/web/htdocs/details/error.mc @@ -1,4 +1,3 @@ -%# $Id: error.mc,v 1.1 2006-09-19 11:11:44 mike Exp $ <%args> $message diff --git a/web/htdocs/details/find.mc b/web/htdocs/details/find.mc index 9edbe8d..dff561f 100644 --- a/web/htdocs/details/find.mc +++ b/web/htdocs/details/find.mc @@ -1,4 +1,3 @@ -%# $Id: find.mc,v 1.12 2009-04-15 18:16:46 wosch Exp $ % if (&utf8param($r,"_search")) { % $m->comp("found.mc"); % } else { diff --git a/web/htdocs/details/found.mc b/web/htdocs/details/found.mc index 307239d..e4312fb 100644 --- a/web/htdocs/details/found.mc +++ b/web/htdocs/details/found.mc @@ -1,4 +1,3 @@ -%# $Id: found.mc,v 1.33 2009-04-15 18:16:46 wosch Exp $ <%once> sub print_navlink { my($params, $cond, $caption, $skip) = @_; diff --git a/web/htdocs/details/full.mc b/web/htdocs/details/full.mc index baec260..19008a9 100644 --- a/web/htdocs/details/full.mc +++ b/web/htdocs/details/full.mc @@ -1,4 +1,3 @@ -%# $Id: full.mc,v 1.30 2007-07-03 13:10:50 mike Exp $ <%args> $id diff --git a/web/htdocs/details/home.mc b/web/htdocs/details/home.mc index fcebe6c..1f42fb2 100644 --- a/web/htdocs/details/home.mc +++ b/web/htdocs/details/home.mc @@ -1,4 +1,3 @@ -%# $Id: home.mc,v 1.11 2008-07-03 08:14:21 mike Exp $

The Z39.50 Target Directory

diff --git a/web/htdocs/details/libtype-list.mc b/web/htdocs/details/libtype-list.mc index 8a40f57..1126b91 100644 --- a/web/htdocs/details/libtype-list.mc +++ b/web/htdocs/details/libtype-list.mc @@ -1,4 +1,3 @@ -%# $Id: libtype-list.mc,v 1.2 2007-03-30 11:14:56 mike Exp $ <%perl> return [ "", qw(Academic Public Corporate Special National Education Other) diff --git a/web/htdocs/details/software.mc b/web/htdocs/details/software.mc index 3039074..3fa573e 100644 --- a/web/htdocs/details/software.mc +++ b/web/htdocs/details/software.mc @@ -1,4 +1,3 @@ -%# $Id: software.mc,v 1.3 2007-04-18 12:13:01 mike Exp $

Index Data uses diff --git a/web/htdocs/details/stats.mc b/web/htdocs/details/stats.mc index f44c349..e50f56a 100644 --- a/web/htdocs/details/stats.mc +++ b/web/htdocs/details/stats.mc @@ -1,4 +1,3 @@ -%# $Id: stats.mc,v 1.9 2007-11-02 12:49:28 mike Exp $ <%doc> Here are the headings in the Z-Spy version: The ten most commonly supported Bib-1 Use attributes @@ -41,12 +40,12 @@ if (!defined $stats || $reload) {

Recalculating stats

% } <& table, stats => $stats, data => "bib1AccessPoints", - title => "The ten most commonly supported Bib-1 Use attributes", - headings => [ "Attribute", "Name"], + title => "The twenty most commonly supported Bib-1 Use attributes", + headings => [ "Attribute", "Name"], maxrows => 20, col3 => sub { bib1_access_point(@_) } &> <& table, stats => $stats, data => "recordSyntaxes", title => "Record syntax support by database", - headings => [ "Record Syntax"] &> + headings => [ "Record Syntax"], maxrows => 30 &> <& table, stats => $stats, data => "explain", title => "Explain Support", headings => [ "Explain Category"] &> @@ -84,7 +83,7 @@ $col3 => undef my $hr; $hr = $stats->{$data}; my @sorted = sort { $hr->{$b} <=> $hr->{$a} || $a <=> $b } keys %$hr; -my $n = @sorted; $n = $maxrows if @sorted > 10; +my $n = @sorted; $n = $maxrows if @sorted > 10 && $n > $maxrows; foreach my $i (1..$n) { my $key = $sorted[$i-1]; diff --git a/web/htdocs/details/upload.mc b/web/htdocs/details/upload.mc index 4a639ad..8324d3f 100644 --- a/web/htdocs/details/upload.mc +++ b/web/htdocs/details/upload.mc @@ -1,4 +1,3 @@ -%# $Id: upload.mc,v 1.3 2009-04-16 18:09:44 wosch Exp $ <%args> $filename => undef diff --git a/web/htdocs/doc.html b/web/htdocs/doc.html index 424d5db..d24a81d 100644 --- a/web/htdocs/doc.html +++ b/web/htdocs/doc.html @@ -1,3 +1,2 @@ -%# $Id: doc.html,v 1.1 2006-09-28 16:48:00 mike Exp $ <& /chrome/layout.mc, %ARGS, component => "/details/doc.mc", title => defined $ARGS{module} ? $ARGS{module} : "Documentation" &> diff --git a/web/htdocs/find.html b/web/htdocs/find.html index 49f32ad..4a341b7 100644 --- a/web/htdocs/find.html +++ b/web/htdocs/find.html @@ -1,3 +1,2 @@ -%# $Id: find.html,v 1.2 2006-09-19 16:30:25 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Find a target", component => "/details/find.mc" &> diff --git a/web/htdocs/full.html b/web/htdocs/full.html index 3924bf4..b6c806b 100644 --- a/web/htdocs/full.html +++ b/web/htdocs/full.html @@ -1,3 +1,2 @@ -%# $Id: full.html,v 1.1 2006-10-20 16:57:40 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Full target record", component => "/details/full.mc" &> diff --git a/web/htdocs/help.html b/web/htdocs/help.html index c211356..4c47c64 100644 --- a/web/htdocs/help.html +++ b/web/htdocs/help.html @@ -1,4 +1,3 @@ -%# $Id: help.html,v 1.5 2006-12-06 14:07:17 mike Exp $ <%args> $help diff --git a/web/htdocs/help/edit.html b/web/htdocs/help/edit.html index bd41d37..7466ccc 100644 --- a/web/htdocs/help/edit.html +++ b/web/htdocs/help/edit.html @@ -1,4 +1,3 @@ -%# $Id: edit.html,v 1.1 2006-12-05 12:26:37 mike Exp $

(The fields used to describe targets in the IRSpy database are for the most part those described in the diff --git a/web/htdocs/help/edit/author.html b/web/htdocs/help/edit/author.html index 33c9302..0de8703 100644 --- a/web/htdocs/help/edit/author.html +++ b/web/htdocs/help/edit/author.html @@ -1,4 +1,3 @@ -%# $Id: author.html,v 1.1 2006-12-06 14:22:22 mike Exp $

The author element should contain the name of the person or organisation to be credited with the creation of the database, and diff --git a/web/htdocs/help/edit/contact.html b/web/htdocs/help/edit/contact.html index 3786d4d..8d5fa5e 100644 --- a/web/htdocs/help/edit/contact.html +++ b/web/htdocs/help/edit/contact.html @@ -1,4 +1,3 @@ -%# $Id: contact.html,v 1.1 2006-12-06 14:22:22 mike Exp $

The contact element is used to record information on a contact person for the database. This should include at least a name and diff --git a/web/htdocs/help/edit/country.html b/web/htdocs/help/edit/country.html index 3db8b2e..9776395 100644 --- a/web/htdocs/help/edit/country.html +++ b/web/htdocs/help/edit/country.html @@ -1,4 +1,3 @@ -%# $Id: country.html,v 1.1 2006-12-06 14:22:22 mike Exp $

The country in which the service is hosted, if known.

diff --git a/web/htdocs/help/edit/dbname.html b/web/htdocs/help/edit/dbname.html index c2a30e7..15e9a76 100644 --- a/web/htdocs/help/edit/dbname.html +++ b/web/htdocs/help/edit/dbname.html @@ -1,4 +1,3 @@ -%# $Id: dbname.html,v 1.1 2006-12-05 12:26:37 mike Exp $

For Z39.50 databases, contains the name of the Z39.50 database which the service provides. If the service provides multiple diff --git a/web/htdocs/help/edit/description.html b/web/htdocs/help/edit/description.html index c5155b6..2e77d21 100644 --- a/web/htdocs/help/edit/description.html +++ b/web/htdocs/help/edit/description.html @@ -1,4 +1,3 @@ -%# $Id: description.html,v 1.1 2006-12-06 14:22:22 mike Exp $

Should contain a description of why this database might be of diff --git a/web/htdocs/help/edit/disabled.html b/web/htdocs/help/edit/disabled.html new file mode 100644 index 0000000..cabb1cc --- /dev/null +++ b/web/htdocs/help/edit/disabled.html @@ -0,0 +1,3 @@ +

The target will not be tested automatically if set and is not zero.

+ +

This option is mainly for the administrator of IRSpy in case of trouble.

diff --git a/web/htdocs/help/edit/extent.html b/web/htdocs/help/edit/extent.html index 8ab835c..7b6a9ef 100644 --- a/web/htdocs/help/edit/extent.html +++ b/web/htdocs/help/edit/extent.html @@ -1,4 +1,3 @@ -%# $Id: extent.html,v 1.1 2006-12-06 14:22:22 mike Exp $

Used to describe the completeness of the database, or the range of material that is included in it. For example a database which diff --git a/web/htdocs/help/edit/history.html b/web/htdocs/help/edit/history.html index 907f31c..899a898 100644 --- a/web/htdocs/help/edit/history.html +++ b/web/htdocs/help/edit/history.html @@ -1,4 +1,3 @@ -%# $Id: history.html,v 1.1 2006-12-06 14:22:22 mike Exp $

Any information which is considered useful regarding the history of the database may be recorded in the history element. This might diff --git a/web/htdocs/help/edit/host.html b/web/htdocs/help/edit/host.html index fcb41d7..f26128f 100644 --- a/web/htdocs/help/edit/host.html +++ b/web/htdocs/help/edit/host.html @@ -1,4 +1,3 @@ -%# $Id: host.html,v 1.2 2006-12-06 14:20:25 mike Exp $

The address of the server which hosts the service. This address should be in a name which will resolve to the correct IP diff --git a/web/htdocs/help/edit/hosturl.html b/web/htdocs/help/edit/hosturl.html index 20f43e5..582a9d6 100644 --- a/web/htdocs/help/edit/hosturl.html +++ b/web/htdocs/help/edit/hosturl.html @@ -1,4 +1,3 @@ -%# $Id: hosturl.html,v 1.1 2006-12-06 14:22:22 mike Exp $

A URL for the organisation which hosts the database. This URL may or not refer to the same host as the diff --git a/web/htdocs/help/edit/language.html b/web/htdocs/help/edit/language.html index df77e8c..88d9b67 100644 --- a/web/htdocs/help/edit/language.html +++ b/web/htdocs/help/edit/language.html @@ -1,4 +1,3 @@ -%# $Id: language.html,v 1.1 2006-12-06 14:22:22 mike Exp $

Used to record the languages used in the database records (as opposed to the target record). For example, in an English-language diff --git a/web/htdocs/help/edit/password.html b/web/htdocs/help/edit/password.html index e898117..80da6d6 100644 --- a/web/htdocs/help/edit/password.html +++ b/web/htdocs/help/edit/password.html @@ -1,2 +1 @@ -%# $Id: password.html,v 1.1 2006-12-06 14:22:22 mike Exp $ <& username.html &> diff --git a/web/htdocs/help/edit/port.html b/web/htdocs/help/edit/port.html index cce93de..631ec62 100644 --- a/web/htdocs/help/edit/port.html +++ b/web/htdocs/help/edit/port.html @@ -1,4 +1,3 @@ -%# $Id: port.html,v 1.1 2006-12-05 12:26:37 mike Exp $

The port that the service is running on. For Z39.50 servers, this is often 210, 3950 or 7090. IRSpy needs this information in order diff --git a/web/htdocs/help/edit/protocol.html b/web/htdocs/help/edit/protocol.html index 4d9fddc..2f493c2 100644 --- a/web/htdocs/help/edit/protocol.html +++ b/web/htdocs/help/edit/protocol.html @@ -1,4 +1,3 @@ -%# $Id: protocol.html,v 1.1 2006-12-05 12:26:37 mike Exp $

Used to record the protocol that should be used to connect to the server. The default value is diff --git a/web/htdocs/help/edit/restrictions.html b/web/htdocs/help/edit/restrictions.html index 2022536..b823e2a 100644 --- a/web/htdocs/help/edit/restrictions.html +++ b/web/htdocs/help/edit/restrictions.html @@ -1,4 +1,3 @@ -%# $Id: restrictions.html,v 1.1 2006-12-06 14:22:22 mike Exp $

If there are any restrictions on the usage or availability of the database or its contents then these should be recorded in this diff --git a/web/htdocs/help/edit/subjects.html b/web/htdocs/help/edit/subjects.html index 4af2fdd..ff95cfe 100644 --- a/web/htdocs/help/edit/subjects.html +++ b/web/htdocs/help/edit/subjects.html @@ -1,4 +1,3 @@ -%# $Id: subjects.html,v 1.1 2006-12-06 14:22:22 mike Exp $

If the database concerns particular subjects from a controlled vocabulary then these may be recorded using this field. These diff --git a/web/htdocs/help/edit/title.html b/web/htdocs/help/edit/title.html index 9b984e4..bc9920a 100644 --- a/web/htdocs/help/edit/title.html +++ b/web/htdocs/help/edit/title.html @@ -1,4 +1,3 @@ -%# $Id: title.html,v 1.1 2006-12-06 14:22:22 mike Exp $

The name by which the database is known.

diff --git a/web/htdocs/help/edit/type.html b/web/htdocs/help/edit/type.html index f5eb71f..2b21736 100644 --- a/web/htdocs/help/edit/type.html +++ b/web/htdocs/help/edit/type.html @@ -1,4 +1,3 @@ -%# $Id: type.html,v 1.1 2006-12-05 16:40:46 mike Exp $

The type of library that this is a catlogue for. This field is not part of the ZeeRex data model, and may require further thought - diff --git a/web/htdocs/help/edit/username.html b/web/htdocs/help/edit/username.html index 003a8d5..302af3e 100644 --- a/web/htdocs/help/edit/username.html +++ b/web/htdocs/help/edit/username.html @@ -1,4 +1,3 @@ -%# $Id: username.html,v 1.1 2006-12-06 14:22:22 mike Exp $

Some Z39.50 servers require authentication before access is allowed. For such servers, a suitable username and password may be diff --git a/web/htdocs/help/info.html b/web/htdocs/help/info.html index 3f40365..3d8711a 100644 --- a/web/htdocs/help/info.html +++ b/web/htdocs/help/info.html @@ -1,2 +1 @@ -%# $Id: info.html,v 1.1 2007-04-26 13:56:46 mike Exp $ diff --git a/web/htdocs/help/info/reliability.html b/web/htdocs/help/info/reliability.html index 6f768e4..e7ffdc6 100644 --- a/web/htdocs/help/info/reliability.html +++ b/web/htdocs/help/info/reliability.html @@ -1,4 +1,3 @@ -%# $Id: reliability.html,v 1.1 2007-04-26 13:56:53 mike Exp $

Measures the reliability of the target only in its ability to respond to connections: the display indicates the number of diff --git a/web/htdocs/help/link.mc b/web/htdocs/help/link.mc index cca692c..a3df767 100644 --- a/web/htdocs/help/link.mc +++ b/web/htdocs/help/link.mc @@ -1,4 +1,3 @@ -%# $Id: link.mc,v 1.9 2007-06-25 10:38:35 mike Exp $ <%args> $help diff --git a/web/htdocs/index.html b/web/htdocs/index.html index 716d916..fed9825 100644 --- a/web/htdocs/index.html +++ b/web/htdocs/index.html @@ -1,3 +1,2 @@ -%# $Id: index.html,v 1.3 2006-09-15 16:47:47 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Welcome", component => "/details/home.mc" &> diff --git a/web/htdocs/offtopic/sru-auth.html b/web/htdocs/offtopic/sru-auth.html index ed911c2..b53f4d8 100644 --- a/web/htdocs/offtopic/sru-auth.html +++ b/web/htdocs/offtopic/sru-auth.html @@ -1,4 +1,3 @@ -%# $Id: sru-auth.html,v 1.1 2007-08-21 09:48:41 mike Exp $ <%doc> In order to test the authentication feature of Simple2ZOOM, we need an authenticator script, accessible via HTTP. The diff --git a/web/htdocs/raw.html b/web/htdocs/raw.html index 694652b..fc8bec2 100644 --- a/web/htdocs/raw.html +++ b/web/htdocs/raw.html @@ -1,4 +1,3 @@ -%# $Id: raw.html,v 1.9 2007-05-11 13:53:40 mike Exp $ <%args> $id diff --git a/web/htdocs/robots-test.txt b/web/htdocs/robots-test.txt new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/web/htdocs/robots-test.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/web/htdocs/robots.txt b/web/htdocs/robots.txt index beef7ee..fed2b3a 100644 --- a/web/htdocs/robots.txt +++ b/web/htdocs/robots.txt @@ -1,8 +1,12 @@ User-agent: * -Disallow: /full.html -Disallow: /all.html +Disallow: /admin/ Disallow: /add.html -Disallow: /upload.html +Disallow: /add_target.html +Disallow: /all.html +Disallow: /doc.html +Disallow: /find.html +Disallow: /full.html Disallow: /raw.html -Disallow: /admin/ +Disallow: /stats.html +Disallow: /upload.html diff --git a/web/htdocs/software.html b/web/htdocs/software.html index 000e048..d32cb78 100644 --- a/web/htdocs/software.html +++ b/web/htdocs/software.html @@ -1,3 +1,2 @@ -%# $Id: software.html,v 1.1 2007-03-30 11:43:59 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "About the IRSpy software", component => "/details/software.mc" &> diff --git a/web/htdocs/stats.html b/web/htdocs/stats.html index bea5cd7..6c0edf6 100644 --- a/web/htdocs/stats.html +++ b/web/htdocs/stats.html @@ -1,3 +1,2 @@ -%# $Id: stats.html,v 1.1 2006-12-14 16:11:03 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Target statistic", component => "/details/stats.mc" &> diff --git a/web/htdocs/style.css b/web/htdocs/style.css index 429bee7..56e1fba 100644 --- a/web/htdocs/style.css +++ b/web/htdocs/style.css @@ -1,4 +1,3 @@ -/* $Id: style.css,v 1.4 2007-03-22 07:57:34 mike Exp $ */ body { color: black; background: white; diff --git a/web/htdocs/upload.html b/web/htdocs/upload.html index 3ba0688..ddfb2a2 100644 --- a/web/htdocs/upload.html +++ b/web/htdocs/upload.html @@ -1,4 +1,3 @@ -%# $Id: upload.html,v 1.1 2007-07-16 11:55:20 mike Exp $ <& /chrome/layout.mc, %ARGS, title => "Upload target record", component => "/details/upload.mc" &> diff --git a/web/images/Makefile b/web/images/Makefile index cecb2a4..bc709e0 100644 --- a/web/images/Makefile +++ b/web/images/Makefile @@ -1,5 +1,3 @@ -# $Id: Makefile,v 1.1 2006-12-05 12:26:37 mike Exp $ - PNG = ../htdocs/help-20px.png ../htdocs/help-16px.png all: $(PNG) diff --git a/xsl/irspy2zeerex.xsl b/xsl/irspy2zeerex.xsl index 76ad5a1..3f22386 100644 --- a/xsl/irspy2zeerex.xsl +++ b/xsl/irspy2zeerex.xsl @@ -1,6 +1,5 @@ diff --git a/zebra/Makefile b/zebra/Makefile index b56e633..710456f 100644 --- a/zebra/Makefile +++ b/zebra/Makefile @@ -5,6 +5,7 @@ DUMP_DIR= irspy-dump ZEBRA_DIR= db ZEBRAIDX= zebraidx-2.0 ZEBRAIDX_TEST= zebraidx-2.0 -c zebra-test.cfg +IRSPY_DATABASE= $${IRSpyDbName-"localhost:8018/IR-Explain---1"} # debugging ZEBRA_TEST_DIR= db-test @@ -41,7 +42,7 @@ dump: rm -rf records-${DATE}.old -test -e records-${DATE} && mv records-${DATE} records-${DATE}.old mkdir records-${DATE} - cd records-${DATE} && ../../bin/irspy-dump.pl localhost:8018/IR-Explain---1 + cd records-${DATE} && ../../bin/irspy-dump.pl ${IRSPY_DATABASE} cd records-${DATE} && ( for i in *.xml; do tidy -xml -i -m -w 140 $$i; done ) > /dev/null 2>&1 tar cf - records-${DATE} | gzip > records-${DATE}.tar.gz diff --git a/zebra/README b/zebra/README index 2ac6c19..af38aa6 100644 --- a/zebra/README +++ b/zebra/README @@ -1,4 +1,3 @@ -$Id: README,v 1.21 2007-05-09 16:48:31 mike Exp $ What's what in this directory: @@ -111,3 +110,4 @@ zebraidx-2.0 update records-2010-04-06 zebraidx-2.0 commit zebrasrv-2.0 -f yazserver.xml + diff --git a/zebra/crontab b/zebra/crontab index 356ecb5..d53a276 100644 --- a/zebra/crontab +++ b/zebra/crontab @@ -1,9 +1,5 @@ -# $Id: crontab,v 1.4 2007-09-18 17:02:19 mike Exp $ +# Example crontab file for a weekly update cycle, testing 1/7 of the +# database every day. See also the script ../bin/irspy-update.sh +# # m h dom mon dow command -0 1 * * 0 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,0 localhost:8018/IR-Explain---1 -0 1 * * 1 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,1 localhost:8018/IR-Explain---1 -0 1 * * 2 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,2 localhost:8018/IR-Explain---1 -0 1 * * 3 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,3 localhost:8018/IR-Explain---1 -0 1 * * 4 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,4 localhost:8018/IR-Explain---1 -0 1 * * 5 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,5 localhost:8018/IR-Explain---1 -0 1 * * 6 cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,6 localhost:8018/IR-Explain---1 +0 1 * * * cd /home/mike/cvs/irspy/bin && YAZ_LOG=irspy ./setrlimit -v -a 500 -- perl -I ../lib irspy.pl -a -n 50 -t Main -m 7,`date '+%w'` localhost:8018/IR-Explain---1 diff --git a/zebra/dom-filter/zebra.cfg b/zebra/dom-filter/zebra.cfg index 76c3946..c06a152 100644 --- a/zebra/dom-filter/zebra.cfg +++ b/zebra/dom-filter/zebra.cfg @@ -1,4 +1,3 @@ -# $Id: zebra.cfg,v 1.12 2006-12-19 12:53:16 sondberg Exp $ # Where to look for config files profilePath: .:/usr/local/share/idzebra-2.0/tab:/usr/share/idzebra-2.0/tab diff --git a/zebra/ezeerex2pqfproperties.pl b/zebra/ezeerex2pqfproperties.pl index 2201b83..bd71e26 100755 --- a/zebra/ezeerex2pqfproperties.pl +++ b/zebra/ezeerex2pqfproperties.pl @@ -1,6 +1,5 @@ #! /usr/bin/perl -w -# $Id: ezeerex2pqfproperties.pl,v 1.7 2006-06-20 11:24:22 mike Exp $ # # Run like this: # ./ezeerex2pqfproperties.pl zeerex.xml diff --git a/zebra/filterconf.xml b/zebra/filterconf.xml index 08b28d5..543a296 100644 --- a/zebra/filterconf.xml +++ b/zebra/filterconf.xml @@ -1,5 +1,4 @@ - diff --git a/zebra/form.html b/zebra/form.html index d782e40..eed61a0 100644 --- a/zebra/form.html +++ b/zebra/form.html @@ -1,5 +1,4 @@ - diff --git a/zebra/pqf.properties b/zebra/pqf.properties index 225b4fa..34c83d2 100644 --- a/zebra/pqf.properties +++ b/zebra/pqf.properties @@ -1,4 +1,3 @@ -# $Id: pqf.properties,v 1.18 2007-03-29 17:14:13 mike Exp $ # # Properties file to drive the YAZ CQL-to-PQF converter for Zebra. # This specifies the interpretation of various CQL indexes, relations, @@ -50,6 +49,7 @@ index.zeerex.default_index = 1=zeerex:default_index # Extensions index.zeerex.libType = 1=zeerex:libType 4=3 index.zeerex.country = 1=zeerex:country 4=3 +index.zeerex.disabled = 1=zeerex:disabled 4=3 # Relation attributes are selected according to the CQL relation by # looking up the "relation." property: diff --git a/zebra/profile b/zebra/profile index 67500c1..a1bcd53 100644 --- a/zebra/profile +++ b/zebra/profile @@ -1,4 +1,3 @@ -$Id: profile,v 1.2 2006-06-16 14:16:12 mike Exp $ Notes on the indexes in the ZeeRex profile, with indications of whether they are yet supported by the Zebra configuration in this diff --git a/zebra/yazserver-test.xml b/zebra/yazserver-test.xml index 92b64cf..8758565 100644 --- a/zebra/yazserver-test.xml +++ b/zebra/yazserver-test.xml @@ -1,5 +1,4 @@ - tcp:@:8019 diff --git a/zebra/yazserver.xml b/zebra/yazserver.xml index 9f4cb1c..f5db2bb 100644 --- a/zebra/yazserver.xml +++ b/zebra/yazserver.xml @@ -1,5 +1,4 @@ - tcp:@:8018 diff --git a/zebra/zeerex-test.xml b/zebra/zeerex-test.xml index aba40eb..800a304 100644 --- a/zebra/zeerex-test.xml +++ b/zebra/zeerex-test.xml @@ -1,5 +1,4 @@ - diff --git a/zebra/zeerex.xml b/zebra/zeerex.xml index abd8de0..c7c0d08 100644 --- a/zebra/zeerex.xml +++ b/zebra/zeerex.xml @@ -1,5 +1,4 @@ - diff --git a/zebra/zeerex2dc.xsl b/zebra/zeerex2dc.xsl index c57aab9..0689a4d 100644 --- a/zebra/zeerex2dc.xsl +++ b/zebra/zeerex2dc.xsl @@ -1,5 +1,4 @@ - - diff --git a/zebra/zeerex2index.xsl b/zebra/zeerex2index.xsl index 7c4f5eb..fd7556f 100644 --- a/zebra/zeerex2index.xsl +++ b/zebra/zeerex2index.xsl @@ -1,5 +1,4 @@ - - + @@ -151,6 +150,9 @@ + + + diff --git a/zebra/zeerex2noauth.xsl b/zebra/zeerex2noauth.xsl index 632cf8d..92c430e 100644 --- a/zebra/zeerex2noauth.xsl +++ b/zebra/zeerex2noauth.xsl @@ -1,7 +1,4 @@ - -