1 /* $Id: client.c,v 1.13 2007-07-03 11:21:48 adam Exp $
2 Copyright (c) 2006-2007, Index Data.
4 This file is part of Pazpar2.
6 Pazpar2 is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 2, or (at your option) any later
11 Pazpar2 is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with Pazpar2; see the file LICENSE. If not, write to the
18 Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA
31 #include <sys/socket.h>
37 #include <yaz/marcdisp.h>
38 #include <yaz/comstack.h>
39 #include <yaz/tcpip.h>
40 #include <yaz/proto.h>
41 #include <yaz/readconf.h>
42 #include <yaz/pquery.h>
43 #include <yaz/otherinfo.h>
44 #include <yaz/yaz-util.h>
46 #include <yaz/query-charset.h>
47 #include <yaz/querytowrbuf.h>
48 #include <yaz/oid_db.h>
49 #include <yaz/diagbib1.h>
57 #include <yaz/timing.h>
60 #include <netinet/in.h>
65 #include "connection.h"
68 /** \brief Represents client state for a connection to one search target */
70 struct session_database *database;
71 struct connection *connection;
72 struct session *session;
73 char *pquery; // Current search
77 int requestid; // ID of current outstanding request
79 enum client_state state;
80 struct show_raw *show_raw;
81 struct client *next; // next client in session or next in free list
85 int active; // whether this request has been sent to the server
89 void (*error_handler)(void *data, const char *addinfo);
90 void (*record_handler)(void *data, const char *buf, size_t sz);
94 static const char *client_states[] = {
98 "Client_Initializing",
103 "Client_Disconnected",
107 static struct client *client_freelist = 0;
109 static int send_apdu(struct client *c, Z_APDU *a)
111 struct session_database *sdb = client_get_database(c);
112 const char *apdulog = session_setting_oneval(sdb, PZ_APDULOG);
113 if (apdulog && *apdulog && *apdulog != '0')
115 ODR p = odr_createmem(ODR_PRINT);
116 yaz_log(YLOG_LOG, "send APDU %s", client_get_url(c));
118 odr_setprint(p, yaz_log_file());
120 odr_setprint(p, stderr);
123 return connection_send_apdu(client_get_connection(c), a);
127 const char *client_get_state_str(struct client *cl)
129 return client_states[cl->state];
132 enum client_state client_get_state(struct client *cl)
137 void client_set_state(struct client *cl, enum client_state st)
142 static void client_show_raw_error(struct client *cl, const char *addinfo);
144 // Close connection and set state to error
145 void client_fatal(struct client *cl)
147 client_show_raw_error(cl, "client connection failure");
148 yaz_log(YLOG_WARN, "Fatal error from %s", client_get_url(cl));
149 connection_destroy(cl->connection);
150 cl->state = Client_Error;
154 static int diag_to_wrbuf(Z_DiagRec **pp, int num, WRBUF w)
158 for (i = 0; i<num; i++)
160 Z_DiagRec *p = pp[i];
163 if (p->which != Z_DiagRec_defaultFormat)
165 wrbuf_puts(w, "? Not in default format");
169 Z_DefaultDiagFormat *r = p->u.defaultFormat;
171 if (!r->diagnosticSetId)
172 wrbuf_puts(w, "? Missing diagset");
176 char diag_name_buf[OID_STR_MAX];
177 const char *diag_name = 0;
178 diag_name = yaz_oid_to_string_buf
179 (r->diagnosticSetId, &oclass, diag_name_buf);
180 wrbuf_puts(w, diag_name);
183 code = *r->condition;
184 wrbuf_printf(w, " %d %s", *r->condition,
185 diagbib1_str(*r->condition));
188 case Z_DefaultDiagFormat_v2Addinfo:
189 wrbuf_printf(w, " -- v2 addinfo '%s'", r->u.v2Addinfo);
191 case Z_DefaultDiagFormat_v3Addinfo:
192 wrbuf_printf(w, " -- v3 addinfo '%s'", r->u.v3Addinfo);
202 struct connection *client_get_connection(struct client *cl)
204 return cl->connection;
207 struct session_database *client_get_database(struct client *cl)
212 struct session *client_get_session(struct client *cl)
217 const char *client_get_pquery(struct client *cl)
222 void client_set_requestid(struct client *cl, int id)
227 int client_show_raw_begin(struct client *cl, int position,
228 const char *syntax, const char *esn,
230 void (*error_handler)(void *data, const char *addinfo),
231 void (*record_handler)(void *data, const char *buf,
236 cl->show_raw = xmalloc(sizeof(*cl->show_raw));
237 cl->show_raw->position = position;
238 cl->show_raw->active = 0;
239 cl->show_raw->data = data;
240 cl->show_raw->error_handler = error_handler;
241 cl->show_raw->record_handler = record_handler;
243 cl->show_raw->syntax = xstrdup(syntax);
245 cl->show_raw->syntax = 0;
247 cl->show_raw->esn = xstrdup(esn);
249 cl->show_raw->esn = 0;
252 if (cl->state == Client_Failed || cl->state == Client_Disconnected)
254 client_show_raw_error(cl, "not connected");
263 void client_show_raw_reset(struct client *cl)
269 static void client_show_raw_error(struct client *cl, const char *addinfo)
273 cl->show_raw->error_handler(cl->show_raw->data, addinfo);
274 client_show_raw_reset(cl);
278 static void client_show_raw_cancel(struct client *cl)
282 cl->show_raw->error_handler(cl->show_raw->data, "cancel");
283 client_show_raw_reset(cl);
287 void client_send_raw_present(struct client *cl)
289 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
291 int start = cl->show_raw->position;
293 assert(cl->show_raw);
295 yaz_log(YLOG_DEBUG, "%s: trying to present %d record(s) from %d",
296 client_get_url(cl), toget, start);
298 a->u.presentRequest->resultSetStartPoint = &start;
299 a->u.presentRequest->numberOfRecordsRequested = &toget;
301 if (cl->show_raw->syntax) // syntax is optional
302 a->u.presentRequest->preferredRecordSyntax =
303 yaz_string_to_oid_odr(yaz_oid_std(),
304 CLASS_RECSYN, cl->show_raw->syntax,
305 global_parameters.odr_out);
306 if (cl->show_raw->esn) // element set is optional
308 Z_ElementSetNames *elementSetNames =
309 odr_malloc(global_parameters.odr_out, sizeof(*elementSetNames));
310 Z_RecordComposition *compo =
311 odr_malloc(global_parameters.odr_out, sizeof(*compo));
312 a->u.presentRequest->recordComposition = compo;
314 compo->which = Z_RecordComp_simple;
315 compo->u.simple = elementSetNames;
317 elementSetNames->which = Z_ElementSetNames_generic;
318 elementSetNames->u.generic =
319 odr_strdup(global_parameters.odr_out, cl->show_raw->esn);
321 if (send_apdu(cl, a) >= 0)
323 cl->show_raw->active = 1;
324 cl->state = Client_Presenting;
328 client_show_raw_error(cl, "send_apdu failed");
329 cl->state = Client_Error;
331 odr_reset(global_parameters.odr_out);
334 void client_send_present(struct client *cl)
336 struct session_database *sdb = client_get_database(cl);
337 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_presentRequest);
339 int start = cl->records + 1;
342 toget = global_parameters.chunk;
343 if (toget > global_parameters.toget - cl->records)
344 toget = global_parameters.toget - cl->records;
345 if (toget > cl->hits - cl->records)
346 toget = cl->hits - cl->records;
348 yaz_log(YLOG_DEBUG, "Trying to present %d record(s) from %d",
351 a->u.presentRequest->resultSetStartPoint = &start;
352 a->u.presentRequest->numberOfRecordsRequested = &toget;
354 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
356 a->u.presentRequest->preferredRecordSyntax =
357 yaz_string_to_oid_odr(yaz_oid_std(),
358 CLASS_RECSYN, recsyn,
359 global_parameters.odr_out);
362 if (send_apdu(cl, a) >= 0)
363 cl->state = Client_Presenting;
365 cl->state = Client_Error;
366 odr_reset(global_parameters.odr_out);
370 void client_send_search(struct client *cl)
372 struct session *se = client_get_session(cl);
373 struct session_database *sdb = client_get_database(cl);
374 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_searchRequest);
378 int ssub = 0, lslb = 100000, mspn = 10;
382 yaz_iconv_t iconv = 0;
384 yaz_log(YLOG_DEBUG, "Sending search to %s", sdb->database->url);
387 // constructing RPN query
388 a->u.searchRequest->query = zquery = odr_malloc(global_parameters.odr_out,
390 zquery->which = Z_Query_type_1;
391 zquery->u.type_1 = p_query_rpn(global_parameters.odr_out,
392 client_get_pquery(cl));
394 // converting to target encoding
395 if ((queryenc = session_setting_oneval(sdb, PZ_QUERYENCODING))){
396 iconv = yaz_iconv_open(queryenc, "UTF-8");
398 yaz_query_charset_convert_rpnquery(zquery->u.type_1,
399 global_parameters.odr_out,
401 yaz_iconv_close(iconv);
403 yaz_log(YLOG_WARN, "Query encoding failed %s %s",
404 client_get_database(cl)->database->url, queryenc);
407 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
409 databaselist = odr_malloc(global_parameters.odr_out, sizeof(char*) * ndb);
410 for (ndb = 0; sdb->database->databases[ndb]; ndb++)
411 databaselist[ndb] = sdb->database->databases[ndb];
413 if (!(piggyback = session_setting_oneval(sdb, PZ_PIGGYBACK))
414 || *piggyback == '1')
416 if ((recsyn = session_setting_oneval(sdb, PZ_REQUESTSYNTAX)))
418 a->u.searchRequest->preferredRecordSyntax =
419 yaz_string_to_oid_odr(yaz_oid_std(),
420 CLASS_RECSYN, recsyn,
421 global_parameters.odr_out);
423 a->u.searchRequest->smallSetUpperBound = &ssub;
424 a->u.searchRequest->largeSetLowerBound = &lslb;
425 a->u.searchRequest->mediumSetPresentNumber = &mspn;
427 a->u.searchRequest->databaseNames = databaselist;
428 a->u.searchRequest->num_databaseNames = ndb;
431 { //scope for sending and logging queries
432 WRBUF wbquery = wrbuf_alloc();
433 yaz_query_to_wrbuf(wbquery, a->u.searchRequest->query);
436 if (send_apdu(cl, a) >= 0)
438 client_set_state(cl, Client_Searching);
439 client_set_requestid(cl, se->requestid);
440 yaz_log(YLOG_LOG, "SearchRequest %s %s %s",
441 client_get_database(cl)->database->url,
442 queryenc ? queryenc : "UTF-8",
443 wrbuf_cstr(wbquery));
446 client_set_state(cl, Client_Error);
447 yaz_log(YLOG_WARN, "Failed SearchRequest %s %s %s",
448 client_get_database(cl)->database->url,
449 queryenc ? queryenc : "UTF-8",
450 wrbuf_cstr(wbquery));
453 wrbuf_destroy(wbquery);
456 odr_reset(global_parameters.odr_out);
459 void client_init_response(struct client *cl, Z_APDU *a)
461 Z_InitResponse *r = a->u.initResponse;
463 yaz_log(YLOG_DEBUG, "Init response %s", cl->database->database->url);
467 cl->state = Client_Idle;
470 cl->state = Client_Failed; // FIXME need to do something to the connection
474 static void ingest_raw_records(struct client *cl, Z_Records *r)
476 Z_NamePlusRecordList *rlist;
477 Z_NamePlusRecord *npr;
481 if (r->which != Z_Records_DBOSD)
483 client_show_raw_error(cl, "non-surrogate diagnostics");
487 rlist = r->u.databaseOrSurDiagnostics;
488 if (rlist->num_records != 1 || !rlist->records || !rlist->records[0])
490 client_show_raw_error(cl, "no records");
493 npr = rlist->records[0];
494 if (npr->which != Z_NamePlusRecord_databaseRecord)
496 client_show_raw_error(cl, "surrogate diagnostic");
500 doc = record_to_xml(client_get_database(cl), npr->u.databaseRecord);
503 client_show_raw_error(cl, "unable to convert record to xml");
507 xmlDocDumpMemory(doc, &buf_out, &len_out);
510 cl->show_raw->record_handler(cl->show_raw->data,
511 (const char *) buf_out, len_out);
518 static void ingest_records(struct client *cl, Z_Records *r)
521 yaz_timing_t t = yaz_timing_create();
524 struct session *s = client_get_session(cl);
525 Z_NamePlusRecordList *rlist;
528 if (r->which != Z_Records_DBOSD)
530 rlist = r->u.databaseOrSurDiagnostics;
531 for (i = 0; i < rlist->num_records; i++)
533 Z_NamePlusRecord *npr = rlist->records[i];
536 if (npr->which != Z_NamePlusRecord_databaseRecord)
539 "Unexpected record type, probably diagnostic %s",
540 cl->database->database->url);
544 rec = ingest_record(cl, npr->u.databaseRecord, cl->records);
548 if (rlist->num_records)
549 session_alert_watch(s, SESSION_WATCH_RECORDS);
553 yaz_log(YLOG_LOG, "ingest_records %6.5f %3.2f %3.2f",
554 yaz_timing_get_real(t), yaz_timing_get_user(t),
555 yaz_timing_get_sys(t));
556 yaz_timing_destroy(&t);
561 void client_search_response(struct client *cl, Z_APDU *a)
563 struct session *se = cl->session;
564 Z_SearchResponse *r = a->u.searchResponse;
566 yaz_log(YLOG_DEBUG, "Search response %s (status=%d)",
567 cl->database->database->url, *r->searchStatus);
569 if (*r->searchStatus)
571 cl->hits = *r->resultCount;
572 se->total_hits += cl->hits;
573 if (r->presentStatus && !*r->presentStatus && r->records)
575 yaz_log(YLOG_DEBUG, "Records in search response %s",
576 cl->database->database->url);
577 ingest_records(cl, r->records);
579 cl->state = Client_Idle;
583 Z_Records *recs = r->records;
585 cl->state = Client_Error;
586 if (recs && recs->which == Z_Records_NSD)
588 WRBUF w = wrbuf_alloc();
590 Z_DiagRec dr, *dr_p = &dr;
591 dr.which = Z_DiagRec_defaultFormat;
592 dr.u.defaultFormat = recs->u.nonSurrogateDiagnostic;
594 wrbuf_printf(w, "Search response NSD %s: ",
595 cl->database->database->url);
597 cl->diagnostic = diag_to_wrbuf(&dr_p, 1, w);
599 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
601 cl->state = Client_Error;
604 else if (recs && recs->which == Z_Records_multipleNSD)
606 WRBUF w = wrbuf_alloc();
608 wrbuf_printf(w, "Search response multipleNSD %s: ",
609 cl->database->database->url);
611 diag_to_wrbuf(recs->u.multipleNonSurDiagnostics->diagRecs,
612 recs->u.multipleNonSurDiagnostics->num_diagRecs,
614 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
615 cl->state = Client_Error;
621 void client_present_response(struct client *cl, Z_APDU *a)
623 Z_PresentResponse *r = a->u.presentResponse;
624 Z_Records *recs = r->records;
626 if (recs && recs->which == Z_Records_NSD)
628 WRBUF w = wrbuf_alloc();
630 Z_DiagRec dr, *dr_p = &dr;
631 dr.which = Z_DiagRec_defaultFormat;
632 dr.u.defaultFormat = recs->u.nonSurrogateDiagnostic;
634 wrbuf_printf(w, "Present response NSD %s: ",
635 cl->database->database->url);
637 cl->diagnostic = diag_to_wrbuf(&dr_p, 1, w);
639 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
641 cl->state = Client_Error;
644 client_show_raw_error(cl, "non surrogate diagnostics");
646 else if (recs && recs->which == Z_Records_multipleNSD)
648 WRBUF w = wrbuf_alloc();
650 wrbuf_printf(w, "Present response multipleNSD %s: ",
651 cl->database->database->url);
653 diag_to_wrbuf(recs->u.multipleNonSurDiagnostics->diagRecs,
654 recs->u.multipleNonSurDiagnostics->num_diagRecs,
656 yaz_log(YLOG_WARN, "%s", wrbuf_cstr(w));
657 cl->state = Client_Error;
660 else if (recs && !*r->presentStatus && cl->state != Client_Error)
662 yaz_log(YLOG_DEBUG, "Good Present response %s",
663 cl->database->database->url);
665 // we can mix show raw and normal show ..
666 if (cl->show_raw && cl->show_raw->active)
668 cl->show_raw->active = 0; // no longer active
669 ingest_raw_records(cl, recs);
672 ingest_records(cl, recs);
673 cl->state = Client_Idle;
675 else if (*r->presentStatus)
677 yaz_log(YLOG_WARN, "Bad Present response %s",
678 cl->database->database->url);
679 cl->state = Client_Error;
680 client_show_raw_error(cl, "bad present response");
684 void client_close_response(struct client *cl, Z_APDU *a)
686 struct connection *co = cl->connection;
687 /* Z_Close *r = a->u.close; */
689 yaz_log(YLOG_WARN, "Close response %s", cl->database->database->url);
691 cl->state = Client_Failed;
692 connection_destroy(co);
695 int client_is_our_response(struct client *cl)
697 struct session *se = client_get_session(cl);
699 if (cl && (cl->requestid == se->requestid ||
700 cl->state == Client_Initializing))
705 // Set authentication token in init if one is set for the client
706 // TODO: Extend this to handle other schemes than open (should be simple)
707 static void init_authentication(struct client *cl, Z_InitRequest *req)
709 struct session_database *sdb = client_get_database(cl);
710 char *auth = session_setting_oneval(sdb, PZ_AUTHENTICATION);
714 struct connection *co = client_get_connection(cl);
715 struct session *se = client_get_session(cl);
716 Z_IdAuthentication *idAuth = odr_malloc(global_parameters.odr_out,
718 idAuth->which = Z_IdAuthentication_open;
719 idAuth->u.open = auth;
720 req->idAuthentication = idAuth;
721 connection_set_authentication(co, nmem_strdup(se->session_nmem, auth));
725 static void init_zproxy(struct client *cl, Z_InitRequest *req)
727 struct session_database *sdb = client_get_database(cl);
728 char *ztarget = sdb->database->url;
729 //char *ztarget = sdb->url;
730 char *zproxy = session_setting_oneval(sdb, PZ_ZPROXY);
733 yaz_oi_set_string_oid(&req->otherInfo,
734 global_parameters.odr_out,
735 yaz_oid_userinfo_proxy,
740 static void client_init_request(struct client *cl)
742 Z_APDU *a = zget_APDU(global_parameters.odr_out, Z_APDU_initRequest);
744 a->u.initRequest->implementationId = global_parameters.implementationId;
745 a->u.initRequest->implementationName = global_parameters.implementationName;
746 a->u.initRequest->implementationVersion =
747 global_parameters.implementationVersion;
748 ODR_MASK_SET(a->u.initRequest->options, Z_Options_search);
749 ODR_MASK_SET(a->u.initRequest->options, Z_Options_present);
750 ODR_MASK_SET(a->u.initRequest->options, Z_Options_namedResultSets);
752 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_1);
753 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_2);
754 ODR_MASK_SET(a->u.initRequest->protocolVersion, Z_ProtocolVersion_3);
756 init_authentication(cl, a->u.initRequest);
757 init_zproxy(cl, a->u.initRequest);
759 if (send_apdu(cl, a) >= 0)
760 client_set_state(cl, Client_Initializing);
762 client_set_state(cl, Client_Error);
763 odr_reset(global_parameters.odr_out);
766 void client_continue(struct client *cl)
768 if (cl->state == Client_Connected) {
769 client_init_request(cl);
771 if (cl->state == Client_Idle)
773 struct session *se = client_get_session(cl);
774 if (cl->requestid != se->requestid && cl->pquery) {
775 // we'll have to abort this because result set is to be deleted
776 client_show_raw_cancel(cl);
777 client_send_search(cl);
779 else if (cl->show_raw)
781 client_send_raw_present(cl);
783 else if (cl->hits > 0 && cl->records < global_parameters.toget &&
784 cl->records < cl->hits) {
785 client_send_present(cl);
790 struct client *client_create(void)
796 client_freelist = client_freelist->next;
799 r = xmalloc(sizeof(struct client));
809 r->state = Client_Disconnected;
815 void client_destroy(struct client *c)
817 struct session *se = c->session;
818 if (c == se->clients)
819 se->clients = c->next;
823 for (cc = se->clients; cc && cc->next != c; cc = cc->next)
831 connection_release(c->connection);
832 c->next = client_freelist;
836 void client_set_connection(struct client *cl, struct connection *con)
838 cl->connection = con;
841 void client_disconnect(struct client *cl)
843 if (cl->state != Client_Idle)
844 cl->state = Client_Disconnected;
845 client_set_connection(cl, 0);
848 // Extract terms from query into null-terminated termlist
849 static void extract_terms(NMEM nmem, struct ccl_rpn_node *query, char **termlist)
853 pull_terms(nmem, query, termlist, &num);
857 // Initialize CCL map for a target
858 static CCL_bibset prepare_cclmap(struct client *cl)
860 struct session_database *sdb = client_get_database(cl);
867 for (s = sdb->settings[PZ_CCLMAP]; s; s = s->next)
869 char *p = strchr(s->name + 3, ':');
872 yaz_log(YLOG_WARN, "Malformed cclmap name: %s", s->name);
877 ccl_qual_fitem(res, s->value, p);
882 // Parse the query given the settings specific to this client
883 int client_parse_query(struct client *cl, const char *query)
885 struct session *se = client_get_session(cl);
886 struct ccl_rpn_node *cn;
888 CCL_bibset ccl_map = prepare_cclmap(cl);
892 cn = ccl_find_str(ccl_map, query, &cerror, &cpos);
893 ccl_qual_rm(&ccl_map);
896 cl->state = Client_Error;
897 yaz_log(YLOG_WARN, "Failed to parse query for %s",
898 client_get_database(cl)->database->url);
901 wrbuf_rewind(se->wrbuf);
902 ccl_pquery(se->wrbuf, cn);
904 cl->pquery = xstrdup(wrbuf_cstr(se->wrbuf));
908 // Initialize relevance structure with query terms
910 extract_terms(se->nmem, cn, p);
911 se->relevance = relevance_create(client_get_database(cl)->pct,
912 se->nmem, (const char **) p,
913 se->expected_maxrecs);
920 void client_set_session(struct client *cl, struct session *se)
923 cl->next = se->clients;
927 int client_is_active(struct client *cl)
929 if (cl->connection && (cl->state == Client_Connecting ||
930 cl->state == Client_Initializing ||
931 cl->state == Client_Searching ||
932 cl->state == Client_Presenting))
937 struct client *client_next_in_session(struct client *cl)
945 int client_get_hits(struct client *cl)
950 int client_get_num_records(struct client *cl)
955 int client_get_diagnostic(struct client *cl)
957 return cl->diagnostic;
960 void client_set_database(struct client *cl, struct session_database *db)
965 struct host *client_get_host(struct client *cl)
967 return client_get_database(cl)->database->host;
970 const char *client_get_url(struct client *cl)
972 return client_get_database(cl)->database->url;
978 * indent-tabs-mode: nil
980 * vim: shiftwidth=4 tabstop=8 expandtab