2 * Copyright (c) 1998-2004, Index Data.
3 * See the file LICENSE for details.
5 * $Id: yaz-proxy.cpp,v 1.93 2004-01-30 00:38:28 adam Exp $
12 #include <yaz/marcdisp.h>
13 #include <yaz/yaz-iconv.h>
15 #include <yaz/diagbib1.h>
16 #include <yaz++/proxy.h>
17 #include <yaz/pquery.h>
20 #include <libxslt/xsltutils.h>
21 #include <libxslt/transform.h>
24 static const char *apdu_name(Z_APDU *apdu)
28 case Z_APDU_initRequest:
30 case Z_APDU_initResponse:
31 return "initResponse";
32 case Z_APDU_searchRequest:
33 return "searchRequest";
34 case Z_APDU_searchResponse:
35 return "searchResponse";
36 case Z_APDU_presentRequest:
37 return "presentRequest";
38 case Z_APDU_presentResponse:
39 return "presentResponse";
40 case Z_APDU_deleteResultSetRequest:
41 return "deleteResultSetRequest";
42 case Z_APDU_deleteResultSetResponse:
43 return "deleteResultSetResponse";
44 case Z_APDU_scanRequest:
46 case Z_APDU_scanResponse:
47 return "scanResponse";
48 case Z_APDU_sortRequest:
50 case Z_APDU_sortResponse:
51 return "sortResponse";
52 case Z_APDU_extendedServicesRequest:
53 return "extendedServicesRequest";
54 case Z_APDU_extendedServicesResponse:
55 return "extendedServicesResponse";
62 static const char *gdu_name(Z_GDU *gdu)
67 return apdu_name(gdu->u.z3950);
68 case Z_GDU_HTTP_Request:
69 return "HTTP Request";
70 case Z_GDU_HTTP_Response:
71 return "HTTP Response";
73 return "Unknown request/response";
76 Yaz_Proxy::Yaz_Proxy(IYaz_PDU_Observable *the_PDU_Observable,
78 Yaz_Z_Assoc(the_PDU_Observable), m_bw_stat(60), m_pdu_stat(60)
80 m_PDU_Observable = the_PDU_Observable;
85 m_keepalive_limit_bw = 500000;
86 m_keepalive_limit_pdu = 1000;
89 m_proxy_authentication = 0;
93 m_client_idletime = 600;
94 m_target_idletime = 600;
95 m_optimize = xstrdup ("1");
96 strcpy(m_session_str, "0 ");
98 m_bytes_sent = m_bytes_recv = 0;
102 m_max_record_retrieve = 0;
106 m_invalid_session = 0;
109 m_stylesheet_schema = 0;
110 m_s2z_stylesheet = 0;
113 m_initRequest_apdu = 0;
114 m_initRequest_mem = 0;
115 m_initRequest_options = 0;
116 m_initRequest_version = 0;
117 m_apdu_invalid_session = 0;
118 m_mem_invalid_session = 0;
120 m_s2z_odr_search = 0;
122 m_s2z_search_apdu = 0;
123 m_s2z_present_apdu = 0;
124 m_http_keepalive = 0;
127 m_s2z_packing = Z_SRW_recordPacking_string;
128 m_time_tv.tv_sec = 0;
129 m_time_tv.tv_usec = 0;
132 Yaz_Proxy::~Yaz_Proxy()
134 yaz_log(LOG_LOG, "%sClosed %d/%d sent/recv bytes total", m_session_str,
135 m_bytes_sent, m_bytes_recv);
136 nmem_destroy(m_initRequest_mem);
137 nmem_destroy(m_mem_invalid_session);
138 xfree (m_proxyTarget);
139 xfree (m_default_target);
140 xfree (m_proxy_authentication);
142 xfree (m_stylesheet_schema);
145 odr_destroy(m_s2z_odr_init);
146 if (m_s2z_odr_search)
147 odr_destroy(m_s2z_odr_search);
151 int Yaz_Proxy::set_config(const char *config)
154 m_config = new Yaz_ProxyConfig();
155 xfree(m_config_fname);
156 m_config_fname = xstrdup(config);
157 int r = m_config->read_xml(config);
159 m_config->get_generic_info(&m_log_mask, &m_max_clients);
163 void Yaz_Proxy::set_default_target(const char *target)
165 xfree (m_default_target);
166 m_default_target = 0;
168 m_default_target = (char *) xstrdup (target);
171 void Yaz_Proxy::set_proxy_authentication (const char *auth)
173 xfree (m_proxy_authentication);
174 m_proxy_authentication = 0;
176 m_proxy_authentication = (char *) xstrdup (auth);
179 Yaz_ProxyConfig *Yaz_Proxy::check_reconfigure()
182 return m_parent->check_reconfigure();
184 Yaz_ProxyConfig *cfg = m_config;
187 yaz_log(LOG_LOG, "reconfigure");
189 if (m_config_fname && cfg)
191 yaz_log(LOG_LOG, "reconfigure config %s", m_config_fname);
192 int r = cfg->read_xml(m_config_fname);
194 yaz_log(LOG_WARN, "reconfigure failed");
198 cfg->get_generic_info(&m_log_mask, &m_max_clients);
202 yaz_log(LOG_LOG, "reconfigure");
208 IYaz_PDU_Observer *Yaz_Proxy::sessionNotify(IYaz_PDU_Observable
209 *the_PDU_Observable, int fd)
212 Yaz_Proxy *new_proxy = new Yaz_Proxy(the_PDU_Observable, this);
213 new_proxy->m_config = 0;
214 new_proxy->m_config_fname = 0;
215 new_proxy->timeout(m_client_idletime);
216 new_proxy->m_target_idletime = m_target_idletime;
217 new_proxy->set_default_target(m_default_target);
218 new_proxy->m_max_clients = m_max_clients;
219 new_proxy->m_log_mask = m_log_mask;
220 new_proxy->set_APDU_log(get_APDU_log());
221 if (m_log_mask & PROXY_LOG_APDU_CLIENT)
222 new_proxy->set_APDU_yazlog(1);
224 new_proxy->set_APDU_yazlog(0);
225 new_proxy->set_proxy_authentication(m_proxy_authentication);
226 sprintf(new_proxy->m_session_str, "%ld:%d ", (long) time(0), m_session_no);
228 yaz_log (LOG_LOG, "%sNew session %s", new_proxy->m_session_str,
229 the_PDU_Observable->getpeername());
233 char *Yaz_Proxy::get_cookie(Z_OtherInformation **otherInfo)
236 Z_OtherInformationUnit *oi;
238 ent.proto = PROTO_Z3950;
239 ent.oclass = CLASS_USERINFO;
240 ent.value = (oid_value) VAL_COOKIE;
241 assert (oid_ent_to_oid (&ent, oid));
243 if (oid_ent_to_oid (&ent, oid) &&
244 (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
245 oi->which == Z_OtherInfo_characterInfo)
246 return oi->information.characterInfo;
250 char *Yaz_Proxy::get_proxy(Z_OtherInformation **otherInfo)
253 Z_OtherInformationUnit *oi;
255 ent.proto = PROTO_Z3950;
256 ent.oclass = CLASS_USERINFO;
257 ent.value = (oid_value) VAL_PROXY;
258 if (oid_ent_to_oid (&ent, oid) &&
259 (oi = update_otherInformation(otherInfo, 0, oid, 1, 1)) &&
260 oi->which == Z_OtherInfo_characterInfo)
261 return oi->information.characterInfo;
265 const char *Yaz_Proxy::load_balance(const char **url)
267 int zurl_in_use[MAX_ZURL_PLEX];
271 for (i = 0; i<MAX_ZURL_PLEX; i++)
273 for (c = m_parent->m_clientPool; c; c = c->m_next)
275 for (i = 0; url[i]; i++)
276 if (!strcmp(url[i], c->get_hostname()))
281 for (i = 0; url[i]; i++)
283 yaz_log(LOG_DEBUG, "%szurl=%s use=%d",
284 m_session_str, url[i], zurl_in_use[i]);
285 if (min > zurl_in_use[i])
288 min = zurl_in_use[i];
294 Yaz_ProxyClient *Yaz_Proxy::get_client(Z_APDU *apdu, const char *cookie,
295 const char *proxy_host)
298 Yaz_Proxy *parent = m_parent;
299 Yaz_ProxyClient *c = m_client;
303 const char *url[MAX_ZURL_PLEX];
304 Yaz_ProxyConfig *cfg = check_reconfigure();
308 /* only to be enabled for debugging... */
309 if (!strcmp(proxy_host, "stop"))
312 xfree(m_default_target);
313 m_default_target = xstrdup(proxy_host);
314 proxy_host = m_default_target;
316 int client_idletime = -1;
317 const char *cql2rpn_fname = 0;
318 url[0] = m_default_target;
323 cfg->get_target_info(proxy_host, url, &m_bw_max,
324 &m_pdu_max, &m_max_record_retrieve,
325 &m_target_idletime, &client_idletime,
326 &parent->m_max_clients,
327 &m_keepalive_limit_bw,
328 &m_keepalive_limit_pdu,
332 if (client_idletime != -1)
334 m_client_idletime = client_idletime;
335 timeout(m_client_idletime);
338 m_cql2rpn.set_pqf_file(cql2rpn_fname);
341 yaz_log(LOG_LOG, "%sNo default target", m_session_str);
344 // we don't handle multiplexing for cookie session, so we just
345 // pick the first one in this case (anonymous users will be able
346 // to use any backend)
347 if (cookie && *cookie)
348 m_proxyTarget = (char*) xstrdup(url[0]);
350 m_proxyTarget = (char*) xstrdup(load_balance(url));
352 if (cookie && *cookie)
353 { // search in sessions with a cookie
354 for (c = parent->m_clientPool; c; c = c->m_next)
357 assert (*c->m_prev == c);
358 if (c->m_cookie && !strcmp(cookie,c->m_cookie) &&
359 !strcmp(m_proxyTarget, c->get_hostname()))
362 // The following handles "cancel"
363 // If connection is busy (waiting for PDU) and
364 // we have an initRequest we can safely do re-open
365 if (c->m_waiting && apdu->which == Z_APDU_initRequest)
367 yaz_log (LOG_LOG, "%s REOPEN target=%s", m_session_str,
374 c->m_last_resultCount = 0;
375 c->m_sr_transform = 0;
377 c->m_resultSetStartPoint = 0;
378 c->m_target_idletime = m_target_idletime;
379 if (c->client(m_proxyTarget))
386 c->m_seqno = parent->m_seqno;
387 if (c->m_server && c->m_server != this)
388 c->m_server->m_client = 0;
391 yaz_log (LOG_DEBUG, "get_client 1 %p %p", this, c);
398 // don't have a client session yet. Search in session w/o cookie
399 for (c = parent->m_clientPool; c; c = c->m_next)
402 assert (*c->m_prev == c);
403 if (c->m_server == 0 && c->m_cookie == 0 &&
405 !strcmp(m_proxyTarget, c->get_hostname()))
408 yaz_log (LOG_LOG, "%sREUSE %s",
409 m_session_str, c->get_hostname());
411 c->m_seqno = parent->m_seqno;
412 assert(c->m_server == 0);
415 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
416 c->set_APDU_yazlog(1);
418 c->set_APDU_yazlog(0);
430 if (apdu->which != Z_APDU_initRequest)
432 yaz_log (LOG_LOG, "%sno init request as first PDU", m_session_str);
435 Z_InitRequest *initRequest = apdu->u.initRequest;
437 if (!initRequest->idAuthentication)
439 if (m_proxy_authentication)
441 initRequest->idAuthentication =
442 (Z_IdAuthentication *)
443 odr_malloc (odr_encode(),
444 sizeof(*initRequest->idAuthentication));
445 initRequest->idAuthentication->which =
446 Z_IdAuthentication_open;
447 initRequest->idAuthentication->u.open =
448 odr_strdup (odr_encode(), m_proxy_authentication);
451 // go through list of clients - and find the lowest/oldest one.
452 Yaz_ProxyClient *c_min = 0;
454 int no_of_clients = 0;
455 if (parent->m_clientPool)
456 yaz_log (LOG_DEBUG, "Existing sessions");
457 for (c = parent->m_clientPool; c; c = c->m_next)
459 yaz_log (LOG_DEBUG, " Session %-3d wait=%d %s cookie=%s", c->m_seqno,
460 c->m_waiting, c->get_hostname(),
461 c->m_cookie ? c->m_cookie : "");
463 if (min_seq < 0 || c->m_seqno < min_seq)
465 min_seq = c->m_seqno;
469 if (no_of_clients >= parent->m_max_clients)
472 if (c->m_waiting || strcmp(m_proxyTarget, c->get_hostname()))
474 yaz_log (LOG_LOG, "%sMAXCLIENTS Destroy %d",
475 m_session_str, c->m_seqno);
476 if (c->m_server && c->m_server != this)
482 yaz_log (LOG_LOG, "%sMAXCLIENTS Reuse %d %d %s",
484 c->m_seqno, parent->m_seqno, c->get_hostname());
488 c->m_cookie = xstrdup(cookie);
489 c->m_seqno = parent->m_seqno;
490 if (c->m_server && c->m_server != this)
492 c->m_server->m_client = 0;
496 c->m_target_idletime = m_target_idletime;
497 c->timeout(m_target_idletime);
499 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
500 c->set_APDU_yazlog(1);
502 c->set_APDU_yazlog(0);
509 yaz_log (LOG_LOG, "%sNEW %d %s",
510 m_session_str, parent->m_seqno, m_proxyTarget);
511 c = new Yaz_ProxyClient(m_PDU_Observable->clone(), parent);
512 c->m_next = parent->m_clientPool;
514 c->m_next->m_prev = &c->m_next;
515 parent->m_clientPool = c;
516 c->m_prev = &parent->m_clientPool;
522 c->m_cookie = xstrdup(cookie);
524 c->m_seqno = parent->m_seqno;
526 c->m_last_resultCount = 0;
529 c->m_sr_transform = 0;
531 c->m_resultSetStartPoint = 0;
533 if (c->client(m_proxyTarget))
538 c->m_target_idletime = m_target_idletime;
541 if (parent->m_log_mask & PROXY_LOG_APDU_SERVER)
542 c->set_APDU_yazlog(1);
544 c->set_APDU_yazlog(0);
546 yaz_log (LOG_DEBUG, "get_client 3 %p %p", this, c);
550 void Yaz_Proxy::display_diagrecs(Z_DiagRec **pp, int num)
553 for (i = 0; i<num; i++)
556 Z_DefaultDiagFormat *r;
557 Z_DiagRec *p = pp[i];
558 if (p->which != Z_DiagRec_defaultFormat)
560 yaz_log(LOG_LOG, "%sError no diagnostics", m_session_str);
564 r = p->u.defaultFormat;
565 if (!(ent = oid_getentbyoid(r->diagnosticSetId)) ||
566 ent->oclass != CLASS_DIAGSET || ent->value != VAL_BIB1)
567 yaz_log(LOG_LOG, "%sError unknown diagnostic set", m_session_str);
570 case Z_DefaultDiagFormat_v2Addinfo:
571 yaz_log(LOG_LOG, "%sError %d %s:%s",
573 *r->condition, diagbib1_str(*r->condition),
576 case Z_DefaultDiagFormat_v3Addinfo:
577 yaz_log(LOG_LOG, "%sError %d %s:%s",
579 *r->condition, diagbib1_str(*r->condition),
586 void Yaz_Proxy::convert_xsl(Z_NamePlusRecordList *p)
588 if (!m_stylesheet_schema)
590 xsltStylesheetPtr xsp;
592 xsp = xsltParseStylesheetFile((const xmlChar *) m_stylesheet_schema);
595 for (i = 0; i < p->num_records; i++)
597 Z_NamePlusRecord *npr = p->records[i];
598 if (npr->which == Z_NamePlusRecord_databaseRecord)
600 Z_External *r = npr->u.databaseRecord;
601 if (r->which == Z_External_octet)
603 xmlDocPtr res, doc = xmlParseMemory(
604 (char*) r->u.octet_aligned->buf,
605 r->u.octet_aligned->len);
607 res = xsltApplyStylesheet(xsp, doc, 0);
611 xmlDocDumpFormatMemory (res, &out_buf, &out_len, 1);
613 p->records[i]->u.databaseRecord =
614 z_ext_record(odr_encode(), VAL_TEXT_XML,
615 (char*) out_buf, out_len);
622 xsltFreeStylesheet(xsp);
625 void Yaz_Proxy::convert_to_marcxml(Z_NamePlusRecordList *p)
629 yaz_marc_t mt = yaz_marc_create();
630 yaz_marc_xml(mt, YAZ_MARC_MARCXML);
631 for (i = 0; i < p->num_records; i++)
633 Z_NamePlusRecord *npr = p->records[i];
634 if (npr->which == Z_NamePlusRecord_databaseRecord)
636 Z_External *r = npr->u.databaseRecord;
637 if (r->which == Z_External_octet)
641 if (yaz_marc_decode_buf(mt, (char*) r->u.octet_aligned->buf,
642 r->u.octet_aligned->len,
645 yaz_iconv_t cd = yaz_iconv_open("UTF-8", "MARC-8");
646 WRBUF wrbuf = wrbuf_alloc();
649 size_t inbytesleft = rlen;
650 const char *inp = result;
651 while (cd && inbytesleft)
653 size_t outbytesleft = sizeof(outbuf);
657 r = yaz_iconv (cd, (char**) &inp,
659 &outp, &outbytesleft);
660 if (r == (size_t) (-1))
662 int e = yaz_iconv_error(cd);
663 if (e != YAZ_ICONV_E2BIG)
665 yaz_log(LOG_WARN, "conversion failure");
669 wrbuf_write(wrbuf, outbuf, outp - outbuf);
674 npr->u.databaseRecord = z_ext_record(odr_encode(),
678 wrbuf_free(wrbuf, 1);
683 yaz_marc_destroy(mt);
686 void Yaz_Proxy::logtime()
688 if (m_time_tv.tv_sec)
691 gettimeofday(&tv, 0);
692 long diff = (tv.tv_sec - m_time_tv.tv_sec)*1000000 +
693 (tv.tv_usec - m_time_tv.tv_usec);
695 yaz_log(LOG_LOG, "%sElapsed %ld.%03ld", m_session_str,
696 diff/1000000, (diff/1000)%1000);
698 m_time_tv.tv_sec = 0;
699 m_time_tv.tv_usec = 0;
702 int Yaz_Proxy::send_http_response(int code)
704 ODR o = odr_encode();
705 Z_GDU *gdu = z_get_HTTP_Response(o, code);
706 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
708 hres->version = odr_strdup(o, m_http_version);
709 if (m_http_keepalive)
710 z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
714 if (m_log_mask & PROXY_LOG_REQ_CLIENT)
716 yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
720 int r = send_GDU(gdu, &len);
725 int Yaz_Proxy::send_srw_response(Z_SRW_PDU *srw_pdu)
727 ODR o = odr_encode();
728 const char *ctype = "text/xml";
729 Z_GDU *gdu = z_get_HTTP_Response(o, 200);
730 Z_HTTP_Response *hres = gdu->u.HTTP_Response;
732 hres->version = odr_strdup(o, m_http_version);
733 z_HTTP_header_add(o, &hres->headers, "Content-Type", ctype);
734 if (m_http_keepalive)
735 z_HTTP_header_add(o, &hres->headers, "Connection", "Keep-Alive");
739 static Z_SOAP_Handler soap_handlers[2] = {
741 {"http://www.loc.gov/zing/srw/", 0,
742 (Z_SOAP_fun) yaz_srw_codec},
747 Z_SOAP *soap_package = (Z_SOAP*) odr_malloc(o, sizeof(Z_SOAP));
748 soap_package->which = Z_SOAP_generic;
749 soap_package->u.generic =
750 (Z_SOAP_Generic *) odr_malloc(o, sizeof(*soap_package->u.generic));
751 soap_package->u.generic->no = 0;
752 soap_package->u.generic->ns = soap_handlers[0].ns;
753 soap_package->u.generic->p = (void *) srw_pdu;
754 soap_package->ns = m_soap_ns;
755 z_soap_codec_enc_xsl(o, &soap_package,
756 &hres->content_buf, &hres->content_len,
757 soap_handlers, 0, m_s2z_stylesheet);
758 if (m_log_mask & PROXY_LOG_REQ_CLIENT)
760 yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
764 int r = send_GDU(gdu, &len);
769 int Yaz_Proxy::send_to_srw_client_error(int srw_error, const char *add)
771 ODR o = odr_encode();
772 Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
773 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
775 srw_res->num_diagnostics = 1;
776 srw_res->diagnostics = (Z_SRW_diagnostic *)
777 odr_malloc(o, sizeof(*srw_res->diagnostics));
778 yaz_mk_std_diagnostic(o, srw_res->diagnostics, srw_error, add);
779 return send_srw_response(srw_pdu);
782 int Yaz_Proxy::z_to_srw_diag(ODR o, Z_SRW_searchRetrieveResponse *srw_res,
783 Z_DefaultDiagFormat *ddf)
785 int bib1_code = *ddf->condition;
786 if (bib1_code == 109)
788 srw_res->num_diagnostics = 1;
789 srw_res->diagnostics = (Z_SRW_diagnostic *)
790 odr_malloc(o, sizeof(*srw_res->diagnostics));
791 yaz_mk_std_diagnostic(o, srw_res->diagnostics,
792 yaz_diag_bib1_to_srw(*ddf->condition),
797 int Yaz_Proxy::send_to_srw_client_ok(int hits, Z_Records *records, int start)
799 ODR o = odr_encode();
800 Z_SRW_PDU *srw_pdu = yaz_srw_get(o, Z_SRW_searchRetrieve_response);
801 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
803 srw_res->numberOfRecords = odr_intdup (o, hits);
804 if (records && records->which == Z_Records_DBOSD)
806 srw_res->num_records =
807 records->u.databaseOrSurDiagnostics->num_records;
809 srw_res->records = (Z_SRW_record *)
810 odr_malloc(o, srw_res->num_records * sizeof(Z_SRW_record));
811 for (i = 0; i < srw_res->num_records; i++)
813 Z_NamePlusRecord *npr = records->u.databaseOrSurDiagnostics->records[i];
814 if (npr->which != Z_NamePlusRecord_databaseRecord)
816 srw_res->records[i].recordSchema = "diagnostic";
817 srw_res->records[i].recordPacking = m_s2z_packing;
818 srw_res->records[i].recordData_buf = "67";
819 srw_res->records[i].recordData_len = 2;
820 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
823 Z_External *r = npr->u.databaseRecord;
824 oident *ent = oid_getentbyoid(r->direct_reference);
825 if (r->which == Z_External_octet && ent->value == VAL_TEXT_XML)
827 srw_res->records[i].recordSchema = m_schema;
828 srw_res->records[i].recordPacking = m_s2z_packing;
829 srw_res->records[i].recordData_buf = (char*)
830 r->u.octet_aligned->buf;
831 srw_res->records[i].recordData_len = r->u.octet_aligned->len;
832 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
836 srw_res->records[i].recordSchema = "diagnostic";
837 srw_res->records[i].recordPacking = m_s2z_packing;
838 srw_res->records[i].recordData_buf = "67";
839 srw_res->records[i].recordData_len = 2;
840 srw_res->records[i].recordPosition = odr_intdup(o, i+start);
844 if (records && records->which == Z_Records_NSD)
847 http_code = z_to_srw_diag(odr_encode(), srw_res,
848 records->u.nonSurrogateDiagnostic);
850 return send_http_response(http_code);
852 return send_srw_response(srw_pdu);
856 int Yaz_Proxy::send_srw_explain_response(Z_SRW_diagnostic *diagnostics,
859 Yaz_ProxyConfig *cfg = check_reconfigure();
863 char *b = cfg->get_explain(odr_encode(), 0 /* target */,
864 m_s2z_database, &len);
867 Z_SRW_PDU *res = yaz_srw_get(odr_encode(), Z_SRW_explain_response);
868 Z_SRW_explainResponse *er = res->u.explain_response;
870 er->record.recordData_buf = b;
871 er->record.recordData_len = len;
872 er->record.recordPacking = m_s2z_packing;
874 er->diagnostics = diagnostics;
875 er->num_diagnostics = num_diagnostics;
876 return send_srw_response(res);
879 return send_http_response(404);
882 int Yaz_Proxy::send_PDU_convert(Z_APDU *apdu, int *len)
886 if (apdu->which == Z_APDU_initResponse)
888 Z_InitResponse *res = apdu->u.initResponse;
889 if (*res->result == 0)
891 send_to_srw_client_error(3, 0);
893 else if (!m_s2z_search_apdu)
895 send_srw_explain_response(0, 0);
899 handle_incoming_Z_PDU(m_s2z_search_apdu);
902 else if (m_s2z_search_apdu && apdu->which == Z_APDU_searchResponse)
904 m_s2z_search_apdu = 0;
905 Z_SearchResponse *res = apdu->u.searchResponse;
906 m_s2z_hit_count = *res->resultCount;
907 if (res->records && res->records->which == Z_Records_NSD)
909 send_to_srw_client_ok(0, res->records, 1);
911 else if (m_s2z_present_apdu && m_s2z_hit_count > 0)
914 Z_PresentRequest *pr = m_s2z_present_apdu->u.presentRequest;
916 if (*pr->resultSetStartPoint <= m_s2z_hit_count)
918 if (*pr->numberOfRecordsRequested+ *pr->resultSetStartPoint
920 *pr->numberOfRecordsRequested =
921 1 + m_s2z_hit_count - *pr->resultSetStartPoint;
923 handle_incoming_Z_PDU(m_s2z_present_apdu);
927 m_s2z_present_apdu = 0;
928 send_to_srw_client_ok(m_s2z_hit_count, res->records, 1);
931 else if (m_s2z_present_apdu && apdu->which == Z_APDU_presentResponse)
934 *m_s2z_present_apdu->u.presentRequest->resultSetStartPoint;
936 m_s2z_present_apdu = 0;
937 Z_PresentResponse *res = apdu->u.presentResponse;
938 send_to_srw_client_ok(m_s2z_hit_count, res->records, start);
943 if (m_log_mask & PROXY_LOG_REQ_CLIENT)
944 yaz_log (LOG_LOG, "%sSending %s to client", m_session_str,
946 int r = send_Z_PDU(apdu, len);
953 int Yaz_Proxy::send_to_client(Z_APDU *apdu)
956 int kill_session = 0;
957 if (apdu->which == Z_APDU_searchResponse)
959 Z_SearchResponse *sr = apdu->u.searchResponse;
960 Z_Records *p = sr->records;
961 if (p && p->which == Z_Records_NSD)
963 Z_DiagRec dr, *dr_p = &dr;
964 dr.which = Z_DiagRec_defaultFormat;
965 dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
967 *sr->searchStatus = 0;
968 display_diagrecs(&dr_p, 1);
972 if (p && p->which == Z_Records_DBOSD)
975 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
976 convert_xsl(p->u.databaseOrSurDiagnostics);
980 yaz_log(LOG_LOG, "%s%d hits", m_session_str,
982 if (*sr->resultCount < 0)
984 m_invalid_session = 1;
987 *sr->searchStatus = 0;
989 create_nonSurrogateDiagnostics(odr_encode(), 2, 0);
990 *sr->resultCount = 0;
995 else if (apdu->which == Z_APDU_presentResponse)
997 Z_PresentResponse *sr = apdu->u.presentResponse;
998 Z_Records *p = sr->records;
999 if (p && p->which == Z_Records_NSD)
1001 Z_DiagRec dr, *dr_p = &dr;
1002 dr.which = Z_DiagRec_defaultFormat;
1003 dr.u.defaultFormat = p->u.nonSurrogateDiagnostic;
1004 if (*sr->presentStatus == Z_PresentStatus_success)
1005 *sr->presentStatus = Z_PresentStatus_failure;
1006 display_diagrecs(&dr_p, 1);
1008 if (p && p->which == Z_Records_DBOSD)
1011 convert_to_marcxml(p->u.databaseOrSurDiagnostics);
1012 convert_xsl(p->u.databaseOrSurDiagnostics);
1015 else if (apdu->which == Z_APDU_initResponse)
1017 if (m_initRequest_options)
1020 (Odr_bitmask *)odr_malloc(odr_encode(),
1021 sizeof(Odr_bitmask));
1022 ODR_MASK_ZERO(nopt);
1025 for (i = 0; i<24; i++)
1026 if (ODR_MASK_GET(m_initRequest_options, i) &&
1027 ODR_MASK_GET(apdu->u.initResponse->options, i))
1028 ODR_MASK_SET(nopt, i);
1029 apdu->u.initResponse->options = nopt;
1031 if (m_initRequest_version)
1033 Z_ProtocolVersion *nopt =
1034 (Odr_bitmask *)odr_malloc(odr_encode(),
1035 sizeof(Odr_bitmask));
1036 ODR_MASK_ZERO(nopt);
1039 for (i = 0; i<8; i++)
1040 if (ODR_MASK_GET(m_initRequest_version, i) &&
1041 ODR_MASK_GET(apdu->u.initResponse->protocolVersion, i))
1042 ODR_MASK_SET(nopt, i);
1043 apdu->u.initResponse->protocolVersion = nopt;
1046 int r = send_PDU_convert(apdu, &len);
1049 m_bytes_sent += len;
1050 m_bw_stat.add_bytes(len);
1055 m_parent->pre_init();
1060 int Yaz_ProxyClient::send_to_target(Z_APDU *apdu)
1063 const char *apdu_name_tmp = apdu_name(apdu);
1064 int r = send_Z_PDU(apdu, &len);
1065 if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
1066 yaz_log (LOG_LOG, "%sSending %s to %s %d bytes",
1068 apdu_name_tmp, get_hostname(), len);
1069 m_bytes_sent += len;
1073 Z_APDU *Yaz_Proxy::result_set_optimize(Z_APDU *apdu)
1075 if (apdu->which == Z_APDU_presentRequest)
1077 Z_PresentRequest *pr = apdu->u.presentRequest;
1078 int toget = *pr->numberOfRecordsRequested;
1079 int start = *pr->resultSetStartPoint;
1081 yaz_log(LOG_LOG, "%sPresent %s %d+%d", m_session_str,
1082 pr->resultSetId, start, toget);
1084 if (*m_parent->m_optimize == '0')
1087 if (!m_client->m_last_resultSetId)
1089 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1090 new_apdu->u.presentResponse->records =
1091 create_nonSurrogateDiagnostics(odr_encode(), 30,
1093 send_to_client(new_apdu);
1096 if (!strcmp(m_client->m_last_resultSetId, pr->resultSetId))
1098 if (start+toget-1 > m_client->m_last_resultCount)
1100 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1101 new_apdu->u.presentResponse->records =
1102 create_nonSurrogateDiagnostics(odr_encode(), 13, 0);
1103 send_to_client(new_apdu);
1106 Z_NamePlusRecordList *npr;
1107 if (m_client->m_cache.lookup (odr_encode(), &npr, start, toget,
1108 pr->preferredRecordSyntax,
1109 pr->recordComposition))
1111 yaz_log (LOG_LOG, "%sReturned cached records for present request",
1113 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1114 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1116 new_apdu->u.presentResponse->numberOfRecordsReturned
1117 = odr_intdup(odr_encode(), toget);
1119 new_apdu->u.presentResponse->records = (Z_Records*)
1120 odr_malloc(odr_encode(), sizeof(Z_Records));
1121 new_apdu->u.presentResponse->records->which = Z_Records_DBOSD;
1122 new_apdu->u.presentResponse->records->u.databaseOrSurDiagnostics = npr;
1123 new_apdu->u.presentResponse->nextResultSetPosition =
1124 odr_intdup(odr_encode(), start+toget);
1126 send_to_client(new_apdu);
1132 if (apdu->which != Z_APDU_searchRequest)
1134 Z_SearchRequest *sr = apdu->u.searchRequest;
1135 Yaz_Z_Query *this_query = new Yaz_Z_Query;
1136 Yaz_Z_Databases this_databases;
1138 this_databases.set(sr->num_databaseNames, (const char **)
1141 this_query->set_Z_Query(sr->query);
1143 char query_str[120];
1144 this_query->print(query_str, sizeof(query_str)-1);
1145 yaz_log(LOG_LOG, "%sSearch %s", m_session_str, query_str);
1147 if (*m_parent->m_optimize != '0' &&
1148 m_client->m_last_ok && m_client->m_last_query &&
1149 m_client->m_last_query->match(this_query) &&
1150 !strcmp(m_client->m_last_resultSetId, sr->resultSetName) &&
1151 m_client->m_last_databases.match(this_databases))
1154 if (m_client->m_last_resultCount > *sr->smallSetUpperBound &&
1155 m_client->m_last_resultCount < *sr->largeSetLowerBound)
1157 Z_NamePlusRecordList *npr;
1158 int toget = *sr->mediumSetPresentNumber;
1159 Z_RecordComposition *comp = 0;
1161 if (toget > m_client->m_last_resultCount)
1162 toget = m_client->m_last_resultCount;
1164 if (sr->mediumSetElementSetNames)
1166 comp = (Z_RecordComposition *)
1167 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1168 comp->which = Z_RecordComp_simple;
1169 comp->u.simple = sr->mediumSetElementSetNames;
1172 if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1173 sr->preferredRecordSyntax, comp))
1175 yaz_log (LOG_LOG, "%sReturned cached records for medium set",
1177 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1178 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1179 new_apdu->u.searchResponse->resultCount =
1180 &m_client->m_last_resultCount;
1182 new_apdu->u.searchResponse->numberOfRecordsReturned
1183 = odr_intdup(odr_encode(), toget);
1185 new_apdu->u.searchResponse->presentStatus =
1186 odr_intdup(odr_encode(), Z_PresentStatus_success);
1187 new_apdu->u.searchResponse->records = (Z_Records*)
1188 odr_malloc(odr_encode(), sizeof(Z_Records));
1189 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1190 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1191 new_apdu->u.searchResponse->nextResultSetPosition =
1192 odr_intdup(odr_encode(), toget+1);
1193 send_to_client(new_apdu);
1199 // send present request (medium size)
1200 yaz_log (LOG_LOG, "%sOptimizing search for medium set",
1203 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1204 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1205 pr->referenceId = sr->referenceId;
1206 pr->resultSetId = sr->resultSetName;
1207 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1208 *pr->numberOfRecordsRequested = toget;
1209 pr->recordComposition = comp;
1210 m_client->m_sr_transform = 1;
1214 else if (m_client->m_last_resultCount >= *sr->largeSetLowerBound ||
1215 m_client->m_last_resultCount <= 0)
1217 // large set. Return pseudo-search response immediately
1218 yaz_log (LOG_LOG, "%sOptimizing search for large set",
1220 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1221 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1222 new_apdu->u.searchResponse->resultCount =
1223 &m_client->m_last_resultCount;
1224 send_to_client(new_apdu);
1229 Z_NamePlusRecordList *npr;
1230 int toget = m_client->m_last_resultCount;
1231 Z_RecordComposition *comp = 0;
1233 // send a present request (small set)
1235 if (sr->smallSetElementSetNames)
1237 comp = (Z_RecordComposition *)
1238 odr_malloc(odr_encode(), sizeof(Z_RecordComposition));
1239 comp->which = Z_RecordComp_simple;
1240 comp->u.simple = sr->smallSetElementSetNames;
1243 if (m_client->m_cache.lookup (odr_encode(), &npr, 1, toget,
1244 sr->preferredRecordSyntax, comp))
1246 yaz_log (LOG_LOG, "%sReturned cached records for small set",
1248 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1249 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1250 new_apdu->u.searchResponse->resultCount =
1251 &m_client->m_last_resultCount;
1253 new_apdu->u.searchResponse->numberOfRecordsReturned
1254 = odr_intdup(odr_encode(), toget);
1256 new_apdu->u.searchResponse->presentStatus =
1257 odr_intdup(odr_encode(), Z_PresentStatus_success);
1258 new_apdu->u.searchResponse->records = (Z_Records*)
1259 odr_malloc(odr_encode(), sizeof(Z_Records));
1260 new_apdu->u.searchResponse->records->which = Z_Records_DBOSD;
1261 new_apdu->u.searchResponse->records->u.databaseOrSurDiagnostics = npr;
1262 new_apdu->u.searchResponse->nextResultSetPosition =
1263 odr_intdup(odr_encode(), toget+1);
1264 send_to_client(new_apdu);
1269 yaz_log (LOG_LOG, "%sOptimizing search for small set",
1271 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentRequest);
1272 Z_PresentRequest *pr = new_apdu->u.presentRequest;
1273 pr->referenceId = sr->referenceId;
1274 pr->resultSetId = sr->resultSetName;
1275 pr->preferredRecordSyntax = sr->preferredRecordSyntax;
1276 *pr->numberOfRecordsRequested = toget;
1277 pr->recordComposition = comp;
1278 m_client->m_sr_transform = 1;
1283 else // query doesn't match
1285 delete m_client->m_last_query;
1286 m_client->m_last_query = this_query;
1287 m_client->m_last_ok = 0;
1288 m_client->m_cache.clear();
1289 m_client->m_resultSetStartPoint = 0;
1291 xfree (m_client->m_last_resultSetId);
1292 m_client->m_last_resultSetId = xstrdup (sr->resultSetName);
1294 m_client->m_last_databases.set(sr->num_databaseNames,
1295 (const char **) sr->databaseNames);
1301 void Yaz_Proxy::inc_request_no()
1303 char *cp = strchr(m_session_str, ' ');
1306 sprintf(cp+1, "%d ", m_request_no);
1309 void Yaz_Proxy::recv_GDU(Z_GDU *apdu, int len)
1313 m_bytes_recv += len;
1315 if (m_log_mask & PROXY_LOG_APDU_CLIENT)
1316 yaz_log (LOG_DEBUG, "%sReceiving %s from client %d bytes",
1317 m_session_str, gdu_name(apdu), len);
1319 if (m_bw_hold_PDU) // double incoming PDU. shutdown now.
1322 m_bw_stat.add_bytes(len);
1323 m_pdu_stat.add_bytes(1);
1325 gettimeofday(&m_time_tv, 0);
1327 int bw_total = m_bw_stat.get_total();
1328 int pdu_total = m_pdu_stat.get_total();
1333 if (bw_total > m_bw_max)
1335 reduce = (bw_total/m_bw_max);
1340 if (pdu_total > m_pdu_max)
1342 int nreduce = (m_pdu_max >= 60) ? 1 : 60/m_pdu_max;
1343 reduce = (reduce > nreduce) ? reduce : nreduce;
1348 yaz_log(LOG_LOG, "%sdelay=%d bw=%d pdu=%d limit-bw=%d limit-pdu=%d",
1349 m_session_str, reduce, bw_total, pdu_total,
1350 m_bw_max, m_pdu_max);
1352 m_bw_hold_PDU = apdu; // save PDU and signal "on hold"
1353 timeout(reduce); // call us reduce seconds later
1355 else if (apdu->which == Z_GDU_Z3950)
1356 handle_incoming_Z_PDU(apdu->u.z3950);
1357 else if (apdu->which == Z_GDU_HTTP_Request)
1358 handle_incoming_HTTP(apdu->u.HTTP_Request);
1361 void Yaz_Proxy::handle_max_record_retrieve(Z_APDU *apdu)
1363 if (m_max_record_retrieve)
1365 if (apdu->which == Z_APDU_presentRequest)
1367 Z_PresentRequest *pr = apdu->u.presentRequest;
1368 if (pr->numberOfRecordsRequested &&
1369 *pr->numberOfRecordsRequested > m_max_record_retrieve)
1370 *pr->numberOfRecordsRequested = m_max_record_retrieve;
1375 Z_Records *Yaz_Proxy::create_nonSurrogateDiagnostics(ODR odr,
1377 const char *addinfo)
1379 Z_Records *rec = (Z_Records *)
1380 odr_malloc (odr, sizeof(*rec));
1382 odr_malloc (odr, sizeof(*err));
1383 Z_DiagRec *drec = (Z_DiagRec *)
1384 odr_malloc (odr, sizeof(*drec));
1385 Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
1386 odr_malloc (odr, sizeof(*dr));
1388 rec->which = Z_Records_NSD;
1389 rec->u.nonSurrogateDiagnostic = dr;
1390 dr->diagnosticSetId =
1391 yaz_oidval_to_z3950oid (odr, CLASS_DIAGSET, VAL_BIB1);
1392 dr->condition = err;
1393 dr->which = Z_DefaultDiagFormat_v2Addinfo;
1394 dr->u.v2Addinfo = odr_strdup (odr, addinfo ? addinfo : "");
1398 Z_APDU *Yaz_Proxy::handle_query_transformation(Z_APDU *apdu)
1400 if (apdu->which == Z_APDU_searchRequest &&
1401 apdu->u.searchRequest->query &&
1402 apdu->u.searchRequest->query->which == Z_Query_type_104 &&
1403 apdu->u.searchRequest->query->u.type_104->which == Z_External_CQL)
1405 Z_RPNQuery *rpnquery = 0;
1406 Z_SearchRequest *sr = apdu->u.searchRequest;
1409 yaz_log(LOG_LOG, "%sCQL: %s", m_session_str,
1410 sr->query->u.type_104->u.cql);
1412 int r = m_cql2rpn.query_transform(sr->query->u.type_104->u.cql,
1413 &rpnquery, odr_encode(),
1416 yaz_log(LOG_LOG, "%sNo CQL to RPN table", m_session_str);
1419 yaz_log(LOG_LOG, "%sCQL Conversion error %d", m_session_str, r);
1420 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1422 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1423 new_apdu->u.searchResponse->records =
1424 create_nonSurrogateDiagnostics(odr_encode(),
1425 yaz_diag_srw_to_bib1(r),
1427 *new_apdu->u.searchResponse->searchStatus = 0;
1429 send_to_client(new_apdu);
1435 sr->query->which = Z_Query_type_1;
1436 sr->query->u.type_1 = rpnquery;
1443 Z_APDU *Yaz_Proxy::handle_query_validation(Z_APDU *apdu)
1445 if (apdu->which == Z_APDU_searchRequest)
1447 Z_SearchRequest *sr = apdu->u.searchRequest;
1451 Yaz_ProxyConfig *cfg = check_reconfigure();
1453 err = cfg->check_query(odr_encode(), m_default_target,
1454 sr->query, &addinfo);
1457 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1459 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1460 new_apdu->u.searchResponse->records =
1461 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1462 *new_apdu->u.searchResponse->searchStatus = 0;
1464 send_to_client(new_apdu);
1472 Z_APDU *Yaz_Proxy::handle_syntax_validation(Z_APDU *apdu)
1475 if (apdu->which == Z_APDU_searchRequest)
1477 Z_SearchRequest *sr = apdu->u.searchRequest;
1480 Yaz_ProxyConfig *cfg = check_reconfigure();
1482 Z_RecordComposition rc_temp, *rc = 0;
1483 if (sr->smallSetElementSetNames)
1485 rc_temp.which = Z_RecordComp_simple;
1486 rc_temp.u.simple = sr->smallSetElementSetNames;
1491 err = cfg->check_syntax(odr_encode(),
1493 sr->preferredRecordSyntax, rc,
1494 &addinfo, &m_stylesheet_schema, &m_schema);
1497 sr->preferredRecordSyntax =
1498 yaz_oidval_to_z3950oid(odr_encode(), CLASS_RECSYN, VAL_USMARC);
1503 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
1505 new_apdu->u.searchResponse->referenceId = sr->referenceId;
1506 new_apdu->u.searchResponse->records =
1507 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1508 *new_apdu->u.searchResponse->searchStatus = 0;
1510 send_to_client(new_apdu);
1515 else if (apdu->which == Z_APDU_presentRequest)
1517 Z_PresentRequest *pr = apdu->u.presentRequest;
1520 Yaz_ProxyConfig *cfg = check_reconfigure();
1523 err = cfg->check_syntax(odr_encode(), m_default_target,
1524 pr->preferredRecordSyntax,
1525 pr->recordComposition,
1526 &addinfo, &m_stylesheet_schema, &m_schema);
1529 pr->preferredRecordSyntax =
1530 yaz_oidval_to_z3950oid(odr_decode(), CLASS_RECSYN, VAL_USMARC);
1535 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_presentResponse);
1537 new_apdu->u.presentResponse->referenceId = pr->referenceId;
1538 new_apdu->u.presentResponse->records =
1539 create_nonSurrogateDiagnostics(odr_encode(), err, addinfo);
1540 *new_apdu->u.presentResponse->presentStatus =
1541 Z_PresentStatus_failure;
1543 send_to_client(new_apdu);
1551 Z_ElementSetNames *Yaz_Proxy::mk_esn_from_schema(ODR o, const char *schema)
1555 Z_ElementSetNames *esn = (Z_ElementSetNames *)
1556 odr_malloc(o, sizeof(Z_ElementSetNames));
1557 esn->which = Z_ElementSetNames_generic;
1558 esn->u.generic = odr_strdup(o, schema);
1562 void Yaz_Proxy::handle_incoming_HTTP(Z_HTTP_Request *hreq)
1566 odr_destroy(m_s2z_odr_init);
1569 if (m_s2z_odr_search)
1571 odr_destroy(m_s2z_odr_search);
1572 m_s2z_odr_search = 0;
1575 m_http_keepalive = 0;
1577 if (!strcmp(hreq->version, "1.0"))
1579 const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1580 if (v && !strcmp(v, "Keep-Alive"))
1581 m_http_keepalive = 1;
1583 m_http_keepalive = 0;
1584 m_http_version = "1.0";
1588 const char *v = z_HTTP_header_lookup(hreq->headers, "Connection");
1589 if (v && !strcmp(v, "close"))
1590 m_http_keepalive = 0;
1592 m_http_keepalive = 1;
1593 m_http_version = "1.1";
1596 Z_SRW_PDU *srw_pdu = 0;
1597 Z_SOAP *soap_package = 0;
1599 Z_SRW_diagnostic *diagnostic = 0;
1600 int num_diagnostic = 0;
1601 if (yaz_srw_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1603 || yaz_sru_decode(hreq, &srw_pdu, &soap_package, odr_decode(),
1604 &charset, &diagnostic, &num_diagnostic) == 0)
1606 m_s2z_odr_init = odr_createmem(ODR_ENCODE);
1607 m_s2z_odr_search = odr_createmem(ODR_ENCODE);
1608 m_soap_ns = odr_strdup(m_s2z_odr_search, soap_package->ns);
1609 m_s2z_init_apdu = 0;
1610 m_s2z_search_apdu = 0;
1611 m_s2z_present_apdu = 0;
1613 m_s2z_stylesheet = 0;
1615 if (srw_pdu->which == Z_SRW_searchRetrieve_request)
1617 Z_SRW_searchRetrieveRequest *srw_req = srw_pdu->u.request;
1619 m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1620 // recordXPath unsupported.
1621 if (srw_req->recordXPath)
1623 yaz_add_srw_diagnostic(odr_decode(),
1624 &diagnostic, &num_diagnostic,
1627 // must have a query
1628 if (!srw_req->query.cql)
1630 yaz_add_srw_diagnostic(odr_decode(),
1631 &diagnostic, &num_diagnostic,
1635 if (srw_req->sort_type != Z_SRW_sort_type_none)
1637 yaz_add_srw_diagnostic(odr_decode(),
1638 &diagnostic, &num_diagnostic,
1642 if (srw_req->stylesheet)
1644 odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1646 // set packing for response records ..
1647 if (srw_req->recordPacking &&
1648 !strcmp(srw_req->recordPacking, "xml"))
1649 m_s2z_packing = Z_SRW_recordPacking_XML;
1651 m_s2z_packing = Z_SRW_recordPacking_string;
1655 Z_SRW_PDU *srw_pdu =
1656 yaz_srw_get(odr_encode(),
1657 Z_SRW_searchRetrieve_response);
1658 Z_SRW_searchRetrieveResponse *srw_res = srw_pdu->u.response;
1660 srw_res->diagnostics = diagnostic;
1661 srw_res->num_diagnostics = num_diagnostic;
1662 send_srw_response(srw_pdu);
1666 // prepare search PDU
1667 m_s2z_search_apdu = zget_APDU(m_s2z_odr_search,
1668 Z_APDU_searchRequest);
1669 Z_SearchRequest *z_searchRequest =
1670 m_s2z_search_apdu->u.searchRequest;
1672 z_searchRequest->num_databaseNames = 1;
1673 z_searchRequest->databaseNames = (char**)
1674 odr_malloc(m_s2z_odr_search, sizeof(char *));
1675 z_searchRequest->databaseNames[0] = odr_strdup(m_s2z_odr_search,
1678 // query transformation
1679 Z_Query *query = (Z_Query *)
1680 odr_malloc(m_s2z_odr_search, sizeof(Z_Query));
1681 z_searchRequest->query = query;
1683 if (srw_req->query_type == Z_SRW_query_type_cql)
1685 Z_External *ext = (Z_External *)
1686 odr_malloc(m_s2z_odr_search, sizeof(*ext));
1687 ext->direct_reference =
1688 odr_getoidbystr(m_s2z_odr_search, "1.2.840.10003.16.2");
1689 ext->indirect_reference = 0;
1690 ext->descriptor = 0;
1691 ext->which = Z_External_CQL;
1692 ext->u.cql = srw_req->query.cql;
1694 query->which = Z_Query_type_104;
1695 query->u.type_104 = ext;
1697 else if (srw_req->query_type == Z_SRW_query_type_pqf)
1699 Z_RPNQuery *RPNquery;
1700 YAZ_PQF_Parser pqf_parser;
1702 pqf_parser = yaz_pqf_create ();
1704 RPNquery = yaz_pqf_parse (pqf_parser, m_s2z_odr_search,
1705 srw_req->query.pqf);
1708 const char *pqf_msg;
1710 int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
1711 yaz_log(LOG_LOG, "%*s^\n", off+4, "");
1712 yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
1714 send_to_srw_client_error(10, 0);
1717 query->which = Z_Query_type_1;
1718 query->u.type_1 = RPNquery;
1720 yaz_pqf_destroy (pqf_parser);
1724 send_to_srw_client_error(7, "query");
1729 m_s2z_present_apdu = 0;
1731 if (srw_req->maximumRecords)
1732 max = *srw_req->maximumRecords;
1734 if (srw_req->startRecord)
1735 start = *srw_req->startRecord;
1738 // Some backend, such as Voyager doesn't honor piggyback
1739 // So we use present always (0 &&).
1740 if (0 && start <= 1) // Z39.50 piggyback
1742 *z_searchRequest->smallSetUpperBound = max;
1743 *z_searchRequest->mediumSetPresentNumber = max;
1744 *z_searchRequest->largeSetLowerBound = 2000000000; // 2e9
1746 z_searchRequest->preferredRecordSyntax =
1747 yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1749 if (srw_req->recordSchema)
1751 z_searchRequest->smallSetElementSetNames =
1752 z_searchRequest->mediumSetElementSetNames =
1753 mk_esn_from_schema(m_s2z_odr_search,
1754 srw_req->recordSchema);
1757 else // Z39.50 present
1759 m_s2z_present_apdu = zget_APDU(m_s2z_odr_search,
1760 Z_APDU_presentRequest);
1761 Z_PresentRequest *z_presentRequest =
1762 m_s2z_present_apdu->u.presentRequest;
1763 *z_presentRequest->resultSetStartPoint = start;
1764 *z_presentRequest->numberOfRecordsRequested = max;
1765 z_presentRequest->preferredRecordSyntax =
1766 yaz_oidval_to_z3950oid(m_s2z_odr_search, CLASS_RECSYN,
1768 z_presentRequest->recordComposition =
1769 (Z_RecordComposition *)
1770 odr_malloc(m_s2z_odr_search,
1771 sizeof(Z_RecordComposition));
1772 if (srw_req->recordSchema)
1774 z_presentRequest->recordComposition->which =
1775 Z_RecordComp_simple;
1776 z_presentRequest->recordComposition->u.simple =
1777 mk_esn_from_schema(m_s2z_odr_search,
1778 srw_req->recordSchema);
1784 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1785 Z_APDU_initRequest);
1787 // prevent m_initRequest_apdu memory from being grabbed
1788 // in Yaz_Proxy::handle_incoming_Z_PDU
1789 m_initRequest_apdu = m_s2z_init_apdu;
1790 handle_incoming_Z_PDU(m_s2z_init_apdu);
1795 handle_incoming_Z_PDU(m_s2z_search_apdu);
1799 else if (srw_pdu->which == Z_SRW_explain_request)
1801 Z_SRW_explainRequest *srw_req = srw_pdu->u.explain_request;
1803 m_s2z_database = odr_strdup(m_s2z_odr_init, srw_req->database);
1806 if (srw_req->stylesheet)
1808 odr_strdup(m_s2z_odr_init, srw_req->stylesheet);
1810 if (srw_req->recordPacking &&
1811 !strcmp(srw_req->recordPacking, "xml"))
1812 m_s2z_packing = Z_SRW_recordPacking_XML;
1814 m_s2z_packing = Z_SRW_recordPacking_string;
1818 send_srw_explain_response(diagnostic, num_diagnostic);
1824 m_s2z_init_apdu = zget_APDU(m_s2z_odr_init,
1825 Z_APDU_initRequest);
1827 // prevent m_initRequest_apdu memory from being grabbed
1828 // in Yaz_Proxy::handle_incoming_Z_PDU
1829 m_initRequest_apdu = m_s2z_init_apdu;
1830 handle_incoming_Z_PDU(m_s2z_init_apdu);
1833 send_srw_explain_response(0, 0);
1836 else if (srw_pdu->which == Z_SRW_scan_request)
1838 m_s2z_database = odr_strdup(m_s2z_odr_init,
1839 srw_pdu->u.scan_request->database);
1841 yaz_add_srw_diagnostic(odr_decode(),
1842 &diagnostic, &num_diagnostic,
1844 Z_SRW_PDU *srw_pdu =
1845 yaz_srw_get(odr_encode(),
1846 Z_SRW_scan_response);
1847 Z_SRW_scanResponse *srw_res = srw_pdu->u.scan_response;
1849 srw_res->diagnostics = diagnostic;
1850 srw_res->num_diagnostics = num_diagnostic;
1851 send_srw_response(srw_pdu);
1858 send_to_srw_client_error(4, 0);
1862 Z_GDU *p = z_get_HTTP_Response(odr_encode(), 400);
1867 void Yaz_Proxy::handle_incoming_Z_PDU(Z_APDU *apdu)
1869 if (!m_client && m_invalid_session)
1871 m_apdu_invalid_session = apdu;
1872 m_mem_invalid_session = odr_extract_mem(odr_decode());
1873 apdu = m_initRequest_apdu;
1876 // Determine our client.
1877 Z_OtherInformation **oi;
1878 get_otherInfoAPDU(apdu, &oi);
1879 m_client = get_client(apdu, get_cookie(oi), get_proxy(oi));
1885 m_client->m_server = this;
1887 if (apdu->which == Z_APDU_initRequest)
1889 if (apdu->u.initRequest->implementationId)
1890 yaz_log(LOG_LOG, "%simplementationId: %s",
1891 m_session_str, apdu->u.initRequest->implementationId);
1892 if (apdu->u.initRequest->implementationName)
1893 yaz_log(LOG_LOG, "%simplementationName: %s",
1894 m_session_str, apdu->u.initRequest->implementationName);
1895 if (apdu->u.initRequest->implementationVersion)
1896 yaz_log(LOG_LOG, "%simplementationVersion: %s",
1897 m_session_str, apdu->u.initRequest->implementationVersion);
1898 if (m_initRequest_apdu == 0)
1900 if (m_initRequest_mem)
1901 nmem_destroy(m_initRequest_mem);
1902 m_initRequest_apdu = apdu;
1903 m_initRequest_mem = odr_extract_mem(odr_decode());
1905 // save init options for the response..
1906 m_initRequest_options = apdu->u.initRequest->options;
1908 apdu->u.initRequest->options =
1909 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1910 sizeof(Odr_bitmask));
1911 ODR_MASK_ZERO(apdu->u.initRequest->options);
1913 for (i = 0; i<= 24; i++)
1914 ODR_MASK_SET(apdu->u.initRequest->options, i);
1915 ODR_MASK_CLEAR(apdu->u.initRequest->options,
1916 Z_Options_negotiationModel);
1919 m_initRequest_version = apdu->u.initRequest->protocolVersion;
1920 apdu->u.initRequest->protocolVersion =
1921 (Odr_bitmask *)nmem_malloc(m_initRequest_mem,
1922 sizeof(Odr_bitmask));
1923 ODR_MASK_ZERO(apdu->u.initRequest->protocolVersion);
1925 for (i = 0; i<= 8; i++)
1926 ODR_MASK_SET(apdu->u.initRequest->protocolVersion, i);
1928 if (m_client->m_init_flag)
1930 if (handle_init_response_for_invalid_session(apdu))
1932 Z_APDU *apdu2 = m_client->m_initResponse;
1933 apdu2->u.initResponse->otherInfo = 0;
1934 if (m_client->m_cookie && *m_client->m_cookie)
1935 set_otherInformationString(apdu2, VAL_COOKIE, 1,
1936 m_client->m_cookie);
1937 apdu2->u.initResponse->referenceId =
1938 apdu->u.initRequest->referenceId;
1939 apdu2->u.initResponse->options = m_client->m_initResponse_options;
1940 apdu2->u.initResponse->protocolVersion =
1941 m_client->m_initResponse_version;
1943 send_to_client(apdu2);
1946 m_client->m_init_flag = 1;
1948 handle_max_record_retrieve(apdu);
1951 apdu = handle_syntax_validation(apdu);
1954 apdu = handle_query_transformation(apdu);
1957 apdu = handle_query_validation(apdu);
1960 apdu = result_set_optimize(apdu);
1963 m_client->timeout(m_target_idletime); // mark it active even
1964 // though we didn't use it
1968 // delete other info part from PDU before sending to target
1969 get_otherInfoAPDU(apdu, &oi);
1973 if (apdu->which == Z_APDU_presentRequest &&
1974 m_client->m_resultSetStartPoint == 0)
1976 Z_PresentRequest *pr = apdu->u.presentRequest;
1977 m_client->m_resultSetStartPoint = *pr->resultSetStartPoint;
1978 m_client->m_cache.copy_presentRequest(apdu->u.presentRequest);
1980 m_client->m_resultSetStartPoint = 0;
1982 if (m_client->send_to_target(apdu) < 0)
1989 m_client->m_waiting = 1;
1992 void Yaz_Proxy::connectNotify()
1996 void Yaz_Proxy::shutdown()
1998 m_invalid_session = 0;
1999 // only keep if keep_alive flag is set...
2001 m_client->m_pdu_recv < m_keepalive_limit_pdu &&
2002 m_client->m_bytes_recv+m_client->m_bytes_sent < m_keepalive_limit_bw &&
2003 m_client->m_waiting == 0)
2005 yaz_log(LOG_LOG, "%sShutdown (client to proxy) keepalive %s",
2007 m_client->get_hostname());
2008 yaz_log(LOG_LOG, "%sbw=%d pdu=%d limit-bw=%d limit-pdu=%d",
2009 m_session_str, m_client->m_pdu_recv,
2010 m_client->m_bytes_sent + m_client->m_bytes_recv,
2011 m_keepalive_limit_bw, m_keepalive_limit_pdu);
2012 assert (m_client->m_waiting != 2);
2013 // Tell client (if any) that no server connection is there..
2014 m_client->m_server = 0;
2015 m_invalid_session = 0;
2019 yaz_log (LOG_LOG, "%sShutdown (client to proxy) close %s",
2021 m_client->get_hostname());
2022 assert (m_client->m_waiting != 2);
2027 yaz_log (LOG_LOG, "%sshutdown (client to proxy) bad state",
2033 yaz_log (LOG_LOG, "%sShutdown (client to proxy)",
2037 m_parent->pre_init();
2041 const char *Yaz_ProxyClient::get_session_str()
2045 return m_server->get_session_str();
2048 void Yaz_ProxyClient::shutdown()
2050 yaz_log (LOG_LOG, "%sShutdown (proxy to target) %s", get_session_str(),
2056 void Yaz_Proxy::failNotify()
2059 yaz_log (LOG_LOG, "%sConnection closed by client",
2064 void Yaz_ProxyClient::failNotify()
2067 m_server->inc_request_no();
2068 yaz_log (LOG_LOG, "%sConnection closed by target %s",
2069 get_session_str(), get_hostname());
2073 void Yaz_ProxyClient::connectNotify()
2075 const char *s = get_session_str();
2076 const char *h = get_hostname();
2077 yaz_log (LOG_LOG, "%sConnection accepted by %s timeout=%d", s, h,
2079 timeout(m_target_idletime);
2084 IYaz_PDU_Observer *Yaz_ProxyClient::sessionNotify(IYaz_PDU_Observable
2085 *the_PDU_Observable, int fd)
2087 return new Yaz_ProxyClient(the_PDU_Observable, 0);
2090 Yaz_ProxyClient::~Yaz_ProxyClient()
2095 m_next->m_prev = m_prev;
2096 m_waiting = 2; // for debugging purposes only.
2097 odr_destroy(m_init_odr);
2098 delete m_last_query;
2099 xfree (m_last_resultSetId);
2103 void Yaz_ProxyClient::pre_init_client()
2105 Z_APDU *apdu = create_Z_PDU(Z_APDU_initRequest);
2106 Z_InitRequest *req = apdu->u.initRequest;
2109 for (i = 0; i<= 24; i++)
2110 ODR_MASK_SET(req->options, i);
2111 ODR_MASK_CLEAR(apdu->u.initRequest->options,
2112 Z_Options_negotiationModel);
2113 for (i = 0; i<= 10; i++)
2114 ODR_MASK_SET(req->protocolVersion, i);
2116 if (send_to_target(apdu) < 0)
2127 void Yaz_Proxy::pre_init()
2130 const char *name = 0;
2131 const char *zurl_in_use[MAX_ZURL_PLEX];
2132 int limit_bw, limit_pdu, limit_req;
2133 int target_idletime, client_idletime;
2135 int keepalive_limit_bw, keepalive_limit_pdu;
2137 const char *cql2rpn = 0;
2139 Yaz_ProxyConfig *cfg = check_reconfigure();
2143 if (m_log_mask & PROXY_LOG_APDU_CLIENT)
2148 for (i = 0; cfg && cfg->get_target_no(i, &name, zurl_in_use,
2149 &limit_bw, &limit_pdu, &limit_req,
2150 &target_idletime, &client_idletime,
2152 &keepalive_limit_bw,
2153 &keepalive_limit_pdu,
2160 for (j = 0; zurl_in_use[j]; j++)
2166 for (c = m_clientPool; c; c = c->m_next)
2168 if (!strcmp(zurl_in_use[j], c->get_hostname()))
2170 if (c->m_cookie == 0)
2172 if (c->m_server == 0)
2181 yaz_log(LOG_LOG, "%spre-init %s %s use=%d other=%d spare=%d "
2182 "preinit=%d",m_session_str,
2183 name, zurl_in_use[j], in_use, other, spare, pre_init);
2184 if (spare < pre_init)
2186 c = new Yaz_ProxyClient(m_PDU_Observable->clone(), this);
2187 c->m_next = m_clientPool;
2189 c->m_next->m_prev = &c->m_next;
2191 c->m_prev = &m_clientPool;
2193 if (m_log_mask & PROXY_LOG_APDU_SERVER)
2194 c->set_APDU_yazlog(1);
2196 c->set_APDU_yazlog(0);
2198 if (c->client(zurl_in_use[j]))
2206 c->m_target_idletime = target_idletime;
2207 c->m_seqno = m_seqno++;
2214 void Yaz_Proxy::timeoutNotify()
2220 timeout(m_client_idletime);
2221 Z_GDU *apdu = m_bw_hold_PDU;
2224 if (apdu->which == Z_GDU_Z3950)
2225 handle_incoming_Z_PDU(apdu->u.z3950);
2226 else if (apdu->which == Z_GDU_HTTP_Request)
2227 handle_incoming_HTTP(apdu->u.HTTP_Request);
2233 yaz_log (LOG_LOG, "%sTimeout (client to proxy)", m_session_str);
2244 void Yaz_Proxy::markInvalid()
2247 m_invalid_session = 1;
2250 void Yaz_ProxyClient::timeoutNotify()
2253 m_server->inc_request_no();
2255 yaz_log (LOG_LOG, "%sTimeout (proxy to target) %s", get_session_str(),
2259 if (m_server && m_init_flag)
2261 // target timed out in a session that was properly initialized
2262 // server object stay alive but we mark it as invalid so it
2263 // gets initialized again
2264 m_server->markInvalid();
2270 Yaz_ProxyClient::Yaz_ProxyClient(IYaz_PDU_Observable *the_PDU_Observable,
2271 Yaz_Proxy *parent) :
2272 Yaz_Z_Assoc (the_PDU_Observable)
2279 m_last_resultSetId = 0;
2280 m_last_resultCount = 0;
2284 m_init_odr = odr_createmem (ODR_DECODE);
2286 m_initResponse_options = 0;
2287 m_initResponse_version = 0;
2288 m_resultSetStartPoint = 0;
2289 m_bytes_sent = m_bytes_recv = 0;
2293 m_target_idletime = 600;
2297 const char *Yaz_Proxy::option(const char *name, const char *value)
2299 if (!strcmp (name, "optimize")) {
2302 m_optimize = xstrdup (value);
2309 void Yaz_ProxyClient::recv_HTTP_response(Z_HTTP_Response *apdu, int len)
2314 void Yaz_ProxyClient::recv_GDU(Z_GDU *apdu, int len)
2316 if (apdu->which == Z_GDU_Z3950)
2317 recv_Z_PDU(apdu->u.z3950, len);
2318 else if (apdu->which == Z_GDU_HTTP_Response)
2319 recv_HTTP_response(apdu->u.HTTP_Response, len);
2324 int Yaz_Proxy::handle_init_response_for_invalid_session(Z_APDU *apdu)
2326 if (!m_invalid_session)
2328 m_invalid_session = 0;
2329 handle_incoming_Z_PDU(m_apdu_invalid_session);
2330 assert (m_mem_invalid_session);
2331 nmem_destroy(m_mem_invalid_session);
2332 m_mem_invalid_session = 0;
2336 void Yaz_ProxyClient::recv_Z_PDU(Z_APDU *apdu, int len)
2338 m_bytes_recv += len;
2341 if (m_root->get_log_mask() & PROXY_LOG_REQ_SERVER)
2342 yaz_log (LOG_LOG, "%sReceiving %s from %s %d bytes", get_session_str(),
2343 apdu_name(apdu), get_hostname(), len);
2344 if (apdu->which == Z_APDU_initResponse)
2346 if (!m_server) // if this is a pre init session , check for more
2348 NMEM nmem = odr_extract_mem (odr_decode());
2349 odr_reset (m_init_odr);
2350 nmem_transfer (m_init_odr->mem, nmem);
2351 m_initResponse = apdu;
2352 m_initResponse_options = apdu->u.initResponse->options;
2353 m_initResponse_version = apdu->u.initResponse->protocolVersion;
2355 Z_InitResponse *ir = apdu->u.initResponse;
2356 char *im0 = ir->implementationName;
2359 odr_malloc(m_init_odr, 20 + (im0 ? strlen(im0) : 0));
2366 strcat(im1, "(YAZ Proxy)");
2367 ir->implementationName = im1;
2369 nmem_destroy (nmem);
2371 if (m_server && m_server->handle_init_response_for_invalid_session(apdu))
2374 if (apdu->which == Z_APDU_searchResponse)
2376 Z_SearchResponse *sr = apdu->u.searchResponse;
2377 m_last_resultCount = *sr->resultCount;
2378 int status = *sr->searchStatus;
2379 if (status && (!sr->records || sr->records->which == Z_Records_DBOSD))
2383 if (sr->records && sr->records->which == Z_Records_DBOSD)
2385 m_cache.add(odr_decode(),
2386 sr->records->u.databaseOrSurDiagnostics, 1,
2391 if (apdu->which == Z_APDU_presentResponse)
2393 Z_PresentResponse *pr = apdu->u.presentResponse;
2397 Z_APDU *new_apdu = create_Z_PDU(Z_APDU_searchResponse);
2398 Z_SearchResponse *sr = new_apdu->u.searchResponse;
2399 sr->referenceId = pr->referenceId;
2400 *sr->resultCount = m_last_resultCount;
2401 sr->records = pr->records;
2402 sr->nextResultSetPosition = pr->nextResultSetPosition;
2403 sr->numberOfRecordsReturned = pr->numberOfRecordsReturned;
2407 pr->records->which == Z_Records_DBOSD && m_resultSetStartPoint)
2409 m_cache.add(odr_decode(),
2410 pr->records->u.databaseOrSurDiagnostics,
2411 m_resultSetStartPoint, -1);
2412 m_resultSetStartPoint = 0;
2416 set_otherInformationString (apdu, VAL_COOKIE, 1, m_cookie);
2419 m_server->send_to_client(apdu);
2421 if (apdu->which == Z_APDU_close)
2427 int Yaz_Proxy::server(const char *addr)
2429 int r = Yaz_Z_Assoc::server(addr);
2432 yaz_log(LOG_LOG, "%sStarted proxy " VERSION " on %s", m_session_str, addr);