1 /* This file is part of Metaproxy.
2 Copyright (C) Index Data
4 Metaproxy is free software; you can redistribute it and/or modify it under
5 the terms of the GNU General Public License as published by the Free
6 Software Foundation; either version 2, or (at your option) any later
9 Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
19 #include <metaproxy/package.hpp>
20 #include <metaproxy/util.hpp>
23 #include <yaz/diagbib1.h>
24 #include <yaz/match_glob.h>
25 #include <boost/scoped_ptr.hpp>
26 #include <boost/thread/mutex.hpp>
27 #include <boost/thread/condition.hpp>
32 namespace mp = metaproxy_1;
33 namespace yf = mp::filter;
35 namespace metaproxy_1 {
37 class SPARQL : public Base {
44 typedef boost::shared_ptr<Session> SessionPtr;
45 typedef boost::shared_ptr<Conf> ConfPtr;
47 typedef boost::shared_ptr<FrontendSet> FrontendSetPtr;
48 typedef std::map<std::string,FrontendSetPtr> FrontendSets;
52 void process(metaproxy_1::Package & package) const;
53 void configure(const xmlNode * ptr, bool test_only,
55 SessionPtr get_session(Package &package, Z_APDU **apdu) const;
56 void release_session(Package &package) const;
57 boost::scoped_ptr<Rep> m_p;
58 std::list<ConfPtr> db_conf;
70 boost::condition m_cond_session_ready;
72 std::map<mp::Session,SessionPtr> m_clients;
74 class SPARQL::Result {
79 friend class FrontendSet;
84 class SPARQL::FrontendSet {
89 std::list<Result> results;
91 class SPARQL::Session {
93 Session(const SPARQL *);
95 void handle_z(Package &package, Z_APDU *apdu);
96 Z_APDU *search(mp::Package &package,
99 const char *sparql_query,
100 ConfPtr conf, FrontendSetPtr fset);
101 int invoke_sparql(mp::Package &package,
102 const char *sparql_query,
109 ODR odr, Odr_oid *preferredRecordSyntax,
110 Z_ElementSetNames *esn,
111 int start, int number, int &error_code, std::string &addinfo,
112 int *number_returned, int *next_position);
115 bool m_support_named_result_sets;
116 FrontendSets m_frontend_sets;
117 const SPARQL *m_sparql;
122 yf::SPARQL::Result::~Result()
128 yf::SPARQL::Result::Result()
133 yf::SPARQL::SPARQL() : m_p(new Rep)
137 yf::SPARQL::~SPARQL()
141 void yf::SPARQL::configure(const xmlNode *xmlnode, bool test_only,
144 const xmlNode *ptr = xmlnode->children;
147 for (; ptr; ptr = ptr->next)
149 if (ptr->type != XML_ELEMENT_NODE)
151 if (!strcmp((const char *) ptr->name, "defaults"))
153 const struct _xmlAttr *attr;
154 for (attr = ptr->properties; attr; attr = attr->next)
156 if (!strcmp((const char *) attr->name, "uri"))
157 uri = mp::xml::get_text(attr->children);
159 throw mp::filter::FilterException(
160 "Bad attribute " + std::string((const char *)
164 else if (!strcmp((const char *) ptr->name, "db"))
166 yaz_sparql_t s = yaz_sparql_create();
167 ConfPtr conf(new Conf);
171 const struct _xmlAttr *attr;
172 for (attr = ptr->properties; attr; attr = attr->next)
174 if (!strcmp((const char *) attr->name, "path"))
175 conf->db = mp::xml::get_text(attr->children);
176 else if (!strcmp((const char *) attr->name, "uri"))
177 conf->uri = mp::xml::get_text(attr->children);
178 else if (!strcmp((const char *) attr->name, "schema"))
179 conf->schema = mp::xml::get_text(attr->children);
181 throw mp::filter::FilterException(
182 "Bad attribute " + std::string((const char *)
185 xmlNode *p = ptr->children;
186 for (; p; p = p->next)
188 if (p->type != XML_ELEMENT_NODE)
190 std::string name = (const char *) p->name;
191 const struct _xmlAttr *attr;
192 for (attr = p->properties; attr; attr = attr->next)
194 if (!strcmp((const char *) attr->name, "type"))
197 name.append(mp::xml::get_text(attr->children));
200 throw mp::filter::FilterException(
201 "Bad attribute " + std::string((const char *)
204 std::string value = mp::xml::get_text(p);
205 if (yaz_sparql_add_pattern(s, name.c_str(), value.c_str()))
207 throw mp::filter::FilterException(
208 "Bad SPARQL config " + name);
211 if (!conf->uri.length())
213 throw mp::filter::FilterException("Missing uri");
215 if (!conf->db.length())
217 throw mp::filter::FilterException("Missing path");
219 db_conf.push_back(conf);
223 throw mp::filter::FilterException
225 + std::string((const char *) ptr->name)
226 + " in sparql filter");
231 yf::SPARQL::Conf::~Conf()
233 yaz_sparql_destroy(s);
236 yf::SPARQL::Session::Session(const SPARQL *sparql) :
238 m_support_named_result_sets(false),
243 yf::SPARQL::Session::~Session()
247 yf::SPARQL::SessionPtr yf::SPARQL::get_session(Package & package,
252 Z_GDU *gdu = package.request().get();
254 boost::mutex::scoped_lock lock(m_p->m_mutex);
256 std::map<mp::Session,SPARQL::SessionPtr>::iterator it;
258 if (gdu && gdu->which == Z_GDU_Z3950)
259 *apdu = gdu->u.z3950;
265 it = m_p->m_clients.find(package.session());
266 if (it == m_p->m_clients.end())
268 if (!it->second->m_in_use)
270 it->second->m_in_use = true;
273 m_p->m_cond_session_ready.wait(lock);
278 // new Z39.50 session ..
279 SessionPtr p(new Session(this));
280 m_p->m_clients[package.session()] = p;
284 void yf::SPARQL::release_session(Package &package) const
286 boost::mutex::scoped_lock lock(m_p->m_mutex);
287 std::map<mp::Session,SessionPtr>::iterator it;
289 it = m_p->m_clients.find(package.session());
290 if (it != m_p->m_clients.end())
292 it->second->m_in_use = false;
294 if (package.session().is_closed())
295 m_p->m_clients.erase(it);
296 m_p->m_cond_session_ready.notify_all();
300 static bool get_result(xmlDoc *doc, Odr_int *sz, Odr_int pos, xmlDoc **ndoc)
302 xmlNode *ptr = xmlDocGetRootElement(doc);
307 *ndoc = xmlNewDoc(BAD_CAST "1.0");
309 if (ptr->type == XML_ELEMENT_NODE &&
310 !strcmp((const char *) ptr->name, "RDF"))
314 q0 = xmlCopyNode(ptr, 2);
315 xmlDocSetRootElement(*ndoc, q0);
319 while (ptr && ptr->type != XML_ELEMENT_NODE)
321 if (ptr && ptr->type == XML_ELEMENT_NODE &&
322 !strcmp((const char *) ptr->name, "Description"))
324 xmlNode *p = ptr->children;
326 while (p && p->type != XML_ELEMENT_NODE)
328 if (p && p->type == XML_ELEMENT_NODE &&
329 !strcmp((const char *) p->name, "type"))
330 { /* SELECT RESULT */
331 for (ptr = ptr->children; ptr; ptr = ptr->next)
332 if (ptr->type == XML_ELEMENT_NODE &&
333 !strcmp((const char *) ptr->name, "solution"))
339 xmlNode *q1 = xmlCopyNode(ptr, 1);
347 { /* CONSTRUCT result */
348 for (; ptr; ptr = ptr->next)
349 if (ptr->type == XML_ELEMENT_NODE &&
350 !strcmp((const char *) ptr->name, "Description"))
356 xmlNode *q1 = xmlCopyNode(ptr, 1);
367 for (; ptr; ptr = ptr->next)
368 if (ptr->type == XML_ELEMENT_NODE &&
369 !strcmp((const char *) ptr->name, "sparql"))
375 q0 = xmlCopyNode(ptr, 2);
376 xmlDocSetRootElement(*ndoc, q0);
378 for (ptr = ptr->children; ptr; ptr = ptr->next)
379 if (ptr->type == XML_ELEMENT_NODE &&
380 !strcmp((const char *) ptr->name, "results"))
388 q1 = xmlCopyNode(ptr, 0);
391 for (ptr = ptr->children; ptr; ptr = ptr->next)
392 if (ptr->type == XML_ELEMENT_NODE &&
393 !strcmp((const char *) ptr->name, "result"))
399 xmlNode *q2 = xmlCopyNode(ptr, 1);
412 Z_Records *yf::SPARQL::Session::fetch(
415 ODR odr, Odr_oid *preferredRecordSyntax,
416 Z_ElementSetNames *esn,
417 int start, int number, int &error_code, std::string &addinfo,
418 int *number_returned, int *next_position)
420 Z_Records *rec = (Z_Records *) odr_malloc(odr, sizeof(Z_Records));
421 std::list<Result>::iterator it = fset->results.begin();
422 const char *schema = 0;
423 bool uri_lookup = false;
424 if (esn && esn->which == Z_ElementSetNames_generic)
425 schema = esn->u.generic;
427 for (; it != fset->results.end(); it++)
429 if (schema && !strcmp(esn->u.generic, it->conf->schema.c_str()))
431 if (yaz_sparql_lookup_schema(it->conf->s, schema))
437 if (it == fset->results.end())
439 rec->which = Z_Records_NSD;
440 rec->u.nonSurrogateDiagnostic =
441 zget_DefaultDiagFormat(
443 YAZ_BIB1_SPECIFIED_ELEMENT_SET_NAME_NOT_VALID_FOR_SPECIFIED_,
447 rec->which = Z_Records_DBOSD;
448 rec->u.databaseOrSurDiagnostics = (Z_NamePlusRecordList *)
449 odr_malloc(odr, sizeof(Z_NamePlusRecordList));
450 rec->u.databaseOrSurDiagnostics->records = (Z_NamePlusRecord **)
451 odr_malloc(odr, sizeof(Z_NamePlusRecord *) * number);
453 for (i = 0; i < number; i++)
455 rec->u.databaseOrSurDiagnostics->records[i] = (Z_NamePlusRecord *)
456 odr_malloc(odr, sizeof(Z_NamePlusRecord));
457 Z_NamePlusRecord *npr = rec->u.databaseOrSurDiagnostics->records[i];
458 npr->databaseName = odr_strdup(odr, fset->db.c_str());
459 npr->which = Z_NamePlusRecord_databaseRecord;
462 if (!get_result(it->doc, 0, start - 1 + i, &ndoc))
468 xmlNode *ndoc_root = xmlDocGetRootElement(ndoc);
477 xmlNode *n = ndoc_root;
480 if (n->type == XML_ELEMENT_NODE)
482 if (!strcmp((const char *) n->name, "uri"))
484 uri = mp::xml::get_text(n->children);
494 rec->which = Z_Records_NSD;
495 rec->u.nonSurrogateDiagnostic =
496 zget_DefaultDiagFormat(
498 YAZ_BIB1_SYSTEM_ERROR_IN_PRESENTING_RECORDS, 0);
504 mp::wrbuf addinfo, query, w;
505 int error = yaz_sparql_from_uri_wrbuf(it->conf->s,
507 uri.c_str(), schema);
510 yaz_log(YLOG_LOG, "query=%s", query.c_str());
511 error = invoke_sparql(package, query.c_str(),
516 rec->which = Z_Records_NSD;
517 rec->u.nonSurrogateDiagnostic =
518 zget_DefaultDiagFormat(
521 addinfo.len() ? addinfo.c_str() : 0);
525 npr->u.databaseRecord =
526 z_ext_record_xml(odr, w.c_str(), w.len());
531 xmlBufferPtr buf = xmlBufferCreate();
532 xmlNodeDump(buf, ndoc, ndoc_root, 0, 0);
533 yaz_log(YLOG_LOG, "record %s %.*s", uri_lookup ? "uri" : "normal",
534 (int) buf->use, (const char *) buf->content);
535 npr->u.databaseRecord =
536 z_ext_record_xml(odr, (const char *) buf->content, buf->use);
541 rec->u.databaseOrSurDiagnostics->num_records = i;
542 *number_returned = i;
543 if (start + number > fset->hits)
546 *next_position = start + number;
550 int yf::SPARQL::Session::invoke_sparql(mp::Package &package,
551 const char *sparql_query,
555 Package http_package(package.session(), package.origin());
558 http_package.copy_filter(package);
559 Z_GDU *gdu = z_get_HTTP_Request_uri(odr, conf->uri.c_str(), 0, 1);
561 z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
562 "Content-Type", "application/x-www-form-urlencoded");
563 z_HTTP_header_add(odr, &gdu->u.HTTP_Request->headers,
564 "Accept", "application/sparql-results+xml,"
565 "application/rdf+xml");
566 const char *names[2];
569 const char *values[1];
570 values[0] = sparql_query;
572 yaz_array_to_uri(&path, odr, (char **) names, (char **) values);
574 gdu->u.HTTP_Request->content_buf = path;
575 gdu->u.HTTP_Request->content_len = strlen(path);
577 yaz_log(YLOG_LOG, "sparql: HTTP request\n%s", sparql_query);
579 http_package.request() = gdu;
582 Z_GDU *gdu_resp = http_package.response().get();
584 if (!gdu_resp || gdu_resp->which != Z_GDU_HTTP_Response)
586 wrbuf_puts(w, "no HTTP response from backend");
587 return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
589 else if (gdu_resp->u.HTTP_Response->code != 200)
591 wrbuf_printf(w, "sparql: HTTP error %d from backend",
592 gdu_resp->u.HTTP_Response->code);
593 return YAZ_BIB1_TEMPORARY_SYSTEM_ERROR;
595 Z_HTTP_Response *resp = gdu_resp->u.HTTP_Response;
596 wrbuf_write(w, resp->content_buf, resp->content_len);
600 Z_APDU *yf::SPARQL::Session::search(mp::Package &package,
603 const char *sparql_query,
604 ConfPtr conf, FrontendSetPtr fset)
606 Z_SearchRequest *req = apdu_req->u.searchRequest;
607 Z_APDU *apdu_res = 0;
610 int error = invoke_sparql(package, sparql_query, conf, w);
613 apdu_res = odr.create_searchResponse(apdu_req, error,
619 xmlDocPtr doc = xmlParseMemory(w.c_str(), w.len());
622 apdu_res = odr.create_searchResponse(
624 YAZ_BIB1_TEMPORARY_SYSTEM_ERROR,
625 "invalid XML from backendbackend");
630 Z_Records *records = 0;
631 int number_returned = 0;
632 int next_position = 0;
638 fset->results.push_back(result);
639 yaz_log(YLOG_LOG, "saving sparql result xmldoc=%p", doc);
641 get_result(result.doc, &fset->hits, -1, 0);
642 m_frontend_sets[req->resultSetName] = fset;
647 const char *element_set_name = 0;
648 mp::util::piggyback_sr(req, fset->hits, number, &element_set_name);
651 Z_ElementSetNames *esn;
653 if (number > *req->smallSetUpperBound)
654 esn = req->mediumSetElementSetNames;
656 esn = req->smallSetElementSetNames;
657 records = fetch(package, fset,
658 odr, req->preferredRecordSyntax, esn,
667 odr.create_searchResponse(
668 apdu_req, error_code, addinfo.c_str());
673 odr.create_searchResponse(apdu_req, 0, 0);
674 Z_SearchResponse *resp = apdu_res->u.searchResponse;
675 *resp->resultCount = fset->hits;
676 *resp->numberOfRecordsReturned = number_returned;
677 *resp->nextResultSetPosition = next_position;
678 resp->records = records;
685 void yf::SPARQL::Session::handle_z(mp::Package &package, Z_APDU *apdu_req)
688 Z_APDU *apdu_res = 0;
689 if (apdu_req->which == Z_APDU_initRequest)
691 apdu_res = odr.create_initResponse(apdu_req, 0, 0);
692 Z_InitRequest *req = apdu_req->u.initRequest;
693 Z_InitResponse *resp = apdu_res->u.initResponse;
695 resp->implementationName = odr_strdup(odr, "sparql");
696 if (ODR_MASK_GET(req->options, Z_Options_namedResultSets))
697 m_support_named_result_sets = true;
699 static const int masks[] = {
700 Z_Options_search, Z_Options_present,
701 Z_Options_namedResultSets, -1
703 for (i = 0; masks[i] != -1; i++)
704 if (ODR_MASK_GET(req->options, masks[i]))
705 ODR_MASK_SET(resp->options, masks[i]);
706 static const int versions[] = {
712 for (i = 0; versions[i] != -1; i++)
713 if (ODR_MASK_GET(req->protocolVersion, versions[i]))
714 ODR_MASK_SET(resp->protocolVersion, versions[i]);
717 *resp->preferredMessageSize = *req->preferredMessageSize;
718 *resp->maximumRecordSize = *req->maximumRecordSize;
720 else if (apdu_req->which == Z_APDU_close)
722 apdu_res = odr.create_close(apdu_req,
723 Z_Close_finished, 0);
724 package.session().close();
726 else if (apdu_req->which == Z_APDU_searchRequest)
728 Z_SearchRequest *req = apdu_req->u.searchRequest;
730 FrontendSets::iterator fset_it =
731 m_frontend_sets.find(req->resultSetName);
732 if (fset_it != m_frontend_sets.end())
734 // result set already exist
735 // if replace indicator is off: we return diagnostic if
736 // result set already exist.
737 if (*req->replaceIndicator == 0)
740 odr.create_searchResponse(
742 YAZ_BIB1_RESULT_SET_EXISTS_AND_REPLACE_INDICATOR_OFF,
744 package.response() = apdu;
746 m_frontend_sets.erase(fset_it);
748 if (req->query->which != Z_Query_type_1)
750 apdu_res = odr.create_searchResponse(
751 apdu_req, YAZ_BIB1_QUERY_TYPE_UNSUPP, 0);
753 else if (req->num_databaseNames != 1)
755 apdu_res = odr.create_searchResponse(
757 YAZ_BIB1_ACCESS_TO_SPECIFIED_DATABASE_DENIED, 0);
761 std::string db = req->databaseNames[0];
762 std::list<ConfPtr>::const_iterator it;
763 FrontendSetPtr fset(new FrontendSet);
765 m_frontend_sets.erase(req->resultSetName);
767 it = m_sparql->db_conf.begin();
768 for (; it != m_sparql->db_conf.end(); it++)
769 if (yaz_match_glob((*it)->db.c_str(), db.c_str()))
771 mp::wrbuf addinfo_wr;
774 yaz_sparql_from_rpn_wrbuf((*it)->s,
775 addinfo_wr, sparql_wr,
776 req->query->u.type_1);
779 apdu_res = odr.create_searchResponse(
781 addinfo_wr.len() ? addinfo_wr.c_str() : 0);
785 Z_APDU *apdu_1 = search(package, apdu_req, odr,
786 sparql_wr.c_str(), *it,
794 apdu_res = odr.create_searchResponse(
795 apdu_req, YAZ_BIB1_DATABASE_DOES_NOT_EXIST, db.c_str());
799 else if (apdu_req->which == Z_APDU_presentRequest)
801 Z_PresentRequest *req = apdu_req->u.presentRequest;
802 FrontendSets::iterator fset_it =
803 m_frontend_sets.find(req->resultSetId);
804 if (fset_it == m_frontend_sets.end())
807 odr.create_presentResponse(
808 apdu_req, YAZ_BIB1_SPECIFIED_RESULT_SET_DOES_NOT_EXIST,
810 package.response() = apdu_res;
813 int number_returned = 0;
814 int next_position = 0;
817 Z_ElementSetNames *esn = 0;
818 if (req->recordComposition)
820 if (req->recordComposition->which == Z_RecordComp_simple)
821 esn = req->recordComposition->u.simple;
825 odr.create_presentResponse(
827 YAZ_BIB1_ONLY_A_SINGLE_ELEMENT_SET_NAME_SUPPORTED,
829 package.response() = apdu_res;
833 Z_Records *records = fetch(
836 odr, req->preferredRecordSyntax, esn,
837 *req->resultSetStartPoint, *req->numberOfRecordsRequested,
844 odr.create_presentResponse(apdu_req, error_code,
850 odr.create_presentResponse(apdu_req, 0, 0);
851 Z_PresentResponse *resp = apdu_res->u.presentResponse;
852 resp->records = records;
853 *resp->numberOfRecordsReturned = number_returned;
854 *resp->nextResultSetPosition = next_position;
859 apdu_res = odr.create_close(apdu_req,
860 Z_Close_protocolError,
861 "sparql: unhandled APDU");
862 package.session().close();
866 package.response() = apdu_res;
869 void yf::SPARQL::process(mp::Package &package) const
872 SessionPtr p = get_session(package, &apdu);
875 p->handle_z(package, apdu);
879 release_session(package);
882 static mp::filter::Base* filter_creator()
884 return new mp::filter::SPARQL;
888 struct metaproxy_1_filter_struct metaproxy_1_filter_sparql = {
899 * c-file-style: "Stroustrup"
900 * indent-tabs-mode: nil
902 * vim: shiftwidth=4 tabstop=8 expandtab