New filter present_chunk
authorAdam Dickmeiss <adam@indexdata.dk>
Fri, 19 Apr 2013 14:42:44 +0000 (16:42 +0200)
committerAdam Dickmeiss <adam@indexdata.dk>
Fri, 19 Apr 2013 14:42:44 +0000 (16:42 +0200)
This filter allows splitting of present requests into smaller
chunks.

etc/config1.xml
src/Makefile.am
src/factory_static.cpp
src/filter_present_chunk.cpp [new file with mode: 0644]
src/filter_present_chunk.hpp [new file with mode: 0644]
xml/schema/filter_present_chunk.rnc [new file with mode: 0644]
xml/schema/metaproxy.rnc

index dd91ae4..d7ff628 100644 (file)
@@ -22,6 +22,9 @@
       <filter type="log">
         <message>log</message>
       </filter>
+      <filter type="present_chunk">
+        <chunk>2</chunk>
+      </filter>
       <filter refid="backend"/>
       <filter type="bounce"/>
     </route>
index 04e2f81..5c1a67d 100644 (file)
@@ -35,6 +35,7 @@ libmetaproxy_la_SOURCES = \
        filter_log.cpp filter_log.hpp \
        filter_multi.cpp filter_multi.hpp \
        filter_query_rewrite.cpp filter_query_rewrite.hpp \
+       filter_present_chunk.cpp filter_present_chunk.hpp \
        filter_record_transform.cpp filter_record_transform.hpp \
        filter_session_shared.cpp filter_session_shared.hpp \
        filter_sort.cpp filter_sort.hpp \
@@ -57,10 +58,11 @@ libmetaproxy_la_SOURCES = \
        torus.cpp torus.hpp \
        url_recipe.cpp \
        util.cpp \
-       xmlutil.cpp 
+       xmlutil.cpp
 
 
-libmetaproxy_la_LIBADD = $(YAZPPLALIB) $(BOOST_LIB) $(BOOST_THREAD_LIB) $(BOOST_REGEX_LIB)
+libmetaproxy_la_LIBADD = $(YAZPPLALIB) $(BOOST_LIB) $(BOOST_THREAD_LIB) \
+       $(BOOST_REGEX_LIB)
 
 # Rules for lib
 
@@ -159,9 +161,3 @@ test_ses_map_LDADD = $(TESTLDADD)
 test_thread_pool_observer_LDADD = $(TESTLDADD)
 test_xmlutil_LDADD = $(TESTLDADD)
 
-# doxygen target
-dox:
-       (cd $(top_srcdir) ; make dox) 
-
-showdox:
-       (cd $(top_srcdir) ; make showdox) 
index 3942637..1d0e5a8 100644 (file)
@@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 #include "filter_load_balance.hpp"
 #include "filter_log.hpp"
 #include "filter_multi.hpp"
+#include "filter_present_chunk.hpp"
 #include "filter_query_rewrite.hpp"
 #include "filter_record_transform.hpp"
 #include "filter_session_shared.hpp"
@@ -71,6 +72,7 @@ mp::FactoryStatic::FactoryStatic()
         &metaproxy_1_filter_load_balance,
         &metaproxy_1_filter_log,
         &metaproxy_1_filter_multi,
+        &metaproxy_1_filter_present_chunk,
         &metaproxy_1_filter_query_rewrite,
         &metaproxy_1_filter_record_transform,
         &metaproxy_1_filter_session_shared,
diff --git a/src/filter_present_chunk.cpp b/src/filter_present_chunk.cpp
new file mode 100644 (file)
index 0000000..c2dc859
--- /dev/null
@@ -0,0 +1,211 @@
+/* This file is part of Metaproxy.
+   Copyright (C) 2005-2013 Index Data
+
+Metaproxy is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#include "config.hpp"
+#include "filter_present_chunk.hpp"
+
+#include <time.h>
+#include <yaz/log.h>
+#include <yaz/copy_types.h>
+#include <metaproxy/package.hpp>
+#include <metaproxy/util.hpp>
+
+namespace mp = metaproxy_1;
+namespace yf = mp::filter;
+
+namespace metaproxy_1 {
+    namespace filter {
+        class PresentChunk::Impl {
+        public:
+            Impl();
+            ~Impl();
+            void process(metaproxy_1::Package & package);
+            void configure(const xmlNode * ptr);
+            void chunk_it(metaproxy_1::Package & package, Z_APDU *apdu);
+        private:
+            Odr_int chunk_number;
+        };
+    }
+}
+
+yf::PresentChunk::PresentChunk() : m_p(new Impl)
+{
+}
+
+yf::PresentChunk::~PresentChunk()
+{  // must have a destructor because of boost::scoped_ptr
+}
+
+void yf::PresentChunk::configure(const xmlNode *xmlnode, bool test_only,
+                          const char *path)
+{
+    m_p->configure(xmlnode);
+}
+
+void yf::PresentChunk::process(mp::Package &package) const
+{
+    m_p->process(package);
+}
+
+yf::PresentChunk::Impl::Impl() : chunk_number(0)
+{
+}
+
+yf::PresentChunk::Impl::~Impl()
+{
+}
+
+void yf::PresentChunk::Impl::configure(const xmlNode *ptr)
+{
+    for (ptr = ptr->children; ptr; ptr = ptr->next)
+    {
+        if (ptr->type != XML_ELEMENT_NODE)
+            continue;
+        if (!strcmp((const char *) ptr->name, "chunk"))
+        {
+            chunk_number = mp::xml::get_int(ptr, 0);
+        }
+        else
+        {
+            throw mp::filter::FilterException("Bad element "
+                                               + std::string((const char *)
+                                                             ptr->name));
+        }
+    }
+}
+
+void yf::PresentChunk::Impl::chunk_it(mp::Package &package,
+                                      Z_APDU *apdu)
+{
+    mp::odr odr;
+    Z_PresentRequest *pr = apdu->u.presentRequest;
+
+    Odr_int total = *pr->numberOfRecordsRequested;
+    Odr_int start = *pr->resultSetStartPoint;
+    Odr_int offset = 0;
+    Z_NamePlusRecordList *npl = (Z_NamePlusRecordList *)
+        odr_malloc(odr, sizeof(*npl));
+    npl->num_records = total;
+    npl->records = (Z_NamePlusRecord **)
+        odr_malloc(odr, sizeof(*npl->records) * total);
+    while (offset < total)
+    {
+        Odr_int left = total - offset;
+
+        Package pp(package.session(), package.origin());
+
+        *pr->numberOfRecordsRequested =
+            left > chunk_number ? chunk_number : left;
+
+        *pr->resultSetStartPoint = start + offset;
+
+        pp.copy_filter(package);
+        pp.request() = apdu;
+
+        pp.move();
+
+        if (pp.session().is_closed())
+        {
+            package.session().close();
+            return;
+        }
+        Z_GDU *gdu_res = pp.response().get();
+        if (gdu_res && gdu_res->which == Z_GDU_Z3950 &&
+            gdu_res->u.z3950->which == Z_APDU_presentResponse)
+        {
+            Z_PresentResponse *pres =
+                gdu_res->u.z3950->u.presentResponse;
+            if (pres->records &&
+                pres->records->which == Z_Records_DBOSD)
+            {
+                Z_NamePlusRecordList *nprl =
+                    pres->records->u.databaseOrSurDiagnostics;
+                int i;
+                for (i = 0; i < nprl->num_records; i++)
+                {
+                    ODR o = odr;
+                    npl->records[offset+i] = yaz_clone_z_NamePlusRecord(
+                        nprl->records[i], o->mem);
+                }
+                offset += nprl->num_records;
+            }
+            else
+            {
+                package.response() = pp.response();
+                return;
+            }
+        }
+        else
+        {
+            package.response() = pp.response();
+            return;
+        }
+    }
+
+    yaz_log(YLOG_LOG, "building response . %lld", offset);
+
+    Z_APDU *a = zget_APDU(odr, Z_APDU_presentResponse);
+    Z_PresentResponse *pres = a->u.presentResponse;
+    pres->records = (Z_Records *)
+        odr_malloc(odr, sizeof(Z_Records));
+    pres->records->which = Z_Records_DBOSD;
+    pres->records->u.databaseOrSurDiagnostics = npl;
+    npl->num_records = offset;
+    *pres->numberOfRecordsReturned = offset;
+
+    package.response() = a;
+}
+
+void yf::PresentChunk::Impl::process(mp::Package &package)
+{
+    Z_GDU *gdu = package.request().get();
+    if (gdu && gdu->which == Z_GDU_Z3950)
+    {
+        Z_APDU *apdu = gdu->u.z3950;
+        if (apdu->which == Z_APDU_presentRequest && chunk_number > 0)
+            chunk_it(package, apdu);
+        else
+            package.move();
+    }
+    else
+        package.move();
+}
+
+static mp::filter::Base* filter_creator()
+{
+    return new mp::filter::PresentChunk;
+}
+
+extern "C" {
+    struct metaproxy_1_filter_struct metaproxy_1_filter_present_chunk = {
+        0,
+        "present_chunk",
+        filter_creator
+    };
+}
+
+
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * c-file-style: "Stroustrup"
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/src/filter_present_chunk.hpp b/src/filter_present_chunk.hpp
new file mode 100644 (file)
index 0000000..7a17b7c
--- /dev/null
@@ -0,0 +1,55 @@
+/* This file is part of Metaproxy.
+   Copyright (C) 2005-2013 Index Data
+
+Metaproxy is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2, or (at your option) any later
+version.
+
+Metaproxy is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+#ifndef FILTER_PRESENT_CHUNK_HPP
+#define FILTER_PRESENT_CHUNK_HPP
+
+#include <boost/scoped_ptr.hpp>
+
+#include <metaproxy/filter.hpp>
+
+namespace metaproxy_1 {
+    namespace filter {
+        class PresentChunk : public Base {
+            class Impl;
+            boost::scoped_ptr<Impl> m_p;
+        public:
+            PresentChunk();
+            ~PresentChunk();
+            void process(metaproxy_1::Package & package) const;
+            void configure(const xmlNode * ptr, bool test_only,
+                           const char *path);
+        };
+    }
+}
+
+extern "C" {
+    extern struct metaproxy_1_filter_struct metaproxy_1_filter_present_chunk;
+}
+
+#endif
+
+/*
+ * Local variables:
+ * c-basic-offset: 4
+ * c-file-style: "Stroustrup"
+ * indent-tabs-mode: nil
+ * End:
+ * vim: shiftwidth=4 tabstop=8 expandtab
+ */
+
diff --git a/xml/schema/filter_present_chunk.rnc b/xml/schema/filter_present_chunk.rnc
new file mode 100644 (file)
index 0000000..877d69e
--- /dev/null
@@ -0,0 +1,10 @@
+# Metaproxy XML config file schema
+
+namespace mp = "http://indexdata.com/metaproxy"
+
+filter_present_chunk =
+  attribute type { "present_chunk" },
+  attribute id { xsd:NCName }?,
+  attribute name { xsd:NCName }?,
+  element mp:chunk { xsd:integer }?
+
index 39a02bd..c817401 100644 (file)
@@ -34,6 +34,7 @@ include "filter_limit.rnc"
 include "filter_load_balance.rnc"
 include "filter_log.rnc"
 include "filter_multi.rnc"
+include "filter_present_chunk.rnc"
 include "filter_query_rewrite.rnc"
 include "filter_record_transform.rnc"
 include "filter_session_shared.rnc"
@@ -83,6 +84,7 @@ filter =
     | filter_load_balance
     | filter_log
     | filter_multi
+    | filter_present_chunk
     | filter_query_rewrite
     | filter_record_transform
     | filter_session_shared