Caches NamePlusRecord BER; works well

But we must be smarter WRT Z39.50 present/search cache lookup

diff --git a/src/zoom-c.c b/src/zoom-c.c
index 41d6e1c..0cc006b 100644 (file)
--- a/src/zoom-c.c
+++ b/src/zoom-c.c
@@ -700,7 +700,7 @@ ZOOM_resultset ZOOM_resultset_create(void)
     yaz_log(log_details0, "%p ZOOM_resultset_create", r);
     r->refcount = 1;
     r->size = 0;
-    r->odr = odr_createmem(ODR_ENCODE);
+    r->odr = odr_createmem(ODR_DECODE);
     r->piggyback = 1;
     r->setname = 0;
     r->step = 0;
@@ -815,6 +815,7 @@ ZOOM_API(ZOOM_resultset)
             wrbuf_destroy(w);
             event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
             ZOOM_connection_put_event(c, event);
+            r->live_set = 1;
             return r;
         }
         else
@@ -842,7 +843,6 @@ ZOOM_API(ZOOM_resultset)
     task->u.search.resultset = r;
     task->u.search.start = start;
     task->u.search.count = count;
-    task->u.search.recv_search_fired = 0;
 
     syntax = ZOOM_options_get(r->options, "preferredRecordSyntax");
     task->u.search.syntax = syntax ? xstrdup(syntax) : 0;

diff --git a/src/zoom-p.h b/src/zoom-p.h
index 5d4ee61..37ef8a7 100644 (file)
--- a/src/zoom-p.h
+++ b/src/zoom-p.h
@@ -143,7 +143,7 @@ struct ZOOM_resultset_p {
     int num_facets;
     char **facets_names;
     WRBUF mc_key;
-    int live_set;
+    int live_set; /* 0=no hit count, 1=cached hit, 2=hits + real set */
 };
 
 struct facet_term_p {
@@ -191,7 +191,6 @@ struct ZOOM_task_p {
             char *syntax;
             char *elementSetName;
             char *schema;
-            int recv_search_fired;
         } search;
 #define ZOOM_TASK_CONNECT 3
 #define ZOOM_TASK_SCAN 4

diff --git a/src/zoom-record-cache.c b/src/zoom-record-cache.c
index 17c4ac5..9e76d66 100644 (file)
--- a/src/zoom-record-cache.c
+++ b/src/zoom-record-cache.c
@@ -130,13 +130,18 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
     record_cache_add(r, npr, pos, syntax, elementSetName, schema, diag);
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (r->connection->mc_st &&
-        !diag && npr->which == Z_NamePlusRecord_databaseRecord &&
-        npr->u.databaseRecord->which == Z_External_octet)
+        !diag && npr->which == Z_NamePlusRecord_databaseRecord)
     {
         WRBUF k = wrbuf_alloc();
         uint32_t flags = 0;
         memcached_return_t rc;
         time_t expiration = 36000;
+        ODR odr = odr_createmem(ODR_ENCODE);
+        char *rec_buf;
+        int rec_len;
+
+        z_NamePlusRecord(odr, &npr, 0, 0);
+        rec_buf = odr_getbuf(odr, &rec_len, 0);
 
         wrbuf_write(k, wrbuf_buf(r->mc_key), wrbuf_len(r->mc_key));
         wrbuf_printf(k, ";%d;%s;%s;%s", pos,
@@ -145,13 +150,13 @@ void ZOOM_record_cache_add(ZOOM_resultset r, Z_NamePlusRecord *npr,
                      schema ? schema : "");
         rc = memcached_set(r->connection->mc_st,
                            wrbuf_buf(k),wrbuf_len(k),
-                           npr->u.databaseRecord->u.octet_aligned->buf,
-                           npr->u.databaseRecord->u.octet_aligned->len,
+                           rec_buf, rec_len,
                            expiration, flags);
 
-        yaz_log(YLOG_LOG, "Store record key=%s rc=%u %s",
-                wrbuf_cstr(k), (unsigned) rc,
+        yaz_log(YLOG_LOG, "Store record lkey=%s len=%d rc=%u %s",
+                wrbuf_cstr(k), rec_len, (unsigned) rc,
                 memcached_last_error_message(r->connection->mc_st));
+        odr_destroy(odr);
         wrbuf_destroy(k);
     }
 #endif
@@ -183,7 +188,7 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos,
     {
         WRBUF k = wrbuf_alloc();
         size_t v_len;
-        char *v;
+        char *v_buf;
         uint32_t flags;
         memcached_return_t rc;
 
@@ -193,33 +198,26 @@ ZOOM_record ZOOM_record_cache_lookup(ZOOM_resultset r, int pos,
                      elementSetName ? elementSetName : "",
                      schema ? schema : "");
 
-        v = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k),
-                          &v_len, &flags, &rc);
+        yaz_log(YLOG_LOG, "Lookup record %s", wrbuf_cstr(k));
+        v_buf = memcached_get(r->connection->mc_st, wrbuf_buf(k), wrbuf_len(k),
+                              &v_len, &flags, &rc);
         wrbuf_destroy(k);
-        if (v)
+        if (v_buf)
         {
-            yaz_log(YLOG_LOG, "Building record from memcached!! syntax=%s",
-                syntax);
-            Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
-                odr_malloc(r->odr, sizeof(Z_NamePlusRecord));
-            npr->databaseName = 0;
-            npr->which = Z_NamePlusRecord_databaseRecord;
-            npr->u.databaseRecord = (Z_External *)
-                odr_malloc(r->odr, sizeof(Z_External));
-            npr->u.databaseRecord->descriptor = 0;
-            npr->u.databaseRecord->direct_reference =
-                syntax ?
-                yaz_string_to_oid_odr(yaz_oid_std(), CLASS_RECSYN,
-                                      syntax, r->odr) : 0;
-            npr->u.databaseRecord->indirect_reference = 0;
-            npr->u.databaseRecord->which = Z_External_octet;
-            npr->u.databaseRecord->u.octet_aligned =
-                odr_create_Odr_oct(r->odr, v, v_len);
-            free(v);
-
-            if (v)
+            Z_NamePlusRecord *npr = 0;
+
+            odr_setbuf(r->odr, v_buf, v_len, 0);
+
+            z_NamePlusRecord(r->odr, &npr, 0, 0);
+            free(v_buf);
+            if (npr)
+            {
+                yaz_log(YLOG_LOG, "returned memcached copy");
                 return record_cache_add(r, npr, pos, syntax, elementSetName,
                                         schema, 0);
+            }
+            yaz_log(YLOG_WARN, "memcached_get npr failed v_len=%ld",
+                    (long) v_len);
         }
     }
 #endif

diff --git a/src/zoom-sru.c b/src/zoom-sru.c
index 8f5feb0..bca1889 100644 (file)
--- a/src/zoom-sru.c
+++ b/src/zoom-sru.c
@@ -154,53 +154,52 @@ zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
     const char *schema = 0;
     Z_Query *z_query;
     Z_FacetList *facet_list = 0;
+
     if (c->error)                  /* don't continue on error */
         return zoom_complete;
     assert(c->tasks);
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        if (!resultset->setname)
-            resultset->setname = xstrdup("default");
-        ZOOM_options_set(resultset->options, "setname", resultset->setname);
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        facets = ZOOM_options_get(resultset->options, "facets");
-        if (facets)
-            facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets);
-        schema = c->tasks->u.search.schema;
-        break;
+    if (c->tasks->which != ZOOM_TASK_SEARCH)
+        return zoom_complete;
 
+    resultset = c->tasks->u.search.resultset;
+    if (!resultset->setname)
+        resultset->setname = xstrdup("default");
+    ZOOM_options_set(resultset->options, "setname", resultset->setname);
+    start = &c->tasks->u.search.start;
+    count = &c->tasks->u.search.count;
+    facets = ZOOM_options_get(resultset->options, "facets");
+    if (facets)
+        facet_list = yaz_pqf_parse_facet_list(c->odr_out, facets);
+    schema = c->tasks->u.search.schema;
+
+    if (resultset->live_set)
+    {
         if (*start >= resultset->size)
             return zoom_complete;
         if (*start + *count > resultset->size)
             *count = resultset->size - *start;
-
-        for (i = 0; i < *count; i++)
+    }
+    for (i = 0; i < *count; i++)
+    {
+        ZOOM_record rec =
+            ZOOM_record_cache_lookup(resultset, i + *start,
+                                     c->tasks->u.search.syntax,
+                                     c->tasks->u.search.elementSetName,
+                                     schema);
+        if (!rec)
+            break;
+        else
         {
-            ZOOM_record rec =
-                ZOOM_record_cache_lookup(resultset, i + *start,
-                                         c->tasks->u.search.syntax,
-                                         c->tasks->u.search.elementSetName,
-                                         schema);
-            if (!rec)
-                break;
-            else
-            {
-                ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
-                ZOOM_connection_put_event(c, event);
-            }
+            ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
+            ZOOM_connection_put_event(c, event);
         }
-        *start += i;
-        *count -= i;
+    }
+    *start += i;
+    *count -= i;
 
-        if (*count == 0)
-            return zoom_complete;
-        break;
-    default:
+    if (*count == 0 && resultset->live_set)
         return zoom_complete;
-    }
+
     assert(resultset->query);
 
     sr = ZOOM_srw_get_pdu(c, Z_SRW_searchRetrieve_request);
@@ -261,34 +260,26 @@ static zoom_ret handle_srw_response(ZOOM_connection c,
     int i;
     NMEM nmem;
     ZOOM_Event event;
-    int *start, *count;
     const char *syntax, *elementSetName, *schema;
 
     if (!c->tasks)
         return zoom_complete;
 
-    switch(c->tasks->which)
-    {
-    case ZOOM_TASK_SEARCH:
-        resultset = c->tasks->u.search.resultset;
-        start = &c->tasks->u.search.start;
-        count = &c->tasks->u.search.count;
-        syntax = c->tasks->u.search.syntax;
-        elementSetName = c->tasks->u.search.elementSetName;
-        schema = c->tasks->u.search.schema;
-        /* Required not for reporting client hit count multiple times into session */
-        if (!c->tasks->u.search.recv_search_fired) {
-            yaz_log(YLOG_DEBUG, "posting ZOOM_EVENT_RECV_SEARCH");
-            event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
-            ZOOM_connection_put_event(c, event);
-            c->tasks->u.search.recv_search_fired = 1;
-        }
-        if (res->facetList)
-            ZOOM_handle_facet_list(resultset, res->facetList);
-        break;
-    default:
+    if (c->tasks->which != ZOOM_TASK_SEARCH)
         return zoom_complete;
+
+    resultset = c->tasks->u.search.resultset;
+    syntax = c->tasks->u.search.syntax;
+    elementSetName = c->tasks->u.search.elementSetName;
+    schema = c->tasks->u.search.schema;
+
+    if (resultset->live_set == 0)
+    {
+        event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
+        ZOOM_connection_put_event(c, event);
     }
+    if (res->facetList)
+        ZOOM_handle_facet_list(resultset, res->facetList);
 
     resultset->size = 0;
 
@@ -299,72 +290,89 @@ static zoom_ret handle_srw_response(ZOOM_connection c,
 
     if (res->num_diagnostics > 0)
     {
+        resultset->live_set = 2;
         set_SRU_error(c, &res->diagnostics[0]);
     }
     else
     {
-        if (res->numberOfRecords) {
+        if (res->numberOfRecords)
+        {
             resultset->size = *res->numberOfRecords;
+#if HAVE_LIBMEMCACHED_MEMCACHED_H
+            if (c->mc_st && resultset->live_set == 0)
+            {
+                uint32_t flags = 0;
+                memcached_return_t rc;
+                time_t expiration = 36000;
+                char str[40];
+
+                sprintf(str, ODR_INT_PRINTF, resultset->size);
+                rc = memcached_set(c->mc_st,
+                                   wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key),
+                                   str, strlen(str), expiration, flags);
+                yaz_log(YLOG_LOG, "Store SRU hit count key=%s value=%s rc=%u %s",
+                        wrbuf_cstr(resultset->mc_key), str, (unsigned) rc,
+                        memcached_last_error_message(c->mc_st));
+            }
+#endif
         }
-        if (res->suggestions) {
-            ZOOM_resultset_option_set(resultset, "suggestions", res->suggestions);
-        }
-        for (i = 0; i<res->num_records; i++)
+        resultset->live_set = 2;
+        if (res->suggestions)
+            ZOOM_resultset_option_set(resultset, "suggestions",
+                                      res->suggestions);
+        for (i = 0; i < res->num_records; i++)
         {
-            int pos = *start + i;
+            int pos = c->tasks->u.search.start + i;
             Z_SRW_record *sru_rec;
             Z_SRW_diagnostic *diag = 0;
             int num_diag;
 
-            Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
-                odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
-
             /* only trust recordPosition if >= calculated position */
             if (res->records[i].recordPosition &&
                 *res->records[i].recordPosition >= pos + 1)
                 pos = *res->records[i].recordPosition - 1;
 
-            sru_rec = &res->records[i];
-
-            npr->databaseName = 0;
-            npr->which = Z_NamePlusRecord_databaseRecord;
-            npr->u.databaseRecord = (Z_External *)
-                odr_malloc(c->odr_in, sizeof(Z_External));
-            npr->u.databaseRecord->descriptor = 0;
-            npr->u.databaseRecord->direct_reference =
-                odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
-            npr->u.databaseRecord->indirect_reference = 0;
-            npr->u.databaseRecord->which = Z_External_octet;
-
-            npr->u.databaseRecord->u.octet_aligned =
-                odr_create_Odr_oct(c->odr_in,
-                                   sru_rec->recordData_buf,
-                                   sru_rec->recordData_len);
-            if (sru_rec->recordSchema
-                && !strcmp(sru_rec->recordSchema,
-                           "info:srw/schema/1/diagnostics-v1.1"))
+            if (!ZOOM_record_cache_lookup(resultset,
+                                          pos,
+                                          syntax, elementSetName, schema))
             {
-                sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
-                                                 sru_rec->recordData_len,
-                                                 &diag, &num_diag,
-                                                 resultset->odr);
+                Z_NamePlusRecord *npr = (Z_NamePlusRecord *)
+                    odr_malloc(c->odr_in, sizeof(Z_NamePlusRecord));
+                sru_rec = &res->records[i];
+
+                npr->databaseName = 0;
+                npr->which = Z_NamePlusRecord_databaseRecord;
+                npr->u.databaseRecord = (Z_External *)
+                    odr_malloc(c->odr_in, sizeof(Z_External));
+                npr->u.databaseRecord->descriptor = 0;
+                npr->u.databaseRecord->direct_reference =
+                    odr_oiddup(c->odr_in, yaz_oid_recsyn_xml);
+                npr->u.databaseRecord->indirect_reference = 0;
+                npr->u.databaseRecord->which = Z_External_octet;
+
+                npr->u.databaseRecord->u.octet_aligned =
+                    odr_create_Odr_oct(c->odr_in,
+                                       sru_rec->recordData_buf,
+                                   sru_rec->recordData_len);
+                if (sru_rec->recordSchema
+                    && !strcmp(sru_rec->recordSchema,
+                               "info:srw/schema/1/diagnostics-v1.1"))
+                {
+                    sru_decode_surrogate_diagnostics(sru_rec->recordData_buf,
+                                                     sru_rec->recordData_len,
+                                                     &diag, &num_diag,
+                                                     resultset->odr);
+                }
+                ZOOM_record_cache_add(resultset, npr,
+                                      pos, syntax, elementSetName,
+                                      schema, diag);
             }
-            ZOOM_record_cache_add(resultset, npr, pos, syntax, elementSetName,
-                                  schema, diag);
         }
-        *count -= i;
-        *start += i;
-        if (*count + *start > resultset->size)
-            *count = resultset->size - *start;
-        yaz_log(YLOG_DEBUG, "SRU result set size " ODR_INT_PRINTF " start %d count %d", resultset->size, *start, *count);
-        if (*count < 0)
-            *count = 0;
         nmem = odr_extract_mem(c->odr_in);
         nmem_transfer(odr_getmem(resultset->odr), nmem);
         nmem_destroy(nmem);
 
-        if (*count > 0)
-            return ZOOM_connection_srw_send_search(c);
+        return ZOOM_connection_srw_send_search(c);
     }
     return zoom_complete;
 }

diff --git a/src/zoom-z3950.c b/src/zoom-z3950.c
index 4ad090d..04f8fdf 100644 (file)
--- a/src/zoom-z3950.c
+++ b/src/zoom-z3950.c
@@ -651,7 +651,7 @@ zoom_ret ZOOM_connection_Z3950_send_search(ZOOM_connection c)
 
     assert(c->tasks->which == ZOOM_TASK_SEARCH);
     r = c->tasks->u.search.resultset;
-    if (r->live_set)
+    if (r->live_set == 2)
         return send_Z3950_present(c);
 
     apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
@@ -1258,11 +1258,13 @@ static void handle_Z3950_search_response(ZOOM_connection c,
     if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
         return;
 
-    event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
-    ZOOM_connection_put_event(c, event);
-
     resultset = c->tasks->u.search.resultset;
 
+    if (resultset->live_set == 0)
+    {
+        event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
+        ZOOM_connection_put_event(c, event);
+    }
     if (sr->resultSetStatus)
     {
         ZOOM_options_set_int(resultset->options, "resultSetStatus",
@@ -1278,7 +1280,7 @@ static void handle_Z3950_search_response(ZOOM_connection c,
     handle_facet_result(c, resultset, sr->additionalSearchInfo);
 
     resultset->size = *sr->resultCount;
-    resultset->live_set = 1;
+    resultset->live_set = 2;
 
 #if HAVE_LIBMEMCACHED_MEMCACHED_H
     if (c->mc_st)
@@ -1292,7 +1294,7 @@ static void handle_Z3950_search_response(ZOOM_connection c,
         rc = memcached_set(c->mc_st,
                            wrbuf_buf(resultset->mc_key),wrbuf_len(resultset->mc_key),
                            str, strlen(str), expiration, flags);
-        yaz_log(YLOG_LOG, "Store hit count key=%s value=%s rc=%u %s",
+        yaz_log(YLOG_LOG, "Store Z39.50 hit count key=%s value=%s rc=%u %s",
                 wrbuf_cstr(resultset->mc_key), str, (unsigned) rc,
                 memcached_last_error_message(c->mc_st));
     }