[GNUnet-SVN] r6294 - in GNUnet: . src/applications/fs/gap

[gnunet]

Author: grothoff
Date: 2008-02-13 23:06:20 -0700 (Wed, 13 Feb 2008)
New Revision: 6294

Modified:
   GNUnet/ChangeLog
   GNUnet/src/applications/fs/gap/gap.c
   GNUnet/src/applications/fs/gap/migration.c
   GNUnet/todo
Log:
do not iterate over all results if we have many

Modified: GNUnet/ChangeLog
===================================================================
--- GNUnet/ChangeLog    2008-02-14 05:55:07 UTC (rev 6293)
+++ GNUnet/ChangeLog    2008-02-14 06:06:20 UTC (rev 6294)
@@ -1,3 +1,8 @@
+Wed Feb 13 22:59:07 MST 2008
+       Modified mySQL and SQLite datastores to return diverse
+       result sets during (partial) hash-based get iterations
+       (should result in more diverse keyword result sets).
+
 Tue Feb 12 23:54:34 MST 2008
        First shot at integrated hostlist HTTP server based
        on libmicrohttpd.

Modified: GNUnet/src/applications/fs/gap/gap.c
===================================================================
--- GNUnet/src/applications/fs/gap/gap.c        2008-02-14 05:55:07 UTC (rev 
6293)
+++ GNUnet/src/applications/fs/gap/gap.c        2008-02-14 06:06:20 UTC (rev 
6294)
@@ -99,6 +99,12 @@
   GNUNET_free (msg);
 }
 
+struct DVPClosure {
+  struct RequestList * request;
+  unsigned int iteration_count;
+  unsigned int result_count;  
+};
+
 /**
  * An iterator over a set of Datastore items.  This
  * function is called whenever GAP is processing a
@@ -122,7 +128,8 @@
                            const GNUNET_DatastoreValue *
                            value, void *closure, unsigned long long uid)
 {
-  struct RequestList *req = closure;
+  struct DVPClosure * cls = closure;
+  struct RequestList *req = cls->request;
   P2P_gap_reply_MESSAGE *msg;
   GNUNET_DatastoreValue *enc;
   unsigned int size;
@@ -131,7 +138,12 @@
   int ret;
   GNUNET_HashCode hc;
   GNUNET_HashCode mhc;
+  int want_more;
 
+  want_more = GNUNET_OK;
+  cls->iteration_count++;
+  if (cls->iteration_count > 10 * (1 + req->value))
+    want_more = GNUNET_SYSERR;
   enc = NULL;
   if (ntohl (value->type) == GNUNET_ECRS_BLOCKTYPE_ONDEMAND)
     {
@@ -145,8 +157,8 @@
       GNUNET_hash (&value[1],
                    ntohl (value->size) - sizeof (GNUNET_DatastoreValue), &hc);
       GNUNET_FS_HELPER_mingle_hash (&hc, req->bloomfilter_mutator, &mhc);
-      if (GNUNET_YES == GNUNET_bloomfilter_test (req->bloomfilter, &mhc))
-        return GNUNET_OK;       /* not useful */
+      if (GNUNET_YES == GNUNET_bloomfilter_test (req->bloomfilter, &mhc))      
+       return want_more;       /* not useful */      
     }
   et = GNUNET_ntohll (value->expirationTime);
   now = GNUNET_get_time ();
@@ -164,13 +176,16 @@
   msg->reserved = htonl (0);
   msg->expiration = et;
   memcpy (&msg[1], &value[1], size - sizeof (P2P_gap_reply_MESSAGE));
+  cls->result_count++;
+  if (cls->result_count > 2 * (1 + req->value))
+    want_more = GNUNET_SYSERR;
   GNUNET_cron_add_job (cron,
                        send_delayed,
                        GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK,
                                           TTL_DECREMENT), 0, msg);
   ret =
     (ntohl (value->type) ==
-     GNUNET_ECRS_BLOCKTYPE_DATA) ? GNUNET_SYSERR : GNUNET_OK;
+     GNUNET_ECRS_BLOCKTYPE_DATA) ? GNUNET_SYSERR : want_more;
   GNUNET_free_non_null (enc);
   return ret;
 }
@@ -204,6 +219,7 @@
 {
   struct RequestList *rl;
   struct RequestList *prev;
+  struct DVPClosure cls;
   PID_INDEX peer;
   unsigned int index;
   GNUNET_CronTime now;
@@ -331,7 +347,10 @@
   rl->policy = policy;
   table[index] = rl;
   /* check local data store */
-  ret = datastore->get (&queries[0], type, datastore_value_processor, rl);
+  cls.request = rl;
+  cls.iteration_count = 0;
+  cls.result_count = 0;
+  ret = datastore->get (&queries[0], type, datastore_value_processor, &cls);
   if ((type == GNUNET_ECRS_BLOCKTYPE_DATA) && (ret != 1))
     ret = datastore->get (&queries[0],
                           GNUNET_ECRS_BLOCKTYPE_ONDEMAND,

Modified: GNUnet/src/applications/fs/gap/migration.c
===================================================================
--- GNUnet/src/applications/fs/gap/migration.c  2008-02-14 05:55:07 UTC (rev 
6293)
+++ GNUnet/src/applications/fs/gap/migration.c  2008-02-14 06:06:20 UTC (rev 
6294)
@@ -28,13 +28,14 @@
 #include "platform.h"
 #include "migration.h"
 #include "fs.h"
+#include "shared.h"
 #include "gnunet_datastore_service.h"
 #include "gnunet_stats_service.h"
 #include "gnunet_protocols.h"
 #include "anonymity.h"
 #include "ondemand.h"
 
-#define ENABLE_MIGRATION GNUNET_NO
+#define ENABLE_MIGRATION GNUNET_YES
 
 #if ENABLE_MIGRATION
 

Modified: GNUnet/todo
===================================================================
--- GNUnet/todo 2008-02-14 05:55:07 UTC (rev 6293)
+++ GNUnet/todo 2008-02-14 06:06:20 UTC (rev 6294)
@@ -4,13 +4,7 @@
   RC == Release Critical
 
 0.7.4 [4'08] (aka "fix search"):
-- test that modified datastores return diverse subsets of large response sets
-- modify gap.c to NOT do full iteration over result set (current code
-  might take forever; add bound in # replies (prio+1?) and 
-  # iterations (10*(prio+1)?)done!)
-- fix pid-table assertion crash (rare, produced once (!) with linear-gap test) 
[RC]
 - tune GAP query planning code [RC]
-- reenable and test GAP migration code [RC]
 - complete IPv4/IPv6 integration of transports (http is missing!) [RC]
 - Insert/Index a "shared directory" automatically [Nils]
   See also: https://gnunet.org/drupal/?q=node/97 [RC]
@@ -19,6 +13,14 @@
   add a special block to the datastore and then trigger
   gnunet-insert only if the file is actually needed!)
 
+TESTING:
+- test that modified datastores return diverse subsets of large response sets
+- test new hostlist application
+- test GAP migration code
+- test GAP code in general (can we reproduce the pid-table assertion crash?)
+
+
+
 0.8.0 [6'08] (aka "advanced features"):
 - gnunet-chat [ RC ]
 - power insert [#854]