[GNUnet-SVN] r6293 - in GNUnet: . src/applications/sqstore_mysql src/applications/sqstore_sqlite

[gnunet]

Author: grothoff
Date: 2008-02-13 22:55:07 -0700 (Wed, 13 Feb 2008)
New Revision: 6293

Modified:
   GNUnet/src/applications/sqstore_mysql/mysql.c
   GNUnet/src/applications/sqstore_sqlite/sqlite.c
   GNUnet/todo
Log:
datastore iterator improvements

Modified: GNUnet/src/applications/sqstore_mysql/mysql.c
===================================================================
--- GNUnet/src/applications/sqstore_mysql/mysql.c       2008-02-14 05:19:17 UTC 
(rev 6292)
+++ GNUnet/src/applications/sqstore_mysql/mysql.c       2008-02-14 05:55:07 UTC 
(rev 6293)
@@ -1292,12 +1292,21 @@
   qbind[1].buffer_type = MYSQL_TYPE_LONGLONG;
   qbind[1].is_unsigned = GNUNET_YES;
   qbind[1].buffer = &last_vkey;
-  qbind[2].buffer_type = MYSQL_TYPE_LONG;
-  qbind[2].is_unsigned = GNUNET_YES;
-  qbind[2].buffer = &type;
-  qbind[3].buffer_type = MYSQL_TYPE_LONG;
-  qbind[3].is_unsigned = GNUNET_YES;
-  qbind[3].buffer = &limit_off;
+  if (type != 0)
+    {
+      qbind[2].buffer_type = MYSQL_TYPE_LONG;
+      qbind[2].is_unsigned = GNUNET_YES;
+      qbind[2].buffer = &type;
+      qbind[3].buffer_type = MYSQL_TYPE_LONG;
+      qbind[3].is_unsigned = GNUNET_YES;
+      qbind[3].buffer = &limit_off;
+    } 
+  else
+    {
+      qbind[2].buffer_type = MYSQL_TYPE_LONG;
+      qbind[2].is_unsigned = GNUNET_YES;
+      qbind[2].buffer = &limit_off;
+    }
   memset (rbind, 0, sizeof (rbind));
   rbind[0].buffer_type = MYSQL_TYPE_LONG;
   rbind[0].buffer = &size;
@@ -1404,10 +1413,9 @@
           GNUNET_mutex_unlock (lock);
         }
       GNUNET_free (datum);
-      off++;
       if (count + off == total)
        last_vkey = 0; /* back to start */
-      if (off == total)
+      if (count == total)
        break;
     }
   mysql_thread_end ();

Modified: GNUnet/src/applications/sqstore_sqlite/sqlite.c
===================================================================
--- GNUnet/src/applications/sqstore_sqlite/sqlite.c     2008-02-14 05:19:17 UTC 
(rev 6292)
+++ GNUnet/src/applications/sqstore_sqlite/sqlite.c     2008-02-14 05:55:07 UTC 
(rev 6293)
@@ -1114,7 +1114,10 @@
      unsigned int type, GNUNET_DatastoreValueIterator iter, void *closure)
 {
   int ret;
-  int count = 0;
+  int count;
+  int total;
+  int off;
+  int limit_off;
   sqlite3_stmt *stmt;
   char scratch[256];
   GNUNET_DatastoreValue *datum;
@@ -1129,16 +1132,67 @@
   GNUNET_mutex_lock (lock);
   handle = getDBHandle ();
   dbh = handle->dbh;
-  strcpy (scratch, "SELECT ");
-  if (iter == NULL)
-    strcat (scratch, "count(*)");
-  else
-    strcat (scratch,
-            "size, type, prio, anonLevel, expire, hash, value, _ROWID_");
-  strcat (scratch, " FROM gn070 WHERE hash = :1 AND _ROWID_ > :2");
+
+  strcpy (scratch, "SELECT count(*) FROM gn070 WHERE hash = :1");
   if (type)
+    strcat (scratch, " AND type = :2");
+  if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK)
+    {
+      LOG_SQLITE (handle,
+                  GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+                  GNUNET_GE_BULK, "sqlite_prepare");
+      GNUNET_mutex_unlock (lock);
+      return GNUNET_SYSERR;
+    }
+  ret = sqlite3_bind_blob (stmt,
+                          1,
+                          key, sizeof (GNUNET_HashCode),
+                          SQLITE_TRANSIENT);
+  if (type && (ret == SQLITE_OK))
+    ret = sqlite3_bind_int (stmt, 2, type);
+  if (ret != SQLITE_OK)
+    {
+      LOG_SQLITE (handle,
+                  GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+                  GNUNET_GE_BULK, "sqlite_bind");
+      sqlite3_reset (stmt);
+      sqlite3_finalize (stmt);
+      GNUNET_mutex_unlock (lock);
+      return GNUNET_SYSERR;
+
+    }
+  ret = sqlite3_step (stmt);
+  if (ret != SQLITE_ROW) 
+    {
+      LOG_SQLITE (handle,
+                  GNUNET_GE_ERROR | GNUNET_GE_ADMIN | GNUNET_GE_USER |
+                  GNUNET_GE_BULK, "sqlite_step");
+      sqlite3_reset (stmt);
+      sqlite3_finalize (stmt);
+      GNUNET_mutex_unlock (lock);
+      return GNUNET_SYSERR;
+
+    }
+  total = sqlite3_column_int (stmt, 0);
+  sqlite3_reset (stmt);
+  sqlite3_finalize (stmt);
+  if ( (iter == NULL) ||
+       (total == 0) )
+    {
+      GNUNET_mutex_unlock (lock);
+      return total;
+    }
+
+  strcpy (scratch, 
+         "SELECT size, type, prio, anonLevel, expire, hash, value, _ROWID_ "
+         "FROM gn070 WHERE hash = :1 AND _ROWID_ >= :2");
+  if (type)
     strcat (scratch, " AND type = :3");
   strcat (scratch, " ORDER BY _ROWID_ ASC LIMIT 1");
+  if (type)
+    strcat(scratch, " OFFSET :4");
+  else
+    strcat(scratch, " OFFSET :3");
   if (sq_prepare (dbh, scratch, &stmt) != SQLITE_OK)
     {
       LOG_SQLITE (handle,
@@ -1149,9 +1203,14 @@
     }
   count = 0;
   last_rowid = 0;
+  off = GNUNET_random_u32(GNUNET_RANDOM_QUALITY_WEAK, total);
   while (1)
     {
-      ret = sqlite3_bind_blob (stmt,
+     if (count == 0)
+       limit_off = off;
+     else
+       limit_off = 0;
+     ret = sqlite3_bind_blob (stmt,
                                1,
                                key, sizeof (GNUNET_HashCode),
                                SQLITE_TRANSIENT);
@@ -1160,20 +1219,14 @@
       if (type && (ret == SQLITE_OK))
         ret = sqlite3_bind_int (stmt, 3, type);
       if (ret == SQLITE_OK)
+       ret = sqlite3_bind_int (stmt, (type == 0) ? 3 : 4, limit_off);
+      if (ret == SQLITE_OK)
         {
           ret = sqlite3_step (stmt);
           if (ret != SQLITE_ROW)
             break;
-          if (iter == NULL)
-            {
-              count = sqlite3_column_int (stmt, 0);
-              sqlite3_reset (stmt);
-              sqlite3_finalize (stmt);
-              GNUNET_mutex_unlock (lock);
-              return count;
-            }
           datum = assembleDatum (handle, stmt, &rkey, &rowid);
-          last_rowid = rowid;
+          last_rowid = rowid + 1;
           sqlite3_reset (stmt);
           if (datum == NULL)
             continue;
@@ -1201,6 +1254,11 @@
             }
           GNUNET_free (datum);
         }
+      off++;
+      if (count + off == total)
+       last_rowid = 0; /* back to start */
+      if (off == total)
+       break;
     }
   sqlite3_reset (stmt);
   sqlite3_finalize (stmt);

Modified: GNUnet/todo
===================================================================
--- GNUnet/todo 2008-02-14 05:19:17 UTC (rev 6292)
+++ GNUnet/todo 2008-02-14 05:55:07 UTC (rev 6293)
@@ -4,21 +4,11 @@
   RC == Release Critical
 
 0.7.4 [4'08] (aka "fix search"):
-- modify datastore to return diverse subsets of large response sets,
-  except when processing for loopback [RC]
-  IDEA: do pre-pass over result set, count # number; then LIMIT with OFFSET 
-        by RAND(total) for the first result and then wrap around
-        to the beginning if the iterator reaches total.
-        + reasonably fast for arbitrary large result sets
-        - except measureable slowdown (?) for result sets of size 1?)
-        + equal chance to get any result first
-        + no need for threads per multi-result query
-        - non-trivial code required in every sqstore implementation
-        o testable, but not trivial to test fully
+- test that modified datastores return diverse subsets of large response sets
 - modify gap.c to NOT do full iteration over result set (current code
   might take forever; add bound in # replies (prio+1?) and 
   # iterations (10*(prio+1)?)done!)
-- fix pid-table assertion crash (rare, produced with linear-gap test) [RC]
+- fix pid-table assertion crash (rare, produced once (!) with linear-gap test) 
[RC]
 - tune GAP query planning code [RC]
 - reenable and test GAP migration code [RC]
 - complete IPv4/IPv6 integration of transports (http is missing!) [RC]