[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [gnunet] branch master updated: extend namestore API to ena
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [gnunet] branch master updated: extend namestore API to enable faster iterations by returning more than one result at a time |
|
Date: |
Wed, 25 Apr 2018 16:18:33 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a commit to branch master
in repository gnunet.
The following commit(s) were added to refs/heads/master by this push:
new 4dc79497d extend namestore API to enable faster iterations by
returning more than one result at a time
4dc79497d is described below
commit 4dc79497d7f745996068e62e973e34d220580323
Author: Christian Grothoff <address@hidden>
AuthorDate: Wed Apr 25 16:18:31 2018 +0200
extend namestore API to enable faster iterations by returning more than one
result at a time
---
src/include/gnunet_namestore_plugin.h | 6 +-
src/include/gnunet_namestore_service.h | 5 +-
src/namestore/gnunet-namestore-fcfsd.c | 12 +-
src/namestore/gnunet-namestore.c | 6 +-
src/namestore/gnunet-service-namestore.c | 41 +++--
src/namestore/namestore.h | 8 +
src/namestore/namestore_api.c | 6 +-
src/namestore/plugin_namestore_flat.c | 100 ++++++++---
src/namestore/plugin_namestore_postgres.c | 36 +++-
src/namestore/plugin_namestore_sqlite.c | 185 ++++++++++++---------
src/namestore/plugin_rest_namestore.c | 6 +-
src/namestore/test_namestore_api.conf | 11 ++
.../test_namestore_api_monitoring_existing.c | 27 +--
src/namestore/test_namestore_api_zone_iteration.c | 3 +-
.../test_namestore_api_zone_iteration_nick.c | 3 +-
...st_namestore_api_zone_iteration_specific_zone.c | 3 +-
.../test_namestore_api_zone_iteration_stop.c | 3 +-
src/namestore/test_plugin_namestore.c | 82 ++++++---
src/zonemaster/gnunet-service-zonemaster.c | 12 +-
19 files changed, 372 insertions(+), 183 deletions(-)
diff --git a/src/include/gnunet_namestore_plugin.h
b/src/include/gnunet_namestore_plugin.h
index 3ebf48987..802d7bac5 100644
--- a/src/include/gnunet_namestore_plugin.h
+++ b/src/include/gnunet_namestore_plugin.h
@@ -109,19 +109,21 @@ struct GNUNET_NAMESTORE_PluginFunctions
/**
* Iterate over the results for a particular zone in the
- * datastore. Will return at most one result to the iterator.
+ * datastore. Will return at most @a limit results to the iterator.
*
* @param cls closure (internal context for the plugin)
* @param zone private key of the zone, NULL for all zones
* @param offset offset in the list of all matching records
+ * @param limit maximum number of results to return to @a iter
* @param iter function to call with the result
* @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results,
#GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results,
#GNUNET_SYSERR on error
*/
int
(*iterate_records) (void *cls,
const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
uint64_t offset,
+ uint64_t limit,
GNUNET_NAMESTORE_RecordIterator iter,
void *iter_cls);
diff --git a/src/include/gnunet_namestore_service.h
b/src/include/gnunet_namestore_service.h
index f8c2eaf3b..4828f72ad 100644
--- a/src/include/gnunet_namestore_service.h
+++ b/src/include/gnunet_namestore_service.h
@@ -267,9 +267,12 @@ GNUNET_NAMESTORE_zone_iteration_start (struct
GNUNET_NAMESTORE_Handle *h,
* for the next record.
*
* @param it the iterator
+ * @param limit number of records to return to the iterator in one shot
+ * (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
*/
void
-GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it);
+GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
+ uint64_t limit);
/**
diff --git a/src/namestore/gnunet-namestore-fcfsd.c
b/src/namestore/gnunet-namestore-fcfsd.c
index ddd609918..6e45f8227 100644
--- a/src/namestore/gnunet-namestore-fcfsd.c
+++ b/src/namestore/gnunet-namestore-fcfsd.c
@@ -326,13 +326,15 @@ iterate_cb (void *cls,
if (1 != rd_len)
{
- GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+ 1);
return;
}
if (GNUNET_GNSRECORD_TYPE_PKEY != rd->record_type)
{
- GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+ 1);
return;
}
@@ -343,7 +345,8 @@ iterate_cb (void *cls,
if (NULL == pkey)
{
GNUNET_break (0);
- GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+ 1);
return;
}
if (bytes_free < (strlen (name) + strlen (pkey) + 40))
@@ -359,7 +362,8 @@ iterate_cb (void *cls,
name,
pkey);
zr->write_offset = strlen (zr->zoneinfo);
- GNUNET_NAMESTORE_zone_iterator_next (zr->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
+ 1);
GNUNET_free (pkey);
}
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 660737595..7b8312b46 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -398,7 +398,8 @@ display_record (void *cls,
if ( (NULL != name) &&
(0 != strcmp (name, rname)) )
{
- GNUNET_NAMESTORE_zone_iterator_next (list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (list_it,
+ 1);
return;
}
FPRINTF (stdout,
@@ -441,7 +442,8 @@ display_record (void *cls,
GNUNET_free (s);
}
FPRINTF (stdout, "%s", "\n");
- GNUNET_NAMESTORE_zone_iterator_next (list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (list_it,
+ 1);
}
diff --git a/src/namestore/gnunet-service-namestore.c
b/src/namestore/gnunet-service-namestore.c
index cc41f071d..2fffbeba7 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -83,8 +83,8 @@ struct ZoneIteration
* Offset of the zone iteration used to address next result of the zone
* iteration in the store
*
- * Initialy set to 0 in handle_iteration_start
- * Incremented with by every call to handle_iteration_next
+ * Initialy set to 0 in #handle_iteration_start
+ * Incremented with by every call to #handle_iteration_next
*/
uint32_t offset;
@@ -1065,6 +1065,7 @@ handle_record_store (void *cls,
GSN_database->iterate_records (GSN_database->cls,
&rp_msg->private_key,
0,
+ 1,
NULL,
0)) )
{
@@ -1323,6 +1324,11 @@ struct ZoneIterationProcResult
struct ZoneIteration *zi;
/**
+ * Number of results left to be returned in this iteration.
+ */
+ uint64_t limit;
+
+ /**
* Iteration result: iteration done?
* #IT_SUCCESS_MORE_AVAILABLE: if there may be more results overall but
* we got one for now and have sent it to the client
@@ -1361,14 +1367,18 @@ zone_iterate_proc (void *cls,
proc->res_iteration_finished = IT_SUCCESS_NOT_MORE_RESULTS_AVAILABLE;
return;
}
- if ((NULL == zone_key) || (NULL == name))
+ if ( (NULL == zone_key) ||
+ (NULL == name) )
{
/* what is this!? should never happen */
proc->res_iteration_finished = IT_START;
GNUNET_break (0);
return;
}
- proc->res_iteration_finished = IT_SUCCESS_MORE_AVAILABLE;
+ GNUNET_assert (proc->limit > 0);
+ proc->limit--;
+ if (0 == proc->limit)
+ proc->res_iteration_finished = IT_SUCCESS_MORE_AVAILABLE;
send_lookup_response (proc->zi->nc,
proc->zi->request_id,
zone_key,
@@ -1389,7 +1399,6 @@ zone_iterate_proc (void *cls,
name,
rd_count,
rd);
-
}
@@ -1397,18 +1406,23 @@ zone_iterate_proc (void *cls,
* Perform the next round of the zone iteration.
*
* @param zi zone iterator to process
+ * @param limit number of results to return in one pass
*/
static void
-run_zone_iteration_round (struct ZoneIteration *zi)
+run_zone_iteration_round (struct ZoneIteration *zi,
+ uint64_t limit)
{
struct ZoneIterationProcResult proc;
struct GNUNET_MQ_Envelope *env;
struct RecordResultMessage *rrm;
int ret;
- memset (&proc, 0, sizeof (proc));
+ memset (&proc,
+ 0,
+ sizeof (proc));
proc.zi = zi;
proc.res_iteration_finished = IT_START;
+ proc.limit = limit;
while (IT_START == proc.res_iteration_finished)
{
if (GNUNET_SYSERR ==
@@ -1419,6 +1433,7 @@ run_zone_iteration_round (struct ZoneIteration *zi)
? NULL
: &zi->zone,
zi->offset,
+ limit,
&zone_iterate_proc,
&proc)))
{
@@ -1433,7 +1448,8 @@ run_zone_iteration_round (struct ZoneIteration *zi)
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"More results available\n");
- return; /* more results later */
+ return; /* more results later after we get the
+ #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT message */
}
/* send empty response to indicate end of list */
env = GNUNET_MQ_msg (rrm,
@@ -1472,7 +1488,8 @@ handle_iteration_start (void *cls,
GNUNET_CONTAINER_DLL_insert (nc->op_head,
nc->op_tail,
zi);
- run_zone_iteration_round (zi);
+ run_zone_iteration_round (zi,
+ 1);
GNUNET_SERVICE_client_continue (nc->client);
}
@@ -1525,6 +1542,7 @@ handle_iteration_next (void *cls,
struct NamestoreClient *nc = cls;
struct ZoneIteration *zi;
uint32_t rid;
+ uint64_t limit;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Received ZONE_ITERATION_NEXT message\n");
@@ -1533,6 +1551,7 @@ handle_iteration_next (void *cls,
1,
GNUNET_NO);
rid = ntohl (zis_msg->gns_header.r_id);
+ limit = GNUNET_ntohll (zis_msg->limit);
for (zi = nc->op_head; NULL != zi; zi = zi->next)
if (zi->request_id == rid)
break;
@@ -1542,7 +1561,8 @@ handle_iteration_next (void *cls,
GNUNET_SERVICE_client_drop (nc->client);
return;
}
- run_zone_iteration_round (zi);
+ run_zone_iteration_round (zi,
+ limit);
GNUNET_SERVICE_client_continue (nc->client);
}
@@ -1665,6 +1685,7 @@ monitor_next (void *cls)
? NULL
: &zm->zone,
zm->offset++,
+ 1,
&monitor_iterate_cb,
zm);
if (GNUNET_SYSERR == ret)
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index ec7f28704..b398af8a9 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -345,6 +345,14 @@ struct ZoneIterationNextMessage
* Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
*/
struct GNUNET_NAMESTORE_Header gns_header;
+
+ /**
+ * Number of records to return to the iterator in one shot
+ * (before #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT
+ * should be send again). In NBO.
+ */
+ uint64_t limit;
+
};
diff --git a/src/namestore/namestore_api.c b/src/namestore/namestore_api.c
index 05bdf17da..65d3d75d0 100644
--- a/src/namestore/namestore_api.c
+++ b/src/namestore/namestore_api.c
@@ -1281,9 +1281,12 @@ GNUNET_NAMESTORE_zone_iteration_start (struct
GNUNET_NAMESTORE_Handle *h,
* for the next record.
*
* @param it the iterator
+ * @param limit number of records to return to the iterator in one shot
+ * (before #GNUNET_NAMESTORE_zone_iterator_next is to be called again)
*/
void
-GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it)
+GNUNET_NAMESTORE_zone_iterator_next (struct GNUNET_NAMESTORE_ZoneIterator *it,
+ uint64_t limit)
{
struct GNUNET_NAMESTORE_Handle *h = it->h;
struct ZoneIterationNextMessage *msg;
@@ -1294,6 +1297,7 @@ GNUNET_NAMESTORE_zone_iterator_next (struct
GNUNET_NAMESTORE_ZoneIterator *it)
env = GNUNET_MQ_msg (msg,
GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT);
msg->gns_header.r_id = htonl (it->op_id);
+ msg->limit = GNUNET_htonll (limit);
GNUNET_MQ_send (h->mq,
env);
}
diff --git a/src/namestore/plugin_namestore_flat.c
b/src/namestore/plugin_namestore_flat.c
index 305fe7ba1..e9ed1cc4f 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -86,6 +86,7 @@ struct Plugin
};
+
struct FlatFileEntry
{
/**
@@ -501,9 +502,7 @@ namestore_flat_lookup_records (void *cls,
size_t key_len;
if (NULL == zone)
- {
return GNUNET_SYSERR;
- }
key_len = strlen (label) + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey);
key = GNUNET_malloc (key_len);
GNUNET_memcpy (key,
@@ -532,30 +531,77 @@ namestore_flat_lookup_records (void *cls,
}
+/**
+ * Closure for #iterate_zones.
+ */
+struct IterateContext
+{
+ /**
+ * How many more records should we skip before returning results?
+ */
+ uint64_t offset;
+
+ /**
+ * How many more records should we return?
+ */
+ uint64_t limit;
+
+ /**
+ * Target zone.
+ */
+ const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone;
+
+ /**
+ * Function to call on each record.
+ */
+ GNUNET_NAMESTORE_RecordIterator iter;
+
+ /**
+ * Closure for @e iter.
+ */
+ void *iter_cls;
+
+};
+
+
+/**
+ * Helper function for #namestore_flat_iterate_records().
+ *
+ * @param cls a `struct IterateContext`
+ * @param key unused
+ * @param value a `struct FlatFileEntry`
+ * @return #GNUNET_YES to continue the iteration
+ */
static int
iterate_zones (void *cls,
const struct GNUNET_HashCode *key,
void *value)
{
- struct Plugin *plugin = cls;
+ struct IterateContext *ic = cls;
struct FlatFileEntry *entry = value;
(void) key;
- if ((plugin->target_offset > plugin->offset) ||
- ( (NULL != plugin->iter_zone) &&
- (0 != memcmp (entry->private_key,
- plugin->iter_zone,
- sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))))) {
- plugin->offset++;
+ if (0 == ic->limit)
+ return GNUNET_NO;
+ if ( (NULL != it->zone) &&
+ (0 != memcmp (entry->private_key,
+ ic->zone,
+ sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) )
+ return GNUNET_YES;
+ if (ic->offset > 0)
+ {
+ ic->offset--;
return GNUNET_YES;
}
- plugin->iter (plugin->iter_cls,
- entry->private_key,
- entry->label,
- entry->record_count,
- entry->record_data);
- plugin->iter_result_found = GNUNET_YES;
- return GNUNET_NO;
+ ic->iter (ic->iter_cls,
+ entry->private_key,
+ entry->label,
+ entry->record_count,
+ entry->record_data);
+ ic->limit--;
+ if (0 == ic->limit)
+ return GNUNET_NO;
+ return GNUNET_YES;
}
@@ -566,31 +612,31 @@ iterate_zones (void *cls,
* @param cls closure (internal context for the plugin)
* @param zone hash of public key of the zone, NULL to iterate over all zones
* @param offset offset in the list of all matching records
+ * @param limit maximum number of results to return to @a iter
* @param iter function to call with the result
* @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results,
#GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results,
#GNUNET_SYSERR on error
*/
static int
namestore_flat_iterate_records (void *cls,
const struct GNUNET_CRYPTO_EcdsaPrivateKey
*zone,
uint64_t offset,
+ uint64_t limit,
GNUNET_NAMESTORE_RecordIterator iter,
void *iter_cls)
{
struct Plugin *plugin = cls;
+ struct IterateContext ic;
- /* FIXME: maybe use separate closure to better handle
- recursive calls? */
- plugin->target_offset = offset;
- plugin->offset = 0;
- plugin->iter = iter;
- plugin->iter_cls = iter_cls;
- plugin->iter_zone = zone;
- plugin->iter_result_found = GNUNET_NO;
+ ic.offset = offset;
+ ic.limit = limit;
+ ic.iter = iter;
+ ic.iter_cls = iter_cls;
+ ic.zone = zone;
GNUNET_CONTAINER_multihashmap_iterate (plugin->hm,
&iterate_zones,
- plugin);
- return plugin->iter_result_found;
+ &ic);
+ return (0 == ic.limit) ? GNUNET_OK : GNUNET_NO;
}
diff --git a/src/namestore/plugin_namestore_postgres.c
b/src/namestore/plugin_namestore_postgres.c
index e38fcafb1..4a24ddf88 100644
--- a/src/namestore/plugin_namestore_postgres.c
+++ b/src/namestore/plugin_namestore_postgres.c
@@ -149,22 +149,28 @@ database_setup (struct Plugin *plugin)
struct GNUNET_PQ_PreparedStatement ps[] = {
GNUNET_PQ_make_prepare ("store_records",
"INSERT INTO ns097records (zone_private_key,
pkey, rvalue, record_count, record_data, label) VALUES "
- "($1, $2, $3, $4, $5, $6)", 6),
+ "($1, $2, $3, $4, $5, $6)",
+ 6),
GNUNET_PQ_make_prepare ("delete_records",
"DELETE FROM ns097records "
- "WHERE zone_private_key=$1 AND label=$2", 2),
+ "WHERE zone_private_key=$1 AND label=$2",
+ 2),
GNUNET_PQ_make_prepare ("zone_to_name",
"SELECT record_count,record_data,label FROM
ns097records"
- " WHERE zone_private_key=$1 AND pkey=$2", 2),
+ " WHERE zone_private_key=$1 AND pkey=$2",
+ 2),
GNUNET_PQ_make_prepare ("iterate_zone",
"SELECT record_count,record_data,label FROM
ns097records "
- "WHERE zone_private_key=$1 ORDER BY rvalue LIMIT
1 OFFSET $2", 2),
+ "WHERE zone_private_key=$1 ORDER BY rvalue
OFFSET $2 LIMIT $3",
+ 3),
GNUNET_PQ_make_prepare ("iterate_all_zones",
"SELECT
record_count,record_data,label,zone_private_key"
- " FROM ns097records ORDER BY rvalue LIMIT 1
OFFSET $1", 1),
+ " FROM ns097records ORDER BY rvalue OFFSET $1
LIMIT $2",
+ 2),
GNUNET_PQ_make_prepare ("lookup_label",
"SELECT record_count,record_data,label "
- "FROM ns097records WHERE zone_private_key=$1 AND
label=$2", 2),
+ "FROM ns097records WHERE zone_private_key=$1 AND
label=$2",
+ 2),
GNUNET_PQ_PREPARED_STATEMENT_END
};
@@ -278,6 +284,12 @@ struct ParserContext
* Zone key, NULL if part of record.
*/
const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key;
+
+ /**
+ * Number of results still to return (counted down by
+ * number of results given to iterator).
+ */
+ uint64_t limit;
};
@@ -360,6 +372,7 @@ parse_result_call_iterator (void *cls,
}
GNUNET_PQ_cleanup_result (rs);
}
+ pc->limit -= num_results;
}
@@ -410,14 +423,16 @@ namestore_postgres_lookup_records (void *cls,
* @param cls closure (internal context for the plugin)
* @param zone hash of public key of the zone, NULL to iterate over all zones
* @param offset offset in the list of all matching records
+ * @param limit maximum number of results to fetch
* @param iter function to call with the result
* @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results,
#GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results,
#GNUNET_SYSERR on error
*/
static int
namestore_postgres_iterate_records (void *cls,
const struct GNUNET_CRYPTO_EcdsaPrivateKey
*zone,
uint64_t offset,
+ uint64_t limit,
GNUNET_NAMESTORE_RecordIterator iter,
void *iter_cls)
{
@@ -428,10 +443,12 @@ namestore_postgres_iterate_records (void *cls,
pc.iter = iter;
pc.iter_cls = iter_cls;
pc.zone_key = zone;
+ pc.limit = limit;
if (NULL == zone)
{
struct GNUNET_PQ_QueryParam params_without_zone[] = {
GNUNET_PQ_query_param_uint64 (&offset),
+ GNUNET_PQ_query_param_uint64 (&limit),
GNUNET_PQ_query_param_end
};
@@ -446,6 +463,7 @@ namestore_postgres_iterate_records (void *cls,
struct GNUNET_PQ_QueryParam params_with_zone[] = {
GNUNET_PQ_query_param_auto_from_type (zone),
GNUNET_PQ_query_param_uint64 (&offset),
+ GNUNET_PQ_query_param_uint64 (&limit),
GNUNET_PQ_query_param_end
};
@@ -458,9 +476,9 @@ namestore_postgres_iterate_records (void *cls,
if (res < 0)
return GNUNET_SYSERR;
- if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res)
+ if ( (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res) ||
+ (pc.limit > 0) )
return GNUNET_NO;
-
return GNUNET_OK;
}
diff --git a/src/namestore/plugin_namestore_sqlite.c
b/src/namestore/plugin_namestore_sqlite.c
index 5ad84688c..f905787b5 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -302,12 +302,12 @@ database_setup (struct Plugin *plugin)
sq_prepare (plugin->dbh,
"SELECT record_count,record_data,label"
" FROM ns097records WHERE zone_private_key=?"
- " ORDER BY rvalue LIMIT 1 OFFSET ?",
+ " ORDER BY rvalue LIMIT ? OFFSET ?",
&plugin->iterate_zone)) ||
(SQLITE_OK !=
sq_prepare (plugin->dbh,
"SELECT record_count,record_data,label,zone_private_key"
- " FROM ns097records ORDER BY rvalue LIMIT 1 OFFSET ?",
+ " FROM ns097records ORDER BY rvalue LIMIT ? OFFSET ?",
&plugin->iterate_all_zones)) ||
(SQLITE_OK !=
sq_prepare (plugin->dbh,
@@ -526,16 +526,18 @@ namestore_sqlite_store_records (void *cls,
* @param plugin plugin context
* @param stmt to run (and then clean up)
* @param zone_key private key of the zone
+ * @param limit maximum number of results to fetch
* @param iter iterator to call with the result
* @param iter_cls closure for @a iter
* @return #GNUNET_OK on success, #GNUNET_NO if there were no results,
#GNUNET_SYSERR on error
*/
static int
-get_record_and_call_iterator (struct Plugin *plugin,
- sqlite3_stmt *stmt,
- const struct GNUNET_CRYPTO_EcdsaPrivateKey
*zone_key,
- GNUNET_NAMESTORE_RecordIterator iter,
- void *iter_cls)
+get_records_and_call_iterator (struct Plugin *plugin,
+ sqlite3_stmt *stmt,
+ const struct GNUNET_CRYPTO_EcdsaPrivateKey
*zone_key,
+ uint64_t limit,
+ GNUNET_NAMESTORE_RecordIterator iter,
+ void *iter_cls)
{
uint32_t record_count;
size_t data_size;
@@ -545,74 +547,86 @@ get_record_and_call_iterator (struct Plugin *plugin,
int ret;
int sret;
- ret = GNUNET_NO;
- if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
+ ret = GNUNET_OK;
+ for (uint64_t i = 0;i<limit ; i++)
{
- struct GNUNET_SQ_ResultSpec rs[] = {
- GNUNET_SQ_result_spec_uint32 (&record_count),
- GNUNET_SQ_result_spec_variable_size (&data, &data_size),
- GNUNET_SQ_result_spec_string (&label),
- GNUNET_SQ_result_spec_end
- };
- struct GNUNET_SQ_ResultSpec rsx[] = {
- GNUNET_SQ_result_spec_uint32 (&record_count),
- GNUNET_SQ_result_spec_variable_size (&data, &data_size),
- GNUNET_SQ_result_spec_string (&label),
- GNUNET_SQ_result_spec_auto_from_type (&zk),
- GNUNET_SQ_result_spec_end
- };
-
- if (NULL == zone_key)
- {
- zone_key = &zk;
- ret = GNUNET_SQ_extract_result (stmt,
- rsx);
- }
- else
- {
- ret = GNUNET_SQ_extract_result (stmt,
- rs);
- }
- if ( (GNUNET_OK != ret) ||
- (record_count > 64 * 1024) )
+ if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
{
- /* sanity check, don't stack allocate far too much just
- because database might contain a large value here */
- GNUNET_break (0);
- ret = GNUNET_SYSERR;
+ struct GNUNET_SQ_ResultSpec rs[] = {
+ GNUNET_SQ_result_spec_uint32 (&record_count),
+ GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+ GNUNET_SQ_result_spec_string (&label),
+ GNUNET_SQ_result_spec_end
+ };
+ struct GNUNET_SQ_ResultSpec rsx[] = {
+ GNUNET_SQ_result_spec_uint32 (&record_count),
+ GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+ GNUNET_SQ_result_spec_string (&label),
+ GNUNET_SQ_result_spec_auto_from_type (&zk),
+ GNUNET_SQ_result_spec_end
+ };
+
+ if (NULL == zone_key)
+ {
+ zone_key = &zk;
+ ret = GNUNET_SQ_extract_result (stmt,
+ rsx);
+ }
+ else
+ {
+ ret = GNUNET_SQ_extract_result (stmt,
+ rs);
+ }
+ if ( (GNUNET_OK != ret) ||
+ (record_count > 64 * 1024) )
+ {
+ /* sanity check, don't stack allocate far too much just
+ because database might contain a large value here */
+ GNUNET_break (0);
+ ret = GNUNET_SYSERR;
+ break;
+ }
+ else
+ {
+ struct GNUNET_GNSRECORD_Data rd[record_count];
+
+ if (GNUNET_OK !=
+ GNUNET_GNSRECORD_records_deserialize (data_size,
+ data,
+ record_count,
+ rd))
+ {
+ GNUNET_break (0);
+ ret = GNUNET_SYSERR;
+ break;
+ }
+ else
+ {
+ if (NULL != iter)
+ iter (iter_cls,
+ zone_key,
+ label,
+ record_count,
+ rd);
+ }
+ }
+ GNUNET_SQ_cleanup_result (rs);
}
else
{
- struct GNUNET_GNSRECORD_Data rd[record_count];
-
- if (GNUNET_OK !=
- GNUNET_GNSRECORD_records_deserialize (data_size,
- data,
- record_count,
- rd))
+ if (SQLITE_DONE != sret)
{
- GNUNET_break (0);
- ret = GNUNET_SYSERR;
+ LOG_SQLITE (plugin,
+ GNUNET_ERROR_TYPE_ERROR,
+ "sqlite_step");
+ ret = GNUNET_SYSERR;
}
else
{
- if (NULL != iter)
- iter (iter_cls,
- zone_key,
- label,
- record_count,
- rd);
- ret = GNUNET_YES;
+ ret = GNUNET_NO;
}
+ break;
}
- GNUNET_SQ_cleanup_result (rs);
- }
- else
- {
- if (SQLITE_DONE != sret)
- LOG_SQLITE (plugin,
- GNUNET_ERROR_TYPE_ERROR,
- "sqlite_step");
}
GNUNET_SQ_reset (plugin->dbh,
stmt);
@@ -656,11 +670,12 @@ namestore_sqlite_lookup_records (void *cls,
plugin->lookup_label);
return GNUNET_SYSERR;
}
- return get_record_and_call_iterator (plugin,
- plugin->lookup_label,
- zone,
- iter,
- iter_cls);
+ return get_records_and_call_iterator (plugin,
+ plugin->lookup_label,
+ zone,
+ 1,
+ iter,
+ iter_cls);
}
@@ -671,14 +686,16 @@ namestore_sqlite_lookup_records (void *cls,
* @param cls closure (internal context for the plugin)
* @param zone hash of public key of the zone, NULL to iterate over all zones
* @param offset offset in the list of all matching records
+ * @param limit maximum number of results to return
* @param iter function to call with the result
* @param iter_cls closure for @a iter
- * @return #GNUNET_OK on success, #GNUNET_NO if there were no results,
#GNUNET_SYSERR on error
+ * @return #GNUNET_OK on success, #GNUNET_NO if there were no more results,
#GNUNET_SYSERR on error
*/
static int
namestore_sqlite_iterate_records (void *cls,
const struct GNUNET_CRYPTO_EcdsaPrivateKey
*zone,
uint64_t offset,
+ uint64_t limit,
GNUNET_NAMESTORE_RecordIterator iter,
void *iter_cls)
{
@@ -689,6 +706,7 @@ namestore_sqlite_iterate_records (void *cls,
if (NULL == zone)
{
struct GNUNET_SQ_QueryParam params[] = {
+ GNUNET_SQ_query_param_uint64 (&limit),
GNUNET_SQ_query_param_uint64 (&offset),
GNUNET_SQ_query_param_end
};
@@ -701,6 +719,7 @@ namestore_sqlite_iterate_records (void *cls,
{
struct GNUNET_SQ_QueryParam params[] = {
GNUNET_SQ_query_param_auto_from_type (zone),
+ GNUNET_SQ_query_param_uint64 (&limit),
GNUNET_SQ_query_param_uint64 (&offset),
GNUNET_SQ_query_param_end
};
@@ -718,11 +737,12 @@ namestore_sqlite_iterate_records (void *cls,
stmt);
return GNUNET_SYSERR;
}
- return get_record_and_call_iterator (plugin,
- stmt,
- zone,
- iter,
- iter_cls);
+ return get_records_and_call_iterator (plugin,
+ stmt,
+ zone,
+ limit,
+ iter,
+ iter_cls);
}
@@ -764,11 +784,12 @@ namestore_sqlite_zone_to_name (void *cls,
LOG (GNUNET_ERROR_TYPE_DEBUG,
"Performing reverse lookup for `%s'\n",
GNUNET_GNSRECORD_z2s (value_zone));
- return get_record_and_call_iterator (plugin,
- plugin->zone_to_name,
- zone,
- iter,
- iter_cls);
+ return get_records_and_call_iterator (plugin,
+ plugin->zone_to_name,
+ zone,
+ 1,
+ iter,
+ iter_cls);
}
@@ -787,7 +808,9 @@ libgnunet_plugin_namestore_sqlite_init (void *cls)
if (NULL != plugin.cfg)
return NULL; /* can only initialize once! */
- memset (&plugin, 0, sizeof (struct Plugin));
+ memset (&plugin,
+ 0,
+ sizeof (struct Plugin));
plugin.cfg = cfg;
if (GNUNET_OK != database_setup (&plugin))
{
diff --git a/src/namestore/plugin_rest_namestore.c
b/src/namestore/plugin_rest_namestore.c
index aedb39159..fd1528a1d 100644
--- a/src/namestore/plugin_rest_namestore.c
+++ b/src/namestore/plugin_rest_namestore.c
@@ -452,7 +452,8 @@ namestore_list_response (void *cls,
"%s does not match %s\n",
rname,
handle->name);
- GNUNET_NAMESTORE_zone_iterator_next (handle->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
+ 1);
return;
}
@@ -483,7 +484,8 @@ namestore_list_response (void *cls,
}
json_decref (result_array);
- GNUNET_NAMESTORE_zone_iterator_next (handle->list_it);
+ GNUNET_NAMESTORE_zone_iterator_next (handle->list_it,
+ 1);
}
diff --git a/src/namestore/test_namestore_api.conf
b/src/namestore/test_namestore_api.conf
index a1a674d89..29ff90b5d 100644
--- a/src/namestore/test_namestore_api.conf
+++ b/src/namestore/test_namestore_api.conf
@@ -1,8 +1,19 @@
address@hidden@ ../../contrib/no_forcestart.conf
address@hidden@ ../../contrib/no_autostart_above_core.conf
+
[PATHS]
GNUNET_TEST_HOME = /tmp/test-gnunet-namestore/
[namestore]
DATABASE = sqlite
+AUTOSTART = YES
+
+[namecache]
+DATABASE = sqlite
+AUTOSTART = YES
+
+[identity]
+AUTOSTART = YES
[namestore-sqlite]
FILENAME = $GNUNET_TEST_HOME/namestore/sqlite_test.db
diff --git a/src/namestore/test_namestore_api_monitoring_existing.c
b/src/namestore/test_namestore_api_monitoring_existing.c
index cd1838b5c..374ad44dd 100644
--- a/src/namestore/test_namestore_api_monitoring_existing.c
+++ b/src/namestore/test_namestore_api_monitoring_existing.c
@@ -71,18 +71,18 @@ do_shutdown ()
if (NULL != ns_ops[0])
{
- GNUNET_NAMESTORE_cancel(ns_ops[0]);
- ns_ops[0] = NULL;
+ GNUNET_NAMESTORE_cancel(ns_ops[0]);
+ ns_ops[0] = NULL;
}
if (NULL != ns_ops[1])
{
- GNUNET_NAMESTORE_cancel(ns_ops[1]);
- ns_ops[1] = NULL;
+ GNUNET_NAMESTORE_cancel(ns_ops[1]);
+ ns_ops[1] = NULL;
}
if (NULL != ns_ops[2])
{
- GNUNET_NAMESTORE_cancel(ns_ops[2]);
- ns_ops[2] = NULL;
+ GNUNET_NAMESTORE_cancel(ns_ops[2]);
+ ns_ops[2] = NULL;
}
if (NULL != nsh)
@@ -274,18 +274,20 @@ put_cont (void *cls, int32_t success, const char *emsg)
static struct GNUNET_GNSRECORD_Data *
create_record (unsigned int count)
{
- unsigned int c;
struct GNUNET_GNSRECORD_Data * rd;
- rd = GNUNET_malloc (count * sizeof (struct GNUNET_GNSRECORD_Data));
- for (c = 0; c < count; c++)
+ rd = GNUNET_new_array (count,
+ struct GNUNET_GNSRECORD_Data);
+ for (unsigned int c = 0; c < count; c++)
{
rd[c].expiration_time = GNUNET_TIME_relative_to_absolute
(GNUNET_TIME_UNIT_HOURS).abs_value_us;
rd[c].record_type = 1111;
rd[c].data_size = 50;
rd[c].data = GNUNET_malloc(50);
rd[c].flags = 0;
- memset ((char *) rd[c].data, 'a', 50);
+ memset ((char *) rd[c].data,
+ 'a',
+ 50);
}
return rd;
}
@@ -300,7 +302,10 @@ run (void *cls,
directory = NULL;
GNUNET_assert (GNUNET_OK ==
- GNUNET_CONFIGURATION_get_value_string(mycfg, "PATHS",
"GNUNET_TEST_HOME", &directory));
+ GNUNET_CONFIGURATION_get_value_string (mycfg,
+ "PATHS",
+ "GNUNET_TEST_HOME",
+ &directory));
GNUNET_DISK_directory_remove (directory);
res = 1;
diff --git a/src/namestore/test_namestore_api_zone_iteration.c
b/src/namestore/test_namestore_api_zone_iteration.c
index 8960be55d..1343d42bf 100644
--- a/src/namestore/test_namestore_api_zone_iteration.c
+++ b/src/namestore/test_namestore_api_zone_iteration.c
@@ -276,7 +276,8 @@ zone_proc (void *cls,
returned_records ++;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Telling namestore to send the next result\n");
- GNUNET_NAMESTORE_zone_iterator_next (zi);
+ GNUNET_NAMESTORE_zone_iterator_next (zi,
+ 1);
}
else
{
diff --git a/src/namestore/test_namestore_api_zone_iteration_nick.c
b/src/namestore/test_namestore_api_zone_iteration_nick.c
index 791702f97..395d1854f 100644
--- a/src/namestore/test_namestore_api_zone_iteration_nick.c
+++ b/src/namestore/test_namestore_api_zone_iteration_nick.c
@@ -245,7 +245,8 @@ zone_proc (void *cls,
returned_records ++;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Telling namestore to send the next result\n");
- GNUNET_NAMESTORE_zone_iterator_next (zi);
+ GNUNET_NAMESTORE_zone_iterator_next (zi,
+ 1);
}
else
{
diff --git a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
index c5ae927b0..d222bf16c 100644
--- a/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
+++ b/src/namestore/test_namestore_api_zone_iteration_specific_zone.c
@@ -238,7 +238,8 @@ zone_proc (void *cls,
returned_records ++;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Telling namestore to send the next result\n");
- GNUNET_NAMESTORE_zone_iterator_next (zi);
+ GNUNET_NAMESTORE_zone_iterator_next (zi,
+ 1);
}
else
{
diff --git a/src/namestore/test_namestore_api_zone_iteration_stop.c
b/src/namestore/test_namestore_api_zone_iteration_stop.c
index a5f040150..d23a5f4cb 100644
--- a/src/namestore/test_namestore_api_zone_iteration_stop.c
+++ b/src/namestore/test_namestore_api_zone_iteration_stop.c
@@ -271,7 +271,8 @@ zone_proc (void *cls,
returned_records ++;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Telling namestore to send the next result\n");
- GNUNET_NAMESTORE_zone_iterator_next (zi);
+ GNUNET_NAMESTORE_zone_iterator_next (zi,
+ 1);
}
else
{
diff --git a/src/namestore/test_plugin_namestore.c
b/src/namestore/test_plugin_namestore.c
index aa5d10809..d591eb325 100644
--- a/src/namestore/test_plugin_namestore.c
+++ b/src/namestore/test_plugin_namestore.c
@@ -47,8 +47,12 @@ unload_plugin (struct GNUNET_NAMESTORE_PluginFunctions *api)
{
char *libname;
- GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name);
- GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
+ GNUNET_asprintf (&libname,
+ "libgnunet_plugin_namestore_%s",
+ plugin_name);
+ GNUNET_break (NULL ==
+ GNUNET_PLUGIN_unload (libname,
+ api));
GNUNET_free (libname);
}
@@ -65,12 +69,17 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
struct GNUNET_NAMESTORE_PluginFunctions *ret;
char *libname;
- GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' namestore plugin\n"),
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Loading `%s' namestore plugin\n"),
plugin_name);
- GNUNET_asprintf (&libname, "libgnunet_plugin_namestore_%s", plugin_name);
+ GNUNET_asprintf (&libname,
+ "libgnunet_plugin_namestore_%s",
+ plugin_name);
if (NULL == (ret = GNUNET_PLUGIN_load (libname, (void*) cfg)))
{
- FPRINTF (stderr, "Failed to load plugin `%s'!\n", plugin_name);
+ FPRINTF (stderr,
+ "Failed to load plugin `%s'!\n",
+ plugin_name);
GNUNET_free (libname);
return NULL;
}
@@ -81,21 +90,22 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
static void
test_record (void *cls,
- const struct
GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
- const char *label,
- unsigned int rd_count,
- const struct
GNUNET_GNSRECORD_Data *rd)
+ const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
+ const char *label,
+ unsigned int rd_count,
+ const struct GNUNET_GNSRECORD_Data *rd)
{
int *idp = cls;
int id = *idp;
struct GNUNET_CRYPTO_EcdsaPrivateKey tzone_private_key;
char tname[64];
unsigned int trd_count = 1 + (id % 1024);
- unsigned int i;
- GNUNET_snprintf (tname, sizeof (tname),
- "a%u", (unsigned int ) id);
- for (i=0;i<trd_count;i++)
+ GNUNET_snprintf (tname,
+ sizeof (tname),
+ "a%u",
+ (unsigned int ) id);
+ for (unsigned int i=0;i<trd_count;i++)
{
GNUNET_assert (rd[i].data_size == id % 10);
GNUNET_assert (0 == memcmp ("Hello World", rd[i].data, id % 10));
@@ -104,31 +114,39 @@ test_record (void *cls,
}
memset (&tzone_private_key, (id % 241), sizeof (tzone_private_key));
GNUNET_assert (0 == strcmp (label, tname));
- GNUNET_assert (0 == memcmp (&tzone_private_key, private_key, sizeof (struct
GNUNET_CRYPTO_EcdsaPrivateKey)));
+ GNUNET_assert (0 == memcmp (&tzone_private_key,
+ private_key,
+ sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey)));
}
static void
-get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
+get_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
+ int id)
{
- GNUNET_assert (GNUNET_OK == nsp->iterate_records (nsp->cls,
- NULL, 0, &test_record, &id));
+ GNUNET_assert (GNUNET_OK ==
+ nsp->iterate_records (nsp->cls,
+ NULL,
+ 0,
+ 1,
+ &test_record,
+ &id));
}
static void
-put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp, int id)
+put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
+ int id)
{
struct GNUNET_CRYPTO_EcdsaPrivateKey zone_private_key;
char label[64];
unsigned int rd_count = 1 + (id % 1024);
struct GNUNET_GNSRECORD_Data rd[rd_count];
struct GNUNET_CRYPTO_EcdsaSignature signature;
- unsigned int i;
GNUNET_snprintf (label, sizeof (label),
"a%u", (unsigned int ) id);
- for (i=0;i<rd_count;i++)
+ for (unsigned int i=0;i<rd_count;i++)
{
rd[i].data = "Hello World";
rd[i].data_size = id % 10;
@@ -147,7 +165,9 @@ put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
int id)
static void
-run (void *cls, char *const *args, const char *cfgfile,
+run (void *cls,
+ char *const *args,
+ const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *cfg)
{
struct GNUNET_NAMESTORE_PluginFunctions *nsp;
@@ -169,7 +189,8 @@ run (void *cls, char *const *args, const char *cfgfile,
int
-main (int argc, char *argv[])
+main (int argc,
+ char *argv[])
{
char cfg_name[128];
char *const xargv[] = {
@@ -187,12 +208,21 @@ main (int argc, char *argv[])
"WARNING",
NULL);
plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
- GNUNET_snprintf (cfg_name, sizeof (cfg_name),
"test_plugin_namestore_%s.conf",
+ GNUNET_snprintf (cfg_name,
+ sizeof (cfg_name),
+ "test_plugin_namestore_%s.conf",
plugin_name);
- GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
- "test-plugin-namestore", "nohelp", options, &run, NULL);
+ GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1,
+ xargv,
+ "test-plugin-namestore",
+ "nohelp",
+ options,
+ &run,
+ NULL);
if (ok != 0)
- FPRINTF (stderr, "Missed some testcases: %d\n", ok);
+ FPRINTF (stderr,
+ "Missed some testcases: %d\n",
+ ok);
//GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite");
return ok;
}
diff --git a/src/zonemaster/gnunet-service-zonemaster.c
b/src/zonemaster/gnunet-service-zonemaster.c
index bf11f7d20..f40f25c3f 100644
--- a/src/zonemaster/gnunet-service-zonemaster.c
+++ b/src/zonemaster/gnunet-service-zonemaster.c
@@ -250,7 +250,8 @@ publish_zone_dht_next (void *cls)
{
zone_publish_task = NULL;
GNUNET_assert (NULL != namestore_iter);
- GNUNET_NAMESTORE_zone_iterator_next (namestore_iter);
+ GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
+ 1);
}
@@ -393,7 +394,10 @@ perform_dht_put (const struct
GNUNET_CRYPTO_EcdsaPrivateKey *key,
rd_public,
rd_public_count);
if (NULL == block)
+ {
+ GNUNET_break (0);
return NULL; /* whoops */
+ }
block_size = ntohl (block->purpose.size)
+ sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
+ sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
@@ -565,8 +569,10 @@ put_gns_record (void *cls,
NULL);
if (NULL == active_put)
{
- GNUNET_break (0);
- dht_put_continuation (NULL, GNUNET_NO);
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ "Could not perform DHT PUT, is the DHT running?\n");
+ dht_put_continuation (NULL,
+ GNUNET_NO);
}
}
--
To stop receiving notification emails like this one, please contact
address@hidden
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] [gnunet] branch master updated: extend namestore API to enable faster iterations by returning more than one result at a time,
gnunet <=