[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] [taler-exchange] branch master updated (18736c4 -> a1f2895)
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] [taler-exchange] branch master updated (18736c4 -> a1f2895) |
|
Date: |
Sat, 11 Aug 2018 11:43:33 +0200 |
This is an automated email from the git hooks/post-receive script.
grothoff pushed a change to branch master
in repository exchange.
from 18736c4 avoid backlog issue by increasing backlog size
new 6d6b223 cache result from log level check
new 17b9f39 disable SECMEM for Taler exchange, it causes massive lock
contention and then scalability issues
new 657afbb leave it to GNUnet
new 30191a8 be aggressive
new 1ee55ea make wirewatch properly handle (soft) failed transations
new 96c2fb8 implement retries for a few more commands
new a1f2895 make wirewatch batch size adaptive to transaction
success/failure
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../.config/taler/account-2.json | 4 +-
src/benchmark/taler-exchange-benchmark.c | 64 ++--
src/exchange-lib/testing_api_cmd_deposit.c | 106 ++++++-
src/exchange-lib/testing_api_cmd_refresh.c | 349 ++++++++++++++++++---
src/exchange-lib/testing_api_cmd_withdraw.c | 11 +-
src/exchange/taler-exchange-httpd.c | 27 +-
src/exchange/taler-exchange-httpd_db.c | 2 +-
src/exchange/taler-exchange-wirewatch.c | 77 ++++-
src/exchangedb/plugin_exchangedb_postgres.c | 2 +-
src/include/taler_testing_lib.h | 42 +++
10 files changed, 581 insertions(+), 103 deletions(-)
diff --git a/src/benchmark/exchange_benchmark_home/.config/taler/account-2.json
b/src/benchmark/exchange_benchmark_home/.config/taler/account-2.json
index b01af76..b56d7c9 100644
--- a/src/benchmark/exchange_benchmark_home/.config/taler/account-2.json
+++ b/src/benchmark/exchange_benchmark_home/.config/taler/account-2.json
@@ -1,5 +1,5 @@
{
"url": "payto://x-taler-bank/localhost:8082/2",
- "salt":
"4362KHFWBXVY0191AQB6CHAG4GSFAJ3HHWQ3GN86SDZ1TMNZS9TC0201ZJDG7S4JP0K9GE5E4ZBSAE8SEMQRN3ZSGVBPGB7HE7XBHXR",
- "master_sig":
"GW122W2JPED3BEPB20T36WN8VMSGDGD7E4NPH8SJ4CBFFY43T1YF2AWE84DA4YNMNYH1NN2C1ZTZNDG08SFG9RQ9V825DDRZ63RWY1R"
+ "salt":
"RJG7PDRM3YFFQ9YXHE5034R2HF9X68PQKC2W0CQWV4BQ50RYQT339GWN33601S53C3GGG35DG9C6479H4PGMZ9SVJ7A22RK99S4BN78",
+ "master_sig":
"H9GKSPFDQVCP8NBW0X4ZMWEFHZGZNTWZETWRTMYK3831HYTDKWR7R5KY2YJ7XW6HNZ27Q9NXW2DGJWJ016WBK01AKWA6MRNFVPQ6G2G"
}
\ No newline at end of file
diff --git a/src/benchmark/taler-exchange-benchmark.c
b/src/benchmark/taler-exchange-benchmark.c
index 9465667..58cff77 100644
--- a/src/benchmark/taler-exchange-benchmark.c
+++ b/src/benchmark/taler-exchange-benchmark.c
@@ -398,18 +398,20 @@ run (void *cls,
create_reserve_label,
AMOUNT_5,
MHD_HTTP_OK));
- unit[1] = TALER_TESTING_cmd_deposit
- ("deposit",
- is->exchange,
- withdraw_label,
- 0, /* Index of the one withdrawn coin in the traits. */
- TALER_TESTING_make_wire_details
- (USER_ACCOUNT_NUMBER,
- exchange_bank_account.hostname),
- order_enc,
- GNUNET_TIME_UNIT_ZERO,
- AMOUNT_1,
- MHD_HTTP_OK);
+ unit[1] =
+ TALER_TESTING_cmd_deposit_with_retry
+ (TALER_TESTING_cmd_deposit
+ ("deposit",
+ is->exchange,
+ withdraw_label,
+ 0, /* Index of the one withdrawn coin in the traits. */
+ TALER_TESTING_make_wire_details
+ (USER_ACCOUNT_NUMBER,
+ exchange_bank_account.hostname),
+ order_enc,
+ GNUNET_TIME_UNIT_ZERO,
+ AMOUNT_1,
+ MHD_HTTP_OK));
if (eval_probability (REFRESH_PROBABILITY))
{
@@ -424,22 +426,28 @@ run (void *cls,
"refresh-reveal-%u-%u",
i,
j);
- unit[2] = TALER_TESTING_cmd_refresh_melt
- (melt_label,
- is->exchange,
- AMOUNT_4,
- withdraw_label,
- MHD_HTTP_OK);
- unit[3] = TALER_TESTING_cmd_refresh_reveal
- (reveal_label,
- is->exchange,
- melt_label,
- MHD_HTTP_OK);
- unit[4] = TALER_TESTING_cmd_refresh_link
- ("refresh-link",
- is->exchange,
- reveal_label,
- MHD_HTTP_OK);
+ unit[2] =
+ TALER_TESTING_cmd_refresh_melt_with_retry
+ (TALER_TESTING_cmd_refresh_melt
+ (melt_label,
+ is->exchange,
+ AMOUNT_4,
+ withdraw_label,
+ MHD_HTTP_OK));
+ unit[3] =
+ TALER_TESTING_cmd_refresh_reveal_with_retry
+ (TALER_TESTING_cmd_refresh_reveal
+ (reveal_label,
+ is->exchange,
+ melt_label,
+ MHD_HTTP_OK));
+ unit[4] =
+ TALER_TESTING_cmd_refresh_link_with_retry
+ (TALER_TESTING_cmd_refresh_link
+ ("refresh-link",
+ is->exchange,
+ reveal_label,
+ MHD_HTTP_OK));
unit[5] = TALER_TESTING_cmd_end ();
}
else
diff --git a/src/exchange-lib/testing_api_cmd_deposit.c
b/src/exchange-lib/testing_api_cmd_deposit.c
index c07e8fb..b3f179f 100644
--- a/src/exchange-lib/testing_api_cmd_deposit.c
+++ b/src/exchange-lib/testing_api_cmd_deposit.c
@@ -82,11 +82,6 @@ struct DepositState
struct TALER_EXCHANGE_DepositHandle *dh;
/**
- * Expected HTTP response code.
- */
- unsigned int expected_response_code;
-
- /**
* Interpreter state.
*/
struct TALER_TESTING_Interpreter *is;
@@ -95,8 +90,60 @@ struct DepositState
* Exchange connection.
*/
struct TALER_EXCHANGE_Handle *exchange;
+
+ /**
+ * Task scheduled to try later.
+ */
+ struct GNUNET_SCHEDULER_Task *retry_task;
+
+ /**
+ * How long do we wait until we retry?
+ */
+ struct GNUNET_TIME_Relative backoff;
+
+ /**
+ * Expected HTTP response code.
+ */
+ unsigned int expected_response_code;
+
+ /**
+ * Should we retry on (transient) failures?
+ */
+ int do_retry;
+
};
+
+/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+deposit_run (void *cls,
+ const struct TALER_TESTING_Command *cmd,
+ struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #deposit_run.
+ *
+ * @param cls a `struct DepositState`
+ */
+static void
+do_retry (void *cls)
+{
+ struct DepositState *ds = cls;
+
+ ds->retry_task = NULL;
+ deposit_run (ds,
+ NULL,
+ ds->is);
+}
+
+
/**
* Callback to analyze the /deposit response, just used to
* check if the response code is acceptable.
@@ -120,6 +167,27 @@ deposit_cb (void *cls,
ds->dh = NULL;
if (ds->expected_response_code != http_status)
{
+ if (GNUNET_YES == ds->do_retry)
+ {
+ if ( (0 == http_status) ||
+ (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+ (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Retrying deposit failed with %u/%d\n",
+ http_status,
+ (int) ec);
+ /* on DB conflicts, do not use backoff */
+ if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+ ds->backoff = GNUNET_TIME_UNIT_ZERO;
+ else
+ ds->backoff = GNUNET_TIME_STD_BACKOFF (ds->backoff);
+ ds->retry_task = GNUNET_SCHEDULER_add_delayed (ds->backoff,
+ &do_retry,
+ ds);
+ return;
+ }
+ }
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected response code %u to command %s in %s:%u\n",
http_status,
@@ -324,7 +392,11 @@ deposit_cleanup (void *cls,
TALER_EXCHANGE_deposit_cancel (ds->dh);
ds->dh = NULL;
}
-
+ if (NULL != ds->retry_task)
+ {
+ GNUNET_SCHEDULER_cancel (ds->retry_task);
+ ds->retry_task = NULL;
+ }
json_decref (ds->wire_details);
GNUNET_free (ds);
}
@@ -441,3 +513,25 @@ TALER_TESTING_cmd_deposit
return cmd;
}
+
+
+/**
+ * Modify a deposit command to enable retries when we get transient
+ * errors from the exchange.
+ *
+ * @param cmd a deposit command
+ * @return the command with retries enabled
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_deposit_with_retry (struct TALER_TESTING_Command cmd)
+{
+ struct DepositState *ds;
+
+ GNUNET_assert (&deposit_run == cmd.run);
+ ds = cmd.cls;
+ ds->do_retry = GNUNET_YES;
+ return cmd;
+}
+
+
+/* end of testing_api_cmd_deposit.c */
diff --git a/src/exchange-lib/testing_api_cmd_refresh.c
b/src/exchange-lib/testing_api_cmd_refresh.c
index 0f8e5fc..788e82a 100644
--- a/src/exchange-lib/testing_api_cmd_refresh.c
+++ b/src/exchange-lib/testing_api_cmd_refresh.c
@@ -56,19 +56,6 @@ struct RefreshMeltState
{
/**
- * if set to GNUNET_YES, then two /refresh/melt operations
- * will be performed. This is needed to trigger the logic
- * that manages those already-made requests. Note: it
- * is not possible to just copy-and-paste a test refresh melt
- * CMD to have the same effect, because every data preparation
- * generates new planchets that (in turn) make the whole "hash"
- * different from any previous one, therefore NOT allowing the
- * exchange to pick any previous /rerfesh/melt operation from
- * the database.
- */
- unsigned int double_melt;
-
- /**
* Information about coins to be melted.
*/
struct MeltDetails melted_coin;
@@ -79,11 +66,6 @@ struct RefreshMeltState
char *refresh_data;
/**
- * Number of bytes in @e refresh_data.
- */
- size_t refresh_data_length;
-
- /**
* Reference to a previous melt command.
*/
const char *melt_reference;
@@ -104,15 +86,48 @@ struct RefreshMeltState
struct TALER_TESTING_Interpreter *is;
/**
+ * Array of the denomination public keys
+ * corresponding to the @e fresh_amounts.
+ */
+ struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
+
+ /**
+ * Task scheduled to try later.
+ */
+ struct GNUNET_SCHEDULER_Task *retry_task;
+
+ /**
+ * How long do we wait until we retry?
+ */
+ struct GNUNET_TIME_Relative backoff;
+
+ /**
+ * Number of bytes in @e refresh_data.
+ */
+ size_t refresh_data_length;
+
+ /**
* Expected HTTP response code.
*/
unsigned int expected_response_code;
/**
- * Array of the denomination public keys
- * corresponding to the @e fresh_amounts.
+ * if set to #GNUNET_YES, then two /refresh/melt operations
+ * will be performed. This is needed to trigger the logic
+ * that manages those already-made requests. Note: it
+ * is not possible to just copy-and-paste a test refresh melt
+ * CMD to have the same effect, because every data preparation
+ * generates new planchets that (in turn) make the whole "hash"
+ * different from any previous one, therefore NOT allowing the
+ * exchange to pick any previous /rerfesh/melt operation from
+ * the database.
*/
- struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
+ unsigned int double_melt;
+
+ /**
+ * Should we retry on (transient) failures?
+ */
+ int do_retry;
/**
* Set by the melt callback as it comes from the exchange.
@@ -137,13 +152,6 @@ struct RefreshRevealState
struct TALER_EXCHANGE_RefreshRevealHandle *rrh;
/**
- * Number of fresh coins withdrawn, set by the
- * reveal callback as it comes from the exchange,
- * it is the length of the @e fresh_coins array.
- */
- unsigned int num_fresh_coins;
-
- /**
* Convenience struct to keep in one place all the
* data related to one fresh coin, set by the reveal callback
* as it comes from the exchange.
@@ -161,9 +169,32 @@ struct RefreshRevealState
struct TALER_TESTING_Interpreter *is;
/**
+ * Task scheduled to try later.
+ */
+ struct GNUNET_SCHEDULER_Task *retry_task;
+
+ /**
+ * How long do we wait until we retry?
+ */
+ struct GNUNET_TIME_Relative backoff;
+
+ /**
+ * Number of fresh coins withdrawn, set by the
+ * reveal callback as it comes from the exchange,
+ * it is the length of the @e fresh_coins array.
+ */
+ unsigned int num_fresh_coins;
+
+ /**
* Expected HTTP response code.
*/
unsigned int expected_response_code;
+
+ /**
+ * Should we retry on (transient) failures?
+ */
+ int do_retry;
+
};
@@ -193,13 +224,59 @@ struct RefreshLinkState
struct TALER_TESTING_Interpreter *is;
/**
+ * Task scheduled to try later.
+ */
+ struct GNUNET_SCHEDULER_Task *retry_task;
+
+ /**
+ * How long do we wait until we retry?
+ */
+ struct GNUNET_TIME_Relative backoff;
+
+ /**
* Expected HTTP response code.
*/
unsigned int expected_response_code;
+
+ /**
+ * Should we retry on (transient) failures?
+ */
+ int do_retry;
+
};
/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_reveal_run (void *cls,
+ const struct TALER_TESTING_Command *cmd,
+ struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_reveal_run.
+ *
+ * @param cls a `struct RefreshRevealState`
+ */
+static void
+do_reveal_retry (void *cls)
+{
+ struct RefreshRevealState *rrs = cls;
+
+ rrs->retry_task = NULL;
+ refresh_reveal_run (rrs,
+ NULL,
+ rrs->is);
+}
+
+
+/**
* "refresh reveal" request callback; it checks that the response
* code is expected and copies into its command's state the data
* coming from the exchange, namely the fresh coins.
@@ -231,6 +308,27 @@ reveal_cb (void *cls,
rrs->rrh = NULL;
if (rrs->expected_response_code != http_status)
{
+ if (GNUNET_YES == rrs->do_retry)
+ {
+ if ( (0 == http_status) ||
+ (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+ (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Retrying refresh reveal failed with %u/%d\n",
+ http_status,
+ (int) ec);
+ /* on DB conflicts, do not use backoff */
+ if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+ rrs->backoff = GNUNET_TIME_UNIT_ZERO;
+ else
+ rrs->backoff = GNUNET_TIME_STD_BACKOFF (rrs->backoff);
+ rrs->retry_task = GNUNET_SCHEDULER_add_delayed (rrs->backoff,
+ &do_reveal_retry,
+ rrs);
+ return;
+ }
+ }
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected response code %u/%d to command %s in %s:%u\n",
http_status,
@@ -258,16 +356,18 @@ reveal_cb (void *cls,
(num_coins, struct FreshCoin);
const struct TALER_EXCHANGE_DenomPublicKey *fresh_pks;
- unsigned int i;
- if (GNUNET_OK != TALER_TESTING_get_trait_denom_pub
- (melt_cmd, 0, &fresh_pks))
+
+ if (GNUNET_OK !=
+ TALER_TESTING_get_trait_denom_pub (melt_cmd,
+ 0,
+ &fresh_pks))
{
GNUNET_break (0);
TALER_TESTING_interpreter_fail (rrs->is);
return;
}
- for (i=0; i<num_coins; i++)
+ for (unsigned int i=0; i<num_coins; i++)
{
struct FreshCoin *fc = &rrs->fresh_coins[i];
@@ -352,6 +452,11 @@ refresh_reveal_cleanup (void *cls,
TALER_EXCHANGE_refresh_reveal_cancel (rrs->rrh);
rrs->rrh = NULL;
}
+ if (NULL != rrs->retry_task)
+ {
+ GNUNET_SCHEDULER_cancel (rrs->retry_task);
+ rrs->retry_task = NULL;
+ }
for (unsigned int j=0; j < rrs->num_fresh_coins; j++)
GNUNET_CRYPTO_rsa_signature_free (rrs->fresh_coins[j].sig.rsa_signature);
@@ -363,6 +468,36 @@ refresh_reveal_cleanup (void *cls,
/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_link_run (void *cls,
+ const struct TALER_TESTING_Command *cmd,
+ struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_link_run.
+ *
+ * @param cls a `struct RefreshLinkState`
+ */
+static void
+do_link_retry (void *cls)
+{
+ struct RefreshLinkState *rls = cls;
+
+ rls->retry_task = NULL;
+ refresh_link_run (rls,
+ NULL,
+ rls->is);
+}
+
+
+/**
* "refresh link" operation callback, checks that HTTP response
* code is expected _and_ that all the linked coins were actually
* withdrawn by the "refresh reveal" CMD.
@@ -402,6 +537,27 @@ link_cb (void *cls,
rls->rlh = NULL;
if (rls->expected_response_code != http_status)
{
+ if (GNUNET_YES == rls->do_retry)
+ {
+ if ( (0 == http_status) ||
+ (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+ (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Retrying refresh link failed with %u/%d\n",
+ http_status,
+ (int) ec);
+ /* on DB conflicts, do not use backoff */
+ if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+ rls->backoff = GNUNET_TIME_UNIT_ZERO;
+ else
+ rls->backoff = GNUNET_TIME_STD_BACKOFF (rls->backoff);
+ rls->retry_task = GNUNET_SCHEDULER_add_delayed (rls->backoff,
+ &do_link_retry,
+ rls);
+ return;
+ }
+ }
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected response code %u/%d to command %s in %s:%u\n",
http_status,
@@ -514,11 +670,9 @@ refresh_link_run (void *cls,
const struct TALER_TESTING_Command *cmd,
struct TALER_TESTING_Interpreter *is)
{
-
struct RefreshLinkState *rls = cls;
struct RefreshRevealState *rrs;
struct RefreshMeltState *rms;
-
const struct TALER_TESTING_Command *reveal_cmd;
const struct TALER_TESTING_Command *melt_cmd;
const struct TALER_TESTING_Command *coin_cmd;
@@ -605,6 +759,41 @@ refresh_link_cleanup (void *cls,
TALER_EXCHANGE_refresh_link_cancel (rls->rlh);
rls->rlh = NULL;
}
+ if (NULL != rls->retry_task)
+ {
+ GNUNET_SCHEDULER_cancel (rls->retry_task);
+ rls->retry_task = NULL;
+ }
+}
+
+
+/**
+ * Run the command.
+ *
+ * @param cls closure.
+ * @param cmd the command to execute.
+ * @param is the interpreter state.
+ */
+static void
+refresh_melt_run (void *cls,
+ const struct TALER_TESTING_Command *cmd,
+ struct TALER_TESTING_Interpreter *is);
+
+
+/**
+ * Task scheduled to re-try #refresh_melt_run.
+ *
+ * @param cls a `struct RefreshMeltState`
+ */
+static void
+do_melt_retry (void *cls)
+{
+ struct RefreshMeltState *rms = cls;
+
+ rms->retry_task = NULL;
+ refresh_melt_run (rms,
+ NULL,
+ rms->is);
}
@@ -634,6 +823,27 @@ melt_cb (void *cls,
rms->rmh = NULL;
if (rms->expected_response_code != http_status)
{
+ if (GNUNET_YES == rms->do_retry)
+ {
+ if ( (0 == http_status) ||
+ (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+ (MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Retrying refresh melt failed with %u/%d\n",
+ http_status,
+ (int) ec);
+ /* on DB conflicts, do not use backoff */
+ if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
+ rms->backoff = GNUNET_TIME_UNIT_ZERO;
+ else
+ rms->backoff = GNUNET_TIME_STD_BACKOFF (rms->backoff);
+ rms->retry_task = GNUNET_SCHEDULER_add_delayed (rms->backoff,
+ &do_melt_retry,
+ rms);
+ return;
+ }
+ }
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Unexpected response code %u/%d to command %s in %s:%u\n",
http_status,
@@ -668,7 +878,7 @@ melt_cb (void *cls,
* @param cmd the command to execute.
* @param is the interpreter state.
*/
-void
+static void
refresh_melt_run (void *cls,
const struct TALER_TESTING_Command *cmd,
struct TALER_TESTING_Interpreter *is)
@@ -819,6 +1029,11 @@ refresh_melt_cleanup (void *cls,
TALER_EXCHANGE_refresh_melt_cancel (rms->rmh);
rms->rmh = NULL;
}
+ if (NULL != rms->retry_task)
+ {
+ GNUNET_SCHEDULER_cancel (rms->retry_task);
+ rms->retry_task = NULL;
+ }
GNUNET_free_non_null (rms->fresh_pks);
rms->fresh_pks = NULL;
GNUNET_free_non_null (rms->refresh_data);
@@ -894,10 +1109,10 @@ TALER_TESTING_cmd_refresh_melt
cmd.run = &refresh_melt_run;
cmd.cleanup = &refresh_melt_cleanup;
cmd.traits = &refresh_melt_traits;
-
return cmd;
}
+
/**
* Create a "refresh melt" CMD that does TWO /refresh/melt
* requests. This was needed to test the replay of a valid melt
@@ -938,10 +1153,28 @@ TALER_TESTING_cmd_refresh_melt_double
cmd.run = &refresh_melt_run;
cmd.cleanup = &refresh_melt_cleanup;
cmd.traits = &refresh_melt_traits;
+ return cmd;
+}
+
+
+/**
+ * Modify a "refresh melt" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_melt_with_retry (struct TALER_TESTING_Command cmd)
+{
+ struct RefreshMeltState *rms;
+ GNUNET_assert (&refresh_melt_run == cmd.run);
+ rms = cmd.cls;
+ rms->do_retry = GNUNET_YES;
return cmd;
}
+
/**
* Offer internal data from a "refresh reveal" CMD.
*
@@ -960,23 +1193,22 @@ refresh_reveal_traits (void *cls,
{
struct RefreshRevealState *rrs = cls;
unsigned int num_coins = rrs->num_fresh_coins;
- #define NUM_TRAITS (num_coins * 3) + 3
+#define NUM_TRAITS (num_coins * 3) + 3
struct TALER_TESTING_Trait traits[NUM_TRAITS];
- unsigned int i;
/* Making coin privs traits */
- for (i=0; i<num_coins; i++)
+ for (unsigned int i=0; i<num_coins; i++)
traits[i] = TALER_TESTING_make_trait_coin_priv
(i, &rrs->fresh_coins[i].coin_priv);
/* Making denom pubs traits */
- for (i=0; i<num_coins; i++)
+ for (unsigned int i=0; i<num_coins; i++)
traits[num_coins + i]
= TALER_TESTING_make_trait_denom_pub
(i, rrs->fresh_coins[i].pk);
/* Making denom sigs traits */
- for (i=0; i<num_coins; i++)
+ for (unsigned int i=0; i<num_coins; i++)
traits[(num_coins * 2) + i]
= TALER_TESTING_make_trait_denom_sig
(i, &rrs->fresh_coins[i].sig);
@@ -998,6 +1230,7 @@ refresh_reveal_traits (void *cls,
index);
}
+
/**
* Create a "refresh reveal" command.
*
@@ -1028,7 +1261,24 @@ TALER_TESTING_cmd_refresh_reveal
cmd.run = &refresh_reveal_run;
cmd.cleanup = &refresh_reveal_cleanup;
cmd.traits = &refresh_reveal_traits;
+ return cmd;
+}
+
+
+/**
+ * Modify a "refresh reveal" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_reveal_with_retry (struct TALER_TESTING_Command cmd)
+{
+ struct RefreshRevealState *rrs;
+ GNUNET_assert (&refresh_reveal_run == cmd.run);
+ rrs = cmd.cls;
+ rrs->do_retry = GNUNET_YES;
return cmd;
}
@@ -1062,6 +1312,23 @@ TALER_TESTING_cmd_refresh_link
cmd.label = label;
cmd.run = &refresh_link_run;
cmd.cleanup = &refresh_link_cleanup;
+ return cmd;
+}
+
+
+/**
+ * Modify a "refresh link" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_link_with_retry (struct TALER_TESTING_Command cmd)
+{
+ struct RefreshLinkState *rls;
+ GNUNET_assert (&refresh_link_run == cmd.run);
+ rls = cmd.cls;
+ rls->do_retry = GNUNET_YES;
return cmd;
}
diff --git a/src/exchange-lib/testing_api_cmd_withdraw.c
b/src/exchange-lib/testing_api_cmd_withdraw.c
index 90a15b8..7aba3ac 100644
--- a/src/exchange-lib/testing_api_cmd_withdraw.c
+++ b/src/exchange-lib/testing_api_cmd_withdraw.c
@@ -169,15 +169,16 @@ reserve_withdraw_cb (void *cls,
{
if (GNUNET_YES == ws->do_retry)
{
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Retrying withdraw failed with %u/%d\n",
- http_status,
- (int) ec);
- if ( (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
+ if ( (0 == http_status) ||
+ (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec) ||
(TALER_EC_WITHDRAW_INSUFFICIENT_FUNDS == ec) ||
(TALER_EC_WITHDRAW_RESERVE_UNKNOWN == ec) ||
(MHD_HTTP_INTERNAL_SERVER_ERROR == http_status) )
{
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Retrying withdraw failed with %u/%d\n",
+ http_status,
+ (int) ec);
/* on DB conflicts, do not use backoff */
if (TALER_EC_DB_COMMIT_FAILED_ON_RETRY == ec)
ws->backoff = GNUNET_TIME_UNIT_ZERO;
diff --git a/src/exchange/taler-exchange-httpd.c
b/src/exchange/taler-exchange-httpd.c
index 19c318d..c6c3b54 100644
--- a/src/exchange/taler-exchange-httpd.c
+++ b/src/exchange/taler-exchange-httpd.c
@@ -721,16 +721,33 @@ handle_mhd_logs (void *cls,
const char *fm,
va_list ap)
{
+ static int cache;
char buf[2048];
+ if (-1 == cache)
+ return;
+ if (0 == cache)
+ {
+ if (0 ==
+ GNUNET_get_log_call_status (GNUNET_ERROR_TYPE_INFO,
+ "libmicrohttpd",
+ __FILE__,
+ __FUNCTION__,
+ __LINE__))
+ {
+ cache = -1;
+ return;
+ }
+ }
+ cache = 1;
vsnprintf (buf,
sizeof (buf),
fm,
ap);
- GNUNET_log_from (GNUNET_ERROR_TYPE_INFO,
- "libmicrohttpd",
- "%s",
- buf);
+ GNUNET_log_from_nocheck (GNUNET_ERROR_TYPE_INFO,
+ "libmicrohttpd",
+ "%s",
+ buf);
}
@@ -948,7 +965,7 @@ main (int argc,
(-1 == fh) ? serve_port : 0,
NULL, NULL,
&handle_mhd_request, NULL,
- MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 24,
+ MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 32,
MHD_OPTION_LISTEN_BACKLOG_SIZE, (unsigned int) 1024,
MHD_OPTION_LISTEN_SOCKET, fh,
MHD_OPTION_EXTERNAL_LOGGER, &handle_mhd_logs, NULL,
diff --git a/src/exchange/taler-exchange-httpd_db.c
b/src/exchange/taler-exchange-httpd_db.c
index 76a45e2..5ba9f98 100644
--- a/src/exchange/taler-exchange-httpd_db.c
+++ b/src/exchange/taler-exchange-httpd_db.c
@@ -30,7 +30,7 @@
* How often should we retry a transaction before giving up
* (for transactions resulting in serialization/dead locks only).
*/
-#define MAX_TRANSACTION_COMMIT_RETRIES 10
+#define MAX_TRANSACTION_COMMIT_RETRIES 100
/**
diff --git a/src/exchange/taler-exchange-wirewatch.c
b/src/exchange/taler-exchange-wirewatch.c
index 9de0597..9f58b28 100644
--- a/src/exchange/taler-exchange-wirewatch.c
+++ b/src/exchange/taler-exchange-wirewatch.c
@@ -140,6 +140,17 @@ static void *last_row_off;
static size_t last_row_off_size;
/**
+ * Latest row offset seen in this transaction, becomes
+ * the new #last_row_off upon commit.
+ */
+static void *latest_row_off;
+
+/**
+ * Number of bytes in #latest_row_off.
+ */
+static size_t latest_row_off_size;
+
+/**
* Should we delay the next request to the wire plugin a bit?
*/
static int delay;
@@ -155,6 +166,16 @@ static int test_mode;
static int reset_mode;
/**
+ * How many transactions do we retrieve per batch?
+ */
+static unsigned int batch_size = 1024;
+
+/**
+ * How many transactions did we see in the current batch?
+ */
+static unsigned int current_batch_size;
+
+/**
* Next task to run, if any.
*/
static struct GNUNET_SCHEDULER_Task *task;
@@ -389,6 +410,35 @@ history_cb (void *cls,
"End of list. Committing progress!\n");
qs = db_plugin->commit (db_plugin->cls,
session);
+ if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Got DB soft error for commit\n");
+ /* do we need to rollback explicitly on commit failure!? */
+ db_plugin->rollback (db_plugin->cls,
+ session);
+ /* reduce transaction size to reduce rollback probability */
+ if (2 > current_batch_size)
+ current_batch_size /= 2;
+ /* try again */
+ GNUNET_assert (NULL == task);
+ task = GNUNET_SCHEDULER_add_now (&find_transfers,
+ NULL);
+ return GNUNET_OK; /* will be ignored anyway */
+ }
+ if (0 < qs)
+ {
+ /* transaction success, update #last_row_off */
+ GNUNET_free_non_null (last_row_off);
+ last_row_off = latest_row_off;
+ last_row_off_size = latest_row_off_size;
+ latest_row_off = NULL;
+ latest_row_off_size = 0;
+
+ /* if successful at limit, try increasing transaction batch size (AIMD)
*/
+ if (current_batch_size == batch_size)
+ batch_size++;
+ }
GNUNET_break (0 <= qs);
if ( (GNUNET_YES == delay) &&
(test_mode) &&
@@ -403,9 +453,6 @@ history_cb (void *cls,
{
wa_pos->delayed_until
= GNUNET_TIME_relative_to_absolute (DELAY);
- GNUNET_free_non_null (last_row_off);
- last_row_off = NULL;
- last_row_off_size = 0;
wa_pos = wa_pos->next;
if (NULL == wa_pos)
wa_pos = wa_head;
@@ -425,13 +472,13 @@ history_cb (void *cls,
TALER_amount2s (&details->amount),
details->wtid_s);
GNUNET_break (0 != row_off_size);
- if (last_row_off_size != row_off_size)
+ if (latest_row_off_size != row_off_size)
{
- GNUNET_free_non_null (last_row_off);
- last_row_off = GNUNET_malloc (row_off_size);
- last_row_off_size = row_off_size;
+ GNUNET_free_non_null (latest_row_off);
+ latest_row_off = GNUNET_malloc (row_off_size);
+ latest_row_off_size = row_off_size;
}
- memcpy (last_row_off,
+ memcpy (latest_row_off,
row_off,
row_off_size);
rtc = GNUNET_new (struct RejectContext);
@@ -459,6 +506,7 @@ history_cb (void *cls,
"Adding wire transfer over %s with subject `%s'\n",
TALER_amount2s (&details->amount),
TALER_B2S (&details->wtid));
+ current_batch_size++;
/* Wire transfer identifier == reserve public key */
GNUNET_assert (sizeof (reserve_pub) == sizeof (details->wtid));
memcpy (&reserve_pub,
@@ -492,13 +540,13 @@ history_cb (void *cls,
return GNUNET_SYSERR;
}
- if (last_row_off_size != row_off_size)
+ if (latest_row_off_size != row_off_size)
{
- GNUNET_free_non_null (last_row_off);
- last_row_off = GNUNET_malloc (row_off_size);
- last_row_off_size = row_off_size;
+ GNUNET_free_non_null (latest_row_off);
+ latest_row_off = GNUNET_malloc (row_off_size);
+ latest_row_off_size = row_off_size;
}
- memcpy (last_row_off,
+ memcpy (latest_row_off,
row_off,
row_off_size);
return GNUNET_OK;
@@ -573,12 +621,13 @@ find_transfers (void *cls)
( (NULL != last_row_off) &&
(0 != last_row_off_size) ) );
delay = GNUNET_YES;
+ current_batch_size = 0;
hh = wa_pos->wire_plugin->get_history (wa_pos->wire_plugin->cls,
wa_pos->section_name,
TALER_BANK_DIRECTION_CREDIT,
last_row_off,
last_row_off_size,
- 1024,
+ batch_size,
&history_cb,
session);
if (NULL == hh)
diff --git a/src/exchangedb/plugin_exchangedb_postgres.c
b/src/exchangedb/plugin_exchangedb_postgres.c
index 02ab5d7..724bf28 100644
--- a/src/exchangedb/plugin_exchangedb_postgres.c
+++ b/src/exchangedb/plugin_exchangedb_postgres.c
@@ -2062,7 +2062,7 @@ postgres_reserves_in_insert (void *cls,
&reserve);
if (0 > reserve_exists)
{
- GNUNET_break (0);
+ GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == reserve_exists);
return reserve_exists;
}
if ( (0 == reserve.balance.value) &&
diff --git a/src/include/taler_testing_lib.h b/src/include/taler_testing_lib.h
index 8e55c05..8db59ee 100644
--- a/src/include/taler_testing_lib.h
+++ b/src/include/taler_testing_lib.h
@@ -836,6 +836,17 @@ TALER_TESTING_cmd_deposit
/**
+ * Modify a deposit command to enable retries when we get transient
+ * errors from the exchange.
+ *
+ * @param cmd a deposit command
+ * @return the command with retries enabled
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_deposit_with_retry (struct TALER_TESTING_Command cmd);
+
+
+/**
* Create a "refresh melt" command.
*
* @param label command label.
@@ -855,6 +866,7 @@ TALER_TESTING_cmd_refresh_melt
const char *coin_reference,
unsigned int expected_response_code);
+
/**
* Create a "refresh melt" CMD that does TWO /refresh/melt
* requests. This was needed to test the replay of a valid melt
@@ -879,6 +891,16 @@ TALER_TESTING_cmd_refresh_melt_double
/**
+ * Modify a "refresh melt" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_melt_with_retry (struct TALER_TESTING_Command cmd);
+
+
+/**
* Create a "refresh reveal" command.
*
* @param label command label.
@@ -897,6 +919,16 @@ TALER_TESTING_cmd_refresh_reveal
/**
+ * Modify a "refresh reveal" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_reveal_with_retry (struct TALER_TESTING_Command cmd);
+
+
+/**
* Create a "refresh link" command.
*
* @param label command label.
@@ -915,6 +947,16 @@ TALER_TESTING_cmd_refresh_link
/**
+ * Modify a "refresh link" command to enable retries.
+ *
+ * @param cmd command
+ * @return modified command.
+ */
+struct TALER_TESTING_Command
+TALER_TESTING_cmd_refresh_link_with_retry (struct TALER_TESTING_Command cmd);
+
+
+/**
* Create a "track transaction" command.
*
* @param label the command label.
--
To stop receiving notification emails like this one, please contact
address@hidden
- [GNUnet-SVN] [taler-exchange] branch master updated (18736c4 -> a1f2895),
gnunet <=
- [GNUnet-SVN] [taler-exchange] 04/07: be aggressive, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 03/07: leave it to GNUnet, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 02/07: disable SECMEM for Taler exchange, it causes massive lock contention and then scalability issues, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 07/07: make wirewatch batch size adaptive to transaction success/failure, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 01/07: cache result from log level check, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 05/07: make wirewatch properly handle (soft) failed transations, gnunet, 2018/08/11
- [GNUnet-SVN] [taler-exchange] 06/07: implement retries for a few more commands, gnunet, 2018/08/11