[exchange] branch master updated: make limits in taler-exchange-sanctionscheck configurable, support incremental runs of the tool and add support for background mode (work on #9053)

[Admin]

This is an automated email from the git hooks/post-receive script.

grothoff pushed a commit to branch master
in repository exchange.

The following commit(s) were added to refs/heads/master by this push:
     new 104d95a77 make limits in taler-exchange-sanctionscheck configurable, 
support incremental runs of the tool and add support for background mode (work 
on #9053)
104d95a77 is described below

commit 104d95a77a3e39f7abc0a8b8e877bd60e2a511a2
Author: Christian Grothoff <christian@grothoff.org>
AuthorDate: Tue Jun 3 16:18:19 2025 +0200

    make limits in taler-exchange-sanctionscheck configurable, support 
incremental runs of the tool and add support for background mode (work on #9053)
---
 contrib/gana                                 |   2 +-
 src/exchange/exchange.conf                   |  23 ++
 src/exchange/taler-exchange-sanctionscheck.c | 322 ++++++++++++++++++++++++---
 3 files changed, 318 insertions(+), 29 deletions(-)

diff --git a/contrib/gana b/contrib/gana
index efd11a2c3..675afe389 160000
--- a/contrib/gana
+++ b/contrib/gana
@@ -1 +1 @@
-Subproject commit efd11a2c3c285ae050432dab5210711e1e3fa7e3
+Subproject commit 675afe389fbc0718577b795d63c9fbcb1d6493b3
diff --git a/src/exchange/exchange.conf b/src/exchange/exchange.conf
index 8c631da33..4f3f43008 100644
--- a/src/exchange/exchange.conf
+++ b/src/exchange/exchange.conf
@@ -129,3 +129,26 @@ PRIVACY_DIR = ${TALER_DATA_HOME}terms/
 
 # Etag / filename for the privacy policy.
 PRIVACY_ETAG = exchange-pp-v0
+
+
+[exchange-sanctionscheck]
+# Name where we store the sanctions check offset.
+MIN_ROW_FILENAME = ${TALER_CACHE_HOME}sanctionscheck-offset.bin
+
+# Sanction list match rating that must be exceeded for an automated
+# freeze of the account (without manual investigation first).
+# Both this and the FREEZE_CONFIDENCE_LIMIT must be met.
+FREEZE_RATING_LIMIT = 0.95
+
+# Sanction list confidence that must be exceeded for an automated
+# freeze of the account (without manual investigation first).
+# Both this and the FREEZE_RATING_LIMIT must be met.
+FREEZE_CONFIDENCE_LIMIT = 0.95
+
+# Value that the rating divided by the confidence must exceed to
+# trigger a (manual) investigation. At 0.95 (default), a modest
+# 0.8 match with low 0.5 confidence will trigger (1.6), but a
+# good 0.9 match with a super-high 1.0 confidence would not (0.9 < 0.95).
+# OTOH, a bad 0.2 match with super-low 0.1 confidence would again
+# trigger (2.0 > 0.95).
+INVESTIGATION_LIMIT = 0.95
\ No newline at end of file
diff --git a/src/exchange/taler-exchange-sanctionscheck.c 
b/src/exchange/taler-exchange-sanctionscheck.c
index ea79da489..405808c73 100644
--- a/src/exchange/taler-exchange-sanctionscheck.c
+++ b/src/exchange/taler-exchange-sanctionscheck.c
@@ -23,6 +23,7 @@
 #include <jansson.h>
 #include <pthread.h>
 #include <microhttpd.h>
+#include "taler_dbevents.h"
 #include "taler_exchangedb_lib.h"
 #include "taler_exchangedb_plugin.h"
 #include "taler_json_lib.h"
@@ -114,6 +115,82 @@ static struct Account *acc_tail;
  */
 static uint64_t min_row_id;
 
+/**
+ * File descriptor with the name of the file where we track our
+ * progress.
+ */
+static int min_row_fd = -1;
+
+/**
+ * '-t' command line flag. Disables background processing mode.
+ */
+static int testmode;
+
+/**
+ * '-r' command line flag. Restarts analysis from scratch (for
+ * fresh sanction list).
+ */
+static int reset;
+
+/**
+ * '-n' command line flag. Do not actually run, only reset.
+ */
+static int norun;
+
+/**
+ * Handler to learn about updated KYC attributes.
+ */
+static struct GNUNET_DB_EventHandler *eh;
+
+/**
+ * Set to true if we should restart immediately after
+ * finishing the current transaction.
+ */
+static bool restart_now;
+
+/**
+ * Set to true while we are in a database transaction iterating
+ * over KYC attributes.
+ */
+static bool in_transaction;
+
+/**
+ * Match quality needed for instantly freezing an account.
+ */
+static float freeze_rating_limit = 0.95;
+
+/**
+ * Match confidence needed for instantly freezing an account.
+ */
+static float freeze_confidence_limit = 0.95;
+
+/**
+ * Rating/confidence threshold that must be passed to begin
+ * an investigation.
+ */
+static float investigation_limit = 0.95;
+
+/**
+ * Write @a min_row_id to @a min_row_fd.
+ */
+static void
+sync_row (void)
+{
+  uint64_t r = GNUNET_htonll (min_row_id);
+
+  GNUNET_break (-1 != min_row_fd);
+  GNUNET_break (0 == lseek (min_row_fd,
+                            0,
+                            SEEK_SET));
+  GNUNET_break (sizeof (r) ==
+                write (min_row_fd,
+                       &r,
+                       sizeof (r)));
+  GNUNET_break (0 ==
+                fsync (min_row_fd));
+}
+
+
 /**
  * We're being aborted with CTRL-C (or SIGTERM). Shut down.
  *
@@ -125,6 +202,12 @@ shutdown_task (void *cls)
   struct Account *acc;
 
   (void) cls;
+  sync_row ();
+  if (-1 != min_row_fd)
+  {
+    GNUNET_break (0 == close (min_row_fd));
+    min_row_fd = -1;
+  }
   while (NULL != (acc = acc_head))
   {
     GNUNET_CONTAINER_DLL_remove (acc_head,
@@ -133,7 +216,12 @@ shutdown_task (void *cls)
     json_decref (acc->properties);
     GNUNET_free (acc);
   }
-
+  if (NULL != eh)
+  {
+    db_plugin->event_listen_cancel (db_plugin->cls,
+                                    eh);
+    eh = NULL;
+  }
   if (NULL != sr)
   {
     TALER_KYCLOGIC_sanction_rater_stop (sr);
@@ -163,6 +251,13 @@ double_to_billion (double d)
 }
 
 
+/**
+ * Start the actual database transaction.
+ */
+static void
+begin_transaction (void);
+
+
 /**
  * Function called with the result of a sanction evaluation.
  *
@@ -185,14 +280,13 @@ sanction_cb (void *cls,
   bool freeze = false;
   bool investigate = false;
 
-  // FIXME-#9053: formulas to be improved and customized via configuration
-  if ( (rating > 0.95) &&
-       (confidence > 0.95) )
+  if ( (rating > (double) freeze_rating_limit) &&
+       (confidence > (double) freeze_confidence_limit) )
   {
     freeze = true;
     investigate = true;
   }
-  else if (rating > 0.95 - confidence)
+  else if (rating > (double) investigation_limit * confidence)
   {
     investigate = true;
   }
@@ -247,6 +341,7 @@ sanction_cb (void *cls,
                                acc_tail,
                                acc);
   json_decref (acc->properties);
+  min_row_id = acc->row_id;
   GNUNET_free (acc);
   if (NULL != acc_head)
     return; /* more work */
@@ -263,7 +358,15 @@ sanction_cb (void *cls,
       return;
     }
   }
-  GNUNET_SCHEDULER_shutdown ();
+  in_transaction = false;
+  sync_row ();
+  if (restart_now)
+  {
+    begin_transaction ();
+    return;
+  }
+  if (testmode)
+    GNUNET_SCHEDULER_shutdown ();
 }
 
 
@@ -404,6 +507,76 @@ init_freeze (void)
 }
 
 
+static void
+begin_transaction ()
+{
+  enum GNUNET_DB_QueryStatus qs;
+
+  restart_now = false;
+  GNUNET_assert (! in_transaction);
+  if (GNUNET_OK !=
+      db_plugin->start (db_plugin->cls,
+                        "sanctionscheck"))
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                "Failed to begin DB transaction\n");
+    global_ret = EXIT_NOTCONFIGURED;
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  in_transaction = true;
+  /* FIXME-10063: we may want to eventually limit the number of
+     records we process in a single transaction. */
+  qs = db_plugin->select_all_kyc_attributes (db_plugin->cls,
+                                             min_row_id,
+                                             &account_cb,
+                                             NULL);
+  if (qs < 0)
+  {
+    global_ret = EXIT_FAILURE;
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  {
+    db_plugin->rollback (db_plugin->cls);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  if (NULL == acc_head)
+    in_transaction = false;
+  if ( (NULL == acc_head) &&
+       (testmode) )
+  {
+    /* no work, not incremental, we are done */
+    GNUNET_SCHEDULER_shutdown ();
+  }
+}
+
+
+/**
+ * Function called on new KYC attributes being available in Postgres.
+ *
+ * @param cls closure
+ * @param extra additional event data provided
+ * @param extra_size number of bytes in @a extra
+ */
+static void
+db_event_cb (void *cls,
+             const void *extra,
+             size_t extra_size)
+{
+  GNUNET_break (NULL == cls);
+  (void) extra;
+  (void) extra_size;
+  if (in_transaction)
+    restart_now = true;
+  else
+    begin_transaction ();
+}
+
+
 /**
  * First task.
  *
@@ -418,8 +591,6 @@ run (void *cls,
      const char *cfgfile,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
-  enum GNUNET_DB_QueryStatus qs;
-
   (void) cls;
   (void) cfgfile;
   cfg = c;
@@ -431,6 +602,42 @@ run (void *cls,
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_float (cfg,
+                                            "exchange-sanctionscheck",
+                                            "FREEZE_RATING_LIMIT",
+                                            &freeze_rating_limit))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "exchange-sanctionscheck",
+                               "FREEZE_RATING_LIMIT");
+    global_ret = EXIT_NOTCONFIGURED;
+    return;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_float (cfg,
+                                            "exchange-sanctionscheck",
+                                            "FREEZE_CONFIDENCE_LIMIT",
+                                            &freeze_confidence_limit))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "exchange-sanctionscheck",
+                               "FREEZE_CONFIDENCE_LIMIT");
+    global_ret = EXIT_NOTCONFIGURED;
+    return;
+  }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_float (cfg,
+                                            "exchange-sanctionscheck",
+                                            "INVESTIGATION_LIMIT",
+                                            &investigation_limit))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               "exchange-sanctionscheck",
+                               "INVESTIGATION_LIMIT");
+    global_ret = EXIT_NOTCONFIGURED;
+    return;
+  }
   if (! init_freeze ())
     return;
   {
@@ -472,34 +679,81 @@ run (void *cls,
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (GNUNET_OK !=
-      db_plugin->start (db_plugin->cls,
-                        "sanctionscheck"))
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Failed to begin DB transaction\n");
-    global_ret = EXIT_NOTCONFIGURED;
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+    char *min_row_fn;
+    uint64_t r;
+
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_string (cfg,
+                                               "exchange-sanctionscheck",
+                                               "MIN_ROW_FILENAME",
+                                               &min_row_fn))
+    {
+      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                                 "exchange-sanctionscheck",
+                                 "MIN_ROW_FILENAME");
+      global_ret = EXIT_NOTCONFIGURED;
+      return;
+    }
+    if (reset &&
+        (0 != unlink (min_row_fn)) &&
+        (ENOENT != errno) )
+    {
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "unlink",
+                                min_row_fn);
+      GNUNET_free (min_row_fn);
+      global_ret = EXIT_NOPERMISSION;
+      return;
+    }
+    min_row_fd = open (min_row_fn,
+                       O_CREAT,
+                       S_IRUSR | S_IWUSR);
+    if (-1 == min_row_fd)
+    {
+      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+                                "open",
+                                min_row_fn);
+      GNUNET_free (min_row_fn);
+      global_ret = EXIT_NOTCONFIGURED;
+      return;
+    }
+    if (sizeof (r) !=
+        read (min_row_fd,
+              &r,
+              sizeof (r)))
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                  "Could not read starting row from `%s', will start from 0\n",
+                  min_row_fn);
+      min_row_id = 0;
+    }
+    else
+    {
+      min_row_id = GNUNET_ntohll (r);
+    }
+    GNUNET_free (min_row_fn);
   }
-  qs = db_plugin->select_all_kyc_attributes (db_plugin->cls,
-                                             min_row_id,
-                                             &account_cb,
-                                             NULL);
-  if (qs < 0)
+  if (norun)
   {
-    global_ret = EXIT_FAILURE;
-    GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     return;
   }
-  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  if (! testmode)
   {
-    db_plugin->rollback (db_plugin->cls);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
+    struct GNUNET_DB_EventHeaderP hdr = {
+      .size = htons (sizeof (hdr)),
+      .type = htons (TALER_DBEVENT_EXCHANGE_NEW_KYC_ATTRIBUTES),
+    };
+
+    eh = db_plugin->event_listen (
+      db_plugin->cls,
+      GNUNET_TIME_UNIT_FOREVER_REL,
+      &hdr,
+      &db_event_cb,
+      NULL);
   }
-  GNUNET_assert (NULL != acc_head);
+  begin_transaction ();
 }
 
 
@@ -516,6 +770,18 @@ main (int argc,
 {
   struct GNUNET_GETOPT_CommandLineOption options[] = {
     GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION),
+    GNUNET_GETOPT_option_flag ('n',
+                               "norun",
+                               "do not actually start a scan (to be used to 
only reset without starting a scan)",
+                               &reset),
+    GNUNET_GETOPT_option_flag ('r',
+                               "reset",
+                               "rescan all records (to be used when the 
sanction list was updated)",
+                               &reset),
+    GNUNET_GETOPT_option_flag ('t',
+                               "test",
+                               "run in test mode and exit when idle",
+                               &testmode),
     GNUNET_GETOPT_OPTION_END
   };
   enum GNUNET_GenericReturnValue ret;

-- 
To stop receiving notification emails like this one, please contact
gnunet@gnunet.org.