Issue 5587: allow configure --enable-ubsan (issue 575180043 by address@hidden)

[nine . fierce . ballads]

Reviewers: ,

Description:
https://sourceforge.net/p/testlilyissues/issues/5587/

The new configuration option --enable-ubsan instruments compiled code
with the Undefined Behavior Sanitizer (UBSan).

GCC UBSan options are documented here:
https://gcc.gnu.org/onlinedocs/gcc-9.2.0/gcc/Instrumentation-Options.html

I ran the regression tests with UBSan enabled, and it found that a bool
value was being read as something other than 0 or 1.  Initializing more
of the members of Constrained_breaking fixed that.

Please review this at https://codereview.appspot.com/575180043/

Affected files (+46, -14 lines):
  M aclocal.m4
  M lily/constrained-breaking.cc
  M lily/include/constrained-breaking.hh


Index: aclocal.m4
diff --git a/aclocal.m4 b/aclocal.m4
index  
beb042ca7bfa484477c97d0e73c2cffad171ebfd..b6452253168e099cdc105236814e2dc0b42e2325  
100644
--- a/aclocal.m4
+++ b/aclocal.m4
@@ -169,6 +169,7 @@ AC_DEFUN(STEPMAKE_COMPILE_BEFORE, [
     profile_b=no
     debug_b=yes
     pipe_b=yes
+    ubsan_b=no

     AC_ARG_ENABLE(debugging,
         [AS_HELP_STRING(
@@ -200,6 +201,12 @@ AC_DEFUN(STEPMAKE_COMPILE_BEFORE, [
             [compile with -pipe.  Default: on])],
         [pipe_b=$enableval])

+    AC_ARG_ENABLE(ubsan,
+        [AS_HELP_STRING(
+            [--enable-ubsan],
+            [instrument with the Undefined Behavior Sanitizer.  Default:  
off])],
+        [ubsan_b=$enableval])
+
     if test "$optimise_b" = yes; then
         OPTIMIZE=" -O2 -finline-functions"
         # following two lines are compatibility while Patchy has not
@@ -248,8 +255,29 @@ AC_DEFUN(STEPMAKE_COMPILE, [
         fi
     fi

-    CFLAGS="$CFLAGS $OPTIMIZE"
+    # If UBSan requested, test if it works and add to CFLAGS.
+    if test "$ubsan_b" = yes; then
+        save_cflags="$CFLAGS"
+        CFLAGS=" -fsanitize=undefined $CFLAGS";
+        AC_CACHE_CHECK([whether compiler understands -fsanitize=undefined],
+            [stepmake_cv_cflags_ubsan],
+            AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[/* UBSan test */]])],
+                [stepmake_cv_cflags_ubsan=yes],
+                [stepmake_cv_cflags_ubsan=no]))
+        CFLAGS=$save_cflags
+        if test $stepmake_cv_cflags_ubsan = yes; then
+            SANITIZE="$SANITIZE -fsanitize=undefined"
+        fi
+    fi
+
+    if test -n "$SANITIZE"; then
+        # "print a verbose error report and exit the program"
+        SANITIZE="$SANITIZE -fno-sanitize-recover"
+    fi
+
+    CFLAGS="$CFLAGS $OPTIMIZE $SANITIZE"
     CPPFLAGS=${CPPFLAGS-""}
+    LDFLAGS="$LDFLAGS $SANITIZE"

     AC_MSG_CHECKING([for IEEE-conformance compiler flags])
     save_cflags="$CFLAGS"
@@ -277,7 +305,7 @@ AC_DEFUN(STEPMAKE_CXX, [
     AC_PROG_CXX
     STEPMAKE_OPTIONAL_REQUIRED(CXX, c++, $1)

-    CXXFLAGS="$CXXFLAGS $OPTIMIZE"
+    CXXFLAGS="$CXXFLAGS $OPTIMIZE $SANITIZE"

     AC_SUBST(CXX)
     AC_SUBST(CXXFLAGS)
Index: lily/constrained-breaking.cc
diff --git a/lily/constrained-breaking.cc b/lily/constrained-breaking.cc
index  
6ec503b1206dc646f1e46a2f88d036c97acf8ad1..e7c05f64ec4afc7c53a4f4b81219fbf195e65677  
100644
--- a/lily/constrained-breaking.cc
+++ b/lily/constrained-breaking.cc
@@ -350,18 +350,14 @@ Constrained_breaking::prepare_solution (vsize start,  
vsize end, vsize sys_count)

 Constrained_breaking::Constrained_breaking (Paper_score *ps)
 {
-  valid_systems_ = systems_ = 0;
   start_.push_back (0);
-  pscore_ = ps;
-  initialize ();
+  initialize (ps);
 }

 Constrained_breaking::Constrained_breaking (Paper_score *ps, vector<vsize>  
const &start)
   : start_ (start)
 {
-  valid_systems_ = systems_ = 0;
-  pscore_ = ps;
-  initialize ();
+  initialize (ps);
 }

 static SCM
@@ -377,13 +373,11 @@ min_permission (SCM perm1, SCM perm2)

 /* find the forces for all possible lines and cache ragged_ and  
ragged_right_ */
 void
-Constrained_breaking::initialize ()
+Constrained_breaking::initialize (Paper_score *ps)
 {
-  if (!pscore_)
-    return;
+  valid_systems_ = systems_ = 0;
+  pscore_ = ps;

-  ragged_right_ = to_boolean (pscore_->layout ()->c_variable  
("ragged-right"));
-  ragged_last_ = to_boolean (pscore_->layout ()->c_variable  
("ragged-last"));
   system_system_space_ = 0;
   system_markup_space_ = 0;
   system_system_padding_ = 0;
@@ -393,6 +387,16 @@ Constrained_breaking::initialize ()
   score_markup_padding_ = 0;
   score_markup_min_distance_ = 0;

+  if (!pscore_)
+    {
+      ragged_right_ = false;
+      ragged_last_ = false;
+      return;
+    }
+
+  ragged_right_ = to_boolean (pscore_->layout ()->c_variable  
("ragged-right"));
+  ragged_last_ = to_boolean (pscore_->layout ()->c_variable  
("ragged-last"));
+
   Output_def *l = pscore_->layout ();

   SCM spacing_spec = l->c_variable ("system-system-spacing");
Index: lily/include/constrained-breaking.hh
diff --git a/lily/include/constrained-breaking.hh  
b/lily/include/constrained-breaking.hh
index  
b0f1e375ea531ea332d5d7ef9a0e519bf7d3c261..15c132828e522497d8bb43aa1acd4e451f760cbd  
100644
--- a/lily/include/constrained-breaking.hh
+++ b/lily/include/constrained-breaking.hh
@@ -194,7 +194,7 @@ private:
   vector<Grob *> all_;
   vector<vsize> breaks_;

-  void initialize ();
+  void initialize (Paper_score *);
   void resize (vsize systems);

   Column_x_positions space_line (vsize start_col, vsize end_col);