[freetype2] master 8165163: [smooth] Fix complex rendering at high ppem.

[Alexei Podtelezhnikov]

branch: master
commit 816516360c7609916a889b5d0f38759e49507e7f
Author: Alexei Podtelezhnikov <address@hidden>
Commit: Alexei Podtelezhnikov <address@hidden>

    [smooth] Fix complex rendering at high ppem.
    
    We used to split large glyphs into horizontal bands and continue
    bisecting them still horizontally if that was not enough.  This is
    guaranteed to fail when a single scanline cannot fit into the
    rendering memory pool.  Now we bisect the bands vertically so that
    the smallest unit is a column of the band height, which is guranteed
    to fit into memory.
    
    * src/smooth/ftgrays.c (gray_convert_glyph): Implement it.
---
 ChangeLog            | 13 +++++++++++
 src/smooth/ftgrays.c | 64 +++++++++++++++++++++++-----------------------------
 2 files changed, 41 insertions(+), 36 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index afe63fc..2f05d8e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2017-11-01  Alexei Podtelezhnikov  <address@hidden>
+
+       [smooth] Fix complex rendering at high ppem.
+
+       We used to split large glyphs into horizontal bands and continue
+       bisecting them still horizontally if that was not enough.  This is
+       guaranteed to fail when a single scanline cannot fit into the
+       rendering memory pool.  Now we bisect the bands vertically so that
+       the smallest unit is a column of the band height, which is guranteed
+       to fit into memory.
+
+       * src/smooth/ftgrays.c (gray_convert_glyph): Implement it.
+
 2017-10-20  Alexei Podtelezhnikov  <address@hidden>
 
        [smooth] Improve complex rendering at high ppem.
diff --git a/src/smooth/ftgrays.c b/src/smooth/ftgrays.c
index fa488ec..91ca6e3 100644
--- a/src/smooth/ftgrays.c
+++ b/src/smooth/ftgrays.c
@@ -1736,35 +1736,43 @@ typedef ptrdiff_t  FT_PtrDist;
   static int
   gray_convert_glyph( RAS_ARG )
   {
+    const TCoord  yMin = ras.min_ey;
+    const TCoord  yMax = ras.max_ey;
+    const TCoord  xMin = ras.min_ex;
+    const TCoord  xMax = ras.max_ex;
+
     TCell    buffer[FT_MAX_GRAY_POOL];
-    TCoord   band_size = FT_MAX_GRAY_POOL / 8;
-    TCoord   count = ras.max_ey - ras.min_ey;
-    int      num_bands;
-    TCoord   min, max, max_y;
+    size_t   height = (size_t)( yMax - yMin );
+    size_t   n = FT_MAX_GRAY_POOL / 8;
+    TCoord   y;
     TCoord   bands[32];  /* enough to accommodate bisections */
     TCoord*  band;
 
 
     /* set up vertical bands */
-    if ( count > band_size )
+    if ( height > n )
     {
       /* two divisions rounded up */
-      num_bands = (int)( ( count + band_size - 1) / band_size );
-      band_size = ( count + num_bands - 1 ) / num_bands;
+      n       = ( height + n - 1 ) / n;
+      height  = ( height + n - 1 ) / n;
     }
 
-    min   = ras.min_ey;
-    max_y = ras.max_ey;
+    /* memory management */
+    n = ( height * sizeof ( PCell ) + sizeof ( TCell ) - 1 ) / sizeof ( TCell 
);
+
+    ras.cells     = buffer + n;
+    ras.max_cells = (FT_PtrDist)( FT_MAX_GRAY_POOL - n );
+    ras.ycells    = (PCell*)buffer;
 
-    for ( ; min < max_y; min = max )
+    for ( y = yMin; y < yMax; )
     {
-      max = min + band_size;
-      if ( max > max_y )
-        max = max_y;
+      ras.min_ey = y;
+      y         += height;
+      ras.max_ey = FT_MIN( y, yMax );
 
       band    = bands;
-      band[1] = min;
-      band[0] = max;
+      band[1] = xMin;
+      band[0] = xMax;
 
       do
       {
@@ -1772,27 +1780,12 @@ typedef ptrdiff_t  FT_PtrDist;
         int     error;
 
 
-        /* memory management */
-        {
-          size_t  ycount = (size_t)width;
-          size_t  cell_start;
-
-
-          cell_start = ( ycount * sizeof ( PCell ) + sizeof ( TCell ) - 1 ) /
-                       sizeof ( TCell );
-
-          ras.cells     = buffer + cell_start;
-          ras.max_cells = (FT_PtrDist)( FT_MAX_GRAY_POOL - cell_start );
-          ras.num_cells = 0;
-
-          ras.ycells = (PCell*)buffer;
-          while ( ycount )
-            ras.ycells[--ycount] = NULL;
-        }
+        FT_MEM_ZERO( ras.ycells, height * sizeof ( PCell ) );
 
+        ras.num_cells = 0;
         ras.invalid   = 1;
-        ras.min_ey    = band[1];
-        ras.max_ey    = band[0];
+        ras.min_ex    = band[1];
+        ras.max_ex    = band[0];
 
         error = gray_convert_glyph_inner( RAS_VAR );
 
@@ -1808,8 +1801,7 @@ typedef ptrdiff_t  FT_PtrDist;
         /* render pool overflow; we will reduce the render band by half */
         width >>= 1;
 
-        /* This is too complex for a single scanline; there must */
-        /* be some problems.                                     */
+        /* this should never happen even with tiny rendering pool */
         if ( width == 0 )
         {
           FT_TRACE7(( "gray_convert_glyph: rotten glyph\n" ));