Merge pull request #7992 from bfredl/mbscreen

Represent Screen state as UTF-8

runtime/doc/mbyte.txt

@@ -1059,8 +1059,7 @@ widespread as file format.
 A composing or combining character is used to change the meaning of the
 character before it.  The combining characters are drawn on top of the
 preceding character.
-Up to two combining characters can be used by default.  This can be changed
-with the 'maxcombine' option.
+Up to six combining characters can be displayed.
 When editing text a composing character is mostly considered part of the
 preceding character.  For example "x" will delete a character and its
 following composing characters by default.

runtime/doc/options.txt

@@ -3888,16 +3888,10 @@ A jump table for the options with a short description can be found at |Q_op|.
 	set a time.  This is to be compatible with Nvi.
 
 						*'maxcombine'* *'mco'*
-'maxcombine' 'mco'	number (default 2)
-			global
-			{only available when compiled with the |+multi_byte|
-			feature}
-	The maximum number of combining characters supported for displaying.
-	The default is OK for most languages.  Hebrew may require 4.
-	Maximum value is 6.
-	Even when this option is set to 2 you can still edit text with more
-	combining characters, you just can't see them.  Use |g8| or |ga|.
-	See |mbyte-combining|.
+'maxcombine' 'mco'	Removed. |vim-differences| {Nvim}
+	Nvim always displays up to 6 combining characters.  You can still edit
+    text with more than 6 combining characters, you just can't see them.
+    Use |g8| or |ga|.  See |mbyte-combining|.
 
 						*'maxfuncdepth'* *'mfd'*
 'maxfuncdepth' 'mfd'	number	(default 100)

runtime/doc/vim_diff.txt

@@ -409,6 +409,7 @@ Options:
   *'macatsui'*
   'maxmem' Nvim delegates memory-management to the OS.
   'maxmemtot' Nvim delegates memory-management to the OS.
+  'maxcombine' (6 is always used)
   *'restorescreen'* *'rs'* *'norestorescreen'* *'nors'*
   'shelltype'
   *'shortname'* *'sn'* *'noshortname'* *'nosn'*

src/nvim/eval.c

@@ -13994,10 +13994,7 @@ static void f_screenchar(typval_T *argvars, typval_T *rettv, FunPtr fptr)
     c = -1;
   } else {
     off = LineOffset[row] + col;
-    if (enc_utf8 && ScreenLinesUC[off] != 0)
-      c = ScreenLinesUC[off];
-    else
-      c = ScreenLines[off];
+    c = utf_ptr2char(ScreenLines[off]);
   }
   rettv->vval.v_number = c;
 }

src/nvim/globals.h

@@ -131,38 +131,39 @@ typedef off_t off_T;
 /*
  * The characters and attributes cached for the screen.
  */
-typedef char_u schar_T;
-typedef unsigned short sattr_T;
+typedef char_u schar_T[(MAX_MCO+1) * 4 + 1];
+typedef int16_t sattr_T;
 
-/*
- * The characters that are currently on the screen are kept in ScreenLines[].
- * It is a single block of characters, the size of the screen plus one line.
- * The attributes for those characters are kept in ScreenAttrs[].
- *
- * "LineOffset[n]" is the offset from ScreenLines[] for the start of line 'n'.
- * The same value is used for ScreenLinesUC[] and ScreenAttrs[].
- *
- * Note: before the screen is initialized and when out of memory these can be
- * NULL.
- */
+/// ScreenLines[] contains a copy of the whole screen, as it currently is
+/// displayed. It is a single block of screen cells, the size of the screen
+/// plus one line. The extra line used as a buffer while redrawing a window
+/// line, so it can be compared with the previous state of that line. This way
+/// we can avoid sending bigger updates than neccessary to the Ul layer.
+///
+/// Screen cells are stored as NUL-terminated UTF-8 strings, and a cell can
+/// contain up to MAX_MCO composing characters after the base character.
+/// The composing characters are to be drawn on top of the original character.
+/// The content after the NUL is not defined (so comparison must be done a
+/// single cell at a time). Double-width characters are stored in the left cell,
+/// and the right cell should only contain the empty string. When a part of the
+/// screen is cleared, the cells should be filled with a single whitespace char.
+///
+/// ScreenAttrs[] contains the highlighting attribute for each cell.
+/// LineOffset[n] is the offset from ScreenLines[] and ScreenAttrs[] for the
+/// start of line 'n'. These offsets are in general not linear, as full screen
+/// scrolling is implemented by rotating the offsets in the LineOffset array.
+/// LineWraps[] is an array of boolean flags indicating if the screen line wraps
+/// to the next line. It can only be true if a window occupies the entire screen
+/// width.
+///
+///
+/// Note: before the screen is initialized and when out of memory these can be
+/// NULL.
 EXTERN schar_T  *ScreenLines INIT(= NULL);
 EXTERN sattr_T  *ScreenAttrs INIT(= NULL);
 EXTERN unsigned *LineOffset INIT(= NULL);
 EXTERN char_u   *LineWraps INIT(= NULL);        /* line wraps to next line */
 
-/*
- * When using Unicode characters (in UTF-8 encoding) the character in
- * ScreenLinesUC[] contains the Unicode for the character at this position, or
- * NUL when the character in ScreenLines[] is to be used (ASCII char).
- * The composing characters are to be drawn on top of the original character.
- * ScreenLinesC[0][off] is only to be used when ScreenLinesUC[off] != 0.
- * Note: These three are only allocated when enc_utf8 is set!
- */
-EXTERN u8char_T *ScreenLinesUC INIT(= NULL);    /* decoded UTF-8 characters */
-EXTERN u8char_T *ScreenLinesC[MAX_MCO];         /* composing characters */
-EXTERN int Screen_mco INIT(= 0);                /* value of p_mco used when
-                                                   allocating ScreenLinesC[] */
-
 EXTERN int screen_Rows INIT(= 0);           /* actual size of ScreenLines[] */
 EXTERN int screen_Columns INIT(= 0);        /* actual size of ScreenLines[] */
 

src/nvim/mbyte.c

@@ -560,7 +560,7 @@ size_t mb_string2cells(const char_u *str)
 /// We make sure that the offset used is less than "max_off".
 int utf_off2cells(unsigned off, unsigned max_off)
 {
-  return (off + 1 < max_off && ScreenLines[off + 1] == 0) ? 2 : 1;
+  return (off + 1 < max_off && ScreenLines[off + 1][0] == 0) ? 2 : 1;
 }
 
 /// Convert a UTF-8 byte sequence to a wide character
@@ -790,27 +790,6 @@ int utfc_ptr2char_len(const char_u *p, int *pcc, int maxlen)
 #undef ISCOMPOSING
 }
 
-/*
- * Convert the character at screen position "off" to a sequence of bytes.
- * Includes the composing characters.
- * "buf" must at least have the length MB_MAXBYTES + 1.
- * Only to be used when ScreenLinesUC[off] != 0.
- * Returns the produced number of bytes.
- */
-int utfc_char2bytes(int off, char_u *buf)
-{
-  int len;
-  int i;
-
-  len = utf_char2bytes(ScreenLinesUC[off], buf);
-  for (i = 0; i < Screen_mco; ++i) {
-    if (ScreenLinesC[i][off] == 0)
-      break;
-    len += utf_char2bytes(ScreenLinesC[i][off], buf + len);
-  }
-  return len;
-}
-
 /// Get the length of a UTF-8 byte sequence representing a single codepoint
 ///
 /// @param[in]  p  UTF-8 string.
@@ -1853,7 +1832,7 @@ int mb_fix_col(int col, int row)
   col = check_col(col);
   row = check_row(row);
   if (ScreenLines != NULL && col > 0
-      && ScreenLines[LineOffset[row] + col] == 0) {
+      && ScreenLines[LineOffset[row] + col][0] == 0) {
     return col - 1;
   }
   return col;

src/nvim/mouse.c

@@ -108,12 +108,13 @@ retnomove:
     goto retnomove;                             // ugly goto...
 
   // Remember the character under the mouse, it might be a '-' or '+' in the
-  // fold column.
+  // fold column. NB: only works for ASCII chars!
   if (row >= 0 && row < Rows && col >= 0 && col <= Columns
-      && ScreenLines != NULL)
-    mouse_char = ScreenLines[LineOffset[row] + (unsigned)col];
-  else
+      && ScreenLines != NULL) {
+    mouse_char = ScreenLines[LineOffset[row] + (unsigned)col][0];
+  } else {
     mouse_char = ' ';
+  }
 
   old_curwin = curwin;
   old_cursor = curwin->w_cursor;

src/nvim/option.c

@@ -4102,11 +4102,7 @@ static char *set_num_option(int opt_idx, char_u *varp, long value,
       errmsg = e_winwidth;
     }
   } else if (pp == &p_mco) {
-    if (value > MAX_MCO) {
-      errmsg = e_invarg;
-    } else if (value < 0) {
-      errmsg = e_positive;
-    }
+    value = MAX_MCO;
   } else if (pp == &p_titlelen) {
     if (value < 0) {
       errmsg = e_positive;
@@ -4268,8 +4264,6 @@ static char *set_num_option(int opt_idx, char_u *varp, long value,
     if (pp == &curbuf->b_p_sw || curbuf->b_p_sw == 0) {
       parse_cino(curbuf);
     }
-  } else if (pp == &p_mco) {
-    screenclear();          // will re-allocate the screen
   } else if (pp == &curbuf->b_p_iminsert) {
     showmode();
     // Show/unshow value of 'keymap' in status lines.

src/nvim/options.lua

@@ -1493,9 +1493,8 @@ return {
       full_name='maxcombine', abbreviation='mco',
       type='number', scope={'global'},
       vi_def=true,
-      redraw={'curswant'},
       varname='p_mco',
-      defaults={if_true={vi=2}}
+      defaults={if_true={vi=6}}
     },
     {
       full_name='maxfuncdepth', abbreviation='mfd',

test/functional/options/num_options_spec.lua

@@ -51,7 +51,6 @@ describe(':set validation', function()
     should_succeed('winminheight', 0)
     should_fail('winwidth', 0, 'E487')
     should_fail('helpheight', -1, 'E487')
-    should_fail('maxcombine', 7, 'E474')
     should_fail('iminsert', 3, 'E474')
     should_fail('imsearch', 3, 'E474')
     should_fail('titlelen', -1, 'E487')
@@ -94,4 +93,18 @@ describe(':set validation', function()
     feed_command('set winminwidth=3')
     eq('E592', eval("v:errmsg"):match("E%d*"))
   end)
+
+  it('set maxcombine resets to 6', function()
+    local function setto(value)
+      feed_command('setglobal maxcombine=' .. value)
+      feed_command('setlocal maxcombine=' .. value)
+      meths.set_option('maxcombine', value)
+      eq(6, meths.get_option('maxcombine'))
+      eq('', eval("v:errmsg"))
+    end
+    setto(0)
+    setto(1)
+    setto(6)
+    setto(7)
+  end)
 end)

test/functional/ui/fold_spec.lua

@@ -27,9 +27,8 @@ describe("folded lines", function()
   end)
 
   it("works with multibyte text", function()
-    -- Soon, we will always use the maximum value of 'maxcombine'.
-    feed_command("set maxcombine=6")
-
+    -- Currently the only allowed value of 'maxcombine'
+    eq(6, meths.get_option('maxcombine'))
     eq(true, meths.get_option('arabicshape'))
     insert([[
       å 语 x̨̣̘̫̲͚͎̎͂̀̂͛͛̾͢͟ العَرَبِيَّة