To: vim_dev@googlegroups.com Subject: Patch 8.1.2027 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 8.1.2027 Problem: MS-Windows: problem with ambiwidth characters. Solution: handle ambiguous width characters in ConPTY on Windows 10 (1903). (Nobuhiro Takasaki, closes #4411) Files: src/Make_mvc.mak, src/Make_cyg_ming.mak, src/libvterm/src/parser.c, src/libvterm/src/state.c, src/libvterm/src/termscreen.c, src/libvterm/src/unicode.c, src/libvterm/src/vterm_internal.h, src/misc2.c, src/os_win32.c, src/proto/misc2.pro, src/proto/os_win32.pro *** ../vim-8.1.2026/src/Make_mvc.mak 2019-09-12 22:26:19.991830017 +0200 --- src/Make_mvc.mak 2019-09-13 22:19:58.262938216 +0200 *************** *** 1716,1721 **** --- 1716,1722 ---- -DVSNPRINTF=vim_vsnprintf \ -DIS_COMBINING_FUNCTION=utf_iscomposing_uint \ -DWCWIDTH_FUNCTION=utf_uint2cells \ + -DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type \ -D_CRT_SECURE_NO_WARNINGS # Create a default rule for libvterm. *** ../vim-8.1.2026/src/Make_cyg_ming.mak 2019-09-07 23:16:16.826370964 +0200 --- src/Make_cyg_ming.mak 2019-09-13 22:29:38.984632426 +0200 *************** *** 1192,1198 **** CCCTERM = $(CC) -c $(CFLAGS) -Ilibvterm/include -DINLINE="" \ -DVSNPRINTF=vim_vsnprintf \ -DIS_COMBINING_FUNCTION=utf_iscomposing_uint \ ! -DWCWIDTH_FUNCTION=utf_uint2cells $(OUTDIR)/%.o : libvterm/src/%.c $(TERM_DEPS) $(CCCTERM) $< -o $@ --- 1192,1199 ---- CCCTERM = $(CC) -c $(CFLAGS) -Ilibvterm/include -DINLINE="" \ -DVSNPRINTF=vim_vsnprintf \ -DIS_COMBINING_FUNCTION=utf_iscomposing_uint \ ! -DWCWIDTH_FUNCTION=utf_uint2cells \ ! -DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type $(OUTDIR)/%.o : libvterm/src/%.c $(TERM_DEPS) $(CCCTERM) $< -o $@ *** ../vim-8.1.2026/src/libvterm/src/parser.c 2019-08-18 20:41:10.692526067 +0200 --- src/libvterm/src/parser.c 2019-09-13 22:24:15.133886984 +0200 *************** *** 127,132 **** --- 127,135 ---- size_t pos = 0; const char *string_start = NULL; // init to avoid gcc warning + vt->in_backspace = 0; // Count down with BS key and activate when + // it reaches 1 + switch(vt->parser.state) { case NORMAL: case CSI_LEADER: *************** *** 172,177 **** --- 175,187 ---- // fallthrough } else if(c < 0x20) { // other C0 + if(vterm_get_special_pty_type() == 2) { + if(c == 0x08) // BS + // Set the trick for BS output after a sequence, to delay backspace + // activation + if(pos + 2 < len && bytes[pos + 1] == 0x20 && bytes[pos + 2] == 0x08) + vt->in_backspace = 2; // Trigger when count down to 1 + } if(vt->parser.state >= STRING) more_string(vt, string_start, bytes + pos - string_start); do_control(vt, c); *** ../vim-8.1.2026/src/libvterm/src/state.c 2019-08-18 20:41:10.692526067 +0200 --- src/libvterm/src/state.c 2019-09-13 22:19:58.262938216 +0200 *************** *** 336,341 **** --- 336,346 ---- for( ; i < glyph_ends; i++) { int this_width; + if(vterm_get_special_pty_type() == 2) { + state->vt->in_backspace -= (state->vt->in_backspace > 0) ? 1 : 0; + if(state->vt->in_backspace == 1) + codepoints[i] = 0; // codepoints under this condition must be 0 + } chars[i - glyph_starts] = codepoints[i]; this_width = vterm_unicode_width(codepoints[i]); #ifdef DEBUG *************** *** 425,430 **** --- 430,441 ---- VTermPos oldpos = state->pos; + VTermScreenCell cell; + + // Preparing to see the leading byte + VTermPos leadpos = state->pos; + leadpos.col -= (leadpos.col >= 2 ? 2 : 0); + switch(control) { case 0x07: // BEL - ECMA-48 8.3.3 if(state->callbacks && state->callbacks->bell) *************** *** 434,439 **** --- 445,456 ---- case 0x08: // BS - ECMA-48 8.3.5 if(state->pos.col > 0) state->pos.col--; + if(vterm_get_special_pty_type() == 2) { + // In 2 cell letters, go back 2 cells + vterm_screen_get_cell(state->vt->screen, leadpos, &cell); + if(vterm_unicode_width(cell.chars[0]) == 2) + state->pos.col--; + } break; case 0x09: // HT - ECMA-48 8.3.60 *************** *** 1019,1024 **** --- 1036,1061 ---- row = CSI_ARG_OR(args[0], 1); col = argcount < 2 || CSI_ARG_IS_MISSING(args[1]) ? 1 : CSI_ARG(args[1]); // zero-based + if(vterm_get_special_pty_type() == 2) { + // Fix a sequence that is not correct right now + if(state->pos.row == row - 1) { + int cnt, ptr = 0; + for(cnt = 0; cnt < col - 1; ++cnt) { + VTermPos p; + VTermScreenCell c0, c1; + p.row = row - 1; + p.col = ptr; + vterm_screen_get_cell(state->vt->screen, p, &c0); + p.col++; + vterm_screen_get_cell(state->vt->screen, p, &c1); + ptr += (c1.chars[0] == (uint32_t)-1) // double cell? + ? (vterm_unicode_is_ambiguous(c0.chars[0])) // is ambiguous? + ? vterm_unicode_width(0x00a1) : 1 // &ambiwidth + : 1; // not ambiguous + } + col = ptr + 1; + } + } state->pos.row = row-1; state->pos.col = col-1; if(state->mode.origin) { *** ../vim-8.1.2026/src/libvterm/src/termscreen.c 2019-08-18 20:41:10.692526067 +0200 --- src/libvterm/src/termscreen.c 2019-09-13 22:19:58.262938216 +0200 *************** *** 770,780 **** cell->fg = intcell->pen.fg; cell->bg = intcell->pen.bg; ! if(pos.col < (screen->cols - 1) && ! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1) ! cell->width = 2; ! else ! cell->width = 1; return 1; } --- 770,797 ---- cell->fg = intcell->pen.fg; cell->bg = intcell->pen.bg; ! if(vterm_get_special_pty_type() == 2) { ! /* Get correct cell width from cell information contained in line buffer */ ! if(pos.col < (screen->cols - 1) && ! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1) { ! if(getcell(screen, pos.row, pos.col)->chars[0] == 0x20) { ! getcell(screen, pos.row, pos.col)->chars[0] = 0; ! cell->width = 2; ! } else if(getcell(screen, pos.row, pos.col)->chars[0] == 0) { ! getcell(screen, pos.row, pos.col + 1)->chars[0] = 0; ! cell->width = 1; ! } else { ! cell->width = 2; ! } ! } else ! cell->width = 1; ! } else { ! if(pos.col < (screen->cols - 1) && ! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1) ! cell->width = 2; ! else ! cell->width = 1; ! } return 1; } *** ../vim-8.1.2026/src/libvterm/src/unicode.c 2019-08-18 20:41:10.692526067 +0200 --- src/libvterm/src/unicode.c 2019-09-13 22:27:24.625145881 +0200 *************** *** 68,79 **** * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c */ - #if !defined(IS_COMBINING_FUNCTION) || !defined(WCWIDTH_FUNCTION) struct interval { int first; int last; }; // sorted list of non-overlapping intervals of non-spacing characters // generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" // Replaced by the combining table from Vim. --- 68,80 ---- * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c */ struct interval { int first; int last; }; + #if !defined(WCWIDTH_FUNCTION) || !defined(IS_COMBINING_FUNCTION) + // sorted list of non-overlapping intervals of non-spacing characters // generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" // Replaced by the combining table from Vim. *************** *** 359,364 **** --- 360,366 ---- {0X1E944, 0X1E94A}, {0XE0100, 0XE01EF} }; + #endif // auxiliary function for binary search in interval table static int bisearch(uint32_t ucs, const struct interval *table, int max) { *************** *** 379,386 **** return 0; } - #endif - /* The following two functions define the column width of an ISO 10646 * character as follows: --- 381,386 ---- *************** *** 478,483 **** --- 478,484 ---- */ static int mk_wcwidth_cjk(uint32_t ucs) { + #endif /* sorted list of non-overlapping intervals of East Asian Ambiguous * characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */ static const struct interval ambiguous[] = { *************** *** 534,539 **** --- 535,541 ---- { 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF }, { 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD } }; + #if 0 // binary search in table of non-spacing characters if (bisearch(ucs, ambiguous, *************** *** 557,562 **** --- 559,570 ---- } #endif + INTERNAL int vterm_unicode_is_ambiguous(uint32_t codepoint) + { + return (bisearch(codepoint, ambiguous, + sizeof(ambiguous) / sizeof(struct interval) - 1)) ? 1 : 0; + } + #ifdef IS_COMBINING_FUNCTION // Use a provided is_combining() function. int IS_COMBINING_FUNCTION(uint32_t codepoint); *************** *** 569,574 **** --- 577,593 ---- } #endif + #ifdef GET_SPECIAL_PTY_TYPE_FUNCTION + int GET_SPECIAL_PTY_TYPE_FUNCTION(void); + #else + # define GET_SPECIAL_PTY_TYPE_FUNCTION vterm_get_special_pty_type_placeholder + static int + vterm_get_special_pty_type_placeholder(void) + { + return 0; + } + #endif + // ################################ // ### The rest added by Paul Evans *************** *** 581,583 **** --- 600,607 ---- { return IS_COMBINING_FUNCTION(codepoint); } + + INTERNAL int vterm_get_special_pty_type(void) + { + return GET_SPECIAL_PTY_TYPE_FUNCTION(); + } *** ../vim-8.1.2026/src/libvterm/src/vterm_internal.h 2019-08-18 20:41:10.692526067 +0200 --- src/libvterm/src/vterm_internal.h 2019-09-13 22:19:58.262938216 +0200 *************** *** 212,217 **** --- 212,219 ---- VTermState *state; VTermScreen *screen; + + int in_backspace; }; struct VTermEncoding { *************** *** 259,263 **** --- 261,267 ---- int vterm_unicode_width(uint32_t codepoint); int vterm_unicode_is_combining(uint32_t codepoint); + int vterm_unicode_is_ambiguous(uint32_t codepoint); + int vterm_get_special_pty_type(void); #endif *** ../vim-8.1.2026/src/misc2.c 2019-09-04 20:59:10.487410001 +0200 --- src/misc2.c 2019-09-13 22:19:58.262938216 +0200 *************** *** 4601,4603 **** --- 4601,4622 ---- } # endif #endif + + /* + * Change the behavior of vterm. + * 0: As usual. + * 1: Windows 10 version 1809 + * The bug causes unstable handling of ambiguous width character. + * 2: Windows 10 version 1903 + * Use the wrong result because each result is different. + * 3: Windows 10 insider preview (current latest logic) + */ + int + get_special_pty_type(void) + { + #ifdef MSWIN + return get_conpty_type(); + #else + return 0; + #endif + } *** ../vim-8.1.2026/src/os_win32.c 2019-07-31 20:53:52.545182527 +0200 --- src/os_win32.c 2019-09-13 22:19:58.266938202 +0200 *************** *** 186,191 **** --- 186,192 ---- static int win32_set_archive(char_u *name); static int conpty_working = 0; + static int conpty_type = 0; static int conpty_stable = 0; static void vtp_flag_init(); *************** *** 7249,7257 **** /* * Support for pseudo-console (ConPTY) was added in windows 10 ! * version 1809 (October 2018 update). However, that version is unstable. */ #define CONPTY_FIRST_SUPPORT_BUILD MAKE_VER(10, 0, 17763) #define CONPTY_STABLE_BUILD MAKE_VER(10, 0, 32767) // T.B.D. static void --- 7250,7274 ---- /* * Support for pseudo-console (ConPTY) was added in windows 10 ! * version 1809 (October 2018 update). */ #define CONPTY_FIRST_SUPPORT_BUILD MAKE_VER(10, 0, 17763) + + /* + * ConPTY differences between versions, need different logic. + * version 1903 (May 2019 update). + */ + #define CONPTY_1903_BUILD MAKE_VER(10, 0, 18362) + + /* + * Confirm until this version. Also the logic changes. + * insider preview. + */ + #define CONPTY_INSIDER_BUILD MAKE_VER(10, 0, 18898) + + /* + * Not stable now. + */ #define CONPTY_STABLE_BUILD MAKE_VER(10, 0, 32767) // T.B.D. static void *************** *** 7281,7286 **** --- 7298,7309 ---- if (ver >= CONPTY_STABLE_BUILD) conpty_stable = 1; + if (ver <= CONPTY_INSIDER_BUILD) + conpty_type = 3; + if (ver <= CONPTY_1903_BUILD) + conpty_type = 2; + if (ver < CONPTY_FIRST_SUPPORT_BUILD) + conpty_type = 1; } #if !defined(FEAT_GUI_MSWIN) || defined(VIMDLL) || defined(PROTO) *************** *** 7503,7508 **** --- 7526,7537 ---- } int + get_conpty_type(void) + { + return conpty_type; + } + + int is_conpty_stable(void) { return conpty_stable; *** ../vim-8.1.2026/src/proto/misc2.pro 2019-09-04 20:59:10.491409987 +0200 --- src/proto/misc2.pro 2019-09-13 22:19:58.266938202 +0200 *************** *** 106,109 **** --- 106,110 ---- int mch_parse_cmd(char_u *cmd, int use_shcf, char ***argv, int *argc); int build_argv_from_string(char_u *cmd, char ***argv, int *argc); int build_argv_from_list(list_T *l, char ***argv, int *argc); + int get_special_pty_type(void); /* vim: set ft=c : */ *** ../vim-8.1.2026/src/proto/os_win32.pro 2019-04-28 22:50:36.157248454 +0200 --- src/proto/os_win32.pro 2019-09-13 22:19:58.266938202 +0200 *************** *** 76,81 **** --- 76,82 ---- int is_term_win32(void); int has_vtp_working(void); int has_conpty_working(void); + int get_conpty_type(void); int is_conpty_stable(void); void resize_console_buf(void); /* vim: set ft=c : */ *** ../vim-8.1.2026/src/version.c 2019-09-13 22:16:16.867909920 +0200 --- src/version.c 2019-09-13 22:28:01.729003238 +0200 *************** *** 759,760 **** --- 759,762 ---- { /* Add new patch number below this line */ + /**/ + 2027, /**/ -- Emacs is a nice OS - but it lacks a good text editor. That's why I am using Vim. --Anonymous /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///