To: vim_dev@googlegroups.com
Subject: Patch 8.1.0529
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------

Patch 8.1.0529
Problem:    Flaky test sometimes fails in different ways.
Solution:   When the second run gives a different error, try running the test
            again, up to five times.
Files:	    src/testdir/runtest.vim


*** ../vim-8.1.0528/src/testdir/runtest.vim	2018-11-11 18:51:39.289611371 +0100
--- src/testdir/runtest.vim	2018-11-16 16:50:36.895763420 +0100
***************
*** 314,341 ****
  for s:test in sort(s:tests)
    " Silence, please!
    set belloff=all
  
    call RunTheTest(s:test)
  
    if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
!     call add(s:messages, 'Found errors in ' . s:test . ':')
!     call extend(s:messages, v:errors)
!     call add(s:messages, 'Flaky test failed, running it again')
!     let first_run = v:errors
! 
!     " Flakiness is often caused by the system being very busy.  Sleep a couple
!     " of seconds to have a higher chance of succeeding the second time.
!     sleep 2
! 
!     let v:errors = []
!     call RunTheTest(s:test)
!     if len(v:errors) > 0
!       let second_run = v:errors
!       let v:errors = ['First run:']
!       call extend(v:errors, first_run)
!       call add(v:errors, 'Second run:')
!       call extend(v:errors, second_run)
!     endif
    endif
  
    call AfterTheTest()
--- 314,360 ----
  for s:test in sort(s:tests)
    " Silence, please!
    set belloff=all
+   let prev_error = ''
+   let total_errors = []
+   let run_nr = 1
  
    call RunTheTest(s:test)
  
+   " Repeat a flaky test.  Give up when:
+   " - it fails again with the same message
+   " - it fails five times (with a different mesage)
    if len(v:errors) > 0 && index(s:flaky, s:test) >= 0
!     while 1
!       call add(s:messages, 'Found errors in ' . s:test . ':')
!       call extend(s:messages, v:errors)
! 
!       call add(total_errors, 'Run ' . run_nr . ':')
!       call extend(total_errors, v:errors)
! 
!       if run_nr == 5 || prev_error == v:errors[0]
!         call add(total_errors, 'Flaky test failed too often, giving up')
!         let v:errors = total_errors
!         break
!       endif
! 
!       call add(s:messages, 'Flaky test failed, running it again')
! 
!       " Flakiness is often caused by the system being very busy.  Sleep a
!       " couple of seconds to have a higher chance of succeeding the second
!       " time.
!       sleep 2
! 
!       let prev_error = v:errors[0]
!       let v:errors = []
!       let run_nr += 1
! 
!       call RunTheTest(s:test)
! 
!       if len(v:errors) == 0
!         " Test passed on rerun.
!         break
!       endif
!     endwhile
    endif
  
    call AfterTheTest()
*** ../vim-8.1.0528/src/version.c	2018-11-16 16:21:01.645310017 +0100
--- src/version.c	2018-11-16 16:51:26.583470837 +0100
***************
*** 794,795 ****
--- 794,797 ----
  {   /* Add new patch number below this line */
+ /**/
+     529,
  /**/

-- 
Life is a gift, living is an art.		(Bram Moolenaar)

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\  an exciting new programming language -- http://www.Zimbu.org        ///
 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///