From 2b7f460716e5c979817f23ba5d9e33d3ab3c9ca0 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Thu, 2 Apr 2015 09:31:41 -0300 Subject: [PATCH 1/4] eval: Add internal_refcount field to dict_T Due to the way vimscript garbage collection handles cyclic references, its not possible to rely on incrementing `dv_refcount` to prevent dicts still used internally from being collected: If a object with dv_refcount > 0 isn't reachable by vimscript code, it will be freed when `garbage_collect()` is called. Add the `internal_refcount` field to prevent this. --- src/nvim/eval.c | 12 ++++++++++-- src/nvim/eval_defs.h | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index 9d8421ef04..dae688eb5a 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -5560,8 +5560,10 @@ static int free_unref_items(int copyID) bool did_free = false; // Go through the list of dicts and free items without the copyID. + // Don't free dicts that are referenced internally. for (dict_T *dd = first_dict; dd != NULL; ) { - if ((dd->dv_copyID & COPYID_MASK) != (copyID & COPYID_MASK)) { + if ((dd->dv_copyID & COPYID_MASK) != (copyID & COPYID_MASK) + && !dd->internal_refcount) { // Free the Dictionary and ordinary items it contains, but don't // recurse into Lists and Dictionaries, they will be in the list // of dicts or list of lists. */ @@ -5671,6 +5673,7 @@ dict_T *dict_alloc(void) FUNC_ATTR_NONNULL_RET d->dv_scope = 0; d->dv_refcount = 0; d->dv_copyID = 0; + d->internal_refcount = 0; return d; } @@ -20064,6 +20067,7 @@ static inline void common_job_callbacks(dict_T *vopts, ufunc_T **on_stdout, return; } + vopts->internal_refcount++; vopts->dv_refcount++; } @@ -20097,7 +20101,11 @@ static inline void free_term_job_data(TerminalJobData *data) { if (data->on_exit) { user_func_unref(data->on_exit); } - dict_unref(data->self); + + if (data->self) { + data->self->internal_refcount--; + dict_unref(data->self); + } free(data); } diff --git a/src/nvim/eval_defs.h b/src/nvim/eval_defs.h index d2de830d6c..34a36004d6 100644 --- a/src/nvim/eval_defs.h +++ b/src/nvim/eval_defs.h @@ -111,6 +111,8 @@ struct dictvar_S { dict_T *dv_copydict; /* copied dict used by deepcopy() */ dict_T *dv_used_next; /* next dict in used dicts list */ dict_T *dv_used_prev; /* previous dict in used dicts list */ + int internal_refcount; // number of internal references to + // prevent garbage collection }; #endif // NVIM_EVAL_DEFS_H From d487dc1a9a63bb1d6d12658ed7ed24382b55a4b1 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Thu, 2 Apr 2015 11:22:24 -0300 Subject: [PATCH 2/4] eval: Ensure all job callbacks are invoked by `jobwait()` A call to `event_poll` is required to ensure the exit callback from the last job is invoked. --- src/nvim/eval.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/nvim/eval.c b/src/nvim/eval.c index dae688eb5a..4ab31985b5 100644 --- a/src/nvim/eval.c +++ b/src/nvim/eval.c @@ -10972,6 +10972,9 @@ static void f_jobwait(typval_T *argvars, typval_T *rettv) } } + // poll to ensure any pending callbacks from the last job are invoked + event_poll(0); + for (listitem_T *arg = args->lv_first; arg != NULL; arg = arg->li_next) { Job *job = NULL; if (arg->li_tv.v_type != VAR_NUMBER From 6abde99ff243d3ead24c26eff24121140a039fb7 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Thu, 2 Apr 2015 11:26:44 -0300 Subject: [PATCH 3/4] test: Remove indeterminism from `jobwait` tests - Use on_exit instead of on_stdout since there's no guarantee that the OS will send the data in time(It fails randomly in slow environments such as travis/valgrind) - Increase the timeout gap for the "jobwait with timeout" test --- test/functional/job/job_spec.lua | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/functional/job/job_spec.lua b/test/functional/job/job_spec.lua index c1c559eb34..c517ae4c1b 100644 --- a/test/functional/job/job_spec.lua +++ b/test/functional/job/job_spec.lua @@ -200,19 +200,22 @@ describe('jobs', function() it('will run callbacks while waiting', function() source([[ let g:dict = {'id': 10} - let g:l = [] - function g:dict.on_stdout(id, data) - call add(g:l, a:data[0]) + let g:exits = 0 + function g:dict.on_exit(id, code) + if a:code != 5 + throw 'Error!' + endif + let g:exits += 1 endfunction call jobwait([ - \ jobstart([&sh, '-c', 'sleep 0.010; echo 4'], g:dict), - \ jobstart([&sh, '-c', 'sleep 0.030; echo 5'], g:dict), - \ jobstart([&sh, '-c', 'sleep 0.050; echo 6'], g:dict), - \ jobstart([&sh, '-c', 'sleep 0.070; echo 7'], g:dict) + \ jobstart([&sh, '-c', 'sleep 0.010; exit 5'], g:dict), + \ jobstart([&sh, '-c', 'sleep 0.030; exit 5'], g:dict), + \ jobstart([&sh, '-c', 'sleep 0.050; exit 5'], g:dict), + \ jobstart([&sh, '-c', 'sleep 0.070; exit 5'], g:dict) \ ]) - call rpcnotify(g:channel, 'wait', g:l) + call rpcnotify(g:channel, 'wait', g:exits) ]]) - eq({'notification', 'wait', {{'4', '5', '6', '7'}}}, next_msg()) + eq({'notification', 'wait', {4}}, next_msg()) end) it('will return status codes in the order of passed ids', function() @@ -250,8 +253,8 @@ describe('jobs', function() it('will return -1 if the wait timed out', function() source([[ call rpcnotify(g:channel, 'wait', jobwait([ - \ jobstart([&sh, '-c', 'sleep 0.05; exit 4']), - \ jobstart([&sh, '-c', 'sleep 0.3; exit 5']), + \ jobstart([&sh, '-c', 'exit 4']), + \ jobstart([&sh, '-c', 'sleep 10000; exit 5']), \ ], 100)) ]]) eq({'notification', 'wait', {{4, -1}}}, next_msg()) From 617878f7473a9f2980df2627601c38fd9f5029ca Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Thu, 2 Apr 2015 11:28:24 -0300 Subject: [PATCH 4/4] runtime: Add vimexpect library and example gdb plugin This library makes it easier to script communication with interactive programs. It is similar to what the "expect" tcl extension does, but uses an object oriented API and is designed to integrate nicely with Neovim job control. --- contrib/neovim_gdb/neovim_gdb.vim | 344 ++++++++++++++++++++++++++++++ runtime/autoload/vimexpect.vim | 154 +++++++++++++ 2 files changed, 498 insertions(+) create mode 100644 contrib/neovim_gdb/neovim_gdb.vim create mode 100644 runtime/autoload/vimexpect.vim diff --git a/contrib/neovim_gdb/neovim_gdb.vim b/contrib/neovim_gdb/neovim_gdb.vim new file mode 100644 index 0000000000..d61e7bc0cc --- /dev/null +++ b/contrib/neovim_gdb/neovim_gdb.vim @@ -0,0 +1,344 @@ +sign define GdbBreakpoint text=● +sign define GdbCurrentLine text=⇒ + + +let s:gdb_port = 7778 +let s:run_gdb = "gdb -q -f build/bin/nvim" +let s:breakpoints = {} +let s:max_breakpoint_sign_id = 0 + + +let s:GdbServer = {} + + +function s:GdbServer.new(gdb) + let this = copy(self) + let this._gdb = a:gdb + return this +endfunction + + +function s:GdbServer.on_exit() + let self._gdb._server_exited = 1 +endfunction + + +let s:GdbPaused = vimexpect#State([ + \ ['Continuing.', 'continue'], + \ ['\v[\o32]{2}([^:]+):(\d+):\d+', 'jump'], + \ ['Remote communication error. Target disconnected.:', 'retry'], + \ ]) + + +function s:GdbPaused.continue(...) + call self._parser.switch(s:GdbRunning) + call self.update_current_line_sign(0) +endfunction + + +function s:GdbPaused.jump(file, line, ...) + if tabpagenr() != self._tab + " Don't jump if we are not in the debugger tab + return + endif + let window = winnr() + exe self._jump_window 'wincmd w' + let self._current_buf = bufnr('%') + let target_buf = bufnr(a:file, 1) + if bufnr('%') != target_buf + exe 'buffer ' target_buf + let self._current_buf = target_buf + endif + exe ':' a:line + let self._current_line = a:line + exe window 'wincmd w' + call self.update_current_line_sign(1) +endfunction + + +function s:GdbPaused.retry(...) + if self._server_exited + return + endif + sleep 1 + call self.attach() + call self.send('continue') +endfunction + + +let s:GdbRunning = vimexpect#State([ + \ ['\v^Breakpoint \d+', 'pause'], + \ ['\v\[Inferior\ +.{-}\ +exited\ +normally', 'disconnected'], + \ ['(gdb)', 'pause'], + \ ]) + + +function s:GdbRunning.pause(...) + call self._parser.switch(s:GdbPaused) + if !self._initialized + call self.send('set confirm off') + call self.send('set pagination off') + if !empty(self._server_addr) + call self.send('set remotetimeout 50') + call self.attach() + call s:RefreshBreakpoints() + call self.send('c') + endif + let self._initialized = 1 + endif +endfunction + + +function s:GdbRunning.disconnected(...) + if !self._server_exited && self._reconnect + " Refresh to force a delete of all watchpoints + call s:RefreshBreakpoints() + sleep 1 + call self.attach() + call self.send('continue') + endif +endfunction + + +let s:Gdb = {} + + +function s:Gdb.kill() + tunmap + tunmap + tunmap + tunmap + call self.update_current_line_sign(0) + exe 'bd! '.self._client_buf + if self._server_buf != -1 + exe 'bd! '.self._server_buf + endif + exe 'tabnext '.self._tab + tabclose + unlet g:gdb +endfunction + + +function! s:Gdb.send(data) + call jobsend(self._client_id, a:data."\") +endfunction + + +function! s:Gdb.attach() + call self.send(printf('target remote %s', self._server_addr)) +endfunction + + +function! s:Gdb.update_current_line_sign(add) + " to avoid flicker when removing/adding the sign column(due to the change in + " line width), we switch ids for the line sign and only remove the old line + " sign after marking the new one + let old_line_sign_id = get(self, '_line_sign_id', 4999) + let self._line_sign_id = old_line_sign_id == 4999 ? 4998 : 4999 + if a:add && self._current_line != -1 && self._current_buf != -1 + exe 'sign place '.self._line_sign_id.' name=GdbCurrentLine line=' + \.self._current_line.' buffer='.self._current_buf + endif + exe 'sign unplace '.old_line_sign_id +endfunction + + +function! s:Spawn(server_cmd, client_cmd, server_addr, reconnect) + if exists('g:gdb') + throw 'Gdb already running' + endif + let gdb = vimexpect#Parser(s:GdbRunning, copy(s:Gdb)) + " gdbserver port + let gdb._server_addr = a:server_addr + let gdb._reconnect = a:reconnect + let gdb._initialized = 0 + " window number that will be displaying the current file + let gdb._jump_window = 1 + let gdb._current_buf = -1 + let gdb._current_line = -1 + let gdb._has_breakpoints = 0 + let gdb._server_exited = 0 + " Create new tab for the debugging view + tabnew + let gdb._tab = tabpagenr() + " create horizontal split to display the current file and maybe gdbserver + sp + let gdb._server_buf = -1 + if type(a:server_cmd) == type('') + " spawn gdbserver in a vertical split + let server = s:GdbServer.new(gdb) + vsp | enew | let gdb._server_id = termopen(a:server_cmd, server) + let gdb._jump_window = 2 + let gdb._server_buf = bufnr('%') + endif + " go to the bottom window and spawn gdb client + wincmd j + enew | let gdb._client_id = termopen(a:client_cmd, gdb) + let gdb._client_buf = bufnr('%') + tnoremap :GdbContinuei + tnoremap :GdbNexti + tnoremap :GdbStepi + tnoremap :GdbFinishi + " go to the window that displays the current file + exe gdb._jump_window 'wincmd w' + let g:gdb = gdb +endfunction + + +function! s:Test(bang, filter) + let cmd = "GDB=1 make test" + if a:bang == '!' + let server_addr = '| vgdb' + let cmd = printf('VALGRIND=1 %s', cmd) + else + let server_addr = printf('localhost:%d', s:gdb_port) + let cmd = printf('GDBSERVER_PORT=%d %s', s:gdb_port, cmd) + endif + if a:filter != '' + let cmd = printf('TEST_SCREEN_TIMEOUT=1000000 TEST_FILTER="%s" %s', a:filter, cmd) + endif + call s:Spawn(cmd, s:run_gdb, server_addr, 1) +endfunction + + +function! s:ToggleBreak() + let file_name = bufname('%') + let file_breakpoints = get(s:breakpoints, file_name, {}) + let linenr = line('.') + if has_key(file_breakpoints, linenr) + call remove(file_breakpoints, linenr) + else + let file_breakpoints[linenr] = 1 + endif + let s:breakpoints[file_name] = file_breakpoints + call s:RefreshBreakpointSigns() + call s:RefreshBreakpoints() +endfunction + + +function! s:ClearBreak() + let s:breakpoints = {} + call s:RefreshBreakpointSigns() + call s:RefreshBreakpoints() +endfunction + + +function! s:RefreshBreakpointSigns() + let buf = bufnr('%') + let i = 5000 + while i <= s:max_breakpoint_sign_id + exe 'sign unplace '.i + let i += 1 + endwhile + let s:max_breakpoint_sign_id = 0 + let id = 5000 + for linenr in keys(get(s:breakpoints, bufname('%'), {})) + exe 'sign place '.id.' name=GdbBreakpoint line='.linenr.' buffer='.buf + let s:max_breakpoint_sign_id = id + let id += 1 + endfor +endfunction + + +function! s:RefreshBreakpoints() + if !exists('g:gdb') + return + endif + if g:gdb._parser.state() == s:GdbRunning + " pause first + call jobsend(g:gdb._client_id, "\") + endif + if g:gdb._has_breakpoints + call g:gdb.send('delete') + endif + let g:gdb._has_breakpoints = 0 + for [file, breakpoints] in items(s:breakpoints) + for linenr in keys(breakpoints) + let g:gdb._has_breakpoints = 1 + call g:gdb.send('break '.file.':'.linenr) + endfor + endfor +endfunction + + +function! s:GetExpression(...) range + let [lnum1, col1] = getpos("'<")[1:2] + let [lnum2, col2] = getpos("'>")[1:2] + let lines = getline(lnum1, lnum2) + let lines[-1] = lines[-1][:col2 - 1] + let lines[0] = lines[0][col1 - 1:] + return join(lines, "\n") +endfunction + + +function! s:Send(data) + if !exists('g:gdb') + throw 'Gdb is not running' + endif + call g:gdb.send(a:data) +endfunction + + +function! s:Eval(expr) + call s:Send(printf('print %s', a:expr)) +endfunction + + +function! s:Watch(expr) + let expr = a:expr + if expr[0] != '&' + let expr = '&' . expr + endif + + call s:Eval(expr) + call s:Send('watch *$') +endfunction + + +function! s:Interrupt() + if !exists('g:gdb') + throw 'Gdb is not running' + endif + call jobsend(g:gdb._client_id, "\info line\") +endfunction + + +function! s:Kill() + if !exists('g:gdb') + throw 'Gdb is not running' + endif + call g:gdb.kill() +endfunction + + +command! GdbDebugNvim call s:Spawn(printf('make && gdbserver localhost:%d build/bin/nvim', s:gdb_port), s:run_gdb, printf('localhost:%d', s:gdb_port), 0) +command! -nargs=1 GdbDebugServer call s:Spawn(0, s:run_gdb, 'localhost:'., 0) +command! -bang -nargs=? GdbDebugTest call s:Test(, ) +command! -nargs=1 -complete=file GdbInspectCore call s:Spawn(0, printf('gdb -q -f -c %s build/bin/nvim', ), 0, 0) +command! GdbDebugStop call s:Kill() +command! GdbToggleBreakpoint call s:ToggleBreak() +command! GdbClearBreakpoints call s:ClearBreak() +command! GdbContinue call s:Send("c") +command! GdbNext call s:Send("n") +command! GdbStep call s:Send("s") +command! GdbFinish call s:Send("finish") +command! GdbFrameUp call s:Send("up") +command! GdbFrameDown call s:Send("down") +command! GdbInterrupt call s:Interrupt() +command! GdbEvalWord call s:Eval(expand('')) +command! -range GdbEvalRange call s:Eval(s:GetExpression()) +command! GdbWatchWord call s:Watch(expand('') +command! -range GdbWatchRange call s:Watch(s:GetExpression()) + + +nnoremap :GdbContinue +nnoremap :GdbNext +nnoremap :GdbStep +nnoremap :GdbFinish +nnoremap :GdbToggleBreakpoint +nnoremap :GdbFrameUp +nnoremap :GdbFrameDown +nnoremap :GdbEvalWord +vnoremap :GdbEvalRange +nnoremap :GdbWatchWord +vnoremap :GdbWatchRange diff --git a/runtime/autoload/vimexpect.vim b/runtime/autoload/vimexpect.vim new file mode 100644 index 0000000000..16e7d30d6c --- /dev/null +++ b/runtime/autoload/vimexpect.vim @@ -0,0 +1,154 @@ +" vimexpect.vim is a small object-oriented library that simplifies the task of +" scripting communication with jobs or any interactive program. The name +" `expect` comes from the famous tcl extension that has the same purpose. +" +" This library is built upon two simple concepts: Parsers and States. +" +" A State represents a program state and associates a set of regular +" expressions(to parse program output) with method names(to deal with parsed +" output). States are created with the vimexpect#State(patterns) function. +" +" A Parser manages data received from the program. It also manages State +" objects by storing them into a stack, where the top of the stack is the +" current State. Parsers are created with the vimexpect#Parser(initial_state, +" target) function +" +" The State methods are defined by the user, and are always called with `self` +" set as the Parser target. Advanced control flow is achieved by changing the +" current state with the `push`/`pop`/`switch` parser methods. +" +" An example of this library in action can be found in Neovim source +" code(contrib/neovim_gdb subdirectory) + +let s:State = {} + + +" Create a new State instance with a list where each item is a [regexp, name] +" pair. A method named `name` must be defined in the created instance. +function s:State.create(patterns) + let this = copy(self) + let this._patterns = a:patterns + return this +endfunction + + +let s:Parser = {} +let s:Parser.LINE_BUFFER_MAX_LEN = 100 + + +" Create a new Parser instance with the initial state and a target. The target +" is a dictionary that will be the `self` of every State method call associated +" with the parser, and may contain options normally passed to +" `jobstart`(on_stdout/on_stderr will be overriden). Returns the target so it +" can be called directly as the second argument of `jobstart`: +" +" call jobstart(prog_argv, vimexpect#Parser(initial_state, {'pty': 1})) +function s:Parser.create(initial_state, target) + let parser = copy(self) + let parser._line_buffer = [] + let parser._stack = [a:initial_state] + let parser._target = a:target + let parser._target.on_stdout = function('s:JobOutput') + let parser._target.on_stderr = function('s:JobOutput') + let parser._target._parser = parser + return parser._target +endfunction + + +" Push a state to the state stack +function s:Parser.push(state) + call add(self._stack, a:state) +endfunction + + +" Pop a state from the state stack. Fails if there's only one state remaining. +function s:Parser.pop() + if len(self._stack) == 1 + throw 'vimexpect:emptystack:State stack cannot be empty' + endif + return remove(self._stack, -1) +endfunction + + +" Replace the state currently in the top of the stack. +function s:Parser.switch(state) + let old_state = self._stack[-1] + let self._stack[-1] = a:state + return old_state +endfunction + + +" Append a list of lines to the parser line buffer and try to match it the +" current state. This will shift old lines if the buffer crosses its +" limit(defined by the LINE_BUFFER_MAX_LEN field). During normal operation, +" this function is called by the job handler provided by this module, but it +" may be called directly by the user for other purposes(testing for example) +function s:Parser.feed(lines) + if empty(a:lines) + return + endif + let lines = a:lines + let linebuf = self._line_buffer + if lines[0] != "\n" && !empty(linebuf) + " continue the previous line + let linebuf[-1] .= lines[0] + call remove(lines, 0) + endif + " append the newly received lines to the line buffer + let linebuf += lines + " keep trying to match handlers while the line isnt empty + while !empty(linebuf) + let match_idx = self.parse(linebuf) + if match_idx == -1 + break + endif + let linebuf = linebuf[match_idx + 1 : ] + endwhile + " shift excess lines from the buffer + while len(linebuf) > self.LINE_BUFFER_MAX_LEN + call remove(linebuf, 0) + endwhile + let self._line_buffer = linebuf +endfunction + + +" Try to match a list of lines with the current state and call the handler if +" the match succeeds. Return the index in `lines` of the first match. +function s:Parser.parse(lines) + let lines = a:lines + if empty(lines) + return -1 + endif + let state = self.state() + " search for a match using the list of patterns + for [pattern, handler] in state._patterns + let matches = matchlist(lines, pattern) + if empty(matches) + continue + endif + let match_idx = match(lines, pattern) + call call(state[handler], matches[1:], self._target) + return match_idx + endfor +endfunction + + +" Return the current state +function s:Parser.state() + return self._stack[-1] +endfunction + + +" Job handler that simply forwards lines to the parser. +function! s:JobOutput(id, lines) + call self._parser.feed(a:lines) +endfunction + +function vimexpect#Parser(initial_state, target) + return s:Parser.create(a:initial_state, a:target) +endfunction + + +function vimexpect#State(patterns) + return s:State.create(a:patterns) +endfunction