diff --git a/src/nvim/lua/stdlib.c b/src/nvim/lua/stdlib.c
index e719d99640..5ebff3a809 100644
--- a/src/nvim/lua/stdlib.c
+++ b/src/nvim/lua/stdlib.c
@@ -619,6 +619,7 @@ static int nlua_with(lua_State *L)
   int rets = 0;
 
   cmdmod_T save_cmdmod = cmdmod;
+  CLEAR_FIELD(cmdmod);
   cmdmod.cmod_flags = flags;
   apply_cmdmod(&cmdmod);
 
diff --git a/test/functional/lua/with_spec.lua b/test/functional/lua/with_spec.lua
index 6127e83619..92e798e7f3 100644
--- a/test/functional/lua/with_spec.lua
+++ b/test/functional/lua/with_spec.lua
@@ -1621,4 +1621,21 @@ describe('vim._with', function()
     matches('Invalid buffer', get_error('{ buf = -1 }, function() end'))
     matches('Invalid window', get_error('{ win = -1 }, function() end'))
   end)
+
+  it('no double-free when called from :filter browse oldfiles #31501', function()
+    exec_lua([=[
+      vim.api.nvim_create_autocmd('BufEnter', {
+        callback = function()
+          vim._with({ lockmarks = true }, function() end)
+        end,
+      })
+      vim.cmd([[
+        let v:oldfiles = ['Xoldfile']
+        call nvim_input('1<CR>')
+        noswapfile filter /Xoldfile/ browse oldfiles
+      ]])
+    ]=])
+    n.assert_alive()
+    eq('Xoldfile', fn.bufname('%'))
+  end)
 end)