diff --git a/runtime/lua/vim/lsp/semantic_tokens.lua b/runtime/lua/vim/lsp/semantic_tokens.lua
index b6b09c58b1..376cac19a7 100644
--- a/runtime/lua/vim/lsp/semantic_tokens.lua
+++ b/runtime/lua/vim/lsp/semantic_tokens.lua
@@ -99,6 +99,7 @@ local function tokens_to_ranges(data, bufnr, client, request)
   local legend = client.server_capabilities.semanticTokensProvider.legend
   local token_types = legend.tokenTypes
   local token_modifiers = legend.tokenModifiers
+  local lines = api.nvim_buf_get_lines(bufnr, 0, -1, false)
   local ranges = {}
 
   local start = uv.hrtime()
@@ -137,11 +138,17 @@ local function tokens_to_ranges(data, bufnr, client, request)
     local modifiers = modifiers_from_number(data[i + 4], token_modifiers)
 
     ---@private
-    local function _get_byte_pos(char_pos)
-      return util._get_line_byte_from_position(bufnr, {
-        line = line,
-        character = char_pos,
-      }, client.offset_encoding)
+    local function _get_byte_pos(col)
+      if col > 0 then
+        local buf_line = lines[line + 1] or ''
+        local ok, result
+        ok, result = pcall(util._str_byteindex_enc, buf_line, col, client.offset_encoding)
+        if ok then
+          return result
+        end
+        return math.min(#buf_line, col)
+      end
+      return col
     end
 
     local start_col = _get_byte_pos(start_char)
diff --git a/runtime/lua/vim/lsp/util.lua b/runtime/lua/vim/lsp/util.lua
index 31af2afb0b..8274361f6d 100644
--- a/runtime/lua/vim/lsp/util.lua
+++ b/runtime/lua/vim/lsp/util.lua
@@ -253,12 +253,17 @@ local function get_lines(bufnr, rows)
   ---@private
   local function buf_lines()
     local lines = {}
-    for _, row in pairs(rows) do
+    for _, row in ipairs(rows) do
       lines[row] = (api.nvim_buf_get_lines(bufnr, row, row + 1, false) or { '' })[1]
     end
     return lines
   end
 
+  -- use loaded buffers if available
+  if vim.fn.bufloaded(bufnr) == 1 then
+    return buf_lines()
+  end
+
   local uri = vim.uri_from_bufnr(bufnr)
 
   -- load the buffer if this is not a file uri
@@ -268,11 +273,6 @@ local function get_lines(bufnr, rows)
     return buf_lines()
   end
 
-  -- use loaded buffers if available
-  if vim.fn.bufloaded(bufnr) == 1 then
-    return buf_lines()
-  end
-
   local filename = api.nvim_buf_get_name(bufnr)
 
   -- get the data from the file