fix(previewer): improve file_maker line splitting and timeouts

Author: jamestrew

lua/telescope/previewers/buffer_previewer.lua

@@ -60,25 +60,6 @@ local function defaulter(f, default_opts)
   }
 end
 
--- modified vim.split to incorporate a timer
-local function split(s, sep, plain, opts)
-  opts = opts or {}
-  local t = {}
-  for c in vim.gsplit(s, sep, plain) do
-    local line = opts.file_encoding and vim.iconv(c, opts.file_encoding, "utf8") or c
-    table.insert(t, line)
-    if opts.preview.timeout then
-      local diff_time = (vim.uv.hrtime() - opts.start_time) / 1e6
-      if diff_time > opts.preview.timeout then
-        return
-      end
-    end
-  end
-  if t[#t] == "" then
-    t[#t] = nil
-  end
-  return t
-end
 local bytes_to_megabytes = math.pow(1024, 2)
 
 local color_hash = {
@@ -205,7 +186,7 @@ local handle_file_preview = function(filepath, bufnr, stat, opts)
       if not api.nvim_buf_is_valid(bufnr) then
         return
       end
-      local processed_data = split(data, "[\r]?\n", nil, opts)
+      local processed_data = putils.timed_split_lines(data, opts)
 
       if processed_data then
         local ok = pcall(api.nvim_buf_set_lines, bufnr, 0, -1, false, processed_data)

lua/telescope/previewers/utils.lua

@@ -235,4 +235,49 @@ utils.binary_mime_type = function(mime_type)
   return true
 end
 
+local CHECK_TIME_INTERVAL = 200
+
+--- Split a string into lines, checking every `CHECK_TIME_INTERVAL` characters
+--- whether to timeout.
+---
+--- Roughly 4-5x faster than using `vim.gsplit` and checking timeout between each line.
+--- The latter approach is also more prone to exceeding timeout if a file has huge lines.
+---@param s string file content to split into lines
+---@param opts {start_time: number, preview: { timeout: number }, file_encoding: string?}
+---@return string[]?
+function utils.timed_split_lines(s, opts)
+  local lines = {}
+  local line_start = 1
+
+  for i = 1, #s do
+    local ch = s:byte(i)
+    if ch == 10 then
+      local line
+      if s:byte(i - 1) ~= 13 then
+        line = s:sub(line_start, i - 1)
+      else
+        line = s:sub(line_start, i - 2)
+      end
+      line_start = i + 1
+      table.insert(lines, opts.file_encoding and vim.iconv(line, opts.file_encoding, "utf8") or line)
+    end
+
+    if i % CHECK_TIME_INTERVAL == 0 then
+      local diff_time = (vim.uv.hrtime() - opts.start_time) / 1e6
+      if diff_time > opts.preview.timeout then
+        return
+      end
+    end
+  end
+
+  -- Only append the tail when it is non-empty.
+  -- neovim treats \n and \r\n as "line terminators" instead of "line separator"
+  local tail = s:sub(line_start)
+  if tail ~= "" then
+    table.insert(lines, opts.file_encoding and vim.iconv(tail, opts.file_encoding, "utf8") or tail)
+  end
+
+  return lines
+end
+
 return utils

lua/telescope/utils.lua

@@ -766,16 +766,8 @@ utils.reverse_table = function(input_table)
   return temp_table
 end
 
-utils.split_lines = (function()
-  if utils.iswin then
-    return function(s, opts)
-      return vim.split(s, "\r?\n", opts)
-    end
-  else
-    return function(s, opts)
-      return vim.split(s, "\n", opts)
-    end
-  end
-end)()
+utils.split_lines = function(s, opts)
+  return vim.split(s, "\r?\n", opts)
+end
 
 return utils

lua/tests/automated/previewer_spec.lua

@@ -0,0 +1,55 @@
+local putils = require "telescope.previewers.utils"
+local utils = require "telescope.utils"
+
+describe("timed_split_lines", function()
+  local expect = {
+    "",
+    "",
+    "line3 of the file",
+    "",
+    "line5 of the file",
+    "",
+    "",
+    "line8 of the file, last line of file",
+  }
+
+  local split_lines = function(s)
+    return putils.timed_split_lines(s, {
+      start_time = vim.uv.hrtime(),
+      preview = {
+        timeout = 250, -- should be more than enough time
+      },
+    })
+  end
+
+  if utils.iswin then
+    describe("handles files on Windows", function()
+      it("reads file ending with \\r\\n (standard Windows line terminator)", function()
+        local file = table.concat(expect, "\r\n") .. "\r\n"
+        assert.are.same(expect, split_lines(file))
+      end)
+
+      it("reads file ending with \\n only", function()
+        local file = table.concat(expect, "\n") .. "\n"
+        assert.are.same(expect, split_lines(file))
+      end)
+
+      it("reads file with no trailing newline", function()
+        local file = table.concat(expect, "\r\n")
+        assert.are.same(expect, split_lines(file))
+      end)
+    end)
+  else
+    describe("handles files on non Windows environment", function()
+      it("reads file ending with \\n (standard Unix line terminator)", function()
+        local file = table.concat(expect, "\n") .. "\n"
+        assert.are.same(expect, split_lines(file))
+      end)
+
+      it("reads file with no trailing newline", function()
+        local file = table.concat(expect, "\n")
+        assert.are.same(expect, split_lines(file))
+      end)
+    end)
+  end
+end)