perf: new file-based cache that ensures rtp is alweays correct and will cache all files, including those after startup

lua/lazy/core/cache.lua

@@ -1,129 +1,208 @@
 local ffi = require("ffi")
----@diagnostic disable-next-line: no-unknown
 local uv = vim.loop
 
 local M = {}
-M.dirty = false
-M.VERSION = "1" .. jit.version
-
----@class LazyCacheConfig
-M.config = {
-  enabled = true,
-  path = vim.fn.stdpath("cache") .. "/lazy/cache",
-  -- Once one of the following events triggers, caching will be disabled.
-  -- To cache all modules, set this to `{}`, but that is not recommended.
-  -- The default is to disable on:
-  --  * VimEnter: not useful to cache anything else beyond startup
-  --  * BufReadPre: this will be triggered early when opening a file from the command line directly
-  disable_events = { "UIEnter", "BufReadPre" },
-  ttl = 3600 * 24 * 5, -- keep unused modules for up to 5 days
-}
-M.debug = false
-
----@type CacheHash
-local cache_hash
 
 ---@alias CacheHash {mtime: {sec:number, nsec:number}, size:number}
----@alias CacheEntry {hash:CacheHash, modpath:string, chunk:string, used:number}
+---@alias CacheEntry {hash:CacheHash, modpath:string, chunk:string}
----@type table<string,CacheEntry?>
+
-M.cache = {}
+M.VERSION = 1
-M.enabled = true
+
----@type string[]
+M.config = {
-M.rtp = nil
+  enabled = false,
-M.rtp_total = 0
+  path = vim.fn.stdpath("cache") .. "/lazy/luac",
-M.stats = {
-  find = { total = 0, time = 0, rtp = 0, unloaded = 0, index = 0, stat = 0, not_found = 0 },
-  autoload = { total = 0, time = 0 },
 }
-M.me = debug.getinfo(1, "S").source:sub(2)
+M._loadfile = loadfile
-M.me = vim.fn.fnamemodify(M.me, ":p:h:h:h:h"):gsub("\\", "/")
----@type table<string, string[]>
-M.topmods = { lazy = { M.me } }
----@type table<string, string[]>
-M.indexed = { [M.me] = { "lazy" } }
-M.indexed_unloaded = false
-M.indexed_rtp = 0
--- selene:allow(global_usage)
-M._loadfile = _G.loadfile
 
--- checks whether the cached modpath is still valid
+M.stats = {
-function M.check_path(modname, modpath)
+  find = { total = 0, time = 0, index = 0, stat = 0, not_found = 0 },
-  -- HACK: never return packer paths
+}
-  if modpath:find("/site/pack/packer/", 1, true) then
+---@type string
-    return false
+M._rtp_key = nil
+---@type string[]
+M._rtp = nil
+---@type table<string, table<string,true>>
+M._topmods = {}
+
+function M.get_rtp()
+  if vim.in_fast_event() then
+    return M._rtp or {}
   end
-
+  local key = vim.go.rtp
-  -- check rtp excluding plugins. This is a very small list, so should be fast
+  if vim.go.rtp ~= M._rtp_key then
-  for _, path in ipairs(M.get_rtp()) do
+    M._rtp = {}
-    if modpath:find(path .. "/", 1, true) == 1 then
+    for _, path in ipairs(vim.api.nvim_get_runtime_file("", true)) do
-      return true
+      path = M.normalize(path)
-    end
+      -- skip after directories
-  end
+      if path:sub(-6, -1) ~= "/after" then
-
+        M._rtp[#M._rtp + 1] = path
-  -- the correct lazy path should be part of rtp.
+      end
-  -- so if we get here, this is folke using the local dev instance ;)
+    end
-  if modname and (modname == "lazy" or modname:sub(1, 5) == "lazy.") then
+    M._rtp_key = key
-    return false
+  end
-  end
+  return M._rtp
-
+end
-  return modname and M.check_autoload(modname, modpath)
+
-end
+-- slightly faster/different version than vim.fs.normalize
-
+-- we also need to have it here, since the cache will load vim.fs
-function M.check_autoload(modname, modpath)
+function M.normalize(path)
-  local start = uv.hrtime()
+  if path:sub(1, 1) == "~" then
-  M.stats.autoload.total = M.stats.autoload.total + 1
+    local home = vim.loop.os_homedir()
-
+    if home:sub(-1) == "\\" or home:sub(-1) == "/" then
-  -- check plugins. Again fast, since we check the plugin name from the path.
+      home = home:sub(1, -2)
-  -- only needed when the plugin mod has been loaded
+    end
-  ---@type LazyCorePlugin
+    path = home .. path:sub(2)
-  local Plugin = package.loaded["lazy.core.plugin"]
+  end
-  if Plugin then
+  path = path:gsub("\\", "/"):gsub("/+", "/")
-    local plugin = Plugin.find(modpath)
+  return path:sub(-1) == "/" and path:sub(1, -2) or path
-    if plugin and modpath:find(plugin.dir, 1, true) == 1 then
+end
-      -- we're not interested in loader time, so calculate delta here
+
-      M.stats.autoload.time = M.stats.autoload.time + uv.hrtime() - start
+function M.reset(path)
-      -- don't load if we're loading specs or if the plugin is already loaded
+  M._topmods[M.normalize(path)] = nil
-      if not (Plugin.loading or plugin._.loaded) then
+end
-        if plugin.module == false then
+
-          error("Plugin " .. plugin.name .. " is not loaded and is configured with module=false")
+-- index the top-level lua modules for this path
-        end
+---@return string[]
-        require("lazy.core.loader").load(plugin, { require = modname })
+function M.get_topmods(path)
+  if not M._topmods[path] then
+    M.stats.find.index = M.stats.find.index + 1
+    M._topmods[path] = {}
+    local handle = vim.loop.fs_scandir(path .. "/lua")
+    while handle do
+      local name, t = vim.loop.fs_scandir_next(handle)
+      if not name then
+        break
+      end
+      ---@cast name string
+      ---@cast t "file"|"directory"|"link"
+      -- HACK: type is not always returned due to a bug in luv
+      t = t or vim.loop.fs_stat(path .. "/" .. name).type
+      ---@type string
+      local topname
+      if name:sub(-4) == ".lua" then
+        topname = name:sub(1, -5)
+      elseif t == "link" or t == "directory" then
+        topname = name
+      end
+      if topname then
+        M._topmods[path][topname] = true
       end
-      return true
     end
   end
-  M.stats.autoload.time = M.stats.autoload.time + uv.hrtime() - start
+  return M._topmods[path]
-  return false
 end
 
-function M.disable()
+---@param modname string
-  if not M.enabled then
+---@param opts? {rtp:string[], patterns:string[]}
-    return
+---@return string?, CacheHash?
+function M.find(modname, opts)
+  opts = opts or {}
+  local start = uv.hrtime()
+  M.stats.find.total = M.stats.find.total + 1
+  modname = modname:gsub("/", ".")
+  local basename = modname:gsub("%.", "/")
+  local idx = modname:find(".", 1, true)
+  local topmod = idx and modname:sub(1, idx - 1) or modname
+
+  -- OPTIM: search for a directory first when topmod == modname
+  local patterns = opts.patterns or (topmod == modname and { "/init.lua", ".lua" } or { ".lua", "/init.lua" })
+  local rtp = opts.rtp or M.get_rtp()
+
+  for _, path in ipairs(rtp) do
+    if M.get_topmods(path)[topmod] then
+      for _, pattern in ipairs(patterns) do
+        local modpath = path .. "/lua/" .. basename .. pattern
+        M.stats.find.stat = M.stats.find.stat + 1
+        local hash = uv.fs_stat(modpath)
+        if hash then
+          M.stats.find.time = M.stats.find.time + uv.hrtime() - start
+          return modpath, hash
+        end
+      end
+    end
   end
-  -- selene:allow(global_usage)
+
-  _G.loadfile = M._loadfile
+  -- module not found
-  M.enabled = false
+  M.stats.find.not_found = M.stats.find.not_found + 1
-  if M.debug and vim.tbl_count(M.topmods) > 1 then
+  M.stats.find.time = M.stats.find.time + uv.hrtime() - start
-    M.log(M.topmods, { level = vim.log.levels.WARN, title = "topmods" })
+end
-  end
+
-  if M.debug and false then
+function M.setup()
-    local stats = vim.deepcopy(M.stats)
+  M.config.enabled = true
-    stats.time = (stats.time or 0) / 1e6
+  vim.fn.mkdir(vim.fn.fnamemodify(M.config.path, ":p"), "p")
-    stats.find.time = (stats.find.time or 0) / 1e6
+  _G.loadfile = M.loadfile
-    stats.autoload.time = (stats.autoload.time or 0) / 1e6
+  table.insert(package.loaders, 2, M.loader)
-    M.log(stats, { title = "stats" })
+end
+
+---@param name string can be a module name, or a file name
+function M.cache_file(name)
+  return M.config.path .. "/" .. name:gsub("[/\\]", "%%") .. ".luac"
+end
+
+---@param h1 CacheHash
+---@param h2 CacheHash
+function M.eq(h1, h2)
+  return h1 and h2 and h1.size == h2.size and h1.mtime.sec == h2.mtime.sec and h1.mtime.nsec == h2.mtime.nsec
+end
+
+---@param entry CacheEntry
+function M.write(name, entry)
+  local cname = M.cache_file(name)
+  local f = assert(uv.fs_open(cname, "w", 438))
+  local header = {
+    M.VERSION,
+    entry.hash.size,
+    entry.hash.mtime.sec,
+    entry.hash.mtime.nsec,
+    #entry.modpath,
+  }
+  uv.fs_write(f, ffi.string(ffi.new("const uint32_t[5]", header), 20))
+  uv.fs_write(f, entry.modpath)
+  uv.fs_write(f, entry.chunk)
+  uv.fs_close(f)
+end
+
+---@return CacheEntry?
+function M.read(name)
+  local cname = M.cache_file(name)
+  local f = uv.fs_open(cname, "r", 438)
+  if f then
+    local hash = uv.fs_fstat(f) --[[@as CacheHash]]
+    local data = uv.fs_read(f, hash.size, 0) --[[@as string]]
+    uv.fs_close(f)
+
+    ---@type integer[]|{[0]:integer}
+    local header = ffi.cast("uint32_t*", ffi.new("const char[20]", data:sub(1, 20)))
+    if header[0] ~= M.VERSION then
+      return
+    end
+    local modpath = data:sub(21, 20 + header[4])
+    return {
+      hash = { size = header[1], mtime = { sec = header[2], nsec = header[3] } },
+      chunk = data:sub(20 + header[4] + 1),
+      modpath = modpath,
+    }
   end
 end
 
----@param msg string|table
+---@param modname string
----@param opts? LazyNotifyOpts
+function M.loader(modname)
-function M.log(msg, opts)
+  modname = modname:gsub("/", ".")
-  if M.debug then
+
-    msg = vim.deepcopy(msg)
+  local modpath, hash = M.find(modname)
-    vim.schedule(function()
+  ---@type function?, string?
-      require("lazy.core.util").debug(msg, opts)
+  local chunk, err
-    end)
+  if modpath then
+    chunk, err = M._load(modname, modpath, { hash = hash })
   end
+  return chunk or err or ("module " .. modname .. " not found")
+end
+
+---@param modpath string
+---@return any, string?
+function M.loadfile(modpath)
+  modpath = M.normalize(modpath)
+  return M._load(modpath, modpath)
 end
 
 function M.check_loaded(modname)
@@ -136,389 +215,43 @@ function M.check_loaded(modname)
   end
 end
 
----@param modname string
----@return fun()|string
-function M.loader(modname)
-  modname = modname:gsub("/", ".")
-  local entry = M.cache[modname]
-
-  local chunk, err
-  if entry then
-    if M.check_path(modname, entry.modpath) then
-      M.stats.find.total = M.stats.find.total + 1
-      chunk, err = M.load(modname, entry.modpath)
-    else
-      M.cache[modname] = nil
-      M.dirty = true
-    end
-  end
-  if not chunk then
-    -- find the modpath and load the module
-    local modpath = M.find(modname)
-    if modpath then
-      M.check_autoload(modname, modpath)
-      if M.enabled then
-        chunk, err = M.load(modname, modpath)
-      else
-        chunk = M.check_loaded(modname)
-        if not chunk then
-          chunk, err = M._loadfile(modpath)
-        end
-      end
-    end
-  end
-  return chunk or err or ("module " .. modname .. " not found")
-end
-
----@param modpath string
----@return any, string?
-function M.loadfile(modpath)
-  modpath = modpath:gsub("\\", "/")
-  return M.load(modpath, modpath)
-end
-
 ---@param modkey string
 ---@param modpath string
+---@param opts? {hash?: CacheHash, entry?:CacheEntry}
 ---@return function?, string? error_message
-function M.load(modkey, modpath)
+function M._load(modkey, modpath, opts)
+  opts = opts or {}
+  if not M.config.enabled then
+    return M._loadfile(modpath)
+  end
+  ---@type function?, string?
   local chunk, err
   chunk = M.check_loaded(modkey)
   if chunk then
     return chunk
   end
-  modpath = modpath:gsub("\\", "/")
+  local hash = opts.hash or uv.fs_stat(modpath)
-  local hash = M.hash(modpath)
   if not hash then
     -- trigger correct error
     return M._loadfile(modpath)
   end
 
-  local entry = M.cache[modkey]
+  local entry = opts.entry or M.read(modkey)
-  if entry then
+  if entry and M.eq(entry.hash, hash) then
-    entry.modpath = modpath
+    -- found in cache and up to date
-    entry.used = os.time()
+    chunk, err = loadstring(entry.chunk --[[@as string]], "@" .. entry.modpath)
-    if M.eq(entry.hash, hash) then
+    if not (err and err:find("cannot load incompatible bytecode", 1, true)) then
-      -- found in cache and up to date
+      return chunk, err
-      chunk, err = loadstring(entry.chunk --[[@as string]], "@" .. entry.modpath)
-      if not (err and err:find("cannot load incompatible bytecode", 1, true)) then
-        return chunk, err
-      end
     end
-  else
-    entry = { hash = hash, modpath = modpath, used = os.time() }
-    M.cache[modkey] = entry
-  end
-  entry.hash = hash
-
-  if M.debug then
-    M.log("`" .. modpath .. "`", { level = vim.log.levels.WARN, title = "Cache.load" })
   end
+  entry = { hash = hash, modpath = modpath }
 
   chunk, err = M._loadfile(entry.modpath)
-  M.dirty = true
   if chunk then
     entry.chunk = string.dump(chunk)
-  else
+    M.write(modkey, entry)
-    M.cache[modkey] = nil
   end
   return chunk, err
 end
 
--- index the top-level lua modules for this path
-function M._index(path)
-  if not M.indexed[path] and path:sub(-6, -1) ~= "/after" then
-    M.stats.find.index = M.stats.find.index + 1
-    ---@type LazyUtilCore
-    local Util = package.loaded["lazy.core.util"]
-    if not Util then
-      return false
-    end
-    M.indexed[path] = {}
-    Util.ls(path .. "/lua", function(_, name, t)
-      local topname
-      if name:sub(-4) == ".lua" then
-        topname = name:sub(1, -5)
-      elseif t == "link" or t == "directory" then
-        topname = name
-      end
-      if topname then
-        M.topmods[topname] = M.topmods[topname] or {}
-        if not vim.tbl_contains(M.topmods[topname], path) then
-          table.insert(M.topmods[topname], path)
-        end
-        if not vim.tbl_contains(M.indexed[path], topname) then
-          table.insert(M.indexed[path], topname)
-        end
-      end
-    end)
-    return true
-  end
-  return false
-end
-
-function M.get_topmods(path)
-  M._index(path)
-  return M.indexed[path] or {}
-end
-
----@param modname string
----@return string?
-function M.find_root(modname)
-  if M.cache[modname] then
-    -- check if modname is in cache
-    local modpath = M.cache[modname].modpath
-    if M.check_path(modname, modpath) and uv.fs_stat(modpath) then
-      local root = modpath:gsub("/init%.lua$", ""):gsub("%.lua$", "")
-      return root
-    end
-  else
-    -- in case modname is just a directory and not a real mod,
-    -- check for any children in the cache
-    for child, entry in pairs(M.cache) do
-      if child:find(modname, 1, true) == 1 then
-        if M.check_path(child, entry.modpath) and uv.fs_stat(entry.modpath) then
-          local basename = modname:gsub("%.", "/")
-          local childbase = child:gsub("%.", "/")
-          local ret = entry.modpath:gsub("/init%.lua$", ""):gsub("%.lua$", "")
-          local idx = assert(ret:find(childbase, 1, true))
-          return ret:sub(1, idx - 1) .. basename
-        end
-      end
-    end
-  end
-
-  -- not found in cache, so find the root with the special pattern
-  local modpath = M.find(modname, { patterns = { "" } })
-  if modpath then
-    local root = modpath:gsub("/init%.lua$", ""):gsub("%.lua$", "")
-    return root
-  end
-end
-
----@param modname string
----@param opts? {patterns?:string[]}
----@return string?
-function M.find(modname, opts)
-  opts = opts or {}
-
-  M.stats.find.total = M.stats.find.total + 1
-  local start = uv.hrtime()
-  local basename = modname:gsub("%.", "/")
-  local idx = modname:find(".", 1, true)
-  local topmod = idx and modname:sub(1, idx - 1) or modname
-
-  -- search for a directory first when topmod == modname
-  local patterns = topmod == modname and { "/init.lua", ".lua" } or { ".lua", "/init.lua" }
-
-  if opts.patterns then
-    vim.list_extend(patterns, opts.patterns)
-  end
-
-  -- check top-level mods to find the module
-  local function _find()
-    for _, toppath in ipairs(M.topmods[topmod] or {}) do
-      for _, pattern in ipairs(patterns) do
-        local path = toppath .. "/lua/" .. basename .. pattern
-        M.stats.find.stat = M.stats.find.stat + 1
-        if uv.fs_stat(path) then
-          return path
-        end
-      end
-    end
-  end
-
-  local modpath = _find()
-  if not modpath then
-    -- update rtp
-    local rtp = M.list_rtp()
-    if #rtp ~= M.indexed_rtp then
-      M.indexed_rtp = #rtp
-      local updated = false
-      for _, path in ipairs(rtp) do
-        updated = M._index(path) or updated
-      end
-      if updated then
-        modpath = _find()
-      end
-    end
-
-    -- update unloaded
-    if not modpath and not M.indexed_unloaded then
-      M.indexed_unloaded = true
-      local updated = false
-      ---@type LazyCoreConfig
-      local Config = package.loaded["lazy.core.config"]
-      if Config and Config.spec then
-        for _, plugin in pairs(Config.spec.plugins) do
-          if not (M.indexed[plugin.dir] or plugin._.loaded or plugin.module == false) then
-            updated = M._index(plugin.dir) or updated
-          end
-        end
-      end
-      if updated then
-        modpath = _find()
-      end
-    end
-
-    -- module not found
-    if not modpath then
-      M.stats.find.not_found = M.stats.find.not_found + 1
-    end
-  end
-
-  M.stats.find.time = M.stats.find.time + uv.hrtime() - start
-  return modpath
-end
-
--- returns the cached RTP excluding plugin dirs
-function M.get_rtp()
-  local rtp = M.list_rtp()
-  if not M.rtp or #rtp ~= M.rtp_total then
-    M.rtp_total = #rtp
-    M.rtp = {}
-    ---@type table<string,true>
-    local skip = {}
-    -- only skip plugins once Config has been setup
-    ---@type LazyCoreConfig
-    local Config = package.loaded["lazy.core.config"]
-    if Config then
-      for _, plugin in pairs(Config.plugins) do
-        if plugin.name ~= "lazy.nvim" then
-          skip[plugin.dir] = true
-        end
-      end
-    end
-    for _, path in ipairs(rtp) do
-      ---@type string
-      path = path:gsub("\\", "/")
-      if not skip[path] and not path:find("after/?$") then
-        M.rtp[#M.rtp + 1] = path
-      end
-    end
-  end
-  return M.rtp
-end
-
-function M.list_rtp()
-  return vim.api.nvim_get_runtime_file("", true)
-end
-
----@param opts? LazyConfig
-function M.setup(opts)
-  -- no fancy deep extend here. just set the options
-  if opts and opts.performance and opts.performance.cache then
-    ---@diagnostic disable-next-line: no-unknown
-    for k, v in pairs(opts.performance.cache) do
-      ---@diagnostic disable-next-line: no-unknown
-      M.config[k] = v
-    end
-  end
-  M.debug = opts and opts.debug
-  M.enabled = M.config.enabled
-
-  if M.enabled then
-    table.insert(package.loaders, 2, M.loader)
-    M.load_cache()
-    -- selene:allow(global_usage)
-    _G.loadfile = M.loadfile
-    if #M.config.disable_events > 0 then
-      vim.api.nvim_create_autocmd(M.config.disable_events, { once = true, callback = M.disable })
-    end
-  else
-    -- we need to always add the loader since this will autoload unloaded modules
-    table.insert(package.loaders, M.loader)
-  end
-
-  return M
-end
-
----@return CacheHash?
-function M.hash(file)
-  local ok, ret = pcall(uv.fs_stat, file)
-  return ok and ret or nil
-end
-
----@param h1 CacheHash
----@param h2 CacheHash
-function M.eq(h1, h2)
-  return h1 and h2 and h1.size == h2.size and h1.mtime.sec == h2.mtime.sec and h1.mtime.nsec == h2.mtime.nsec
-end
-
-function M.save_cache()
-  vim.fn.mkdir(vim.fn.fnamemodify(M.config.path, ":p:h"), "p")
-  local f = assert(uv.fs_open(M.config.path, "w", 438))
-  uv.fs_write(f, M.VERSION)
-  uv.fs_write(f, "\0")
-  for modname, entry in pairs(M.cache) do
-    if entry.used > os.time() - M.config.ttl then
-      entry.modname = modname
-      local header = {
-        entry.hash.size,
-        entry.hash.mtime.sec,
-        entry.hash.mtime.nsec,
-        #modname,
-        #entry.chunk,
-        #entry.modpath,
-        entry.used,
-      }
-      uv.fs_write(f, ffi.string(ffi.new("const uint32_t[7]", header), 28))
-      uv.fs_write(f, modname)
-      uv.fs_write(f, entry.chunk)
-      uv.fs_write(f, entry.modpath)
-    end
-  end
-  uv.fs_close(f)
-end
-
-function M.load_cache()
-  M.cache = {}
-  local f = uv.fs_open(M.config.path, "r", 438)
-  if f then
-    cache_hash = uv.fs_fstat(f) --[[@as CacheHash]]
-    local data = uv.fs_read(f, cache_hash.size, 0) --[[@as string]]
-    uv.fs_close(f)
-
-    local zero = data:find("\0", 1, true)
-    if not zero then
-      return
-    end
-
-    if M.VERSION ~= data:sub(1, zero - 1) then
-      return
-    end
-
-    local offset = zero + 1
-    while offset + 1 < #data do
-      local header = ffi.cast("uint32_t*", ffi.new("const char[28]", data:sub(offset, offset + 27)))
-      offset = offset + 28
-      local modname = data:sub(offset, offset + header[3] - 1)
-      offset = offset + header[3]
-      local chunk = data:sub(offset, offset + header[4] - 1)
-      offset = offset + header[4]
-      local file = data:sub(offset, offset + header[5] - 1)
-      offset = offset + header[5]
-      M.cache[modname] = {
-        hash = { size = header[0], mtime = { sec = header[1], nsec = header[2] } },
-        chunk = chunk,
-        modpath = file,
-        used = header[6],
-      }
-    end
-  end
-end
-
-function M.autosave()
-  vim.api.nvim_create_autocmd("VimLeavePre", {
-    callback = function()
-      if M.dirty then
-        local hash = M.hash(M.config.path)
-        -- abort when the file was changed in the meantime
-        if hash == nil or M.eq(cache_hash, hash) then
-          M.save_cache()
-        end
-      end
-    end,
-  })
-end
-
 return M

lua/lazy/core/config.lua

@@ -226,7 +226,6 @@ function M.setup(opts)
     pattern = "VeryLazy",
     once = true,
     callback = function()
-      require("lazy.core.cache").autosave()
       require("lazy.view.commands").setup()
       if M.options.change_detection.enabled then
         require("lazy.manage.reloader").enable()

lua/lazy/core/loader.lua

@@ -72,7 +72,7 @@ function M.install_missing()
       -- remove and installed plugins from indexed, so cache will index again
       for _, p in pairs(Config.plugins) do
         if p._.installed then
-          Cache.indexed[p.dir] = nil
+          Cache.reset(p.dir)
         end
       end
       -- reload plugins
@@ -341,7 +341,7 @@ function M.get_main(plugin)
   local normname = Util.normname(plugin.name)
   ---@type string[]
   local mods = {}
-  for _, modname in ipairs(Cache.get_topmods(plugin.dir)) do
+  for modname, _ in pairs(Cache.get_topmods(plugin.dir)) do
     mods[#mods + 1] = modname
     local modnorm = Util.normname(modname)
     -- if we found an exact match, then use that
@@ -450,4 +450,22 @@ function M.colorscheme(name)
   end
 end
 
+---@param modname string
+function M.loader(modname)
+  local modpath = Cache.find(modname, { rtp = Util.get_unloaded_rtp(modname) })
+  if modpath then
+    local plugin = Plugin.find(modpath)
+    if plugin and modpath:find(plugin.dir, 1, true) == 1 then
+      -- don't load if we're loading specs or if the plugin is already loaded
+      if not (Plugin.loading or plugin._.loaded) then
+        if plugin.module == false then
+          error("Plugin " .. plugin.name .. " is not loaded and is configured with module=false")
+        end
+        M.load(plugin, { require = modname })
+      end
+      return Cache._load(modname, modpath)
+    end
+  end
+end
+
 return M

lua/lazy/core/util.lua

@@ -217,11 +217,44 @@ function M.walkmods(root, fn, modname)
   end)
 end
 
+---@param modname string
+function M.get_unloaded_rtp(modname)
+  modname = modname:gsub("/", ".")
+  local idx = modname:find(".", 1, true)
+  local topmod = idx and modname:sub(1, idx - 1) or modname
+  topmod = M.normname(topmod)
+
+  local rtp = {}
+  local Config = require("lazy.core.config")
+  if Config.spec then
+    for _, plugin in pairs(Config.spec.plugins) do
+      if not (plugin._.loaded or plugin.module == false) then
+        if topmod == M.normname(plugin.name) then
+          table.insert(rtp, 1, plugin.dir)
+        else
+          table.insert(rtp, plugin.dir)
+        end
+      end
+    end
+  end
+  return rtp
+end
+
+function M.find_root(modname)
+  local Cache = require("lazy.core.cache")
+  local rtp = vim.deepcopy(Cache.get_rtp())
+  vim.list_extend(rtp, M.get_unloaded_rtp(modname))
+  local modpath = Cache.find(modname, { rtp = rtp, patterns = { "", ".lua" } })
+  if modpath then
+    local root = modpath:gsub("/init%.lua$", ""):gsub("%.lua$", "")
+    return root
+  end
+end
+
 ---@param modname string
 ---@param fn fun(modname:string, modpath:string)
 function M.lsmod(modname, fn)
-  local Cache = require("lazy.core.cache")
+  local root = M.find_root(modname)
-  local root = Cache.find_root(modname)
   if not root then
     return
   end

lua/lazy/init.lua

@@ -34,13 +34,14 @@ function M.setup(spec, opts)
   local start = vim.loop.hrtime()
 
   -- load module cache before anything else
-  require("lazy.core.cache").setup(opts)
+  require("lazy.core.cache").setup()
 
   require("lazy.stats").track("LazyStart")
 
   local Util = require("lazy.core.util")
   local Config = require("lazy.core.config")
   local Loader = require("lazy.core.loader")
+  table.insert(package.loaders, 3, Loader.loader)
 
   Util.track({ plugin = "lazy.nvim" }) -- setup start
   Util.track("module", vim.loop.hrtime() - start)

lua/lazy/manage/reloader.lua

@@ -41,7 +41,7 @@ function M.check(start)
   -- spec is a module
   local function check(_, modpath)
     checked[modpath] = true
-    local hash = Cache.hash(modpath)
+    local hash = vim.loop.fs_stat(modpath)
     if hash then
       if M.files[modpath] then
         if not Cache.eq(M.files[modpath], hash) then

lua/lazy/view/render.lua

@@ -680,25 +680,6 @@ function M:debug()
     { "not found", Cache.stats.find.not_found, "Number" },
   }, { indent = 2 })
   self:nl()
-
-  self:append("Cache.autoload()", "LazyH2"):nl()
-  self:props({
-    { "total", Cache.stats.autoload.total, "Number" },
-    { "time", self:ms(Cache.stats.autoload.time, 3), "Bold" },
-    { "avg time", self:ms(Cache.stats.autoload.time / Cache.stats.autoload.total, 3), "Bold" },
-  }, { indent = 2 })
-  self:nl()
-
-  self:append("Cache", "LazyH2"):nl()
-  local Cache = require("lazy.core.cache")
-  Util.foreach(Cache.cache, function(modname, entry)
-    local kb = math.floor(#entry.chunk / 10.24) / 100
-    self:append("● ", "LazySpecial", { indent = 2 }):append(modname):append(" " .. kb .. "Kb", "Bold")
-    if entry.modpath ~= modname then
-      self:append(" " .. vim.fn.fnamemodify(entry.modpath, ":p:~:."), "LazyComment")
-    end
-    self:nl()
-  end)
 end
 
 return M

tests/core/util_spec.lua

@@ -52,9 +52,9 @@ describe("util", function()
 
       -- test with empty cache
       Cache.cache = {}
-      Cache.indexed = {}
+      Cache._topmods = {}
-      Cache.indexed_rtp = false
+      Cache.topmods_rtp = false
-      local root = Cache.find_root(test.mod)
+      local root = Util.find_root(test.mod)
       assert(root, "no root found for " .. test.mod .. " (test " .. t .. ")")
       assert.same(Helpers.path(test.root), root)
       local mods = {}
@@ -69,9 +69,9 @@ describe("util", function()
       for i, file in ipairs(files) do
         Cache.cache[test.mods[i]] = { modpath = file }
       end
-      Cache.indexed = {}
+      Cache._topmods = {}
-      Cache.indexed_rtp = false
+      Cache.topmods_rtp = false
-      root = Cache.find_root(test.mod)
+      root = Util.find_root(test.mod)
       assert(root, "no root found for " .. test.mod .. " (test " .. t .. ")")
       assert.same(Helpers.path(test.root), root)
       mods = {}
@@ -85,12 +85,12 @@ describe("util", function()
 
   it("find the correct root with dels", function()
     Cache.cache = {}
-    Cache.indexed = {}
+    Cache._topmods = {}
-    Cache.indexed_rtp = false
+    Cache.topmods_rtp = false
     vim.opt.rtp:append(Helpers.path("old"))
     Helpers.fs_create({ "old/lua/foobar/init.lua" })
     Cache.cache["foobar"] = { modpath = Helpers.path("old/lua/foobar/init.lua") }
-    local root = Cache.find_root("foobar")
+    local root = Util.find_root("foobar")
     assert(root, "foobar root not found")
     assert.same(Helpers.path("old/lua/foobar"), root)
 
@@ -98,24 +98,22 @@ describe("util", function()
     assert(not vim.loop.fs_stat(Helpers.path("old/lua/foobar")), "old/lua/foobar should not exist")
 
     -- vim.opt.rtp = rtp
-    Cache.indexed = {}
+    Cache._topmods = {}
-    Cache.indexed_rtp = false
     vim.opt.rtp:append(Helpers.path("new"))
     Helpers.fs_create({ "new/lua/foobar/init.lua" })
-    root = Cache.find_root("foobar")
+    root = Util.find_root("foobar")
     assert(root, "foobar root not found")
     assert.same(Helpers.path("new/lua/foobar"), root)
   end)
 
   it("find the correct root with mod dels", function()
     Cache.cache = {}
-    Cache.indexed = {}
+    Cache._topmods = {}
-    Cache.indexed_rtp = false
     Cache.enabled = true
     vim.opt.rtp:append(Helpers.path("old"))
     Helpers.fs_create({ "old/lua/foobar/test.lua" })
     Cache.cache["foobar.test"] = { modpath = Helpers.path("old/lua/foobar/test.lua") }
-    local root = Cache.find_root("foobar")
+    local root = Util.find_root("foobar")
     assert(root, "foobar root not found")
     assert.same(Helpers.path("old/lua/foobar"), root)
     assert(not Cache.cache["foobar"], "foobar should not be in cache")
@@ -124,11 +122,10 @@ describe("util", function()
     Helpers.fs_rm("old")
 
     -- vim.opt.rtp = rtp
-    Cache.indexed = {}
+    Cache._topmods = {}
-    Cache.indexed_rtp = false
     vim.opt.rtp:append(Helpers.path("new"))
     Helpers.fs_create({ "new/lua/foobar/test.lua" })
-    root = Cache.find_root("foobar")
+    root = Util.find_root("foobar")
     assert(root, "foobar root not found")
     assert.same(Helpers.path("new/lua/foobar"), root)
   end)