diff --git a/lua/lazy/core/loader.lua b/lua/lazy/core/loader.lua
index 6f426d9..7209362 100644
--- a/lua/lazy/core/loader.lua
+++ b/lua/lazy/core/loader.lua
@@ -524,7 +524,7 @@ function M.colorscheme(name)
 end
 
 function M.auto_load(modname, modpath)
-  local plugin = Plugin.find(modpath)
+  local plugin = Plugin.find(modpath, { fast = not M.did_handlers })
   if plugin and modpath:find(plugin.dir, 1, true) == 1 then
     plugin._.rtp_loaded = true
     -- don't load if:
@@ -545,8 +545,14 @@ end
 
 ---@param modname string
 function M.loader(modname)
-  local paths = Util.get_unloaded_rtp(modname)
+  local paths, cached = Util.get_unloaded_rtp(modname, { cache = true })
   local ret = Cache.find(modname, { rtp = false, paths = paths })[1]
+
+  if not ret and cached then
+    paths = Util.get_unloaded_rtp(modname)
+    ret = Cache.find(modname, { rtp = false, paths = paths })[1]
+  end
+
   if ret then
     M.auto_load(modname, ret.modpath)
     local mod = package.loaded[modname]
diff --git a/lua/lazy/core/meta.lua b/lua/lazy/core/meta.lua
index 679a16f..4645f1b 100644
--- a/lua/lazy/core/meta.lua
+++ b/lua/lazy/core/meta.lua
@@ -117,6 +117,14 @@ end
 --- Rebuild all plugins based on dirty fragments,
 --- or dirty plugins. Will remove plugins that no longer have fragments.
 function M:rebuild()
+  local frag_count = vim.tbl_count(self.fragments.dirty)
+  local plugin_count = vim.tbl_count(self.dirty)
+  if frag_count == 0 and plugin_count == 0 then
+    return
+  end
+  if Config.options.debug then
+    Util.track("rebuild plugins frags=" .. frag_count .. " plugins=" .. plugin_count)
+  end
   for fid in pairs(self.fragments.dirty) do
     local meta = self.frag_to_meta[fid]
     if meta then
@@ -143,6 +151,9 @@ function M:rebuild()
   for n, _ in pairs(self.dirty) do
     self:_rebuild(n)
   end
+  if Config.options.debug then
+    Util.track()
+  end
 end
 
 --- Rebuild a single plugin.
@@ -150,6 +161,10 @@ end
 --- This also resolves dependencies, dep, optional, dir, dev, and url.
 ---@param name string
 function M:_rebuild(name)
+  if not self.dirty[name] then
+    return
+  end
+  self.dirty[name] = nil
   local plugin = self.plugins[name]
   if not plugin or #plugin._.frags == 0 then
     self.plugins[name] = nil
@@ -225,7 +240,6 @@ function M:_rebuild(name)
 
   setmetatable(plugin, { __index = super })
 
-  self.dirty[plugin.name] = nil
   return plugin
 end
 
diff --git a/lua/lazy/core/plugin.lua b/lua/lazy/core/plugin.lua
index ecf6d64..062d5d2 100644
--- a/lua/lazy/core/plugin.lua
+++ b/lua/lazy/core/plugin.lua
@@ -362,17 +362,24 @@ end
 
 -- Finds the plugin that has this path
 ---@param path string
-function M.find(path)
+---@param opts? {fast?:boolean}
+function M.find(path, opts)
   if not Config.spec then
     return
   end
+  opts = opts or {}
   local lua = path:find("/lua/", 1, true)
   if lua then
     local name = path:sub(1, lua - 1)
     local slash = name:reverse():find("/", 1, true)
     if slash then
       name = name:sub(#name - slash + 2)
-      return name and Config.plugins[name] or Config.spec.plugins[name] or nil
+      if name then
+        if opts.fast then
+          return Config.spec.meta.plugins[name]
+        end
+        return Config.spec.plugins[name]
+      end
     end
   end
 end
diff --git a/lua/lazy/core/util.lua b/lua/lazy/core/util.lua
index 0504dd8..9a65a85 100644
--- a/lua/lazy/core/util.lua
+++ b/lua/lazy/core/util.lua
@@ -238,18 +238,33 @@ function M.walkmods(root, fn, modname)
 end
 
 ---@param modname string
-function M.get_unloaded_rtp(modname)
-  modname = modname:gsub("/", ".")
-  local idx = modname:find(".", 1, true)
-  local topmod = idx and modname:sub(1, idx - 1) or modname
-  topmod = M.normname(topmod)
+---@return string
+function M.topmod(modname)
+  return modname:match("^[^./]+") or modname
+end
 
+---@type table<string, string[]>
+M.unloaded_cache = {}
+
+---@param modname string
+---@param opts? {cache?:boolean}
+function M.get_unloaded_rtp(modname, opts)
+  opts = opts or {}
+
+  local topmod = M.topmod(modname)
+  if opts.cache and M.unloaded_cache[topmod] then
+    return M.unloaded_cache[topmod], true
+  end
+
+  local norm = M.normname(topmod)
+
+  ---@type string[]
   local rtp = {}
   local Config = require("lazy.core.config")
   if Config.spec then
     for _, plugin in pairs(Config.spec.plugins) do
       if not (plugin._.loaded or plugin.module == false) then
-        if topmod == M.normname(plugin.name) then
+        if norm == M.normname(plugin.name) then
           table.insert(rtp, 1, plugin.dir)
         else
           table.insert(rtp, plugin.dir)
@@ -257,15 +272,27 @@ function M.get_unloaded_rtp(modname)
       end
     end
   end
-  return rtp
+  M.unloaded_cache[topmod] = rtp
+  return rtp, false
 end
 
 function M.find_root(modname)
+  local paths, cached = M.get_unloaded_rtp(modname, { cache = true })
+
   local ret = require("lazy.core.cache").find(modname, {
     rtp = true,
-    paths = M.get_unloaded_rtp(modname),
+    paths = paths,
     patterns = { "", ".lua" },
   })[1]
+
+  if not ret and cached then
+    paths = M.get_unloaded_rtp(modname)
+    ret = require("lazy.core.cache").find(modname, {
+      rtp = false,
+      paths = paths,
+      patterns = { "", ".lua" },
+    })[1]
+  end
   if ret then
     local root = ret.modpath:gsub("/init%.lua$", ""):gsub("%.lua$", "")
     return root