perf: split caching in state, cache and module

This commit is contained in:
Folke Lemaitre 2022-11-22 21:12:33 +01:00
parent a543134b8c
commit 54d5ff18f5
No known key found for this signature in database
GPG Key ID: 41F8B1FBACAE2040
8 changed files with 455 additions and 395 deletions

View File

@ -1,363 +0,0 @@
local M = {}
---@class CacheOptions
M.options = {
module = "config.plugins",
cache = vim.fn.stdpath("state") .. "/lazy/plugins.state",
trim = false,
}
M.dirty = false
M.cache_hash = ""
---@alias ModEntry {file: string, hash?:string, chunk?: string|fun(), used?: boolean}
---@type table<string,ModEntry>
M.modules = {}
---@type LazyState?
M.state = nil
---@alias DirEntry {name: string, type: "file"|"directory"|"link"}
function M.walk(dir, modname, fn)
local d = vim.loop.fs_opendir(dir, nil, 100)
if d then
---@type DirEntry[]
local entries = vim.loop.fs_readdir(d)
while entries do
for _, entry in ipairs(entries) do
local path = dir .. "/" .. entry.name
if entry.type == "directory" then
M.walk(path, modname .. "." .. entry.name, fn)
else
local child = entry.name == "init.lua" and modname or (modname .. "." .. entry.name:match("^(.*)%.lua$"))
if child then
fn(child, path)
end
end
end
entries = vim.loop.fs_readdir(d)
end
vim.loop.fs_closedir(d)
end
end
function M.hash(modpath)
local stat = vim.loop.fs_stat(modpath)
if stat then
return stat.mtime.sec .. stat.mtime.nsec .. stat.size
end
end
---@param opts? CacheOptions
function M.boot(opts)
if opts then
for k, _ in pairs(M.options) do
M.options[k] = opts[k] or M.options[k]
end
end
M.load_state()
-- preload core modules
local root = debug.getinfo(1, "S").source:sub(2)
root = vim.fn.fnamemodify(root, ":p:h")
for _, modname in ipairs({ "util", "config", "plugin", "loader" }) do
local file = root .. "/" .. modname .. ".lua"
modname = "lazy." .. modname
if not M.modules[modname] then
M.modules[modname] = { file = file }
end
package.preload[modname] = function()
return M.load(modname)
end
end
end
function M.setup()
vim.api.nvim_create_autocmd("User", {
pattern = "LazyDone",
once = true,
callback = function()
vim.api.nvim_create_autocmd("VimLeavePre", {
callback = function()
if M.dirty then
local hash = M.hash(M.options.cache)
-- abort when the file was changed in the meantime
if M.hash == nil or M.cache_hash == hash then
M.compile()
end
end
end,
})
end,
})
if M.state and M.load_plugins() then
return true
else
M.dirty = true
-- FIXME: what if module is a file
local root = vim.fn.stdpath("config") .. "/lua/" .. M.options.module:gsub("%.", "/")
if vim.loop.fs_stat(root .. ".lua") then
if not M.modules[M.options.module] then
M.modules[M.options.module] = { file = root .. ".lua" }
end
end
M.walk(root, M.options.module, function(modname, modpath)
if not M.modules[modname] then
M.modules[modname] = { file = modpath }
end
end)
end
end
---@param modname string
function M.load(modname)
local info = M.modules[modname]
if type(package.loaded[modname]) == "table" then
if info then
info.used = true
end
return package.loaded[modname]
end
if info then
local hash = M.hash(info.file)
if hash ~= info.hash then
info.chunk = nil
end
local err
if not info.chunk then
vim.schedule(function()
vim.notify("loading " .. modname)
end)
info.chunk, err = loadfile(info.file)
info.hash = hash
M.dirty = true
end
if type(info.chunk) == "string" then
info.chunk, err = loadstring(info.chunk --[[@as string]], "@" .. info.file)
end
if not info.chunk then
error(err)
end
info.used = true
---@type table
local mod = info.chunk()
package.loaded[modname] = mod
return mod
end
end
---@param state LazyState
function M.write(state)
local chunks = state.chunks
state.chunks = nil
local header = loadstring("return " .. M.dump(state))
assert(header)
table.insert(chunks, string.dump(header, true))
vim.fn.mkdir(vim.fn.fnamemodify(M.options.cache, ":p:h"), "p")
local f = assert(io.open(M.options.cache, "wb"))
for _, chunk in ipairs(chunks) do
f:write(tostring(#chunk), "\0", chunk)
end
f:close()
end
---@return LazyState?
function M.read()
M.cache_hash = M.hash(M.options.cache)
local f = io.open(M.options.cache, "rb")
if f then
---@type string
local data = f:read("*a")
f:close()
local from = 1
local to = data:find("\0", from, true)
---@type string[]
local chunks = {}
while to do
local len = tonumber(data:sub(from, to - 1))
from = to + 1
local chunk = data:sub(from, from + len - 1)
table.insert(chunks, chunk)
from = from + len
to = data:find("\0", from, true)
end
local state = loadstring(table.remove(chunks))
assert(state)
---@type LazyState
local ret = state()
ret.chunks = chunks
return ret
end
end
function M.compile()
local Config = require("lazy.config")
---@class LazyState
local state = {
---@type LazyPlugin[]
plugins = {},
---@type table<string, {file:string, hash:string, chunk:number}>
modules = {},
loaders = require("lazy.loader").loaders,
-- config = Config.options,
---@type string[]
chunks = {},
}
local skip = { installed = true, loaded = true }
-- plugins
for _, plugin in pairs(Config.plugins) do
-- mark module as used
if M.modules[plugin.modname] then
---@diagnostic disable-next-line: no-unknown
M.modules[plugin.modname].used = true
end
---@type LazyPlugin | {_chunks: string[] | table<string, number>}
local save = {}
table.insert(state.plugins, save)
for k, v in pairs(plugin) do
if type(v) == "function" then
save._chunks = save._chunks or {}
if plugin.modname then
table.insert(save._chunks, k)
else
table.insert(state.chunks, string.dump(v, M.options.trim))
save._chunks[k] = #state.chunks
end
elseif not skip[k] then
save[k] = v
end
end
end
-- modules
for modname, entry in pairs(M.modules) do
if entry.used and entry.chunk then
table.insert(
state.chunks,
type(entry.chunk) == "string" and entry.chunk or string.dump(entry.chunk --[[@as fun()]], M.options.trim)
)
state.modules[modname] = { file = entry.file, hash = entry.hash, chunk = #state.chunks }
end
end
M.write(state)
end
function M._dump(value, result)
local t = type(value)
if t == "number" or t == "boolean" then
table.insert(result, tostring(value))
elseif t == "string" then
table.insert(result, ("%q"):format(value))
elseif t == "table" then
table.insert(result, "{")
local i = 1
---@diagnostic disable-next-line: no-unknown
for k, v in pairs(value) do
if k == i then
elseif type(k) == "string" then
table.insert(result, ("[%q]="):format(k))
else
table.insert(result, k .. "=")
end
M._dump(v, result)
table.insert(result, ",")
i = i + 1
end
table.insert(result, "}")
else
error("Unsupported type " .. t)
end
end
function M.dump(value)
local result = {}
M._dump(value, result)
return table.concat(result, "")
end
function M.load_state()
M.state = M.read()
if not M.state then
return
end
local reload = false
for modname, entry in pairs(M.state.modules) do
entry.chunk = M.state.chunks[entry.chunk]
---@cast entry ModEntry
if M.hash(entry.file) ~= entry.hash then
-- keep loading modules, but reset state (reload plugins)
reload = true
end
M.modules[modname] = entry
end
if reload then
M.state = nil
end
end
local function load_plugin(plugin, fun, ...)
local mod = M.load(plugin.modname)
for k, v in pairs(mod) do
if type(v) == "function" then
plugin[k] = v
end
end
return mod[fun](...)
end
function M.load_plugins()
local Config = require("lazy.config")
if not vim.deepcopy(Config.options, M.state.config) then
return false
end
-- plugins
for _, plugin in ipairs(M.state.plugins) do
plugin.loaded = false
plugin.installed = vim.loop.fs_stat(plugin.dir) and true
if plugin._chunks then
if plugin.modname then
for _, fun in ipairs(plugin._chunks) do
plugin[fun] = function(...)
return load_plugin(plugin, fun, ...)
end
end
else
for fun, value in pairs(plugin._chunks) do
plugin[fun] = function(...)
plugin[fun] = loadstring(M.state.chunks[value])
return plugin[fun](...)
end
end
end
plugin._chunks = nil
end
end
-- loaders
local Loader = require("lazy.loader")
Loader.loaders = M.state.loaders
-- save plugins
Config.plugins = {}
for _, plugin in ipairs(M.state.plugins) do
Config.plugins[plugin.name] = plugin
end
return true
end
return M

109
lua/lazy/core/cache.lua Normal file
View File

@ -0,0 +1,109 @@
-- Simple string cache with fast saving and loading from file
local M = {}
local cache_path = vim.fn.stdpath("state") .. "/lazy/plugins.state"
---@type string
local cache_hash = nil
local dirty = false
---@type table<string,boolean>
local used = {}
---@type table<string,string>
local cache = {}
function M.get(key)
if cache[key] then
used[key] = true
return cache[key]
end
end
function M.set(key, value)
cache[key] = value
used[key] = true
dirty = true
end
function M.del(key)
cache[key] = nil
dirty = true
end
function M.dirty()
dirty = true
end
function M.use(pattern)
for key, _ in pairs(cache) do
if key:find(pattern) then
used[key] = true
end
end
end
function M.hash(file)
local stat = vim.loop.fs_stat(file)
return stat and (stat.mtime.sec .. stat.mtime.nsec .. stat.size)
end
function M.setup()
M.load()
vim.api.nvim_create_autocmd("User", {
pattern = "LazyDone",
once = true,
callback = function()
vim.api.nvim_create_autocmd("VimLeavePre", {
callback = function()
if dirty then
local hash = M.hash(cache_path)
-- abort when the file was changed in the meantime
if hash == nil or cache_hash == hash then
M.save()
end
end
end,
})
end,
})
end
function M.save()
require("lazy.core.state").save()
require("lazy.core.module").save()
vim.fn.mkdir(vim.fn.fnamemodify(cache_path, ":p:h"), "p")
local f = assert(io.open(cache_path, "wb"))
for key, value in pairs(cache) do
if used[key] then
f:write(key, "\0", tostring(#value), "\0", value)
end
end
f:close()
end
function M.load()
cache = {}
local f = io.open(cache_path, "rb")
if f then
cache_hash = M.hash(cache_path)
---@type string
local data = f:read("*a")
f:close()
local from = 1
local to = data:find("\0", from, true)
while to do
local key = data:sub(from, to - 1)
from = to + 1
to = data:find("\0", from, true)
local len = tonumber(data:sub(from, to - 1))
from = to + 1
cache[key] = data:sub(from, from + len - 1)
from = from + len
to = data:find("\0", from, true)
end
end
end
return M

130
lua/lazy/core/module.lua Normal file
View File

@ -0,0 +1,130 @@
local Cache = require("lazy.core.cache")
local M = {}
---@type table<string, {file: string, hash?:string}>
M.modules = {}
function M.add(modname, file)
if not M.modules[modname] then
M.modules[modname] = { file = file }
end
end
---@param modname string
function M.load(modname)
if type(package.loaded[modname]) == "table" then
return package.loaded[modname]
end
local info = M.modules[modname]
if info then
local err
---@type string|fun()|nil
local chunk = Cache.get(modname)
if not chunk then
vim.schedule(function()
vim.notify("loading " .. modname)
end)
chunk, err = loadfile(info.file)
if chunk then
Cache.set(modname, string.dump(chunk))
info.hash = info.hash or Cache.hash(info.file)
end
end
if type(chunk) == "string" then
chunk, err = loadstring(chunk --[[@as string]], "@" .. info.file)
end
if not chunk then
error(err)
end
---@type table
local mod = chunk()
package.loaded[modname] = mod
return mod
end
end
local function _add_module(dir, modname)
local d = vim.loop.fs_opendir(dir, nil, 100)
if d then
---@type {name: string, type: "file"|"directory"|"link"}[]
local entries = vim.loop.fs_readdir(d)
while entries do
for _, entry in ipairs(entries) do
local path = dir .. "/" .. entry.name
if entry.type == "directory" then
_add_module(path, modname .. "." .. entry.name)
else
local childname = entry.name:match("^(.*)%.lua$")
if childname then
local child = entry.name == "init.lua" and modname or (modname .. "." .. childname)
if child then
M.add(child, path)
end
end
end
end
entries = vim.loop.fs_readdir(d)
end
vim.loop.fs_closedir(d)
end
end
function M.add_module(path)
---@type string
local modname = path:match("/lua/(.*)/?")
assert(modname)
modname = modname:gsub("/", ".")
if vim.loop.fs_stat(path .. ".lua") then
M.add(modname, path .. ".lua")
end
_add_module(path, modname)
end
function M.setup()
-- load cache
local value = Cache.get("cache.modules")
if value then
M.modules = vim.json.decode(value)
for k, v in pairs(M.modules) do
if Cache.hash(v.file) ~= v.hash then
Cache.del(k)
M.changed = true
M.modules[k] = nil
end
end
end
-- preload core modules
local root = vim.fn.fnamemodify(debug.getinfo(1, "S").source:sub(2), ":p:h:h")
for _, name in ipairs({ "util", "config", "plugin", "loader", "core.state" }) do
local modname = "lazy." .. name
M.add(modname, root .. "/" .. name:gsub("%.", "/") .. ".lua")
end
table.insert(package.loaders, 2, function(modname)
if M.modules[modname] then
return function()
return M.load(modname)
end
end
end)
return M
end
function M.save()
local value = {}
for k, v in pairs(M.modules) do
if v.hash then
value[k] = v
end
end
Cache.set("cache.modules", vim.json.encode(value))
end
return M

127
lua/lazy/core/state.lua Normal file
View File

@ -0,0 +1,127 @@
local Cache = require("lazy.core.cache")
local Module = require("lazy.core.module")
local M = {}
M.functions = { "init", "config", "run" }
M.changed = true
function M.save()
local Config = require("lazy.config")
---@class LazyState
local state = {
---@type LazyPlugin[]
plugins = {},
loaders = require("lazy.loader").loaders,
config = Config.options,
}
local skip = { installed = true, loaded = true, tasks = true, dirty = true, [1] = true, dir = true }
local funcount = 0
for _, plugin in pairs(Config.plugins) do
---@type LazyPlugin | {_chunks: string[] | table<string, number>}
local save = {}
table.insert(state.plugins, save)
for k, v in pairs(plugin) do
if type(v) == "function" then
if vim.tbl_contains(M.functions, k) then
if plugin.modname then
save[k] = true
else
funcount = funcount + 1
Cache.set("cache.state.fun." .. funcount, string.dump(v))
save[k] = funcount
end
end
elseif not skip[k] then
save[k] = v
end
end
end
Cache.set("cache.state", vim.json.encode(state))
end
local function load_plugin(plugin, fun, ...)
local mod = Module.load(plugin.modname)
for k, v in pairs(mod) do
if type(v) == "function" then
plugin[k] = v
end
end
return mod[fun](...)
end
function M.load()
---@type boolean, LazyState
local ok, state = pcall(vim.json.decode, Cache.get("cache.state"))
if not ok then
Cache.dirty()
return false
end
local Util = require("lazy.util")
local Config = require("lazy.config")
if not vim.deep_equal(Config.options, state.config) then
Cache.dirty()
return false
end
-- Check for installed plugins
---@type table<"opt"|"start", table<string,boolean>>
local installed = { opt = {}, start = {} }
for _, opt in ipairs({ "opt", "start" }) do
for _, entry in ipairs(Util.scandir(Config.options.package_path .. "/" .. opt)) do
if entry.type == "directory" or entry.type == "link" then
installed[opt][entry.name] = true
end
end
end
-- plugins
for _, plugin in ipairs(state.plugins) do
---@cast plugin LazyPlugin|{_chunks:table}
Config.plugins[plugin.name] = plugin
plugin.loaded = false
plugin.dir = Config.options.package_path .. "/" .. (plugin.opt and "opt" or "start") .. "/" .. plugin.pack
plugin.installed = installed[plugin.opt and "opt" or "start"][plugin.pack]
if plugin.modname then
-- mark module as used
if not Cache.get(plugin.modname) then
Util.error("Module missing for " .. plugin.name)
end
for _, fun in ipairs(M.functions) do
if plugin[fun] == true then
plugin[fun] = function(...)
return load_plugin(plugin, fun, ...)
end
end
end
else
for _, fun in ipairs(M.functions) do
if type(plugin[fun]) == "number" then
local chunk = Cache.get("cache.state.fun." .. plugin[fun])
if not chunk then
Util.error("Chunk missing for " .. plugin.name)
end
plugin[fun] = function(...)
plugin[fun] = loadstring(chunk)
return plugin[fun](...)
end
end
end
end
end
-- loaders
local Loader = require("lazy.loader")
Loader.loaders = state.loaders
M.changed = false
return true
end
return M

View File

@ -2,31 +2,47 @@ local M = {}
---@param opts? LazyConfig
function M.setup(opts)
--FIXME: preload()
local done = false
-- table.insert(package.loaders, 1, function(modname)
-- if not done and modname:find("lazy") == 1 then
-- dd(modname)
-- end
-- end)
-- Loading order
-- 1. load module cache
-- 2. if changes, then reload
local Cache = require("lazy.cache")
local cache_start = vim.loop.hrtime()
require("lazy.core.cache").setup()
local start = vim.loop.hrtime()
Cache.boot()
local module_start = vim.loop.hrtime()
local Module = require("lazy.core.module").setup()
local require_start = vim.loop.hrtime()
local Util = require("lazy.util")
local Config = require("lazy.config")
local Plugin = require("lazy.plugin")
local Loader = require("lazy.loader")
local State = require("lazy.core.state")
Util.track("lazy_boot", vim.loop.hrtime() - start)
Util.track("cache.setup", module_start - cache_start)
Util.track("module.setup", require_start - module_start)
Util.track("require.core", vim.loop.hrtime() - require_start)
Util.track("lazy_setup")
Util.track("setup")
Util.track("lazy_config")
Util.track("config")
Config.setup(opts)
Util.track()
Util.track("lazy_plugins")
if not Cache.setup() then
Util.track("plugins")
if Module.changed or not State.load() then
-- rebuild state
local Plugin = require("lazy.plugin")
Module.add_module(vim.fn.stdpath("config") .. "/lua/" .. Config.options.plugins:gsub("%.", "/"))
vim.schedule(function()
vim.notify("Reloading")
end)
Util.track("plugin_normalize")
Util.track("normalize")
Plugin.normalize(require(Config.options.plugins))
if not Config.plugins.lazy then
Plugin.plugin({
@ -36,13 +52,13 @@ function M.setup(opts)
end
Util.track()
Util.track("plugin_process")
Util.track("process")
Plugin.process()
Util.track()
end
Util.track()
Util.track("lazy_install")
Util.track("install")
for _, plugin in pairs(Config.plugins) do
if not plugin.installed then
require("lazy.manager").install({
@ -53,14 +69,15 @@ function M.setup(opts)
end
Util.track()
Util.track("loader_setup")
local Loader = require("lazy.loader")
Util.track("loader")
Loader.setup()
Util.track()
Loader.init_plugins()
Util.track() -- end setup
Loader.init_plugins()
done = true
vim.cmd("do User LazyDone")
end

View File

@ -13,23 +13,18 @@ M.types = {
"cmd",
}
---@class LazyLoaders: table<LoaderType, table<string, string[]>>|{init: string[]}
M.loaders = { init = {} }
for _, type in ipairs(M.types) do
M.loaders[type] = {}
end
---@type table<LoaderType, table<string, string[]>>|{init: string[]}
M.loaders = nil
---@param plugin LazyPlugin
function M.add(plugin)
if plugin.init or plugin.opt == false and plugin.config then
if plugin.init or (plugin.opt == false and plugin.config) then
table.insert(M.loaders.init, plugin.name)
end
for _, loader_type in ipairs(M.types) do
---@type string[]|string
local loaders = plugin[loader_type]
if loaders then
if plugin[loader_type] then
loaders = type(loaders) == "table" and loaders or { loaders }
---@cast loaders string[]
for _, loader in ipairs(loaders) do
@ -43,6 +38,16 @@ function M.add(plugin)
end
function M.setup()
if not M.loaders then
M.loaders = { init = {} }
for _, type in ipairs(M.types) do
M.loaders[type] = {}
end
for _, plugin in pairs(Config.plugins) do
M.add(plugin)
end
end
local group = vim.api.nvim_create_augroup("lazy_loader", {
clear = true,
})
@ -126,7 +131,7 @@ function M.setup()
end
function M.init_plugins()
Util.track("loader_plugin_init")
Util.track("plugin_init")
for _, name in ipairs(M.loaders.init) do
local plugin = Config.plugins[name]
if plugin.init then
@ -167,14 +172,15 @@ end
---@param plugins string|LazyPlugin|string[]|LazyPlugin[]
function M.load(plugins)
if type(plugins) == "string" or plugins.name then
---@diagnostic disable-next-line: assign-type-mismatch
plugins = { plugins }
end
---@cast plugins (string|LazyPlugin)[]
for _, plugin in ipairs(plugins) do
if type(plugin) == "string" then
plugin = Config.plugins[plugin]
end
---@cast plugin LazyPlugin
if not plugin.loaded then
plugin.loaded = true
@ -191,6 +197,9 @@ function M.load(plugins)
end
Util.track()
vim.schedule(function()
vim.cmd("do User LazyRender")
end)
end
end
end

View File

@ -1,7 +1,6 @@
local Config = require("lazy.config")
local Util = require("lazy.util")
local Loader = require("lazy.loader")
local Cache = require("lazy.cache")
local Module = require("lazy.core.module")
local M = {}
@ -77,7 +76,7 @@ function M.process_config(plugin)
local name = plugin.name
local modname = Config.options.plugins .. "." .. name
local spec = Cache.load(modname)
local spec = Module.load(modname)
if spec then
-- add to loaded modules
if spec.requires then
@ -130,7 +129,6 @@ function M.process()
plugin.dir = Config.options.package_path .. "/" .. (plugin.opt and "opt" or "start") .. "/" .. plugin.pack
plugin.installed = Util.file_exists(plugin.dir)
M.process_local(plugin)
Loader.add(plugin)
end
end

View File

@ -185,4 +185,37 @@ function M.error(msg)
})
end
function M._dump(value, result)
local t = type(value)
if t == "number" or t == "boolean" then
table.insert(result, tostring(value))
elseif t == "string" then
table.insert(result, ("%q"):format(value))
elseif t == "table" then
table.insert(result, "{")
local i = 1
---@diagnostic disable-next-line: no-unknown
for k, v in pairs(value) do
if k == i then
elseif type(k) == "string" then
table.insert(result, ("[%q]="):format(k))
else
table.insert(result, k .. "=")
end
M._dump(v, result)
table.insert(result, ",")
i = i + 1
end
table.insert(result, "}")
else
error("Unsupported type " .. t)
end
end
function M.dump(value)
local result = {}
M._dump(value, result)
return table.concat(result, "")
end
return M