main.lua

lrl = require 'luarocks.loader'  -- seems useless??
-- the next 2 lines actually do what `require 'luarocks.loader'` seems to be supposed to do
-- they are needed to import any luarocks installed module
package.path = package.path .. ";" .. "C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\lua\\?.lua;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\lua\\?\\init.lua;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\?.lua;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\?\\init.lua;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\..\\share\\lua\\5.1\\?.lua;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\..\\share\\lua\\5.1\\?\\init.lua;.\\?.lua;.\\?\\init.lua;C:\\Program Files (x86)\\Lua\\5.1\\lua\\?.luac;C:\\Users\\Lars\\AppData\\Roaming\\luarocks\\share\\lua\\5.1\\?.lua;C:\\Users\\Lars\\AppData\\Roaming\\luarocks\\share\\lua\\5.1\\?\\init.lua"
package.cpath = package.cpath .. ";" .. "C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\?.dll;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\..\\lib\\lua\\5.1\\?.dll;C:\\Users\\Lars\\AppData\\Local\\Programs\\Lua\\luarocks-3.13.0-windows-64\\loadall.dll;.\\?.dll;C:\\Users\\Lars\\AppData\\Roaming\\luarocks\\lib\\lua\\5.1\\?.dll"
lfs = require 'lfs'

os = require 'ext.os'  -- From thenumbernine
ftcsv = require 'ftcsv'
Parser = require 'thenumbernine-parser.parser'  -- Using a modified https://github.com/larswijn/lua-parser/tree/local-fixes, for Lua 5.1 - 5.4, and luaJIT, AST seems useless though
Parser2 = require 'lua-parser.parser'  -- Using https://github.com/Validark/lua-parser fork of andremm's parser, only supports Lua 5.3
ast_pp = require 'lua-parser.pp'

github_api = require 'github_api'
os_capture = require 'os_capture'
tprint = require 'tprint'


table.unpack = table.unpack or unpack  -- ironic

REPO_DIR = "cloned-repos"
MANUALLY_EXCLUDE = {
		FILES = {
			-- Files manually excluded, because e.g. the file is _supposed_ to not compile
			"invalid-module.lua",  -- From Kong, unparsable on purpose
			"moonscript/spec/outputs/bubbling.lua",
			"moonscript/spec/outputs/literals.lua",
			"moonscript/spec/outputs/syntax.lua",
			"cl_compile_fail.lua",  -- From busted, unparsable on purpose
			"wincent/aspects/nvim/files/.config/nvim/pack/bundle/opt/nvim-treesitter/tests/indent/lua/cond.lua",
			"wincent/aspects/nvim/files/.config/nvim/pack/bundle/opt/nvim-treesitter/tests/indent/lua/func.lua",
			"hawkthorne-journey/templates/credits.lua",
			"not_lua.lua",  -- From MoonGen, unparsable on purpose
			"MoonGen/examples/moonsniff/pkt-matcher.lua",  -- These MoonGen files use 0b notation, seems like a Luau thing
			"MoonGen/examples/moonsniff/sniffer.lua",
			"MoonGen/examples/moonsniff/traffic-gen.lua",
			"Styles/Bright.lua",  -- 3 MineOS styling files that are supposed to be loaded as string, not file
			"Styles/Dark.lua",
			"Styles/Default.lua",
			"ldoc/tests/example/style/simple.lua",
			"ldoc/tests/factory/factory.lua",
			"ldoc/tests/mod1.lua",
			"EmptyEpsilon/artemis_mission_convert_template.lua",
			"FairyGUI-Editor/plugin/LuaAPI/CS_FairyGUI_Timers.lua",
			"lua_missions/missions/coroutines.lua",
			"ReaScripts/Development/Lokasenna_Script Compiler.lua",  -- seems like someone fucked up 8 years ago
		},
		REPOS = {
			-- Repos that are manually excluded due to e.g. being FULL of unparseable files
			["xmake-io/xmake"] = true,  -- multiple files that have `${FAQ}` in them
			["ntop/ntopng"] = true,  -- multiple files have non-lua syntax, why are they .lua files?
			["tarantool/tarantool"] = true,  -- multiple files that cannot be parsed, though they do look Lua-like
			["snabbco/snabb"] = true,  -- uses RaptorJIT (fork of LuaJIT) and cannot be parsed by even our JIT parser
			["viruscamp/luadec"] = true,  -- multiple compiled lua files stored as .lua
			["Facepunch/garrysmod"] = true,  -- files are a lot more C-like than Lua
			["citizenlab/chat-censorship"] = true,  -- no idea why these files are .lua
			["wiremod/wire"] = true,  -- garrysmod add-on
			["Quenty/NevermoreEngine"] = true,  -- not Lua
			["davidgiven/wordgrinder"] = true,  -- seems like some sort of typehinted lua, not vanilla
			["FPtje/DarkRP"] = true,  -- garrysmod gamemode
			["TACC/Lmod"] = true,  -- multiple lua-like files with small syntax errors; they seem to be never executed
			
			["ntop/nProbe"] = true,  -- filenames with colons which cannot be cloned on windows, I can't be bothered to fix it
			["yutkat/dotfiles"] = true  -- same as above, filenames with colons
		}
}

function string:endswith(suffix)
	return self:sub(-#suffix) == suffix
end

function decrease_date(date_table)
	-- where date_table is a table formatted as {YYYY, MM} or {YYYY, MM, DD}
	-- returns a new table
	assert(type(date_table) == "table", ("date_table is type '%s', expected table"):format(type(date_table)))
	local new_date_table = {table.unpack(date_table)}
	
	local index = #new_date_table  -- last index by default
	while index ~= 0 do
		new_date_table[index] = new_date_table[index] - 1
		
		if new_date_table[index] > 0 then
			break
		end
		
		if index == 2 then
			-- month
			new_date_table[index] = 12
		elseif index == 3 then
			-- day
			local nxt_month = new_date_table[index-1]-1
			if nxt_month == 0 then
				-- dec
				new_date_table[index] = 31
			elseif nxt_month == 2 then
				-- feb
				local current_year = new_date_table[index-2]
				local is_leap_year = current_year % 4 == 0 and (current_year % 100 ~= 0 or current_year % 400 == 0)
				new_date_table[index] = is_leap_year and 29 or 28
			elseif (nxt_month % 2 == 1 and nxt_month <= 7) or nxt_month == 8 or nxt_month == 10 or nxt_month == 12 then
				-- jan, mar, may, july, aug, oct, dec
				new_date_table[index] = 31
			else
				-- apr, june, sep, nov
				assert(nxt_month == 4 or nxt_month == 6 or nxt_month == 9 or nxt_month == 11, "incorrect month was calculated")
				new_date_table[index] = 30
			end
		else
			error("index is not supposed to be " .. index)
		end
		index = index - 1
	end
	
	assert(new_date_table ~= date_table)
	return new_date_table
end

function get_repo_names(count, filename)
	local count = count or 100
	filename = filename or "list_of_repos.txt"
	if not lfs.attributes("list_of_repos.txt") then
		-- create a list of repos we should clone and iterate over
		print("Searching for up to " .. count .. " suitable repositories on GitHub...")
		local response = github_api.search_repositories(count)
		local repos = {}
		for _, repo in pairs(response.items) do
			if not MANUALLY_EXCLUDE.REPOS[repo.full_name] then
				repos[#repos+1] = repo.full_name
			end
		end
		local f = assert(io.open("list_of_repos.txt", "w"))
		f:write("# Start a line with `#` to exclude it\n")
		for _, repo in ipairs(repos) do
			f:write(repo .. "\n")
		end
		f:close()
	end
	local repos = {}
	local f = assert(io.open("list_of_repos.txt", "r"))
	for line in f:lines() do
		if line and line:sub(0, 1) ~= "#" and line:match("%S") then
			-- there's a non-comment non-whitespace entry 
			repos[#repos+1] = line
		end
	end
	f:close()
	return repos
end

function directory_apply_recursive(path, _rel_path)
	-- uses `git ls-tree`, and coroutine.yield
	local rel_path = (_rel_path and _rel_path.."/" or "") .. path
	_rel_path = nil
	
	local command = ('cd "%s" & git ls-tree --format="%%(objectmode) %%(objectname) %%(path)" HEAD'):format(rel_path)  -- escape % with %%
	local ls_output = os_capture(command, true)
	for line in ls_output:gmatch("[^\r\n]+") do
		local mode, hash, file = line:match("(%d+)%s+(%x+)%s+(.+)")
		if not mode or not hash or not file then
			error(("unable to parse line '%s'"):format(line))
		end
		if mode == "040000" then
			-- directory
			directory_apply_recursive(file, rel_path)
		elseif mode == "100644" or mode == "100664" or mode == "100755" then
			-- "regular non-executable file" or "regular non-executable group-writeable file" or "regular executable file"
			-- TL;DR: different types of normal files
			coroutine.yield(rel_path .. "/" .. file, hash)
		elseif mode == "120000" or mode == "160000" then
			-- "symbolic link" or "gitlink" (aka a git submodule)
			-- print(("   mode '%s' at '%s', skip!"):format(mode, file))
		else
			error(("unknown mode '%s' for file '%s' at '%s'"):format(mode, file, rel_path))
		end
	end
end

function analyze_repo(repo)
	local repo_name = repo:gsub(".*/", "")
	local full_path = REPO_DIR .. "/" .. repo_name
	local files_seen = {}
	local data = {}
	local version_date = {2026, 6}
	
	github_api.clone_repository(repo, REPO_DIR)
	local current_commit = os_capture(('cd "%s" & git rev-list -1 HEAD'):format(full_path))
	
	while version_date[1] > 2011 do
		local timestamp = table.concat(version_date, "-")
		print(("  checking out %-7s aka %s  (%ds)"):format(timestamp, current_commit, os.clock()))
		
		analyze_dir(full_path, data, timestamp, files_seen)  -- update data and files_seen in-place
		
		local previous_commit = current_commit
		while previous_commit == current_commit do
			version_date = decrease_date(decrease_date(decrease_date(version_date)))
			timestamp = table.concat(version_date, "-")
			previous_commit = os_capture(('cd "%s" & git rev-list -1 --before="%s" HEAD'):format(full_path, timestamp.."-01"))
			if previous_commit == nil or #previous_commit == 0 then
				break
			end
			-- print(("   timestamp, previous_commit = %-7s, %s"):format(timestamp, previous_commit))
		end
		if previous_commit == nil or #previous_commit == 0 then
			break
		end
		local checkout = os_capture(('cd "%s" & git checkout %s'):format(full_path, previous_commit))
		current_commit = previous_commit
		assert(
			checkout:find("HEAD is now at") and not checkout:find("error: pathspec"), 
			("failed to checkout to commit '%s'"):format(previous_commit)
		)
	end
	
	github_api.remove_repository(full_path)
	
	return data
end

function analyze_dir(dir_path, data, timestamp, files_seen)
	-- MODIFIES data, files_seen IN-PLACE
	data = data or {}
	timestamp = timestamp or "latest"
	files_seen = files_seen or {}
	
	local dir_min_version, dir_max_version
	local iterator = coroutine.create(directory_apply_recursive)
	while true do
		if coroutine.status(iterator) == "dead" then
			break
		end
		local _, path, hash = assert(coroutine.resume(iterator, dir_path))
		if path and hash then
			local file_content = read_lua_file(path)
			if file_content then
				-- print("    hash, path = " .. tprint(hash) .. ", " .. tprint(path))
				local file_info
				if files_seen[hash] then
					file_info = data[ files_seen[hash] ].file_info
					data[#data+1] = {timestamp=timestamp, path=path:sub(#REPO_DIR+2), file_info=file_info}
				else
					local file_ast, file_min_version, file_max_version, error_msgs = parse_code(file_content, path)
					file_max_version = file_max_version == "5.4" and "5.5" or file_max_version  -- because we cannot parse 5.5, assume parsing 5.4 -> 5.5 compat
					file_info = {min_version = file_min_version, max_version = file_max_version}
					if file_ast and #file_ast > 0 then
						local analysis = analyze_ast(file_ast, file_min_version, file_max_version)
						file_info.analysis = analysis
						file_info.min_version = file_info.min_version > analysis.min_version and file_info.min_version or analysis.min_version
						file_info.max_version = file_info.max_version < analysis.max_version and file_info.max_version or analysis.max_version
					end
					local data_entry = {timestamp=timestamp, path=path:sub(#REPO_DIR+2), file_info=file_info}
					data[#data+1] = data_entry
					files_seen[hash] = #data
				end
				if dir_min_version == nil or (dir_min_version ~= "jit" and file_info.min_version ~= "?" and file_info.min_version > dir_min_version) then
					dir_min_version = file_info.min_version
				end
				if dir_max_version == nil or (dir_max_version ~= "jit" and file_info.max_version ~= "?" and file_info.max_version < dir_max_version) then
					dir_max_version = file_info.max_version
				end
				if file_info.min_version == "jit" or file_info.max_version == "jit" then
					dir_min_version, dir_max_version = file_info.min_version, file_info.max_version
				end
				if dir_min_version > dir_max_version then
					print(("WARNING: dir min/max versions are incorrect! dir_path = %s, min_v = %s, max_v = %s"):format(
						dir_path, dir_min_version, dir_max_version))
				end
			end
		end
	end
	data[#data+1] = {timestamp=timestamp, path=".", file_info={min_version=dir_min_version, max_version=dir_max_version}}
end

function read_lua_file(path)
	if not path:endswith(".lua") then
		-- print("  File at '" .. path .. "' is not a lua file")
		return
	end
	for _, invalid_file in pairs(MANUALLY_EXCLUDE.FILES) do
		if path:endswith(invalid_file) then
			-- print("  Found manually excluded file " .. path)
			return
		end
	end
	local filesize = lfs.attributes(path, "size")
	if filesize and filesize > 10 * 1024^2 then
		-- above ~10 MB our parser starts running into memory issues
		print(("  File at '%s' is too large! (%d MB)"):format(path, lfs.attributes(path, "size")/1024^2))
		return
	end
	
	local file = io.open(path, "r")
	if file == nil and filesize == nil then
		-- "lua: No such file or directory". Presumably due to unicode filenames etc
		-- print(("  Unable to open file '%s' (unicode?)"):format(path))
		return
	end
	local content = file:read("*all")
	file:close()
	
	local bom = "\239\187\191"
	if content:sub(1, #bom) == bom then
        content = content:sub(#bom+1)
    end
	content = content .. "\n"  -- add single newline for parser to find; parser crashes on files ending with a comment
	
	return content
end

function parse_code(content, source)
	-- returns table, str, str, str
	-- representing AST, min version, max version, error
	-- any of the first 3 returns might be nil if unable to parse; 4th return could always be nil
	local status, ast, error_msg
	local min_version, max_version
	local error_msgs = ''
	for _, lua_version in ipairs({"5.4", "5.3", "5.2", "5.1"}) do
		-- very inefficient, could probably do some sort of smart log approach instead (more relevant if we could parse more versions)
		local optional
		if lua_version == "5.3" then
			status, ast, error_msg = pcall(Parser2.parse, content, source)
			if not status then
				ast = nil
			end
		else
			status, optional, error_msg = pcall(Parser.parse, content, source, lua_version)
		end
		if status and not error_msg then
			-- success!
			if min_version == nil or lua_version < min_version then
				min_version = lua_version
			end
			if max_version == nil or lua_version > max_version then
				max_version = lua_version
			end
		elseif error_msg then
			error_msgs = error_msgs .. "  " .. lua_version .. " recoverable error: '" .. error_msg .. "'"
		end
	end
	if not min_version and not max_version then
		-- at least try JIT
		status, _, error_msg = pcall(Parser.parse, content, source, "5.4", true)
		if status and not error_msg then
			min_version, max_version = "jit", "jit"
		end
	end
	
	if not ast and not min_version and not max_version then
		print("  ERROR: could not parse " .. source .. "  -  " .. error_msgs:sub(2))
	end

	return ast or {}, min_version or "?", max_version or "?", error_msgs:sub(2)
end

function analyze_ast(ast, min_version, max_version)
	-- only works for analyzing the AST from andremm lua parser (Parser2), which only supports lua 5.3
	assert(type(ast) == "table", ("argument `ast` is type '%s', but expected table"):format(type(ast)))
	local min_version, min_reason, min_pos = min_version or "5.0", nil, {}
	local max_version, max_reason, max_pos = max_version or "5.5", nil, {}
	local notes = {}
	
	local tag = ast.tag
	
	if tag == "VarList" or tag == "NameList" then
		-- when defining global or local vars respectively
		for _, variable in ipairs(ast) do
			local variable_name = variable[1]
			if max_version >= "5.3" and variable_name == "global" then
				max_version, max_reason, max_pos = "5.4", "variable named 'global'", {variable[pos], variable[end_pos]}
				notes[max_reason] = true
			elseif max_version >= "5.1" and variable_name == "goto" then
				max_version, max_reason, max_pos = "5.1", "variable named 'goto'", {variable[pos], variable[end_pos]}
				notes[max_reason] = true
			end
		end
	end
	
	if tag == "Op" then
		local op = ast[1]
		if min_version <= "5.1" and (op == "len" or op == "mod") then
			min_version, min_reason, min_pos = "5.1", op .. "operator added in 5.1", {ast[pos], ast[end_pos]}
			notes[min_reason] = true
		elseif min_version <= "5.3" and (op == "band" or op == "bor" or op == "bxor" or op == "shr" or op == "shl" or op == "bnot" or op == "idiv") then
			min_version, min_reason, min_pos = "5.3", op .. " operator added in 5.3", {ast[pos], ast[end_pos]}
			notes[min_reason] = true
		end
	end
	
	if min_version <= "5.3" and (tag == "Goto" or tag == "Label") then
		min_version, min_reason, min_pos = "5.3", tag .. " syntax added in 5.3", {ast[pos], ast[end_pos]}
		notes[min_reason] = true
	end
	
	for i, data in ipairs(ast) do
		if type(data) == "table" then
			local recursive = analyze_ast(data, version)
			if recursive.min_version >= min_version then
				min_version, min_reason, min_pos = recursive.min_version, recursive.min_reason, recursive.min_pos
			end
			if recursive.max_version <= max_version then
				max_version, max_reason, max_pos = recursive.max_version, recursive.max_reason, recursive.max_pos
			end
			if recursive.notes then
				for _, note in ipairs(recursive.notes) do
					notes[note] = true
				end
			end
		end
	end
	
	assert(
		min_version <= max_version, 
		("min_version %s > max_version %s somehow min_pos = %s, max_pos = %s"):format(min_version, max_version, tprint(min_pos), tprint(max_pos))
	)
	
	return {
		min_version = min_version, --min_pos = min_pos, min_reason = min_reason,
		max_version = max_version, --max_pos = max_pos, max_reason = max_reason,
		notes = notes,
	}
end

function test()
	-- add support for varargs?
	local code = [==[
function TargetModeInfo()
  ImGui.Image(ctx, getImage("indicator_insert_forward"),20,20);; _TT("Insert (add notes and shift)", false, "Stretch"); SL()
end
]==]
	local ast, minimum_version, maximum_version, errs = parse_code(code, "string")
	local a, e = Parser.parse(code, "string", "5.4")
	print(tprint(e))
	if true then return end
	--print("ast = " .. tprint(ast, 12))
	--print("ppast = " .. ast_pp.tostring(ast))
	print("min_v, max_v = " .. minimum_version .. ", " .. maximum_version)
	print("errs = " .. tprint(errs))
	local result = analyze_ast(ast, minimum_version, maximum_version)
	print("result = " .. tprint(result, 4))
end

function main()
	print("STARTED")
	
	local attrs_repo_dir = lfs.attributes(REPO_DIR)
	if not attrs_repo_dir then 
		-- dir does not exist
		lfs.mkdir(REPO_DIR)
	end
	
	local repos = get_repo_names()  -- might print a single line
	
	for i, repo in ipairs(repos) do
		local repo_name = repo:gsub(".*/", "")
		print(("Analyzing repository %d/%d: %s"):format(i, #repos, repo))
		local data = analyze_repo(repo)
		local savefile = REPO_DIR .. "/" .. repo_name .. ".txt"
		local file = assert(io.open(savefile, "wb"))
		file:write(tprint(data))
		file:close()
		-- print(" repo data = " .. tprint(data, 4))
		print(" Written repo data to file!")
	end
	
	print(string.format("ENDED (took %.2f seconds)", os.clock()))
end

main()