Module:scripts/charToScript

Documentation for this module may be created at Module:scripts/charToScript/doc

local subexport = {}

local require_when_needed = require("Module:require when needed")

local cp = require_when_needed("Module:string utilities", "codepoint")
local floor = math.floor
local get_plaintext = require_when_needed("Module:utilities", "get_plaintext")
local get_script = require_when_needed("Module:scripts", "getByCode")
local insert = table.insert
local ipairs = ipairs
local min = math.min
local pairs = pairs
local setmetatable = setmetatable
local sort = table.sort
local split = require_when_needed("Module:string utilities", "split")
local table_len = require_when_needed("Module:table", "length")
local type = type

-- Copied from [[Module:Unicode data]].
local function binaryRangeSearch(codepoint, ranges)
	local low, mid, high
	low, high = 1, ranges.length or table_len(ranges)
	while low <= high do
		mid = floor((low + high) / 2)
		local range = ranges[mid]
		if codepoint < range[1] then
			high = mid - 1
		elseif codepoint <= range[2] then
			return range, mid
		else
			low = mid + 1
		end
	end
	return nil, mid
end

-- Copied from [[Module:Unicode data]].
local function linearRangeSearch(codepoint, ranges)
	for i, range in ipairs(ranges) do
		if codepoint < range[1] then
			break
		elseif codepoint <= range[2] then
			return range
		end
	end
end

local function compareRanges(range1, range2)
	return range1[1] < range2[1]
end

-- Save previously used codepoint ranges in case another character is in the
-- same range.
local rangesCache = {}

--[=[
	Takes a codepoint or a character and finds the script code(s) (if any) that are appropriate for it based on the codepoint, using the data module [[Module:scripts/recognition data]]. The data module was generated from the patterns in [[Module:scripts/data]] using [[Module:User:Erutuon/script recognition]].
	
	By default, it returns only the first script code if there are multiple matches (i.e. the code we take to be the default). If `all_scripts` is set, then a table of all matching codes is returned.
]=]

local charToScriptData
function subexport.charToScript(char, all_scripts)
	charToScriptData = charToScriptData or mw.loadData("Module:scripts/recognition data")
	local t = type(char)
	local codepoint
	if t == "string" then
		local etc
		codepoint, etc = cp(char, 1, 2)
		if etc then
			error("bad argument #1 to 'charToScript' (expected a single character)")
		end
	elseif t == "number" then
		codepoint = char
	else
		error(("bad argument #1 to 'charToScript' (expected string or a number, got %s)")
			:format(t))
	end
	
	local ret = {}
	local individualMatch = charToScriptData.individual[codepoint]
	if individualMatch then
		ret = split(individualMatch, "%s*,%s*", true)
	else
		local range
		if rangesCache[1] then
			range = linearRangeSearch(codepoint, rangesCache)
			if range then
				for i, script in ipairs(range) do
					if i > 2 then
						insert(ret, script)
						if not all_scripts then
							break
						end
					end
				end
			end
		end
		if not ret[1] then
			local index = floor(codepoint / 0x1000)
			range = linearRangeSearch(index, charToScriptData.blocks)
			if not range and charToScriptData[index] then
				range = binaryRangeSearch(codepoint, charToScriptData[index])
				if range then
					insert(rangesCache, range)
					sort(rangesCache, compareRanges)
				end
			end
			if range then
				for i, script in ipairs(range) do
					if i > 2 then
						insert(ret, script)
						if not all_scripts then
							break
						end
					end
				end
			end
		end
	end
	if not ret[1] then
		insert(ret, "None")
	end
	if all_scripts then
		return ret
	else
		return ret[1]
	end
end

--[=[
	Finds the best script for a string in a language-agnostic way.
	
	Converts each character to a codepoint. Iterates the counter for the script code if the codepoint is in the list
	of individual characters, or if it is in one of the defined ranges in the 4096-character block that it belongs to.
	
	Each script has a two-part counter, for primary and secondary matches. Primary matches are when the script is the
	first one listed; otherwise, it's a secondary match. When comparing scripts, first the total of both are compared
	(i.e. the overall number of matches). If these are the same, the number of primary and then secondary matches are
	used as tiebreakers. For example, this is used to ensure that `Grek` takes priority over `Polyt` if no characters
	which exclusively match `Polyt` are found, as `Grek` is a subset of `Polyt`.
	
	If `none_is_last_resort_only` is specified, this will never return None if any characters in `text` belong to a
	script. Otherwise, it will return None if there are more characters that don't belong to a script than belong to
	any individual script. (FIXME: This behavior is probably wrong, and `none_is_last_resort_only` should probably
	become the default.)
]=]
function subexport.findBestScriptWithoutLang(text, none_is_last_resort_only)
	-- `scripts` contains counters for any scripts detected so far. Jpan and Kore are handled as special-cases, as they are combinations of other scripts.
	local scripts_mt = 	{Jpan = true, Kore = true}
	
	local weights_mt = {
		__lt = function(a, b)
			if a[1] + a[2] ~= b[1] + b[2] then
				return a[1] + a[2] < b[1] + b[2]
			elseif a[1] ~= b[1] then
				return a[1] < b[1]
			elseif a[2] ~= b[2] then
				return a[2] < b[2]
			else
				return false
			end
		end
	}
	scripts_mt.__index = function(t, k)
		local ret = {}
		if k == "Jpan" and scripts_mt.Jpan then
			for i = 1, 2 do
				ret[i] = t["Hani"][i] + t["Hira"][i] + t["Kana"][i]
			end
		elseif k == "Kore" and scripts_mt.Kore then
			for i = 1, 2 do
				ret[i] = t["Hani"][i] + t["Hang"][i]
			end
		else
			for i = 1, 2 do
				insert(ret, 0)
			end
		end
		return setmetatable(ret, weights_mt)
	end
	
	local scripts = setmetatable({}, scripts_mt)
	
	text = get_plaintext(text)
	
	local combined_scripts = {
		Jpan = {["Hani"] = true, ["Hira"] = true, ["Kana"] = true},
		Kore = {["Hani"] = true, ["Hang"] = true}
	}
	
	for character in text:gmatch(".[\128-\191]*") do
		for i, script in ipairs(subexport.charToScript(character, true)) do
			if not none_is_last_resort_only or script ~= "None" then
				scripts[script] = scripts[script]
				local weight = min(i, 2)
				scripts[script][weight] = scripts[script][weight] + 1
			end
		end
	end
	
	-- Check the combined script counts. If a single constituent has the same count (i.e. it's the only one), discard the combined script.
	for combined_script, set in pairs(combined_scripts) do
		for script in pairs(set) do
			scripts[combined_script] = scripts[combined_script]
			if (scripts[script][1] + scripts[script][2]) == (scripts[combined_script][1] + scripts[combined_script][2]) then
				scripts[combined_script] = nil
				break
			end
		end
	end
	
	local bestScript
	local greatestCount
	for script, count in pairs(scripts) do
		if (not greatestCount) or greatestCount < count then
			bestScript = script
			greatestCount = count
		end
	end
	
	bestScript = bestScript or "None"
	
	return get_script(bestScript)
end

return subexport