Module:Technique/Comparison

From Wikimedia Commons, the free media repository
Jump to navigation Jump to search

Documentation for this module may be created at Module:Technique/Comparison/doc

--[=[
TODO:
* adjectives
* for performance: consider not to preprocess and to read templates like
  {{c}} (in en), {{wf}} (in fr), {{w}} (in nl) with rules
* consider to put information of the wikitable in a Lua table and concatenate,
  e.g. have a header table with headers['wd1'] = '[[blah]]' etc.
* bring back the "#default" from [[Template:Technique/ru/adjectives]] and
  [[Template:Technique/mk/adjectives]]
]=]

require('Module:No globals') -- used for debugging purposes as it detects cases of unintended global variables
local material_LUT = require('Module:Artwork/Technique LUT')
local material_LUT1 = require('Module:Technique/WikidataLUT').nouns
local synonyms = require('Module:Technique/synonyms')
local plurals = require('Module:Technique/WikidataLUT').plurals
local declension = require('Module:Declension')

local p = {}

-- provide the keys of a table in sorted order
local function getSortedKeys(inTable)
	local sortedKeys = {}
	for key,_ in pairs(inTable) do
		table.insert(sortedKeys, key)
	end
	table.sort(sortedKeys)
	return sortedKeys
end

local function getlanguageSelector(notpiol)
	local languageSelector = mw.getCurrentFrame():expandTemplate{
		title='LanguageSelector',
		args = {'ar','ca','cs','da','de','el','en','es','et','fi','fr',
			'gl', 'he','hu','it','ja','mk','nds','nl','no','pl','pt',
			'ro','ru','scn','sk','sl','sr','sv','sw','vec','vi','zh',
			notpiol=notpiol
		}}
	return languageSelector
end

local function syntaxHighlightLuaCode(code)
	local res = '<syntaxhighlight lang="lua">' .. code .. '</syntaxhighlight>'
	return mw.getCurrentFrame():preprocess(res)
end

local function termSearchLink(term)
	local outText = '[[File:Vector search icon.svg|10x10px|link='
		.. 'https://commons.wikimedia.org/w/index.php?search='
		.. mw.uri.encode('insource:"'
			.. term .. '" hastemplate:technique insource:/\\|((1|2|on|and[1-5]?|mounted|over)=)? *'
			.. term .. ' *(}|\\|)/i', 'PATH')
		.. ']]'
	return outText
end

local function stripHtmlComments(inText)
	return table.concat(mw.text.split(inText,'<!%-%-.-%-%->'))
end

local function escapeSingleQuote(inText)
	return inText:gsub("'", "\\'")
end

local function findTermsArea(inText)
	local area = mw.ustring.match(inText, '{{#switch:%s*{{{1|}}}.-(|.*#default%s*=.)')
	return area
end

local function analyzeTermLine(line)
	local before, comment, after = mw.ustring.match(line, '^(.-)%s*<!%-%-%s*(.-)%s*%-%->%s*(.*)$')
	if before then
		line = before .. after
	end
	local left, right =  mw.ustring.match(line,'^%s*|%s*(.-)%s*=%s*(.-)%s*$')
	-- get rid of standard keys
	local standardKeys = {['']=1, adj=1, basic=1, over=1, on=1, mounted=1, order=1, case=1, ['#default']=1}
	if left and right and not standardKeys[left] then
		local lineData = {left=left}
		lineData.synonyms = mw.text.split(left, '%s*|%s*')
		lineData.right = right
		lineData.comment = comment
		return lineData
	end
end

-- parse a language subpage of [[Template:Technique]] and
-- return its mapping as a table
local function analyzeLangSubtemplate(wordtype, lang, selectFunction)
	local pageWordtypePart
	if wordtype == 'nouns' then
		pageWordtypePart = ''
	elseif wordtype == 'adjectives' then
		pageWordtypePart = '/adjectives'
	end
	local langPage = mw.title.new('Technique/' .. lang .. pageWordtypePart, 'Template'):getContent()
	local nounTerms = {}
	local synTerms = {}
	local mappingCount, totalMappingCount, parserFunctionCount, templateCount = 0, 0, 0, 0
	
	langPage = stripHtmlComments(langPage)
	local area = findTermsArea(langPage)
	for line in mw.ustring.gmatch(area, '|[^\n]-\n') do
	-- for some reason that gets slow with a simpler pattern, gsplit is even worse
		local lineData = analyzeTermLine(line)
		if lineData then
			-- count mappings:
			mappingCount = mappingCount + 1
			-- if the value has a '{{' which indicates a template or a parser function
			-- it has to be preprocessed:
			if mw.ustring.find(lineData.right, '^[^{]*{{[^{]') then
				-- count mappings with parser functions
				if mw.ustring.find(lineData.right, '{{#') then
					parserFunctionCount = parserFunctionCount + 1
				end
				-- count mappings with templates
				-- add a '_' to find cases with '{{' in front
				if mw.ustring.find('_' .. lineData.right, '[^{]{{[^{#]') then
					templateCount = templateCount + 1
				end
				lineData.right = mw.getCurrentFrame():preprocess(lineData.right)
			end
			local synonyms = lineData.synonyms
			for _,s in ipairs(synonyms) do
				if not selectFunction or selectFunction(s) then
					nounTerms[s] = lineData.right
					totalMappingCount = totalMappingCount + 1
				end
			end
			-- map to the first term given
			for i = #synonyms, 2, -1 do
				synTerms[synonyms[i]] = synonyms[1]
			end
		end
	end
	return nounTerms, mappingCount, parserFunctionCount, templateCount, totalMappingCount, synTerms
end


-- copying instead of this function would have done it too …
-- parse [[Template:Technique/lang]] to get all language codes linked from there
local function subpageLangs()
	local content = mw.title.new('Technique/lang', 'Template'):getContent()
	local subpageLangs = {}
	for langCode in mw.ustring.gmatch(content, '{{fullurl:Template:Technique/([a-z][a-z][a-z]?)}} {{#language:') do
		table.insert(subpageLangs, langCode)
	end
	return subpageLangs
end

local function collectQIDs(terms)
	local qids = {}
	for _, term in pairs(terms) do
		if term['qid1'] then
			table.insert(qids, term['qid1'])
		end
	end
	return qids
end

local function constructTechniqueSPARQL(languages, qids)
	local vars = ''
	local optionals = ''
	local optionalform = [=[
  OPTIONAL { ?item rdfs:label ?label%s. FILTER(LANG(?label%s)='%s') }
  OPTIONAL { ?link%s schema:about ?item;
                     schema:isPartOf <https://%s.wikipedia.org/> . }
]=]
	for i, lang in ipairs(languages) do
		vars = vars .. '?label' .. lang .. ' ?link' .. lang .. ' '
		optionals = optionals .. string.format(optionalform, lang, lang, lang, lang, lang)
	end
	local values = ''
	for i, q in ipairs(qids) do
		values = values .. ' wd:' .. q
	end
	
	local queryform = [=[
SELECT ?item
%s
WHERE {
  VALUES ?item { %s }
%s
}
LIMIT 10000
]=]
	local sparqlQ = string.format(queryform, vars, values, optionals)
	local queryURL = 'https://query.wikidata.org/#' .. mw.uri.encode(sparqlQ, 'PATH')
	return queryURL
end


--[=[ for debugging:
local wdMap = {
	Q207849 = {fr = {label = "peinture acrylique", link = "Peinture acrylique"}, en = {label = "acrylic paint", link = "Acrylic paint"}, },
	Q123314 = {fr = {label = "agate", link = "Agate"}, en = {label = "agate", link = "Agate"}, },
	Q143447 = {},
	Q2144394 = {fr = {label = "", link = ""}, en = {label = "", link = ""}, },
	
	Q127583 = {sk={label='zafír', link='barLink'}}, -- same label
	Q42329 = {sk={label='vlna', link=nil}}, -- same label, no link
	Q14298 = {sk={label='satén DIFFERENT', link='barLink'}}, -- different label
	}
wdMap.Q207849.sk = {label='fooLabel', link='barLink'}
wdMap.Q123314.sk = {label=nil, link='barLink'}
wdMap.Q143447.sk = {label='fooLabel', link=nil}
wdMap.Q2144394.sk = {label='fooLabel', link='barLink'}
--]=]

-- @treturn string linkedLabel
local function getLinkedLabel(pagename, prefix, label)
	-- prefix must be a string if pagename isn't nil
	-- pagename and label can always be nil
	local linkedLabel = ''
	if pagename then
		local labelpart = ''
		if label then
			labelpart = '|' .. label
		end
		linkedLabel = '[[:' .. prefix .. ':' .. pagename .. labelpart .. ']]'
	else
		linkedLabel = label or ''
	end
	return linkedLabel
end

-- finds wikilinks to 2 or 3 letter prefixed Wikipedia versions in given wikitext string
local function analyzeWikilink(wikitext)
	local _, _, prefix, link, label = mw.ustring.find(wikitext,
		'^%[%[:?w?:(%l%l%l?):([^%[%]]-)%|([^%[%]]-)%]%]%s*$')
		-- pattern issues:
		-- * something like '[[::foo|bar]]' would get matched
		-- * something like '[[:en:foo[1]|bar]]' wouldn't get matched
	-- if label couldn't be detected as a link label, take the bare text:
	if not label then
		label = wikitext
	end
	-- make first character of link uppercase as Wikipedia title
	-- standard and stored titles in Wikidata:
	if link then
		link = mw.ustring.upper(mw.ustring.sub(link,1,1)) .. mw.ustring.sub(link,2)
	end
	return prefix, link, label
end

local function getBestSitelink(lang, qid, altQids)
	local sitelink = mw.wikibase.getSitelink(qid, lang .. 'wiki')
	local sitelinkIsAlt = false
	if not sitelink and altQids then
		for _,qid in ipairs(altQids) do
			local link = mw.wikibase.getSitelink(qid, lang .. 'wiki')
			if link then
				sitelink = link
				sitelinkIsAlt = true
				break
			end
		end
	end
	return sitelink, sitelinkIsAlt
end

local function compareToWD(termTable, languages)
	for _, lang in ipairs(languages) do
		local translation = termTable[lang]
		translation.wdSitelink, translation.wdSitelinkIsAlt = getBestSitelink(lang, termTable.qid1, termTable.altQids)
		translation.wdLabel = mw.wikibase.getLabelByLang(termTable.qid1, lang)
		translation.wdLinkedLabel = getLinkedLabel(
			translation.wdSitelink, lang, translation.wdLabel)
		if translation.wdSitelinkIsAlt then
			translation.wdLinkedLabel = "''" .. translation.wdLinkedLabel
				.. [['' (<span title="link from an alternative item">alt</span>)]]
		end
		
		local prefix, link, commonsLabel = analyzeWikilink(translation['com'] or '')
		local commonsLink
		if prefix == lang then
			commonsLink = link
		end
		
		-- get colors for marking comparison with Wikidata
		-- (colors from https://en.wikipedia.org/wiki/Help:Using_colours#Wikimedia_colour_schemes)
		-- note: commonsLink (if found) won't be ''!
		if  commonsLink and translation.wdSitelink == commonsLink then
			-- they are same -> get a GREEN
			translation.linksComparisonColor = '#CEF2E0'
		elseif commonsLink then
			-- they are different -> get a RED
			translation.linksComparisonColor = '#F2CEE0'
		end
		if translation.wdLabel and translation.wdLabel ~= '' and commonsLabel and commonsLabel ~= '' then
			if translation.wdLabel ~= commonsLabel:gsub('­', '') then
				-- ignore soft hyphens in comparison
				-- they are different -> get a YELLOW
				translation.labelsComparisonColor = '#faecc8'
			else
				-- they are same -> get a GREEN
				translation.labelsComparisonColor = '#CEF2E0'
			end
		end
		if commonsLabel and commonsLabel ~= '' and not translation.wdLabel then
			translation.exportWdLabel = commonsLabel
		end
		if commonsLabel and commonsLabel ~= '' and translation.wdLabel
				and commonsLabel ~= translation.wdLabel
				and commonsLabel == mw.ustring.gsub(translation.wdLabel, '^%u', mw.ustring.lower) then
			translation.exportWdLabelLowercase = commonsLabel
		end
	end
	return termTable
end

-- source: [[Template:Technique/list]]
local function getTechniqueList(terms, selectFunction)
	local allTemplate = mw.title.new('Technique/list', 'Template'):getContent()
	local extract = mw.ustring.match(allTemplate, '<table(.-)</table>')
	for term in mw.ustring.gmatch(extract, '<tr><td>(.-)</td><td>{{technique/{{int:lang}}') do
		if selectFunction(term) then
			terms[term] = {isInListTemplate='true'}
		end
	end
	return terms
end

-- source: [[Module:Technique/WikidataLUT]]
local function getQIDs1(terms, selectFunction)
	local counterWD1 = 0
	for term, termData in pairs(material_LUT1) do
		if selectFunction(term) then
			if terms[term] then
				terms[term]['qid1'] = termData['qid']
				terms[term]['altQids'] = termData['altQids']
				counterWD1 = counterWD1 + 1
			else
				mw.log(term)
				-- stuff in [[Module:Technique/WikidataLUT]] that perhaps shouldn't be there
			end
		end
	end
	return terms, counterWD1
end

-- source: [[Module:Artwork/Technique LUT]]
local function getQIDs2(terms)
	for qid, term in pairs(material_LUT) do
		term = mw.ustring.lower(term)
		-- fix some uppercase terms in [[Module:Artwork/Technique LUT]]
		if not terms[term] then
			terms[term] = {}
		end
		terms[term]['qid'] = qid
	end
	return terms
end

-- source: language subpages of [[Template:Technique]]
local function getTechniqueLangPages(terms, languages, selectFunction)
	local langMappings = {}
	local synTerms = {}
	local counters = {}
	counters.mappings = {}
	counters.totalMappings = {}
	counters.parserfunction = {}
	counters.template = {}
	counters.unlistedTerm = {}

	for _, lang in ipairs(languages) do
		langMappings[lang],
			counters.mappings[lang], 
			counters.parserfunction[lang],
			counters.template[lang],
			counters.totalMappings[lang],
			synTerms[lang] = analyzeLangSubtemplate('nouns', lang, selectFunction)
		counters.unlistedTerm[lang] = 0
		for term, value in pairs(langMappings[lang]) do
			if selectFunction(term) and not terms[term] then
				terms[term] = {}
			end
			if selectFunction(term) and not terms[term]['isInListTemplate'] then
				counters.unlistedTerm[lang] = counters.unlistedTerm[lang] + 1
			end
		end
	end
	return terms, langMappings, synTerms, counters
end

local function makeLuaSynonyms(synonyms)
	local synsKeys = getSortedKeys(synonyms)
	local synonymPairs = {}
	for _,v in ipairs(synsKeys) do
		table.insert(synonymPairs, "['" .. escapeSingleQuote(v) .. "'] = '" .. escapeSingleQuote(synonyms[v]) .. "',")
	end
	local luaCode = table.concat(synonymPairs, '\n')
	local styledLuaCode = string.format('<pre>%s</pre>', luaCode)
	-- nicer, but slower:
--	local styledLuaCode = syntaxHighlightLuaCode(luaCode)
	return styledLuaCode
end

local function termsVisualizer(languages, compare_to_WD, selectFunction, showQs)
	local selectFunction = selectFunction or function(term) return true end
	
	-- collect terms from some sources
	local terms = {} -- the term (noun) table for all sources
	terms = getTechniqueList(terms, selectFunction)
	local terms, counterWD1 = getQIDs1(terms, selectFunction)
--	terms = getQIDs2(terms)
	local terms, langMappings, synTerms, counters = getTechniqueLangPages(terms, languages, selectFunction)
	
	-- construct a SPARQL query showing information from Wikidata
	local qids = collectQIDs(terms)
	local queryURL = constructTechniqueSPARQL(languages, qids)
	
	-- get a table of the keys of terms in sorted order to access them in that order
	local sortedTerms = getSortedKeys(terms)
	
	-- prepare the parts for the output wikitext which need language looping
	local allHeaderCells = ''
	local figuresCells = ''
	local manyPFCount = 0
	for _, lang in ipairs(languages) do
		if compare_to_WD then
			for _, term in ipairs(sortedTerms) do
				terms[term][lang] = {['com'] = langMappings[lang][term]}
				--[[ CAVE: seems to use a lot of memory
				that's why langMappings[lang][term] is used directly and
				this is only assigned here if compare_to_WD and thus needed
				to hand the value over to function compareToWD() ]]
			end
		end
		
		local langHeaderCells = '! [[Template:Technique/' .. lang .. '|' .. lang ..
			']] [[File:Commons-logo.svg|15x15px|link=]]\n'
		if compare_to_WD then
			langHeaderCells = langHeaderCells .. '!' .. lang ..
			' [[File:Wikidata-logo.svg|15x15px|link=]]\n'
		end
		allHeaderCells = allHeaderCells .. langHeaderCells
		
		local synsPrint = string.format([[
{| class="wikitable mw-collapsible mw-collapsed"
|+ class="nowrap" style="text-align:left;font-weight:normal; | synonyms:
|-
| %s
|}
]], makeLuaSynonyms(synTerms[lang]))
		
		local parserfunctionRatio = math.ceil(counters.parserfunction[lang] / counters.mappings[lang] * 1000) / 10
		local templateRatio = math.ceil(counters.template[lang] / counters.mappings[lang] * 1000) / 10
		local manyPFColor = ''
		if parserfunctionRatio > 20 then
			manyPFColor = '#d0e5f5' -- a blue
			manyPFCount = manyPFCount + 1
		end
		figuresCells = figuresCells ..
			'!style="text-align:left;font-weight:normal;background:' ..
			manyPFColor .. ';"| ' ..
			'<abbr title="number of mappings">MP</abbr>:&nbsp;' ..
			tostring(counters.mappings[lang]) .. 
			[=[ + ''<abbr title="synonyms">s</abbr>'' = ]=] ..
			tostring(counters.totalMappings[lang]) .. '<br />' ..
			'<abbr title="parser function ratio">PF</abbr>:&nbsp;' ..
			tostring(parserfunctionRatio) .. '<br />' ..
			'<abbr title="template ratio">TP</abbr>:&nbsp;' .. 
			tostring(templateRatio) .. '<br />' ..
			'<abbr title="number of unlisted terms">UL</abbr>:&nbsp;' .. 
			tostring(counters.unlistedTerm[lang]) .. '\n' .. synsPrint .. '\n'
		if compare_to_WD then
			figuresCells = figuresCells .. '!\n'
		end
	end

	local counterWD1synonyms = 0
	local termRows = ''
	for _, term in pairs(sortedTerms) do
--		local labelWdLink = ''
		local wdLink = ''
		local qsString = ''
		local qsStringLowercase = ''
		if terms[term]['qid1'] then
			wdLink = '[[d:' .. terms[term]['qid1'] .. '|' .. terms[term]['qid1'] .. ']]'
			-- here come many of: linksComparisonColor, labelsComparisonColor, wdLinkedLabel
			-- CAVE: this is expensive!!!
			if compare_to_WD then
				local compResult = compareToWD(terms[term], languages)
				terms[term]['linksComparisonColor'] = compResult['linksComparisonColor']
				terms[term]['labelsComparisonColor'] = compResult['labelsComparisonColor']
				terms[term]['wdLinkedLabel'] = compResult['wdLinkedLabel']
				terms[term]['exportWdLabel'] = compResult['exportWdLabel']
				terms[term]['exportWdLabelLowercase'] = compResult['exportWdLabelLowercase']
				-- TODO: clean this up! or understand it first!
				--       actually, I don't understand why this works
			end
		elseif synonyms.nouns[term] then
			wdLink = "see ''" .. synonyms.nouns[term] .. "''"
			counterWD1synonyms = counterWD1synonyms + 1
		elseif plurals[term] then
			wdLink = "see singular ''" .. plurals[term] .. "''"
			counterWD1synonyms = counterWD1synonyms + 1
		end
		
		-- construct cells for the languages
		local langCells = ''
		terms[term].counterItemsFromTranslations = 0
		for _, lang in ipairs(languages) do
			local translation = terms[term][lang]
			
			local labelsComparisonColor = ''
			local wdCell = ''
			if compare_to_WD then
				wdCell = '\n| '
				if translation.wdLinkedLabel then
					labelsComparisonColor = translation.labelsComparisonColor or ''
					wdCell = wdCell .. 'style="background:' ..
						(translation.linksComparisonColor or '') .. '"| ' ..
						(translation.wdLinkedLabel)
				elseif not synonyms.nouns[term] and not plurals[term] then
					local prefix, link, commonsLabel = analyzeWikilink(translation['com'] or '')
					if prefix == lang then
						local qid = mw.wikibase.getEntityIdForTitle(link, lang .. 'wiki')
						if qid then
							local wdLabel = mw.wikibase.getLabelByLang(qid, lang)
							local color = ''
							if wdLabel and wdLabel == commonsLabel then
								-- they are same -> get a GREEN
								color = '#CEF2E0'
							elseif wdLabel and wdLabel ~= commonsLabel then
								-- they are different -> get a YELLOW
								color = '#faecc8'
							end
							wdCell = wdCell .. 'data-sort-value="<' .. term ..
								'"|<' .. qid .. '?> ([[:d:' .. qid ..
								'|d]]; <span style="background-color:' .. color ..
								'"><abbr title="label">l</abbr>: ' ..
								(wdLabel or '') .. '</span>)'
							terms[term].counterItemsFromTranslations = terms[term].counterItemsFromTranslations + 1
						end
					end
				else
					wdCell = wdCell .. ' style="background-color:lightgray |'
				end
				if translation.exportWdLabel then
					qsString = qsString .. '\n' .. terms[term]['qid1'] .. '\tL' .. lang .. '\t"' .. translation.exportWdLabel .. '"'
				end
				if translation.exportWdLabelLowercase then
					qsStringLowercase = qsStringLowercase .. '\n' .. terms[term]['qid1'] .. '\tL' .. lang .. '\t"' .. translation.exportWdLabelLowercase .. '"'
				end
			end
			
			langCells = langCells .. string.format('\n|style="background:%s" title="' .. lang .. '" | %s%s',
				labelsComparisonColor, (langMappings[lang][term] or ''), wdCell)
		end
		
		local inListYesNo = ''
		if terms[term].isInListTemplate then
			inListYesNo = 'yes'
		else
			inListYesNo = 'no'
		end
		
		local qsLink = ''
		local qsLinkLowercase = ''
		if showQs and qsString ~= '' and qsString ~= terms[term]['qid1'] then
			qsLink = [=[[[File:Commons to Wikidata QuickStatements.svg|15x15px|link=https://tools.wmflabs.org/quickstatements/index_old.html#v1=]=] .. mw.uri.encode(qsString, 'PATH') .. ']]'
		end
		if showQs and qsStringLowercase ~= '' and qsStringLowercase ~= terms[term]['qid1'] then
			qsLinkLowercase = [=[ lc:[[File:Commons to Wikidata QuickStatements.svg|15x15px|link=https://tools.wmflabs.org/quickstatements/index_old.html#v1=]=] .. mw.uri.encode(qsStringLowercase, 'PATH') .. ']]'
		end
		
		local termRow ='|-' ..
			'\n| ' .. term ..
			'\n| ' .. inListYesNo .. ' ' .. termSearchLink(term) ..
			'\n| ' .. wdLink .. qsLink .. qsLinkLowercase ..
			((wdLink == '' and compare_to_WD and ('<span title="number of suggestions">' ..
				terms[term].counterItemsFromTranslations .. '</span>')) or '') ..
--			'\n| ' .. labelWdLink ..
			langCells ..
			'\n'
		termRows = termRows .. termRow
	end
	
	
	-- construct wikitext for output
	local languageSelector = ''
	local comparisonDescription = ''
	if compare_to_WD then
		languageSelector = '<small>' .. getlanguageSelector() .. '</small>\n\n'
		local languageSelectorNotpiol = getlanguageSelector('1')
		local userLang = mw.getCurrentFrame():callParserFunction('int', 'lang')
		comparisonDescription = string.format([=[
The [[File:Wikidata-logo.svg|15x15px|link=]] Wikidata columns show
how Wikidata IDs are automatically processed to nice linked text.

The Commons and Wikidata columns compare the labels and links from
[[Template:Technique]] subtemplates:
* If for a given language the '''labels''' match the <!--
-->[[File:Commons-logo.svg|15x15px|link=]] Commons column has a <!--
--><span style="background-color:#CEF2E0">green background</span>, <!--
-->if they don't a <!--
--><span style="background-color:#faecc8">yellow background</span>.
* If for a given language the '''links''' match the <!--
-->[[File:Wikidata-logo.svg|15x15px|link=]] Wikidata column has a <!--
--><span style="background-color:#CEF2E0">green background</span>, <!--
-->if Commons holds a link, but it doesn't match the one from Wikidata a <!--
--><span style="background-color:#F2CEE0">red background</span>.
* If the term isn't in [[Module:Technique/WikidataLUT]] or <!--
-->[[Module:Technique/synonyms]], but the language subtemplate contains a link <!--
-->to a Wikipedia page for the term, <!--
-->and if a Wikidata item can be found for the link, <!--
-->the item's ID is shown <in brackets>. <!--
-->It is followed by a link to the Wikidata item and the label it holds <!--
-->with the label's color signifying if it matches the label on Commons or not.

'''You can help by aligning information on Commons and Wikidata'''
which improves quality of both and helps
to migrate as much of the data for [[Template:Technique]] as possible to Wikidata
where it is easier to maintain and serves a wider use.
That way we get translations and links for more terms and for more languages.
Follow these steps:
# '''Select the language''' in which you want to align translations and links <!--
-->in the language selector: <small>%s</small>
# '''Sort the table''' by the Commons or Wikidata column for the chosen language.
# Now scan the cells of the columns and if a one is yellow or red <!--
-->'''fix the label or link/sitelink''' either on the language subtemplate <!--
-->[[Template:Technique/%s]] on Commons or <!--
-->at the Wikidata item that translates the term which is linked in the "WD1" column. <!--
--><small>(It is possible that the Wikidata item isn't a good choice for the term, <!--
-->it can be changed on [[Module:Technique/WikidataLUT]]. <!--
-->It will probably be a good idea to consult <!--
-->[[Template:Technique/translation dashboard]] <!--
-->for the translation in other languages before changing it.)</small>
]=], languageSelectorNotpiol, userLang)
	end
	
	local headerRow = string.format([=[
|-
! term
! [[Template:Technique/list|listed]]?
! [[Module:Technique/WikidataLUT|WD1]]
%s]=], allHeaderCells)
	
	local counterWD1all = counterWD1 + counterWD1synonyms
	
	local outText = string.format([=[
%sThe first column of the table below shows
%s
translatable technique noun terms from [[Template:Technique/list]],
[[Module:Technique/WikidataLUT]] and
%s
language subtemplates of [[Template:Technique]]
(of those linked from [[Template:Technique/lang]] like [[Template:Technique/en]]).

The [[File:Commons-logo.svg|15x15px|link=]] Commons columns show
how the language subtemplates process the terms.
%s
<small>In %s out of %s language subtemplates in more than 20%% of the mappings
parser functions were detected.
That means that for those it probably won't be possible to replace them
completely with Wikidata querying before
[[phab:T212843|Lua access to lexemes is possible]].

It should be possible to update the data from Wikidata with [%s this query].</small>	

<small>
{|class="wikitable sortable"
|+ Comparison of different renderings of techniques
%s
|-
! style="text-align:left;font-weight:normal; | number of mappings (MP):<br />
parser function ratio (PF):<br />
template ratio (TP):<br />
number of unlisted terms (UL):
!
! style="text-align:left;font-weight:normal;" | <abbr title="number of mappings">MP</abbr>:&nbsp;%s + %s <abbr title="synonyms">s</abbr> = %s
%s
%s
%s
|}</small>
]=],
		languageSelector, tostring(#sortedTerms), tostring(#languages),
		comparisonDescription, manyPFCount, tostring(#languages), queryURL,
		headerRow, counterWD1, counterWD1synonyms, counterWD1all, figuresCells, termRows, headerRow
		)
	
	return outText
end


-- ===========================================================================
-- === functions to convert the language subpages of
-- === [[Template:Technique]] to Lua versions
-- ===========================================================================

local function fillToLength(inString, minimalLength)
	return inString .. mw.ustring.rep(' ', minimalLength - mw.ustring.len(inString))
end

-- make a Lua table string from a list that may contain nil elements,
-- which isn't possible with a simple table.concat.
-- expects only string values in the inList.
local function makeLuaTable(inList)
	local ps = {}
	local lastKey = 0
	for _,k in ipairs(getSortedKeys(inList)) do
		if k == lastKey+1 then
			lastKey = lastKey + 1
			ps[#ps+1] = string.format("'%s'", inList[k])
		else
			ps[#ps+1] = string.format("[%s] = '%s'", k, inList[k])
		end
	end
	return '{' .. table.concat(ps, ', ') .. '}'
end

local function analyzeParserFunctionSwitch(cases)
	cases = cases or ''
	local curlyBracketCount, squareBracketCount, currentPosition = 0, 0, 0
	local isComment = false
	local part, startPosition
	local keys = {}
	local lastDelimiter = '|'
	local switchData = {}
	for c in mw.ustring.gmatch(cases .. '|', '.') do
		currentPosition = currentPosition + 1
		local isFirstLevel = (curlyBracketCount == 0
			and squareBracketCount == 0
			and not isComment)
		if c == '{' then
			curlyBracketCount = curlyBracketCount + 1
		elseif c == '}' then
			curlyBracketCount = curlyBracketCount - 1
		elseif c == '[' then
			squareBracketCount = squareBracketCount + 1
		elseif c == ']' then
			squareBracketCount = squareBracketCount - 1
		elseif c == '<' and mw.ustring.sub(cases, currentPosition, currentPosition+3) == '<!--' then
			isComment = true
		elseif c == '>' and mw.ustring.sub(cases, currentPosition-2, currentPosition) == '-->' then
			isComment = false
		elseif isFirstLevel and c == '|'  and lastDelimiter == '|' and currentPosition == mw.ustring.len(cases)+1 then
			-- default value as last value without a key "#default" given
			switchData['#default'] = mw.text.trim(mw.ustring.sub(cases, startPosition, currentPosition-1))
		elseif isFirstLevel and (c == '=' or c == '|') and lastDelimiter == '|' then
			-- set key before '=' or in multi-assignment
			keys[#keys+1] = mw.text.trim(mw.ustring.sub(cases, startPosition, currentPosition-1))
			startPosition = currentPosition + 1
			lastDelimiter = c
		elseif isFirstLevel and c == '|' then
			-- set value before '|'
			if keys ~= {} then
				for _,key in ipairs(keys) do
					switchData[key] = mw.text.trim(mw.ustring.sub(cases, startPosition, currentPosition-1))
				end
				keys = {}
			end
			startPosition = currentPosition + 1
			lastDelimiter = c
		end
	end
	return switchData
end

local function getDefaultPrefixedLink(prefix, term)
	term = plurals[term] or term
	term = synonyms.nouns[term] or term
	local termData = material_LUT1[term]
	local link = termData and termData.qid and getBestSitelink(prefix, termData.qid, termData.altQids)
	return (link and prefix .. ':' .. link)
end

local function makeLuaNounData(wikiTextNounData, term, lang)
	local termData, prefix, prefixedLink, right
	local warnings = {}
	local cases =  mw.ustring.match(wikiTextNounData,'^{{#switch:%s*{{{query|}}}%s*|(.-)%s*}}%s*$')
	if cases then
		termData = analyzeParserFunctionSwitch(cases)
	end
	if termData then
		local expectedKeys = {['#default']=1, L=1, l=1, d=1, p=1, a=1, ad=1, al=1, gender=1}
		for k,_ in pairs(termData) do
			if not expectedKeys[k] then
				warnings[#warnings+1] = 'FIXME: unexpected key "' .. k .. '"'
			end
		end
		termData.default = termData['#default'] --or 'WARNING: NO DEFAULT!!!'
		
		if termData.gender == 'p' then
			termData.gender = nil
			termData.number = 'p'
		elseif termData.gender then
			local gender, number =  mw.ustring.match(termData.gender,'^([mfn])([sp])$')
			if gender then
				termData.gender = gender
				termData.number = number
			end
		end
		
		local lastPefixedLink
		expectedKeys['#default'] = nil
		expectedKeys['gender'] = nil
		expectedKeys['default'] = 1
		for k,_ in pairs(expectedKeys) do
			local fullCaseLink
			if termData[k] then
				local casePrefix, unprefixedCaseLink, label = analyzeWikilink(termData[k])
				if casePrefix then
					local prefixedCaseLink = casePrefix .. ':' .. unprefixedCaseLink
					if lastPefixedLink and prefixedCaseLink ~= lastPefixedLink then
						warnings[#warnings+1] = 'FIXME: mismatching links'
					end
					prefix = casePrefix
					prefixedLink = prefixedCaseLink
					lastPefixedLink = prefixedCaseLink
					termData[k] = label
				end
			end
		end
		if prefix and (prefixedLink ~= getDefaultPrefixedLink(prefix, term)) then
			termData.link = prefixedLink
		end
	else
		-- look what label [[Module:Technique/WikidataLUT]] provides
		local canonicalTerm = synonyms.nouns[term] or term
		local wdTermData = material_LUT1[canonicalTerm]
		local wdLabel = wdTermData and wdTermData.qid and mw.wikibase.getLabelByLang(wdTermData.qid, lang)
		
		local commonsPrefix, link, label = analyzeWikilink(wikiTextNounData)
		if commonsPrefix then
			prefix = commonsPrefix
			prefixedLink = commonsPrefix .. ':' .. link
			label = (label ~= wdLabel) and label
			if prefixedLink == getDefaultPrefixedLink(commonsPrefix, term) then
				if label then
					return "'" .. label .. "'", ''
				end
			else
				termData = {default = label, link = prefixedLink}
			end
		else
			if wikiTextNounData ~= wdLabel then
				return "'" .. wikiTextNounData .. "'", ''
			end
		end
	end
		
	if termData then
		local termDataPairs = {}
		for _,k in ipairs({'link', 'default', 'L', 'l', 'd', 'p', 'a', 'ad', 'al', 'gender', 'number'}) do
			if termData[k] then
				termDataPairs[#termDataPairs+1] = mw.ustring.lower(k) .. " = '" .. termData[k] .. "'"
			end
		end
		right = '{' .. table.concat(termDataPairs, ', ') .. '}'
	end
	return right, table.concat(warnings, ', ')
end

local function makeLuaAdjectiveData(wikiTextData, lang)
	local warnings = {}
	local luaAdjectiveData
	local before, cases, behind =  mw.ustring.match(wikiTextData,'^(.-){{%s*#switch:%s*{{{agreement|?}}}%s*|(.-)%s*}}(.*)$')
	if cases then
		local caseData = analyzeParserFunctionSwitch(cases)
		local usedGrammaticalFeatures = {
			default = {numbers={'', 'p'}, genders={'m', 'f'}, cases={''}},
			da = {numbers={''}, genders={'c', 'n', 'p'},
				out = {numbers={'', 'p'}, genders={'c', 'n'}}
			},
			de = {numbers={''}, genders={'m', 'f', 'n', 'p'}, cases={'', 'd'},
				out = {numbers={'', 'p'}, genders={'m', 'f', 'n'}}
			},
			et = {numbers={''}, genders={''}, cases={'', 'ad', 'al'}},
			mk = {numbers={'', 'p'}, genders={'m', 'f', 'n'}},
			pl = {numbers={'s', 'p'}, genders={'m', 'f', 'n'}, cases={'', 'L'}},
			ro = {numbers={'', 'p'}, genders={'m', 'f', 'n'}},
			ru = {numbers={''}, genders={'m', 'f', 'n', 'p'}, cases={'', 'd', 'p', 'a'},
				out = {numbers={'', 'p'}, genders={'m', 'f', 'n'}}
			},
		}
		for _,feature in ipairs({'numbers', 'genders', 'cases'}) do
			if usedGrammaticalFeatures[lang] and usedGrammaticalFeatures[lang][feature] then
				usedGrammaticalFeatures[feature] = usedGrammaticalFeatures[lang][feature]
			else
				usedGrammaticalFeatures[feature] = usedGrammaticalFeatures.default[feature]
			end
		end
		
		local caseTable = {}
		for key,value in pairs(caseData) do
			local number, gender, case
			local keystring = ''
			for _,g in ipairs(usedGrammaticalFeatures.genders) do
				local keystring = keystring .. g
				local genderConvert = {['']=1, m=1, f=2, n=3, c=1, p='p'}
				g = genderConvert[g]
				for _,n in ipairs(usedGrammaticalFeatures.numbers) do
					local keystring = keystring .. n
					if n == '' then n = 's' end
					if g == 'p' then n = 'p' end
					for _,c in ipairs(usedGrammaticalFeatures.cases) do
						local keystring = keystring .. c
						if key == keystring then
							caseTable[n] = caseTable[n] or {}
							caseTable[n][c] = caseTable[n][c] or {}
							if g ~= 'p' then
								if caseTable[n][c][g] then
									-- theoretically possible e.g. in Danish
									warnings[#warnings+1] = 'ERROR: VALUE GIVEN TWICE'
								end
								caseTable[n][c][g] = value
							else
								caseTable[n][c] = {value, value, value}
							end
							number = n
							gender = g
							case = c
						end
					end
				end
			end
			if (not number or not gender or not case) and key ~= '#default' then
				warnings[#warnings+1] = 'FIXME: unexpected key "' .. key .. '"'
			end
		end
		local numberStrings = {}
		local regular = declension.langlist[lang] and declension.langlist[lang].adjective
		if regular then -- still needed for pl, can be removed if not needed any more
			for number,_ in pairs(regular) do
				for case,v in pairs(regular[number]) do
					regular[number][type(case)=='string' and mw.ustring.lower(case) or case] = v
				end
			end
		end
		
		local out = usedGrammaticalFeatures[lang] and usedGrammaticalFeatures[lang].out
		usedGrammaticalFeatures.numbers = out and out.numbers or usedGrammaticalFeatures.numbers
		usedGrammaticalFeatures.genders = out and out.genders or usedGrammaticalFeatures.genders
		local isRegular = true
		for _,n in ipairs(usedGrammaticalFeatures.numbers) do
			if n == '' then n = 's' end
			local caseStrings = {}
			for _,c in ipairs(usedGrammaticalFeatures.cases) do
				local genderStrings = {}
				for _,g in ipairs(usedGrammaticalFeatures.genders) do
					local genderConvert = {['']=1, m=1, f=2, n=3, c=1}
					g = genderConvert[g]
					local form
					local caseTableTail
					if caseTable[n] then caseTableTail = caseTable[n] end
					if caseTableTail and caseTableTail[c or ''] then caseTableTail = caseTableTail[c or ''] end
					if caseTableTail and caseTableTail[g] then form = caseTableTail[g] end
					genderStrings[g] = form or caseData['#default'] or ''
					local numberTranslate = {s='singular',p='plural'}
					local regularForm = regular and declension.selectAdjectiveForm(regular, {number=numberTranslate[n], case=mw.ustring.lower((c=='' and 'n') or c), gender=g})
					if genderStrings[g] ~= regularForm then
						-- mw.log(before,n,c,g)
						isRegular = false
					end
				end
				local formsAreIdentical = true
				local sortedGenderNumbers = getSortedKeys(genderStrings)
				for i = 1, sortedGenderNumbers[#sortedGenderNumbers] do
					if genderStrings[i] and (genderStrings[i] ~= genderStrings[1]) then
						formsAreIdentical = false
					end
				end
				local allGenderStrings
				if formsAreIdentical then
					allGenderStrings = string.format("{'%s'}", genderStrings[1])
				else
					allGenderStrings = makeLuaTable(genderStrings)
				end
				local cl = mw.ustring.lower(c)
				caseStrings[#caseStrings+1] = ((cl~='' and (cl .. '=')) or '') .. allGenderStrings
			end
			numberStrings[#numberStrings+1] = ((n~='' and (n .. '=')) or '') .. '{' .. table.concat(caseStrings, ', ') .. '}'
		end
		local luaSwitchString
		if isRegular then
			luaSwitchString = 'regular()'
		else
			luaSwitchString = '{' .. table.concat(numberStrings, ', ') .. '}'
		end
		local parts = {before ~= '' and "'" .. before .. "'" or nil}
		parts[#parts+1] = luaSwitchString
		parts[#parts+1] = behind ~= '' and "'" .. behind .. "'" or nil
		luaAdjectiveData = mw.ustring.format("{parts = {%s}}", table.concat(parts, ', '))
	end
	return luaAdjectiveData, table.concat(warnings, ', ')
end

local function makeLuaLine(lineData, wordtype, lang)
	local value, warning
	if wordtype == 'nouns' then
		value, warning = makeLuaNounData(lineData.value, lineData.key, lang)
	elseif wordtype == 'adjectives' then
		value, warning = makeLuaAdjectiveData(lineData.value, lang)
		if not value then
			value = "'" .. lineData.value .. "'"
		end
	end
	local commentString = ''
	if lineData.comment then
		commentString = ' -- ' .. lineData.comment
	end
	local warningString = ''
	if warning ~= '' then
		warningString = '\n-- ' .. warning
	end
	if value then
		return fillToLength("['" .. lineData.key .. "']", 20)
			.. ' = ' .. value .. ',' .. commentString .. warningString
	end
end

local function findComments(inText)
	local comments = {}
	for comment in mw.ustring.gmatch(inText, '<!%-%-%s*(.-)%s*%-%->') do
		comments[#comments+1] = comment
	end
	local outText = table.concat(mw.text.split(inText,'<!%-%-.-%-%->'))
	return outText, comments
end

local function processSecondAdjectiveSwitch(inText, key)
	local area, rest = mw.ustring.match(inText, '^{{#switch: {{{1|}}}(.-)}}(.*)')
	if area then
		area = mw.ustring.match(area, '^%s*<!%-%-.-%-%->(.*)$') or area
		area = mw.ustring.match(area, '^%s*<!%-%-.-%-%->(.*)$') or area
		local mappings2 = analyzeParserFunctionSwitch(area)
		return (mappings2[key] or 'ERROR: TERM NOT FOUND IN SWITCH') .. rest
	end
end

local function processTermsArea(inText, wordtype, lang, suppressundifferentiated)
	inText = mw.ustring.match(inText, '{{#switch: {{{1|}}}.-|(.*)\n%s*|%s*#default%s*=') or ''
	inText = table.concat(mw.text.split(inText,'\n%s*<!%-%-.-%-%->'))
	local mappings = analyzeParserFunctionSwitch(inText)
	local standardKeys = {['']=1, adj=1, basic=1, over=1, on=1, mounted=1, order=1, case=1, ['#default']=1}
	for k,_ in pairs(standardKeys) do
		mappings[k] = nil
	end
	local outLines = {}
	local mappings2 = {}
	for key,value in pairs(mappings) do
		if wordtype == 'adjectives' then
			value = processSecondAdjectiveSwitch(value, key) or value
		end
		local canonicalTerm = synonyms[wordtype][key] or key
		local escapedTerm = escapeSingleQuote(canonicalTerm)
		if not mappings2[escapedTerm] then
			mappings2[escapedTerm] = value
		elseif mappings2[escapedTerm] and mappings2[escapedTerm] ~= value then
			outLines[#outLines+1] = '-- ERROR: MISMATCH: DATA FOR SYNONYM "' .. key
			.. '" AND NORMALIZED TERM "'
			.. escapedTerm .. '" DIFFERS:'
			.. '\n-- synonym:    ' .. value
			.. '\n-- normalized: ' .. mappings2[escapedTerm]
		end
	end
	local sortedTerms = getSortedKeys(mappings2)
	for _,key in ipairs(sortedTerms) do
		local value = mappings2[key]
		
		local comments, valueComments, bareKey, bareValue
		bareKey, comments = findComments(key)
		bareValue, valueComments = findComments(value)
		for _,v in ipairs(valueComments or {}) do
			comments[#comments+1] = v
		end
		local commentsString = table.concat(comments, '; ')
		commentsString = (commentsString ~= '') and commentsString
		
		local escapedValue = escapeSingleQuote(bareValue)
		local termData = {key=bareKey, value=escapedValue, comment=commentsString}
		outLines[#outLines+1] = makeLuaLine(termData, wordtype, lang)
		
		for k,v in pairs(mappings2) do
			local undifferentiatedConditions = (value == v)
				and (synonyms[wordtype][k] or k) ~= (synonyms[wordtype][key] or key)
				and not suppressundifferentiated
			if undifferentiatedConditions then
				outLines[#outLines+1] = mw.ustring.format(
					'-- ERROR: UNDIFFERENTIATED: for the terms "%s" and "%s" the same mapping is used:\n-- %s',
					key, k, v)
			end
		end
	end
	local outText = table.concat(outLines, '\n')	
	return outText
end

local function convert(args)
	local lang = args.lang
	local workingArea = mw.title.new('Technique/' .. lang, 'Template'):getContent()
	local nouns = processTermsArea(workingArea, 'nouns', lang, args.suppressundifferentiated)
	
	local workingArea = mw.title.new('Technique/' .. lang .. '/adjectives', 'Template'):getContent()
	local adjectives = processTermsArea(workingArea, 'adjectives', lang, args.suppressundifferentiated)
	local regularFunctionString = string.format([[
local function regular()
	return require('Module:Declension').langlist.%s.adjective
end]], lang)
	
	local res = syntaxHighlightLuaCode(string.format(
		'p.nouns = {\n%s\n}\n\n%s\n\np.adjectives = {\n%s\n}\n\nreturn p',
		nouns, regularFunctionString, adjectives))
	return res
end

function p.convert(frame)
	local args = frame.args
	local userLang = frame:callParserFunction('int', 'lang')
	local lang, suppressundifferentiated
--	local lang = 'pl'
--	local lang = 'de'
--	local suppressundifferentiated = true
	local lang = lang or args.lang or userLang
	local suppressundifferentiated = suppressundifferentiated or args.suppressundifferentiated
	local result = getlanguageSelector() .. convert({lang=lang, suppressundifferentiated=suppressundifferentiated})
	return result
end

-- ===========================================================================
-- === functions to be called from wikitext
-- === and small direct helper functions for them
-- ===========================================================================

local function cutTable(inTable, expressionNumber)
	local outTable = {}
	for i,v in ipairs(inTable) do
		if i <= 25 then
			table.insert(outTable,v)
		end
	end
	return outTable
end

local function deleteValue(inTable, value)
	local outTable = {}
	for _,k in ipairs(inTable) do
		if k ~= value then
			table.insert(outTable, k)
		end
	end
	return outTable
end

function p.getSynonyms(frame)
	local args = frame.args
	local userLang = frame:callParserFunction('int', 'lang')
	local lang
--	local lang = 'en'
	local lang = lang or args.lang or userLang
	local wordtype = args.wordtype or 'adjectives'
	local _,_,_,_,_, synTerms = analyzeLangSubtemplate(wordtype, lang, nil)
	return makeLuaSynonyms(synTerms)
end

function p.comparison(frame)
	local args = frame.args
	local userLangs = frame:callParserFunction('int', 'lang')
	userLangs = mw.text.split(userLangs, '%s*,%s*')
	local languages = {'en'}
--[[ for debugging
	local allLanguages = {'ar','ca','cs','da','de','el','en','es','et','fi','fr',
						'gl', 'he','hu','it','ja','mk','nds','nl','nb','pl','pt',
						'ro','ru','scn','sk','sl','sr','sv','sw','vec','vi','zh'}
	local lowercaseLanguages = {'ca','cs','da','el','en','es','et','fi','fr',
						'gl', 'hu','it','mk','nl','nb','pl','pt',
						'ro','ru','scn','sk','sl','sr','sv'}
	local languages = cutTable(allLanguages, 4)
	local languages = {'en', 'fr', 'pl', 'ru'}
	local languages = {'en', 'de', 'fr', 'es', 'it', 'pl', 'ru'}
	args.uselangfirst = 'yes'
	local languages = allLanguages
--]]
	languages = args.languages and mw.text.split(args.languages, '%s*,%s*') or languages
	-- put the user language in the first columns
	if args.uselangfirst == 'yes' then
		for i = #userLangs, 1, -1 do
			languages = deleteValue(languages, userLangs[i])
			table.insert(languages, 1, userLangs[i])
		end
	end
	
	local selectFunction
	local startTerm = args.startTerm
	local endTerm = args.endTerm
	selectFunction = function (term)
		local term = mw.ustring.lower(term)
		local startTerm = startTerm or 'a'
		local endTerm = endTerm or 'al'
		local result = (term >= startTerm and string.sub(term, 1, #endTerm) <= endTerm)
		return result
	end
	if not startTerm and not endTerm then
		selectFunction = function(term) return true end
	end
	local showQs
--	local showQs = true
	showQs = args.showQs or showQs
	
	return termsVisualizer(languages, true, selectFunction, showQs)
end

function p.translation_dashboard(frame)
	local args = frame.args
	local languages = subpageLangs()
--	local languages = {'ar','ca','cs','da','de','el','en','es','et','fi','fr','he','hu'}
--	local languages = cutTable(subpageLangs(), 25)
	if args.uselangfirst == 'yes' then
		local userLang = frame:callParserFunction('int', 'lang')
		languages = deleteValue(languages, userLang)
		table.insert(languages, 1, userLang)
	end
	return termsVisualizer(languages, false)
end

return p