Module:Project portal/sandbox

This is the module sandbox page for Module:Project portal (diff).
Module documentation
This Lua module is used in system messages.
Changes to it can cause immediate changes to the Wikimedia user interface.
To avoid major disruption, any changes should be tested in the module's /sandbox or /testcases subpages, or in your own module sandbox. The tested changes can be added to this page in a single edit. Please discuss changes on the talk page before implementing them.
This module provides a dashboard() function listing things that need attention in project portals such as www.wikibooks.org template, based on the daily statistics at Wiktionary (and similar tables). This module automatically generates the HTML source code to copy and paste into the live portals.
local p = {}
local cssmin = require("Module:Cssmin")
local yesno = require("Module:Yesno")
local timeAgo = require("Module:TimeAgo")._main
local projectStats = require("Module:Project statistics")

local dataTitle = "Module:Project portal/wikis"
local dataByWiki = require(dataTitle)
local viewsTitle = "Module:Project portal/views.json"
local viewsByProject = mw.text.jsonDecode( mw.getCurrentFrame():expandTemplate{ title = viewsTitle } )
viewsByProject.wikipedia = viewsByProject.wiki

local contentLang = mw.getContentLanguage()
local statsBotName = "EmausBot"

-- Returns the project of the current page.
local function currentProject()
	return mw.ustring.match(mw.title.getCurrentTitle().rootText,
		"^Www%.(.-)%.org template$") or "wikipedia"
end

-- Returns true if the given wiki is closed.
local function isClosed(wiki, project)
	project = project or "wikipedia"
	local projectData = dataByWiki[wiki] and dataByWiki[wiki][project]
	return projectData and projectData.closed
end

-- Get the top ten wikis:
-- convert the wikis to sorted pairs ordered by the number of articles
-- fetch the first ten that are not closed
local function getTop10Wikis( statsByWiki )
	local pairsTable = {}
	for k, v in pairs( statsByWiki ) do
		table.insert( pairsTable, {k, v} )
	end
	table.sort(
		pairsTable,
		function ( wikiA, wikiB )
			return wikiA[2].articles > wikiB[2].articles
		end
	)
	local results = {}
	for i, v in ipairs( pairsTable ) do
		if #results < 10 and not v[2].closed and v[1] ~= nil then
			table.insert( results, v[1] )
		end
	end
	return results
end

-- minify contents of style tags
local function minifyCSS(inputHtml)
	local result = mw.ustring.gsub(
		inputHtml,
		"(<style.->\n?)(.-)(\n?</style>)",
		function (openTag, css, closeTag)
			return openTag .. cssmin.cssmin(css) .. closeTag
		end
	)
	return result
end

-- Round down the number of articles to the nearest thousand, unless its
-- less than a thousand, it which case its rounded down to the nearest power
-- of ten, with no rounding for numbers less than 10
local function roundNumArticles( rawNumArticles )
	local roundedNum = math.floor(rawNumArticles / 1000) * 1000
	if roundedNum == 0 then
		roundedNum = math.floor(rawNumArticles / 100) * 100
	end
	if roundedNum == 0 then
		roundedNum = math.floor(rawNumArticles / 10) * 10
	end
	if roundedNum == 0 then
		roundedNum = rawNumArticles
	end
	return roundedNum
end

-- call `timeAgo` for the result of the
-- REVISIONTIMESTAMP parser function with a title
local function getTimeAgo( frame, page )
	local timestamp = frame:callParserFunction("REVISIONTIMESTAMP", page )
	return timeAgo{ timestamp }
end

-- Retrieves statistics about the project.
function p._getStatistics(page, project)
	local statistics = projectStats.getStatistics(project)
	local statsByWiki = statistics.wikis
	
	local isoDate = table.concat(statistics.date, "-")
	
	local bucketsByWiki = {}
	local numArticlesByWiki = {}
	for wiki, stats in pairs(statsByWiki) do
		bucketsByWiki[wiki] = math.floor(math.log10(stats.articles))
		numArticlesByWiki[wiki] = stats.articles
	end
	
	-- Sometimes the statistics pages don’t get updated right after a wiki is
	-- created. The workaround is to temporarily set [project].numArticles in
	-- /wikis to the number of articles.
	for wiki, data in pairs(dataByWiki) do
		if not bucketsByWiki[wiki] and data[project] and data[project].numArticles then
			bucketsByWiki[wiki] = math.floor(math.log10(data[project].numArticles))
			numArticlesByWiki[wiki] = data[project].numArticles
		end
	end
	
	local top10Wikis = getTop10Wikis( statsByWiki )
	
	return isoDate, bucketsByWiki, numArticlesByWiki, top10Wikis
end

-- Scrape interesting data from the given portal source and return it in several
-- tables.
function p._getPortal(page, project)
	local bucketsByWiki = {}
	local minBucket = math.huge
	local maxBucket
	for skipRepeats, operator, minCount, links
			in mw.ustring.gmatch(page,
				"<!%-%- (m?o?r?e?)wikis|[au]l?|([>≥]?)(%d+) %-%->(.-)<!%-%- /wikis %-%->") do
		local bucket = math.log10( tonumber(minCount, 10) )
		if bucket < minBucket then
			minBucket = bucket
		end
		
		if ( skipRepeats == "more" ) and ( operator == ">" or operator == "≥" ) then
			maxBucket = 0
		end
		
		for wiki in mw.ustring.gmatch(links, "<a href=[\"']//([%a-]+)%." .. project .. "%.org%W") do
			bucketsByWiki[wiki] = bucket
		end
	end
	
	local numArticlesByWiki = {}
	local top10Wikis = {}
	local top10Metric, linkBoxes = mw.ustring.match(page,
		"<!%-%- topn|%d+|(%w+) %-%->(.-)<!%-%- /topn %-%->")
	for wiki, box in mw.ustring.gmatch(linkBoxes or "",
		"<!%-%- #%d+%. ([%w-]+)%." .. project .. "%.org[^>]+%-%->%s*<div[^>]*>%s*(.-)%s*</div>") do
		table.insert(top10Wikis, wiki)
		
		local span = mw.ustring.match(box, "<small[^>]*>(.-)</small>")
		assert(span or metric ~= "articles", "No article count for " .. wiki)
		
		if span then
			local numArticles = mw.ustring.gsub(span,
				"[ " .. mw.ustring.char(0xa0) .. "+]", "")
			numArticles = tonumber(mw.ustring.match(numArticles, "%d+"))
			assert(numArticles, "Invalid article count for " .. wiki)
			
			numArticlesByWiki[wiki] = numArticles
		end
	end
	
	return bucketsByWiki, minBucket, maxBucket, numArticlesByWiki, top10Wikis, top10Metric
end

-- Returns the most frequently viewed wikis of the given project.
local function topViewedWikis(project, count)
	assert(viewsByProject[project], "No view statistics for " .. project)
	local views = {}
	for wiki, data in pairs(viewsByProject[project]) do
		if not isClosed(wiki, project) then
			table.insert(views, {wiki, data.views})
		end
	end
	assert(count <= #views, "Missing view statistics for " .. project ..
		" top " .. count .. " wikis")
	
	table.sort(
		views,
		function (a, b)
			return a[2] > b[2]
		end
	)
	
	local topWikis = {}
	for i = 1, count do
		table.insert(topWikis, views[i][1])
	end
	return topWikis
end

-- Source list with links and edit info for edit notice, and with links
-- for summary in both edit notice and preload
-- returns summarySourceList, sourceList, statsOldId
local function getSourcesLists(frame)
	local project = currentProject()
	local unusedVar, sources
	local statsTitle
	if project == "wikimedia" then
		sources = { mw.title.getCurrentTitle().rootText .. "/temp" }
	else
		unusedVar, sources = p._generatedPortal(frame)
		statsTitle = projectStats.statsTitleForProject(project)
	end

	local sourceList = {}
	local summarySourceList = {}
	local statsOldId
	for i, source in ipairs(sources) do
		if type(source) == "string" then
			source = mw.title.new(source)
		end
		local oldId = frame:callParserFunction("REVISIONID", source.fullText)
		if source == statsTitle then
			statsOldId = oldId
		end
		local user = frame:callParserFunction("REVISIONUSER", source.fullText)
		local userLink
		if user:match("^%d%d?%d?%.%d%d?%d?%.%d%d?%d?%.%d%d?%d?%$") then
			userLink = string.format("[[Special:Contributions/%s|<strong class='error'>%s</strong>]]",
				user, user)
		elseif source == statsTitle and user ~= statsBotName then
			userLink = string.format("[[User:%s|<strong class='error'>%s</strong>]]",
				user, user)
		else
			userLink = string.format("[[User:%s|%s]]", user, user)
		end
		table.insert(
			sourceList,
			mw.ustring.format(
				"* [[%s]], last modified [[Special:Diff/%i|%s]] by %s",
				source.fullText,
				oldId,
				getTimeAgo( frame, source.fullText ),
				userLink
			)
		)
		table.insert(
			summarySourceList,
			mw.ustring.format(
				"%s%s (&#x5b;[Special:Diff/%i|%i]])",
				(source.isSubpage and "/") or "",
				source.subpageText,
				oldId,
				oldId
			)
		)
		
		if source == statsTitle and user ~= statsBotName then
			table.insert(
				sourceList,
				mw.ustring.format(
					"** This page is normally only edited by [[User:%s|%s]]. " ..
						"Please ensure that the statistics have not been updated piecemeal.",
					statsBotName,
					statsBotName
				)
			)
		end
	end
	
	return summarySourceList, sourceList, statsOldId
end

local function makeSummaryFromSourcesList(sourcesList)
	-- Shared between edit notice and dashboard
	local summary = mw.text.listToText( sourcesList, ", ", ", " )
	return 'Updated using &#x5b;[/auto]] based on ' .. summary
end

function p.checkEdit(frame)
	local countTotalIssues, display = p._getStatus(frame)
	
	if countTotalIssues > 0 then
		return "Needs update"
	else
		return "Up to date"
	end
end

function p.dashboard(frame)
	local countTotalIssues, display = p._getStatus(frame)
	return display
end

function p._getStatus(frame)
	local project = frame.args.project or currentProject()
	local statsTitle = projectStats.statsTitleForProject(project)
	local statsPage = statsTitle:getContent()
	if not statsPage then
		if project == 'wikimedia' then
			return 0, 'There is no automated detection for the wikimedia.org template'
		else
			error("[[" .. statsTitle.fullText .. "]] not found.")
		end
	end

	local date, botBucketsByWiki, botNumArticlesByWiki, botTop10Wikis =
		p._getStatistics(statsPage, project)
	botTop10Wikis = table.concat(botTop10Wikis, ", ")
	local botAge = getTimeAgo( frame, statsTitle.fullText )
	local viewsAge = getTimeAgo( frame, viewsTitle )
	
	local portalTitle = mw.title.new("Www." .. project .. ".org template", 0)
	local portalPage = portalTitle:getContent()
	if not portalPage then
		error("[[" .. portalTitle.fullText .. "]] not found.")
	end
	
	local bucketsByWiki, minBucket, maxBucket, numArticlesByWiki, top10Wikis, liveTop10Metric =
		p._getPortal(portalPage, project)
	top10Wikis = table.concat(top10Wikis, ", ")
	
	local additions = {}
	local changes = {}
	local reminders = {}
	
	local templateTitle = portalTitle:subPageTitle("temp")
	local templateSource = templateTitle:getContent()
	if not templateSource then
		error("[[" .. templateTitle.fullText .. "]] not found.")
	end
	local top10Metric = mw.ustring.match(templateSource,
		"{{%s*topn%s*|%s*%d+%s*|%s*(%w+)%s*}}")
	if liveTop10Metric ~= top10Metric then
		table.insert(changes, mw.ustring.format("* Rearrange top 10 by %s rather than %s.",
			top10Metric, liveTop10Metric))
	end
	
	mw.logObject(bucketsByWiki)
	for wiki, botBucket in pairs(botBucketsByWiki) do
		local data = dataByWiki[wiki]
		local codeLink = mw.ustring.format("<code>[[%s:Special:Statistics|%s:]]</code>", wiki, wiki)
		
		local bucket = bucketsByWiki[wiki]
		if bucket then
			local issue
			if botBucket > bucket and (not maxBucket or bucket ~= maxBucket) then
				issue = mw.ustring.format("* Promote %s from %s+ up to %s+.",
					codeLink, contentLang:formatNum(10 ^ bucket), contentLang:formatNum(10 ^ botBucket))
			elseif botBucket < bucket then
				if botBucket < minBucket then
					issue = mw.ustring.format("* Remove %s from %s+.",
						codeLink, contentLang:formatNum(10 ^ bucket))
				else
					issue = mw.ustring.format("* Demote %s from %s+ down to %s+.",
						codeLink, contentLang:formatNum(10 ^ bucket), contentLang:formatNum(10 ^ botBucket))
				end
			end
			
			if issue then
				table.insert(changes, issue)
			end
			
			if data and data[project] and data[project].numArticles then
				issue = mw.ustring.format("* ''Check if %s is still %s+.''",
					codeLink, contentLang:formatNum(10 ^ bucket))
				table.insert(reminders, issue)
			end
		elseif botBucket >= minBucket and not isClosed(wiki, project) then
			local issue = mw.ustring.format("* Add %s to %s+.",
				codeLink, contentLang:formatNum(10 ^ math.min(botBucket, maxBucket or botBucket)))
			if not dataByWiki[wiki] then
				if mw.language.isKnownLanguageTag(wiki) then
					-- Format the language name
					local newLang = mw.getLanguage(wiki)
					local name = mw.language.fetchLanguageName(wiki)
					name = mw.text.split(name, " ", true)
					for i, word in ipairs(name) do
						name[i] = newLang:ucfirst(word)
					end
					name = table.concat(name, " ")
					if newLang:isRTL() then
						name = mw.ustring.format("<bdi dir=\"rtl\">%s</bdi>", name)
					end
					
					local propsLua = mw.ustring.format( "name = \"%s\",", name )
					local newLua = frame:extensionTag{
						name = "syntaxhighlight",
						args = {
							lang = "lua",
						},
						content = mw.ustring.format("\t%s = {\n\t\t%s\n\t},", wiki, propsLua),
					}
					
					issue = mw.ustring.format("%s Add this entry to [[%s/wikis]]:%s",
						issue, frame:getTitle(), newLua)
				else
					issue = mw.ustring.format("%s Add the corresponding entry to [[%s/wikis]]. " ..
						"See [[w:ISO 639:%s]] for the language name.",
						issue, frame:getTitle(), wiki)
				end
			end
			table.insert(additions, issue)
		end
	end
	table.sort(additions)
	
	local correctTop10Wikis
	if top10Metric == "articles" then
		correctTop10Wikis = botTop10Wikis
	elseif top10Metric == "views" then
		correctTop10Wikis = table.concat(topViewedWikis(project, 10), ", ")
	else
		error("Unrecognized wiki ranking metric “" .. top10Metric .. "”")
	end
	if top10Wikis ~= correctTop10Wikis then
		local issue = mw.ustring.format(
			"* Update top 10 ring:%s",
			frame:expandTemplate{
				title = "TextDiff",
				args = {
					top10Wikis,
					correctTop10Wikis,
				},
			}
		)
		table.insert(changes, issue)
	end
	table.sort(reminders)
	
	local minorChanges = {}
	for wiki, numArticles in pairs(numArticlesByWiki) do
		if botNumArticlesByWiki[wiki] then
			local botNumArticles = roundNumArticles(botNumArticlesByWiki[wiki])
			if numArticles ~= botNumArticles then
				local issue = mw.ustring.format(
					"* ''Change <code>%s:</code> article count from %s+ to %s+ in top 10 ring.''",
					wiki,
					contentLang:formatNum(numArticles),
					contentLang:formatNum(botNumArticles)
				)
				local firstDigit = math.floor(numArticles / 10 ^ math.floor(math.log10(numArticles)))
				local botFirstDigit = math.floor(botNumArticles / 10 ^ math.floor(math.log10(botNumArticles)))
				if math.abs(firstDigit - botFirstDigit) >= 1 then
					table.insert(changes, (mw.ustring.gsub(issue, "''", "")))
				else
					table.insert(minorChanges, issue)
				end
			end
		end
	end
	table.sort(changes)
	table.sort(minorChanges)
	
	local sourcesList, unusedOne, unusedTwo = getSourcesLists( frame )
	local summaryPreload = makeSummaryFromSourcesList( sourcesList )
	
	-- Replace the rendering of an opening brace with an actual opening brace
	summaryPreload = string.gsub( summaryPreload, '&#x5b;', '[')
	
	local remindersStr = mw.ustring.format([=[

; Reminders
%s
This report was automatically generated based on article counts in [[%s]] (last updated %s) and page view statistics in [[%s]] (last updated %s).

'''Administrators:''' See [%s instructions for updating the portal].]=],
		table.concat(reminders, "\n"),
		statsTitle.fullText,
		botAge,
		viewsTitle,
		viewsAge,
		portalTitle:fullUrl{action = "edit", summary = summaryPreload}
	)
	local result = ''
	local countTotalIssues = #additions + #changes + #minorChanges
	
	if countTotalIssues > 0 then
		result = mw.ustring.format([=[
[[Module:Project portal]] identified the following issues with the [[%s]]:
; Additions
%s
; Changes
%s
%s
%s]=],
			portalTitle.fullText,
			table.concat(additions, "\n"),
			table.concat(changes, "\n"),
			table.concat(minorChanges, "\n"),
			frame:expandTemplate{
				title = "Edit Protected",
				args = {
					auto = "yes",
				},
			}
		)
	else
		result = mw.ustring.format([=[
[[Module:Project portal]] identified no issues with the [[%s]].
]=],
			portalTitle.fullText
		)
	end
	result = result .. remindersStr
	return countTotalIssues, result
end

function p.findMissingLanguages(frame)
	local sourceTitle = mw.title.new(frame.args[1], 0)
	local sourcePage = sourceTitle:getContent()
	if not sourcePage then
		error("[[" .. sourceTitle.fullText .. "]] not found.")
	end
	
	local sourceWikis = {}
	for wiki in mw.text.gsplit(sourcePage, "\n", true) do
		sourceWikis[wiki] = true
	end
	
	local statsTitle = mw.title.new(frame.args["stats page"] or "List of Wikipedias/Table", 0)
	local statsPage = statsTitle:getContent()
	if not statsPage then
		error("[[" + statsTitle.fullText + "]] not found.")
	end
	
	local date, botBucketsByWiki, _, _ = p._getStatistics(statsPage)
	
	local changes = {}
	for wiki, _ in pairs(botBucketsByWiki) do
		if not sourceWikis[wiki] and not isClosed(wiki) then
			local name = mw.language.fetchLanguageName(wiki)
			if name then
				issue = mw.ustring.format(
					"* Add <code>%s</code> (%s).",
					wiki,
					name
				)
			else
				issue = mw.ustring.format(
					"* Add <code>%s</code>. See [[w:ISO 639:%s]] for the language name.",
					wiki,
					wiki
				)
			end
			table.insert(changes, issue)
		end
	end
	
	for wiki, _ in pairs(sourceWikis) do
		if not (botBucketsByWiki[wiki] or isClosed(wiki) or
				#mw.title.new(wiki .. ":", 0).interwiki > 0) then
			table.insert(changes, mw.ustring.format("* Remove <code>%s</code>.", wiki))
		end
	end
	
	local issues = mw.ustring.format([=[
[[Module:Project portal]] identified the following issues in [[%s]]:
%s
This report was automatically generated based on article count data in [[%s]], which was last updated on %s.
Note that this report only identifies issues with missing or unrecognized language codes, not sorting issues.
'''Administrators:''' Review all changes before deploying them.]=],
		sourceTitle.fullText,
		table.concat(changes, "\n"),
		statsTitle.fullText,
		date
	)
	
	if #changes > 0 then
		issues = mw.ustring.format("%s\n%s", frame:expandTemplate{
			title = "Edit Protected",
			args = {
				auto = "yes",
			},
		}, issues)
	end
	return issues
end

-- Generates a translated version of a portal like [[List of Wikipedias/Table]].
function p.translatePortal(frame)
	local title = mw.title.getCurrentTitle()
	local srcTitle = mw.title.new(title.baseText, 0)
	local src = srcTitle:getContent()
	if not src then
		error("[[" .. srcTitle.fullText .. "]] not found.")
	end
	
	-- Translate table headers
	src = mw.ustring.gsub(
		src,
		"\n! ([^\n]+)",
		function (header)
			return mw.ustring.format("\n! %s", frame.args[header] or header)
		end
	)
	
	-- Translate language names
	local langCode = title.subpageText
	local lang = mw.getLanguage(langCode)
	src = mw.ustring.gsub(
		src,
		"| %[%[(%a+):(.-) language|%2%]%]\n| (.-)\n| %[%[.-:(.-):|%4%]%]",
		function (projCode, rowEnglishName, nativeArticleRow, rowLangCode)
			local nativeName = mw.language.fetchLanguageName(rowLangCode)
			local localName = mw.language.fetchLanguageName(rowLangCode, langCode)
			if localName == nativeName and rowLangCode ~= langCode then
				localName = rowEnglishName
			end
			local articleLangCode = langCode
			local articleFormat = frame.args._articleFmt or "%s language"
			if localName == rowEnglishName then
				articleLangCode = "en"
				articleFormat = "%s language"
			end
			local article = mw.ustring.format(articleFormat, localName)
			return mw.ustring.format(
				"| [[%s:%s:%s|%s]]\n| %s\n| [[%s:%s:|%s]]",
				projCode,
				articleLangCode,
				article,
				localName,
				nativeArticleRow,
				projCode,
				rowLangCode,
				rowLangCode
			)
		end
	)
	
	-- Localize numbers
	src = mw.ustring.gsub(
		src,
		"(%d[%d,]*)",
		function (num)
			local rawNum = contentLang:parseFormattedNumber(num)
			if not rawNum or (rawNum == 0 and num ~= "0") then
				return num
			end
			return lang:formatNum(rawNum)
		end
	)
	src = mw.ustring.gsub(
		src,
		"(1[0 ]*)(%+ articles)",
		function (num, suffix)
			local formattedNum = lang:formatNum(tonumber((num:gsub(" ", ""))))
			return mw.ustring.format(
				frame.args._headingFmt or "%s" .. suffix,
				formattedNum
			)
		end
	)
	
	return frame:preprocess(src)
end

function p.minifiedPortal(frame)
	local project = currentProject()
	local tempPortalTitle = mw.title.new("Www." .. project .. ".org template/temp", 0)
	local html = tempPortalTitle:getContent()
	if not html then
		error("[[" .. tempPortalTitle.fullText .. "]] not found.")
	end
	
	html = minifyCSS( html )
	if yesno(frame.args.pretty) then
		html = frame:callParserFunction{
			name = "#tag",
			args = {
				"syntaxhighlight",
				lang = "html5",
				html,
			},
		}
	end
	return html
end

local defaultWiktionaryLogo = "tiles"

-- Generates a full portal based on current statistics and language data.
function p.generatedPortal(frame)
	local html, sources = p._generatedPortal(frame)
	return html
end
function p._generatedPortal(frame)
	local project = frame.args.project or currentProject()
	local statsTitle = projectStats.statsTitleForProject(project)
	local statsPage = statsTitle:getContent()
	if not statsPage then
		error("[[" .. statsTitle.fullText .. "]] not found.")
	end
	
	local date, botBucketsByWiki, botNumArticlesByWiki, botTop10Wikis = p._getStatistics(statsPage, project)
	
	local htmlTitle = mw.title.new(frame.args.template or "Www." .. project .. ".org template/temp", 0)
	local html = htmlTitle:getContent()
	if not html then
		error("[[" .. htmlTitle.fullText .. "]] not found.")
	end
	
	local thisModuleTitle = mw.title.new( "Project portal", 828 )
	local sources = {htmlTitle, dataTitle, statsTitle, thisModuleTitle}
	
	-- Remove pretty printing.
	html = mw.ustring.gsub(html, "^<syntaxhighlight.->\n?", "", 1)
	html = mw.ustring.gsub(html, "</syntaxhighlight>\n?$", "", 1)
	
	-- Returns HTML for the given attribute table.
	local function htmlFromAttr(attrs)
		local html = {}
		for k, v in pairs(attrs) do
			table.insert(html, mw.ustring.format(" %s=\"%s\"", k, v))
		end
		return table.concat(html, "")
	end
	
	local wikisByOccurrences = {}
	
	-- Substitute top n wikis.
	html = mw.ustring.gsub(html, "{{%s*topn%s*|%s*(%d+)%s*|%s*(%w+)%s*}}", function (count, metric)
		count = tonumber(count)
		
		-- Figure out which wikis to list.
		local topWikis
		if metric == "articles" then
			topWikis = botTop10Wikis
		elseif metric == "views" then
			table.insert(sources, viewsTitle)
			topWikis = topViewedWikis(project, count)
		else
			error("Unrecognized wiki ranking metric “" .. metric .. "”")
		end
		
		-- Create a link box for each wiki.
		local linkBoxes = {}
		for i, wiki in ipairs(topWikis) do
			wikisByOccurrences[wiki] = (wikisByOccurrences[wiki] or 0) + 1
			local data = dataByWiki[wiki]
			local lang = data.lang or wiki
			local name = data.topName or data.name
			local digraphic = mw.ustring.find(data.name, " / ", 1, true)
			local class = string.format("central-featured-lang lang%i", i)
			if digraphic then
				class = class .. " digraphic"
			end
			
			local attrs = {}
			if project == "wiktionary" then
				attrs["data-logo"] = (data.wiktionary and data.wiktionary.logo) or defaultWiktionaryLogo
			end
			-- Replace <bdi> around name with dir= on entire link box, but not
			-- for digraphic languages.
			name = mw.ustring.gsub(
				name,
				"^<bdi dir=\"(%w+)\">([^<]+)</bdi>$",
				function (dir, rawName)
					attrs.dir = dir
					return rawName
				end
			)
			
			local sort = ""
			if metric == "views" then
				sort = mw.ustring.format(
					" – %s views/day",
					contentLang:formatNum(viewsByProject[project][wiki].views)
				)
			end
			local latin = data.latin
			local siteName = (data[project] and data[project].siteName)
				or contentLang:ucfirst(project)
			local slogan = data[project] and data[project].slogan
			local createPage = data[project] and data[project].createPage
			assert(slogan or createPage, "No slogan or “create an article” page for " .. wiki .. " " .. project)
			if createPage then
				local create = data[project] and data[project].create
				if not create then
					create = mw.ustring.gsub(createPage, "^.-:", "", 1)
				end
				local createAttrs = (data[project] and data[project].createAttrs) or {}
				slogan = mw.ustring.format(
					"<a href=\"//%s.%s.org/wiki/%s\"%s>%s</a>",
					wiki,
					project,
					mw.ustring.gsub(createPage, " ", "_"),
					htmlFromAttr(createAttrs),
					create
				)
			end
			local sloganAttrs = (data[project] and data[project].sloganAttrs) or {}
			
			assert(botNumArticlesByWiki[wiki], "Article count unavailable for " .. wiki)
			local botNumArticles = roundNumArticles(botNumArticlesByWiki[wiki])
			local numArticles = contentLang:formatNum(botNumArticles):gsub(",", " ") .. "+"
			if attrs.dir then
				numArticles = mw.ustring.format("<bdi dir=\"ltr\">%s</bdi>", numArticles)
			end
			
			local articles = ""
			if project ~= "wikivoyage" then
				articles = data[project] and data[project].articles
				assert(articles, "No word for " .. project .. " articles in " .. wiki)
				local articlesAttrs = (data[project] and data[project].articlesAttrs)
				if articlesAttrs then
					articles = mw.ustring.format(
						"<span%s>%s</span>",
						htmlFromAttr(articlesAttrs),
						articles
					)
				end
				articles = mw.ustring.format("<br>\n<small>%s %s</small>", numArticles, articles)
			end
			
			local linkBoxAttrs = (data[project] and data[project].linkBoxAttrs) or {}
			if not createPage then
				 linkBoxAttrs.class = "link-box"
			end
			if project == "wikipedia" then
				assert(siteName, "No site name for " .. wiki .. " " .. project)
				assert(slogan, "No slogan for " .. wiki .. " " .. project)
				linkBoxAttrs.title = mw.ustring.format(
					"%s — %s — %s",
					latin or data.name,
					siteName,
					slogan
				)
			elseif latin then
				linkBoxAttrs.title = latin
			end
			
			local earlyLinkClose = ""
			local lateLinkClose = ""
			if createPage then
				earlyLinkClose = "</a>"
			else
				lateLinkClose = "</a>"
			end
			
			local linkBox = mw.ustring.format(
				"<!-- #%i. %s.%s.org%s -->\n" ..
				"<div class=\"%s\" lang=\"%s\"%s>\n" ..
				"<a href=\"//%s.%s.org/\"%s><strong>%s</strong>%s<br>\n" ..
				"<em%s>%s</em>" ..
				"%s%s\n" ..
				"</div>",
				i, wiki, project, sort,
				class, lang, htmlFromAttr(attrs),
				wiki, project, htmlFromAttr(linkBoxAttrs), name, earlyLinkClose,
				htmlFromAttr(sloganAttrs), slogan,
				articles, lateLinkClose)
			table.insert(linkBoxes, linkBox)
		end
		return mw.ustring.format(
			"<!-- topn|%i|%s -->\n%s\n<!-- /topn -->",
			count,
			metric,
			table.concat(linkBoxes, "\n\n")
		)
	end)
	
	-- Wikis written in different orthographies of the same language.
	-- This table assumes that the codes are listed in order according to the
	-- sort keys on the data page.
	local macroLangs = {
		be = {"be", "be-tarask"},
		pa = {"pa", "pnb"},
		no = {"no", "nn"},
	}
	
	-- Returns HTML for a link to the given wiki.
	local function linkForWiki(wiki, data)
		local attrs = data.attrs or {}
		local name = data.name or mw.language.fetchLanguageName(wiki) or wiki
		if data.latin then
			attrs.title = data.latin
		end
		return mw.ustring.format(
			"<a href=\"//%s.%s.org/\" lang=\"%s\"%s>%s</a>",
			wiki,
			project,
			data.lang or wiki,
			htmlFromAttr(attrs),
			name
		)
	end
	
	-- Substitute link lists.
	html = mw.ustring.gsub(html, "{{%s*(m?o?r?e?)wikis%s*|%s*(%w+)%s*|%s*([>≥]?)(%d+)%s*}}", function (skipRepeats, tag, operator, minCount)
		if skipRepeats ~= "" and skipRepeats ~= "more" then
			return
		end
		skipRepeats = skipRepeats == "more"
		
		local greater = operator == ">" or operator == "≥"
		local targetBucket = math.floor(math.log10(tonumber(minCount)))
		local wikis = {}
		for wiki, bucket in pairs(botBucketsByWiki) do
			if not isClosed(wiki, project)
					and (bucket == targetBucket or (greater and bucket > targetBucket))
					and not (skipRepeats and wikisByOccurrences[wiki]) then
				table.insert(wikis, wiki)
				if tag ~= "option" then
					wikisByOccurrences[wiki] = (wikisByOccurrences[wiki] or 0) + 1
				end
			end
		end
		
		local function compWikis(a, b)
			-- Handle missing data
			local aSort = ''
			local bSort = ''
			if dataByWiki[a] then
				aSort = (dataByWiki[a].sort or dataByWiki[a].latin or dataByWiki[a].name)
			end
			if dataByWiki[b] then
				bSort = (dataByWiki[b].sort or dataByWiki[b].latin or dataByWiki[b].name)
			end
			return aSort:lower() < bSort:lower()
		end
		table.sort(wikis, compWikis)
		
		-- Coalesce consecutive wikis written in the same language and writing
		-- system (only with different orthographies).
		if tag ~= "option" then
			wikis = table.concat(wikis, ",")
			for wiki, subWikis in pairs(macroLangs) do
				table.sort(subWikis, compWikis)
				
				-- If only one wiki belonging to the macrolanguage is present,
				-- treat that wiki as a representative of the whole
				-- macrolanguage.
				local presentSubWikis = 0
				for i, subWiki in ipairs(subWikis) do
					if botNumArticlesByWiki[subWiki] then
						presentSubWikis = presentSubWikis + 1
					end
				end
				
				-- If the wikis don’t all share the same macrolanguage name, for
				-- example due to digraphia, treat them as independent wikis.
				local macroName
				for i, subWiki in ipairs(subWikis) do
					local name = dataByWiki[subWiki].name:match("^(.-) -%(")
					if not macroName then
						macroName = name
					elseif name ~= macroName then
						macroName = nil
						break
					end
				end
				
				local pattern = table.concat(subWikis, ","):gsub("-", "%-")
				if presentSubWikis > 1 and macroName then
					-- Replace consecutive members of the macrolanguage with the
					-- macrolanguage itself, marked as such.
					wikis = wikis:gsub(pattern, "*" .. wiki, 1)
				else
					-- Mark each subwiki with the name of the digraphic
					-- macrolanguage.
					for i, subWiki in ipairs(subWikis) do
						wikis = wikis:gsub("^" .. subWiki:gsub("-", "%-") .. ",",
							wiki .. ":" .. subWiki .. ",", 1)
						wikis = wikis:gsub("," .. subWiki:gsub("-", "%-") .. ",",
							"," .. wiki .. ":" .. subWiki .. ",", 1)
					end
				end
			end
			wikis = mw.text.split(wikis, ",", true)
		end
		
		local links = {}
		if tag == "option" then
			for i, wiki in ipairs(wikis) do
				local data = dataByWiki[wiki]
				local attrs = ""
				if project == "wiktionary" then
					attrs = mw.ustring.format("%s data-logo=\"%s\"", attrs,
						(data.wiktionary and data.wiktionary.logo) or defaultWiktionaryLogo)
				end
				-- TODO: Select the largest language by default. This is
				-- overridden with the browser language anyways.
				if wiki == "en" then
					attrs = attrs .. " selected"
				end
				local comment = ""
				if data.latin then
					comment = mw.ustring.format("<!-- %s -->", data.latin)
				end
				local name = data.name
				if name then
					name = mw.ustring.gsub(name, "</?%w.->", "")
				else
					name = mw.language.fetchLanguageName(wiki) or wiki
				end
				local link = mw.ustring.format(
					"<option value=\"%s\" lang=\"%s\"%s>%s</option>%s",
					wiki,
					data.lang or wiki,
					attrs,
					name,
					comment
				)
				table.insert(links, link)
			end
		elseif tag == "a" or tag == "ul" then
			for i, wiki in ipairs(wikis) do
				-- All this just for a couple languages that have different
				-- orthographies spread out over multiple wikis.
				local isMacro = wiki:sub(1, 1) == "*"
				if isMacro then
					wiki = wiki:sub(2)
					local subWikis = macroLangs[wiki]
					local macroName = dataByWiki[subWikis[1]].name:gsub("%s*%b()", "")
					local macroLatin = dataByWiki[subWikis[1]].latin
					if macroLatin then
						macroLatin = macroLatin:gsub("%s*%b()", "")
						macroName = mw.text.tag("span", { title = macroLatin }, macroName)
					end
					local subLinks = {}
					for i, subWiki in ipairs(subWikis) do
						local data = mw.clone(dataByWiki[subWiki])
						data.name = mw.ustring.match(data.name, "%((.+)%)$")
						if data.latin then
							data.latin = mw.ustring.match(data.latin, "%((.+)%)$")
						end
						local link = linkForWiki(subWiki, data)
						if tag == "ul" then
							link = mw.text.tag("li", {}, link)
						end
						table.insert(subLinks, link)
					end
					if tag == "ul" then
						subLinks = mw.text.tag(tag, { lang = wiki },
							table.concat(subLinks, "\n"))
						local list = mw.text.tag("li", { lang = wiki }, macroName)
						table.insert(links, list .. subLinks)
					else
						subLinks = table.concat(subLinks, "&nbsp;• ")
						local list = mw.text.tag("span", { lang = wiki },
							mw.ustring.format("%s (%s)", macroName, subLinks))
						table.insert(links, list)
					end
				else
					local macro
					if wiki:find(":", 1, true) then
						local codes = mw.text.split(wiki, ":", true)
						macro, wiki = codes[1], codes[2]
					end
					local data = mw.clone(dataByWiki[wiki] or {})
					-- If a member of a digraphic macrolanguage, delete the
					-- qualifier.
					if macro then
						if data.lang then
							data.lang = macro
						end
						data.name = data.name:gsub("%s*%b()", "")
						if data.latin then
							data.latin = data.latin:gsub("%s*%b()", "")
						end
					end
					local link = linkForWiki(wiki, data)
					if tag == "ul" then
						link = mw.text.tag("li", {}, link)
					end
					table.insert(links, link)
				end
			end
		else
			error("Invalid wiki listing tag “" .. tag .. "”")
		end
		if tag == "option" or tag == "ul" then
			links = table.concat(links, "\n")
		else
			links = table.concat(links, "&nbsp;•\n")
		end
		if tag == "ul" then
			links = mw.text.tag(tag, {}, links)
		end
		
		local commentTag = (skipRepeats and "more") or ""
		return mw.ustring.format(
			"<!-- %swikis|%s|%s%i -->\n%s\n<!-- /%swikis -->",
			commentTag,
			tag,
			operator,
			minCount,
			links,
			commentTag
		)
	end)
	
	html = mw.ustring.gsub(html, "{{%s*colophon%s*}}", function ()
		local links = {}
		for i, source in ipairs(sources) do
			if type(source) == "string" then
				source = mw.title.new(source)
			end
			local oldId = frame:callParserFunction("REVISIONID", source.fullText)
			table.insert(
				links,
				mw.ustring.format(
					"<%s>",
					source:fullUrl({ oldid = oldId }, "canonical")
				)
			)
		end
		return mw.ustring.format(
			"Generated by <https://meta.wikimedia.org/wiki/Module:Project_portal> based on %s.",
			table.concat(links, ", ")
		)
	end)
	
	-- Minify CSS blocks.
	html = minifyCSS( html )
	
	return html, sources
end

-- Returns instructions for updating the portal along with generated source if
-- applicable.
function p.instructions(frame)
	-- Pretty print the full generated code.
	local project = currentProject()
	local html, sources
	local statsTitle
	if project == "wikimedia" then
		html = p.minifiedPortal(frame)
		sources = { mw.title.getCurrentTitle().rootText .. "/temp" }
	else
		html, sources = p._generatedPortal(frame)
		statsTitle = projectStats.statsTitleForProject(project)
	end
	html = frame:callParserFunction{
		name = "#tag",
		args = {
			"syntaxhighlight",
			lang = "html5",
			html,
		},
	}
	
	local summarySourceList, sourceList, statsOldId = getSourcesLists( frame )

	assert(statsOldId or project == "wikimedia", "No revision ID for stats page")
	
	local functionName = project == "wikimedia" and "minifiedPortal" or "generatedPortal"
	local instructions = mw.ustring.format(
		"Up-to-date HTML5 source code has been generated for [[www.%s.org template]]. Just replace the entire contents of the live portal’s edit field with this <code>subst:</code> call:\n\n" ..
		" &#x7b;{subst:#invoke:Project portal|%s}}\n\n" ..
		"Use this edit summary:\n\n" ..
		" %s\n\n" ..
		"'''Important:''' Review all changes before deploying them. The generated code draws from the following unprotected pages:\n\n%s" ..
		"",
		project,
		functionName,
		makeSummaryFromSourcesList( summarySourceList ),
		table.concat(sourceList, "\n")
	)
	
	local output = html
	if mw.ustring.match(mw.title.getCurrentTitle().fullText, "^Www%..-%.org template$") then
		output = mw.ustring.format(
			"Before saving, press '''%s''' or '''%s''' to give yourself a chance to catch any mistakes.",
			frame:expandTemplate{
				title = "Button",
				args = { mw.message.new("showpreview"):plain() },
			},
			frame:expandTemplate{
				title = "Button",
				args = { mw.message.new("showdiff"):plain() },
			},
			project
		)
	end
	
	return instructions .. "\n\n" .. output
end

return p