Module:UtilsSchema

local p = {} local h = {}

local utilsError = require("Module:UtilsError") local utilsString = require("Module:UtilsString") local utilsTable = require("Module:UtilsTable") local utilsValidate = require("Module:UtilsValidate")

local SYMBOLS = { optional = "[%s]", required = "%s!", default = "%s=%s", array = "{%s}", map = "<%s, %s>", oneOf = "%s|%s", allOf = "%s&%s", combinationGroup = "(%s)", }

p.TYPES = { oneOf = "oneOf", array = "array", record = "record", map = "map" }

local function code(s) return string.format(" ", s) end

function p.validate(schema, schemaName, data, dataName) -- first validate the schema itself h.collectReferences(p.Schemas["Schema"]) local schemaErrors = h.validate(p.Schemas["Schema"], schema, schemaName) if #schemaErrors > 0 then utilsError.warn(string.format("Schema  is invalid.", schemaName)) end -- then validate the data h.collectReferences(schema) local errors = h.validate(schema, data, dataName) if #errors > 0 then utilsError.warn(string.format(" is invalid according to schema  ", dataName, schemaName)) return errors end end

function p.getTypeDefinitions(schema, schemaName, formattingFn) h.collectReferences(schema) h.minRefDepths(schema) return h.getTypeDefs(schemaName, schema, formattingFn) end

function h.collectReferences(schema) h.references = {} h.references["#"] = schema for k, v in pairs(schema.definitions or {}) do h.references["#/definitions/" .. k] = v		h.collectIdReferences(v) end h.collectIdReferences(schema) end function h.collectIdReferences(schema) h.walkSchema(function(schemaNode)		if schemaNode._id then			h.references[schemaNode._id] = schemaNode		end	end, schema) end

function h.resolveReference(schemaNode) if schemaNode._ref then local referenceNode = h.references[schemaNode._ref] if not referenceNode then mw.addWarning(string.format("%s not found", code(mw.text.nowiki(schemaNode._ref)))) else local resolvedSchema = utilsTable.merge({}, h.references[schemaNode._ref], schemaNode) schemaNode = utilsTable.merge({}, schemaNode, resolvedSchema) schemaNode._ref = nil end end return schemaNode end

function h.walkSchema(fn, schemaNode, path) path = path or {} local continue = fn(schemaNode, path) if continue == false then return end if schemaNode.items then h.walkSchema(fn, schemaNode.items, utilsTable.concat(path, "items")) end if schemaNode.keys then h.walkSchema(fn, schemaNode.keys, utilsTable.concat(path, "keys")) end if schemaNode.values then h.walkSchema(fn, schemaNode.values, utilsTable.concat(path, "values")) end if schemaNode.properties then for i, v in ipairs(schemaNode.properties) do			local keyPath = utilsTable.concat(path, "properties", v.name) h.walkSchema(fn, v, keyPath) end end if schemaNode.oneOf then for k, v in pairs(schemaNode.oneOf) do			h.walkSchema(fn, v, utilsTable.concat(path, "oneOf")) end end if schemaNode.allOf then for i, v in ipairs(schemaNode.allOf) do			h.walkSchema(fn, v, utilsTable.concat(path, "allOf")) end end if schemaNode.definitions then for k, v in pairs(schemaNode.definitions) do			h.walkSchema(fn, v, utilsTable.concat(path, "definitions", k)) end end end

-- This is to ensure that the documentation for recursive refs is shown only once. function h.minRefDepths(schema) h.minDepthNode = {} local minDepths = {} h.walkSchema(function(schemaNode, path)		if schemaNode._ref then			minDepths[schemaNode._ref] = math.min(minDepths[schemaNode._ref] or 9000, #path)			if #path == minDepths[schemaNode._ref] then				h.minDepthNode[schemaNode._ref] = schemaNode			end		end	end, schema) end function h.hasRefs(schema) local hasRefs = false h.walkSchema(function(schemaNode)		if schemaNode._ref then			hasRefs = true			return false		end	end, schema) return hasRefs end function h.showSubkeys(schemaNode) local ref = schemaNode._ref if schemaNode._hideSubkeys then return false elseif not ref then return true elseif not utilsString.startsWith(ref, "#/definitions") then return false else return h.minDepthNode[ref] == schemaNode or not h.hasRefs(h.references[ref]) end end

function h.getTypeDefs(schemaName, schema, formattingFn, parentSchema, isSubschema) local typeLabel = schema.typeLabel local showSubkeys = true if schema._ref then typeLabel = typeLabel or string.gsub(schema._ref, "#/definitions/", "") typeLabel = typeLabel or string.gsub(typeLabel, "#", "") showSubkeys = h.showSubkeys(schema) end schema = h.resolveReference(schema) local rawType = schema.type local symbolicType local subkeys if showSubkeys then if schema.type == p.TYPES.record then for _, prop in ipairs(schema.properties) do				subkeys = subkeys or {} local propDef = h.getTypeDefs(prop.name, prop, formattingFn, schema) table.insert(subkeys, propDef) end end if schema.type == p.TYPES.array then local subtypeKeys, subtype = h.getTypeDefs(nil, schema.items, formattingFn, schema) if #subtypeKeys > 0 then subkeys = subtypeKeys end symbolicType = string.format(SYMBOLS.array, subtype) end if schema.type == p.TYPES.map then local _, keyType = h.getTypeDefs(nil, schema.keys, formattingFn, schema) local valueDef, valueType = h.getTypeDefs(nil, schema.values, formattingFn, schema) symbolicType = string.format(SYMBOLS.map, keyType, valueType) subkeys = valueDef end if schema.oneOf then subkeys = subkeys or {} subkeys.oneOf = subkeys.oneOf or {} local subtypes = {} local i = 1 for k, subschema in pairs(schema.oneOf) do				if type(k) == "string" then subschema.typeLabel = k				end local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema, true) if string.find(subtype, "|") or string.find(subtype, "&") then subtype = string.format(SYMBOLS.combinationGroup, subtype) end table.insert(subtypes, subtype) subkeys.oneOf[i] = keys i = i + 1 end symbolicType = subtypes[1] for i, subtype in ipairs(utilsTable.tail(subtypes)) do				symbolicType = string.format(SYMBOLS.oneOf, symbolicType, subtype) end end if schema.allOf then subkeys = subkeys or {} subkeys.allOf = {} local subtypes = {} for i, subschema in ipairs(schema.allOf) do				local keys, subtype = h.getTypeDefs(nil, subschema, formattingFn, schema) if string.find(subtype, "|") or string.find(subtype, "&") then subtype = string.format(SYMBOLS.combinationGroup, subtype) end table.insert(subtypes, subtype) subkeys.allOf[i] = keys end subtypes = utilsTable.unique(subtypes) symbolicType = subtypes[1] for i, subtype in ipairs(utilsTable.tail(subtypes)) do				symbolicType = string.format(SYMBOLS.allOf, symbolicType, subtype) end end end symbolicType = symbolicType or rawType or typeLabel local parentType = parentSchema and parentSchema.type if parentType == "array" and typeLabel then symbolicType = typeLabel end local key = schemaName if schema.default then key = key and string.format(SYMBOLS.default, schemaName, tostring(schema.default)) end if parentSchema == nil or not (parentSchema.allOf or parentSchema.oneOf or parentType == p.TYPES.array or parentType == p.TYPES.map) then -- otherwise leads to nonsense like [[string], instead of [{string}|string]		if schema.required then			symbolicType = string.format(SYMBOLS.required, symbolicType)		else			symbolicType = string.format(SYMBOLS.optional, symbolicType)			key = key and string.format(SYMBOLS.optional, key)		end	end	local formattedDef = formattingFn({		key = key,		subkeys = subkeys,		rawType = rawType,		typeLabel = typeLabel,		symbolicType = symbolicType,		desc = schema.desc,		parentType = parentType,		isSubschema = isSubschema,	})	return formattedDef, symbolicType end

function h.validate(schemaNode, data, dataName, dataPath, parentSchema, quiet) dataPath = dataPath or {} local value = utilsTable.property(data, dataPath) local errPath = dataName .. utilsTable.printPath(dataPath) schemaNode = h.resolveReference(schemaNode) local errors = h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet) if #errors > 0 then return errors end if schemaNode.allOf and value then local subschemaErrors = h.validateSubschemas(schemaNode.allOf, data, dataName, dataPath, schemaNode) local invalidSubschemas = utilsTable.keys(subschemaErrors) if #invalidSubschemas > 0 then invalidSubschemas = utilsTable.map(invalidSubschemas, code) invalidSubschemas = mw.text.listToText(invalidSubschemas) local msg = string.format("%s does not match  sub-schemas %s", code(errPath), invalidSubschemas) if not quiet then utilsError.warn(msg, { traceBack = false }) h.logSubschemaErrors(subschemaErrors) end table.insert(errors, {				path = errPath,				msg = msg,				errors = subschemaErrors			}) end end if schemaNode.oneOf and value then local subschemaErrors, validSubschemas = h.validateSubschemas(schemaNode.oneOf, data, dataName, dataPath, schemaNode, parentSchema) local invalidSubschemas = utilsTable.keys(subschemaErrors) if #validSubschemas == 0 then local msg = string.format("%s does not match any  sub-schemas.", code(errPath)) if not quiet then utilsError.warn(msg, { traceBack = false }) h.logSubschemaErrors(subschemaErrors) end table.insert(errors, {				path = errPath,				msg = msg,				errors = subschemaErrors			}) end if #validSubschemas > 1 then validSubschemas = utilsTable.map(validSubschemas, code) validSubschemas = mw.text.listToText(validSubschemas) local msg = string.format("%s matches  sub-schemas %s, but must match only one.", code(errPath), validSubschemas) if not quiet then utilsError.warn(msg, { traceBack = false }) end table.insert(errors, {				path = errPath,				msg = msg,			}) end end if schemaNode.properties and value then for _, propSchema in pairs(schemaNode.properties) do			local keyPath = utilsTable.concat(dataPath, propSchema.name) errors = utilsTable.concat(errors, h.validate(propSchema, data, dataName, keyPath, schemaNode, quiet)) end if not schemaNode.additionalProperties and not (parentSchema and parentSchema.allOf) then local schemaProps = schemaNode.properties if parentSchema and parentSchema.allOfProps then schemaProps = utilsTable.concat(schemaProps, parentSchema.allOfProps) end local schemaProps = utilsTable.map(schemaProps, "name") local dataProps = utilsTable.keys(value) local undefinedProps = utilsTable.difference(dataProps, schemaProps) if #undefinedProps > 0 then undefinedProps = mw.text.listToText(utilsTable.map(undefinedProps, code)) local msg = string.format("Record %s has undefined properties: %s", code(errPath), undefinedProps) if not quiet then utilsError.warn(msg, { traceBack = false }) end table.insert(errors, {					path = errPath,					msg = msg,				}) end end end if schemaNode.items and value then local props = utilsTable.stringKeys(value) if #props > 0 and not (parentSchema and parentSchema.allOf) then local msg = string.format("%s is supposed to be an array only, but it has string keys: %s", code(errPath), utilsTable.print(props)) if not quiet then utilsError.warn(msg, { traceBack = false }) end table.insert(errors, {				path = errPath,				msg = msg,			}) end for i, item in ipairs(value) do			local itemPath = utilsTable.concat(dataPath, i)			errors = utilsTable.concat(errors, h.validate(schemaNode.items, data, dataName, itemPath, schemaNode, quiet)) end end if schemaNode.keys and schemaNode.values and value then for k, v in pairs(value) do			local keyPath = utilsTable.concat(dataPath, string.format('["%s"]', k)) errors = utilsTable.concat(errors, h.validatePrimitive(schemaNode.keys, k, dataName, dataPath, quiet, true)) errors = utilsTable.concat(errors, h.validate(schemaNode.values, data, dataName, keyPath, schemaNode, quiet)) end end return errors end

function h.validateSubschemas(subschemas, data, dataName, dataPath, schemaNode, parentSchema) local errors = {} local valids = {} for k, subschema in pairs(subschemas) do		local key = subschema._ref and string.gsub(subschema._ref, "#/definitions/", "") or k		if parentSchema and parentSchema.allOf then local commonProps = {} for _, v in ipairs(parentSchema.allOf) do				commonProps = utilsTable.concat(commonProps, v.properties) end schemaNode.allOfProps = commonProps end local err = h.validate(subschema, data, dataName, dataPath, schemaNode, true) if #err > 0 then errors[key] = err else table.insert(valids, key) end end return errors, valids end function h.logSubschemaErrors(subschemaErrors, path) for schemaKey, schemaErrors in pairs(subschemaErrors) do local subpath = utilsTable.concat(path or {}, '["' .. schemaKey .. '"]') local indent = string.rep(":", #subpath) for _, err in pairs(schemaErrors) do			local msg = string.format("%s: %s", code(utilsTable.printPath(subpath)), err.msg) utilsError.warn(indent .. msg, { traceBack = false }) if err.errors then h.logSubschemaErrors(err.errors, subpath) end end end end

function h.validatePrimitive(schemaNode, value, dataName, dataPath, quiet, isKey) local errors = {} local addIfError = h.errorCollector(errors, value, dataName, dataPath, quiet, isKey) local validatorOptions = { quiet = quiet, stackTrace = false, }	if schemaNode.type and schemaNode.type ~= "any" then local expectedType = h.getLuaType(schemaNode.type) addIfError(utilsValidate.type(expectedType)) end if schemaNode.required then addIfError(utilsValidate.required) end if schemaNode.deprecated then addIfError(utilsValidate.deprecated) end if schemaNode.enum then addIfError(utilsValidate.enum(schemaNode.enum)) end return errors end

function h.errorCollector(errorTbl, value, dataName, dataPath, quiet, isKey) return function(validator) local errorMessages = validator(value, dataName, dataPath, isKey, {			quiet = quiet,			stackTrace = false,		}) if type(errorMessages) ~= "table" then --errMsg can either be a single message, or an array of messages (as is the case with enum) errorMessages = {errorMessages} end for _, errMsg in ipairs(errorMessages) do			table.insert(errorTbl, {				path = dataName .. utilsTable.printPath(dataPath),				msg = errMsg,			}) end end end

function h.getLuaType(schemaType) if schemaType == p.TYPES.array or schemaType == p.TYPES.record or schemaType == p.TYPES.map then return "table" end return schemaType end

p.Schemas = { Schema = { allOf = { {				type = "record", properties = { {						name = "definitions", desc = "Schema fragments for referencing.", type = "map", keys = { type = "string" }, values = { _ref = "#/definitions/Schema" }, }				},			},			{				_ref = "#/definitions/Schema", },		},		definitions = { Schema = { required = true, oneOf = { { _ref = "#/definitions/Primitive Schema" }, { _ref = "#/definitions/Array Schema" }, { _ref = "#/definitions/Record Schema" }, { _ref = "#/definitions/Map Schema" }, { _ref = "#/definitions/oneOf" }, { _ref = "#/definitions/allOf" }, { _ref = "#/definitions/Reference" }, }			},			base = { type = "record", properties = { {						name = "_id", type = "string", desc = "An ID to use for referencing.", },					{						name = "required", type = "boolean", desc = "If, the value cannot be  ." },					{						name = "deprecated", type = "boolean", desc = "If, validation fails when this value is present. A deprecation warning is logged." },					{						name = "desc", type = "string", desc = "Description of the schema, for documentation." },				},			},			["Primitive Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "string", "number", "boolean", "any" }, desc = ', ,  , or  ' },							{								name = "enum", desc = "An array of values that are considered acceptable, plus an optional  key.", allOf = { {										type = "array", items = { type = "any" }, },									{										type = "record", properties = { {												name = "reference", type = "string", desc = "A link to a page that lists the accepted values." }										}									}								}							},						},					}				}			},			["Array Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "array" }, desc = 'The string .' },							{								name = "items", required = true, _ref = "#/definitions/Schema", desc = "A schema that all items in the array must adhere to.", },						},					}				}			},			name = { type = "record", properties = { {						name = "name", required = true, type = "string", desc = "The key for the record entry." },				},			},			["Record Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "record" }, desc = 'The string .' },							{								name = "properties", required = true, type = "array", desc = "An array of schemas for each record entry, plus an additional field  for the name of the record entry.", items = { allOf = { { _ref = "#/definitions/name" }, { _ref = "#/definitions/Schema" }, }								},							},						},						{							name = "additionalProperties", type = "boolean", desc = "If true, the record is considered valid when it has additional properties other than the ones specified in . True by default for   subschemas; false by default otherwise." },					},				},			},			["Map Schema"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "type", type = "string", required = true, enum = { "map" }, desc = 'The string .' },							{								name = "keys", type = "record", required = true, _ref = "#/definitions/Schema", desc = "The schema for the keys of the map." },							{								name = "values", type = "record", required = true, _ref = "#/definitions/Schema", desc = "The schema for the values of the map.", },						},					}				}			},			["oneOf"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "oneOf", required = true, desc = "A table of subschemas. The data must be valid against exactly one of them.", oneOf = { {										type = "array", items = { _ref = "#/definitions/Schema" }, },									{										type = "map", keys = { type = "string" }, values = { _ref = "#/definitions/Schema" }, }								},							}						},					}				}			},			["allOf"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "allOf", required = true, type = "array", desc = "An array of subschemas. The data must be valid against all of them.", items = { _ref = "#/definitions/Schema" }, },						},					}				}			},			["Reference"] = { allOf = { { _ref = "#/definitions/base" }, {						type = "record", properties = { {								name = "_ref", required = true, type = "string", desc = "A reference to another part of the schema.", },							{								name = "_hideSubkeys", type = "boolean", desc = "Ignore reference when generating documentation. Use sparingly as there is already a mechanism for determining whether to show referenced schema fragments." }						},					}				}			},		},	} }

p.Documentation = { validate = { params = {"schema", "schemaName", "data", "dataName"}, returns = "An array of error paths and messages, or nil if there are none.", cases = { outputOnly = true, {				desc = "Basic record with four fields.", args = { {						type = "record", properties = { {								name = "name", type = "string", required = true, },							{								name = "games", type = "array", items = { type = "string", enum = { "TLoZ", "TAoL", "ALttP"}, },							},							{								name = "cost", type = "number", },							{								name = "deprecatedField", type = "string", deprecated = true, }						},											},					"gameItem", {						cost = "50 Rupees", games = { "YY", "ZZ" }, nonsenseField = "foo", deprecatedField = "bar", },					"itemVariable" },				expect = { {						path = 'itemVariable.name', msg = ' is required but is  .', },					{						path = 'itemVariable.games[1]', msg = ' has unexpected value. The accepted values are: ', },					{						path = 'itemVariable.games[2]', msg = ' has unexpected value. The accepted values are: ', },					{						path = 'itemVariable.cost', msg = ' is type   but type   was expected.', },					{						msg = ' is deprecated but has value  .', path = 'itemVariable.deprecatedField', },					{						path = "itemVariable", msg = "Record  has undefined properties:  ", },				},			},			{				desc = "Map validation.", args = { {						type = "map", keys = { type = "string" }, values = { type = "string" }, },					"Games", {						OoA = "Oracle of Ages", OoT = 5, [3] = "A Link to the Past", },					"games" },				expect = { {						path = "games", msg = " key   is type   but type   was expected.", },					{						path = 'games["OoT"]', msg = ' is type   but type   was expected.', },				}			},			{				desc = "A schema using  and an ID-based reference.", args = { {						oneOf = { {								_id = "#num", type = "number", },							{								type = "array", items = { _ref = "#num"}, },						}					},					"numberOrArray", { "foo" }, "arg", },				expect = { {						msg = " does not match any   sub-schemas.", path = "arg", errors = { {								{									path = "arg", msg = " is type   but type   was expected.", },							},							{								{									path = "arg[1]", msg = " is type   but type   was expected.", },							},						},					},				},			},			{				desc = "Schema using  and   references. A simplification of Module:Documentation's schema.", args = { {						oneOf = { { _ref = "#/definitions/functions", }, { _ref = "#/definitions/sections", }, },						definitions = { functions = { type = "array", items = { type = "string" }, },							sections = { type = "record", properties = { {										name = "sections", type = "array", items = { _ref = "#/definitions/functions" } },								},							},						}					},					"Documentation", {						"foo", sections = { {"bar", "baz"}, { "quux" }, },					},					"doc", },				expect = { {						msg = " does not match any   sub-schemas.", path = "doc", errors = { functions = { {									msg = ' is supposed to be an array only, but it has string keys: {"sections"}', path = "doc", },							},							sections = { {									path = "doc", msg = "Record  has undefined properties:  ", },							},						},					},				},			},			{				desc = "Data is invalid if it matches more than one .", args = { {						oneOf = {{ type = "string" }, { type = "string" }} },					"Schema", "Fooloo Limpah", "magicWords", },				expect = { {						path = "magicWords", msg = " matches   sub-schemas   and , but must match only one.", },				},			},			{				desc = "A schema using .", args = { {						allOf = { { 								type = "record", properties = { {										name = "foo", type = "string", },								},							},							{ 								type = "array", items = { type = "number" }, }						},					},					"Schema", {1, 2, 3, foo = 4}, "arg", },				expect = { {						msg = " does not match   sub-schemas  ", path = "arg", errors = { {								{									path = "arg.foo", msg = ' is type   but type   was expected.', },							},						}					},				}			},		},	}, }

return p