From bcde3816906303044f1fad30a75581521b936857 Mon Sep 17 00:00:00 2001 From: Malware Date: Sun, 22 Jul 2018 10:35:37 +0200 Subject: [PATCH] Working on an API documentation generator --- Source/DocGen/DocGen.csproj | 5 + Source/DocGen/MDKUtilityFramework.cs | 55 +++++++++ Source/DocGen/Program.cs | 18 ++- Source/DocGen/ProgrammableBlockApi.cs | 169 ++++++++++++++++++++++++++ Source/DocGen/Terminals.cs | 8 +- Source/DocGen/Whitelist.cs | 59 +++++++++ Source/DocGen/WhitelistKey.cs | 169 ++++++++++++++++++++++++++ 7 files changed, 472 insertions(+), 11 deletions(-) create mode 100644 Source/DocGen/MDKUtilityFramework.cs create mode 100644 Source/DocGen/ProgrammableBlockApi.cs create mode 100644 Source/DocGen/Whitelist.cs create mode 100644 Source/DocGen/WhitelistKey.cs diff --git a/Source/DocGen/DocGen.csproj b/Source/DocGen/DocGen.csproj index bd77e81..ba2ef1d 100644 --- a/Source/DocGen/DocGen.csproj +++ b/Source/DocGen/DocGen.csproj @@ -21,6 +21,7 @@ prompt MinimumRecommendedRules.ruleset true + latest bin\x64\Release\ @@ -43,9 +44,13 @@ + + + + diff --git a/Source/DocGen/MDKUtilityFramework.cs b/Source/DocGen/MDKUtilityFramework.cs new file mode 100644 index 0000000..4ab7fb0 --- /dev/null +++ b/Source/DocGen/MDKUtilityFramework.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +namespace DocGen +{ + /// + /// Framework initialization and configuration + /// + public class MDKUtilityFramework + { + static readonly Dictionary AssemblyNames = new Dictionary(); + + /// + /// Gets the game binary path as defined through . + /// + public static string GameBinPath { get; internal set; } + + /// + /// Initializes the mock system. Pass in the path to the Space Engineers Bin64 folder. + /// + /// The path to the MDK options file + public static void Load(string gameBinPath) + { + var directory = new DirectoryInfo(gameBinPath); + + foreach (var dllFileName in directory.EnumerateFiles("*.dll")) + { + AssemblyName assemblyName; + try + { + assemblyName = AssemblyName.GetAssemblyName(dllFileName.FullName); + } + catch (BadImageFormatException) + { + // Not a .NET assembly or wrong platform, ignore + continue; + } + AssemblyNames[assemblyName.FullName] = assemblyName; + //AssemblyNames[dllFileName.FullName.Replace("\\", "\\\\")] = assemblyName; + } + AppDomain.CurrentDomain.AssemblyResolve += OnResolveAssembly; + + GameBinPath = directory.FullName; + } + + static Assembly OnResolveAssembly(object sender, ResolveEventArgs args) + { + if (AssemblyNames.TryGetValue(args.Name, out AssemblyName assemblyName)) + return Assembly.Load(assemblyName); + return null; + } + } +} \ No newline at end of file diff --git a/Source/DocGen/Program.cs b/Source/DocGen/Program.cs index 29980ab..440e6c5 100644 --- a/Source/DocGen/Program.cs +++ b/Source/DocGen/Program.cs @@ -10,13 +10,13 @@ namespace DocGen { public class Program { - public static int Main() + public static async Task Main() { var commandLine = new CommandLine(Environment.CommandLine); try { var program = new Program(); - return Task.Run(async () => await program.Run(commandLine).ConfigureAwait(false)).GetAwaiter().GetResult(); + return await program.Run(commandLine); } catch (Exception e) { @@ -38,11 +38,13 @@ async Task UpdateCaches(string path) var pluginPath = Path.GetFullPath("MDKWhitelistExtractor.dll"); var whitelistTarget = path; var terminalTarget = path; + var apiTarget = path; var directoryInfo = new DirectoryInfo(whitelistTarget); if (!directoryInfo.Exists) directoryInfo.Create(); whitelistTarget = Path.Combine(whitelistTarget, "whitelist.cache"); terminalTarget = Path.Combine(terminalTarget, "terminal.cache"); + apiTarget = Path.Combine(apiTarget, "api.cache"); var args = new List { @@ -52,7 +54,9 @@ async Task UpdateCaches(string path) "-whitelistcaches", $"\"{whitelistTarget}\"", "-terminalcaches", - $"\"{terminalTarget}\"" + $"\"{terminalTarget}\"", + "-pbapi", + $"\"{apiTarget}\"" }; var process = new Process @@ -121,13 +125,17 @@ async Task Run(CommandLine commandLine) if (outputIndex >= 0) output = Path.GetFullPath(commandLine[outputIndex + 1]); if (output != null) - GenerateDocs(path, output); + await GenerateDocs(path, output); return 0; } - void GenerateDocs(string path, string output) + async Task GenerateDocs(string path, string output) { + var api = new ProgrammableBlockApi(); + await api.Scan(Path.Combine(path, "whitelist.cache")); + await api.SaveAsync(Path.Combine(output, "api")); + //var whitelistTarget = path; var terminalTarget = path; //whitelistTarget = Path.Combine(whitelistTarget, "whitelist.cache"); diff --git a/Source/DocGen/ProgrammableBlockApi.cs b/Source/DocGen/ProgrammableBlockApi.cs new file mode 100644 index 0000000..3641ae1 --- /dev/null +++ b/Source/DocGen/ProgrammableBlockApi.cs @@ -0,0 +1,169 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Xml.XPath; +using Malware.MDKUtilities; + +namespace DocGen +{ + class ProgrammableBlockApi + { + List _members = new List(); + List _assemblies; + List>> _groupings; + + public async Task Scan(string whitelistCacheFileName) + { + await Task.Run(() => + { + var whitelist = Whitelist.Load(whitelistCacheFileName); + var spaceEngineers = new SpaceEngineers(); + var installPath = Path.Combine(spaceEngineers.GetInstallPath(), "bin64"); + MDKUtilityFramework.Load(installPath); + var dllFiles = Directory.EnumerateFiles(installPath, "*.dll", SearchOption.TopDirectoryOnly) + .ToList(); + + foreach (var dllFile in dllFiles) + Visit(whitelist, dllFile); + + _groupings = _members.GroupBy(m => m.DeclaringType).GroupBy(m => m.Key.Assembly) + .ToList(); + }); + } + + void Visit(Whitelist whitelist, string dllFile) + { + try + { + var assemblyName = AssemblyName.GetAssemblyName(dllFile); + var assembly = Assembly.Load(assemblyName); + Visit(whitelist, assembly); + } + catch (FileLoadException e) + { } + catch (BadImageFormatException e) + { } + } + + void Visit(Whitelist whitelist, Assembly assembly) + { + if (!whitelist.IsWhitelisted(assembly)) + return; + if (assembly.GetName().Name == "mscorlib") + return; + var companyAttribute = assembly.GetCustomAttribute(); + if (companyAttribute?.Company == "Microsoft Corporation") + return; + var types = assembly.GetExportedTypes(); + foreach (var type in types) + Visit(whitelist, type); + } + + void Visit(Whitelist whitelist, Type type) + { + if (!type.IsPublic) + return; + var members = type.GetMembers(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly); + foreach (var member in members) + Visit(whitelist, member); + } + + void Visit(Whitelist whitelist, MemberInfo member) + { + if (!whitelist.IsWhitelisted(member)) + return; + _members.Add(member); + } + + public async Task SaveAsync(string path) + { + var directory = new DirectoryInfo(Path.Combine(path, "api")); + if (!directory.Exists) + directory.Create(); + var fileName = Path.Combine(directory.FullName, "index.md"); + using (var file = File.CreateText(fileName)) + { + await file.WriteLineAsync("#Index"); + await file.WriteLineAsync(); + + foreach (var assemblyGroup in _groupings.OrderBy(g => g.Key.GetName().Name)) + { + var assemblyPath = new Uri(assemblyGroup.Key.CodeBase).LocalPath; + var assemblyFileName = Path.GetFileName(assemblyPath); + var xmlFileName = Path.ChangeExtension(assemblyPath, "xml"); + XDocument documentation; + if (File.Exists(xmlFileName)) + documentation = XDocument.Load(xmlFileName); + else + documentation = null; + + await file.WriteLineAsync($"##{assemblyFileName}"); + foreach (var typeGroup in assemblyGroup.OrderBy(g => g.Key.FullName)) + { + var typeKey = WhitelistKey.ForType(typeGroup.Key); + var mdPath = ToMdFileName(typeKey.Path); + await file.WriteLineAsync($"**[`{typeKey.Path}`]({mdPath})**"); + foreach (var member in typeGroup.OrderBy(m => m.Name)) + { + var memberKey = WhitelistKey.ForMember(member, false); + await file.WriteLineAsync($"* [`{memberKey.Path}`]({mdPath})"); + } + await file.WriteLineAsync(); + await WriteTypeFileAsync(typeGroup, Path.Combine(directory.FullName, mdPath), documentation); + } + } + + file.Flush(); + } + } + + async Task WriteTypeFileAsync(IGrouping typeGroup, string fileName, XDocument documentation) + { + using (var file = File.CreateText(fileName)) + { + var typeKey = WhitelistKey.ForType(typeGroup.Key); + await file.WriteLineAsync($"#{typeKey.Path}"); + await file.WriteLineAsync(); + foreach (var member in typeGroup.OrderBy(m => m.Name)) + { + var fullMemberKey = WhitelistKey.ForMember(member); + var xmlKey = fullMemberKey.ToXmlDoc(); + var memberKey = WhitelistKey.ForMember(member, false); + var doc = documentation?.XPathSelectElement($"/doc/members/member[@name='{xmlKey}']"); + string summary; + if (doc != null) + summary = doc.Element("summary")?.Value ?? ""; + else + summary = ""; + await file.WriteLineAsync($"* `{memberKey.Path}`"); + await file.WriteLineAsync($" " + Trim(summary)); + await file.WriteLineAsync(); + } + } + } + + string Trim(string summary) => Regex.Replace(summary.Trim(), @"\s{2,}", " "); + + readonly HashSet _invalidCharacters = new HashSet(Path.GetInvalidFileNameChars()); + + string ToMdFileName(string path) + { + var builder = new StringBuilder(path); + for (var i = 0; i < builder.Length; i++) + { + if (_invalidCharacters.Contains(builder[i])) + builder[i] = '_'; + } + + builder.Append(".md"); + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/Source/DocGen/Terminals.cs b/Source/DocGen/Terminals.cs index 1b149ff..15a4030 100644 --- a/Source/DocGen/Terminals.cs +++ b/Source/DocGen/Terminals.cs @@ -1,5 +1,4 @@ -using System; -using System.IO; +using System.IO; using System.Linq; using System.Text; using System.Xml.Linq; @@ -30,6 +29,7 @@ void Load(XElement root) return; var blocks = root.Elements("block").OrderBy(block => GetBlockName((string)block.Attribute("type"))).ToArray(); _document.AppendLine($"## Overview"); + _document.AppendLine("**Note: Terminal actions and properties are for all intents and purposes obsolete since all vanilla block interfaces now contain proper API access to all this information. It is highly recommended you use those for less overhead.**"); _document.AppendLine(); foreach (var block in blocks) { @@ -50,9 +50,7 @@ void Load(XElement root) _document.AppendLine("|Name|Description|"); _document.AppendLine("|-|-|"); foreach (var action in elements) - { _document.AppendLine($"|{(string)action.Attribute("name")}|{(string)action.Attribute("text")}|"); - } _document.AppendLine(); } elements = block.Elements("property").OrderBy(e => (string)e.Attribute("name")).ToArray(); @@ -63,9 +61,7 @@ void Load(XElement root) _document.AppendLine("|Name|Type|"); _document.AppendLine("|-|-|"); foreach (var action in elements) - { _document.AppendLine($"|{(string)action.Attribute("name")}|{TranslateType((string)action.Attribute("type"))}|"); - } _document.AppendLine(); } } diff --git a/Source/DocGen/Whitelist.cs b/Source/DocGen/Whitelist.cs new file mode 100644 index 0000000..44c2b52 --- /dev/null +++ b/Source/DocGen/Whitelist.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace DocGen +{ + class Whitelist + { + public static Whitelist Load(string fileName) + { + var lines = File.ReadAllLines(fileName).Where(line => !string.IsNullOrWhiteSpace(line)).Select(line => line.Trim()).ToList(); + return new Whitelist(lines); + } + + List _entries; + HashSet _assemblyNames; + + Whitelist(List lines) + { + _entries = lines.Select(line => + { + if (WhitelistKey.TryParse(line, out var entry)) + return entry; + return null; + }).Where(entry => entry != null) + .ToList(); + _assemblyNames = new HashSet(_entries.Select(e => e.AssemblyName).Distinct(StringComparer.OrdinalIgnoreCase), StringComparer.OrdinalIgnoreCase); + } + + public bool IsWhitelisted(Assembly assembly) + { + return IsWhitelisted(assembly.GetName()); + } + + public bool IsWhitelisted(AssemblyName assemblyName) + { + return _assemblyNames.Contains(assemblyName.Name); + } + + public bool IsWhitelisted(Type type) + { + var typeKey = WhitelistKey.ForType(type); + if (typeKey == null) + return false; + return _entries.Any(key => key.IsMatchFor(typeKey)); + } + + public bool IsWhitelisted(MemberInfo memberInfo) + { + if (!IsWhitelisted(memberInfo.DeclaringType.Assembly)) + return false; + var typeKey = WhitelistKey.ForMember(memberInfo); + return _entries.Any(key => key.IsMatchFor(typeKey)); + } + } +} \ No newline at end of file diff --git a/Source/DocGen/WhitelistKey.cs b/Source/DocGen/WhitelistKey.cs new file mode 100644 index 0000000..87b5e08 --- /dev/null +++ b/Source/DocGen/WhitelistKey.cs @@ -0,0 +1,169 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace DocGen +{ + class WhitelistKey + { + public static bool TryParse(string text, out WhitelistKey entry) + { + var assemblyNameIndex = text.LastIndexOf(','); + if (assemblyNameIndex < 0) + { + entry = null; + return false; + } + + var assemblyName = text.Substring(assemblyNameIndex + 1).Trim(); + var path = text.Substring(0, assemblyNameIndex).Trim(); + var regexPattern = "\\A" + Regex.Escape(path.Replace('+', '.')).Replace("\\*", ".*") + "\\z"; + var regex = new Regex(regexPattern, RegexOptions.Singleline | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + entry = new WhitelistKey(assemblyName, path, regex, null); + return true; + } + + public static WhitelistKey ForType(Type type, bool includeNamespace = true) + { + if (!type.IsPublic && !type.IsNestedPublic && !type.IsNestedFamily && !type.IsNestedFamANDAssem && !type.IsNestedFamORAssem) + return null; + //if (type.IsGenericType) + // type = type.GetGenericTypeDefinition(); + if (type.IsGenericTypeDefinition || type.IsGenericType) + return ForGenericType(type, includeNamespace); + var assemblyName = type.Assembly.GetName().Name; + var path = includeNamespace? type.FullName : type.Name; + return new WhitelistKey(assemblyName, path, null, "T"); + } + + static WhitelistKey ForGenericType(Type type, bool includeNamespace = true) + { + var assemblyName = type.Assembly.GetName().Name; + var path = type.IsGenericType? (includeNamespace? type.GetGenericTypeDefinition().FullName : type.GetGenericTypeDefinition().Name) : (includeNamespace ? type.FullName : type.Name); + var separatorIndex = path.IndexOf('`'); + if (separatorIndex >= 0) + path = path.Substring(0, separatorIndex); + var genericArguments = type.GetGenericArguments(); + path += "<" + string.Join(", ", genericArguments.Select(arg => arg.IsGenericParameter? arg.Name : arg.FullName)) + ">"; + return new WhitelistKey(assemblyName, path, null, "T"); + } + + public static WhitelistKey ForMember(MemberInfo memberInfo, bool includeTypeName = true) + { + switch (memberInfo) + { + case ConstructorInfo constructorInfo: + return ForConstructor(constructorInfo, includeTypeName); + case EventInfo eventInfo: + return ForEvent(eventInfo, includeTypeName); + case FieldInfo fieldInfo: + return ForField(fieldInfo, includeTypeName); + case PropertyInfo propertyInfo: + return ForProperty(propertyInfo, includeTypeName); + case MethodInfo methodInfo: + return ForMethod(methodInfo, includeTypeName); + default: + return null; + } + } + + static WhitelistKey ForConstructor(ConstructorInfo constructorInfo, bool includeTypeName) + { + if (constructorInfo.IsSpecialName || !constructorInfo.IsPublic && !constructorInfo.IsFamily && !constructorInfo.IsFamilyAndAssembly && !constructorInfo.IsFamilyOrAssembly) + return null; + var basis = ForType(constructorInfo.DeclaringType); + var path = includeTypeName? basis.Path + "." + constructorInfo.Name : constructorInfo.Name; + var parameters = constructorInfo.GetParameters(); + path += "(" + string.Join(", ", parameters.Select(ParameterStr)) + ")"; + return new WhitelistKey(basis.AssemblyName, path, null, "C"); + } + + static WhitelistKey ForEvent(EventInfo eventInfo, bool includeTypeName) + { + if (eventInfo.IsSpecialName || !(eventInfo.AddMethod?.IsPublic ?? false) && !(eventInfo.AddMethod?.IsFamily ?? false) && !(eventInfo.AddMethod?.IsFamilyAndAssembly ?? false) && !(eventInfo.AddMethod?.IsFamilyOrAssembly ?? false) + && !(eventInfo.RemoveMethod?.IsPublic ?? false) && !(eventInfo.RemoveMethod?.IsFamily ?? false) && !(eventInfo.RemoveMethod?.IsFamilyAndAssembly ?? false) && !(eventInfo.RemoveMethod?.IsFamilyOrAssembly ?? false)) + return null; + var basis = ForType(eventInfo.DeclaringType); + return new WhitelistKey(basis.AssemblyName, includeTypeName? basis.Path + "." + eventInfo.Name: eventInfo.Name, null, "E"); + } + + static WhitelistKey ForField(FieldInfo fieldInfo, bool includeTypeName) + { + if (fieldInfo.IsSpecialName || !fieldInfo.IsPublic && !fieldInfo.IsFamily && !fieldInfo.IsFamilyAndAssembly && !fieldInfo.IsFamilyOrAssembly) + return null; + var basis = ForType(fieldInfo.DeclaringType); + return new WhitelistKey(basis.AssemblyName, includeTypeName? basis.Path + "." + fieldInfo.Name: fieldInfo.Name, null, "F"); + } + + static WhitelistKey ForProperty(PropertyInfo propertyInfo, bool includeTypeName) + { + if (propertyInfo.IsSpecialName || !(propertyInfo.GetMethod?.IsPublic ?? false) && !(propertyInfo.GetMethod?.IsFamily ?? false) && !(propertyInfo.GetMethod?.IsFamilyAndAssembly ?? false) && !(propertyInfo.GetMethod?.IsFamilyOrAssembly ?? false) + && !(propertyInfo.SetMethod?.IsPublic ?? false) && !(propertyInfo.SetMethod?.IsFamily ?? false) && !(propertyInfo.SetMethod?.IsFamilyAndAssembly ?? false) && !(propertyInfo.SetMethod?.IsFamilyOrAssembly ?? false)) + return null; + var basis = ForType(propertyInfo.DeclaringType); + return new WhitelistKey(basis.AssemblyName, includeTypeName? basis.Path + "." + propertyInfo.Name : propertyInfo.Name, null, "P"); + } + + static WhitelistKey ForMethod(MethodInfo methodInfo, bool includeTypeName) + { + if (methodInfo.IsSpecialName || !methodInfo.IsPublic && !methodInfo.IsFamily && !methodInfo.IsFamilyAndAssembly && !methodInfo.IsFamilyOrAssembly) + return null; + + //if (methodInfo.IsGenericMethod) + // methodInfo = methodInfo.GetGenericMethodDefinition(); + if (methodInfo.IsGenericMethodDefinition || methodInfo.IsGenericMethod) + return ForGenericMethod(methodInfo, includeTypeName); + var basis = ForType(methodInfo.DeclaringType); + var path = includeTypeName? basis.Path + "." + methodInfo.Name : methodInfo.Name; + var parameters = methodInfo.GetParameters(); + path += "(" + string.Join(", ", parameters.Select(ParameterStr)) + ")"; + return new WhitelistKey(basis.AssemblyName, path, null, "M"); + } + + static WhitelistKey ForGenericMethod(MethodInfo methodInfo, bool includeTypeName) + { + var basis = ForType(methodInfo.DeclaringType); + var path = includeTypeName? basis.Path + "." + methodInfo.Name : methodInfo.Name; + var separatorIndex = path.IndexOf('`'); + if (separatorIndex >= 0) + path = path.Substring(0, separatorIndex); + var genericArguments = methodInfo.GetGenericArguments(); + path += "<" + string.Join(", ", genericArguments.Select(arg => arg.Name)) + ">"; + var parameters = methodInfo.GetParameters(); + path += "(" + string.Join(", ", parameters.Select(ParameterStr)) + ")"; + return new WhitelistKey(basis.AssemblyName, path, null, "M"); + } + + static string ParameterStr(ParameterInfo parameterInfo) + { + var prefix = parameterInfo.ParameterType.IsByRef ? "ref " : parameterInfo.ParameterType.IsPointer ? "*" : parameterInfo.IsOut ? "out " : ""; + var type = parameterInfo.ParameterType.IsByRef || parameterInfo.ParameterType.IsPointer ? parameterInfo.ParameterType.GetElementType() : parameterInfo.ParameterType; + return prefix + ForType(type).Path; + } + + readonly Regex _regex; + readonly string _typeChar; + + WhitelistKey(string assemblyName, string path, Regex regex, string typeChar) + { + _regex = regex; + _typeChar = typeChar; + AssemblyName = assemblyName; + Path = path; + } + + public string AssemblyName { get; } + + public string Path { get; } + + public bool IsMatchFor(WhitelistKey typeKey) + { + if (typeKey == null) + return false; + return string.Equals(AssemblyName, typeKey.AssemblyName, StringComparison.OrdinalIgnoreCase) && _regex.IsMatch(typeKey.Path); + } + + public string ToXmlDoc() => $"{_typeChar}:{Path.Replace('<', '{').Replace('>', '}')}"; + } +} \ No newline at end of file