diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..8b8f9fe --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "idapopulate/FFXIVClientStructs"] + path = idapopulate/FFXIVClientStructs + url = https://github.com/awgil/FFXIVClientStructs.git diff --git a/idapopulate/FFXIVClientStructs b/idapopulate/FFXIVClientStructs new file mode 160000 index 0000000..b67538d --- /dev/null +++ b/idapopulate/FFXIVClientStructs @@ -0,0 +1 @@ +Subproject commit b67538d428032c0fab9bce8062f2c2b79bc6e7fc diff --git a/idapopulate/idapopulate.sln b/idapopulate/idapopulate.sln new file mode 100644 index 0000000..13e497a --- /dev/null +++ b/idapopulate/idapopulate.sln @@ -0,0 +1,40 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.4.33205.214 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "idapopulate", "idapopulate\idapopulate.csproj", "{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}" + ProjectSection(ProjectDependencies) = postProject + {A8C9ED42-A7CF-4A7C-894C-803030860E21} = {A8C9ED42-A7CF-4A7C-894C-803030860E21} + EndProjectSection +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FFXIVClientStructs", "FFXIVClientStructs\FFXIVClientStructs\FFXIVClientStructs.csproj", "{A8C9ED42-A7CF-4A7C-894C-803030860E21}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FFXIVClientStructs.InteropSourceGenerators", "FFXIVClientStructs\FFXIVClientStructs.InteropSourceGenerators\FFXIVClientStructs.InteropSourceGenerators.csproj", "{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Release|Any CPU.Build.0 = Release|Any CPU + {A8C9ED42-A7CF-4A7C-894C-803030860E21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8C9ED42-A7CF-4A7C-894C-803030860E21}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8C9ED42-A7CF-4A7C-894C-803030860E21}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8C9ED42-A7CF-4A7C-894C-803030860E21}.Release|Any CPU.Build.0 = Release|Any CPU + {8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8B4CC935-E684-4D8D-91EF-56B88D28E7C3} + EndGlobalSection +EndGlobal diff --git a/idapopulate/idapopulate/CSImport.cs b/idapopulate/idapopulate/CSImport.cs new file mode 100644 index 0000000..e29b2f2 --- /dev/null +++ b/idapopulate/idapopulate/CSImport.cs @@ -0,0 +1,339 @@ +using FFXIVClientStructs.Attributes; +using FFXIVClientStructs.Interop; +using FFXIVClientStructs.Interop.Attributes; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace idapopulate; + +internal static class CSImportExt +{ + public static string WithoutPrefix(this string str, string prefix) => str.StartsWith(prefix) ? str.Substring(prefix.Length) : str; + + // if [FieldOffset] is not specified, assume sequential layout... + public static int GetFieldOffset(this FieldInfo fi) => fi.GetCustomAttribute()?.Value ?? Marshal.OffsetOf(fi.DeclaringType!, fi.Name).ToInt32(); + + public static (Type?, int) GetFixedBufferInfo(this FieldInfo fi) + { + var attr = fi.GetCustomAttribute(); + return (attr?.ElementType, attr?.Length ?? 0); + } + + public static (Type?, int) GetFixedArrayInfo(this FieldInfo fi) + { + var attr = fi.GetCustomAttribute(typeof(FixedSizeArrayAttribute<>)); + if (attr == null) + return (null, 0); + var len = (int?)attr.GetType().GetProperty("Count", BindingFlags.Instance | BindingFlags.Public)?.GetValue(attr) ?? 0; + var t = attr.GetType().GetGenericArguments()[0]; + if (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Pointer<>)) + t = t.GetGenericArguments()[0].MakePointerType(); + return (t, len); + } +} + +internal class CSImport +{ + private HashSet _processedTypes = new(); + + public void Populate(Result res, SigResolver resolver) + { + var typesToProcess = GetAssemblyTypes("FFXIVClientStructs").Where(IsTypeExportable).ToList(); + for (int i = 0; i < typesToProcess.Count; i++) // note: more types could be added while processing types + { + PopulateType(typesToProcess[i], typesToProcess, res, resolver); + } + } + + private static IEnumerable GetAssemblyTypes(string assemblyName) + { + var assembly = AppDomain.CurrentDomain.Load(assemblyName); + try + { + return assembly.DefinedTypes.Select(ti => ti.AsType()); + } + catch (ReflectionTypeLoadException ex) + { + return ex.Types.Cast(); + } + } + + private static bool IsTypeExportable(Type type) + { + if (type.FullName == null) + return false; + if (!type.FullName.StartsWith("FFXIVClientStructs.FFXIV.") && !type.FullName.StartsWith("FFXIVClientStructs.Havok.")) + return false; + if (type.DeclaringType != null && (type.Name is "Addresses" or "MemberFunctionPointers" or "StaticAddressPointers" || type.Name == type.DeclaringType.Name + "VTable")) + return false; + if (type.Name.EndsWith("e__FixedBuffer")) + return false; + return true; + } + + private void QueueType(Type type, List queue) + { + while (DerefPointer(type) is var derefType && derefType != null) + type = derefType; + if (type.IsPrimitive || type == typeof(void)) + return; // void can appear as eg void* fields + queue.Add(type); + } + + private void PopulateType(Type type, List queue, Result res, SigResolver resolver) + { + if (!_processedTypes.Add(type)) + return; // already processed + + var tn = TypeName(type); + if (type.IsEnum) + { + var (width, signed) = GetEnumSignWidth(type); + var e = new Result.Enum() { IsBitfield = type.GetCustomAttribute() != null, IsSigned = signed, Width = width }; + foreach (var f in type.GetFields().Where(f => f.Name != "value__")) + e.Values.Add(new() { Name = f.Name, Value = Convert.ToInt64(f.GetRawConstantValue()) }); + res.Enums.Add(tn, e); + } + else + { + if (type.IsGenericType && type.ContainsGenericParameters) + { + //Debug.WriteLine($"Skipping generic struct: {type}"); + return; // we don't care about unspecialized templates + } + + var addresses = type.GetNestedType("Addresses"); + + var s = new Result.Struct() { Size = SizeOf(type) }; + if ((type.StructLayoutAttribute?.Size ?? 0) is var layoutSize && layoutSize != 0 && layoutSize != s.Size) + Debug.WriteLine($"Size mismatch for {type}: layout says 0x{layoutSize:X}, actual is 0x{s.Size:X}"); + + // see whether there are any vtable definitions or virtual functions + if (type.GetCustomAttribute() is var attrVTable && attrVTable != null) + { + if (attrVTable.IsPointer) + Debug.WriteLine($"VTable for {type} is stored as a pointer, wtf does it even mean?"); + s.PrimaryVTable = new() { Address = new(attrVTable.Signature, attrVTable.Offset), Ea = GetResolvedAddress(addresses, "VTable", resolver) }; + } + + // process methods (both virtual and non-virtual) + foreach (var method in type.GetMethods(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public)) + { + if (method.GetCustomAttribute() is var vfAttr && vfAttr != null) + { + s.PrimaryVTable ??= new(); + s.PrimaryVTable.VFuncs.Add(vfAttr.Index, new() { Name = method.Name, Signature = ExtractFuncSig(method, tn) }); + } + else if (method.GetCustomAttribute() is var mfAttr && mfAttr != null) + { + var ea = GetResolvedAddress(addresses, method.Name, resolver); + if (ea == 0) + Debug.WriteLine($"Failed to find method {type}.{method.Name}: sig={mfAttr.Signature}"); + else if (res.Functions.ContainsKey(ea)) + Debug.WriteLine($"Multiple functions resolve to same address 0x{ea:X}: {type}.{method.Name} sig={mfAttr.Signature} vs. {res.Functions[ea]}"); + else + res.Functions[ea] = new() { Name = $"{tn}.{method.Name}", Address = new(mfAttr.Signature), Signature = ExtractFuncSig(method, method.IsStatic ? "" : tn) }; + } + else if (method.GetCustomAttribute() is var saAttr && saAttr != null) + { + var ea = GetResolvedAddress(addresses, method.Name, resolver); + if (ea == 0) + Debug.WriteLine($"Failed to find global {type}.{method.Name}: sig={saAttr.Signature}+0x{saAttr.Offset}"); + else if (res.Globals.ContainsKey(ea)) + Debug.WriteLine($"Multiple globals resolve to same address 0x{ea:X}: {type}.{method.Name} sig={saAttr.Signature}+0x{saAttr.Offset} vs. {res.Globals[ea]}"); + else + res.Globals[ea] = new() { Type = saAttr.IsPointer ? tn + "*" : tn, Name = $"g_{tn}_{method.Name}", Address = new(saAttr.Signature, saAttr.Offset), Size = saAttr.IsPointer ? 8 : s.Size }; // note: name currently matches idarename + } + } + + var fields = type.GetFields().Where(f => !f.IsLiteral && !f.IsStatic && f.GetCustomAttribute() == null && f.GetCustomAttribute() == null); + int nextOff = 0; + int prevSize = 0; + foreach (var (f, off) in fields.Select(f => (f, f.GetFieldOffset())).OrderBy(pair => pair.Item2)) + { + if (off == 0 && f.Name == "VTable") + { + // this is not a particularly interesting field - just mark struct as having a vtable (if it has neither known vtable address nor known virtual functions) and continue + s.PrimaryVTable ??= new(); + continue; + } + + if (off < nextOff && (s.Fields.Count == 0 || off != nextOff - prevSize)) // first check covers a situation where previous field is a base + { + Debug.WriteLine($"Skipping field {type}.{f.Name} at offset 0x{off:X}: previous field ended at 0x{nextOff:X} (0x{nextOff - prevSize:X}+0x{prevSize:X})"); + continue; + } + + var ftype = f.FieldType; + int fsize = 0; + int arrLen = 0; + + var (fixedBufferElem, fixedBufferLength) = f.GetFixedBufferInfo(); + if (fixedBufferElem != null) + { + fsize = SizeOf(fixedBufferElem) * fixedBufferLength; + + var (fixedArrayElem, fixedArrayLength) = f.GetFixedArrayInfo(); + if (fixedArrayElem != null) + { + QueueType(fixedArrayElem, queue); + var fixedArraySize = SizeOf(fixedArrayElem) * fixedArrayLength; + if (fixedArraySize != fsize) + { + Debug.WriteLine($"Array size mismatch for {type}.{f.Name}: raw is {fixedBufferElem}[{fixedBufferLength}] (0x{fsize:X}), typed is {fixedArrayElem}[{fixedArrayLength}] (0x{fixedArraySize:X})"); + fsize = fixedArraySize; + } + + ftype = fixedArrayElem; + arrLen = fixedArrayLength; + } + else + { + ftype = fixedBufferElem; + arrLen = fixedBufferLength; + } + } + else + { + QueueType(ftype, queue); + fsize = SizeOf(f.FieldType); + } + bool isStruct = ftype.IsValueType && !ftype.IsPrimitive && !ftype.IsEnum && DerefPointer(ftype) == null; + + bool fieldCanBeBase = isStruct && s.Fields.Count == 0 && off == nextOff; // no gaps or fields between bases allowed + bool isBaseClass = f.GetCustomAttribute()?.IsBase ?? (fieldCanBeBase && off == 0 && ftype.Name == f.Name); // implicit base-class logic: single-inheritance, field name matches baseclass name + if (isBaseClass && !fieldCanBeBase) + { + Debug.WriteLine($"Field {type}.{f.Name} is marked as a base class, but can't be one"); + isBaseClass = false; + } + + if (isBaseClass) + s.Bases.Add(new() { Type = TypeName(ftype), Offset = off, Size = fsize }); + else + s.Fields.Add(new() { Name = f.Name, Type = TypeName(ftype), IsStruct = isStruct, Offset = off, ArrayLength = arrLen, Size = fsize }); + + if (off >= nextOff) + { + nextOff = off + fsize; + prevSize = fsize; + } + else + { + nextOff = Math.Max(nextOff, off + fsize); + prevSize = Math.Max(prevSize, fsize); + } + } + res.Structs.Add(tn, s); + } + } + + private string TypeName(Type type) + { + if (DerefPointer(type) is var derefType && derefType != null) + return TypeName(derefType) + "*"; + else if (type == typeof(void)) + return "void"; + else if (type == typeof(bool)) + return "bool"; + else if (type == typeof(char)) + return "char"; // note: despite c# char being a wchar, CS seems to use it as a normal char, go figure... + else if (type == typeof(sbyte)) + return "char"; + else if (type == typeof(byte)) + return "uchar"; + else if (type == typeof(short)) + return "short"; + else if (type == typeof(ushort)) + return "ushort"; + else if (type == typeof(int)) + return "int"; + else if (type == typeof(uint)) + return "uint"; + else if (type == typeof(long) || type == typeof(nint)) + return "__int64"; + else if (type == typeof(ulong) || type == typeof(nuint)) + return "unsigned __int64"; + else if (type == typeof(float)) + return "float"; + else if (type == typeof(double)) + return "double"; + else + return TypeNameComplex(type); + } + + private string TypeNameComplex(Type type) + { + var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs.").WithoutPrefix("FFXIV.").WithoutPrefix("Havok.").Replace(".", "::") ?? ""; + var leafName = type.Name; + if (type.IsGenericType) + { + leafName = leafName.Split('`')[0]; + if (!type.ContainsGenericParameters) + { + leafName += $"${string.Join("$", type.GetGenericArguments().Select(arg => TypeName(arg).Replace("*", "_ptr")))}$"; + } + } + var fullName = baseName.Length > 0 ? $"{baseName}::{leafName}" : leafName; + + // hack for std + if (fullName.StartsWith("STD::Std")) + { + fullName = fullName.WithoutPrefix("STD::Std"); + fullName = "std::"+ fullName.Substring(0, 1).ToLower() + fullName.Substring(1); + } + return fullName; + } + + private int SizeOf(Type type) + { + if (DerefPointer(type) != null) + return 8; // assume 64-bit + // Marshal.SizeOf doesn't work correctly because the assembly is unmarshaled, and more specifically, it sets bools as 4 bytes long... + return (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0; + } + + private (int, bool) GetEnumSignWidth(Type enumType) + { + var underlying = enumType.GetEnumUnderlyingType(); + if (underlying == typeof(sbyte)) + return (1, true); + else if (underlying == typeof(byte)) + return (1, false); + else if (underlying == typeof(short)) + return (2, true); + else if (underlying == typeof(ushort)) + return (2, false); + else if (underlying == typeof(int)) + return (4, true); + else if (underlying == typeof(uint)) + return (4, false); + else if (underlying == typeof(long)) + return (8, true); + else if (underlying == typeof(ulong)) + return (8, false); + else + throw new Exception($"Unsupported underlying enum type {underlying} for {enumType}"); + } + + private Type? DerefPointer(Type type) => type.IsPointer ? type.GetElementType()! : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Pointer<>) ? type.GetGenericArguments()[0] : null; + + private Result.FuncSig ExtractFuncSig(MethodInfo m, string thisType) + { + var res = new Result.FuncSig() { RetType = TypeName(m.ReturnType), Arguments = m.GetParameters().Select(p => new Result.FuncArg() { Type = TypeName(p.ParameterType), Name = p.Name ?? "" }).ToList() }; + if (thisType.Length > 0) + res.Arguments.Insert(0, new() { Type = thisType + "*", Name = "this" }); + return res; + } + + private ulong GetResolvedAddress(Type? addresses, string name, SigResolver resolver) + { + var addr = addresses?.GetField(name, BindingFlags.Static | BindingFlags.Public)?.GetValue(null) as Address; + var res = addr != null ? resolver.ToEA(addr) : 0; + if (res == 0) + Debug.WriteLine($"Failed to resolve address for {addresses?.FullName}.{name}"); + return res; + } +} diff --git a/idapopulate/idapopulate/DataYmlImport.cs b/idapopulate/idapopulate/DataYmlImport.cs new file mode 100644 index 0000000..3f5bc3a --- /dev/null +++ b/idapopulate/idapopulate/DataYmlImport.cs @@ -0,0 +1,169 @@ +using System.Diagnostics; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace idapopulate; + +internal class DataYmlImport +{ + private class DataYmlClassInstance + { + public ulong Ea = 0; + public bool Pointer = true; // not sure what is the correct default, but this at least is safer... + public string Name = "Instance"; + } + + private class DataYmlClassVtbl + { + public ulong Ea = 0; + public string Base = ""; + } + + private class DataYmlClass + { + public List Instances = new(); + public List Vtbls = new(); + public SortedDictionary Vfuncs = new(); // index -> name + public SortedDictionary Funcs = new(); // ea -> name + } + + private class DataYml + { + public string Version = ""; + public SortedDictionary Globals = new(); // ea -> name + public SortedDictionary Functions = new(); // ea -> name + public SortedDictionary Classes = new(); // name -> data + } + + public void Populate(Result res, FileInfo fi) + { + var data = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance).Build().Deserialize(File.ReadAllText(fi.FullName)); + foreach (var (ea, name) in data.Globals) + PopulateGlobal(res, ea, name, "", 0); + foreach (var (ea, name) in data.Functions) + PopulateFunction(res, ea, name); + + // we process classes in several passes: + // 1. ensure all struct entries exist for each class defined in yml + foreach (var name in data.Classes.Keys.Select(FixClassName)) + if (!res.Structs.ContainsKey(name)) + res.Structs[name] = new(); + // 2. process class data + foreach (var (name, cls) in data.Classes) + PopulateClass(res, FixClassName(name), cls ?? new()); // note: some class entries are completely empty + } + + + private static string FixClassName(string name) => name.Replace('<', '$').Replace('>', '$').Replace(',', '$').Replace("*", "_ptr"); + + private void PopulateGlobal(Result res, ulong ea, string name, string type, int size) + { + var match = res.Globals.GetValueOrDefault(ea); + if (match != null) + { + // we assume that if CS defines a global, it always has known type & size + // any conflicts between CS and yaml are resolved for CS + if (name != match.Name) + Debug.WriteLine($"Name mismatch for global @ 0x{ea:X}: CS={match.Name}, yml={name}"); + if (type.Length > 0 && type != match.Type) + Debug.WriteLine($"Type mismatch for global @ 0x{ea:X}: CS={match.Type}, yml={type}"); + if (size != 0 && size != match.Size) + Debug.WriteLine($"Size mismatch for global @ 0x{ea:X}: CS={match.Size}, yml={size}"); + } + else + { + res.Globals[ea] = new() { Type = type, Name = name, Size = size }; + } + } + + private void PopulateFunction(Result res, ulong ea, string name) + { + var match = res.Functions.GetValueOrDefault(ea); + if (match != null) + { + // any conflicts between CS and yaml are resolved for CS + // TODO: consider overriding CS names if yml has mangled function name (starting from ?) + if (name != match.Name) + Debug.WriteLine($"Name mismatch for function @ 0x{ea:X}: CS={match.Name}, yml={name}"); + } + else + { + res.Functions[ea] = new() { Name = name }; + } + } + + private void PopulateClass(Result res, string name, DataYmlClass data) + { + var s = res.Structs[name]; + + // add placeholder for missing bases + foreach (var vt in data.Vtbls.Where(vt => vt.Base.Length > 0)) + { + var baseName = FixClassName(vt.Base); + if (!res.Structs.ContainsKey(baseName)) + { + Debug.WriteLine($"Base class {baseName} for {name} is not defined"); + res.Structs[baseName] = new() { Size = 8, PrimaryVTable = new() }; // assume base also contains a vtable + } + } + + var primaryVT = data.Vtbls.FirstOrDefault(); + if (primaryVT != null) + { + if (s.PrimaryVTable == null) + { + s.PrimaryVTable = new() { Ea = primaryVT.Ea }; + s.Size = Math.Max(s.Size, 8); + } + else if (s.PrimaryVTable.Ea == 0) + { + s.PrimaryVTable.Ea = primaryVT.Ea; + } + else if (s.PrimaryVTable.Ea != primaryVT.Ea) + { + Debug.WriteLine($"Primary VT address mismatch for {name}: CS=0x{s.PrimaryVTable.Ea}, yml={primaryVT.Ea}"); + } + + if (primaryVT.Base.Length == 0) + { + // nothing to do here, yml doesn't define a base here, if CS defines one - so be it + } + else if (s.Bases.Count == 0) + { + // yml defines a base that CS doesn't know about - synthesize one + var primaryBase = FixClassName(primaryVT.Base); + s.Bases.Add(new() { Type = primaryBase, Size = res.Structs[primaryBase].Size }); + } + else if (FixClassName(primaryVT.Base) is var baseName && baseName != s.Bases[0].Type) + { + Debug.WriteLine($"Main base mismatch for {name}: CS={s.Bases[0].Type}, yml={baseName}"); + } + } + + foreach (var secondaryVT in data.Vtbls.Skip(1)) + { + if (secondaryVT.Base.Length == 0) + Debug.WriteLine($"Unexpected null secondary base name in yml for {name}"); + else if (s.Bases.Count == 0) + Debug.WriteLine($"Class {name} has no known primary base, but has secondary base {secondaryVT.Base}"); + else + s.SecondaryVTables.Add(new() { Ea = secondaryVT.Ea, Base = FixClassName(secondaryVT.Base) }); + } + + foreach (var (index, fname) in data.Vfuncs) + { + s.PrimaryVTable ??= new(); + s.Size = Math.Max(s.Size, 8); // ensure we can fit vtable + if (!s.PrimaryVTable.VFuncs.ContainsKey(index)) + s.PrimaryVTable.VFuncs[index] = new() { Name = fname }; + else if (s.PrimaryVTable.VFuncs[index].Name != fname) + Debug.WriteLine($"VF name mismatch for {name} #{index}: CS={s.PrimaryVTable.VFuncs[index].Name}, yml={fname}"); + } + + foreach (var (ea, fname) in data.Funcs) + PopulateFunction(res, ea, fname.StartsWith('?') ? fname : $"{name}.{fname}"); + + foreach (var inst in data.Instances) + PopulateGlobal(res, inst.Ea, inst.Name.StartsWith('?') ? inst.Name : $"g_{name}_{inst.Name}", inst.Pointer ? name + "*" : name, inst.Pointer ? 8 : s.Size); + } +} diff --git a/idapopulate/idapopulate/PathUtils.cs b/idapopulate/idapopulate/PathUtils.cs new file mode 100644 index 0000000..9b341a1 --- /dev/null +++ b/idapopulate/idapopulate/PathUtils.cs @@ -0,0 +1,49 @@ +using Microsoft.Win32; +using System.Reflection; + +namespace idapopulate; + +internal static class PathUtils +{ + public static string FindGameRoot() + { + // stolen from FFXIVLauncher/src/XIVLauncher/AppUtil.cs + foreach (var registryView in new RegistryView[] { RegistryView.Registry32, RegistryView.Registry64 }) + { + using (var hklm = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, registryView)) + { + // Should return "C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\boot\ffxivboot.exe" if installed with default options. + using (var subkey = hklm.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\{2B41E132-07DF-4925-A3D3-F2D1765CCDFE}")) + { + if (subkey != null && subkey.GetValue("DisplayIcon", null) is string path) + { + // DisplayIcon includes "boot\ffxivboot.exe", need to remove it + var basePath = Directory.GetParent(path)?.Parent?.FullName; + if (basePath != null) + { + var gamePath = Path.Join(basePath, "game"); + if (Directory.Exists(gamePath)) + { + return gamePath; + } + } + } + } + } + } + return "D:\\installed\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game"; + } + + public static FileInfo? FindFileAmongParents(string suffix) + { + var dir = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory; + while (dir != null) + { + var yml = new FileInfo(dir.FullName + suffix); + if (yml.Exists) + return yml; + dir = dir.Parent; + } + return null; + } +} diff --git a/idapopulate/idapopulate/Program.cs b/idapopulate/idapopulate/Program.cs new file mode 100644 index 0000000..f0da2b1 --- /dev/null +++ b/idapopulate/idapopulate/Program.cs @@ -0,0 +1,39 @@ +using idapopulate; +using System.Diagnostics; + +var outDir = PathUtils.FindFileAmongParents("/idbtoolkit/populate_idb.py")?.Directory; +if (outDir == null) +{ + Debug.WriteLine("Failed to find output location"); + return; +} + +var testRes = new Result(); +var testEnum1 = new Result.Enum() { IsSigned = true, Width = 2 }; +testEnum1.Values.Add(new() { Name = "E1V1", Value = 1 }); +testEnum1.Values.Add(new() { Name = "E1V1dup", Value = 1 }); +testEnum1.Values.Add(new() { Name = "E1V2", Value = 2 }); +testEnum1.Values.Add(new() { Name = "E1V3", Value = 10 }); +testRes.Enums["test::Enum1"] = testEnum1; +var testEnum2 = new Result.Enum() { IsBitfield = true, IsSigned = false, Width = 4 }; +testEnum2.Values.Add(new() { Name = "E2V1", Value = 0x1 }); +testEnum2.Values.Add(new() { Name = "E2V2", Value = 0x4 }); +testRes.Enums["test::Enum2"] = testEnum2; +testRes.Enums["test::Enum3"] = new() { Width = 1 }; +testRes.Write(outDir.FullName + "/test.yml"); + +var gameRoot = PathUtils.FindGameRoot(); +var resolver = new SigResolver(gameRoot + "\\ffxiv_dx11.exe"); + +var res = new Result(); +new CSImport().Populate(res, resolver); + +var dataYml = PathUtils.FindFileAmongParents("/FFXIVClientStructs/ida/data.yml"); +if (dataYml != null) + new DataYmlImport().Populate(res, dataYml); +else + Debug.WriteLine("Failed to find data.yml"); + +res.ValidateUniqueEaName(); +res.ValidateLayout(); +res.Write(outDir.FullName + "/info.yml"); diff --git a/idapopulate/idapopulate/Result.cs b/idapopulate/idapopulate/Result.cs new file mode 100644 index 0000000..fb7f57e --- /dev/null +++ b/idapopulate/idapopulate/Result.cs @@ -0,0 +1,211 @@ +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization; +using System.Diagnostics; + +namespace idapopulate; + +// assumption about vtables: +// - we don't support virtual bases (for now?) +// - trivial: a struct can have 0 vtables - if neither it nor its bases have virtual functions +// - trivial: a struct with virtual functions has a vptr at offset 0; a struct with a base has base at offset 0; vptr is also natually at offset 0 +// in IDA we model that as a 'baseclass' special field (and no vptr field) -or- explicit vptr field at offset 0 +// - edge case: if a base has no virtual functions, but derived has - derived will have vptr at offset 0 and base at offset 8 +// I don't think IDA really supports this, so we model it as a field rather than a baseclass +// - trivial: any new vfuncs in derived are added to the vtable - we model that as derived-vtable having base-vtable as a base class +// - trivial: if there are several bases, these are in order +// in IDA we model that as several 'baseclass' special fields ('baseclass_X', where X is offset) +// - tricky: any new vfuncs are added only to the main vtable (at offset 0); so we don't need to create extra types for vtables from secondary bases (these contain thunks) +internal class Result +{ + public class EnumValue + { + public string Name = ""; + public long Value; + + public override string ToString() => $"{Name} = 0x{Value:X}"; + } + + public class Enum + { + public bool IsBitfield; + public bool IsSigned; + public int Width; + public List Values = new(); // sorted by value + + public override string ToString() => $"{(IsSigned ? "signed" : "unsigned")} {Width}-byte-wide {(IsBitfield ? "bitfield" : "enum")}"; + } + + public class Address + { + public string Sig; + public int SigOffset; + + public Address(string sig, int sigOffset = 0) + { + Sig = sig; + SigOffset = sigOffset; + } + + public override string ToString() => $"'{Sig}' +{SigOffset}"; + } + + public class FuncArg + { + public string Type = ""; + public string Name = ""; + + public override string ToString() => $"{Type} {Name}"; + } + + public class FuncSig + { + public string RetType = ""; + public List Arguments = new(); + + public override string ToString() => $"({string.Join(", ", Arguments)}) -> {RetType}"; + } + + public class Function + { + public string Name = ""; + public Address? Address; + public FuncSig? Signature; + + public override string ToString() => $"auto {Name}{Signature} @ {Address}"; + } + + public class VTable + { + public ulong Ea; + public Address? Address; + public SortedDictionary VFuncs = new(); + + public override string ToString() => $"0x{Ea:X} {Address}"; + } + + public class SecondaryVTable + { + public ulong Ea; + public string Base = ""; + + public override string ToString() => $"0x{Ea:X} {Base}"; + } + + public class StructBase + { + public string Type = ""; + public int Offset; + public int Size; + + public override string ToString() => $"[0x{Offset:X}] {Type} (size=0x{Size:X})"; + } + + public class StructField + { + public string Name = ""; + public string Type = ""; + public bool IsStruct; // if true, type is another struct that has to be defined before this struct + public int Offset; + public int ArrayLength; // 0 if not an array + public int Size; + + public override string ToString() => $"[0x{Offset:X}] {Type} {Name}{(ArrayLength > 0 ? $"[{ArrayLength}]" : "")}{(IsStruct ? " (struct)" : "")} (size=0x{Size:X})"; + } + + public class Struct + { + public int Size; + public VTable? PrimaryVTable; + public List SecondaryVTables = new(); + public List Bases = new(); // sorted by offset + public List Fields = new(); // sorted by offset, non overlapping with bases + + public override string ToString() => $"size=0x{Size:X}"; + } + + public class Global + { + public string Type = ""; + public string Name = ""; + public Address? Address; + public int Size; + + public override string ToString() => $"{Type} {Name} (size=0x{Size:X}) @ {Address}"; + } + + public SortedDictionary Enums = new(); + public SortedDictionary Structs = new(); + public SortedDictionary Globals = new(); // key = ea + public SortedDictionary Functions = new(); // key = ea + + public void Write(string path) + { + var data = new SerializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance).Build().Serialize(this); + File.WriteAllText(path, data); + } + + public bool ValidateUniqueEaName() + { + Dictionary uniqueEAs = new(); + Dictionary uniqueNames = new(); + Func validate = (ea, name, text) => + { + bool success = true; + if (ea != 0 && !uniqueEAs.TryAdd(ea, text)) + { + Debug.WriteLine($"Duplicate EA 0x{ea:X}: {text} and {uniqueEAs[ea]}"); + success = false; + } + if (name.Length > 0 && !uniqueNames.TryAdd(name, text)) + { + Debug.WriteLine($"Duplicate name {name}: {text} and {uniqueNames[name]}"); + success = false; + } + return success; + }; + + bool success = true; + foreach (var (ea, g) in Globals) + success &= validate(ea, g.Name, $"global {g.Name}"); + foreach (var (ea, f) in Functions) + success &= validate(ea, f.Name, $"function {f.Name}"); + foreach (var (name, s) in Structs) + { + if (s.PrimaryVTable != null) + success &= validate(s.PrimaryVTable.Ea, $"vtbl_{name}", $"vtbl_{name}"); + foreach (var vt in s.SecondaryVTables) + success &= validate(vt.Ea, $"vtbl_{name}__{vt.Base}", $"vtbl_{name}__{vt.Base}"); + } + return success; + } + + public bool ValidateLayout() + { + bool success = true; + foreach (var (name, s) in Structs) + { + int minFieldOffset = 0; + foreach (var b in s.Bases) + { + if (b.Offset != minFieldOffset) + { + Debug.WriteLine($"Structure {name} has incorrect placement for base {b}"); + success = false; + } + minFieldOffset = b.Offset + b.Size; + } + + if (s.PrimaryVTable != null) + { + minFieldOffset = Math.Max(minFieldOffset, 8); + } + + if (s.Fields.Count > 0 && s.Fields[0].Offset < minFieldOffset) + { + Debug.WriteLine($"Structure {name} has first field {s.Fields[0]} overlapping bases or vtable"); + success = false; + } + } + return success; + } +} diff --git a/idapopulate/idapopulate/SigResolver.cs b/idapopulate/idapopulate/SigResolver.cs new file mode 100644 index 0000000..d8924d5 --- /dev/null +++ b/idapopulate/idapopulate/SigResolver.cs @@ -0,0 +1,42 @@ +using FFXIVClientStructs.Interop; +using System.Reflection.PortableExecutable; + +namespace idapopulate; + +internal class SigResolver +{ + private ulong _baseAddress; + private nint _resolverBase; + private SectionHeader _text; + private SectionHeader _data; + private SectionHeader _rdata; + + public unsafe SigResolver(string exePath, ulong baseAddress = 0x140000000) + { + _baseAddress = baseAddress; + + var contents = File.ReadAllBytes(exePath); + var headers = new PEHeaders(new MemoryStream(contents)); + _text = headers.SectionHeaders.First(h => h.Name == ".text"); + _data = headers.SectionHeaders.First(h => h.Name == ".data"); + _rdata = headers.SectionHeaders.First(h => h.Name == ".rdata"); + fixed (byte* p = contents) + { + _resolverBase = (nint)p; + Resolver.GetInstance.SetupSearchSpace(_resolverBase, contents.Length, _text.PointerToRawData, _text.SizeOfRawData, _data.PointerToRawData, _data.SizeOfRawData, _rdata.PointerToRawData, _rdata.SizeOfRawData); + Resolver.GetInstance.Resolve(); + } + } + + public ulong ToEA(Address address) + { + if (address.Value == 0) + return 0; + // note: looking at the resolver code, any found addresses are relative to text section + return _baseAddress + (ulong)_text.VirtualAddress + (ulong)((nint)address.Value - _resolverBase - _text.PointerToRawData); + //var rva = (nint)address.Value - _resolverBase; + //return ToSectionEA(rva, _text) ?? ToSectionEA(rva, _data) ?? ToSectionEA(rva, _rdata) ?? throw new Exception("Weird resolved address"); + } + + //private ulong? ToSectionEA(nint rva, SectionHeader header) => rva >= header.PointerToRawData && rva < header.PointerToRawData + header.SizeOfRawData ? _baseAddress + (ulong)header.VirtualAddress + (ulong)(rva - header.PointerToRawData) : null; +} diff --git a/idapopulate/idapopulate/idapopulate.csproj b/idapopulate/idapopulate/idapopulate.csproj new file mode 100644 index 0000000..33fa7c8 --- /dev/null +++ b/idapopulate/idapopulate/idapopulate.csproj @@ -0,0 +1,19 @@ + + + + Exe + net7.0-windows + enable + enable + true + + + + + + + + + + + diff --git a/idbtoolkit/ii.py b/idbtoolkit/ii.py new file mode 100644 index 0000000..212a0df --- /dev/null +++ b/idbtoolkit/ii.py @@ -0,0 +1,209 @@ +# This is a wrapper for IDA API, it is quite inconvenient to use... +import idaapi +import ida_bytes +import ida_enum +import ida_funcs +import ida_lines +import ida_nalt +import ida_name +import ida_struct +import ida_typeinf +import ida_xref +import ida_ua +import datetime +from contextlib import contextmanager + +# enumerate (ea, name) tuples for all global names in database +def enumerate_names(): + for i in range(ida_name.get_nlist_size()): + yield (ida_name.get_nlist_ea(i), ida_name.get_nlist_name(i)) + +# find EA of a global name; return idaapi.BADADDR on failure +def find_global_name_ea(name): + return ida_name.get_name_ea(idaapi.BADADDR, name) + +# check whether we have a data offset at given EA +def ea_has_data_offset(ea): + flags = ida_bytes.get_full_flags(ea) + return (flags & ida_bytes.FF_DATA) != 0 and ida_bytes.is_off0(flags) + +# get instruction at EA; return None or insn_t instance +def decode_instruction(ea): + insn = ida_ua.insn_t() + ilen = ida_ua.decode_insn(insn, ea) + return insn if ilen > 0 else None + +# get instruction's operand as immediate +def get_instruction_operand_immediate(ea, opIndex): + insn = decode_instruction(ea) + return insn.ops[opIndex].addr if insn else None + +# given EA of a call instruction, return EAs of instructions setting arguments in proper order +def get_call_argument_assignment_eas(call_ea): + return idaapi.get_arg_addrs(call_ea) + +# enumerate xrefs to address +def enumerate_xrefs_to(ea): + addr = ida_xref.get_first_cref_to(ea) + while addr != idaapi.BADADDR: + yield addr + addr = ida_xref.get_next_cref_to(ea, addr) + +# create typeinfo from cdecl; return None on failure +# cdecl should not have a trailing semicolon, it is added automatically +def parse_cdecl(cdecl): + tif = ida_typeinf.tinfo_t() + return tif if ida_typeinf.parse_decl(tif, None, cdecl + ';', 0) == '' else None + +# speed up mass creation of enums or structs +@contextmanager +def mass_type_updater(utpFlag): + print(f'{datetime.datetime.now()} Starting mass type updates: {utpFlag}') + ida_typeinf.begin_type_updating(utpFlag) + try: + yield None + finally: + print(f'{datetime.datetime.now()} Comitting mass type updates: {utpFlag}') + ida_typeinf.end_type_updating(utpFlag) + print(f'{datetime.datetime.now()} Done with mass type updates: {utpFlag}') + +# create enum type; returns idaapi.BADADDR on failure +def add_enum(name, bitfield, signed, width): + # flags seem to be undocumented, here's what I found: + # signed is 0x20000 + # hexa is FF_0NUMH | FF_1NUMH (0x1100000), decimal is FF_0NUMD | FF_1NUMD (0x2200000), octal is FF_0NUMO | FF_1NUMO (0x7700000), binary is FF_0NUMB | FF_1NUMB (0x6600000), character is FF_0CHAR | FF_1CHAR (0x3300000) + flags = 0x1100000 + if signed: + flags |= 0x20000 + eid = ida_enum.add_enum(idaapi.BADADDR, name, flags) + if eid != idaapi.BADADDR: + ida_enum.set_enum_bf(eid, bitfield) + ida_enum.set_enum_width(eid, width) + return eid + +# get structure by name +def get_struct_by_name(name): + return ida_struct.get_struc(ida_struct.get_struc_id(name)) + +# create struct type; returns struc_t or None on failure +def add_struct(name, isUnion = False): + sid = ida_struct.add_struc(idaapi.BADADDR, name, isUnion) + return ida_struct.get_struc(sid) if sid != idaapi.BADADDR else None + +# create structure member of primitive types +def add_struct_member_primitive(s, offset, name, flag, size): + return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | flag, None, size) == 0 + +def add_struct_member_byte(s, offset, name, arraySize = 1): + return add_struct_member_primitive(s, offset, name, ida_bytes.FF_BYTE, arraySize) + +def add_struct_member_word(s, offset, name, arraySize = 1): + return add_struct_member_primitive(s, offset, name, ida_bytes.FF_WORD, arraySize * 2) + +def add_struct_member_dword(s, offset, name, arraySize = 1): + return add_struct_member_primitive(s, offset, name, ida_bytes.FF_DWORD, arraySize * 4) + +def add_struct_member_qword(s, offset, name, arraySize = 1): + return add_struct_member_primitive(s, offset, name, ida_bytes.FF_QWORD, arraySize * 8) + +# create structure member of pointer type +def add_struct_member_ptr(s, offset, name, arraySize = 1): + opinfo = ida_nalt.opinfo_t() + opinfo.ri = ida_nalt.refinfo_t() + opinfo.ri.base = opinfo.ri.target = idaapi.BADADDR + opinfo.ri.flags = ida_nalt.REF_OFF64 + return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | ida_bytes.FF_QWORD | ida_bytes.FF_0OFF | ida_bytes.FF_1OFF, opinfo, arraySize * 8) == 0 + +# create structure member of enumeration type +def add_struct_member_enum(s, offset, name, type, arraySize = 1): + opinfo = ida_nalt.opinfo_t() + opinfo.tid = ida_enum.get_enum(type) + if opinfo.tid == idaapi.BADADDR: + return False + w = ida_enum.get_enum_width(opinfo.tid) + if w == 1: + flag = ida_bytes.FF_BYTE + elif w == 2: + flag = ida_bytes.FF_WORD + elif w == 8: + flag = ida_bytes.FF_QWORD + else: + flag = ida_bytes.FF_DWORD # default + return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | flag | ida_bytes.FF_0ENUM | ida_bytes.FF_1ENUM, opinfo, w * arraySize) == 0 + +# create structure member that is an instance of a different structure; returns success +def add_struct_member_substruct(s, offset, name, type, arraySize = 1): + opinfo = ida_nalt.opinfo_t() + opinfo.tid = ida_struct.get_struc_id(type) + size = ida_struct.get_struc_size(opinfo.tid) if opinfo.tid != idaapi.BADADDR else 0 + if size == 0: # note: adding 0-sized struct field leads to problems + return False + return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | ida_bytes.FF_STRUCT, opinfo, size * arraySize) == 0 + +# create base class field for a structure +def add_struct_baseclass(s, type): + offset = ida_struct.get_struc_size(s) + success = add_struct_member_substruct(s, offset, f'baseclass_{hex(offset)[2:]}', type) + if success: + ida_struct.get_member(s, offset).props |= ida_struct.MF_BASECLASS + # TODO: we might need to call save_struc here... + return success + +# create structure member of custom type +def add_struct_member_typed(s, offset, name, type): + return add_struct_member_byte(s, offset, name) and set_struct_member_by_offset_type(s, offset, type) + +def get_struct_member_tinfo(m): + tif = ida_typeinf.tinfo_t() + return tif if ida_struct.get_member_tinfo(tif, m) else None + +# set structure member type info +def set_struct_member_tinfo(s, m, tif): + # note: SET_MEMTI_* flags + return ida_struct.set_member_tinfo(s, m, 0, tif, 0) > 0 + +# set structure member type from string +def set_struct_member_type(s, m, type): + tif = parse_cdecl(type) + return set_struct_member_tinfo(s, m, tif) if tif else False + +# set structure member type +def set_struct_member_by_offset_type(s, offset, type): + m = ida_struct.get_member(s, offset) + return set_struct_member_type(s, m, type) if m else False + +# add comment; if one already exists, append as new line (unless existing comment already contains what we're trying to add) +def add_comment_enum(id, comment, repeatable = False): + existing = ida_enum.get_enum_cmt(id, repeatable) + if not existing or comment not in existing: + ida_enum.set_enum_cmt(id, f'{existing}\n{comment}' if existing else comment, repeatable) + +def add_comment_func(func, comment, repeatable = False): + existing = ida_funcs.get_func_cmt(func, repeatable) + if not existing or comment not in existing: + ida_funcs.set_func_cmt(func, f'{existing}\n{comment}' if existing else comment, repeatable) + +def add_comment_inline(ea, comment, repeatable = False): + existing = ida_bytes.get_cmt(ea, repeatable) + if not existing or comment not in existing: + ida_bytes.set_cmt(ea, f'{existing}\n{comment}' if existing else comment, repeatable) + +def add_comment_outline(ea, comment, posterior = False): + line = ida_lines.E_NEXT if posterior else ida_lines.E_PREV + while True: + existing = ida_lines.get_extra_cmt(ea, line) + if not existing: + ida_lines.add_extra_cmt(ea, not posterior, comment) + return + elif comment in existing: + return + else: + line += 1 + +# func or outline +def add_comment_ea_auto(ea, comment): + func = ida_funcs.get_func(ea) + if func: + add_comment_func(func, comment) + else: + add_comment_outline(ea, comment) diff --git a/idbtoolkit/populate_idb.py b/idbtoolkit/populate_idb.py new file mode 100644 index 0000000..02b0352 --- /dev/null +++ b/idbtoolkit/populate_idb.py @@ -0,0 +1,554 @@ +import idaapi +import ida_bytes +import ida_enum +import ida_name +import ida_nalt +import ida_funcs +import ida_xref +import ida_typeinf +import ida_struct +import ida_funcs +import yaml +import os +import collections +import datetime +import itertools + +# not import to simplify development - we want to reload my modules if they change +idaapi.require('ii') +import ii + +def msg(text): + print(f'{datetime.datetime.now()} {text}') + +def has_custom_name(ea): + name = ida_name.get_ea_name(ea) + if not ida_name.is_uname(name): + return False # it's an officially IDA dummy name (see https://hex-rays.com/blog/igors-tip-of-the-week-34-dummy-names/) + if name.startswith('nullsub_'): + return False # nullsub_ is not considered a dummy, but it's autogenerated nonetheless + if name.startswith('staticinit_'): + return False # staticinit_ is our custom prefix which is functionally a dummy + if len(name) >= 2 and name[0] == 'a': + byte = ida_bytes.get_byte(ea) + if byte < 0x80 and name[1] == chr(byte).upper(): + return False # string constant: foo => aFoo + return True + +def set_custom_name(ea, name, sig): + if not has_custom_name(ea): + dupEA = ii.find_global_name_ea(name) + if dupEA != idaapi.BADADDR: + msg(f'Skipping rename for {hex(ea)}: same name {name} is already used by {hex(dupEA)}') + ii.add_comment_ea_auto(ea, f'duplicate name: {name}') + elif ida_name.set_name(ea, name) == 0: + # no need to write a message, we get a messagebox anyway... + ii.add_comment_ea_auto(ea, f'rename failed: {name}') + elif ida_name.get_ea_name(ea) != name: + msg(f'Skipping rename for existing name: {hex(ea)} {ida_name.get_ea_name(ea)} -> {name}') + ii.add_comment_ea_auto(ea, f'alt name: {name}') + if sig: + ii.add_comment_ea_auto(ea, f'signature: {sig["sig"]} +{sig["sigOffset"]}') + +# ensure specified ea is a function start; returns success (either function already existed or was created) +def ensure_function_at(ea, message): + if ida_funcs.get_func(ea): + return True + added = ida_funcs.add_func(ea) + if message: + msg(f'Created new {message} at {hex(ea)}' if added else f'Failed to create {message} at {hex(ea)}') + return added + +def calc_vtable_length(ea): + # assume that length is from start until next name + end = ea + 8 + while ida_name.get_ea_name(end) == '' and ii.ea_has_data_offset(end): + end += 8 + return (end - ea) >> 3 + +def convert_exported_sig(sig, name = None): + args = ', '.join(f'{a["type"]} {a["name"]}' for a in sig['arguments']) + return f'{sig["retType"]} {name if name else "func"}({args})' + +def apply_function_type(ea, tinfo): + func = ida_funcs.get_func(ea) + if not func: + raise Exception(f'Function not defined') + existing = ida_typeinf.print_type(ea, 0) + if existing: + msg(f'Skipping function type assignment @ {hex(ea)}: {existing} -> {tinfo}') + ii.add_comment_func(func, f'alt sig: {tinfo}') + elif not ida_typeinf.apply_tinfo(ea, tinfo, 0): + raise Exception(f'Apply failed') + +# return offset of the base of specified type, or None if not found +def find_base_offset(structId, baseId): + s = ida_struct.get_struc(structId) + if not s or baseId == idaapi.BADADDR: + return None + offset = 0 + while True: + m = ida_struct.get_member(s, offset) + if not m or ida_struct.get_member_name(m.id) != f'baseclass_{offset}': + return None + op = ida_nalt.opinfo_t() + ida_struct.retrieve_member_info(op, m) + nestedOffset = 0 if op.tid == baseId else find_base_offset(op.tid, baseId) + if nestedOffset != None: + return offset + nestedOffset + offset += ida_struct.get_member_size(m) + +def populate_static_initializers(): + msg('*** Populating static initializers ***') + + def find_main_iniiterm(): + eaInitterm = ii.find_global_name_ea('_initterm') + if eaInitterm == idaapi.BADADDR: + raise Exception('Failed to find _initterm address') + # there are several _initterm calls, interesting one is from main, others are purely framework ones + mainXrefs = [xref for xref in ii.enumerate_xrefs_to(eaInitterm) if ida_funcs.get_func_name(xref).startswith('?__scrt_common_main')] + if len(mainXrefs) != 1: + raise Exception(f'Found {len(mainXrefs)} calls to _initterm from main, 1 expected') + return mainXrefs[0] + + def parse_initterm_arguments(callEA): + # assume arguments are of the form 'lea rcx/rdx, addr' + args = ii.get_call_argument_assignment_eas(callEA) + if len(args) != 2: + raise Exception(f'Unexpected args for _initterm call: 2 expected, got {len(args)}') + start = ii.get_instruction_operand_immediate(args[0], 1) + end = ii.get_instruction_operand_immediate(args[1], 1) + # both start and end have 0's, between them have function pointers + if start + 8 * calc_vtable_length(start) != end: + raise Exception('Unexpected _initterm table contents') + return (start, end) + + try: + eaInittermCall = find_main_iniiterm() + start, end = parse_initterm_arguments(eaInittermCall) + for ea in range(start + 8, end, 8): + func = ida_bytes.get_qword(ea) + ensure_function_at(func, "static initializer") + if not has_custom_name(func): + ida_name.set_name(func, f'staticinit_{(ea - start) >> 3}') + except Exception as e: + msg(f'Static initializer error: {e}') + +def populate_global_names(data): + msg('** Populating exported global names **') + for ea, g in data.items(): + set_custom_name(ea, g['name'], g['address']) + +def populate_function_names(data): + msg('** Populating exported function names **') + for ea, g in data.items(): + ensure_function_at(ea, "function") # if function is not referenced from anywhere, define one manually + set_custom_name(ea, g['name'], g['address']) + +def populate_enums(data): + msg('** Populating exported enums **') + + def populate_enum(name, isBitfield, isSigned, width, values): + if ida_enum.get_enum(name) != idaapi.BADADDR: + raise Exception(f'{name} already exists in database') + eid = ii.add_enum(name, isBitfield, isSigned, width) + if (eid == idaapi.BADADDR): + raise Exception(f'Failed to create {name}') + for val in values: + en = val['name'] + ev = val['value'] + qn = f'{name}.{en}' # enum names in ida are global, so qualify them + res = ida_enum.add_enum_member(eid, qn, ev, ev if isBitfield else -1) + if res != 0: + msg(f'Failed to add enum member {name}.{en} = {ev}: {res}') + ii.add_comment_enum(eid, f'could not add field {en} = {ev}') + + with ii.mass_type_updater(ida_typeinf.UTP_ENUM): + for name, e in data.items(): + try: + populate_enum(name, e['isBitfield'], e['isSigned'], e['width'], e['values']) + except Exception as e: + msg(f'Enum error: {e}') + +def populate_vtables(data): + # vtable population is done in several passes + Vtable = collections.namedtuple("VTable", "primaryEA secondaryEAs vFuncs base") + vtables = {} # base always ordered before derived; key = class name + + # first pass: build an ordered set of classes with vtables (base before derived) and assign names for all known addresses + # note that we don't immediately calculate vtable size on the off chance that some of the vtables-to-be-renamed has no known xrefs + def pass1(): + msg('** Populating exported vtables: pass 1 **') + def populate_vtable(cname): + if cname in vtables: + return # class already processed, since it's a base of some earlier-defined class + cdata = data[cname] + primary = cdata['primaryVTable'] + secondary = cdata['secondaryVTables'] + bases = cdata['bases'] + # ensure we process all bases first - both direct and indirect (from secondary vtables) - we might not have correct inheritance chain set up + for b in bases: + populate_vtable(b['type']) + for v in secondary: + populate_vtable(v['base']) + # if primary base has vtable, this class should have one too + # TODO: this should be handled during generation + primaryBase = bases[0]['type'] if len(bases) > 0 else None + if primaryBase and primaryBase in vtables and not primary: + msg(f'Class {cname} has no primary vtable, but has base {bases[0]["type"]} with one') + primary = { 'ea': 0, 'address': None, 'vFuncs': [] } + if not primary: + return # skip, this class has no vtables + primaryEA = primary['ea'] + if primaryEA != 0: + set_custom_name(primaryEA, f'vtbl_{cname}', primary['address']) + for v in secondary: + secEA = v['ea'] + secBase = v['base'] + set_custom_name(secEA, f'vtbl_{cname}___{secBase}', None) + if secBase in vtables: + vtables[secBase].secondaryEAs.append(secEA) + else: + msg(f'Indirect base {secBase} has no known vtables') + vtables[cname] = Vtable(primaryEA, [], primary['vFuncs'], primaryBase) + for name in data.keys(): + populate_vtable(name) + + # second pass: determine vtable sizes and create structures + def pass2(): + msg('** Populating exported vtables: pass 2 **') + + def common_vtable_length(primaryEA, secondaryEAs): + primaryLen = calc_vtable_length(primaryEA) if primaryEA != 0 else 0 + vlen = primaryLen + for ea in secondaryEAs: + secLen = calc_vtable_length(ea) + if vlen == 0: + vlen = secLen + if primaryLen != 0 and primaryLen != secLen: + msg(f'Mismatch between vtable sizes at {hex(primaryEA)} ({primaryLen}) and {hex(ea)} ({secLen})') + if vlen == 0 or vlen > secLen: + vlen = secLen + return vlen + + def calc_vf_name(vtable, vfuncs, idx): + if idx in vfuncs: + custom = vfuncs[idx]['name'] + if not ida_struct.get_member_by_name(vtable, custom): + return custom + msg(f'Duplicate vtable field {ida_struct.get_struc_name(vtable.id)}.{custom}, using fallback for {idx}') + return f'vf{idx}' + + def create_vtable(cname, vtbl): + vlen = common_vtable_length(vtbl.primaryEA, vtbl.secondaryEAs) + if vlen == 0: + return # don't bother creating a vtable if there are no instances + + # create structure + vtable = ii.add_struct(f'{cname}_vtbl') + if not vtable: + raise Exception(f'Failed to create vtable structure for {cname}') + + # add base, if any + if vtbl.base and not ii.add_struct_baseclass(vtable, f'{vtbl.base}_vtbl'): + msg(f'Failed to add base for vtable for {cname}') + + # check that all custom vfuncs are in range + firstNewVF = ida_struct.get_struc_size(vtable) >> 3 + for idx in vtbl.vFuncs.keys(): + if idx < firstNewVF: + msg(f'Class {cname} overrides vfunc {idx} inherited from base {vtbl.base}') + elif idx >= vlen: + msg(f'Class {cname} defines vfunc {idx} which is outside bounds ({vlen})') + + # add fields + for idx in range(firstNewVF, vlen): + name = calc_vf_name(vtable, vtbl.vFuncs, idx) + if not ii.add_struct_member_ptr(vtable, idx << 3, name): + msg(f'Failed to add vfunc {idx} to vtable {cname}') + #ida_struct.save_struc(vtable) + + with ii.mass_type_updater(ida_typeinf.UTP_STRUCT): + for cname, vtbl in vtables.items(): + try: + create_vtable(cname, vtbl) + except Exception as e: + msg(f'Create vtable error: {e}') + + # third pass: rename functions, add crossrefs for vtable instances + # the only reason to split it into separate pass is to do it outside mass type update + def pass3(): + msg('** Populating exported vtables: pass 3 **') + + def create_vtable_instance(vtable, numVFs, ea, prefix, signatures): + # note: i feel that creating vtable global is, while correct, makes viewing it slightly worse (not seeing vf offsets etc) + # but at very least create custom xref (so that find-refs on vtable struct works) + # TODO: reconsider... + ida_xref.add_dref(ea, vtable.id, ida_xref.XREF_USER | ida_xref.dr_I) + + for idx in range(0, numVFs): + vfuncEA = ida_bytes.get_qword(ea + idx * 8) + vfuncName = ida_name.get_ea_name(vfuncEA) + if vfuncName == "_purecall": + continue # abstract virtual function + + inner = ida_struct.get_innermost_member(vtable, idx * 8) + if not inner: + msg(f'Failed to find field for vfunc {idx} of {cname}') + continue + leafName = ida_struct.get_member_name(inner[0].id) + if f'.{leafName}' in vfuncName: + continue # this function is probably not overridden + + set_custom_name(vfuncEA, f'{prefix}.{leafName}', signatures[idx]['address'] if signatures and idx in signatures else None) + + for cname, vtbl in vtables.items(): + vtable = ii.get_struct_by_name(f'{cname}_vtbl') + numVFs = ida_struct.get_struc_size(vtable) >> 3 + + if vtbl.primaryEA != 0: + create_vtable_instance(vtable, numVFs, vtbl.primaryEA, cname, vtbl.vFuncs) + for ea in vtbl.secondaryEAs: + create_vtable_instance(vtable, numVFs, ea, ida_name.get_ea_name(ea)[5:], None) # remove vtbl_ prefix, leaving derived___cname + + pass1() + pass2() + pass3() + return vtables + +def populate_structs(data): + # structure creation is done in two passes + res = {} # base/substruct always ordered before referencing struct; key = class name + + # first pass: build an ordered set of structures (base/subfield before containing structures) and create empty structs + # these empty structs can be used as a kind of 'forward declarations' for pointers + def pass1(): + msg('** Populating exported structs: pass 1 **') + def populate_struct(cname): + if cname in res: + return # class already processed, since it's a base/substruct of some earlier-defined class + cdata = data[cname] + # ensure we process all bases and struct fields first + for b in cdata['bases']: + populate_struct(b['type']) + for f in cdata['fields']: + if f['isStruct']: + populate_struct(f['type']) + res[cname] = cdata # tbd + # add struct + s = ii.add_struct(cname) + if not s: + raise Exception(f'Failed to create structure {cname}') + + with ii.mass_type_updater(ida_typeinf.UTP_STRUCT): + for name in data.keys(): + try: + populate_struct(name) + except Exception as e: + msg(f'Struct create error: {e}') + + # second pass: fill structure bases and fields + def pass2(): + msg('** Populating exported structs: pass 2 **') + + def add_base(s, type, offset, size): + curSize = ida_struct.get_struc_size(s) + if curSize != offset: + # treat this as a warning... + msg(f'Unexpected offset for {ida_struct.get_struc_name(s.id)} base {type}: expected {hex(offset)}, got {hex(curSize)}') + if not ii.add_struct_baseclass(s, type): + msg(f'Failed to add {ida_struct.get_struc_name(s.id)} base {type}') + return + actualSize = ida_struct.get_member_size(ida_struct.get_member(s, curSize)) + if actualSize != size: + msg(f'Unexpected size for {ida_struct.get_struc_name(s.id)} base {type}: expected {hex(size)}, got {hex(actualSize)}') + + def add_vptr(s, vtname): + if not ii.add_struct_member_ptr(s, 0, "__vftable"): + msg(f'Failed to add vtable pointer to {ida_struct.get_struc_name(s.id)}') + elif not ii.set_struct_member_by_offset_type(s, 0, vtname + '*' if ida_struct.get_struc_id(vtname) != idaapi.BADADDR else 'void*'): + msg(f'Failed to set vtable pointer type for {ida_struct.get_struc_name(s.id)} (vtbl-struct-id={hex(ida_struct.get_struc_id(vtname))})') + + def add_field(s, offset, fdata, checkSize): + name = fdata['name'] + type = fdata['type'] + arrLen = fdata['arrayLength'] + if fdata['isStruct']: + success = ii.add_struct_member_substruct(s, offset, name, type, arrLen if arrLen > 0 else 1) + else: + typeSuffix = f'[{arrLen}]' if arrLen > 0 else '' + success = ii.add_struct_member_typed(s, offset, name, type + typeSuffix) + + if not success: + msg(f'Failed to add field {ida_struct.get_struc_name(s.id)}.{name}') + elif checkSize: + actualSize = ida_struct.get_member_size(ida_struct.get_member(s, offset)) + expectedSize = fdata['size'] + if actualSize != expectedSize: + msg(f'Unexpected size for {ida_struct.get_struc_name(s.id)}.{name}: expected {hex(expectedSize)}, got {hex(actualSize)}') + + with ii.mass_type_updater(ida_typeinf.UTP_STRUCT): + for cname, cdata in res.items(): + s = ii.get_struct_by_name(cname) + if not s: + continue + # start with bases (if any) + for b in cdata['bases']: + add_base(s, b['type'], b['offset'], b['size']) + # now add primary vtable, if needed + if ida_struct.get_struc_size(s) == 0 and cdata['primaryVTable']: + add_vptr(s, cname + '_vtbl') + # now add fields + for offset, fgroup in itertools.groupby(cdata['fields'], lambda f: f['offset']): + if offset < ida_struct.get_struc_size(s): + msg(f'Unexpected offset for {cname}+{hex(offset)}, current size if {ida_struct.get_struc_size(s)}') + continue + flist = [f for f in fgroup] # group can only be iterated over once + if len(flist) == 1: + add_field(s, offset, flist[0], True) + else: + uname = f'union{hex(offset)[2:]}' + su = ii.add_struct(f'{cname}_{uname}', True) + for f in flist: + add_field(su, 0, f, False) + ii.add_struct_member_substruct(s, offset, uname, f'{cname}_{uname}') + # add tail, if structure is larger than last field + finalSize = ida_struct.get_struc_size(s) + expectedSize = cdata['size'] + if finalSize > expectedSize: + msg(f'Structure {cname} is too large: {hex(finalSize)} > {hex(expectedSize)}') + elif finalSize < expectedSize and not ii.add_struct_member_byte(s, finalSize, f'tail_{hex(finalSize)[2:]}', expectedSize - finalSize): + msg(f'Failed to extend structure {cname}') + + pass1() + pass2() + +def populate_global_types(data): + msg('** Populating exported global types **') + + def process_global(ea, type, expectedSize): + if not type: + return 0 # nothing to do, type unknown + tif = ii.parse_cdecl(type) + if not tif: + raise Exception(f'Failed to parse type {type}') + actualSize = tif.get_size() + if actualSize != expectedSize: + msg(f'Mismatched global size {type} @ {hex(ea)}: expected {hex(expectedSize)}, got {hex(actualSize)}') + for nameEA, name in ii.enumerate_names(): + if nameEA > ea and nameEA < ea + actualSize: + msg(f'Existing global {name} is now a part of global {type} @ {hex(ea)} at offset {hex(nameEA - ea)}') + if not ida_typeinf.apply_tinfo(ea, tif, 0): + msg(f'Failed to apply {type} @ {hex(ea)}') + return 0 + return actualSize + + minEA = 0 + for ea, g in data.items(): + if ea < minEA: + msg(f'Skipping global {g["type"]} {g["name"]} @ {hex(ea)}, since it is a part of another global') + continue # this global was already consumed by another global + + minEA = ea + try: + minEA += process_global(ea, g['type'], g['size']) + except Exception as e: + msg(f'Global type error: {e}') + +def populate_function_types(data): + msg('** Populating exported function types **') + for ea, f in data.items(): + sig = f['signature'] + if not sig: + continue + tif = ii.parse_cdecl(convert_exported_sig(sig)) + try: + apply_function_type(ea, tif) + except Exception as e: + msg(f'Failed to apply function type {tif} @ {hex(ea)}: {e}') + +def populate_vfunc_types(vtables): + msg('** Populating exported virtual function types **') + + def update_vtable_fields(cname, vtable, vtbl): + for idx, vfunc in vtbl.vFuncs.items(): + sig = vfunc['signature'] + if not sig: + continue + m = ida_struct.get_member(vtable, idx * 8) + if not m or ida_struct.get_member_name(m.id) == 'baseclass_0': + continue + type = convert_exported_sig(sig, '(*)') + if not ii.set_struct_member_type(vtable, m, type): + msg(f'Failed to set vtable {cname} entry #{idx} type to {type}') + + def propagate_vfunc_type(eaRef, tinfo, cname, shift): + vfuncEA = ida_bytes.get_qword(eaRef) + vfuncName = ida_name.get_ea_name(vfuncEA) + if vfuncName == "_purecall": + return # abstract virtual function + # replace 'this' pointer type with proper one + try: + fi = ida_typeinf.func_type_data_t() + if not tinfo.get_func_details(fi): + raise Exception('Failed to get func details') + elif fi.size() == 0: + raise Exception('Func has 0 args') + elif fi[0].name != 'this': + raise Exception(f'First arg is not this: {fi[0].name}') + elif not fi[0].type.is_ptr(): + raise Exception(f'First arg has unexpected type {fi[0].type}') + + if shift == 0: + fi[0].type = ii.parse_cdecl(cname + '*') + elif shift > 0: + fi[0].type = ii.parse_cdecl(f'{fi[0].type} __shifted({cname}, {shift})') + # else: failed to find base, keep base* + + tifAdj = ida_typeinf.tinfo_t() + if not tifAdj.create_func(fi): + raise Exception(f'Failed to build updated tinfo: {tifAdj}') + + apply_function_type(vfuncEA, tifAdj) + except Exception as e: + msg(f'Failed to apply virtual function type {tinfo} @ {hex(vfuncEA)}: {e}') + + def propagate_types_to_instances(cname, vtable, vtbl): + numVFs = ida_struct.get_struc_size(vtable) >> 3 + for idx in range(numVFs): + inner = ida_struct.get_innermost_member(vtable, idx * 8) + itype = ii.get_struct_member_tinfo(inner[0]) if inner else None + if not itype or not itype.is_funcptr(): + continue + itype = ida_typeinf.remove_pointer(itype) + if vtbl.primaryEA != 0: + propagate_vfunc_type(vtbl.primaryEA + idx * 8, itype, cname, 0) + for ea in vtbl.secondaryEAs: + derivedName = ida_name.get_ea_name(ea)[5:-3-len(cname)] # remove vtbl_ prefix and ___cname suffix, leaving derived + shift = find_base_offset(ida_struct.get_struc_id(derivedName), ida_struct.get_struc_id(cname)) + propagate_vfunc_type(ea + idx * 8, itype, derivedName, shift if shift != None else -1) + + for cname, vtbl in vtables.items(): + vtable = ii.get_struct_by_name(f'{cname}_vtbl') + if not vtable: + continue + update_vtable_fields(cname, vtable, vtbl) + propagate_types_to_instances(cname, vtable, vtbl) + +def populate_exported(yamlName): + msg(f'*** Populating exported items from {yamlName} ***') + with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), yamlName), 'r') as fd: + data = yaml.safe_load(fd) + populate_global_names(data['globals']) + populate_function_names(data['functions']) + populate_enums(data['enums']) + vtables = populate_vtables(data['structs']) + populate_structs(data['structs']) + populate_global_types(data['globals']) + populate_function_types(data['functions']) + populate_vfunc_types(vtables) + +breakpoint() +populate_static_initializers() +populate_exported('info.yml') +msg('*** Finished! ***') diff --git a/upgrade b/upgrade new file mode 100644 index 0000000..cb612b9 --- /dev/null +++ b/upgrade @@ -0,0 +1,32 @@ +1. Open new binary, wait for analysis +2. Names (funcs, globals, vtables): run (alt-f7) idarename.py, fix failures manually +3. Structures & enums: build & run CExporter, import _arrays.h +4. EXD +5. Network +6. Globals +7. Signatures + +consider this flow: +1. c# parses ffxivclientstructs and extracts to yaml: + - enums + - structs (normal + dependent template specializations) + - fields: offset+name+type + - methods: name+sig+type + - vtable sig + - vfuncs: name+ordinal+type + - globals (sigs+type) +2. c# parses lumina and extracts to yaml: + - enum SheetID + - sheets + - bitfield enums + - structs +3. ida-python parses data.yml + ffxivclientstruct yaml + lumina yaml + custom-patches yaml + - create enums (from ffxiv/lumina) + - create local types / structs (ordering!) (from ffxiv/lumina) + - create globals + - ea from data.yml (alt: sig from ffxiv) + - type from ffxiv + - rename global funcs & methods + - ea from data.yml (alt: sig from ffxiv) + - type from ffxiv + - vtables! diff --git a/vnetlog/vnetlog/PacketDecoder.cs b/vnetlog/vnetlog/PacketDecoder.cs index 271a6df..dac3729 100644 --- a/vnetlog/vnetlog/PacketDecoder.cs +++ b/vnetlog/vnetlog/PacketDecoder.cs @@ -170,6 +170,7 @@ public unsafe class PacketDecoder ActorControlCategory.SetTarget => $"{ObjStr(targetID)}", ActorControlCategory.SetAnimationState => $"#{p1} = {p2}", ActorControlCategory.SetModelState => $"{p1}", + ActorControlCategory.ForcedMovement => $"dest={Vec3Str(IntToFloatCoords((ushort)p1, (ushort)p2, (ushort)p3))}, rot={IntToFloatAngleDeg((ushort)p4):f1}deg over {p5 * 0.0001:f4}s, type={p6}", ActorControlCategory.PlayActionTimeline => $"{p1:X4}", ActorControlCategory.EObjSetState => $"{p1:X4}, housing={(p3 != 0 ? p4 : null)}", ActorControlCategory.EObjAnimation => $"{p1:X4} {p2:X4}", diff --git a/vnetlog/vnetlog/ServerIPC.cs b/vnetlog/vnetlog/ServerIPC.cs index 38c8574..0159136 100644 --- a/vnetlog/vnetlog/ServerIPC.cs +++ b/vnetlog/vnetlog/ServerIPC.cs @@ -336,6 +336,7 @@ public enum ActorControlCategory : ushort DespawnZoneScreenMsg = 207, // from dissector InstanceSelectDlg = 210, // from dissector ActorDespawnEffect = 212, // from dissector + ForcedMovement = 226, CompanionUnlock = 253, // from dissector ObtainBarding = 254, // from dissector EquipBarding = 255, // from dissector