IDB migration WIP

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "idapopulate/FFXIVClientStructs"]
+	path = idapopulate/FFXIVClientStructs
+	url = https://github.com/awgil/FFXIVClientStructs.git

idapopulate/FFXIVClientStructs

@@ -0,0 +1 @@
+Subproject commit b67538d428032c0fab9bce8062f2c2b79bc6e7fc

idapopulate/idapopulate.sln

@@ -0,0 +1,40 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.4.33205.214
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "idapopulate", "idapopulate\idapopulate.csproj", "{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}"
+	ProjectSection(ProjectDependencies) = postProject
+		{A8C9ED42-A7CF-4A7C-894C-803030860E21} = {A8C9ED42-A7CF-4A7C-894C-803030860E21}
+	EndProjectSection
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FFXIVClientStructs", "FFXIVClientStructs\FFXIVClientStructs\FFXIVClientStructs.csproj", "{A8C9ED42-A7CF-4A7C-894C-803030860E21}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FFXIVClientStructs.InteropSourceGenerators", "FFXIVClientStructs\FFXIVClientStructs.InteropSourceGenerators\FFXIVClientStructs.InteropSourceGenerators.csproj", "{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6EF10816-5C9E-49DA-B2A7-E26CF95994C2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A8C9ED42-A7CF-4A7C-894C-803030860E21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A8C9ED42-A7CF-4A7C-894C-803030860E21}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A8C9ED42-A7CF-4A7C-894C-803030860E21}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A8C9ED42-A7CF-4A7C-894C-803030860E21}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8D06EC02-F65E-4E6E-88F9-DC69CA583B88}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {8B4CC935-E684-4D8D-91EF-56B88D28E7C3}
+	EndGlobalSection
+EndGlobal

idapopulate/idapopulate/CSImport.cs

@@ -0,0 +1,339 @@
+using FFXIVClientStructs.Attributes;
+using FFXIVClientStructs.Interop;
+using FFXIVClientStructs.Interop.Attributes;
+using System.Diagnostics;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace idapopulate;
+
+internal static class CSImportExt
+{
+    public static string WithoutPrefix(this string str, string prefix) => str.StartsWith(prefix) ? str.Substring(prefix.Length) : str;
+
+    // if [FieldOffset] is not specified, assume sequential layout...
+    public static int GetFieldOffset(this FieldInfo fi) => fi.GetCustomAttribute<FieldOffsetAttribute>()?.Value ?? Marshal.OffsetOf(fi.DeclaringType!, fi.Name).ToInt32();
+
+    public static (Type?, int) GetFixedBufferInfo(this FieldInfo fi)
+    {
+        var attr = fi.GetCustomAttribute<FixedBufferAttribute>();
+        return (attr?.ElementType, attr?.Length ?? 0);
+    }
+
+    public static (Type?, int) GetFixedArrayInfo(this FieldInfo fi)
+    {
+        var attr = fi.GetCustomAttribute(typeof(FixedSizeArrayAttribute<>));
+        if (attr == null)
+            return (null, 0);
+        var len = (int?)attr.GetType().GetProperty("Count", BindingFlags.Instance | BindingFlags.Public)?.GetValue(attr) ?? 0;
+        var t = attr.GetType().GetGenericArguments()[0];
+        if (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Pointer<>))
+            t = t.GetGenericArguments()[0].MakePointerType();
+        return (t, len);
+    }
+}
+
+internal class CSImport
+{
+    private HashSet<Type> _processedTypes = new();
+
+    public void Populate(Result res, SigResolver resolver)
+    {
+        var typesToProcess = GetAssemblyTypes("FFXIVClientStructs").Where(IsTypeExportable).ToList();
+        for (int i = 0; i < typesToProcess.Count; i++) // note: more types could be added while processing types
+        {
+            PopulateType(typesToProcess[i], typesToProcess, res, resolver);
+        }
+    }
+
+    private static IEnumerable<Type> GetAssemblyTypes(string assemblyName)
+    {
+        var assembly = AppDomain.CurrentDomain.Load(assemblyName);
+        try
+        {
+            return assembly.DefinedTypes.Select(ti => ti.AsType());
+        }
+        catch (ReflectionTypeLoadException ex)
+        {
+            return ex.Types.Cast<Type>();
+        }
+    }
+
+    private static bool IsTypeExportable(Type type)
+    {
+        if (type.FullName == null)
+            return false;
+        if (!type.FullName.StartsWith("FFXIVClientStructs.FFXIV.") && !type.FullName.StartsWith("FFXIVClientStructs.Havok."))
+            return false;
+        if (type.DeclaringType != null && (type.Name is "Addresses" or "MemberFunctionPointers" or "StaticAddressPointers" || type.Name == type.DeclaringType.Name + "VTable"))
+            return false;
+        if (type.Name.EndsWith("e__FixedBuffer"))
+            return false;
+        return true;
+    }
+
+    private void QueueType(Type type, List<Type> queue)
+    {
+        while (DerefPointer(type) is var derefType && derefType != null)
+            type = derefType;
+        if (type.IsPrimitive || type == typeof(void))
+            return; // void can appear as eg void* fields
+        queue.Add(type);
+    }
+
+    private void PopulateType(Type type, List<Type> queue, Result res, SigResolver resolver)
+    {
+        if (!_processedTypes.Add(type))
+            return; // already processed
+
+        var tn = TypeName(type);
+        if (type.IsEnum)
+        {
+            var (width, signed) = GetEnumSignWidth(type);
+            var e = new Result.Enum() { IsBitfield = type.GetCustomAttribute<FlagsAttribute>() != null, IsSigned = signed, Width = width };
+            foreach (var f in type.GetFields().Where(f => f.Name != "value__"))
+                e.Values.Add(new() { Name = f.Name, Value = Convert.ToInt64(f.GetRawConstantValue()) });
+            res.Enums.Add(tn, e);
+        }
+        else
+        {
+            if (type.IsGenericType && type.ContainsGenericParameters)
+            {
+                //Debug.WriteLine($"Skipping generic struct: {type}");
+                return; // we don't care about unspecialized templates
+            }
+
+            var addresses = type.GetNestedType("Addresses");
+
+            var s = new Result.Struct() { Size = SizeOf(type) };
+            if ((type.StructLayoutAttribute?.Size ?? 0) is var layoutSize && layoutSize != 0 && layoutSize != s.Size)
+                Debug.WriteLine($"Size mismatch for {type}: layout says 0x{layoutSize:X}, actual is 0x{s.Size:X}");
+
+            // see whether there are any vtable definitions or virtual functions
+            if (type.GetCustomAttribute<VTableAddressAttribute>() is var attrVTable && attrVTable != null)
+            {
+                if (attrVTable.IsPointer)
+                    Debug.WriteLine($"VTable for {type} is stored as a pointer, wtf does it even mean?");
+                s.PrimaryVTable = new() { Address = new(attrVTable.Signature, attrVTable.Offset), Ea = GetResolvedAddress(addresses, "VTable", resolver) };
+            }
+
+            // process methods (both virtual and non-virtual)
+            foreach (var method in type.GetMethods(BindingFlags.Instance | BindingFlags.Static | BindingFlags.Public))
+            {
+                if (method.GetCustomAttribute<VirtualFunctionAttribute>() is var vfAttr && vfAttr != null)
+                {
+                    s.PrimaryVTable ??= new();
+                    s.PrimaryVTable.VFuncs.Add(vfAttr.Index, new() { Name = method.Name, Signature = ExtractFuncSig(method, tn) });
+                }
+                else if (method.GetCustomAttribute<MemberFunctionAttribute>() is var mfAttr && mfAttr != null)
+                {
+                    var ea = GetResolvedAddress(addresses, method.Name, resolver);
+                    if (ea == 0)
+                        Debug.WriteLine($"Failed to find method {type}.{method.Name}: sig={mfAttr.Signature}");
+                    else if (res.Functions.ContainsKey(ea))
+                        Debug.WriteLine($"Multiple functions resolve to same address 0x{ea:X}: {type}.{method.Name} sig={mfAttr.Signature} vs. {res.Functions[ea]}");
+                    else
+                        res.Functions[ea] = new() { Name = $"{tn}.{method.Name}", Address = new(mfAttr.Signature), Signature = ExtractFuncSig(method, method.IsStatic ? "" : tn) };
+                }
+                else if (method.GetCustomAttribute<StaticAddressAttribute>() is var saAttr && saAttr != null)
+                {
+                    var ea = GetResolvedAddress(addresses, method.Name, resolver);
+                    if (ea == 0)
+                        Debug.WriteLine($"Failed to find global {type}.{method.Name}: sig={saAttr.Signature}+0x{saAttr.Offset}");
+                    else if (res.Globals.ContainsKey(ea))
+                        Debug.WriteLine($"Multiple globals resolve to same address 0x{ea:X}: {type}.{method.Name} sig={saAttr.Signature}+0x{saAttr.Offset} vs. {res.Globals[ea]}");
+                    else
+                        res.Globals[ea] = new() { Type = saAttr.IsPointer ? tn + "*" : tn, Name = $"g_{tn}_{method.Name}", Address = new(saAttr.Signature, saAttr.Offset), Size = saAttr.IsPointer ? 8 : s.Size }; // note: name currently matches idarename
+                }
+            }
+
+            var fields = type.GetFields().Where(f => !f.IsLiteral && !f.IsStatic && f.GetCustomAttribute<IDAIgnoreAttribute>() == null && f.GetCustomAttribute<ObsoleteAttribute>() == null);
+            int nextOff = 0;
+            int prevSize = 0;
+            foreach (var (f, off) in fields.Select(f => (f, f.GetFieldOffset())).OrderBy(pair => pair.Item2))
+            {
+                if (off == 0 && f.Name == "VTable")
+                {
+                    // this is not a particularly interesting field - just mark struct as having a vtable (if it has neither known vtable address nor known virtual functions) and continue
+                    s.PrimaryVTable ??= new();
+                    continue;
+                }
+
+                if (off < nextOff && (s.Fields.Count == 0 || off != nextOff - prevSize)) // first check covers a situation where previous field is a base
+                {
+                    Debug.WriteLine($"Skipping field {type}.{f.Name} at offset 0x{off:X}: previous field ended at 0x{nextOff:X} (0x{nextOff - prevSize:X}+0x{prevSize:X})");
+                    continue;
+                }
+
+                var ftype = f.FieldType;
+                int fsize = 0;
+                int arrLen = 0;
+
+                var (fixedBufferElem, fixedBufferLength) = f.GetFixedBufferInfo();
+                if (fixedBufferElem != null)
+                {
+                    fsize = SizeOf(fixedBufferElem) * fixedBufferLength;
+
+                    var (fixedArrayElem, fixedArrayLength) = f.GetFixedArrayInfo();
+                    if (fixedArrayElem != null)
+                    {
+                        QueueType(fixedArrayElem, queue);
+                        var fixedArraySize = SizeOf(fixedArrayElem) * fixedArrayLength;
+                        if (fixedArraySize != fsize)
+                        {
+                            Debug.WriteLine($"Array size mismatch for {type}.{f.Name}: raw is {fixedBufferElem}[{fixedBufferLength}] (0x{fsize:X}), typed is {fixedArrayElem}[{fixedArrayLength}] (0x{fixedArraySize:X})");
+                            fsize = fixedArraySize;
+                        }
+
+                        ftype = fixedArrayElem;
+                        arrLen = fixedArrayLength;
+                    }
+                    else
+                    {
+                        ftype = fixedBufferElem;
+                        arrLen = fixedBufferLength;
+                    }
+                }
+                else
+                {
+                    QueueType(ftype, queue);
+                    fsize = SizeOf(f.FieldType);
+                }
+                bool isStruct = ftype.IsValueType && !ftype.IsPrimitive && !ftype.IsEnum && DerefPointer(ftype) == null;
+
+                bool fieldCanBeBase = isStruct && s.Fields.Count == 0 && off == nextOff; // no gaps or fields between bases allowed
+                bool isBaseClass = f.GetCustomAttribute<IDABaseClassAttribute>()?.IsBase ?? (fieldCanBeBase && off == 0 && ftype.Name == f.Name); // implicit base-class logic: single-inheritance, field name matches baseclass name
+                if (isBaseClass && !fieldCanBeBase)
+                {
+                    Debug.WriteLine($"Field {type}.{f.Name} is marked as a base class, but can't be one");
+                    isBaseClass = false;
+                }
+
+                if (isBaseClass)
+                    s.Bases.Add(new() { Type = TypeName(ftype), Offset = off, Size = fsize });
+                else
+                    s.Fields.Add(new() { Name = f.Name, Type = TypeName(ftype), IsStruct = isStruct, Offset = off, ArrayLength = arrLen, Size = fsize });
+
+                if (off >= nextOff)
+                {
+                    nextOff = off + fsize;
+                    prevSize = fsize;
+                }
+                else
+                {
+                    nextOff = Math.Max(nextOff, off + fsize);
+                    prevSize = Math.Max(prevSize, fsize);
+                }
+            }
+            res.Structs.Add(tn, s);
+        }
+    }
+
+    private string TypeName(Type type)
+    {
+        if (DerefPointer(type) is var derefType && derefType != null)
+            return TypeName(derefType) + "*";
+        else if (type == typeof(void))
+            return "void";
+        else if (type == typeof(bool))
+            return "bool";
+        else if (type == typeof(char))
+            return "char"; // note: despite c# char being a wchar, CS seems to use it as a normal char, go figure...
+        else if (type == typeof(sbyte))
+            return "char";
+        else if (type == typeof(byte))
+            return "uchar";
+        else if (type == typeof(short))
+            return "short";
+        else if (type == typeof(ushort))
+            return "ushort";
+        else if (type == typeof(int))
+            return "int";
+        else if (type == typeof(uint))
+            return "uint";
+        else if (type == typeof(long) || type == typeof(nint))
+            return "__int64";
+        else if (type == typeof(ulong) || type == typeof(nuint))
+            return "unsigned __int64";
+        else if (type == typeof(float))
+            return "float";
+        else if (type == typeof(double))
+            return "double";
+        else
+            return TypeNameComplex(type);
+    }
+
+    private string TypeNameComplex(Type type)
+    {
+        var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs.").WithoutPrefix("FFXIV.").WithoutPrefix("Havok.").Replace(".", "::") ?? "";
+        var leafName = type.Name;
+        if (type.IsGenericType)
+        {
+            leafName = leafName.Split('`')[0];
+            if (!type.ContainsGenericParameters)
+            {
+                leafName += $"${string.Join("$", type.GetGenericArguments().Select(arg => TypeName(arg).Replace("*", "_ptr")))}$";
+            }
+        }
+        var fullName = baseName.Length > 0 ? $"{baseName}::{leafName}" : leafName;
+
+        // hack for std
+        if (fullName.StartsWith("STD::Std"))
+        {
+            fullName = fullName.WithoutPrefix("STD::Std");
+            fullName = "std::"+ fullName.Substring(0, 1).ToLower() + fullName.Substring(1);
+        }
+        return fullName;
+    }
+
+    private int SizeOf(Type type)
+    {
+        if (DerefPointer(type) != null)
+            return 8; // assume 64-bit
+        // Marshal.SizeOf doesn't work correctly because the assembly is unmarshaled, and more specifically, it sets bools as 4 bytes long...
+        return (int?)typeof(Unsafe).GetMethod("SizeOf")?.MakeGenericMethod(type).Invoke(null, null) ?? 0;
+    }
+
+    private (int, bool) GetEnumSignWidth(Type enumType)
+    {
+        var underlying = enumType.GetEnumUnderlyingType();
+        if (underlying == typeof(sbyte))
+            return (1, true);
+        else if (underlying == typeof(byte))
+            return (1, false);
+        else if (underlying == typeof(short))
+            return (2, true);
+        else if (underlying == typeof(ushort))
+            return (2, false);
+        else if (underlying == typeof(int))
+            return (4, true);
+        else if (underlying == typeof(uint))
+            return (4, false);
+        else if (underlying == typeof(long))
+            return (8, true);
+        else if (underlying == typeof(ulong))
+            return (8, false);
+        else
+            throw new Exception($"Unsupported underlying enum type {underlying} for {enumType}");
+    }
+
+    private Type? DerefPointer(Type type) => type.IsPointer ? type.GetElementType()! : type.IsGenericType && type.GetGenericTypeDefinition() == typeof(Pointer<>) ? type.GetGenericArguments()[0] : null;
+
+    private Result.FuncSig ExtractFuncSig(MethodInfo m, string thisType)
+    {
+        var res = new Result.FuncSig() { RetType = TypeName(m.ReturnType), Arguments = m.GetParameters().Select(p => new Result.FuncArg() { Type = TypeName(p.ParameterType), Name = p.Name ?? "" }).ToList() };
+        if (thisType.Length > 0)
+            res.Arguments.Insert(0, new() { Type = thisType + "*", Name = "this" });
+        return res;
+    }
+
+    private ulong GetResolvedAddress(Type? addresses, string name, SigResolver resolver)
+    {
+        var addr = addresses?.GetField(name, BindingFlags.Static | BindingFlags.Public)?.GetValue(null) as Address;
+        var res = addr != null ? resolver.ToEA(addr) : 0;
+        if (res == 0)
+            Debug.WriteLine($"Failed to resolve address for {addresses?.FullName}.{name}");
+        return res;
+    }
+}

idapopulate/idapopulate/DataYmlImport.cs

@@ -0,0 +1,169 @@
+using System.Diagnostics;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace idapopulate;
+
+internal class DataYmlImport
+{
+    private class DataYmlClassInstance
+    {
+        public ulong Ea = 0;
+        public bool Pointer = true; // not sure what is the correct default, but this at least is safer...
+        public string Name = "Instance";
+    }
+
+    private class DataYmlClassVtbl
+    {
+        public ulong Ea = 0;
+        public string Base = "";
+    }
+
+    private class DataYmlClass
+    {
+        public List<DataYmlClassInstance> Instances = new();
+        public List<DataYmlClassVtbl> Vtbls = new();
+        public SortedDictionary<uint, string> Vfuncs = new(); // index -> name
+        public SortedDictionary<ulong, string> Funcs = new(); // ea -> name
+    }
+
+    private class DataYml
+    {
+        public string Version = "";
+        public SortedDictionary<ulong, string> Globals = new(); // ea -> name
+        public SortedDictionary<ulong, string> Functions = new(); // ea -> name
+        public SortedDictionary<string, DataYmlClass?> Classes = new(); // name -> data
+    }
+
+    public void Populate(Result res, FileInfo fi)
+    {
+        var data = new DeserializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance).Build().Deserialize<DataYml>(File.ReadAllText(fi.FullName));
+        foreach (var (ea, name) in data.Globals)
+            PopulateGlobal(res, ea, name, "", 0);
+        foreach (var (ea, name) in data.Functions)
+            PopulateFunction(res, ea, name);
+
+        // we process classes in several passes:
+        // 1. ensure all struct entries exist for each class defined in yml
+        foreach (var name in data.Classes.Keys.Select(FixClassName))
+            if (!res.Structs.ContainsKey(name))
+                res.Structs[name] = new();
+        // 2. process class data
+        foreach (var (name, cls) in data.Classes)
+            PopulateClass(res, FixClassName(name), cls ?? new()); // note: some class entries are completely empty
+    }
+
+
+    private static string FixClassName(string name) => name.Replace('<', '$').Replace('>', '$').Replace(',', '$').Replace("*", "_ptr");
+
+    private void PopulateGlobal(Result res, ulong ea, string name, string type, int size)
+    {
+        var match = res.Globals.GetValueOrDefault(ea);
+        if (match != null)
+        {
+            // we assume that if CS defines a global, it always has known type & size
+            // any conflicts between CS and yaml are resolved for CS
+            if (name != match.Name)
+                Debug.WriteLine($"Name mismatch for global @ 0x{ea:X}: CS={match.Name}, yml={name}");
+            if (type.Length > 0 && type != match.Type)
+                Debug.WriteLine($"Type mismatch for global @ 0x{ea:X}: CS={match.Type}, yml={type}");
+            if (size != 0 && size != match.Size)
+                Debug.WriteLine($"Size mismatch for global @ 0x{ea:X}: CS={match.Size}, yml={size}");
+        }
+        else
+        {
+            res.Globals[ea] = new() { Type = type, Name = name, Size = size };
+        }
+    }
+
+    private void PopulateFunction(Result res, ulong ea, string name)
+    {
+        var match = res.Functions.GetValueOrDefault(ea);
+        if (match != null)
+        {
+            // any conflicts between CS and yaml are resolved for CS
+            // TODO: consider overriding CS names if yml has mangled function name (starting from ?)
+            if (name != match.Name)
+                Debug.WriteLine($"Name mismatch for function @ 0x{ea:X}: CS={match.Name}, yml={name}");
+        }
+        else
+        {
+            res.Functions[ea] = new() { Name = name };
+        }
+    }
+
+    private void PopulateClass(Result res, string name, DataYmlClass data)
+    {
+        var s = res.Structs[name];
+
+        // add placeholder for missing bases
+        foreach (var vt in data.Vtbls.Where(vt => vt.Base.Length > 0))
+        {
+            var baseName = FixClassName(vt.Base);
+            if (!res.Structs.ContainsKey(baseName))
+            {
+                Debug.WriteLine($"Base class {baseName} for {name} is not defined");
+                res.Structs[baseName] = new() { Size = 8, PrimaryVTable = new() }; // assume base also contains a vtable
+            }
+        }
+
+        var primaryVT = data.Vtbls.FirstOrDefault();
+        if (primaryVT != null)
+        {
+            if (s.PrimaryVTable == null)
+            {
+                s.PrimaryVTable = new() { Ea = primaryVT.Ea };
+                s.Size = Math.Max(s.Size, 8);
+            }
+            else if (s.PrimaryVTable.Ea == 0)
+            {
+                s.PrimaryVTable.Ea = primaryVT.Ea;
+            }
+            else if (s.PrimaryVTable.Ea != primaryVT.Ea)
+            {
+                Debug.WriteLine($"Primary VT address mismatch for {name}: CS=0x{s.PrimaryVTable.Ea}, yml={primaryVT.Ea}");
+            }
+
+            if (primaryVT.Base.Length == 0)
+            {
+                // nothing to do here, yml doesn't define a base here, if CS defines one - so be it
+            }
+            else if (s.Bases.Count == 0)
+            {
+                // yml defines a base that CS doesn't know about - synthesize one
+                var primaryBase = FixClassName(primaryVT.Base);
+                s.Bases.Add(new() { Type = primaryBase, Size = res.Structs[primaryBase].Size });
+            }
+            else if (FixClassName(primaryVT.Base) is var baseName && baseName != s.Bases[0].Type)
+            {
+                Debug.WriteLine($"Main base mismatch for {name}: CS={s.Bases[0].Type}, yml={baseName}");
+            }
+        }
+
+        foreach (var secondaryVT in data.Vtbls.Skip(1))
+        {
+            if (secondaryVT.Base.Length == 0)
+                Debug.WriteLine($"Unexpected null secondary base name in yml for {name}");
+            else if (s.Bases.Count == 0)
+                Debug.WriteLine($"Class {name} has no known primary base, but has secondary base {secondaryVT.Base}");
+            else
+                s.SecondaryVTables.Add(new() { Ea = secondaryVT.Ea, Base = FixClassName(secondaryVT.Base) });
+        }
+
+        foreach (var (index, fname) in data.Vfuncs)
+        {
+            s.PrimaryVTable ??= new();
+            s.Size = Math.Max(s.Size, 8); // ensure we can fit vtable
+            if (!s.PrimaryVTable.VFuncs.ContainsKey(index))
+                s.PrimaryVTable.VFuncs[index] = new() { Name = fname };
+            else if (s.PrimaryVTable.VFuncs[index].Name != fname)
+                Debug.WriteLine($"VF name mismatch for {name} #{index}: CS={s.PrimaryVTable.VFuncs[index].Name}, yml={fname}");
+        }
+
+        foreach (var (ea, fname) in data.Funcs)
+            PopulateFunction(res, ea, fname.StartsWith('?') ? fname : $"{name}.{fname}");
+
+        foreach (var inst in data.Instances)
+            PopulateGlobal(res, inst.Ea, inst.Name.StartsWith('?') ? inst.Name : $"g_{name}_{inst.Name}", inst.Pointer ? name + "*" : name, inst.Pointer ? 8 : s.Size);
+    }
+}

idapopulate/idapopulate/PathUtils.cs

@@ -0,0 +1,49 @@
+using Microsoft.Win32;
+using System.Reflection;
+
+namespace idapopulate;
+
+internal static class PathUtils
+{
+    public static string FindGameRoot()
+    {
+        // stolen from FFXIVLauncher/src/XIVLauncher/AppUtil.cs
+        foreach (var registryView in new RegistryView[] { RegistryView.Registry32, RegistryView.Registry64 })
+        {
+            using (var hklm = RegistryKey.OpenBaseKey(RegistryHive.LocalMachine, registryView))
+            {
+                // Should return "C:\Program Files (x86)\SquareEnix\FINAL FANTASY XIV - A Realm Reborn\boot\ffxivboot.exe" if installed with default options.
+                using (var subkey = hklm.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall\{2B41E132-07DF-4925-A3D3-F2D1765CCDFE}"))
+                {
+                    if (subkey != null && subkey.GetValue("DisplayIcon", null) is string path)
+                    {
+                        // DisplayIcon includes "boot\ffxivboot.exe", need to remove it
+                        var basePath = Directory.GetParent(path)?.Parent?.FullName;
+                        if (basePath != null)
+                        {
+                            var gamePath = Path.Join(basePath, "game");
+                            if (Directory.Exists(gamePath))
+                            {
+                                return gamePath;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return "D:\\installed\\SquareEnix\\FINAL FANTASY XIV - A Realm Reborn\\game";
+    }
+
+    public static FileInfo? FindFileAmongParents(string suffix)
+    {
+        var dir = new FileInfo(Assembly.GetExecutingAssembly().Location).Directory;
+        while (dir != null)
+        {
+            var yml = new FileInfo(dir.FullName + suffix);
+            if (yml.Exists)
+                return yml;
+            dir = dir.Parent;
+        }
+        return null;
+    }
+}

idapopulate/idapopulate/Program.cs

@@ -0,0 +1,39 @@
+using idapopulate;
+using System.Diagnostics;
+
+var outDir = PathUtils.FindFileAmongParents("/idbtoolkit/populate_idb.py")?.Directory;
+if (outDir == null)
+{
+    Debug.WriteLine("Failed to find output location");
+    return;
+}
+
+var testRes = new Result();
+var testEnum1 = new Result.Enum() { IsSigned = true, Width = 2 };
+testEnum1.Values.Add(new() { Name = "E1V1", Value = 1 });
+testEnum1.Values.Add(new() { Name = "E1V1dup", Value = 1 });
+testEnum1.Values.Add(new() { Name = "E1V2", Value = 2 });
+testEnum1.Values.Add(new() { Name = "E1V3", Value = 10 });
+testRes.Enums["test::Enum1"] = testEnum1;
+var testEnum2 = new Result.Enum() { IsBitfield = true, IsSigned = false, Width = 4 };
+testEnum2.Values.Add(new() { Name = "E2V1", Value = 0x1 });
+testEnum2.Values.Add(new() { Name = "E2V2", Value = 0x4 });
+testRes.Enums["test::Enum2"] = testEnum2;
+testRes.Enums["test::Enum3"] = new() { Width = 1 };
+testRes.Write(outDir.FullName + "/test.yml");
+
+var gameRoot = PathUtils.FindGameRoot();
+var resolver = new SigResolver(gameRoot + "\\ffxiv_dx11.exe");
+
+var res = new Result();
+new CSImport().Populate(res, resolver);
+
+var dataYml = PathUtils.FindFileAmongParents("/FFXIVClientStructs/ida/data.yml");
+if (dataYml != null)
+    new DataYmlImport().Populate(res, dataYml);
+else
+    Debug.WriteLine("Failed to find data.yml");
+
+res.ValidateUniqueEaName();
+res.ValidateLayout();
+res.Write(outDir.FullName + "/info.yml");

idapopulate/idapopulate/Result.cs

@@ -0,0 +1,211 @@
+using YamlDotNet.Serialization.NamingConventions;
+using YamlDotNet.Serialization;
+using System.Diagnostics;
+
+namespace idapopulate;
+
+// assumption about vtables:
+// - we don't support virtual bases (for now?)
+// - trivial: a struct can have 0 vtables - if neither it nor its bases have virtual functions
+// - trivial: a struct with virtual functions has a vptr at offset 0; a struct with a base has base at offset 0; vptr is also natually at offset 0
+//   in IDA we model that as a 'baseclass' special field (and no vptr field) -or- explicit vptr field at offset 0
+// - edge case: if a base has no virtual functions, but derived has - derived will have vptr at offset 0 and base at offset 8
+//   I don't think IDA really supports this, so we model it as a field rather than a baseclass
+// - trivial: any new vfuncs in derived are added to the vtable - we model that as derived-vtable having base-vtable as a base class
+// - trivial: if there are several bases, these are in order
+//   in IDA we model that as several 'baseclass' special fields ('baseclass_X', where X is offset)
+// - tricky: any new vfuncs are added only to the main vtable (at offset 0); so we don't need to create extra types for vtables from secondary bases (these contain thunks)
+internal class Result
+{
+    public class EnumValue
+    {
+        public string Name = "";
+        public long Value;
+
+        public override string ToString() => $"{Name} = 0x{Value:X}";
+    }
+
+    public class Enum
+    {
+        public bool IsBitfield;
+        public bool IsSigned;
+        public int Width;
+        public List<EnumValue> Values = new(); // sorted by value
+
+        public override string ToString() => $"{(IsSigned ? "signed" : "unsigned")} {Width}-byte-wide {(IsBitfield ? "bitfield" : "enum")}";
+    }
+
+    public class Address
+    {
+        public string Sig;
+        public int SigOffset;
+
+        public Address(string sig, int sigOffset = 0)
+        {
+            Sig = sig;
+            SigOffset = sigOffset;
+        }
+
+        public override string ToString() => $"'{Sig}' +{SigOffset}";
+    }
+
+    public class FuncArg
+    {
+        public string Type = "";
+        public string Name = "";
+
+        public override string ToString() => $"{Type} {Name}";
+    }
+
+    public class FuncSig
+    {
+        public string RetType = "";
+        public List<FuncArg> Arguments = new();
+
+        public override string ToString() => $"({string.Join(", ", Arguments)}) -> {RetType}";
+    }
+
+    public class Function
+    {
+        public string Name = "";
+        public Address? Address;
+        public FuncSig? Signature;
+
+        public override string ToString() => $"auto {Name}{Signature} @ {Address}";
+    }
+
+    public class VTable
+    {
+        public ulong Ea;
+        public Address? Address;
+        public SortedDictionary<uint, Function> VFuncs = new();
+
+        public override string ToString() => $"0x{Ea:X} {Address}";
+    }
+
+    public class SecondaryVTable
+    {
+        public ulong Ea;
+        public string Base = "";
+
+        public override string ToString() => $"0x{Ea:X} {Base}";
+    }
+
+    public class StructBase
+    {
+        public string Type = "";
+        public int Offset;
+        public int Size;
+
+        public override string ToString() => $"[0x{Offset:X}] {Type} (size=0x{Size:X})";
+    }
+
+    public class StructField
+    {
+        public string Name = "";
+        public string Type = "";
+        public bool IsStruct; // if true, type is another struct that has to be defined before this struct
+        public int Offset;
+        public int ArrayLength; // 0 if not an array
+        public int Size;
+
+        public override string ToString() => $"[0x{Offset:X}] {Type} {Name}{(ArrayLength > 0 ? $"[{ArrayLength}]" : "")}{(IsStruct ? " (struct)" : "")} (size=0x{Size:X})";
+    }
+
+    public class Struct
+    {
+        public int Size;
+        public VTable? PrimaryVTable;
+        public List<SecondaryVTable> SecondaryVTables = new();
+        public List<StructBase> Bases = new(); // sorted by offset
+        public List<StructField> Fields = new(); // sorted by offset, non overlapping with bases
+
+        public override string ToString() => $"size=0x{Size:X}";
+    }
+
+    public class Global
+    {
+        public string Type = "";
+        public string Name = "";
+        public Address? Address;
+        public int Size;
+
+        public override string ToString() => $"{Type} {Name} (size=0x{Size:X}) @ {Address}";
+    }
+
+    public SortedDictionary<string, Enum> Enums = new();
+    public SortedDictionary<string, Struct> Structs = new();
+    public SortedDictionary<ulong, Global> Globals = new(); // key = ea
+    public SortedDictionary<ulong, Function> Functions = new(); // key = ea
+
+    public void Write(string path)
+    {
+        var data = new SerializerBuilder().WithNamingConvention(CamelCaseNamingConvention.Instance).Build().Serialize(this);
+        File.WriteAllText(path, data);
+    }
+
+    public bool ValidateUniqueEaName()
+    {
+        Dictionary<ulong, string> uniqueEAs = new();
+        Dictionary<string, string> uniqueNames = new();
+        Func<ulong, string, string, bool> validate = (ea, name, text) =>
+        {
+            bool success = true;
+            if (ea != 0 && !uniqueEAs.TryAdd(ea, text))
+            {
+                Debug.WriteLine($"Duplicate EA 0x{ea:X}: {text} and {uniqueEAs[ea]}");
+                success = false;
+            }
+            if (name.Length > 0 && !uniqueNames.TryAdd(name, text))
+            {
+                Debug.WriteLine($"Duplicate name {name}: {text} and {uniqueNames[name]}");
+                success = false;
+            }
+            return success;
+        };
+
+        bool success = true;
+        foreach (var (ea, g) in Globals)
+            success &= validate(ea, g.Name, $"global {g.Name}");
+        foreach (var (ea, f) in Functions)
+            success &= validate(ea, f.Name, $"function {f.Name}");
+        foreach (var (name, s) in Structs)
+        {
+            if (s.PrimaryVTable != null)
+                success &= validate(s.PrimaryVTable.Ea, $"vtbl_{name}", $"vtbl_{name}");
+            foreach (var vt in s.SecondaryVTables)
+                success &= validate(vt.Ea, $"vtbl_{name}__{vt.Base}", $"vtbl_{name}__{vt.Base}");
+        }
+        return success;
+    }
+
+    public bool ValidateLayout()
+    {
+        bool success = true;
+        foreach (var (name, s) in Structs)
+        {
+            int minFieldOffset = 0;
+            foreach (var b in s.Bases)
+            {
+                if (b.Offset != minFieldOffset)
+                {
+                    Debug.WriteLine($"Structure {name} has incorrect placement for base {b}");
+                    success = false;
+                }
+                minFieldOffset = b.Offset + b.Size;
+            }
+
+            if (s.PrimaryVTable != null)
+            {
+                minFieldOffset = Math.Max(minFieldOffset, 8);
+            }
+
+            if (s.Fields.Count > 0 && s.Fields[0].Offset < minFieldOffset)
+            {
+                Debug.WriteLine($"Structure {name} has first field {s.Fields[0]} overlapping bases or vtable");
+                success = false;
+            }
+        }
+        return success;
+    }
+}

idapopulate/idapopulate/SigResolver.cs

@@ -0,0 +1,42 @@
+using FFXIVClientStructs.Interop;
+using System.Reflection.PortableExecutable;
+
+namespace idapopulate;
+
+internal class SigResolver
+{
+    private ulong _baseAddress;
+    private nint _resolverBase;
+    private SectionHeader _text;
+    private SectionHeader _data;
+    private SectionHeader _rdata;
+
+    public unsafe SigResolver(string exePath, ulong baseAddress = 0x140000000)
+    {
+        _baseAddress = baseAddress;
+
+        var contents = File.ReadAllBytes(exePath);
+        var headers = new PEHeaders(new MemoryStream(contents));
+        _text = headers.SectionHeaders.First(h => h.Name == ".text");
+        _data = headers.SectionHeaders.First(h => h.Name == ".data");
+        _rdata = headers.SectionHeaders.First(h => h.Name == ".rdata");
+        fixed (byte* p = contents)
+        {
+            _resolverBase = (nint)p;
+            Resolver.GetInstance.SetupSearchSpace(_resolverBase, contents.Length, _text.PointerToRawData, _text.SizeOfRawData, _data.PointerToRawData, _data.SizeOfRawData, _rdata.PointerToRawData, _rdata.SizeOfRawData);
+            Resolver.GetInstance.Resolve();
+        }
+    }
+
+    public ulong ToEA(Address address)
+    {
+        if (address.Value == 0)
+            return 0;
+        // note: looking at the resolver code, any found addresses are relative to text section
+        return _baseAddress + (ulong)_text.VirtualAddress + (ulong)((nint)address.Value - _resolverBase - _text.PointerToRawData);
+        //var rva = (nint)address.Value - _resolverBase;
+        //return ToSectionEA(rva, _text) ?? ToSectionEA(rva, _data) ?? ToSectionEA(rva, _rdata) ?? throw new Exception("Weird resolved address");
+    }
+
+    //private ulong? ToSectionEA(nint rva, SectionHeader header) => rva >= header.PointerToRawData && rva < header.PointerToRawData + header.SizeOfRawData ? _baseAddress + (ulong)header.VirtualAddress + (ulong)(rva - header.PointerToRawData) : null;
+}

idapopulate/idapopulate/idapopulate.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0-windows</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="YamlDotNet" Version="13.1.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FFXIVClientStructs\FFXIVClientStructs\FFXIVClientStructs.csproj" />
+  </ItemGroup>
+
+</Project>

idbtoolkit/ii.py

@@ -0,0 +1,209 @@
+# This is a wrapper for IDA API, it is quite inconvenient to use...
+import idaapi
+import ida_bytes
+import ida_enum
+import ida_funcs
+import ida_lines
+import ida_nalt
+import ida_name
+import ida_struct
+import ida_typeinf
+import ida_xref
+import ida_ua
+import datetime
+from contextlib import contextmanager
+
+# enumerate (ea, name) tuples for all global names in database
+def enumerate_names():
+	for i in range(ida_name.get_nlist_size()):
+		yield (ida_name.get_nlist_ea(i), ida_name.get_nlist_name(i))
+
+# find EA of a global name; return idaapi.BADADDR on failure
+def find_global_name_ea(name):
+	return ida_name.get_name_ea(idaapi.BADADDR, name)
+
+# check whether we have a data offset at given EA
+def ea_has_data_offset(ea):
+	flags = ida_bytes.get_full_flags(ea)
+	return (flags & ida_bytes.FF_DATA) != 0 and ida_bytes.is_off0(flags)
+
+# get instruction at EA; return None or insn_t instance
+def decode_instruction(ea):
+	insn = ida_ua.insn_t()
+	ilen = ida_ua.decode_insn(insn, ea)
+	return insn if ilen > 0 else None
+
+# get instruction's operand as immediate
+def get_instruction_operand_immediate(ea, opIndex):
+	insn = decode_instruction(ea)
+	return insn.ops[opIndex].addr if insn else None
+
+# given EA of a call instruction, return EAs of instructions setting arguments in proper order
+def get_call_argument_assignment_eas(call_ea):
+	return idaapi.get_arg_addrs(call_ea)
+
+# enumerate xrefs to address
+def enumerate_xrefs_to(ea):
+	addr = ida_xref.get_first_cref_to(ea)
+	while addr != idaapi.BADADDR:
+		yield addr
+		addr = ida_xref.get_next_cref_to(ea, addr)
+
+# create typeinfo from cdecl; return None on failure
+# cdecl should not have a trailing semicolon, it is added automatically
+def parse_cdecl(cdecl):
+	tif = ida_typeinf.tinfo_t()
+	return tif if ida_typeinf.parse_decl(tif, None, cdecl + ';', 0) == '' else None
+
+# speed up mass creation of enums or structs
+@contextmanager
+def mass_type_updater(utpFlag):
+	print(f'{datetime.datetime.now()} Starting mass type updates: {utpFlag}')
+	ida_typeinf.begin_type_updating(utpFlag)
+	try:
+		yield None
+	finally:
+		print(f'{datetime.datetime.now()} Comitting mass type updates: {utpFlag}')
+		ida_typeinf.end_type_updating(utpFlag)
+		print(f'{datetime.datetime.now()} Done with mass type updates: {utpFlag}')
+
+# create enum type; returns idaapi.BADADDR on failure
+def add_enum(name, bitfield, signed, width):
+	# flags seem to be undocumented, here's what I found:
+	# signed is 0x20000
+	# hexa is FF_0NUMH | FF_1NUMH (0x1100000), decimal is FF_0NUMD | FF_1NUMD (0x2200000), octal is FF_0NUMO | FF_1NUMO (0x7700000), binary is FF_0NUMB | FF_1NUMB (0x6600000), character is FF_0CHAR | FF_1CHAR (0x3300000)
+	flags = 0x1100000
+	if signed:
+		flags |= 0x20000
+	eid = ida_enum.add_enum(idaapi.BADADDR, name, flags)
+	if eid != idaapi.BADADDR:
+		ida_enum.set_enum_bf(eid, bitfield)
+		ida_enum.set_enum_width(eid, width)
+	return eid
+
+# get structure by name
+def get_struct_by_name(name):
+	return ida_struct.get_struc(ida_struct.get_struc_id(name))
+
+# create struct type; returns struc_t or None on failure
+def add_struct(name, isUnion = False):
+	sid = ida_struct.add_struc(idaapi.BADADDR, name, isUnion)
+	return ida_struct.get_struc(sid) if sid != idaapi.BADADDR else None
+
+# create structure member of primitive types
+def add_struct_member_primitive(s, offset, name, flag, size):
+	return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | flag, None, size) == 0
+
+def add_struct_member_byte(s, offset, name, arraySize = 1):
+	return add_struct_member_primitive(s, offset, name, ida_bytes.FF_BYTE, arraySize)
+
+def add_struct_member_word(s, offset, name, arraySize = 1):
+	return add_struct_member_primitive(s, offset, name, ida_bytes.FF_WORD, arraySize * 2)
+
+def add_struct_member_dword(s, offset, name, arraySize = 1):
+	return add_struct_member_primitive(s, offset, name, ida_bytes.FF_DWORD, arraySize * 4)
+
+def add_struct_member_qword(s, offset, name, arraySize = 1):
+	return add_struct_member_primitive(s, offset, name, ida_bytes.FF_QWORD, arraySize * 8)
+
+# create structure member of pointer type
+def add_struct_member_ptr(s, offset, name, arraySize = 1):
+	opinfo = ida_nalt.opinfo_t()
+	opinfo.ri = ida_nalt.refinfo_t()
+	opinfo.ri.base = opinfo.ri.target = idaapi.BADADDR
+	opinfo.ri.flags = ida_nalt.REF_OFF64
+	return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | ida_bytes.FF_QWORD | ida_bytes.FF_0OFF | ida_bytes.FF_1OFF, opinfo, arraySize * 8) == 0
+
+# create structure member of enumeration type
+def add_struct_member_enum(s, offset, name, type, arraySize = 1):
+	opinfo = ida_nalt.opinfo_t()
+	opinfo.tid = ida_enum.get_enum(type)
+	if opinfo.tid == idaapi.BADADDR:
+		return False
+	w = ida_enum.get_enum_width(opinfo.tid)
+	if w == 1:
+		flag = ida_bytes.FF_BYTE
+	elif w == 2:
+		flag = ida_bytes.FF_WORD
+	elif w == 8:
+		flag = ida_bytes.FF_QWORD
+	else:
+		flag = ida_bytes.FF_DWORD # default
+	return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | flag | ida_bytes.FF_0ENUM | ida_bytes.FF_1ENUM, opinfo, w * arraySize) == 0
+
+# create structure member that is an instance of a different structure; returns success
+def add_struct_member_substruct(s, offset, name, type, arraySize = 1):
+	opinfo = ida_nalt.opinfo_t()
+	opinfo.tid = ida_struct.get_struc_id(type)
+	size = ida_struct.get_struc_size(opinfo.tid) if opinfo.tid != idaapi.BADADDR else 0
+	if size == 0: # note: adding 0-sized struct field leads to problems
+		return False
+	return ida_struct.add_struc_member(s, name, offset, ida_bytes.FF_DATA | ida_bytes.FF_STRUCT, opinfo, size * arraySize) == 0
+
+# create base class field for a structure
+def add_struct_baseclass(s, type):
+	offset = ida_struct.get_struc_size(s)
+	success = add_struct_member_substruct(s, offset, f'baseclass_{hex(offset)[2:]}', type)
+	if success:
+		ida_struct.get_member(s, offset).props |= ida_struct.MF_BASECLASS
+		# TODO: we might need to call save_struc here...
+	return success
+
+# create structure member of custom type
+def add_struct_member_typed(s, offset, name, type):
+	return add_struct_member_byte(s, offset, name) and set_struct_member_by_offset_type(s, offset, type)
+
+def get_struct_member_tinfo(m):
+	tif = ida_typeinf.tinfo_t()
+	return tif if ida_struct.get_member_tinfo(tif, m) else None
+
+# set structure member type info
+def set_struct_member_tinfo(s, m, tif):
+	# note: SET_MEMTI_* flags
+	return ida_struct.set_member_tinfo(s, m, 0, tif, 0) > 0
+
+# set structure member type from string
+def set_struct_member_type(s, m, type):
+	tif = parse_cdecl(type)
+	return set_struct_member_tinfo(s, m, tif) if tif else False
+
+# set structure member type
+def set_struct_member_by_offset_type(s, offset, type):
+	m = ida_struct.get_member(s, offset)
+	return set_struct_member_type(s, m, type) if m else False
+
+# add comment; if one already exists, append as new line (unless existing comment already contains what we're trying to add)
+def add_comment_enum(id, comment, repeatable = False):
+	existing = ida_enum.get_enum_cmt(id, repeatable)
+	if not existing or comment not in existing:
+		ida_enum.set_enum_cmt(id, f'{existing}\n{comment}' if existing else comment, repeatable)
+
+def add_comment_func(func, comment, repeatable = False):
+	existing = ida_funcs.get_func_cmt(func, repeatable)
+	if not existing or comment not in existing:
+		ida_funcs.set_func_cmt(func, f'{existing}\n{comment}' if existing else comment, repeatable)
+
+def add_comment_inline(ea, comment, repeatable = False):
+	existing = ida_bytes.get_cmt(ea, repeatable)
+	if not existing or comment not in existing:
+		ida_bytes.set_cmt(ea, f'{existing}\n{comment}' if existing else comment, repeatable)
+
+def add_comment_outline(ea, comment, posterior = False):
+	line = ida_lines.E_NEXT if posterior else ida_lines.E_PREV
+	while True:
+		existing = ida_lines.get_extra_cmt(ea, line)
+		if not existing:
+			ida_lines.add_extra_cmt(ea, not posterior, comment)
+			return
+		elif comment in existing:
+			return
+		else:
+			line += 1
+
+# func or outline
+def add_comment_ea_auto(ea, comment):
+	func = ida_funcs.get_func(ea)
+	if func:
+		add_comment_func(func, comment)
+	else:
+		add_comment_outline(ea, comment)

idbtoolkit/populate_idb.py

@@ -0,0 +1,554 @@
+import idaapi
+import ida_bytes
+import ida_enum
+import ida_name
+import ida_nalt
+import ida_funcs
+import ida_xref
+import ida_typeinf
+import ida_struct
+import ida_funcs
+import yaml
+import os
+import collections
+import datetime
+import itertools
+
+# not import to simplify development - we want to reload my modules if they change
+idaapi.require('ii')
+import ii
+
+def msg(text):
+	print(f'{datetime.datetime.now()} {text}')
+
+def has_custom_name(ea):
+	name = ida_name.get_ea_name(ea)
+	if not ida_name.is_uname(name):
+		return False # it's an officially IDA dummy name (see https://hex-rays.com/blog/igors-tip-of-the-week-34-dummy-names/)
+	if name.startswith('nullsub_'):
+		return False # nullsub_ is not considered a dummy, but it's autogenerated nonetheless
+	if name.startswith('staticinit_'):
+		return False # staticinit_ is our custom prefix which is functionally a dummy
+	if len(name) >= 2 and name[0] == 'a':
+		byte = ida_bytes.get_byte(ea)
+		if byte < 0x80 and name[1] == chr(byte).upper():
+			return False # string constant: foo => aFoo
+	return True
+
+def set_custom_name(ea, name, sig):
+	if not has_custom_name(ea):
+		dupEA = ii.find_global_name_ea(name)
+		if dupEA != idaapi.BADADDR:
+			msg(f'Skipping rename for {hex(ea)}: same name {name} is already used by {hex(dupEA)}')
+			ii.add_comment_ea_auto(ea, f'duplicate name: {name}')
+		elif ida_name.set_name(ea, name) == 0:
+			# no need to write a message, we get a messagebox anyway...
+			ii.add_comment_ea_auto(ea, f'rename failed: {name}')
+	elif ida_name.get_ea_name(ea) != name:
+		msg(f'Skipping rename for existing name: {hex(ea)} {ida_name.get_ea_name(ea)} -> {name}')
+		ii.add_comment_ea_auto(ea, f'alt name: {name}')
+	if sig:
+		ii.add_comment_ea_auto(ea, f'signature: {sig["sig"]} +{sig["sigOffset"]}')
+
+# ensure specified ea is a function start; returns success (either function already existed or was created)
+def ensure_function_at(ea, message):
+	if ida_funcs.get_func(ea):
+		return True
+	added = ida_funcs.add_func(ea)
+	if message:
+		msg(f'Created new {message} at {hex(ea)}' if added else f'Failed to create {message} at {hex(ea)}')
+	return added
+
+def calc_vtable_length(ea):
+	# assume that length is from start until next name
+	end = ea + 8
+	while ida_name.get_ea_name(end) == '' and ii.ea_has_data_offset(end):
+		end += 8
+	return (end - ea) >> 3
+
+def convert_exported_sig(sig, name = None):
+	args = ', '.join(f'{a["type"]} {a["name"]}' for a in sig['arguments'])
+	return f'{sig["retType"]} {name if name else "func"}({args})'
+
+def apply_function_type(ea, tinfo):
+	func = ida_funcs.get_func(ea)
+	if not func:
+		raise Exception(f'Function not defined')
+	existing = ida_typeinf.print_type(ea, 0)
+	if existing:
+		msg(f'Skipping function type assignment @ {hex(ea)}: {existing} -> {tinfo}')
+		ii.add_comment_func(func, f'alt sig: {tinfo}')
+	elif not ida_typeinf.apply_tinfo(ea, tinfo, 0):
+		raise Exception(f'Apply failed')
+
+# return offset of the base of specified type, or None if not found
+def find_base_offset(structId, baseId):
+	s = ida_struct.get_struc(structId)
+	if not s or baseId == idaapi.BADADDR:
+		return None
+	offset = 0
+	while True:
+		m = ida_struct.get_member(s, offset)
+		if not m or ida_struct.get_member_name(m.id) != f'baseclass_{offset}':
+			return None
+		op = ida_nalt.opinfo_t()
+		ida_struct.retrieve_member_info(op, m)
+		nestedOffset = 0 if op.tid == baseId else find_base_offset(op.tid, baseId)
+		if nestedOffset != None:
+			return offset + nestedOffset
+		offset += ida_struct.get_member_size(m)
+
+def populate_static_initializers():
+	msg('*** Populating static initializers ***')
+
+	def find_main_iniiterm():
+		eaInitterm = ii.find_global_name_ea('_initterm')
+		if eaInitterm == idaapi.BADADDR:
+			raise Exception('Failed to find _initterm address')
+		# there are several _initterm calls, interesting one is from main, others are purely framework ones
+		mainXrefs = [xref for xref in ii.enumerate_xrefs_to(eaInitterm) if ida_funcs.get_func_name(xref).startswith('?__scrt_common_main')]
+		if len(mainXrefs) != 1:
+			raise Exception(f'Found {len(mainXrefs)} calls to _initterm from main, 1 expected')
+		return mainXrefs[0]
+
+	def parse_initterm_arguments(callEA):
+		# assume arguments are of the form 'lea rcx/rdx, addr'
+		args = ii.get_call_argument_assignment_eas(callEA)
+		if len(args) != 2:
+			raise Exception(f'Unexpected args for _initterm call: 2 expected, got {len(args)}')
+		start = ii.get_instruction_operand_immediate(args[0], 1)
+		end = ii.get_instruction_operand_immediate(args[1], 1)
+		# both start and end have 0's, between them have function pointers
+		if start + 8 * calc_vtable_length(start) != end:
+			raise Exception('Unexpected _initterm table contents')
+		return (start, end)
+
+	try:
+		eaInittermCall = find_main_iniiterm()
+		start, end = parse_initterm_arguments(eaInittermCall)
+		for ea in range(start + 8, end, 8):
+			func = ida_bytes.get_qword(ea)
+			ensure_function_at(func, "static initializer")
+			if not has_custom_name(func):
+				ida_name.set_name(func, f'staticinit_{(ea - start) >> 3}')
+	except Exception as e:
+		msg(f'Static initializer error: {e}')
+
+def populate_global_names(data):
+	msg('** Populating exported global names **')
+	for ea, g in data.items():
+		set_custom_name(ea, g['name'], g['address'])
+
+def populate_function_names(data):
+	msg('** Populating exported function names **')
+	for ea, g in data.items():
+		ensure_function_at(ea, "function") # if function is not referenced from anywhere, define one manually
+		set_custom_name(ea, g['name'], g['address'])
+
+def populate_enums(data):
+	msg('** Populating exported enums **')
+
+	def populate_enum(name, isBitfield, isSigned, width, values):
+		if ida_enum.get_enum(name) != idaapi.BADADDR:
+			raise Exception(f'{name} already exists in database')
+		eid = ii.add_enum(name, isBitfield, isSigned, width)
+		if (eid == idaapi.BADADDR):
+			raise Exception(f'Failed to create {name}')
+		for val in values:
+			en = val['name']
+			ev = val['value']
+			qn = f'{name}.{en}' # enum names in ida are global, so qualify them
+			res = ida_enum.add_enum_member(eid, qn, ev, ev if isBitfield else -1)
+			if res != 0:
+				msg(f'Failed to add enum member {name}.{en} = {ev}: {res}')
+				ii.add_comment_enum(eid, f'could not add field {en} = {ev}')
+				
+	with ii.mass_type_updater(ida_typeinf.UTP_ENUM):
+		for name, e in data.items():
+			try:
+				populate_enum(name, e['isBitfield'], e['isSigned'], e['width'], e['values'])
+			except Exception as e:
+				msg(f'Enum error: {e}')
+
+def populate_vtables(data):
+	# vtable population is done in several passes
+	Vtable = collections.namedtuple("VTable", "primaryEA secondaryEAs vFuncs base")
+	vtables = {}  # base always ordered before derived; key = class name
+
+	# first pass: build an ordered set of classes with vtables (base before derived) and assign names for all known addresses
+	# note that we don't immediately calculate vtable size on the off chance that some of the vtables-to-be-renamed has no known xrefs
+	def pass1():
+		msg('** Populating exported vtables: pass 1 **')
+		def populate_vtable(cname):
+			if cname in vtables:
+				return # class already processed, since it's a base of some earlier-defined class
+			cdata = data[cname]
+			primary = cdata['primaryVTable']
+			secondary = cdata['secondaryVTables']
+			bases = cdata['bases']
+			# ensure we process all bases first - both direct and indirect (from secondary vtables) - we might not have correct inheritance chain set up
+			for b in bases:
+				populate_vtable(b['type'])
+			for v in secondary:
+				populate_vtable(v['base'])
+			# if primary base has vtable, this class should have one too
+			# TODO: this should be handled during generation
+			primaryBase = bases[0]['type'] if len(bases) > 0 else None
+			if primaryBase and primaryBase in vtables and not primary:
+				msg(f'Class {cname} has no primary vtable, but has base {bases[0]["type"]} with one')
+				primary = { 'ea': 0, 'address': None, 'vFuncs': [] }
+			if not primary:
+				return # skip, this class has no vtables
+			primaryEA = primary['ea']
+			if primaryEA != 0:
+				set_custom_name(primaryEA, f'vtbl_{cname}', primary['address'])
+			for v in secondary:
+				secEA = v['ea']
+				secBase = v['base']
+				set_custom_name(secEA, f'vtbl_{cname}___{secBase}', None)
+				if secBase in vtables:
+					vtables[secBase].secondaryEAs.append(secEA)
+				else:
+					msg(f'Indirect base {secBase} has no known vtables')
+			vtables[cname] = Vtable(primaryEA, [], primary['vFuncs'], primaryBase)
+		for name in data.keys():
+			populate_vtable(name)
+
+	# second pass: determine vtable sizes and create structures
+	def pass2():
+		msg('** Populating exported vtables: pass 2 **')
+
+		def common_vtable_length(primaryEA, secondaryEAs):
+			primaryLen = calc_vtable_length(primaryEA) if primaryEA != 0 else 0
+			vlen = primaryLen
+			for ea in secondaryEAs:
+				secLen = calc_vtable_length(ea)
+				if vlen == 0:
+					vlen = secLen
+				if primaryLen != 0 and primaryLen != secLen:
+					msg(f'Mismatch between vtable sizes at {hex(primaryEA)} ({primaryLen}) and {hex(ea)} ({secLen})')
+				if vlen == 0 or vlen > secLen:
+					vlen = secLen
+			return vlen
+		
+		def calc_vf_name(vtable, vfuncs, idx):
+			if idx in vfuncs:
+				custom = vfuncs[idx]['name']
+				if not ida_struct.get_member_by_name(vtable, custom):
+					return custom
+				msg(f'Duplicate vtable field {ida_struct.get_struc_name(vtable.id)}.{custom}, using fallback for {idx}')
+			return f'vf{idx}'
+		
+		def create_vtable(cname, vtbl):
+			vlen = common_vtable_length(vtbl.primaryEA, vtbl.secondaryEAs)
+			if vlen == 0:
+				return # don't bother creating a vtable if there are no instances
+
+			# create structure
+			vtable = ii.add_struct(f'{cname}_vtbl')
+			if not vtable:
+				raise Exception(f'Failed to create vtable structure for {cname}')
+
+			# add base, if any
+			if vtbl.base and not ii.add_struct_baseclass(vtable, f'{vtbl.base}_vtbl'):
+				msg(f'Failed to add base for vtable for {cname}')
+
+			# check that all custom vfuncs are in range
+			firstNewVF = ida_struct.get_struc_size(vtable) >> 3
+			for idx in vtbl.vFuncs.keys():
+				if idx < firstNewVF:
+					msg(f'Class {cname} overrides vfunc {idx} inherited from base {vtbl.base}')
+				elif idx >= vlen:
+					msg(f'Class {cname} defines vfunc {idx} which is outside bounds ({vlen})')
+
+			# add fields
+			for idx in range(firstNewVF, vlen):
+				name = calc_vf_name(vtable, vtbl.vFuncs, idx)
+				if not ii.add_struct_member_ptr(vtable, idx << 3, name):
+					msg(f'Failed to add vfunc {idx} to vtable {cname}')
+			#ida_struct.save_struc(vtable)
+
+		with ii.mass_type_updater(ida_typeinf.UTP_STRUCT):
+			for cname, vtbl in vtables.items():
+				try:
+					create_vtable(cname, vtbl)
+				except Exception as e:
+					msg(f'Create vtable error: {e}')
+
+	# third pass: rename functions, add crossrefs for vtable instances
+	# the only reason to split it into separate pass is to do it outside mass type update
+	def pass3():
+		msg('** Populating exported vtables: pass 3 **')
+
+		def create_vtable_instance(vtable, numVFs, ea, prefix, signatures):
+			# note: i feel that creating vtable global is, while correct, makes viewing it slightly worse (not seeing vf offsets etc)
+			# but at very least create custom xref (so that find-refs on vtable struct works)
+			# TODO: reconsider...
+			ida_xref.add_dref(ea, vtable.id, ida_xref.XREF_USER | ida_xref.dr_I)
+
+			for idx in range(0, numVFs):
+				vfuncEA = ida_bytes.get_qword(ea + idx * 8)
+				vfuncName = ida_name.get_ea_name(vfuncEA)
+				if vfuncName == "_purecall":
+					continue # abstract virtual function
+
+				inner = ida_struct.get_innermost_member(vtable, idx * 8)
+				if not inner:
+					msg(f'Failed to find field for vfunc {idx} of {cname}')
+					continue
+				leafName = ida_struct.get_member_name(inner[0].id)
+				if f'.{leafName}' in vfuncName:
+					continue # this function is probably not overridden
+
+				set_custom_name(vfuncEA, f'{prefix}.{leafName}', signatures[idx]['address'] if signatures and idx in signatures else None)
+
+		for cname, vtbl in vtables.items():
+			vtable = ii.get_struct_by_name(f'{cname}_vtbl')
+			numVFs = ida_struct.get_struc_size(vtable) >> 3
+
+			if vtbl.primaryEA != 0:
+				create_vtable_instance(vtable, numVFs, vtbl.primaryEA, cname, vtbl.vFuncs)
+			for ea in vtbl.secondaryEAs:
+				create_vtable_instance(vtable, numVFs, ea, ida_name.get_ea_name(ea)[5:], None) # remove vtbl_ prefix, leaving derived___cname
+
+	pass1()
+	pass2()
+	pass3()
+	return vtables
+
+def populate_structs(data):
+	# structure creation is done in two passes
+	res = {} # base/substruct always ordered before referencing struct; key = class name
+
+	# first pass: build an ordered set of structures (base/subfield before containing structures) and create empty structs
+	# these empty structs can be used as a kind of 'forward declarations' for pointers
+	def pass1():
+		msg('** Populating exported structs: pass 1 **')
+		def populate_struct(cname):
+			if cname in res:
+				return # class already processed, since it's a base/substruct of some earlier-defined class
+			cdata = data[cname]
+			# ensure we process all bases and struct fields first
+			for b in cdata['bases']:
+				populate_struct(b['type'])
+			for f in cdata['fields']:
+				if f['isStruct']:
+					populate_struct(f['type'])
+			res[cname] = cdata # tbd
+			# add struct
+			s = ii.add_struct(cname)
+			if not s:
+				raise Exception(f'Failed to create structure {cname}')
+
+		with ii.mass_type_updater(ida_typeinf.UTP_STRUCT):
+			for name in data.keys():
+				try:
+					populate_struct(name)
+				except Exception as e:
+					msg(f'Struct create error: {e}')
+				
+	# second pass: fill structure bases and fields
+	def pass2():
+		msg('** Populating exported structs: pass 2 **')
+
+		def add_base(s, type, offset, size):
+			curSize = ida_struct.get_struc_size(s)
+			if curSize != offset:
+				# treat this as a warning...
+				msg(f'Unexpected offset for {ida_struct.get_struc_name(s.id)} base {type}: expected {hex(offset)}, got {hex(curSize)}')
+			if not ii.add_struct_baseclass(s, type):
+				msg(f'Failed to add {ida_struct.get_struc_name(s.id)} base {type}')
+				return
+			actualSize = ida_struct.get_member_size(ida_struct.get_member(s, curSize))
+			if actualSize != size:
+				msg(f'Unexpected size for {ida_struct.get_struc_name(s.id)} base {type}: expected {hex(size)}, got {hex(actualSize)}')
+			
+		def add_vptr(s, vtname):
+			if not ii.add_struct_member_ptr(s, 0, "__vftable"):
+				msg(f'Failed to add vtable pointer to {ida_struct.get_struc_name(s.id)}')
+			elif not ii.set_struct_member_by_offset_type(s, 0, vtname + '*' if ida_struct.get_struc_id(vtname) != idaapi.BADADDR else 'void*'):
+				msg(f'Failed to set vtable pointer type for {ida_struct.get_struc_name(s.id)} (vtbl-struct-id={hex(ida_struct.get_struc_id(vtname))})')
+
+		def add_field(s, offset, fdata, checkSize):
+			name = fdata['name']
+			type = fdata['type']
+			arrLen = fdata['arrayLength']
+			if fdata['isStruct']:
+				success = ii.add_struct_member_substruct(s, offset, name, type, arrLen if arrLen > 0 else 1)
+			else:
+				typeSuffix = f'[{arrLen}]' if arrLen > 0 else ''
+				success = ii.add_struct_member_typed(s, offset, name, type + typeSuffix)
+
+			if not success:
+				msg(f'Failed to add field {ida_struct.get_struc_name(s.id)}.{name}')
+			elif checkSize:
+				actualSize = ida_struct.get_member_size(ida_struct.get_member(s, offset))
+				expectedSize = fdata['size']
+				if actualSize != expectedSize:
+					msg(f'Unexpected size for {ida_struct.get_struc_name(s.id)}.{name}: expected {hex(expectedSize)}, got {hex(actualSize)}')
+
+		with ii.mass_type_updater(ida_typeinf.UTP_STRUCT):
+			for cname, cdata in res.items():
+				s = ii.get_struct_by_name(cname)
+				if not s:
+					continue
+				# start with bases (if any)
+				for b in cdata['bases']:
+					add_base(s, b['type'], b['offset'], b['size'])
+				# now add primary vtable, if needed
+				if ida_struct.get_struc_size(s) == 0 and cdata['primaryVTable']:
+					add_vptr(s, cname + '_vtbl')
+				# now add fields
+				for offset, fgroup in itertools.groupby(cdata['fields'], lambda f: f['offset']):
+					if offset < ida_struct.get_struc_size(s):
+						msg(f'Unexpected offset for {cname}+{hex(offset)}, current size if {ida_struct.get_struc_size(s)}')
+						continue
+					flist = [f for f in fgroup] # group can only be iterated over once
+					if len(flist) == 1:
+						add_field(s, offset, flist[0], True)
+					else:
+						uname = f'union{hex(offset)[2:]}'
+						su = ii.add_struct(f'{cname}_{uname}', True)
+						for f in flist:
+							add_field(su, 0, f, False)
+						ii.add_struct_member_substruct(s, offset, uname, f'{cname}_{uname}')
+				# add tail, if structure is larger than last field
+				finalSize = ida_struct.get_struc_size(s)
+				expectedSize = cdata['size']
+				if finalSize > expectedSize:
+					msg(f'Structure {cname} is too large: {hex(finalSize)} > {hex(expectedSize)}')
+				elif finalSize < expectedSize and not ii.add_struct_member_byte(s, finalSize, f'tail_{hex(finalSize)[2:]}', expectedSize - finalSize):
+					msg(f'Failed to extend structure {cname}')
+
+	pass1()
+	pass2()
+
+def populate_global_types(data):
+	msg('** Populating exported global types **')
+
+	def process_global(ea, type, expectedSize):
+		if not type:
+			return 0 # nothing to do, type unknown
+		tif = ii.parse_cdecl(type)
+		if not tif:
+			raise Exception(f'Failed to parse type {type}')
+		actualSize = tif.get_size()
+		if actualSize != expectedSize:
+			msg(f'Mismatched global size {type} @ {hex(ea)}: expected {hex(expectedSize)}, got {hex(actualSize)}')
+		for nameEA, name in ii.enumerate_names():
+			if nameEA > ea and nameEA < ea + actualSize:
+				msg(f'Existing global {name} is now a part of global {type} @ {hex(ea)} at offset {hex(nameEA - ea)}')
+		if not ida_typeinf.apply_tinfo(ea, tif, 0):
+			msg(f'Failed to apply {type} @ {hex(ea)}')
+			return 0
+		return actualSize
+
+	minEA = 0
+	for ea, g in data.items():
+		if ea < minEA:
+			msg(f'Skipping global {g["type"]} {g["name"]} @ {hex(ea)}, since it is a part of another global')
+			continue # this global was already consumed by another global
+
+		minEA = ea
+		try:
+			minEA += process_global(ea, g['type'], g['size'])
+		except Exception as e:
+			msg(f'Global type error: {e}')
+
+def populate_function_types(data):
+	msg('** Populating exported function types **')
+	for ea, f in data.items():
+		sig = f['signature']
+		if not sig:
+			continue
+		tif = ii.parse_cdecl(convert_exported_sig(sig))
+		try:
+			apply_function_type(ea, tif)
+		except Exception as e:
+			msg(f'Failed to apply function type {tif} @ {hex(ea)}: {e}')
+
+def populate_vfunc_types(vtables):
+	msg('** Populating exported virtual function types **')
+
+	def update_vtable_fields(cname, vtable, vtbl):
+		for idx, vfunc in vtbl.vFuncs.items():
+			sig = vfunc['signature']
+			if not sig:
+				continue
+			m = ida_struct.get_member(vtable, idx * 8)
+			if not m or ida_struct.get_member_name(m.id) == 'baseclass_0':
+				continue
+			type = convert_exported_sig(sig, '(*)')
+			if not ii.set_struct_member_type(vtable, m, type):
+				msg(f'Failed to set vtable {cname} entry #{idx} type to {type}')
+
+	def propagate_vfunc_type(eaRef, tinfo, cname, shift):
+		vfuncEA = ida_bytes.get_qword(eaRef)
+		vfuncName = ida_name.get_ea_name(vfuncEA)
+		if vfuncName == "_purecall":
+			return # abstract virtual function
+		# replace 'this' pointer type with proper one
+		try:
+			fi = ida_typeinf.func_type_data_t()
+			if not tinfo.get_func_details(fi):
+				raise Exception('Failed to get func details')
+			elif fi.size() == 0:
+				raise Exception('Func has 0 args')
+			elif fi[0].name != 'this':
+				raise Exception(f'First arg is not this: {fi[0].name}')
+			elif not fi[0].type.is_ptr():
+				raise Exception(f'First arg has unexpected type {fi[0].type}')
+	
+			if shift == 0:
+				fi[0].type = ii.parse_cdecl(cname + '*')
+			elif shift > 0:
+				fi[0].type = ii.parse_cdecl(f'{fi[0].type} __shifted({cname}, {shift})')
+			# else: failed to find base, keep base*
+
+			tifAdj = ida_typeinf.tinfo_t()
+			if not tifAdj.create_func(fi):
+				raise Exception(f'Failed to build updated tinfo: {tifAdj}')
+
+			apply_function_type(vfuncEA, tifAdj)
+		except Exception as e:
+			msg(f'Failed to apply virtual function type {tinfo} @ {hex(vfuncEA)}: {e}')
+
+	def propagate_types_to_instances(cname, vtable, vtbl):
+		numVFs = ida_struct.get_struc_size(vtable) >> 3
+		for idx in range(numVFs):
+			inner = ida_struct.get_innermost_member(vtable, idx * 8)
+			itype = ii.get_struct_member_tinfo(inner[0]) if inner else None
+			if not itype or not itype.is_funcptr():
+				continue
+			itype = ida_typeinf.remove_pointer(itype)
+			if vtbl.primaryEA != 0:
+				propagate_vfunc_type(vtbl.primaryEA + idx * 8, itype, cname, 0)
+			for ea in vtbl.secondaryEAs:
+				derivedName = ida_name.get_ea_name(ea)[5:-3-len(cname)] # remove vtbl_ prefix and ___cname suffix, leaving derived
+				shift = find_base_offset(ida_struct.get_struc_id(derivedName), ida_struct.get_struc_id(cname))
+				propagate_vfunc_type(ea + idx * 8, itype, derivedName, shift if shift != None else -1)
+
+	for cname, vtbl in vtables.items():
+		vtable = ii.get_struct_by_name(f'{cname}_vtbl')
+		if not vtable:
+			continue
+		update_vtable_fields(cname, vtable, vtbl)
+		propagate_types_to_instances(cname, vtable, vtbl)
+
+def populate_exported(yamlName):
+	msg(f'*** Populating exported items from {yamlName} ***')
+	with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), yamlName), 'r') as fd:
+		data = yaml.safe_load(fd)
+	populate_global_names(data['globals'])
+	populate_function_names(data['functions'])
+	populate_enums(data['enums'])
+	vtables = populate_vtables(data['structs'])
+	populate_structs(data['structs'])
+	populate_global_types(data['globals'])
+	populate_function_types(data['functions'])
+	populate_vfunc_types(vtables)
+
+breakpoint()
+populate_static_initializers()
+populate_exported('info.yml')
+msg('*** Finished! ***')

upgrade

@@ -0,0 +1,32 @@
+1. Open new binary, wait for analysis
+2. Names (funcs, globals, vtables): run (alt-f7) idarename.py, fix failures manually
+3. Structures & enums: build & run CExporter, import _arrays.h
+4. EXD
+5. Network
+6. Globals
+7. Signatures
+
+consider this flow:
+1. c# parses ffxivclientstructs and extracts to yaml:
+ - enums
+ - structs (normal + dependent template specializations)
+  - fields: offset+name+type
+  - methods: name+sig+type
+  - vtable sig
+  - vfuncs: name+ordinal+type
+ - globals (sigs+type)
+2. c# parses lumina and extracts to yaml:
+ - enum SheetID
+ - sheets
+  - bitfield enums
+  - structs
+3. ida-python parses data.yml + ffxivclientstruct yaml + lumina yaml + custom-patches yaml
+ - create enums (from ffxiv/lumina)
+ - create local types / structs (ordering!) (from ffxiv/lumina)
+ - create globals
+  - ea from data.yml (alt: sig from ffxiv)
+  - type from ffxiv
+ - rename global funcs & methods
+  - ea from data.yml (alt: sig from ffxiv)
+  - type from ffxiv
+ - vtables!

vnetlog/vnetlog/PacketDecoder.cs

@@ -170,6 +170,7 @@ public unsafe class PacketDecoder
             ActorControlCategory.SetTarget => $"{ObjStr(targetID)}",
             ActorControlCategory.SetAnimationState => $"#{p1} = {p2}",
             ActorControlCategory.SetModelState => $"{p1}",
+            ActorControlCategory.ForcedMovement => $"dest={Vec3Str(IntToFloatCoords((ushort)p1, (ushort)p2, (ushort)p3))}, rot={IntToFloatAngleDeg((ushort)p4):f1}deg over {p5 * 0.0001:f4}s, type={p6}",
             ActorControlCategory.PlayActionTimeline => $"{p1:X4}",
             ActorControlCategory.EObjSetState => $"{p1:X4}, housing={(p3 != 0 ? p4 : null)}",
             ActorControlCategory.EObjAnimation => $"{p1:X4} {p2:X4}",

vnetlog/vnetlog/ServerIPC.cs

@@ -336,6 +336,7 @@ public enum ActorControlCategory : ushort
     DespawnZoneScreenMsg = 207, // from dissector
     InstanceSelectDlg = 210, // from dissector
     ActorDespawnEffect = 212, // from dissector
+    ForcedMovement = 226,
     CompanionUnlock = 253, // from dissector
     ObtainBarding = 254, // from dissector
     EquipBarding = 255, // from dissector