Fixes

- bumped CS version to contain a bunch of fixes
- correctly strip namespace element if it is the last one
- do not export types marked as obsolete
- removed test.yml generation
- added a pass to dump nested unions
- a bunch of TODOs on python side

.gitignore

@@ -3,3 +3,5 @@ obj/
 bin/
 *.user
 logs/
+idbtoolkit/info.yml
+idbtoolkit/__pycache__/

idapopulate/FFXIVClientStructs

@@ -1 +1 @@
-Subproject commit b67538d428032c0fab9bce8062f2c2b79bc6e7fc
+Subproject commit 9375140795326c929c5eaeb5283082f1eb1909a1

idapopulate/idapopulate/CSImport.cs

@@ -10,7 +10,7 @@ namespace idapopulate;
 
 internal static class CSImportExt
 {
-    public static string WithoutPrefix(this string str, string prefix) => str.StartsWith(prefix) ? str.Substring(prefix.Length) : str;
+    public static string WithoutPrefix(this string str, string prefix, string sep) => str == prefix ? "" : str.StartsWith(prefix + sep) ? str.Substring(prefix.Length + sep.Length) : str;
 
     // if [FieldOffset] is not specified, assume sequential layout...
     public static int GetFieldOffset(this FieldInfo fi) => fi.GetCustomAttribute<FieldOffsetAttribute>()?.Value ?? Marshal.OffsetOf(fi.DeclaringType!, fi.Name).ToInt32();
@@ -64,6 +64,8 @@ internal class CSImport
     {
         if (type.FullName == null)
             return false;
+        if (type.GetCustomAttribute<ObsoleteAttribute>() != null)
+            return false;
         if (!type.FullName.StartsWith("FFXIVClientStructs.FFXIV.") && !type.FullName.StartsWith("FFXIVClientStructs.Havok."))
             return false;
         if (type.DeclaringType != null && (type.Name is "Addresses" or "MemberFunctionPointers" or "StaticAddressPointers" || type.Name == type.DeclaringType.Name + "VTable"))
@@ -266,7 +268,7 @@ internal class CSImport
 
     private string TypeNameComplex(Type type)
     {
-        var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs.").WithoutPrefix("FFXIV.").WithoutPrefix("Havok.").Replace(".", "::") ?? "";
+        var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs", ".").WithoutPrefix("FFXIV", ".").WithoutPrefix("Havok", ".").Replace(".", "::") ?? "";
         var leafName = type.Name;
         if (type.IsGenericType)
         {
@@ -281,7 +283,7 @@ internal class CSImport
         // hack for std
         if (fullName.StartsWith("STD::Std"))
         {
-            fullName = fullName.WithoutPrefix("STD::Std");
+            fullName = fullName.WithoutPrefix("STD::Std", "");
             fullName = "std::"+ fullName.Substring(0, 1).ToLower() + fullName.Substring(1);
         }
         return fullName;

idapopulate/idapopulate/Program.cs

@@ -8,25 +8,12 @@ if (outDir == null)
     return;
 }
 
-var testRes = new Result();
-var testEnum1 = new Result.Enum() { IsSigned = true, Width = 2 };
-testEnum1.Values.Add(new() { Name = "E1V1", Value = 1 });
-testEnum1.Values.Add(new() { Name = "E1V1dup", Value = 1 });
-testEnum1.Values.Add(new() { Name = "E1V2", Value = 2 });
-testEnum1.Values.Add(new() { Name = "E1V3", Value = 10 });
-testRes.Enums["test::Enum1"] = testEnum1;
-var testEnum2 = new Result.Enum() { IsBitfield = true, IsSigned = false, Width = 4 };
-testEnum2.Values.Add(new() { Name = "E2V1", Value = 0x1 });
-testEnum2.Values.Add(new() { Name = "E2V2", Value = 0x4 });
-testRes.Enums["test::Enum2"] = testEnum2;
-testRes.Enums["test::Enum3"] = new() { Width = 1 };
-testRes.Write(outDir.FullName + "/test.yml");
-
 var gameRoot = PathUtils.FindGameRoot();
 var resolver = new SigResolver(gameRoot + "\\ffxiv_dx11.exe");
 
 var res = new Result();
 new CSImport().Populate(res, resolver);
+res.DumpNestedUnions();
 
 var dataYml = PathUtils.FindFileAmongParents("/FFXIVClientStructs/ida/data.yml");
 if (dataYml != null)

idapopulate/idapopulate/Result.cs

@@ -1,6 +1,7 @@
 using YamlDotNet.Serialization.NamingConventions;
 using YamlDotNet.Serialization;
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 
 namespace idapopulate;
 
@@ -144,6 +145,28 @@ internal class Result
         File.WriteAllText(path, data);
     }
 
+    public void DumpNestedUnions()
+    {
+        foreach (var (name, s) in Structs)
+        {
+            if (s.Fields.All(f => f.Offset == 0))
+                continue; // it's a simple union, we handle that on IDA side
+
+            foreach (var g in s.Fields.GroupBy(f => f.Offset))
+            {
+                if (g.Count() > 1)
+                {
+                    var firstType = g.First().Type;
+                    if (!g.All(f => f.Type == firstType))
+                    {
+                        Debug.WriteLine($"Nested union found at {name}+0x{g.Key:X}: {string.Join(", ", g.Select(f => f.Name))}");
+                    }
+                    // else: all elements of the same type, we don't create union for that on IDA side
+                }
+            }
+        }
+    }
+
     public bool ValidateUniqueEaName()
     {
         Dictionary<ulong, string> uniqueEAs = new();

idbtoolkit/populate_idb.py

@@ -145,6 +145,7 @@ def populate_function_names(data):
 		ensure_function_at(ea, "function") # if function is not referenced from anywhere, define one manually
 		set_custom_name(ea, g['name'], g['address'])
 
+# TODO: calculate masks for bitfield values properly
 def populate_enums(data):
 	msg('** Populating exported enums **')
 
@@ -170,6 +171,7 @@ def populate_enums(data):
 			except Exception as e:
 				msg(f'Enum error: {e}')
 
+# TODO: have c# app generate correctly ordered list
 def populate_vtables(data):
 	# vtable population is done in several passes
 	Vtable = collections.namedtuple("VTable", "primaryEA secondaryEAs vFuncs base")
@@ -316,6 +318,9 @@ def populate_vtables(data):
 	pass3()
 	return vtables
 
+# TODO: if all fields are at offset 0, just create union directly
+# TODO: if all fields in the nested union are of the same type, don't create a union, instead just add member comment with alt names
+# TODO: have c# app generate correctly ordered list
 def populate_structs(data):
 	# structure creation is done in two passes
 	res = {} # base/substruct always ordered before referencing struct; key = class name