From 1d4f9ff7f3a30ad7fff239280e7afbba66a9cf6c Mon Sep 17 00:00:00 2001 From: Andrew Gilewsky Date: Wed, 10 May 2023 21:59:01 +0100 Subject: [PATCH] Fixes - bumped CS version to contain a bunch of fixes - correctly strip namespace element if it is the last one - do not export types marked as obsolete - removed test.yml generation - added a pass to dump nested unions - a bunch of TODOs on python side --- .gitignore | 2 ++ idapopulate/FFXIVClientStructs | 2 +- idapopulate/idapopulate/CSImport.cs | 8 +++++--- idapopulate/idapopulate/Program.cs | 15 +-------------- idapopulate/idapopulate/Result.cs | 23 +++++++++++++++++++++++ idbtoolkit/populate_idb.py | 5 +++++ 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 2aabaab..e1002f5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ obj/ bin/ *.user logs/ +idbtoolkit/info.yml +idbtoolkit/__pycache__/ diff --git a/idapopulate/FFXIVClientStructs b/idapopulate/FFXIVClientStructs index b67538d..9375140 160000 --- a/idapopulate/FFXIVClientStructs +++ b/idapopulate/FFXIVClientStructs @@ -1 +1 @@ -Subproject commit b67538d428032c0fab9bce8062f2c2b79bc6e7fc +Subproject commit 9375140795326c929c5eaeb5283082f1eb1909a1 diff --git a/idapopulate/idapopulate/CSImport.cs b/idapopulate/idapopulate/CSImport.cs index e29b2f2..635da64 100644 --- a/idapopulate/idapopulate/CSImport.cs +++ b/idapopulate/idapopulate/CSImport.cs @@ -10,7 +10,7 @@ namespace idapopulate; internal static class CSImportExt { - public static string WithoutPrefix(this string str, string prefix) => str.StartsWith(prefix) ? str.Substring(prefix.Length) : str; + public static string WithoutPrefix(this string str, string prefix, string sep) => str == prefix ? "" : str.StartsWith(prefix + sep) ? str.Substring(prefix.Length + sep.Length) : str; // if [FieldOffset] is not specified, assume sequential layout... public static int GetFieldOffset(this FieldInfo fi) => fi.GetCustomAttribute()?.Value ?? Marshal.OffsetOf(fi.DeclaringType!, fi.Name).ToInt32(); @@ -64,6 +64,8 @@ internal class CSImport { if (type.FullName == null) return false; + if (type.GetCustomAttribute() != null) + return false; if (!type.FullName.StartsWith("FFXIVClientStructs.FFXIV.") && !type.FullName.StartsWith("FFXIVClientStructs.Havok.")) return false; if (type.DeclaringType != null && (type.Name is "Addresses" or "MemberFunctionPointers" or "StaticAddressPointers" || type.Name == type.DeclaringType.Name + "VTable")) @@ -266,7 +268,7 @@ internal class CSImport private string TypeNameComplex(Type type) { - var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs.").WithoutPrefix("FFXIV.").WithoutPrefix("Havok.").Replace(".", "::") ?? ""; + var baseName = type.DeclaringType != null ? TypeNameComplex(type.DeclaringType) : type.Namespace?.WithoutPrefix("FFXIVClientStructs", ".").WithoutPrefix("FFXIV", ".").WithoutPrefix("Havok", ".").Replace(".", "::") ?? ""; var leafName = type.Name; if (type.IsGenericType) { @@ -281,7 +283,7 @@ internal class CSImport // hack for std if (fullName.StartsWith("STD::Std")) { - fullName = fullName.WithoutPrefix("STD::Std"); + fullName = fullName.WithoutPrefix("STD::Std", ""); fullName = "std::"+ fullName.Substring(0, 1).ToLower() + fullName.Substring(1); } return fullName; diff --git a/idapopulate/idapopulate/Program.cs b/idapopulate/idapopulate/Program.cs index f0da2b1..e10718d 100644 --- a/idapopulate/idapopulate/Program.cs +++ b/idapopulate/idapopulate/Program.cs @@ -8,25 +8,12 @@ if (outDir == null) return; } -var testRes = new Result(); -var testEnum1 = new Result.Enum() { IsSigned = true, Width = 2 }; -testEnum1.Values.Add(new() { Name = "E1V1", Value = 1 }); -testEnum1.Values.Add(new() { Name = "E1V1dup", Value = 1 }); -testEnum1.Values.Add(new() { Name = "E1V2", Value = 2 }); -testEnum1.Values.Add(new() { Name = "E1V3", Value = 10 }); -testRes.Enums["test::Enum1"] = testEnum1; -var testEnum2 = new Result.Enum() { IsBitfield = true, IsSigned = false, Width = 4 }; -testEnum2.Values.Add(new() { Name = "E2V1", Value = 0x1 }); -testEnum2.Values.Add(new() { Name = "E2V2", Value = 0x4 }); -testRes.Enums["test::Enum2"] = testEnum2; -testRes.Enums["test::Enum3"] = new() { Width = 1 }; -testRes.Write(outDir.FullName + "/test.yml"); - var gameRoot = PathUtils.FindGameRoot(); var resolver = new SigResolver(gameRoot + "\\ffxiv_dx11.exe"); var res = new Result(); new CSImport().Populate(res, resolver); +res.DumpNestedUnions(); var dataYml = PathUtils.FindFileAmongParents("/FFXIVClientStructs/ida/data.yml"); if (dataYml != null) diff --git a/idapopulate/idapopulate/Result.cs b/idapopulate/idapopulate/Result.cs index fb7f57e..52c528b 100644 --- a/idapopulate/idapopulate/Result.cs +++ b/idapopulate/idapopulate/Result.cs @@ -1,6 +1,7 @@ using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.Serialization; using System.Diagnostics; +using System.Runtime.InteropServices; namespace idapopulate; @@ -144,6 +145,28 @@ internal class Result File.WriteAllText(path, data); } + public void DumpNestedUnions() + { + foreach (var (name, s) in Structs) + { + if (s.Fields.All(f => f.Offset == 0)) + continue; // it's a simple union, we handle that on IDA side + + foreach (var g in s.Fields.GroupBy(f => f.Offset)) + { + if (g.Count() > 1) + { + var firstType = g.First().Type; + if (!g.All(f => f.Type == firstType)) + { + Debug.WriteLine($"Nested union found at {name}+0x{g.Key:X}: {string.Join(", ", g.Select(f => f.Name))}"); + } + // else: all elements of the same type, we don't create union for that on IDA side + } + } + } + } + public bool ValidateUniqueEaName() { Dictionary uniqueEAs = new(); diff --git a/idbtoolkit/populate_idb.py b/idbtoolkit/populate_idb.py index 02b0352..6e30a0c 100644 --- a/idbtoolkit/populate_idb.py +++ b/idbtoolkit/populate_idb.py @@ -145,6 +145,7 @@ def populate_function_names(data): ensure_function_at(ea, "function") # if function is not referenced from anywhere, define one manually set_custom_name(ea, g['name'], g['address']) +# TODO: calculate masks for bitfield values properly def populate_enums(data): msg('** Populating exported enums **') @@ -170,6 +171,7 @@ def populate_enums(data): except Exception as e: msg(f'Enum error: {e}') +# TODO: have c# app generate correctly ordered list def populate_vtables(data): # vtable population is done in several passes Vtable = collections.namedtuple("VTable", "primaryEA secondaryEAs vFuncs base") @@ -316,6 +318,9 @@ def populate_vtables(data): pass3() return vtables +# TODO: if all fields are at offset 0, just create union directly +# TODO: if all fields in the nested union are of the same type, don't create a union, instead just add member comment with alt names +# TODO: have c# app generate correctly ordered list def populate_structs(data): # structure creation is done in two passes res = {} # base/substruct always ordered before referencing struct; key = class name