Basic schema converter

SchemaConverter/ColumnInfo.cs

@@ -0,0 +1,31 @@
+using Lumina.Data.Structs.Excel;
+
+namespace SchemaConverter;
+
+public class ColumnInfo
+{
+	public int BitOffset { get; set; }
+	public string Name { get; set; }
+	public int Index;
+	public string? Type { get; set; } // icon, color etc
+	public ExcelColumnDataType DataType { get; set; }
+	public bool IsArrayMember { get; set; }
+	public int? ArrayIndex { get; set; }
+	public New.Link? Link { get; set; }
+	
+	public ColumnInfo() { }
+
+	public ColumnInfo(Old.Definition def, int index, bool isArrayMember, int? arrayIndex, New.Link link)
+	{
+		var converterType = def.Converter?.Type;
+		var nameSuffix = isArrayMember ? $"[{arrayIndex}]" : "";
+		Name = Util.StripDefinitionName(def.Name);// + nameSuffix;
+		Index = index;
+		Type = converterType;
+		IsArrayMember = isArrayMember;
+		ArrayIndex = arrayIndex;
+		Link = link;
+	}
+
+	public override string ToString() => $"{Name} ({Index}@{BitOffset / 8}&{BitOffset % 8}) {Type} {IsArrayMember} {ArrayIndex}";
+}

SchemaConverter/DetectedArraySpecs.cs

@@ -0,0 +1,9 @@
+namespace SchemaConverter;
+
+public class DetectedArraySpecs
+{
+	public int Count;
+	public int StartOffset { get; set; }
+	public int EndOffset { get; set; }
+	public List<ColumnInfo> Members { get; set; } = new();
+}

SchemaConverter/NewSheetDefinition.cs

@@ -0,0 +1,41 @@
+// ReSharper disable UnusedMember.Global
+// ReSharper disable InconsistentNaming
+namespace SchemaConverter.New;
+
+public enum FieldType
+{
+	scalar,
+	array,
+	icon,
+	modelId,
+	color,
+}
+
+public class Sheet
+{
+	public string Name { get; set; }
+	public string? DisplayField { get; set; }
+	public List<Field> Fields { get; set; }
+}
+
+public class Field
+{
+	public string? Name { get; set; }
+	public int? Count { get; set; }
+	public string? Comment { get; set; }
+	public FieldType Type { get; set; }
+	public List<Field>? Fields { get; set; }
+	public Link? Link { get; set; }
+}
+
+public class Link
+{
+	public List<string> Target { get; set; }
+	public Condition? Condition { get; set; }
+}
+
+public class Condition
+{
+	public string Switch { get; set; }
+	public Dictionary<int, List<string>> Cases { get; set; } 
+}

SchemaConverter/OldSheetDefinition.cs

@@ -0,0 +1,54 @@
+using Newtonsoft.Json;
+
+namespace SchemaConverter.Old;
+
+public class When
+{
+    [JsonProperty( "key" )] public string Key { get; set; }
+    [JsonProperty( "value" )] public int Value { get; set; }
+}
+
+public class Link
+{
+    [JsonProperty( "when" )] public When When { get; set; }
+    [JsonProperty( "project" )] public string Project { get; set; }
+    [JsonProperty( "key" )] public string Key { get; set; }
+    [JsonProperty( "sheet" )] public string LinkedSheet { get; set; }
+    [JsonProperty( "sheets" )] public List< string > Sheets { get; set; }
+}
+
+public class Converter
+{
+    [JsonProperty( "type" )] public string Type { get; set; }
+    [JsonProperty( "target" )] public string Target { get; set; }
+    [JsonProperty( "links" )] public List< Link > Links { get; set; }
+    [JsonProperty( "targets" )] public List< string > Targets { get; set; }
+}
+
+public class Definition
+{
+    [JsonProperty( "index" )] public uint Index { get; set; }
+    [JsonProperty( "name" )] public string? Name { get; set; }
+    [JsonProperty( "converter" )] public Converter Converter { get; set; }
+    [JsonProperty( "type" )] public string Type { get; set; }
+    [JsonProperty( "count" )] public int Count { get; set; }
+    
+    // Valid for repeats only
+    [JsonProperty( "definition" )] public Definition RepeatDefinition { get; set; }
+    
+    // Valid for groups only
+    [JsonProperty( "members" )] public List< Definition > GroupDefinitions { get; set; }
+
+    public string GetName()
+    {
+        return Name ?? $"Unknown{Type.FirstCharToUpper()}{Index}";
+    }
+}
+
+public class Sheet
+{
+    [JsonProperty( "sheet" )] public string SheetName { get; set; }
+    [JsonProperty( "defaultColumn" )] public string? DefaultColumn { get; set; }
+    [JsonProperty( "isGenericReferenceTarget" )] public bool IsGenericReferenceTarget { get; set; }
+    [JsonProperty( "definitions" )] public List< Definition > Definitions { get; set; }
+}

SchemaConverter/SchemaConverter.cs

@@ -0,0 +1,238 @@
+using Lumina;
+using Lumina.Data.Files.Excel;
+using Lumina.Data.Structs.Excel;
+using Newtonsoft.Json;
+using SchemaConverter.New;
+using SchemaConverter.Old;
+
+namespace SchemaConverter;
+
+public class SchemaConverter
+{
+	private static readonly New.Link _genericReferenceLink = new() {Target = new List<string>()};
+
+	private static void EnumerateGenericReferenceTargets(string oldSchemaDir)
+	{
+		var targets = new List<string>();
+		foreach (var oldSchemaPath in Directory.GetFiles(oldSchemaDir, "*.json"))
+		{
+			var oldSchema = JsonConvert.DeserializeObject<Old.Sheet>(File.ReadAllText(oldSchemaPath));
+			if (oldSchema is { IsGenericReferenceTarget: true })
+			{
+				targets.Add(oldSchema.SheetName);
+			}
+		}
+		_genericReferenceLink.Target.AddRange(targets);
+	}
+	
+	public static void Main(string[] args)
+	{
+		// we need 3 args
+		if (args.Length != 3)
+		{
+			Console.WriteLine("Usage: SchemaConverter.exe <game install directory> <schema directory> <output directory>");
+			return;
+		}
+		
+		var gameDir = args[0];
+		var oldSchemaDir = args[1];
+		var newSchemaDir = args[2];
+		
+		var gameData = new GameData(gameDir);
+
+		EnumerateGenericReferenceTargets(oldSchemaDir);
+		
+		foreach (var oldSchemaPath in Directory.GetFiles(oldSchemaDir, "*.json"))
+		{
+			var sheetName = Path.GetFileNameWithoutExtension(oldSchemaPath);
+			var newSchemaPath = Path.Combine(newSchemaDir, $"{sheetName}.yml");
+			Directory.CreateDirectory(Path.GetDirectoryName(newSchemaPath));
+			var exh = gameData.GetFile<ExcelHeaderFile>($"exd/{sheetName}.exh");
+			if (exh == null)
+			{
+				Console.Error.WriteLine($"Sheet {sheetName} does not exist!");
+				continue;
+			}
+			var result = Convert(exh, oldSchemaPath, newSchemaPath);
+			var strResult = result ? "succeeded!" : "failed...";
+			Console.WriteLine($"Conversion of {sheetName} {strResult}");
+		}
+	}
+
+	public static bool Convert(ExcelHeaderFile exh, string oldSchemaPath, string newSchemaPath)
+	{
+		// Loading and validation
+		var oldSchema = JsonConvert.DeserializeObject<Old.Sheet>(File.ReadAllText(oldSchemaPath));
+		if (oldSchema == null)
+		{
+			Console.Error.WriteLine($"Failed to parse old schema for {exh.FilePath}!");
+			return false;
+		}
+		if (oldSchema.Definitions.Count == 0)
+		{
+			Console.WriteLine($"{exh.FilePath.Path} has no column definitions in old schema!");
+		}
+
+		// Load and parse the old schema to supplement exh information
+		var columnInfos = new List<ColumnInfo>();
+		Emit(columnInfos, oldSchema);
+		var definedColumnMap = columnInfos.ToDictionary(c => c.Index, c => c);
+		
+		for (int i = 0; i < exh.ColumnDefinitions.Length; i++)
+		{
+			var definition = exh.ColumnDefinitions[i];
+			if (definedColumnMap.TryGetValue(i, out var columnInfo))
+			{
+				columnInfo.BitOffset = Util.GetBitOffset(exh.ColumnDefinitions[i].Offset, exh.ColumnDefinitions[i].Type);
+				columnInfo.DataType = definition.IsBoolType ? ExcelColumnDataType.Bool : definition.Type;
+				
+				// fixup ulongs lol
+				if (columnInfo.DataType == ExcelColumnDataType.Int64)
+					columnInfo.Type = "quad";
+			}
+			else
+			{
+				columnInfos.Add(
+					new ColumnInfo
+					{
+						Name = $"Unknown{i}",
+						Index = i,
+						Type = definition.Type == ExcelColumnDataType.Int64 ? null : "quad",
+						DataType = definition.IsBoolType ? ExcelColumnDataType.Bool : definition.Type,
+						BitOffset = Util.GetBitOffset(definition.Offset, definition.Type)
+					});
+			}
+		}
+
+		columnInfos.Sort((c1, c2) => c1.BitOffset.CompareTo(c2.BitOffset));
+
+		var name = oldSchema?.SheetName;
+		if (name == null)
+			name = Path.GetFileNameWithoutExtension(exh.FilePath.Path).FirstCharToUpper();
+		var newSchema = new New.Sheet
+		{
+			Name = name,
+			DisplayField = oldSchema?.DefaultColumn,
+			Fields = new List<New.Field>(),
+		};
+		
+		foreach (var col in columnInfos)
+		{
+			var field = new New.Field
+			{
+				Name = col.Name,
+				Type = col.Type switch
+				{
+					"quad" => FieldType.modelId,
+					"icon" => FieldType.icon,
+					"color" => FieldType.color,
+					_ => FieldType.scalar,
+				},
+				Link = col.Link,
+			};
+			newSchema.Fields.Add(field);
+		}
+		
+		var newSchemaStr = SerializeUtil.Serialize(newSchema);
+		File.WriteAllText(newSchemaPath, newSchemaStr);
+		
+		return true;
+	}
+
+	private static void Emit(List<ColumnInfo> infos, Old.Sheet sheet)
+	{
+		var index = 0;
+		foreach (var definition in sheet.Definitions)
+		{
+			if (index != definition.Index)
+			{
+				for (int i = index; i < definition.Index; i++)
+				{
+					infos.Add(new ColumnInfo {Name = $"Unknown{i}", Index = i });
+				}
+				Console.WriteLine($"{sheet.SheetName}: skipped and generated {definition.Index - index} columns from {index} to {definition.Index}");
+			}
+			index = (int)definition.Index;
+			if (definition.Type == null)
+				EmitSingle(infos, definition, false, null, ref index);
+			else if (definition.Type == "repeat")
+				EmitRepeat(infos, definition, ref index);
+			else if (definition.Type == "group")
+				EmitGroup(infos, definition, ref index);
+			else
+				throw new Exception($"Unknown type {definition.Type}!");
+		}
+	}
+
+	private static void EmitSingle(List<ColumnInfo> infos, Old.Definition definition, bool isArray, int? arrayIndex, ref int index)
+	{
+		infos.Add(new ColumnInfo(definition, index++, isArray, arrayIndex, ConvertLink(definition.Converter)));
+	}
+	
+	private static void EmitRepeat(List<ColumnInfo> infos, Old.Definition definition, ref int index)
+	{
+		for (int i = 0; i < definition.Count; i++)
+		{
+			if (definition.RepeatDefinition.Type == null)
+				EmitSingle(infos, definition.RepeatDefinition, true, i, ref index);
+			else if (definition.RepeatDefinition.Type == "repeat")
+				EmitRepeat(infos, definition.RepeatDefinition, ref index);
+			else if (definition.RepeatDefinition.Type == "group")
+				EmitGroup(infos, definition.RepeatDefinition, ref index);
+			else
+				throw new Exception($"Unknown repeat type {definition.Type}!");
+		}
+	}
+
+	private static void EmitGroup(List<ColumnInfo> infos, Old.Definition definition, ref int index)
+	{
+		foreach (var member in definition.GroupDefinitions)
+		{
+			if (member.Type == null)
+				EmitSingle(infos, member, false, null, ref index);
+			else if (member.Type == "repeat")
+				EmitRepeat(infos, member, ref index);
+			else if (member.Type == "group")
+				EmitGroup(infos, member, ref index);
+			else
+				throw new Exception($"Unknown group member type {member.Type}!");
+		}
+	}
+	
+	private static New.Link ConvertLink(Old.Converter oldLink)
+	{
+		if (oldLink == null) return null;
+		
+		var newLink = new New.Link();
+		if (oldLink.Type == "generic")
+		{
+			return _genericReferenceLink;
+		}
+		else if (oldLink.Type == "link")
+		{
+			newLink.Target = new List<string>() {oldLink.Target};
+		}
+		else if (oldLink.Type == "multiref")
+		{
+			newLink.Target = oldLink.Targets;
+		}
+		else if (oldLink.Type == "complexlink")
+		{
+			if (oldLink.Links[0].Project != null)
+			{
+				return null;
+			}
+			newLink.Condition = new Condition();
+			newLink.Condition.Switch = oldLink.Links[0].When.Key;
+			newLink.Condition.Cases = new Dictionary<int, List<string>>();
+			foreach (var oldLinkLink in oldLink.Links)
+				newLink.Condition.Cases.Add(oldLinkLink.When.Value, oldLinkLink.Sheets);
+		}
+		else
+		{
+			return null;
+		}
+
+		return newLink;
+	}
+}

SchemaConverter/SchemaConverter.csproj

@@ -7,4 +7,11 @@
         <Nullable>enable</Nullable>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="Lumina" Version="3.11.0" />
+      <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+      <PackageReference Include="YamlDotNet" Version="13.3.1" />
+    </ItemGroup>
+
+
 </Project>

SchemaConverter/SerializeUtil.cs

@@ -0,0 +1,47 @@
+using YamlDotNet.Core;
+using YamlDotNet.Core.Events;
+using YamlDotNet.Serialization;
+using YamlDotNet.Serialization.EventEmitters;
+using YamlDotNet.Serialization.NamingConventions;
+
+namespace SchemaConverter;
+
+public static class SerializeUtil
+{
+	private static readonly ISerializer _serializer;
+
+	static SerializeUtil()
+	{
+		_serializer = new SerializerBuilder()
+			.WithIndentedSequences()
+			.WithNamingConvention(CamelCaseNamingConvention.Instance)
+			.ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults)
+			// .WithEventEmitter(nextEmitter => new FlowEverythingEmitter(nextEmitter))
+			.Build();
+	}
+
+	public static string Serialize(object o)
+	{
+		return _serializer.Serialize(o);
+	}
+	
+	public class FlowEverythingEmitter : ChainedEventEmitter
+	{
+		public FlowEverythingEmitter(IEventEmitter nextEmitter) : base(nextEmitter) { }
+
+		public override void Emit(MappingStartEventInfo eventInfo, IEmitter emitter)
+		{
+			Console.WriteLine($"Type: {eventInfo.Source.Type} Style: {eventInfo.Source.StaticType} Value: {eventInfo.Source.Value}");
+			
+			eventInfo.Style = MappingStyle.Flow;
+			base.Emit(eventInfo, emitter);
+		}
+
+		public override void Emit(SequenceStartEventInfo eventInfo, IEmitter emitter)
+		{
+			Console.WriteLine($"Type: {eventInfo.Source.Type} StaticType: {eventInfo.Source.StaticType} Value: {eventInfo.Source.Value}");
+			eventInfo.Style = SequenceStyle.Flow;
+			nextEmitter.Emit(eventInfo, emitter);
+		}
+	}
+}

SchemaConverter/Util.cs

@@ -0,0 +1,62 @@
+using Lumina.Data.Structs.Excel;
+
+namespace SchemaConverter;
+
+public static class Util
+{
+	public static string FirstCharToUpper(this string input) =>
+		input switch
+		{
+			null => throw new ArgumentNullException(nameof(input)),
+			"" => throw new ArgumentException($"{nameof(input)} cannot be empty", nameof(input)),
+			_ => string.Concat(input[0].ToString().ToUpper(), input.AsSpan(1))
+		};
+	
+	public static string StripDefinitionName(string str) 
+	{
+		if( string.IsNullOrWhiteSpace(str))
+			return null;
+
+		str = str
+			.Replace("<", "")
+			.Replace(">", "")
+			.Replace("{", "")
+			.Replace("}", "")
+			.Replace("(", "")
+			.Replace(")", "")
+			.Replace("/", "")
+			.Replace("[", "")
+			.Replace("]", "")
+			.Replace(" ", "")
+			.Replace("'", "")
+			.Replace("-", "")
+			.Replace("%", "Pct");
+
+		if(char.IsDigit(str[0]))
+		{
+			var index = str[0] - '0';
+			var words = new[] {"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"};
+
+			str = $"{words[index]}{str[1..]}";
+		}
+
+		return str;
+	}
+	
+	public static int GetBitOffset(int offset, ExcelColumnDataType dataType)
+	{
+		var bitOffset = offset * 8;
+		return dataType switch
+		{
+			ExcelColumnDataType.PackedBool0 => bitOffset + 0,
+			ExcelColumnDataType.PackedBool1 => bitOffset + 1,
+			ExcelColumnDataType.PackedBool2 => bitOffset + 2,
+			ExcelColumnDataType.PackedBool3 => bitOffset + 3,
+			ExcelColumnDataType.PackedBool4 => bitOffset + 4,
+			ExcelColumnDataType.PackedBool5 => bitOffset + 5,
+			ExcelColumnDataType.PackedBool6 => bitOffset + 6,
+			ExcelColumnDataType.PackedBool7 => bitOffset + 7,
+			_ => bitOffset,
+		};
+	}
+}