From 1c663a7f1ff777872288d3c17d7c683614c3c613 Mon Sep 17 00:00:00 2001 From: liam Date: Mon, 18 Sep 2023 19:55:35 -0400 Subject: [PATCH] Basic schema converter --- SchemaConverter/ColumnInfo.cs | 31 ++++ SchemaConverter/DetectedArraySpecs.cs | 9 + SchemaConverter/NewSheetDefinition.cs | 41 +++++ SchemaConverter/OldSheetDefinition.cs | 54 ++++++ SchemaConverter/SchemaConverter.cs | 238 +++++++++++++++++++++++++ SchemaConverter/SchemaConverter.csproj | 7 + SchemaConverter/SerializeUtil.cs | 47 +++++ SchemaConverter/Util.cs | 62 +++++++ 8 files changed, 489 insertions(+) create mode 100644 SchemaConverter/ColumnInfo.cs create mode 100644 SchemaConverter/DetectedArraySpecs.cs create mode 100644 SchemaConverter/NewSheetDefinition.cs create mode 100644 SchemaConverter/OldSheetDefinition.cs create mode 100644 SchemaConverter/SchemaConverter.cs create mode 100644 SchemaConverter/SerializeUtil.cs create mode 100644 SchemaConverter/Util.cs diff --git a/SchemaConverter/ColumnInfo.cs b/SchemaConverter/ColumnInfo.cs new file mode 100644 index 0000000..7ce153b --- /dev/null +++ b/SchemaConverter/ColumnInfo.cs @@ -0,0 +1,31 @@ +using Lumina.Data.Structs.Excel; + +namespace SchemaConverter; + +public class ColumnInfo +{ + public int BitOffset { get; set; } + public string Name { get; set; } + public int Index; + public string? Type { get; set; } // icon, color etc + public ExcelColumnDataType DataType { get; set; } + public bool IsArrayMember { get; set; } + public int? ArrayIndex { get; set; } + public New.Link? Link { get; set; } + + public ColumnInfo() { } + + public ColumnInfo(Old.Definition def, int index, bool isArrayMember, int? arrayIndex, New.Link link) + { + var converterType = def.Converter?.Type; + var nameSuffix = isArrayMember ? $"[{arrayIndex}]" : ""; + Name = Util.StripDefinitionName(def.Name);// + nameSuffix; + Index = index; + Type = converterType; + IsArrayMember = isArrayMember; + ArrayIndex = arrayIndex; + Link = link; + } + + public override string ToString() => $"{Name} ({Index}@{BitOffset / 8}&{BitOffset % 8}) {Type} {IsArrayMember} {ArrayIndex}"; +} \ No newline at end of file diff --git a/SchemaConverter/DetectedArraySpecs.cs b/SchemaConverter/DetectedArraySpecs.cs new file mode 100644 index 0000000..dcde597 --- /dev/null +++ b/SchemaConverter/DetectedArraySpecs.cs @@ -0,0 +1,9 @@ +namespace SchemaConverter; + +public class DetectedArraySpecs +{ + public int Count; + public int StartOffset { get; set; } + public int EndOffset { get; set; } + public List Members { get; set; } = new(); +} \ No newline at end of file diff --git a/SchemaConverter/NewSheetDefinition.cs b/SchemaConverter/NewSheetDefinition.cs new file mode 100644 index 0000000..1e73326 --- /dev/null +++ b/SchemaConverter/NewSheetDefinition.cs @@ -0,0 +1,41 @@ +// ReSharper disable UnusedMember.Global +// ReSharper disable InconsistentNaming +namespace SchemaConverter.New; + +public enum FieldType +{ + scalar, + array, + icon, + modelId, + color, +} + +public class Sheet +{ + public string Name { get; set; } + public string? DisplayField { get; set; } + public List Fields { get; set; } +} + +public class Field +{ + public string? Name { get; set; } + public int? Count { get; set; } + public string? Comment { get; set; } + public FieldType Type { get; set; } + public List? Fields { get; set; } + public Link? Link { get; set; } +} + +public class Link +{ + public List Target { get; set; } + public Condition? Condition { get; set; } +} + +public class Condition +{ + public string Switch { get; set; } + public Dictionary> Cases { get; set; } +} \ No newline at end of file diff --git a/SchemaConverter/OldSheetDefinition.cs b/SchemaConverter/OldSheetDefinition.cs new file mode 100644 index 0000000..828d601 --- /dev/null +++ b/SchemaConverter/OldSheetDefinition.cs @@ -0,0 +1,54 @@ +using Newtonsoft.Json; + +namespace SchemaConverter.Old; + +public class When +{ + [JsonProperty( "key" )] public string Key { get; set; } + [JsonProperty( "value" )] public int Value { get; set; } +} + +public class Link +{ + [JsonProperty( "when" )] public When When { get; set; } + [JsonProperty( "project" )] public string Project { get; set; } + [JsonProperty( "key" )] public string Key { get; set; } + [JsonProperty( "sheet" )] public string LinkedSheet { get; set; } + [JsonProperty( "sheets" )] public List< string > Sheets { get; set; } +} + +public class Converter +{ + [JsonProperty( "type" )] public string Type { get; set; } + [JsonProperty( "target" )] public string Target { get; set; } + [JsonProperty( "links" )] public List< Link > Links { get; set; } + [JsonProperty( "targets" )] public List< string > Targets { get; set; } +} + +public class Definition +{ + [JsonProperty( "index" )] public uint Index { get; set; } + [JsonProperty( "name" )] public string? Name { get; set; } + [JsonProperty( "converter" )] public Converter Converter { get; set; } + [JsonProperty( "type" )] public string Type { get; set; } + [JsonProperty( "count" )] public int Count { get; set; } + + // Valid for repeats only + [JsonProperty( "definition" )] public Definition RepeatDefinition { get; set; } + + // Valid for groups only + [JsonProperty( "members" )] public List< Definition > GroupDefinitions { get; set; } + + public string GetName() + { + return Name ?? $"Unknown{Type.FirstCharToUpper()}{Index}"; + } +} + +public class Sheet +{ + [JsonProperty( "sheet" )] public string SheetName { get; set; } + [JsonProperty( "defaultColumn" )] public string? DefaultColumn { get; set; } + [JsonProperty( "isGenericReferenceTarget" )] public bool IsGenericReferenceTarget { get; set; } + [JsonProperty( "definitions" )] public List< Definition > Definitions { get; set; } +} \ No newline at end of file diff --git a/SchemaConverter/SchemaConverter.cs b/SchemaConverter/SchemaConverter.cs new file mode 100644 index 0000000..455b945 --- /dev/null +++ b/SchemaConverter/SchemaConverter.cs @@ -0,0 +1,238 @@ +using Lumina; +using Lumina.Data.Files.Excel; +using Lumina.Data.Structs.Excel; +using Newtonsoft.Json; +using SchemaConverter.New; +using SchemaConverter.Old; + +namespace SchemaConverter; + +public class SchemaConverter +{ + private static readonly New.Link _genericReferenceLink = new() {Target = new List()}; + + private static void EnumerateGenericReferenceTargets(string oldSchemaDir) + { + var targets = new List(); + foreach (var oldSchemaPath in Directory.GetFiles(oldSchemaDir, "*.json")) + { + var oldSchema = JsonConvert.DeserializeObject(File.ReadAllText(oldSchemaPath)); + if (oldSchema is { IsGenericReferenceTarget: true }) + { + targets.Add(oldSchema.SheetName); + } + } + _genericReferenceLink.Target.AddRange(targets); + } + + public static void Main(string[] args) + { + // we need 3 args + if (args.Length != 3) + { + Console.WriteLine("Usage: SchemaConverter.exe "); + return; + } + + var gameDir = args[0]; + var oldSchemaDir = args[1]; + var newSchemaDir = args[2]; + + var gameData = new GameData(gameDir); + + EnumerateGenericReferenceTargets(oldSchemaDir); + + foreach (var oldSchemaPath in Directory.GetFiles(oldSchemaDir, "*.json")) + { + var sheetName = Path.GetFileNameWithoutExtension(oldSchemaPath); + var newSchemaPath = Path.Combine(newSchemaDir, $"{sheetName}.yml"); + Directory.CreateDirectory(Path.GetDirectoryName(newSchemaPath)); + var exh = gameData.GetFile($"exd/{sheetName}.exh"); + if (exh == null) + { + Console.Error.WriteLine($"Sheet {sheetName} does not exist!"); + continue; + } + var result = Convert(exh, oldSchemaPath, newSchemaPath); + var strResult = result ? "succeeded!" : "failed..."; + Console.WriteLine($"Conversion of {sheetName} {strResult}"); + } + } + + public static bool Convert(ExcelHeaderFile exh, string oldSchemaPath, string newSchemaPath) + { + // Loading and validation + var oldSchema = JsonConvert.DeserializeObject(File.ReadAllText(oldSchemaPath)); + if (oldSchema == null) + { + Console.Error.WriteLine($"Failed to parse old schema for {exh.FilePath}!"); + return false; + } + if (oldSchema.Definitions.Count == 0) + { + Console.WriteLine($"{exh.FilePath.Path} has no column definitions in old schema!"); + } + + // Load and parse the old schema to supplement exh information + var columnInfos = new List(); + Emit(columnInfos, oldSchema); + var definedColumnMap = columnInfos.ToDictionary(c => c.Index, c => c); + + for (int i = 0; i < exh.ColumnDefinitions.Length; i++) + { + var definition = exh.ColumnDefinitions[i]; + if (definedColumnMap.TryGetValue(i, out var columnInfo)) + { + columnInfo.BitOffset = Util.GetBitOffset(exh.ColumnDefinitions[i].Offset, exh.ColumnDefinitions[i].Type); + columnInfo.DataType = definition.IsBoolType ? ExcelColumnDataType.Bool : definition.Type; + + // fixup ulongs lol + if (columnInfo.DataType == ExcelColumnDataType.Int64) + columnInfo.Type = "quad"; + } + else + { + columnInfos.Add( + new ColumnInfo + { + Name = $"Unknown{i}", + Index = i, + Type = definition.Type == ExcelColumnDataType.Int64 ? null : "quad", + DataType = definition.IsBoolType ? ExcelColumnDataType.Bool : definition.Type, + BitOffset = Util.GetBitOffset(definition.Offset, definition.Type) + }); + } + } + + columnInfos.Sort((c1, c2) => c1.BitOffset.CompareTo(c2.BitOffset)); + + var name = oldSchema?.SheetName; + if (name == null) + name = Path.GetFileNameWithoutExtension(exh.FilePath.Path).FirstCharToUpper(); + var newSchema = new New.Sheet + { + Name = name, + DisplayField = oldSchema?.DefaultColumn, + Fields = new List(), + }; + + foreach (var col in columnInfos) + { + var field = new New.Field + { + Name = col.Name, + Type = col.Type switch + { + "quad" => FieldType.modelId, + "icon" => FieldType.icon, + "color" => FieldType.color, + _ => FieldType.scalar, + }, + Link = col.Link, + }; + newSchema.Fields.Add(field); + } + + var newSchemaStr = SerializeUtil.Serialize(newSchema); + File.WriteAllText(newSchemaPath, newSchemaStr); + + return true; + } + + private static void Emit(List infos, Old.Sheet sheet) + { + var index = 0; + foreach (var definition in sheet.Definitions) + { + if (index != definition.Index) + { + for (int i = index; i < definition.Index; i++) + { + infos.Add(new ColumnInfo {Name = $"Unknown{i}", Index = i }); + } + Console.WriteLine($"{sheet.SheetName}: skipped and generated {definition.Index - index} columns from {index} to {definition.Index}"); + } + index = (int)definition.Index; + if (definition.Type == null) + EmitSingle(infos, definition, false, null, ref index); + else if (definition.Type == "repeat") + EmitRepeat(infos, definition, ref index); + else if (definition.Type == "group") + EmitGroup(infos, definition, ref index); + else + throw new Exception($"Unknown type {definition.Type}!"); + } + } + + private static void EmitSingle(List infos, Old.Definition definition, bool isArray, int? arrayIndex, ref int index) + { + infos.Add(new ColumnInfo(definition, index++, isArray, arrayIndex, ConvertLink(definition.Converter))); + } + + private static void EmitRepeat(List infos, Old.Definition definition, ref int index) + { + for (int i = 0; i < definition.Count; i++) + { + if (definition.RepeatDefinition.Type == null) + EmitSingle(infos, definition.RepeatDefinition, true, i, ref index); + else if (definition.RepeatDefinition.Type == "repeat") + EmitRepeat(infos, definition.RepeatDefinition, ref index); + else if (definition.RepeatDefinition.Type == "group") + EmitGroup(infos, definition.RepeatDefinition, ref index); + else + throw new Exception($"Unknown repeat type {definition.Type}!"); + } + } + + private static void EmitGroup(List infos, Old.Definition definition, ref int index) + { + foreach (var member in definition.GroupDefinitions) + { + if (member.Type == null) + EmitSingle(infos, member, false, null, ref index); + else if (member.Type == "repeat") + EmitRepeat(infos, member, ref index); + else if (member.Type == "group") + EmitGroup(infos, member, ref index); + else + throw new Exception($"Unknown group member type {member.Type}!"); + } + } + + private static New.Link ConvertLink(Old.Converter oldLink) + { + if (oldLink == null) return null; + + var newLink = new New.Link(); + if (oldLink.Type == "generic") + { + return _genericReferenceLink; + } + else if (oldLink.Type == "link") + { + newLink.Target = new List() {oldLink.Target}; + } + else if (oldLink.Type == "multiref") + { + newLink.Target = oldLink.Targets; + } + else if (oldLink.Type == "complexlink") + { + if (oldLink.Links[0].Project != null) + { + return null; + } + newLink.Condition = new Condition(); + newLink.Condition.Switch = oldLink.Links[0].When.Key; + newLink.Condition.Cases = new Dictionary>(); + foreach (var oldLinkLink in oldLink.Links) + newLink.Condition.Cases.Add(oldLinkLink.When.Value, oldLinkLink.Sheets); + } + else + { + return null; + } + + return newLink; + } +} \ No newline at end of file diff --git a/SchemaConverter/SchemaConverter.csproj b/SchemaConverter/SchemaConverter.csproj index 2b14c81..728bf7c 100644 --- a/SchemaConverter/SchemaConverter.csproj +++ b/SchemaConverter/SchemaConverter.csproj @@ -7,4 +7,11 @@ enable + + + + + + + diff --git a/SchemaConverter/SerializeUtil.cs b/SchemaConverter/SerializeUtil.cs new file mode 100644 index 0000000..3342084 --- /dev/null +++ b/SchemaConverter/SerializeUtil.cs @@ -0,0 +1,47 @@ +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.EventEmitters; +using YamlDotNet.Serialization.NamingConventions; + +namespace SchemaConverter; + +public static class SerializeUtil +{ + private static readonly ISerializer _serializer; + + static SerializeUtil() + { + _serializer = new SerializerBuilder() + .WithIndentedSequences() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) + // .WithEventEmitter(nextEmitter => new FlowEverythingEmitter(nextEmitter)) + .Build(); + } + + public static string Serialize(object o) + { + return _serializer.Serialize(o); + } + + public class FlowEverythingEmitter : ChainedEventEmitter + { + public FlowEverythingEmitter(IEventEmitter nextEmitter) : base(nextEmitter) { } + + public override void Emit(MappingStartEventInfo eventInfo, IEmitter emitter) + { + Console.WriteLine($"Type: {eventInfo.Source.Type} Style: {eventInfo.Source.StaticType} Value: {eventInfo.Source.Value}"); + + eventInfo.Style = MappingStyle.Flow; + base.Emit(eventInfo, emitter); + } + + public override void Emit(SequenceStartEventInfo eventInfo, IEmitter emitter) + { + Console.WriteLine($"Type: {eventInfo.Source.Type} StaticType: {eventInfo.Source.StaticType} Value: {eventInfo.Source.Value}"); + eventInfo.Style = SequenceStyle.Flow; + nextEmitter.Emit(eventInfo, emitter); + } + } +} \ No newline at end of file diff --git a/SchemaConverter/Util.cs b/SchemaConverter/Util.cs new file mode 100644 index 0000000..3f986b7 --- /dev/null +++ b/SchemaConverter/Util.cs @@ -0,0 +1,62 @@ +using Lumina.Data.Structs.Excel; + +namespace SchemaConverter; + +public static class Util +{ + public static string FirstCharToUpper(this string input) => + input switch + { + null => throw new ArgumentNullException(nameof(input)), + "" => throw new ArgumentException($"{nameof(input)} cannot be empty", nameof(input)), + _ => string.Concat(input[0].ToString().ToUpper(), input.AsSpan(1)) + }; + + public static string StripDefinitionName(string str) + { + if( string.IsNullOrWhiteSpace(str)) + return null; + + str = str + .Replace("<", "") + .Replace(">", "") + .Replace("{", "") + .Replace("}", "") + .Replace("(", "") + .Replace(")", "") + .Replace("/", "") + .Replace("[", "") + .Replace("]", "") + .Replace(" ", "") + .Replace("'", "") + .Replace("-", "") + .Replace("%", "Pct"); + + if(char.IsDigit(str[0])) + { + var index = str[0] - '0'; + var words = new[] {"zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"}; + + str = $"{words[index]}{str[1..]}"; + } + + return str; + } + + public static int GetBitOffset(int offset, ExcelColumnDataType dataType) + { + var bitOffset = offset * 8; + return dataType switch + { + ExcelColumnDataType.PackedBool0 => bitOffset + 0, + ExcelColumnDataType.PackedBool1 => bitOffset + 1, + ExcelColumnDataType.PackedBool2 => bitOffset + 2, + ExcelColumnDataType.PackedBool3 => bitOffset + 3, + ExcelColumnDataType.PackedBool4 => bitOffset + 4, + ExcelColumnDataType.PackedBool5 => bitOffset + 5, + ExcelColumnDataType.PackedBool6 => bitOffset + 6, + ExcelColumnDataType.PackedBool7 => bitOffset + 7, + _ => bitOffset, + }; + } +} \ No newline at end of file