Oni2/OniSplit/InstanceFileWriter.cs

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
using Oni.Collections;
using Oni.Metadata;

namespace Oni
{
    internal sealed class InstanceFileWriter
    {
        #region Private data
        private static readonly byte[] padding = new byte[512];
        private static byte[] copyBuffer1 = new byte[32768];
        private static byte[] copyBuffer2 = new byte[32768];
        private StreamCache streamCache;
        private readonly bool bigEndian;
        private readonly Dictionary<string, int> namedInstancedIdMap;
        private readonly FileHeader header;
        private readonly List<DescriptorTableEntry> descriptorTable;
        private NameDescriptorTable nameIndex;
        private TemplateDescriptorTable templateTable;
        private NameTable nameTable;
        private readonly Dictionary<InstanceDescriptor, InstanceDescriptor> sharedMap;
        private readonly Dictionary<InstanceFile, int[]> linkMaps;
        private readonly Dictionary<InstanceFile, Dictionary<int, int>> rawOffsetMaps;
        private readonly Dictionary<InstanceFile, Dictionary<int, int>> sepOffsetMaps;
        private int rawOffset;
        private int sepOffset;
        private List<BinaryPartEntry> rawParts;
        private List<BinaryPartEntry> sepParts;
        #endregion

        #region private class FileHeader

        private class FileHeader
        {
            public const int Size = 64;

            public long TemplateChecksum;
            public int Version;
            public int InstanceCount;
            public int NameCount;
            public int TemplateCount;
            public int DataTableOffset;
            public int DataTableSize;
            public int NameTableOffset;
            public int NameTableSize;
            public int RawTableOffset;
            public int RawTableSize;

            public void Write(BinaryWriter writer)
            {
                writer.Write(TemplateChecksum);
                writer.Write(Version);
                writer.Write(InstanceFileHeader.Signature);
                writer.Write(InstanceCount);
                writer.Write(NameCount);
                writer.Write(TemplateCount);
                writer.Write(DataTableOffset);
                writer.Write(DataTableSize);
                writer.Write(NameTableOffset);
                writer.Write(NameTableSize);
                writer.Write(RawTableOffset);
                writer.Write(RawTableSize);
                writer.Write(0);
                writer.Write(0);
            }
        }

        #endregion
        #region private class DescriptorTableEntry

        private class DescriptorTableEntry
        {
            public const int Size = 20;

            public readonly InstanceDescriptor SourceDescriptor;
            public readonly int Id;
            public int DataOffset;
            public int NameOffset;
            public int DataSize;
            public bool AnimationPositionPointHack;

            public DescriptorTableEntry(int id, InstanceDescriptor descriptor)
            {
                Id = id;
                SourceDescriptor = descriptor;
            }

            public bool HasName => SourceDescriptor.HasName;
            public string Name => SourceDescriptor.FullName;
            public TemplateTag Code => SourceDescriptor.Template.Tag;
            public InstanceFile SourceFile => SourceDescriptor.File;

            public void Write(BinaryWriter writer, bool shared)
            {
                writer.Write((int)Code);
                writer.Write(DataOffset);
                writer.Write(NameOffset);
                writer.Write(DataSize);

                var flags = InstanceDescriptorFlags.None;

                if (!SourceDescriptor.HasName)
                    flags |= InstanceDescriptorFlags.Private;

                if (DataOffset == 0)
                    flags |= InstanceDescriptorFlags.Placeholder;

                if (shared)
                    flags |= InstanceDescriptorFlags.Shared;

                writer.Write((int)flags);
            }
        }

        #endregion
        #region private class NameDescriptorTable

        private class NameDescriptorTable
        {
            private List<Entry> entries;

            #region private class Entry

            private class Entry : IComparable<Entry>
            {
                public const int Size = 8;

                public int InstanceNumber;
                public string Name;

                public void Write(BinaryWriter writer)
                {
                    writer.Write(InstanceNumber);
                    writer.Write(0);
                }

                #region IComparable<NameIndexEntry> Members

                int IComparable<Entry>.CompareTo(Entry other)
                {
                    //
                    // Note: Oni is case sensitive so we need to sort names accordingly.
                    //

                    return string.CompareOrdinal(Name, other.Name);
                }

                #endregion
            }

            #endregion

            public static NameDescriptorTable CreateFromDescriptors(List<DescriptorTableEntry> descriptorTable)
            {
                NameDescriptorTable nameIndex = new NameDescriptorTable();

                nameIndex.entries = new List<Entry>();

                for (int i = 0; i < descriptorTable.Count; i++)
                {
                    DescriptorTableEntry descriptor = descriptorTable[i];

                    if (descriptor.HasName)
                    {
                        Entry entry = new Entry();
                        entry.Name = descriptor.Name;
                        entry.InstanceNumber = i;
                        nameIndex.entries.Add(entry);
                    }
                }

                nameIndex.entries.Sort();

                return nameIndex;
            }

            public int Count => entries.Count;
            public int Size => entries.Count * Entry.Size;

            public void Write(BinaryWriter writer)
            {
                foreach (Entry entry in entries)
                    entry.Write(writer);
            }
        }

        #endregion
        #region private class TemplateDescriptorTable

        private class TemplateDescriptorTable
        {
            private List<Entry> entries;

            #region private class Entry

            private class Entry : IComparable<Entry>
            {
                public const int Size = 16;

                public long Checksum;
                public TemplateTag Code;
                public int Count;

                public void Write(BinaryWriter writer)
                {
                    writer.Write(Checksum);
                    writer.Write((int)Code);
                    writer.Write(Count);
                }

                int IComparable<Entry>.CompareTo(Entry other) => Code.CompareTo(other.Code);
            }

            #endregion

            public static TemplateDescriptorTable CreateFromDescriptors(InstanceMetadata metadata, List<DescriptorTableEntry> descriptorTable)
            {
                Dictionary<TemplateTag, int> templateCount = new Dictionary<TemplateTag, int>();

                foreach (DescriptorTableEntry entry in descriptorTable)
                {
                    int count;
                    templateCount.TryGetValue(entry.Code, out count);
                    templateCount[entry.Code] = count + 1;
                }

                TemplateDescriptorTable templateTable = new TemplateDescriptorTable();
                templateTable.entries = new List<Entry>(templateCount.Count);

                foreach (KeyValuePair<TemplateTag, int> pair in templateCount)
                {
                    Entry entry = new Entry();
                    entry.Checksum = metadata.GetTemplate(pair.Key).Checksum;
                    entry.Code = pair.Key;
                    entry.Count = pair.Value;
                    templateTable.entries.Add(entry);
                }

                templateTable.entries.Sort();

                return templateTable;
            }

            public int Count => entries.Count;

            public int Size => entries.Count * Entry.Size;

            public void Write(BinaryWriter writer)
            {
                foreach (Entry entry in entries)
                    entry.Write(writer);
            }
        }

        #endregion
        #region private class NameTable

        private class NameTable
        {
            private List<string> names;
            private int size;

            public static NameTable CreateFromDescriptors(List<DescriptorTableEntry> descriptors)
            {
                NameTable nameTable = new NameTable();

                nameTable.names = new List<string>();

                int nameTableSize = 0;

                foreach (DescriptorTableEntry descriptor in descriptors)
                {
                    if (!descriptor.HasName)
                        continue;

                    string name = descriptor.Name;

                    nameTable.names.Add(name);
                    descriptor.NameOffset = nameTableSize;
                    nameTableSize += name.Length + 1;

                    if (name.Length > 63)
                        Console.WriteLine("Warning: name '{0}' too long.", name);
                }

                nameTable.size = nameTableSize;

                return nameTable;
            }

            public int Size => size;

            public void Write(BinaryWriter writer)
            {
                byte[] copyBuffer = new byte[256];

                foreach (string name in names)
                {
                    int length = Encoding.UTF8.GetBytes(name, 0, name.Length, copyBuffer, 0);
                    copyBuffer[length] = 0;
                    writer.Write(copyBuffer, 0, length + 1);
                }
            }
        }

        #endregion
        #region private class BinaryPartEntry

        private class BinaryPartEntry : IComparable<BinaryPartEntry>
        {
            public readonly int SourceOffset;
            public readonly string SourceFile;
            public readonly int DestinationOffset;
            public readonly int Size;
            public readonly BinaryPartField Field;

            public BinaryPartEntry(string sourceFile, int sourceOffset, int size, int destinationOffset, Field field)
            {
                SourceFile = sourceFile;
                SourceOffset = sourceOffset;
                Size = size;
                DestinationOffset = destinationOffset;
                Field = (BinaryPartField)field;
            }

            #region IComparable<BinaryPartEntry> Members

            int IComparable<BinaryPartEntry>.CompareTo(BinaryPartEntry other)
            {
                //
                // Sort the binary parts by destination offset in an attempt to streamline the write IO
                //

                return DestinationOffset.CompareTo(other.DestinationOffset);
            }

            #endregion
        }

        #endregion
        #region private class ChecksumStream

        private class ChecksumStream : Stream
        {
            private int checksum;
            private int position;

            public int Checksum => checksum;

            public override bool CanRead => false;
            public override bool CanSeek => false;
            public override bool CanWrite => true;

            public override void Flush()
            {
            }

            public override long Length => position;

            public override long Position
            {
                get
                {
                    return position;
                }
                set
                {
                    throw new NotSupportedException();
                }
            }

            public override int Read(byte[] buffer, int offset, int count)
            {
                throw new NotSupportedException();
            }

            public override long Seek(long offset, SeekOrigin origin)
            {
                throw new NotSupportedException();
            }

            public override void SetLength(long value)
            {
                throw new NotSupportedException();
            }

            public override void Write(byte[] buffer, int offset, int count)
            {
                for (int i = offset; i < offset + count; i++)
                    checksum += buffer[i] ^ (i + position);

                position += count;
            }
        }

        #endregion
        #region private class StreamCache

        private class StreamCache : IDisposable
        {
            private const int maxCacheSize = 32;
            private Dictionary<string, CacheEntry> cacheEntries = new Dictionary<string, CacheEntry>();

            private class CacheEntry
            {
                public BinaryReader Stream;
                public long LastTimeUsed;
            }

            public BinaryReader GetReader(InstanceDescriptor descriptor)
            {
                CacheEntry entry;

                if (!cacheEntries.TryGetValue(descriptor.FilePath, out entry))
                    entry = OpenStream(descriptor);

                entry.LastTimeUsed = DateTime.Now.Ticks;
                entry.Stream.Position = descriptor.DataOffset;

                return entry.Stream;
            }

            private CacheEntry OpenStream(InstanceDescriptor descriptor)
            {
                CacheEntry oldestEntry = null;
                string oldestDescriptor = null;

                if (cacheEntries.Count >= maxCacheSize)
                {
                    foreach (KeyValuePair<string, CacheEntry> pair in cacheEntries)
                    {
                        if (oldestEntry == null || pair.Value.LastTimeUsed < oldestEntry.LastTimeUsed)
                        {
                            oldestDescriptor = pair.Key;
                            oldestEntry = pair.Value;
                        }
                    }
                }

                if (oldestEntry == null)
                {
                    oldestEntry = new CacheEntry();
                }
                else
                {
                    oldestEntry.Stream.Dispose();
                    cacheEntries.Remove(oldestDescriptor);
                }

                oldestEntry.Stream = new BinaryReader(descriptor.FilePath);
                cacheEntries.Add(descriptor.FilePath, oldestEntry);

                return oldestEntry;
            }

            public void Dispose()
            {
                foreach (CacheEntry entry in cacheEntries.Values)
                    entry.Stream.Dispose();
            }
        }

        #endregion

        public static InstanceFileWriter CreateV31(long templateChecksum, bool bigEndian)
        {
            return new InstanceFileWriter(templateChecksum, InstanceFileHeader.Version31, bigEndian);
        }

        public static InstanceFileWriter CreateV32(List<InstanceDescriptor> descriptors)
        {
            long templateChecksum;

            if (descriptors.Exists(x => x.Template.Tag == TemplateTag.SNDD && x.IsMacFile))
                templateChecksum = InstanceFileHeader.OniMacTemplateChecksum;
            else
                templateChecksum = InstanceFileHeader.OniPCTemplateChecksum;

            var writer = new InstanceFileWriter(templateChecksum, InstanceFileHeader.Version32, false);
            writer.AddDescriptors(descriptors, false);
            return writer;
        }

        private InstanceFileWriter(long templateChecksum, int version, bool bigEndian)
        {
            if (templateChecksum != InstanceFileHeader.OniPCTemplateChecksum
                && templateChecksum != InstanceFileHeader.OniMacTemplateChecksum
                && templateChecksum != 0)
            {
                throw new ArgumentException("Unknown template checksum", "templateChecksum");
            }

            this.bigEndian = bigEndian;

            header = new FileHeader
            {
                TemplateChecksum = templateChecksum,
                Version = version
            };

            descriptorTable = new List<DescriptorTableEntry>();
            namedInstancedIdMap = new Dictionary<string, int>();

            linkMaps = new Dictionary<InstanceFile, int[]>();
            rawOffsetMaps = new Dictionary<InstanceFile, Dictionary<int, int>>();
            sepOffsetMaps = new Dictionary<InstanceFile, Dictionary<int, int>>();

            sharedMap = new Dictionary<InstanceDescriptor, InstanceDescriptor>();
        }

        public void AddDescriptors(List<InstanceDescriptor> descriptors, bool removeDuplicates)
        {
            if (removeDuplicates)
            {
                Console.WriteLine("Removing duplicates");

                using (streamCache = new StreamCache())
                    descriptors = RemoveDuplicates(descriptors);
            }

            //
            // Initialize LinkMap table of each source file.
            //

            var inputFiles = new Set<InstanceFile>();

            foreach (var descriptor in descriptors)
                inputFiles.Add(descriptor.File);

            foreach (var inputFile in inputFiles)
            {
                linkMaps[inputFile] = new int[inputFile.Descriptors.Count];
                rawOffsetMaps[inputFile] = new Dictionary<int, int>();
                sepOffsetMaps[inputFile] = new Dictionary<int, int>();
            }

            foreach (var descriptor in descriptors)
            {
                AddDescriptor(descriptor);
            }

            CreateHeader();
        }

        private void AddDescriptor(InstanceDescriptor descriptor)
        {
            //
            // SNDD instances are special because they are different between PC 
            // and Mac/PC Demo versions so we need to check if the instance type matches the 
            // output file type. If the file type wasn't specified then we will set it when
            // the first SNDD instance is seen.
            //

            if (descriptor.Template.Tag == TemplateTag.SNDD)
            {
                if (header.TemplateChecksum == 0)
                {
                    header.TemplateChecksum = descriptor.TemplateChecksum;
                }
                else if (header.TemplateChecksum != descriptor.TemplateChecksum)
                {
                    if (header.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum)
                        throw new NotSupportedException(string.Format("File {0} cannot be imported due to conflicting template checksums", descriptor.FilePath));
                }
            }

            //
            // Create a new id for this descriptor and remember it and the old one 
            // in the LinkMap table of the source file.
            // 

            int id = MakeInstanceId(descriptorTable.Count);

            linkMaps[descriptor.File][descriptor.Index] = id;

            //
            // If the descriptor has a name we will need to know later what is the new id
            // for its name.
            //

            if (descriptor.HasName)
                namedInstancedIdMap[descriptor.FullName] = id;

            //
            // Create and add new table entry for this descriptor.
            //

            var entry = new DescriptorTableEntry(id, descriptor);

            if (!descriptor.IsPlaceholder)
            {
                //
                // .oni files have only one non empty named descriptor. The rest are
                // forced to be empty and their contents stored in separate .oni files.
                // 

                if (!IsV32
                    || !descriptor.HasName
                    || descriptorTable.Count == 0
                    || descriptorTable[0].SourceDescriptor == descriptor)
                {
                    int dataSize = descriptor.DataSize;

                    if (descriptor.Template.Tag == TemplateTag.SNDD
                        && header.TemplateChecksum == InstanceFileHeader.OniPCTemplateChecksum
                        && descriptor.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum)
                    {
                        //
                        // HACK: when converting SNDD instances from PC Demo to PC Retail the resulting
                        // data size differs from the original.
                        //

                        dataSize = 0x60;
                    }
                    else if (descriptor.Template.Tag == TemplateTag.AKDA)
                    {
                        dataSize = 0x20;
                    }

                    entry.DataSize = dataSize;
                    entry.DataOffset = header.DataTableSize + 8;

                    header.DataTableSize += entry.DataSize;
                }
            }

            descriptorTable.Add(entry);
        }

        private void CreateHeader()
        {
            if (header.TemplateChecksum == 0)
                throw new InvalidOperationException("Target file format was not specified and cannot be autodetected.");

            header.InstanceCount = descriptorTable.Count;

            int offset = FileHeader.Size + descriptorTable.Count * DescriptorTableEntry.Size;

            if (IsV31)
            {
                nameIndex = NameDescriptorTable.CreateFromDescriptors(descriptorTable);

                header.NameCount = nameIndex.Count;
                offset += nameIndex.Size;

                templateTable = TemplateDescriptorTable.CreateFromDescriptors(
                    InstanceMetadata.GetMetadata(header.TemplateChecksum),
                    descriptorTable);

                header.TemplateCount = templateTable.Count;
                offset += templateTable.Size;

                header.DataTableOffset = Utils.Align32(offset);

                nameTable = NameTable.CreateFromDescriptors(descriptorTable);

                header.NameTableSize = nameTable.Size;
                header.NameTableOffset = Utils.Align32(header.DataTableOffset + header.DataTableSize);
            }
            else
            {
                //
                // .oni files do not need the name index and the template table.
                // They consume space, complicate things and the information 
                // contained in them can be recreated anyway.
                //

                nameTable = NameTable.CreateFromDescriptors(descriptorTable);

                header.NameTableSize = nameTable.Size;
                header.NameTableOffset = Utils.Align32(offset);
                header.DataTableOffset = Utils.Align32(header.NameTableOffset + nameTable.Size);
                header.RawTableOffset = Utils.Align32(header.DataTableOffset + header.DataTableSize);
            }
        }

        public void Write(string filePath)
        {
            string outputDirPath = Path.GetDirectoryName(filePath);

            Directory.CreateDirectory(outputDirPath);

            int fileId = IsV31 ? MakeFileId(filePath) : 0;

            using (streamCache = new StreamCache())
            using (var outputStream = new FileStream(filePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 65536))
            using (var writer = new BinaryWriter(outputStream))
            {
                outputStream.Position = FileHeader.Size;

                foreach (DescriptorTableEntry entry in descriptorTable)
                {
                    entry.Write(writer, sharedMap.ContainsKey(entry.SourceDescriptor));
                }

                if (IsV31)
                {
                    nameIndex.Write(writer);
                    templateTable.Write(writer);
                }
                else
                {
                    // 
                    // For .oni files write the name table before the data table
                    // for better reading performance at import time.
                    //

                    writer.Position = header.NameTableOffset;
                    nameTable.Write(writer);
                }

                WriteDataTable(writer, fileId);

                if (IsV31)
                {
                    writer.Position = header.NameTableOffset;
                    nameTable.Write(writer);
                }

                WriteBinaryParts(writer, filePath);

                if (IsV32 && outputStream.Length > header.RawTableOffset)
                {
                    //
                    // The header was created with a RawTable size of 0 because
                    // we don't know the size in advance. Fix that now.
                    //

                    header.RawTableSize = (int)outputStream.Length - header.RawTableOffset;
                }

                outputStream.Position = 0;
                header.Write(writer);
            }
        }

        private void WriteDataTable(BinaryWriter writer, int fileId)
        {
            writer.Position = header.DataTableOffset;

            //
            // Raw and sep parts will be added as they are found. The initial offset
            // is 32 because a 0 offset means "NULL".
            //

            rawOffset = 32;
            rawParts = new List<BinaryPartEntry>();

            sepOffset = 32;
            sepParts = new List<BinaryPartEntry>();

            var entries = descriptorTable.ToArray();
            Array.Sort(entries, (x, y) => x.DataOffset.CompareTo(y.DataOffset));

            foreach (var entry in entries)
            {
                if (entry.DataSize == 0)
                    continue;

                int padSize = header.DataTableOffset + entry.DataOffset - 8 - writer.Position;

                if (padSize <= 512)
                    writer.Write(padding, 0, padSize);
                else
                    writer.Position = header.DataTableOffset + entry.DataOffset - 8;

                writer.Write(entry.Id);
                writer.Write(fileId);

                var template = entry.SourceDescriptor.Template;

                if (template.Tag == TemplateTag.SNDD
                    && entry.SourceDescriptor.File.Header.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum
                    && header.TemplateChecksum == InstanceFileHeader.OniPCTemplateChecksum)
                {
                    //
                    // Special case to convert PC Demo SNDD files to PC Retail SNDD files.
                    //

                    ConvertSNDDHack(entry, writer);
                }
                else
                {
                    template.Type.Copy(streamCache.GetReader(entry.SourceDescriptor), writer, state =>
                    {
                        if (state.Type == MetaType.RawOffset)
                            RemapRawOffset(entry, state);
                        else if (state.Type == MetaType.SepOffset)
                            RemapSepOffset(entry, state);
                        else if (state.Type is MetaPointer)
                            RemapLinkId(entry, state);
                    });

                    if (entry.Code == TemplateTag.TXMP)
                    {
                        //
                        // HACK: All .oni files use the PC format except the SNDD ones. Most
                        // differences between PC and Mac formats are handled by the metadata
                        // but the TXMP is special because the raw/sep offset field is at a
                        // different offset.
                        //

                        ConvertTXMPHack(entry, writer.BaseStream);
                    }
                }
            }
        }

        private void ConvertSNDDHack(DescriptorTableEntry entry, BinaryWriter writer)
        {
            var reader = streamCache.GetReader(entry.SourceDescriptor);

            int flags = reader.ReadInt32();
            int duration = reader.ReadInt32();
            int dataSize = reader.ReadInt32();
            int dataOffset = reader.ReadInt32();

            int channelCount = (flags == 3) ? 2 : 1;

            writer.Write(8);
            writer.WriteInt16(2);
            writer.WriteInt16(channelCount);
            writer.Write(22050);
            writer.Write(11155);
            writer.WriteInt16(512);
            writer.WriteInt16(4);
            writer.WriteInt16(32);
            writer.Write(new byte[] {
                0xf4, 0x03, 0x07, 0x00, 0x00, 0x01, 0x00, 0x00,
                0x00, 0x02, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00,
                0xc0, 0x00, 0x40, 0x00, 0xf0, 0x00, 0x00, 0x00,
                0xcc, 0x01, 0x30, 0xff, 0x88, 0x01, 0x18, 0xff
            });

            writer.Write((short)duration);
            writer.Write(dataSize);
            writer.Write(RemapRawOffsetCore(entry, dataOffset, null));
        }

        private void ConvertTXMPHack(DescriptorTableEntry entry, Stream stream)
        {
            stream.Position = header.DataTableOffset + entry.DataOffset + 0x80;
            stream.Read(copyBuffer1, 0, 28);

            if (header.TemplateChecksum == InstanceFileHeader.OniPCTemplateChecksum)
            {
                //
                // Swap Bytes is always set for PC files.
                //

                copyBuffer1[1] |= 0x10;
            }
            else if (IsV31 && header.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum)
            {
                //
                // Swap Bytes if always set for MacPPC files except if the format is RGBA 
                // which requires no conversion.
                //

                if (bigEndian && copyBuffer1[8] == (byte)Motoko.TextureFormat.RGBA)
                    copyBuffer1[1] &= 0xef;
                else
                    copyBuffer1[1] |= 0x10;
            }

            if (entry.SourceDescriptor.TemplateChecksum != header.TemplateChecksum)
            {
                //
                // Swap the 0x94 and 0x98 fields to convert between Mac and PC TXMPs
                //

                for (int i = 20; i < 24; i++)
                {
                    byte b = copyBuffer1[i];
                    copyBuffer1[i] = copyBuffer1[i + 4];
                    copyBuffer1[i + 4] = b;
                }
            }

            stream.Position = header.DataTableOffset + entry.DataOffset + 0x80;
            stream.Write(copyBuffer1, 0, 28);
        }

        private bool ZeroTRAMPositionPointsHack(DescriptorTableEntry entry, CopyVisitor state)
        {
            if (entry.Code != TemplateTag.TRAM)
                return false;

            int offset = state.GetInt32();

            if (state.Position == 0x04)
            {
                entry.AnimationPositionPointHack = (offset == 0);
            }
            else if (state.Position == 0x28 && entry.AnimationPositionPointHack)
            {
                if (offset != 0)
                {
                    InstanceFile input = entry.SourceFile;
                    int size = input.GetRawPartSize(offset);
                    offset = AllocateRawPart(null, 0, size, null);
                    state.SetInt32(offset);
                }

                return true;
            }

            return false;
        }

        private void RemapRawOffset(DescriptorTableEntry entry, CopyVisitor state)
        {
            if (ZeroTRAMPositionPointsHack(entry, state))
                return;

            state.SetInt32(RemapRawOffsetCore(entry, state.GetInt32(), state.Field));
        }

        private int RemapRawOffsetCore(DescriptorTableEntry entry, int oldOffset, Field field)
        {
            if (oldOffset == 0)
                return 0;

            InstanceFile input = entry.SourceFile;
            Dictionary<int, int> rawOffsetMap = rawOffsetMaps[input];
            int newOffset;

            if (!rawOffsetMap.TryGetValue(oldOffset, out newOffset))
            {
                int size = input.GetRawPartSize(oldOffset);

                //
                // .oni files are always in PC format (except SNDD files) so when importing
                // to a Mac file we need to allocate some binary parts in the sep file
                // instead of the raw file.
                //

                if (header.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum
                    && (entry.Code == TemplateTag.TXMP
                       || entry.Code == TemplateTag.OSBD
                       || entry.Code == TemplateTag.BINA))
                {
                    newOffset = AllocateSepPart(input.RawFilePath, oldOffset + input.Header.RawTableOffset, size, null);
                }
                else
                {
                    newOffset = AllocateRawPart(input.RawFilePath, oldOffset + input.Header.RawTableOffset, size, field);
                }

                rawOffsetMap[oldOffset] = newOffset;
            }

            return newOffset;
        }

        private void RemapSepOffset(DescriptorTableEntry entry, CopyVisitor state)
        {
            int oldOffset = state.GetInt32();

            if (oldOffset == 0)
                return;

            InstanceFile input = entry.SourceFile;
            Dictionary<int, int> sepOffsetMap = sepOffsetMaps[input];
            int newOffset;

            if (!sepOffsetMap.TryGetValue(oldOffset, out newOffset))
            {
                int size = input.GetSepPartSize(oldOffset);

                //
                // If we're writing a PC file then there is no sep file, everything gets allocated
                // in the raw file
                //

                if (header.TemplateChecksum == InstanceFileHeader.OniPCTemplateChecksum)
                    newOffset = AllocateRawPart(input.SepFilePath, oldOffset, size, null);
                else
                    newOffset = AllocateSepPart(input.SepFilePath, oldOffset, size, null);

                sepOffsetMap[oldOffset] = newOffset;
            }

            state.SetInt32(newOffset);
        }

        private int AllocateRawPart(string sourceFile, int sourceOffset, int size, Field field)
        {
            var entry = new BinaryPartEntry(sourceFile, sourceOffset, size, rawOffset, field);
            rawOffset = Utils.Align32(rawOffset + size);
            rawParts.Add(entry);
            return entry.DestinationOffset;
        }

        private int AllocateSepPart(string sourceFile, int sourceOffset, int size, Field field)
        {
            var entry = new BinaryPartEntry(sourceFile, sourceOffset, size, sepOffset, field);
            sepOffset = Utils.Align32(sepOffset + size);
            sepParts.Add(entry);
            return entry.DestinationOffset;
        }

        private void RemapLinkId(DescriptorTableEntry entry, CopyVisitor state)
        {
            int oldId = state.GetInt32();

            if (oldId != 0)
            {
                int newId = RemapLinkIdCore(entry.SourceDescriptor, oldId);
                state.SetInt32(newId);
            }
        }

        private int RemapLinkIdCore(InstanceDescriptor descriptor, int id)
        {
            var file = descriptor.File;

            if (IsV31)
            {
                InstanceDescriptor oldDescriptor = file.GetDescriptor(id);
                InstanceDescriptor newDescriptor;
                int newDescriptorId;

                if (oldDescriptor.HasName)
                {
                    //
                    // Always lookup named instances, this deals with cases where an instance from one source file
                    // is replaced by one from another source file.
                    //

                    if (namedInstancedIdMap.TryGetValue(oldDescriptor.FullName, out newDescriptorId))
                        return newDescriptorId;
                }

                if (sharedMap.TryGetValue(oldDescriptor, out newDescriptor))
                    return linkMaps[newDescriptor.File][newDescriptor.Index];
            }

            return linkMaps[file][id >> 8];
        }

        private void WriteBinaryParts(BinaryWriter writer, string filePath)
        {
            if (IsV32)
            {
                //
                // For .oni files the raw/sep parts are written to the .oni file.
                // Separate .raw/.sep files are not used.
                //

                WriteParts(writer, rawParts);
                return;
            }

            string rawFilePath = Path.ChangeExtension(filePath, ".raw");

            Console.WriteLine("Writing {0}", rawFilePath);

            using (var rawOutputStream = new FileStream(rawFilePath, FileMode.Create, FileAccess.Write, FileShare.None, 65536))
            using (var rawWriter = new BinaryWriter(rawOutputStream))
            {
                WriteParts(rawWriter, rawParts);
            }

            if (header.TemplateChecksum == InstanceFileHeader.OniMacTemplateChecksum)
            {
                //
                // Only Mac/PC Demo files have a .sep file.
                //

                string sepFilePath = Path.ChangeExtension(filePath, ".sep");

                Console.WriteLine("Writing {0}", sepFilePath);

                using (var sepOutputStream = new FileStream(sepFilePath, FileMode.Create, FileAccess.Write, FileShare.None, 65536))
                using (var sepWriter = new BinaryWriter(sepOutputStream))
                {
                    WriteParts(sepWriter, sepParts);
                }
            }
        }

        private void WriteParts(BinaryWriter writer, List<BinaryPartEntry> binaryParts)
        {
            if (binaryParts.Count == 0)
            {
                writer.Write(padding, 0, 32);
                return;
            }

            binaryParts.Sort();

            int fileLength = 0;

            foreach (BinaryPartEntry entry in binaryParts)
            {
                if (entry.DestinationOffset + entry.Size > fileLength)
                    fileLength = entry.DestinationOffset + entry.Size;
            }

            if (IsV31)
                writer.BaseStream.SetLength(fileLength);
            else
                writer.BaseStream.SetLength(fileLength + header.RawTableOffset);

            BinaryReader reader = null;

            foreach (BinaryPartEntry entry in binaryParts)
            {
                if (entry.SourceFile == null)
                    continue;

                if (reader == null)
                {
                    reader = new BinaryReader(entry.SourceFile);
                }
                else if (reader.Name != entry.SourceFile)
                {
                    reader.Dispose();
                    reader = new BinaryReader(entry.SourceFile);
                }

                reader.Position = entry.SourceOffset;

                //
                // Smart change of output's stream position. This assumes that
                // the binary parts are sorted by destination offset.
                //

                int padSize = entry.DestinationOffset + header.RawTableOffset - writer.Position;

                if (padSize <= 32)
                    writer.Write(padding, 0, padSize);
                else
                    writer.Position = entry.DestinationOffset + header.RawTableOffset;

                if (entry.Field == null || entry.Field.RawType == null)
                {
                    //
                    // If we don't know the field or the fieldtype for this binary part
                    // we just copy it over without cleaning up garbage.
                    //

                    if (copyBuffer1.Length < entry.Size)
                        copyBuffer1 = new byte[entry.Size * 2];

                    reader.Read(copyBuffer1, 0, entry.Size);
                    writer.Write(copyBuffer1, 0, entry.Size);
                }
                else
                {
                    int size = entry.Size;

                    while (size > 0)
                    {
                        int copiedSize = entry.Field.RawType.Copy(reader, writer, null);

                        if (copiedSize > size)
                            throw new InvalidOperationException(string.Format("Bad metadata copying field {0}", entry.Field.Name));

                        size -= copiedSize;
                    }
                }
            }

            if (reader != null)
                reader.Dispose();
        }

        private List<InstanceDescriptor> RemoveDuplicates(List<InstanceDescriptor> descriptors)
        {
            var checksums = new Dictionary<int, List<InstanceDescriptor>>();
            var newDescriptorList = new List<InstanceDescriptor>(descriptors.Count);

            foreach (var descriptor in descriptors)
            {
                //
                // We only handle duplicates for these types of instances.
                // These are the most common cases and they are simple to handle
                // because they do not contain links to other instances.
                //

                if (!(descriptor.Template.Tag == TemplateTag.IDXA
                    || descriptor.Template.Tag == TemplateTag.PNTA
                    || descriptor.Template.Tag == TemplateTag.VCRA
                    || descriptor.Template.Tag == TemplateTag.TXCA
                    || descriptor.Template.Tag == TemplateTag.TRTA
                    || descriptor.Template.Tag == TemplateTag.TRIA
                    || descriptor.Template.Tag == TemplateTag.ONCP
                    || descriptor.Template.Tag == TemplateTag.ONIA))
                {
                    newDescriptorList.Add(descriptor);
                    continue;
                }

                int checksum = GetInstanceChecksum(descriptor);

                List<InstanceDescriptor> existingDescriptors;

                if (!checksums.TryGetValue(checksum, out existingDescriptors))
                {
                    existingDescriptors = new List<InstanceDescriptor>();
                    checksums.Add(checksum, existingDescriptors);
                }
                else
                {
                    InstanceDescriptor existing = existingDescriptors.Find(x => AreInstancesEqual(descriptor, x));

                    if (existing != null)
                    {
                        sharedMap.Add(descriptor, existing);
                        continue;
                    }
                }

                existingDescriptors.Add(descriptor);
                newDescriptorList.Add(descriptor);
            }

            return newDescriptorList;
        }

        private int GetInstanceChecksum(InstanceDescriptor descriptor)
        {
            using (var checksumStream = new ChecksumStream())
            using (var writer = new BinaryWriter(checksumStream))
            {
                descriptor.Template.Type.Copy(streamCache.GetReader(descriptor), writer, null);
                return checksumStream.Checksum;
            }
        }

        private bool AreInstancesEqual(InstanceDescriptor d1, InstanceDescriptor d2)
        {
            if (d1.File == d2.File && d1.Index == d2.Index)
                return true;

            if (d1.Template.Tag != d2.Template.Tag
                || d1.DataSize != d2.DataSize)
                return false;

            if (copyBuffer1.Length < d1.DataSize)
                copyBuffer1 = new byte[d1.DataSize * 2];

            if (copyBuffer2.Length < d2.DataSize)
                copyBuffer2 = new byte[d2.DataSize * 2];

            MetaType type = d1.Template.Type;

            //return type.Compare(streamCache.GetStream(d1), streamCache.GetStream(d2));

            using (var writer1 = new BinaryWriter(new MemoryStream(copyBuffer1)))
            using (var writer2 = new BinaryWriter(new MemoryStream(copyBuffer2)))
            {
                int s1 = type.Copy(streamCache.GetReader(d1), writer1, null);
                int s2 = type.Copy(streamCache.GetReader(d2), writer2, null);

                if (s1 != s2)
                    return false;

                for (int i = 0; i < s1; i++)
                {
                    if (copyBuffer1[i] != copyBuffer2[i])
                        return false;
                }
            }

            return true;
        }

        private bool IsV31 => header.Version == InstanceFileHeader.Version31;
        private bool IsV32 => header.Version == InstanceFileHeader.Version32;

        private static int MakeFileId(string filePath)
        {
            //
            // File id is generated from the filename. The filename is expected to be in
            // XXXXXN_YYYYY.dat format where the XXXXX (5 characters) part is ignored, N is treated
            // as a number (level number) and the YYYYY (this part can have any length) is hashed.
            // The file extension is ignored.
            //

            string fileName = Path.GetFileNameWithoutExtension(filePath);

            if (fileName.Length < 6)
                return 0;

            fileName = fileName.Substring(5);

            int levelNumber = 0;
            int buildTypeHash = 0;

            int i = fileName.IndexOf('_');

            if (i != -1)
            {
                int.TryParse(fileName.Substring(0, i), out levelNumber);

                if (!string.Equals(fileName.Substring(i + 1), "Final", StringComparison.Ordinal))
                {
                    for (int j = 1; i + j < fileName.Length; j++)
                        buildTypeHash += (char.ToUpperInvariant(fileName[i + j]) - 0x40) * j;
                }
            }

            return (((levelNumber << 24) | (buildTypeHash & 0xffffff)) << 1) | 1;
        }

        public static int MakeInstanceId(int index) => (index << 8) | 1;
    }
}
Revision:	1114
Committed:	Wed Jan 22 14:08:57 2020 UTC (5 years, 8 months ago) by iritscen
File size:	47295 byte(s)
Log Message:	Adding OniSplit source code (v0.9.99.0). Many thanks to Neo for all his work over the years.