From 9d03dbb7b11a1df56e46277704b21d7f6a33f94b Mon Sep 17 00:00:00 2001 From: Deterous <138427222+Deterous@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:37:15 +0900 Subject: [PATCH] Perform hashing in one read pass --- IRDKit/IRDKit.csproj | 2 +- IRDKit/Program.cs | 42 +----- LibIRD/IRD.cs | 349 +++++++++++++++++++++++-------------------- LibIRD/LibIRD.csproj | 2 +- LibIRD/ReIRD.cs | 2 +- 5 files changed, 198 insertions(+), 199 deletions(-) diff --git a/IRDKit/IRDKit.csproj b/IRDKit/IRDKit.csproj index ef75eda..172a274 100644 --- a/IRDKit/IRDKit.csproj +++ b/IRDKit/IRDKit.csproj @@ -9,7 +9,7 @@ win-x86;win-x64;win-arm64;linux-x64;linux-arm64;osx-x64;osx-arm64 latest true - 0.5.0 + 0.6.0 Deterous diff --git a/IRDKit/Program.cs b/IRDKit/Program.cs index 1354bd0..26e53c6 100644 --- a/IRDKit/Program.cs +++ b/IRDKit/Program.cs @@ -704,16 +704,11 @@ namespace IRDKit return irdPath; } } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } // Create new reproducible redump-style IRD with a given key file @@ -740,16 +735,11 @@ namespace IRDKit return irdPath; } } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } // Create new reproducible redump-style IRD with a given GetKey log @@ -764,16 +754,11 @@ namespace IRDKit ird1.Print(); return irdPath; } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } // No key provided, try search for .key file @@ -798,16 +783,11 @@ namespace IRDKit return irdPath; } } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } // No key provided, try search for .getkey.log file @@ -825,16 +805,11 @@ namespace IRDKit ird1.Print(); return irdPath; } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } // No key provided, try get key from redump.org @@ -913,16 +888,11 @@ namespace IRDKit ird.Print(); return irdPath; } - catch (ArgumentException e) + catch (Exception e) { Console.Error.WriteLine(e.Message + ", failed to create IRD"); return null; } - catch (FileNotFoundException) - { - Console.Error.WriteLine("File not found, failed to create IRD"); - return null; - } } #endregion diff --git a/LibIRD/IRD.cs b/LibIRD/IRD.cs index be553c1..980ea80 100644 --- a/LibIRD/IRD.cs +++ b/LibIRD/IRD.cs @@ -2,6 +2,7 @@ using DiscUtils.Iso9660; using DiscUtils.Streams; using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.IO.Hashing; @@ -621,7 +622,7 @@ namespace LibIRD private protected void GenerateIRD(string isoPath, bool redump = false) { // Parse ISO file as a file stream - using FileStream fs = new FileStream(isoPath, FileMode.Open, FileAccess.Read) ?? throw new FileNotFoundException(isoPath); + using FileStream fs = new FileStream(isoPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.SequentialScan) ?? throw new FileNotFoundException(isoPath); // Validate ISO file stream if (!CDReader.Detect(fs)) throw new InvalidFileSystemException("Not a valid ISO file"); @@ -707,14 +708,15 @@ namespace LibIRD DiscDirectoryInfo rootDir = reader.GetDirectoryInfo("\\"); FileCount = 0; CountFiles(rootDir); + // Pre-allocate arrays and reset file count FileKeys = new long[FileCount]; FileExtents = new Range[FileCount][]; uint fileCount = FileCount; FileCount = 0; + // Determine file offsets GetFiles(fs, reader, rootDir); - FileHashes = new byte[FileCount][]; // Resize arrays if non-contiguous files were detected if (FileCount != fileCount) @@ -730,13 +732,9 @@ namespace LibIRD Array.Sort(FileKeys, FileExtents); // Calculate CRC32 hash of ISO only if generating a redump IRD and the UID is not already set + RegionHashes = new byte[RegionCount][]; + FileHashes = new byte[FileCount][]; HashISO(fs, redump && UID == 0x00000000); - - // Determine region hashes - HashRegions(fs); - - // Determine file hashes - HashFiles(fs); } #endregion @@ -876,7 +874,6 @@ namespace LibIRD RegionCount = (byte)(2 * ((uint)decRegionCount[3]) - 1); if (RegionCount <= 0) throw new InvalidFileSystemException("No regions detected in ISO"); - RegionHashes = new byte[RegionCount][]; RegionStart = new long[RegionCount]; RegionEnd = new long[RegionCount]; @@ -907,53 +904,6 @@ namespace LibIRD RegionEnd[^1] = (UpdateEnd / SectorSize) - 1; } - /// - /// Determines and stores the hashes for each disc region - /// - /// ISO filestream - /// - private void HashRegions(FileStream fs) - { - // Determine MD5 hashes for each region - int bufSectors = 1024; - byte[] buf = new byte[bufSectors * SectorSize]; - for (int i = 0; i < RegionCount; i++) - { - // Start reading data from first sector of region - fs.Seek(SectorSize * RegionStart[i], SeekOrigin.Begin); - - // Compute MD5 hash for just the region portion of the ISO file - int numBytes; - using MD5 md5 = MD5.Create(); - int regionSectors = (int)(RegionEnd[i] - RegionStart[i]) + 1; - for (int j = bufSectors; j <= regionSectors; j += bufSectors) - { - // Read into buffer - numBytes = fs.Read(buf, 0, buf.Length); - // TODO: Process partial buffer if non-zero is returned - if (numBytes < buf.Length) - throw new InvalidFileSystemException("Disc region ended unexpectedly"); - - // Process MD5 sum - md5.TransformBlock(buf, 0, buf.Length, null, 0); - } - - // Read any remaining sectors - int bufRemainder = (int)(SectorSize * (regionSectors % bufSectors)); - if (bufRemainder != 0) - { - numBytes = fs.Read(buf, 0, bufRemainder); - // TODO: Process partial buffer if non-zero is returned - if (numBytes < bufRemainder) - throw new InvalidFileSystemException("Disc region ended unexpectedly"); - } - - // Compute and store MD5 hash of region - md5.TransformFinalBlock(buf, 0, bufRemainder); - RegionHashes[i] = md5.Hash; - } - } - /// /// Determine and store file extents for all files and files within subdirectories recursively /// @@ -1008,91 +958,29 @@ namespace LibIRD } } - /// - /// Calculate hashes for all file extents - /// - /// ISO filestream - private void HashFiles(FileStream fs) - { - // Process all files in current directory - int bufSectors = 1024; - byte[] buf = new byte[bufSectors * SectorSize]; - for (int file = 0; file < FileCount; file++) - { - // Determine whether file is in encrypted or decrypted region - bool encrypted = false; - for (int i = RegionCount - 1; i > 0; i--) - { - if (RegionStart[i] <= FileKeys[file]) - { - encrypted = i % 2 == 1; - break; - } - } - - // Read one sector at a time for small files - //int bufSectors = FileExtents[file][0].Count > 1024 * SectorSize ? 1024 : 32; - //byte[] buf = new byte[bufSectors * SectorSize]; - - // Hash each non-contiguous portion of the file - using MD5 md5 = MD5.Create(); - for (int i = 0; i < FileExtents[file].Length; i++) - { - // Start reading data from the beginning of the current extent - fs.Seek(FileExtents[file][i].Offset * SectorSize, SeekOrigin.Begin); - int numBytes; - // Read file in buffers - long fileSectors = FileExtents[file][i].Count / SectorSize; - for (int j = 0; j <= fileSectors - bufSectors; j += bufSectors) - { - numBytes = fs.Read(buf, 0, buf.Length); - // TODO: Process partial buffer if non-zero is returned - if (numBytes != buf.Length) - throw new InvalidFileSystemException("Disc region ended unexpectedly"); - - // Decrypt buffer if necessary - if (encrypted) - DecryptSectors(ref buf, (int)FileExtents[file][i].Offset + j); - - // Hash sector - md5.TransformBlock(buf, 0, buf.Length, null, 0); - } - // Read remaining partial buffer - if ((FileExtents[file][i].Count % buf.Length) > 0) - { - numBytes = fs.Read(buf, 0, buf.Length); - // TODO: Process partial buffer if non-zero is returned - if (numBytes != buf.Length) - throw new InvalidFileSystemException("Disc region ended unexpectedly"); - - // Decrypt buffer if necessary - if (encrypted) - DecryptSectors(ref buf, (int)FileExtents[file][i].Offset + bufSectors * (int)(FileExtents[file][i].Count / buf.Length)); - - // Hash partial buffer - md5.TransformBlock(buf, 0, (int)(FileExtents[file][i].Count % buf.Length), null, 0); - } - } - - // Finalise and store MD5 hash - md5.TransformFinalBlock(buf, 0, 0); - FileHashes[file] = md5.Hash; - } - } - /// /// Decrypts a given byte array of sector(s) /// /// Byte array to be decrypted - /// Number of bytes to decrypt + /// Sector number of first sector being decrypted + /// Number of sectors to skip + /// Number of sectors to decrypt, beginning from offset /// - private protected void DecryptSectors(ref byte[] buffer, int sectorNumber) + private protected void DecryptSectors(ref byte[] buffer, int sectorNumber, int offset = 0, int? count = null) { ArgumentNullException.ThrowIfNull(buffer); if (buffer.Length == 0 || buffer.Length % SectorSize != 0) throw new ArgumentException("Encrypted buffer must be multiple of SectorSize"); + if (offset < 0 || offset >= buffer.Length / SectorSize) + throw new ArgumentException("Offset sector must be within buffer"); + + count ??= (int)(buffer.Length / SectorSize) - offset; + + if (count < 0 || count > (buffer.Length / SectorSize) - offset) + throw new ArgumentException("Number of sectors must be within buffer"); + // Setup AES decryption using Aes aes = Aes.Create() ?? throw new InvalidOperationException("AES not available. Change your system settings"); // Set AES settings @@ -1100,8 +988,11 @@ namespace LibIRD aes.Padding = PaddingMode.None; aes.Mode = CipherMode.CBC; + // Convert offset and count to number of bytes + offset *= (int)SectorSize; + count *= (int)SectorSize; // Decrypt buffer one sector at a time - for (int i = 0; i < buffer.Length; i += (int)SectorSize) + for (int i = offset; i < offset + count; i += (int)SectorSize) { // Determine AES Initial Value based on sector number byte[] iv = new byte[16]; @@ -1149,53 +1040,186 @@ namespace LibIRD Crc32 isoHasher = new(); // Initialise MD5 region hashes - int currentRegion = 0; - MD5[] regionHashers = new MD5[RegionCount]; + List regions = []; + MD5[] regionMD5 = new MD5[RegionCount]; for (int i = 0; i < RegionCount; i++) - regionHashers[i] = MD5.Create(); + { + regions.Add(i); + regionMD5[i] = MD5.Create(); + } // Initialise MD5 file hashes - int currentFile = 0; - MD5[] fileHashers = new MD5[FileCount]; + List files = []; + MD5[] fileMD5 = new MD5[FileCount]; for (int i = 0; i < FileCount; i++) - fileHashers[i] = MD5.Create(); + { + files.Add(i); + fileMD5[i] = MD5.Create(); + } // Start hashing from beginning of ISO - fs.Seek(0, SeekOrigin.Begin); + long currentSector = 0; + fs.Seek(currentSector, SeekOrigin.Begin); - // Read from ISO 1024 sectors at a time + // Read from ISO, 1024 sectors at a time int bufSectors = 1024; byte[] buf = new byte[bufSectors * SectorSize]; while (true) { // Attempt to read a full buffer + bufSectors = 1024; int numBytes = fs.Read(buf, 0, buf.Length); + // If end of ISO reached, stop reading if (numBytes == 0) - break; - // Partial buffer read, process partial read - else if (numBytes == buf.Length) { + // If making redump-style IRD, save CRC32 hash to UID field if (redump) - isoHasher.Append(buf); + { + crc32 = isoHasher.GetCurrentHash(); + UID = BitConverter.ToUInt32(crc32, 0); + } + return; } - // Process full buffer - else - { - if (redump) - isoHasher.Append(buf[..numBytes]); - } - } - currentRegion = currentFile; - currentFile = currentRegion; - // Save CRC32 hash to UID field, if making redump-style IRD - if (redump) - { - crc32 = isoHasher.GetCurrentHash(); - UID = BitConverter.ToUInt32(crc32, 0); - } - + // Keep trying to read to fill buffer, remove once partial buffer hashing is supported + if (numBytes != buf.Length) + { + while (numBytes % SectorSize != 0) + { + int newNumBytes = fs.Read(buf, numBytes, buf.Length - numBytes); + numBytes += newNumBytes; + + // If end of ISO reached, trim buffer and hash + if (newNumBytes == 0 && numBytes % SectorSize != 0) + { + //numBytes -= numBytes % (int)SectorSize; + Console.Error.WriteLine("ERROR: ISO filestream ended early"); + break; + } + } + // Only hash portion of buffer + bufSectors = numBytes / (int)SectorSize; + if (bufSectors == 0) + Console.Error.WriteLine("ERROR: Trailing partial sector in ISO filestream"); + if (numBytes > buf.Length) + throw new InvalidFileSystemException("ERROR: Read more bytes than buffer size???"); + } + + // Hash ISO + if (redump) + isoHasher.Append(new ReadOnlySpan(buf, 0, numBytes)); + + // Hash regions + List regionsEnded = []; + foreach (int i in regions) + { + // Stop hashing regions if current region has not yet started (assumes regions are ordered) + if (RegionStart[i] > currentSector + bufSectors) + break; + + // Skip region if it has already ended + //if (RegionEnd[i] < currentSector) + // continue; + + // Check if region has ended in this buffer [We know: Start is not in the future, Ending is not in the past] + if (RegionEnd[i] < currentSector + bufSectors) + { + // Determine start byte, if region is entirely within the buffer + int startByte = RegionStart[i] > currentSector ? (int)(SectorSize * (RegionStart[i] - currentSector)) : 0; + // Determine end byte + int endByte = (int)(SectorSize * (RegionEnd[i] - currentSector + 1)); + // Close region hash + regionMD5[i].TransformFinalBlock(buf, startByte, endByte - startByte); + RegionHashes[i] = regionMD5[i].Hash; + regionMD5[i].Clear(); + regionsEnded.Add(i); + } + // Check if region has already begun + else if (RegionStart[i] <= currentSector) + { + // Hash buffer + regionMD5[i].TransformBlock(buf, 0, (int)SectorSize * bufSectors, null, 0); + } + // Region Start is in this buffer, ending is in the future + else + { + // Hash partial buffer + int regionStart = (int)(SectorSize * (RegionStart[i] - currentSector)); + regionMD5[i].TransformBlock(buf, regionStart, (int)SectorSize * bufSectors - regionStart, null, 0); + } + } + if (regionsEnded.Count > 0) + regions.RemoveAll(item => regionsEnded.Contains(item)); + + // Decrypt any encrypted sectors of buffer + for (int i = 1; i < RegionCount; i += 2) + { + // If the current encrypted region is within the buffer + if (RegionStart[i] < currentSector + bufSectors + && RegionEnd[i] >= currentSector) + { + // First sector to decrypt from + int encOffset = 0; + + // Don't decrypt initial sectors if the encrypted region starts within this buffer + if (RegionStart[i] > currentSector) + encOffset = (int)(RegionStart[i] - currentSector); + + // Number of sectors to decrypt + int encCount = bufSectors - encOffset; + + // Don't decrypt last sectors if the encrypted region ends within this buffer + if (RegionEnd[i] < currentSector + bufSectors) + encCount -= (int)(currentSector + bufSectors - RegionEnd[i] + 1); + + // Decrypt encrypted sectors + DecryptSectors(ref buf, (int)currentSector + encOffset, encOffset, encCount); + } + } + + // Hash files + List filesEnded = []; + foreach (int i in files) + { + // Stop hashing files if current file has not yet started (assumes FileKeys are sorted) + if (FileKeys[i] > currentSector + bufSectors) + break; + + // Hash each file extent for each file + for (int j = 0; j < FileExtents[i].Length; j++) + { + // Skip hashing file extent if it has not yet started or already ended + if (FileExtents[i][j].Offset > currentSector + bufSectors + || SectorSize* FileExtents[i][j].Offset + FileExtents[i][j].Count < SectorSize * currentSector) + continue; + // Determine first file byte location in buffer + int startByte = FileExtents[i][j].Offset > currentSector ? (int)(SectorSize * (FileExtents[i][j].Offset - currentSector)) : 0; + // Determine last file byte location in buffer + int endByte = (int)(FileExtents[i][j].Count - SectorSize * (currentSector - FileExtents[i][j].Offset)); + // Don't hash more than the buffer size + endByte = endByte < bufSectors * (int)SectorSize ? endByte : bufSectors * (int)SectorSize; + // Hash portion of buffer that file exists in + fileMD5[i].TransformBlock(buf, startByte, endByte - startByte, null, 0); + } + + // Check if current file has ended in this buffer (assumes last extent contains last byte) + long lastByte = SectorSize * FileExtents[i][^1].Offset + FileExtents[i][^1].Count; + if (lastByte < SectorSize * (currentSector + bufSectors) + && lastByte > SectorSize * currentSector) + { + // Close file hash + fileMD5[i].TransformFinalBlock(buf, 0, 0); + FileHashes[i] = fileMD5[i].Hash; + fileMD5[i].Clear(); + filesEnded.Add(i); + } + } + if (filesEnded.Count > 0) + files.RemoveAll(item => filesEnded.Contains(item)); + + currentSector += bufSectors; + } } #endregion @@ -1265,7 +1289,12 @@ namespace LibIRD // Hashes for each region for (int i = 0; i < RegionCount; i++) - bw.Write(RegionHashes[i], 0, 16); + { + if (RegionHashes[i] == null) + bw.Write(NullMD5); + else + bw.Write(RegionHashes[i], 0, 16); + } // Number of files hashed bw.Write(FileCount); diff --git a/LibIRD/LibIRD.csproj b/LibIRD/LibIRD.csproj index 7b50cf2..244f2e6 100644 --- a/LibIRD/LibIRD.csproj +++ b/LibIRD/LibIRD.csproj @@ -6,7 +6,7 @@ win-x86;win-x64;win-arm64;linux-x64;linux-arm64;osx-x64;osx-arm64 latest true - 0.5.0 + 0.6.0 ../nupkg diff --git a/LibIRD/ReIRD.cs b/LibIRD/ReIRD.cs index 954c19c..8d36195 100644 --- a/LibIRD/ReIRD.cs +++ b/LibIRD/ReIRD.cs @@ -131,7 +131,7 @@ namespace LibIRD if (layerbreak >= 2 * BDLayerSize || layerbreak % SectorSize != 0) throw new ArgumentException("Unexpected layerbreak value", nameof(size)); } - else + else if (size > BDLayerSize) { // If no layerbreak provided, ensure ISO is not BD-Video hybrid using FileStream fs = new FileStream(isoPath, FileMode.Open, FileAccess.Read) ?? throw new FileNotFoundException(isoPath);