More assembly store + assemblies

dotnet · grendello · Jul 3, 2025 · Jul 4, 2025 · Jul 7, 2025 · Jul 7, 2025
commit 8edb4b3185e978553653817710223198b103213e
@@ -5,21 +5,134 @@ namespace ApplicationUtility;
 
 public class ApplicationAssembly : IAspect
 {
+	const string LogTag = "ApplicationAssembly";
+	const uint COMPRESSED_MAGIC = 0x5A4C4158; // 'XALZ', little-endian
+	const ushort MSDOS_EXE_MAGIC = 0x5A4D; // 'MZ'
+	const uint PE_EXE_MAGIC = 0x00004550; // 'PE\0\0'
+
 	public static string AspectName { get; } = "Application assembly";
 
-	public bool IsCompressed    { get; private set; }
-	public string Name          { get; private set; } = "";
-	public ulong CompressedSize { get; private set; }
-	public ulong Size           { get; private set; }
-	public bool IgnoreOnLoad    { get; private set; }
+	public bool IsCompressed    { get; }
+	public string Name          { get; }
+	public ulong CompressedSize { get; }
+	public ulong Size           { get; }
+	public bool IgnoreOnLoad    { get; }
+	public ulong NameHash       { get; internal set; }
+
+	readonly Stream? assemblyStream;
+
+	ApplicationAssembly (Stream stream, uint uncompressedSize, string? description, bool isCompressed)
+	{
+		assemblyStream = stream;
+		Size = uncompressedSize;
+		CompressedSize = isCompressed ? (ulong)stream.Length : 0;
+		IsCompressed = isCompressed;
+		Name = NameMe (description);
+	}
+
+	ApplicationAssembly (string? description, bool isIgnored)
+	{
+		IgnoreOnLoad = isIgnored;
+		Name = NameMe (description);
+	}
+
+	static string NameMe (string? description) => String.IsNullOrEmpty (description) ? "Unnamed" : description;
+
+	// This is a special case, as much as I hate to have one. Ignored assemblies exist only in the assembly store's
+	// index. They have an associated descriptor, but no data whatsoever. For that reason, we can't go the `ProbeAspect`
+	// + `LoadAspect` route, so `AssemblyStore` will call this method for them.
+	public static IAspect CreateIgnoredAssembly (string? description, ulong nameHash)
+	{
+		Log.Debug ($"{LogTag}: stream ('{description}') is an ignored assembly.");
+		return new ApplicationAssembly (description, isIgnored: true) {
+			NameHash = nameHash,
+		};
+	}
 
 	public static IAspect LoadAspect (Stream stream, IAspectState state, string? description)
 	{
-		throw new NotImplementedException ();
+		using var reader = Utilities.GetReaderAndRewindStream (stream);
+		if (ReadCompressedHeader (reader, out uint uncompressedLength)) {
+			return new ApplicationAssembly (stream, uncompressedLength, description, isCompressed: true);
+		}
+
+		return new ApplicationAssembly (stream, (uint)stream.Length, description, isCompressed: false);
 	}
 
 	public static IAspectState ProbeAspect (Stream stream, string? description)
+	{
+		Log.Debug ($"{LogTag}: probing stream ('{description}')");
+		if (stream.Length == 0) {
+			// It can happen if the assembly store index or name table are corrupted and we cannot
+			// determine if an assembly is ignored or not. If it is ignored, it will have no data
+			// available and so the stream will have length of 0
+			return new BasicAspectState (false);
+		}
+
+		// If we detect compressed assembly signature, we won't proceed with checking whether
+		// the rest of data is actually a valid managed assembly. This is to avoid doing a
+		// costly operation of decompressing when e.g. loading data from an assemblystore, when
+		// we potentially create a lot of `ApplicationAssembly` instances. Presence of the compression
+		// header is enough for the probing stage.
+
+		using var reader = Utilities.GetReaderAndRewindStream (stream);
+		if (ReadCompressedHeader (reader, out _)) {
+			Log.Debug ($"{LogTag}: stream ('{description}') is a compressed assembly.");
+			return new BasicAspectState (true);
+		}
+
+		// We could use PEReader (https://learn.microsoft.com/en-us/dotnet/api/system.reflection.portableexecutable.pereader)
+		// but it would be too heavy for our purpose here.
+		reader.BaseStream.Seek (0, SeekOrigin.Begin);
+		ushort mzExeMagic = reader.ReadUInt16 ();
+		if (mzExeMagic != MSDOS_EXE_MAGIC) {
+			return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have MS-DOS executable signature.");
+		}
+
+		const long PE_HEADER_OFFSET = 0x3c;
+		if (reader.BaseStream.Length <= PE_HEADER_OFFSET) {
+			return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted MS-DOS executable image (too short, offset {PE_HEADER_OFFSET} is bigger than stream size).");
+		}
+
+		// Offset at 0x3C is where we can read the 32-bit offset to the PE header
+		reader.BaseStream.Seek (PE_HEADER_OFFSET, SeekOrigin.Begin);
+		uint uintVal = reader.ReadUInt32 ();
+		if (reader.BaseStream.Length <= (long)uintVal) {
+			return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted PE executable image (too short, offset {uintVal} is bigger than stream size).");
+		}
+
+		reader.BaseStream.Seek ((long)uintVal, SeekOrigin.Begin);
+		uintVal = reader.ReadUInt32 ();
+		if (uintVal != PE_EXE_MAGIC) {
+			return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have PE executable signature.");
+		}
+		// This is good enough for us
+
+		Log.Debug ($"{LogTag}: stream ('{description}') appears to be a PE image.");
+		return new BasicAspectState (true);
+	}
+
+	/// <summary>
+	/// Writes assembly data to the indicated file, uncompressing it if necessary. If the destination
+	/// file exists, it will be overwritten.
+	/// </summary>
+	public void SaveToFile (string filePath)
 	{
 		throw new NotImplementedException ();
 	}
+
+	// We don't care about the descriptor index here, it's only needed during the run time
+	static bool ReadCompressedHeader (BinaryReader reader, out uint uncompressedLength)
+	{
+		uncompressedLength = 0;
+
+		uint uintVal = reader.ReadUInt32 ();
+		if (uintVal != COMPRESSED_MAGIC) {
+			return false;
+		}
+
+		uintVal = reader.ReadUInt32 (); // descriptor index
+		uncompressedLength = reader.ReadUInt32 ();
+		return true;
+	}
 }
@@ -0,0 +1,4 @@
+namespace ApplicationUtility;
+
+abstract class AssemblyStoreIndexEntry
+{}
@@ -1,6 +1,15 @@
 namespace ApplicationUtility;
 
-class AssemblyStoreIndexEntryV3
+class AssemblyStoreIndexEntryV3 : AssemblyStoreIndexEntry
 {
+	public ulong NameHash { get; }
+	public uint DescriptorIndex { get; }
+	public bool Ignore { get; }
 
+	public AssemblyStoreIndexEntryV3 (ulong nameHash, uint descriptorIndex, byte ignore)
+	{
+		NameHash = nameHash;
+		DescriptorIndex = descriptorIndex;
+		Ignore = ignore != 0;
+	}
 }
@@ -32,7 +32,7 @@ protected FormatBase (Stream storeStream, string? description)
 	public bool Read ()
 	{
 		bool success = true;
-		using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true);
+		using var reader = Utilities.GetReaderAndRewindStream (StoreStream);
 
 		// They can be `null` if `Validate` wasn't called for some reason.
 		if (Header == null) {
@@ -67,7 +67,7 @@ public bool Read ()
 
 	public IAspectState Validate ()
 	{
-		using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true);
+		using var reader = Utilities.GetReaderAndRewindStream (StoreStream);
 
 		if (ReadHeader (reader, out AssemblyStoreHeader? header)) {
 			Header = header;
@@ -82,12 +82,6 @@ public IAspectState Validate ()
 
 	protected abstract IAspectState ValidateInner ();
 
-	protected BasicAspectState ValidationFailed (string message)
-	{
-		Log.Debug (message);
-		return new BasicAspectState (false);
-	}
-
 	protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? header)
 	{
 		header = null;

@@ -4,6 +4,8 @@
 using System.IO;
 using System.Text;
 
+using Xamarin.Android.Tasks;
+
 namespace ApplicationUtility;
 
 class Format_V3 : FormatBase
@@ -25,12 +27,12 @@ protected bool EnsureValidState (string where, out IAspectState? retval)
 	{
 		retval = null;
 		if (Header == null || Header.EntryCount == null || Header.IndexEntryCount == null || Header.IndexSize == null) {
-			retval = ValidationFailed ($"{LogTag}: invalid header data in {where}.");
+			retval = Utilities.GetFailureAspectState ($"{LogTag}: invalid header data in {where}.");
 			return false;
 		}
 
 		if (Descriptors == null || Descriptors.Count == 0) {
-			retval = ValidationFailed ($"{LogTag}: no descriptors read in {where}.");
+			retval = Utilities.GetFailureAspectState ($"{LogTag}: no descriptors read in {where}.");
 			return false;
 		}
 
@@ -47,11 +49,12 @@ protected override IAspectState ValidateInner ()
 		// Repetitive to `EnsureValidState`, but it's better than using `!` all over the place below...
 		Debug.Assert (Header != null);
 		Debug.Assert (Header.EntryCount != null);
+		Debug.Assert (Header.IndexSize != null);
 		Debug.Assert (Header.IndexEntryCount != null);
 		Debug.Assert (Descriptors != null);
 
 		ulong indexEntrySize = Header.Version.Is64Bit ? IndexEntrySize64 : IndexEntrySize32;
-		ulong indexSize = (indexEntrySize * (ulong)Header.IndexEntryCount!);
+		ulong indexSize = (ulong)Header.IndexSize; // (indexEntrySize * (ulong)Header.IndexEntryCount!);
 		ulong descriptorsSize = AssemblyDescriptorSize * (ulong)Header.EntryCount!;
 		ulong requiredStreamSize = HeaderSize + indexSize + descriptorsSize;
 
@@ -72,11 +75,11 @@ protected override IAspectState ValidateInner ()
 		Log.Debug ($"{LogTag}: calculated the required stream size to be {requiredStreamSize}");
 
 		if (requiredStreamSize > Int64.MaxValue) {
-			return ValidationFailed ($"{LogTag}: required stream size is too long for the stream API to handle.");
+			return Utilities.GetFailureAspectState ($"{LogTag}: required stream size is too long for the stream API to handle.");
 		}
 
 		if ((long)requiredStreamSize != StoreStream.Length) {
-			return ValidationFailed ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead.");
+			return Utilities.GetFailureAspectState ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead.");
 		} else {
 			Log.Debug ($"{LogTag}: stream size is valid.");
 		}
@@ -107,6 +110,8 @@ protected override IList<string> ReadAssemblyNames (BinaryReader reader)
 
 	protected override bool ReadAssemblies (BinaryReader reader, out IList<ApplicationAssembly>? assemblies)
 	{
+		Debug.Assert (Header != null);
+		Debug.Assert (Header.IndexEntryCount != null);
 		Debug.Assert (Descriptors != null);
 
 		assemblies = null;
@@ -115,27 +120,80 @@ protected override bool ReadAssemblies (BinaryReader reader, out IList<Applicati
 		}
 
 		IList<string> assemblyNames = ReadAssemblyNames (reader);
-		if (assemblyNames.Count != Descriptors.Count) {
-			Log.Debug ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})");
-			return false;
+		bool assemblyNamesUnreliable = assemblyNames.Count != Descriptors.Count;
+		if (assemblyNamesUnreliable) {
+			Log.Error ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})");
+		}
+
+		bool is64Bit = Header.Version.Is64Bit;
+		var index = new Dictionary<ulong, AssemblyStoreIndexEntryV3> ();
+		reader.BaseStream.Seek ((long)HeaderSize, SeekOrigin.Begin);
+		for (uint i = 0; i < Header.IndexEntryCount; i++) {
+			ulong hash = is64Bit ? reader.ReadUInt64 () : reader.ReadUInt32 ();
+			uint descIdx = reader.ReadUInt32 ();
+			byte ignore = reader.ReadByte ();
+
+			if (index.ContainsKey (hash)) {
+				Log.Error ($"{LogTag}: duplicate assembly name hash (0x{hash:x}) found in the '{Description}' assembly store.");
+				continue;
+			}
+			Log.Debug ($"{LogTag}: index entry {i} hash == 0x{hash:x}");
+			index.Add (hash, new AssemblyStoreIndexEntryV3 (hash, descIdx, ignore));
 		}
 
 		var ret = new List<ApplicationAssembly> ();
 		for (int i = 0; i < Descriptors.Count; i++) {
 			var desc = (AssemblyStoreAssemblyDescriptorV3)Descriptors[i];
-			string name = assemblyNames[i];
+			string name = assemblyNamesUnreliable ? "" : assemblyNames[i];
 			var assemblyStream = new SubStream (reader.BaseStream, (long)desc.DataOffset, (long)desc.DataSize);
+
+			ulong hash = NameHash (name);
+			Log.Debug ($"{LogTag}: hash for assembly '{name}' is 0x{hash:x}");
+
+			bool isIgnored = CheckIgnored (hash);
+			if (isIgnored) {
+				ret.Add ((ApplicationAssembly)ApplicationAssembly.CreateIgnoredAssembly (name, hash));
+				continue;
+			}
+
 			IAspectState assemblyState = ApplicationAssembly.ProbeAspect (assemblyStream, name);
 			if (!assemblyState.Success) {
 				assemblyStream.Dispose ();
 				continue;
 			}
 
 			var assembly = (ApplicationAssembly)ApplicationAssembly.LoadAspect (assemblyStream, assemblyState, name);
+			assembly.NameHash = hash;
 			ret.Add (assembly);
 		}
 
 		assemblies = ret.AsReadOnly ();
 		return true;
+
+		ulong NameHash (string name)
+		{
+			if (String.IsNullOrEmpty (name)) {
+				return 0;
+			}
+
+			if (name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
+				name = Path.GetFileNameWithoutExtension (name);
+			}
+			return MonoAndroidHelper.GetXxHash (name, is64Bit);
+		}
+
+		bool CheckIgnored (ulong hash)
+		{
+			if (hash == 0) {
+				return false;
+			}
+
+			if (!index.TryGetValue (hash, out AssemblyStoreIndexEntryV3? entry)) {
+				Log.Debug ($"{LogTag}: hash 0x{hash:x} not found in the index");
+				return false;
+			}
+
+			return entry.Ignore;
+		}
 	}
 }
@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Text;
 
 namespace ApplicationUtility;
 
@@ -31,4 +32,16 @@ public static void CloseAndDeleteFile (FileStream stream, bool quiet = true)
 
 		DeleteFile (path);
 	}
+
+	public static BinaryReader GetReaderAndRewindStream (Stream stream)
+	{
+		stream.Seek (0, SeekOrigin.Begin);
+		return new BinaryReader (stream, Encoding.UTF8, leaveOpen: true);
+	}
+
+	public static BasicAspectState GetFailureAspectState (string message)
+	{
+		Log.Debug (message);
+		return new BasicAspectState (false);
+	}
 }