From de137adf9ec300bf12fc081a05d638a6d2e5e204 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 3 Dec 2025 15:47:05 +0800 Subject: [PATCH 1/8] Add SHA verification for Hugging Face/Github Model Download --- .../HuggingFaceModelFileDetails.cs | 24 +++ AIDevGallery.Utils/ModelFileDetails.cs | 26 +++ AIDevGallery.Utils/ModelInformationHelper.cs | 24 ++- AIDevGallery.Utils/SourceGenerationContext.cs | 1 + .../Controls/DownloadProgressList.xaml | 18 ++ .../Controls/DownloadProgressList.xaml.cs | 57 +++++ .../ModelIntegrityVerificationFailedEvent.cs | 44 ++++ AIDevGallery/Utils/ModelDownload.cs | 196 +++++++++++++++++- AIDevGallery/ViewModels/DownloadableModel.cs | 65 +++++- 9 files changed, 440 insertions(+), 15 deletions(-) create mode 100644 AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs diff --git a/AIDevGallery.Utils/HuggingFaceModelFileDetails.cs b/AIDevGallery.Utils/HuggingFaceModelFileDetails.cs index 9a982820..942f767e 100644 --- a/AIDevGallery.Utils/HuggingFaceModelFileDetails.cs +++ b/AIDevGallery.Utils/HuggingFaceModelFileDetails.cs @@ -27,4 +27,28 @@ public class HuggingFaceModelFileDetails /// [JsonPropertyName("path")] public string? Path { get; init; } + + /// + /// Gets the LFS (Large File Storage) information for the file. + /// + [JsonPropertyName("lfs")] + public HuggingFaceLfsInfo? Lfs { get; init; } +} + +/// +/// LFS (Large File Storage) information for a Hugging Face file. +/// +public class HuggingFaceLfsInfo +{ + /// + /// Gets the OID (SHA256 hash) of the file. Format: "sha256:abc123..." + /// + [JsonPropertyName("oid")] + public string? Oid { get; init; } + + /// + /// Gets the size of the file in LFS. + /// + [JsonPropertyName("size")] + public long Size { get; init; } } \ No newline at end of file diff --git a/AIDevGallery.Utils/ModelFileDetails.cs b/AIDevGallery.Utils/ModelFileDetails.cs index f5e717df..03bc4742 100644 --- a/AIDevGallery.Utils/ModelFileDetails.cs +++ b/AIDevGallery.Utils/ModelFileDetails.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; + namespace AIDevGallery.Utils; /// @@ -27,4 +29,28 @@ public class ModelFileDetails /// Gets the relative path to the file /// public string? Path { get; init; } + + /// + /// Gets the expected SHA256 hash of the file (from Hugging Face LFS). + /// + public string? Sha256 { get; init; } + + /// + /// Gets the expected Git blob SHA-1 hash of the file (from GitHub API). + /// This is NOT the same as SHA1 of the file content - it includes a "blob {size}\0" prefix. + /// + public string? GitBlobSha1 { get; init; } + + /// + /// Gets a value indicating whether this file should be verified for integrity (main model files like .onnx). + /// + public bool ShouldVerifyIntegrity => Name != null && + (Name.EndsWith(".onnx", StringComparison.OrdinalIgnoreCase) || + Name.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase) || + Name.EndsWith(".safetensors", StringComparison.OrdinalIgnoreCase)); + + /// + /// Gets a value indicating whether this file has any hash available for verification. + /// + public bool HasVerificationHash => !string.IsNullOrEmpty(Sha256) || !string.IsNullOrEmpty(GitBlobSha1); } \ No newline at end of file diff --git a/AIDevGallery.Utils/ModelInformationHelper.cs b/AIDevGallery.Utils/ModelInformationHelper.cs index 08293a94..550e59e7 100644 --- a/AIDevGallery.Utils/ModelInformationHelper.cs +++ b/AIDevGallery.Utils/ModelInformationHelper.cs @@ -63,7 +63,8 @@ public static async Task> GetDownloadFilesFromGitHub(GitH DownloadUrl = f.DownloadUrl, Size = f.Size, Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(), - Path = f.Path + Path = f.Path, + GitBlobSha1 = f.Sha }).ToList(); } @@ -172,13 +173,26 @@ public static async Task> GetDownloadFilesFromHuggingFace } return hfFiles.Where(f => f.Type != "directory").Select(f => - new ModelFileDetails() + { + string? sha256 = null; + if (f.Lfs?.Oid != null) + { + sha256 = f.Lfs.Oid; + if (sha256.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + sha256 = sha256.Substring(7); + } + } + + return new ModelFileDetails() { DownloadUrl = $"https://huggingface.co/{hfUrl.Organization}/{hfUrl.Repo}/resolve/{hfUrl.Ref}/{f.Path}", - Size = f.Size, + Size = f.Lfs?.Size ?? f.Size, Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(), - Path = f.Path - }).ToList(); + Path = f.Path, + Sha256 = sha256 + }; + }).ToList(); } /// diff --git a/AIDevGallery.Utils/SourceGenerationContext.cs b/AIDevGallery.Utils/SourceGenerationContext.cs index 2fa31fda..6c02f5f5 100644 --- a/AIDevGallery.Utils/SourceGenerationContext.cs +++ b/AIDevGallery.Utils/SourceGenerationContext.cs @@ -9,6 +9,7 @@ namespace AIDevGallery.Utils; [JsonSourceGenerationOptions(WriteIndented = true, AllowTrailingCommas = true)] [JsonSerializable(typeof(List))] [JsonSerializable(typeof(List))] +[JsonSerializable(typeof(HuggingFaceLfsInfo))] internal partial class SourceGenerationContext : JsonSerializerContext { } \ No newline at end of file diff --git a/AIDevGallery/Controls/DownloadProgressList.xaml b/AIDevGallery/Controls/DownloadProgressList.xaml index f6268a71..2f5bfb88 100644 --- a/AIDevGallery/Controls/DownloadProgressList.xaml +++ b/AIDevGallery/Controls/DownloadProgressList.xaml @@ -169,6 +169,24 @@ Tag="{x:Bind}" ToolTipService.ToolTip="Cancel" Visibility="{x:Bind vm:DownloadableModel.VisibleWhenDownloading(Status), Mode=OneWay}" /> + diff --git a/AIDevGallery/Controls/DownloadProgressList.xaml.cs b/AIDevGallery/Controls/DownloadProgressList.xaml.cs index 1cbfe46c..e86f757c 100644 --- a/AIDevGallery/Controls/DownloadProgressList.xaml.cs +++ b/AIDevGallery/Controls/DownloadProgressList.xaml.cs @@ -5,6 +5,7 @@ using AIDevGallery.ViewModels; using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Controls; +using System; using System.Collections.ObjectModel; using System.Linq; @@ -86,4 +87,60 @@ private void ClearHistory_Click(object sender, RoutedEventArgs e) } } } + + private async void VerificationFailedClicked(object sender, RoutedEventArgs e) + { + if (sender is Button button && button.Tag is DownloadableModel downloadableModel) + { + var dialog = new ContentDialog + { + Title = "Integrity verification failed", + Content = new StackPanel + { + Spacing = 12, + Children = + { + new TextBlock + { + Text = "The downloaded model file(s) did not match the expected hash. This could indicate the file was corrupted during download or has been tampered with.", + TextWrapping = TextWrapping.Wrap + }, + new TextBlock + { + Text = downloadableModel.VerificationFailureMessage ?? "Unknown verification error", + TextWrapping = TextWrapping.Wrap, + Foreground = (Microsoft.UI.Xaml.Media.Brush)Application.Current.Resources["SystemFillColorCautionBrush"], + FontSize = 12 + }, + new TextBlock + { + Text = "Would you like to keep the model anyway or delete it?", + TextWrapping = TextWrapping.Wrap + } + } + }, + PrimaryButtonText = "Delete model", + SecondaryButtonText = "Keep anyway", + CloseButtonText = "Cancel", + DefaultButton = ContentDialogButton.Primary, + XamlRoot = this.XamlRoot + }; + + var result = await dialog.ShowAsync(); + + if (result == ContentDialogResult.Primary) + { + // Delete the model + downloadableModel.DeleteVerificationFailedModel(); + downloadProgresses.Remove(downloadableModel); + } + else if (result == ContentDialogResult.Secondary) + { + // Keep the model despite verification failure + downloadableModel.KeepVerificationFailedModel(); + } + + // If Cancel, do nothing - leave the item in the list + } + } } \ No newline at end of file diff --git a/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs new file mode 100644 index 00000000..5487e1d4 --- /dev/null +++ b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using Microsoft.Diagnostics.Telemetry; +using Microsoft.Diagnostics.Telemetry.Internal; +using System; +using System.Diagnostics.Tracing; + +namespace AIDevGallery.Telemetry.Events; + +[EventData] +internal class ModelIntegrityVerificationFailedEvent : EventBase +{ + internal ModelIntegrityVerificationFailedEvent(string modelUrl, string fileName, string expectedHash, string actualHash) + { + ModelUrl = modelUrl; + FileName = fileName; + ExpectedHash = expectedHash; + ActualHash = actualHash; + EventTime = DateTime.UtcNow; + } + + public string ModelUrl { get; private set; } + public string FileName { get; private set; } + public string ExpectedHash { get; private set; } + public string ActualHash { get; private set; } + public DateTime EventTime { get; private set; } + + public override PartA_PrivTags PartA_PrivTags => PrivTags.ProductAndServiceUsage; + + public override void ReplaceSensitiveStrings(Func replaceSensitiveStrings) + { + } + + public static void Log(string modelUrl, string fileName, string expectedHash, string actualHash) + { + TelemetryFactory.Get().LogError( + "ModelIntegrityVerificationFailed_Event", + LogLevel.Critical, + new ModelIntegrityVerificationFailedEvent(modelUrl, fileName, expectedHash, actualHash)); + } +} + + diff --git a/AIDevGallery/Utils/ModelDownload.cs b/AIDevGallery/Utils/ModelDownload.cs index 6dfd7dc7..d2f19553 100644 --- a/AIDevGallery/Utils/ModelDownload.cs +++ b/AIDevGallery/Utils/ModelDownload.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Security.Cryptography; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; @@ -50,6 +51,22 @@ protected set } } + private string? _verificationFailureMessage; + public string? VerificationFailureMessage + { + get => _verificationFailureMessage; + protected set + { + _verificationFailureMessage = value; + StateChanged?.Invoke(this, new ModelDownloadEventArgs + { + Progress = DownloadProgress, + Status = DownloadStatus, + VerificationFailureMessage = _verificationFailureMessage + }); + } + } + protected CancellationTokenSource CancellationTokenSource { get; } public void Dispose() @@ -73,6 +90,11 @@ internal class OnnxModelDownload : ModelDownload { public ModelUrl ModelUrl { get; set; } + /// + /// Gets the list of files that failed integrity verification. + /// + public List<(string FileName, string ExpectedHash, string ActualHash)> FailedVerifications { get; } = []; + public OnnxModelDownload(ModelDetails details) : base(details) { @@ -108,12 +130,15 @@ public override async Task StartDownload() if (cachedModel == null) { - DownloadStatus = DownloadStatus.Canceled; - - var localPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); - if (Directory.Exists(localPath)) + if (DownloadStatus != DownloadStatus.VerificationFailed) { - Directory.Delete(localPath, true); + DownloadStatus = DownloadStatus.Canceled; + + var localPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); + if (Directory.Exists(localPath)) + { + Directory.Delete(localPath, true); + } } return false; @@ -130,7 +155,57 @@ public override void CancelDownload() DownloadStatus = DownloadStatus.Canceled; } - private async Task DownloadModel(string cacheDir, IProgress? progress = null) + /// + /// Deletes the downloaded model files after user chooses not to keep a verification-failed model. + /// + public void DeleteFailedModel() + { + var localPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); + if (Directory.Exists(localPath)) + { + Directory.Delete(localPath, true); + } + } + + /// + /// Keeps the downloaded model files despite verification failure (user's choice). + /// + /// A task representing the asynchronous operation. + public async Task KeepModelDespiteVerificationFailure() + { + var localFolderPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); + var filesToDownload = await GetFilesToDownloadAsync(); + long modelSize = filesToDownload.Sum(f => f.Size); + + var cachedModel = new CachedModel(Details, ModelUrl.IsFile ? $"{localFolderPath}\\{filesToDownload.First().Name}" : localFolderPath, ModelUrl.IsFile, modelSize); + await App.ModelCache.CacheStore.AddModel(cachedModel); + DownloadStatus = DownloadStatus.Completed; + } + + private async Task> GetFilesToDownloadAsync() + { + var cancellationToken = CancellationTokenSource.Token; + List filesToDownload; + + if (Details.Url.StartsWith("https://github.com", StringComparison.InvariantCulture)) + { + var ghUrl = new GitHubUrl(Details.Url); + filesToDownload = await ModelInformationHelper.GetDownloadFilesFromGitHub(ghUrl, cancellationToken); + } + else + { + var hfUrl = new HuggingFaceUrl(Details.Url); + using var socketsHttpHandler = new SocketsHttpHandler + { + MaxConnectionsPerServer = 4 + }; + filesToDownload = await ModelInformationHelper.GetDownloadFilesFromHuggingFace(hfUrl, socketsHttpHandler, cancellationToken); + } + + return ModelInformationHelper.FilterFiles(filesToDownload, Details.FileFilters); + } + + private async Task DownloadModel(string cacheDir, IProgress? progress = null) { ModelUrl url; List filesToDownload; @@ -171,6 +246,9 @@ private async Task DownloadModel(string cacheDir, IProgress? using var client = new HttpClient(); + // Track files that need verification + List<(string FilePath, ModelFileDetails FileDetails)> filesToVerify = []; + foreach (var downloadableFile in filesToDownload) { if (downloadableFile.DownloadUrl == null) @@ -187,6 +265,12 @@ private async Task DownloadModel(string cacheDir, IProgress? var existingFileInfo = new FileInfo(existingFile); if (existingFileInfo.Length == downloadableFile.Size) { + // Still need to verify existing files if they have a hash + if (downloadableFile.ShouldVerifyIntegrity && downloadableFile.HasVerificationHash) + { + filesToVerify.Add((filePath, downloadableFile)); + } + continue; } } @@ -201,16 +285,109 @@ private async Task DownloadModel(string cacheDir, IProgress? var fileInfo = new FileInfo(filePath); if (fileInfo.Length != downloadableFile.Size) { - // file did not download properly, should retry + throw new IOException($"File size mismatch for {downloadableFile.Name}: expected {downloadableFile.Size}, got {fileInfo.Length}"); + } + + // Add to verification list if it's a main model file with hash + if (downloadableFile.ShouldVerifyIntegrity && downloadableFile.HasVerificationHash) + { + filesToVerify.Add((filePath, downloadableFile)); } bytesDownloaded += downloadableFile.Size; } + // Verify integrity of main model files + if (filesToVerify.Count > 0) + { + DownloadStatus = DownloadStatus.Verifying; + + foreach (var (filePath, fileDetails) in filesToVerify) + { + bool verified; + string expectedHash; + string actualHash; + + if (!string.IsNullOrEmpty(fileDetails.Sha256)) + { + // Hugging Face: use SHA256 of file content + expectedHash = fileDetails.Sha256; + actualHash = await ComputeSha256Async(filePath, cancellationToken); + verified = string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase); + } + else if (!string.IsNullOrEmpty(fileDetails.GitBlobSha1)) + { + // GitHub: use Git blob SHA-1 (includes "blob {size}\0" prefix) + expectedHash = fileDetails.GitBlobSha1; + actualHash = await ComputeGitBlobSha1Async(filePath, cancellationToken); + verified = string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase); + } + else + { + continue; + } + + if (!verified) + { + FailedVerifications.Add((fileDetails.Name ?? filePath, expectedHash, actualHash)); + ModelIntegrityVerificationFailedEvent.Log(Details.Url, fileDetails.Name ?? filePath, expectedHash, actualHash); + } + } + + if (FailedVerifications.Count > 0) + { + var failedFileNames = string.Join(", ", FailedVerifications.Select(f => f.FileName)); + VerificationFailureMessage = $"Integrity verification failed for: {failedFileNames}"; + DownloadStatus = DownloadStatus.VerificationFailed; + return null; + } + } + var modelDirectory = url.GetLocalPath(cacheDir); return new CachedModel(Details, url.IsFile ? $"{modelDirectory}\\{filesToDownload.First().Name}" : modelDirectory, url.IsFile, modelSize); } + + private static async Task ComputeSha256Async(string filePath, CancellationToken cancellationToken) + { + using var sha256 = SHA256.Create(); + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, useAsync: true); + var hashBytes = await sha256.ComputeHashAsync(stream, cancellationToken); + return Convert.ToHexString(hashBytes).ToLowerInvariant(); + } + + /// + /// Computes the Git blob SHA-1 hash of a file. + /// Git blob format: "blob {size}\0{content}". + /// + /// + /// SHA-1 is used here because Git uses SHA-1 for blob hashing. + /// This is required for compatibility with GitHub's API which returns Git blob SHA-1 hashes. + /// +#pragma warning disable CA5350 // Do not use weak cryptographic algorithms - Git requires SHA-1 for blob hashing + private static async Task ComputeGitBlobSha1Async(string filePath, CancellationToken cancellationToken) + { + var fileInfo = new FileInfo(filePath); + var header = System.Text.Encoding.ASCII.GetBytes($"blob {fileInfo.Length}\0"); + + using var sha1 = SHA1.Create(); + using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, useAsync: true); + + // Create a combined stream with header + file content + sha1.TransformBlock(header, 0, header.Length, null, 0); + + var buffer = new byte[81920]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) + { + sha1.TransformBlock(buffer, 0, bytesRead, null, 0); + } + + sha1.TransformFinalBlock([], 0, 0); + + return Convert.ToHexString(sha1.Hash!).ToLowerInvariant(); + } +#pragma warning restore CA5350 } internal class FoundryLocalModelDownload : ModelDownload @@ -263,12 +440,15 @@ internal enum DownloadStatus { Waiting, InProgress, + Verifying, Completed, - Canceled + Canceled, + VerificationFailed } internal class ModelDownloadEventArgs { public required float Progress { get; init; } public required DownloadStatus Status { get; init; } + public string? VerificationFailureMessage { get; init; } } \ No newline at end of file diff --git a/AIDevGallery/ViewModels/DownloadableModel.cs b/AIDevGallery/ViewModels/DownloadableModel.cs index 316485ca..777a434f 100644 --- a/AIDevGallery/ViewModels/DownloadableModel.cs +++ b/AIDevGallery/ViewModels/DownloadableModel.cs @@ -22,6 +22,9 @@ internal partial class DownloadableModel : BaseModel [ObservableProperty] public partial DownloadStatus Status { get; set; } = DownloadStatus.Waiting; + [ObservableProperty] + public partial string? VerificationFailureMessage { get; set; } + public bool IsDownloadEnabled => Compatibility.CompatibilityState != ModelCompatibilityState.NotCompatible; private ModelDownload? _modelDownload; @@ -52,6 +55,7 @@ public ModelDownload? ModelDownload _modelDownload.StateChanged += ModelDownload_StateChanged; Status = _modelDownload.DownloadStatus; Progress = _modelDownload.DownloadProgress; + VerificationFailureMessage = _modelDownload.VerificationFailureMessage; CanDownload = false; } } @@ -87,6 +91,36 @@ public void CancelDownload() ModelDownload?.CancelDownload(); } + /// + /// Deletes the model files when user rejects a verification-failed model. + /// + public void DeleteVerificationFailedModel() + { + if (ModelDownload is OnnxModelDownload onnxDownload) + { + onnxDownload.DeleteFailedModel(); + } + + Status = DownloadStatus.Canceled; + ModelDownload = null; + VerificationFailureMessage = null; + } + + /// + /// Keeps the model despite verification failure (user's choice). + /// + public async void KeepVerificationFailedModel() + { + if (ModelDownload is OnnxModelDownload onnxDownload) + { + await onnxDownload.KeepModelDespiteVerificationFailure(); + } + + Status = DownloadStatus.Completed; + ModelDownload = null; + VerificationFailureMessage = null; + } + private void ModelDownload_StateChanged(object? sender, ModelDownloadEventArgs e) { if (!_progressTimer.IsEnabled) @@ -94,7 +128,13 @@ private void ModelDownload_StateChanged(object? sender, ModelDownloadEventArgs e _progressTimer.Start(); } - if (e.Progress == 1) + if (e.Progress == 1 && e.Status == DownloadStatus.InProgress) + { + // Download complete, but may still need verification + return; + } + + if (e.Status == DownloadStatus.Completed) { Status = DownloadStatus.Completed; ModelDownload = null; @@ -106,6 +146,17 @@ private void ModelDownload_StateChanged(object? sender, ModelDownloadEventArgs e ModelDownload = null; Progress = 0; } + + if (e.Status == DownloadStatus.VerificationFailed) + { + Status = DownloadStatus.VerificationFailed; + VerificationFailureMessage = e.VerificationFailureMessage; + } + + if (e.Status == DownloadStatus.Verifying) + { + Status = DownloadStatus.Verifying; + } } private void ProgressTimer_Tick(object? sender, object e) @@ -115,6 +166,7 @@ private void ProgressTimer_Tick(object? sender, object e) { Progress = ModelDownload.DownloadProgress * 100; Status = ModelDownload.DownloadStatus; + VerificationFailureMessage = ModelDownload.VerificationFailureMessage; } } @@ -144,7 +196,7 @@ public static Visibility DownloadStatusButtonVisibility(ModelDownload download) public static Visibility VisibleWhenDownloading(DownloadStatus status) { - return status is DownloadStatus.InProgress or DownloadStatus.Waiting ? Visibility.Visible : Visibility.Collapsed; + return status is DownloadStatus.InProgress or DownloadStatus.Waiting or DownloadStatus.Verifying ? Visibility.Visible : Visibility.Collapsed; } public static Visibility VisibleWhenCanceled(DownloadStatus status) @@ -157,6 +209,11 @@ public static Visibility VisibleWhenDownloaded(DownloadStatus status) return status == DownloadStatus.Completed ? Visibility.Visible : Visibility.Collapsed; } + public static Visibility VisibleWhenVerificationFailed(DownloadStatus status) + { + return status == DownloadStatus.VerificationFailed ? Visibility.Visible : Visibility.Collapsed; + } + public static Visibility BoolToVisibilityInverse(bool value) { return value ? Visibility.Collapsed : Visibility.Visible; @@ -180,10 +237,14 @@ public static string StatusToText(DownloadStatus status) return "Waiting.."; case DownloadStatus.InProgress: return "Downloading.."; + case DownloadStatus.Verifying: + return "Verifying integrity.."; case DownloadStatus.Completed: return "Downloaded"; case DownloadStatus.Canceled: return "Canceled"; + case DownloadStatus.VerificationFailed: + return "Verification failed"; default: return string.Empty; } From 6a801d0d568d0e278e8ba4d6680cab04c9913619 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Wed, 10 Dec 2025 19:48:23 +0800 Subject: [PATCH 2/8] Github Model SHA-256 verification --- AIDevGallery.Utils/ModelFileDetails.cs | 14 ++- AIDevGallery.Utils/ModelInformationHelper.cs | 89 +++++++++++++++++++- AIDevGallery/Utils/ModelDownload.cs | 57 ++----------- 3 files changed, 95 insertions(+), 65 deletions(-) diff --git a/AIDevGallery.Utils/ModelFileDetails.cs b/AIDevGallery.Utils/ModelFileDetails.cs index 03bc4742..5456c4db 100644 --- a/AIDevGallery.Utils/ModelFileDetails.cs +++ b/AIDevGallery.Utils/ModelFileDetails.cs @@ -31,16 +31,12 @@ public class ModelFileDetails public string? Path { get; init; } /// - /// Gets the expected SHA256 hash of the file (from Hugging Face LFS). + /// Gets the expected SHA256 hash of the file. + /// For Hugging Face: from LFS oid field. + /// For GitHub: from LFS pointer file (for LFS-tracked files). /// public string? Sha256 { get; init; } - /// - /// Gets the expected Git blob SHA-1 hash of the file (from GitHub API). - /// This is NOT the same as SHA1 of the file content - it includes a "blob {size}\0" prefix. - /// - public string? GitBlobSha1 { get; init; } - /// /// Gets a value indicating whether this file should be verified for integrity (main model files like .onnx). /// @@ -50,7 +46,7 @@ public class ModelFileDetails Name.EndsWith(".safetensors", StringComparison.OrdinalIgnoreCase)); /// - /// Gets a value indicating whether this file has any hash available for verification. + /// Gets a value indicating whether this file has a hash available for verification. /// - public bool HasVerificationHash => !string.IsNullOrEmpty(Sha256) || !string.IsNullOrEmpty(GitBlobSha1); + public bool HasVerificationHash => !string.IsNullOrEmpty(Sha256); } \ No newline at end of file diff --git a/AIDevGallery.Utils/ModelInformationHelper.cs b/AIDevGallery.Utils/ModelInformationHelper.cs index 550e59e7..d70cc966 100644 --- a/AIDevGallery.Utils/ModelInformationHelper.cs +++ b/AIDevGallery.Utils/ModelInformationHelper.cs @@ -57,15 +57,96 @@ public static async Task> GetDownloadFilesFromGitHub(GitH return []; } - return files.Select(f => - new ModelFileDetails() + var result = new List(); + + foreach (var f in files) + { + string? sha256 = null; + + // Check if this is a Git LFS file by checking if download_url points to media.githubusercontent.com + // LFS files have their actual content stored separately and we need to get SHA256 from the LFS pointer +#if NET8_0_OR_GREATER + if (f.DownloadUrl != null && f.DownloadUrl.Contains("media.githubusercontent.com", StringComparison.OrdinalIgnoreCase)) +#else + if (f.DownloadUrl != null && f.DownloadUrl.IndexOf("media.githubusercontent.com", StringComparison.OrdinalIgnoreCase) >= 0) +#endif + { + // This is an LFS file - fetch the LFS pointer to get SHA256 + sha256 = await GetLfsFileSha256Async(client, url, f.Path, cancellationToken); + } + + result.Add(new ModelFileDetails() { DownloadUrl = f.DownloadUrl, Size = f.Size, Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(), Path = f.Path, - GitBlobSha1 = f.Sha - }).ToList(); + Sha256 = sha256 + }); + } + + return result; + } + + /// + /// Retrieves the SHA256 hash from a Git LFS pointer file. + /// + /// The HTTP client to use. + /// The GitHub URL. + /// The path to the file in the repository. + /// A token to monitor for cancellation requests. + /// The SHA256 hash if found, otherwise null. + private static async Task GetLfsFileSha256Async(HttpClient client, GitHubUrl url, string? filePath, CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(filePath)) + { + return null; + } + + try + { + // Fetch the raw LFS pointer file from raw.githubusercontent.com + var lfsPointerUrl = $"https://raw.githubusercontent.com/{url.Organization}/{url.Repo}/{url.Ref}/{filePath}"; +#if NET8_0_OR_GREATER + var lfsPointerContent = await client.GetStringAsync(lfsPointerUrl, cancellationToken); +#else + var response = await client.GetAsync(lfsPointerUrl, cancellationToken); + var lfsPointerContent = await response.Content.ReadAsStringAsync(); +#endif + + // Parse the LFS pointer format: + // version https://git-lfs.github.com/spec/v1 + return ParseLfsPointerSha256(lfsPointerContent); + } + catch (Exception ex) + { + Debug.WriteLine($"Failed to get LFS pointer for {filePath}: {ex.Message}"); + return null; + } + } + + /// + /// Parses a Git LFS pointer file content to extract the SHA256 hash. + /// + /// The content of the LFS pointer file. + /// The SHA256 hash if found, otherwise null. + private static string? ParseLfsPointerSha256(string lfsPointerContent) + { + if (string.IsNullOrEmpty(lfsPointerContent)) + { + return null; + } + + var lines = lfsPointerContent.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries); + foreach (var line in lines) + { + if (line.StartsWith("oid sha256:", StringComparison.OrdinalIgnoreCase)) + { + return line.Substring("oid sha256:".Length).Trim(); + } + } + + return null; } /// diff --git a/AIDevGallery/Utils/ModelDownload.cs b/AIDevGallery/Utils/ModelDownload.cs index d2f19553..3022a21c 100644 --- a/AIDevGallery/Utils/ModelDownload.cs +++ b/AIDevGallery/Utils/ModelDownload.cs @@ -304,29 +304,15 @@ private async Task> GetFilesToDownloadAsync() foreach (var (filePath, fileDetails) in filesToVerify) { - bool verified; - string expectedHash; - string actualHash; - - if (!string.IsNullOrEmpty(fileDetails.Sha256)) - { - // Hugging Face: use SHA256 of file content - expectedHash = fileDetails.Sha256; - actualHash = await ComputeSha256Async(filePath, cancellationToken); - verified = string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase); - } - else if (!string.IsNullOrEmpty(fileDetails.GitBlobSha1)) - { - // GitHub: use Git blob SHA-1 (includes "blob {size}\0" prefix) - expectedHash = fileDetails.GitBlobSha1; - actualHash = await ComputeGitBlobSha1Async(filePath, cancellationToken); - verified = string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase); - } - else + if (string.IsNullOrEmpty(fileDetails.Sha256)) { continue; } + var expectedHash = fileDetails.Sha256; + var actualHash = await ComputeSha256Async(filePath, cancellationToken); + var verified = string.Equals(actualHash, expectedHash, StringComparison.OrdinalIgnoreCase); + if (!verified) { FailedVerifications.Add((fileDetails.Name ?? filePath, expectedHash, actualHash)); @@ -355,39 +341,6 @@ private static async Task ComputeSha256Async(string filePath, Cancellati var hashBytes = await sha256.ComputeHashAsync(stream, cancellationToken); return Convert.ToHexString(hashBytes).ToLowerInvariant(); } - - /// - /// Computes the Git blob SHA-1 hash of a file. - /// Git blob format: "blob {size}\0{content}". - /// - /// - /// SHA-1 is used here because Git uses SHA-1 for blob hashing. - /// This is required for compatibility with GitHub's API which returns Git blob SHA-1 hashes. - /// -#pragma warning disable CA5350 // Do not use weak cryptographic algorithms - Git requires SHA-1 for blob hashing - private static async Task ComputeGitBlobSha1Async(string filePath, CancellationToken cancellationToken) - { - var fileInfo = new FileInfo(filePath); - var header = System.Text.Encoding.ASCII.GetBytes($"blob {fileInfo.Length}\0"); - - using var sha1 = SHA1.Create(); - using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, useAsync: true); - - // Create a combined stream with header + file content - sha1.TransformBlock(header, 0, header.Length, null, 0); - - var buffer = new byte[81920]; - int bytesRead; - while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(), cancellationToken)) > 0) - { - sha1.TransformBlock(buffer, 0, bytesRead, null, 0); - } - - sha1.TransformFinalBlock([], 0, 0); - - return Convert.ToHexString(sha1.Hash!).ToLowerInvariant(); - } -#pragma warning restore CA5350 } internal class FoundryLocalModelDownload : ModelDownload From eb022f3d1d54e7285219ff0b28d000e4b13ef803 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Thu, 11 Dec 2025 15:56:50 +0800 Subject: [PATCH 3/8] Advance github SHA access --- AIDevGallery.Utils/GitHubModelFileDetails.cs | 13 ++++ AIDevGallery.Utils/ModelInformationHelper.cs | 70 +++++--------------- 2 files changed, 29 insertions(+), 54 deletions(-) diff --git a/AIDevGallery.Utils/GitHubModelFileDetails.cs b/AIDevGallery.Utils/GitHubModelFileDetails.cs index fca956e8..4c5b64a8 100644 --- a/AIDevGallery.Utils/GitHubModelFileDetails.cs +++ b/AIDevGallery.Utils/GitHubModelFileDetails.cs @@ -45,4 +45,17 @@ public class GitHubModelFileDetails /// [JsonPropertyName("type")] public string? Type { get; init; } + + /// + /// Gets the base64-encoded content of the file. + /// For LFS files, this contains the LFS pointer with SHA256. + /// + [JsonPropertyName("content")] + public string? Content { get; init; } + + /// + /// Gets the encoding of the content (usually "base64"). + /// + [JsonPropertyName("encoding")] + public string? Encoding { get; init; } } \ No newline at end of file diff --git a/AIDevGallery.Utils/ModelInformationHelper.cs b/AIDevGallery.Utils/ModelInformationHelper.cs index d70cc966..971d4cf6 100644 --- a/AIDevGallery.Utils/ModelInformationHelper.cs +++ b/AIDevGallery.Utils/ModelInformationHelper.cs @@ -57,72 +57,34 @@ public static async Task> GetDownloadFilesFromGitHub(GitH return []; } - var result = new List(); - - foreach (var f in files) + return files.Select(f => { string? sha256 = null; - // Check if this is a Git LFS file by checking if download_url points to media.githubusercontent.com - // LFS files have their actual content stored separately and we need to get SHA256 from the LFS pointer -#if NET8_0_OR_GREATER - if (f.DownloadUrl != null && f.DownloadUrl.Contains("media.githubusercontent.com", StringComparison.OrdinalIgnoreCase)) -#else - if (f.DownloadUrl != null && f.DownloadUrl.IndexOf("media.githubusercontent.com", StringComparison.OrdinalIgnoreCase) >= 0) -#endif + // For LFS files, the API returns the LFS pointer content in base64. + // We can extract SHA256 directly without additional HTTP requests. + if (f.Content != null && f.Encoding == "base64") { - // This is an LFS file - fetch the LFS pointer to get SHA256 - sha256 = await GetLfsFileSha256Async(client, url, f.Path, cancellationToken); + try + { + var decodedContent = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(f.Content)); + sha256 = ParseLfsPointerSha256(decodedContent); + } + catch + { + // Not a valid base64 or not an LFS pointer, ignore + } } - result.Add(new ModelFileDetails() + return new ModelFileDetails() { DownloadUrl = f.DownloadUrl, Size = f.Size, Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(), Path = f.Path, Sha256 = sha256 - }); - } - - return result; - } - - /// - /// Retrieves the SHA256 hash from a Git LFS pointer file. - /// - /// The HTTP client to use. - /// The GitHub URL. - /// The path to the file in the repository. - /// A token to monitor for cancellation requests. - /// The SHA256 hash if found, otherwise null. - private static async Task GetLfsFileSha256Async(HttpClient client, GitHubUrl url, string? filePath, CancellationToken cancellationToken) - { - if (string.IsNullOrEmpty(filePath)) - { - return null; - } - - try - { - // Fetch the raw LFS pointer file from raw.githubusercontent.com - var lfsPointerUrl = $"https://raw.githubusercontent.com/{url.Organization}/{url.Repo}/{url.Ref}/{filePath}"; -#if NET8_0_OR_GREATER - var lfsPointerContent = await client.GetStringAsync(lfsPointerUrl, cancellationToken); -#else - var response = await client.GetAsync(lfsPointerUrl, cancellationToken); - var lfsPointerContent = await response.Content.ReadAsStringAsync(); -#endif - - // Parse the LFS pointer format: - // version https://git-lfs.github.com/spec/v1 - return ParseLfsPointerSha256(lfsPointerContent); - } - catch (Exception ex) - { - Debug.WriteLine($"Failed to get LFS pointer for {filePath}: {ex.Message}"); - return null; - } + }; + }).ToList(); } /// From 212f992957200e51a9ab9a04d605120e3255c1a6 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Fri, 12 Dec 2025 12:22:08 +0800 Subject: [PATCH 4/8] Update logic --- AIDevGallery.Utils/GitHubModelFileDetails.cs | 2 +- AIDevGallery.Utils/ModelFileDetails.cs | 2 +- AIDevGallery.Utils/ModelInformationHelper.cs | 6 +-- .../ModelIntegrityVerificationFailedEvent.cs | 6 +-- AIDevGallery/Utils/ModelDownload.cs | 42 +++++++------------ 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/AIDevGallery.Utils/GitHubModelFileDetails.cs b/AIDevGallery.Utils/GitHubModelFileDetails.cs index 4c5b64a8..beaef13f 100644 --- a/AIDevGallery.Utils/GitHubModelFileDetails.cs +++ b/AIDevGallery.Utils/GitHubModelFileDetails.cs @@ -47,7 +47,7 @@ public class GitHubModelFileDetails public string? Type { get; init; } /// - /// Gets the base64-encoded content of the file. + /// Gets the encoded content of the file. /// For LFS files, this contains the LFS pointer with SHA256. /// [JsonPropertyName("content")] diff --git a/AIDevGallery.Utils/ModelFileDetails.cs b/AIDevGallery.Utils/ModelFileDetails.cs index 5456c4db..e9ad7419 100644 --- a/AIDevGallery.Utils/ModelFileDetails.cs +++ b/AIDevGallery.Utils/ModelFileDetails.cs @@ -33,7 +33,7 @@ public class ModelFileDetails /// /// Gets the expected SHA256 hash of the file. /// For Hugging Face: from LFS oid field. - /// For GitHub: from LFS pointer file (for LFS-tracked files). + /// For GitHub: from LFS pointer file. /// public string? Sha256 { get; init; } diff --git a/AIDevGallery.Utils/ModelInformationHelper.cs b/AIDevGallery.Utils/ModelInformationHelper.cs index 971d4cf6..be6ebe46 100644 --- a/AIDevGallery.Utils/ModelInformationHelper.cs +++ b/AIDevGallery.Utils/ModelInformationHelper.cs @@ -61,8 +61,6 @@ public static async Task> GetDownloadFilesFromGitHub(GitH { string? sha256 = null; - // For LFS files, the API returns the LFS pointer content in base64. - // We can extract SHA256 directly without additional HTTP requests. if (f.Content != null && f.Encoding == "base64") { try @@ -70,9 +68,9 @@ public static async Task> GetDownloadFilesFromGitHub(GitH var decodedContent = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(f.Content)); sha256 = ParseLfsPointerSha256(decodedContent); } - catch + catch (FormatException) { - // Not a valid base64 or not an LFS pointer, ignore + Debug.WriteLine($"Failed to decode base64 content for {f.Path}"); } } diff --git a/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs index 5487e1d4..b04f6b21 100644 --- a/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs +++ b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs @@ -36,9 +36,7 @@ public static void Log(string modelUrl, string fileName, string expectedHash, st { TelemetryFactory.Get().LogError( "ModelIntegrityVerificationFailed_Event", - LogLevel.Critical, + LogLevel.Info, new ModelIntegrityVerificationFailedEvent(modelUrl, fileName, expectedHash, actualHash)); } -} - - +} \ No newline at end of file diff --git a/AIDevGallery/Utils/ModelDownload.cs b/AIDevGallery/Utils/ModelDownload.cs index 3022a21c..f47cd453 100644 --- a/AIDevGallery/Utils/ModelDownload.cs +++ b/AIDevGallery/Utils/ModelDownload.cs @@ -95,6 +95,11 @@ internal class OnnxModelDownload : ModelDownload /// public List<(string FileName, string ExpectedHash, string ActualHash)> FailedVerifications { get; } = []; + /// + /// Cached model info from the download process + /// + private CachedModel? _pendingCachedModel; + public OnnxModelDownload(ModelDetails details) : base(details) { @@ -173,36 +178,14 @@ public void DeleteFailedModel() /// A task representing the asynchronous operation. public async Task KeepModelDespiteVerificationFailure() { - var localFolderPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); - var filesToDownload = await GetFilesToDownloadAsync(); - long modelSize = filesToDownload.Sum(f => f.Size); - - var cachedModel = new CachedModel(Details, ModelUrl.IsFile ? $"{localFolderPath}\\{filesToDownload.First().Name}" : localFolderPath, ModelUrl.IsFile, modelSize); - await App.ModelCache.CacheStore.AddModel(cachedModel); - DownloadStatus = DownloadStatus.Completed; - } - - private async Task> GetFilesToDownloadAsync() - { - var cancellationToken = CancellationTokenSource.Token; - List filesToDownload; - - if (Details.Url.StartsWith("https://github.com", StringComparison.InvariantCulture)) + if (_pendingCachedModel == null) { - var ghUrl = new GitHubUrl(Details.Url); - filesToDownload = await ModelInformationHelper.GetDownloadFilesFromGitHub(ghUrl, cancellationToken); - } - else - { - var hfUrl = new HuggingFaceUrl(Details.Url); - using var socketsHttpHandler = new SocketsHttpHandler - { - MaxConnectionsPerServer = 4 - }; - filesToDownload = await ModelInformationHelper.GetDownloadFilesFromHuggingFace(hfUrl, socketsHttpHandler, cancellationToken); + throw new InvalidOperationException("Cannot keep model: no pending cached model from verification failure."); } - return ModelInformationHelper.FilterFiles(filesToDownload, Details.FileFilters); + await App.ModelCache.CacheStore.AddModel(_pendingCachedModel); + _pendingCachedModel = null; + DownloadStatus = DownloadStatus.Completed; } private async Task DownloadModel(string cacheDir, IProgress? progress = null) @@ -324,6 +307,11 @@ private async Task> GetFilesToDownloadAsync() { var failedFileNames = string.Join(", ", FailedVerifications.Select(f => f.FileName)); VerificationFailureMessage = $"Integrity verification failed for: {failedFileNames}"; + + // Cache the model info for potential KeepModelDespiteVerificationFailure call + var failedModelDirectory = url.GetLocalPath(cacheDir); + _pendingCachedModel = new CachedModel(Details, url.IsFile ? $"{failedModelDirectory}\\{filesToDownload.First().Name}" : failedModelDirectory, url.IsFile, modelSize); + DownloadStatus = DownloadStatus.VerificationFailed; return null; } From ebeb7687bdef95db84156f1867d5d644f4817a21 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Fri, 12 Dec 2025 16:07:01 +0800 Subject: [PATCH 5/8] Clean up VerificationFailedModel --- AIDevGallery/MainWindow.xaml.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/AIDevGallery/MainWindow.xaml.cs b/AIDevGallery/MainWindow.xaml.cs index d913160b..3494633d 100644 --- a/AIDevGallery/MainWindow.xaml.cs +++ b/AIDevGallery/MainWindow.xaml.cs @@ -46,6 +46,8 @@ public MainWindow(object? obj = null) Closed += async (sender, args) => { + CleanupVerificationFailedModels(); + if (SampleContainer.AnySamplesLoading()) { this.Hide(); @@ -491,4 +493,16 @@ private void UpdateResources() ModelPickerDefinition.Definitions["openai"].Icon = $"ms-appx:///Assets/ModelIcons/OpenAI{AppUtils.GetThemeAssetSuffix()}.png"; }); } + + private static void CleanupVerificationFailedModels() + { + foreach (var download in App.ModelDownloadQueue.GetDownloads()) + { + if (download.DownloadStatus == DownloadStatus.VerificationFailed && + download is OnnxModelDownload onnxDownload) + { + onnxDownload.DeleteFailedModel(); + } + } + } } \ No newline at end of file From be223aa086218661f5d0f8d24cd73b383771555f Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Fri, 12 Dec 2025 16:20:39 +0800 Subject: [PATCH 6/8] Address comments to use Size --- AIDevGallery.Utils/ModelInformationHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AIDevGallery.Utils/ModelInformationHelper.cs b/AIDevGallery.Utils/ModelInformationHelper.cs index be6ebe46..d5e69e59 100644 --- a/AIDevGallery.Utils/ModelInformationHelper.cs +++ b/AIDevGallery.Utils/ModelInformationHelper.cs @@ -228,7 +228,7 @@ public static async Task> GetDownloadFilesFromHuggingFace return new ModelFileDetails() { DownloadUrl = $"https://huggingface.co/{hfUrl.Organization}/{hfUrl.Repo}/resolve/{hfUrl.Ref}/{f.Path}", - Size = f.Lfs?.Size ?? f.Size, + Size = f.Size, Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(), Path = f.Path, Sha256 = sha256 From 41ef1ef4d5d5500be1b0d5e557ea7cf294579200 Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Fri, 12 Dec 2025 16:53:57 +0800 Subject: [PATCH 7/8] Add more validation files --- AIDevGallery.Utils/ModelFileDetails.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AIDevGallery.Utils/ModelFileDetails.cs b/AIDevGallery.Utils/ModelFileDetails.cs index e9ad7419..1cce49bb 100644 --- a/AIDevGallery.Utils/ModelFileDetails.cs +++ b/AIDevGallery.Utils/ModelFileDetails.cs @@ -38,10 +38,11 @@ public class ModelFileDetails public string? Sha256 { get; init; } /// - /// Gets a value indicating whether this file should be verified for integrity (main model files like .onnx). + /// Gets a value indicating whether this file should be verified for integrity. /// public bool ShouldVerifyIntegrity => Name != null && (Name.EndsWith(".onnx", StringComparison.OrdinalIgnoreCase) || + Name.EndsWith(".onnx.data", StringComparison.OrdinalIgnoreCase) || Name.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase) || Name.EndsWith(".safetensors", StringComparison.OrdinalIgnoreCase)); From 753339ea931ecd2644b44300bc58cc056b9e126c Mon Sep 17 00:00:00 2001 From: Hao Liu Date: Fri, 12 Dec 2025 17:41:47 +0800 Subject: [PATCH 8/8] Do not keep the source model file when verification failed --- .../Controls/DownloadProgressList.xaml | 6 +- .../Controls/DownloadProgressList.xaml.cs | 57 ++---------- AIDevGallery/MainWindow.xaml.cs | 14 --- .../ModelIntegrityVerificationFailedEvent.cs | 16 ++-- AIDevGallery/Utils/ModelDownload.cs | 89 ++++++++----------- AIDevGallery/ViewModels/DownloadableModel.cs | 32 +------ 6 files changed, 54 insertions(+), 160 deletions(-) diff --git a/AIDevGallery/Controls/DownloadProgressList.xaml b/AIDevGallery/Controls/DownloadProgressList.xaml index b507a0a8..38a238d3 100644 --- a/AIDevGallery/Controls/DownloadProgressList.xaml +++ b/AIDevGallery/Controls/DownloadProgressList.xaml @@ -176,16 +176,16 @@ Height="32" Padding="0" VerticalAlignment="Center" - AutomationProperties.Name="Verification failed - click to resolve" + AutomationProperties.Name="Verification failed - click to retry" Click="VerificationFailedClicked" Style="{StaticResource SubtleButtonStyle}" Tag="{x:Bind}" - ToolTipService.ToolTip="Verification failed - click to resolve" + ToolTipService.ToolTip="Verification failed - click to retry" Visibility="{x:Bind vm:DownloadableModel.VisibleWhenVerificationFailed(Status), Mode=OneWay}"> + Glyph="" /> diff --git a/AIDevGallery/Controls/DownloadProgressList.xaml.cs b/AIDevGallery/Controls/DownloadProgressList.xaml.cs index e86f757c..0baf849b 100644 --- a/AIDevGallery/Controls/DownloadProgressList.xaml.cs +++ b/AIDevGallery/Controls/DownloadProgressList.xaml.cs @@ -5,7 +5,6 @@ using AIDevGallery.ViewModels; using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Controls; -using System; using System.Collections.ObjectModel; using System.Linq; @@ -81,66 +80,20 @@ private void ClearHistory_Click(object sender, RoutedEventArgs e) { foreach (DownloadableModel model in downloadProgresses.ToList()) { - if (model.Status is DownloadStatus.Completed or DownloadStatus.Canceled) + if (model.Status is DownloadStatus.Completed or DownloadStatus.Canceled or DownloadStatus.VerificationFailed) { downloadProgresses.Remove(model); } } } - private async void VerificationFailedClicked(object sender, RoutedEventArgs e) + private void VerificationFailedClicked(object sender, RoutedEventArgs e) { + // Retry download when verification failed if (sender is Button button && button.Tag is DownloadableModel downloadableModel) { - var dialog = new ContentDialog - { - Title = "Integrity verification failed", - Content = new StackPanel - { - Spacing = 12, - Children = - { - new TextBlock - { - Text = "The downloaded model file(s) did not match the expected hash. This could indicate the file was corrupted during download or has been tampered with.", - TextWrapping = TextWrapping.Wrap - }, - new TextBlock - { - Text = downloadableModel.VerificationFailureMessage ?? "Unknown verification error", - TextWrapping = TextWrapping.Wrap, - Foreground = (Microsoft.UI.Xaml.Media.Brush)Application.Current.Resources["SystemFillColorCautionBrush"], - FontSize = 12 - }, - new TextBlock - { - Text = "Would you like to keep the model anyway or delete it?", - TextWrapping = TextWrapping.Wrap - } - } - }, - PrimaryButtonText = "Delete model", - SecondaryButtonText = "Keep anyway", - CloseButtonText = "Cancel", - DefaultButton = ContentDialogButton.Primary, - XamlRoot = this.XamlRoot - }; - - var result = await dialog.ShowAsync(); - - if (result == ContentDialogResult.Primary) - { - // Delete the model - downloadableModel.DeleteVerificationFailedModel(); - downloadProgresses.Remove(downloadableModel); - } - else if (result == ContentDialogResult.Secondary) - { - // Keep the model despite verification failure - downloadableModel.KeepVerificationFailedModel(); - } - - // If Cancel, do nothing - leave the item in the list + downloadProgresses.Remove(downloadableModel); + App.ModelDownloadQueue.AddModel(downloadableModel.ModelDetails); } } } \ No newline at end of file diff --git a/AIDevGallery/MainWindow.xaml.cs b/AIDevGallery/MainWindow.xaml.cs index 3494633d..d913160b 100644 --- a/AIDevGallery/MainWindow.xaml.cs +++ b/AIDevGallery/MainWindow.xaml.cs @@ -46,8 +46,6 @@ public MainWindow(object? obj = null) Closed += async (sender, args) => { - CleanupVerificationFailedModels(); - if (SampleContainer.AnySamplesLoading()) { this.Hide(); @@ -493,16 +491,4 @@ private void UpdateResources() ModelPickerDefinition.Definitions["openai"].Icon = $"ms-appx:///Assets/ModelIcons/OpenAI{AppUtils.GetThemeAssetSuffix()}.png"; }); } - - private static void CleanupVerificationFailedModels() - { - foreach (var download in App.ModelDownloadQueue.GetDownloads()) - { - if (download.DownloadStatus == DownloadStatus.VerificationFailed && - download is OnnxModelDownload onnxDownload) - { - onnxDownload.DeleteFailedModel(); - } - } - } } \ No newline at end of file diff --git a/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs index b04f6b21..c9cbfb1a 100644 --- a/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs +++ b/AIDevGallery/Telemetry/Events/ModelIntegrityVerificationFailedEvent.cs @@ -11,19 +11,21 @@ namespace AIDevGallery.Telemetry.Events; [EventData] internal class ModelIntegrityVerificationFailedEvent : EventBase { - internal ModelIntegrityVerificationFailedEvent(string modelUrl, string fileName, string expectedHash, string actualHash) + internal ModelIntegrityVerificationFailedEvent(string modelUrl, string fileName, string verificationType, string expectedValue, string actualValue) { ModelUrl = modelUrl; FileName = fileName; - ExpectedHash = expectedHash; - ActualHash = actualHash; + VerificationType = verificationType; + ExpectedValue = expectedValue; + ActualValue = actualValue; EventTime = DateTime.UtcNow; } public string ModelUrl { get; private set; } public string FileName { get; private set; } - public string ExpectedHash { get; private set; } - public string ActualHash { get; private set; } + public string VerificationType { get; private set; } + public string ExpectedValue { get; private set; } + public string ActualValue { get; private set; } public DateTime EventTime { get; private set; } public override PartA_PrivTags PartA_PrivTags => PrivTags.ProductAndServiceUsage; @@ -32,11 +34,11 @@ public override void ReplaceSensitiveStrings(Func replaceSensi { } - public static void Log(string modelUrl, string fileName, string expectedHash, string actualHash) + public static void Log(string modelUrl, string fileName, string verificationType, string expectedValue, string actualValue) { TelemetryFactory.Get().LogError( "ModelIntegrityVerificationFailed_Event", LogLevel.Info, - new ModelIntegrityVerificationFailedEvent(modelUrl, fileName, expectedHash, actualHash)); + new ModelIntegrityVerificationFailedEvent(modelUrl, fileName, verificationType, expectedValue, actualValue)); } } \ No newline at end of file diff --git a/AIDevGallery/Utils/ModelDownload.cs b/AIDevGallery/Utils/ModelDownload.cs index f47cd453..201fa8c1 100644 --- a/AIDevGallery/Utils/ModelDownload.cs +++ b/AIDevGallery/Utils/ModelDownload.cs @@ -6,6 +6,7 @@ using AIDevGallery.Telemetry.Events; using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Net.Http; @@ -90,16 +91,6 @@ internal class OnnxModelDownload : ModelDownload { public ModelUrl ModelUrl { get; set; } - /// - /// Gets the list of files that failed integrity verification. - /// - public List<(string FileName, string ExpectedHash, string ActualHash)> FailedVerifications { get; } = []; - - /// - /// Cached model info from the download process - /// - private CachedModel? _pendingCachedModel; - public OnnxModelDownload(ModelDetails details) : base(details) { @@ -160,34 +151,6 @@ public override void CancelDownload() DownloadStatus = DownloadStatus.Canceled; } - /// - /// Deletes the downloaded model files after user chooses not to keep a verification-failed model. - /// - public void DeleteFailedModel() - { - var localPath = ModelUrl.GetLocalPath(App.AppData.ModelCachePath); - if (Directory.Exists(localPath)) - { - Directory.Delete(localPath, true); - } - } - - /// - /// Keeps the downloaded model files despite verification failure (user's choice). - /// - /// A task representing the asynchronous operation. - public async Task KeepModelDespiteVerificationFailure() - { - if (_pendingCachedModel == null) - { - throw new InvalidOperationException("Cannot keep model: no pending cached model from verification failure."); - } - - await App.ModelCache.CacheStore.AddModel(_pendingCachedModel); - _pendingCachedModel = null; - DownloadStatus = DownloadStatus.Completed; - } - private async Task DownloadModel(string cacheDir, IProgress? progress = null) { ModelUrl url; @@ -268,7 +231,23 @@ public async Task KeepModelDespiteVerificationFailure() var fileInfo = new FileInfo(filePath); if (fileInfo.Length != downloadableFile.Size) { - throw new IOException($"File size mismatch for {downloadableFile.Name}: expected {downloadableFile.Size}, got {fileInfo.Length}"); + // Size mismatch - log telemetry + ModelIntegrityVerificationFailedEvent.Log( + Details.Url, + downloadableFile.Name ?? filePath, + verificationType: "Size", + expectedValue: downloadableFile.Size.ToString(CultureInfo.InvariantCulture), + actualValue: fileInfo.Length.ToString(CultureInfo.InvariantCulture)); + VerificationFailureMessage = $"Size verification failed for: {downloadableFile.Name}"; + DownloadStatus = DownloadStatus.VerificationFailed; + + var localPath = url.GetLocalPath(cacheDir); + if (Directory.Exists(localPath)) + { + Directory.Delete(localPath, true); + } + + return null; } // Add to verification list if it's a main model file with hash @@ -298,22 +277,24 @@ public async Task KeepModelDespiteVerificationFailure() if (!verified) { - FailedVerifications.Add((fileDetails.Name ?? filePath, expectedHash, actualHash)); - ModelIntegrityVerificationFailedEvent.Log(Details.Url, fileDetails.Name ?? filePath, expectedHash, actualHash); - } - } - - if (FailedVerifications.Count > 0) - { - var failedFileNames = string.Join(", ", FailedVerifications.Select(f => f.FileName)); - VerificationFailureMessage = $"Integrity verification failed for: {failedFileNames}"; - - // Cache the model info for potential KeepModelDespiteVerificationFailure call - var failedModelDirectory = url.GetLocalPath(cacheDir); - _pendingCachedModel = new CachedModel(Details, url.IsFile ? $"{failedModelDirectory}\\{filesToDownload.First().Name}" : failedModelDirectory, url.IsFile, modelSize); + ModelIntegrityVerificationFailedEvent.Log( + Details.Url, + fileDetails.Name ?? filePath, + verificationType: "SHA256", + expectedValue: expectedHash, + actualValue: actualHash); + VerificationFailureMessage = $"Integrity verification failed for: {fileDetails.Name ?? filePath}"; + DownloadStatus = DownloadStatus.VerificationFailed; + + // Delete the downloaded files + var localPath = url.GetLocalPath(cacheDir); + if (Directory.Exists(localPath)) + { + Directory.Delete(localPath, true); + } - DownloadStatus = DownloadStatus.VerificationFailed; - return null; + return null; + } } } diff --git a/AIDevGallery/ViewModels/DownloadableModel.cs b/AIDevGallery/ViewModels/DownloadableModel.cs index c1eccd36..aa946bed 100644 --- a/AIDevGallery/ViewModels/DownloadableModel.cs +++ b/AIDevGallery/ViewModels/DownloadableModel.cs @@ -91,36 +91,6 @@ public void CancelDownload() ModelDownload?.CancelDownload(); } - /// - /// Deletes the model files when user rejects a verification-failed model. - /// - public void DeleteVerificationFailedModel() - { - if (ModelDownload is OnnxModelDownload onnxDownload) - { - onnxDownload.DeleteFailedModel(); - } - - Status = DownloadStatus.Canceled; - ModelDownload = null; - VerificationFailureMessage = null; - } - - /// - /// Keeps the model despite verification failure (user's choice). - /// - public async void KeepVerificationFailedModel() - { - if (ModelDownload is OnnxModelDownload onnxDownload) - { - await onnxDownload.KeepModelDespiteVerificationFailure(); - } - - Status = DownloadStatus.Completed; - ModelDownload = null; - VerificationFailureMessage = null; - } - private void ModelDownload_StateChanged(object? sender, ModelDownloadEventArgs e) { if (!_progressTimer.IsEnabled) @@ -151,6 +121,8 @@ private void ModelDownload_StateChanged(object? sender, ModelDownloadEventArgs e { Status = DownloadStatus.VerificationFailed; VerificationFailureMessage = e.VerificationFailureMessage; + ModelDownload = null; + Progress = 0; } if (e.Status == DownloadStatus.Verifying)