Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions AIDevGallery.Utils/GitHubModelFileDetails.cs
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,17 @@ public class GitHubModelFileDetails
/// </summary>
[JsonPropertyName("type")]
public string? Type { get; init; }

/// <summary>
/// Gets the encoded content of the file.
/// For LFS files, this contains the LFS pointer with SHA256.
/// </summary>
[JsonPropertyName("content")]
public string? Content { get; init; }

/// <summary>
/// Gets the encoding of the content (usually "base64").
/// </summary>
[JsonPropertyName("encoding")]
public string? Encoding { get; init; }
}
24 changes: 24 additions & 0 deletions AIDevGallery.Utils/HuggingFaceModelFileDetails.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,28 @@ public class HuggingFaceModelFileDetails
/// </summary>
[JsonPropertyName("path")]
public string? Path { get; init; }

/// <summary>
/// Gets the LFS (Large File Storage) information for the file.
/// </summary>
[JsonPropertyName("lfs")]
public HuggingFaceLfsInfo? Lfs { get; init; }
}

/// <summary>
/// LFS (Large File Storage) information for a Hugging Face file.
/// </summary>
public class HuggingFaceLfsInfo
{
/// <summary>
/// Gets the OID (SHA256 hash) of the file. Format: "sha256:abc123..."
/// </summary>
[JsonPropertyName("oid")]
public string? Oid { get; init; }

/// <summary>
/// Gets the size of the file in LFS.
/// </summary>
[JsonPropertyName("size")]
public long Size { get; init; }
}
23 changes: 23 additions & 0 deletions AIDevGallery.Utils/ModelFileDetails.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using System;

namespace AIDevGallery.Utils;

/// <summary>
Expand All @@ -27,4 +29,25 @@ public class ModelFileDetails
/// Gets the relative path to the file
/// </summary>
public string? Path { get; init; }

/// <summary>
/// Gets the expected SHA256 hash of the file.
/// For Hugging Face: from LFS oid field.
/// For GitHub: from LFS pointer file.
/// </summary>
public string? Sha256 { get; init; }

/// <summary>
/// Gets a value indicating whether this file should be verified for integrity.
/// </summary>
public bool ShouldVerifyIntegrity => Name != null &&
(Name.EndsWith(".onnx", StringComparison.OrdinalIgnoreCase) ||
Name.EndsWith(".onnx.data", StringComparison.OrdinalIgnoreCase) ||
Name.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase) ||
Name.EndsWith(".safetensors", StringComparison.OrdinalIgnoreCase));

/// <summary>
/// Gets a value indicating whether this file has a hash available for verification.
/// </summary>
public bool HasVerificationHash => !string.IsNullOrEmpty(Sha256);
}
67 changes: 61 additions & 6 deletions AIDevGallery.Utils/ModelInformationHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,55 @@ public static async Task<List<ModelFileDetails>> GetDownloadFilesFromGitHub(GitH
}

return files.Select(f =>
new ModelFileDetails()
{
string? sha256 = null;

if (f.Content != null && f.Encoding == "base64")
{
try
{
var decodedContent = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(f.Content));
sha256 = ParseLfsPointerSha256(decodedContent);
}
catch (FormatException)
{
Debug.WriteLine($"Failed to decode base64 content for {f.Path}");
}
}

return new ModelFileDetails()
{
DownloadUrl = f.DownloadUrl,
Size = f.Size,
Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(),
Path = f.Path
}).ToList();
Path = f.Path,
Sha256 = sha256
};
}).ToList();
}

/// <summary>
/// Parses a Git LFS pointer file content to extract the SHA256 hash.
/// </summary>
/// <param name="lfsPointerContent">The content of the LFS pointer file.</param>
/// <returns>The SHA256 hash if found, otherwise null.</returns>
private static string? ParseLfsPointerSha256(string lfsPointerContent)
{
if (string.IsNullOrEmpty(lfsPointerContent))
{
return null;
}

var lines = lfsPointerContent.Split(['\n', '\r'], StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
if (line.StartsWith("oid sha256:", StringComparison.OrdinalIgnoreCase))
{
return line.Substring("oid sha256:".Length).Trim();
}
}

return null;
}

/// <summary>
Expand Down Expand Up @@ -172,13 +214,26 @@ public static async Task<List<ModelFileDetails>> GetDownloadFilesFromHuggingFace
}

return hfFiles.Where(f => f.Type != "directory").Select(f =>
new ModelFileDetails()
{
string? sha256 = null;
if (f.Lfs?.Oid != null)
{
sha256 = f.Lfs.Oid;
if (sha256.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
sha256 = sha256.Substring(7);
}
}

return new ModelFileDetails()
{
DownloadUrl = $"https://huggingface.co/{hfUrl.Organization}/{hfUrl.Repo}/resolve/{hfUrl.Ref}/{f.Path}",
Size = f.Size,
Name = (f.Path ?? string.Empty).Split(["/"], StringSplitOptions.RemoveEmptyEntries).LastOrDefault(),
Path = f.Path
}).ToList();
Path = f.Path,
Sha256 = sha256
};
}).ToList();
}

/// <summary>
Expand Down
1 change: 1 addition & 0 deletions AIDevGallery.Utils/SourceGenerationContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ namespace AIDevGallery.Utils;
[JsonSourceGenerationOptions(WriteIndented = true, AllowTrailingCommas = true)]
[JsonSerializable(typeof(List<GitHubModelFileDetails>))]
[JsonSerializable(typeof(List<HuggingFaceModelFileDetails>))]
[JsonSerializable(typeof(HuggingFaceLfsInfo))]
internal partial class SourceGenerationContext : JsonSerializerContext
{
}
18 changes: 18 additions & 0 deletions AIDevGallery/Controls/DownloadProgressList.xaml
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,24 @@
Tag="{x:Bind}"
ToolTipService.ToolTip="Cancel"
Visibility="{x:Bind vm:DownloadableModel.VisibleWhenDownloading(Status), Mode=OneWay}" />
<Button
Grid.RowSpan="3"
Grid.Column="2"
Width="32"
Height="32"
Padding="0"
VerticalAlignment="Center"
AutomationProperties.Name="Verification failed - click to retry"
Click="VerificationFailedClicked"
Style="{StaticResource SubtleButtonStyle}"
Tag="{x:Bind}"
ToolTipService.ToolTip="Verification failed - click to retry"
Visibility="{x:Bind vm:DownloadableModel.VisibleWhenVerificationFailed(Status), Mode=OneWay}">
<FontIcon
FontSize="14"
Foreground="{ThemeResource SystemFillColorCautionBrush}"
Glyph="&#xE72C;" />
</Button>
</Grid>
</toolkit:SettingsCard>
</DataTemplate>
Expand Down
12 changes: 11 additions & 1 deletion AIDevGallery/Controls/DownloadProgressList.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,20 @@ private void ClearHistory_Click(object sender, RoutedEventArgs e)
{
foreach (DownloadableModel model in downloadProgresses.ToList())
{
if (model.Status is DownloadStatus.Completed or DownloadStatus.Canceled)
if (model.Status is DownloadStatus.Completed or DownloadStatus.Canceled or DownloadStatus.VerificationFailed)
{
downloadProgresses.Remove(model);
}
}
}

private void VerificationFailedClicked(object sender, RoutedEventArgs e)
{
// Retry download when verification failed
if (sender is Button button && button.Tag is DownloadableModel downloadableModel)
{
downloadProgresses.Remove(downloadableModel);
App.ModelDownloadQueue.AddModel(downloadableModel.ModelDetails);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Microsoft.Diagnostics.Telemetry;
using Microsoft.Diagnostics.Telemetry.Internal;
using System;
using System.Diagnostics.Tracing;

namespace AIDevGallery.Telemetry.Events;

[EventData]
internal class ModelIntegrityVerificationFailedEvent : EventBase
{
internal ModelIntegrityVerificationFailedEvent(string modelUrl, string fileName, string verificationType, string expectedValue, string actualValue)
{
ModelUrl = modelUrl;
FileName = fileName;
VerificationType = verificationType;
ExpectedValue = expectedValue;
ActualValue = actualValue;
EventTime = DateTime.UtcNow;
}

public string ModelUrl { get; private set; }
public string FileName { get; private set; }
public string VerificationType { get; private set; }
public string ExpectedValue { get; private set; }
public string ActualValue { get; private set; }
public DateTime EventTime { get; private set; }

public override PartA_PrivTags PartA_PrivTags => PrivTags.ProductAndServiceUsage;

public override void ReplaceSensitiveStrings(Func<string?, string?> replaceSensitiveStrings)
{
}

public static void Log(string modelUrl, string fileName, string verificationType, string expectedValue, string actualValue)
{
TelemetryFactory.Get<ITelemetry>().LogError(
"ModelIntegrityVerificationFailed_Event",
LogLevel.Info,
new ModelIntegrityVerificationFailedEvent(modelUrl, fileName, verificationType, expectedValue, actualValue));
}
}
Loading