Surface F# compiler timing data in regression test pipeline via binlog extraction

azure-pipelines-PR.yml

@@ -900,7 +900,7 @@ stages:
         testMatrix:
         - repo: marklam/SlowBuildRepro
           commit: bbe2dec4d0379b5d7d0480997858c30d442fbb42
-          buildScript: dotnet build
+          buildScript: dotnet build -bl
           displayName: UMX_Slow_Repro        
         - repo: fsprojects/FSharpPlus
           commit: f614035b75922aba41ed6a36c2fc986a2171d2b8
@@ -913,16 +913,16 @@ stages:
           useVmImage: $(UbuntuMachineQueueName)
         - repo: fsprojects/FSharpPlus
           commit: 2648efe
-          buildScript: build.cmd
-          displayName: FsharpPlus_NET10
+          buildScript: dotnet build tests/FSharpPlus.Tests/FSharpPlus.Tests.fsproj -c Release -bl
+          displayName: FsharpPlus_NET10_Build_Lib_Tests
          # remove this before merging
         - repo: fsprojects/FSharpPlus
           commit: 2648efe
-          buildScript: dotnet msbuild build.proj -t:Build;Test
+          buildScript: dotnet msbuild build.proj -t:Build;Test -bl
           displayName: FsharpPlus_NET10_Test
         - repo: fsprojects/FSharpPlus
           commit: 2648efe
-          buildScript: dotnet msbuild build.proj -t:Build;AllDocs
+          buildScript: dotnet msbuild build.proj -t:Build;AllDocs -bl
           displayName: FsharpPlus_NET10_Docs
         - repo: fsprojects/FSharpPlus
           commit: 2648efe
@@ -931,21 +931,21 @@ stages:
           useVmImage: $(UbuntuMachineQueueName)
         - repo: TheAngryByrd/IcedTasks
           commit: 863bf91cdee93d8c4c875bb5d321dd92eb20d5a9
-          buildScript: dotnet build IcedTasks.sln
+          buildScript: dotnet build IcedTasks.sln -bl
           displayName: IcedTasks_Build
         - repo: TheAngryByrd/IcedTasks
           commit: 863bf91cdee93d8c4c875bb5d321dd92eb20d5a9
-          buildScript: dotnet test IcedTasks.sln
+          buildScript: dotnet test IcedTasks.sln -bl
           displayName: IcedTasks_Test
         - repo: demystifyfp/FsToolkit.ErrorHandling
           commit: 9cd957e335767df03e2fb0aa2f7b0fed782c5091
-          buildScript: dotnet build FsToolkit.ErrorHandling.sln
+          buildScript: dotnet build FsToolkit.ErrorHandling.sln -bl
           displayName: FsToolkit_ErrorHandling_Build
         - repo: demystifyfp/FsToolkit.ErrorHandling
           commit: 9cd957e335767df03e2fb0aa2f7b0fed782c5091
-          buildScript: dotnet test FsToolkit.ErrorHandling.sln
+          buildScript: dotnet test FsToolkit.ErrorHandling.sln -bl
           displayName: FsToolkit_ErrorHandling_Test
         - repo: opentk/opentk
           commit: 60c20cca65a7df6e8335e8d6060d91b30909fbea
-          buildScript: dotnet build tests/OpenTK.Tests/OpenTK.Tests.fsproj -c Release ;; dotnet build tests/OpenTK.Tests.Integration/OpenTK.Tests.Integration.fsproj -c Release
+          buildScript: dotnet build tests/OpenTK.Tests/OpenTK.Tests.fsproj -c Release -bl:build_1.binlog ;; dotnet build tests/OpenTK.Tests.Integration/OpenTK.Tests.Integration.fsproj -c Release -bl:build_2.binlog
           displayName: OpenTK_FSharp_Build

eng/scripts/ExtractTimingsFromBinlog.fsx

@@ -0,0 +1,65 @@
+/// Script to extract F# compiler timing data from MSBuild binary logs
+/// Usage: dotnet fsi ExtractTimingsFromBinlog.fsx <path-to-binlog-file>
+
+#r "nuget: MSBuild.StructuredLogger"
+
+open System
+open Microsoft.Build.Logging.StructuredLogger
+
+let binlogPath = 
+    let args = Environment.GetCommandLineArgs()
+    // When running with dotnet fsi, args are: [0]=dotnet; [1]=fsi.dll; [2]=script.fsx; [3...]=args
+    let scriptArgs = args |> Array.skipWhile (fun a -> not (a.EndsWith(".fsx"))) |> Array.skip 1
+    if scriptArgs.Length > 0 then
+        scriptArgs.[0]
+    else
+        failwith "Usage: dotnet fsi ExtractTimingsFromBinlog.fsx <path-to-binlog-file>"
+
+if not (IO.File.Exists(binlogPath)) then
+    printfn "Binlog file not found: %s" binlogPath
+    exit 1
+
+let rec findProject (node: TreeNode) =
+    match node with
+    | null -> None
+    | :? Project as p -> Some p.Name
+    | _ -> findProject node.Parent
+
+let build = BinaryLog.ReadBuild(binlogPath)
+
+let isTimingLine (s: string) =
+    s.StartsWith("|") || (s.Length > 10 && s.StartsWith("-") && s.TrimEnd() |> Seq.forall (fun c -> c = '-'))
+
+let mutable foundFscTasks = false
+let mutable foundTimingData = false
+
+build.VisitAllChildren<Task>(fun task ->
+    if task.Name = "Fsc" then
+        foundFscTasks <- true
+
+        let projectName = 
+            match findProject task with
+            | Some name -> name
+            | None -> "Unknown Project"
+
+        let timingMessages = 
+            task.Children
+            |> Seq.choose (function
+                | :? Message as m when isTimingLine m.Text -> Some m.Text
+                | _ -> None)
+            |> Seq.toList
+
+        if timingMessages.Length > 0 then
+            foundTimingData <- true
+            printfn "=== %s ===" projectName
+            for msg in timingMessages do
+                printfn "  %s" msg
+            printfn ""
+)
+
+if not foundFscTasks then
+    printfn "No Fsc tasks found in binlog."
+elif not foundTimingData then
+    printfn "Fsc tasks found but no timing data present. Was --times flag set?"
+
+exit 0

eng/scripts/PrepareRepoForRegressionTesting.fsx

@@ -1,16 +1,12 @@
 /// Script to inject UseLocalCompiler.Directory.Build.props import into a third-party repository's Directory.Build.props
 /// Usage: dotnet fsi PrepareRepoForRegressionTesting.fsx <path-to-UseLocalCompiler.Directory.Build.props>
-/// 
-/// This script is designed to be run in the root of a third-party repository
-/// It modifies the Directory.Build.props to import the UseLocalCompiler.Directory.Build.props
 
 open System
 open System.IO
 open System.Xml
 
 let propsFilePath = "Directory.Build.props"
 
-// Get the path to UseLocalCompiler.Directory.Build.props from command line args
 let useLocalCompilerPropsPath = 
     let args = Environment.GetCommandLineArgs()
     // When running with dotnet fsi, args are: [0]=dotnet; [1]=fsi.dll; [2]=script.fsx; [3...]=args
@@ -24,60 +20,87 @@ printfn "PrepareRepoForRegressionTesting.fsx"
 printfn "==================================="
 printfn "UseLocalCompiler props path: %s" useLocalCompilerPropsPath
 
-// Verify the UseLocalCompiler props file exists
 if not (File.Exists(useLocalCompilerPropsPath)) then
     failwithf "UseLocalCompiler.Directory.Build.props not found at: %s" useLocalCompilerPropsPath
 
 printfn "✓ UseLocalCompiler.Directory.Build.props found"
 
-// Convert to absolute path and normalize slashes for MSBuild
 let absolutePropsPath = 
     Path.GetFullPath(useLocalCompilerPropsPath).Replace("\\", "/")
 printfn "Absolute path: %s" absolutePropsPath
 
 if File.Exists(propsFilePath) then
     printfn "Directory.Build.props exists, modifying it..."
 
-    // Load the existing XML
     let doc = XmlDocument()
     doc.PreserveWhitespace <- true
     doc.Load(propsFilePath)
 
-    // Find the Project element
     let projectElement = doc.SelectSingleNode("/Project")
     if isNull projectElement then
         failwith "Could not find Project element in Directory.Build.props"
 
-    // Check if our import already exists
-    let xpath = sprintf "//Import[contains(@Project, 'UseLocalCompiler.Directory.Build.props')]"
+    let xpath = "//Import[contains(@Project, 'UseLocalCompiler.Directory.Build.props')]"
     let existingImport = doc.SelectSingleNode(xpath)
 
     if isNull existingImport then
-        // Create Import element
         let importElement = doc.CreateElement("Import")
         importElement.SetAttribute("Project", absolutePropsPath)
 
-        // Insert as first child of Project element
         if projectElement.HasChildNodes then
             projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore
         else
             projectElement.AppendChild(importElement) |> ignore
 
-        // Add newline for formatting
         let newline = doc.CreateTextNode("\n  ")
         projectElement.InsertAfter(newline, importElement) |> ignore
 
         doc.Save(propsFilePath)
         printfn "✓ Added UseLocalCompiler import to Directory.Build.props"
     else
         printfn "✓ UseLocalCompiler import already exists"
+
+    let otherFlagsWithTimes = doc.SelectSingleNode("//OtherFlags[contains(text(), '--times')]")
+
+    if isNull otherFlagsWithTimes then
+        let propertyGroup = doc.CreateElement("PropertyGroup")
+        let otherFlags = doc.CreateElement("OtherFlags")
+        otherFlags.InnerText <- "$(OtherFlags) --nowarn:75 --times"
+        propertyGroup.AppendChild(otherFlags) |> ignore
+
+        let importNode = doc.SelectSingleNode(xpath)
+
+        // PreserveWhitespace=true causes XML DOM to keep text nodes (newlines/indentation) between elements;
+        // skip past the whitespace text node after the import to position the PropertyGroup correctly
+        let nodeAfterImport = 
+            if not (isNull importNode) && not (isNull importNode.NextSibling) && importNode.NextSibling.NodeType = XmlNodeType.Text then
+                importNode.NextSibling
+            else
+                null
+
+        if not (isNull nodeAfterImport) then
+            projectElement.InsertAfter(propertyGroup, nodeAfterImport) |> ignore
+        else
+            projectElement.InsertAfter(propertyGroup, importNode) |> ignore
+
+        let newlineAfter = doc.CreateTextNode("\n  ")
+        projectElement.InsertAfter(newlineAfter, propertyGroup) |> ignore
+
+        doc.Save(propsFilePath)
+        printfn "✓ Added --times flag to OtherFlags"
+    else
+        if not (otherFlagsWithTimes.InnerText.Contains("--nowarn:75")) then
+            otherFlagsWithTimes.InnerText <- otherFlagsWithTimes.InnerText.Replace("--times", "--nowarn:75 --times")
+            doc.Save(propsFilePath)
+            printfn "✓ Added --nowarn:75 to existing OtherFlags"
+        else
+            printfn "✓ --times and --nowarn:75 already exist in OtherFlags"
 else
     printfn "Directory.Build.props does not exist, creating it..."
-    let newContent = sprintf "<Project>\n  <Import Project=\"%s\" />\n</Project>\n" absolutePropsPath
+    let newContent = sprintf "<Project>\n  <Import Project=\"%s\" />\n  <PropertyGroup>\n    <OtherFlags>$(OtherFlags) --nowarn:75 --times</OtherFlags>\n  </PropertyGroup>\n</Project>\n" absolutePropsPath
     File.WriteAllText(propsFilePath, newContent)
-    printfn "✓ Created Directory.Build.props with UseLocalCompiler import"
+    printfn "✓ Created Directory.Build.props with UseLocalCompiler import and --times flag"
 
-// Print the final content
 printfn ""
 printfn "Final Directory.Build.props content:"
 printfn "-----------------------------------"

eng/templates/regression-test-jobs.yml

@@ -206,7 +206,9 @@ jobs:
                 Process-BuildOutput $_
               }
             } else {
-              bash -c "$cmd" 2>&1 | Tee-Object -FilePath $fullLogPath -Append | ForEach-Object {
+              # Escape semicolons for bash -c to prevent them being treated as command separators
+              $escapedCmd = $cmd -replace ';', '\;'
+              bash -c "$escapedCmd" 2>&1 | Tee-Object -FilePath $fullLogPath -Append | ForEach-Object {
                 Process-BuildOutput $_
               }
             }
@@ -273,6 +275,38 @@ jobs:
       condition: always()
       continueOnError: true
 
+    - pwsh: |
+        $binlogDir = "$(Pipeline.Workspace)/BinaryLogs"
+        $script = "$(Build.SourcesDirectory)/eng/scripts/ExtractTimingsFromBinlog.fsx"
+
+        if (-not (Test-Path $binlogDir)) {
+          Write-Host "No binary logs directory found, skipping timing extraction"
+          exit 0
+        }
+
+        $binlogs = Get-ChildItem -Path $binlogDir -Filter "*.binlog" -ErrorAction SilentlyContinue
+        if ($binlogs.Count -eq 0) {
+          Write-Host "No binlog files found, skipping timing extraction"
+          exit 0
+        }
+
+        # Copy script to BinaryLogs dir so dotnet fsi doesn't inherit fsharp repo's global.json
+        # (which pins an SDK version not installed in the regression test job)
+        Copy-Item $script $binlogDir/
+        $localScript = Join-Path $binlogDir (Split-Path $script -Leaf)
+
+        Write-Host "##[group]F# Compiler Timing Summary for ${{ item.displayName }}"
+        foreach ($bl in $binlogs) {
+          Write-Host ""
+          Write-Host "--- Timings from $($bl.Name) ---"
+          dotnet fsi $localScript "$($bl.FullName)"
+        }
+        Write-Host "##[endgroup]"
+      displayName: Extract compiler timing from binlogs for ${{ item.displayName }}
+      condition: always()
+      continueOnError: true
+      workingDirectory: '$(Pipeline.Workspace)/BinaryLogs'
+
     - task: PublishPipelineArtifact@1
       displayName:  Publish ${{ item.displayName }} Binary Logs
       inputs: 
@@ -291,7 +325,7 @@ jobs:
         Write-Host "Commit: ${{ item.commit }}"
         Write-Host "Build Script: ${{ item.buildScript }}"
 
-        if ($env:AGENT_JOBSTATUS -eq "Succeeded") {
+        if ($env:AGENT_JOBSTATUS -eq "Succeeded" -or $env:AGENT_JOBSTATUS -eq "SucceededWithIssues") {
           Write-Host "Status: SUCCESS"
           Write-Host "The ${{ item.displayName }} library builds successfully with the new F# compiler"
         } else {