Skip fourslash at runtime instead of statically (#2344)

Author: jakebailey

internal/fourslash/_scripts/convertFourslash.mts

@@ -9,18 +9,12 @@ const stradaFourslashPath = path.resolve(import.meta.dirname, "../", "../", "../
 
 let inputFileSet: Set<string> | undefined;
 
-const failingTestsPath = path.join(import.meta.dirname, "failingTests.txt");
 const manualTestsPath = path.join(import.meta.dirname, "manualTests.txt");
 
 const outputDir = path.join(import.meta.dirname, "../", "tests", "gen");
 
 const unparsedFiles: string[] = [];
 
-function getFailingTests(): Set<string> {
-    const failingTestsList = fs.readFileSync(failingTestsPath, "utf-8").split("\n").map(line => line.trim().substring(4)).filter(line => line.length > 0);
-    return new Set(failingTestsList);
-}
-
 function getManualTests(): Set<string> {
     if (!fs.existsSync(manualTestsPath)) {
         return new Set();
@@ -43,13 +37,13 @@ export function main() {
     fs.rmSync(outputDir, { recursive: true, force: true });
     fs.mkdirSync(outputDir, { recursive: true });
 
-    parseTypeScriptFiles(getFailingTests(), getManualTests(), stradaFourslashPath);
+    parseTypeScriptFiles(getManualTests(), stradaFourslashPath);
     console.log(unparsedFiles.join("\n"));
     const gofmt = which.sync("go");
     cp.execFileSync(gofmt, ["tool", "mvdan.cc/gofumpt", "-lang=go1.25", "-w", outputDir]);
 }
 
-function parseTypeScriptFiles(failingTests: Set<string>, manualTests: Set<string>, folder: string): void {
+function parseTypeScriptFiles(manualTests: Set<string>, folder: string): void {
     const files = fs.readdirSync(folder);
 
     files.forEach(file => {
@@ -60,14 +54,14 @@ function parseTypeScriptFiles(failingTests: Set<string>, manualTests: Set<string
         }
 
         if (stat.isDirectory()) {
-            parseTypeScriptFiles(failingTests, manualTests, filePath);
+            parseTypeScriptFiles(manualTests, filePath);
         }
         else if (file.endsWith(".ts") && !manualTests.has(file.slice(0, -3))) {
             const content = fs.readFileSync(filePath, "utf-8");
             const test = parseFileContent(file, content);
             const isServer = filePath.split(path.sep).includes("server");
             if (test) {
-                const testContent = generateGoTest(failingTests, test, isServer);
+                const testContent = generateGoTest(test, isServer);
                 const testPath = path.join(outputDir, `${test.name}_test.go`);
                 fs.writeFileSync(testPath, testContent, "utf-8");
             }
@@ -3273,7 +3267,7 @@ interface GoTest {
     commands: Cmd[];
 }
 
-function generateGoTest(failingTests: Set<string>, test: GoTest, isServer: boolean): string {
+function generateGoTest(test: GoTest, isServer: boolean): string {
     const testName = (test.name[0].toUpperCase() + test.name.substring(1)).replaceAll("-", "_").replaceAll(/[^a-zA-Z0-9_]/g, "");
     const content = test.content;
     const commands = test.commands.map(cmd => generateCmd(cmd)).join("\n");
@@ -3305,8 +3299,8 @@ import (
 )
 
 func Test${testName}(t *testing.T) {
+    fourslash.SkipIfFailing(t)
     t.Parallel()
-    ${failingTests.has(testName) ? "t.Skip()" : ""}
     defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = ${content}
     f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content)

internal/fourslash/_scripts/updateFailing.mts

@@ -2,25 +2,24 @@ import * as cp from "child_process";
 import * as fs from "fs";
 import path from "path";
 import which from "which";
-import { main as convertFourslash } from "./convertFourslash.mts";
 
 const failingTestsPath = path.join(import.meta.dirname, "failingTests.txt");
 
 function main() {
-    const oldFailingTests = fs.readFileSync(failingTestsPath, "utf-8");
-    fs.writeFileSync(failingTestsPath, "", "utf-8");
-    convertFourslash();
     const go = which.sync("go");
     let testOutput: string;
     try {
-        testOutput = cp.execFileSync(go, ["test", "-v", "./internal/fourslash/tests/gen"], { encoding: "utf-8" });
+        // Run tests with TSGO_FOURSLASH_IGNORE_FAILING=1 to run all tests including those in failingTests.txt
+        testOutput = cp.execFileSync(go, ["test", "-v", "./internal/fourslash/tests/gen"], {
+            encoding: "utf-8",
+            env: { ...process.env, TSGO_FOURSLASH_IGNORE_FAILING: "1" },
+        });
     }
     catch (error) {
         testOutput = (error as { stdout: string; }).stdout as string;
     }
     const panicRegex = /^panic/m;
     if (panicRegex.test(testOutput)) {
-        fs.writeFileSync(failingTestsPath, oldFailingTests, "utf-8");
         throw new Error("Unrecovered panic detected in tests\n" + testOutput);
     }
     const failRegex = /--- FAIL: ([\S]+)/gm;
@@ -32,7 +31,6 @@ function main() {
     }
 
     fs.writeFileSync(failingTestsPath, failingTests.sort((a, b) => a.localeCompare(b, "en-US")).join("\n") + "\n", "utf-8");
-    convertFourslash();
 }
 
 main();

internal/fourslash/skip_if_failing.go

@@ -0,0 +1,53 @@
+package fourslash
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+)
+
+var failingTests = sync.OnceValue(func() map[string]struct{} {
+	failingTestsSet := make(map[string]struct{})
+
+	// Get the path to failingTests.txt relative to this source file
+	_, thisFile, _, ok := runtime.Caller(0)
+	if !ok {
+		return failingTestsSet
+	}
+
+	failingTestsPath := filepath.Join(filepath.Dir(thisFile), "_scripts", "failingTests.txt") //nolint:forbidigo
+
+	file, err := os.Open(failingTestsPath) //nolint:forbidigo
+	if err != nil {
+		return failingTestsSet
+	}
+	defer file.Close() //nolint:forbidigo
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line != "" {
+			failingTestsSet[line] = struct{}{}
+		}
+	}
+	return failingTestsSet
+})
+
+// SkipIfFailing checks if the current test is in the failingTests.txt file
+// and skips it unless the TSGO_FOURSLASH_IGNORE_FAILING environment variable is set.
+// This allows tests to be marked as failing without modifying the test files themselves.
+func SkipIfFailing(t *testing.T) {
+	t.Helper()
+
+	if os.Getenv("TSGO_FOURSLASH_IGNORE_FAILING") != "" { //nolint:forbidigo
+		return
+	}
+
+	if _, found := failingTests()[t.Name()]; found {
+		t.Skip("Test is in failingTests.txt")
+	}
+}

internal/fourslash/tests/gen/addDeclareToFunction_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddDeclareToFunction(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `/*1*/function parseInt(s/*2*/:string):number;`
 	f, done := fourslash.NewFourslash(t, nil /*capabilities*/, content)

internal/fourslash/tests/gen/addDeclareToModule_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddDeclareToModule(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `/**/module mAmbient {
     module m3 { }

internal/fourslash/tests/gen/addDuplicateSetter_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddDuplicateSetter(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `class C {
     set foo(value) { }

internal/fourslash/tests/gen/addFunctionAboveMultiLineLambdaExpression_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddFunctionAboveMultiLineLambdaExpression(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `/**/
 () =>

internal/fourslash/tests/gen/addFunctionInDuplicatedConstructorClassBody_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddFunctionInDuplicatedConstructorClassBody(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `class Foo {
     constructor() { }

internal/fourslash/tests/gen/addInterfaceMemberAboveClass_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddInterfaceMemberAboveClass(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `
 interface Intersection {

internal/fourslash/tests/gen/addInterfaceToNotSatisfyConstraint_test.go

@@ -8,8 +8,8 @@ import (
 )
 
 func TestAddInterfaceToNotSatisfyConstraint(t *testing.T) {
+	fourslash.SkipIfFailing(t)
 	t.Parallel()
-
 	defer testutil.RecoverAndFail(t, "Panic on fourslash test")
 	const content = `interface A {
 	a: number;