diff --git a/datamodel/document_config.go b/datamodel/document_config.go index cf3207d0..29b7f2e7 100644 --- a/datamodel/document_config.go +++ b/datamodel/document_config.go @@ -81,6 +81,12 @@ type DocumentConfiguration struct { // FileFilter should be used to limit the scope of the rolodex. AllowFileReferences bool + // SkipExternalRefResolution will skip resolving external $ref references (those not starting with #). + // When enabled, external references will be left as-is during model building. Schema proxies will + // report IsReference()=true and GetReference() will return the ref string, but Schema() will return nil. + // This is useful for code generators that handle external refs via import mappings. + SkipExternalRefResolution bool + // AllowRemoteReferences will allow the index to lookup remote references. This is disabled by default. // // This behavior is now driven by the inclusion of a BaseURL. If a BaseURL is set, then the diff --git a/datamodel/low/base/schema.go b/datamodel/low/base/schema.go index 10cc08bf..aac0204c 100644 --- a/datamodel/low/base/schema.go +++ b/datamodel/low/base/schema.go @@ -2,6 +2,7 @@ package base import ( "context" + "errors" "fmt" "hash/maphash" "sort" @@ -1403,14 +1404,16 @@ func buildPropertyMap(ctx context.Context, parent *Schema, root *yaml.Node, idx refString := "" var refNode *yaml.Node if h, _, l := utils.IsNodeRefValue(prop); h { - ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, prop, foundIdx) + ref, fIdx, err, fctx := low.LocateRefNodeWithContext(foundCtx, prop, foundIdx) if ref != nil { - refNode = prop prop = ref refString = l foundCtx = fctx foundIdx = fIdx + } else if errors.Is(err, low.ErrExternalRefSkipped) { + refString = l + refNode = prop } else { return nil, fmt.Errorf("schema properties build failed: cannot find reference %s, line %d, col %d", prop.Content[1].Value, prop.Content[1].Line, prop.Content[1].Column) @@ -1554,12 +1557,14 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label h := false if h, _, refLocation = utils.IsNodeRefValue(valueNode); h { isRef = true - ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, valueNode, foundIdx) + ref, fIdx, err, fctx := low.LocateRefNodeWithContext(foundCtx, valueNode, foundIdx) if ref != nil { refNode = valueNode valueNode = ref foundCtx = fctx foundIdx = fIdx + } else if err == low.ErrExternalRefSkipped { + refNode = valueNode } else { errors <- fmt.Errorf("build schema failed: reference cannot be found: %s, line %d, col %d", valueNode.Content[1].Value, valueNode.Content[1].Line, valueNode.Content[1].Column) @@ -1587,16 +1592,17 @@ func buildSchema(ctx context.Context, schemas chan schemaProxyBuildResult, label foundCtx = ctx if h, _, refLocation = utils.IsNodeRefValue(vn); h { isRef = true - ref, fIdx, _, fctx := low.LocateRefNodeWithContext(foundCtx, vn, foundIdx) + ref, fIdx, err, fctx := low.LocateRefNodeWithContext(foundCtx, vn, foundIdx) if ref != nil { refNode = vn vn = ref foundCtx = fctx foundIdx = fIdx + } else if err == low.ErrExternalRefSkipped { + refNode = vn } else { - err := fmt.Errorf("build schema failed: reference cannot be found: %s, line %d, col %d", + errors <- fmt.Errorf("build schema failed: reference cannot be found: %s, line %d, col %d", vn.Content[1].Value, vn.Content[1].Line, vn.Content[1].Column) - errors <- err return } } @@ -1633,14 +1639,22 @@ func ExtractSchema(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) ( foundIndex := idx foundCtx := ctx - if rf, rl, _ := utils.IsNodeRefValue(root); rf { + if rf, rl, rv := utils.IsNodeRefValue(root); rf { // locate reference in index. - ref, fIdx, _, nCtx := low.LocateRefNodeWithContext(ctx, root, idx) + ref, fIdx, err, nCtx := low.LocateRefNodeWithContext(ctx, root, idx) if ref != nil { schNode = ref schLabel = rl foundCtx = nCtx foundIndex = fIdx + } else if errors.Is(err, low.ErrExternalRefSkipped) { + refLocation = rv + schema := &SchemaProxy{kn: root, vn: root, idx: idx, ctx: ctx} + _ = schema.Build(ctx, root, root, idx) + n := &low.NodeReference[*SchemaProxy]{Value: schema, KeyNode: root, ValueNode: root} + n.SetReference(refLocation, root) + schema.SetReference(refLocation, root) + return n, nil } else { v := root.Content[1].Value if root.Content[1].Value == "" { @@ -1654,7 +1668,7 @@ func ExtractSchema(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) ( if schNode != nil { h := false if h, _, refLocation = utils.IsNodeRefValue(schNode); h { - ref, fIdx, _, nCtx := low.LocateRefNodeWithContext(foundCtx, schNode, foundIndex) + ref, fIdx, lerr, nCtx := low.LocateRefNodeWithContext(foundCtx, schNode, foundIndex) if ref != nil { refNode = schNode schNode = ref @@ -1662,6 +1676,8 @@ func ExtractSchema(ctx context.Context, root *yaml.Node, idx *index.SpecIndex) ( foundIndex = fIdx } foundCtx = nCtx + } else if errors.Is(lerr, low.ErrExternalRefSkipped) { + refNode = schNode } else { v := schNode.Content[1].Value if schNode.Content[1].Value == "" { diff --git a/datamodel/low/base/schema_proxy.go b/datamodel/low/base/schema_proxy.go index 603875ac..ae037e67 100644 --- a/datamodel/low/base/schema_proxy.go +++ b/datamodel/low/base/schema_proxy.go @@ -145,6 +145,12 @@ func (sp *SchemaProxy) Schema() *Schema { return sp.rendered } + // If this proxy represents an unresolved external ref, return nil without error. + if sp.IsReference() && sp.idx != nil && sp.idx.GetConfig() != nil && + sp.idx.GetConfig().SkipExternalRefResolution && utils.IsExternalRef(sp.GetReference()) { + return nil + } + // handle property merging for references with sibling properties buildNode := sp.vn if sp.idx != nil && sp.idx.GetConfig() != nil { diff --git a/datamodel/low/base/schema_proxy_test.go b/datamodel/low/base/schema_proxy_test.go index 7033f413..7debdfc9 100644 --- a/datamodel/low/base/schema_proxy_test.go +++ b/datamodel/low/base/schema_proxy_test.go @@ -700,3 +700,50 @@ type: array`), &node) result := sp.attemptPropertyMerging(node.Content[0], config) assert.Nil(t, result) // when merge fails, nil is returned } + +func TestSchemaProxy_SkipExternalRef_ReturnsNil(t *testing.T) { + yml := `components: + schemas: + Local: + type: object` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + sp := &SchemaProxy{idx: idx} + sp.SetReference("./models/Pet.yaml#/Pet", nil) + _ = sp.Build(context.Background(), nil, &yaml.Node{Kind: yaml.MappingNode}, idx) + + // Schema() should return nil without setting a build error + result := sp.Schema() + assert.Nil(t, result) + assert.Nil(t, sp.GetBuildError()) + assert.True(t, sp.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", sp.GetReference()) +} + +func TestSchemaProxy_SkipExternalRef_LocalRefNotBlocked(t *testing.T) { + yml := `components: + schemas: + Local: + type: object + properties: + name: + type: string` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + // local ref - should NOT be skipped by the guard + sp := &SchemaProxy{idx: idx} + sp.SetReference("#/components/schemas/Local", nil) + + assert.True(t, sp.IsReference()) + assert.Equal(t, "#/components/schemas/Local", sp.GetReference()) +} diff --git a/datamodel/low/base/schema_test.go b/datamodel/low/base/schema_test.go index 80cac525..1d75a244 100644 --- a/datamodel/low/base/schema_test.go +++ b/datamodel/low/base/schema_test.go @@ -1400,6 +1400,7 @@ func TestExtractSchema_OneOfRef(t *testing.T) { } func TestSchema_Hash_Equal(t *testing.T) { + low.ClearHashCache() left := `schema: $schema: https://athing.com multipleOf: 1 @@ -3043,3 +3044,165 @@ contentSchema: // Different contentSchema types should produce different hashes assert.NotEqual(t, hash1, hash2) } + +func TestBuildPropertyMap_SkipExternalRef(t *testing.T) { + // Schema with a property that has an external $ref + schemaYml := `type: object +properties: + local: + type: string + external: + $ref: './models/Pet.yaml#/Pet'` + + var schemaNode yaml.Node + _ = yaml.Unmarshal([]byte(schemaYml), &schemaNode) + + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&schemaNode, cfg) + + var schema Schema + _ = low.BuildModel(schemaNode.Content[0], &schema) + err := schema.Build(context.Background(), schemaNode.Content[0], idx) + assert.Nil(t, err) // parent builds successfully + + // Check properties + assert.NotNil(t, schema.Properties.Value) + found := false + for k, v := range schema.Properties.Value.FromOldest() { + if k.Value == "external" { + found = true + proxy := v.Value + assert.True(t, proxy.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", proxy.GetReference()) + // Schema() should return nil for unresolved external ref + assert.Nil(t, proxy.Schema()) + assert.Nil(t, proxy.GetBuildError()) + } + } + assert.True(t, found, "expected to find 'external' property") +} + +func TestBuildSchema_AllOf_SkipExternalRef(t *testing.T) { + schemaYml := `allOf: + - $ref: './models/Base.yaml#/Base' + - type: object + properties: + name: + type: string` + + var schemaNode yaml.Node + _ = yaml.Unmarshal([]byte(schemaYml), &schemaNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&schemaNode, cfg) + + var schema Schema + _ = low.BuildModel(schemaNode.Content[0], &schema) + err := schema.Build(context.Background(), schemaNode.Content[0], idx) + assert.Nil(t, err) + + assert.NotNil(t, schema.AllOf.Value) + assert.Len(t, schema.AllOf.Value, 2) + + // First allOf item should be the external ref + first := schema.AllOf.Value[0].Value + assert.True(t, first.IsReference()) + assert.Equal(t, "./models/Base.yaml#/Base", first.GetReference()) + assert.Nil(t, first.Schema()) + assert.Nil(t, first.GetBuildError()) +} + +func TestBuildSchema_OneOf_SkipExternalRef(t *testing.T) { + schemaYml := `oneOf: + - $ref: 'https://example.com/Cat.yaml' + - type: object + properties: + bark: + type: boolean` + + var schemaNode yaml.Node + _ = yaml.Unmarshal([]byte(schemaYml), &schemaNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&schemaNode, cfg) + + var schema Schema + _ = low.BuildModel(schemaNode.Content[0], &schema) + err := schema.Build(context.Background(), schemaNode.Content[0], idx) + assert.Nil(t, err) + + assert.NotNil(t, schema.OneOf.Value) + assert.Len(t, schema.OneOf.Value, 2) + + first := schema.OneOf.Value[0].Value + assert.True(t, first.IsReference()) + assert.Equal(t, "https://example.com/Cat.yaml", first.GetReference()) + assert.Nil(t, first.Schema()) + assert.Nil(t, first.GetBuildError()) +} + +func TestBuildSchema_AllOfMap_SkipExternalRef(t *testing.T) { + // allOf as a single map $ref (not an array) exercises the map branch of buildSchema (Site B) + schemaYml := `allOf: + $ref: './models/Base.yaml#/Base'` + + var schemaNode yaml.Node + _ = yaml.Unmarshal([]byte(schemaYml), &schemaNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&schemaNode, cfg) + + var schema Schema + _ = low.BuildModel(schemaNode.Content[0], &schema) + err := schema.Build(context.Background(), schemaNode.Content[0], idx) + assert.Nil(t, err) + + assert.NotNil(t, schema.AllOf.Value) + assert.Len(t, schema.AllOf.Value, 1) + + first := schema.AllOf.Value[0].Value + assert.True(t, first.IsReference()) + assert.Equal(t, "./models/Base.yaml#/Base", first.GetReference()) + assert.Nil(t, first.Schema()) + assert.Nil(t, first.GetBuildError()) +} + +func TestExtractSchema_RootRef_SkipExternalRef(t *testing.T) { + yml := `$ref: './models/Pet.yaml#/Pet'` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&root, cfg) + + result, err := ExtractSchema(context.Background(), root.Content[0], idx) + assert.Nil(t, err) + assert.NotNil(t, result) + assert.True(t, result.Value.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", result.Value.GetReference()) + assert.Nil(t, result.Value.Schema()) + assert.Nil(t, result.Value.GetBuildError()) +} + +func TestExtractSchema_SchemaKeyRef_SkipExternalRef(t *testing.T) { + yml := `schema: + $ref: './models/Pet.yaml#/Pet'` + + var root yaml.Node + _ = yaml.Unmarshal([]byte(yml), &root) + + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&root, cfg) + + result, err := ExtractSchema(context.Background(), root.Content[0], idx) + assert.Nil(t, err) + assert.NotNil(t, result) + assert.True(t, result.Value.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", result.Value.GetReference()) + assert.Nil(t, result.Value.Schema()) + assert.Nil(t, result.Value.GetBuildError()) +} diff --git a/datamodel/low/extraction_functions.go b/datamodel/low/extraction_functions.go index 92d667f1..2fcc8e72 100644 --- a/datamodel/low/extraction_functions.go +++ b/datamodel/low/extraction_functions.go @@ -5,6 +5,7 @@ package low import ( "context" + "errors" "fmt" "hash/maphash" "net/url" @@ -38,6 +39,10 @@ var stringBuilderPool = sync.Pool{ // Uses sync.Map for thread-safe concurrent access. var hashCache sync.Map +// ErrExternalRefSkipped is returned by LocateRefNodeWithContext when +// SkipExternalRefResolution is enabled and the reference is external. +var ErrExternalRefSkipped = errors.New("external reference resolution skipped") + // ClearHashCache clears the global hash cache. This should be called before // starting a new document comparison to ensure clean state. func ClearHashCache() { @@ -116,6 +121,10 @@ func LocateRefNodeWithContext(ctx context.Context, root *yaml.Node, idx *index.S root.Line, root.Column), ctx } + if idx != nil && idx.GetConfig() != nil && idx.GetConfig().SkipExternalRefResolution && utils.IsExternalRef(rv) { + return nil, idx, ErrExternalRefSkipped, ctx + } + origRef := rv resolvedRef := rv if scope := index.GetSchemaIdScope(ctx); scope != nil && scope.BaseUri != "" { @@ -381,6 +390,10 @@ func ExtractObjectRaw[T Buildable[N], N any](ctx context.Context, key, root *yam if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + var n T = new(N) + SetReference(n, rv, root) + return n, nil, true, rv } else { if err != nil { return nil, fmt.Errorf("object extraction failed: %s", err.Error()), isReference, referenceValue @@ -431,6 +444,12 @@ func ExtractObject[T Buildable[N], N any](ctx context.Context, label string, roo if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + var n T = new(N) + SetReference(n, refVal, root) + res := NodeReference[T]{Value: n, KeyNode: rl, ValueNode: root} + res.SetReference(refVal, root) + return res, nil } else { if err != nil { return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", err.Error()) @@ -453,6 +472,12 @@ func ExtractObject[T Buildable[N], N any](ctx context.Context, label string, roo if lerr != nil { circError = lerr } + } else if errors.Is(lerr, ErrExternalRefSkipped) { + var n T = new(N) + SetReference(n, rVal, vn) + res := NodeReference[T]{Value: n, KeyNode: ln, ValueNode: vn} + res.SetReference(rVal, vn) + return res, nil } else { if lerr != nil { return NodeReference[T]{}, fmt.Errorf("object extraction failed: %s", lerr.Error()) @@ -498,11 +523,37 @@ func SetReference(obj any, ref string, refNode *yaml.Node) { return } + // Ensure the embedded *Reference is initialized before calling SetReference. + // Buildable types embed *Reference (a pointer) which is nil after new(T). + // Calling SetReference on a nil *Reference would panic. + initEmbeddedReference(obj) + if r, ok := obj.(SetReferencer); ok { r.SetReference(ref, refNode) } } +// initEmbeddedReference uses reflection to find and initialize a nil *Reference +// field embedded in obj. This is needed when objects are created via new(T) without +// calling Build(), which normally initializes the embedded *Reference. +func initEmbeddedReference(obj any) { + v := reflect.ValueOf(obj) + if v.Kind() != reflect.Ptr || v.IsNil() { + return + } + v = v.Elem() + if v.Kind() != reflect.Struct { + return + } + f := v.FieldByName("Reference") + if !f.IsValid() || f.Kind() != reflect.Ptr || !f.IsNil() { + return + } + if f.Type() == reflect.TypeOf((*Reference)(nil)) { + f.Set(reflect.ValueOf(new(Reference))) + } +} + // ExtractArray will extract a slice of []ValueReference[T] from a root yaml.Node that is defined as a sequence. // Used when the value being extracted is an array. func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root *yaml.Node, idx *index.SpecIndex) ([]ValueReference[T], @@ -523,6 +574,8 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + return []ValueReference[T]{}, rl, root, nil } else { return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", root.Content[1].Value) @@ -540,6 +593,8 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + return []ValueReference[T]{}, ln, vn, nil } else { if err != nil { return []ValueReference[T]{}, nil, nil, @@ -587,6 +642,13 @@ func ExtractArray[T Buildable[N], N any](ctx context.Context, label string, root if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + var n T = new(N) + SetReference(n, rv, node) + v := ValueReference[T]{Value: n, ValueNode: node} + v.SetReference(rv, node) + items = append(items, v) + continue } else { if err != nil { return []ValueReference[T]{}, nil, nil, fmt.Errorf("array build failed: reference cannot be found: %s", @@ -688,6 +750,13 @@ func ExtractMapNoLookupExtensions[PT Buildable[N], N any]( if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + var n PT = new(N) + SetReference(n, rv, node) + v := ValueReference[PT]{Value: n, ValueNode: node} + v.SetReference(rv, node) + valueMap.Set(KeyReference[string]{Value: currentKey.Value, KeyNode: currentKey}, v) + continue } else { if err != nil { return nil, fmt.Errorf("map build failed: reference cannot be found: %s", err.Error()) @@ -781,6 +850,8 @@ func ExtractMapExtensions[PT Buildable[N], N any]( if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + return nil, rl, root, nil } else { return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", root.Content[1].Value) @@ -798,6 +869,8 @@ func ExtractMapExtensions[PT Buildable[N], N any]( if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + return nil, labelNode, valueNode, nil } else { if err != nil { return nil, labelNode, valueNode, fmt.Errorf("map build failed: reference cannot be found: %s", @@ -888,6 +961,15 @@ func ExtractMapExtensions[PT Buildable[N], N any]( if err != nil { circError = err } + } else if errors.Is(err, ErrExternalRefSkipped) { + var n PT = new(N) + SetReference(n, refVal, en) + v := ValueReference[PT]{Value: n, ValueNode: en} + v.SetReference(refVal, en) + return mappingResult[PT]{ + k: KeyReference[string]{KeyNode: input.label, Value: input.label.Value}, + v: v, + }, nil } else { if err != nil { return mappingResult[PT]{}, fmt.Errorf("flat map build failed: reference cannot be found: %s", diff --git a/datamodel/low/extraction_functions_test.go b/datamodel/low/extraction_functions_test.go index 533ec973..cc88a777 100644 --- a/datamodel/low/extraction_functions_test.go +++ b/datamodel/low/extraction_functions_test.go @@ -144,6 +144,18 @@ func (p *pizza) Build(_ context.Context, _, _ *yaml.Node, _ *index.SpecIndex) er return nil } +// refPizza embeds *Reference like real buildable types (Parameter, Header, etc.) +// to test that SetReference initializes the nil *Reference before use. +type refPizza struct { + *Reference + Description NodeReference[string] +} + +func (p *refPizza) Build(_ context.Context, _, _ *yaml.Node, _ *index.SpecIndex) error { + p.Reference = new(Reference) + return nil +} + func TestExtractObject(t *testing.T) { yml := `components: schemas: @@ -3692,3 +3704,437 @@ func TestHashNodeTree_ConcurrentModification(t *testing.T) { // If we get here without panic, the fix works } + +func TestLocateRefNodeWithContext_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + // external ref should be skipped + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + node, retIdx, err, _ := LocateRefNodeWithContext(context.Background(), cNode.Content[0], idx) + assert.Nil(t, node) + assert.Equal(t, idx, retIdx) + assert.ErrorIs(t, err, ErrExternalRefSkipped) +} + +func TestLocateRefNodeWithContext_SkipExternalRef_LocalNotSkipped(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + // local ref should still resolve normally + refYml := `$ref: '#/components/schemas/cake'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + node, _, err, _ := LocateRefNodeWithContext(context.Background(), cNode.Content[0], idx) + assert.NotNil(t, node) + assert.Nil(t, err) +} + +func TestLocateRefNodeWithContext_SkipExternalRef_FlagNotSet(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + // flag NOT set + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + // external ref without the flag: should try to resolve and fail + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + node, _, err, _ := LocateRefNodeWithContext(context.Background(), cNode.Content[0], idx) + assert.Nil(t, node) + assert.NotNil(t, err) + assert.NotErrorIs(t, err, ErrExternalRefSkipped) // regular not-found error, not sentinel +} + +func TestLocateRefEnd_SkipExternalRef_Propagates(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: 'https://example.com/schema.yaml'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + node, retIdx, err, _ := LocateRefEnd(context.Background(), cNode.Content[0], idx, 0) + assert.Nil(t, node) + assert.Equal(t, idx, retIdx) + assert.ErrorIs(t, err, ErrExternalRefSkipped) +} + +func TestExtractObjectRaw_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + tag, err, isRef, rv := ExtractObjectRaw[*pizza](context.Background(), nil, cNode.Content[0], idx) + assert.Nil(t, err) + assert.True(t, isRef) + assert.Equal(t, "./models/Pet.yaml#/Pet", rv) + assert.NotNil(t, tag) +} + +func TestExtractObject_RootRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + // Pass the mapping node directly (not the document node) + res, err := ExtractObject[*pizza](context.Background(), "tags", cNode.Content[0], idx) + assert.Nil(t, err) + assert.NotNil(t, res.Value) + assert.True(t, res.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", res.GetReference()) +} + +func TestExtractObject_ValueRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `tags: + $ref: 'https://example.com/tags.yaml'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + // Pass the mapping node + res, err := ExtractObject[*pizza](context.Background(), "tags", cNode.Content[0], idx) + assert.Nil(t, err) + assert.NotNil(t, res.Value) + assert.True(t, res.IsReference()) + assert.Equal(t, "https://example.com/tags.yaml", res.GetReference()) +} + +func TestExtractArray_RootRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + // Pass the mapping node directly so IsNodeRefValue sees the $ref + refYml := `$ref: './models/Tags.yaml#/Tags'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + items, _, _, err := ExtractArray[*pizza](context.Background(), "tags", cNode.Content[0], idx) + assert.Nil(t, err) + assert.Empty(t, items) +} + +func TestExtractArray_ValueRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `parameters: + $ref: './models/Params.yaml#/Params'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + items, _, _, err := ExtractArray[*pizza](context.Background(), "parameters", cNode.Content[0], idx) + assert.Nil(t, err) + assert.Empty(t, items) +} + +func TestExtractArray_PerItem_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `tags: + - $ref: './models/Tag.yaml#/Tag' + - description: local tag` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + // Pass the mapping node (not the document node) + items, _, _, err := ExtractArray[*pizza](context.Background(), "tags", cNode.Content[0], idx) + assert.Nil(t, err) + assert.Len(t, items, 2) + assert.True(t, items[0].IsReference()) + assert.Equal(t, "./models/Tag.yaml#/Tag", items[0].GetReference()) +} + +func TestExtractMapExtensions_RootRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: 'https://example.com/paths.yaml'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + m, _, _, err := ExtractMapExtensions[*pizza, pizza](context.Background(), "paths", cNode.Content[0], idx, false) + assert.Nil(t, err) + assert.Nil(t, m) +} + +func TestExtractMapExtensions_ValueRef_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `paths: + $ref: 'https://example.com/paths.yaml'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + // Pass the mapping node + m, _, _, err := ExtractMapExtensions[*pizza, pizza](context.Background(), "paths", cNode.Content[0], idx, false) + assert.Nil(t, err) + assert.Nil(t, m) +} + +func TestExtractMapExtensions_PerItem_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `responses: + pet: + $ref: './models/Pet.yaml#/Pet' + local: + description: local thing` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + m, _, _, err := ExtractMapExtensions[*pizza, pizza](context.Background(), "responses", cNode.Content[0], idx, false) + assert.Nil(t, err) + assert.NotNil(t, m) + assert.Equal(t, 2, m.Len()) + + petRef := FindItemInOrderedMap[*pizza]("pet", m) + assert.NotNil(t, petRef) + assert.True(t, petRef.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", petRef.GetReference()) +} + +func TestExtractMapNoLookupExtensions_PerItem_SkipExternalRef(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `pet: + $ref: './models/Pet.yaml#/Pet' +local: + description: local thing` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + m, err := ExtractMapNoLookupExtensions[*pizza, pizza](context.Background(), cNode.Content[0], idx, false) + assert.Nil(t, err) + assert.NotNil(t, m) + assert.Equal(t, 2, m.Len()) + + // check the external ref entry + petRef := FindItemInOrderedMap[*pizza]("pet", m) + assert.NotNil(t, petRef) + assert.True(t, petRef.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", petRef.GetReference()) +} + +func TestSetReference_NilEmbeddedReference(t *testing.T) { + // new(refPizza) has a nil *Reference. SetReference must not panic. + rp := new(refPizza) + assert.Nil(t, rp.Reference, "Reference should be nil before SetReference") + + node := &yaml.Node{Value: "test"} + assert.NotPanics(t, func() { + SetReference(rp, "./models/Pet.yaml", node) + }) + assert.NotNil(t, rp.Reference, "Reference should be initialized after SetReference") + assert.Equal(t, "./models/Pet.yaml", rp.GetReference()) + assert.True(t, rp.IsReference()) +} + +func TestInitEmbeddedReference_NilObj(t *testing.T) { + // Should not panic on nil + assert.NotPanics(t, func() { + initEmbeddedReference(nil) + }) +} + +func TestInitEmbeddedReference_NonStruct(t *testing.T) { + // Should not panic on non-struct + s := "hello" + assert.NotPanics(t, func() { + initEmbeddedReference(&s) + }) +} + +func TestInitEmbeddedReference_NoReferenceField(t *testing.T) { + // pizza has no *Reference field, should be a no-op + p := new(pizza) + assert.NotPanics(t, func() { + initEmbeddedReference(p) + }) +} + +func TestInitEmbeddedReference_AlreadyInitialized(t *testing.T) { + rp := &refPizza{Reference: new(Reference)} + rp.Reference.SetReference("original", nil) + + // Should not overwrite an already-initialized Reference + initEmbeddedReference(rp) + assert.Equal(t, "original", rp.GetReference()) +} + +func TestExtractObjectRaw_SkipExternalRef_EmbeddedReference(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + tag, err, isRef, rv := ExtractObjectRaw[*refPizza](context.Background(), nil, cNode.Content[0], idx) + assert.Nil(t, err) + assert.True(t, isRef) + assert.Equal(t, "./models/Pet.yaml#/Pet", rv) + require.NotNil(t, tag) + assert.True(t, tag.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", tag.GetReference()) +} + +func TestExtractObject_RootRef_SkipExternalRef_EmbeddedReference(t *testing.T) { + yml := `components: + schemas: + cake: + description: hello` + + var idxNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &idxNode) + cfg := index.CreateClosedAPIIndexConfig() + cfg.SkipExternalRefResolution = true + idx := index.NewSpecIndexWithConfig(&idxNode, cfg) + + refYml := `$ref: './models/Pet.yaml#/Pet'` + var cNode yaml.Node + _ = yaml.Unmarshal([]byte(refYml), &cNode) + + result, err := ExtractObject[*refPizza](context.Background(), "cake", cNode.Content[0], idx) + assert.Nil(t, err) + assert.True(t, result.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", result.GetReference()) + require.NotNil(t, result.Value) + assert.True(t, result.Value.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", result.Value.GetReference()) +} diff --git a/datamodel/low/v2/swagger.go b/datamodel/low/v2/swagger.go index d40f9cd9..835836ad 100644 --- a/datamodel/low/v2/swagger.go +++ b/datamodel/low/v2/swagger.go @@ -145,6 +145,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur idxConfig.IgnoreArrayCircularReferences = config.IgnoreArrayCircularReferences idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences idxConfig.AllowUnknownExtensionContentDetection = config.AllowUnknownExtensionContentDetection + idxConfig.SkipExternalRefResolution = config.SkipExternalRefResolution idxConfig.AvoidCircularReferenceCheck = true idxConfig.BaseURL = config.BaseURL idxConfig.BasePath = config.BasePath diff --git a/datamodel/low/v3/create_document.go b/datamodel/low/v3/create_document.go index e253bcb5..953252c9 100644 --- a/datamodel/low/v3/create_document.go +++ b/datamodel/low/v3/create_document.go @@ -49,6 +49,7 @@ func createDocument(info *datamodel.SpecInfo, config *datamodel.DocumentConfigur idxConfig.IgnorePolymorphicCircularReferences = config.IgnorePolymorphicCircularReferences idxConfig.AllowUnknownExtensionContentDetection = config.AllowUnknownExtensionContentDetection idxConfig.TransformSiblingRefs = config.TransformSiblingRefs + idxConfig.SkipExternalRefResolution = config.SkipExternalRefResolution idxConfig.AvoidCircularReferenceCheck = true // handle $self field for OpenAPI 3.2+ documents diff --git a/document_examples_test.go b/document_examples_test.go index c185dbd6..33d89868 100644 --- a/document_examples_test.go +++ b/document_examples_test.go @@ -639,3 +639,224 @@ func ExampleNewDocument_modifyAndReRender() { // Output: There were 13 original paths. There are now 14 paths in the document // The new spec has 31406 bytes } + +// TestDocument_SkipExternalRefResolution_Issue519 verifies that when SkipExternalRefResolution is enabled, +// external $ref references are left unresolved while the rest of the model is built correctly. +// This is the scenario from https://github.com/pb33f/libopenapi/issues/519 — code generators like +// oapi-codegen need to parse specs with external refs without actually resolving them, using import +// mappings instead. +func TestDocument_SkipExternalRefResolution_Issue519(t *testing.T) { + // An OpenAPI 3.1 spec where: + // - Order schema has a property "product" that is an external $ref to ./models/product.yaml + // - Order schema has a local property "id" (integer) that should be fully accessible + // - Pet schema is a top-level external $ref to ./models/pet.yaml + // - There is a local schema (ErrorResponse) with no external refs at all + spec := `openapi: "3.1.0" +info: + title: Test External Refs + version: "1.0" +paths: + /orders: + get: + summary: List orders + operationId: listOrders + responses: + '200': + description: A list of orders + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Order' + '500': + description: Server error + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + /pets: + get: + summary: List pets + operationId: listPets + responses: + '200': + description: A list of pets + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Pet' +components: + schemas: + Order: + type: object + properties: + id: + type: integer + description: The order ID + product: + $ref: './models/product.yaml' + customer: + $ref: 'https://example.com/schemas/customer.yaml' + required: + - id + - product + Pet: + $ref: './models/pet.yaml' + ErrorResponse: + type: object + properties: + code: + type: integer + message: + type: string +` + + // ---- First: demonstrate the problem WITHOUT the flag ---- + // Without SkipExternalRefResolution, the rolodex tries to resolve external refs, + // fails because no BasePath/BaseURL is set, and the model build either fails entirely + // or produces schemas where Schema() returns nil for anything containing external refs. + configWithout := datamodel.NewDocumentConfiguration() + + docWithout, err := NewDocumentWithConfiguration([]byte(spec), configWithout) + assert.NoError(t, err, "document creation should succeed") + + modelWithout, buildErr := docWithout.BuildV3Model() + // The build produces errors because external refs can't be resolved + if buildErr == nil && modelWithout != nil { + // If somehow the model builds, the Order schema with external refs in properties + // should have Schema() returning nil — this is the core problem from issue #519 + orderWithout := modelWithout.Model.Components.Schemas.GetOrZero("Order") + if orderWithout != nil { + orderSchemaWithout := orderWithout.Schema() + assert.Nil(t, orderSchemaWithout, "Without SkipExternalRefResolution, Order.Schema() "+ + "should be nil because external refs cannot be resolved") + } + } + // Either way, the build is broken when external refs can't be resolved. + // This is what issue #519 is about. + + // ---- Now: enable SkipExternalRefResolution and verify the fix ---- + config := datamodel.NewDocumentConfiguration() + config.SkipExternalRefResolution = true + + doc, err := NewDocumentWithConfiguration([]byte(spec), config) + assert.NoError(t, err, "document creation should succeed with SkipExternalRefResolution") + + model, errs := doc.BuildV3Model() + assert.NoError(t, errs, "building model should not produce errors with SkipExternalRefResolution") + assert.NotNil(t, model, "model should be non-nil") + + // Verify we can access components + assert.NotNil(t, model.Model.Components, "Components should be non-nil") + assert.NotNil(t, model.Model.Components.Schemas, "Schemas should be non-nil") + + // Verify paths are parsed correctly + assert.NotNil(t, model.Model.Paths, "Paths should be non-nil") + assert.Equal(t, 2, orderedmap.Len(model.Model.Paths.PathItems), "Should have 2 paths") + + // ---- Check the Order schema (object with external refs in properties) ---- + orderProxy := model.Model.Components.Schemas.GetOrZero("Order") + assert.NotNil(t, orderProxy, "Order SchemaProxy should exist") + + // THIS is the key assertion from issue #519: Schema() should NOT be nil + orderSchema := orderProxy.Schema() + assert.NotNil(t, orderSchema, "Order.Schema() should NOT be nil when SkipExternalRefResolution is enabled") + + if orderSchema != nil { + // The local "id" property should be fully accessible + idProxy := orderSchema.Properties.GetOrZero("id") + assert.NotNil(t, idProxy, "Order.properties.id should exist") + if idProxy != nil { + idSchema := idProxy.Schema() + assert.NotNil(t, idSchema, "id schema should be buildable") + if idSchema != nil { + assert.Equal(t, "integer", idSchema.Type[0], "id should be type integer") + assert.Equal(t, "The order ID", idSchema.Description, "id description should be set") + } + } + + // The "product" property should be an unresolved external ref + productProxy := orderSchema.Properties.GetOrZero("product") + assert.NotNil(t, productProxy, "Order.properties.product should exist") + if productProxy != nil { + assert.True(t, productProxy.IsReference(), "product should report IsReference()=true") + assert.Equal(t, "./models/product.yaml", productProxy.GetReference(), + "product GetReference() should return the external ref string") + // Schema() should be nil for unresolved external refs — the ref is preserved but not resolved + assert.Nil(t, productProxy.Schema(), "product.Schema() should be nil (external ref not resolved)") + } + + // The "customer" property should also be an unresolved external ref (remote URL) + customerProxy := orderSchema.Properties.GetOrZero("customer") + assert.NotNil(t, customerProxy, "Order.properties.customer should exist") + if customerProxy != nil { + assert.True(t, customerProxy.IsReference(), "customer should report IsReference()=true") + assert.Equal(t, "https://example.com/schemas/customer.yaml", customerProxy.GetReference(), + "customer GetReference() should return the remote ref URL") + assert.Nil(t, customerProxy.Schema(), "customer.Schema() should be nil (external ref not resolved)") + } + + // Verify required fields are preserved + assert.Contains(t, orderSchema.Required, "id") + assert.Contains(t, orderSchema.Required, "product") + } + + // ---- Check the Pet schema (top-level external $ref) ---- + petProxy := model.Model.Components.Schemas.GetOrZero("Pet") + assert.NotNil(t, petProxy, "Pet SchemaProxy should exist") + if petProxy != nil { + assert.True(t, petProxy.IsReference(), "Pet should report IsReference()=true") + assert.Equal(t, "./models/pet.yaml", petProxy.GetReference(), + "Pet GetReference() should return the external ref string") + // Top-level ref: Schema() is nil because the entire schema is an unresolved external ref + assert.Nil(t, petProxy.Schema(), "Pet.Schema() should be nil (entire schema is external ref)") + } + + // ---- Check the ErrorResponse schema (fully local, no external refs) ---- + errorProxy := model.Model.Components.Schemas.GetOrZero("ErrorResponse") + assert.NotNil(t, errorProxy, "ErrorResponse SchemaProxy should exist") + if errorProxy != nil { + errorSchema := errorProxy.Schema() + assert.NotNil(t, errorSchema, "ErrorResponse.Schema() should be non-nil (no external refs)") + if errorSchema != nil { + codeProxy := errorSchema.Properties.GetOrZero("code") + assert.NotNil(t, codeProxy, "ErrorResponse.properties.code should exist") + if codeProxy != nil && codeProxy.Schema() != nil { + assert.Equal(t, "integer", codeProxy.Schema().Type[0]) + } + + msgProxy := errorSchema.Properties.GetOrZero("message") + assert.NotNil(t, msgProxy, "ErrorResponse.properties.message should exist") + if msgProxy != nil && msgProxy.Schema() != nil { + assert.Equal(t, "string", msgProxy.Schema().Type[0]) + } + } + } + + // ---- Check that the response schema references work via paths ---- + ordersPath := model.Model.Paths.PathItems.GetOrZero("/orders") + assert.NotNil(t, ordersPath, "/orders path should exist") + if ordersPath != nil && ordersPath.Get != nil { + resp200 := ordersPath.Get.Responses.Codes.GetOrZero("200") + assert.NotNil(t, resp200, "200 response should exist") + if resp200 != nil { + jsonContent := resp200.Content.GetOrZero("application/json") + assert.NotNil(t, jsonContent, "application/json content should exist") + if jsonContent != nil && jsonContent.Schema != nil { + arrSchema := jsonContent.Schema.Schema() + assert.NotNil(t, arrSchema, "array schema should be non-nil") + if arrSchema != nil { + assert.Equal(t, "array", arrSchema.Type[0]) + // Items should reference Order via local $ref + assert.NotNil(t, arrSchema.Items, "items should exist") + if arrSchema.Items != nil && arrSchema.Items.A != nil { + assert.True(t, arrSchema.Items.A.IsReference(), "items should be a reference to Order") + } + } + } + } + } +} diff --git a/document_test.go b/document_test.go index 0abf3689..85f8127d 100644 --- a/document_test.go +++ b/document_test.go @@ -1928,3 +1928,109 @@ components: t.Fatal("components or schemas not found in reloaded model") } } + +func TestSkipExternalRefResolution_Integration(t *testing.T) { + spec := `openapi: "3.1.0" +info: + title: Test + version: "1.0" +paths: + /pets: + get: + operationId: listPets + responses: + '200': + description: ok + content: + application/json: + schema: + type: object + properties: + localProp: + type: string + externalProp: + $ref: './models/Pet.yaml#/Pet' + allOf: + - $ref: './models/Base.yaml#/Base' + - type: object + properties: + name: + type: string +components: + schemas: + LocalSchema: + type: object + properties: + id: + type: integer` + + config := datamodel.NewDocumentConfiguration() + config.SkipExternalRefResolution = true + + doc, err := NewDocumentWithConfiguration([]byte(spec), config) + require.NoError(t, err) + + model, errs := doc.BuildV3Model() + _ = errs + require.NotNil(t, model) + + // Navigate to the response schema + path := model.Model.Paths.PathItems.GetOrZero("/pets") + require.NotNil(t, path) + + getOp := path.Get + require.NotNil(t, getOp) + + resp := getOp.Responses.Codes.GetOrZero("200") + require.NotNil(t, resp) + + jsonContent := resp.Content.GetOrZero("application/json") + require.NotNil(t, jsonContent) + + schema := jsonContent.Schema + require.NotNil(t, schema) + + schemaResolved := schema.Schema() + require.NotNil(t, schemaResolved) + + // Check properties are iterable + require.NotNil(t, schemaResolved.Properties) + + // Check external ref property + externalProp := schemaResolved.Properties.GetOrZero("externalProp") + require.NotNil(t, externalProp, "externalProp should exist in properties") + assert.True(t, externalProp.IsReference()) + assert.Equal(t, "./models/Pet.yaml#/Pet", externalProp.GetReference()) + assert.Nil(t, externalProp.Schema()) // unresolved external ref + assert.Nil(t, externalProp.GetBuildError()) + + // Check local properties still work + localProp := schemaResolved.Properties.GetOrZero("localProp") + require.NotNil(t, localProp, "localProp should exist in properties") + assert.False(t, localProp.IsReference()) + localSchema := localProp.Schema() + require.NotNil(t, localSchema) + assert.Contains(t, localSchema.Type, "string") + + // Check allOf with external ref + require.NotNil(t, schemaResolved.AllOf) + require.Len(t, schemaResolved.AllOf, 2) + + allOfFirst := schemaResolved.AllOf[0] + assert.True(t, allOfFirst.IsReference()) + assert.Equal(t, "./models/Base.yaml#/Base", allOfFirst.GetReference()) + assert.Nil(t, allOfFirst.Schema()) + assert.Nil(t, allOfFirst.GetBuildError()) + + // Second allOf should build normally (local) + allOfSecond := schemaResolved.AllOf[1] + assert.False(t, allOfSecond.IsReference()) + secondSchema := allOfSecond.Schema() + require.NotNil(t, secondSchema) + + // Check that the LocalSchema component still resolves + localSchemaComp := model.Model.Components.Schemas.GetOrZero("LocalSchema") + require.NotNil(t, localSchemaComp, "LocalSchema component should exist") + resolved := localSchemaComp.Schema() + require.NotNil(t, resolved) +} diff --git a/index/extract_refs.go b/index/extract_refs.go index 31cf4000..1f83c238 100644 --- a/index/extract_refs.go +++ b/index/extract_refs.go @@ -840,6 +840,10 @@ func (index *SpecIndex) ExtractComponentsFromRefs(ctx context.Context, refs []*R } index.refLock.Unlock() } else { + // If SkipExternalRefResolution is enabled, don't record errors for external refs + if index.config != nil && index.config.SkipExternalRefResolution && utils.IsExternalRef(ref.Definition) { + continue + } // Record error for definitive failure _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) index.errorLock.Lock() @@ -959,6 +963,10 @@ func (index *SpecIndex) ExtractComponentsFromRefs(ctx context.Context, refs []*R } index.refLock.Unlock() } else { + // If SkipExternalRefResolution is enabled, don't record errors for external refs + if index.config != nil && index.config.SkipExternalRefResolution && utils.IsExternalRef(ref.Definition) { + continue + } // Definitive failure - record error _, path := utils.ConvertComponentIdIntoFriendlyPathSearch(ref.Definition) index.errorLock.Lock() diff --git a/index/find_component.go b/index/find_component.go index a14af8b6..4234b719 100644 --- a/index/find_component.go +++ b/index/find_component.go @@ -150,6 +150,11 @@ func (index *SpecIndex) lookupRolodex(ctx context.Context, uri []string) *Refere return nil } + // If SkipExternalRefResolution is enabled, don't attempt rolodex lookups + if index.config != nil && index.config.SkipExternalRefResolution { + return nil + } + if len(uri) > 0 { // split string to remove file reference diff --git a/index/find_component_test.go b/index/find_component_test.go index c71d4c75..5b534fc4 100644 --- a/index/find_component_test.go +++ b/index/find_component_test.go @@ -544,3 +544,33 @@ components: assert.True(t, strings.Contains(logOutput, "external_schema.yaml"), "Expected log to contain the file location") } + +func TestLookupRolodex_SkipExternalRefResolution(t *testing.T) { + spec := `openapi: 3.0.2 +info: + title: Test + version: 1.0.0 +components: + schemas: + thang: + type: object` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(spec), &rootNode) + + c := CreateOpenAPIIndexConfig() + c.SkipExternalRefResolution = true + + index := NewSpecIndexWithConfig(&rootNode, c) + r := NewRolodex(c) + index.rolodex = r + + // lookupRolodex should return nil immediately when SkipExternalRefResolution is set, + // without attempting to open files via the rolodex + n := index.lookupRolodex(context.Background(), []string{"./models/pet.yaml"}) + assert.Nil(t, n, "lookupRolodex should return nil when SkipExternalRefResolution is enabled") + + // Also test with a remote URL reference + n = index.lookupRolodex(context.Background(), []string{"https://example.com/schemas/pet.yaml"}) + assert.Nil(t, n, "lookupRolodex should return nil for remote refs when SkipExternalRefResolution is enabled") +} diff --git a/index/index_model.go b/index/index_model.go index 376cdaff..73ee111c 100644 --- a/index/index_model.go +++ b/index/index_model.go @@ -177,6 +177,10 @@ type SpecIndexConfig struct { // the file is a JSON Schema. To allow JSON Schema files to be included set this to true. SkipDocumentCheck bool + // SkipExternalRefResolution will skip resolving external $ref references (those not starting with #). + // When enabled, external references will be left as-is during model building. + SkipExternalRefResolution bool + // ExtractRefsSequentially will extract all references sequentially, which means the index will look up references // as it finds them, vs looking up everything asynchronously. // This is a more thorough way of building the index, but it's slower. It's required building a document @@ -270,6 +274,7 @@ func (s *SpecIndexConfig) ToDocumentConfiguration() *datamodel.DocumentConfigura TransformSiblingRefs: s.TransformSiblingRefs, MergeReferencedProperties: s.MergeReferencedProperties, PropertyMergeStrategy: strategy, + SkipExternalRefResolution: s.SkipExternalRefResolution, Logger: s.Logger, } } diff --git a/index/index_model_test.go b/index/index_model_test.go index 5b82de16..1203cf2e 100644 --- a/index/index_model_test.go +++ b/index/index_model_test.go @@ -107,3 +107,19 @@ func TestSpecIndexConfig_ToDocumentConfiguration_AllFields(t *testing.T) { assert.True(t, result.TransformSiblingRefs) assert.False(t, result.MergeReferencedProperties) // default disabled for index configs } + +func TestSpecIndexConfig_ToDocumentConfiguration_SkipExternalRefResolution(t *testing.T) { + config := &SpecIndexConfig{ + SkipExternalRefResolution: true, + } + result := config.ToDocumentConfiguration() + assert.NotNil(t, result) + assert.True(t, result.SkipExternalRefResolution) +} + +func TestSpecIndexConfig_ToDocumentConfiguration_SkipExternalRefResolution_False(t *testing.T) { + config := &SpecIndexConfig{} + result := config.ToDocumentConfiguration() + assert.NotNil(t, result) + assert.False(t, result.SkipExternalRefResolution) +} diff --git a/index/resolver.go b/index/resolver.go index 8933dd46..7a0fcba5 100644 --- a/index/resolver.go +++ b/index/resolver.go @@ -599,6 +599,14 @@ func (resolver *Resolver) extractRelatives(ref *Reference, node, parent *yaml.No value := node.Content[i+1].Value value = strings.ReplaceAll(value, "\\\\", "\\") + + // If SkipExternalRefResolution is enabled, skip external refs entirely + if resolver.specIndex != nil && resolver.specIndex.config != nil && + resolver.specIndex.config.SkipExternalRefResolution && utils.IsExternalRef(value) { + skip = true + continue + } + var locatedRef *Reference var fullDef string var definition string diff --git a/index/resolver_test.go b/index/resolver_test.go index dc92e847..628d8973 100644 --- a/index/resolver_test.go +++ b/index/resolver_test.go @@ -1781,4 +1781,43 @@ func TestVisitReference_Nil(t *testing.T) { assert.Nil(t, n) } +func TestResolver_SkipExternalRefResolution(t *testing.T) { + // Spec with external $ref that cannot be resolved + yml := `openapi: 3.0.0 +info: + title: Test + version: 1.0.0 +components: + schemas: + Order: + type: object + properties: + id: + type: integer + product: + $ref: './models/product.yaml' + customer: + $ref: 'https://example.com/schemas/customer.yaml'` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + config := CreateOpenAPIIndexConfig() + config.SkipExternalRefResolution = true + idx := NewSpecIndexWithConfig(&rootNode, config) + + resolver := NewResolver(idx) + assert.NotNil(t, resolver) + + errs := resolver.Resolve() + // With SkipExternalRefResolution, the resolver should NOT produce errors + // for external refs that can't be resolved + for _, err := range errs { + assert.NotContains(t, err.ErrorRef.Error(), "product.yaml", + "resolver should not report errors for external file refs when SkipExternalRefResolution is enabled") + assert.NotContains(t, err.ErrorRef.Error(), "customer.yaml", + "resolver should not report errors for external URL refs when SkipExternalRefResolution is enabled") + } +} + // func (resolver *Resolver) VisitReference(ref *Reference, seen map[string]bool, journey []*Reference, resolve bool) []*yaml.Node { diff --git a/index/spec_index_test.go b/index/spec_index_test.go index 528a1124..ad346a1f 100644 --- a/index/spec_index_test.go +++ b/index/spec_index_test.go @@ -1062,6 +1062,68 @@ components: assert.Len(t, index.GetReferenceIndexErrors(), 1) } +func TestSpecIndex_ExtractComponentsFromRefs_SkipExternalRef_Sequential(t *testing.T) { + yml := `openapi: 3.0.0 +info: + title: Test + version: 1.0.0 +components: + schemas: + exists: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + config := CreateOpenAPIIndexConfig() + config.ExtractRefsSequentially = true + config.SkipExternalRefResolution = true + index := NewSpecIndexWithConfig(&rootNode, config) + + // Create an external reference that cannot be found locally + ref := &Reference{ + FullDefinition: "./models/pet.yaml", + Definition: "./models/pet.yaml", + } + + result := index.ExtractComponentsFromRefs(context.Background(), []*Reference{ref}) + assert.Empty(t, result, "should return no results for unresolvable external ref") + // The key assertion: no errors should be recorded because external ref errors are skipped + assert.Len(t, index.GetReferenceIndexErrors(), 0, + "should not record errors for external refs when SkipExternalRefResolution is enabled") +} + +func TestSpecIndex_ExtractComponentsFromRefs_SkipExternalRef_Async(t *testing.T) { + yml := `openapi: 3.0.0 +info: + title: Test + version: 1.0.0 +components: + schemas: + exists: + type: string` + + var rootNode yaml.Node + _ = yaml.Unmarshal([]byte(yml), &rootNode) + + config := CreateOpenAPIIndexConfig() + config.ExtractRefsSequentially = false // async mode + config.SkipExternalRefResolution = true + index := NewSpecIndexWithConfig(&rootNode, config) + + // Create an external reference that cannot be found locally + ref := &Reference{ + FullDefinition: "https://example.com/schemas/pet.yaml", + Definition: "https://example.com/schemas/pet.yaml", + } + + result := index.ExtractComponentsFromRefs(context.Background(), []*Reference{ref}) + assert.Empty(t, result, "should return no results for unresolvable external ref") + // The key assertion: no errors should be recorded because external ref errors are skipped + assert.Len(t, index.GetReferenceIndexErrors(), 0, + "should not record errors for external refs when SkipExternalRefResolution is enabled") +} + func TestSpecIndex_FindComponent_WithACrazyAssPath(t *testing.T) { yml := `paths: /crazy/ass/references: diff --git a/utils/utils.go b/utils/utils.go index 8a7afe87..b0827e5d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1122,6 +1122,12 @@ func CheckForMergeNodes(node *yaml.Node) { } } +// IsExternalRef returns true if the reference string points to an external resource +// (i.e., it is non-empty and does not start with '#'). +func IsExternalRef(ref string) bool { + return ref != "" && !strings.HasPrefix(ref, "#") +} + type RemoteURLHandler = func(url string) (*http.Response, error) // GenerateAlphanumericString creates a random alphanumeric string of length n diff --git a/utils/utils_test.go b/utils/utils_test.go index 06c0003b..c205384b 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1857,3 +1857,23 @@ func TestGetRefValueNode_EmptyNode(t *testing.T) { result := GetRefValueNode(node) assert.Nil(t, result) } + +func TestIsExternalRef_RelativeFile(t *testing.T) { + assert.True(t, IsExternalRef("./file.yaml#/Foo")) +} + +func TestIsExternalRef_AbsoluteURL(t *testing.T) { + assert.True(t, IsExternalRef("https://example.com/schema.yaml")) +} + +func TestIsExternalRef_LocalFragment(t *testing.T) { + assert.False(t, IsExternalRef("#/components/schemas/Foo")) +} + +func TestIsExternalRef_AnchorOnly(t *testing.T) { + assert.False(t, IsExternalRef("#SomeAnchor")) +} + +func TestIsExternalRef_Empty(t *testing.T) { + assert.False(t, IsExternalRef("")) +}