feat(tracing): add OpenTelemetry tracing support

audit/audit.go

@@ -23,9 +23,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"strings"
 	"sync"
+
+	"github.com/nuts-foundation/nuts-node/tracing"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -61,6 +63,15 @@ const auditLogLevel = "audit"
 var auditLoggerInstance *logrus.Logger
 var initAuditLoggerOnce = &sync.Once{}
 
+func init() {
+	// Register callback so tracing can add hooks to the audit logger.
+	// This is needed because the audit logger is a separate logrus instance,
+	// and we can't import audit from tracing due to circular dependencies.
+	tracing.RegisterAuditLogHook(func(hook logrus.Hook) {
+		auditLogger().AddHook(hook)
+	})
+}
+
 // auditLogger returns the initialized logger instance intended for audit logging.
 func auditLogger() *logrus.Logger {
 	initAuditLoggerOnce.Do(func() {
@@ -180,7 +191,7 @@ func Log(ctx context.Context, logger *logrus.Entry, eventName string) *logrus.En
 		panic("audit: eventName is empty")
 	}
 
-	return auditLogger().WithFields(logger.Data).
+	return auditLogger().WithContext(ctx).WithFields(logger.Data).
 		WithField("actor", info.Actor).
 		WithField("operation", info.Operation).
 		WithField("event", eventName)

auth/api/iam/api.go

@@ -318,7 +318,7 @@ func callbackRequestToError(request CallbackRequestObject, redirectURI *url.URL)
 	return requestErr
 }
 
-func (r Wrapper) RetrieveAccessToken(_ context.Context, request RetrieveAccessTokenRequestObject) (RetrieveAccessTokenResponseObject, error) {
+func (r Wrapper) RetrieveAccessToken(ctx context.Context, request RetrieveAccessTokenRequestObject) (RetrieveAccessTokenResponseObject, error) {
 	// get access token from store
 	var token TokenResponse
 	err := r.accessTokenClientStore().Get(request.SessionID, &token)
@@ -336,7 +336,7 @@ func (r Wrapper) RetrieveAccessToken(_ context.Context, request RetrieveAccessTo
 	// change this when tokens can be cached
 	err = r.accessTokenClientStore().Delete(request.SessionID)
 	if err != nil {
-		log.Logger().WithError(err).Warn("Failed to delete access token")
+		log.Logger().WithContext(ctx).WithError(err).Warn("Failed to delete access token")
 	}
 	// return access token
 	return RetrieveAccessToken200JSONResponse(token), nil

cmd/root.go

@@ -61,6 +61,8 @@ import (
 	"github.com/nuts-foundation/nuts-node/policy"
 	"github.com/nuts-foundation/nuts-node/storage"
 	storageCmd "github.com/nuts-foundation/nuts-node/storage/cmd"
+	"github.com/nuts-foundation/nuts-node/tracing"
+	tracingCmd "github.com/nuts-foundation/nuts-node/tracing/cmd"
 	"github.com/nuts-foundation/nuts-node/vcr"
 	openid4vciAPI "github.com/nuts-foundation/nuts-node/vcr/api/openid4vci/v0"
 	vcrAPI "github.com/nuts-foundation/nuts-node/vcr/api/vcr/v2"
@@ -224,6 +226,9 @@ func CreateSystem(shutdownCallback context.CancelFunc) *core.System {
 	system.RegisterRoutes(&discoveryServerAPI.Wrapper{Server: discoveryInstance})
 
 	// Register engines
+	// Tracing engine MUST be registered first to ensure tracing is active before other engines configure/start,
+	// and shuts down last (due to reverse shutdown order) to capture all logs/spans.
+	system.RegisterEngine(tracing.New())
 	// without dependencies
 	system.RegisterEngine(pkiInstance)
 	system.RegisterEngine(storageInstance)
@@ -340,6 +345,7 @@ func serverConfigFlags() *pflag.FlagSet {
 	set.AddFlagSet(goldenHammerCmd.FlagSet())
 	set.AddFlagSet(discoveryCmd.FlagSet())
 	set.AddFlagSet(policy.FlagSet())
+	set.AddFlagSet(tracingCmd.FlagSet())
 
 	return set
 }

cmd/root_test.go

@@ -165,7 +165,7 @@ func Test_CreateSystem(t *testing.T) {
 	system.VisitEngines(func(engine core.Engine) {
 		numEngines++
 	})
-	assert.Equal(t, 17, numEngines)
+	assert.Equal(t, 18, numEngines)
 }
 
 func Test_ClientCommand_ErrorHandlers(t *testing.T) {

core/engine.go

@@ -22,10 +22,11 @@ package core
 import (
 	"context"
 	"fmt"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/pflag"
 	"os"
 	"strings"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/pflag"
 )
 
 // Routable enables connecting a REST API to the echo server. The API wrappers should implement this interface
@@ -95,6 +96,7 @@ func (system *System) Start() error {
 }
 
 // Shutdown shuts down all engines in the system.
+// Engines are shut down in reverse order of registration.
 func (system *System) Shutdown() error {
 	var engines []Runnable
 	system.VisitEngines(func(engine Engine) {
@@ -115,24 +117,23 @@ func (system *System) Shutdown() error {
 }
 
 // Configure configures all engines in the system.
+// Engines are configured in order of registration (tracing engine should be first).
 func (system *System) Configure() error {
 	coreLogger.Debugf("Creating datadir: %s", system.Config.Datadir)
-	var err error
-	if err = os.MkdirAll(system.Config.Datadir, os.ModePerm); err != nil {
+	if err := os.MkdirAll(system.Config.Datadir, os.ModePerm); err != nil {
 		return fmt.Errorf("unable to create datadir (dir=%s): %w", system.Config.Datadir, err)
 	}
 	return system.VisitEnginesE(func(engine Engine) error {
 		// only if Engine is dynamically configurable
 		name := engineName(engine)
 		if m, ok := engine.(Configurable); ok {
 			coreLogger.Debugf("Configuring %s", name)
-			err = m.Configure(*system.Config)
+			if err := m.Configure(*system.Config); err != nil {
+				return fmt.Errorf("unable to configure %s: %w", name, err)
+			}
 			coreLogger.Debugf("Configured %s", name)
 		}
-		if err != nil {
-			err = fmt.Errorf("unable to configure %s: %w", name, err)
-		}
-		return err
+		return nil
 	})
 }
 

core/http_client.go

@@ -22,11 +22,17 @@ package core
 import (
 	"context"
 	"fmt"
-	"github.com/sirupsen/logrus"
 	"io"
 	"net/http"
+
+	"github.com/sirupsen/logrus"
 )
 
+// TracingHTTPTransport wraps an http.RoundTripper with OpenTelemetry tracing instrumentation.
+// It is set by the tracing package when tracing is enabled, and nil when disabled.
+// This callback pattern avoids circular imports between core and tracing packages.
+var TracingHTTPTransport func(http.RoundTripper) http.RoundTripper
+
 // HttpResponseBodyLogClipAt is the maximum length of a response body to log.
 // If the response body is longer than this, it will be truncated.
 const HttpResponseBodyLogClipAt = 200
@@ -98,8 +104,14 @@ func (w httpRequestDoerAdapter) Do(req *http.Request) (*http.Response, error) {
 // If the given authorization token builder is non-nil, it calls it and passes the resulting token as bearer token with requests.
 func CreateHTTPInternalClient(cfg ClientConfig, generator AuthorizationTokenGenerator) (HTTPRequestDoer, error) {
 	var result *httpRequestDoerAdapter
-	client := &http.Client{}
-	client.Timeout = cfg.Timeout
+	var transport http.RoundTripper = http.DefaultTransport
+	if TracingHTTPTransport != nil {
+		transport = TracingHTTPTransport(transport)
+	}
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   cfg.Timeout,
+	}
 
 	result = &httpRequestDoerAdapter{
 		fn: client.Do,

core/server_config.go

@@ -25,15 +25,16 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"net/url"
+	"reflect"
+	"strings"
+	"time"
+
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/posflag"
 	"github.com/knadh/koanf/v2"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
-	"net/url"
-	"reflect"
-	"strings"
-	"time"
 )
 
 const defaultConfigFile = "./config/nuts.yaml"

crypto/storage/external/client.go

@@ -30,6 +30,8 @@ import (
 	"github.com/nuts-foundation/nuts-node/core"
 	"github.com/nuts-foundation/nuts-node/crypto/storage/spi"
 	"github.com/nuts-foundation/nuts-node/crypto/util"
+	"github.com/nuts-foundation/nuts-node/tracing"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
 // StorageType is the name of this storage type, used in health check reports and configuration.
@@ -82,8 +84,21 @@ func NewAPIClient(config Config) (spi.Storage, error) {
 	if _, err := url.ParseRequestURI(config.Address); err != nil {
 		return nil, err
 	}
-	client, _ := NewClientWithResponses(config.Address, WithHTTPClient(&http.Client{Timeout: config.Timeout}))
-	return &APIClient{httpClient: client}, nil
+	var transport http.RoundTripper = http.DefaultTransport
+	if tracing.Enabled() {
+		transport = otelhttp.NewTransport(http.DefaultTransport,
+			otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string {
+				return "crypto-storage: " + r.Method + " " + r.URL.Path
+			}),
+			otelhttp.WithTracerProvider(tracing.GetTracerProvider()),
+		)
+	}
+	httpClient := &http.Client{
+		Transport: transport,
+		Timeout:   config.Timeout,
+	}
+	apiClient, _ := NewClientWithResponses(config.Address, WithHTTPClient(httpClient))
+	return &APIClient{httpClient: apiClient}, nil
 }
 
 func (c APIClient) GetPrivateKey(ctx context.Context, keyName string, _ string) (crypto.Signer, error) {

crypto/storage/vault/vault.go

@@ -23,13 +23,17 @@ import (
 	"crypto"
 	"errors"
 	"fmt"
+	"net/http"
+	"path/filepath"
+	"time"
+
 	vault "github.com/hashicorp/vault/api"
 	"github.com/nuts-foundation/nuts-node/core"
 	"github.com/nuts-foundation/nuts-node/crypto/log"
 	"github.com/nuts-foundation/nuts-node/crypto/storage/spi"
 	"github.com/nuts-foundation/nuts-node/crypto/util"
-	"path/filepath"
-	"time"
+	"github.com/nuts-foundation/nuts-node/tracing"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
 const privateKeyPathName = "nuts-private-keys"
@@ -110,6 +114,18 @@ func (v vaultKVStorage) NewPrivateKey(ctx context.Context, keyPath string) (cryp
 func configureVaultClient(cfg Config) (*vault.Client, error) {
 	vaultConfig := vault.DefaultConfig()
 	vaultConfig.Timeout = cfg.Timeout
+
+	// Add tracing if enabled
+	if tracing.Enabled() {
+		vaultConfig.HttpClient.Transport = otelhttp.NewTransport(
+			vaultConfig.HttpClient.Transport,
+			otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string {
+				return "vault: " + r.Method + " " + r.URL.Path
+			}),
+			otelhttp.WithTracerProvider(tracing.GetTracerProvider()),
+		)
+	}
+
 	client, err := vault.NewClient(vaultConfig)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize Vault client: %w", err)

docs/pages/deployment/monitoring.rst

@@ -178,6 +178,62 @@ The Nuts service executable exports the following metric namespaces:
 * ``go_`` contains Go metrics related to the process
 * ``promhttp_`` contains metrics related to HTTP calls to the Nuts node's ``/metrics`` endpoint
 
+Tracing
+*******
+
+The Nuts node supports distributed tracing via OpenTelemetry. When enabled, it exports traces to an OTLP-compatible backend
+(e.g., Jaeger, Zipkin, .NET Aspire Dashboard, Grafana Tempo).
+
+Configuration
+=============
+
+Enable tracing by configuring the OTLP endpoint:
+
+.. code-block:: yaml
+
+    tracing:
+      endpoint: localhost:4318
+
+Or via environment variables:
+
+.. code-block:: shell
+
+    NUTS_TRACING_ENDPOINT=localhost:4318
+
+Configuration options:
+
+* ``tracing.endpoint`` - OTLP HTTP endpoint (e.g., ``localhost:4318``). Tracing is disabled when empty.
+* ``tracing.insecure`` - Disable TLS for the OTLP connection (default: ``false``). Only use in trusted networks or development environments, as trace data may contain sensitive information.
+* ``tracing.servicename`` - Service name reported to the tracing backend (default: ``nuts-node``). Useful for distinguishing multiple instances in distributed tracing.
+
+What is traced
+==============
+
+The following are automatically instrumented:
+
+* **Inbound HTTP requests** - All API calls to the Nuts node create spans (except ``/health``, ``/metrics``, ``/status``)
+* **Outbound HTTP requests** - HTTP calls to external services (e.g., fetching DID documents, OAuth flows)
+* **SQL database** - Database queries via GORM
+* **Hashicorp Vault** - Key storage operations when using Vault backend
+* **Log correlation** - Log entries include ``trace_id`` and ``span_id`` fields when tracing is enabled
+* **OTLP log export** - Logs are also exported to the OTLP backend for unified observability
+
+Trace context propagation
+=========================
+
+The Nuts node uses W3C Trace Context (``traceparent`` header) for propagating trace context across service boundaries.
+When calling the Nuts node from another traced service, include the ``traceparent`` header to link spans.
+
+Known limitations
+=================
+
+The following components are not yet instrumented:
+
+* **Azure Key Vault** - Azure managed keys backend is not instrumented. The Azure SDK supports OpenTelemetry via the ``azotel`` package (see `Azure SDK tracing <https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/tracing/azotel>`_).
+* **gRPC network layer** - P2P communication between nodes (``did:nuts``) does not include tracing as it's for v5 and deprecated
+
+These limitations may be addressed in future releases.
+
 CPU profiling
 *************