diff --git a/.tools/run_node_tests.sh b/.tools/run_node_tests.sh index 66be72dd..c485eaf5 100755 --- a/.tools/run_node_tests.sh +++ b/.tools/run_node_tests.sh @@ -10,6 +10,7 @@ function npm_install_check() { } npm_install_check $PROJECT_ROOT/typescript/basics +npm_install_check $PROJECT_ROOT/typescript/tracing npm_install_check $PROJECT_ROOT/typescript/templates/node npm_install_check $PROJECT_ROOT/typescript/templates/lambda diff --git a/typescript/tracing/otel/.gitignore b/typescript/tracing/otel/.gitignore new file mode 100644 index 00000000..39895742 --- /dev/null +++ b/typescript/tracing/otel/.gitignore @@ -0,0 +1,19 @@ +# Node +node_modules +dist + +# screenshots +*.png + +# debug +npm-debug.log* + +# env files +.env* + +# typescript +*.tsbuildinfo + +# Restate +.restate +restate-data diff --git a/typescript/tracing/otel/README.md b/typescript/tracing/otel/README.md new file mode 100644 index 00000000..e5eeecf2 --- /dev/null +++ b/typescript/tracing/otel/README.md @@ -0,0 +1,135 @@ +# End-to-End OpenTelemetry Tracing with Restate + +This example demonstrates distributed tracing across a fictional multi-tier system: + +``` +┌──────────┐ ┌─────────────┐ ┌─────────────────┐ ┌────────────┐ +│ Client │────▶│ Restate │────▶│ Greeter Service │────▶│ Downstream │ +│ App │ │ Server │ │ (SDK/Node) │ │ Service │ +└──────────┘ └─────────────┘ └─────────────────┘ └────────────┘ + │ │ │ │ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌────────────────────────────────────────────────────────────────────────┐ +│ Jaeger │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +**What gets traced:** + +1. **Client App** - Creates the root span and injects W3C trace context into the Restate request +2. **Restate Server** - Receives trace context, emits spans for ingress requests and handler invocations +3. **Greeter Service** - SDK handler that creates custom spans and propagates context to downstream calls +4. **Downstream Service** - Receives and logs the propagated trace headers + +## Prerequisites + +- Node.js 18+ +- Docker (for Jaeger) + +## Setup + +### 1. Start Jaeger + +```bash +docker run -d --name jaeger \ + -p 4317:4317 \ + -p 16686:16686 \ + jaegertracing/all-in-one:latest +``` + +Jaeger UI will be available at `http://localhost:16686` + +### 2. Install dependencies + +```bash +npm install +``` + +### 3. Start Restate Server with tracing enabled + +```bash +npx @restatedev/restate-server --tracing-endpoint http://localhost:4317 +``` + +### 4. Start the downstream service (terminal 1) + +```bash +npm run downstream +``` + +### 5. Start the Greeter service (terminal 2) + +```bash +npm run service +``` + +### 6. Register the service with Restate + +```bash +npx @restatedev/restate deployments register http://localhost:9080 +``` + +### 7. Run the client + +```bash +npm run client Alice +``` + +## Viewing Traces + +After running the client, you'll see output like: + +``` +Root Trace ID: abc123... +View in Jaeger: `http://localhost:16686/trace/abc123...` +``` + +Open the Jaeger link to see the complete distributed trace spanning all four components. + +## What You'll See in Jaeger + +The trace will show spans from all four services: + +- **client-app**: The root `client-request` span +- **Greeter**: Restate server spans for ingress, invoke, and journal operations +- **restate-greeter-service**: Custom `Greeter.greet` span with events +- **downstream-service**: `handle-request` span (may show errors due to 50% failure rate) + +## Key Pattern: Extracting Trace Context in TypeScript SDK + +The Restate server propagates W3C trace context to handlers via HTTP headers. In the TypeScript SDK, you need to manually extract this from `ctx.request().attemptHeaders`. + +**Why not use Node.js auto-instrumentation?** Unlike Java and Go, Node.js does have OTEL auto-instrumentation packages (e.g. `@opentelemetry/auto-instrumentations-node`). However, they operate at the raw HTTP transport layer, which Restate wraps internally. More importantly, Restate provides durable execution — a handler may be invoked multiple times due to retries. Extracting trace context from `ctx.request().attemptHeaders` ensures exactly one span per logical invocation, correctly positioned in the trace hierarchy regardless of retries. + +```typescript +import { context, propagation, type Context } from "@opentelemetry/api"; + +function extractTraceContext(ctx: restate.Context): Context { + const headers = ctx.request().attemptHeaders; + // TextMapGetter lets any propagator format (W3C, B3, Jaeger…) work automatically + return propagation.extract(context.active(), headers, { + get: (carrier, key) => { + const val = carrier.get(key); + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => [...carrier.keys()], + }); +} +``` + +Then run your handler logic within that context: + +```typescript +const traceContext = extractTraceContext(ctx); +return context.with(traceContext, () => { + const span = tracer.startSpan("MyHandler"); + // ... your logic here, span is now a child of Restate's span +}); +``` + +## Files + +- `src/client.ts` - Client app that initiates traced requests +- `src/restate-service.ts` - Restate Greeter service with OpenTelemetry instrumentation +- `src/downstream.ts` - HTTP server with tracing and random failure rate diff --git a/typescript/tracing/otel/package.json b/typescript/tracing/otel/package.json new file mode 100644 index 00000000..ebbe23b4 --- /dev/null +++ b/typescript/tracing/otel/package.json @@ -0,0 +1,28 @@ +{ + "name": "@restatedev/examples-tracing", + "version": "0.0.1", + "description": "End-to-end OpenTelemetry tracing with Restate", + "license": "MIT", + "author": "Restate developers", + "email": "code@restate.dev", + "type": "commonjs", + "scripts": { + "build": "tsc --noEmitOnError", + "service": "tsx ./src/restate-service.ts", + "client": "tsx ./src/client.ts", + "downstream": "tsx ./src/downstream.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-node": "^0.57.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@restatedev/restate-sdk": "^1.10.2" + }, + "devDependencies": { + "@types/node": "^20.14.2", + "tsx": "^4.19.2", + "typescript": "^5.4.5" + } +} diff --git a/typescript/tracing/otel/src/client.ts b/typescript/tracing/otel/src/client.ts new file mode 100644 index 00000000..73e3447b --- /dev/null +++ b/typescript/tracing/otel/src/client.ts @@ -0,0 +1,91 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "client-app", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const RESTATE_INGRESS = "http://localhost:8080"; +const tracer = trace.getTracer("client-app"); + +async function main() { + const name = process.argv[2] || "World"; + + console.log("=== Client App ==="); + console.log(`Calling Restate Greeter service with name: ${name}`); + + // Create the root span for this request + const rootSpan = tracer.startSpan("client-request", { + kind: SpanKind.CLIENT, + attributes: { + "request.name": name, + }, + }); + + try { + const result = await context.with( + trace.setSpan(context.active(), rootSpan), + async () => { + const headers: Record = { + "Content-Type": "application/json", + }; + + propagation.inject(context.active(), headers); + console.log(`Injected W3C trace context headers:`, headers); + + const traceId = rootSpan.spanContext().traceId; + console.log(`Root Trace ID: ${traceId}`); + console.log(`View in Jaeger: http://localhost:16686/trace/${traceId}`); + console.log(""); + + const response = await fetch(`${RESTATE_INGRESS}/Greeter/greet`, { + method: "POST", + headers, + body: JSON.stringify(name), + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${await response.text()}`); + } + + return response.json(); + }, + ); + + rootSpan.addEvent("response_received", { + "response.value": JSON.stringify(result), + }); + rootSpan.setStatus({ code: SpanStatusCode.OK }); + + console.log(`Response: ${JSON.stringify(result)}`); + } catch (err) { + rootSpan.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : "Unknown error", + }); + console.error("Error:", err); + process.exitCode = 1; + } finally { + rootSpan.end(); + await sdk.shutdown(); + } +} + +main(); diff --git a/typescript/tracing/otel/src/downstream.ts b/typescript/tracing/otel/src/downstream.ts new file mode 100644 index 00000000..4074a631 --- /dev/null +++ b/typescript/tracing/otel/src/downstream.ts @@ -0,0 +1,93 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, +} from "@opentelemetry/api"; +import { createServer } from "node:http"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "downstream-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +const PORT = 3000; +const FAILURE_RATE = 0.5; // 50% chance + +const tracer = trace.getTracer("downstream-service"); + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const server = createServer((req, res) => { + // Extract trace context from incoming HTTP headers + const traceContext = propagation.extract(context.active(), req.headers, { + get: (carrier, key) => { + const val = carrier[key]; + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => Object.keys(carrier), + }); + + // Run request handling within the extracted trace context + context.with(traceContext, async () => { + const span = tracer.startSpan("handle-request", { + kind: SpanKind.SERVER, + attributes: { + "http.method": req.method, + "http.url": req.url, + }, + }); + + try { + // Simulate some work + await sleep(50 + Math.random() * 100); + + // Random failure + if (Math.random() < FAILURE_RATE) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: "Random failure", + }); + span.addEvent("failure_triggered", { rate: FAILURE_RATE }); + + res.writeHead(500, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + error: "Random failure", + receivedTrace: !!req.headers["traceparent"], + }), + ); + return; + } + + span.addEvent("processing_complete"); + span.setStatus({ code: SpanStatusCode.OK }); + + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ status: "ok", receivedTrace: !!req.headers["traceparent"] })); + } finally { + span.end(); + } + }); +}); + +server.listen(PORT, () => { + console.log(`Downstream service listening on http://localhost:${PORT}`); + console.log(`Failure rate: ${FAILURE_RATE * 100}%`); +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/tracing/otel/src/restate-service.ts b/typescript/tracing/otel/src/restate-service.ts new file mode 100644 index 00000000..0f61feb6 --- /dev/null +++ b/typescript/tracing/otel/src/restate-service.ts @@ -0,0 +1,130 @@ +// OpenTelemetry must be initialized before other imports +import { NodeSDK } from "@opentelemetry/sdk-node"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc"; +import { Resource } from "@opentelemetry/resources"; +import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; +import { + trace, + context, + propagation, + SpanKind, + SpanStatusCode, + type Context, +} from "@opentelemetry/api"; + +const sdk = new NodeSDK({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: "restate-greeter-service", + }), + traceExporter: new OTLPTraceExporter({ + url: "http://localhost:4317", + }), +}); + +sdk.start(); + +import * as restate from "@restatedev/restate-sdk"; + +const DOWNSTREAM_URL = "http://localhost:3000/api/process"; + +const tracer = trace.getTracer("greeter-service"); + +// Extract trace context propagated by Restate via attempt headers. +// +// Unlike Java/Go, Node.js does have OTEL auto-instrumentation packages, but they +// operate at the raw HTTP transport level. Restate wraps the HTTP layer and provides +// durable execution semantics — a handler may be replayed multiple times. Extracting +// from ctx.request().attemptHeaders ensures one span per logical invocation, +// correctly positioned in the trace hierarchy regardless of retries. +function extractTraceContext(ctx: restate.Context): Context { + const headers = ctx.request().attemptHeaders; + // Use a TextMapGetter so any propagator format (W3C, B3, Jaeger…) is supported + return propagation.extract(context.active(), headers, { + get: (carrier, key) => { + const val = carrier.get(key); + return Array.isArray(val) ? val[0] : (val ?? undefined); + }, + keys: (carrier) => [...carrier.keys()], + }); +} + +const greeter = restate.service({ + name: "Greeter", + handlers: { + greet: async (ctx: restate.Context, name: string): Promise => { + const traceContext = extractTraceContext(ctx); + + // Create span under the extracted trace context + const span = tracer.startSpan( + "Greeter.greet", + { kind: SpanKind.INTERNAL, attributes: { "greeter.name": name } }, + traceContext, + ); + + // Create context with our span as parent for downstream calls + const spanContext = trace.setSpan(traceContext, span); + + return context.with(spanContext, () => { + return (async () => { + try { + span.addEvent("processing_started", { name }); + + const greeting = `Hello, ${name}!`; + + // Call downstream - our span becomes the parent + const downstreamResult = await ctx.run("call-downstream", () => + callDownstreamWithTrace(name, spanContext), + ); + + span.addEvent("downstream_completed", { + "downstream.result": JSON.stringify(downstreamResult), + }); + + span.setStatus({ code: SpanStatusCode.OK }); + return greeting; + } catch (err) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: err instanceof Error ? err.message : "Unknown error", + }); + throw err; + } finally { + span.end(); + } + })(); + }); + }, + }, +}); + +async function callDownstreamWithTrace( + name: string, + traceContext: Context, +): Promise<{ status: string; receivedTrace: boolean }> { + const headers: Record = { + "Content-Type": "application/json", + }; + propagation.inject(traceContext, headers); + + const response = await fetch(DOWNSTREAM_URL, { + method: "POST", + headers, + body: JSON.stringify({ name }), + }); + + if (!response.ok) { + const body = (await response.json()) as { error?: string }; + throw new Error(`Downstream failed: ${body.error ?? response.statusText}`); + } + + return response.json() as Promise<{ status: string; receivedTrace: boolean }>; +} + +restate.serve({ + services: [greeter], + port: 9080, +}); + +process.on("SIGTERM", () => { + sdk.shutdown().then(() => process.exit(0)); +}); diff --git a/typescript/tracing/otel/tsconfig.json b/typescript/tracing/otel/tsconfig.json new file mode 100644 index 00000000..c2946b24 --- /dev/null +++ b/typescript/tracing/otel/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "module": "nodenext", + "allowJs": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true + } +} diff --git a/typescript/tracing/package.json b/typescript/tracing/package.json new file mode 100644 index 00000000..ebbe23b4 --- /dev/null +++ b/typescript/tracing/package.json @@ -0,0 +1,28 @@ +{ + "name": "@restatedev/examples-tracing", + "version": "0.0.1", + "description": "End-to-end OpenTelemetry tracing with Restate", + "license": "MIT", + "author": "Restate developers", + "email": "code@restate.dev", + "type": "commonjs", + "scripts": { + "build": "tsc --noEmitOnError", + "service": "tsx ./src/restate-service.ts", + "client": "tsx ./src/client.ts", + "downstream": "tsx ./src/downstream.ts" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.57.0", + "@opentelemetry/resources": "^1.30.0", + "@opentelemetry/sdk-node": "^0.57.0", + "@opentelemetry/semantic-conventions": "^1.28.0", + "@restatedev/restate-sdk": "^1.10.2" + }, + "devDependencies": { + "@types/node": "^20.14.2", + "tsx": "^4.19.2", + "typescript": "^5.4.5" + } +} diff --git a/typescript/tracing/tsconfig.json b/typescript/tracing/tsconfig.json new file mode 100644 index 00000000..c2946b24 --- /dev/null +++ b/typescript/tracing/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "lib": ["esnext"], + "module": "nodenext", + "allowJs": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipDefaultLibCheck": true, + "skipLibCheck": true + } +}