Skip to content

Commit dcc9909

Browse files
authored
Opentelemetry improvements for llm tracing + hono api request tracing (#1125)
1 parent cf6817e commit dcc9909

File tree

10 files changed

+379
-217
lines changed

10 files changed

+379
-217
lines changed

deno.lock

+150-65
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

toolshed/deno.json

+14-10
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,30 @@
99
},
1010
"imports": {
1111
"@/": "./",
12-
"@ai-sdk/anthropic": "npm:@ai-sdk/anthropic@^1.1.6",
13-
"@ai-sdk/google-vertex": "npm:@ai-sdk/google-vertex@^2.1.12",
14-
"@ai-sdk/xai": "npm:@ai-sdk/xai@^1.2.13",
12+
"@ai-sdk/anthropic": "npm:@ai-sdk/anthropic@^1.2.10",
13+
"@ai-sdk/google-vertex": "npm:@ai-sdk/google-vertex@^2.2.17",
14+
"@ai-sdk/xai": "npm:@ai-sdk/xai@^1.2.15",
1515
"@std/cli": "jsr:@std/cli@^1.0.12",
1616
"gcp-metadata": "npm:gcp-metadata@6.1.0",
17-
"@ai-sdk/groq": "npm:@ai-sdk/groq@^1.1.7",
18-
"@ai-sdk/openai": "npm:@ai-sdk/openai@^1.3.16",
19-
"@arizeai/openinference-semantic-conventions": "npm:@arizeai/openinference-semantic-conventions@^1.0.0",
17+
"@ai-sdk/groq": "npm:@ai-sdk/groq@^1.2.8",
18+
"@ai-sdk/openai": "npm:@ai-sdk/openai@^1.3.20",
19+
"@opentelemetry/context-async-hooks": "npm:@opentelemetry/context-async-hooks@^1.19.0",
20+
"@opentelemetry/core": "npm:@opentelemetry/core@^1.19.0",
21+
"@opentelemetry/sdk-trace-base": "npm:@opentelemetry/sdk-trace-base@^1.19.0",
22+
"@opentelemetry/semantic-conventions": "npm:@opentelemetry/semantic-conventions@^1.19.0",
23+
"@arizeai/openinference-semantic-conventions": "npm:@arizeai/openinference-semantic-conventions@^1.1.0",
2024
"@arizeai/openinference-vercel": "npm:@arizeai/openinference-vercel@^2.0.1",
2125
"@fal-ai/client": "npm:@fal-ai/client@^1.2.2",
2226
"@hono/sentry": "npm:@hono/sentry@^1.2.0",
2327
"@hono/zod-openapi": "npm:@hono/zod-openapi@^0.18.3",
2428
"@hono/zod-validator": "npm:@hono/zod-validator@^0.4.2",
25-
"@opentelemetry/api": "npm:@opentelemetry/api@^1.9.0",
26-
"@opentelemetry/exporter-trace-otlp-proto": "npm:@opentelemetry/exporter-trace-otlp-proto@^0.57.1",
27-
"@opentelemetry/resources": "npm:@opentelemetry/resources@^1.30.1",
29+
"@opentelemetry/api": "npm:@opentelemetry/api@^1.7.0",
30+
"@opentelemetry/exporter-trace-otlp-proto": "npm:@opentelemetry/exporter-trace-otlp-proto@^0.46.0",
31+
"@opentelemetry/resources": "npm:@opentelemetry/resources@^1.19.0",
2832
"@scalar/hono-api-reference": "npm:@scalar/hono-api-reference@^0.5.165",
2933
"@sentry/deno": "npm:@sentry/deno@^9.3.0",
3034
"@vercel/otel": "npm:@vercel/otel@^1.10.1",
31-
"ai": "npm:ai@^4.3.9",
35+
"ai": "npm:ai@^4.3.10",
3236
"jsonschema": "npm:jsonschema@^1.5.0",
3337
"hono-pino": "npm:hono-pino@^0.7.0",
3438
"pino": "npm:pino@^9.6.0",

toolshed/lib/create-app.ts

+4
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { otelTracing } from "@/middlewares/opentelemetry.ts";
77
import { sentry } from "@hono/sentry";
88
import env from "@/env.ts";
99
import type { AppBindings, AppOpenAPI } from "@/lib/types.ts";
10+
import { initOpenTelemetry } from "@/lib/otel.ts";
1011

1112
export function createRouter() {
1213
return new OpenAPIHono<AppBindings>({
@@ -16,6 +17,9 @@ export function createRouter() {
1617
}
1718

1819
export default function createApp() {
20+
// Initialize OpenTelemetry before creating the app
21+
initOpenTelemetry();
22+
1923
const app = createRouter();
2024

2125
app.use("*", sentry({ dsn: env.SENTRY_DSN }));

toolshed/lib/otel.ts

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import { context, diag, trace } from "@opentelemetry/api";
2+
import {
3+
BasicTracerProvider,
4+
BatchSpanProcessor,
5+
ReadableSpan,
6+
} from "@opentelemetry/sdk-trace-base";
7+
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
8+
import { Resource } from "@opentelemetry/resources";
9+
import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks";
10+
import env from "@/env.ts";
11+
import {
12+
isOpenInferenceSpan,
13+
OpenInferenceBatchSpanProcessor,
14+
} from "@arizeai/openinference-vercel";
15+
16+
// Ensure we only register once even during hot-reload
17+
let _providerRegistered = false;
18+
let _provider: BasicTracerProvider | undefined;
19+
20+
export const otlpExporter = new OTLPTraceExporter({
21+
url: env.OTEL_EXPORTER_OTLP_ENDPOINT
22+
? `${env.OTEL_EXPORTER_OTLP_ENDPOINT.replace(/\/$/, "")}/v1/traces`
23+
: "http://localhost:4318/v1/traces",
24+
});
25+
26+
export const provider = new BasicTracerProvider({
27+
resource: new Resource({
28+
"service.name": env.OTEL_SERVICE_NAME || "toolshed-dev",
29+
"service.version": "1.0.0",
30+
"deployment.environment": env.ENV || "development",
31+
"openinference.project.name": env.CTTS_AI_LLM_PHOENIX_PROJECT,
32+
}),
33+
spanProcessors: [
34+
new OpenInferenceBatchSpanProcessor({
35+
exporter: otlpExporter,
36+
spanFilter: (span) => {
37+
return isOpenInferenceSpan(span);
38+
},
39+
}),
40+
],
41+
});
42+
43+
export function getTracerProvider() {
44+
return _provider;
45+
}
46+
47+
// Prefer Deno's built-in context manager when running on Deno ≥2.2.
48+
// It properly hooks into the runtime's AsyncContext implementation so
49+
// tracing context survives across *all* async boundaries.
50+
// Falls back to the Node/async-hooks based manager when not available
51+
// (eg. unit tests executed under Node).
52+
const getContextManager = () => {
53+
try {
54+
// deno-lint-ignore no-explicit-any
55+
const cm = (globalThis as any)?.Deno?.telemetry?.contextManager;
56+
if (cm && typeof cm.enable === "function") {
57+
diag.debug("Using Deno's built-in telemetry context manager");
58+
return cm;
59+
}
60+
} catch (_) {
61+
// ignored – not running on Deno with telemetry support
62+
}
63+
diag.debug("Falling back to AsyncHooksContextManager");
64+
return new AsyncHooksContextManager();
65+
};
66+
67+
export function initOpenTelemetry() {
68+
if (_providerRegistered || !env.OTEL_ENABLED) {
69+
if (!env.OTEL_ENABLED) {
70+
console.log("OpenTelemetry is disabled via OTEL_ENABLED env var");
71+
} else {
72+
console.log("OpenTelemetry already initialized, skipping");
73+
}
74+
return;
75+
}
76+
77+
try {
78+
// Set up context manager
79+
const contextManager = getContextManager();
80+
context.setGlobalContextManager(contextManager.enable());
81+
82+
// Register provider globally
83+
trace.setGlobalTracerProvider(provider);
84+
_provider = provider;
85+
_providerRegistered = true;
86+
87+
console.log(
88+
`OpenTelemetry initialized successfully with endpoint: ${env.OTEL_EXPORTER_OTLP_ENDPOINT}`,
89+
);
90+
91+
diag.debug("OpenTelemetry configuration details:", {
92+
exporter: env.OTEL_EXPORTER_OTLP_ENDPOINT,
93+
service: env.OTEL_SERVICE_NAME || "toolshed-dev",
94+
environment: env.ENV || "development",
95+
});
96+
} catch (error) {
97+
console.error("Failed to initialize OpenTelemetry:", error);
98+
// Don't crash the app if telemetry fails
99+
}
100+
}

toolshed/llm_documentation/full-idea.md

-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,6 @@ toolshed/
9191
│ │ └── img.test.ts # Test cases
9292
│ │ └── llm/ # Language model endpoints
9393
│ │ ├── cache.ts # llm-specific caching
94-
│ │ ├── instrumentation.ts # Telemetry
9594
│ │ ├── llm.handlers.ts # Request handlers
9695
│ │ ├── llm.index.ts # Route definitions
9796
│ │ ├── llm.routes.ts # Route schemas

toolshed/middlewares/opentelemetry.ts

+78-78
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
import { Context, MiddlewareHandler } from "@hono/hono";
22
import { context, Span, SpanStatusCode, trace } from "@opentelemetry/api";
3+
import { getTracerProvider } from "@/lib/otel.ts";
34

4-
const tracer = trace.getTracer("toolshed-middleware", "1.0.0");
5+
// Dynamically resolve the tracer so we don't capture the no-op global tracer
6+
const obtainTracer = () => {
7+
const provider = getTracerProvider();
8+
return provider
9+
? provider.getTracer("toolshed-middleware", "1.0.0")
10+
: trace.getTracer("toolshed-middleware", "1.0.0");
11+
};
512

613
export interface OtelConfig {
714
/**
@@ -29,95 +36,88 @@ export function otelTracing(config: OtelConfig = {}): MiddlewareHandler {
2936
return async (c, next) => {
3037
const path = c.req.path;
3138
const method = c.req.method;
39+
const route = c.req.routePath || path;
3240

33-
// Create and configure span
34-
const span = tracer.startSpan(`${method} ${path}`);
35-
span.setAttribute("http.method", method);
36-
span.setAttribute("http.route", path);
37-
span.setAttribute("http.target", path);
38-
span.setAttribute("http.host", c.req.header("host") || "unknown");
39-
span.setAttribute(
40-
"http.user_agent",
41-
c.req.header("user-agent") || "unknown",
42-
);
41+
await obtainTracer().startActiveSpan(`${method} ${path}`, async (span) => {
42+
span.setAttribute("http.method", method);
43+
span.setAttribute("http.route", path + c.req.routePath);
44+
span.setAttribute("http.host", c.req.header("host") || "unknown");
45+
span.setAttribute(
46+
"http.user_agent",
47+
c.req.header("user-agent") || "unknown",
48+
);
4349

44-
// Add request ID if it exists in headers
45-
const requestId = c.req.header("x-request-id");
46-
if (requestId) {
47-
span.setAttribute("http.request_id", requestId);
48-
}
49-
50-
// Add custom attributes if configured
51-
if (config.additionalAttributes) {
52-
Object.entries(config.additionalAttributes).forEach(([key, value]) => {
53-
span.setAttribute(key, value);
54-
});
55-
}
56-
57-
// Include request body if configured
58-
if (config.includeRequestBody) {
59-
try {
60-
const bodyClone = c.req.raw.clone();
61-
const body = await bodyClone.text();
62-
if (body) {
63-
span.setAttribute("http.request.body", body);
64-
}
65-
} catch (e) {
66-
// Ignore if body can't be parsed
50+
// Add request ID if it exists in headers
51+
const requestId = c.req.header("x-request-id");
52+
if (requestId) {
53+
span.setAttribute("http.request_id", requestId);
6754
}
68-
}
6955

70-
try {
71-
// Set the span in context for nested spans
72-
await context.with(trace.setSpan(context.active(), span), async () => {
73-
// Need to capture response status after next() is called
74-
await next();
75-
});
76-
77-
// Capture status code from response if available
78-
if (c.res?.status) {
79-
span.setAttribute("http.status_code", c.res.status);
56+
// Add custom attributes if configured
57+
if (config.additionalAttributes) {
58+
Object.entries(config.additionalAttributes).forEach(([key, value]) => {
59+
span.setAttribute(key, value);
60+
});
8061
}
8162

82-
// Include response body if configured
83-
if (config.includeResponseBody && c.res?.body) {
63+
// Include request body if configured
64+
if (config.includeRequestBody) {
8465
try {
85-
// Try to get the response body content
86-
const clonedResponse = c.res.clone();
87-
const text = await clonedResponse.text();
88-
if (text) {
89-
span.setAttribute("http.response.body", text);
66+
const bodyClone = c.req.raw.clone();
67+
const body = await bodyClone.text();
68+
if (body) {
69+
span.setAttribute("http.request.body", body);
9070
}
91-
} catch (e) {
92-
// Ignore if body can't be accessed
71+
} catch (_) {
72+
/* swallow */
9373
}
9474
}
95-
} catch (error) {
96-
// Handle errors
97-
span.setAttribute("error", true);
98-
span.setAttribute(
99-
"error.message",
100-
error instanceof Error ? error.message : String(error),
101-
);
102-
span.setAttribute(
103-
"error.type",
104-
error instanceof Error ? error.name : "UnknownError",
105-
);
106-
span.setStatus({
107-
code: SpanStatusCode.ERROR,
108-
message: error instanceof Error ? error.message : String(error),
109-
});
11075

111-
if (error instanceof Error && error.stack) {
112-
span.setAttribute("error.stack", error.stack);
113-
}
76+
try {
77+
// Execute the downstream handlers while this span is active
78+
await next();
11479

115-
span.end();
116-
throw error;
117-
}
80+
// Capture status code from response if available
81+
if (c.res?.status) {
82+
span.setAttribute("http.status_code", c.res.status);
83+
}
11884

119-
// End the span
120-
span.end();
85+
// Include response body if configured
86+
if (config.includeResponseBody && c.res?.body) {
87+
try {
88+
const clonedResponse = c.res.clone();
89+
const text = await clonedResponse.text();
90+
if (text) {
91+
span.setAttribute("http.response.body", text);
92+
}
93+
} catch (_) {
94+
/* swallow */
95+
}
96+
}
97+
} catch (error) {
98+
span.setAttribute("error", true);
99+
span.setAttribute(
100+
"error.message",
101+
error instanceof Error ? error.message : String(error),
102+
);
103+
span.setAttribute(
104+
"error.type",
105+
error instanceof Error ? error.name : "UnknownError",
106+
);
107+
span.setStatus({
108+
code: SpanStatusCode.ERROR,
109+
message: error instanceof Error ? error.message : String(error),
110+
});
111+
112+
if (error instanceof Error && error.stack) {
113+
span.setAttribute("error.stack", error.stack);
114+
}
115+
116+
throw error;
117+
} finally {
118+
span.end();
119+
}
120+
});
121121
};
122122
}
123123

@@ -137,7 +137,7 @@ export async function createSpan<T>(
137137
attributes: Record<string, string | number | boolean> = {},
138138
): Promise<T> {
139139
const parentSpan = getCurrentSpan();
140-
const span = tracer.startSpan(
140+
const span = obtainTracer().startSpan(
141141
name,
142142
undefined,
143143
parentSpan ? trace.setSpan(context.active(), parentSpan) : undefined,

0 commit comments

Comments
 (0)