Skip to content

Commit 540c798

Browse files
andreiborzaclaudechargome
authored
feat(server-utils): Record LangChain TypeSafeClassifier runs as gen_ai.evaluate spans (#25120)
## What `TypeSafeClassifier` runs from `@langchain/typesafe` now show up as `gen_ai.evaluate` spans with model, provider, token usage, and the state, questions and answers when gen_ai recording is on. - Standalone `classifier.invoke()` calls are traced automatically. - Calls from inside agents (for example the TypeSafe model router and auto-mode middleware) are traced too. ## Why The classifier calls Jev with `fetch` directly, so the TypeSafe integration did not see it, and LangChain reported it as a generic `invoke_agent` span without any evaluation data. Closes: #24859 --------- Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: Charly Gomez <[email protected]>
1 parent de43b56 commit 540c798

11 files changed

Lines changed: 393 additions & 15 deletions

File tree

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
2+
import { RunnableLambda } from '@langchain/core/runnables';
3+
import { TypeSafeClassifier } from '@langchain/typesafe';
4+
// Sets up the AsyncLocalStorage that passes a parent run's config to its children, as it is inside `createAgent`.
5+
import '@langchain/langgraph';
6+
import * as Sentry from '@sentry/node';
7+
import express from 'express';
8+
9+
function startMockTypeSafeServer() {
10+
const app = express();
11+
app.use(express.json());
12+
13+
app.post('/v1/systemone', (req, res) => {
14+
res.json({
15+
model: 'jev-1.13',
16+
answers: { urgent: { type: 'noul', noul: 0.9 } },
17+
usage: { input_tokens: 30, output_tokens: 2 },
18+
});
19+
});
20+
21+
return new Promise(resolve => {
22+
const server = app.listen(0, () => {
23+
resolve(server);
24+
});
25+
});
26+
}
27+
28+
// Stands in for a tracer such as LangSmith, which the user passes to the parent run.
29+
class RecordingHandler extends BaseCallbackHandler {
30+
name = 'RecordingHandler';
31+
runs = [];
32+
33+
handleChainStart(chain, _inputs, _runId, parentRunId) {
34+
this.runs.push(`${chain.id.at(-1)}:${parentRunId ? 'child' : 'root'}`);
35+
}
36+
}
37+
38+
async function run() {
39+
const server = await startMockTypeSafeServer();
40+
const baseUrl = `http://localhost:${server.address().port}`;
41+
42+
await Sentry.startSpan({ op: 'function', name: 'main' }, async span => {
43+
const classifier = new TypeSafeClassifier({
44+
apiKey: 'mock-api-key',
45+
baseUrl,
46+
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
47+
});
48+
49+
// Like the `@langchain/typesafe` middlewares, call the classifier without a config.
50+
const parent = RunnableLambda.from(input => classifier.invoke(input)).withConfig({ runName: 'parent' });
51+
const recorder = new RecordingHandler();
52+
await parent.invoke('My payouts have been failing.', { callbacks: [recorder] });
53+
54+
span.setAttribute('test.recorded_runs', recorder.runs.join(','));
55+
});
56+
57+
await Sentry.flush(2000);
58+
server.close();
59+
}
60+
61+
run();
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { HumanMessage } from '@langchain/core/messages';
2+
import { TypeSafeClassifier } from '@langchain/typesafe';
3+
import * as Sentry from '@sentry/node';
4+
import express from 'express';
5+
6+
function startMockTypeSafeServer() {
7+
const app = express();
8+
app.use(express.json());
9+
10+
app.post('/v1/systemone', (req, res) => {
11+
res.json({
12+
model: 'jev-1.13',
13+
answers: { urgent: { type: 'noul', noul: 0.9 } },
14+
usage: { input_tokens: 30, output_tokens: 2 },
15+
});
16+
});
17+
18+
return new Promise(resolve => {
19+
const server = app.listen(0, () => {
20+
resolve(server);
21+
});
22+
});
23+
}
24+
25+
async function run() {
26+
const server = await startMockTypeSafeServer();
27+
const baseUrl = `http://localhost:${server.address().port}`;
28+
29+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
30+
const classifier = new TypeSafeClassifier({
31+
apiKey: 'mock-api-key',
32+
baseUrl,
33+
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
34+
});
35+
36+
await classifier.invoke('My payouts have been failing.');
37+
await classifier.invoke(new HumanMessage('My card was charged twice.'));
38+
});
39+
40+
server.close();
41+
}
42+
43+
run();

‎dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import {
33
GEN_AI_CONVERSATION_ID,
44
GEN_AI_INPUT_MESSAGES,
55
GEN_AI_OPERATION_NAME,
6+
GEN_AI_OUTPUT_MESSAGES,
67
GEN_AI_PROVIDER_NAME,
78
GEN_AI_REQUEST_MAX_TOKENS,
89
GEN_AI_REQUEST_MODEL,
@@ -18,6 +19,7 @@ import {
1819
SENTRY_OP,
1920
SENTRY_ORIGIN,
2021
} from '@sentry/conventions/attributes';
22+
import { GEN_AI_EVALUATE } from '@sentry/conventions/op';
2123
import { GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE } from '../../../../../../packages/server-utils/src/ai/core/gen-ai-attributes';
2224
import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner';
2325
import { createEsmTests } from '../../../../utils/runner/createEsmAndCjsTests';
@@ -233,6 +235,88 @@ describe('LangChain integration (v1)', () => {
233235
},
234236
);
235237

238+
createEsmAndCjsTests(
239+
__dirname,
240+
'scenario-typesafe-classifier.mjs',
241+
'instrument-with-pii.mjs',
242+
(createRunner, test) => {
243+
test('records a TypeSafeClassifier run as a gen_ai.evaluate span', async () => {
244+
const runner = createRunner().ignore('event');
245+
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
246+
247+
await runner.start().completed();
248+
249+
const spans = (await spansPromise).filter(
250+
span => span.attributes[SENTRY_ORIGIN]?.value === 'auto.ai.langchain',
251+
);
252+
expect(spans.map(span => span.name)).toEqual(['evaluate jev-latest', 'evaluate jev-latest']);
253+
254+
const [evaluateSpan, messageEvaluateSpan] = spans.sort((a, b) => a.start_timestamp - b.start_timestamp);
255+
expect(evaluateSpan.attributes[SENTRY_OP].value).toBe(GEN_AI_EVALUATE);
256+
expect(evaluateSpan.attributes[SENTRY_ORIGIN].value).toBe('auto.ai.langchain');
257+
expect(evaluateSpan.attributes[GEN_AI_OPERATION_NAME].value).toBe('evaluate');
258+
expect(evaluateSpan.attributes[GEN_AI_PROVIDER_NAME].value).toBe('typesafe');
259+
expect(evaluateSpan.attributes[GEN_AI_REQUEST_MODEL].value).toBe('jev-latest');
260+
expect(evaluateSpan.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('jev-1.13');
261+
expect(evaluateSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(30);
262+
expect(evaluateSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(2);
263+
expect(evaluateSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(32);
264+
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
265+
{
266+
type: 'evaluation',
267+
state: 'My payouts have been failing.',
268+
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
269+
},
270+
]);
271+
// A message is recorded as the transcript line the classifier sends, not as LangChain's serialized form.
272+
expect(JSON.parse(messageEvaluateSpan!.attributes[GEN_AI_INPUT_MESSAGES].value)).toEqual([
273+
{
274+
type: 'evaluation',
275+
state: 'user: My card was charged twice.',
276+
questions: { urgent: { type: 'noul', instructions: 'Is this urgent?' } },
277+
},
278+
]);
279+
expect(JSON.parse(evaluateSpan.attributes[GEN_AI_OUTPUT_MESSAGES].value)).toEqual([
280+
{ type: 'evaluation', answers: { urgent: { type: 'noul', noul: 0.9 } } },
281+
]);
282+
});
283+
},
284+
{
285+
additionalDependencies: {
286+
langchain: '^1.0.0',
287+
'@langchain/core': '^1.0.0',
288+
'@langchain/typesafe': '^0.0.2',
289+
},
290+
},
291+
);
292+
293+
createEsmAndCjsTests(
294+
__dirname,
295+
'scenario-typesafe-classifier-inherited-callbacks.mjs',
296+
'instrument-with-pii.mjs',
297+
(createRunner, test) => {
298+
test('keeps the parent run callbacks for a TypeSafeClassifier called without config', async () => {
299+
const runner = createRunner().ignore('event');
300+
const spansPromise = runner.collectStreamedSpansUntilSegment('main');
301+
302+
await runner.start().completed();
303+
304+
const spans = await spansPromise;
305+
const segment = spans.find(span => span.is_segment && span.name === 'main');
306+
expect(segment!.attributes['test.recorded_runs'].value).toBe('RunnableLambda:root,TypeSafeClassifier:child');
307+
expect(spans.filter(span => span.attributes[SENTRY_OP]?.value === GEN_AI_EVALUATE)).toHaveLength(1);
308+
});
309+
},
310+
{
311+
additionalDependencies: {
312+
langchain: '^1.0.0',
313+
'@langchain/core': '^1.0.0',
314+
'@langchain/typesafe': '^0.0.2',
315+
'@langchain/langgraph': '^1.0.0',
316+
},
317+
},
318+
);
319+
236320
createEsmTests(
237321
__dirname,
238322
'scenario-openai-before-langchain.mjs',

‎packages/server-utils/src/ai/langchain/index.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ import {
1515
import { GEN_AI_CHAT, GEN_AI_EXECUTE_TOOL, GEN_AI_INVOKE_AGENT } from '@sentry/conventions/op';
1616
import { resolveAIRecordingOptions } from '../core/utils';
1717
import { LANGCHAIN_ORIGIN } from './constants';
18+
import {
19+
addTypeSafeClassifierResponseAttributes,
20+
isTypeSafeClassifier,
21+
startTypeSafeClassifierSpan,
22+
} from './typesafe-classifier';
1823
import type {
1924
LangChainCallbackHandler,
2025
LangChainLLMResult,
@@ -43,6 +48,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
4348

4449
// Internal state - single instance tracks all spans
4550
const spanMap = new Map<string, Span>();
51+
const evaluateRunIds = new Set<string>();
4652

4753
/**
4854
* Exit a span and clean up
@@ -51,8 +57,9 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
5157
const span = spanMap.get(runId);
5258
if (span?.isRecording()) {
5359
span.end();
54-
spanMap.delete(runId);
5560
}
61+
spanMap.delete(runId);
62+
evaluateRunIds.delete(runId);
5663
};
5764

5865
/**
@@ -215,6 +222,13 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
215222
_runType?: string,
216223
runName?: string,
217224
) {
225+
// A Jev call is a real model call, so it is recorded inside an agent too.
226+
if (isTypeSafeClassifier(chain)) {
227+
spanMap.set(runId, startTypeSafeClassifierSpan(chain, inputs, metadata, recordInputs));
228+
evaluateRunIds.add(runId);
229+
return;
230+
}
231+
218232
// Skip chain spans when inside an agent context (createReactAgent).
219233
// The agent already creates an invoke_agent span; internal chain steps
220234
// (ChannelWrite, Branch, prompt, etc.) are noise.
@@ -264,14 +278,16 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
264278
handleChainEnd(outputs: unknown, runId: string) {
265279
const span = spanMap.get(runId);
266280
if (span?.isRecording()) {
267-
// Add outputs if recordOutputs is enabled
268-
if (recordOutputs) {
281+
if (evaluateRunIds.has(runId)) {
282+
addTypeSafeClassifierResponseAttributes(span, outputs, recordOutputs);
283+
} else if (recordOutputs) {
269284
span.setAttributes({
270285
'langchain.chain.outputs': JSON.stringify(outputs),
271286
});
272287
}
273-
exitSpan(runId);
274288
}
289+
// Also for a sampled-out span, which would otherwise stay tracked.
290+
exitSpan(runId);
275291
},
276292

277293
// Chain Error Handler
@@ -281,8 +297,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}):
281297
const span = spanMap.get(runId);
282298
if (span?.isRecording()) {
283299
span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' });
284-
exitSpan(runId);
285300
}
301+
exitSpan(runId);
286302
},
287303

288304
// Tool Start Handler
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import { SENTRY_ORIGIN } from '@sentry/conventions/attributes';
2+
import type { Span } from '@sentry/core';
3+
import { isObjectLike } from '@sentry/core';
4+
import { addResponseAttributes, startEvaluateSpan } from '../typesafe';
5+
import { LANGCHAIN_ORIGIN } from './constants';
6+
import { getAgentNameFromMetadata, getConversationIdFromMetadata } from './utils';
7+
8+
// `TypeSafeClassifier` from `@langchain/typesafe` calls Jev with `fetch`, not through `@typesafe-ai/sdk`,
9+
// so the TypeSafe integration does not see it. Its serialized id is `[...lc_namespace, lc_name()]`; only
10+
// the class name is matched, as the namespace is the part that changes between versions.
11+
const TYPESAFE_CLASSIFIER_NAME = 'TypeSafeClassifier';
12+
13+
/** The package's default, used when the classifier is constructed without a `model`. */
14+
const DEFAULT_TYPESAFE_CLASSIFIER_MODEL = 'jev-latest';
15+
16+
// The `state` the classifier sends to Jev (messages rendered as transcript lines), keyed by the input
17+
// passed to `invoke()`. LangChain hands that same object to `handleChainStart`, so the span records what
18+
// Jev received rather than LangChain's serialized messages.
19+
const wireStates = new WeakMap<object, unknown>();
20+
21+
/** Record the `state` a `TypeSafeClassifier` sends for `input`, using the classifier's own serializer. */
22+
export function recordTypeSafeClassifierState(classifier: unknown, input: unknown): void {
23+
// A string state is sent as is, so there is nothing to record.
24+
if (!isObjectLike(input) || !isObjectLike(classifier) || typeof classifier.payload !== 'function') {
25+
return;
26+
}
27+
28+
try {
29+
const body: unknown = JSON.parse(classifier.payload(input));
30+
if (isObjectLike(body)) {
31+
wireStates.set(input, body.state);
32+
}
33+
} catch {
34+
// The classifier rejects the same input itself; the span keeps LangChain's form of it.
35+
}
36+
}
37+
38+
// Typed loosely: LangChain's `Serialized` union does not match our handler's chain type.
39+
export function isTypeSafeClassifier(chain: unknown): boolean {
40+
return isObjectLike(chain) && Array.isArray(chain.id) && chain.id[chain.id.length - 1] === TYPESAFE_CLASSIFIER_NAME;
41+
}
42+
43+
/** Start an `evaluate` span for a `TypeSafeClassifier` run, from its serialized constructor arguments. */
44+
export function startTypeSafeClassifierSpan(
45+
chain: unknown,
46+
inputs: Record<string, unknown>,
47+
metadata: Record<string, unknown> | undefined,
48+
recordInputs: boolean,
49+
): Span {
50+
const kwargs = isObjectLike(chain) && isObjectLike(chain.kwargs) ? chain.kwargs : {};
51+
const model = typeof kwargs.model === 'string' ? kwargs.model : DEFAULT_TYPESAFE_CLASSIFIER_MODEL;
52+
53+
return startEvaluateSpan({ model, state: getState(inputs), questions: kwargs.questions }, undefined, recordInputs, {
54+
[SENTRY_ORIGIN]: LANGCHAIN_ORIGIN,
55+
...getAgentNameFromMetadata(metadata),
56+
...getConversationIdFromMetadata(metadata),
57+
});
58+
}
59+
60+
/** The classifier result reports usage in camelCase, the TypeSafe helpers read the API's snake_case. */
61+
export function addTypeSafeClassifierResponseAttributes(span: Span, outputs: unknown, recordOutputs: boolean): void {
62+
if (!isObjectLike(outputs)) {
63+
return;
64+
}
65+
66+
const usage = isObjectLike(outputs.usage) ? outputs.usage : {};
67+
addResponseAttributes(
68+
span,
69+
{
70+
model: outputs.model,
71+
answers: outputs.answers,
72+
usage: { input_tokens: usage.inputTokens, output_tokens: usage.outputTokens },
73+
},
74+
recordOutputs,
75+
);
76+
}
77+
78+
function getState(inputs: Record<string, unknown>): unknown {
79+
// LangChain hands a string or array input to callbacks wrapped as `{ input }`.
80+
const keys = Object.keys(inputs);
81+
const input = inputs.input;
82+
const state =
83+
keys.length === 1 && keys[0] === 'input' && (typeof input === 'string' || Array.isArray(input)) ? input : inputs;
84+
85+
return isObjectLike(state) && wireStates.has(state) ? wireStates.get(state) : state;
86+
}

‎packages/server-utils/src/ai/langchain/utils.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,22 @@ export function extractToolDefinitions(extraParams?: Record<string, unknown>): s
564564
return JSON.stringify(toolDefs);
565565
}
566566

567+
// A run started without config inherits the parent run's config, which LangChain keeps in a global
568+
// AsyncLocalStorage. `ensureConfig` lets an explicit `callbacks` key replace the inherited one.
569+
const LANGCHAIN_ALS_KEY = Symbol.for('ls:tracing_async_local_storage');
570+
const LANGCHAIN_CHILD_CONFIG_KEY = Symbol.for('lc:child_config');
571+
572+
/** The callbacks LangChain gives a run started without config: the ones of the parent run, if any. */
573+
export function getInheritedLangChainCallbacks(): unknown {
574+
const storage = (globalThis as Record<symbol, unknown>)[LANGCHAIN_ALS_KEY] as
575+
| { getStore?: () => unknown }
576+
| undefined;
577+
const store = typeof storage?.getStore === 'function' ? storage.getStore() : undefined;
578+
const extra = isObjectLike(store) ? (store.extra as Record<symbol, unknown> | undefined) : undefined;
579+
const config = isObjectLike(extra) ? extra[LANGCHAIN_CHILD_CONFIG_KEY] : undefined;
580+
return isObjectLike(config) ? config.callbacks : undefined;
581+
}
582+
567583
/** Duck-types a LangChain `CallbackManager` (avoids coupling to a specific `@langchain/core` resolution). */
568584
function isCallbackManager(value: unknown): value is {
569585
addHandler: (handler: unknown, inherit?: boolean) => void;

0 commit comments

Comments
 (0)