Skip to content

Commit a19bd07

Browse files
Google APIscopybara-github
Google APIs
authored and
copybara-github
committed
feat: add OnlineEvaluator API and update Evaluation API
PiperOrigin-RevId: 903986874
1 parent 469d785 commit a19bd07

7 files changed

Lines changed: 1465 additions & 191 deletions

File tree

‎google/cloud/aiplatform/v1beta1/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ proto_library(
5959
"entity_type.proto",
6060
"env_var.proto",
6161
"evaluated_annotation.proto",
62+
"evaluation_agent_data.proto",
63+
"evaluation_rubric.proto",
6264
"evaluation_service.proto",
6365
"event.proto",
6466
"example.proto",
@@ -130,6 +132,8 @@ proto_library(
130132
"notebook_runtime_template_ref.proto",
131133
"notebook_service.proto",
132134
"notebook_software_config.proto",
135+
"online_evaluator.proto",
136+
"online_evaluator_service.proto",
133137
"openapi.proto",
134138
"operation.proto",
135139
"persistent_resource.proto",

‎google/cloud/aiplatform/v1beta1/aiplatform_v1beta1.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ apis:
2929
- name: google.cloud.aiplatform.v1beta1.ModelMonitoringService
3030
- name: google.cloud.aiplatform.v1beta1.ModelService
3131
- name: google.cloud.aiplatform.v1beta1.NotebookService
32+
- name: google.cloud.aiplatform.v1beta1.OnlineEvaluatorService
3233
- name: google.cloud.aiplatform.v1beta1.PersistentResourceService
3334
- name: google.cloud.aiplatform.v1beta1.PipelineService
3435
- name: google.cloud.aiplatform.v1beta1.PredictionService
@@ -46,6 +47,7 @@ apis:
4647
- name: google.longrunning.Operations
4748

4849
types:
50+
- name: google.cloud.aiplatform.v1beta1.ActivateOnlineEvaluatorOperationMetadata
4951
- name: google.cloud.aiplatform.v1beta1.AssignNotebookRuntimeOperationMetadata
5052
- name: google.cloud.aiplatform.v1beta1.BatchCancelPipelineJobsResponse
5153
- name: google.cloud.aiplatform.v1beta1.BatchCreateFeaturesOperationMetadata
@@ -75,13 +77,15 @@ types:
7577
- name: google.cloud.aiplatform.v1beta1.CreateModelMonitorOperationMetadata
7678
- name: google.cloud.aiplatform.v1beta1.CreateNotebookExecutionJobOperationMetadata
7779
- name: google.cloud.aiplatform.v1beta1.CreateNotebookRuntimeTemplateOperationMetadata
80+
- name: google.cloud.aiplatform.v1beta1.CreateOnlineEvaluatorOperationMetadata
7881
- name: google.cloud.aiplatform.v1beta1.CreatePersistentResourceOperationMetadata
7982
- name: google.cloud.aiplatform.v1beta1.CreateRegistryFeatureOperationMetadata
8083
- name: google.cloud.aiplatform.v1beta1.CreateSpecialistPoolOperationMetadata
8184
- name: google.cloud.aiplatform.v1beta1.CreateTensorboardOperationMetadata
8285
- name: google.cloud.aiplatform.v1beta1.DeleteFeatureValuesOperationMetadata
8386
- name: google.cloud.aiplatform.v1beta1.DeleteFeatureValuesResponse
8487
- name: google.cloud.aiplatform.v1beta1.DeleteMetadataStoreOperationMetadata
88+
- name: google.cloud.aiplatform.v1beta1.DeleteOnlineEvaluatorOperationMetadata
8589
- name: google.cloud.aiplatform.v1beta1.DeleteOperationMetadata
8690
- name: google.cloud.aiplatform.v1beta1.DeployIndexOperationMetadata
8791
- name: google.cloud.aiplatform.v1beta1.DeployIndexResponse
@@ -122,6 +126,7 @@ types:
122126
- name: google.cloud.aiplatform.v1beta1.StartNotebookRuntimeOperationMetadata
123127
- name: google.cloud.aiplatform.v1beta1.SuggestTrialsMetadata
124128
- name: google.cloud.aiplatform.v1beta1.SuggestTrialsResponse
129+
- name: google.cloud.aiplatform.v1beta1.SuspendOnlineEvaluatorOperationMetadata
125130
- name: google.cloud.aiplatform.v1beta1.UndeployIndexOperationMetadata
126131
- name: google.cloud.aiplatform.v1beta1.UndeployIndexResponse
127132
- name: google.cloud.aiplatform.v1beta1.UndeployModelOperationMetadata
@@ -137,6 +142,7 @@ types:
137142
- name: google.cloud.aiplatform.v1beta1.UpdateIndexOperationMetadata
138143
- name: google.cloud.aiplatform.v1beta1.UpdateModelDeploymentMonitoringJobOperationMetadata
139144
- name: google.cloud.aiplatform.v1beta1.UpdateModelMonitorOperationMetadata
145+
- name: google.cloud.aiplatform.v1beta1.UpdateOnlineEvaluatorOperationMetadata
140146
- name: google.cloud.aiplatform.v1beta1.UpdatePersistentResourceOperationMetadata
141147
- name: google.cloud.aiplatform.v1beta1.UpdateSpecialistPoolOperationMetadata
142148
- name: google.cloud.aiplatform.v1beta1.UpdateTensorboardOperationMetadata
@@ -347,6 +353,7 @@ http:
347353
- post: '/v1beta1/{name=projects/*/locations/*/notebookExecutionJobs/*/operations/*}:cancel'
348354
- post: '/v1beta1/{name=projects/*/locations/*/notebookRuntimes/*/operations/*}:cancel'
349355
- post: '/v1beta1/{name=projects/*/locations/*/notebookRuntimeTemplates/*/operations/*}:cancel'
356+
- post: '/v1beta1/{name=projects/*/locations/*/onlineEvaluators/*/operations/*}:cancel'
350357
- post: '/v1beta1/{name=projects/*/locations/*/persistentResources/*/operations/*}:cancel'
351358
- post: '/v1beta1/{name=projects/*/locations/*/ragEngineConfig/operations/*}:cancel'
352359
- post: '/v1beta1/{name=projects/*/locations/*/ragCorpora/*/operations/*}:cancel'
@@ -364,6 +371,7 @@ http:
364371
- post: '/v1beta1/{name=projects/*/locations/*/tensorboards/*/experiments/*/operations/*}:cancel'
365372
- post: '/v1beta1/{name=projects/*/locations/*/tensorboards/*/experiments/*/runs/*/operations/*}:cancel'
366373
- post: '/v1beta1/{name=projects/*/locations/*/tensorboards/*/experiments/*/runs/*/timeSeries/*/operations/*}:cancel'
374+
- post: '/v1beta1/{name=onlineEvaluators/*/operations/*}:cancel'
367375
- selector: google.longrunning.Operations.DeleteOperation
368376
delete: '/ui/{name=projects/*/locations/*/operations/*}'
369377
additional_bindings:
@@ -451,6 +459,7 @@ http:
451459
- delete: '/v1beta1/{name=projects/*/locations/*/notebookExecutionJobs/*/operations/*}'
452460
- delete: '/v1beta1/{name=projects/*/locations/*/notebookRuntimes/*/operations/*}'
453461
- delete: '/v1beta1/{name=projects/*/locations/*/notebookRuntimeTemplates/*/operations/*}'
462+
- delete: '/v1beta1/{name=projects/*/locations/*/onlineEvaluators/*/operations/*}'
454463
- delete: '/v1beta1/{name=projects/*/locations/*/persistentResources/*/operations/*}'
455464
- delete: '/v1beta1/{name=projects/*/locations/*/ragEngineConfig/operations/*}'
456465
- delete: '/v1beta1/{name=projects/*/locations/*/ragCorpora/*/operations/*}'
@@ -474,6 +483,7 @@ http:
474483
- delete: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/features/*/operations/*}'
475484
- delete: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/featureMonitors/*/operations/*}'
476485
- delete: '/v1beta1/{name=projects/*/locations/*/featureOnlineStores/*/featureViews/*/operations/*}'
486+
- delete: '/v1beta1/{name=onlineEvaluators/*/operations/*}'
477487
- selector: google.longrunning.Operations.GetOperation
478488
get: '/ui/{name=projects/*/locations/*/operations/*}'
479489
additional_bindings:
@@ -563,6 +573,7 @@ http:
563573
- get: '/v1beta1/{name=projects/*/locations/*/notebookExecutionJobs/*/operations/*}'
564574
- get: '/v1beta1/{name=projects/*/locations/*/notebookRuntimes/*/operations/*}'
565575
- get: '/v1beta1/{name=projects/*/locations/*/notebookRuntimeTemplates/*/operations/*}'
576+
- get: '/v1beta1/{name=projects/*/locations/*/onlineEvaluators/*/operations/*}'
566577
- get: '/v1beta1/{name=projects/*/locations/*/persistentResources/*/operations/*}'
567578
- get: '/v1beta1/{name=projects/*/locations/*/ragEngineConfig/operations/*}'
568579
- get: '/v1beta1/{name=projects/*/locations/*/ragCorpora/*/operations/*}'
@@ -586,6 +597,7 @@ http:
586597
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/operations/*}'
587598
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/features/*/operations/*}'
588599
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/featureMonitors/*/operations/*}'
600+
- get: '/v1beta1/{name=onlineEvaluators/*/operations/*}'
589601
- selector: google.longrunning.Operations.ListOperations
590602
get: '/ui/{name=projects/*/locations/*}/operations'
591603
additional_bindings:
@@ -674,6 +686,7 @@ http:
674686
- get: '/v1beta1/{name=projects/*/locations/*/notebookExecutionJobs/*}/operations'
675687
- get: '/v1beta1/{name=projects/*/locations/*/notebookRuntimes/*}/operations'
676688
- get: '/v1beta1/{name=projects/*/locations/*/notebookRuntimeTemplates/*}/operations'
689+
- get: '/v1beta1/{name=projects/*/locations/*/onlineEvaluators/*}/operations'
677690
- get: '/v1beta1/{name=projects/*/locations/*/persistentResources/*}/operations'
678691
- get: '/v1beta1/{name=projects/*/locations/*/ragEngineConfig}/operations'
679692
- get: '/v1beta1/{name=projects/*/locations/*/ragCorpora/*}/operations'
@@ -697,6 +710,7 @@ http:
697710
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*}/operations'
698711
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/features/*}/operations'
699712
- get: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/featureMonitors/*}/operations'
713+
- get: '/v1beta1/{name=onlineEvaluators/*}/operations'
700714
- selector: google.longrunning.Operations.WaitOperation
701715
post: '/ui/{name=projects/*/locations/*/operations/*}:wait'
702716
additional_bindings:
@@ -785,6 +799,7 @@ http:
785799
- post: '/v1beta1/{name=projects/*/locations/*/notebookExecutionJobs/*/operations/*}:wait'
786800
- post: '/v1beta1/{name=projects/*/locations/*/notebookRuntimes/*/operations/*}:wait'
787801
- post: '/v1beta1/{name=projects/*/locations/*/notebookRuntimeTemplates/*/operations/*}:wait'
802+
- post: '/v1beta1/{name=projects/*/locations/*/onlineEvaluators/*/operations/*}:wait'
788803
- post: '/v1beta1/{name=projects/*/locations/*/persistentResources/*/operations/*}:wait'
789804
- post: '/v1beta1/{name=projects/*/locations/*/ragEngineConfig/operations/*}:wait'
790805
- post: '/v1beta1/{name=projects/*/locations/*/ragCorpora/*/operations/*}:wait'
@@ -807,6 +822,7 @@ http:
807822
- post: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/operations/*}:wait'
808823
- post: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/features/*/operations/*}:wait'
809824
- post: '/v1beta1/{name=projects/*/locations/*/featureGroups/*/featureMonitors/*/operations/*}:wait'
825+
- post: '/v1beta1/{name=onlineEvaluators/*/operations/*}:wait'
810826

811827
authentication:
812828
rules:
@@ -931,6 +947,10 @@ authentication:
931947
oauth:
932948
canonical_scopes: |-
933949
https://www.googleapis.com/auth/cloud-platform
950+
- selector: 'google.cloud.aiplatform.v1beta1.OnlineEvaluatorService.*'
951+
oauth:
952+
canonical_scopes: |-
953+
https://www.googleapis.com/auth/cloud-platform
934954
- selector: 'google.cloud.aiplatform.v1beta1.PersistentResourceService.*'
935955
oauth:
936956
canonical_scopes: |-

‎google/cloud/aiplatform/v1beta1/evaluation_agent_data.proto‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.aiplatform.v1beta1;
18+
19+
import "google/api/field_behavior.proto";
20+
import "google/cloud/aiplatform/v1beta1/content.proto";
21+
import "google/cloud/aiplatform/v1beta1/tool.proto";
22+
import "google/protobuf/struct.proto";
23+
import "google/protobuf/timestamp.proto";
24+
25+
option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
26+
option go_package = "cloud.google.com/go/aiplatform/apiv1beta1/aiplatformpb;aiplatformpb";
27+
option java_multiple_files = true;
28+
option java_outer_classname = "EvaluationAgentDataProto";
29+
option java_package = "com.google.cloud.aiplatform.v1beta1";
30+
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
31+
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";
32+
33+
// Represents data specific to multi-turn agent evaluations.
34+
message AgentData {
35+
// Optional. A map containing the static configurations for each agent in the
36+
// system. Key: agent_id (matches the `author` field in events). Value: The
37+
// static configuration of the agent.
38+
map<string, AgentConfig> agents = 1 [(google.api.field_behavior) = OPTIONAL];
39+
40+
// Optional. A chronological list of conversation turns.
41+
// Each turn represents a logical execution cycle (e.g., User Input -> Agent
42+
// Response).
43+
repeated ConversationTurn turns = 2 [(google.api.field_behavior) = OPTIONAL];
44+
}
45+
46+
// Represents configuration for an Agent.
47+
message AgentConfig {
48+
// Required. Unique identifier of the agent.
49+
// This ID is used to refer to this agent, e.g., in AgentEvent.author, or in
50+
// the `sub_agents` field. It must be unique within the `agents` map.
51+
optional string agent_id = 1 [(google.api.field_behavior) = REQUIRED];
52+
53+
// Optional. The type or class of the agent (e.g., "LlmAgent", "RouterAgent",
54+
// "ToolUseAgent"). Useful for the autorater to understand the expected
55+
// behavior of the agent.
56+
string agent_type = 2 [(google.api.field_behavior) = OPTIONAL];
57+
58+
// Optional. A high-level description of the agent's role and
59+
// responsibilities. Critical for evaluating if the agent is routing tasks
60+
// correctly.
61+
string description = 3 [(google.api.field_behavior) = OPTIONAL];
62+
63+
// Optional. Provides instructions for the LLM model, guiding the agent's
64+
// behavior. Can be static or dynamic. Dynamic instructions can contain
65+
// placeholders like {variable_name} that will be resolved at runtime using
66+
// the `AgentEvent.state_delta` field.
67+
string instruction = 4 [(google.api.field_behavior) = OPTIONAL];
68+
69+
// Optional. The list of tools available to this agent.
70+
repeated Tool tools = 5 [(google.api.field_behavior) = OPTIONAL];
71+
72+
// Optional. The list of valid agent IDs that this agent can delegate to.
73+
// This defines the directed edges in the multi-agent system graph topology.
74+
repeated string sub_agents = 6 [(google.api.field_behavior) = OPTIONAL];
75+
}
76+
77+
// Represents a single turn/invocation in the conversation.
78+
message ConversationTurn {
79+
// Required. The 0-based index of the turn in the conversation sequence.
80+
optional int32 turn_index = 1 [(google.api.field_behavior) = REQUIRED];
81+
82+
// Optional. A unique identifier for the turn.
83+
// Useful for referencing specific turns across systems.
84+
string turn_id = 2 [(google.api.field_behavior) = OPTIONAL];
85+
86+
// Optional. The list of events that occurred during this turn.
87+
repeated AgentEvent events = 3 [(google.api.field_behavior) = OPTIONAL];
88+
}
89+
90+
// Represents a single event in the execution trace.
91+
message AgentEvent {
92+
// Required. The ID of the agent or entity that generated this event.
93+
// Use "user" to denote events generated by the end-user.
94+
optional string author = 1 [(google.api.field_behavior) = REQUIRED];
95+
96+
// Required. The content of the event (e.g., text response, tool call, tool
97+
// response).
98+
optional Content content = 2 [(google.api.field_behavior) = REQUIRED];
99+
100+
// Optional. The timestamp when the event occurred.
101+
google.protobuf.Timestamp event_time = 3
102+
[(google.api.field_behavior) = OPTIONAL];
103+
104+
// Optional. The change in the session state caused by this event. This is a
105+
// key-value map of fields that were modified or added by the event.
106+
google.protobuf.Struct state_delta = 4
107+
[(google.api.field_behavior) = OPTIONAL];
108+
109+
// Optional. The list of tools that were active/available to the agent at the
110+
// time of this event. This overrides the `AgentConfig.tools` if set.
111+
repeated Tool active_tools = 5 [(google.api.field_behavior) = OPTIONAL];
112+
}

‎google/cloud/aiplatform/v1beta1/evaluation_rubric.proto‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Copyright 2026 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package google.cloud.aiplatform.v1beta1;
18+
19+
import "google/api/field_behavior.proto";
20+
21+
option csharp_namespace = "Google.Cloud.AIPlatform.V1Beta1";
22+
option java_multiple_files = true;
23+
option java_outer_classname = "EvaluationRubricProto";
24+
option java_package = "com.google.cloud.aiplatform.v1beta1";
25+
option php_namespace = "Google\\Cloud\\AIPlatform\\V1beta1";
26+
option ruby_package = "Google::Cloud::AIPlatform::V1beta1";
27+
28+
// Message representing a single testable criterion for evaluation.
29+
// One input prompt could have multiple rubrics.
30+
message Rubric {
31+
// Content of the rubric, defining the testable criteria.
32+
message Content {
33+
// Defines criteria based on a specific property.
34+
message Property {
35+
// Description of the property being evaluated.
36+
// Example: "The model's response is grammatically correct."
37+
string description = 1;
38+
}
39+
40+
oneof content_type {
41+
// Evaluation criteria based on a specific property.
42+
Property property = 1;
43+
}
44+
}
45+
46+
// Importance level of the rubric.
47+
enum Importance {
48+
// Importance is not specified.
49+
IMPORTANCE_UNSPECIFIED = 0;
50+
51+
// High importance.
52+
HIGH = 1;
53+
54+
// Medium importance.
55+
MEDIUM = 2;
56+
57+
// Low importance.
58+
LOW = 3;
59+
}
60+
61+
// Unique identifier for the rubric.
62+
// This ID is used to refer to this rubric, e.g., in RubricVerdict.
63+
string rubric_id = 1;
64+
65+
// Required. The actual testable criteria for the rubric.
66+
Content content = 2;
67+
68+
// Optional. A type designator for the rubric, which can inform how it's
69+
// evaluated or interpreted by systems or users.
70+
// It's recommended to use consistent, well-defined, upper snake_case strings.
71+
// Examples: "SUMMARIZATION_QUALITY", "SAFETY_HARMFUL_CONTENT",
72+
// "INSTRUCTION_ADHERENCE".
73+
optional string type = 3;
74+
75+
// Optional. The relative importance of this rubric.
76+
optional Importance importance = 4;
77+
}
78+
79+
// A group of rubrics, used for grouping rubrics based on a metric or a version.
80+
message RubricGroup {
81+
// Unique identifier for the group.
82+
string group_id = 1;
83+
84+
// Human-readable name for the group. This should be unique
85+
// within a given context if used for display or selection.
86+
// Example: "Instruction Following V1", "Content Quality - Summarization
87+
// Task".
88+
string display_name = 2;
89+
90+
// Rubrics that are part of this group.
91+
repeated Rubric rubrics = 3;
92+
}
93+
94+
// Represents the verdict of an evaluation against a single rubric.
95+
message RubricVerdict {
96+
// Required. The full rubric definition that was evaluated.
97+
// Storing this ensures the verdict is self-contained and understandable,
98+
// especially if the original rubric definition changes or was dynamically
99+
// generated.
100+
Rubric evaluated_rubric = 1;
101+
102+
// Required. Outcome of the evaluation against the rubric, represented as a
103+
// boolean. `true` indicates a "Pass", `false` indicates a "Fail".
104+
bool verdict = 2;
105+
106+
// Optional. Human-readable reasoning or explanation for the verdict.
107+
// This can include specific examples or details from the evaluated content
108+
// that justify the given verdict.
109+
optional string reasoning = 3;
110+
}

0 commit comments

Comments
 (0)