aiplatform/v1/model_evaluation_slice.proto

// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.aiplatform.v1;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/aiplatform/v1/explanation.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";

option csharp_namespace = "Google.Cloud.AIPlatform.V1";
option go_package = "cloud.google.com/go/aiplatform/apiv1/aiplatformpb;aiplatformpb";
option java_multiple_files = true;
option java_outer_classname = "ModelEvaluationSliceProto";
option java_package = "com.google.cloud.aiplatform.v1";
option php_namespace = "Google\\Cloud\\AIPlatform\\V1";
option ruby_package = "Google::Cloud::AIPlatform::V1";

// A collection of metrics calculated by comparing Model's predictions on a
// slice of the test data against ground truth annotations.
message ModelEvaluationSlice {
  option (google.api.resource) = {
    type: "aiplatform.googleapis.com/ModelEvaluationSlice"
    pattern: "projects/{project}/locations/{location}/models/{model}/evaluations/{evaluation}/slices/{slice}"
  };

  // Definition of a slice.
  message Slice {
    // Specification for how the data should be sliced.
    message SliceSpec {
      // Specification message containing the config for this SliceSpec.
      // When `kind` is selected as `value` and/or `range`, only a single slice
      // will be computed.
      // When `all_values` is present, a separate slice will be computed for
      // each possible label/value for the corresponding key in `config`.
      // Examples, with feature zip_code with values 12345, 23334, 88888 and
      // feature country with values "US", "Canada", "Mexico" in the dataset:
      //
      // Example 1:
      //
      //     {
      //       "zip_code": { "value": { "float_value": 12345.0 } }
      //     }
      //
      // A single slice for any data with zip_code 12345 in the dataset.
      //
      // Example 2:
      //
      //     {
      //       "zip_code": { "range": { "low": 12345, "high": 20000 } }
      //     }
      //
      // A single slice containing data where the zip_codes between 12345 and
      // 20000 For this example, data with the zip_code of 12345 will be in this
      // slice.
      //
      // Example 3:
      //
      //     {
      //       "zip_code": { "range": { "low": 10000, "high": 20000 } },
      //       "country": { "value": { "string_value": "US" } }
      //     }
      //
      // A single slice containing data where the zip_codes between 10000 and
      // 20000 has the country "US". For this example, data with the zip_code of
      // 12345 and country "US" will be in this slice.
      //
      // Example 4:
      //
      //     { "country": {"all_values": { "value": true } } }
      //
      // Three slices are computed, one for each unique country in the dataset.
      //
      // Example 5:
      //
      //     {
      //       "country": { "all_values": { "value": true } },
      //       "zip_code": { "value": { "float_value": 12345.0 } }
      //     }
      //
      // Three slices are computed, one for each unique country in the dataset
      // where the zip_code is also 12345. For this example, data with zip_code
      // 12345 and country "US" will be in one slice, zip_code 12345 and country
      // "Canada" in another slice, and zip_code 12345 and country "Mexico" in
      // another slice, totaling 3 slices.
      message SliceConfig {
        oneof kind {
          // A unique specific value for a given feature.
          // Example: `{ "value": { "string_value": "12345" } }`
          Value value = 1;

          // A range of values for a numerical feature.
          // Example: `{"range":{"low":10000.0,"high":50000.0}}`
          // will capture 12345 and 23334 in the slice.
          Range range = 2;

          // If all_values is set to true, then all possible labels of the keyed
          // feature will have another slice computed.
          // Example: `{"all_values":{"value":true}}`
          google.protobuf.BoolValue all_values = 3;
        }
      }

      // A range of values for slice(s).
      // `low` is inclusive, `high` is exclusive.
      message Range {
        // Inclusive low value for the range.
        float low = 1;

        // Exclusive high value for the range.
        float high = 2;
      }

      // Single value that supports strings and floats.
      message Value {
        oneof kind {
          // String type.
          string string_value = 1;

          // Float type.
          float float_value = 2;
        }
      }

      // Mapping configuration for this SliceSpec.
      // The key is the name of the feature.
      // By default, the key will be prefixed by "instance" as a dictionary
      // prefix for Vertex Batch Predictions output format.
      map<string, SliceConfig> configs = 1;
    }

    // Output only. The dimension of the slice.
    // Well-known dimensions are:
    //   * `annotationSpec`: This slice is on the test data that has either
    //     ground truth or prediction with
    //     [AnnotationSpec.display_name][google.cloud.aiplatform.v1.AnnotationSpec.display_name]
    //     equals to
    //     [value][google.cloud.aiplatform.v1.ModelEvaluationSlice.Slice.value].
    //   * `slice`: This slice is a user customized slice defined by its
    //     SliceSpec.
    string dimension = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. The value of the dimension in this slice.
    string value = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

    // Output only. Specification for how the data was sliced.
    SliceSpec slice_spec = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  }

  // Output only. The resource name of the ModelEvaluationSlice.
  string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. The slice of the test data that is used to evaluate the Model.
  Slice slice = 2 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Points to a YAML file stored on Google Cloud Storage
  // describing the
  // [metrics][google.cloud.aiplatform.v1.ModelEvaluationSlice.metrics] of this
  // ModelEvaluationSlice. The schema is defined as an OpenAPI 3.0.2 [Schema
  // Object](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#schemaObject).
  string metrics_schema_uri = 3 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Sliced evaluation metrics of the Model. The schema of the
  // metrics is stored in
  // [metrics_schema_uri][google.cloud.aiplatform.v1.ModelEvaluationSlice.metrics_schema_uri]
  google.protobuf.Value metrics = 4 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Timestamp when this ModelEvaluationSlice was created.
  google.protobuf.Timestamp create_time = 5
      [(google.api.field_behavior) = OUTPUT_ONLY];

  // Output only. Aggregated explanation metrics for the Model's prediction
  // output over the data this ModelEvaluation uses. This field is populated
  // only if the Model is evaluated with explanations, and only for tabular
  // Models.
  ModelExplanation model_explanation = 6
      [(google.api.field_behavior) = OUTPUT_ONLY];
}