retail/v2alpha/import_config.proto

// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.cloud.retail.v2alpha;

import "google/api/field_behavior.proto";
import "google/api/resource.proto";
import "google/cloud/retail/v2alpha/product.proto";
import "google/cloud/retail/v2alpha/user_event.proto";
import "google/protobuf/field_mask.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/status.proto";
import "google/type/date.proto";

option csharp_namespace = "Google.Cloud.Retail.V2Alpha";
option go_package = "cloud.google.com/go/retail/apiv2alpha/retailpb;retailpb";
option java_multiple_files = true;
option java_outer_classname = "ImportConfigProto";
option java_package = "com.google.cloud.retail.v2alpha";
option objc_class_prefix = "RETAIL";
option php_namespace = "Google\\Cloud\\Retail\\V2alpha";
option ruby_package = "Google::Cloud::Retail::V2alpha";

// Google Cloud Storage location for input content.
message GcsSource {
  // Required. Google Cloud Storage URIs to input files. URI can be up to
  // 2000 characters long. URIs can match the full object path (for example,
  // `gs://bucket/directory/object.json`) or a pattern matching one or more
  // files, such as `gs://bucket/directory/*.json`. A request can
  // contain at most 100 files, and each file can be up to 2 GB. See
  // [Importing product
  // information](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog)
  // for the expected file format and setup instructions.
  repeated string input_uris = 1 [(google.api.field_behavior) = REQUIRED];

  // The schema to use when parsing the data from the source.
  //
  // Supported values for product imports:
  //
  // * `product` (default): One JSON
  // [Product][google.cloud.retail.v2alpha.Product] per line. Each product must
  //   have a valid [Product.id][google.cloud.retail.v2alpha.Product.id].
  // * `product_merchant_center`: See [Importing catalog data from Merchant
  //   Center](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog#mc).
  //
  // Supported values for user events imports:
  //
  // * `user_event` (default): One JSON
  // [UserEvent][google.cloud.retail.v2alpha.UserEvent] per line.
  // * `user_event_ga360`: Using
  //   https://support.google.com/analytics/answer/3437719.
  //
  // Supported values for control imports:
  //
  // * `control` (default): One JSON
  // [Control][google.cloud.retail.v2alpha.Control] per line.
  //
  // Supported values for catalog attribute imports:
  //
  // * `catalog_attribute` (default): One CSV
  // [CatalogAttribute][google.cloud.retail.v2alpha.CatalogAttribute] per line.
  string data_schema = 2;
}

// BigQuery source import data from.
message BigQuerySource {
  // BigQuery table partition info. Leave this empty if the BigQuery table
  // is not partitioned.
  oneof partition {
    // BigQuery time partitioned table's _PARTITIONDATE in YYYY-MM-DD format.
    //
    // Only supported in
    // [ImportProductsRequest][google.cloud.retail.v2alpha.ImportProductsRequest].
    google.type.Date partition_date = 6;
  }

  // The project ID (can be project # or ID) that the BigQuery source is in with
  // a length limit of 128 characters. If not specified, inherits the project
  // ID from the parent request.
  string project_id = 5;

  // Required. The BigQuery data set to copy the data from with a length limit
  // of 1,024 characters.
  string dataset_id = 1 [(google.api.field_behavior) = REQUIRED];

  // Required. The BigQuery table to copy the data from with a length limit of
  // 1,024 characters.
  string table_id = 2 [(google.api.field_behavior) = REQUIRED];

  // Intermediate Cloud Storage directory used for the import with a length
  // limit of 2,000 characters. Can be specified if one wants to have the
  // BigQuery export to a specific Cloud Storage directory.
  string gcs_staging_dir = 3;

  // The schema to use when parsing the data from the source.
  //
  // Supported values for product imports:
  //
  // * `product` (default): One JSON
  // [Product][google.cloud.retail.v2alpha.Product] per line. Each product must
  //   have a valid [Product.id][google.cloud.retail.v2alpha.Product.id].
  // * `product_merchant_center`: See [Importing catalog data from Merchant
  //   Center](https://cloud.google.com/retail/recommendations-ai/docs/upload-catalog#mc).
  //
  // Supported values for user events imports:
  //
  // * `user_event` (default): One JSON
  // [UserEvent][google.cloud.retail.v2alpha.UserEvent] per line.
  // * `user_event_ga360`:
  //   The schema is available here:
  //   https://support.google.com/analytics/answer/3437719.
  // * `user_event_ga4`:
  //   The schema is available here:
  //   https://support.google.com/analytics/answer/7029846.
  //
  // Supported values for autocomplete imports:
  //
  // * `suggestions` (default): One JSON completion suggestion per line.
  // * `denylist`:  One JSON deny suggestion per line.
  // * `allowlist`:  One JSON allow suggestion per line.
  string data_schema = 4;
}

// The inline source for the input config for ImportProducts method.
message ProductInlineSource {
  // Required. A list of products to update/create. Each product must have a
  // valid [Product.id][google.cloud.retail.v2alpha.Product.id]. Recommended max
  // of 100 items.
  repeated Product products = 1 [(google.api.field_behavior) = REQUIRED];
}

// The inline source for the input config for ImportUserEvents method.
message UserEventInlineSource {
  // Required. A list of user events to import. Recommended max of 10k items.
  repeated UserEvent user_events = 1 [(google.api.field_behavior) = REQUIRED];
}

// Configuration of destination for Import related errors.
message ImportErrorsConfig {
  // Required. Errors destination.
  oneof destination {
    // Google Cloud Storage prefix for import errors. This must be an empty,
    // existing Cloud Storage directory. Import errors are written to
    // sharded files in this directory, one per line, as a JSON-encoded
    // `google.rpc.Status` message.
    string gcs_prefix = 1;
  }
}

// Request message for Import methods.
message ImportProductsRequest {
  // Indicates how imported products are reconciled with the existing products
  // created or imported before.
  enum ReconciliationMode {
    // Defaults to INCREMENTAL.
    RECONCILIATION_MODE_UNSPECIFIED = 0;

    // Inserts new products or updates existing products.
    INCREMENTAL = 1;

    // Calculates diff and replaces the entire product dataset. Existing
    // products may be deleted if they are not present in the source location.
    FULL = 2;
  }

  // Required.
  // `projects/1234/locations/global/catalogs/default_catalog/branches/default_branch`
  //
  // If no updateMask is specified, requires products.create permission.
  // If updateMask is specified, requires products.update permission.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = { type: "retail.googleapis.com/Branch" }
  ];

  // Deprecated. This field has no effect.
  string request_id = 6 [deprecated = true];

  // Required. The desired input location of the data.
  ProductInputConfig input_config = 2 [(google.api.field_behavior) = REQUIRED];

  // The desired location of errors incurred during the Import.
  ImportErrorsConfig errors_config = 3;

  // Indicates which fields in the provided imported `products` to update. If
  // not set, all fields are updated.
  google.protobuf.FieldMask update_mask = 4;

  // The mode of reconciliation between existing products and the products to be
  // imported. Defaults to
  // [ReconciliationMode.INCREMENTAL][google.cloud.retail.v2alpha.ImportProductsRequest.ReconciliationMode.INCREMENTAL].
  ReconciliationMode reconciliation_mode = 5;

  // Full Pub/Sub topic name for receiving notification. If this field is set,
  // when the import is finished, a notification is sent to
  // specified Pub/Sub topic. The message data is JSON string of a
  // [Operation][google.longrunning.Operation].
  //
  // Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`. It has
  // to be within the same project as
  // [ImportProductsRequest.parent][google.cloud.retail.v2alpha.ImportProductsRequest.parent].
  // Make sure that `service-<project
  // number>@gcp-sa-retail.iam.gserviceaccount.com` has the
  // `pubsub.topics.publish` IAM permission on the topic.
  string notification_pubsub_topic = 7;

  // If true, this performs the FULL import even if it would delete a large
  // proportion of the products in the default branch, which could potentially
  // cause outages if you have live predict/search traffic.
  //
  // Only supported when
  // [ImportProductsRequest.reconciliation_mode][google.cloud.retail.v2alpha.ImportProductsRequest.reconciliation_mode]
  // is set to `FULL`.
  bool skip_default_branch_protection = 8;
}

// Request message for the ImportUserEvents request.
message ImportUserEventsRequest {
  // Required. `projects/1234/locations/global/catalogs/default_catalog`
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = { type: "retail.googleapis.com/Catalog" }
  ];

  // Required. The desired input location of the data.
  UserEventInputConfig input_config = 2
      [(google.api.field_behavior) = REQUIRED];

  // The desired location of errors incurred during the Import. Cannot be set
  // for inline user event imports.
  ImportErrorsConfig errors_config = 3;
}

// Request message for ImportCompletionData methods.
message ImportCompletionDataRequest {
  // Required. The catalog which the suggestions dataset belongs to.
  //
  // Format: `projects/1234/locations/global/catalogs/default_catalog`.
  string parent = 1 [
    (google.api.field_behavior) = REQUIRED,
    (google.api.resource_reference) = { type: "retail.googleapis.com/Catalog" }
  ];

  // Required. The desired input location of the data.
  CompletionDataInputConfig input_config = 2
      [(google.api.field_behavior) = REQUIRED];

  // Pub/Sub topic for receiving notification. If this field is set,
  // when the import is finished, a notification is sent to
  // specified Pub/Sub topic. The message data is JSON string of a
  // [Operation][google.longrunning.Operation].
  // Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`.
  string notification_pubsub_topic = 3;
}

// The input config source for products.
message ProductInputConfig {
  // Required. The source of the input.
  oneof source {
    // The Inline source for the input content for products.
    ProductInlineSource product_inline_source = 1;

    // Google Cloud Storage location for the input content.
    GcsSource gcs_source = 2;

    // BigQuery input source.
    BigQuerySource big_query_source = 3;
  }
}

// The input config source for user events.
message UserEventInputConfig {
  // The source of the input.
  oneof source {
    // Required. The Inline source for the input content for UserEvents.
    UserEventInlineSource user_event_inline_source = 1
        [(google.api.field_behavior) = REQUIRED];

    // Required. Google Cloud Storage location for the input content.
    GcsSource gcs_source = 2 [(google.api.field_behavior) = REQUIRED];

    // Required. BigQuery input source.
    BigQuerySource big_query_source = 3
        [(google.api.field_behavior) = REQUIRED];
  }
}

// The input config source for completion data.
message CompletionDataInputConfig {
  // The source of the input.
  //
  // Supported
  // [BigQuerySource.data_schema][google.cloud.retail.v2alpha.BigQuerySource.data_schema]
  // values for suggestions imports:
  //
  // * `suggestions` (default): One JSON completion suggestion per line.
  // * `denylist`:  One JSON deny suggestion per line.
  // * `allowlist`:  One JSON allow suggestion per line.
  oneof source {
    // Required. BigQuery input source.
    //
    // Add the IAM permission "BigQuery Data Viewer" for
    // cloud-retail-customer-data-access@system.gserviceaccount.com before
    // using this feature otherwise an error is thrown.
    BigQuerySource big_query_source = 1
        [(google.api.field_behavior) = REQUIRED];
  }
}

// Metadata related to the progress of the Import operation. This is
// returned by the google.longrunning.Operation.metadata field.
message ImportMetadata {
  // Operation create time.
  google.protobuf.Timestamp create_time = 1;

  // Operation last update time. If the operation is done, this is also the
  // finish time.
  google.protobuf.Timestamp update_time = 2;

  // Count of entries that were processed successfully.
  int64 success_count = 3;

  // Count of entries that encountered errors while processing.
  int64 failure_count = 4;

  // Deprecated. This field is never set.
  string request_id = 5 [deprecated = true];

  // Pub/Sub topic for receiving notification. If this field is set,
  // when the import is finished, a notification is sent to
  // specified Pub/Sub topic. The message data is JSON string of a
  // [Operation][google.longrunning.Operation].
  // Format of the Pub/Sub topic is `projects/{project}/topics/{topic}`.
  string notification_pubsub_topic = 6;

  // Metadata related to transform user events.
  TransformedUserEventsMetadata transformed_user_events_metadata = 7;
}

// Metadata related to transform user events operation.
message TransformedUserEventsMetadata {
  // Count of entries in the source user events BigQuery table.
  int64 source_events_count = 1;

  // Count of entries in the transformed user events BigQuery table, which could
  // be different from the actually imported number of user events.
  int64 transformed_events_count = 2;
}

// Response of the
// [ImportProductsRequest][google.cloud.retail.v2alpha.ImportProductsRequest].
// If the long running operation is done, then this message is returned by the
// google.longrunning.Operations.response field if the operation was successful.
message ImportProductsResponse {
  // A sample of errors encountered while processing the request.
  repeated google.rpc.Status error_samples = 1;

  // Echoes the destination for the complete errors in the request if set.
  ImportErrorsConfig errors_config = 2;
}

// Response of the ImportUserEventsRequest. If the long running
// operation was successful, then this message is returned by the
// google.longrunning.Operations.response field if the operation was successful.
message ImportUserEventsResponse {
  // A sample of errors encountered while processing the request.
  repeated google.rpc.Status error_samples = 1;

  // Echoes the destination for the complete errors if this field was set in
  // the request.
  ImportErrorsConfig errors_config = 2;

  // Aggregated statistics of user event import status.
  UserEventImportSummary import_summary = 3;
}

// A summary of import result. The UserEventImportSummary summarizes
// the import status for user events.
message UserEventImportSummary {
  // Count of user events imported with complete existing catalog information.
  int64 joined_events_count = 1;

  // Count of user events imported, but with catalog information not found
  // in the imported catalog.
  int64 unjoined_events_count = 2;
}

// Response of the
// [ImportCompletionDataRequest][google.cloud.retail.v2alpha.ImportCompletionDataRequest].
// If the long running operation is done, this message is returned by the
// google.longrunning.Operations.response field if the operation is successful.
message ImportCompletionDataResponse {
  // A sample of errors encountered while processing the request.
  repeated google.rpc.Status error_samples = 1;
}