syntax = "proto3"; import "proto/clarifai/api/status/status.proto"; import "proto/clarifai/api/status/status_code.proto"; import "proto/clarifai/api/utils/extensions.proto"; import "proto/clarifai/api/utils/matrix.proto"; import "proto/clarifai/api/utils/time.proto"; import "proto/clarifai/auth/util/extension.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; package clarifai.api; option go_package = "github.com/Clarifai/clarifai-go-grpc/proto/clarifai/api"; option java_multiple_files = true; option java_package = "com.clarifai.grpc.api"; option objc_class_prefix = "CAIP"; // Annotation of an asset with metadata message Annotation { reserved 4, 5, 6, 11, 12, 20; // The ID for the annotation string id = 1; // ID of the input this annotation is tied to string input_id = 2; // The data passed along in this annotation. Data data = 3; // task_id is deprecated in annotation_info. Use task_id google.protobuf.Struct annotation_info = 13; // DEPRECATED: Use worker.user.id instead. string user_id = 15 [deprecated = true]; // DEPRECATED: Use worker.model.model_version.id instead string model_version_id = 16 [deprecated = true]; // DEPRECATED. string embed_model_version_id = 14 [deprecated = true]; // Annotation Status clarifai.api.status.Status status = 7; // When the annotation was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 8; // When the annotation was modified. google.protobuf.Timestamp modified_at = 9; // Whether or not this annotation is trusted // Will be deprecated bool trusted = 10 [deprecated = true]; // Is this the input level annotation. bool input_level = 17; // Consensus review related information, e.g. // * annotation group // * id of annotation parent, in case the annotation was split from another annotation google.protobuf.Struct consensus_info = 18; // The id of the task annotation belongs to string task_id = 19; // Worker is the worker that created the annotation. Worker worker = 21; } // AnnotationTrack of an asset with metadata message AnnotationTrack { reserved 2, 5, 7, 8, 11; // The ID for the annotation track string id = 1; // ID of the asset this annotation track is tied to string input_id = 3; // Concept this annotation track Concept concept = 4; // AnnotationTrack Status clarifai.api.status.Status status = 6; // Start frame number (in original video) of the annotation track, inclusive. uint32 start_frame_nr = 16; // End frame number (in original video) of the annotation track, inclusive. uint32 end_frame_nr = 17; // Start time (in milliseconds of original video) of the annotation track, inclusive. uint32 start_frame_ms = 14; // End time (in milliseconds of original video) of the annotation track, inclusive. uint32 end_frame_ms = 15; // When the annotation track was created. google.protobuf.Timestamp created_at = 9; // When the annotation track was modified. google.protobuf.Timestamp modified_at = 10; // Sampling rate of the annotation track in milliseconds. uint32 sample_rate_ms = 12; // Sampling frame rate of the video track in frame number increments // increment of 1 means it matches the original video FPS // increment of 2 means every second frame is sampled, etc. // So if you have 30fps original video and frame_rate=3, your annotations in a track are stored at 30fps/3frame_rate=10 frames per second // Useful if client relies on simple frame access. // Useful if video has variable frame rate (VFR), then annotations are also sampled with VFR in mind uint32 sample_rate_frame = 13; } // Worker is the author of an annotation. message Worker { oneof worker { // User is the human that created the annotation. // // By default no real names of users are returned in responses. These can // be requested with the 'names' additional field. User user = 1; // Model is the model that created the annotation. Model model = 2; // Workflow is the workflow that created the annotation. Workflow workflow = 3; } } // Application with tasks and datasets message App { reserved 10, 11, 12; string id = 1; string name = 2; string default_language = 3; // Deprecated, use default_workflow instead. string default_workflow_id = 4 [deprecated = true]; // @exclude TODO (EAGLE-4506): Remove this field and associated code Workflow default_workflow = 23; // why is user_id present here when this message type is used in PostApps but completely ignored there? PostApp already // specifies the userid in path but doesn't even actually use neither of userids, it instead used the id from auth context. // This creates a lot of ambiguity, should always have different message types for Post/Get endpoints so that the minimum interface for each op can be described string user_id = 5; // When the app was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 6; // When the app was last modified google.protobuf.Timestamp modified_at = 17; // if user accept legal consent for face recognition uint32 legal_consent_status = 7; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 13; // short description about the app. string description = 14; // Default value for model predictions on video: Sample delay for video predicting (1 frame per N milliseconds) uint32 sample_ms = 15; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 16; // data tier id this app is using. string data_tier_id = 18; // Is starred by the requesting user (only showed on get/list requests) // Please use PostAppStars/DeleteAppStars endpoints to star/unstar an app bool is_starred = 19; // How many users have starred the app (only showed on get/list requests) // Computed value, not editable int32 star_count = 20; // Notes for the application // This field should be used for in-depth notes and supports up to 64Kbs. string notes = 21; // Representative image for this app Image image = 22; // An app marked as a template can be duplicated by any user that can see it, // including all visible resources within it. google.protobuf.BoolValue is_template = 25; AppExtraInfo extra_info = 24; // Where app embeddings are stored // postgres (default), qdrant EmbeddingsStorage embeddings_storage = 26; enum EmbeddingsStorage { EMBEDDING_STORAGE_NOT_SET = 0; POSTGRES = 1; QDRANT = 2; } } message AppExtraInfo { // Revision marker for this application. // The value of the revision changes when // * inputs are added, updated or deleted // * annotations are added, updated or deleted // * inputs are added to or removed from datasets // For example, this value can be used to detect if client side caches related to searching should be invalidated. // Field not filled in for list endpoints, use GetApp string search_revision_marker = 1; AppResourceCounts counts = 2; } // App query message AppQuery { // Query by application name. This supports wildcard queries like "gen*" to match "general" as an example. string name = 1; } message AppResourceCounts { reserved 4; int64 datasets = 1; int64 models = 2; int64 workflows = 3; int64 inputs = 5; int64 pipelines = 6; } // Collaborator - invited user, who shares an access to an application message Collaborator { // id of this collaborator string id = 1; // the app this collaborator has access to // FIXME(zeiler): this should be in the user_app_id.app_id already from the endpoint. clarifai.api.App app = 2; // who is this collaborator clarifai.api.User user = 3; // the permission this collaborator repeated string scopes = 4; repeated string endpoints = 5; // When the app was shared with. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 6; // When the collaborator was updated. google.protobuf.Timestamp modified_at = 7; // When the collaborator was removed from app. google.protobuf.Timestamp deleted_at = 8; } // collaboration includes an app you're invited to work on. message Collaboration { // the application App app = 1; // the app owner's info(including user_unique_id, first_name, last_name, primary_email) User app_owner = 2; // the low-level scope users are shared with for this collaboration repeated string scopes = 3; // the endpoint-level scopes users are shared with for this collaboration repeated string endpoints = 4; // when is the collaboration created google.protobuf.Timestamp created_at = 5; } // Audio asset struct message Audio { // This is a URL to a publicly accessible image file. The platform will download this file server // side and then process. string url = 1; // The base64 field is using image file bytes directly in the request. // NOTE: if you're sending a json request, then this MUST be base64 encoded before sending (hence // the name here). // When using our grpc clients, you DO NOT need to base64 encode // it yourself since the clients know how to do this for you automatically and will avoid the // base64 encoding if they send a binary request. bytes base64 = 2; // If True then you will be allowed to have multiple urls. bool allow_duplicate_url = 4; // The hosted field lists original audio hosted in Clarifai storage. This field is currently used // only in response. HostedURL hosted = 5; // audio info AudioInfo audio_info = 6; } message AudioInfo { // audio format string audio_format = 1; // sample rate int32 sample_rate = 2; // audio track duration in seconds float duration_seconds = 3; // audio track bit rate int32 bit_rate = 4; } // Track proto encodes information of a track over a number of frames message Track { reserved 3; // track id string id = 1; // This is a recursive definition which can contain all the concepts, // embeddings, etc. that are computed within this track. Data data = 2; TimeInfo time_info = 4; float quality = 5; } // Cluster data message Cluster { string id = 1; // Number of annotations tied to the cluster in the app uint32 count = 2; // The score assigned to this cluster. // For List Clusters endpoint, this represents percentage of inputs in the app assigned to this cluster. float score = 3; // Representative hits for cluster (for now we only return 1) repeated Hit hits = 4; repeated float projection = 5; } // Color data message Color { string raw_hex = 1; W3C w3c = 2; float value = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; } message W3C { string hex = 1; string name = 2; } // Common message to identify the app in a url endpoint. message UserAppIDSet { // Note user_id 'me' is reserved - it is the alias for the id of authorized user string user_id = 1; string app_id = 2; } // WorkerIDSet helps to identify different types of workers. // To identify a user, set user_id field. // To identify a model from current application, set model_id and model_version_id fields. // To identify a model from another application, set user_id, app_id, model_id and model_version_id fields. // To identify a workflow from current application, set workflow_id and workflow_version_id fields. // To identify a workflow from another application, set user_id, app_id, workflow_id and workflow_version_id fields. message WorkerIDSet { string user_id = 1; string app_id = 2; string model_id = 3; string model_version_id = 4; string workflow_id = 5; string workflow_version_id = 6; } // PatchAction message PatchAction { // The operation to perform on the patched metadata given a path // For now only operations 'overwrite', 'delete, and 'merge' is supported string op = 1; // If the action is 'merge' and there is a conflict, how to resolve it. // The options are // 'overwrite_by_id', 'remove_by_id', 'merge_by_id','overwrite', 'append' and 'do_nothing' // Note that for conflict resolutions '*_by_id' to work on a list, the list should contain // objects with an 'id' field which will be used to uniquely identify each field. For example // Patching existing json // { // "tag": [ // { // "id": "1", // "data": 1 // }, // { // "id": "2", // "data": 2 // } // ] // } // with op 'merge' and merge_conflict_resolution 'overwrite_by_id' // { // "tag": [ // { // "id": "2", // "data": 3 // } // ] // } // would produce // { // "tag": [ // { // "id": "1", // "data": 1 // }, // { // "id": "2", // "data": 3 // } // ] // } // while with merge_conflict_resolution 'remove_by_id' it would produce // { // "tag": [ // { // "id": "1", // "data": 1 // } // ] // } // // Option 'append' will simply create a list on conflicts. For example in above example // the final result would be // { // "tag": [ // { // "id": "1", // "data": 1 // }, // { // "id": "2", // "data": [2, 3] // } // ] // } string merge_conflict_resolution = 2; // Path for the change. For example 'tag[1].data' is a valid path in above example. // Default path is root level i.e. ''. string path = 3; } //////////////////////////////////////////////////////////////////////////////// // Concept or tag message Concept { // The concept's unique id. string id = 1; // The name of the concept in the given language. string name = 2; // Used to indicate presence (1.0) or not (0.0) of this concept when making a request. // This is also the prediction probability when returning predictions from our API. // For convenience we use the default of 1.0 when making requests so the concept you provide is // is treated as a positive (1.0) and not a negative (which would be value == 0.0). float value = 3 [(clarifai.api.utils.cl_default_float) = 1.0, (clarifai.api.utils.cl_show_if_empty) = true]; // When the concept was created. The format is https://www.ietf.org/rfc/rfc3339.txt . // Example: "2006-01-02T15:04:05.999999Z". This field is used only in a response. google.protobuf.Timestamp created_at = 4; // The language in which the concept name is in. This is *ONLY* used in the response and setting // it in a request is ignored since the default language of your app is used when creating // or patching a Concept. To set other languages for your concept use the ConceptLanguage object // and its corresponding endpoints. string language = 5; // The application id that this concept is within. This can be ignored by most users. string app_id = 6; // The definition for the concept. Similar to name. This can be ignored by most users. string definition = 7; // The vocabulary that this concept belongs to. This is useful if you have different unique sets // of concepts that you can separate out based on this field. For example "age_appearance" vs // "gender_appearance" in a list of concept returned from the demographics model. string vocab_id = 8; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 9; // The user the concept belongs to. string user_id = 10; // Information about keypoints for this concept KeypointInfo keypoint_info = 11; // Optional extra info. ConceptExtraInfo extra_info = 12; // To handle arbitrary json metadata: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 13; // Representative image for the concept Image image = 22; } message KeypointInfo { // Names of the keypoints repeated string keypoint_names = 1; // Defines the connections between keypoint_names. Each value represents the index in keypoint_names. repeated KeypointEdge skeleton = 2; } message KeypointEdge { uint32 k1 = 1; uint32 k2 = 2; } // ConceptExtraInfo represents extra information related to a concept that is context-dependent. // It is only set when requested in ConceptExtraInfoRequest. message ConceptExtraInfo { // Whether this concept is rankable based on ConceptExtraInfoRequest configuration. bool is_rankable = 1; } // ConceptCount message ConceptCount { // The concept's unique id. string id = 1; // The name of the concept. string name = 2; // The total count for concepts labeled for all asset statues (processing, to_process, processed, error) ConceptTypeCount concept_type_count = 3; // The detail count for different assets status DetailConceptCount detail_concept_count = 4; } // ConceptTypeCount message ConceptTypeCount { // The number of inputs that have a concept with a value of 1.0 (indicating presence of the // concept in an input). uint32 positive = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; // The number of inputs that have a concept with a value of 0.0 (indicating absence of the // concept in an input). uint32 negative = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; } // DetailConceptCount message DetailConceptCount { // The concept count for processed assets ConceptTypeCount processed = 1; // The concept count for to process assets ConceptTypeCount to_process = 2; // The concept count for assets with status error ConceptTypeCount errors = 3; // The concept count for processing assets ConceptTypeCount processing = 4; } // ConceptQuery message ConceptQuery { // The name of the concept to search. string name = 1; // The language of the concept name in a search. Defaults to English. string language = 2; // Deprecated: Use workflow.id instead. string workflow_id = 3 [deprecated = true]; // @exclude TODO (EAGLE-4506): Remove this field and associated code // The concepts must belong to models with specified use cases. // Multiple values are joined using an OR condition. repeated WorkflowModelUseCase use_cases = 4; // Search for concepts in a specific model or workflow. // ########## Supported fields ########## // - model.id - fetch concepts from this model // - model.model_version.id - if set, then use the specified model version. if not set, use latest model version // // - workflow.id - fetch concepts from this workflow // - workflow.version.id - if set, then use the specified workflow version. if not set, use latest workflow version // ###################################### oneof source { Model model = 5; Workflow workflow = 6; } // By default, we return app concepts combined with source (model or workflow) concepts. // If source is not set, then we only return app concepts. // If ignore_app_concepts is true, then we only return source concepts. // When use_cases are set, then ignore_app_concepts is always true, because // concept use cases can only be determined in relation to a model or a workflow. bool ignore_app_concepts = 7; } enum WorkflowModelUseCase { WORKFLOW_MODEL_USE_CASE_NOT_SET = 0; // Classifier models without a detector parent (recursive check) in a workflow // are used for classification. CLASSIFICATION = 1; // Detector models in a workflow are used for detection. // Classifier models that run after a detector model are also used for detection. DETECTION = 2; } // This represents a relation (i.e. edge) between the subject concept and the object concept message ConceptRelation { // ID of the concept relation string id = 1; // The subject concept (i.e. source) of the concept relation Concept subject_concept = 2; // The subject concept (i.e. destination) of the concept relation Concept object_concept = 3; // The predicate (i.e. edge) linking the subject and the object // Both subject_concept and object_concept are concepts. // The predicate is the type of relationship. // That predicate acts on the subject. // // There are three current types of predicates: // 1) "hyponym" // 2) "hypernym" // 3) "synonym" // // 1) For example, 'hyponym' is a type of predicate which represents 'is_a_kind_of' relation so // the following relationship: // 'honey' (subject), 'hyponym' (predicate), 'food' (object) // Can more easily be read as: // 'honey' 'is a kind of' 'food' // // // 2) The 'hypernym' relation is the opposite of 'hyponym' and when you add one of the // relationships the opposite will automatically appear for you in queries. // // The 'hypernym' can be read as 'is a parent of' so: // 'food' (subject), 'hypernym' (predicate), 'honey' (object) // Can more easily be read as: // 'food' is a parent of 'honey' // // 3) The 'synonym' relation defines two concepts that essential mean the same thing. This // is more like a "is" relationship. So for example a 'synonym' relationship could be: // "puppy" is "pup" // The reverse is also true once the former is added so: // "pup" is "puppy" // will appear in queries as well. string predicate = 4; // The knowledge graph id that this edge belongs to. If using the app's global knowledge graph // and not a specific one then this should be the empty string "". string knowledge_graph_id = 5; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 6; } // A Knowledge Graph is a logical subsets of edges in the overall Concept Graph message KnowledgeGraph { // ID of the knowledge graph string id = 1; // Name of the knowledge graph string name = 2; // Human readable description of the knowledge graph string description = 3; // The app that contains the images that correspond to the concepts in the knowledge graph string examples_app_id = 4; // The app that contains the sample images that we want to show the customer for the concepts in the knowledge graph string sampled_examples_app_id = 5; } // This represents a link to an outside source for the given concept. // The values from here are sticked into Concept message into the name and definition fields when // returning from the API in your default language. The "id" field here becomes the "language" // field of the Concept message which is a little weird. message ConceptLanguage { // This is the language code for the language such as "en". string id = 1; // The type of the outside source. string name = 2; // The ID that is referenced in the source. string definition = 3; } // Data message Data { reserved 4, 10; // Input and output images. Image image = 1; // Input and output videos. Video video = 2; // A list of concepts. repeated Concept concepts = 3; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 5; // Geography information. Geo geo = 6; // The dominant colors within an image. repeated Color colors = 7; // Clustering centroids for inputs. repeated Cluster clusters = 8; // Embedding vectors representing each input. repeated Embedding embeddings = 9; // For recursing into localized regions of an input. repeated Region regions = 11; // For temporal content like video. repeated Frame frames = 12; // Input, output or annotation text. Text text = 13; // Input and output audio. Audio audio = 14; // Track information. repeated Track tracks = 15; // Time segments information. repeated TimeSegment time_segments = 16; // Holds score, rank, and user, app, input IDs and search hit data repeated Hit hits = 17; // Heatmap as 2d image repeated Image heatmaps = 18; // For data messages that have multiple parts such as multi-modal // requests, we allow you to specify those as a list of Data objects. repeated Part parts = 19; // A proto representation for numpy arrays, useful to pass information from python SDK to a // python based model implementation. NDArray ndarray = 20; // Input and output integer number int64 int_value = 21; // Input and output floating number double float_value = 22; // Input and output bytes data bytes bytes_value = 23; // Input and output bool data bool bool_value = 24; // Input and output string data string string_value = 25; // To handle Input and output json google.protobuf.Struct struct_value = 26; } // A part of data used for multi-modal processing. message Part { // The data for this part. Data data = 1; // A unique id for the part. string id = 2; } // A region within the data. message Region { // A unique id for the region. string id = 1; // The details about the location of the region. RegionInfo region_info = 2; // A recursive definition of the data within the Region. For example, this will contain // data.concepts if the region also has annotations or predictions of concepts within it. Data data = 3; // This is the confidence score of the overall Region. float value = 4; // For tracking algorithsm and annotations we tie regions together with this track id. string track_id = 5; } // The information of the location of the Region. message RegionInfo { reserved 2, 3; // Details of the region's rectangular bounding box. BoundingBox bounding_box = 1; // Details of the region's segmentation mask. Mask mask = 4; // A polygon of points. Polygon polygon = 5; // A landmark point location. Point point = 6; // Span char sequence for NLP. Span span = 7; // Token char sequence for NLP. Token token = 8; // The locations of detected keypoints, which are to be used in conjunction with the detected concept's skeleton to connect the keypoint locations. // These will be in the same order as the respective keypoint_names inside the concept. repeated Point keypoint_locations = 9; } // Rectangular bounding box for a region. message BoundingBox { // The top left of the bounding box normalized to the data dimension to be within [0-1.0] float top_row = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; // The left column of the bounding box normalized to the data dimension to be within [0-1.0] float left_col = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; // The bottom row of the bounding box normalized to the data dimension to be within [0-1.0] float bottom_row = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; // The right col of the bounding box normalized to the data dimension to be within [0-1.0] float right_col = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; } // The information of the location of the Frame. message FrameInfo { // Deprecated. Use Time instead. // The index of the frame, informational and optional. // Depends on the sampling rate used during processing // May be 0 for interpolated frames that are generated for brief time (training) or if new frame is manually added uint32 index = 1 [(clarifai.api.utils.cl_show_if_empty) = true, deprecated = true]; // time in the video in milliseconds. This is independent of the sampling rates used during // processing. Changed from uint32 to uint64 to support video livestreams longer than 49.7 days. uint64 time = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; // The absolute number of the frame in the (original) video // Different from index. Index is just the order in which frames were processed for search (and can be 0 for manual annotations) uint32 number = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; } // A Frame of time-series Data such as a Video. message Frame { // Information aboue frame such as number and time. FrameInfo frame_info = 1; // A recursive definition of the data within the Frame. For example, this will contain // data.concepts if the Frame also has annotations or predictions of concepts within it. // This can also have data.regions for annotation or predictions of detection regions, which can // then recursively have their data field filled in as well. Data data = 2; // An ID for the frame. string id = 3; } // A representation of a numpy array as a proto. // To convert a numpy array 'ndarray' to this proto do: // NDArray(buffer=ndarray.tobytes(), shape=ndarray.shape, dtype=str(ndarray.dtype)) // // To convert this proto 'ndarray_proto' to a numpy array: // array = np.frombuffer(ndarray_proto.buffer, dtype=ndarray_proto.dtype) // array = array.reshape(tuple(ndarray_proto.shape)) message NDArray { // The bytes of data from the array from array.tobytes() bytes buffer = 1; // Simply the shape of the numpy array. array.shape. repeated uint32 shape = 2; // Dtype for numpy. You can get it back from this string format using: // np.dtype('float32') for example. string dtype = 3; } // Segmentation mask. message Mask { reserved 1; // The image of the mask in a non-raster format. Image image = 2; } // Polygon message Polygon { // A list of points connected together to form the polygon. repeated Point points = 1; } // Point message Point { // The row location of the point. This has a [0.0-1.0] range with 0.0 being top row and 1.0 // being the bottom row. float row = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; // The column location of the point. This has a [0.0-1.0] range with 0.0 being left col and 1.0 // being the right col. float col = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; // Depth if applicable for the point. float z = 3; // Whether this point is visible or occluded enum Visibility { NOT_SET = 0; // Visibility of the point is not set VISIBLE = 1; // Point is visible NOT_VISIBLE = 2; // Point is occluded NOT_PRESENT = 3; // Point is not in the image } Visibility visibility = 4; } message Span { uint32 char_start = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 char_end = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; string raw_text = 3; } message Token { uint32 char_start = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 char_end = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; string raw_text = 3; } // Embedding message Embedding { repeated float vector = 1 [packed = true]; uint32 num_dimensions = 2; } // GeoPoint message GeoPoint { float longitude = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; float latitude = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; } // GeoLimit message GeoLimit { string type = 1; float value = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; } // GeoBoxedPoint message GeoBoxedPoint {GeoPoint geo_point = 1;} // Geo message Geo { GeoPoint geo_point = 1; GeoLimit geo_limit = 2; // NOTE: inconsistency: should have been geo_boxed_points repeated GeoBoxedPoint geo_box = 3; } // Image message Image { reserved 3; // This is a URL to a publicly accessible image file. The platform will download this file server // side and then process. string url = 1; // The base64 field is using image file bytes directly in the request. // NOTE: if you're sending a json request, then this MUST be base64 encoded before sending (hence // the name here). // When using our grpc clients, you DO NOT need to base64 encode // it yourself since the clients know how to do this for you automatically and will avoid the // base64 encoding if they send a binary request. bytes base64 = 2; bool allow_duplicate_url = 4; // The hosted field lists images in different sizes hosted in Clarifai storage. HostedURL hosted = 5; // image info for original size. for image info for other sizes, use hosted_image_info ImageInfo image_info = 6; // The map of hosted image info of different sizes (see hosted.sizes), excluding the original image. // Note: keys(hosted_image_info) = hosted.sizes - "orig" map hosted_image_info = 7; // For internal processing of already decoded bytes. bytes decoded_bytes = 8; } message ImageInfo { // width int32 width = 1; // height int32 height = 2; // image format string format = 3; // image color mode string color_mode = 4; // mode (when used for decoded_bytes) (RGB, RGBA, P, L, etc.) string mode = 5; } // HostedURL message HostedURL { // Prefix of the URL of every hosted image. string prefix = 1; // Suffix of an image stored in different sizes. string suffix = 2; // The sizes field lists which images of the different sizes are hosted in our storage. The URL // of each hosted image can be obtained by joining the prefix, one of the sizes and suffix. repeated string sizes = 3; // The crossorigin property of html media tag // For Secure Data Hosting this needs to be set to 'use-credentials' string crossorigin = 4; } // Input message Input { reserved 3; // The ID for the input string id = 1; // The data passed along in this input. Data data = 2; // When the input was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 4; // When the input was modified. google.protobuf.Timestamp modified_at = 5; // This is the status at a per Input level which allows for // partial failures. clarifai.api.status.Status status = 6; // List of dataset IDs that this input is part of // Currently, this field is ONLY used to // * search inputs part of dataset(s), e.g. in `PostSearches`, `PostInputsSearches` and `PostAnnotationsSearches` endpoints, and // * to add inputs to dataset(s) in `PostInputs` endpoint. // Note that this field is ignored for other endpoints, e.g. `GetInput`, `ListInputs` and `PatchInputs`. repeated string dataset_ids = 7; // Global settings for annotation tracks. InputSettings settings = 8; } message InputSettings { // Default model used for annotation generation (SAM2, etc.) // Make sure to set correct model version id, app id and user id etc. // Workflow is not supported here yet Worker worker = 1; // Sampling settings used uint32 sample_rate_ms = 2; uint32 sample_rate_frame = 3; // Pinned concept ids repeated Concept pinned_concepts = 4; } // InputBatch is a batch of Input resources. Large amounts of inputs are usually // divided into multiple InputBatches. message InputBatch {repeated Input inputs = 1;} // NOTE: inconsistency: this is weird mix of plural and singular words. message InputCount { uint32 processed = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 to_process = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 errors = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 processing = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 reindexed = 5 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 to_reindex = 6 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 reindex_errors = 7 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 reindexing = 8 [(clarifai.api.utils.cl_show_if_empty) = true]; } // Dataset message Dataset { reserved 6, 10; // The ID for the dataset string id = 1; // When the dataset was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // When the dataset was modified. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // The app the dataset belongs to. string app_id = 4; // The user the dataset belongs to. string user_id = 5; // Description of the dataset string description = 7; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 8; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 9; // Default annotation filter used for this dataset. AnnotationFilter default_annotation_filter = 12; // Default processing info used for this dataset. DatasetVersionProcessingInfo default_processing_info = 16; // Notes for the dataset // This field should be used for in-depth notes and supports up to 64Kbs. string notes = 11; // Dataset version associated with this dataset. This is used in listing Datasets // and including the latest version. DatasetVersion version = 13; // Whether the dataset is starred by the requesting user. bool is_starred = 14; // Number of users that starred this dataset. int32 star_count = 15; // bookmark info. When set, this dataset is a bookmarked dataset of this app. // Info in this field will allow you to find/access original dataset. BookmarkOrigin bookmark_origin = 17; // Representative image for this dataset Image image = 18; } // AnnotationFilter is used to create a new dataset version. // For now, the filter is simply a wrapper over a Search. // In the future, we may add extra fields to customize the filtering. message AnnotationFilter { reserved 6, 7, 8; // The ID for the annotation filter string id = 1; // When the annotation filter was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // When the annotation filter was modified. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // The user the annotation filter belongs to. string user_id = 4; // The app the annotation filter belongs to. string app_id = 5; // The search that this filter uses. Search search = 9; } // DatasetInput message DatasetInput { // When the input was added to the dataset. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 1; // The input data. Input input = 2; } enum DatasetVersionRequestOrigin { DATASET_VERSION_REQUEST_ORIGIN_NOT_SET = 0; MANUAL = 1; TRAINING = 2; EVAL_GROUND_TRUTH = 3; EVAL_PREDICTIONS = 4; } // DatasetVersion message DatasetVersion { reserved 7, 9, 11; // The ID for the dataset version string id = 1; // When the dataset version was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // When the dataset version was modified. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // The app the dataset version belongs to. string app_id = 4; // The user the dataset version belongs to. string user_id = 5; // The dataset the dataset version belongs to. string dataset_id = 6; // Data config reveals how the dataset version is generated. oneof data_config { // The dataset version will be generated based on a single annotation filter. AnnotationFilterConfig annotation_filter_config = 15; // The dataset version will be generated based on model version inferences. ModelPredictConfig model_predict_config = 18; } // Status for this dataset version. clarifai.api.status.Status status = 8; // Description of the dataset version string description = 10; // Dataset version processing. If this is not set when the dataset version is // created, then the dataset default_processing_info is copied instead. Later // updates to default_processing_info will not apply to existing versions. DatasetVersionProcessingInfo processing_info = 19; // Dataset version metrics map metrics = 16; // Dataset version exports DatasetVersionExportInfo export_info = 17; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 12; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 13; // The embedding models to return embeddings for. If empty, no embeddings are returned. repeated string embed_model_version_ids = 14; // Read Only. Cannot be Set // Origin of request for new dataset version DatasetVersionRequestOrigin request_origin = 20; } message AnnotationFilterConfig { // The annotation filter that is used. AnnotationFilter annotation_filter = 1; // If true, empty inputs are not included in the dataset version. // If false, empty inputs are included in the dataset version. // We define an empty input as an input without any annotations after annotation filter is applied. bool ignore_empty_inputs = 2; } message ModelPredictConfig { // Assumed to be owned by the calling users app unless user_id and app_id are filled out. Model model = 1; } message DatasetVersionMetrics { reserved 2, 3, 4, 5, 7; // Number of inputs google.protobuf.UInt64Value inputs_count = 1; // Number of unlabeled inputs // An input is considered unlabeled if it there are no annotations with positive labels for that input. google.protobuf.UInt64Value unlabeled_inputs_count = 6; // Number of inputs that have metadata google.protobuf.UInt64Value inputs_with_metadata_count = 8; // Number of inputs that have geo information google.protobuf.UInt64Value inputs_with_geo_count = 9; // Number of regions google.protobuf.UInt64Value regions_count = 20; // The matrix shows where the regions are located. // Example: If the matrix has 2x2 dimensions, then // * region_location_matrix[0][0] = the number of regions that appear in the top left corner, i.e. [0,0]..(0.5,0.5) // * region_location_matrix[0][1] = the number of regions that appear in the top right corner, i.e. [0,0.5]..[0.5,1] // * region_location_matrix[1][0] = the number of regions that appear in the bottom left corner, i.e. [0.5,0]..[1,0.5) // * region_location_matrix[1][1] = the number of regions that appear in the bottom right corner, i.e. [0.5,0.5]..[1,1] MatrixUint64 region_location_matrix = 21; // Number of bounding boxes google.protobuf.UInt64Value bounding_boxes_count = 22; // Number of polygons google.protobuf.UInt64Value polygons_count = 23; // Number of points google.protobuf.UInt64Value points_count = 24; // Number of masks google.protobuf.UInt64Value masks_count = 25; // Number of inputs that have regions attached // Note that this is not a recursive count: if an input contains frames that contains regions, then the region_frames_count is increased, but region_inputs_count is not increased. google.protobuf.UInt64Value region_inputs_count = 60; // Number of frames that have regions attached google.protobuf.UInt64Value region_frames_count = 61; // Number of frames google.protobuf.UInt64Value frames_count = 30; // Number of inputs that have frames attached google.protobuf.UInt64Value frame_inputs_count = 70; // Number of embeddings google.protobuf.UInt64Value embeddings_count = 40; // Number of positive tags added at input-level google.protobuf.UInt64Value positive_input_tags_count = 50; // Number of positive tags added at region-level google.protobuf.UInt64Value positive_region_tags_count = 51; // Number of positive tags added at frame-level google.protobuf.UInt64Value positive_frame_tags_count = 52; } message DatasetVersionMetricsGroup { string parent_path = 1; DatasetVersionMetricsGroupType type = 2; google.protobuf.Value value = 3; DatasetVersionMetrics metrics = 4; } enum DatasetVersionMetricsGroupType { DATASET_VERSION_METRICS_GROUP_TYPE_NOT_SET = 0; // Group data examples by input type. // Examples: images, videos, text, audio. INPUT_TYPE = 2; // Group data examples by concept ID. // Examples: inputs with cat concept, inputs with dog concept. CONCEPT_ID = 10; // Group data examples by concepts count. // Examples: inputs with 20 concepts, inputs with 21 concepts. CONCEPTS_COUNT = 11; // Group data examples by bounding boxes count. // Examples: inputs with 20 bounding boxes, inputs with 21 bounding boxes. BOUNDING_BOXES_COUNT = 20; // Group data examples by polygons count. // Examples: inputs with 20 polygons, inputs with 21 polygons. POLYGONS_COUNT = 21; // Group data examples by points count. // Examples: inputs with 20 points, inputs with 21 points. POINTS_COUNT = 22; // Group data examples by masks count. // Examples: inputs with 20 masks, inputs with 21 masks. MASKS_COUNT = 23; // Group data examples by pixels count. // In order to reduce the number of groups, we use bins. // Examples for bin size = 400: inputs with [200000, 200400) pixels, inputs with [200400, 200800) pixels. PIXELS_COUNT = 30; // Group data examples by aspect ratio. // In order to reduce the number of groups, we use bins. // Examples for bin size = 0.1: inputs with [0.5, 0.6) aspect ratio, inputs with [0.6, 0.7) aspect ratio. ASPECT_RATIO = 31; } // DatasetVersionExportInfo contains information about all exports of a dataset version. // // If the dataset version has not been exported in a format, then the DatasetVersionExport // field for that format is empty instead of having a "not exported" status. message DatasetVersionExportInfo { // clarifai_data_protobuf is a CLARIFAI_DATA_PROTOBUF export of the dataset version. DatasetVersionExport clarifai_data_protobuf = 1; // clarifai_data_json is a CLARIFAI_DATA_JSON export of the dataset version. DatasetVersionExport clarifai_data_json = 3; // coco is a COCO export of the dataset version. DatasetVersionExport coco = 2; } // DatasetVersionExport contains metadata for a single dataset version export. message DatasetVersionExport { // format is the format of the dataset version export. DatasetVersionExportFormat format = 1; // status is the current status of the dataset version export. clarifai.api.status.Status status = 2; // url is the URL from where the dataset version export can be downloaded. string url = 3; // size is the size of the dataset version export in number of bytes. uint64 size = 4; // whether to include embeddings in the export or not. bool include_embeddings = 5; } enum DatasetVersionExportFormat { DATASET_VERSION_EXPORT_FORMAT_NOT_SET = 0; // CLARIFAI_DATA_PROTOBUF is the proprietary Clarifai API Data format. It // is a ZIP-archive containing batches of serialized InputBatch protobuf messages. // // Note that only the "id" and "data" fields of exported inputs are set. CLARIFAI_DATA_PROTOBUF = 1; // CLARIFAI_DATA_JSON is the proprietary Clarifai API Data format in JSON. It // is a ZIP-archive containing batches of serialized InputBatch JSON messages. // // Note that only the "id" and "data" fields of exported inputs are set. CLARIFAI_DATA_JSON = 3; // COCO is the data format used by Common Objects in Context. It is a // ZIP-archive containing JSON files with the dataset version annotations. // See https://cocodataset.org/#format-data. COCO = 2; } // DatasetVersionProcessingInfo contains information about processing applied // to a dataset version. message DatasetVersionProcessingInfo { // If frame_interpolation_info is set, then these settings are used to // interpolate new frame annotation from other video annotations. // // If frame_interpolation_info is set in the dataset default_processing_info, // then it can be disabled for a single dataset version by setting // processing_info but not setting processing_info.frame_interpolation_info. FrameInterpolationInfo frame_interpolation_info = 1; } // FrameInterpolationInfo contains information about frame annotations // interpolated from other video annotations, such as image object-detection // regions generated from video object-tracking regions. message FrameInterpolationInfo { // sample_ms is the sampling rate at which frame annotations are interpolated. // If sample_ms is zero, then the dataset default_processing_info value is used. // If the dataset default is zero or not set, then the input frame prediction // sampling rate is used. uint32 sample_ms = 1; } // Key message Key { // The id of this key, it is used for authorization. string id = 1; // The type of key, it can be app_specific_key (default) or personal_access_token string type = 8; // The description string description = 2; // The low-level scopes this key has repeated string scopes = 3; // The endpoint-level scopes this key has repeated string endpoints = 7; // The apps that this key give you access to, it is empty if this key is personal_access_token // API key can only give you access to a single app. repeated App apps = 4; // When the key was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 5; // When does the key expires, the key won't expire if this is empty google.protobuf.Timestamp expires_at = 6; // list of idp ids at which key is currently authorized repeated string authorized_idp_ids = 9; string organization_id = 10; // The organization associated with the key, if any. This is applicable only for PAT keys. } enum LicenseType { UNKNOWN_LICENSE_TYPE = 0; FIRST_PARTY = 1; OPEN_SOURCE = 2; CLOSED_SOURCE = 3; } // This is the Model object which represents a created model in the platform. // Each model has a particular type denoted by the model_type_id. // When creating a Model with PostModels the following happens: // - if the ModelType is trainable, then a new ModelVersion is created that is // - UNTRAINED status by default // - TRAINED status if a ModelVersion was included with PretrainedModelConfig in PostModels // - if the ModelType is not trainable, then a new ModelVersion is created with TRAINED status. // To modify config settings like OutputInfo for the Model you an use PatchModels. This will // also create a new ModelVersion, potentially UNTRAINED following the same rules as above. // The fields that are patchable include Model.name, Model.display_name and Model.output_info // (except the Model.output_info.type and Model.output_info.type_ext). message Model { reserved 8, 10, 11, 12, 13, 24, 28, 39; // The model's ID. Must be unique within a particular app and URL-friendly. string id = 1; // DEPRECATED: Please use the model id to name the model. string name = 2 [deprecated = true]; // When the model was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 3; // When was the most recent model version created at google.protobuf.Timestamp modified_at = 19; // The app the model belongs to. string app_id = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; // Info about the model's output and configuration. // DEPRECATED: Will be moved to model version OutputInfo output_info = 5 [deprecated = true]; // A particular version of the model, e.g., to specify the version when creating a workflow or // when listing Models to include the latest ModelVersion of the model in the response. ModelVersion model_version = 6; // DEPRECATED: Please use the model id to name the model. string display_name = 7 [deprecated = true]; // The user id that the model belongs to. string user_id = 9; // The default evaluation info. Can be overwritten by eval request. EvalInfo default_eval_info = 30; // The ModelType.Id that is used for this model. This is used for all versions and you cannot // change model_type_id between versions of the same model. string model_type_id = 14; // The task the model was trained to do string task = 26; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 15; // Short description about this model string description = 16; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 17; google.protobuf.Struct presets = 27; // Notes for the model // This field should be used for in-depth notes and supports up to 64Kbs. string notes = 18; // Tags from toolkits category repeated string toolkits = 20; // Tags from use_cases category repeated string use_cases = 21; // Tags from languages category. repeated string languages = 25; // Tags from languages category with names, only used in responses. repeated FullTag languages_full = 31; repeated string check_consents = 32; // Is starred by the requesting user (only showed on get/list requests) // Please use PostModelStars/DeleteModelStars endpoints to star/unstar a model bool is_starred = 22; // How many users have starred the model (only showed on get/list requests) // Computed value, not editable int32 star_count = 23; // Whether it's recommended that this model is used within a workflow google.protobuf.BoolValue workflow_recommended = 29; // bookmark info. When set, this model is a bookmarked model of this app. // Info in this field will allow you to find/access original model. BookmarkOrigin bookmark_origin = 33; // Representative image for this model Image image = 34; // License Type LicenseType license_type = 35; // Source of Model enum Source { UNKNOWN_SOURCE = 0; HOSTED = 1; WRAPPED = 2; } Source source = 36; // Creator of Model string creator = 37; int32 version_count = 38; enum BillingType { Unknown = 0; Tokens = 1; Ops = 2; } BillingType billing_type = 40; // Whether the model should be featured, and if so, the order in which it should be featured. // The order is relative to other models that are also featured. // Models with a higher order will be featured first. google.protobuf.Int32Value featured_order = 41; // Deploy restriction for the model. DeployRestriction deploy_restriction = 42; // replica_count indicates the number of deployed model replicas. // This field is populated when `show_replicas` is true in ListModelsRequest. uint32 replica_count = 43; // OpenRouter Info for the model OpenRouterInfo open_router_info = 44; } // Tracks special handling reason and whether it's been done. message SpecialHandling { string id = 1; enum Reason { REASON_NOT_SET = 0; CONTACT_SALES = 1; INTERNAL_ONLY = 2; } Reason reason = 2; // Whether special handling is done. bool done = 3; } message OpenRouterInfo { google.protobuf.Struct params = 1; } enum DeployRestriction { USAGE_RESTRICTION_NOT_SET = 0; // Default uninitialized state. NO_LIMITS = 1; // No restrictions on usage. SHARED_COMPUTE_ONLY = 2; // Model can only be used on shared compute resources. DEDICATED_COMPUTE_ONLY = 3; // Model can only be used on dedicated compute resources. MODEL_OWNER_DEDICATED_COMPUTE_ONLY = 4; // Model can only be used on dedicated compute resources owned by the model owner. } // A link to a html/markdown/text file that stores reference material tied to a model. message ModelReference { // Id of the reference string id = 1; // The id of the model this Model reference is tied to. string model_id = 2; // address of resource string url = 3; // name of link string name = 4; // To handle arbitrary json metadata: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 5; } // ModelVersionInputExample message ModelVersionInputExample { // user unique id string id = 1; // external id of model string model_id = 2; // external id of model version string model_version_id = 3; // data to store as example input for model Data data = 4; // name of link for display string name = 5; // description of link contents string description = 6; } // OutputInfo defines some of the settings for each model version that PatchModels can effect. These // parameters control some of the training or inference operations that this model can do. // As the number of parameters continued to grow when we launched more ModelTypes we decided to move // to using the OutputInfo.params field which is a Struct (or JSON object if you're using // our JSON REST APIs). This allows each ModelType to define the set of fields, their default values // and description of each field so that we can display those in Portal and make the creation of // Model's very extensible. The OutputConfig object will eventually go away in favor of // output_info.params struct. message OutputInfo { reserved 4, 5; // List of concepts or other output related data for the model. Data data = 1; // Model configuration...going away in favor of output_info.params and train_params over time. // TO BE DEPRECATED OutputConfig output_config = 2; // For returning where to look for the Output info if not returning it. string message = 3; // Map from the api.Data field names to the underlying model graph's outputs. When using a // PretrainedModelConfig the values in this map need to match the Triton config.pbtxt output names. google.protobuf.Struct fields_map = 6; // For predicting with the various ModelType's we accept a Struct (JSON object) worth of args // that the ModelTypeField defines. During inference, the settings contained within are sent // to the model predictor to alter predictions from this Model. google.protobuf.Struct params = 7; // These allow you to specifcy addition fields that a specific model supports beyond those defined // in it's ModelType. This field is to be deprecated and will be replaced by MethodSignature // proto. repeated ModelTypeField params_specs = 8; } // InputInfo message InputInfo { // Map from the api.Data field names to the underlying model graph's inputs. When using a // PretrainedModelConfig the values in this map need to match the Triton config.pbtxt input names. google.protobuf.Struct fields_map = 1; // To control the inputs to the given model we allow a list of parameters // defined for each ModelType as a Struct (JSON object) here. During training or inference, the // settings contained within are sent to the training processor to alter the training process. google.protobuf.Struct params = 2; // For base model to get embeddings from for transfer learned models. Model base_embed_model = 3; } message TrainInfo { // To control the training process when PostModelVersions is used we allow a list of parameters // defined for each ModelType as a Struct (JSON object) here. During training, the settings // contained within are sent to the training processor to alter the training process. google.protobuf.Struct params = 1; // The dataset and dataset version this model version was or will be trained on Dataset dataset = 2; // The model to resume training from. Model resume_from_model = 3; } message EvalInfo { // To control the evaluation process. // Allow a list of parameters. google.protobuf.Struct params = 1; } // DEPRECATED: no longer support importing models from third-party toolkits message ImportInfo { // Used to configure model imports from third-party toolkits. // DEPRECATED: no longer support importing models from third party toolkits google.protobuf.Struct params = 1; } // OutputConfig is a collection of parameters controlling either inference or training settings for // the given Model. This message will be deprecated over time in favor or output_info.params and // train_params in OutputInfo which are cleaner and more extensible for many ModelTypes. message OutputConfig { reserved 11, 12, 16, 18; // For custom concept model training: whether the concept predictions must sum to 1. bool concepts_mutually_exclusive = 1 [deprecated = true]; // DEPRECATED: For custom models, this is the base model to use for image embeddings. // Default is general model. string existing_model_id = 3 [deprecated = true]; // For concept model predictions: Overrides the default_language for the app in a predict call. string language = 4; // DEPRECATED: Hyper-parameters for custom training. // Use new hyper_params field instead. string hyper_parameters = 5 [deprecated = true]; // For concept model predictions: Maximum number of concepts in result. Defaults to 0 which under // the hood will return default of 20. We do a server side default in order to control this // feature in the future. uint32 max_concepts = 6 [(clarifai.api.utils.cl_show_if_empty) = true]; // For concept model predictions: Minimum value of concept's probability score in result. // Defaults to 0.0 which means we won't do any thresholding as all probabilities will // likely be > 0.0. float min_value = 7 [(clarifai.api.utils.cl_show_if_empty) = true]; // For concept model predictions: Select concepts in result by name or by id repeated Concept select_concepts = 8; // For custom concept model training: Training timeout of the model (in seconds) uint32 training_timeout = 9; // For model predictions on video: Sample delay for video predicting (1 frame per N milliseconds) uint32 sample_ms = 10; // For custom model training: Hyperparameters for custom training google.protobuf.Struct hyper_params = 13; // For custom model training: this is the base model version to use for image embeddings. // This has to be one of the embed models in the app workflow. string embed_model_version_id = 14 [deprecated = true]; // Use input_info.base_embed_model instead. // For custom model training: Use this flag to fail on missing positive examples // By default we fill in the missing with random examples bool fail_on_missing_positive_examples = 15; // For custom model training: This is any additional metadata as a JSON object that we want // want to persist in the model's output config. This is a useful quick way to set fields for // introducing fields for new model types so we don't have to add a new proto field and DB field // each time. Please refer to the documentation or model implementation internally for more // details on what fields are supported for which models. // TODO(zeiler): remove this field after Portal is updated. google.protobuf.Struct model_metadata = 17 [deprecated = true]; } // ModelType is a definition of a set of models that generally have the same input and output fields. // This is used to understand more about the possible models in our platform. message ModelType { reserved 7, 4, 13, 14, 15; // A unique identifier for this model type. string id = 1; // A display title for this model. string title = 2; // Description of this model type. string description = 3; // For both input_fields and output_fields below, the following hold true: // - They are both lists of strings that name the fields from the Data proto // - Individual entries in the list can be comma-separated lists. Each element of it is expected to be present in an incoming data example. // - There is not currently a notion of fields that can be "OR separated". "OR separated" meaning any combination of the fields can be present. // - Multiple entries in the list imply that inputs come from different input sources. // - Both are used to validate which models can be chained before and after each other inside a workflow. // The list of input fields that this model expects as inputs. // Used to validate that request input data has the expected fields. repeated string input_fields = 5; // The list of output fields that this model accepts. repeated string output_fields = 6; // Is this model trainable in our platform. bool trainable = 8; // Is this model creatable. We have some pre-trained model types that users cannot create yet in // model mode. bool creatable = 9; // Is this model type only for internal users at this time. bool internal_only = 10; // The remaining fields are definitions of the configurable fields that exist. repeated ModelTypeField model_type_fields = 11; // For sequence models we need to know when processing that they require temporal time frames // in sequential order. This will be true for model types like trackers as an example. bool requires_sequential_frames = 12; // Expected input layers of an uploaded model. repeated ModelLayerInfo expected_input_layers = 16; // Expected output layers of an uploaded model repeated ModelLayerInfo expected_output_layers = 17; // What type of evaluation is supported for this model type. EvaluationType evaluation_type = 18; // method signature for this model type // This will be used in the future to replace input_fields, output_fields, and model_type_fields // as it can define any python function call. repeated MethodSignature method_signatures = 19; } message ModelLayerInfo { // The api.Data field this layer will be parsed into string data_field_name = 1; // Description of the expected shape. Can support multiple support layer shapes. repeated LayerShape shapes = 2; // Brief description about the layer if needed string description = 3; // Whether this layer should have a label_filename specified and provided bool requires_label_filename = 4; } message TritonCondaEnvInfo { string conda_pack_url = 1; string conda_yaml_url = 2; } enum DataType { UNDEFINED = 0; // Default value, should not be used STRING = 1; UINT8 = 2; INT32 = 3; INT64 = 4; FP32 = 5; } message LayerShape { // Supported dimensions // Example: [-1,4] is a 2-dimensional array with the first dimension of variablesize, but second dimension with a static size: [[1,2,3,4],[4,5,6,7],...] repeated int32 dims = 1; // Max dimension size, applicable to layers that can have flexible sizes. repeated int32 max_dims = 2; // The triton data type DataType data_type = 3; // Description about the dimensions string description = 4; } // ModelTypeField stores a field value of a configurable type. message ModelTypeField { // The path where the value of the field will be stored in the model version object. // Example: // "output_info.data" would be the Data message in the OutputInfo message. // "output_info.output_config.language" is in the OutputConfig message within OutputInfo // "input_info.params" is in the params struct within InputInfo. // "output_info.params" is in the params struct within OutputInfo. // "train_info.params" is in the params struct within TrainInfo. // and so on. string path = 1; // These are various types of fields that we have UIs for. enum ModelTypeFieldType { reserved 6; INVALID_MODEL_TYPE_FIELD_TYPE = 0; BOOLEAN = 1; STRING = 2; NUMBER = 3; // For auto-completing to concepts in the app. This goes into an data.concepts field. ARRAY_OF_CONCEPTS = 4; // For auto-completing to concepts in the app. This goes into an data.concepts field. ARRAY_OF_CONCEPTS_WITH_THRESHOLD = 5; // A range for a float value. RANGE = 7; // If ENUM is used then the "enum_options" field should also be filled in with the respective ID and description // for the different ENUM options. ENUM = 8; // For listing collaborators of the app. The field is a string of the collaborator's user_id. COLLABORATORS = 9; // For arbitrary json object: "{...}" JSON = 10; // Such as [1.0, 2.0, 3.5] ARRAY_OF_NUMBERS = 11; // For selecting the embed_model_version_id for context based models. WORKFLOW_EMBED_MODELS = 12; // Such as ['a', 'b', 'cantaloupe'] ARRAY_OF_STRINGS = 13; // If RECURSIVE_ENUM is used then the "enum_options" field should also be filled in with the respective ID and // description for the different RECURSIVE_ENUM options, as well as model_type_fields for each enum choice. RECURSIVE_ENUM = 14; // For blocks of code that need to be specified by the user for setup or execution during workflow runs. PYTHON_CODE = 15; // For selecting a dataset id in model parameters. String in API request. DATASET_ID = 16; // For selecting a dataset version id. String. DATASET_VERSION_ID = 17; // For auto-completing to concepts in the model. ARRAY_OF_MODEL_CONCEPTS = 18; // For selecting a dataset DATASET = 19; // For selecting a dataset version DATASET_VERSION = 20; // To pass a string downstream, that is encrypted in the DB and API. ENCRYPTED_STRING = 21; // For selecting a model version of the same model type to resume training from. CHECKPOINT_MODEL = 22; // For model secrets ARRAY_OF_SECRETS = 23; } // The field for this field. This is often used for displaying the field in the UI whereas // the DataType enum below defines the specific type of datain the Python function. ModelTypeFieldType field_type = 2; // A default value. We use the Value field because we want to have structured data (just like // google.protobuf.Struct but this is just a single value). google.protobuf.Value default_value = 3; // Description for this field. string description = 4; // Placeholder text for the UI element. string placeholder = 5; // List of options of the ENUM type and potentially additional fields they bring with them. repeated ModelTypeEnumOption model_type_enum_options = 6; // If this field should appear for internal users only. bool internal_only = 7; // If this field is a required field. If True then during validation you won't be able to create // a model of this type with providing a value for this field. When False, the ModelType's // default_value will be used for this field. bool required = 8; // If the field_type is RANGE, this must be filled in. ModelTypeRangeInfo model_type_range_info = 9; ///////////////////////////////////// // New fields for MethodSignature ///////////////////////////////////// // The fields below were added when we introduced ability to call an arbitrary python function // from the client side. This is used to define the method signature of the python function // that we want to call. The above fields are still relevant to decide what a UI should display. // Some of the above fields above will be relevant like definition, required, etc. // name of method signature argument string name = 10; // DataType is used in MethodSignature to define all the possible types that a python function // may have that we want to support. These include built-ins like int, float, str, bool, and // more complex types like JSON, numpy arrays, List, Tuple, Dict (as Named Fields), as well as Clarifai provided // unstructured types like Image, Video, Text, etc. enum DataType { NOT_SET = 0; // A string value. STR = 1; // A byte string. This is used for binary data. BYTES = 2; // An integer value. INT = 3; // A float value. FLOAT = 4; // A boolean value. BOOL = 5; // A proto representation for numpy arrays. NDARRAY = 6; // For arbitrary json object: "{...}" JSON_DATA = 7; // For text data TEXT = 8; // A image is a image proto for url or bytes. IMAGE = 9; // A concept is a concept proto that represents a concept in the app. CONCEPT = 10; // A region is a bounding box in an image or video frame. REGION = 11; // A frame is a single image in a video stream FRAME = 12; // A audio is a audio proto for url or bytes. AUDIO = 13; // A video is a video proto for url or bytes. VIDEO = 14; // this can be used to store named fields with values similar to Dict NAMED_FIELDS = 15; // An arg that is a tuple. TUPLE = 16; // An arg that is a list. LIST = 17; } // The type of the argument. DataType type = 11; // type enum, and recursively set type_args with // the inner type argumets in complex objects (e.g. List[Tuple[int, str]]) repeated ModelTypeField type_args = 12; // this will be use to define whether the method argument supports streaming as an iterator. bool iterator = 13; // This specify the default value of the method argument // If this argument is not passed, the input is required. If it is explicitly set to None, the input is optional // We define this as a string // because the default value can be a string, int, float, bool, or a complex object like a JSON // The default_value field above should not also be used. string default = 14; // wheather it's a inference param or a modeltype field bool is_param = 15; ///////////////////////////////////// } // ModelTypeRangeInfo message ModelTypeRangeInfo { // The start of the range as a float. float min = 1; // The end of the range as a float. float max = 2; // An optional step size for the range. If provided then only values at that step size will be // rounded to. For example if step is 0.02 then 0.0245 will round to 0.02. float step = 3; } // ModelTypeEnumOption message ModelTypeEnumOption { // The unique value of the enum option. string id = 1; // List of other ID values that are equivalent with this ID. // This allows the user to choose this option by multiple IDs. // Example: if enum is "Phone Number Prefix", you could add an option that is selectable by two values: // 1. ID: "Estonia" // 2. Alias: 37 repeated ModelTypeEnumOptionAlias aliases = 5; // Optional description for this enum option. string description = 2; // These are additional fields that are specific to this enum choice. This allows // us to use enums to control configuration settings as well. repeated ModelTypeField model_type_fields = 3; // If this enum option should be internal only. bool internal_only = 4; // Whether this is the recommended enum option. Set to `true` when there // are multiple options, and one is shown to be better than the others. bool recommended = 6; } message ModelTypeEnumOptionAlias { // Integer alias for id. int64 id_int = 1; // String that can contain wild cards and the regex needs to match. string wildcard_string = 2; } // ModelQuery message ModelQuery { reserved 2; // The name ofthe field. This supports wilcard queries like "gen*" to match "general" as an example. string name = 1; // Filter models by the specific model_type_id. See ListModelTypes for the list of ModelType.Id's // supported. string model_type_id = 3; } enum ValueComparator { CONCEPT_THRESHOLD_NOT_SET = 0; // input > value GREATER_THAN = 1; // input >= value GREATER_THAN_OR_EQUAL = 2; // input < value LESS_THAN = 3; // input <= value LESS_THAN_OR_EQUAL = 4; // input == value EQUAL = 5; } enum EvaluationType { Undefined = 0; Classification = 1; // default Detection = 2; Segmentation = 3; Clustering = 4; Tracker = 5; Generation = 6; } // ModelVersion message ModelVersion { reserved 9, 18; string id = 1; // When the version was created. google.protobuf.Timestamp created_at = 2; // The status of the version (whether it's untrained, training, trained, etc.). clarifai.api.status.Status status = 3; uint32 active_concept_count = 4; EvalMetrics metrics = 5; // number of inputs in the model version uint32 total_input_count = 6; // When a model has already been trained externally, you can upload // it directly to the platform as a model version directly by // setting the PretrainedModelConfig PretrainedModelConfig pretrained_model_config = 7; // Detailed training stats. // When training of this version was completed. google.protobuf.Timestamp completed_at = 10; // Description about this version string description = 11; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 12; // The app the model version belongs to. string app_id = 13; // The user the model version belongs to. string user_id = 14; // When this model version was last modified google.protobuf.Timestamp modified_at = 15; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 16; string license = 17; // Info about the model's output. Besides `output_info.data`, these fields should // be reserved for parameters that affect the models outputs when inferencing. // `output_info.data` is used to specify the training concepts for this model version. OutputInfo output_info = 19; // Info about preprocessing the models inputs, before they are sent to this model for training or inferencing. // E.g.: `input_info.base_embed_model` lets us know inputs should be ran through a base model before being sent to an embedding-classifier. InputInfo input_info = 20; // Configuration for the training process of this model version. TrainInfo train_info = 21; // Configuration used to import model from third-party toolkits // DEPRECATED: no longer support importing models from third party toolkits ImportInfo import_info = 22 [deprecated = true]; // Contains the training logs if available string train_log = 23; // The minimum required compute resource for this model for inference. ComputeInfo inference_compute_info = 24; // Build information for the model version BuildInfo build_info = 25; // Model signature information for the model version repeated MethodSignature method_signatures = 26; // List of special handling instructions for this model version. repeated SpecialHandling special_handling = 27; // The number of threads to use for this model version. int32 num_threads = 28; } // MethodSignature is a definition of a method that a model can have. // This is used to communicate between a python method definition of any arbitrary function // to the client or UI on how to call that function from the client side. message MethodSignature { // The name of the method on the server. string name = 1; // whether the method is for predict(unary-unary), generate(unary-stream), stream(stream-stream) RunnerMethodType method_type = 2; // description from the docstring of the method on the server. string description = 3; // input fields and signature of every method arguments repeated ModelTypeField input_fields = 4; // output signature of method repeated ModelTypeField output_fields = 5; } message BuildInfo { // Docker image name string docker_image_name = 1; // Docker image tag string docker_image_tag = 2; // Docker image digest string docker_image_digest = 3; // Platform(s) the model was built for (e.g., "linux/amd64,linux/arm64") string platform = 4; } // ModelVersionExport contains metadata for a single Model version export. message ModelVersionExport { // status is the current status of the dataset version export. clarifai.api.status.Status status = 1; // url is the URL from where the model version export can be downloaded. string url = 2; // size of model file int64 size = 3; } // PretrainedModelConfig message PretrainedModelConfig { reserved 2, 5; // This is the internal id of the pretrained model. // Map from the api.Data field names to the Triton config.pbtxt input. google.protobuf.Struct input_fields_map = 3; // Map from the api.Data field names to the Triton config.pbtxt output. google.protobuf.Struct output_fields_map = 4; // Url to a zipped up model in triton format with the following files and folders at the root: // config.pbtxt // version 1 folder that contains model files (onnx graph, torch script, python BE model, and etc.) string model_zip_url = 6; // Whether to overwrite the model for the existing internal id // If this is a local dev model that runs external to the platform, set this to true. // This helps during development of models before uploading them to the platform. // These models MUST run in an associated compute cluster with cluster_type = "local-dev" bool local_dev = 8; } // TrainStats message TrainStats {repeated LossCurveEntry loss_curve = 1;} // LossCurveEntry message LossCurveEntry { // current epoch uint32 epoch = 1; // current global step uint32 global_step = 2; // current cost // FIXME(rigel): this should be loss instead of cost. float cost = 3; } // LabelCount message LabelCount { string concept_name = 1 [deprecated = true]; uint32 count = 2; Concept concept = 3; } // LabelDistribution message LabelDistribution {repeated LabelCount positive_label_counts = 1;} // NOTE: this is inefficient, should just have the order of the rows/cols message CooccurrenceMatrixEntry { // concept_id for the row string row = 1; // concept_id for the col string col = 2; uint32 count = 3; } // CooccurrenceMatrix message CooccurrenceMatrix { repeated CooccurrenceMatrixEntry matrix = 1; // These concept_ids are ordered by the strength of the diagonal in the ConfusionMatrix. repeated string concept_ids = 2; } // ConfusionMatrixEntry message ConfusionMatrixEntry { string predicted = 1; string actual = 2; float value = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; Concept predicted_concept = 5; Concept actual_concept = 6; } // ConfusionMatrix message ConfusionMatrix { repeated ConfusionMatrixEntry matrix = 1; // These concept_ids are ordered by the strength of the diagonal in the ConfusionMatrix. repeated string concept_ids = 2; } // ROC message ROC { repeated float fpr = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; repeated float tpr = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; repeated float thresholds = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; repeated float fpr_per_image = 4; repeated float fpr_per_object = 5; } // PrecisionRecallCurve message PrecisionRecallCurve { repeated float recall = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; repeated float precision = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; repeated float thresholds = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; } // BinaryMetrics message BinaryMetrics { uint32 num_pos = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 num_neg = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 num_tot = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; float roc_auc = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; float f1 = 5 [(clarifai.api.utils.cl_show_if_empty) = true]; Concept concept = 6; ROC roc_curve = 7; PrecisionRecallCurve precision_recall_curve = 8; float avg_precision = 9; string area_name = 10; double area_min = 11; double area_max = 12; float iou = 13; } // TrackerMetrics message TrackerMetrics { // Multiple object tracking accuracy float mot_mota = 1; // Number of switches between tracks int32 mot_num_switches = 2; // MORSE fragmentation rate (a.k.a unique switch rate, only calculated in public sector) float morse_frag = 3; // Average precision calculated from all processed frames float avg_precision = 4; // The concept that we are evaluating the tracker string aiid = 5; // Same as morse_frag but calculated using MOT mapping/metrics float unique_switch_rate = 6; } // EvalTestSetEntry message EvalTestSetEntry { reserved 1, 2; Input input = 6; // the input information repeated Concept predicted_concepts = 3; // All the ground truth concepts will be show on the top level repeated Concept ground_truth_concepts = 4; // Only region-based/frame-based app contains this annotation // Each annotation only contains one region // And the concepts is in ground_truth_concepts instead of this annotation Annotation annotation = 5 [deprecated = true]; // For region based models, region and associated concepts are stored together. Annotation predicted_annotation = 7; Annotation ground_truth_annotation = 8; } // MetricsSummary message MetricsSummary { float top1_accuracy = 1 [deprecated = true]; float top5_accuracy = 2 [deprecated = true]; float macro_avg_roc_auc = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; float macro_std_roc_auc = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; float macro_avg_f1_score = 5 [(clarifai.api.utils.cl_show_if_empty) = true]; float macro_std_f1_score = 6 [(clarifai.api.utils.cl_show_if_empty) = true]; float macro_avg_precision = 7 [(clarifai.api.utils.cl_show_if_empty) = true]; float macro_avg_recall = 8 [(clarifai.api.utils.cl_show_if_empty) = true]; float mean_avg_precision_iou_50 = 10; float mean_avg_precision_iou_range = 11; } // EvalMetrics message EvalMetrics { clarifai.api.status.Status status = 1; // user id that owns this evaluation string user_id = 15; // app id that owns this evaluation string app_id = 16; // Id of this evaluation string id = 10; // Model to evaluate Model model = 13; // The ground truth dataset Dataset ground_truth_dataset = 14; // The dataset with predictions Dataset predictions_dataset = 18; MetricsSummary summary = 2; ConfusionMatrix confusion_matrix = 3; CooccurrenceMatrix cooccurrence_matrix = 4; LabelDistribution label_counts = 5; repeated BinaryMetrics binary_metrics = 6; repeated EvalTestSetEntry test_set = 7; repeated BinaryMetrics metrics_by_area = 8; repeated BinaryMetrics metrics_by_class = 9; repeated TrackerMetrics tracker_metrics = 11; // Evaluation parameters to pass. Expected to match what // is defined in the model type for the respective model. EvalInfo eval_info = 12; ExtendedMetrics extended_metrics = 17; } message ExtendedMetrics { google.protobuf.Struct user_metrics = 1; } // FieldsValue message FieldsValue { bool confusion_matrix = 1; bool cooccurrence_matrix = 2; bool label_counts = 3; bool binary_metrics = 4; bool test_set = 5; bool metrics_by_area = 6; bool metrics_by_class = 7; } // Output message Output { // One of these outputs per Input string id = 1; clarifai.api.status.Status status = 2; // When the object was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 3; // The model that created this Output. Model model = 4; // The input that was passed to the model to create this Output. For example if we have an image // model then it will take as input here an Input object with Image filled in. Input input = 5; // The output data for this Output. For example if we have a concept model then the predicted // concepts will appear here. Data data = 6; // Number of prompt tokens as reported by the model or third-party API. uint32 prompt_tokens = 7; // Number of completion tokens as reported by the model or third-party API. uint32 completion_tokens = 8; // Number of cached prompt tokens as reported by the model (subset of prompt_tokens). // Cached tokens are prompt tokens served from the model's KV cache rather than recomputed. uint32 cached_tokens = 9; } // ScopeDeps message ScopeDeps { // The scope string scope = 1; // Other scopes that are required. repeated string depending_scopes = 2; } // EndpointDeps message EndpointDeps { // The fully qualified endpoint to string endpoint = 1; // Other scopes that are required. repeated string depending_scopes = 2; } // Hit message Hit { // This is the score for the ranked Hit results of the search query. This score is a number // between 0.0 and 1.0 as it represents a confidence in the search Hit. For example, if you search // for "car" and get a close matching Hit, the score should be close to 1.0. If you get a score // of close to 0.0 that means it's very disimilar to your query, in this case NOT a "car". There // is a special intermediate score of 0.5 that means that the Hit is not really correlated with // your search query (ie. not similar or dissimlar to the query) which is a common occurrence // when using negate queries. // Note: some queries that are just filtering down your app of inputs may just return a score of // 1.0 for all Hits. float score = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; // This is the matched input returned from the search query. This will contain information about // the Input such as the url, created_at time and trusted annotation information (for backwards // compatibility with apps that existed before Annotations were introduced. Input input = 2; // We also provide back the specific matched annotation for the above input. We do this in order // to support more complex Annotation queries in the And message below. For example if we match // the search results to a region in your input, or a frame in a video input, this annotation // field will be that matched annotation info and the input will be the image/video that the user // originally added which contains those regions / frames. Annotation annotation = 3; // The customer-facing id of the user who owns the app the asset came from. string user_id = 4; // The cfid of the app the asset came from. string app_id = 5; } message HitCount { // The estimated total number of hits for the search query, not just the current page. uint64 estimated_total = 1; } // This is the common building block of a query which is a sequence of And messages ANDed together. // Note that some fields are used too RANK results (affect the scores) and some are used to FILTER // results (unordered subset of your app's contents). In general, FILTER operations are more // efficient queries at scale and when combined with RANK operations can speed up search performance // as you effectively operate on a smaller sub-set of your entire app. message And { // FILTER by input.data... information. // This can include human provided concepts, geo location info, metadata, etc. // This is effectively searching over only the trusted annotation attached to an input in your // app. To search by more specific annotation fields use the Annotation object here. // ########## Supported fields ########## // - data.concepts[].id // - data.concepts[].name // - data.concepts[].value // - data.geo.geo_box[].geo_point.latitude // - data.geo.geo_box[].geo_point.longitude // - data.geo.geo_limit.type // - data.geo.geo_limit.value // - data.geo.geo_point.latitude // - data.geo.geo_point.longitude // - data.image.url // - data.metadata.fields - filter by metadata. metadata key&value fields are OR-ed. // - dataset_ids[] - filter by dataset IDs // - id - filter by input ID // - status.code - filter by input status Input input = 1; // RANK based predicted outputs from models such as custom trained models, pre-trained models, // etc. This is also where you enter the image url for a visual search because what we're asking // the system to do is find output embedding most visually similar to the provided input (that // input being in And.output.input.data.image.url for example). This will return the Hits // sorted by visual similarity (1.0 being very similar or exact match and 0.0 being very // dissimlar). For a search by Output concept, this means we're asking the system to rank // the Hits by confidence of our model's predicted Outputs. So for example if the model // predicts an image is 0.95 likely there is a "dog" present, that should related directly // to the score returned if you search for Output concept "dog" in your query. This provides // a natural ranking to search results based on confidence of predictions from the models and // is used when ANDing multiple of these types of RANK by Output queries together as well. // // ########## Supported fields ########## // - data.clusters[].id // - data.concepts[].id // - data.concepts[].name // - data.concepts[].value // - input.data.image.base64[] // - input.data.image.url // - input.id Output output = 2; // If True then this will flip the meaning of this part of the // query. This allow for queries such as dog AND ! metadata=={"blah":"value"} bool negate = 3; // FILTER by annotation information. This is more flexible than just filtering by // Input information because in the general case each input can have several annotations. // Some example use cases for filtering by annotations: // 1) find all the inputs annotated "dog" by worker_id = "XYZ" // 2) find all the annotations associated with embed_model_version_id = "123" // 3) find all the annotations that are trusted, etc. // // Since all the annotations under the hood are joined to the embedding model's annotation // using worker_id's of other models like cluster models or concept models should be // combinable with queries like visual search (a query with Output filled in). // // ########## Supported fields ########## // - annotation_info.fields - filter by annotation info // - data.concepts[].id // - data.concepts[].name // - data.concepts[].value // - data.geo.geo_box[].geo_point.latitude // - data.geo.geo_box[].geo_point.longitude // - data.geo.geo_limit.type // - data.geo.geo_limit.value // - data.geo.geo_point.latitude // - data.geo.geo_point.longitude // - data.image.url // - data.metadata.fields - filter by metadata. metadata key&value fields are OR-ed. // - input_id // - input_level // - model_version_id // - status.code // - task_id // - trusted // - user_id Annotation annotation = 4; } // This is the search query used in /searches, model training requests, bulk data exports, etc. message Query { // The query syntax is simply a list of And operatiosn that will be ANDed together to fetch // results which are returned to the user as Hit messages. // // Deprecated: Only used by the deprecated PostSearches endpoint. Use filters // and ranks instead with PostInputsSearches or PostAnnotationsSearches. repeated And ands = 1 [deprecated = true]; // This allows the query to override any default language the app was setup in when doing Concept // based searches. This currently only affects public Models Output searches when those public // Models have translations for their Concepts. string language = 2; // filters in this query // e.q. only fetch annotations that have certain metadata repeated Filter filters = 3; // rankings in this query // e.g. visual search by a url repeated Rank ranks = 4; } // This is the new Search object used in saved searches. message Search { // Search query. Query query = 1; // Customer facing, external ID for search to be saved. Provided by the user, e.g. "saved-search-1. // It is unique per application. string id = 2; // Application that owns this saved search. string application_id = 3; // Human readable display name of the saved search. string name = 4; // "As of" timestamp, indicating a time in the past as of which we want to // retrieve the annotations satisfying the query. google.protobuf.Timestamp as_of = 5; // Git hash of the code that ran the filter. string git_hash = 6; // When the saved search was created. google.protobuf.Timestamp created_at = 7; // When the saved search was updated. google.protobuf.Timestamp modified_at = 8; // The search algorithm to be used. // Options are are 'nearest_neighbor', 'brute_force', and 'avg_concept_brute_force' // The last two perform a brute force search visual search instead of a more scalable distributed // nearest neighbor search and should be used by advanced users only. // If not specified we default to nearest neighbor string algorithm = 9; // If true, save this search, and exit without executing the search. // If false execute the query bool save = 10; // Minimum value of confidence threshold score in result. // Defaults to 0.0 which means we won't do any thresholding as all probabilities will // likely be > 0.0. float min_value = 11; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 12; // Metric used for search. Can be EUCLIDEAN_DISTANCE (default) or COSINE_DISTANCE. // Currently only brute force search supports non-eudlicean metrics. enum Metric { METRIC_NOT_SET = 0; EUCLIDEAN_DISTANCE = 1; COSINE_DISTANCE = 2; } Metric metric = 13; } // Filter message Filter { // If True then this will flip the meaning of this part of the // query. This allow for queries such as dog AND ! metadata=={"blah":"value"} bool negate = 3; // FILTER by annotation information. // ########## Supported fields ########## // # Filter by ID fields // - id - example: `{"id": "xyz"}` // - input_id // - model_version_id // - task_id // - user_id // // # Filter by worker fields such as model, workflow and user IDs // - worker.model.model_version.id // - worker.user.id // - worker.workflow.version.id // // # Filter by other top-level fields // - annotation_info.fields - filter by annotation info // - input_level - filter only input-level annotations // - status.code - filter by annotation status code // - trusted - filter only trusted annotations // // # Filter by space-time info fields, i.e. region, frames and time-segments // - data - filter only annotations without space-time info, e.g. classifications // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {}}` // - data.frames[].frame_info - filter only frame annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"frames": [{"frame_info": {}}]}}` // - data.regions[].region_info.bounding_box - filter only bounding box annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"regions": [{"region_info": {"bounding_box":{}}}]}}` // - data.regions[].region_info.mask - filter only mask annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"regions": [{"region_info": {"mask":{}}}]}}` // - data.regions[].region_info.point - filter only point annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"regions": [{"region_info": {"point":{}}}]}}` // - data.regions[].region_info.polygon - filter only polygon annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"regions": [{"region_info": {"polygon":{}}}]}}` // - data.regions[].region_info.span - filter only span annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"regions": [{"region_info": {"span":{}}}]}}` // - data.regions[].track_id - filter annotations by track_id // - in order to enable this, you need to provide "track_id_value" i.e. `{"data": {"regions": [{"track_id" : "track_id_value"}]}}` // - data.time_segments[].time_info - filter only time-segment annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"time_segments": [{"time_info": {}}]}}` // // # Filter by other data fields // - data.clusters[].id // - data.concepts[].id // - data.concepts[].name // - data.concepts[].value // - data.geo.geo_box[].geo_point.latitude // - data.geo.geo_box[].geo_point.longitude // - data.geo.geo_limit.type // - data.geo.geo_limit.value // - data.geo.geo_point.latitude // - data.geo.geo_point.longitude // - data.metadata.fields - filter by metadata // - Important to note: metadata key&value fields are OR-ed. // - example with 1 metadata key: searching by // `{ // ` "data": { // ` "metadata": { // ` "fields": { // ` "foo": { // ` "string_value": "bar" // ` }, // ` } // ` } // ` } // `} // will result in a search condition like `metadata includes {"foo": "bar}`; // - example with 2 metadata keys: searching by // `{ // ` "data": { // ` "metadata": { // ` "fields": { // ` "foo1": { // ` "string_value": "bar2" // ` }, // ` "foo2": { // ` "string_value": "bar2" // ` } // ` } // ` } // ` } // `} // will result in a search condition like `(metadata includes {"foo1": "bar1"}) OR (metadata includes {"foo2": "bar2"})`. // - data.text - filter only text annotations // - in order to enable this, you need to set the field to an empty object, i.e. `{"data": {"text": {}}}` Annotation annotation = 4; // FILTER by input information. // ########## Supported fields ########## // - data.audio - filter only audio inputs // - in order to enable this, you need to set the field to an empty object, i.e. `{"audio": {}}` // - data.image - filter only image inputs // - enable using `{"image": {}}` // - data.text - filter only text inputs // - enable using `{"text": {}}` // - data.video - filter only video inputs // - enable using `{"video": {}}` // - dataset_ids[] - filter by dataset IDs // - example: `{"dataset_ids": ["d1", "d2"]}` will filter for inputs in d1 OR d2 // - status.code - filter by input status // - example: `{"status": {"code": 30000}}` to filter only for SUCCESS inputs Input input = 5; // Filter by annotation last updated time range. TimeRange last_updated_time_range = 6; } // TimeRange message TimeRange { google.protobuf.Timestamp start_time = 1; // Begin of the time range, optional, inclusive. google.protobuf.Timestamp end_time = 2; // End of the time range, optional, inclusive. } // Rank message Rank { // If True then this will flip the meaning of this part of the // query. This allow for queries such as !dog bool negate = 3; // RANK by annotation information. // ########## Supported fields ########## // - data.concepts[].id // - data.concepts[].name // - data.concepts[].value // - data.embeddings[].num_dimensions // - data.embeddings[].vector[] // - data.image.base64[] // - data.image.url // - data.text.raw // - input_id // - model_version_id Annotation annotation = 4; } // Text message Text { // This is a raw text string. string raw = 1; // Url to a text file string url = 2; bool allow_duplicate_url = 3; // The hosted field lists original text hosted in Clarifai storage. This field is currently used // only in response. HostedURL hosted = 4; // text info TextInfo text_info = 5; } message TextInfo { // count of characters in text int32 char_count = 1; // text encoding string encoding = 2; } enum APIEventType { API_EVENT_TYPE_NOT_SET = 0; // On Prem event types ON_PREM_PREDICT = 1; ON_PREM_TRAIN = 2; ON_PREM_SEARCH = 3; // Platform event types } enum UsageIntervalType { // undef UsageIntervalType is so that the interval field can be forced to be included undef = 0; day = 1; month = 2; year = 3; } // User message User { reserved 13, 14; string id = 1; string primary_email = 2 [deprecated = true]; string first_name = 3; string last_name = 4; string company_name = 5; string job_title = 19; string job_role = 20; // This specifies user intent when registering on clarifai string intention = 24; // This specifies how one got to know about clarifai string referral_source = 25; string bill_type = 7 [deprecated = true]; // When the user was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 6; google.protobuf.Timestamp date_gdpr_consent = 8 [deprecated = true]; google.protobuf.Timestamp date_tos_consent = 9 [deprecated = true]; google.protobuf.Timestamp date_marketing_consent = 10 [deprecated = true]; google.protobuf.Timestamp date_pii_consent = 23 [deprecated = true]; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 11 [deprecated = true]; repeated EmailAddress email_addresses = 12 [deprecated = true]; bool two_factor_auth_enabled = 15 [deprecated = true]; uint32 teams_count = 16 [deprecated = true]; // Is starred by the requesting user (only showed on get/list requests) // Please use PostUserStars/DeleteUserStars endpoints to star/unstar an user bool is_starred = 21; // How many users have starred the user (only showed on get/list requests) // Computed value, not editable int32 star_count = 22; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 17; // This is all the personal information of a user. GetUser/ListUsers will not return this // information unless the caller has the UserAccounts_Get scope on their key or is the user // themselves. UserDetail user_detail = 18; // Representative image for this User (a.k.a. Profile or cover photo) Image image = 26; } // This message holds the confidential information from the User object that we don't want to expose // to other users. It will be accessible only from /users/{user_id}/account and with the User scopes. message UserDetail { reserved 8; string primary_email = 1; string bill_type = 2; google.protobuf.Timestamp date_gdpr_consent = 3; google.protobuf.Timestamp date_tos_consent = 4; google.protobuf.Timestamp date_marketing_consent = 5; google.protobuf.Timestamp date_pii_consent = 13; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 6; repeated EmailAddress email_addresses = 7; bool two_factor_auth_enabled = 9; uint32 teams_count = 10; string country = 11; string state = 12; CommitmentValue commitment_value = 14; // For phone number verification, true if the phone number has been verified bool phone_verified = 15; } // EmailAddress message EmailAddress { string email = 1 [(clarifai.api.utils.cl_show_if_empty) = true]; bool primary = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; bool verified = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; } // Password message Password { // unencrypted password string string plaintext = 1; } // PasswordViolations message PasswordViolations { // when new password length is shorter than minimum length set bool minimum_length = 1; // when new password length is longer than maximum length set bool maximum_length = 2; // there is no upper case letter in the new password when there should be at least one bool upper_case_needed = 3; // there is no lower case letter in the new password when there should be at least one bool lower_case_needed = 4; // there is no numerics in the new password when there should be at least one bool numeric_needed = 5; // there is no special character in the new password when there should be at least one bool non_alphanumeric_needed = 6; // when one of the N most recent old password is reused, N is specified by password_reuse_epoch in db.password_policies bool password_reuse = 7; // when either user's first, middle or last name is used in the new password bool exclude_names = 8; // when first part of user's email (exact string or after removing special characters) is used in the new password bool exclude_email = 9; // when there are confusing letters in the new password, such as o (first character of 'omega') vs 0 (zero) bool no_confusing_letters = 10; // when there are simple password patterns used, such as 12345678 or aaaaaaa1 bool no_simple_passwords = 11; // when there are common vocabs from the common vocab list used bool no_common_vocabs = 12; // when the current password is contained in the new password or vice versa bool no_overlap_with_old = 13; // when password has to be changed becauase it's too old bool password_lifespan = 14; } // Commitment Value - used to track users' commitment amount and type message CommitmentValue { // The commitment value, in cents int32 value = 1; // The commitment type, can be either 'monthly' or 'annual' CommitmentType type = 2; enum CommitmentType { TYPE_NOT_SET = 0; MONTHLY = 1; ANNUAL = 2; } } // Video message Video { // This is a URL to a publicly accessible video file. The platform will download this file server // side and then process. string url = 1; // The base64 field is using video file bytes directly in the request. // NOTE: if you're sending a json request, then this MUST be base64 encoded before sending (hence // the name here). // When using our grpc clients, you DO NOT need to base64 encode // it yourself since the clients know how to do this for you automatically and will avoid the // base64 encoding if they send a binary request. bytes base64 = 2; bool allow_duplicate_url = 4; // URL of thumbnail image, which is currently frame at position of 1s. This field is currently // used only in response. // Deprecated in favour of thumbnail_hosted, which also contains alternate sizes of thumbnail string thumbnail_url = 5 [deprecated = true]; // The hosted field lists original video hosted in Clarifai storage. This field is currently used // only in response. HostedURL hosted = 6; // The hosted field lists various sizes of the vide thumbnail hosted in Clarifai storage, with 'thumbnail' as the full size // This field is currently used only in response. HostedURL hosted_thumbnail = 8; // video info VideoInfo video_info = 7; } message VideoInfo { // width int32 width = 1; // height int32 height = 2; // Frames per second of the video. float fps = 3; // video format string video_format = 4; // video track bit rate int32 bit_rate = 5; // video frame count int32 frame_count = 6; // video duration in seconds float duration_seconds = 7; } // Workflow message Workflow { // The workflows's unique id. string id = 1; // The app the workflow belongs to string app_id = 2; // When the workflow was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 3; // The list of nodes retrieved from latest workflow version. // Each node can specify an input node that it connects to in order to define the graph. repeated WorkflowNode nodes = 4; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 5; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 6; // The user the workflow belongs to string user_id = 7; // When the workflow was last modified google.protobuf.Timestamp modified_at = 8; // Info about the workflow version used to return the latest version when listing Workflows. WorkflowVersion version = 9; // Is starred by the requesting user (only showed on get/list requests) // Please use PostWorkflowStars/DeleteWorkflowStars endpoints to star/unstar a workflow bool is_starred = 10; // How many users have starred the workflow (only showed on get/list requests) // Computed value, not editable int32 star_count = 11; // Short description about this workflow string description = 12; // Notes for the workflow // This field should be used for in-depth notes and supports up to 64Kbs. string notes = 13; // Tags from use_cases category repeated string use_cases = 14 [(clarifai.api.utils.cl_show_if_empty) = true]; // Tags for check consents repeated string check_consents = 15 [(clarifai.api.utils.cl_show_if_empty) = true]; // bookmark info. When set, this workflow is a bookmarked workflow of this app. // Info in this field will allow you to find/access original workflow. BookmarkOrigin bookmark_origin = 16; // Representative image for this workflow Image image = 17; } // WorkflowVersion message WorkflowVersion { // Id of this version. string id = 1; // Workflow id for this version. string workflow_id = 2; // When the version was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 3; // Most recent time when the version was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 4; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 5; // The list of nodes that make up the workflow version. Each node can specify an input node // that it connects to in order to define the graph. repeated WorkflowNode nodes = 6; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 7; // The app the workflow version belongs to. string app_id = 8; // The user the workflow version belongs to. string user_id = 9; // Short description about this workflow version string description = 10; // License associated to this workflow version string license = 11; // If a model version associated with the workflow version is deleted, the workflow version // will be marked as deprecated. bool is_deprecated = 12; } // WorkflowNode message WorkflowNode { // An identifier for this node in the graph. This is used when connecting NodeInputs // together. string id = 1; // The model that will do the processing at this node. We only vlidate the model.id and // model.model_version.id fields. Model model = 2; // Each WorkflowNode can connect to multiple input nodes so that we can handle multi-model data // and more complex workflow operations. repeated NodeInput node_inputs = 3; // suppress the output for workflow prediction bool suppress_output = 4; // Used to override the output_info.data and output_info.params of the model specified by the node. // Values for fields_map, message, and output_config are ignored. OutputInfo output_info_override = 5; } // NodeInput represents inputs to a node of the graph. message NodeInput { // The id to a connected WorkflowNode which will be used as an input for current WorkflowNode. string node_id = 1; } // WorkflowResult // One result per input in the workflow. message WorkflowResult { reserved 4; string id = 1; clarifai.api.status.Status status = 2; // When the object was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 3; // The input that ran through the workflow to generate the outputs in this WorkflowResult. Input input = 5; // For each model in the workflow we return an Output. repeated Output outputs = 6; // Indicate if the output of this model is suppressed. bool suppress_output = 7; } // WorkflowState message WorkflowState { // A unique ID for the workflow state. // To start saving a state in a PostWorkflowResults request set this ID to "init" // and it will return a newly generated unique state id that you can then pass in subsequent // PostWorkflowResults calls. These state expire after 5 minutes between calls. string id = 1; } // Deprecated: App duplication is no longer supported. // AppDuplication message AppDuplication { option deprecated = true; // The unique identifier of an app duplication job. string id = 1; // The destination application where resources are written. App destination_app = 10; // Deprecated: Use destination_app.id with an existing ID instead. string existing_app_id = 8 [deprecated = true]; // Deprecated: Use destination_app.id with a new ID instead. string new_app_id = 2 [deprecated = true]; // Deprecated: Application names are deprecated, use application IDs instead. string new_app_name = 3 [deprecated = true]; // The status of the app duplication job. clarifai.api.status.Status status = 4; // The time when the app duplication job was created. google.protobuf.Timestamp created_at = 5; // The last time when the app duplication job status was updated. google.protobuf.Timestamp last_modified_at = 6; // The filter specifies which resources are copied by the app duplication job. AppDuplicationFilters filter = 7; // Copy progress for each resource type requested by the filter. repeated AppCopyProgress progress = 9; } message AppCopyProgress { string field = 1; int32 value = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; } // Deprecated: App duplication is no longer supported. // AppDuplicationFilters message AppDuplicationFilters { option deprecated = true; reserved 6; // Copy inputs. bool copy_inputs = 1; // Copy concepts. bool copy_concepts = 2; // Copy annotations. bool copy_annotations = 3; // Copy models. bool copy_models = 4; // Copy workflows. bool copy_workflows = 5; } // LabelOrder message LabelOrder { // id of the order string id = 1; // name of the order string name = 2; // status of the order. // pending (QA lead review the order), // in progress (labeling in progress), // ready for release (passed clarifai QA and client can review) // success (released) clarifai.api.status.Status status = 3; // if set to true, automatically release the labels once passed clarifai review. bool auto_release = 4; // allow input without any tag. bool allow_empty_tag = 5; // User desired estimation when the task should be done google.protobuf.Timestamp desired_fulfill_time = 6; // Clarifai estimation when the task should be done . google.protobuf.Timestamp estimate_fulfill_time = 7; // task for this label order Task task = 8; // When the label order was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 9; // Most recent time when the label order was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 10; } // Task is the work that needs to be done for labeling the inputs in an app. message Task { // Unique ID for the task. string id = 1; // When the task was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // Most recent time when the task was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // Task type. TaskType type = 4; // Description of the task. string description = 5; // Worker details. TaskWorker worker = 6; // List of concept ids used in the work of this task. // DEPRECATED: Use task.concepts instead. repeated string concept_ids = 7 [deprecated = true]; // List of inputs used in this task will be taken from this source. TaskInputSource input_source = 8; // For model predictions on video: Sample delay for video predicting (1 frame per N milliseconds) uint32 sample_ms = 9; // AI assistant details. TaskAIAssistant ai_assistant = 10; // Review details. TaskReview review = 11; // Status of this task. clarifai.api.status.Status status = 12; // Add a title for this task to quickly recognise it in a list of tasks. string name = 13; AiAssistParameters ai_assist_params = 14; enum TaskType { TYPE_NOT_SET = 0; // Concepts classification tasks annotate concepts for the overall image, frame of video or section of text. CONCEPTS_CLASSIFICATION = 1; // Bounding box detection tasks annotate rectangular bounding box regions around each concept in an image, frame of video or section of text. BOUNDING_BOX_DETECTION = 2; // Polygon detection tasks annotate free-form regions around concepts in an image, frame of video or section of text. POLYGON_DETECTION = 3; } // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 15; // The app the task belongs to. string app_id = 16; // The user the task belongs to. string user_id = 17; // The label order the task belongs to. string label_order_id = 18; // Ignore Task.concept_ids field if Task.TaskConcept are supplied. repeated TaskConcept concepts = 19; // Specify whether existing Annotations within the same app that are generated by other auto annotation tasks // with the specified Concept from the selected Model or Workflow should deleted before executing the Task bool delete_previous_annotations = 20; // Tasks metrics are filled in upon user-request. TaskMetrics metrics = 21; // Priority of the task TaskPriority priority = 23; enum TaskPriority { TASK_PRIORITY_NOT_SET = 0; LOW = 1; MEDIUM = 2; HIGH = 3; } // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 24; } // AiAssistParameters message AiAssistParameters { // Min and max threshold values for approving annotations by default based on prediction score float min_threshold = 1; float max_threshold = 2; // ids of concept relations. Used in AI assist workflow repeated string concept_relation_ids = 3; } // TaskWorker includes information about the workers that will work on this task. message TaskWorker { reserved 5, 6; // Worker strategy. TaskWorkerStrategy strategy = 1; // Who will work on this task. // DEPRECATED: Use workers.user.id instead. repeated string user_ids = 2 [deprecated = true]; // Users who will work on this task. // When the 'worker.users' field is additionally requested, then all user // info is filled for the workers. Otherwise, only the user 'id' is filled. // DEPRECATED: Use workers.user instead. repeated User users = 4 [deprecated = true]; // Info based on the worker strategy, oneof strategy_info {TaskWorkerPartitionedStrategyInfo partitioned_strategy_info = 3;} enum TaskWorkerStrategy { reserved 1; WORKER_STRATEGY_NOT_SET = 0; // The inputs will be partitioned in several partitions. // Each worker will label one or more input partitions. // All inputs are assigned at task creation. PARTITIONED = 2; // Each worker will label all inputs from input source. // All inputs are assigned at task creation. FULL = 3; // Each worker will dynamically get 10 inputs assigned at a time. // No inputs are assigned at task creation. DYNAMIC = 4; } // Workers that will work on this task. // // For Auto Annotation Tasks: // the worker can be either a model or a workflow; // currently only supports 1 worker. // For manual labeling Tasks: // the workers can only be users; // no limitation on number of workers. repeated Worker workers = 7; // Who is doing annotations - human Worker or auto-annotation via Model/Workflow. // If set, worker must have be set accordingly to either human worker or model/workflow worker WorkerType type = 8; enum WorkerType { // for backward compatibility when task is not setting any type and only sets workers WORKER_TYPE_NOT_SET = 0; // only human workers WORKER_HUMAN = 1; // auto-annotation tasks. Task must set worker as model or workflow WORKER_AUTO = 2; } // Runner selectors is used to select specific runners for the workers of this task. // The index of runner_selectors corresponds to the index of workers. // In other words: runner_selectors[i] is the runner selector for workers[i]. repeated RunnerSelector runner_selectors = 9; } // TaskWorkerPartitionedStrategyInfo message TaskWorkerPartitionedStrategyInfo { // Define how the partitioning should work. TaskWorkerPartitionedStrategy type = 1; // How many workers will label each input. int32 workers_per_input = 2; // In case of weighted partitioning, map user ids to weights. // Each labeler will be assigned work proportional to its own weight as compared to the sum of total weight. // // EXAMPLE: // If we have 3 workers, and weights = {1: 30, 2: 30, 3: 40}, // then first worker will have assigned 30% of the work, // second worker will have assigned 30% of the work, // and third worker will have assigned 40% of the work. // You may use weights which add up to 100, but it's not necessary. // For example, weights {1: 30, 2: 30, 3: 40} are equivalent with {1: 3, 2: 3, 3: 4} // because they represent the same percentages: {1: 30%, 2: 30%, 3: 40%}. // // NOTE: // Note that no worker should be assigned a weight percentage greater than 1/workers_per_input. // It is mathematically impossible to partition the work in such a case. // Why? Say, we have 3 workers. And workers_per_input = 2, i.e. each input must be labeled by 2 workers. // Let's assign weights {1: 51%, 2: 25%, 3: 24%}. // Note that first worker has a weight percentage higher than 1/workers_per_input = 1/2 = 50%. // If we have 100 inputs, then a total of 100 * workers_per_input = 200 cumulative inputs will be labeled by these 3 workers. // Worker 1 should label 102 cumulative inputs, while worker 2 and worker 3 will label 98 cumulative inputs together. // No matter how we assign the 98 cumulative inputs, the 2 workers will be able to label up to 98 actual inputs. // This means the remaining 2 inputs will be labeled only by worker 1. This contradicts the worker_per_input = 2 requirement. google.protobuf.Struct weights = 3; enum TaskWorkerPartitionedStrategy { PARTITIONED_WORKER_STRATEGY_NOT_SET = 0; // Each worker will label (approximately) the same number of inputs. EVENLY = 1; // Each worker will have an assigned weight. // See weights field for more details. WEIGHTED = 2; } } // TaskInputSource message TaskInputSource { // Type of input source. TaskInputSourceType type = 1; // If type is SAVED_SEARCH, then this is the saved search id. string id = 2; enum TaskInputSourceType { INPUT_SOURCE_TYPE_NOT_SET = 0; // Use all inputs in the app. ALL_INPUTS = 1; // Use the inputs from a saved search. SAVED_SEARCH = 2; // Inputs from a dataset. DATASET = 3; // Single input (e.g. video livestream input) INPUT = 4; } } // TaskReview message TaskReview { // Task review strategy. TaskReviewStrategy strategy = 1; // Who will review this task. // DEPRECATED: Use users.id instead. repeated string user_ids = 2 [deprecated = true]; // Users who will review this task. // When the 'review.users' field is additionally requested, then all user // info is filled for the reviewers. Otherwise, only the user 'id' is filled. repeated User users = 5; // Info based on the review strategy, oneof strategy_info { TaskReviewManualStrategyInfo manual_strategy_info = 3; TaskReviewConsensusStrategyInfo consensus_strategy_info = 4; } enum TaskReviewStrategy { TASK_REVIEW_STRATEGY_NOT_SET = 0; // No review is needed. // When a labeler labels an input, the annotations are immediately approved. NONE = 1; // Human reviewers will review the work done by labelers. MANUAL = 2; // Automatically approve inputs when labelers reach consensus. // If consensus is not reached, then it will fallback to human reviewers. CONSENSUS = 3; } } // TaskReviewManualStrategyInfo message TaskReviewManualStrategyInfo { // This field represents the percentage of inputs that will be reviewed by reviewers. It is a value between 0 and 1. // Deprecated: Not used. float sample_percentage = 1 [deprecated = true]; // Deprecated: Use consensus_strategy_info.approval_threshold_reviewers. int32 approval_threshold = 2 [deprecated = true]; } // TaskReviewConsensusStrategyInfo message TaskReviewConsensusStrategyInfo { reserved 1; // Deprecated: Use approval_threshold_labelers. uint32 approval_threshold = 2 [deprecated = true]; // The number of labelers that need to agree in order to automatically approve an annotation. // When 0, labelers consensus is disabled. // When 1, the labels are automatically approved once a single labeler labels the input. // When greater than 1, the labels are automatically approved when the specified number of labelers agree. // If the number of labelers that agree is less than the specified number, then the input will reviewed by reviewers. uint32 approval_threshold_labelers = 3; // The number of reviewers that need to agree in order to approve an input. // Currently, the only allowed values are: // 0 - when not set, it defaults to 1 // 1 - only a single reviewer needs to approve each labeled input // -1 - an input will be approved when all reviewers approve it int32 approval_threshold_reviewers = 4; } // TaskAIAssistant message TaskAIAssistant { // The worker is helped by an AI assistant. // This field is the workflow id which is used to assist the worker with predictions. // If empty, then AI assistant is disabled. string workflow_id = 1 [deprecated = true]; Workflow workflow = 2; } message TaskAssignment { string id = 1; // Creation time. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // Most recent modification time. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // Assigned worker. Worker worker = 4; // Assigned input. Input input = 5; // Assignment status. // Read as: This is the status of the work assigned to worker W, on input I in task T. clarifai.api.status.Status status = 6; } // TaskStatusCountPerUser can represent one of the following: // * count of task annotations created by a worker for each valid status, // * count of task inputs assigned to a worker (i.e. task assignments) for each valid status message TaskStatusCountPerUser { // Deprecated: Use worker instead. string user_id = 1 [deprecated = true]; uint32 pending = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 awaiting_review = 3 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 success = 4 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 review_denied = 5 [(clarifai.api.utils.cl_show_if_empty) = true]; uint32 awaiting_consensus_review = 6 [(clarifai.api.utils.cl_show_if_empty) = true]; Worker worker = 7; } message ThresholdRange { // The range used to filter over concept values. // e.g. GREATER_THAN_OR_EQUAL_TO 0.7 -> is_lower_inclusive = true, lower = 0.7, is_upper_inclusive = true, upper = 1.0 // e.g. (0.3, 0.75] -> is_lower_inclusive = false, lower = 0.3, is_upper_inclusive = true, upper = 0.75 bool is_lower_inclusive = 1; bool is_upper_inclusive = 2; float lower = 3; float upper = 4; } message TaskConceptAutoAnnotationConfig { // Filter annotations by their annotation data type. // This is a bit-mask field that holds multiple AnnotationDataType values that are combined in an OR fashion. // Example: if annotation_data_types = 34, then we filter annotations that appear as a mask or a bounding box, // because MASK = 32 and BOUNDING_BOX = 2. uint32 annotation_data_types = 1; // Filter annotations by concept value. // Only concepts that fit in the threshold will be used to generate annotations. ThresholdRange threshold_range = 2; // The output annotations will be created using this status code. clarifai.api.status.StatusCode status_code = 3; // Task auto annotation for this concept only applies to this time of day range. // When out of range, the task will not generate annotations for the concept. clarifai.api.utils.TimeOfDayRange time_of_day_range = 4; // Filter out annotations that are out of this polygon. // When it's set, only detection annotations that intersect the polygon will pass the filter. Polygon polygon = 5; } message TaskConcept { // For auto annotation, id/name and value, user + app id must be specified. For other tasks, only the id field is required. Concept concept = 1; TaskConceptAutoAnnotationConfig auto_annotation_config = 2; } enum AnnotationDataType { ANNOTATION_DATA_TYPE_NOT_SET = 0; TAG = 1; BOUNDING_BOX = 2; POLYGON = 4; POINT = 8; SPAN = 16; MASK = 32; } message TaskMetrics { reserved 1; TaskWorkMetrics work = 2; TaskReviewMetrics review = 3; TaskInputSourceMetrics input_source = 4; } message TaskWorkMetrics { // Estimated number of inputs that workers have worked on. uint64 inputs_count_estimated = 1; // Estimated percent of inputs that workers have worked on. // This is a value between 0 and 100, where 0 = 0% and 100 = 100%. uint32 inputs_percent_estimated = 2; } message TaskReviewMetrics { // Estimated number of fully reviewed inputs. // An input is considered fully reviewed if it has been reviewed by all necessary reviewers. // Example: if task has no review, then an input is considered fully reviewed right after it's labeled (as review is skipped). // Example: if task has manual review with single-reviewer per input, then an input is considered fully reviewed when 1 reviewer has approved/rejected it. // Example: if task has consensus review with 3 reviewers per input, then an input is considered fully reviewed when 3 reviewers have approved it or 1 reviewer has rejected it. uint64 inputs_count_estimated = 1; // Estimated percent of review work that was finished. // This is a value between 0 and 100, where 0 = 0% and 100 = 100%. // Calculated as inputs_count_estimated/task.metrics.input_source.inputs_count_estimated. // As the counts are estimated, the percentage is also estimated. // However, additional checks are made to ensure that 100% percentage is only returned when all inputs are reviewed - giving a guarantee that the 100% percentage is always accurate. uint32 inputs_percent_estimated = 2; // Estimated number of reviewed inputs per reviewer index. // The reviewer indexes are based on task.review.users. // An input is considered reviewed by a reviewer if: // * the reviewer approved the input // * the reviewer rejected the input // Note that when a reviewer requests changes for an input, the input is sent to back to work again, so the whole work & review process is restarted. // The reviewer will have to review the input again after work has been completed. // As such, the review that requests changes for an input is immediately dis-regarded and not counted in this metric. repeated uint64 inputs_count_estimated_per_reviewer = 3; // The number of inputs actually available for review for each reviewer. // Most times, this equals task.metrics.input_source.inputs_count_estimated. // Several situations may result in different values: // * When task has no review, then this is 0 for each reviewer. // * When task has auto-annotation, then this number equals the inputs that have been auto-annotated with AWAITING_REVIEW status. All other inputs are considered completed by the auto-annotation process. // * When task has consensus review with approval_threshold_labelers > 0, then it's possible that labelers will approve inputs through consensus, which skips review. In this case, the number of inputs available for review is less than task.metrics.input_source.inputs_count_estimated. // * When task has consensus review with approval_threshold_reviewers = 1, then all inputs are assigned only to one reviewer, so each reviewer will get only a part of the inputs to review. It's expected that the sum(inputs_reviewable_count_estimated) = task.metrics.input_source.inputs_count_estimated. // * When task has consensus review with approval_threshold_reviewers = -1, then all inputs are assigned to all reviewers. However, if an input is rejected, then rejection is final and all other reviewers will not review it. In this case, the number of inputs available for review for other reviewers will be less than task.metrics.input_source.inputs_count_estimated. repeated uint64 inputs_reviewable_count_estimated_per_reviewer = 4; // Estimated percent of review work that was finished per reviewer. // This is a value between 0 and 100, where 0 = 0% and 100 = 100%. // Calculated as inputs_count_estimated_per_reviewer/inputs_reviewable_count_estimated_per_reviewer. // As the counts are estimated, the percentage is also estimated. // However, additional checks are made to ensure that 100% percentage is only returned when all inputs are reviewed - giving a guarantee that the 100% percentage is always accurate. repeated uint32 inputs_percent_estimated_per_reviewer = 5; } message TaskInputSourceMetrics { // Estimated number of inputs that are in the source of data uint64 inputs_count_estimated = 1; } // Role represents a list of permissions message Role { string id = 1; // When the role was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // Most recent time when the role was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; string name = 4; string description = 5; // The low-level scopes this role has repeated string scopes = 6; // The endpoint-level scopes this role has repeated string endpoints = 7; // Type of the role 'team' or 'org' RoleType type = 8; } enum RoleType { TEAM = 0; ORG = 1; } // Represents a group of users. message Team { // Identify the team (unique). string id = 1; // When the team was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // Most recent time when the team was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // Name the team. string name = 4; // When a new application is assigned to the team without an explicit role, then the default role will be used string default_role_id = 5; } // PCAProjectionComparator message PCAProjectionComparator { // Within what distance do we consider two annotations duplicates float distance_threshold = 1; // What cluster model version generated these string model_version_id = 2; } // DuplicateAnnotationsResults message DuplicateAnnotationsResults { repeated string duplicate_cfid = 1; int32 unique_count = 2; } // Visibility represents how visible the given resource is to other users. // When authenticating a request we can tell if a user is a collaborator or a teammate for the // the app that contains the resource and set their allowed visibility. We use that to restrict // what they are allowed to see: // If AllowedVisibility is PRIVATE then we allow PRIVATE (10), ORG (30), PUBLIC (50) // If AllowedVisibility is ORG then we allow ORG (30), PUBLIC (50) // If AllowedVisibility is PUBLIC then we allow PUBLIC (50) only. message Visibility { // Gettable defined the level of access for GET operations for this resource. enum Gettable { // Default value not allowed. UNKNOWN_VISIBILITY = 0; // PRIVATE requires collaborator or team permissions in order to GET this resource. PRIVATE = 10; // ORG requires you to be in the same org in order to GET this resource, but don't have to be a // teammate or collaborator. ORG = 30; // PUBLIC opens up GET access to the resource to any user on the platform even if they are not // a teammate or collaborator. PUBLIC = 50; } Gettable gettable = 1; } enum ValidationErrorType { VALIDATION_ERROR_TYPE_NOT_SET = 0; RESTRICTED = 1; DATABASE = 2; FORMAT = 3; } message FullTag { // Display name of the tag. Ex. "English" string name = 1; // Id value for referencing. Ex. "en" string id = 2; } // TimeSegment message TimeSegment { // A unique id for the time segment. string id = 1; Data data = 2; TimeInfo time_info = 3; } // TimeInfo message TimeInfo { reserved 2, 3; // Number of frames uint32 num_frames = 1; // Timestamp where track begins. float begin_time = 4; // Timestamp where track ends. float end_time = 5; } message BulkOperation { // id of the Bulk Operation task string id = 1; // Input Source could be list of input ids or a Search whose results will be a list of input ids. // InputIDs: // List of input ids to which operation to be applied // clarifai.api.Search: // A Search(either filter or rank with min value) to allow filtering down the entire app's // sub-assets(image, region in image, frame in video, region in frame in video) // and perform operation to only the results of this search query. See our search // documentation for more details about the search Query message. // For eg., filters the asset/sub-asset matching the search and performs specified operation. // Dataset: // A dataset, whose inputs will have the operation applied to. This does not support dataset versions. oneof input_source { InputIDs input_ids = 2; clarifai.api.Search search = 10; Dataset dataset = 11; } // Annotation Source can be a Search whose results will be a list of annotation ids. oneof annotation_source { clarifai.api.Search annotation_search = 12; } // Operation to perform Operation operation = 3; // Application ID that this Operation was created from string app_id = 4; // Status (pending, in-progress, completed, failed) of the operation clarifai.api.status.Status status = 5; // Progress of an on-going Bulk Operation task Progress progress = 6; // User id that created this operation string created_by = 7; // When the operation was created. We follow the XXXX timestamp // format. We use https://www.ietf.org/rfc/rfc3339.txt format: // "2006-01-02T15:04:05.999999Z" so you can expect results like // the following from the API: // "2017-04-11T21:50:50.223962Z" google.protobuf.Timestamp created_at = 8; // Last time the status got updated google.protobuf.Timestamp last_modified_at = 9; } message InputIDs {repeated string input_ids = 1;} message Progress { uint32 processed = 1; string last_processed_id = 2; } message Operation { // Bulk Operations supported: // Concepts: // Operations: add_concepts, delete_concepts // AddConcepts: // If new concepts are given, add concepts operation creates new concepts in the app and adds them to the given inputs' annotations. // If the given concept already exist, the label value of the concept is updated with the given value. // DeleteConcepts: // Remove the matching concept(s) for all the inputs in input source (mentioned above). // If user IDs are set, concepts will be deleted only from annotations created by given user ids. // If the user IDs are not set, the list will be automatically set with 1 element that is the caller user ID. // Input Source: // Input ids of assets(images) (or) search on sub-assets(region in image, frame in video, region in frame in video) // Metadata: // Operations: add_metadata, delete_metadata // AddMetadata: // Add the provided metadata to the input level annotation for all the inputs in input source (mentioned above). // If the key(s) already exists, it will overwrite the key(s) with the corresponding new value(s). // DeleteMetadata: // Remove the key, value pairs that match the given metadata from the existing input level Annotations' metadata // for all the inputs in input source (mentioned above). // Input Source: // Input ids of assets(images, videos) (or) search on sub-assets(region in image, frame in video, region in frame in video) // Geo: // Operations: overwrite_geo, delete_geo // OverwriteGeo: // Add the provided geo info for all the inputs in input source (mentioned above). // DeleteGeo: // Delete Geo info for all the inputs in input source (mentioned above). // Input Source: // Input ids of assets(images, videos) (or) search on sub-assets(region in image, frame in video, region in frame in video) // Dataset Inputs: // Operations: add_to_dataset, delete_from_dataset, split_into_datasets // AddToDataset: // Add inputs to a dataset // DeleteFromDataset: // Delete inputs from a dataset // SplitIntoDatasets: // Randomly split inputs into provided dataset ID's with provided percentages. oneof operation { AddConcepts add_concepts = 1; DeleteConcepts delete_concepts = 2; AddMetadata add_metadata = 3; DeleteMetadata delete_metadata = 4; OverwriteGeo overwrite_geo = 5; DeleteGeo delete_geo = 6; DeleteFromDataset delete_from_dataset = 7; AddToDataset add_to_dataset = 8; SplitIntoDatasets split_into_datasets = 9; } // Bulk Operations for Annotations: // Operations: delete_annotations // DeleteAnnotations: // Delete all the annotations matching the provided annotation source oneof annotation_operation { DeleteAnnotations delete_annotations = 10; } } message AddConcepts {repeated Concept concepts = 1;} message DeleteConcepts { repeated Concept concepts = 1; repeated string user_ids = 2; } message AddMetadata { // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 1; } message DeleteMetadata { // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 1; } message OverwriteGeo { // Geo info Geo geo = 1; } message DeleteGeo {} message AddToDataset {string dataset_id = 1;} message DeleteFromDataset {string dataset_id = 1;} message SplitIntoDatasets { repeated DatasetSplit dataset_splits = 1; DatasetSplitMethod method = 2; enum DatasetSplitMethod { NOT_SET = 0; // We will randomly split inputs into the datasets RANDOM_PERCENTAGE_SPLIT = 1; } } message DatasetSplit { // Expected to have ID Dataset dataset = 1; oneof method_info { // For RANDOM_PERCENTAGE_SPLIT. // Values from (0,100] uint32 percentage = 2; } } message DeleteAnnotations { // This operation takes no data (payload). } message InputsAddJob { reserved 2, 5, 6; // id of the job string id = 1; // If call back url is set, we will send a Post request to this endpoint with job status. string call_back_url = 3; // Personal Access Token to the application to which inputs are added string app_pat = 4; // Progress of an on-going Input Ingestion task InputsAddJobProgress progress = 7; // When the job was created. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 8; // Most recent time when the job was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 9; // Sub-jobs that extract inputs from the cloud and/or archives repeated InputsExtractionJob extraction_jobs = 10; // Archive uploads repeated Upload uploads = 11; // Status of the job clarifai.api.status.Status status = 12; } message InputsAddJobProgress { uint64 pending_count = 1; uint64 in_progress_count = 2; uint64 success_count = 3; uint64 failed_count = 4; } message Upload { // ID of upload string id = 1; // When the upload was started. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 2; // Most recent time when the upload was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 3; // When the upload will expire and be deleted // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp expires_at = 4; // Status of the upload clarifai.api.status.Status status = 5; // name of uploaded content (e.g. filename) string content_name = 8; // Total size of the upload content uint64 content_length = 6; // Url of uploaded content string content_url = 7; } message UploadContentPart { uint64 range_start = 1; int64 part_number = 2; bytes data = 3; } message InputsExtractionJob { clarifai.api.status.Status status = 1; // ID of extraction job string id = 2; // Url of archive or bucket string url = 3; // Progress counts of the job InputsExtractionJobProgress progress = 4; // When the extraction job was started. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp created_at = 5; // Most recent time when the extraction job was updated. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp modified_at = 6; // How to handle input ID conflicts. InputIDConflictResolution input_id_conflict_resolution = 7; // Fields set in the template are added to all generated inputs Input input_template = 8; } message InputsExtractionJobProgress { reserved 1; uint64 audio_inputs_count = 2; uint64 image_inputs_count = 3; uint64 video_inputs_count = 4; uint64 text_inputs_count = 5; uint64 pending_archives_count = 6; uint64 in_progress_archives_count = 7; uint64 completed_archives_count = 8; uint64 failed_archives_count = 9; } message InputsDataSource { // Collect statistics about created inputs in job with given ID. // On Post call: // * If job ID is empty, then job is automatically created with random ID. // * If job ID is non-empty, then a new job will be created with given ID. string inputs_add_job_id = 1; DataSourceURL url = 2; // How to handle input ID conflicts. InputIDConflictResolution input_id_conflict_resolution = 3; // Fields set in the template will also be added to all generated inputs Input input_template = 4; } message DataSourceURL { // Supported providers are AWS S3, Azure blob, GCP cloud storage. string url = 1; // Credentials that would allow access to the provided url DataSourceCredentials credentials = 2; } message DataSourceCredentials { reserved 3; oneof credentials { // AWS S3 credentials for authentication. AWSCreds s3_creds = 1; // GCP Cloud Storage uses service account key data(creds.json) as Byte array for authentication. bytes gcp_creds = 2; // Azure Blob credentials for authentication. AzureBlobCreds azure_blob_creds = 4; } } // AWS S3 storage credentials. message AWSCreds { reserved 1; string region = 2; string id = 3; string secret = 4; string token = 5; } // Azure Blob storage credentials. message AzureBlobCreds { string account_name = 1; string account_key = 2; } message InputsUpload { // Collect statistics about created inputs in job with given ID. // * If job ID is empty, then job is automatically created with random ID. // * If job ID is non-empty, then a new job will be created with given ID. string inputs_add_job_id = 1; // Personal Access Token to the application to which inputs are added // Deprecated: No need to send app_pat, it will be generated internally if not present string app_pat = 2 [deprecated = true]; Upload upload = 3; // How to handle input ID conflicts. InputIDConflictResolution input_id_conflict_resolution = 4; // Fields set in the template will also be added to all generated inputs Input input_template = 5; } enum InputIDConflictResolution { INPUT_ID_CONFLICT_RESOLUTION_NOT_SET = 0; // Defaults to SKIP SKIP = 1; // Mark duplicate inputs as error and skip processing them. SUFFIX = 2; // Add a suffix to inputs with conflicting IDs. Attempts numeric suffixes "-1" to "-9" and then a randomized suffix. Identical ID's in the same request are still treated as errors. } message BookmarkOrigin { // original resource id string id = 1; // original resource app id string app_id = 2; // original resource user id string user_id = 3; enum BookmarkType { reserved 4; unknown = 0; model = 1; workflow = 2; dataset = 3; } // resource type. BookmarkType resource_type = 4; } // Moving the runner label matching into it's own object so that it can be used // next to other resource types that a Runner might match work on. // message RunnerLabels { // FUTURE // repeated string labels = 1; // } // RunnerMetrics captures metrics and status for a Runner's underlying k8s deployment. // This allows tracking of deployment health, replica counts, and other relevant metrics. message RunnerMetrics { uint32 pods_total = 1; uint32 pods_running = 2; // Cumulative total time (in seconds) that pods have been running for this runner. // This accumulates across scale-up/down cycles and is reported by the agent. uint32 total_pods_running_time_s = 3; // Cumulative count of pods that have been preempted by the k8s scheduler. // This accumulates across reconcile cycles and is reported by the agent. uint32 pods_preempted_total = 4; } // A worker for compute within a nodepool of instances. // This asks the API for work message Runner { reserved 6; // A unique ID for this runner. // This is a UUID since runners can be automatically orchestrated. string id = 1; // short description about the runner. string description = 2; // When the runner was created. google.protobuf.Timestamp created_at = 3; // When the runner was last modified. google.protobuf.Timestamp modified_at = 4; // To handle arbitrary json metadata you can use a struct field: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto // This is an optional arg. google.protobuf.Struct metadata = 5; // Labels to match in order to find work. repeated string labels = 7 [deprecated = true]; // Instead of just matching on labels we might want to have more explicit matching of what // work this runner is looking for. // The thing that the autoscaling config applies to for this nodepool. Worker worker = 8; // Runners are defined within nodepools so this field needs the id and user_id of the nodepool // to be provided when creating a Runner. // This nodepool must be accessible to you or an org you are part of. Nodepool nodepool = 9; //////////////////////////// // Need resources on the runner so we can schedule this Runner into the Nodepool. // If this runner is being orchestrated for a model then the orchestrator will set this to the // model resource requirements. If a workflow then it'll compute those requirements and set // populate this resource field. // Having this on the underlying object like Model and Workflow allows us to represent the minimum // requirements on those object, which may be less than what the Runner allocates (as a safety // margin for the runner to for sure run the resource). ComputeInfo compute_info = 10; // Number of replicas that this runner should have up. // We keep it separate from ComputeInfo which defines how many resources each replica needs. uint32 num_replicas = 11; // List of special handling instructions for this runner. repeated SpecialHandling special_handling = 12; // Metrics and status for the underlying k8s deployment. // Each Runner is 1:1 with a k8s deployment, so this field tracks deployment health and metrics. RunnerMetrics runner_metrics = 13; // Hard minimum replicas from the deployment's autoscale config. // The agent uses this to determine how many replicas are non-preemptable. uint32 min_replicas = 14; // The scheduling priority for this runner's k8s deployment. // Valid values are 0-9, where higher values indicate higher priority. // This value comes from the deployment nodepool's priority setting. uint32 priority = 15; } // A nodepool is a set of nodes dedicated for a given user's compute needs. // This compute will typically be consumed by runners and in the future other objects // like UI modules may be assigned to node pools. message Nodepool { reserved 5; enum NodepoolStatus { NODEPOOL_STATUS_UNKNOWN = 0; NODEPOOL_STATUS_PENDING = 1; NODEPOOL_STATUS_READY = 2; NODEPOOL_STATUS_ERROR = 3; NODEPOOL_STATUS_DELETED = 4; } // The user defined ID of the nodepool. string id = 1; // Short description about the nodepool. string description = 2; // When the nodepool was created. google.protobuf.Timestamp created_at = 3; // When the nodepool was last modified. google.protobuf.Timestamp modified_at = 4; // Which cluster this nodepool is within. ComputeCluster compute_cluster = 6; NodeCapacityType node_capacity_type = 7; repeated InstanceType instance_types = 8; // Minimum number of instances in this nodepool. This allows the nodepool to scale down to this // amount. This is the user desired minimum. uint32 min_instances = 9; // An upper limit on the number of instances in this nodepool. This allows the nodepool to scale // up to this amount. This is the user desired maximum. uint32 max_instances = 10; // The actual minimum number of instances. Enforced by the user's plan limits. uint32 enforced_min_instances = 13; // The actual maximum number of instances. Enforced by the user's plan limits. uint32 enforced_max_instances = 14; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 11; // To handle arbitrary json metadata: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 12; // List of special handling instructions for this nodepool. repeated SpecialHandling special_handling = 15; // The current number of nodes in this nodepool. This value is queried from Prometheus metrics // and represents the most recent node count reported by the compute plane. uint32 node_count = 16; // Current status of the nodepool. NodepoolStatus status = 17; // Human-readable status description. Contains error details when status is NODEPOOL_STATUS_ERROR. string status_description = 18; // Other future configuration we can add: // - disk information // - architecture type (ARM/x86) // - base OS // - repeated availability zones. } // Type of nodes that are ok for instances in this pool. // If both spot and on-demand are provided then the runner will be able to run on either // with a preference for spot until they are not available. message NodeCapacityType { enum CapacityType { UKNOWN_CAPACITY_TYPE = 0; ON_DEMAND_TYPE = 1; SPOT_TYPE = 2; } repeated CapacityType capacity_types = 1; } // The instance types that will be available in this pool of nodes. // Clarifai offers multiple different choices that combine cpu cores, memory and accelerator. message InstanceType { string id = 1; // Short description of instance type. string description = 2; ComputeInfo compute_info = 3; string price = 4; // The cloud provider where this instance type is available, if any. CloudProvider cloud_provider = 5; // The region where this instance type is available, if any. string region = 6; // The capacity types allowed for this instance type. If empty - all capacity types are allowed. NodeCapacityType allowed_capacity_types = 7; // The feature flag group associated with this instance type. string feature_flag_group = 8; // List of special handling instructions for this instance type. repeated SpecialHandling special_handling = 9; // Hardware architecture of the instance type (e.g., "linux/amd64", "linux/arm64"). string architecture = 10; // Available compute info after accounting for system overhead (daemonsets, kubelet, etc.). // This represents the actual resources available for user workloads. ComputeInfo available_compute_info = 11; } // CloudProvider represents the entity that provides the infrastructure where the Nodepools are deployed. // This could be a public cloud provider like AWS, GCP, Azure, etc., or a self-hosted infrastructure. message CloudProvider { // Unique identifier of the cloud provider. string id = 1; // Name of the cloud provider. string name = 2; // List of special handling instructions for this cloud provider. repeated SpecialHandling special_handling = 3; } // CloudRegion represents a region where compute resources can be deployed. message CloudRegion { // Unique identifier of the region. string id = 1; // List of special handling instructions for this region. repeated SpecialHandling special_handling = 2; } // We define a cluster here to be used in Nodepools and by the cloud provider. // There will be one cloud provider per Cluster. // This allows us to define Clusters that are VPCs within one physical cloud and have that // managed by one cloud provider which can list all nodepools for that VPC to deploy them and // orchestrate work within them. message ComputeCluster { string id = 1; // Short description of cluster region. string description = 2; // The cloud provider where this cluster is hosted. // Some example cloud provider IDs may be aws, gcp, azure, local, kubernetes, etc. CloudProvider cloud_provider = 3; // The region. The naming here depends on the cluster choice above and will be validated // against which clusters+regions that Clarifai currently supports. string region = 4; // The user/org that this compute cluster belongs to. string user_id = 5; // When the compute cluster was created. google.protobuf.Timestamp created_at = 6; // When the compute cluster was last modified. google.protobuf.Timestamp modified_at = 7; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 8; // We offer different types of compute clusters such as: // 'shared' which only Clarifai can create. // 'dedicated' where you're in control of defining the nodepools within the cluster // 'local-dev' which means you're responsible for starting runners manually which is great for local // development but not recommended for production use cases. string cluster_type = 9; // Managed by represents who is responsible for the cluster. // This is currently either "clarifai" where we fully manage the infrastructure. // Or, "user" where the user is responsible for the underlying infrastructure. string managed_by = 10; // Key to use within the compute cluster for all requests to the API. // You can post with the key.id filled in to set the key for the compute cluster. // The responses will intentionaly only return the description of the key for security // purposes since you may have other people through orgs/teams having access to this compute // cluster who should not view your key. // This must be a valid key created before creating the ComputeCluster. // Deleting this key will not be prevented, which means all resources in this ComputeCluster // will lose connection to the API, so delete keys at your own risk. // The user_id who owns the key must match the user_id provided in the ComputeCluster. Key key = 11; // For future support of different VPC. // To support splitting a region into multiple VPCs. // string vpc_id = 12; // FUTURE // The user/org that this VPC belongs to. // string vpc_user_id = 13; // FUTURE // this is the base url for the API for all runners in this nodepool to communicate with. // This is necesary for multi-region / multi-cloud support since runners should only pick up // work from the region/cloud that they are within. The runners will be orchestrated within the // nodepool so can have this base_api_url injected as the CLARIFAI_API_BASE env variable. // We include this in the proto in case a user needs to create a nodepool on their own // infrastructure that talks with a specific region. // string base_api_url = 14; // FUTURE // The available set of instance types in this cluster. // InstanceType instance_types = 15; // FUTURE } // These are the resource needs of a given API object such as a model. // This is what they require as a minimum to run and will be used upon scheduling // as the request and limit for the k8s pod. If we want to separate limits and requests in the // future we can allow setting a limits ComputeInfo and a requests ComputeInfo. message ComputeInfo { reserved 1; // Amount of CPUs to use as a limit. This follows kubernetes notation like: "1", "100m", "4.5", etc. // See https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ // For instances, this is the instance's CPU count. // For runners, this is the maximum amount of CPU that the runner pod can use. string cpu_limit = 6; // Amount of CPU memory to use as a limit. This follows kubernetes notation like: // 1Ki, 1500Mi, 3Gi, 4Ti, etc. // See https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ // For instances, this is the instance's CPU memory. // For runners, this is the maximum amount of CPU memory that the runner pod can use. string cpu_memory = 2; // Amount of CPUs to use as a minimum. This follows kubernetes notation like: "1", "100m", "4.5", etc. // See https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ // For runners, this is the minimum amount of CPU requested for the runner pod. // Optional. If unspecified, a small default is used. string cpu_requests = 7; // Amount of CPU memory to use as a minimum. This follows kubernetes notation like: // 1Ki, 1500Mi, 3Gi, 4Ti, etc. // For runners, this is the minimum amount of CPU memory requested for the runner pod. // Optional. If unspecified, a small default is used. string cpu_memory_requests = 8; // Amount of GPU/TPUs to use. uint32 num_accelerators = 3; // Amount of accelerator/GPU memory to use as a minimum. // This is defined per accelerator. // This follows the format used by kubernetes like 1Ki, 2Mi, 3Gi, 4Ti. // See https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ string accelerator_memory = 4; // The supported accelerators that the resource can run on. If the resource requires a specific // accelerator type then it will only be scheduled on nodes that have that type of accelerator. // If there is no hard requirements beyond the number of accelerators and their memory then this // field can be left empty. // This setting will cause an error if num_accelerators == 0 // Or should it be removed completely and use the nodepool accelerator type itself. // These are the supported accelerators that the model can run on. repeated string accelerator_type = 5; // Future potential things we may add: // - disk information // - architecture type (ARM/x86) // - number of nodes for each replica of this resource to run on for multi-node things. // - classes of accelerators like Nvidia or "Ampere", etc. // The number of nodes of this type needed to serve this resource. // Since 0 is not valid, this will default to 1 if 0 is provided. // uint32 num_nodes = 6; // FUTURE // For multi-host accelerators (i.e., TPU Slices), this defines the slice topology. // Corresponds to the tpu.googleapis.com/topology annotation. // Example: "2x2x1" for a 16-chip slice using v4 TPUs. // Leave empty for single-host accelerators like GPUs or non-slice TPUs. repeated string accelerator_topology = 10; } // It might be better to do this as runner autoscaling so that resources // of a model are very simply what that model needs at minimum. // Note that resources for things like modes inside runners are not // related to how many replicas of those runners are needed to handle traffic. message AutoscaleConfig { reserved 6; // The minimum number of replicas for the runner to have. // Defaults to 0 which means autoscaling can scale down to zero. // If you want a replica always up then set to >= 1. uint32 min_replicas = 1; // The maximium number of replicas to scale up the runner to. uint32 max_replicas = 2; // The number of seconds of traffic history to consider when autoscaling. uint32 traffic_history_seconds = 3; // The time to wait before scaling down after the last request. uint32 scale_down_delay_seconds = 4; // The time to wait between scaling up replicas without burst traffic. uint32 scale_up_delay_seconds = 5; // Depending on your plan you may be able to enable packing of resources into a single node // for more compute and cost efficiency. bool disable_packing = 7; // The idle time before scaling down to zero uint32 scale_to_zero_delay_seconds = 8; // Additional minimum replicas added on top of min_replicas. "Soft" refers to scheduling // priority (preemptable), not to whether the floor is enforced. The orchestrator always // maintains min_replicas + soft_min_replicas (capped to max_replicas). // A value of 0 means not set / disabled. uint32 soft_min_replicas = 9; } // DeploymentMetricsSummary holds the core deployment metrics fields, reusable as both // an aggregate across all nodepools and per-individual-nodepool metrics. message DeploymentMetricsSummary { // The number of replicas desired by the orchestrator. uint32 desired_replicas = 1; // The actual number of live replicas connected and ready to process requests. uint32 live_replicas = 2; // If true, a new version is currently being rolled out. bool rollout_in_progress = 3; } // NodepoolDeploymentMetrics captures metrics for a single nodepool within a Deployment. message NodepoolDeploymentMetrics { // The user-facing compute cluster ID. string compute_cluster_id = 1; // The user-facing nodepool ID. string nodepool_id = 2; // Metrics for this nodepool. DeploymentMetricsSummary metrics = 3; } // DeploymentMetrics captures metrics and status for a Deployment's underlying runners. // This allows tracking of the desired replica count and the actual live replica count. message DeploymentMetrics { // Deprecated: use aggregate instead. uint32 desired_replicas = 1 [deprecated = true]; // Deprecated: use aggregate instead. uint32 live_replicas = 2 [deprecated = true]; // Deprecated: use aggregate instead. bool rollout_in_progress = 3 [deprecated = true]; // Per-nodepool breakdown of metrics. repeated NodepoolDeploymentMetrics nodepool_metrics = 4; // Aggregate metrics across all nodepools. DeploymentMetricsSummary aggregate = 5; } // A deployment allows you to configure how runners for a particular type of resource will // scale up and down. These are unique per user_id, nodepool and model so for differnet nodepools // you can scale differently. message Deployment { reserved 5, 6; // Status of the deployment. enum Status { ENABLED = 0; // Default: deployment is enabled and can serve traffic. DISABLED = 1; // Deployment is disabled: scaled to zero, traffic rejected. } // An id for this configured deployment. string id = 1; // The user who owns the deployment. These live in the user/org account. string user_id = 2; // How to autoscale the object. AutoscaleConfig autoscale_config = 3; // You can configure different autoscaling per nodepool(s). // These nodepools have to be also owned by the same user_id/org as this deployment. // If there is more than one nodepool we use the model's ComputeInfo to match // with what the nodepool provides to decide which one can handle it combined with the // NodepoolRank below. Note: even within a single nodepool if it is heterogeneous then // we need a way to rank scheduling choices when we don't know how to decide (like a model // supports // Use DeploymentNodepools field instead repeated Nodepool nodepools = 4 [deprecated = true]; // In some scenarios it may not be obvous how we should schedule a resource to underlying nodes // within the nodepool(s) above. The SchedulerChoice allows us to specify how to decide which // nodepool to use when there are multiple nodepools and how to decide which type of node // within a nodepool if there are multiple types. // If here are multiple nondepools then a decision on which to use comes into play // if it is not specified in the prediction request. // Even with a single nodepool a choice may come up such as when a resource that needs scheduling // has not specified the accelerator types it supports and the nodepool has multiple types. enum SchedulingChoice { UNKNOWN_SCHEDULING_CHOICE = 0; FAIL = 1; // fail if there is any ambiguity. RANDOM = 2; // randomly pick amongst the nodepools/nodes to assign. PRICE = 3; // choose the compute that is cheaper PERFORMANCE = 4; // schedule to the fastest known iption. NETWORK = 5; // optimize based on network latency. UTILIZATION = 6; // send to the least used nodepool based on it's capacity. PREFER_SPOT = 7; // choose spot instances over on-demand PREFER_ONDEMAND = 8; // choose on-demand instances over spot. } SchedulingChoice scheduling_choice = 7; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 8; // To handle arbitrary json metadata: // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 9; // Short description of deployment. string description = 10; // The thing that the autoscaling config applies to for this nodepool. // For a given user_id, nodepool_id, and object ID we can only have one deployment as it defines Worker worker = 11; Worker desired_worker = 16; // When the deployment was created. google.protobuf.Timestamp created_at = 12; // When the deployment was last modified. google.protobuf.Timestamp modified_at = 13; // Whether to always deploy latest model version bool deploy_latest_version = 14; // List of special handling instructions for this deployment. repeated SpecialHandling special_handling = 15; // Duration after which to send a reminder email if pods are still running. // If set, users will receive a one-time email notification when any runner // under this deployment has been running longer than this duration. google.protobuf.Duration email_reminder_after = 17; // Whether to gracefully deploy a new worker bool graceful_deploy = 18; // Per-nodepool settings including priority. If set, 'nodepools' must not also be set. // When any priority value is non-zero, the response will populate this field instead // of 'nodepools'. repeated DeploymentNodepool deployment_nodepools = 19; // Real-time metrics for this deployment, including the desired and live replica counts. DeploymentMetrics deployment_metrics = 20; // The current status of the deployment. // When disabled, all infrastructure is scaled to zero and prediction requests are rejected. // The autoscale_config is preserved for when the deployment is enabled. Status status = 21; } // DeploymentNodepool associates a nodepool with a deployment and holds per-nodepool settings. message DeploymentNodepool { // Nodepool ID correlates with nodepools in deployment. string id = 1; // The compute cluster that owns this nodepool. ComputeCluster compute_cluster = 2; // The scheduling priority for this deployment on the given nodepool. // Valid values are 0-9, where higher values indicate higher priority. // Default is 0 (lowest priority). uint32 priority = 3; // ------------------------------------------------------------------- // OUTPUT FIELDS (Server populates these so the client gets the data) // ------------------------------------------------------------------- Nodepool nodepool = 4; } ////////////////////////////////////////// // Don't need RunnerSelector if we're opening up endpoints for deployments. ////////////////////////////////////////// // The RunnerSelector is an optional field we can provide during runtime // of model/workflow predictions to specify which particular runner we want to process the work. // This can optionally be used to select a particular nodepool and then within that nodepool // a particular runner. message RunnerSelector { // A particular nodepool for the resource to be run within. This request the id and user_id of // the nodepool to be specified. Runners will be scaled according to a deployment for the given // resource to be run. There should not be more than one deployment in this nodepool for the // particular resource to run. If no deployments then default autoscaling will be used. Nodepool nodepool = 1; // Optionally a particular runner within the nodepool. Runner runner = 2; // Optionally a particular deployment within the nodepool. Deployment deployment = 3; // In future as we support matching runners based on just labels: // RunnerLabels runner_labels = 3; // FUTURE } enum RunnerMethodType { UNKNOWN = 0; UNARY_UNARY = 1; // single request, single response. predict() in code UNARY_STREAMING = 2; // single request, streamed response. generate() in code STREAMING_UNARY = 3; // stream of requests, single response. STREAMING_STREAMING = 4; // stream of requests, stream of responses. stream() in code // In the future we may support other things here like long running processes, etc. } // Processing info tells the runner how to process a RunnerItem message ProcessingInfo { // The type of method witin the runner to call. RunnerMethodType runner_method_type = 1; // A status of the processing. We use this for signalling end of a request stream, a runner // item's processing should be cancelled, etc. clarifai.api.status.Status status = 2; // Internal field to track processing. Runners will not have access to this. string processing_id = 3; } enum EventType { reserved 200, 201, 202, 203, 204, 205; EVENT_TYPE_NOT_SET = 0; // Event types related to organization memberships and teams: 100 - 199 ORGANIZATION_MEMBER_ADD = 100 [(clarifai.api.utils.description) = "Organization members added"]; ORGANIZATION_MEMBER_CHANGE = 101 [(clarifai.api.utils.description) = "Organization members' roles changed"]; ORGANIZATION_MEMBER_REMOVE = 102 [(clarifai.api.utils.description) = "Organization members removed"]; ORGANIZATION_MEMBER_INVITATION_CREATE = 103 [(clarifai.api.utils.description) = "Organization member invitations created"]; ORGANIZATION_MEMBER_INVITATION_CANCEL = 104 [(clarifai.api.utils.description) = "Organization member invitations canceled"]; ORGANIZATION_MEMBER_INVITATION_ACCEPT = 105 [(clarifai.api.utils.description) = "Organization member invitations accepted"]; ORGANIZATION_MEMBER_INVITATION_DECLINE = 106 [(clarifai.api.utils.description) = "Organization member invitations declined"]; ORGANIZATION_TEAM_CREATE = 107 [(clarifai.api.utils.description) = "Organization teams created"]; ORGANIZATION_TEAM_UPDATE = 108 [(clarifai.api.utils.description) = "Organization teams updated"]; ORGANIZATION_TEAM_DELETE = 109 [(clarifai.api.utils.description) = "Organization teams deleted"]; ORGANIZATION_TEAM_MEMBER_ADD = 110 [(clarifai.api.utils.description) = "Organization team members added"]; ORGANIZATION_TEAM_MEMBER_REMOVE = 111 [(clarifai.api.utils.description) = "Organization team members removed"]; ORGANIZATION_TEAM_APP_ADD = 112 [(clarifai.api.utils.description) = "Organization team applications added"]; ORGANIZATION_TEAM_APP_REMOVE = 113 [(clarifai.api.utils.description) = "Organization team applications removed"]; // Event types related to models: 300 - 399 MODEL_CREATE = 300 [(clarifai.api.utils.description) = "Models created"]; MODEL_UPDATE = 301 [(clarifai.api.utils.description) = "Models updated"]; MODEL_DELETE = 302 [(clarifai.api.utils.description) = "Models deleted"]; MODEL_VERSION_CREATE = 303 [(clarifai.api.utils.description) = "Model versions created"]; MODEL_VERSION_UPDATE = 304 [(clarifai.api.utils.description) = "Model versions updated"]; MODEL_VERSION_DELETE = 305 [(clarifai.api.utils.description) = "Model versions deleted"]; // Event types related to workflows: 400 - 499 WORKFLOW_CREATE = 400 [(clarifai.api.utils.description) = "Workflows created"]; WORKFLOW_UPDATE = 401 [(clarifai.api.utils.description) = "Workflows updated"]; WORKFLOW_DELETE = 402 [(clarifai.api.utils.description) = "Workflows deleted"]; WORKFLOW_VERSION_CREATE = 403 [(clarifai.api.utils.description) = "Workflow versions created"]; WORKFLOW_VERSION_UPDATE = 404 [(clarifai.api.utils.description) = "Workflow versions updated"]; WORKFLOW_VERSION_DELETE = 405 [(clarifai.api.utils.description) = "Workflow versions deleted"]; // Event types related to datasets: 500 - 599 // Event types related to applications: 600 - 699 APPLICATION_CREATE = 600 [(clarifai.api.utils.description) = "Applications created"]; APPLICATION_UPDATE = 601 [(clarifai.api.utils.description) = "Applications updated"]; APPLICATION_DELETE = 602 [(clarifai.api.utils.description) = "Applications deleted"]; // Event types related to collaborators: 700 - 799 COLLABORATOR_ADD = 700 [(clarifai.api.utils.description) = "Collaborators added"]; COLLABORATOR_UPDATE = 701 [(clarifai.api.utils.description) = "Collaborators updated"]; COLLABORATOR_REMOVE = 702 [(clarifai.api.utils.description) = "Collaborators removed"]; // Event types related to users: 800 - 899 USER_UPDATE = 800 [(clarifai.api.utils.description) = "Users updated"]; // Event types related to compute clusters: 900-999 COMPUTE_CLUSTER_CREATE = 900 [(clarifai.api.utils.description) = "Compute clusters created"]; COMPUTE_CLUSTER_UPDATE = 902 [(clarifai.api.utils.description) = "Compute clusters updated"]; COMPUTE_CLUSTER_DELETE = 901 [(clarifai.api.utils.description) = "Compute clusters deleted"]; // Event types related to nodepools: 1000-1099 NODEPOOL_CREATE = 1000 [(clarifai.api.utils.description) = "Nodepools created"]; NODEPOOL_UPDATE = 1001 [(clarifai.api.utils.description) = "Nodepools updated"]; NODEPOOL_DELETE = 1002 [(clarifai.api.utils.description) = "Nodepools deleted"]; // Event types related to deployments: 1100-1199 DEPLOYMENT_CREATE = 1100 [(clarifai.api.utils.description) = "Deployments created"]; DEPLOYMENT_UPDATE = 1101 [(clarifai.api.utils.description) = "Deployments updated"]; DEPLOYMENT_DELETE = 1102 [(clarifai.api.utils.description) = "Deployments deleted"]; } // AuditLogTarget is a resource on which an operation recorded in an // audit log was performed. message AuditLogTarget { reserved 5, 6; oneof target { User user = 1; Role role = 2; Team team = 3; App app = 4; Workflow workflow = 7; WorkflowVersion workflow_version = 8; Model model = 9; ModelVersion model_version = 10; ComputeCluster compute_cluster = 11; Nodepool nodepool = 12; Deployment deployment = 13; } } // AuditLogEntry is a single operation recorded in an audit log. message AuditLogEntry { google.protobuf.Timestamp timestamp = 1; // Time of the operation. User user = 2; // User that performed the operation. EventType operation = 3; // Type of operation that was performed. string description = 4; // A human-readable description of the operation. // Targets of the operation. For example, // - when creating a new model, the targets would be the application and the model, // - when adding a team member, the targets would be the team and the member. repeated AuditLogTarget targets = 5; // Additional human-readable details of the operation. For example, // when patching a resource, these would list what was changed. repeated string details = 6; bool success = 7 [(clarifai.api.utils.cl_show_if_empty) = true]; // Was the operation successful? string req_id = 8; // Request that triggered the operation. string source_ip = 9; // IP address where the request originated from. } // AuditLogQuery is a query for audit log entries. message AuditLogQuery { google.protobuf.Timestamp timestamp_from = 1; // Query operations within this time range. google.protobuf.Timestamp timestamp_to = 2; repeated string user_ids = 3; // Query operations by these users. repeated EventType operations = 4; // Query these types of operations. repeated AuditLogTarget targets = 5; // Query operations with these targets. google.protobuf.BoolValue success = 6; // Query operations by success. repeated string source_ips = 7; // Query operations by source IP address. } message WorkflowVersionEvaluationMetric { string id = 1; string summary = 2; string description = 3; // Metric data type - string, float, int DataType data_type = 4; VisualisationType visualisation_type = 5; // Enum for data types enum DataType { DATA_TYPE_NOT_SET = 0; FLOAT = 1; } // Enum for visualization types enum VisualisationType { VISUALIZATION_TYPE_NOT_SET = 0; CONFUSION_MATRIX = 1; PRECISION_RECALL_CURVE = 2; ROC_AUC_CURVE = 3; } } message WorkflowVersionEvaluationTemplate { string id = 1; string description = 2; // Applicable for the task types like TextClassification, TextGeneration, etc repeated TaskType task_types = 3; // The Workflow Evaluation template metrics repeated WorkflowVersionEvaluationMetric workflow_version_evaluation_metrics = 4; enum TaskType { TASK_TYPE_NOT_SET = 0; TEXT_CLASSIFICATION = 1; } } // ComputePlaneMetrics captures the compute plane metrics to send back to the control plane. // Each message should have the meta filled and one or more of the other fields. message ComputePlaneMetrics { // Who and where the metrics are from. ComputeSourceMetadata meta = 1; // required. string cloud = 2; // e.g. aws, azure, on-prem. string region = 3; // e.g. us-east, us-west. string instance_type = 4; // e.g. t3a.medium, g5.xlarge. string reservation_type = 5; // e.g. spot, on-demand. // Metrics billing float reservation_price = 6; // cost of the reservation. Spot prices may change over time. int32 runtime_s = 7; // Runtime in seconds. // Metrics for latency. google.protobuf.Timestamp timestamp = 8; // Time of the event. string event_type = 9; // e.g. NodeProvisioned, NodeTerminated, ModelDeployed, ModelScheduled, ModelReady. repeated GpuMetrics gpu_metrics = 10; // GPU metrics. string hostname = 11; // Hostname of the node. repeated CpuMetrics cpu_metrics = 12; // CPU metrics. // When true, indicates this is the final metrics submission from a node that is being // terminated. The handler should skip node count aggregation to avoid corrupting the // count with a partial batch from the dying node. bool is_final_metrics = 13; } message GpuMetrics { string uuid = 1; // GPU UUID. string model_name = 2; // GPU model name. e.g. NVIDIA_A10G float utilization_pct = 3; // GPU utilization. e.g. DCGM_FI_DEV_GPU_UTIL float tensor_utilization_pct = 4; // Tensor utilization. e.g. DCGM_FI_PROF_PIPE_TENSOR_ACTIVE float memory_utilization_pct = 5; // Memory utilization. e.g. DCGM_FI_PROF_DRAM_ACTIVE } message CpuMetrics { google.protobuf.Timestamp timestamp = 1; // Time of the event. float cpu_utilization_pct = 2; // CPU utilization. float memory_utilization_pct = 3; // Memory utilization. int64 millicores = 4; // CPU millicores. int64 memory_bytes = 5; // Memory bytes. } // RunnerReplicaTaskMetrics captures auto-annotation task metrics reported by an individual // runner replica. Each entry corresponds to a specific task running on a specific replica. message RunnerReplicaTaskMetrics { string task_id = 1; // Time window these metrics cover. google.protobuf.Timestamp start_time = 2; google.protobuf.Timestamp end_time = 3; // Frame time range in milliseconds (nullable — absent when no frames have been processed yet). google.protobuf.UInt64Value start_frame_time = 4; google.protobuf.UInt64Value end_frame_time = 5; // Frame counts. uint32 frames_processed = 10; uint32 frames_dropped = 11; uint32 frames_skipped = 12; // Annotation counts. uint32 annotations_uploaded = 20; uint32 annotations_failed = 21; uint32 batches_uploaded = 30; uint32 batches_dropped = 31; // Upload latency metrics (nullable — absent means not yet measured). google.protobuf.Duration avg_upload_latency = 40; google.protobuf.Duration max_upload_latency = 41; google.protobuf.Duration last_upload_latency = 42; // Stream delay metrics (nullable — absent means not yet measured). google.protobuf.Duration avg_stream_delay = 50; google.protobuf.Duration max_stream_delay = 51; google.protobuf.Duration last_stream_delay = 52; // Queue metrics. uint32 predict_queue_size = 60; uint32 upload_queue_size = 61; // Current probability of dropping a frame (0.0 – 1.0). float current_frame_drop_chance = 70; } // LogEntry is a single technical log entry (e.g. service log, stack traces, etc). message LogEntry { // Text of the log entry. string message = 1; // The type of log entry. Examples: model, agent, build, training. string log_type = 2; // URL to log file or stream. string url = 3; // Deprecated metadata fields. reserved 4, 5, 6, 7, 8; // Who and where the metrics are from. ComputeSourceMetadata meta = 9; } // ComputeSourceMetadata describes the source of something computed. The who and where. message ComputeSourceMetadata { reserved 5; // The user app id, if any. UserAppIDSet user_app_id = 1; // The Model ID, if any. string model_id = 2; // The Version ID, if any. string model_version_id = 3; // Workflow Id, if any. string workflow_id = 4; // Compute Cluster, Nodepool, Runner. string compute_cluster_user_id = 14; string compute_cluster_id = 6; string nodepool_id = 7; string runner_id = 8; // Pipeline related data, if any string pipeline_id = 9; string pipeline_version_id = 10; string pipeline_version_run_id = 11; // Pipeline step related data for pipeline step builds. string pipeline_step_id = 12; string pipeline_step_version_id = 13; } message WorkflowVersionEvaluation { // Customer-Facing / External ID of the workflow version evaluation. string id = 1; // Workflow version that is being evaluated. WorkflowVersion workflow_version = 2; // The target node id that is being evaluated. string target_node_id = 3; // The dataset version that contains the ground-truth and is used for evaluation. DatasetVersion ground_truth_dataset_version = 4; // The dataset version that contains the predictions and is used for evaluation. DatasetVersion predictions_dataset_version = 5; // Evaluation template that is used for evaluation. WorkflowVersionEvaluationTemplate workflow_version_evaluation_template = 6; // The user the workflow version evaluation belongs to. string user_id = 7; // The app the workflow version evaluation belongs to. string app_id = 8; // Results of the evaluation. WorkflowEvaluationResult workflow_evaluation_result = 9; // Status of the evaluation clarifai.api.status.Status status = 10; // When the workflow version evaluation was created. google.protobuf.Timestamp created_at = 11; // When the workflow version evaluation was modified. google.protobuf.Timestamp modified_at = 12; } message WorkflowEvaluationResult { // The summary of the evaluation result. WorkflowEvaluationResultSummary summary = 1; } message WorkflowEvaluationResultSummary { // The evaluation metrics. repeated EvaluationMetricValue evaluation_metric_values = 2; } message EvaluationMetricValue { // The metric values string evaluation_metric_id = 1; // Aggregated metric value MetricValue metric_value = 2; // explanation for the value string explanation = 3; // Metric values for each concept map per_concept_values = 4; } message InputEvaluationMetricValue { // The metric values string evaluation_metric_id = 1; // Aggregated metric value MetricValue metric_value = 2; // explanation for the value string explanation = 3; // Metric values for each region map per_region_values = 4; } message MetricValue { oneof metric_value { string string_value = 1; float float_value = 2; int32 int_value = 3; } } // The evaluation result at the input/sample level message WorkflowEvaluationInputResult { repeated InputEvaluationMetricValue input_evaluation_metric_values = 1; } message WorkflowVersionEvaluationData { string id = 1; // the data example id Input input = 2; repeated Data ground_truths = 3; repeated Data predictions = 4; WorkflowEvaluationInputResult workflow_evaluation_sample_result = 5; } message ArgoParameterOverride { string name = 1; string value = 2; // Argo parameters are always strings } message ArgoArgsOverride { repeated ArgoParameterOverride parameters = 1; } message OrchestrationArgsOverride { oneof override { ArgoArgsOverride argo_args_override = 1; // Future: KubeFlowArgsOverride, AirflowArgsOverride, etc. } } message ArgoOrchestrationSpec { // The API version of the orchestration specification. // Example: "argoproj.io/v1alpha1", "argoproj.io/v1beta1" string api_version = 2; // The JSON representation of the Argo orchestration specification. string spec_json = 3; } message OrchestrationSpec { oneof orchestration { ArgoOrchestrationSpec argo_orchestration_spec = 1; // Argo orchestration specification // Enable adding other orchestration engines spec here in the future e.g. flyte_orchestration_spec and so on } } message PipelineStepInputParam { // The name of the input parameter. string name = 1; // The default value of the input parameter. string default_value = 2; // The description of the input parameter. string description = 3; // The accepted values for the input parameter. repeated string accepted_values = 4; } message PipelineStep { // The ID of the pipeline step. string id = 1; // The user the pipeline step belongs to string user_id = 2; // Description of the pipeline step string description = 3; // Latest Pipeline Step Version PipelineStepVersion pipeline_step_version = 4; // The visibility field represents whether this is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 5; // When the pipeline step was created google.protobuf.Timestamp created_at = 6; // When the pipeline step was last modified google.protobuf.Timestamp modified_at = 7; } message OrchestrationStepSpec { oneof orchestration { ArgoOrchestrationStepSpec argo_orchestration_step_spec = 1; // Argo orchestration step template } } message ArgoOrchestrationStepSpec { // The API version of the orchestration W. // Example: "argoproj.io/v1alpha1", "argoproj.io/v1beta1" string api_version = 1; // The JSON representation of the Argo Workflow Template string spec_json = 2; } message PipelineStepVersion { // The ID of the pipeline step version. string id = 1; // The user the pipeline step version belongs to string user_id = 2; // The app the pipeline step version belongs to string app_id = 3; // Description of the pipeline step version string description = 4; // Pipeline Step PipelineStep pipeline_step = 5; // Orchestration Step Specification using oneof OrchestrationStepSpec orchestration_step_spec = 6; // The pipeline step version input parameters repeated PipelineStepInputParam pipeline_step_input_params = 7; // Pipeline step Status - Created, Building Artifacts, Completed, Failed clarifai.api.status.Status status = 8; // The minimum required compute resource to run the pipeline step as part of a Pipeline ComputeInfo pipeline_step_compute_info = 9; // Build information for the pipeline step BuildInfo build_info = 10; // The visibility field represents whether this is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 11; // When the pipeline step was created google.protobuf.Timestamp created_at = 12; // When the pipeline step was last modified google.protobuf.Timestamp modified_at = 13; } message Pipeline { string id = 1; // The user the pipeline belongs to string user_id = 2; // The app the pipeline belongs to string app_id = 3; // Latest Pipeline Version PipelineVersion pipeline_version = 5; // Short description about this pipeline string description = 6; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 7; // Notes for the Pipeline. This field should be used for in-depth notes and supports up to 64Kbs. string notes = 8; // To handle arbitrary json metadata, use a struct field // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 9; // When the pipeline was created google.protobuf.Timestamp created_at = 10; // When the pipeline was last modified google.protobuf.Timestamp modified_at = 11; } message PipelineVersionConfig { // StepVersionSecrets maps step names to their secret configurations // Using google.protobuf.Struct to create the desired flat JSON structure // This produces: {stepName: {secretName: "users/user-name/secrets/key"}} // example: {"step-0": {"API_KEY": "users/user-name/secrets/key"}} map step_version_secrets = 1; } message PipelineVersion { string id = 1; // The app the pipeline version belongs to. string app_id = 2; // The user the pipeline version belongs to. string user_id = 3; // Orchestration Specification using oneof OrchestrationSpec orchestration_spec = 4; // Pipeline's Id string pipeline_id = 5; // Short description about this pipeline version string description = 6; // The visibility field represents whether this message is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 7; // To handle arbitrary json metadata, use a struct field // https://github.com/google/protobuf/blob/master/src/google/protobuf/struct.proto google.protobuf.Struct metadata = 8; // When the pipeline was created google.protobuf.Timestamp created_at = 9; // When the pipeline was last modified google.protobuf.Timestamp modified_at = 10; // Pipeline version configuration including step secrets PipelineVersionConfig config = 11; } message OrchestrationStatus { // Oneof field for extensibility, supporting different orchestration systems oneof status_details { ArgoOrchestrationStatus argo_status = 1; // Status for Argo Workflow // Add other orchestration statuses here as needed similar to OrchestrationSpec } // This will help us with filtering the PipelineVersionRuns based on status clarifai.api.status.Status status = 2; } // Argo Workflow Status message message ArgoOrchestrationStatus { // Refer https://pkg.go.dev/github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1#WorkflowStatus string status = 1; } message PipelineVersionRun { string id = 1; // Pipeline Version associated with this run PipelineVersion pipeline_version = 2; // Nodepool(s) used for the Pipeline Version Run repeated Nodepool nodepools = 3; // Orchestration Status for this run, supporting multiple orchestration systems OrchestrationStatus orchestration_status = 4; // The user the pipeline belongs to string user_id = 5; // The app the pipeline belongs to string app_id = 6; // When the pipeline was created google.protobuf.Timestamp created_at = 7; // When the pipeline was last modified google.protobuf.Timestamp modified_at = 8; // Optional: Overrides to input arguments for the orchestration system. OrchestrationArgsOverride input_args_override = 9; // Final merged orchestration spec snapshot submitted to backend. // This field is read-only and cannot be set during creation. OrchestrationSpec orchestration_spec = 10; // When the pipeline run started executing (first JOB_RUNNING status) google.protobuf.Timestamp started_at = 11; // When the pipeline run finished executing (terminal status) google.protobuf.Timestamp ended_at = 12; // Total cumulative run duration excluding pause time google.protobuf.Duration total_run_duration = 13; } message PipelineVersionRunStatusLog { // Unique ID for this status record string id = 1; // The PipelineVersionRun this log entry belongs to string pipeline_version_run_id = 2; // Clarifai status object representing the current status clarifai.api.status.Status status = 3; // Optional human-readable reason or message (e.g. "Paused due to resource limit") string message = 4; // Who triggered this status change string triggered_by_user_id = 5; // When this status was recorded google.protobuf.Timestamp created_at = 6; } message PipelineTemplateParameter { // Name of the parameter string name = 1; enum Type { PARAMETER_TYPE_UNSPECIFIED = 0; STRING = 1; INT = 2; FLOAT = 3; BOOL = 4; } // Type of the parameter (string, int, float, bool) Type type = 2; // Whether this parameter is required bool required = 3; oneof default_value { // Default value when type is STRING string default_string = 4; // Default value when type is INT int64 default_int = 5; // Default value when type is FLOAT double default_float = 6; // Default value when type is BOOL bool default_bool = 7; } // Description of the parameter string description = 8; message Constraints { message StringConstraints { // List of allowed values for the string parameter repeated string allowed_values = 1; // Whether free text input is allowed beyond allowed_values bool allow_free_text = 2; // Minimum length of the string uint32 min_length = 3; // Maximum length of the string uint32 max_length = 4; } message IntConstraints { // Minimum value for the int parameter int64 min = 1; // Maximum value for the int parameter int64 max = 2; } message FloatConstraints { // Minimum value for the float parameter double min = 1; // Maximum value for the float parameter double max = 2; } oneof constraints { StringConstraints string_constraints = 1; IntConstraints int_constraints = 2; FloatConstraints float_constraints = 3; } } // Constraints to validate parameter values Constraints constraints = 9; message UIHints { enum UIWidget { UI_WIDGET_UNSPECIFIED = 0; TEXT_INPUT = 1; NUMBER_INPUT = 2; SLIDER = 3; DROPDOWN = 4; TOGGLE = 5; } // The UI widget to use for displaying this parameter UIWidget widget = 1; // Used for dropdowns populated dynamically string data_source = 2; // Used when backend expects string even if UI shows bool/int bool serialize_as_string = 3; } // UI hints for rendering this parameter in the frontend UIHints ui_hints = 10; } message PipelineTemplate { // Name of the template string name = 1; // Description of the template string description = 2; enum PipelineTemplateType { UNKNOWN = 0; TRAINING = 1; } // Type of the pipeline template PipelineTemplateType pipeline_template_type = 3; // Compute resources required to run this pipeline template ComputeInfo compute_info = 4; // List of parameters that can be configured for this template repeated PipelineTemplateParameter parameters = 5; // Map from pipeline step name to pipeline step reference map pipeline_steps = 6; } message Secret { reserved 2, 3, 9; // The name of the secret, corresponds to id in model_version.output_info.params.secrets string id = 1; // The user the secret belongs to. string user_id = 11; // The value of the secret. string value = 4; // The version of the secret. uint32 version = 5; // The description of the secret. string description = 6; // When the secret was created. google.protobuf.Timestamp created_at = 7; // When the secret was last modified. google.protobuf.Timestamp modified_at = 8; // When the secret will expire. google.protobuf.Timestamp expires_at = 10; } message MetricData { message Label { MetricLabel name = 1; string value = 2; } message MetricSample { // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". google.protobuf.Timestamp timestamp = 1; double value = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; } message MatrixData { message TimeSeries { repeated Label labels = 1; repeated MetricSample value = 2; } repeated TimeSeries series = 1; } oneof data { MatrixData matrix_data = 1; } } enum MetricType { METRIC_TYPE_NOT_SET = 0; // Model metrics: 1-50 MODEL_TTFT = 1; MODEL_THROUGHPUT = 2; MODEL_PROMPT_TOKEN_THROUGHPUT = 3; MODEL_COMPLETION_TOKEN_THROUGHPUT = 4; MODEL_REQUEST_COUNT = 5; MODEL_E2E_LATENCY = 6; // Nodepool metrics: 51-100 NODEPOOL_NODE_COUNT = 51; NODEPOOL_GPU_UTILIZATION = 52; NODEPOOL_GPU_TENSOR_UTILIZATION = 53; NODEPOOL_GPU_MEMORY_UTILIZATION = 54; NODEPOOL_CPU_UTILIZATION = 55; NODEPOOL_MEMORY_UTILIZATION = 56; } enum MetricLabel { METRIC_LABEL_NOT_SET = 0; APP_ID = 1; MODEL_ID = 2; MODEL_VERSION_ID = 3; HTTP_STATUS = 4; COMPUTE_CLUSTER_ID = 5; NODEPOOL_ID = 6; } message MetricAggregate { enum Operator { OPERATOR_NOT_SET = 0; AVG = 1; SUM = 2; MAX = 3; MIN = 4; P95 = 5; P99 = 6; P50 = 7; COUNT = 8; } Operator operator = 1; repeated MetricLabel labels = 2; } message MetricFilter { message MultiValues { repeated string in = 1; } MetricLabel label = 1; oneof value { string equals = 2; MultiValues in = 3; } } message MetricSearchQuery { MetricType metric_type = 1; // start time of the search window google.protobuf.Timestamp start_time = 2; // end time of the search window google.protobuf.Timestamp end_time = 3; // duration string https://pkg.go.dev/time#ParseDuration string resolution = 4; repeated MetricFilter filters = 5; // ANDed MetricAggregate aggregate = 6; } message MetricTypeLabels { message LabelWithValues { MetricLabel label = 1; repeated string values = 2 [(clarifai.api.utils.cl_show_if_empty) = true]; // sample values for this label } MetricType metric_type = 1; repeated LabelWithValues labels = 2; } // Artifact is a resource that represents a file stored in Clarifai's storage system. // It should have a reference to upload resource which contains the actual file location and metadata. message Artifact { // ID of artifact string id = 1; // User ID that this Artifact belongs to string user_id = 2; // Application ID that this Artifact belongs to string app_id = 3; // Reference to the artifact version resource that represents a specific version of the artifact ArtifactVersion artifact_version = 5; // When the artifact was created. google.protobuf.Timestamp created_at = 6; // Most recent time when the artifact was updated. google.protobuf.Timestamp modified_at = 7; // When the artifact was deleted. google.protobuf.Timestamp deleted_at = 8; } // ArtifactVersion represents a specific version of the artifact. message ArtifactVersion { // ID of artifact version string id = 1; // Description of the artifact version string description = 2; // Reference to the artifact resource Artifact artifact = 3; // Reference to the upload resource which contains the actual file location and metadata Upload upload = 4; // The visibility field represents whether this is privately/publicly visible. // To be visible to the public the App that contains it AND the User that contains the App must // also be publicly visible. Visibility visibility = 5; // When the artifact version will expire and be deleted. // The format is https://www.ietf.org/rfc/rfc3339.txt. // Example: "2006-01-02T15:04:05.999999Z". // If not set, the artifact version will be retained indefinitely // and will not be automatically deleted by lifecycle policies. google.protobuf.Timestamp expires_at = 6; // When the artifact version was created. google.protobuf.Timestamp created_at = 7; // Most recent time when the artifact version was updated. google.protobuf.Timestamp modified_at = 8; // When the artifact version was deleted. google.protobuf.Timestamp deleted_at = 9; }