BackendService.thrift

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

namespace cpp doris
namespace java org.apache.doris.thrift

include "Status.thrift"
include "Types.thrift"
include "PlanNodes.thrift"
include "AgentService.thrift"
include "PaloInternalService.thrift"
include "DorisExternalService.thrift"
include "FrontendService.thrift"

struct TTabletStat {
    1: required i64 tablet_id
    // local data size = local inverted index file size + local segment file size
    2: optional i64 data_size
    3: optional i64 row_count
    4: optional i64 total_version_count
    // remote data size = remote inverted index file size + remote segment file size
    5: optional i64 remote_data_size
    6: optional i64 visible_version_count
    7: optional i64 visible_version
    8: optional i64 local_index_size = 0      // .idx
    9: optional i64 local_segment_size = 0    // .dat
    10: optional i64 remote_index_size = 0    // .idx
    11: optional i64 remote_segment_size = 0  // .dat
}

struct TTabletStatResult {
    1: required map<i64, TTabletStat> tablets_stats
    2: optional list<TTabletStat> tablet_stat_list
}

struct TKafkaLoadInfo {
    1: required string brokers;
    2: required string topic;
    3: required map<i32, i64> partition_begin_offset;
    4: optional map<string, string> properties;
}

struct TRoutineLoadTask {
    1: required Types.TLoadSourceType type
    2: required i64 job_id
    3: required Types.TUniqueId id
    4: required i64 txn_id
    5: required i64 auth_code
    6: optional string db
    7: optional string tbl
    8: optional string label
    9: optional i64 max_interval_s
    10: optional i64 max_batch_rows
    11: optional i64 max_batch_size
    12: optional TKafkaLoadInfo kafka_load_info
    // 13: optional PaloInternalService.TExecPlanFragmentParams params # deprecated
    14: optional PlanNodes.TFileFormatType format
    15: optional PaloInternalService.TPipelineFragmentParams pipeline_params 
    16: optional bool is_multi_table
    17: optional bool memtable_on_sink_node;
    18: optional string qualified_user
    19: optional string cloud_cluster
}

struct TKafkaMetaProxyRequest {
    1: optional TKafkaLoadInfo kafka_info
}

struct TKafkaMetaProxyResult {
    1: optional list<i32> partition_ids
}

struct TProxyRequest {
    1: optional TKafkaMetaProxyRequest kafka_meta_request;
}

struct TProxyResult {
    1: required Status.TStatus status;
    2: optional TKafkaMetaProxyResult kafka_meta_result;
}

struct TStreamLoadRecord {
    1: optional string cluster
    2: required string user
    3: required string passwd
    4: required string db
    5: required string tbl
    6: optional string user_ip
    7: required string label
    8: required string status
    9: required string message
    10: optional string url
    11: optional i64 auth_code;
    12: required i64 total_rows
    13: required i64 loaded_rows
    14: required i64 filtered_rows
    15: required i64 unselected_rows
    16: required i64 load_bytes
    17: required i64 start_time
    18: required i64 finish_time
    19: optional string comment
    20: optional string first_error_msg
}

struct TStreamLoadRecordResult {
    1: required map<string, TStreamLoadRecord> stream_load_record
}

struct TDiskTrashInfo {
    1: required string root_path
    2: required string state
    3: required i64 trash_used_capacity
}

struct TCheckStorageFormatResult {
    1: optional list<i64> v1_tablets;
    2: optional list<i64> v2_tablets;
}

struct TWarmUpCacheAsyncRequest {
    1: required string host
    2: required i32 brpc_port
    3: required list<i64> tablet_ids
    4: optional string cloud_compute_group_id
}

struct TWarmUpCacheAsyncResponse {
    1: required Status.TStatus status
}

struct TCheckWarmUpCacheAsyncRequest {
    1: optional list<i64> tablets
}

struct TCheckWarmUpCacheAsyncResponse {
    1: required Status.TStatus status
    2: optional map<i64, bool> task_done;
}

struct TSyncLoadForTabletsRequest {
    1: required list<i64> tablet_ids
}

struct TSyncLoadForTabletsResponse {
}

struct THotPartition {
    1: required i64 partition_id
    2: required i64 last_access_time
    3: optional i64 query_per_day
    4: optional i64 query_per_week
}

struct THotTableMessage {
    1: required i64 table_id
    2: required i64 index_id
    3: optional list<THotPartition> hot_partitions
}

struct TGetTopNHotPartitionsRequest {
}

struct TGetTopNHotPartitionsResponse {
    1: required i64 file_cache_size
    2: optional list<THotTableMessage> hot_tables
}

enum TDownloadType {
    BE = 0,
    S3 = 1,
}

enum TWarmUpEventType {
    LOAD = 0,
    QUERY = 1,
}

enum TWarmUpTabletsRequestType {
    SET_JOB = 0,
    SET_BATCH = 1,
    GET_CURRENT_JOB_STATE_AND_LEASE = 2,
    CLEAR_JOB = 3
}

struct TJobMeta {
    1: required TDownloadType download_type
    2: optional string be_ip
    3: optional i32 brpc_port
    4: optional list<i64> tablet_ids
}

struct TWarmUpTabletsRequest {
    1: required i64 job_id
    2: required i64 batch_id
    3: optional list<TJobMeta> job_metas
    4: required TWarmUpTabletsRequestType type
    5: optional TWarmUpEventType event
}

struct TWarmUpTabletsResponse {
    1: required Status.TStatus status;
    2: optional i64 job_id
    3: optional i64 batch_id
    4: optional i64 pending_job_size
    5: optional i64 finish_job_size
}

struct TIngestBinlogRequest {
    1: optional i64 txn_id;
    2: optional i64 remote_tablet_id;
    3: optional i64 binlog_version;
    4: optional string remote_host;
    5: optional string remote_port;
    6: optional i64 partition_id;
    7: optional i64 local_tablet_id;
    8: optional Types.TUniqueId load_id;
}

struct TIngestBinlogResult {
    1: optional Status.TStatus status;
    2: optional bool is_async;
}

struct TQueryIngestBinlogRequest {
    1: optional i64 txn_id;
    2: optional i64 partition_id;
    3: optional i64 tablet_id;
    4: optional Types.TUniqueId load_id;
}

enum TIngestBinlogStatus {
    ANALYSIS_ERROR = 0,
    UNKNOWN = 1,
    NOT_FOUND = 2,
    OK = 3,
    FAILED = 4,
    DOING = 5
}

struct TQueryIngestBinlogResult {
    1: optional TIngestBinlogStatus status;
    2: optional string err_msg;
}

enum TTopicInfoType {
    WORKLOAD_GROUP = 0,
    MOVE_QUERY_TO_GROUP = 1,
    WORKLOAD_SCHED_POLICY = 2
}

enum TWgSlotMemoryPolicy {
    NONE = 0,
    FIXED = 1,
    DYNAMIC = 2
}

struct TWorkloadGroupInfo {
  1: optional i64 id
  2: optional string name
  3: optional i64 version
  4: optional i32 min_cpu_percent
  5: optional i32 max_cpu_percent
  6: optional string mem_limit  // deprecated
  7: optional bool enable_memory_overcommit // deprecated
  8: optional bool enable_cpu_hard_limit    // deprecated
  9: optional i32 scan_thread_num
  10: optional i32 max_remote_scan_thread_num
  11: optional i32 min_remote_scan_thread_num
  12: optional i32 memory_low_watermark
  13: optional i32 memory_high_watermark
  14: optional i64 read_bytes_per_second
  15: optional i64 remote_read_bytes_per_second
  16: optional string tag // deprecated
  17: optional i32 total_query_slot_count
  18: optional i32 write_buffer_ratio
  19: optional TWgSlotMemoryPolicy slot_memory_policy
  20: optional i32 min_memory_percent
  21: optional i32 max_memory_percent
}

enum TWorkloadMetricType {
    QUERY_TIME = 0,
    BE_SCAN_ROWS = 1,
    BE_SCAN_BYTES = 2,
    QUERY_BE_MEMORY_BYTES = 3
}

enum TCompareOperator {
    EQUAL = 0,
    GREATER = 1,
    GREATER_EQUAL = 2,
    LESS = 3,
    LESS_EQUAL = 4
}

struct TWorkloadCondition {
    1: optional TWorkloadMetricType metric_name
    2: optional TCompareOperator op
    3: optional string value
}

enum TWorkloadActionType {
    MOVE_QUERY_TO_GROUP = 0,
    CANCEL_QUERY = 1
}

struct TWorkloadAction {
    1: optional TWorkloadActionType action
    2: optional string action_args
}

struct TWorkloadSchedPolicy {
    1: optional i64 id
    2: optional string name
    3: optional i32 version
    4: optional i32 priority
    5: optional bool enabled
    6: optional list<TWorkloadCondition> condition_list
    7: optional list<TWorkloadAction> action_list
    8: optional list<i64> wg_id_list
}

struct TopicInfo {
    1: optional TWorkloadGroupInfo workload_group_info
    2: optional TWorkloadSchedPolicy workload_sched_policy
}

struct TPublishTopicRequest {
    1: required map<TTopicInfoType, list<TopicInfo>> topic_map
}

struct TPublishTopicResult {
    1: required Status.TStatus status
}

struct TGetRealtimeExecStatusRequest {
    // maybe query id or other unique id
    1: optional Types.TUniqueId id
    2: optional string req_type // "stats" or "profile"
}

struct TGetRealtimeExecStatusResponse {
    1: optional Status.TStatus status
    2: optional FrontendService.TReportExecStatusParams report_exec_status_params
    // query_stats is for getting real-time query statistics of a certain query
    3: optional FrontendService.TQueryStatistics query_stats
}

struct TDictionaryStatus {
    1: optional i64 dictionary_id
    2: optional i64 version_id
    3: optional i64 dictionary_memory_size
}

struct TDictionaryStatusList {
    1: optional list<TDictionaryStatus> dictionary_status_list
}

struct TTestStorageConnectivityRequest {
    1: optional Types.TStorageBackendType type;
    2: optional map<string, string> properties;
}

struct TTestStorageConnectivityResponse {
    1: optional Status.TStatus status;
}

struct TPythonEnvInfo {
    1: optional string env_name       // e.g. "myenv"
    2: optional string full_version   // e.g. "3.9.16"
    3: optional string env_type       // "conda" or "venv"
    4: optional string base_path      // e.g. "/opt/miniconda3/envs/myenv"
    5: optional string executable_path
}

struct TPythonPackageInfo {
    1: optional string package_name
    2: optional string version
}

service BackendService {
    AgentService.TAgentResult submit_tasks(1:list<AgentService.TAgentTaskRequest> tasks);

    AgentService.TAgentResult make_snapshot(1:AgentService.TSnapshotRequest snapshot_request);

    AgentService.TAgentResult release_snapshot(1:string snapshot_path);

    AgentService.TAgentResult publish_cluster_state(1:AgentService.TAgentPublishRequest request);

    TTabletStatResult get_tablet_stat();

    i64 get_trash_used_capacity();

    list<TDiskTrashInfo> get_disk_trash_used_capacity();

    Status.TStatus submit_routine_load_task(1:list<TRoutineLoadTask> tasks);

    // doris will build  a scan context for this session, context_id returned if success
    DorisExternalService.TScanOpenResult open_scanner(1: DorisExternalService.TScanOpenParams params);

    // return the batch_size of data
    DorisExternalService.TScanBatchResult get_next(1: DorisExternalService.TScanNextBatchParams params);

    // release the context resource associated with the context_id
    DorisExternalService.TScanCloseResult close_scanner(1: DorisExternalService.TScanCloseParams params);

    TStreamLoadRecordResult get_stream_load_record(1: i64 last_stream_record_time);

    // check tablet rowset type
    TCheckStorageFormatResult check_storage_format();

    TWarmUpCacheAsyncResponse warm_up_cache_async(1: TWarmUpCacheAsyncRequest request);

    TCheckWarmUpCacheAsyncResponse check_warm_up_cache_async(1: TCheckWarmUpCacheAsyncRequest request);

    TSyncLoadForTabletsResponse sync_load_for_tablets(1: TSyncLoadForTabletsRequest request);

    TGetTopNHotPartitionsResponse get_top_n_hot_partitions(1: TGetTopNHotPartitionsRequest request);

    TWarmUpTabletsResponse warm_up_tablets(1: TWarmUpTabletsRequest request);

    TIngestBinlogResult ingest_binlog(1: TIngestBinlogRequest ingest_binlog_request);
    TQueryIngestBinlogResult query_ingest_binlog(1: TQueryIngestBinlogRequest query_ingest_binlog_request);

    TPublishTopicResult publish_topic_info(1:TPublishTopicRequest topic_request);

    TGetRealtimeExecStatusResponse get_realtime_exec_status(1:TGetRealtimeExecStatusRequest request);

    // if empty, return all dictionary status.
    TDictionaryStatusList get_dictionary_status(1:list<i64> dictionary_ids);

    // Test storage connectivity (S3, HDFS, etc.)
    TTestStorageConnectivityResponse test_storage_connectivity(1:TTestStorageConnectivityRequest request);

    // Get Python environments available on this BE
    list<TPythonEnvInfo> get_python_envs();

    // Get installed pip packages for a specific Python version
    list<TPythonPackageInfo> get_python_packages(1:string python_version);
}