# HELP python_gc_objects_collected_total Objects collected during gc
# TYPE python_gc_objects_collected_total counter
python_gc_objects_collected_total{generation="0"} 8515.0
python_gc_objects_collected_total{generation="1"} 949.0
python_gc_objects_collected_total{generation="2"} 55.0
# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
# TYPE python_gc_objects_uncollectable_total counter
python_gc_objects_uncollectable_total{generation="0"} 0.0
python_gc_objects_uncollectable_total{generation="1"} 0.0
python_gc_objects_uncollectable_total{generation="2"} 0.0
# HELP python_gc_collections_total Number of times this generation was collected
# TYPE python_gc_collections_total counter
python_gc_collections_total{generation="0"} 2252.0
python_gc_collections_total{generation="1"} 204.0
python_gc_collections_total{generation="2"} 9.0
# HELP python_info Python platform information
# TYPE python_info gauge
python_info{implementation="CPython",major="3",minor="12",patchlevel="3",version="3.12.3"} 1.0
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 1.375297536e+09
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 1.2169216e+08
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.7762519185e+09
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 89.03
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 46.0
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 65536.0
# HELP target_info Target metadata
# TYPE target_info gauge
target_info{service_name="unknown_service",telemetry_sdk_language="python",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="1.39.1"} 1.0
# HELP ray_operation_run_time_ms operation execution time
# TYPE ray_operation_run_time_ms histogram
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 210299.0
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 210400.0
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 210401.0
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 210401.0
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 210402.0
ray_operation_run_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 210402.0
ray_operation_run_time_ms_count{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 210402.0
ray_operation_run_time_ms_sum{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 111260.0
# HELP ray_operation_queue_time_ms operation queuing time
# TYPE ray_operation_queue_time_ms histogram
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 208991.0
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 210224.0
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 210398.0
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 210398.0
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 210402.0
ray_operation_queue_time_ms_bucket{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 210402.0
ray_operation_queue_time_ms_count{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 210402.0
ray_operation_queue_time_ms_sum{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 142847.0
# HELP ray_pull_manager_object_request_time_ms Time between initial object pull request and local pinning of the object. 
# TYPE ray_pull_manager_object_request_time_ms histogram
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="10.0"} 42.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="100.0"} 156.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="1000.0"} 265.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="10000.0"} 403.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId="",le="+Inf"} 543.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId=""} 543.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="MemoryAvailableToPin",Version="2.54.0",WorkerId=""} 3.625451e+06
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="10.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="100.0"} 10.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="1000.0"} 142.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="10000.0"} 341.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId="",le="+Inf"} 827.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId=""} 827.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToCancel",Version="2.54.0",WorkerId=""} 1.088765e+07
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="10.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="100.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="1000.0"} 9.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="10000.0"} 321.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId="",le="+Inf"} 543.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId=""} 543.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Type="StartToPin",Version="2.54.0",WorkerId=""} 6.16095e+06
# HELP ray_object_store_dist The distribution of object size in bytes
# TYPE ray_object_store_dist histogram
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="33554432.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="67108864.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="134217728.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="268435456.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="536870912.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="1073741824.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="2147483648.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="4294967296.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="8589934592.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="17179869184.0"} 151.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId="",le="+Inf"} 151.0
ray_object_store_dist_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId=""} 151.0
ray_object_store_dist_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="CreatedByWorker",Version="2.54.0",WorkerId=""} 2.533359616e+09
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="33554432.0"} 302.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="67108864.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="134217728.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="268435456.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="536870912.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="1073741824.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="2147483648.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="4294967296.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="8589934592.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="17179869184.0"} 311.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId="",le="+Inf"} 311.0
ray_object_store_dist_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId=""} 311.0
ray_object_store_dist_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Source="ReceivedFromRemoteRaylet",Version="2.54.0",WorkerId=""} 5.519704064e+09
# HELP ray_grpc_server_req_process_time_ms Request latency in grpc server
# TYPE ray_grpc_server_req_process_time_ms histogram
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 280.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 332.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 335.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 335.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 335.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 335.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 335.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 605.0000000000005
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 1129.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 1158.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 780.4500000000033
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 5786.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 5810.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 5810.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 5810.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 5810.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 5810.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 5810.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 3314.2999999999975
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 6869.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 6934.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 6936.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 6936.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 6936.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 6936.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6936.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 4245.449999999997
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 45.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 47.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 47.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 47.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 47.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 47.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 35.75000000000001
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 1124.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 1158.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 1158.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 805.2000000000035
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 215.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 334.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 497.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 704.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 742.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 742.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 742.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 332587.74999999994
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 159.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 413.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 511.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 511.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 511.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 511.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 511.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6874.450000000008
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 169.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 372.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 372.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 372.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 372.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 372.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 372.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1209.449999999998
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 11295.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 11528.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 11535.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 11535.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 11535.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 11535.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 11535.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 7878.749999999994
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 102.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 104.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 67.1
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 103.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 104.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 104.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 62.15
# HELP ray_worker_register_time_ms end to end latency of register a worker process.
# TYPE ray_worker_register_time_ms histogram
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="100.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="1000.0"} 22.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="10000.0"} 47.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId="",le="+Inf"} 47.0
ray_worker_register_time_ms_count{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_worker_register_time_ms_sum{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 149600.0
# HELP ray_grpc_server_req_new_total New request number in grpc server
# TYPE ray_grpc_server_req_new_total counter
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6936.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 5810.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 742.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 511.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 335.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 372.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 11535.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
# HELP ray_grpc_server_req_handling_total Request number are handling in grpc server
# TYPE ray_grpc_server_req_handling_total counter
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 335.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 5810.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6936.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 742.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 511.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 372.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 11535.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
# HELP ray_operation_count_total operation count
# TYPE ray_operation_count_total counter
ray_operation_count_total{Component="raylet",Name="raylet_main_io_context",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 210395.0
# HELP ray_grpc_server_req_finished_total Finished request number in grpc server
# TYPE ray_grpc_server_req_finished_total counter
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 335.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 5810.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6936.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 742.0
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 511.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 372.0
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 11535.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
# HELP ray_grpc_server_req_succeeded_total Succeeded request count in grpc server
# TYPE ray_grpc_server_req_succeeded_total counter
ray_grpc_server_req_succeeded_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 335.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetWorkerPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 5810.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 6936.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 47.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetAgentPIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 1158.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 742.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 511.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 372.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 11535.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorkerLease",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 104.0
# HELP ray_internal_num_processes_started The total number of worker processes the worker pool has created.
# TYPE ray_internal_num_processes_started gauge
ray_internal_num_processes_started{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 14470.0
# HELP ray_internal_num_processes_started_from_cache The total number of workers started from a cached worker process.
# TYPE ray_internal_num_processes_started_from_cache gauge
ray_internal_num_processes_started_from_cache{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 17184.0
# HELP ray_grpc_client_req_failed_total Number of gRPC client failures (non-OK response statuses).
# TYPE ray_grpc_client_req_failed_total counter
ray_grpc_client_req_failed_total{Component="raylet",Method="CoreWorkerService.grpc_client.UpdateObjectLocationBatch",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 20.0
ray_grpc_client_req_failed_total{Component="raylet",Method="CoreWorkerService.grpc_client.PubsubLongPolling",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 20.0
# HELP ray_internal_num_processes_skipped_job_mismatch The total number of cached workers skipped due to job mismatch.
# TYPE ray_internal_num_processes_skipped_job_mismatch gauge
ray_internal_num_processes_skipped_job_mismatch{Component="raylet",NodeAddress="172.31.30.77",SessionName="session_2026-04-15_09-17-21_264021_4107",Version="2.54.0",WorkerId=""} 7008.0