Spark JMX Metrics
Every 10 seconds, the collectd service gathers the following Spark JMX
metrics on each node in the cluster.
For detailed information about Spark metrics, see the Apache Spark documentation.
- mapr.spark.driver_block_manager_disk_space_used_mb
- mapr.spark.driver_block_manager_memory_max_mem_mbb
- mapr.spark.driver_block_manager_memory_max_off_heap_mem_mb
- mapr.spark.driver_block_manager_memory_max_on_heap_mem_mb
- mapr.spark.driver_block_manager_memory_mem_used_mb
- mapr.spark.driver_block_manager_memory_off_heap_mem_used_mb
- mapr.spark.driver_block_manager_memory_on_heap_mem_used_mb
- mapr.spark.driver_block_manager_memory_remaining_mem_mb
- mapr.spark.driver_block_manager_memory_remaining_off_heap_mem_mb
- mapr.spark.driver_block_manager_memory_remaining_on_heap_mem_mb
- mapr.spark.driver_dag_scheduler_job_active_jobs
- mapr.spark.driver_dag_scheduler_job_all_jobs
- mapr.spark.driver_dag_scheduler_stage_failed_stages
- mapr.spark.driver_dag_scheduler_stage_running_stages
- mapr.spark.driver_dag_scheduler_stage_waiting_stages
- mapr.spark.driver_live_listener_bus_queue_app_status_size
- mapr.spark.driver_live_listener_bus_queue_event_log_size
- mapr.spark.driver_live_listener_bus_queue_executor_management_size
- mapr.spark.driver_hive_external_catalog_file_cache_hits
- mapr.spark.driver_hive_external_catalog_files_discovered
- mapr.spark.driver_hive_external_catalog_hive_client_calls
- mapr.spark.driver_hive_external_catalog_parallel_listing_job_count
- mapr.spark.driver_hive_external_catalog_partitions_fetched
- mapr.spark.driver_live_listener_bus_num_events_posted
- mapr.spark.driver_live_listener_bus_queue_app_status_num_dropped_events
- mapr.spark.driver_live_listener_bus_queue_event_log_num_dropped_events
- mapr.spark.driver_live_listener_bus_queue_executor_management_num_dropped_events
- mapr.spark.driver_code_generator_compilation_time_count
- mapr.spark.driver_code_generator_compilation_time_max
- mapr.spark.driver_code_generator_compilation_time_mean
- mapr.spark.driver_code_generator_compilation_time_min
- mapr.spark.driver_code_generator_compilation_time_p50
- mapr.spark.driver_code_generator_compilation_time_p75
- mapr.spark.driver_code_generator_compilation_time_p95
- mapr.spark.driver_code_generator_compilation_time_p98
- mapr.spark.driver_code_generator_compilation_time_p99
- mapr.spark.driver_code_generator_compilation_time_p999
- mapr.spark.driver_code_generator_compilation_time_stddev
- mapr.spark.driver_code_generator_generated_class_size_count
- mapr.spark.driver_code_generator_generated_class_size_max
- mapr.spark.driver_code_generator_generated_class_size_mean
- mapr.spark.driver_code_generator_generated_class_size_min
- mapr.spark.driver_code_generator_generated_class_size_p50
- mapr.spark.driver_code_generator_generated_class_size_p75
- mapr.spark.driver_code_generator_generated_class_size_p95
- mapr.spark.driver_code_generator_generated_class_size_p98
- mapr.spark.driver_code_generator_generated_class_size_p99
- mapr.spark.driver_code_generator_generated_class_size_p999
- mapr.spark.driver_code_generator_generated_class_size_stddev
- mapr.spark.driver_code_generator_generated_method_size_count
- mapr.spark.driver_code_generator_generated_method_size_max
- mapr.spark.driver_code_generator_generated_method_size_mean
- mapr.spark.driver_code_generator_generated_method_size_min
- mapr.spark.driver_code_generator_generated_method_size_p50
- mapr.spark.driver_code_generator_generated_method_size_p75
- mapr.spark.driver_code_generator_generated_method_size_p95
- mapr.spark.driver_code_generator_generated_method_size_p98
- mapr.spark.driver_code_generator_generated_method_size_p99
- mapr.spark.driver_code_generator_generated_method_size_p999
- mapr.spark.driver_code_generator_generated_method_size_stddev
- mapr.spark.driver_code_generator_source_code_size_count
- mapr.spark.driver_code_generator_source_code_size_max
- mapr.spark.driver_code_generator_source_code_size_mean
- mapr.spark.driver_code_generator_source_code_size_min
- mapr.spark.driver_code_generator_source_code_size_p50
- mapr.spark.driver_code_generator_source_code_size_p75
- mapr.spark.driver_code_generator_source_code_size_p95
- mapr.spark.driver_code_generator_source_code_size_p98
- mapr.spark.driver_code_generator_source_code_size_p99
- mapr.spark.driver_code_generator_source_code_size_p999
- mapr.spark.driver_code_generator_source_code_size_stddev
- mapr.spark.driver_dag_scheduler_message_processing_time_count
- mapr.spark.driver_dag_scheduler_message_processing_time_max
- mapr.spark.driver_dag_scheduler_message_processing_time_mean
- mapr.spark.driver_dag_scheduler_message_processing_time_min
- mapr.spark.driver_dag_scheduler_message_processing_time_p50
- mapr.spark.driver_dag_scheduler_message_processing_time_p75
- mapr.spark.driver_dag_scheduler_message_processing_time_p95
- mapr.spark.driver_dag_scheduler_message_processing_time_p98
- mapr.spark.driver_dag_scheduler_message_processing_time_p99
- mapr.spark.driver_dag_scheduler_message_processing_time_p999
- mapr.spark.driver_dag_scheduler_message_processing_time_stddev
- mapr.spark.driver_dag_scheduler_message_processing_time_m15_rate
- mapr.spark.driver_dag_scheduler_message_processing_time_m1_rate
- mapr.spark.driver_dag_scheduler_message_processing_time_m5_rate
- mapr.spark.driver_dag_scheduler_message_processing_time_mean_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_count
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_max
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_min
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p50
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p75
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p95
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p98
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p99
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_p999
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_stddev
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m15_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m1_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_m5_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_heartbeat_receiver_mean_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_count
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_max
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_min
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p50
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p75
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p95
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p98
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p99
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_p999
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_stddev
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m15_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m1_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_m5_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_event_logging_listener_mean_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_count
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_max
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_min
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p50
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p75
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p95
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p98
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p99
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_p999
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_stddev
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m15_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m1_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_m5_rate
- mapr.spark.driver_live_listener_bus_listener_processing_time_app_status_listener_mean_rate
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_count
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_max
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_min
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p50
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p75
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p95
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p98
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p99
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_p999
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_stddev
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m15_rat
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m1_rate
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_m5_rate
- mapr.spark.driver_live_listener_bus_queue_app_status_listener_processing_time_mean_rate
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_count
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_max
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_min
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p50
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p75
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p95
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p98
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p99
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_p999
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_stddev
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m15_rate
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m1_rate
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_m5_rate
- mapr.spark.driver_live_listener_bus_queue_event_log_listener_processing_time_mean_rate
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_count
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_max
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_min
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p50
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p75
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p95
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p98
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p99
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_p990
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_stddev
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m15_rate
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m1_rate
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_m5_rate
- mapr.spark.driver_live_listener_bus_queue_executor_management_listener_processing_time_mean_rate
- mapr.spark.driver_jvm_ps_mark_sweep_count
- mapr.spark.driver_jvm_ps_mark_sweep_time
- mapr.spark.driver_jvm_ps_scavenge_count
- mapr.spark.driver_jvm_ps_scavenge_time
- mapr.spark.driver_jvm_direct_capacity
- mapr.spark.driver_jvm_direct_count
- mapr.spark.driver_jvm_direct_used
- mapr.spark.driver_jvm_direct_heap_committed
- mapr.spark.driver_jvm_direct_heap_init
- mapr.spark.driver_jvm_direct_heap_max
- mapr.spark.driver_jvm_direct_heap_usage
- mapr.spark.driver_jvm_direct_heap_used
- mapr.spark.driver_jvm_direct_mapped_capacity
- mapr.spark.driver_jvm_direct_mapped_count
- mapr.spark.driver_jvm_direct_mapped_used
- mapr.spark.driver_jvm_direct_non_heap_committed
- mapr.spark.driver_jvm_direct_non_heap_init
- mapr.spark.driver_jvm_direct_non_heap_max
- mapr.spark.driver_jvm_direct_non_heap_usage
- mapr.spark.driver_jvm_direct_non_heap_used
- mapr.spark.driver_jvm_pools_code_cache_committed
- mapr.spark.driver_jvm_pools_code_cache_init
- mapr.spark.driver_jvm_pools_code_cache_max
- mapr.spark.driver_jvm_pools_code_cache_usage
- mapr.spark.driver_jvm_pools_code_cache_used
- mapr.spark.driver_jvm_pools_compressed_class_space_committed
- mapr.spark.driver_jvm_pools_compressed_class_space_init
- mapr.spark.driver_jvm_pools_compressed_class_space_max
- mapr.spark.driver_jvm_pools_compressed_class_space_usage
- mapr.spark.driver_jvm_pools_compressed_class_space_used
- mapr.spark.driver_jvm_pools_metaspace_committed
- mapr.spark.driver_jvm_pools_metaspace_init
- mapr.spark.driver_jvm_pools_metaspace_max
- mapr.spark.driver_jvm_pools_metaspace_usage
- mapr.spark.driver_jvm_pools_metaspace_used
- mapr.spark.driver_jvm_pools_ps_eden_space_committed
- mapr.spark.driver_jvm_pools_ps_eden_space_init
- mapr.spark.driver_jvm_pools_ps_eden_space_max
- mapr.spark.driver_jvm_pools_ps_eden_space_usage
- mapr.spark.driver_jvm_pools_ps_eden_space_used
- mapr.spark.driver_jvm_pools_ps_old_gen_committed
- mapr.spark.driver_jvm_pools_ps_old_gen_init
- mapr.spark.driver_jvm_pools_ps_old_gen_max
- mapr.spark.driver_jvm_pools_ps_old_gen_usage
- mapr.spark.driver_jvm_pools_ps_old_gen_used
- mapr.spark.driver_jvm_pools_ps_survivor_space_committed
- mapr.spark.driver_jvm_pools_ps_survivor_space_init
- mapr.spark.driver_jvm_pools_ps_survivor_space_max
- mapr.spark.driver_jvm_pools_ps_survivor_space_usage
- mapr.spark.driver_jvm_pools_ps_survivor_space_used
- mapr.spark.driver_jvm_total_committed
- mapr.spark.driver_jvm_total_init
- mapr.spark.driver_jvm_total_max
- mapr.spark.driver_jvm_total_used