Skip to content

Commit

Permalink
Merge pull request #1961 from RazCrimson/docker-plugin-updates
Browse files Browse the repository at this point in the history 
Docker plugin updates
  • Loading branch information
@nicolargo
nicolargo authored Nov 28, 2021
2 parents ad3daca + 14caec0 commit d95a64c
Showing 1 changed file with 85 additions and 116 deletions.
201 changes: 85 additions & 116 deletions glances/plugins/glances_docker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
import time
from copy import deepcopy

from glances.logger import logger
from glances.compat import iterkeys, itervalues, nativestr
from glances.timer import getTimeSinceLastUpdate
from glances.logger import logger
from glances.plugins.glances_plugin import GlancesPlugin
from glances.processes import sort_stats as sort_stats_processes, glances_processes
from glances.timer import getTimeSinceLastUpdate

# Docker-py library (optional and Linux-only)
# https://github.com/docker/docker-py
Expand Down Expand Up @@ -66,7 +66,6 @@
'description': 'Container CPU consumption in %',
'y_unit': '%'}]


# List of key to remove before export
export_exclude_list = ['cpu', 'io', 'memory', 'network']

Expand Down Expand Up @@ -95,11 +94,21 @@ def __init__(self, args=None, config=None):
# Init the Docker API
self.docker_client = self.connect()

# Dict of thread (to grab stats asynchronously, one thread is created by container)
# Dict of thread (to grab stats asynchronously, one thread is created per container)
# key: Container Id
# value: instance of ThreadDockerGrabber
self.thread_list = {}

# Dict of Network stats (Storing previous network stats to compute Rx/s and Tx/s)
# key: Container Id
# value: network stats dict
self.network_old = {}

# Dict of Disk IO stats (Storing previous disk_io stats to compute Rx/s and Tx/s)
# key: Container Id
# value: network stats dict
self.io_old = {}

# Force a first update because we need two update to have the first stat
self.update()
self.refresh_timer.set(0)
Expand Down Expand Up @@ -217,8 +226,8 @@ def update(self):
t.start()

# Stop threads for non-existing containers
nonexisting_containers = set(iterkeys(self.thread_list)) - set([c.id for c in containers])
for container_id in nonexisting_containers:
absent_containers = set(iterkeys(self.thread_list)) - set([c.id for c in containers])
for container_id in absent_containers:
# Stop the thread
logger.debug("{} plugin - Stop thread for old container {}".format(self.plugin_name, container_id[:12]))
self.thread_list[container_id].stop()
Expand Down Expand Up @@ -267,7 +276,8 @@ def update(self):
container_stats['cpu'] = self.get_docker_cpu(container.id, self.thread_list[container.id].stats)
container_stats['cpu_percent'] = container_stats['cpu'].get('total', None)
# MEM
container_stats['memory'] = self.get_docker_memory(container.id, self.thread_list[container.id].stats)
container_stats['memory'] = self.get_docker_memory(container.id,
self.thread_list[container.id].stats)
container_stats['memory_usage'] = container_stats['memory'].get('usage', None)
if container_stats['memory'].get('cache', None) is not None:
container_stats['memory_usage'] -= container_stats['memory']['cache']
Expand All @@ -276,7 +286,8 @@ def update(self):
container_stats['io_r'] = container_stats['io'].get('ior', None)
container_stats['io_w'] = container_stats['io'].get('iow', None)
# NET
container_stats['network'] = self.get_docker_network(container.id, self.thread_list[container.id].stats)
container_stats['network'] = self.get_docker_network(container.id,
self.thread_list[container.id].stats)
container_stats['network_rx'] = container_stats['network'].get('rx', None)
container_stats['network_tx'] = container_stats['network'].get('tx', None)
else:
Expand Down Expand Up @@ -310,51 +321,41 @@ def get_docker_cpu(self, container_id, all_stats):
all_stats is the output of the stats method of the Docker API
Output: a dict {'total': 1.49}
"""
ret = {'total': 0.0}
cpu_stats = {'total': 0.0}

# Read the stats
# For each container, you will find a pseudo-file cpuacct.stat,
# containing the CPU usage accumulated by the processes of the container.
# Those times are expressed in ticks of 1/USER_HZ of a second.
# On x86 systems, USER_HZ is 100.
cpu_new = {}
precpu_new = {}
try:
cpu_new['total'] = all_stats['cpu_stats']['cpu_usage'].get(
'total_usage', None)
precpu_new['total'] = all_stats['precpu_stats']['cpu_usage'].get(
'total_usage', None)
cpu_new['system'] = all_stats['cpu_stats'].get(
'system_cpu_usage', None)
precpu_new['system'] = all_stats['precpu_stats'].get(
'system_cpu_usage', None)
cpu = {
'system': all_stats['cpu_stats']['system_cpu_usage'],
'total': all_stats['cpu_stats']['cpu_usage']['total_usage']
}
precpu = {
'system': all_stats['precpu_stats']['system_cpu_usage'],
'total': all_stats['precpu_stats']['cpu_usage']['total_usage']
}
# Issue #1857
# If either precpu_stats.online_cpus or cpu_stats.online_cpus is nil
# then for compatibility with older daemons the length of
# the corresponding cpu_usage.percpu_usage array should be used.
if 'online_cpus' in all_stats['cpu_stats'] and \
all_stats['cpu_stats']['online_cpus'] is not None:
cpu_new['nb_core'] = all_stats['cpu_stats']['online_cpus']
else:
cpu_new['nb_core'] = len(all_stats['cpu_stats']['cpu_usage']['percpu_usage'] or [])
cpu['nb_core'] = all_stats['cpu_stats'].get('online_cpus', None)
if cpu['nb_core'] is None:
cpu['nb_core'] = len(all_stats['cpu_stats']['cpu_usage']['percpu_usage'] or [])
except KeyError as e:
logger.debug(
"docker plugin - Cannot grab CPU usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
else:
try:
cpu_delta = cpu_new['total'] - precpu_new['total']
system_cpu_delta = cpu_new['system'] - precpu_new['system']
cpu_delta = cpu['total'] - precpu['total']
system_cpu_delta = cpu['system'] - precpu['system']
# CPU usage % = (cpu_delta / system_cpu_delta) * number_cpus * 100.0
ret['total'] = (cpu_delta / system_cpu_delta) * \
cpu_new['nb_core'] * 100.0
cpu_stats['total'] = (cpu_delta / system_cpu_delta) * cpu['nb_core'] * 100.0
except TypeError as e:
logger.debug(
"docker plugin - Cannot compute CPU usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)

# Return the stats
return ret
return cpu_stats

def get_docker_memory(self, container_id, all_stats):
"""Return the container MEMORY.
Expand All @@ -363,30 +364,28 @@ def get_docker_memory(self, container_id, all_stats):
all_stats is the output of the stats method of the Docker API
Output: a dict {'rss': 1015808, 'cache': 356352, 'usage': ..., 'max_usage': ...}
"""
ret = {}
memory_stats = {}
# Read the stats
try:
# Mandatory fields
memory_stats['usage'] = all_stats['memory_stats']['usage']
memory_stats['limit'] = all_stats['memory_stats']['limit']
# Issue #1857
# Some stats are not always available in ['memory_stats']['stats']
if 'rss' in all_stats['memory_stats']['stats']:
ret['rss'] = all_stats['memory_stats']['stats']['rss']
memory_stats['rss'] = all_stats['memory_stats']['stats']['rss']
elif 'total_rss' in all_stats['memory_stats']['stats']:
ret['rss'] = all_stats['memory_stats']['stats']['total_rss']
memory_stats['rss'] = all_stats['memory_stats']['stats']['total_rss']
else:
ret['rss'] = None
ret['cache'] = all_stats['memory_stats']['stats'].get(
'cache', None)
ret['max_usage'] = all_stats['memory_stats'].get(
'max_usage', None)
# Mandatory fields
ret['usage'] = all_stats['memory_stats']['usage']
ret['limit'] = all_stats['memory_stats']['limit']
memory_stats['rss'] = None
memory_stats['cache'] = all_stats['memory_stats']['stats'].get('cache', None)
memory_stats['max_usage'] = all_stats['memory_stats'].get('max_usage', None)
except (KeyError, TypeError) as e:
# all_stats do not have MEM information
logger.debug("docker plugin - Cannot grab MEM usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
# Return the stats
return ret
return memory_stats

def get_docker_network(self, container_id, all_stats):
"""Return the container network usage using the Docker API (v1.0 or higher).
Expand All @@ -403,45 +402,32 @@ def get_docker_network(self, container_id, all_stats):

# Read the rx/tx stats (in bytes)
try:
netcounters = all_stats["networks"]
net_stats = all_stats["networks"]
except KeyError as e:
# all_stats do not have NETWORK information
logger.debug("docker plugin - Cannot grab NET usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
# No fallback available...
return network_new

# Previous network interface stats are stored in the network_old variable
if not hasattr(self, 'netcounters_old'):
# First call, we init the network_old var
self.netcounters_old = {}
try:
self.netcounters_old[container_id] = netcounters
except (IOError, UnboundLocalError):
pass

if container_id not in self.netcounters_old:
try:
self.netcounters_old[container_id] = netcounters
except (IOError, UnboundLocalError):
pass
# Previous network interface stats are stored in the self.network_old variable
# By storing time data we enable Rx/s and Tx/s calculations in the XML/RPC API, which would otherwise
# be overly difficult work for users of the API
try:
network_new['cumulative_rx'] = net_stats["eth0"]["rx_bytes"]
network_new['cumulative_tx'] = net_stats["eth0"]["tx_bytes"]
except KeyError as e:
# all_stats do not have INTERFACE information
logger.debug("docker plugin - Cannot grab network interface usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
else:
# By storing time data we enable Rx/s and Tx/s calculations in the
# XML/RPC API, which would otherwise be overly difficult work
# for users of the API
try:
network_new['time_since_update'] = getTimeSinceLastUpdate('docker_net_{}'.format(container_id))
network_new['rx'] = netcounters["eth0"]["rx_bytes"] - self.netcounters_old[container_id]["eth0"]["rx_bytes"]
network_new['tx'] = netcounters["eth0"]["tx_bytes"] - self.netcounters_old[container_id]["eth0"]["tx_bytes"]
network_new['cumulative_rx'] = netcounters["eth0"]["rx_bytes"]
network_new['cumulative_tx'] = netcounters["eth0"]["tx_bytes"]
except KeyError as e:
# all_stats do not have INTERFACE information
logger.debug("docker plugin - Cannot grab network interface usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
network_new['time_since_update'] = getTimeSinceLastUpdate('docker_net_{}'.format(container_id))
if container_id in self.network_old:
network_new['rx'] = network_new['cumulative_rx'] - self.network_old[container_id]['cumulative_rx']
network_new['tx'] = network_new['cumulative_tx'] - self.network_old[container_id]['cumulative_tx']

# Save stats to compute next bitrate
self.netcounters_old[container_id] = netcounters
self.network_old[container_id] = network_new

# Return the stats
return network_new
Expand All @@ -461,53 +447,35 @@ def get_docker_io(self, container_id, all_stats):

# Read the ior/iow stats (in bytes)
try:
iocounters = all_stats["blkio_stats"]
io_stats = all_stats["blkio_stats"]
except KeyError as e:
# all_stats do not have io information
logger.debug("docker plugin - Cannot grab block IO usage for container {} ({})".format(container_id, e))
logger.debug(all_stats)
# No fallback available...
return io_new

# Previous io interface stats are stored in the io_old variable
if not hasattr(self, 'iocounters_old'):
# First call, we init the io_old var
self.iocounters_old = {}
try:
self.iocounters_old[container_id] = iocounters
except (IOError, UnboundLocalError):
pass
# Previous io interface stats are stored in the self.io_old variable
# By storing time data we enable IoR/s and IoW/s calculations in the
# XML/RPC API, which would otherwise be overly difficult work
# for users of the API
try:
io_service_bytes_recursive = io_stats['io_service_bytes_recursive']

if container_id not in self.iocounters_old:
try:
self.iocounters_old[container_id] = iocounters
except (IOError, UnboundLocalError):
pass
# Read IOR and IOW value in the structure list of dict
io_new['cumulative_ior'] = [i for i in io_service_bytes_recursive if i['op'].lower() == 'read'][0]['value']
io_new['cumulative_iow'] = [i for i in io_service_bytes_recursive if i['op'].lower() == 'write'][0]['value']
except (TypeError, IndexError, KeyError, AttributeError) as e:
# all_stats do not have io information
logger.debug("docker plugin - Cannot grab block IO usage for container {} ({})".format(container_id, e))
else:
# By storing time data we enable IoR/s and IoW/s calculations in the
# XML/RPC API, which would otherwise be overly difficult work
# for users of the API
try:
new_io_service_bytes_recursive = iocounters['io_service_bytes_recursive']
old_io_service_bytes_recursive = self.iocounters_old[container_id]['io_service_bytes_recursive']

# Read IOR and IOW value in the structure list of dict
ior = [i for i in new_io_service_bytes_recursive if i['op'].lower() == 'read'][0]['value']
iow = [i for i in new_io_service_bytes_recursive if i['op'].lower() == 'write'][0]['value']
ior_old = [i for i in old_io_service_bytes_recursive if i['op'].lower() == 'read'][0]['value']
iow_old = [i for i in old_io_service_bytes_recursive if i['op'].lower() == 'write'][0]['value']
except (TypeError, IndexError, KeyError, AttributeError) as e:
# all_stats do not have io information
logger.debug("docker plugin - Cannot grab block IO usage for container {} ({})".format(container_id, e))
else:
io_new['time_since_update'] = getTimeSinceLastUpdate('docker_io_{}'.format(container_id))
io_new['ior'] = ior - ior_old
io_new['iow'] = iow - iow_old
io_new['cumulative_ior'] = ior
io_new['cumulative_iow'] = iow
io_new['time_since_update'] = getTimeSinceLastUpdate('docker_io_{}'.format(container_id))
if container_id in self.io_old:
io_new['ior'] = io_new['cumulative_ior'] - self.io_old[container_id]['cumulative_ior']
io_new['iow'] = io_new['cumulative_iow'] - self.io_old[container_id]['cumulative_iow']

# Save stats to compute next bitrate
self.iocounters_old[container_id] = iocounters
# Save stats to compute next bitrate
self.io_old[container_id] = io_new

# Return the stats
return io_new
Expand Down Expand Up @@ -569,8 +537,8 @@ def msg_curse(self, args=None, max_width=None):

# Only process if stats exist (and non null) and display plugin enable...
if not self.stats \
or 'containers' not in self.stats or len(self.stats['containers']) == 0 \
or self.is_disabled():
or 'containers' not in self.stats or len(self.stats['containers']) == 0 \
or self.is_disabled():
return ret

# Build the string message
Expand All @@ -591,7 +559,7 @@ def msg_curse(self, args=None, max_width=None):
default=20)
if self.config is not None else 20,
len(max(self.stats['containers'],
key=lambda x: len(x['name']))['name']))
key=lambda x: len(x['name']))['name']))
msg = ' {:{width}}'.format('Name', width=name_max_width)
ret.append(self.curse_add_line(msg))
msg = '{:>10}'.format('Status')
Expand Down Expand Up @@ -663,7 +631,8 @@ def msg_curse(self, args=None, max_width=None):
unit = 'b'
for r in ['rx', 'tx']:
try:
value = self.auto_unit(int(container['network'][r] // container['network']['time_since_update'] * to_bit)) + unit
value = self.auto_unit(
int(container['network'][r] // container['network']['time_since_update'] * to_bit)) + unit
msg = '{:>7}'.format(value)
except KeyError:
msg = '{:>7}'.format('_')
Expand Down

0 comments on commit d95a64c

Please sign in to comment.