File Manager
Viewing File: analyzers.py
# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
import logging
from lvestat import LVEStat
from lvestats.core.plugin import LveStatsPlugin
KB = 1024
FAULTS = ("cpu_fault", "memphy_fault", "mem_fault", "io_fault", "iops_fault", "mep_fault", "nproc_fault")
LIMITS = ("lcpu", "lmemphy", "lmem", "io", "liops", "lep", "lnproc")
STATS = ("cpu_usage", "memphy", "mem_usage", "io_usage", "iops", "mep", "nproc")
LVEUSAGESLOTS = ("lve_version", "time", "has_changed_limits", "has_changed_nproc") + STATS + LIMITS + FAULTS
__author__ = "iseletsk"
log = logging.getLogger("LVEUsage")
class LVEUsageAnalyzer(LveStatsPlugin):
"""
Calculates lve usage and updates it in lve_data, in field 'lve_usage'
"""
def set_config(self, config):
self.config = config # pylint: disable=attribute-defined-outside-init
def execute(self, lve_data):
"""
:param dict lve_data:
"""
# from /proc/lve
new_stats = lve_data["stats"]
old_stats = lve_data["old_stats"]
del lve_data["old_stats"]
old_now = lve_data.get("old_now", None)
lve_data["old_now"] = self.now
total_hz = lve_data["totalHz"]
procs = lve_data["procs"]
result = {}
for _id, lve_stat in new_stats.items():
u = LVEUsage(lve_version=lve_data["LVE_VERSION"])
lve_stat_old = old_stats.get(_id, None)
u.calc(lve_stat_old, lve_stat, total_hz, self.now - old_now if old_now else None, procs)
# LVES-210: clear all data if EP and NPROC == 0 and all other data not changed
if (
_id == 0
or lve_stat.nproc > 0
or lve_stat.mep > 0
or (lve_stat_old is not None and u.is_lve_data_changed(lve_stat_old, lve_stat))
):
result[_id] = u
lve_usages = lve_data.get("lve_usages", [])
lve_usages.append(result)
lve_data["lve_usages_5s"] = [result]
lve_data["lve_active_ids"] = list(result.keys())
lve_data["lve_usages"] = lve_usages
class LVEUsage(object):
__slots__ = LVEUSAGESLOTS
def __init__(self, lve_version):
self.lve_version = lve_version
self.time = None
self.has_changed_limits = False
self.has_changed_nproc = False
# if LVE_VERSION >= 4
self.cpu_usage = 0
self.cpu_fault = 0
self.mep = 0
self.mem_fault = 0
self.mep_fault = 0
self.mem_usage = 0
self.lep = 0
self.lcpu = 0
self.lmem = 0
# if LVE_VERSION >=6
self.io = 0
self.io_usage = 0
self.io_fault = 0
self.lmemphy = 0
self.memphy = 0
self.memphy_fault = 0
self.lnproc = 0
self.nproc = 0
self.nproc_fault = 0
# if LVE_VERSION >= 8
self.liops = 0
self.iops = 0
self.iops_fault = 0
def init_limits_from_lvestat(self, lvestat: LVEStat, procs):
"""
Prepares LVEUsage obj taking limits from LVEStat
"""
self.lep = lvestat.lep
self.lcpu = self.convert_lcpu_to_new_kernel_format(lvestat.cpu, self.lve_version, procs)
# CMT-509: the incorrect LVE I/O units on CMT dashboard
# Proc saves this value is in KBps,
# but we should send it is in Bps,
# because for an active user this value is sent in Bps
self.io = lvestat.io * KB
self.lmemphy = lvestat.lmemphy
self.lnproc = lvestat.lnproc
if self.lve_version >= 8:
self.liops = lvestat.liops
return self
def __repr__(self):
representation = "<"
for k in (k for k in dir(self) if not k.startswith("_")):
v = getattr(self, k)
if isinstance(v, (str, int, float, bool, type(None))):
representation += f"{k}:{v}, "
representation += ">"
return representation
def has_interesting_values(self):
if any((self.cpu_usage, self.nproc, self.io_usage, self.iops)):
return True
if any((getattr(self, k) for k in FAULTS)):
return True
return False
def _has_changed_limits(self, new, old):
"""
:type old: lvestat.LVEStat
:type new: lvestat.LVEStat
"""
for attr in LIMITS:
if getattr(new, attr, None) != getattr(old, attr, None):
return True
return False
def _has_changed_nproc(self, new, old):
"""
:type old: lvestat.LVEStat
:type new: lvestat.LVEStat
"""
return old.nproc != new.nproc
def convert_lcpu_to_new_kernel_format(self, lcpu, lve_ver, procs):
"""
New kernel format lcpu is 100 times bigger than previous. So that it's 0-10000
"""
if lve_ver >= 8:
return lcpu
else:
# new_kernel_format = (old_format * 10000 * procs / 100) = old_format * 100 * procs
return lcpu * procs * 100
def is_lve_data_changed(self, old, new):
"""
Check if LVE data is changed
:param old: Previous data
:param new: Current data
:return: True - changes, false - not
"""
# Common data
if (
old.cpu_usage != new.cpu_usage
or old.mep != new.mep
or old.mem_fault != new.mem_fault
or old.mep_fault != new.mep_fault
or old.mem_usage != new.mem_usage
or old.lep != new.lep
or old.io != new.io
or old.io_usage != new.io_usage
):
return True
# LVE6+ data
if (
old.memphy != new.memphy
or old.memphy_fault != new.memphy_fault
or old.nproc != new.nproc
or old.nproc_fault != new.nproc_fault
):
return True
# LVE8+ data
if self.lve_version >= 8 and old.iops != new.iops:
return True
return False
def calc(self, old, new, hz, time, procs):
self.time = time
# LCPU 100% now is 100*100 = 10 000
# LCPU 0% is 0
self.lcpu = self.convert_lcpu_to_new_kernel_format(new.cpu, self.lve_version, procs)
if bool(old):
# Previous data present
self.has_changed_limits = self._has_changed_limits(new, old)
self.has_changed_nproc = self._has_changed_nproc(new, old)
# CPU USAGE is 0-10000*procs
# new_kernel_format = (old_format * 10000 * procs / 100) = old_format * 100 * procs;
# (100 * (new.cpu_usage - self.cpu_usage) / (hz * time)) = old_format
assert time is not None, "oldstats is not None, but time is unknown"
assert time != 0, "oldstats is not None, but time is 0"
if new.cpu_usage > old.cpu_usage:
self.cpu_usage = int(round(10000 * (new.cpu_usage - old.cpu_usage) * procs / (hz * time)))
if new.mem_fault > old.mem_fault:
self.mem_fault = new.mem_fault - old.mem_fault
if new.mep_fault > old.mep_fault:
self.mep_fault = new.mep_fault - old.mep_fault
if new.memphy_fault > old.memphy_fault:
self.memphy_fault = new.memphy_fault - old.memphy_fault
if new.nproc_fault > old.nproc_fault:
self.nproc_fault = new.nproc_fault - old.nproc_fault
if new.io_usage > old.io_usage:
# bytes per second
self.io_usage = int(round(KB * (new.io_usage - old.io_usage) / time))
if self.lve_version >= 8:
if new.iops > old.iops:
self.iops = int(round((new.iops - old.iops) / time))
# Fault-counter convention (applies to every *_fault assignment below):
# each is a per-interval boolean — 1 if usage was >= cap during this interval, otherwise 0.
# `LVEUsage` is instantiated fresh per snapshot pair and `calc()` runs exactly once on it,
# so each fault field is written at most once here.
# Aggregators (AggregatedLveUsage.add) sum these per-interval 1s
# across all sub-intervals in the reporting window,
# yielding "number of intervals at-or-above the cap".
if self.cpu_usage < 0:
self.cpu_usage = 0
elif self.cpu_usage >= self.lcpu > 0:
self.cpu_usage = self.lcpu
self.cpu_fault = 1
# CLOS-4011: synthesise fault=1 when usage is at-or-above the limit but the
# /proc delta produced no fault. The kernel does not propagate end-user fault
# events into a reseller LVP entry's mem/mep/memphy/nproc fault counters, so
# without this fallback those fields always read 0 in reseller statistics
# even when end-users are actively hitting their limits.
#
# `>=` (not `>`) matches the cpu_fault/io_fault semantics above and covers
# the at-limit boundary: under normal kernel enforcement reseller memphy/nproc
# are pinned at-or-below the cap.
# The `not self.X_fault` guard prevents double-counting on regular-user LVEs
# whose /proc-delta path already set X_fault > 0 earlier in this method.
self.lmem = new.lmem
if self.mem_fault > 0 or new.mem_usage >= new.lmem > 0:
self.mem_usage = new.lmem
if not self.mem_fault and new.mem_usage >= new.lmem > 0:
self.mem_fault = 1
else:
self.mem_fault = 0
self.mem_usage = new.mem_usage
self.lep = new.lep
if self.mep_fault > 0 or new.mep >= new.lep > 0:
self.mep = new.lep
if not self.mep_fault and new.mep >= new.lep > 0:
self.mep_fault = 1
else:
self.mep_fault = 0
self.mep = new.mep
if self.lve_version >= 8:
self.liops = new.liops
if self.iops >= self.liops > 0:
self.iops = self.liops
self.iops_fault = 1
self.io = KB * new.io # limit in bytes per second
if self.io_usage < 0:
self.io_usage = 0
elif self.io_usage >= self.io > 0:
self.io_usage = self.io
self.io_fault = 1
self.lmemphy = new.lmemphy
self.lnproc = new.lnproc
# CLOS-4011: synthesise memphy_fault for reseller LVPs.
# This is the field surfaced as PMemF in `lveinfo --reseller-name` and was the
# primary symptom in the customer report (PMemF stayed at 0 on the reseller view
# even when end-users were OOM-killed by the reseller PMEM cap).
if self.memphy_fault > 0 or new.memphy >= new.lmemphy > 0:
self.memphy = new.lmemphy
if not self.memphy_fault and new.memphy >= new.lmemphy > 0:
self.memphy_fault = 1
else:
self.memphy_fault = 0
self.memphy = new.memphy
if self.nproc_fault > 0 or new.nproc >= new.lnproc > 0:
self.nproc = new.lnproc
if not self.nproc_fault and new.nproc >= new.lnproc > 0:
self.nproc_fault = 1
else:
self.nproc_fault = 0
self.nproc = new.nproc
def __str__(self):
return "(cpu_usage=" + str(self.cpu_usage) + ", mep=" + str(self.mep) + ")"
def __getitem__(self, key):
return getattr(self, key)