File Manager
Viewing File: clwpos_get.py
# -*- coding: utf-8 -*-
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
"""
This module contains a wrapper around `clwpos-user get` local utility
"""
import json
import logging
import os
import select
import subprocess
import time
import multiprocessing
from typing import Optional
from xray.internal.utils import build_clwpos_user_cmd
from clsummary.summary import CloudlinuxSummary
try:
from clwpos.papi import is_wpos_visible
except ImportError:
# case when wpos is not installed yet
is_wpos_visible = lambda username: None
from ..apiclient import get_client
from ..internal.nginx_utils import NginxUserCache
from xray.smart_advice_plugin_helpers import get_plugin_status
from xray.internal.clwpos_safe_imports import any_suite_allowed_on_server
logger = logging.getLogger('clwpos_util')
# Hard cap on the bytes we buffer from a single `clwpos-user scan` invocation.
# The scan JSON is derived from the user's website tree, so its size is
# attacker-influenceable; this collector runs in a shared root-context process
# (root cron looping over every user) with no memory cgroup, so an unbounded
# buffer here is a cross-tenant DoS. 8 MiB is comfortably above any legitimate
# per-domain scan payload while still preventing memory exhaustion.
_CLWPOS_SCAN_MAX_OUTPUT_BYTES = 8 * 1024 * 1024
# Wall-clock cap on a single scan, matching the subprocess timeout convention
# used elsewhere in py/xray (e.g. user_agent.py, agent/daemon.py).
_CLWPOS_SCAN_TIMEOUT = 120
# Upper bound on how much subprocess output is logged. The scan runs inside the
# user's own site tree, so its stdout/stderr can carry docroot paths, site
# config and tracebacks; logging the full payload would forward it off-box via
# the Sentry LoggingIntegration. Log only a short, truncated snippet instead.
_CLWPOS_LOG_SNIPPET_MAX = 200
def _log_snippet(output) -> str:
"""Return a bounded, redaction-marked snippet of subprocess output for logging."""
text = '' if output is None else str(output)
if len(text) <= _CLWPOS_LOG_SNIPPET_MAX:
return text
return text[:_CLWPOS_LOG_SNIPPET_MAX] + '...[truncated]'
class ClWposGetter:
util = "/usr/bin/clwpos-user"
awp_info_collector = '/usr/sbin/clwpos_collect_information.py'
def post_metadata(self, username: str, domain: str) -> None:
"""Construct and POST metadata to Smart Advice microservice"""
if self.nginx_cache_for_user(username):
logger.info('ea-nginx detected, skipping metadata send')
return
json_data = self.construct_metadata(username, domain)
logger.debug('Got WPOS: %s', str(json_data))
if json_data:
self.send(json_data)
else:
logger.error('Metadata for user %s with domain %s will not be sent', username, domain)
@staticmethod
def nginx_cache_for_user(username: str) -> bool:
"""
Check nginx cache status for given user
"""
return NginxUserCache(username).is_enabled
def _build_cmd(self, username: str, domainname: str) -> list:
"""Build command list for clwpos-user invocation"""
return build_clwpos_user_cmd(username, ['scan', '--website', domainname])
def utility(self, username: str, domainname: str) -> Optional[dict]:
"""
External call of `clwpos-user get` utility
"""
if not os.path.isfile(self.util):
return
if not username or not domainname:
return
try:
_exec = self._build_cmd(username, domainname)
except ValueError as e:
logger.error('Invalid username %r for %s: %s', username, self.util, e)
return None
# The child output is attacker-influenceable and this runs in a shared
# root process, so stream stdout with a hard byte cap (subprocess.run
# would buffer everything before we could check the size) and a
# wall-clock timeout. On overflow/timeout we kill the child and fail
# this one scan gracefully (return None) — the same failure shape used
# below for a non-zero return code or invalid JSON — so the collector
# keeps serving the other tenants.
try:
stdout, returncode, overflow = self._run_scan_bounded(_exec)
# in case something really bad happened to process (killed/..etc)
except (OSError, ValueError, subprocess.SubprocessError) as e:
logger.error('Error running %s: %s', self.util, e)
return None
if overflow:
logger.error('Metadata collection via %s aborted: output exceeded %d bytes',
self.util, _CLWPOS_SCAN_MAX_OUTPUT_BYTES)
return None
if returncode != 0:
logger.error('Metadata collection via %s failed. returncode: %s, stdout: %s',
self.util, returncode, _log_snippet(stdout))
return None
try:
return json.loads(stdout.strip())
except json.JSONDecodeError:
logger.error('Invalid JSON from %s for metadata collection. stdout: %s',
self.util, _log_snippet(stdout))
return None
@staticmethod
def _run_scan_bounded(cmd: list):
"""Run ``cmd`` capping stdout at ``_CLWPOS_SCAN_MAX_OUTPUT_BYTES`` and
wall-clock time at ``_CLWPOS_SCAN_TIMEOUT`` seconds.
Returns ``(stdout, returncode, overflow)`` where ``stdout`` is decoded
text. On overflow the child is killed and ``overflow`` is True so the
caller refuses to parse the partial/oversized payload. On timeout the
child is killed and ``subprocess.TimeoutExpired`` is raised, reusing the
existing SubprocessError failure path.
Cleanup is in an explicit ``finally`` rather than ``with proc:`` on
purpose: ``Popen.__exit__`` performs a blocking ``wait()`` with no
timeout, so a child that hangs without closing its stdout would stall
there forever — defeating the wall-clock cap. We always kill a
still-running child BEFORE any wait, and every wait is itself bounded,
so no exit path can block indefinitely.
"""
deadline = time.monotonic() + _CLWPOS_SCAN_TIMEOUT
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, bufsize=0)
buf = bytearray()
overflow = False
try:
stdout = proc.stdout
assert stdout is not None
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise subprocess.TimeoutExpired(cmd, _CLWPOS_SCAN_TIMEOUT)
# select() bounds the read so a child that hangs while
# producing little/no output still hits the wall clock.
readable, _, _ = select.select([stdout], [], [], remaining)
if not readable:
raise subprocess.TimeoutExpired(cmd, _CLWPOS_SCAN_TIMEOUT)
# os.read returns whatever is currently buffered (up to the
# size) without blocking to fill it, unlike BufferedReader.
chunk = os.read(stdout.fileno(), 65536)
if not chunk:
break
if len(buf) + len(chunk) > _CLWPOS_SCAN_MAX_OUTPUT_BYTES:
overflow = True
break
buf.extend(chunk)
if overflow:
# Kill happens in finally; return the overflow sentinel here.
return '', proc.returncode, True
# Child closed stdout; it should be exiting. Bounded wait so even
# this reap cannot hang forever.
returncode = proc.wait(timeout=max(0.0, deadline - time.monotonic()))
finally:
# Kill a still-running child BEFORE any blocking wait so neither the
# overflow nor the timeout path can stall the shared root collector.
if proc.poll() is None:
proc.kill()
try:
# Bounded reap: even a wedged child can't block us indefinitely.
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass # child was killed; reaping is best-effort
if proc.stdout is not None:
proc.stdout.close()
return buf.decode('utf-8', errors='replace'), returncode, False
@staticmethod
def get_advices_for_website(advices, user, domain, website):
"""
Iterate through advices and return only those which relate to current
user-domain-site
"""
return [
advice
for advice in advices
if advice['metadata']['username'] == user and
advice['metadata']['domain'] == domain and
advice['metadata']['website'] == website
]
def get_updated_extended_metadata(self, username, domain, current_advices):
"""
For getting extended metadata which will be sent daily by cron
"""
final_metadata = {}
websites_metadata = self.utility(username, domain)
if not websites_metadata:
return final_metadata
websites = []
for site, issues in websites_metadata['issues'].items():
path = f'/{site}'
website_issues = websites_metadata['issues'].get(site, [])
advice_for_website = self.get_advices_for_website(current_advices, username, domain, path)
try:
smart_advice_plugin_status = get_plugin_status(username,
domain,
path,
website_issues,
advice_for_website)
except Exception:
logger.exception('Getting Smart Advice plugin status failed')
websites.append(dict(path=path,
issues=website_issues))
else:
websites.append(dict(path=path,
issues=website_issues,
wp_plugin_status=smart_advice_plugin_status))
final_metadata = {
'username': username,
'domain': domain,
'websites': websites,
'server_load_rate': self.server_load_rate(),
}
return final_metadata
def construct_metadata(self, username: str, domainname: str) -> dict:
"""
Ensure format accepted by Smart Advice POST requests/metadata endpoint
"""
dummy_result = None
data = self.utility(username, domainname)
if data is not None:
dummy_result = {'username': username,
'domain': domainname,
'websites':
[dict(path=f"/{site}",
issues=data['issues'].get(site, []))
for site, issues in data['issues'].items()
],
'server_load_rate': self.server_load_rate()}
return dummy_result
@staticmethod
def server_load_rate() -> float:
""""""
try:
domains_total = CloudlinuxSummary._get_total_domains_amount()
except Exception as e:
# something went wrong while querying Summary
logger.error('Unable to get domains_total stats: %s', str(e))
return -1.0
# returns None is cpu_count is undetermined, assume 1 CPU thus
cpu_count = multiprocessing.cpu_count() or 1.0
return domains_total / cpu_count
@staticmethod
def send(stat: dict) -> None:
"""
Send gathered metadata to adviser miscroservice.
Ignore sending if websites are empty
"""
if stat['websites']:
client = get_client('adviser')
client().send_stat(data=stat)
def save_clwpos_info(self):
"""
Needed to save PHP info to work without clwpos_monitoring demon
when AccelerateWP is turned off completely
"""
if not any_suite_allowed_on_server():
subprocess.run([self.awp_info_collector, '--force'], capture_output=True)