Linux evo.fastest-server.com 5.14.0-284.1101.el9.tuxcare.11.els11.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Aug 14 13:30:35 UTC 2026 x86_64
LiteSpeed
Server IP : 103.249.112.113 & Your IP : 216.73.217.135
Domains : 988 Domain
User : tanishks
Terminal
Auto Root
Create File
Create Folder
Localroot Suggester
Backdoor Destroyer
Readme
/
usr /
share /
lve /
dbgovernor /
Delete
Unzip
Name
Size
Permission
Date
Action
__pycache__
[ DIR ]
drwxr-xr-x
2026-09-04 11:34
cpanel
[ DIR ]
drwxr-xr-x
2026-09-04 11:33
history
[ DIR ]
drwxr-xr-x
2026-08-06 17:01
modules
[ DIR ]
drwxr-xr-x
2026-09-04 11:33
scripts
[ DIR ]
drwxr-xr-x
2026-09-04 11:34
storage
[ DIR ]
drwx------
2026-08-06 17:01
tmp
[ DIR ]
drwxr-xr-x
2026-09-04 11:34
utils
[ DIR ]
drwxr-xr-x
2026-09-04 11:33
CLOS-2049
0
B
-rw-r--r--
2026-09-04 11:33
MYSQLG-178
0
B
-rw-r--r--
2026-09-04 11:34
cl-mysql.repo.default
194
B
-rw-r--r--
2026-08-06 17:01
creates_individual_limits_vector_list.py
3.52
KB
-rwxr-xr-x
2026-08-06 17:01
db-select-mysql
651
B
-rwxr-xr-x
2026-08-06 17:01
governor_connected
0
B
-rw-------
2026-09-25 15:33
governor_install.log
7.3
KB
-rw-------
2026-09-25 01:15
governor_package_limitting.py
42.39
KB
-rwxr-xr-x
2026-08-06 17:01
list_problem_files.txt
418
B
-rw-------
2026-08-06 17:01
map_rebuilder.log
13.83
KB
-rw-r--r--
2026-09-04 12:44
mysqlgovernor.py
18.01
KB
-rwxr-xr-x
2026-08-06 17:01
sentry-dsn
0
B
-rw-------
2026-08-06 17:01
utilities.py
66.08
KB
-rw-r--r--
2026-08-06 17:01
vectors_are_already_present
0
B
-rw-r--r--
2026-09-04 11:35
Save
Rename
#!/opt/cloudlinux/venv/bin/python3 # coding:utf-8 # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2024 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT # import argparse import functools import itertools import json import logging import os import re import subprocess import sys import time from collections import defaultdict from typing import Dict, List, Optional, Tuple from clcommon.cpapi import cpinfo, admin_packages, resellers_packages, cpusers, getCPName from utilities import acquire_lock, LockFailedException, set_path_environ CURRENT_PROGRAM_NAME = os.path.basename(sys.argv[0]) DBCTL_BIN = '/usr/share/lve/dbgovernor/utils/dbctl_orig' PACKAGE_LIMIT_CONFIG = '/etc/container/governor_package_limit.json' LOCK_FILE = '/var/run/governor_package_limit' DBCTL_SYNC_LOCK_FILE = '/var/run/governor_package_sync' DEBUG = False ENCODING = 'utf-8' # Control-panel account/reseller names must be plain identifiers. Reject # anything outside this set (and any leading '-') so a name can never be # parsed as a dbctl option by its permuting getopt_long (option/argument # injection -> LVE limit bypass). SAFE_NAME_RE = re.compile(r'^[A-Za-z0-9._][A-Za-z0-9._-]*$') def is_safe_panel_name(name) -> bool: """ True if `name` is a safe control-panel account/reseller name: a non-empty string of [A-Za-z0-9._-] that does not start with '-'. """ return isinstance(name, str) and bool(SAFE_NAME_RE.match(name)) def init_logging(): """ Messages related to governor_package_limitting """ logging.basicConfig( stream = sys.stderr, format=f"{CURRENT_PROGRAM_NAME} %(levelname)s: %(message)s", level=logging.ERROR ) def _process_call_error(call_err): if isinstance(call_err.cmd, list): cmd_str = " ".join(call_err.cmd) else: cmd_str = call_err.cmd logging.error(f'command \'{cmd_str}\' exit code = {call_err.returncode}') logging.info('stdout:\n' + str(call_err.stdout)) if call_err.stderr is not None: logging.info('stderr:\n' + str(call_err.stderr)) return call_err.returncode def debug_log(line, end='\n'): """ Debug output log """ global DEBUG if DEBUG: print(line, end=end) def build_parser(): """ Build CLI parser """ parser = argparse.ArgumentParser( prog="Configure mysqlgovernor limits with Control Panel Packages", description="Description: Configure governor-mysql with Control Panel Package limits", add_help=False, usage='governor_package_limitting.py [COMMAND] [OPTIONS]' ) parser._positionals.title = 'Commands' parser._optionals.title = 'Options' parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='Governor package limiting') parser.add_argument('--debug', action='store_true', help='Turn on debug mode') subparser = parser.add_subparsers(dest='command') ################ SET ################ _set = subparser.add_parser( 'set', help='Set limits for governor packages limits', description='Description: Set cpu limit for governor package' ) _set.add_argument('--package', help='Package name', type=str, required=True) _set.add_argument('--cpu', help='limit CPU (pct) usage', nargs='+',) _set.add_argument('--read', help='limit READ (MB/s) usage', nargs='+') _set.add_argument('--write', help='limit WRITE (MB/s) usage', nargs='+') _set.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ DELETE ################ delete = subparser.add_parser( 'delete', help='Delete governor package limits', description='Description: Delete governor package limit', usage='governor_package_limitting.py delete [PACKAGE_NAME]' ) delete._optionals.title = 'Options' delete.add_argument('--package', help='Package name', type=str, required=True) delete.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ GET ################ get = subparser.add_parser( 'get', help='Get governor package limits', description='Description: Get governor package limits', usage='governor_package_limitting.py get [OPTIONS]', formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, width=99999) ) get._optionals.title = 'Options' g = get.add_mutually_exclusive_group() g.add_argument('--package', help='Get package limits', type=str, ) g.add_argument('--all', help="Get all package limits", action='store_true') get.add_argument('--format', help='Show limits in formats: (KB/s), (BB/s), (MB/s). Default is mb', default='mb',choices=['bb', 'kb', 'mb']) get._optionals.title = 'Options' get.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ GET INDIVIDUAL ################ get_individual = subparser.add_parser( 'get_individual', help='Get governor package limits vector', description='The vector describes which limit is an individual and which is not', usage='governor_package_limitting.py get_individual --user=[USER_NAME]' ) get_individual._optionals.title = 'Options' get_individual.add_argument('--user', help='Shows a particular user\'s vector', type=str,) get_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ SET INDIVIDUAL ################ _set_individual = subparser.add_parser( 'set_individual', help='Set governor package limits vector', description='The vector describes which limit is an individual and which is not', usage='governor_package_limitting.py set_individual --user=[USER_NAME] --cpu=[False]*4 --read=[True]*4 --write=[True]*4' ) _set_individual.add_argument('--user', help='Package name', type=str, required=True) _set_individual.add_argument('--cpu', help='True/False vector, four values', nargs='+',) _set_individual.add_argument('--read', help='True/False vector, four values', nargs='+') _set_individual.add_argument('--write', help='True/False vector, four values', nargs='+') _set_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ RESET INDIVIDUAL ################ reset_individual = subparser.add_parser( 'reset_individual', help='Erases information about individual limits', description='The user will use a package limits or default limits as result', usage='governor_package_limitting.py reset_individual --user=[USER_NAME] --limits=mysql-io,mysql-cpu' ) reset_individual._optionals.title = 'Options' reset_individual.add_argument('--user', help='User name', type=str, required=True) reset_individual.add_argument('--limits', help='mysql-io or mysql-cpu', type=str) reset_individual.add_argument('--metadata-only', action='store_true', help='Only drop the individual_limits JSON record; skip the live dbctl_orig delete ' 'and the post-reset sync. Used by dbctl for transactional rollback of a ' 'brand-new record so no phantom key or live change is left behind.') reset_individual.add_argument('--debug', action='store_true', help='Turn on debug mode') ################ SYNC ################ sync = subparser.add_parser( 'sync', help='Synchronize governor package limits', description='Description: Apply package limitting configuration to governor', usage='governor_package_limitting.py sync' ) sync._optionals.title = 'Options' sync.add_argument('--package', help='Package name', type=str, required=False) sync.add_argument('--user', help='User name', type=str, required=False) sync.add_argument('--debug', action='store_true', help='Turn on debug mode') return parser def cp_packages(user: Optional[str]) -> Dict[str, list]: """ Get packages and it's users. Returns: (dict): package_name and contained users list """ package_users = defaultdict(list) # user's packages for package_user, package in cpinfo(user, ("cplogin", "package")): # Drop names that could be parsed as a dbctl option (see SAFE_NAME_RE). if not is_safe_panel_name(package_user): logging.error(f'Skipping account with unsafe name: {package_user!r}') continue package_users[package].append(package_user) # reseller's packages for reseller, packages in resellers_packages().items(): if not is_safe_panel_name(reseller): logging.error(f'Skipping reseller with unsafe name: {reseller!r}') continue for package in packages: package_users[package].append(reseller) return package_users def limits_serializer(args: List, set_vector=False): """Convert user input list of string to list of int <set_vector=True> -> when use the function to prepare vector instead of values For example: ['1,2,3,4'] to -> [1,2,3,4] """ __limits = [] try: if args and isinstance(args, list): for i in args[0].split(','): if set_vector and 'true' in i.lower(): __limits.append(True) elif set_vector and 'false' in i.lower(): __limits.append(False) else: __limits.append(i) except Exception as err: logging.error(f'Some parameters are incorrect: {err}') sys.exit(1) if len(__limits) > 4: logging.error('Some parameters are incorrect. Provided options is more than 4') sys.exit(1) if __limits and set_vector and len(__limits) != 4: logging.error('Some parameters are incorrect. Provided options must be equal 4 when you set the vector!') sys.exit(1) if not __limits and set_vector: while len(__limits) < 4: __limits.append(False) if len(__limits) < 4: while len(__limits) < 4: __limits.append(0) return __limits def write_config_to_json_file(cfg: dict): with acquire_lock(LOCK_FILE, exclusive=True): with open(PACKAGE_LIMIT_CONFIG, 'w', encoding=ENCODING) as jsonfile: json.dump(cfg, jsonfile) def read_config_file(): with acquire_lock(LOCK_FILE, exclusive=True): with open(PACKAGE_LIMIT_CONFIG, 'r', encoding=ENCODING) as jsonfile: try: cfg = json.load(jsonfile) except json.JSONDecodeError: return {} return cfg if cfg else {} def get_item_from_config_dict(name: str, cfg: dict): """Get package from cfg dictionary if exists""" cfg = cfg.get(name) if cfg: cfg = {name: cfg} return cfg def show_default_value_instead_of_zero(name: str, lim_list: list, default_limits: tuple): """ Replaces 'zero' values by default values Args: name - cpu or read or write set of limits lim_list - lim_list is a current set of limits we'll modify if 'zero' values are present default_limits - 'default limits' we'll use instead of 'zero' values """ d_cpu, d_read, d_write = default_limits d_limits = {'cpu': d_cpu, 'read': d_read, 'write': d_write} # to know which values have not been changed indices_of_unchanged_elements = list() for _indx, num in enumerate(lim_list): if num < 1: lim_list[_indx] = d_limits[name][_indx] else: indices_of_unchanged_elements.append(_indx) return indices_of_unchanged_elements def print_individual_in_json_format(cfg: dict): print(json.dumps(cfg, ensure_ascii=False)) def print_config_in_json_format(cfg: dict, size_format, only_section=True): default_limits = get_dbctl_limits(format=size_format) if only_section: cfg = cfg['package_limits'] for key, val in cfg.items(): for k, v in val.items(): # returns the default values instead of 'zero' index_list = show_default_value_instead_of_zero( k, v, default_limits) if k != 'cpu': for i in index_list: cfg[key][k][i] = byte_size_convertor( v[i], from_format='bb', to_format=size_format ) print(json.dumps(cfg, ensure_ascii=False)) def check_if_values_are_not_less_than_zero(cfg): for k, v in cfg.items(): for i in v: if i < 0: print(f'Value for {k} must be >= 0') debug_log(f'Incorrect parameters for {k}') debug_log(f'Applied config is {cfg}') sys.exit(1) def fill_gpl_json(entity_name: str, cpu: list = None, io_read: list = None, io_write: list = None, serialize: bool = True, set_vector: bool = False) -> None: """ Setting 'package limits' or 'individual limits' to the governor_package_limit.json Args: entity_name (str): Package or user name cpu (list): cpu limits io_read (list): io read limits io_write (list): io write limits serialize (bool): To serialize or not set_vector (bool): True if setting individual limits Return: None """ # check package existence if not set_vector and entity_name not in get_all_packages(): logging.error(f'Package name {entity_name} not found in {getCPName()} packages') sys.exit(1) # check user existence if set_vector and entity_name not in cpusers(): logging.error(f'User {entity_name} not found in {getCPName()} users') sys.exit(1) cfg = { entity_name: { 'cpu': limits_serializer(cpu, set_vector) if serialize else cpu, 'read': limits_serializer(io_read, set_vector) if serialize else io_read, 'write': limits_serializer(io_write, set_vector) if serialize else io_write, } } if not set_vector: convert_io_rw_to_bb(cfg[entity_name]) check_if_values_are_not_less_than_zero(cfg[entity_name]) section = 'package_limits' else: section = 'individual_limits' config = get_package_limit() if config[section].get(entity_name): if cpu: config[section][entity_name]['cpu'] = cfg[entity_name]['cpu'] if io_read: config[section][entity_name]['read'] = cfg[entity_name]['read'] if io_write: config[section][entity_name]['write'] = cfg[entity_name]['write'] else: config[section][entity_name] = cfg[entity_name] debug_log(f'Setting package limit with config: {config}\n') write_config_to_json_file(config) return def update_default_limits(): """ Update default values in our json from the dbctl list """ config = get_package_limit() try: config['package_limits'].update({'default': {"cpu": [0] * 4, "read": [0] * 4, "write": [0] * 4} } ) write_config_to_json_file(config) except (KeyError, AttributeError): return def delete_package_limit(package: str): """Delete package limits Args: package (str): Name of package limit to delete Returns: None """ config = get_package_limit() try: config['package_limits'].pop(package) write_config_to_json_file(config) debug_log(f'Deleting package {package} from config') except (KeyError, AttributeError) as err: logging.error(f'Package name {package} not found') debug_log(err) sys.exit(1) def reset_individual(username: str, certain_limits: str = None, metadata_only: bool = False): """ Delete user individual limits vector from the governor_package_limit.json, 'individual_limits' section. metadata_only: when True, only drop the JSON record - skip the live dbctl_orig delete (the caller is rolling back a brand-new record whose live limit was never applied, so there is no live state to undo). """ help_msg = f"""Vector {username} not found""" def reset_all(): # remove the individual limits from the mysql-governor.xml, unless the # caller asked for a metadata-only removal (transactional rollback): then # leave live dbctl_orig untouched and only drop the JSON record below. if not metadata_only: run_dbctl_command([username], 'delete') config = get_package_limit() individual = config.get('individual_limits') if not isinstance(individual, dict): # Malformed config: 'individual_limits' missing or not a dict. # Fail loud so corruption is not silently swallowed. logging.error(help_msg) debug_log(f'individual_limits is {type(individual).__name__}') sys.exit(1) if username not in individual: # CLOS-3496: idempotent no-op so the caller can still proceed to # dbctl_sync('set', user=...) and re-apply the package limit. # Exiting here used to leave the user at db_governor defaults # instead of the configured package value. debug_log(f'{help_msg} (no-op)') return individual.pop(username) write_config_to_json_file(config) debug_log(f'Deleting vector {username} from config') if not certain_limits or certain_limits.lower() == 'all': reset_all() else: _limits = certain_limits.split(',') if len(_limits) > 1: reset_all() else: reset_a_certain_limit(username, _limits) def change_vector_for_a_user(_user: str, limits: list): """ Marks that a particular limits are not individual anymore. We can reset individual limits for mysql-cpu or mysql-io. Args: _user: is a particular user we makes changes for limits: vector where non individual limits have to be set """ # This is the persistent-vector sink reached via reset_a_certain_limit(), # which does NOT funnel _user through run_dbctl_command(). Validate the name # here too so an unsafe ('-'-leading / metachar) account name can never # mutate governor_package_limit.json without a matching, validated dbctl # step - keeping the persisted vector and dbctl state from diverging # (F-02). See SAFE_NAME_RE and the guards at the other dbctl sinks. if not is_safe_panel_name(_user): logging.error(f'Skipping account with unsafe name: {_user!r}') return config = get_package_limit() dictionary_to_update = {'mysql-cpu': {'cpu': [False] * 4}, 'mysql-io': {'read': [False] * 4, 'write': [False] * 4} } for lim in limits: update_vector = dictionary_to_update.get(lim) try: config['individual_limits'][_user].update(update_vector) except (KeyError, AttributeError) as err: return write_config_to_json_file(config) def return_the_individual_limits_to_dbctl(_user: str, limits: list, saved_limits: tuple): """ Sets the individual limits as they were before because the individual limits must not be changed Args: _user: the user for which individual limits should be set limits: mysql-io (read/write) or mysql-cpu set of limits saved_limits: individual limits that were set before the general reset of limits for a particular user """ # This is a separate dbctl sink reached via reset_a_certain_limit(), which # does NOT funnel its argv through run_dbctl_command(). Validate _user here # too so a '-'-leading / metachar name can never be re-interpreted as a # dbctl option by its permuting getopt_long (option injection -> limit # bypass). See SAFE_NAME_RE. if not is_safe_panel_name(_user): logging.error(f'Skipping account with unsafe name: {_user!r}') return _cpu_limit, _read_limit, _write_limit = saved_limits for lim in limits: # if we change mysql read/write limits - the individual cpu limits # must not be changed! if lim == 'mysql-io': _cpu_limits = ",".join([str(i) + 'b' for i in _cpu_limit]) _limit = [f'--cpu={_cpu_limits}'] # the same situation as described above elif lim == 'mysql-cpu': _read_limit = ",".join([str(i) + 'b' for i in _read_limit]) _read = f'--read={_read_limit}' _write_limit = ",".join([str(i) + 'b' for i in _write_limit]) _write = f'--write={_write_limit}' _limit = [_read, _write] else: return # _user is validated by is_safe_panel_name() at the top of this function, so # it can never be a '-'-leading / metachar string that dbctl's getopt could # re-interpret as an option. dbctl reads the target user from a positional # argv[2], so the user MUST stay immediately after the subcommand. dbctlset = [DBCTL_BIN, 'set', _user, *_limit] try: # shell=False: _user is passed as an argv element, never via a shell subprocess.run(dbctlset, shell=False, text=True, check=True, capture_output=True) except subprocess.CalledProcessError as call_err: return _process_call_error(call_err) def reset_a_certain_limit(username: str, limits: list): """ Makes possible to reset the individual limits for a particular limit. For instance we can reset only mysql-cpu or only mysql-io (read/write), not both. """ # Saves current individual limits saved_limits = get_dbctl_limits(username) # Removes all limits (dbctl individual) for a particular user run_dbctl_command([username], 'delete') # Marks non individual limits (set as <False>) # governor_package_limit.json -> individual_limits -> # user name -> cpu or read/write change_vector_for_a_user(username, limits) # Return individual limits for a set of limits which still individual return_the_individual_limits_to_dbctl(username, limits, saved_limits) def get_package_limit(package: str = None, size_format: str = 'mb', print_to_stdout=False, cfg=None): """Get package limits Args: package (str): name of package to get print_to_stdout (bool): Print to stdout is used for cli. size_format: Print values in specified format. mb, kb, bb Returns: Package limit configurations or provided package configuration """ cfg = cfg or read_config_file() if package: cfg = get_item_from_config_dict(package, cfg['package_limits']) only_section = False else: only_section = True if print_to_stdout and cfg: print_config_in_json_format(cfg, size_format, only_section) return cfg def get_individual(username: str = None, print_to_stdout=False): """ Get individual limits Args: username: name of user to get print_to_stdout (bool): Print to stdout is used for cli. Returns: Individual limits vector -> {"individual_limits": {"user1": {"cpu": [false, false, false, false], "read": [true, true, true, true], "write": [true, true, true, true]}}} """ cfg = read_config_file() if username: cfg = get_item_from_config_dict(username, cfg['individual_limits']) if not cfg: cfg = {username: {'cpu': [False] * 4, 'read': [False] * 4, 'write': [False] * 4} } else: cfg = cfg['individual_limits'] if print_to_stdout: print_individual_in_json_format(cfg) return cfg def run_dbctl_command(users: list, action: str, limits: dict = None) -> int: """Run dbctl command in os, return exit code Set or delete limits for all users specified in package Args: users (list): List of users to apply configuration action (str): Set or delete. Set is used both for set and update. limits (dict): cpu, read, write in format [int] """ return_code = 0 if action == 'set' and not limits: logging.error("Limits for dbctl have not been set") sys.exit(1) # Central validation at the dbctl sink: every code path that runs dbctl for # a control-panel account funnels through here (cp_packages sync, # reset_individual CLI, sync --user). Drop any name that could be parsed as # a dbctl option by its permuting getopt_long (option/argument injection -> # LVE limit bypass) before it ever reaches argv. See SAFE_NAME_RE. if users: safe_users = [] for user in users: if is_safe_panel_name(user): safe_users.append(user) else: logging.error(f'Skipping account with unsafe name: {user!r}') users = safe_users if action == 'delete' and users: for user in users: # user is validated by is_safe_panel_name() at the top of this # function, so a '-'-leading / metachar name can never reach here. # dbctl reads the target from a positional argv[2], so user MUST # follow 'delete'. command = [DBCTL_BIN, 'delete', user] debug_log(f'Running command: {command}') try: # shell=False: user is passed as an argv element, never via a shell command_result = subprocess.run(command, shell=False, text=True, check=True, capture_output=True) return_code += command_result.returncode except subprocess.CalledProcessError as call_err: # we shouldn't do exit here because this function is called in loop in dbctl_sync return_code += _process_call_error(call_err) if action == 'set' and limits and users: for user in users: # Returns an empty list when user is absent in dbctl # so we need to skip all dbctl actions prepare_limits_out = prepare_limits( user, package_limit=limits, individual_limit=get_individual(user) ) if prepare_limits_out: cpu, io_read, io_write = prepare_limits_out # user is validated by is_safe_panel_name() at the top of this # function, closing the option-injection (-> limit bypass) # vector. dbctl reads the target from a positional argv[2], so # user MUST follow 'set'. command = [DBCTL_BIN, 'set', user, f'--cpu={cpu}', f'--read={io_read}', f'--write={io_write}'] debug_log(f'Running command: {command}') try: # shell=False: user is passed as an argv element, never via a shell command_result = subprocess.run(command, shell=False, text=True, check=True, capture_output=True) return_code += command_result.returncode except subprocess.CalledProcessError as call_err: # we shouldn't do exit here because this function is called in loop in dbctl_sync return_code += _process_call_error(call_err) return return_code def dbctl_sync(action: str, package: str = None, user: str = None): """Sync package configuration with dbgovernor Args: action (str): Set or Delete package (str): Package name is used with action delete. package (str): User name is used with action 'set' to synchronise for a specific user only """ if not action: logging.error("sync action not specified") sys.exit(1) exit_code = 0 debug_log("Syncing with dbctl") __cp_packages = cp_packages(user) if action == 'delete' and package: users_to_apply_package = __cp_packages.get(package) if users_to_apply_package: debug_log(f'Deleting package limits for users: {users_to_apply_package}') exit_code += run_dbctl_command(users_to_apply_package, action) sys.exit(exit_code) if action == 'set': _package_limits = get_package_limit(package) # Use a specific package (including all users of these package) # if package for sync is specified # Use all packages otherwise package_limits = _package_limits if package else _package_limits['package_limits'] # To avoid traceback if incorrect package name has been set if not package_limits: logging.warning(f'package limits for \'{package}\' are empty, probably incorrect package name') return for package_name, limits in package_limits.items(): users_to_apply_package = __cp_packages.get(package_name) if users_to_apply_package: if not user: # Sets package limits for all package's users debug_log(f'Setting package limits for users: {users_to_apply_package}') exit_code += run_dbctl_command(users_to_apply_package, action, limits) continue if user in users_to_apply_package: # Sets package limits only for a specific user # if user for sync is specified debug_log(f'Setting package limits for user: {user}') exit_code += run_dbctl_command([user], action, limits) sys.exit(exit_code) def byte_size_convertor(value: int, from_format: str, to_format: str): """Converting integer between formats mb|kb|bb Args: value (int): Integer value to convert from_format (str): To convert from mb|kb|bb to_format (str): To convert to mb|kb|bb """ if value < 0: return value if from_format == 'bb': if to_format == 'kb': value /= 1024 elif to_format == 'mb': value = value / 1024 / 1024 elif from_format == 'kb': if to_format == 'bb': value *= 1024 elif to_format == 'mb': value /= 1024 elif from_format == 'mb': if to_format == 'kb': value *= 1024 elif to_format == 'bb': value = value * 1024 * 1024 return int(value) def convert_io_rw_to_bb(cfg: dict): """Converting MB or KB to bytes Parameters from user cli can be bytes, MB or KB. Symbol 'b' - bytes, symbol 'k' - kilobytes, symbol 'm' or no symbols - megabytes [1, 50m, 52428800b, 100k] Args: cfg (dict): {cpu: [x,x,x,x], read: [x,x,x,x], write: [x,x,x,x,]} Returns: cfg (dict) """ for k, v in cfg.items(): if k != 'cpu': for i in range(4): if isinstance(v[i], str): if 'b' in v[i]: v[i] = int(v[i].replace('b', '')) elif 'k' in v[i]: debug_log(f'Converting kilobytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i].replace('k', '')), from_format='kb', to_format='bb' ) elif 'm' in v[i]: debug_log(f'Converting megabytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i].replace('m', '')), from_format='mb', to_format='bb' ) else: debug_log(f'Converting megabytes `{v[i]}` to bytes') v[i] = byte_size_convertor( int(v[i]), from_format='mb', to_format='bb' ) else: for i in range(4): v[i] = int(v[i]) debug_log('\n') return cfg def trying_to_get_user_in_dbctl_list(user: str, format: str) -> Dict: """ Extract user limits from dbctl list output Make additional attempt in case of fail and fallback to default limits Args: user -> web panel user for which we perform actions format -> formats that we use in dbctl (bb, kb, mb) Returns: specific user limits (cpu, read, write) """ data = _get_dbctl_list_json(format).get(user) # make one more attempt in case of new user that wasn't added to dbctl list yet if not data: default_limits = { "cpu": {"current": 0, "short": 0, "mid": 0, "long": 0}, "read": {"current": 0, "short": 0, "mid": 0, "long": 0}, "write": {"current": 0, "short": 0, "mid": 0, "long": 0} } # restart db_governor service to add the user to the dbctl list immediately os.system('/usr/share/lve/dbgovernor/mysqlgovernor.py --dbupdate') time.sleep(1) os.system('service db_governor restart &> /dev/null') time.sleep(1) # if user limits still unavailable - just return a structure with zero values data = _get_dbctl_list_json(format).get(user) or default_limits return data def _get_dbctl_list_json(format: str) -> Dict: # shell=False: format passed as argv element dbctl_limits = [DBCTL_BIN, 'list-json', f'--{format}'] output = subprocess.run(dbctl_limits, shell=False, text=True, capture_output=True) _data = output.stdout try: data = json.loads(_data) except json.JSONDecodeError: logging.debug(f"<{DBCTL_BIN} list-json --{format}> returns non json data!") logging.debug(f"stdout -> {output.stdout}") logging.debug(f"stderr -> {output.stderr}") sys.exit(1) return data def get_dbctl_limits(user: str = 'default', format: str = 'bb') -> Tuple: """ Gets dbctl limits list and returns a tuple of individual limits (cpu, read, write) for a specific user. Running this function without 'user' argument returns the default limits. Args: user -> to get an individual user limits, returns 'default' if hasn't been set format -> all available formats we use in dbctl: bb - bytes kb - kilobytes mb - megabytes Returns: a tuple of lists ([cpu],[read],[write]), or blank tuple if user has not been found """ individual_limits = trying_to_get_user_in_dbctl_list(user, format) individual_cpu_limit = ( individual_limits['cpu']['current'], individual_limits['cpu']['short'], individual_limits['cpu']['mid'], individual_limits['cpu']['long'] ) individual_read_limit = ( individual_limits['read']['current'], individual_limits['read']['short'], individual_limits['read']['mid'], individual_limits['read']['long'] ) individual_write_limit = ( individual_limits['write']['current'], individual_limits['write']['short'], individual_limits['write']['mid'], individual_limits['write']['long'] ) debug_log(f'{user}\'s individual limits are: ', end='') debug_log(f'cpu: {individual_cpu_limit}, read: {individual_read_limit}, write: {individual_write_limit}') return individual_cpu_limit, individual_read_limit, individual_write_limit def get_the_limits_set(limits_set: list) -> Tuple: """ Here we have twelve values: -> four per cpu -> four per read -> four per write Here we divide them to conviniate representation """ cpu = tuple(limits_set[0:4]) read = tuple(limits_set[4:8]) write = tuple(limits_set[8:]) return cpu, read, write def set_default_limit(package_limit: dict) -> Tuple: """ Defines the limits (cpu, read, write). Uses package limits or default limits depending on values. If package value is greater than zero - use it, otherwise use default value. """ package_cpu_limit = package_limit.get('cpu') package_read_limit = package_limit.get('read') package_write_limit = package_limit.get('write') all_limits = list() for tuple_of_limits_lists in (package_cpu_limit, package_read_limit, package_write_limit): for n in tuple_of_limits_lists: # values of package limit and default limit (0) respectively all_limits.append(n) if n > 0 else all_limits.append(0) return get_the_limits_set(all_limits) def ensures_the_individual_limits_still_set( vector: dict, default_or_package_limit: tuple, individual_limits: tuple) -> Tuple: """ Sets the individual limit instead of <package limit/default limit> if vector value is <True> for the certain limit. """ all_limits = list() individual_cpu_limit, individual_read_limit, individual_write_limit = individual_limits dorp_cpu_limit, dorp_read_limit, dorp_write_limit = default_or_package_limit vectors = vector.values() for vector_dict in vectors: cpu_vectors = vector_dict['cpu'] read_vectors = vector_dict['read'] write_vectors = vector_dict['write'] # <data_set> contains a 'vector list', an 'individual limits' list # and a current list of limits collected from the package limits # or default limits for data_set in ( (cpu_vectors, individual_cpu_limit, dorp_cpu_limit), (read_vectors, individual_read_limit, dorp_read_limit), (write_vectors, individual_write_limit, dorp_write_limit) ): for n in zip(data_set[0], data_set[1], data_set[2]): # If vector is True if n[0]: all_limits.append(n[1]) # If vector is False else: all_limits.append(n[2]) return get_the_limits_set(all_limits) def prepare_limits(user: str, package_limit: Dict, individual_limit: Dict) -> List: """Prepare Limits with algorithm First of all sets the package limits or default limits, than sets the individual limits (dependin on vector values) For example: Pack1 Package limit for cpu is: [100, 0, 65, 40] Default Package limit is: [50, 90, 0, 50] Individual limit is: [150, not_set, not_set, not_set] Result will be: [150, 90, 65, 40] """ cpu_limit, read_limit, write_limit = ensures_the_individual_limits_still_set( individual_limit, set_default_limit(package_limit), get_dbctl_limits(user) ) try: # with 'b' symbol to set bytes using 'dbctl' tool cpu = ','.join(str(x) for x in cpu_limit) io_read = ','.join(str(x) + 'b' for x in read_limit) io_write = ','.join(str(x) + 'b' for x in write_limit) debug_log(f'Limits after calculation for {user} is:') debug_log(f'cpu: [{cpu}], read: [{io_read}], write: [{io_write}]') return cpu, io_read, io_write except TypeError as err: logging.error(f'Some limits are not given: {err}') sys.exit(1) @functools.cache def get_all_packages(): """ Gets all packages: admin packages and resellers packages either. """ all_packages = set(admin_packages()) reseller_packages_iter = itertools.chain.from_iterable(resellers_packages().values()) all_packages.update(reseller_packages_iter) return list(all_packages) def sync_with_panel(): # won't do MYSQLG-789 """ Just getting package names and applying default values [0,0,0,0] """ cfg = read_config_file() for package in get_all_packages(): if not get_package_limit(package, cfg=cfg): fill_gpl_json( entity_name=package ) cfg = read_config_file() def turn_on_debug_if_user_enabled_debug_mode(opts): if opts.debug: global DEBUG DEBUG = True def ensure_json_presence(): """ The governor_package_limit.json file must always be present! Even if someone delete the json config and its content. """ content = {"package_limits": {}, "individual_limits": {}} if not os.path.exists(PACKAGE_LIMIT_CONFIG): write_config_to_json_file(content) return json_ = read_config_file() for element in content.keys(): if not json_.get(element): json_.update({element: {}}) write_config_to_json_file(json_) def main(argv): """ Run main actions """ # Sanitize PATH so non-absolute shell commands run by this root entry # point resolve only through trusted dirs, not the inherited PATH. set_path_environ() parser = build_parser() if not argv: parser.print_help() sys.exit(1) opts = parser.parse_args(argv) turn_on_debug_if_user_enabled_debug_mode(opts) ensure_json_presence() init_logging() if opts.command == 'set': fill_gpl_json(opts.package, opts.cpu, opts.read, opts.write) dbctl_sync('set', package=opts.package) elif opts.command == 'get': sync_with_panel() update_default_limits() get_package_limit(opts.package, opts.format, print_to_stdout=True) elif opts.command == 'delete': delete_package_limit(opts.package) dbctl_sync('delete', opts.package) elif opts.command == 'sync': # to avoid excessive calls of sync command just ignore too frequent calls try: with acquire_lock(DBCTL_SYNC_LOCK_FILE, exclusive=True, attempts=1): dbctl_sync('set', opts.package, opts.user) except LockFailedException: debug_log('Excessive sync call ignored') elif opts.command == 'get_individual': get_individual(opts.user, print_to_stdout=True) elif opts.command == 'set_individual': fill_gpl_json(opts.user, opts.cpu, opts.read, opts.write, serialize=True, set_vector=True) # There is no call to 'sync' after this, since in the main case of using this command, # 'dbctl set' is called immediately after it # https://docs.google.com/document/d/1KH3MiHVcqJduvw6Vid8UiOvv9-6-CHDAtwkaUQh8_vU elif opts.command == 'reset_individual': metadata_only = getattr(opts, 'metadata_only', False) reset_individual(opts.user, opts.limits, metadata_only) # A metadata-only reset (transactional rollback) must not touch live # limits, so skip the re-sync that would otherwise re-apply package limits. if not metadata_only: dbctl_sync('set', user=opts.user) else: parser.print_help() sys.exit(1) if "__main__" == __name__: main(sys.argv[1:])