random-mirrors/smaeul-u-boot
1
0
Fork 0
mirror of https://github.com/smaeul/u-boot.git synced 2025-10-14 12:56:00 +01:00
Code Issues Packages Projects Releases Wiki Activity

buildman: Tidy up pylint problems in boards module

Browse Source 
Fix all the pylint warnings. Also tidy up the comments so that they show
type information, as required.

Signed-off-by: Simon Glass <sjg@chromium.org>
This commit is contained in:
Simon Glass 2022-07-11 19:04:05 -06:00 committed by Tom Rini
parent a8a0141bc2
commit 969fd333ba
1 changed files with 204 additions and 148 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

276
tools/buildman/boards.py
View File

@ -24,19 +24,26 @@ from buildman import kconfiglib
OUTPUT_FILE = 'boards.cfg' OUTPUT_FILE = 'boards.cfg'
CONFIG_DIR = 'configs' CONFIG_DIR = 'configs'
SLEEP_TIME = 0.03 SLEEP_TIME = 0.03
COMMENT_BLOCK = '''# COMMENT_BLOCK = f'''#
# List of boards # List of boards
# Automatically generated by %s: don't edit # Automatically generated by {__file__}: don't edit
# #
# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
''' % __file__ '''
def try_remove(f): def try_remove(fname):
"""Remove a file ignoring 'No such file or directory' error.""" """Remove a file ignoring 'No such file or directory' error.
Args:
fname (str): Filename to remove
Raises:
OSError: output file exists but could not be removed
"""
try: try:
os.remove(f) os.remove(fname)
except OSError as exception: except OSError as exception:
# Ignore 'No such file or directory' error # Ignore 'No such file or directory' error
if exception.errno != errno.ENOENT: if exception.errno != errno.ENOENT:
@ -46,20 +53,30 @@ def try_remove(f):
def output_is_new(output): def output_is_new(output):
"""Check if the output file is up to date. """Check if the output file is up to date.
Looks at defconfig and Kconfig files to make sure none is newer than the
output file. Also ensures that the boards.cfg does not mention any removed
boards.
Args:
output (str): Filename to check
Returns: Returns:
True if the given output file exists and is newer than any of True if the given output file exists and is newer than any of
*_defconfig, MAINTAINERS and Kconfig*. False otherwise. *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
Raises:
OSError: output file exists but could not be opened
""" """
# pylint: disable=too-many-branches
try: try:
ctime = os.path.getctime(output) ctime = os.path.getctime(output)
except OSError as exception: except OSError as exception:
if exception.errno == errno.ENOENT: if exception.errno == errno.ENOENT:
# return False on 'No such file or directory' error # return False on 'No such file or directory' error
return False return False
else:
raise raise
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'): for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'): if fnmatch.fnmatch(filename, '.*'):
continue continue
@ -67,7 +84,7 @@ def output_is_new(output):
if ctime < os.path.getctime(filepath): if ctime < os.path.getctime(filepath):
return False return False
for (dirpath, dirnames, filenames) in os.walk('.'): for (dirpath, _, filenames) in os.walk('.'):
for filename in filenames: for filename in filenames:
if (fnmatch.fnmatch(filename, '*~') or if (fnmatch.fnmatch(filename, '*~') or
not fnmatch.fnmatch(filename, 'Kconfig*') and not fnmatch.fnmatch(filename, 'Kconfig*') and
@ -79,8 +96,8 @@ def output_is_new(output):
# Detect a board that has been removed since the current board database # Detect a board that has been removed since the current board database
# was generated # was generated
with open(output, encoding="utf-8") as f: with open(output, encoding="utf-8") as inf:
for line in f: for line in inf:
if line[0] == '#' or line == '\n': if line[0] == '#' or line == '\n':
continue continue
defconfig = line.split()[6] + '_defconfig' defconfig = line.split()[6] + '_defconfig'
@ -97,7 +114,7 @@ class Expr:
"""Set up a new Expr object. """Set up a new Expr object.
Args: Args:
expr: String cotaining regular expression to store expr (str): String cotaining regular expression to store
""" """
self._expr = expr self._expr = expr
self._re = re.compile(expr) self._re = re.compile(expr)
@ -106,7 +123,7 @@ class Expr:
"""Check if any of the properties match the regular expression. """Check if any of the properties match the regular expression.
Args: Args:
props: List of properties to check props (list of str): List of properties to check
Returns: Returns:
True if any of the properties match the regular expression True if any of the properties match the regular expression
""" """
@ -132,7 +149,7 @@ class Term:
"""Add an Expr object to the list to check. """Add an Expr object to the list to check.
Args: Args:
expr: New Expr object to add to the list of those that must expr (Expr): New Expr object to add to the list of those that must
match for a board to be built. match for a board to be built.
""" """
self._expr_list.append(Expr(expr)) self._expr_list.append(Expr(expr))
@ -147,7 +164,7 @@ class Term:
Each of the expressions in the term is checked. All must match. Each of the expressions in the term is checked. All must match.
Args: Args:
props: List of properties to check props (list of str): List of properties to check
Returns: Returns:
True if all of the expressions in the Term match, else False True if all of the expressions in the Term match, else False
""" """
@ -178,6 +195,7 @@ class KconfigScanner:
os.environ['srctree'] = os.getcwd() os.environ['srctree'] = os.getcwd()
os.environ['UBOOTVERSION'] = 'dummy' os.environ['UBOOTVERSION'] = 'dummy'
os.environ['KCONFIG_OBJDIR'] = '' os.environ['KCONFIG_OBJDIR'] = ''
self._tmpfile = None
self._conf = kconfiglib.Kconfig(warn=False) self._conf = kconfiglib.Kconfig(warn=False)
def __del__(self): def __del__(self):
@ -188,17 +206,17 @@ class KconfigScanner:
the temporary file might be left over. In that case, it should be the temporary file might be left over. In that case, it should be
deleted in this destructor. deleted in this destructor.
""" """
if hasattr(self, '_tmpfile') and self._tmpfile: if self._tmpfile:
try_remove(self._tmpfile) try_remove(self._tmpfile)
def scan(self, defconfig): def scan(self, defconfig):
"""Load a defconfig file to obtain board parameters. """Load a defconfig file to obtain board parameters.
Arguments: Args:
defconfig: path to the defconfig file to be processed defconfig (str): path to the defconfig file to be processed
Returns: Returns:
A dictionary of board parameters. It has a form of: Dictionary of board parameters. It has a form:
{ {
'arch': <arch_name>, 'arch': <arch_name>,
'cpu': <cpu_name>, 'cpu': <cpu_name>,
@ -211,14 +229,15 @@ class KconfigScanner:
} }
""" """
# strip special prefixes and save it in a temporary file # strip special prefixes and save it in a temporary file
fd, self._tmpfile = tempfile.mkstemp() outfd, self._tmpfile = tempfile.mkstemp()
with os.fdopen(fd, 'w') as f: with os.fdopen(outfd, 'w') as outf:
for line in open(defconfig): with open(defconfig, encoding='utf-8') as inf:
for line in inf:
colon = line.find(':CONFIG_') colon = line.find(':CONFIG_')
if colon == -1: if colon == -1:
f.write(line) outf.write(line)
else: else:
f.write(line[colon + 1:]) outf.write(line[colon + 1:])
self._conf.load_config(self._tmpfile) self._conf.load_config(self._tmpfile)
try_remove(self._tmpfile) try_remove(self._tmpfile)
@ -237,7 +256,7 @@ class KconfigScanner:
defconfig = os.path.basename(defconfig) defconfig = os.path.basename(defconfig)
params['target'], match, rear = defconfig.partition('_defconfig') params['target'], match, rear = defconfig.partition('_defconfig')
assert match and not rear, '%s : invalid defconfig' % defconfig assert match and not rear, f'{defconfig} : invalid defconfig'
# fix-up for aarch64 # fix-up for aarch64
if params['arch'] == 'arm' and params['cpu'] == 'armv8': if params['arch'] == 'arm' and params['cpu'] == 'armv8':
@ -256,7 +275,15 @@ class KconfigScanner:
class MaintainersDatabase: class MaintainersDatabase:
"""The database of board status and maintainers.""" """The database of board status and maintainers.
Properties:
database: dict:
key: Board-target name (e.g. 'snow')
value: tuple:
str: Board status (e.g. 'Active')
str: List of maintainers, separated by :
"""
def __init__(self): def __init__(self):
"""Create an empty database.""" """Create an empty database."""
@ -269,51 +296,56 @@ class MaintainersDatabase:
Display a warning message and return '-' if status information Display a warning message and return '-' if status information
is not found. is not found.
Args:
target (str): Build-target name
Returns: Returns:
'Active', 'Orphan' or '-'. str: 'Active', 'Orphan' or '-'.
""" """
if not target in self.database: if not target in self.database:
print("WARNING: no status info for '%s'" % target, file=sys.stderr) print(f"WARNING: no status info for '{target}'", file=sys.stderr)
return '-' return '-'
tmp = self.database[target][0] tmp = self.database[target][0]
if tmp.startswith('Maintained'): if tmp.startswith('Maintained'):
return 'Active' return 'Active'
elif tmp.startswith('Supported'): if tmp.startswith('Supported'):
return 'Active' return 'Active'
elif tmp.startswith('Orphan'): if tmp.startswith('Orphan'):
return 'Orphan' return 'Orphan'
else: print(f"WARNING: {tmp}: unknown status for '{target}'", file=sys.stderr)
print(("WARNING: %s: unknown status for '%s'" %
(tmp, target)), file=sys.stderr)
return '-' return '-'
def get_maintainers(self, target): def get_maintainers(self, target):
"""Return the maintainers of the given board. """Return the maintainers of the given board.
Args:
target (str): Build-target name
Returns: Returns:
Maintainers of the board. If the board has two or more maintainers, str: Maintainers of the board. If the board has two or more
they are separated with colons. maintainers, they are separated with colons.
""" """
if not target in self.database: if not target in self.database:
print("WARNING: no maintainers for '%s'" % target, file=sys.stderr) print(f"WARNING: no maintainers for '{target}'", file=sys.stderr)
return '' return ''
return ':'.join(self.database[target][1]) return ':'.join(self.database[target][1])
def parse_file(self, file): def parse_file(self, fname):
"""Parse a MAINTAINERS file. """Parse a MAINTAINERS file.
Parse a MAINTAINERS file and accumulates board status and Parse a MAINTAINERS file and accumulate board status and maintainers
maintainers information. information in the self.database dict.
Arguments: Args:
file: MAINTAINERS file to be parsed fname (str): MAINTAINERS file to be parsed
""" """
targets = [] targets = []
maintainers = [] maintainers = []
status = '-' status = '-'
for line in open(file, encoding="utf-8"): with open(fname, encoding="utf-8") as inf:
for line in inf:
# Check also commented maintainers # Check also commented maintainers
if line[:3] == '#M:': if line[:3] == '#M:':
line = line[1:] line = line[1:]
@ -322,8 +354,8 @@ class MaintainersDatabase:
maintainers.append(rest) maintainers.append(rest)
elif tag == 'F:': elif tag == 'F:':
# expand wildcard and filter by 'configs/*_defconfig' # expand wildcard and filter by 'configs/*_defconfig'
for f in glob.glob(rest): for item in glob.glob(rest):
front, match, rear = f.partition('configs/') front, match, rear = item.partition('configs/')
if not front and match: if not front and match:
front, match, rear = rear.rpartition('_defconfig') front, match, rear = rear.rpartition('_defconfig')
if match and not rear: if match and not rear:
@ -353,7 +385,7 @@ class Boards:
The board's target member must not already exist in the board list. The board's target member must not already exist in the board list.
Args: Args:
brd: board to add brd (Board): board to add
""" """
self._boards.append(brd) self._boards.append(brd)
@ -363,7 +395,7 @@ class Boards:
Create a Board object for each and add it to our _boards list. Create a Board object for each and add it to our _boards list.
Args: Args:
fname: Filename of boards.cfg file fname (str): Filename of boards.cfg file
""" """
with open(fname, 'r', encoding='utf-8') as inf: with open(fname, 'r', encoding='utf-8') as inf:
for line in inf: for line in inf:
@ -452,9 +484,10 @@ class Boards:
a board to be selected. a board to be selected.
Args: Args:
args: List of command line arguments args (list of str): List of command line arguments
Returns: Returns:
A list of Term objects list of Term: A list of Term objects
""" """
syms = [] syms = []
for arg in args: for arg in args:
@ -493,11 +526,12 @@ class Boards:
If brds and args are both empty, all boards are selected. If brds and args are both empty, all boards are selected.
Args: Args:
args: List of strings specifying boards to include, either named, args (list of str): List of strings specifying boards to include,
or by their target, architecture, cpu, vendor or soc. If either named, or by their target, architecture, cpu, vendor or
empty, all boards are selected. soc. If empty, all boards are selected.
exclude: List of boards to exclude, regardless of 'args' exclude (list of str): List of boards to exclude, regardless of
brds: List of boards to build 'args', or None for none
brds (list of Board): List of boards to build, or None/[] for all
Returns: Returns:
Tuple Tuple
@ -505,21 +539,21 @@ class Boards:
due to each argument, arranged by argument. due to each argument, arranged by argument.
List of errors found List of errors found
""" """
result = OrderedDict() def _check_board(brd):
warnings = [] """Check whether to include or exclude a board
terms = self._build_terms(args)
result['all'] = [] Checks the various terms and decide whether to build it or not (the
for term in terms: 'build_it' variable).
result[str(term)] = []
exclude_list = [] If it is built, add the board to the result[term] list so we know
if exclude: which term caused it to be built. Add it to result['all'] also.
for expr in exclude:
exclude_list.append(Expr(expr))
found = [] Keep a list of boards we found in 'found', so we can report boards
for brd in self._boards: which appear in self._boards but not in brds.
Args:
brd (Board): Board to check
"""
matching_term = None matching_term = None
build_it = False build_it = False
if terms: if terms:
@ -547,6 +581,23 @@ class Boards:
result[matching_term].append(brd.target) result[matching_term].append(brd.target)
result['all'].append(brd.target) result['all'].append(brd.target)
result = OrderedDict()
warnings = []
terms = self._build_terms(args)
result['all'] = []
for term in terms:
result[str(term)] = []
exclude_list = []
if exclude:
for expr in exclude:
exclude_list.append(Expr(expr))
found = []
for brd in self._boards:
_check_board(brd)
if brds: if brds:
remaining = set(brds) - set(found) remaining = set(brds) - set(found)
if remaining: if remaining:
@ -554,37 +605,40 @@ class Boards:
return result, warnings return result, warnings
def scan_defconfigs_for_multiprocess(self, queue, defconfigs): @classmethod
def scan_defconfigs_for_multiprocess(cls, queue, defconfigs):
"""Scan defconfig files and queue their board parameters """Scan defconfig files and queue their board parameters
This function is intended to be passed to This function is intended to be passed to multiprocessing.Process()
multiprocessing.Process() constructor. constructor.
Arguments: Args:
queue: An instance of multiprocessing.Queue(). queue (multiprocessing.Queue): The resulting board parameters are
The resulting board parameters are written into it. written into this.
defconfigs: A sequence of defconfig files to be scanned. defconfigs (sequence of str): A sequence of defconfig files to be
scanned.
""" """
kconf_scanner = KconfigScanner() kconf_scanner = KconfigScanner()
for defconfig in defconfigs: for defconfig in defconfigs:
queue.put(kconf_scanner.scan(defconfig)) queue.put(kconf_scanner.scan(defconfig))
def read_queues(self, queues, params_list): @classmethod
def read_queues(cls, queues, params_list):
"""Read the queues and append the data to the paramers list""" """Read the queues and append the data to the paramers list"""
for q in queues: for que in queues:
while not q.empty(): while not que.empty():
params_list.append(q.get()) params_list.append(que.get())
def scan_defconfigs(self, jobs=1): def scan_defconfigs(self, jobs=1):
"""Collect board parameters for all defconfig files. """Collect board parameters for all defconfig files.
This function invokes multiple processes for faster processing. This function invokes multiple processes for faster processing.
Arguments: Args:
jobs: The number of jobs to run simultaneously jobs (int): The number of jobs to run simultaneously
""" """
all_defconfigs = [] all_defconfigs = []
for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'): for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'): if fnmatch.fnmatch(filename, '.*'):
continue continue
@ -596,42 +650,43 @@ class Boards:
for i in range(jobs): for i in range(jobs):
defconfigs = all_defconfigs[total_boards * i // jobs : defconfigs = all_defconfigs[total_boards * i // jobs :
total_boards * (i + 1) // jobs] total_boards * (i + 1) // jobs]
q = multiprocessing.Queue(maxsize=-1) que = multiprocessing.Queue(maxsize=-1)
p = multiprocessing.Process( proc = multiprocessing.Process(
target=self.scan_defconfigs_for_multiprocess, target=self.scan_defconfigs_for_multiprocess,
args=(q, defconfigs)) args=(que, defconfigs))
p.start() proc.start()
processes.append(p) processes.append(proc)
queues.append(q) queues.append(que)
# The resulting data should be accumulated to this list # The resulting data should be accumulated to this list
params_list = [] params_list = []
# Data in the queues should be retrieved preriodically. # Data in the queues should be retrieved preriodically.
# Otherwise, the queues would become full and subprocesses would get stuck. # Otherwise, the queues would become full and subprocesses would get stuck.
while any([p.is_alive() for p in processes]): while any(p.is_alive() for p in processes):
self.read_queues(queues, params_list) self.read_queues(queues, params_list)
# sleep for a while until the queues are filled # sleep for a while until the queues are filled
time.sleep(SLEEP_TIME) time.sleep(SLEEP_TIME)
# Joining subprocesses just in case # Joining subprocesses just in case
# (All subprocesses should already have been finished) # (All subprocesses should already have been finished)
for p in processes: for proc in processes:
p.join() proc.join()
# retrieve leftover data # retrieve leftover data
self.read_queues(queues, params_list) self.read_queues(queues, params_list)
return params_list return params_list
def insert_maintainers_info(self, params_list): @classmethod
def insert_maintainers_info(cls, params_list):
"""Add Status and Maintainers information to the board parameters list. """Add Status and Maintainers information to the board parameters list.
Arguments: Args:
params_list: A list of the board parameters params_list (list of dict): A list of the board parameters
""" """
database = MaintainersDatabase() database = MaintainersDatabase()
for (dirpath, dirnames, filenames) in os.walk('.'): for (dirpath, _, filenames) in os.walk('.'):
if 'MAINTAINERS' in filenames: if 'MAINTAINERS' in filenames:
database.parse_file(os.path.join(dirpath, 'MAINTAINERS')) database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
@ -641,51 +696,52 @@ class Boards:
params['maintainers'] = database.get_maintainers(target) params['maintainers'] = database.get_maintainers(target)
params_list[i] = params params_list[i] = params
def format_and_output(self, params_list, output): @classmethod
def format_and_output(cls, params_list, output):
"""Write board parameters into a file. """Write board parameters into a file.
Columnate the board parameters, sort lines alphabetically, Columnate the board parameters, sort lines alphabetically,
and then write them to a file. and then write them to a file.
Arguments: Args:
params_list: The list of board parameters params_list (list of dict): The list of board parameters
output: The path to the output file output (str): The path to the output file
""" """
FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target', fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
'options', 'maintainers') 'options', 'maintainers')
# First, decide the width of each column # First, decide the width of each column
max_length = dict([ (f, 0) for f in FIELDS]) max_length = {f: 0 for f in fields}
for params in params_list: for params in params_list:
for f in FIELDS: for field in fields:
max_length[f] = max(max_length[f], len(params[f])) max_length[field] = max(max_length[field], len(params[field]))
output_lines = [] output_lines = []
for params in params_list: for params in params_list:
line = '' line = ''
for f in FIELDS: for field in fields:
# insert two spaces between fields like column -t would # insert two spaces between fields like column -t would
line += ' ' + params[f].ljust(max_length[f]) line += ' ' + params[field].ljust(max_length[field])
output_lines.append(line.strip()) output_lines.append(line.strip())
# ignore case when sorting # ignore case when sorting
output_lines.sort(key=str.lower) output_lines.sort(key=str.lower)
with open(output, 'w', encoding="utf-8") as f: with open(output, 'w', encoding="utf-8") as outf:
f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
def ensure_board_list(self, output, jobs=1, force=False, quiet=False): def ensure_board_list(self, output, jobs=1, force=False, quiet=False):
"""Generate a board database file if needed. """Generate a board database file if needed.
Arguments: Args:
output: The name of the output file output (str): The name of the output file
jobs: The number of jobs to run simultaneously jobs (int): The number of jobs to run simultaneously
force: Force to generate the output even if it is new force (bool): Force to generate the output even if it is new
quiet: True to avoid printing a message if nothing needs doing quiet (bool): True to avoid printing a message if nothing needs doing
""" """
if not force and output_is_new(output): if not force and output_is_new(output):
if not quiet: if not quiet:
print("%s is up to date. Nothing to do." % output) print(f'{output} is up to date. Nothing to do.')
return return
params_list = self.scan_defconfigs(jobs) params_list = self.scan_defconfigs(jobs)
self.insert_maintainers_info(params_list) self.insert_maintainers_info(params_list)