diff --git a/.pydocstyle b/.pydocstyle new file mode 100644 index 000000000..0b6c5f94b --- /dev/null +++ b/.pydocstyle @@ -0,0 +1,2 @@ +[pydocstyle] +convention = google \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index fb2c4858b..6fcc90041 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,6 @@ "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": true, "python.linting.flake8Enabled": true, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.linting.pydocstyleEnabled": true } diff --git a/BasicDevTests.py b/BasicDevTests.py index 1ec9b65ef..0a268221d 100644 --- a/BasicDevTests.py +++ b/BasicDevTests.py @@ -9,6 +9,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Check that python code in the package aligns with pep8 and file encoding.""" import glob import os @@ -17,7 +18,7 @@ import re -def TestEncodingOk(apath, encodingValue): +def TestEncodingOk(apath, encodingValue): # noqa try: with open(apath, "rb") as f_obj: f_obj.read().decode(encodingValue) @@ -28,7 +29,7 @@ def TestEncodingOk(apath, encodingValue): return True -def TestFilenameLowercase(apath): +def TestFilenameLowercase(apath): # noqa if apath != apath.lower(): logging.critical(f"Lowercase failure: file {apath} not lower case path") logging.error(f"\n\tLOWERCASE: {apath.lower()}\n\tINPUTPATH: {apath}") @@ -37,8 +38,7 @@ def TestFilenameLowercase(apath): def PackageAndModuleValidCharacters(apath): - ''' check pep8 recommendations for package and module names''' - + """Check pep8 recommendations for package and module names.""" match = re.match('^[a-z0-9_/.]+$', apath.replace("\\", "/")) if match is None: logging.critical( @@ -47,14 +47,14 @@ def PackageAndModuleValidCharacters(apath): return True -def TestNoSpaces(apath): +def TestNoSpaces(apath): # noqa if " " in apath: logging.critical(f"NoSpaces failure: file {apath} has spaces in path") return False return True -def TestRequiredLicense(apath): +def TestRequiredLicense(apath): # noqa licenses = ["SPDX-License-Identifier: BSD-2-Clause-Patent", ] try: with open(apath, "rb") as f_obj: diff --git a/ConfirmVersionAndTag.py b/ConfirmVersionAndTag.py index 57b3fc679..6be5f2279 100644 --- a/ConfirmVersionAndTag.py +++ b/ConfirmVersionAndTag.py @@ -6,7 +6,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## - +"""Script to check that the wheel/package created is aligned on a git tag.""" import glob import os import sys diff --git a/azure-pipelines/templates/build-test-job.yml b/azure-pipelines/templates/build-test-job.yml index 23f501bca..6bb1f8770 100644 --- a/azure-pipelines/templates/build-test-job.yml +++ b/azure-pipelines/templates/build-test-job.yml @@ -33,6 +33,8 @@ jobs: - template: flake8-test-steps.yml + - template: pydocstyle-test-steps.yml + - template: spell-test-steps.yml - template: markdown-lint-steps.yml diff --git a/azure-pipelines/templates/pydocstyle-test-steps.yml b/azure-pipelines/templates/pydocstyle-test-steps.yml new file mode 100644 index 000000000..cb30f1bc9 --- /dev/null +++ b/azure-pipelines/templates/pydocstyle-test-steps.yml @@ -0,0 +1,27 @@ +# File pydocstyle-test-steps.yml +# +# template file to run pydocstyle and if error publish log +# +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +parameters: + none: '' + +steps: +- script: pydocstyle . + displayName: 'Run pydocstyle' + condition: succeededOrFailed() + +# Only capture and archive the lint log on failures. +- script: pydocstyle . > pydocstyle.err.log + displayName: 'Capture pydocstyle failures' + condition: Failed() + +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: 'pydocstyle.err.log' + artifactName: 'Pydocstyle Error log file' + continueOnError: true + condition: Failed() \ No newline at end of file diff --git a/docs/developing.md b/docs/developing.md index 5b04e8d14..0b3161eec 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -94,13 +94,19 @@ out all the different parts. (whitespace, indentation, etc). In VSCode open the py file and use ++alt+shift+f++ to auto format. -2. Run the `BasicDevTests.py` script to check file encoding, file naming, etc +2. Run a Basic Python docstring Check (using pydocstring) and resolve any issues + + ``` cmd + pydocstyle . + ``` + +3. Run the `BasicDevTests.py` script to check file encoding, file naming, etc ```cmd BasicDevTests.py ``` -3. Run pytest with coverage data collected +4. Run pytest with coverage data collected ``` cmd pytest -v --junitxml=test.junit.xml --html=pytest_report.html --self-contained-html --cov=edk2toolext --cov-report html:cov_html --cov-report xml:cov.xml --cov-config .coveragerc @@ -112,17 +118,17 @@ out all the different parts. Coverage is uploaded to `codecov.io`. For more information, review `coverage.md` in the docs folder. -4. Look at the reports +5. Look at the reports * pytest_report.html * cov_html/index.html -5. Run the spell checker +6. Run the spell checker ```cmd cspell -c .cspell.json "**/*.py" "**/*.md" ``` -6. Run the markdown linter +7. Run the markdown linter ```cmd markdownlint "**/*.md" diff --git a/edk2toolext/__init__.py b/edk2toolext/__init__.py index 0a69010b8..9f73dc662 100644 --- a/edk2toolext/__init__.py +++ b/edk2toolext/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa \ No newline at end of file diff --git a/edk2toolext/base_abstract_invocable.py b/edk2toolext/base_abstract_invocable.py index 07573b4ec..cfb175eb8 100644 --- a/edk2toolext/base_abstract_invocable.py +++ b/edk2toolext/base_abstract_invocable.py @@ -5,6 +5,8 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +"""The Base abstract Invocable that all other invocables should inherit from.""" import os import sys import logging @@ -16,69 +18,151 @@ class BaseAbstractInvocable(object): + """The Abstract Invocable. + + The base abstract invocable that other invocables should inherit from. + Provides base functionality to configure logging and the environment. + Attributes: + log_filename (str): logfile path + plugin_manager (PluginManager): the plugin manager + helper (HelperFunctions): container for all helper functions + """ def __init__(self): + """Init the Invocable.""" self.log_filename = None return def ParseCommandLineOptions(self): - ''' parse arguments ''' + """Parse command line arguments. + + TIP: Required Override in a subclass + + HINT: argparse.ArgumentParser + """ raise NotImplementedError() def GetWorkspaceRoot(self): - ''' Return the workspace root for initializing the SDE ''' + """Return the workspace root for initializing the SDE. + + TIP: Required Override in a subclass + + The absolute path to the root of the workspace + + Returns: + (str): path to workspace root + + """ raise NotImplementedError() def GetActiveScopes(self): - ''' return tuple containing scopes that should be active for this process ''' + """Return tuple containing scopes that should be active for this process. + + TIP: Required Override in a subclass + + TIP: A single scope should end in a comma i.e. (scope,) + + Returns: + (Tuple): scopes + """ raise NotImplementedError() def GetSkippedDirectories(self): - ''' Return tuple containing workspace-relative directory paths that should be skipped for processing. - Absolute paths are not supported. ''' + """Return tuple containing workspace-relative directory paths that should be skipped for processing. + + TIP: Optional Override in a subclass + + WARNING: Absolute paths are not supported. + + TIP: A single directory should end with a comma i.e. (dir,) + + Returns: + (Tuple): directories + """ return () def GetLoggingLevel(self, loggerType): - ''' Get the logging level for a given type (return Logging.Level) + """Get the logging level depending on logger type. + + TIP: Required Override in a subclass + + Returns: + (Logging.Level): The logging level + + HINT: loggerType possible values base == lowest logging level supported con == Screen logging txt == plain text file logging md == markdown file logging - ''' + HINT: Return None for no logging for this type. + """ raise NotImplementedError() def GetLoggingFolderRelativeToRoot(self): - ''' Return a path to folder for log files ''' + """Return the path to a directory to hold all log files. + + TIP: Required Override in a subclass + + Returns: + (str): path to the directory + """ raise NotImplementedError() def InputParametersConfiguredCallback(self): - ''' This function is called once all the input parameters - are collected and can be used to initialize environment - ''' + """A Callback once all input parameters are collected. + + TIP: Optional override in subclass + If you need to do something after input variables have been configured. + """ pass def GetVerifyCheckRequired(self): - ''' Will call self_describing_environment.VerifyEnvironment if this returns True ''' + """Will call self_describing_environment.VerifyEnvironment if this returns True. + + TIP: Optional override in a subclass + + Returns: + (bool): whether verify check is required or not + """ return True def GetLoggingFileName(self, loggerType): - ''' Get the logging file name for the type. - Return None if the logger shouldn't be created + """Get the logging File name. - base == lowest logging level supported - con == Screen logging - txt == plain text file logging - md == markdown file logging - ''' + TIP: Required Override this in a subclass + Provides logger file name customization. + + Args: + loggerType: values can be base, con, txt, md. See hint below + + Returns: + (str): filename + + HINT: Return None if the logger shouldn't be created + + HINT: loggerType possible values + base == lowest logging level supported + con == Screen logging + txt == plain text file logging + md == markdown file logging + """ raise NotImplementedError() def Go(self): - ''' Main function to run ''' + """Main function to run. + + Main function to run after the environment and logging has been configured. + + TIP: Required Override in a subclass + """ raise NotImplementedError() def ConfigureLogging(self): - ''' Set up the logging. This function only needs to be overridden if new behavior is needed''' + """Sets up the logging. + TIP: Optional override in a subclass + Only if new behavior is needed. + """ logger = logging.getLogger('') logger.setLevel(self.GetLoggingLevel("base")) @@ -107,8 +191,12 @@ def ConfigureLogging(self): return def Invoke(self): - ''' Main process function. Should not need to be overwritten ''' + """Main process function. + + What actually configure logging and the environment. + WARNING: Do not override this method + """ self.ParseCommandLineOptions() self.ConfigureLogging() self.InputParametersConfiguredCallback() diff --git a/edk2toolext/bin/__init__.py b/edk2toolext/bin/__init__.py index 0a69010b8..9f73dc662 100644 --- a/edk2toolext/bin/__init__.py +++ b/edk2toolext/bin/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa \ No newline at end of file diff --git a/edk2toolext/bin/nuget.py b/edk2toolext/bin/nuget.py index e2013bd4c..9513448d4 100644 --- a/edk2toolext/bin/nuget.py +++ b/edk2toolext/bin/nuget.py @@ -5,6 +5,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This module contains code that knows how to download nuget.""" import os import urllib.error import urllib.request @@ -17,11 +18,18 @@ def DownloadNuget(unpack_folder: str = None) -> list: - ''' - Downloads a version of NuGet to the specific folder as NuGet.exe - If the file already exists, it won't be redownloaded. - The file will be checked against a SHA256 hash for accuracy - ''' + """Downloads a version of NuGet to the specific folder as Nuget.exe. + + If the file already exists, it won't be redownloaded. + The file will be checked against a SHA256 hash for accuracy + + Args: + unpack_folder (str): where to download NuGet + + Raises: + (HTTPError): Issue downloading NuGet + (RuntimeError): Sha256 did not match + """ if unpack_folder is None: unpack_folder = os.path.dirname(__file__) diff --git a/edk2toolext/capsule/capsule_helper.py b/edk2toolext/capsule/capsule_helper.py index 96ca3c5f5..865821855 100644 --- a/edk2toolext/capsule/capsule_helper.py +++ b/edk2toolext/capsule/capsule_helper.py @@ -8,8 +8,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Helper Functions for building EDK2 FMP UEFI Capsules. - +This module contains helper functions for building EDK2 FMP UEFI capsules from +binary payloads, along with the functions to standardize the creation of the Windows +driver installation files +""" import uuid import os import struct @@ -31,19 +35,19 @@ @dataclass class CapsulePayload: - '''Stores information about a specific capsule payload. - - CapsulePayload instances have the following attributes: - payload - an instance of UefiCapsuleHeaderClass that represents the payload data. - payload_filename - the payload filename as a string - esrt_guid - the payload ESRT guid as a uuid.UUID instance. - version - the 32-bit ESRT version for the payload. - firmware_description - the firmware payload description. - tag - a string uniquely identifying the payload. optional, if not present, will be auto-generated. - rollback - indicates whether this is a rollback payload. optional, defaults to false. - integrity_data - integrity data for this payload. optional. - integrity_filename - integrity filename. optional if integrity_data is None, required otherwise. - ''' + """Stores information about a specific capsule payload. + + Attributes: + payload (UefiCapsuleHeaderClass): an instance of UefiCapsuleHeaderClass that represents the payload data. + payload_filename (str): the payload filename as a string + esrt_guid (uuid.UUID): the payload ESRT guid as a uuid.UUID instance. + version (int): the 32-bit ESRT version for the payload. + firmware_description (str): the firmware payload description. + tag (str): a uniquely identifying the payload. optional, if not present, will be auto-generated. + rollback (bool): indicates whether this is a rollback payload. optional, defaults to false. + integrity_data (bytes): integrity data for this payload. optional. + integrity_filename (str): integrity filename. optional if integrity_data is None, required otherwise. + """ payload: UefiCapsuleHeaderClass payload_filename: str esrt_guid: uuid.UUID @@ -57,19 +61,18 @@ class CapsulePayload: @dataclass class Capsule: - '''Stores information about a capsule (potentially with multiple payloads) - - Capsule instances have the following attributes: - version_string - the version of the entire capsule driver package as a string (e.g. 1.0.0.1) - name - the name of the capsule package - provider_name - the name of the capsule provider - arch - the architecture targeted by the capsule - os - the OS targeted by the capsule. - manufacturer_name - name of the capsule manufacturer. optional, defaults to provider_name if None. - date - a datetime.date object indicating when the capsule was built. optional, defaults to - datetime.date.today(). - payloads - a list of capsule payloads. optional, defaults to empty list - ''' + """Stores information about a capsule (potentially with multiple payloads). + + Attributes: + version_string (str): the version of the entire capsule driver package as a string (e.g. 1.0.0.1) + name (str): the name of the capsule package + provider_name (str): the name of the capsule provider + arch (str): the architecture targeted by the capsule + os (str): the OS targeted by the capsule. + manufacturer_name (str): name of the capsule manufacturer. optional, defaults to provider_name if None. + date (datetime.date): when the capsule was built. optional, defaults to datetime.date.today(). + payloads (List[CapsulePayload]): a list of capsule payloads. optional, defaults to empty list + """ version_string: str name: str provider_name: str @@ -81,12 +84,12 @@ class Capsule: def get_capsule_file_name(capsule_options: dict) -> str: - '''from the shared capsule_options dictionary, returns the formatted capsule file name''' + """Returns the formatted capsule file name from the capsule_options dict.""" return f"{capsule_options['fw_name']}_{capsule_options['fw_version_string']}.bin" def get_normalized_version_string(version_string: str) -> str: - '''takes in a version string and returns a normalized version that is compatible with inf and cat files''' + """Normalizes a version string that is compatible with INF and CAT files.""" # 19H1 HLK requires a 4 digit version string, or it will fail while (version_string.count('.') < 3): version_string += '.0' @@ -94,19 +97,18 @@ def get_normalized_version_string(version_string: str) -> str: def get_default_arch() -> str: - '''helper function to consistently return the default architecture for windows files''' + """Consistently return the default architecture for windows files.""" return 'amd64' def get_default_os_string() -> str: - '''helper function to consistently return the default os for windows files''' + """Consistently return the default os for windows files.""" return 'Win10' def build_capsule(capsule_data: bytes, capsule_options: dict, signer_module: object, signer_options: dict) -> UefiCapsuleHeaderClass: - ''' - goes through all of the steps of capsule generation for a single-payload FMP capsule + """Goes through all steps of capsule generation for a single-payload FMP capsule. takes in capsule_data as a byte string, a signer module, and capsule and signer options, and produces all of the headers necessary. Will use the signer module to produce the cert data @@ -114,17 +116,18 @@ def build_capsule(capsule_data: bytes, capsule_options: dict, signer_module: obj NOTE: Uses a fixed MonotonicCount of 1. - capsule_data - a byte string for the innermost payload - capsule_options - a dictionary that will be used for all the capsule payload fields. Must include - 'fw_version', 'lsv_version', and 'esrt_guid' at a minimum. These should all be - strings and the two versions should be strings of hex number (e.g. 0x12345) - signer_module - a capsule signer module that implements the sign() function (see pyopenssl_signer or - signtool_signer built-in modules for examples) - signer_options - a dictionary of options that will be passed to the signer_module. The required values - depend on the expectations of the signer_module provided - - returns a UefiCapsuleHeaderClass object containing all of the provided data - ''' + Args: + capsule_data (bytes): innermost payload + capsule_options (dict): contains the capsule payload fields. Must include + 'fw_version', 'lsv_version', and 'esrt_guid' at minimum. These should + all be strings and the two versions should be strings of hex number + signer_module (object): a capsule signer module that implements the sign() function + (see pyopenssl_signer or signtool_signer build-in modules for examples) + signer_options (dict): options that will be passed to the signer_module. + + Returns: + (UefiCapsuleHeaderClass): the capsule header containing all data provided. + """ # Start building the capsule as we go. # Create the FMP Payload and set all the necessary options. fmp_payload_header = FmpPayloadHeaderClass() @@ -166,13 +169,19 @@ def build_capsule(capsule_data: bytes, capsule_options: dict, signer_module: obj def save_capsule(uefi_capsule_header: UefiCapsuleHeaderClass, capsule_options: dict, save_path: str) -> str: - ''' + """Serializes the capsule object to the target directory. + takes in a UefiCapsuleHeaderClass object, a dictionary of capsule_options, and a filesystem directory path and serializes the capsule object to the target directory - will use get_capsule_file_name() to determine the final filename - will create all intermediate directories if save_path does not already exist - ''' + Args: + uefi_capsule_header (UefiCapsuleHeaderClass): header to encode + capsule_options (dict): used to get the capsule file name + save_path (str): where to save the file to. + + Returns: + (str): path the file was saved to. + """ # Expand the version string prior to creating the payload file. capsule_options['fw_version_string'] = get_normalized_version_string(capsule_options['fw_version_string']) @@ -188,14 +197,17 @@ def save_capsule(uefi_capsule_header: UefiCapsuleHeaderClass, capsule_options: d def save_multinode_capsule(capsule: Capsule, save_path: str) -> str: - ''' + """Generates capsule files from capsule object. + takes in a Capsule object and a filesystem directory path and generates the capsule files at that path. - capsule - a Capsule object containing the capsule details. - save_path - directory path to save the capsule contents into + Args: + capsule (Capsule): a Capsule object containing the capsule details. + save_path (str): directory path to save the capsule contents into - returns save_path - ''' + Returns: + (str): the save path + """ os.makedirs(save_path, exist_ok=True) for capsule_payload in capsule.payloads: payload_file_path = os.path.join(save_path, capsule_payload.payload_filename) @@ -212,13 +224,20 @@ def save_multinode_capsule(capsule: Capsule, save_path: str) -> str: def create_inf_file(capsule_options: dict, save_path: str) -> str: - ''' + """Creates the Windows INF file for the UEFI capsule. + takes in a dictionary of capsule_options and creates the Windows INF file for the UEFI capsule according to the provided options - will save the final file to the save_path with a name determined from the capsule_options - ''' + Args: + capsule_options (dict): options to create the INF file + save_path (str): directory to save the file. + Returns: + (str): INF file path + + NOTE: will save the final file to the save_path with a name determined from the capsule_options + """ # Expand the version string prior to creating INF file. capsule_options['fw_version_string'] = get_normalized_version_string(capsule_options['fw_version_string']) @@ -254,14 +273,17 @@ def create_inf_file(capsule_options: dict, save_path: str) -> str: def create_multinode_inf_file(capsule: Capsule, save_path: str) -> str: - ''' + """Creates the Windows INF file from a capsule object. + Takes in a capsule object containing payload information and creates the Windows INF file in save_path - capsule - capsule object containing payload information - save_path - path to directory where inf file will be created. + Args: + capsule (Capsule): capsule object containing payload information + save_path (str): path to directory where inf file will be created - returns the name of the created inf file. - ''' + Returns: + name of created inf file + """ # Expand the version string prior to creating INF file. capsule.version_string = get_normalized_version_string(capsule.version_string) @@ -303,12 +325,20 @@ def create_multinode_inf_file(capsule: Capsule, save_path: str) -> str: def create_cat_file(capsule_options: dict, save_path: str) -> str: - ''' - takes in a dictionary of capsule_options and creates the Windows CAT file for the UEFI capsule according + """Creates the Windows CAT file for the UEFI capsule. + + Takes in a dictionary of capsule_options and creates the Windows CAT file for the UEFI capsule according to the provided options - will save the final file to the save_path with a name determined from the capsule_options - ''' + Args: + capsule_options (dict): options to create the cat file + save_path (str): directory to save file. + + Returns: + (str): cat file path + + NOTE: will save the final file to the save_path with a name determined from the capsule_options + """ # Deal with optional parameters when creating the CAT file. capsule_options['arch'] = capsule_options.get('arch', get_default_arch()) capsule_options['os_string'] = capsule_options.get('os_string', get_default_os_string()) diff --git a/edk2toolext/capsule/capsule_tool.py b/edk2toolext/capsule/capsule_tool.py index 3a5ed787a..bf17f0877 100644 --- a/edk2toolext/capsule/capsule_tool.py +++ b/edk2toolext/capsule/capsule_tool.py @@ -7,7 +7,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## - +"""CLI interface for creating EDK FMP UEFI Capsules from payload files.""" import os import sys @@ -31,10 +31,7 @@ def get_cli_options(args=None): - ''' - will parse the primary options from the command line. If provided, will take the options as - an array in the first parameter - ''' + """Parse the primary options from the command line.""" parser = argparse.ArgumentParser(description=TOOL_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) # Add the group for the signer specifier. @@ -65,10 +62,7 @@ def get_cli_options(args=None): def load_options_file(in_file): - ''' - takes in a string to a file path and loads it as a json-/yaml-encoded options - file, returning the contents in a dictionary - ''' + """Loads a yaml file into a dictionary and returns it.""" if not hasattr(in_file, 'read'): return None @@ -76,11 +70,12 @@ def load_options_file(in_file): def update_options(file_options, capsule_options, signer_options): - ''' - takes in a pre-loaded options dictionary and walks add all corresponding options - from the command line. Command line options will be organized by type (into capsule_options - or signer_options) and in lists of strings that look like '=' - ''' + """Takes in a pre-loaded options dictionary and adds all corresponding options from the command line. + + Command line options will be organized by type (into capsule_options or + signer_options) and in lists of strings that look like + '='. + """ if file_options is not None: updated_options = copy.copy(file_options) else: @@ -100,6 +95,7 @@ def update_options(file_options, capsule_options, signer_options): def main(): + """Main entry point into the Capsule Tool.""" args = get_cli_options() final_options = update_options(load_options_file(args.options_file), args.capsule_options, args.signer_options) diff --git a/edk2toolext/capsule/pyopenssl_signer.py b/edk2toolext/capsule/pyopenssl_signer.py index a81f82bc7..67d6b7c0b 100644 --- a/edk2toolext/capsule/pyopenssl_signer.py +++ b/edk2toolext/capsule/pyopenssl_signer.py @@ -8,8 +8,11 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Abstracted signing interface for pyopenssl. - +This interface abstraction takes in the signature_options and signer_options +dictionaries that are used by capsule_tool and capsule_helper. +""" import logging import warnings @@ -17,12 +20,22 @@ def sign(data: bytes, signature_options: dict, signer_options: dict) -> bytes: - ''' - primary signing interface. Takes n the signature_options and signer_options - dictionaries that are used by capsule_tool and capsule_helper - ''' - # NOTE: Currently, we only support the necessary algorithms for capsules. + """Primary signing interface. + + Takes in the signature_options and signer_options + dictionaries that are used by capsule_tool and capsule_helper. + + Args: + data (bytes): data to write into the sign tool. + signature_options (dict): dictionary containing signature options + signer_options (dict): dictionary containing signer options + + Raises: + (ValueError()): Unsupported signature or signer options + (RuntimeError()): Signtool.exe returned with error + NOTE: Currently, we only support the necessary algorithms for capsules. + """ # The following _if_ clause handles the deprecated signature_option 'sign_alg' for backwards compatibility # when the deprecated option is supplied, this code adds the new, required options based on prior code behavior if 'sign_alg' in signature_options: diff --git a/edk2toolext/capsule/signing_helper.py b/edk2toolext/capsule/signing_helper.py index 61e57ba28..675996cfd 100644 --- a/edk2toolext/capsule/signing_helper.py +++ b/edk2toolext/capsule/signing_helper.py @@ -9,7 +9,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code to help with the selection and loading of a signer module. +These signer modules can be built-in to the edk2toolext module, loaded from +other Pip or Python modules that are on the current pypath, or passed as a +file path to a local Python module that should be dynamically loaded. +""" import importlib @@ -24,16 +29,18 @@ def get_signer(type, specifier=None): - ''' - based on the type and optional specifier, load a signer module and return it + """Load a signer module based on the arguments. - if type is PYPATH_MODULE_SIGNER, the specifier should be the Python module - package/namespace path - example: edk2toolext.capsule.pyopenssl_signer + if type is PYPATH_MODULE_SIGNER, the specifier should be the python module. + i.e. edk2toolext.capsule.pyopenssl_signer if the type is LOCAL_MODULE_SIGNER, the specifier should be a filesystem - path to a Python module that can be loaded as the signer - ''' + path to a Python module that can be loaded as the signer. + + Args: + type (str): PYOPENSSL_SIGNER, SIGNTOOL_SIGNER, PYPATH_MODULE_SIGNER, LOCAL_MODULE_SIGNER + specifier (module): python module to import + """ if type == PYOPENSSL_SIGNER: try: from edk2toolext.capsule import pyopenssl_signer diff --git a/edk2toolext/capsule/signtool_signer.py b/edk2toolext/capsule/signtool_signer.py index f4e292879..ba79c54de 100644 --- a/edk2toolext/capsule/signtool_signer.py +++ b/edk2toolext/capsule/signtool_signer.py @@ -11,8 +11,14 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Abstracted signing interface for Windows Signtool. +This interface abstraction takes in the signature_options and signer_options +dictionaries that are used by capsule_tool and capsule_helper. +Will attempt to locate a valid installation of Windows Signtool using the +utility_functions provided by edk2toollib. +""" import os import tempfile import warnings @@ -27,11 +33,11 @@ def get_signtool_path(): - ''' - helper function to locate a valid installation of Windows Signtool. Will - attempt to reuse a previously located version, since this call can be - lengthy - ''' + """Locates a valid installation of Windows Signtool. + + Will attempt to reuse a previously located version, since this call can be + lengthy. + """ global GLOBAL_SIGNTOOL_PATH if GLOBAL_SIGNTOOL_PATH is None: @@ -41,13 +47,22 @@ def get_signtool_path(): def sign(data: bytes, signature_options: dict, signer_options: dict) -> bytes: - ''' - primary signing interface. Takes n the signature_options and signer_options + """Primary signing interface. + + Takes in the signature_options and signer_options dictionaries that are used by capsule_tool and capsule_helper - ''' - # NOTE: Currently, we only support the necessary options for capsules & Windows Firmware Policies + Args: + data (bytes): data to write into the sign tool. + signature_options (dict): dictionary containing signature options + signer_options (dict): dictionary containing signer options + Raises: + (ValueError()): Unsupported signature or signer options + (RuntimeError()): Signtool.exe returned with error + + Note: Currently, only support the necessary options for capsules & Windows Firmware Policies + """ # The following _if_ clause handles the deprecated signature_option 'sign_alg' for backwards compatibility # when the deprecated option is supplied, this code adds the new, required options based on prior code behavior if 'sign_alg' in signature_options: @@ -131,11 +146,22 @@ def sign(data: bytes, signature_options: dict, signer_options: dict) -> bytes: def sign_in_place(sign_file_path, signature_options, signer_options): - ''' - alternate module-specific signing interface to support particular signatures associated - with Windows capsule files (e.g. CAT files). Takes n the signature_options and signer_options - dictionaries that are used by capsule_tool and capsule_helper - ''' + """Alternate module-specific signing interface. + + Supports particular signatures associated with windows capsule files + (e.g. CAT files). Takes in the signature_options and signer_options + dictionaries that are used by capsule_tool and capsule_helper. + + Args: + sign_file_path (str): sign file path + signature_options (Dict): dictionary containing signature options + signer_options (Dict): dictionary containing signer options + + Raises: + (ValueError()): Unsupported signature or signer options + (RuntimeError()): Signtool.exe returned with error. + + """ # NOTE: Currently, we only support the necessary algorithms for capsules. if signature_options['sign_alg'] != 'pkcs12': raise ValueError(f"Unsupported signature algorithm: {signature_options['sign_alg']}!") diff --git a/edk2toolext/edk2_git.py b/edk2toolext/edk2_git.py index ee921a4cc..f58cfc2ec 100644 --- a/edk2toolext/edk2_git.py +++ b/edk2toolext/edk2_git.py @@ -7,7 +7,10 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This module contains code that supports simple git operations. +This should not be used as an extensive git lib, but as what is needed for CI/CD builds. +""" import os import logging from io import StringIO @@ -15,15 +18,19 @@ class ObjectDict(object): + """A class representing an ObjectDict.""" def __init__(self): + """Inits an empty ObjectDict.""" self.__values = list() def __setattr__(self, key, value): + """Sets an attribute.""" if not key.startswith("_"): self.__values.append(key) super().__setattr__(key, value) def __str__(self): + """String representation of ObjectDict.""" result = list() result.append("ObjectDict:") for value in self.__values: @@ -31,12 +38,14 @@ def __str__(self): return "\n".join(result) def set(self, key, value): + """Sets a key value pair.""" self.__setattr__(key, value) class Repo(object): - + """A class representing a git repo.""" def __init__(self, path=None): + """Inits an empty Repo object.""" self._path = path # the path that the repo is pointed at self.active_branch = None # the active branch or none if detached self.bare = True # if the repo is bare @@ -169,6 +178,7 @@ def _get_initalized(self): return os.path.isdir(os.path.join(self._path, ".git")) def submodule(self, command, *args): + """Performs a git command on a submodule.""" self._logger.debug( "Calling command on submodule {0} with {1}".format(command, args)) return_buffer = StringIO() @@ -186,6 +196,7 @@ def submodule(self, command, *args): return True def fetch(self, remote="origin", branch=None): + """Performs a git fetch.""" return_buffer = StringIO() param_list = ["fetch", remote] @@ -205,6 +216,7 @@ def fetch(self, remote="origin", branch=None): return True def pull(self): + """Performs a git pull.""" return_buffer = StringIO() params = "pull" @@ -220,6 +232,7 @@ def pull(self): return True def checkout(self, branch=None, commit=None): + """Checks out a branch or commit.""" return_buffer = StringIO() if branch is not None: params = "checkout %s" % branch @@ -237,6 +250,7 @@ def checkout(self, branch=None, commit=None): @classmethod def clone_from(self, url, to_path, branch=None, shallow=False, reference=None, **kwargs): + """Clones a repository.""" _logger = logging.getLogger("git.repo") _logger.debug("Cloning {0} into {1}".format(url, to_path)) # make sure we get the commit if diff --git a/edk2toolext/edk2_invocable.py b/edk2toolext/edk2_invocable.py index 16ec0c159..d941b81ca 100644 --- a/edk2toolext/edk2_invocable.py +++ b/edk2toolext/edk2_invocable.py @@ -8,6 +8,14 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Edk2 Invocable Interface to be overriden in a subclass. + +Provides two main classes, the Edk2InvocableSettingsInterface and the +Edk2Invocable, which should be used as subclasses to create invocables that +perform tasks associated with the EDK2 build system. Any Edk2Invocable subclass +should be platform agnostic and work for any platform. Platform specific data +is provided via the Edk2InvocableSettingsInterface. +""" import os import sys import logging @@ -25,61 +33,146 @@ class Edk2InvocableSettingsInterface(): - ''' Settings APIs to support an Edk2Invocable - - This is an interface definition only - to show which functions are required to be implemented - and can be implemented in a settings manager. - ''' + """Settings APIs to support an Edk2Invocable. + + This is an interface definition only to show which functions are + required to be implemented and can be implemented in a settings + manager. + + Example: Overriding Edk2InvocableSettingsInterface + ``` python + import os + import logging + import argparse + from typing import Iterable, Tuple + from edk2toolext.edk2_invocable import Edk2InvocableSettingsInterface + class NewInvocableSettingsManager(Edk2InvocableSettingsInterface): + def GetWorkspaceRoot(self) -> str: + return os.path.abspath(__file__) + def GetPackagesPath(self) -> Iterable[os.PathLike] + return ["C:/src/MU_BASECORE", "Common/MU"] + def GetActiveScopes(self) -> Tuple[str]: + return ("edk2-build", "pipbuild-win") + def GetLoggingLevel(self, loggerType: str) -> str: + if loggerType == 'txt': + return logging.WARNING + else: + return logging.INFO + def AddCommandLineOptions(self, parserObj: object) -> None: + parserObj.add_argument('-a', "--arch", dest="build_arch", type=str, default="IA32,X64") + def RetrieveCommandLineOptions(self, args: object) -> None: + self.arch = args.build_arch + def GetSkippedDirectories(self) -> Tuple[str]: + return ("Downloads/Extra") + ``` + + WARNING: This interface should not be subclassed directly unless you are creating a new invocable + Other invocables subclass from this interface, so you have the ability to + call the functions in this class as a part of those invocable settings managers. + """ def GetWorkspaceRoot(self) -> str: - ''' get absolute path to WorkspaceRoot ''' + """Return the workspace root for initializing the SDE. + + TIP: Required Override in a subclass + + The absolute path to the root of the workspace + + Returns: + (str): path to workspace root + """ raise NotImplementedError() def GetPackagesPath(self) -> Iterable[os.PathLike]: - ''' Optional API to return an Iterable of paths that should be mapped as edk2 PackagesPath ''' + """Provides an iterable of paths should should be mapped as edk2 PackagePaths. + + TIP: Optional Override in a subclass + + Returns: + (Iterable[os.PathLike]): paths + """ return [] def GetActiveScopes(self) -> Tuple[str]: - ''' Optional API to return Tuple containing scopes that should be active for this process ''' + """Provides scopes that should be active for this process. + + TIP: Optional Override in a subclass + + Returns: + (Tuple[str]): scopes + """ return () def GetLoggingLevel(self, loggerType: str) -> str: - ''' Get the logging level for a given type + """Get the logging level depending on logger type. + + TIP: Optional Override in a subclass + + Returns: + (Logging.Level): The logging level + + HINT: loggerType possible values base == lowest logging level supported con == Screen logging txt == plain text file logging md == markdown file logging - ''' + """ return None def AddCommandLineOptions(self, parserObj: object) -> None: - ''' Implement in subclass to add command line options to the argparser ''' + """Add command line options to the argparser. + + TIP: Optional override in a subclass + + Args: + parserObj: Argparser object. + """ pass def RetrieveCommandLineOptions(self, args: object) -> None: - ''' Implement in subclass to retrieve command line options from the argparser namespace ''' + """Retrieve Command line options from the argparser. + + TIP: Optional override in a subclass + + Args: + args: argparser args namespace containing command line options + """ pass def GetSkippedDirectories(self) -> Tuple[str]: - ''' Implement in subclass to return a Tuple containing workspace-relative directories that should be skipped. - Absolute paths are not supported. ''' + """Returns a tuple containing workspace-relative directories to be skipped. + + TIP: Optional override in a subclass + + Returns: + (Tuple[str]): directories to be skipped. + """ return () class Edk2Invocable(BaseAbstractInvocable): - ''' Base class for Edk2 based invocables. + """Base class for Edk2 based invocables. - Edk2 means it has common features like workspace, packagespath, - scopes, and other name value pairs - ''' + Edk2 means it has common features like workspace, packagespath, + scopes, and other name value pairs + + Attributes: + PlatformSettings(Edk2InvocableSettingsInterface): A settings class + PlatformModule(object): The platform module + Verbose(bool): CLI Argument to determine whether or not to have verbose + + TIP: Checkout BaseAbstractInvocable Attributes + to find any additional attributes that might exist. + + WARNING: This Invocable should only be subclassed if creating a new invocable + """ @classmethod def collect_python_pip_info(cls): - ''' Class method to collect all pip packages names and - versions and report them to the global version_aggregator as - well as print them to the screen. - ''' + """Class method to collect all pip packages names and versions. + + Reports them to the global version_aggregator as well as print them to the screen. + """ # Get the current python version cur_py = "%d.%d.%d" % sys.version_info[:3] ver_agg = version_aggregator.GetVersionAggregator() @@ -93,21 +186,41 @@ def collect_python_pip_info(cls): ver_agg.ReportVersion(package.project_name, version, version_aggregator.VersionTypes.PIP) def GetWorkspaceRoot(self) -> os.PathLike: - ''' Use the SettingsManager to get the absolute path to the workspace root ''' + """Returns the absolute path to the workspace root. + + HINT: Workspace Root is platform specific and thus provided by the PlatformSettings. + + Returns: + (PathLike): absolute path + """ try: return self.PlatformSettings.GetWorkspaceRoot() except AttributeError: raise RuntimeError("Can't call this before PlatformSettings has been set up!") def GetPackagesPath(self) -> Iterable[os.PathLike]: - ''' Use the SettingsManager to an iterable of paths to be used as Edk2 Packages Path''' + """Returns an iterable of packages path. + + HINT: PackagesPath is platform specific and thus provided by the PlatformSettings. + + Returns: + Iterable[os.PathLike]: Packages path + """ try: return [x for x in self.PlatformSettings.GetPackagesPath() if x not in self.GetSkippedDirectories()] except AttributeError: raise RuntimeError("Can't call this before PlatformSettings has been set up!") def GetActiveScopes(self) -> Tuple[str]: - ''' Use the SettingsManager to return tuple containing scopes that should be active for this process.''' + """Returns an iterable of Active scopes. + + HINT: Scopes are platform specific and thus provided by the PlatformSettings. + + Adds an os specific scope in addition to scopes provided by SettingsManager + + Returns: + Tuple[str]: scopes + """ try: scopes = self.PlatformSettings.GetActiveScopes() except AttributeError: @@ -123,12 +236,13 @@ def GetActiveScopes(self) -> Tuple[str]: return scopes def GetLoggingLevel(self, loggerType): - ''' Get the logging level for a given type - base == lowest logging level supported - con == Screen logging - txt == plain text file logging - md == markdown file logging - ''' + """Get the logging level for a given logger type. + + HINT: Logging Level is platform specific and thus provided by the PlatformSettings. + + Returns: + (logging.Level): logging level + """ try: level = self.PlatformSettings.GetLoggingLevel(loggerType) if level is not None: @@ -141,35 +255,57 @@ def GetLoggingLevel(self, loggerType): return logging.DEBUG def AddCommandLineOptions(self, parserObj): - ''' Implement in subclass to add command line options to the argparser ''' + """Add command line options to the argparser. + + HINT: Optional Override to add functionality + """ pass def RetrieveCommandLineOptions(self, args): - ''' Implement in subclass to retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser. + + HINT: Optional Override to add functionality + """ pass def GetSkippedDirectories(self): - ''' Implement in subclass to return a Tuple containing workspace-relative directories that should be skipped. - Absolute paths are not supported. ''' + """Returns a Tuple containing workspace-relative directories that should be skipped. + + HINT: Override in a subclass to add invocable specific directories to skip + + HINT: Skipped Directories are platform specific and thus provided by the PlatformSettings. + + WARNING: Absolute paths are not supported + + Returns: + Tuple[str]: skipped directories + """ try: return self.PlatformSettings.GetSkippedDirectories() except AttributeError: raise RuntimeError("Can't call this before PlatformSettings has been set up!") def GetSettingsClass(self): - ''' Child class should provide the class that contains their required settings ''' + """The required settings manager for the invocable. + + HINT: Required override to define Edk2InvocableSettingsInterface subclass specific to the invocable. + + Returns: + (Edk2InvocableSettingsInterface): Subclass of Edk2InvocableSettingsInterface + """ raise NotImplementedError() def GetLoggingFolderRelativeToRoot(self): + """Directory containing all logging files.""" return "Build" def ParseCommandLineOptions(self): - ''' - Parses command line options. + """Parses command line options. + Sets up argparser specifically to get PlatformSettingsManager instance. Then sets up second argparser and passes it to child class and to PlatformSettingsManager. Finally, parses all known args and then reads the unknown args in to build vars. - ''' + """ # first argparser will only get settings manager and help will be disabled settingsParserObj = argparse.ArgumentParser(add_help=False) # instantiate the second argparser that will get passed around diff --git a/edk2toolext/edk2_logging.py b/edk2toolext/edk2_logging.py index d24fefe03..6d290f873 100644 --- a/edk2toolext/edk2_logging.py +++ b/edk2toolext/edk2_logging.py @@ -6,6 +6,10 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Handles basic logging config for builds. + +Splits logs into a master log and per package log. +""" import logging import os import shutil @@ -42,6 +46,7 @@ # sub_directory is relative to ws argument def clean_build_logs(ws, sub_directory=None): + """Removes all build logs.""" # Make sure that we have a clean environment. if sub_directory is None: sub_directory = os.path.join("Build", "BuildLogs") @@ -50,18 +55,22 @@ def clean_build_logs(ws, sub_directory=None): def get_section_level(): + """Returns SECTION.""" return SECTION def get_subsection_level(): + """Returns SUB_SECTION.""" return SUB_SECTION def get_progress_level(): + """Returns PROGRESS.""" return PROGRESS def get_edk2_filter(verbose=False): + """Returns an edk2 filter.""" gEdk2Filter = Edk2LogFilter() if verbose: gEdk2Filter.setVerbose(verbose) @@ -69,10 +78,12 @@ def get_edk2_filter(verbose=False): def log_progress(message): + """Creates a logging message at the progress section level.""" logging.log(get_progress_level(), message) def setup_section_level(): + """Sets up different sections to log to.""" # todo define section level # add section as a level to the logger section_level = get_section_level() @@ -89,6 +100,7 @@ def setup_section_level(): # creates the the plaintext logger def setup_txt_logger(directory, filename="log", logging_level=logging.INFO, formatter=None, logging_namespace='', isVerbose=False): + """Configures a text logger.""" logger = logging.getLogger(logging_namespace) log_formatter = formatter if log_formatter is None: @@ -112,7 +124,7 @@ def setup_txt_logger(directory, filename="log", logging_level=logging.INFO, # creates the markdown logger def setup_markdown_logger(directory, filename="log", logging_level=logging.INFO, formatter=None, logging_namespace='', isVerbose=False): - + """Configures a markdown logger.""" logger = logging.getLogger(logging_namespace) log_formatter = formatter if log_formatter is None: @@ -144,7 +156,7 @@ def setup_markdown_logger(directory, filename="log", logging_level=logging.INFO, # sets up a colored console logger def setup_console_logging(logging_level=logging.INFO, formatter=None, logging_namespace='', isVerbose=False, use_azure_colors=False, use_color=True): - + """Configures a console logger.""" if formatter is None and isVerbose: formatter_msg = "%(name)s: %(levelname)s - %(message)s" elif formatter is None: @@ -178,6 +190,7 @@ def setup_console_logging(logging_level=logging.INFO, formatter=None, logging_na def stop_logging(loghandle, logging_namespace=''): + """Stops logging on a log handle.""" logger = logging.getLogger(logging_namespace) if loghandle is None: return @@ -192,6 +205,7 @@ def stop_logging(loghandle, logging_namespace=''): def create_output_stream(level=logging.INFO, logging_namespace=''): + """Creates an output stream to log to.""" # creates an output stream that is in memory if string_handler: handler = string_handler.StringStreamHandler() @@ -204,6 +218,7 @@ def create_output_stream(level=logging.INFO, logging_namespace=''): def remove_output_stream(handler, logging_namespace=''): + """Removes an output stream to log to.""" logger = logging.getLogger(logging_namespace) if isinstance(handler, list): for single_handler in handler: @@ -213,6 +228,12 @@ def remove_output_stream(handler, logging_namespace=''): def scan_compiler_output(output_stream): + """Scans the compiler for errors and warnings. + + Returns: + (list[Tuple[logging.Type, str]]): list of tuples containing the type + of issue (Error, warning) and the description. + """ # seek to the start of the output stream def output_compiler_error(match, line, start_txt="Compiler"): start, end = match.span() @@ -253,22 +274,27 @@ def output_compiler_error(match, line, start_txt="Compiler"): class Edk2LogFilter(logging.Filter): + """Subclass of logging.Filter.""" _allowedLoggers = ["root"] def __init__(self): + """Inits a filter.""" logging.Filter.__init__(self) self._verbose = False self._currentSection = "root" def setVerbose(self, isVerbose=True): + """Sets the filter verbosity.""" self._verbose = isVerbose def addSection(self, section): + """Adds a section to the filter.""" # TODO request the global singleton? # how to make this class static Edk2LogFilter._allowedLoggers.append(section) def filter(self, record): + """Adds a filter for a record if it doesn't already exist.""" # check to make sure we haven't already filtered this record if record.name not in Edk2LogFilter._allowedLoggers and record.levelno < logging.WARNING and not self._verbose: return False diff --git a/edk2toolext/environment/__init__.py b/edk2toolext/environment/__init__.py index 0a69010b8..9f73dc662 100644 --- a/edk2toolext/environment/__init__.py +++ b/edk2toolext/environment/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa \ No newline at end of file diff --git a/edk2toolext/environment/conf_mgmt.py b/edk2toolext/environment/conf_mgmt.py index e8a386428..63df0b234 100644 --- a/edk2toolext/environment/conf_mgmt.py +++ b/edk2toolext/environment/conf_mgmt.py @@ -6,7 +6,10 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Handle Edk2 Conf Management. +Customized for edk2-pytool-extensions based build and support dynamic Visual studio support 2017++ +""" import os import logging import shutil @@ -15,29 +18,30 @@ class ConfMgmt(): - + """Handles Edk2 Conf Management.""" def __init__(self): + """Init an empty ConfMgmt object.""" self.Logger = logging.getLogger("ConfMgmt") self.delay_time_in_seconds = 30 def _set_delay_time(self, time_in_seconds): - ''' allow changing the warning time for out of date templates''' + """Allow changing the warning time for out of date templates.""" self.delay_time_in_seconds = time_in_seconds def populate_conf_dir(self, conf_folder_path: str, override_conf: bool, conf_template_source_list: list) -> None: - ''' compare the conf dir files to the template files. - copy files if they are not present in the conf dir or the override - parameter is set. - - param: - conf_folder_path: folder path to output conf location (absolute path) - override_conf: boolean to indicate if templates files should replace conf files - regardless of existence or version. - conf_template_source_list: priority list of folder path that might contain a "Conf" - folder with template files to use - - When complete the conf_folder_path dir must be setup for edk2 builds - ''' + """Compare the conf dir files to the template files. + + Copy files if they are not present in the conf dir or the override parameter is set. + + Args: + conf_folder_path (str): folder path to output conf location (absolute path) + override_conf (bool): boolean to indicate if templates files should replace conf files + regardless of existence or version. + conf_template_source_list (list): priority list of folder path that might contain a "Conf" + older with template files to use + + Note: When complete the conf_folder_path dir must be setup for edk2 builds + """ # make folder to conf path if needed os.makedirs(conf_folder_path, exist_ok=True) @@ -75,11 +79,12 @@ def populate_conf_dir(self, conf_folder_path: str, override_conf: bool, conf_tem version_aggregator.VersionTypes.INFO) def _get_version(self, conf_file: str) -> str: - ''' parse the version from the conf_file - version should be in #!VERSION={value} format + """Parse the version from the conf_file. - "0.0" is returned if no version is found - ''' + version should be in #!VERSION={value} format + + NOTE: "0.0" is returned if no version is found + """ version = "0.0" with open(conf_file, "r") as f: for line in f.readlines(): @@ -92,14 +97,16 @@ def _get_version(self, conf_file: str) -> str: return version def _is_older_version(self, conf_file: str, template_file: str) -> bool: - ''' given a conf_file and a template_file file determine if - the conf_file has an older version than the template. - - param: - conf_file: path to current conf_file - template_file: path to template file that is basis - for the the conf_file. - ''' + """Determines whether the conf or template file is older. + + Given a conf_file and a template_file file determine if + the conf_file has an older version than the template. + + Args: + conf_file (str): path to current conf_file + template_file (str): path to template file that is basis + for the the conf_file. + """ conf = 0 template = 0 @@ -114,14 +121,13 @@ def _is_older_version(self, conf_file: str, template_file: str) -> bool: return (conf < template) def _copy_conf_file_if_necessary(self, conf_file: str, template_file: str, override_conf: bool) -> None: - ''' Copy template_file to conf_file if policy applies - - param: - conf_file: path to final conf file location - template_file: template file to copy if policy applies - override_conf: flag indication to override regardless of policy - ''' + """Copy template_file to conf_file if policy applies. + Args: + conf_file (str): path to final conf file location + template_file (str): template file to copy if policy applies + override_conf (bool): flag indication to override regardless of policy + """ if not os.path.isfile(conf_file): # file doesn't exist. copy template self.Logger.debug(f"{conf_file} file not found. Creating from Template file {template_file}") diff --git a/edk2toolext/environment/environment_descriptor_files.py b/edk2toolext/environment/environment_descriptor_files.py index ce7917eba..af6793be3 100644 --- a/edk2toolext/environment/environment_descriptor_files.py +++ b/edk2toolext/environment/environment_descriptor_files.py @@ -8,12 +8,26 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This module contains code for working with the JSON environment descriptor files. + +It can parse the files, validate them, and return objects representing their contents. +""" import os import yaml class PathEnv(object): + """Path env object that is created from the descriptor file. + + Attributes: + scope (string): scope the path env is associated with + flags (List[str]): flags associated with the path env + var_name (string): ENV var to set with the object + descriptor_location (string): location of the PathEnv + published_path (string): location of the PathEnv + """ def __init__(self, descriptor): + """Init with the descriptor information.""" super(PathEnv, self).__init__() # @@ -29,7 +43,21 @@ def __init__(self, descriptor): class DescriptorFile(object): + """The base class for the different types of descriptor files. + + Attributes: + file_path (str): descriptor file path + descriptor_contents (Dict): Contents of the descriptor file + """ def __init__(self, file_path): + """Loads the contents of the descriptor file and validates. + + Args: + file_path (str): path to descriptor file + + Raises: + (ValueError): Missing specified value from descriptor file + """ super(DescriptorFile, self).__init__() self.file_path = file_path @@ -70,16 +98,22 @@ def __init__(self, file_path): if (isinstance(v, str)): self.descriptor_contents[k] = self.sanitize_string(v) - # - # Clean up a string "value" in the descriptor file. - # def sanitize_string(self, s): + """Clean up a string "value" in the descriptor file.""" # Perform any actions needed to clean the string. return s.strip() class PathEnvDescriptor(DescriptorFile): + """Descriptor File for a PATH ENV.""" def __init__(self, file_path): + """Inits the descriptor as a PathEnvDescriptor from the provided path. + + Loads the contents of the filepath into descriptor_contents + + Args: + file_path (str): path to the yaml descriptor file + """ super(PathEnvDescriptor, self).__init__(file_path) # @@ -93,7 +127,20 @@ def __init__(self, file_path): class ExternDepDescriptor(DescriptorFile): + """Descriptor File for a External Dependency. + + Attributes: + descriptor_contents (Dict): Contents of the Descriptor yaml file + file_path (PathLike): path to the descriptor file + """ def __init__(self, file_path): + """Inits the descriptor as a ExternDepDescriptor from the provided path. + + Loads the contents of the filepath into descriptor_contents + + Args: + file_path (str): path to the yaml descriptor file + """ super(ExternDepDescriptor, self).__init__(file_path) # @@ -107,7 +154,20 @@ def __init__(self, file_path): class PluginDescriptor(DescriptorFile): + """Descriptor File for a Plugin. + + Attributes: + descriptor_contents (Dict): Contents of the Descriptor yaml file + file_path (PathLike): path to the descriptor file + """ def __init__(self, file_path): + """Inits the descriptor as a PluginDescriptor from the provided path. + + Loads the contents of the filepath into descriptor_contents + + Args: + file_path (str): path to the yaml descriptor file + """ super(PluginDescriptor, self).__init__(file_path) # diff --git a/edk2toolext/environment/extdeptypes/__init__.py b/edk2toolext/environment/extdeptypes/__init__.py index 0a69010b8..9f73dc662 100644 --- a/edk2toolext/environment/extdeptypes/__init__.py +++ b/edk2toolext/environment/extdeptypes/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa \ No newline at end of file diff --git a/edk2toolext/environment/extdeptypes/az_cli_universal_dependency.py b/edk2toolext/environment/extdeptypes/az_cli_universal_dependency.py index 701198b63..92f787352 100644 --- a/edk2toolext/environment/extdeptypes/az_cli_universal_dependency.py +++ b/edk2toolext/environment/extdeptypes/az_cli_universal_dependency.py @@ -8,6 +8,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An ExternalDependency subclass able to download from an Azure feed.""" import os import logging import shutil @@ -20,16 +21,19 @@ class AzureCliUniversalDependency(ExternalDependency): - ''' - ext_dep fields: - - feed: feed name - - version: semantic version - - source: url of organization (example: https://dev.azure.com/tianocore) - - project: - - name: name of artifact - - file-filter: filter for folders and files. - - pat_var: shell_var name for PAT for this ext_dep - ''' + """An ExternalDependency subclass able to download from an Azure feed. + + Attributes: + feed (str): feed name + version (str): semantic version + source (str): url of organization (example: https://dev.azure.com/tianocore) + project (str) + name (str): name of artifact + file-filter (str): filter for folders and files. + pat_var (str): shell_var name for PAT for this ext_dep + + TIP: The attributes are what must be described in the ext_dep yaml file! + """ TypeString = "az-universal" # https://docs.microsoft.com/en-us/azure/devops/cli/log-in-via-pat?view=azure-devops&tabs=windows @@ -39,9 +43,10 @@ class AzureCliUniversalDependency(ExternalDependency): @classmethod def VerifyToolDependencies(cls): - """ Verify any tool environment or dependencies requirements are met. - Log to Version Aggregator the Tool Versions""" + """Verify any tool environment or dependencies requirements are met. + Log to Version Aggregator the Tool Versions + """ if cls.VersionLogged: return results = StringIO() @@ -77,6 +82,7 @@ def VerifyToolDependencies(cls): cls.VersionLogged = True def __init__(self, descriptor): + """Inits a Azure CLI dependency based off the provided descriptior.""" super().__init__(descriptor) self.global_cache_path = None self.organization = self.source @@ -95,7 +101,7 @@ def _fetch_from_cache(self, package_name): return False def __str__(self): - """ return a string representation of this """ + """Return a string representation.""" return f"AzCliUniversalDependency: {self.name}@{self.version}" def _attempt_universal_install(self, install_dir): @@ -134,6 +140,7 @@ def _attempt_universal_install(self, install_dir): (downloaded_version, self.version)) def fetch(self): + """Fetches the dependency using internal state from the init.""" # # Before trying anything with we should check # to see whether the package is already in @@ -171,9 +178,11 @@ def fetch(self): self.published_path = self.compute_published_path() def get_temp_dir(self): + """Returns the temporary directory the Azure CLI feed is downloaded to.""" return self.contents_dir + "_temp" def clean(self): + """Removes the temporary directory the NuGet package is downloaded to.""" super(AzureCliUniversalDependency, self).clean() if os.path.isdir(self.get_temp_dir()): RemoveTree(self.get_temp_dir()) diff --git a/edk2toolext/environment/extdeptypes/git_dependency.py b/edk2toolext/environment/extdeptypes/git_dependency.py index d4bebfcce..15a7f4331 100644 --- a/edk2toolext/environment/extdeptypes/git_dependency.py +++ b/edk2toolext/environment/extdeptypes/git_dependency.py @@ -7,6 +7,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An ExternalDependency subclass able to clone from git.""" import os import logging from urllib.parse import urlsplit, urlunsplit @@ -17,16 +18,19 @@ class GitDependency(ExternalDependency): - ''' - ext_dep fields: - - source: url for git clone - - version: commit from git repo - - url_creds_var: shell_var name for credential updating [optional] - ''' + """An ExternalDependency subclass able to clone from git. + Attributes: + source (str): url for git clone + version (str): commit from git repo + url_creds_var (str): shell_var name for credential updating. Optional + + TIP: The attributes are what must be described in the ext_dep yaml file! + """ TypeString = "git" def __init__(self, descriptor): + """Inits a git dependency based off the provided descriptor.""" super().__init__(descriptor) # Check to see whether this URL should be patched. @@ -55,11 +59,11 @@ def __init__(self, descriptor): self._repo_resolver_dep_obj = {"Path": self.name, "Url": self.repo_url, "Commit": self.commit} def __str__(self): - """ return a string representation of this """ + """Return a string representation.""" return f"GitDependecy: {self.repo_url}@{self.commit}" def fetch(self): - + """Fetches the dependency using internal state from the init.""" # def resolve(file_system_path, dependency, force=False, ignore=False, update_ok=False): repo_resolver.resolve(self._local_repo_root_path, self._repo_resolver_dep_obj, update_ok=True) @@ -67,6 +71,7 @@ def fetch(self): self.update_state_file() def clean(self): + """Removes the local clone of the repo.""" self.logger.debug("Cleaning git dependency directory for '%s'..." % self.name) if os.path.isdir(self._local_repo_root_path): @@ -76,8 +81,8 @@ def clean(self): # Let super class clean up common dependency stuff super().clean() - # override verify due to different scheme with git def verify(self): + """Verifies the clone was successful.""" result = True if not os.path.isdir(self._local_repo_root_path): @@ -105,6 +110,6 @@ def verify(self): self.logger.debug("Verify '%s' returning '%s'." % (self.name, result)) return result - # Override to include the repository name in published path def compute_published_path(self): + """Override to include the repository name in the published path.""" return os.path.join(super().compute_published_path(), self.name) diff --git a/edk2toolext/environment/extdeptypes/nuget_dependency.py b/edk2toolext/environment/extdeptypes/nuget_dependency.py index 155c3ed73..da2b517f5 100644 --- a/edk2toolext/environment/extdeptypes/nuget_dependency.py +++ b/edk2toolext/environment/extdeptypes/nuget_dependency.py @@ -6,6 +6,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An ExternalDependency subclass able to download from NuGet.""" import os import logging import semantic_version @@ -19,28 +20,38 @@ class NugetDependency(ExternalDependency): + """An ExternalDependency subclass able to download from NuGet. + + Attributes: + source (str): Source of the nuget dependency. + version (str): Version of the web dependency. + + TIP: The attributes are what must be described in the ext_dep yaml file! + """ TypeString = "nuget" - ''' Env variable name for path to folder containing NuGet.exe''' + # Env variable name for path to folder containing NuGet.exe NUGET_ENV_VAR_NAME = "NUGET_PATH" def __init__(self, descriptor): + """Inits a nuget dependency based off the provided descriptor.""" super().__init__(descriptor) self.nuget_cache_path = None - #### - # Add mono to front of command and resolve full path of exe for mono, - # Used to add nuget support on posix platforms. - # https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools - # - # Note that the strings returned might not be pathlike given they may be - # quoted for use on command line - # - # @return list containing either ["nuget.exe"] or ["mono", "/PATH/TO/nuget.exe"] - # @return none if not found - #### @classmethod def GetNugetCmd(cls) -> List[str]: + """Appends mono to the command and resolves the full path of the exe for mono. + + Used to add nuget support on posix platforms. + https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools + + Note: Strings returned might not be pathlike given they may be quoted + for use on the command line. + + Returns: + (list): ["nuget.exe"] or ["mono", "/PATH/TO/nuget.exe"] + (None): none was found + """ file_name = "NuGet.exe" cmd = [] if GetHostInfo().os == "Linux": @@ -69,13 +80,13 @@ def GetNugetCmd(cls) -> List[str]: @staticmethod def normalize_version(version, nuget_name=""): - # The following link describes where NuGet versioning diverges from - # Semantic Versioning: - # https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#where-nugetversion-diverges-from-semantic-versioning - # - # These cases will be handled before a "Semantic Version compatible" - # set of data is passed to the Semantic Version checker. + """Normalizes the version as NuGet versioning diverges from Semantic Versioning. + + https://learn.microsoft.com/en-us/nuget/concepts/package-versioning#where-nugetversion-diverges-from-semantic-versioning + These cases will be handled befpre a Semantic Version Compatible" set + of data is passed to the Semantic Version checker. + """ # 1. NuGetVersion requires the major segment to be defined if not version: raise ValueError("String is empty. At least major version is " @@ -195,7 +206,7 @@ def _fetch_from_nuget_cache(self, package_name): return result def __str__(self): - """ return a string representation of this """ + """Return a string representation.""" return f"NugetDependecy: {self.name}@{self.version}" def _attempt_nuget_install(self, install_dir, non_interactive=True): @@ -235,6 +246,7 @@ def _attempt_nuget_install(self, install_dir, non_interactive=True): raise RuntimeError(f"[Nuget] We failed to install this version {self.version} of {package_name}") def fetch(self): + """Fetches the dependency using internal state from the init.""" package_name = self.name # First, check the global cache to see if it's present. @@ -294,9 +306,11 @@ def fetch(self): self.published_path = self.compute_published_path() def get_temp_dir(self): + """Returns the temporary directory the NuGet package is downloaded to.""" return self.contents_dir + "_temp" def clean(self): + """Removes the temporary directory the NuGet package is downloaded to.""" super(NugetDependency, self).clean() if os.path.isdir(self.get_temp_dir()): RemoveTree(self.get_temp_dir()) diff --git a/edk2toolext/environment/extdeptypes/web_dependency.py b/edk2toolext/environment/extdeptypes/web_dependency.py index 78389c283..82ea89e95 100644 --- a/edk2toolext/environment/extdeptypes/web_dependency.py +++ b/edk2toolext/environment/extdeptypes/web_dependency.py @@ -6,6 +6,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An ExternalDependency subclass able to download from a url.""" import os import logging @@ -20,19 +21,29 @@ class WebDependency(ExternalDependency): - ''' - ext_dep fields: - - internal_path: Describes layout of what we're downloading. Include / at the beginning - if the ext_dep is a directory. Item located at internal_path will - unpacked into the ext_dep folder and this is what the path/shell vars - will point to when compute_published_path is run. - - compression_type: optional. supports zip and tar. If the file isn't compressed, do not include this field. - - sha256: optional. hash of downloaded file to be checked against. - ''' + """An ExternalDependency subclass able to download from a url. + + Able to download an uncrompressed, zip, or tar file from the web and can + optionally compare the hash to a provided hash. + + Attributes: + internal_path (str): Describes layout of what we're downloading. + Include / at the beginning if the ext_dep is a + directory. Item located at internal_path will + unpacked into the ext_dep folder and this is what + the path/shell vars will point to when + compute_published_path is run. + compression_type (str): Supports zip and tar. If the file isn't + compressed, do not include this field. Optional + sha256 (str): Hash of downloaded file to be checked against. Optional + + TIP: The attributes are what must be described in the ext_dep yaml file! + """ TypeString = "web" def __init__(self, descriptor): + """Inits a web dependency based off the provided descriptor.""" super().__init__(descriptor) self.internal_path = os.path.normpath(descriptor['internal_path']) self.compression_type = descriptor.get('compression_type', None) @@ -45,25 +56,28 @@ def __init__(self, descriptor): self.internal_path = self.internal_path.strip(os.path.sep) def __str__(self): - """ return a string representation of this """ + """Returns a string representation.""" return f"WebDependecy: {self.source}@{self.version}" @staticmethod def linuxize_path(path): - ''' - path: path that uses os.sep, to be replaced with / for compatibility with zipfile - ''' + """Replaces windows style separators with linux style separators. + + Arguments: + path (str): the path + """ return "/".join(path.split("\\")) @staticmethod def unpack(compressed_file_path, destination, internal_path, compression_type): - ''' - compressed_file_path: name of compressed file to unpack. - destination: directory you would like it unpacked into. - internal_path: internal structure of the compressed volume that you would like extracted. - compression_type: type of compression. tar and zip supported. - ''' - + """Unpacks a compressed file to the specified location; zip or tar supported. + + Arguments: + compressed_file_path (str): Name of compressed file to unpack. + destination (str): directory to unpacked into. + internal_path (str): internal structure of the compressed volume that you would like extracted. + compression_type (str): type of compression. tar and zip supported. + """ # First, we will open the file depending on the type of compression we're dealing with. # tarfile and zipfile both use the Linux path seperator / instead of using os.sep @@ -91,6 +105,7 @@ def unpack(compressed_file_path, destination, internal_path, compression_type): _ref.close() def fetch(self): + """Fetches the dependency using internal state from the init.""" # First, check the global cache to see if it's present. if super().fetch(): return diff --git a/edk2toolext/environment/external_dependency.py b/edk2toolext/environment/external_dependency.py index e1522490c..4a0c9ccd0 100644 --- a/edk2toolext/environment/external_dependency.py +++ b/edk2toolext/environment/external_dependency.py @@ -8,6 +8,11 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This module contains helper object for manipulating external dependencies. + +These manipulations include retrieving, validating, and cleaning external +dependencies for the build environment. +""" import os import logging @@ -19,19 +24,26 @@ class ExternalDependency(object): - ''' - ext_dep fields: - - scope: Determines if descriptor is included on a particular build. - - type: Type of ext_dep. - - name: Name of ext_dep, used to name the folder the ext_dep will be unpacked in to - - source: Source to query for ext_dep. - - version: Version string to keep track of what version is currently installed. - - flags: Flags dictating what actions should be taken once this dependency is resolved - More info: (docs/feature_extdep/) - - var_name: Used with set_*_var flag. Determines name of var to be set. - ''' + """Baseclass to allow downloading external dependencies during the update phase. + + Specific External dependency types (git, nuget, etc.) are subclassed from + this class. Additional external dependency types may be created. + + Attributes: + scope (str): Determines if descriptor is included on a particular build. + type (str): Type of ext_dep. + name (str): Name of ext_dep, used to name the folder the ext_dep will be unpacked in to + source (str): Source to query for ext_dep. + version (str): Version string to keep track of what version is currently installed. + flags (list[str]): Flags dictating what actions should be taken once this dependency is resolved + More info: (docs/feature_extdep/) + var_name (str): Used with set_*_var flag. Determines name of var to be set. + + TIP: The attributes are what must be described in the ext_dep yaml file! + """ def __init__(self, descriptor): + """Inits a web dependency based off the provided descriptor.""" super(ExternalDependency, self).__init__() # @@ -56,10 +68,16 @@ def __init__(self, descriptor): self.published_path = self.compute_published_path() def set_global_cache_path(self, global_cache_path): + """Sets the global cache path to locate already downloaded dependencies. + + Arguments: + global_cache_path (str): directory of the global cache + """ self.global_cache_path = os.path.abspath(global_cache_path) return self def compute_published_path(self): + """Determines the published path.""" new_published_path = self.contents_dir if self.flags and "host_specific" in self.flags and self.verify(): @@ -101,11 +119,13 @@ def compute_published_path(self): return new_published_path def clean(self): + """Removes the local directory for the external dependency.""" logging.debug("Cleaning dependency directory for '%s'..." % self.name) if os.path.isdir(self.contents_dir): RemoveTree(self.contents_dir) def determine_cache_path(self): + """Determines the cache path is global_cache_path is not none.""" result = None if self.global_cache_path is not None and os.path.isdir(self.global_cache_path): subpath_calc = hashlib.sha1() @@ -116,6 +136,7 @@ def determine_cache_path(self): return result def fetch(self): + """Fetches the dependency using internal state from the init.""" cache_path = self.determine_cache_path() if cache_path is None or not os.path.isdir(cache_path): return False @@ -126,6 +147,11 @@ def fetch(self): return True def copy_from_global_cache(self, dest_path: str): + """Copies the dependency from global cache if present. + + Arguments: + dest_path (str): path to copy to + """ cache_path = self.determine_cache_path() if cache_path is None: return @@ -133,6 +159,11 @@ def copy_from_global_cache(self, dest_path: str): shutil.copytree(cache_path, dest_path, dirs_exist_ok=True) def copy_to_global_cache(self, source_path: str): + """Copies the dependency to global cache if present. + + Arguments: + source_path (str): source to copy into global cache. + """ cache_path = self.determine_cache_path() if cache_path is None: return @@ -143,6 +174,7 @@ def copy_to_global_cache(self, source_path: str): shutil.copytree(source_path, cache_path, dirs_exist_ok=True) def verify(self): + """Verifies the dependency was successfully downloaded.""" result = True state_data = None @@ -168,18 +200,24 @@ def verify(self): return result def report_version(self): + """Reports the version of the external dependency.""" version_aggregator.GetVersionAggregator().ReportVersion(self.name, self.version, version_aggregator.VersionTypes.INFO, self.descriptor_location) def update_state_file(self): + """Updates the file representing the state of the dependency.""" with open(self.state_file_path, 'w+') as file: yaml.dump({'version': self.version}, file) def ExtDepFactory(descriptor): - # Add all supported external dependencies here to avoid import errors. + """External Dependency Factory capable of generating each type of dependency. + + Note: Ensure all external dependencies are imported in this class to + avoid errors. + """ from edk2toolext.environment.extdeptypes.web_dependency import WebDependency from edk2toolext.environment.extdeptypes.nuget_dependency import NugetDependency from edk2toolext.environment.extdeptypes.git_dependency import GitDependency diff --git a/edk2toolext/environment/multiple_workspace.py b/edk2toolext/environment/multiple_workspace.py index df9f7a7d3..ca010ef05 100644 --- a/edk2toolext/environment/multiple_workspace.py +++ b/edk2toolext/environment/multiple_workspace.py @@ -11,49 +11,49 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent # +"""This file is required to make Python interpreter treat the directory as a containing package. -import os +File is slightly modified from Edk2 BaseTools/Source/Python/Common/MultipleWorkspace.py +""" -# MultipleWorkspace -# -# This class manage multiple workspace behavior -# -# @param class: -# -# @var WORKSPACE: defined the current WORKSPACE -# @var PACKAGES_PATH: defined the other WORKSPACE, if current WORKSPACE is invalid, -# search valid WORKSPACE from PACKAGES_PATH -# +import os class MultipleWorkspace(object): + """This class manages multiple workspace behavior. + + Attributes: + WORKSPACE (str): defined the current workspace + PACKAGES_PATH (str): defined the other WORKSPACE + """ WORKSPACE = '' PACKAGES_PATH = None - # convertPackagePath() - # - # Convert path to match workspace. - # - # @param cls The class pointer - # @param Ws The current WORKSPACE - # @param Path Path to be converted to match workspace. - # @classmethod def convertPackagePath(cls, Ws, Path): + """Convert path to match workspace. + + Args: + cls (obj): The class pointer + Ws (str): The current WORKSPACE + Path (str): Path to be converted to match workspace + + Returns: + (str) Converted path. + """ if str(os.path.normcase(Path)).startswith(Ws): return os.path.join(Ws, Path[len(Ws) + 1:]) return Path - # setWs() - # - # set WORKSPACE and PACKAGES_PATH environment - # - # @param cls The class pointer - # @param Ws initialize WORKSPACE variable - # @param PackagesPath initialize PackagesPath variable - # @classmethod def setWs(cls, Ws, PackagesPath=None): + """Set WORKSPACE and PACKAGES_PATH environment. + + Args: + cls (obj): The class pointer + Ws (str): initialize WORKSPACE variable + PackagesPath (str): initialize PackagesPath variable + """ cls.WORKSPACE = Ws if PackagesPath: cls.PACKAGES_PATH = [cls.convertPackagePath(Ws, os.path.normpath( @@ -61,17 +61,18 @@ def setWs(cls, Ws, PackagesPath=None): else: cls.PACKAGES_PATH = [] - # join() - # - # rewrite os.path.join function - # - # @param cls The class pointer - # @param Ws the current WORKSPACE - # @param *p path of the inf/dec/dsc/fdf/conf file - # @retval Path the absolute path of specified file - # @classmethod def join(cls, Ws, *p): + """Rewrite os.path.join. + + Args: + cls (obj): The class pointer + Ws (str): the current WORKSPACE + *p (str): path of the inf/dec/dsc/fdf/conf file + + Returns: + (str): absolute path of the specified file + """ Path = os.path.join(Ws, *p) if not os.path.exists(Path): for Pkg in cls.PACKAGES_PATH: @@ -81,17 +82,18 @@ def join(cls, Ws, *p): Path = os.path.join(Ws, *p) return Path - # relpath() - # - # rewrite os.path.relpath function - # - # @param cls The class pointer - # @param Path path of the inf/dec/dsc/fdf/conf file - # @param Ws the current WORKSPACE - # @retval Path the relative path of specified file - # @classmethod def relpath(cls, Path, Ws): + """Rewrite os.path.relpath. + + Args: + cls (obj): The class pointer + Path (str): path of the inf/dec/dsc/fdf/conf file + Ws (str): the current WORKSPACE + + Returns: + (str): the relative path of specified file + """ for Pkg in cls.PACKAGES_PATH: if Path.lower().startswith(Pkg.lower()): Path = os.path.relpath(Path, Pkg) @@ -100,17 +102,18 @@ def relpath(cls, Path, Ws): Path = os.path.relpath(Path, Ws) return Path - # getWs() - # - # get valid workspace for the path - # - # @param cls The class pointer - # @param Ws the current WORKSPACE - # @param Path path of the inf/dec/dsc/fdf/conf file - # @retval Ws the valid workspace relative to the specified file path - # @classmethod def getWs(cls, Ws, Path): + """Get valid workspace for the path. + + Args: + cls (obj): The class pointer + ws (str): the current WORKSPACE + Path (str): path of the inf/dec/dsc/fdf/conf file + + Returns: + (str): valid workspace relative to the specified file path + """ absPath = os.path.join(Ws, Path) if not os.path.exists(absPath): for Pkg in cls.PACKAGES_PATH: @@ -119,15 +122,19 @@ def getWs(cls, Ws, Path): return Pkg return Ws - # handleWsMacro() - # - # handle the $(WORKSPACE) tag, if current workspace is invalid path relative the tool, replace it. - # - # @param cls The class pointer - # @retval PathStr Path string include the $(WORKSPACE) - # @classmethod def handleWsMacro(cls, PathStr): + """Handle the $(WORKSPACE) tag. + + If current workspace is an invalid path relative to the tool, replace + it. + + Args: + cls (obj): The class pointer + + Returns: + (Str): Path string including the $(WORKSPACE) + """ TAB_WORKSPACE = '$(WORKSPACE)' if TAB_WORKSPACE in PathStr: PathList = PathStr.split() @@ -146,12 +153,15 @@ def handleWsMacro(cls, PathStr): PathStr = ' '.join(PathList) return PathStr - # getPkgPath() - # - # get all package pathes. - # - # @param cls The class pointer - # @classmethod def getPkgPath(cls): + """Get all package paths. + + Args: + cls (obj): class pointer + + Returns: + (obj): Packages Path + + """ return cls.PACKAGES_PATH diff --git a/edk2toolext/environment/plugin_manager.py b/edk2toolext/environment/plugin_manager.py index f77a5411e..b1699e6af 100644 --- a/edk2toolext/environment/plugin_manager.py +++ b/edk2toolext/environment/plugin_manager.py @@ -5,6 +5,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This module contains code that supports Build Plugins.""" import sys import os @@ -14,25 +15,38 @@ class PluginDescriptor(object): + """Plugin Descripter. + + Attributes: + descripter(Dict): descriptor + Obj (obj): Object + Name (str): name attribute from descriptor + Module (obj): module attribute from descriptor + """ def __init__(self, t): + """Inits the Plugin descriptor with the Descriptor.""" self.descriptor = t self.Obj = None self.Name = t["name"] self.Module = t["module"] def __str__(self): + """String representation of the plugin descriptor.""" return "PLUGIN DESCRIPTOR:{0}".format(self.Name) class PluginManager(object): + """A class that manages all plugins in the environment. + Attributes: + Descriptors (List[PluginDescriptor]): list of plugin descriptors + """ def __init__(self): + """Inits an empty plugin manager.""" self.Descriptors = [] - # - # Pass tuple of Environment Descriptor dictionaries to be loaded as plugins - # def SetListOfEnvironmentDescriptors(self, newlist): + """Passes a tuple of environment descriptor dictionaries to be loaded as plugins.""" env = shell_environment.GetBuildVars() failed = [] if newlist is None: @@ -49,26 +63,24 @@ def SetListOfEnvironmentDescriptors(self, newlist): failed.append(a) return failed - # - # Return List of all plugins of a given class - # def GetPluginsOfClass(self, classobj): + """Return list of all plugins of a given class.""" temp = [] for a in self.Descriptors: if (isinstance(a.Obj, classobj)): temp.append(a) return temp - # - # Return List of all plugins - # def GetAllPlugins(self): + """Return list of all plugins.""" return self.Descriptors - # - # Load and Instantiate the plugin - # def _load(self, PluginDescriptor): + """Load and instantiate the plugin. + + Arguments: + PluginDescriptor(PluginDescriptor): the plugin descriptor + """ PluginDescriptor.Obj = None PythonFileName = PluginDescriptor.descriptor["module"] + ".py" PyModulePath = os.path.join(os.path.dirname(os.path.abspath( diff --git a/edk2toolext/environment/plugintypes/__init__.py b/edk2toolext/environment/plugintypes/__init__.py index 0a69010b8..9f73dc662 100644 --- a/edk2toolext/environment/plugintypes/__init__.py +++ b/edk2toolext/environment/plugintypes/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa \ No newline at end of file diff --git a/edk2toolext/environment/plugintypes/ci_build_plugin.py b/edk2toolext/environment/plugintypes/ci_build_plugin.py index 51b373408..23b495bb1 100644 --- a/edk2toolext/environment/plugintypes/ci_build_plugin.py +++ b/edk2toolext/environment/plugintypes/ci_build_plugin.py @@ -5,7 +5,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## - +"""Plugin that supports adding tests or operations to the ci environment.""" import os import logging @@ -13,63 +13,70 @@ class ICiBuildPlugin(object): - - ## - # External function of plugin. This function is used to perform the task of the CiBuild Plugin - # - # - package is the edk2 path to package. This means workspace/package path relative. - # - edk2path object configured with workspace and packages path - # - PkgConfig Object (dict) for the pkg - # - EnvConfig Object - # - Plugin Manager Instance - # - Plugin Helper Obj Instance - # - tc - test case that needs state configured for reporting by plugin. - # - output_stream the StringIO output stream from this plugin via logging - # - # Returns >0 : number of errors found - # 0 : passed successfully - # -1 : skipped for missing prereq - # - # + """Plugin that supports adding tests or operations to the ci environment.""" def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream): + """External function of plugin. + + This function is used to perform the task of the CiBuild Plugin + + Args: + packagename (str): edk2 path to package (workspace/package path relative) + Edk2pathObj (Edk2Path): Edk2Path configured with workspace and package path + pkgconfig (dict): Package config + environment (EnvConfig): Environment config + PLM (PluginManager): Plugin manager instance + PLMHelper (HelperFunctions): Plugin helper object instace + tc (obj): test case that needs state configured for reporting by plugin + output_stream (StringIO): output stream from this plugin via logging + + Returns: + (int): >0 - number of errors found + (int): 0 - passed successfully + (int): -1 - skipped for missing prereq + """ pass def GetTestName(self, packagename: str, environment: object) -> Tuple[str, str]: - ''' Given the package name and configuration provide the caller - the name of the test case and the class name. These are both used in logging - and reporting test status. + """Provides the test case and class name. + + Given the package name and configuration provide the caller + the name of the test case and the class name. These are both used in logging + and reporting test status. - @packagename: String - Package Name - @environment: EnvDict Object - Environment Dictionary configuration + Args: + packagename (str): Package Name + environment (EnvDict): Environment Dictionary configuration - @returns tuple of (test case name, test case base class name) - ''' + Returns: + (Tuple[str, str]): (test case name, test case base class name) + """ pass def RunsOnTargetList(self) -> List[str]: - ''' Returns a list of edk2 TARGETs that this plugin would like to run on + """Returns a list of edk2 TARGETs that this plugin would like to run on. - KNOWN TARGET VALUES: - DEBUG - RELEASE - NOOPT - NO-TARGET + HINT: known target values: + DEBUG + RELEASE + NOOPT + NO-TARGET - If the plugin is not Target specific it should return a list of - one element of "NO-TARGET" - ''' + HINT: If the plugin is not Target specific it should return a list of one element of "NO-TARGET" + """ return ["NO-TARGET"] def WalkDirectoryForExtension(self, extensionlist: List[str], directory: os.PathLike, ignorelist: List[str] = None) -> List[os.PathLike]: - ''' Walks a file directory recursively for all items ending in certain extension + """Walks a file directory recursively for all items ending in certain extension. - @extensionlist: List[str] list of file extensions - @directory: Path - absolute path to directory to start looking - @ignorelist: List[str] or None. optional - default is None: a list of case insensitive filenames to ignore + Args: + extensionlist (List[str]): list of file extensions + directory (PathLike): absolute path to directory to start looking + ignorelist (List[str]): a list of case insensitive filenames to ignore (Optional) - @returns a List of file paths to matching files - ''' + Returns: + (List): file paths to matching files + """ if not isinstance(extensionlist, list): logging.critical("Expected list but got " + str(type(extensionlist))) raise TypeError("extensionlist must be a list") diff --git a/edk2toolext/environment/plugintypes/dsc_processor_plugin.py b/edk2toolext/environment/plugintypes/dsc_processor_plugin.py index c71465187..22dc56a09 100644 --- a/edk2toolext/environment/plugintypes/dsc_processor_plugin.py +++ b/edk2toolext/environment/plugintypes/dsc_processor_plugin.py @@ -1,32 +1,36 @@ # @file dsc_processor_plugin -# Plugin for for parsing DSCs +# Plugin for parsing DSCs ## # Copyright (c) Microsoft Corporation # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Plugin for parsing DSCs.""" class IDscProcessorPlugin(object): + """Plugin for parsing DSCs.""" - ## - # does the transform on the DSC - # - # @param dsc - the in-memory model of the DSC - # @param thebuilder - UefiBuild object to get env information - # - # @return 0 for success NonZero for error. - ## def do_transform(self, dsc, thebuilder): + """Does the transform on a DSC. + + Args: + dsc (obj): the in-memory model of the DSC + thebuilder (UefiBuilder): UefiBuild object for env information + + Returns: + (int): 0 or NonZero for success or failure + """ return 0 - ## - # gets the level that this transform operates at - # - # @param thebuilder - UefiBuild object to get env information - # - # @return 0 for the most generic level - ## def get_level(self, thebuilder): + """Gets the level that this transform operates at. + + Args: + thebuilder (UefiBuilder): UefiBuild object for env information + + Returns: + (int): the level + """ return 0 diff --git a/edk2toolext/environment/plugintypes/uefi_build_plugin.py b/edk2toolext/environment/plugintypes/uefi_build_plugin.py index 524fb50a6..847aedb52 100644 --- a/edk2toolext/environment/plugintypes/uefi_build_plugin.py +++ b/edk2toolext/environment/plugintypes/uefi_build_plugin.py @@ -5,29 +5,30 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Plugin that supports Pre and Post Build Steps.""" class IUefiBuildPlugin(object): + """Plugin that supports Pre and Post Build Steps.""" - ## - # Run Post Build Operations - # - # @param thebuilder - UefiBuild object to get env information - # - # @return 0 for success NonZero for error. - ## def do_post_build(self, thebuilder): + """Runs Post Build Plugin Operations. + + Args: + thebuilder (UefiBuilder): UefiBuild object for env information + + Returns: + (int): 0 or NonZero for success or failure + """ return 0 - ## - # Run Pre Build Operations - # - # @param thebuilder - UefiBuild object to get env information - # - # @return 0 for success NonZero for error. - ## def do_pre_build(self, thebuilder): - ''' - Run Pre build Operation - ''' + """Runs Pre Build Plugin Operations. + + Args: + thebuilder (UefiBuilder): UefiBuild object for env information + + Returns: + (int): 0 or NonZero for success or failure + """ return 0 diff --git a/edk2toolext/environment/plugintypes/uefi_helper_plugin.py b/edk2toolext/environment/plugintypes/uefi_helper_plugin.py index f367d8c2c..393fbe1d1 100644 --- a/edk2toolext/environment/plugintypes/uefi_helper_plugin.py +++ b/edk2toolext/environment/plugintypes/uefi_helper_plugin.py @@ -6,47 +6,58 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## - +"""Plugin that supports adding Extension or helper methods to the build environment.""" import logging class IUefiHelperPlugin(object): + """The class that should be subclassed when creating a UEFI Helper Plugin.""" - ## - # Function that allows plugin to register its functions with the - # obj. - # @param obj[in, out]: HelperFunctions object that allows functional - # registration. - # def RegisterHelpers(self, obj): - pass + """Allows a plugin to register its functions. -# Supports IUefiHelperPlugin type + TIP: obj.Register() + + Args: + obj (HelperFunctions): HelperFunctions object that allows functional registration + """ + pass class HelperFunctions(object): + """A class that contains all registed functions. + + Attributes: + RegisteredFunctions(dict): registered functions + """ def __init__(self): + """Initializes instance.""" self.RegisteredFunctions = {} - # - # Function to logging.debug all registered functions and their source path - # def DebugLogRegisteredFunctions(self): + """Logs all registered functions and their source path. + + Uses logging.debug to write all registered functions and their source path. + """ logging.debug("Logging all Registered Helper Functions:") for name, file in self.RegisteredFunctions.items(): logging.debug(" Function %s registered from file %s", name, file) logging.debug("Finished logging %d functions", len(self.RegisteredFunctions)) - # - # Plugins that want to register a helper function should call - # this routine for each function - # - # @param name[in]: name of function - # @param function[in] function being registered - # @param filepath[in] filepath registering function. used for tracking and debug purposes - # def Register(self, name, function, filepath): + """Registers a plugin. + + Plugins that want to register a helper function should call + this routine for each function + + Args: + name (str): name of the function + function (func): function being registered + filepath (str): filepath to this file + + TIP: os.path.abspath(__file__) + """ if (name in self.RegisteredFunctions.keys()): raise Exception("Function %s already registered from plugin file %s. Can't register again from %s" % ( name, self.RegisteredFunctions[name], filepath)) @@ -54,12 +65,31 @@ def Register(self, name, function, filepath): self.RegisteredFunctions[name] = filepath def HasFunction(self, name): + """Returns if a function exists. + + Args: + name (str): name of the function + + Returns: + (bool): if the function is registered or not. + """ if (name in self.RegisteredFunctions.keys()): return True else: return False def LoadFromPluginManager(self, pm): + """Load all IUefiHelperPlugins into the class. + + Uses the PluginManager class to get all IUefiHelperPlugins in the environment, + then stores them all in a dict. + + Args: + pm (PluginManager): class holding all plugins + + Returns: + (int): number of plugins that failed to be loaded. + """ error = 0 for Descriptor in pm.GetPluginsOfClass(IUefiHelperPlugin): logging.info(Descriptor) diff --git a/edk2toolext/environment/repo_resolver.py b/edk2toolext/environment/repo_resolver.py index 52053f623..b3d7a23b1 100644 --- a/edk2toolext/environment/repo_resolver.py +++ b/edk2toolext/environment/repo_resolver.py @@ -5,7 +5,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## - +"""This module supports git operations.""" import os import logging from edk2toolext import edk2_logging @@ -17,6 +17,18 @@ # checks out dependency at git_path def resolve(file_system_path, dependency, force=False, ignore=False, update_ok=False): + """Resolves a particular repo. + + Args: + file_system_path (Pathlike): path to repo + dependency (Dict): contains Path, Commit, Branch + force (bool): If it is OK to update the commit or branch + ignore (bool): If it is OK to ignore errors or not. + update_ok (bool): If it is OK to update the commit or branch + + Raises: + (Exception): An error resolving a repo and ignore=False + """ logger = logging.getLogger("git") logger.info("Checking for dependency {0}".format(dependency["Path"])) git_path = os.path.abspath(file_system_path) @@ -110,6 +122,19 @@ def resolve(file_system_path, dependency, force=False, ignore=False, update_ok=F def resolve_all(workspace_path, dependencies, force=False, ignore=False, update_ok=False, omnicache_dir=None): + """Resolves all repos. + + Args: + workspace_path (Pathlike): workspace root + dependencies (List[Dict]): Dict contains Path, Commit, Branch + force (bool): If it is OK to update the commit or branch + ignore (bool): If it is OK to ignore errors or not. + update_ok (bool): If it is OK to update the commit or branch + omnicache_dir (:obj:`bool`, optional): Omnicache path, if used + + Raises: + (Exception): An error resolving a repo and ignore=False + """ logger = logging.getLogger("git") repos = [] if force: @@ -138,8 +163,15 @@ def resolve_all(workspace_path, dependencies, force=False, ignore=False, update_ return repos -# Gets the details of a particular repo def get_details(abs_file_system_path): + """Gets the Url, Branch, and Commit of a particular repo. + + Args: + abs_file_system_path (PathLike): repo directory + + Returns: + (Dict): Url, Branch, Commit + """ repo = Repo(abs_file_system_path) url = repo.remotes.origin.url active_branch = repo.active_branch @@ -148,14 +180,27 @@ def get_details(abs_file_system_path): def clear_folder(abs_file_system_path): + """Cleans the folder. + + Args: + abs_file_system_path (PathLike): Directory to delete. + """ logger = logging.getLogger("git") logger.warning("WARNING: Deleting contents of folder {0} to make way for Git repo".format( abs_file_system_path)) RemoveTree(abs_file_system_path) -# Clones the repo in the folder we need using the dependency object from the json def clone_repo(abs_file_system_path, DepObj): + """Clones the repo in the folder using the dependency object. + + Args: + abs_file_system_path (PathLike): destination to clone + DepObj (Dict): dict containing Commit, Full, Branch, etc + + Returns: + (Tuple[PathLike, bool]): (destination, result) + """ logger = logging.getLogger("git") logger.log(edk2_logging.get_progress_level(), "Cloning repo: {0}".format(DepObj["Url"])) dest = abs_file_system_path @@ -188,6 +233,21 @@ def clone_repo(abs_file_system_path, DepObj): def checkout(abs_file_system_path, dep, repo, update_ok=False, ignore_dep_state_mismatch=False, force=False): + """Checks out a commit or branch. + + Args: + abs_file_system_path (PathLike): The path to the repo + dep (Dict): A dictionary containing a either a Commit or Branch, and also a Path + repo (Repo): A valid repo object + update_ok (bool): If it is OK to update the commit or branch + ignore_dep_state_mismatch (bool): Whether a mismatch will result in an exception or not. + force (bool): If it is OK to update the commit or branch + + Raises: + (Exception): dependency state mismatch if ignore_dep_state_mismatch = False + + TIP: Either abs_file_system_path or repo is necessary. Not both. + """ logger = logging.getLogger("git") if repo is None: repo = Repo(abs_file_system_path) diff --git a/edk2toolext/environment/self_describing_environment.py b/edk2toolext/environment/self_describing_environment.py index 567ddddc3..91367fba6 100644 --- a/edk2toolext/environment/self_describing_environment.py +++ b/edk2toolext/environment/self_describing_environment.py @@ -6,7 +6,11 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An environment capable of scanning the source tree. +Scans the environment for files that describe the source and dependencies +and then acts upon those files. +""" import os import logging import pathlib @@ -22,7 +26,13 @@ class self_describing_environment(object): + """An environment capable of scanning the source tree. + + Scans the environment for files that describe the source and dependencies + and then acts upon those files. + """ def __init__(self, workspace_path, scopes=(), skipped_dirs=()): + """Inits an empty self describing environment.""" logging.debug("--- self_describing_environment.__init__()") logging.debug(f"Skipped directories specified = {skipped_dirs}") super(self_describing_environment, self).__init__() @@ -73,6 +83,7 @@ def _gather_env_files(self, ext_strings, base_path): return matches def load_workspace(self): + """Loads the workspace.""" logging.debug("--- self_describing_environment.load_workspace()") logging.debug("Loading workspace: %s" % self.workspace) logging.debug(" Including scopes: %s" % ', '.join(self.scopes)) @@ -205,21 +216,25 @@ def _apply_descriptor_object_to_env(self, desc_object, env_object): desc_object.var_name, desc_object.published_path) def update_simple_paths(self, env_object): + """Updates simple paths.""" logging.debug("--- self_describing_environment.update_simple_paths()") for path in self._get_paths(): self._apply_descriptor_object_to_env(path, env_object) def update_extdep_paths(self, env_object): + """Updates external dependency paths.""" logging.debug("--- self_describing_environment.update_extdep_paths()") for extdep in self._get_extdeps(env_object): self._apply_descriptor_object_to_env(extdep, env_object) def report_extdep_version(self, env_object): + """Reports the version of all external dependencies.""" logging.debug("--- self_describing_environment.report_extdep_version()") for extdep in self._get_extdeps(env_object): extdep.report_version() def update_extdeps(self, env_object): + """Updates external dependencies.""" logging.debug("--- self_describing_environment.update_extdeps()") # This function is called by our thread pool @@ -285,11 +300,13 @@ def update_extdep(self, extdep): return success_count, failure_count def clean_extdeps(self, env_object): + """Cleans external dependencies.""" for extdep in self._get_extdeps(env_object): extdep.clean() # TODO: Determine whether we want to update the env. def verify_extdeps(self, env_object): + """Verifies external dependencies.""" result = True for extdep in self._get_extdeps(env_object): if not extdep.verify(): @@ -300,7 +317,7 @@ def verify_extdeps(self, env_object): def DestroyEnvironment(): - ''' Destroys global environment state ''' + """Destroys global environment state.""" global ENVIRONMENT_BOOTSTRAP_COMPLETE, ENV_STATE ENVIRONMENT_BOOTSTRAP_COMPLETE = False @@ -308,6 +325,21 @@ def DestroyEnvironment(): def BootstrapEnvironment(workspace, scopes=(), skipped_dirs=()): + """Performs a multistage bootstrap of the environment. + + 1. Locate and load all environment description files + 2. Parse all PATH-related descriptor files + 3. Load modules that had dependencies + 4. Report versions into the version aggregator + + Args: + workspace (str): workspace root + scopes (Tuple): scopes being built against + skipped_dirs (Tuple): directories to ignore + + WARNING: if only one scope or skipped_dir, the tuple should end with a comma + example: '(myscope,)' + """ global ENVIRONMENT_BOOTSTRAP_COMPLETE, ENV_STATE if not ENVIRONMENT_BOOTSTRAP_COMPLETE: @@ -349,6 +381,19 @@ def BootstrapEnvironment(workspace, scopes=(), skipped_dirs=()): def CleanEnvironment(workspace, scopes=(), skipped_dirs=()): + """Cleans all external dependencies based on environment. + + Environment is bootstrapped from provided arguments and all dependencies + are cleaned from that. + + Args: + workspace (str): workspace root + scopes (Tuple): scopes being built against + skipped_dirs (Tuple): directories to ignore + + WARNING: if only one scope or skipped_dir, the tuple should end with a comma + example: '(myscope,)' + """ # Bootstrap the environment. (build_env, shell_env) = BootstrapEnvironment(workspace, scopes, skipped_dirs) @@ -357,16 +402,42 @@ def CleanEnvironment(workspace, scopes=(), skipped_dirs=()): def UpdateDependencies(workspace, scopes=(), skipped_dirs=()): + """Updates all external dependencies based on environment. + + Environment is bootstrapped from provided arguments and all dependencies + are updated from that. + + Args: + workspace (str): workspace root + scopes (Tuple): scopes being built against + skipped_dirs (Tuple): directories to ignore + + WARNING: if only one scope or skipped_dir, the tuple should end with a comma + example: '(myscope,)' + """ # Bootstrap the environment. (build_env, shell_env) = BootstrapEnvironment(workspace, scopes, skipped_dirs) - # Clean all the dependencies. + # Update all the dependencies. return build_env.update_extdeps(shell_env) def VerifyEnvironment(workspace, scopes=(), skipped_dirs=()): + """Verifies all external dependencies based on environment. + + Environment is bootstrapped from provided arguments and all dependencies + are verified from that. + + Args: + workspace (str): workspace root + scopes (Tuple): scopes being built against + skipped_dirs (Tuple): directories to ignore + + WARNING: if only one scope or skipped_dir, the tuple should end with a comma + example: '(myscope,)' + """ # Bootstrap the environment. (build_env, shell_env) = BootstrapEnvironment(workspace, scopes, skipped_dirs) - # Clean all the dependencies. + # Verify all the dependencies. return build_env.verify_extdeps(shell_env) diff --git a/edk2toolext/environment/shell_environment.py b/edk2toolext/environment/shell_environment.py index 78c35d25c..6ee826615 100644 --- a/edk2toolext/environment/shell_environment.py +++ b/edk2toolext/environment/shell_environment.py @@ -6,7 +6,10 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code that helps manage and maintain the build environment. +This management includes PATH, PYTHONPATH and ENV Variables. +""" import os import sys import copy @@ -21,21 +24,28 @@ # Copy the Singleton pattern from... # https://stackoverflow.com/a/6798042 # -class Singleton(type): +class Singleton(type): # noqa _instances = {} - def __call__(cls, *args, **kwargs): + def __call__(cls, *args, **kwargs): # noqa if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] class ShellEnvironment(metaclass=Singleton): + """An active copy of the current OS environment. + + Allows for easy manipulation of the environment including taking + screenshots (checkpoints) that are stored and can be accessed + later. + """ # Easy definition for the very first checkpoint # when the environment is first created. INITIAL_CHECKPOINT = 0 def __init__(self): + """Inits the local environment with the initial os environment.""" # Add all of our logging to the EnvDict group. self.logger = logging.getLogger(LOGGING_GROUP) @@ -57,6 +67,7 @@ def __init__(self): # These methods manage the singleton, the surrounding environment, and checkpoints. # def import_environment(self): + """Loads the local environment with os environment.""" # Create a complete copy of os.environ self.active_environ = dict() for key, value in os.environ.items(): @@ -79,6 +90,7 @@ def import_environment(self): self.active_environ.pop("PYTHONPATH", None) def export_environment(self): + """Exports enviornment to the OS.""" # Purge all keys that aren't in the export. for key, value in os.environ.items(): if key not in self.active_environ: @@ -95,6 +107,7 @@ def export_environment(self): sys.path = self.active_pypath def log_environment(self): + """Logs the current environment to the logger.""" self.logger.debug("FINAL PATH:") self.logger.debug(", ".join(self.active_path)) @@ -108,6 +121,14 @@ def log_environment(self): self.logger.debug(", ".join(environ_list)) def checkpoint(self): + """Creates a checkpoint in time. + + Checkpoint stores the following: + 1. active_environment + 2. active_path + 3. active_pypath + 4. active_buildvars + """ new_index = len(self.checkpoints) self.checkpoints.append({ 'environ': copy.copy(self.active_environ), @@ -119,6 +140,7 @@ def checkpoint(self): return new_index def restore_checkpoint(self, index): + """Restore a specific checkpoint.""" if index < len(self.checkpoints): check_point = self.checkpoints[index] self.active_environ = copy.copy(check_point['environ']) @@ -132,6 +154,7 @@ def restore_checkpoint(self, index): raise IndexError("Checkpoint %s does not exist" % index) def restore_initial_checkpoint(self): + """Restore the initial checkpoint made.""" self.restore_checkpoint(ShellEnvironment.INITIAL_CHECKPOINT) # @@ -148,23 +171,36 @@ def _internal_set_pypath(self, path_elements): sys.path = self.active_pypath def set_path(self, new_path): + """Set the path. + + Args: + new_path (str): path to override with + """ self.logger.debug("Overriding PATH with new value.") if type(new_path) is str: new_path = list(new_path.split(os.pathsep)) self._internal_set_path(new_path) def set_pypath(self, new_path): + """Set the pypath. + + Args: + new_path (str): path to override with + """ self.logger.debug("Overriding PYTHONPATH with new value.") if type(new_path) is str: new_path = list(new_path.split(os.pathsep)) self._internal_set_pypath(new_path) def append_path(self, path_element): - ''' append to the end of path + """Append to the end of path. + if path_element already exists within path it will be removed from the current location and appended to the end - ''' + Args: + path_element (str): path element to append + """ self.logger.debug("Appending PATH element '%s'." % path_element) if path_element in self.active_path: # remove so we don't have duplicates but we respect the order @@ -173,11 +209,14 @@ def append_path(self, path_element): self._internal_set_path(self.active_path + [path_element]) def insert_path(self, path_element): - '''insert at front of the path + """Insert at front of the path. + if path_element already exists within path it will be removed from the current location and prepended to the front - ''' + Args: + path_element (str): path element to insert + """ self.logger.debug("Inserting PATH element '%s'." % path_element) if path_element in self.active_path: # remove so we don't have duplicates but we respect the order @@ -186,10 +225,14 @@ def insert_path(self, path_element): self._internal_set_path([path_element] + self.active_path) def append_pypath(self, path_element): - ''' append to the end of pypath + """Append to the end of pypath. + if path_element already exists within pypath it will be removed from the current location and appended to the end - ''' + + Args: + path_element (str): path element to append + """ self.logger.debug("Appending PYTHONPATH element '%s'." % path_element) if path_element in self.active_pypath: # remove so we don't have duplicates but we respect the order @@ -198,10 +241,14 @@ def append_pypath(self, path_element): self._internal_set_pypath(self.active_pypath + [path_element]) def insert_pypath(self, path_element): - '''insert at front of the pypath + """Insert at front of the pypath. + if path_element already exists within pypath it will be removed from the current location and prepended to the front - ''' + + Args: + path_element (str): path element to insert + """ self.logger.debug("Inserting PYTHONPATH element '%s'." % path_element) if path_element in self.active_pypath: # remove so we don't have duplicates but we respect the order @@ -210,41 +257,101 @@ def insert_pypath(self, path_element): self._internal_set_pypath([path_element] + self.active_pypath) def replace_path_element(self, old_path_element, new_path_element): - # Generate a new PATH by iterating through the old PATH and replacing - # old_path_element with new_path_element where it is found. + """Replaces the PATH element. + + Generates a new PATH by iterating through the old PATH and replacing + old_path_element with new_path_element where it is found. + + Args: + old_path_element (str): element to replace + new_path_element (str): element to replace with + + """ self.logger.debug("Replacing PATH element {0} with {1}".format(old_path_element, new_path_element)) self._internal_set_path([x if x != old_path_element else new_path_element for x in self.active_path]) def replace_pypath_element(self, old_pypath_element, new_pypath_element): - # Generate a new PYPATH by iterating through the old PYPATH and replacing - # old_pypath_element with new_pypath_element where it is found. + """Replaces the PYPATH element. + + Generates a new PYPATH by iterating through the old PYPATH and replacing + old_pypath_element with new_pypath_element where it is found. + + Args: + old_pypath_element (str): element to replace + new_pypath_element (str): element to replace with + + """ self.logger.debug("Replacing PYPATH element {0} with {1}".format(old_pypath_element, new_pypath_element)) self._internal_set_pypath([x if x != old_pypath_element else new_pypath_element for x in self.active_pypath]) def remove_path_element(self, path_element): - # Generate a new PATH by iterating through the old PYPATH and removing - # path_element if it is found. + """Removes the PATH element. + + Generates a new PATH by iterating through the old PATH and removing + path_element if it is found. + + Args: + path_element (str): path element to remove + """ self.logger.debug("Removing PATH element {0}".format(path_element)) self._internal_set_path([x for x in self.active_path if x != path_element]) def remove_pypath_element(self, pypath_element): - # Generate a new PYPATH by iterating through the old PYPATH and removing - # pypath_element if it is found. + """Removes the PYPATH element. + + Generates a new PYPATH by iterating through the old PYPATH and removing + pypath_element if it is found. + + Args: + pypath_element (str): pypath element to remove + """ self.logger.debug("Removing PYPATH element {0}".format(pypath_element)) self._internal_set_pypath([x for x in self.active_pypath if x != pypath_element]) def get_build_var(self, var_name): + """Gets the build variable. + + Args: + var_name (str): variable to get the value of + Returns: + (obj): value associated with the var_name + """ return self.active_buildvars.GetValue(var_name) def set_build_var(self, var_name, var_data): + """Sets the build var. + + Args: + var_name (str): variable to set the value for + var_data (obj): data to set + + WARNING: Unlike `set_shell_var`, this only sets the variable in the + `VarDict` + """ self.logger.debug( "Updating BUILD VAR element '%s': '%s'." % (var_name, var_data)) self.active_buildvars.SetValue(var_name, var_data, '', overridable=True) def get_shell_var(self, var_name): + """Gets the shell variable. + + Args: + var_name (str): variable to get the value of + + Returns: + (obj): value associated with the var name + """ return self.active_environ.get(var_name, None) def set_shell_var(self, var_name, var_data): + """Sets the shell variable. + + Args: + var_name (str): variable to set the value for + var_data (obj): data to set + + The variable is set both in the `VarDict` and in the OS + """ # Check for the "special" shell vars. if var_name.upper() == 'PATH': self.set_path(var_data) @@ -258,10 +365,20 @@ def set_shell_var(self, var_name, var_data): def GetEnvironment(): + """Returns the environment. + + Returns: + (ShellEnvironment): Singleton class + """ return ShellEnvironment() def GetBuildVars(): + """The current checkpoint buildvar values. + + Returns: + (VarDict): A special dictionary containing build vars + """ # # Tricky! # Define a wrapper class that always forwards commands to the @@ -287,6 +404,7 @@ def __getattr__(self, attrname): def CheckpointBuildVars(): + """Creates a checkpoint [a screenshot in time] of all current build var values.""" global checkpoint_list new_checkpoint = ShellEnvironment().checkpoint() checkpoint_list.append(new_checkpoint) @@ -294,6 +412,7 @@ def CheckpointBuildVars(): def RevertBuildVars(): + """Reverts all build var values to the most recent checkpoint.""" global checkpoint_list if len(checkpoint_list) > 0: last_checkpoint = checkpoint_list.pop() diff --git a/edk2toolext/environment/uefi_build.py b/edk2toolext/environment/uefi_build.py index a0c04ca0a..0863b12a4 100644 --- a/edk2toolext/environment/uefi_build.py +++ b/edk2toolext/environment/uefi_build.py @@ -7,7 +7,11 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code that supports the Tianocore Edk2 build system. +This class is designed to be subclassed by a platform to allow more extensive +and custom behavior. +""" import os import logging @@ -26,8 +30,34 @@ class UefiBuilder(object): + """Object responsible for the full build process. + + The following steps are completed by the `UefiBuilder` and is overridable + by the platform: + + 1. `PlatformPreBuild()` + 2. `UefiBuildPlugins` that implement `do_pre_build()` + 3. `Build()` (should not be overridden) + 4. `UefiBuildPlugins` that implement `do_post_build()` + 5. `PlatformFlashImage()` + + Attributes: + SkipPreBuild (bool): Skip Pre Build or not + SkipPostBuild (bool): Skip Post Build or not + SkipBuild (bool): Skip Build or not + FlashImage (bool): Flash the image not + Clean (bool): Clean the build directory or not + Update Conf (bool): Update the conf or not + env (VarDict): Special dictionary containing build and env vars + mws (MultipleWorkspace): multiple workspace manager + ws (str): Workspace root dir + pp (str): packagespath separated by os.pathsep + Helper (HelperFunctions): object containing registered helper functions + pm (PluginManager): The plugin manager + """ def __init__(self): + """Inits an empty UefiBuilder.""" self.SkipBuild = False self.SkipPreBuild = False self.SkipPostBuild = False @@ -39,7 +69,11 @@ def __init__(self): self.OutputConfig = None def AddPlatformCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' + """Adds command line options to the argparser. + + Args: + parserObj(argparser): argparser object + """ parserObj.add_argument("--SKIPBUILD", "--skipbuild", "--SkipBuild", dest="SKIPBUILD", action='store_true', default=False, help="Skip the build process") parserObj.add_argument("--SKIPPREBUILD", "--skipprebuild", "--SkipPrebuild", dest="SKIPPREBUILD", @@ -64,7 +98,11 @@ def AddPlatformCommandLineOptions(self, parserObj): help='Provide shell variables in a file') def RetrievePlatformCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser''' + """Retrieve command line options from the argparser. + + Args: + args (Namespace): namespace containing gathered args from argparser + """ self.OutputConfig = os.path.abspath(args.OutputConfig) if args.OutputConfig else None self.SkipBuild = args.SKIPBUILD @@ -87,6 +125,7 @@ def RetrievePlatformCommandLineOptions(self, args): self.FlashImage = False def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager): + """Core executable that performs all build steps.""" self.env = shell_environment.GetBuildVars() self.mws = MultipleWorkspace() self.mws.setWs(WorkSpace, PackagesPath) @@ -189,6 +228,11 @@ def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager): return 0 def CleanTree(self, RemoveConfTemplateFilesToo=False): + """Cleans the build directory. + + Args: + RemoveConfTemplateFilesToo (bool): deletes conf files used for building makefiles + """ ret = 0 # loop thru each build target set. edk2_logging.log_progress("Cleaning All Output for Build") @@ -221,11 +265,8 @@ def CleanTree(self, RemoveConfTemplateFilesToo=False): return ret - # - # Build step - # - def Build(self): + """Adds all arguments to the build command and runs it.""" BuildType = self.env.GetValue("TARGET") edk2_logging.log_progress("Running Build %s" % BuildType) @@ -284,6 +325,10 @@ def Build(self): return 0 def PreBuild(self): + """Performs internal PreBuild steps. + + This including calling the platform overridable `PlatformPreBuild()` + """ edk2_logging.log_progress("Running Pre Build") # # Run the platform pre-build steps. @@ -313,6 +358,10 @@ def PreBuild(self): return ret def PostBuild(self): + """Performs internal PostBuild steps. + + This includes calling the platform overridable `PlatformPostBuild()`. + """ edk2_logging.log_progress("Running Post Build") # # Run the platform post-build steps. @@ -344,6 +393,10 @@ def PostBuild(self): return ret def SetEnv(self): + """Performs internal SetEnv steps. + + This includes platform overridable `SetPlatformEnv()` and `SetPlatformEnvAfterTarget(). + """ edk2_logging.log_progress("Setting up the Environment") shell_environment.GetEnvironment().set_shell_var("WORKSPACE", self.ws) shell_environment.GetBuildVars().SetValue("WORKSPACE", self.ws, "Set in SetEnv") @@ -427,6 +480,7 @@ def SetEnv(self): return 0 def FlashRomImage(self): + """Executes platform overridable `PlatformFlashImage()`.""" return self.PlatformFlashImage() # ----------------------------------------------------------------------- @@ -435,42 +489,59 @@ def FlashRomImage(self): @classmethod def PlatformPreBuild(self): + """Perform Platform PreBuild Steps.""" return 0 @classmethod def PlatformPostBuild(self): + """Perform Platform PostBuild Steps.""" return 0 @classmethod def SetPlatformEnv(self): + """Set and read Platform Env variables. + + This is performed before platform files like the DSC and FDF have been parsed. + + TIP: If a platform file (DSC, FDF, etc) relies on a variable set in + the `UefiBuilder`, it must be set here, before the platform files + have been parsed and values have been set. + """ return 0 @classmethod def SetPlatformEnvAfterTarget(self): + """Set and read Platform Env variables after platform files have been parsed.""" return 0 @classmethod def PlatformBuildRom(self): + """Build the platform Rom. + + TIP: Typically called by the platform in PlatformFlashImage. Not called + automatically by the `UefiBuilder`. + """ return 0 @classmethod def PlatformFlashImage(self): + """Flashes the image to the system.""" return 0 @classmethod def PlatformGatedBuildShouldHappen(self): + """Specifies if a gated build should happen.""" return True # ------------------------------------------------------------------------ # HELPER FUNCTIONS # ------------------------------------------------------------------------ # - - # - # Parse the TargetText file and add them as env settings. - # set them so they can be overridden. - # def ParseTargetFile(self): + """Parses the target.txt file and adds values as env settings. + + "Sets them so they can be overriden. + """ if (os.path.isfile(self.mws.join(self.ws, "Conf", "target.txt"))): # parse TargetTxt File logging.debug("Parse Target.txt file") @@ -494,6 +565,10 @@ def ParseTargetFile(self): return 0 def ParseToolsDefFile(self): + """Parses the tools_def.txt file and adds values as env settings. + + "Sets them so they can be overriden. + """ if (os.path.isfile(self.mws.join(self.ws, "Conf", "tools_def.txt"))): # parse ToolsdefTxt File logging.debug("Parse tools_def.txt file") @@ -513,11 +588,12 @@ def ParseToolsDefFile(self): return 0 - # - # Parse the Active platform DSC file. This will get lots of variable info to - # be used in the build. This makes it so we don't have to define things twice - # def ParseDscFile(self): + """Parses the active platform DSC file. + + This will get lots of variable info to be used in the build. This + makes it so we don't have to define things twice. + """ if self.env.GetValue("ACTIVE_PLATFORM") is None: logging.error("The DSC file was not set. Please set ACTIVE_PLATFORM") return -1 @@ -545,13 +621,13 @@ def ParseDscFile(self): return 0 - # - # Parse the Active platform FDF file. This will get lots of variable info to - # be used in the build. This makes it so we don't have to define things twice - # the FDF file usually comes from the Active Platform DSC file so it needs to - # be parsed first. - # def ParseFdfFile(self): + """Parses the active platform FDF file. + + This will get lots of variable info to be used in the build. This makes + it so we don't have to define things twice the FDF file usually comes + from the Active Platform DSC file so it needs to be parsed first. + """ if (self.env.GetValue("FLASH_DEFINITION") is None): logging.debug("No flash definition set") return 0 @@ -577,11 +653,8 @@ def ParseFdfFile(self): return 0 - # - # Function used to set default values for numerous build - # flow control variables - # def SetBasicDefaults(self): + """Sets default values for numerous build control flow variables.""" self.env.SetValue("WORKSPACE", self.ws, "DEFAULT") if (self.pp is not None): self.env.SetValue("PACKAGES_PATH", self.pp, "DEFAULT") diff --git a/edk2toolext/environment/var_dict.py b/edk2toolext/environment/var_dict.py index 3bec95887..101f83873 100644 --- a/edk2toolext/environment/var_dict.py +++ b/edk2toolext/environment/var_dict.py @@ -8,17 +8,36 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""A special overridable dictionary. + +Stores most of the build configuration data and allows extensive config +sharing for the build process, pre-build, and post-build. +""" import logging class EnvEntry(object): + """A single Environment Variable entry for VarDict. + + Attributes: + Value (obj): The value to store in the dictionary + Comment (str): A debug comment specifying where / how the value was set + overridable (bool): If the value can be overwritten in the future + """ + def __init__(self, value, comment, overridable=False): + """Inits an entry with the specified values.""" self.Value = value self.Comment = comment self.Overrideable = overridable def PrintEntry(self, f=None): + """Prints the value. + + Args: + f (:obj:`str`, optional): a file to write to instead of the terminal. + """ print("Value: %s" % self.Value, file=f) print("Comment: %s" % self.Comment, file=f) if (self.Overrideable): @@ -29,6 +48,16 @@ def PrintEntry(self, f=None): # def SetValue(self, value, comment, overridable=False): + """Sets the value of the entry if it os overridable. + + Args: + value (obj): value to set + comment (str): A debug comment specifying where / how the value was set + overridable (bool): If the value can be overwritten in the future + + WARNING: Even if you set a value as overridable=False, another entity can + call `AllowOverride()` and change the value anyway. + """ if (value == self.Value) and (overridable == self.Overrideable): return True @@ -43,22 +72,29 @@ def SetValue(self, value, comment, overridable=False): return True def AllowOverride(self): + """Allows the value to be overwritten in the future.""" self.Overrideable = True return True def GetValue(self): + """Returns the value.""" return self.Value class VarDict(object): + """An overridable dictionary to store build configuration data.""" + def __init__(self): + """Inits an empty VarDict.""" self.Logger = logging.getLogger("EnvDict") self.Dstore = {} # a set of envs def GetEntry(self, key): + """Returns an entry in the Dstore Dict.""" return self.Dstore.get(key.upper()) def __copy__(self): + """Copies data into a new VarDict.""" new_copy = VarDict() new_copy.Logger = self.Logger @@ -72,6 +108,18 @@ def __copy__(self): return new_copy def GetValue(self, k, default=None): + """Gets a value from the variable dictionary that was set during build. + + Note: + Values set in DSC, FDF, and CLI stored as strings + + Args: + k (str): The key the value was stored as + default (varied): default value if key is not present + + Returns: + (varied): The value of the key, if present, else default value + """ if (k is None): logging.debug( "GetValue - Invalid Parameter key is None.") @@ -87,6 +135,19 @@ def GetValue(self, k, default=None): return default def SetValue(self, k, v, comment, overridable=False): + """Sets an environment variable to be used throughout the build. + + Args: + k (str): The key to store the value under + v (varied): The value to store + comment (str): A comment to show where / how the variable was stored. + Useful for debugging + overrideable (bool): Specifies if the variable is allowed to be override + elsewhere in the build + + Returns: + (bool): If the variable was successfully stored or not + """ key = k.upper() en = self.GetEntry(key) value = str(v) @@ -100,6 +161,17 @@ def SetValue(self, k, v, comment, overridable=False): return en.SetValue(value, comment, overridable) def AllowOverride(self, k): + """Forces the key/value pair to be overridable. + + Note: Even if overridable was specifically set to False, + it still allows it. + + Args: + k (str): The key the value was stored as + + Returns: + (bool): if the key existed or not + """ key = k.upper() en = self.GetEntry(key) if (en is not None): @@ -108,18 +180,22 @@ def AllowOverride(self, k): return True return False - # - # function used to get a build var value for given key and buildtype - # - # if BuildType is None - # Build vars are defined by vars that start with BLD_ - # BLD_*_ means all build types - # BLD_DEBUG_ means build of debug type - # BLD_RELEASE_ means build of release type - # etc - # - def GetBuildValue(self, key, BuildType=None): + """Get a build var value for given key and buildtype. + + TIP: Build vars are defined by vars that start with BLD_ + BLD_*_ means all build types + BLD_DEBUG_ means build of debug type + BLD_RELEASE_ means build of release type + etc + + Args: + key (str): The key the value was stored as + BuildType (:obj:`str`, optional): DEBUG/RELEASE + + Returns: + (str): The value of the key, if present, else None + """ rv = None if (BuildType is None): @@ -148,16 +224,22 @@ def GetBuildValue(self, key, BuildType=None): # return value...if not found should return None return rv - # - # function used to get a dictionary for all build vars - # - # Build vars are defined by vars that start with BLD_ - # BLD_*_ means all build types - # BLD_DEBUG_ means build of debug type - # BLD_RELEASE_ means build of release type - # etc - # def GetAllBuildKeyValues(self, BuildType=None): + """Gets a dictionary for all build vars. + + TIP: Build vars are defined by vars that start with BLD_ + BLD_*_ means all build types + BLD_DEBUG_ means build of debug type + BLD_RELEASE_ means build of release type + etc + + Args: + BuildType (:obj:`str`, optional): DEBUG/RELEASE + + Returns: + (dict): all keys, values in the environment which are build keys + + """ returndict = {} if (BuildType is None): BuildType = self.GetValue("TARGET") @@ -187,9 +269,11 @@ def GetAllBuildKeyValues(self, BuildType=None): return returndict def GetAllNonBuildKeyValues(self): - ''' Return a copy of the dictionary of all keys, values - in the environment which are not Build Keys - ''' + """Returns a dict of non Build Key values. + + Return a copy of the dictionary of all keys, values in the environment + which are not Build Keys. + """ returndict = {} # get all the generic build options for key, value in self.Dstore.items(): @@ -198,6 +282,13 @@ def GetAllNonBuildKeyValues(self): return returndict def PrintAll(self, fp=None): + """Prints all variables. + + If fp is not none, writes to a fp also + + Args: + fp (:obj:`str`, optional): file pointer to print to + """ f = None if (fp is not None): f = open(fp, 'a+') diff --git a/edk2toolext/environment/version_aggregator.py b/edk2toolext/environment/version_aggregator.py index a1ddbb1a0..9c8a5ae14 100644 --- a/edk2toolext/environment/version_aggregator.py +++ b/edk2toolext/environment/version_aggregator.py @@ -6,7 +6,11 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Used to facilitate the collection of information. +Used to facilitate the collection of information regarding the tools, +binaries, submodule configuration used in a build. +""" import copy import logging from enum import Enum @@ -15,18 +19,25 @@ class version_aggregator(object): + """Used to facilitate the collection of information. + + Used to facilitate the collection of information regarding the tools, + binaries, submodule configuration used in a build. + """ def __init__(self): + """Inits an empty verion aggregator.""" super(version_aggregator, self).__init__() self._Versions = {} self._logger = logging.getLogger("version_aggregator") def ReportVersion(self, key, value, versionType, path=None): - """ - Report the version of something. + """Report the version of something. - key -- The name of what you are reporting. - value -- The value of what you are reporting. - versionType -- The method of categorizing what is being reported. See VersionTypes for details. + Args: + key (str): the name of what you are reporting. + value (str): The value of what you are reporting. + versionType (str): The method of categorizing what is being reported. See VersionTypes for details. + path (:obj:'str', optional): the associated path. """ if key in self._Versions: old_version = self._Versions[key] @@ -49,7 +60,7 @@ def ReportVersion(self, key, value, versionType, path=None): self._logger.debug("version_aggregator logging version: {0}".format(str(self._Versions[key]))) def Print(self): - """ Prints out the current information from the version aggregator """ + """Prints out the current information from the version aggregator.""" for version_key in self._Versions: version = self._Versions[version_key] print(f"{version['type']} - {version['name']}: {version['version']}") @@ -57,22 +68,23 @@ def Print(self): print("VERSION AGGREGATOR IS EMPTY") def GetAggregatedVersionInformation(self): - """ - Returns a copy of the aggregated information. - """ + """Returns a copy of the aggregated information.""" return copy.deepcopy(self._Versions) def Reset(self): + """Resets all versions.""" self._Versions = {} class VersionTypes(Enum): - """ - COMMIT is for the commit hash of a repository. - BINARY is for a pre-packaged binary that is distributed with a version number. - TOOL is for recording the version number of a tool that was used during the build process. - INFO is for recording miscellaneous information. - PIP is for recording a python pip package. + """Enumerator representing the different version types for recording. + + Attributes: + COMMIT: the commit hash of a repository. + BINARY: pre-packaged binary that is distributed with a version number. + TOOL: the version number of a tool that was used during the build process. + INFO: miscellaneous information. + PIP: a python pip package. """ TOOL = 1 COMMIT = 2 @@ -82,9 +94,7 @@ class VersionTypes(Enum): def GetVersionAggregator(): - """ - Returns a singleton instance of this class for global use. - """ + """Returns a singleton instance of this class for global use.""" global VERSION_AGGREGATOR if VERSION_AGGREGATOR is None: @@ -95,7 +105,5 @@ def GetVersionAggregator(): def ResetVersionAggregator(): - ''' - Resets the version Aggregator singleton - ''' + """Resets the version Aggregator singleton.""" GetVersionAggregator().Reset() diff --git a/edk2toolext/image_validation.py b/edk2toolext/image_validation.py index 7c64c9ba3..da2d2828d 100644 --- a/edk2toolext/image_validation.py +++ b/edk2toolext/image_validation.py @@ -6,6 +6,10 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""This tool allows the user to validate a PE/COFF file against specific requirements. + +It also provides CLI functions to set and clear the nx_compat flag. +""" import os from pefile import PE, SECTION_CHARACTERISTICS, MACHINE_TYPE, SUBSYSTEM_TYPE @@ -21,18 +25,22 @@ def has_characteristic(data, mask): + """Checks if data has a specific mask.""" return ((data & mask) == mask) def set_bit(data, bit): + """Sets a specific bit.""" return data | (1 << bit) def clear_bit(data, bit): + """Clears a specific bit.""" return data & ~(1 << bit) def set_nx_compat_flag(pe): + """Sets the nx_compat flag to 1 in the PE/COFF file.""" dllchar = pe.OPTIONAL_HEADER.DllCharacteristics dllchar = set_bit(dllchar, 8) # 8th bit is the nx_compat_flag pe.OPTIONAL_HEADER.DllCharacteristics = dllchar @@ -41,6 +49,7 @@ def set_nx_compat_flag(pe): def get_nx_compat_flag(pe): + """Reads the nx_compat flag of the PE/COFF file.""" dllchar = pe.OPTIONAL_HEADER.DllCharacteristics if has_characteristic(dllchar, 256): # 256 (8th bit) is the mask @@ -52,6 +61,7 @@ def get_nx_compat_flag(pe): def clear_nx_compat_flag(pe): + """Sets the nx_compat flag to 0 in the PE/COFF file.""" dllchar = pe.OPTIONAL_HEADER.DllCharacteristics dllchar = clear_bit(dllchar, 8) # 8th bit is the nx_compat_flag pe.OPTIONAL_HEADER.DllCharacteristics = dllchar @@ -60,6 +70,12 @@ def clear_nx_compat_flag(pe): def fill_missing_requirements(default, target): + """Fills missing requirements for a specific test config with default config. + + As an example, If there are specific requirements for an APP PE/COFF, those + will take override the default requirements.Any requirement not specified + by the APP config will be filled by the DEFAULT config. + """ for key in default: if key not in target: target[key] = default[key] @@ -67,6 +83,7 @@ def fill_missing_requirements(default, target): class Result: + """Test results.""" PASS = '[PASS]' WARN = '[WARNING]' SKIP = '[SKIP]' @@ -74,24 +91,38 @@ class Result: class TestInterface: - + """Interface for creating tests to execute on parsed PE/COFF files.""" def name(self): - """Returns the name of the test""" + """Returns the name of the test. + + "WARNING: Implement in a subclass. + """ raise NotImplementedError("Must Override Test Interface") def execute(self, pe, config_data): - """ - Executes the test + """Executes the test on the pefile. + + Arguments: + pe (pefile): a parsed PE/COFF image file + config_data (dict): config data for the test + + Returns: + (Result): SKIP, WARN, FAIL, PASS - @param pe: The parser pefile - @param config_data: Configuration data for the specific target machine - and profile + WARNING: Implement in a subclass. """ raise NotImplementedError("Must Override Test Interface") class TestManager(object): + """Manager responsible for executing all tests on all parsed PE/COFF files.""" def __init__(self, config_data=None): + """Inits the TestManager with configuration data. + + Args: + config_data (dict, optional): the configuration data, loads default + data if not provided. + """ self.tests = [] if config_data: self.config_data = config_data @@ -213,40 +244,41 @@ def __init__(self, config_data=None): } def add_test(self, test): - """ - Adds a test to the test manager. Will be executed in the order added + """Adds a test to the test manager. - @param test: [Test(TestInterface)] A class that inherits and overrides - the TestInterface class + Will be executed in the order added. + + Args: + test (TestInterface): A subclasses of the TestInterface """ self.tests.append(test) def add_tests(self, tests): - """ - Adds multiple test to the test manager. Tests will be executed in the - order added. + """Adds multiple test to the test manager. - @param test: [List[Test(TestInterface)]] A list of classes that - inherits and overrides the TestInterface class + Tests will be executed in the order added. + + Args: + tests (List[TestInterface]): A list of subclasses of the TestInterface """ self.tests.extend(tests) def run_tests(self, pe, profile="DEFAULT"): - """ - Runs all tests that have been added to the test manager. Tests will be - executed in the order added - - @param pe : [PE] The parsed pe - @param target_info: [Dict] A Dict that contains MACHINE_TYPE and - PROFILE information. If MachineType is not present, it will be - pulled from the parsed pe, however the user must provide the Module - Type - - @return Result.PASS : All tests passed successfully (including warnings) - @return Result.SKIP : There is no information in the config file for the target and fv file type - @return Result.ERROR: At least one test failed. Error messages can be found in the log - """ + """Runs all tests that have been added to the test manager. + + Tests will be executed in the order added + Args: + pe (pefile): The parsed pe + target_info (dict): MACHINE_TYPE and PROFILE information. If + MachineType is not present, it will be pulled from the parsed + pe, however the user must provide the ModuleType + + Returns: + (Result.PASS): All tests passed successfully (including warnings) + (Result.SKIP): There is no information in the config file for the target and fv file type + (Result.FAIL): At least one test failed. Error messages can be found in the log + """ # Catch any invalid profiles machine_type = MACHINE_TYPE[pe.FILE_HEADER.Machine] if not self.config_data[machine_type].get(profile): @@ -291,15 +323,13 @@ def run_tests(self, pe, profile="DEFAULT"): # TESTS START # ########################### class TestWriteExecuteFlags(TestInterface): - """ - Test: Section data / code separation verification + """Section data / code separation verification Test. - Detailed Description: - This test ensures that each section of the binary is not both - write-able and execute-able. Sections can only be one or the other - (or neither).This test is done by iterating over each section and - checking the characteristics label for the Write Mask (0x80000000) - and Execute Mask (0x20000000). + This test ensures that each section of the binary is not both + write-able and execute-able. Sections can only be one or the other + (or neither).This test is done by iterating over each section and + checking the characteristics label for the Write Mask (0x80000000) + and Execute Mask (0x20000000). Output: @Success: Only one (or neither) of the two masks (Write, Execute) are @@ -314,9 +344,19 @@ class TestWriteExecuteFlags(TestInterface): """ def name(self): + """Returns the name of the test.""" return 'Section data / code separation verification' def execute(self, pe, config_data): + """Executes the test on the pefile. + + Arguments: + pe (pefile): a parsed PE/COFF image file + config_data (dict): config data for the test + + Returns: + (Result): SKIP, WARN, FAIL, PASS + """ target_requirements = config_data["TARGET_REQUIREMENTS"] if target_requirements.get("DATA_CODE_SEPARATION", False) is False: @@ -333,13 +373,11 @@ def execute(self, pe, config_data): class TestSectionAlignment(TestInterface): - """ - Test: Section alignment verification + """Section alignment verification Test. - Detailed Description: - Checks the section alignment of the binary by accessing the optional - header, then the section alignment. This value must meet the - requirements specified in the config file. + Checks the section alignment of the binary by accessing the optional + header, then the section alignment. This value must meet the + requirements specified in the config file. Output: @Success: Image alignment meets the requirement specified in the @@ -349,15 +387,26 @@ class TestSectionAlignment(TestInterface): @Skip: No Alignment requirements specified in the config file @Fail: Image alignment does not meet the requirements specified in the config file + Possible Solution: Update the section alignment of the binary to match the requirements specified in the config file """ def name(self): + """Returns the name of the test.""" return 'Section alignment verification' def execute(self, pe, config_data): + """Executes the test on the pefile. + + Arguments: + pe (pefile): a parsed PE/COFF image file + config_data (dict): config data for the test + + Returns: + (Result): SKIP, WARN, FAIL, PASS + """ target_requirements = config_data["TARGET_REQUIREMENTS"] target_info = config_data["TARGET_INFO"] @@ -405,13 +454,11 @@ def execute(self, pe, config_data): class TestSubsystemValue(TestInterface): - """ - Test: Subsystem type verification + """Subsystem type verification Test. - Detailed Description: - Checks the subsystem value by accessing the optional header, then - subsystem value. This value must match one of the allowed subsystem - described in the config file + Checks the subsystem value by accessing the optional header, then + subsystem value. This value must match one of the allowed subsystem + described in the config file Output: @Success: Subsystem type found in the optional header matches one of @@ -426,9 +473,19 @@ class TestSubsystemValue(TestInterface): """ def name(self): + """Returns the name of the test.""" return 'Subsystem type verification' def execute(self, pe, config_data): + """Executes the test on the pefile. + + Arguments: + pe (pefile): a parsed PE/COFF image file + config_data (dict): config data for the test + + Returns: + (Result): SKIP, WARN, FAIL, PASS + """ target_requirements = config_data["TARGET_REQUIREMENTS"] subsystems = target_requirements.get("ALLOWED_SUBSYSTEMS") @@ -461,10 +518,8 @@ def execute(self, pe, config_data): ########################### -# -# Command Line Interface configuration -# def get_cli_args(args): + """Adds CLI arguments for using the image validation tool.""" parser = argparse.ArgumentParser(description='A Image validation tool for memory mitigation') parser.add_argument('-i', '--file', @@ -502,6 +557,7 @@ def get_cli_args(args): def main(): + """Main entry point into the image validation tool.""" # setup main console as logger logger = logging.getLogger('') logger.setLevel(logging.INFO) diff --git a/edk2toolext/invocables/__init__.py b/edk2toolext/invocables/__init__.py index 0a69010b8..a7e3a2a8b 100644 --- a/edk2toolext/invocables/__init__.py +++ b/edk2toolext/invocables/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa diff --git a/edk2toolext/invocables/edk2_ci_build.py b/edk2toolext/invocables/edk2_ci_build.py index aa98121a0..f51550791 100644 --- a/edk2toolext/invocables/edk2_ci_build.py +++ b/edk2toolext/invocables/edk2_ci_build.py @@ -7,6 +7,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code that supports CI/CD via the ci_build invocable. + +Contains a CIBuildSettingsManager that must be subclassed in a build settings +file. This provides platform specific information to Edk2CiBuild invocable +while allowing the invocable itself to remain platform agnostic. +""" import os import sys import logging @@ -24,29 +30,68 @@ class CiBuildSettingsManager(MultiPkgAwareSettingsInterface): - ''' Platform settings will be accessed through this implementation. ''' + """Platform specific settings for Edk2CiBuild. + + Provide information necessary for `stuart_ci_build.exe` or + `edk2_ci_build.py` to successfully execute. + + Example: Example: Overriding CiBuildSettingsManager + ```python + from edk2toolext.invocables.edk2_ci_build import CiBuildSettingsManager + import yaml + class CiManager(CiBuildSettingsManager): + def GetDependencies(self): + return { + "Path": "/Common/MU", + "Url": "https://github.com/Microsoft/mu_tiano_plus.git" + } + ``` + """ def GetName(self) -> str: - ''' Get the name of the repo, platform, or product being build by CI ''' + """Get the name of the repo, platform, or product being build by CI. + + TIP: Required Override in a subclass + + Returns: + (str): repo, platform, product + """ raise NotImplementedError() def GetPluginSettings(self) -> Dict[str, Any]: - ''' Implement in subclass to pass dictionary of settings for individual plugins ''' + """Provide a dictionary of global settings for individual plugins. + + TIP: Optional Override in a subclass + + WARNING: + This sets the global plugin configurations. Edk2CiBuild automatically searches for, + and loads, the package ci settings file if it exists. This file will override these + settings. This file must be located at the base of the package named .ci.yaml. + + Ex: EmbeddedPkg/EmbeddedPkg.ci.yaml. + + Returns: + (Dict[str, Any]): plugin settings + """ return {} class Edk2CiBuild(Edk2MultiPkgAwareInvocable): - ''' Invocable supporting an iterative multi-package build and test process - leveraging CI build plugins - ''' + """Invocable supporting an iterative multi-package build and test process leveraging CI build plugins.""" def GetSettingsClass(self): + """Returns the CiBuildSettingsManager class. + + WARNING: CiBuildSettingsManager must be subclassed in your platform settings file. + """ return CiBuildSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename (CI_BUILDLOG) of where the logs for the Edk2CiBuild invocable are stored in.""" return "CI_BUILDLOG" def Go(self): + """Executes the core functionality of the Edk2CiBuild invocable.""" log_directory = os.path.join(self.GetWorkspaceRoot(), self.GetLoggingFolderRelativeToRoot()) Edk2CiBuild.collect_python_pip_info() @@ -220,10 +265,14 @@ def Go(self): @staticmethod def merge_config(gbl_config, pkg_config, descriptor={}): - ''' Merge two configurations. One global and one specific - to the package to create the proper config for a plugin - to execute. - ''' + """Merge two configurations. + + One global and one specificto the package to create the proper config + for a plugin to execute. + + Returns: + (dict): Dictionary of config settings + """ plugin_name = "" config = dict() if "module" in descriptor: @@ -244,4 +293,5 @@ def merge_config(gbl_config, pkg_config, descriptor={}): def main(): + """Entry point to invoke Edk2CiBuild.""" Edk2CiBuild().Invoke() diff --git a/edk2toolext/invocables/edk2_ci_setup.py b/edk2toolext/invocables/edk2_ci_setup.py index 285285c43..1507919df 100644 --- a/edk2toolext/invocables/edk2_ci_setup.py +++ b/edk2toolext/invocables/edk2_ci_setup.py @@ -6,6 +6,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code that supports CI/CD via the ci_setup invocable. + +Contains a CISetupSettingsManager that must be subclassed in a build settings +file. This provides platform specific information to Edk2CiSetup invocable +while allowing the invocable itself to remain platform agnostic. +""" import os import logging @@ -15,13 +21,31 @@ class CiSetupSettingsManager(MultiPkgAwareSettingsInterface): - ''' Platform settings will be accessed through this implementation. ''' + """Platform specific settings for Edk2CiSetup. + + Provide information necessary for `stuart_ci_setup.exe` or + `edk2_ci_setup.py` to successfully execute. + + Example: Example: Overriding CiSetupSettingsManager + ```python + from edk2toolext.invocables.edk2_ci_setup import CiSetupSettingsManager + class CiManager(CiSetupSettingsManager): + def GetDependencies(self): + return { + "Path": "/Common/MU", + "Url": "https://github.com/Microsoft/mu_tiano_plus.git" + } + ``` + """ def GetDependencies(self): - ''' Return Git Repository Dependendencies + """Get any Git Repository Dependendencies. + This list of repositories will be resolved during the setup step. - Return an iterable of dictionary objects with the following fields + TIP: Optional Override in subclass + + TIP: Return an iterable of dictionary objects with the following fields { Path: Workspace relative path Url: Url of git repo @@ -30,13 +54,17 @@ def GetDependencies(self): Full: Boolean to do shallow or Full checkout. (default is False) ReferencePath: Workspace relative path to git repo to use as "reference" } - ''' + """ return [] class Edk2CiBuildSetup(Edk2MultiPkgAwareInvocable): + """Invocable supporting an iterative multi-package build and test process leveraging CI build plugins. + Edk2CiBuildSetup sets up the necessary environment for Edk2CiBuild by preparing all necessary submodules. + """ def AddCommandLineOptions(self, parser): + """Adds command line arguments to Edk2CiBuild.""" parser.add_argument('-ignore', '--ignore-git', dest="git_ignore", action="store_true", help="Whether to ignore errors in the git cloning process", default=False) parser.add_argument('--omnicache', '--reference', dest='omnicache_path', @@ -48,7 +76,7 @@ def AddCommandLineOptions(self, parser): super().AddCommandLineOptions(parser) def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser.""" self.git_ignore = args.git_ignore self.git_force = args.git_force self.git_update = args.git_update @@ -60,16 +88,22 @@ def RetrieveCommandLineOptions(self, args): super().RetrieveCommandLineOptions(args) def GetVerifyCheckRequired(self): - ''' Will not verify environment ''' + """Will not verify environment.""" return False def GetSettingsClass(self): + """Returns the CiSetupSettingsManager class. + + WARNING: CiSetupSettingsManager must be subclassed in your platform settings file. + """ return CiSetupSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename (CISETUP) of where the logs for the Edk2CiBuild invocable are stored in.""" return "CISETUP" def Go(self): + """Executes the core functionality of the Edk2CiSetup invocable.""" setup_dependencies = self.PlatformSettings.GetDependencies() logging.debug(f"Dependencies list {setup_dependencies}") repos = repo_resolver.resolve_all(self.GetWorkspaceRoot(), @@ -83,4 +117,5 @@ def Go(self): def main(): + """Entry point invoke Edk2CiBuild.""" Edk2CiBuildSetup().Invoke() diff --git a/edk2toolext/invocables/edk2_multipkg_aware_invocable.py b/edk2toolext/invocables/edk2_multipkg_aware_invocable.py index 814bae7db..1c0d75df7 100644 --- a/edk2toolext/invocables/edk2_multipkg_aware_invocable.py +++ b/edk2toolext/invocables/edk2_multipkg_aware_invocable.py @@ -10,71 +10,166 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""An intermediate class that supports a multi-package aware invocable process. + +Provides two main classes, the MultiPkgAwareSettingsInterface and the +Edk2MultiPkgAwareInvocable that act as an intermediate class that other +invocables that require a multi-package aware invocable process. These classes +should only be subclassed if a new invocable is being developed. Any +Edk2MultiPkgAwareInvocable should be platform agnostic and work for any +platform. Platform specific data is provided via the +MultiPkgAwareSettingsInterface +""" from edk2toolext.edk2_invocable import Edk2Invocable, Edk2InvocableSettingsInterface class MultiPkgAwareSettingsInterface(Edk2InvocableSettingsInterface): - ''' Settings to support Multi-Pkg functionality. - This is an interface definition only - to show which functions are required to be implemented - and which functions can be implemented. - ''' + """Settings to support Multi-Pkg functionality. + + This is an interface definition only to show which functions are required + to be implemented and which functions can be implemented. + + Example: Overriding MultiPkgAwareSettingsInterface + ``` python + import os + import logging + import argparse + from typing import Iterable, Tuple + from edk2toolext.edk2_multipkg_aware_invocable import MultiPkgAwareSettingsInterface + class NewInvocableSettingsManager(MultiPkgAwareSettingsInterface): + + def GetPackagesSupported(self): + return ("PlatformPkg",) + + def GetArchitecturesSupported(self): + return ("IA32","X64) + + def GetTargetsSupported(self): + return ("TARGET", "RELEASE") + + def SetPackages(self, list_of_requested_packages): + + if len(filter(lambda pkg: pkg in self.GetPackagesSupported(), list_of_requested_packages)) != + len(list_of_requested_packages): + raise Exception("Requested Packages contains unsupported Package") + else: + self.pkgs = list_of_requested_packages + + def SetArchitectures(self, list_of_requested_architectures): + if list_of_requested_architectures != self.GetPackagesSupported(): + raise Exception("Only Support IA32,X64 combination") + def SetTargets(self, list_of_requested_targets): + if list_of_requested_targets != self.GetArchitecturesSupported(): + raise Exception("Only Support "TARGET", "RELEASE combination") + ``` + + Warning: + This interface should not be subclassed directly unless creating a new invocable type. Override these + methods as a part of other subclasses invocable settings managers such as SetupSettingsManager, etc. + """ # ####################################################################################### # # Supported Values and Defaults # # ####################################################################################### # def GetPackagesSupported(self): - ''' return iterable of edk2 packages supported by this build. - These should be edk2 workspace relative paths ''' + """Returns an iterable of edk2 packages supported by this build. + + TIP: Required Override in a subclass + + Returns: + (Iterable): edk2 packages + + Note: + packages should be relative to workspace or package path + """ raise NotImplementedError() def GetArchitecturesSupported(self): - ''' return iterable of edk2 architectures supported by this build ''' + """Returns an iterable of edk2 architectures supported by this build. + + TIP: Required Override in a subclass + + Returns: + (Iterable): architectures (X64, I32, etc.) + """ raise NotImplementedError() def GetTargetsSupported(self): - ''' return iterable of edk2 target tags supported by this build ''' + """Returns an iterable of edk2 target tags supported by this build. + + TIP: Required Override in a subclass + + Returns: + (Iterable): targets (DEBUG, RELEASE, etc) + """ raise NotImplementedError() # ####################################################################################### # # Verify and Save requested Config # # ####################################################################################### # def SetPackages(self, list_of_requested_packages): - ''' Confirm the requests package list is valid and configure SettingsManager - to build only the requested packages. + """Confirms the requested package list is valid. + + TIP: Optional Override in a subclass - Raise Exception if a requested_package is not supported - ''' + Args: + list_of_requested_packages (list[str]): packages to be built + + Raises: + Exception: A requested package is not supported + """ pass def SetArchitectures(self, list_of_requested_architectures): - ''' Confirm the requests architecture list is valid and configure SettingsManager - to run only the requested architectures. + """Confirms the requested architecture list is valid. + + TIP: Optional Override in a subclass - Raise Exception if a requested_architecture is not supported - ''' + Args: + list_of_requested_architectures (list[str]): architectures to be built + + Raises: + Exception: A requested architecture is not supported + """ pass def SetTargets(self, list_of_requested_target): - ''' Confirm the requests target list is valid and configure SettingsManager - to run only the requested targets. + """Confirms the requested target list is valid. + + TIP: Optional Override in a subclass - Raise Exception if a requested_target is not supported - ''' + Args: + list_of_requested_targets (list[str]): targets to use + + Raises: + Exception: A requested target is not supported + """ pass class Edk2MultiPkgAwareInvocable(Edk2Invocable): - ''' Base class for Multi-Pkg aware invocable ''' + """Base class for Multi-Pkg aware invocable. + + Attributes: + requested_architecture_list (list): requested architectures to build + requested_package_list (list): requested packages to build + requested_target_list (list): requested targets to use + + TIP: Checkout Edk2Invocable Attributes + to find any additional attributes that might exist. + + WARNING: This invocable should only be subclassed if creating a new invocable + """ def __init__(self): + """Initializes the Invocable.""" self.requested_architecture_list = [] self.requested_package_list = [] self.requested_target_list = [] super().__init__() def AddCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' + """Adds command line options to the argparser.""" # This will parse the packages that we are going to update parserObj.add_argument('-p', '--pkg', '--pkg-dir', dest='packageList', type=str, help='Optional - A package or folder you want to update (workspace relative).' @@ -86,7 +181,7 @@ def AddCommandLineOptions(self, parserObj): help="Optional - CSV of targets requested to update. Example: -t DEBUG,NOOPT") def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser .""" packageListSet = set() for item in args.packageList: # Parse out the individual packages item_list = item.split(",") @@ -107,9 +202,7 @@ def RetrieveCommandLineOptions(self, args): self.requested_target_list = [] def InputParametersConfiguredCallback(self): - ''' This function is called once all the input parameters are collected - and can be used to initialize environment - ''' + """Initializes the environment once input parameters are collected.""" if (len(self.requested_package_list) == 0): self.requested_package_list = list(self.PlatformSettings.GetPackagesSupported()) self.PlatformSettings.SetPackages(self.requested_package_list) diff --git a/edk2toolext/invocables/edk2_platform_build.py b/edk2toolext/invocables/edk2_platform_build.py index 78d772840..5c4c1b8fc 100644 --- a/edk2toolext/invocables/edk2_platform_build.py +++ b/edk2toolext/invocables/edk2_platform_build.py @@ -6,6 +6,13 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Invocable class that does a build. + +Contains a BuildSettingsManager that must be subclassed in a build settings +file, along with a UefiBuilder subclass. This provides platform specific +information to the Edk2PlatformBuild invocable while allowing the invocable +itself to remain platform agnostic. +""" import os import sys import logging @@ -20,19 +27,36 @@ class BuildSettingsManager(Edk2InvocableSettingsInterface): - ''' Platform settings will be accessed through this implementation. ''' + """Platform specific settings for Edk2PlatformBuild. + + Provides information necessary for `stuart_build.exe` + or `edk2_platform_build.py` to successfully execute. + + Example: Example: Overriding BuildSettingsManager + ```python + from edk2toolext.invocables.edk2_platform_build import BuildSettingsManager + class PlatformManager(BuildSettingsManager): + def GetName(self) -> str: + return "QemuQ35" + ``` + """ + + def GetName(self) -> str: + """Get the name of the repo, platform, or product being build. - def GetName(self): - ''' Get the name of the repo, platform, or product being build ''' + TIP: Optional Override in subclass + + Returns: + (str): Name of the repo, platform, or product + """ return None class Edk2PlatformBuild(Edk2Invocable): - ''' Imports UefiBuilder and calls go ''' + """Invocable that performs some environment setup,Imports UefiBuilder and calls go.""" def AddCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' - + """Adds command line options to the argparser.""" # PlatformSettings could also be a subclass of UefiBuilder, who knows! if isinstance(self.PlatformSettings, UefiBuilder): self.PlatformBuilder = self.PlatformSettings @@ -46,20 +70,25 @@ def AddCommandLineOptions(self, parserObj): self.PlatformBuilder.AddPlatformCommandLineOptions(parserObj) def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser.""" self.PlatformBuilder.RetrievePlatformCommandLineOptions(args) def GetSettingsClass(self): - ''' Providing BuildSettingsManager ''' + """Returns the BuildSettingsManager class. + + WARNING: CiSetupSettingsManager must be subclassed in your platform settings file. + """ return BuildSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename of where the logs for the Edk2PlatformBuild invocable are stored in.""" name = self.PlatformSettings.GetName() if name is not None: return f"BUILDLOG_{name}" return "BUILDLOG" def Go(self): + """Executes the core functionality of the Edk2PlatformBuild invocable.""" logging.info("Running Python version: " + str(sys.version_info)) Edk2PlatformBuild.collect_python_pip_info() @@ -110,4 +139,5 @@ def Go(self): def main(): + """Entry point invoke Edk2PlatformBuild.""" Edk2PlatformBuild().Invoke() diff --git a/edk2toolext/invocables/edk2_pr_eval.py b/edk2toolext/invocables/edk2_pr_eval.py index e2673a615..3690094a5 100644 --- a/edk2toolext/invocables/edk2_pr_eval.py +++ b/edk2toolext/invocables/edk2_pr_eval.py @@ -6,7 +6,14 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Invocable that checks the diff between a branch and head. +Identifies any packages that needs to be build. + +Contains a PrEvalSettingsManager that must be subclassed in a build settings +file. This provides platform specific information to Edk2PrEval invocable +while allowing the invocable itself to remain platform agnostic. +""" import os import logging from io import StringIO @@ -21,29 +28,70 @@ class PrEvalSettingsManager(MultiPkgAwareSettingsInterface): - ''' Platform settings will be accessed through this implementation. ''' + """Platform specific Settings for Edk2PrEval. + + provide information necessary for `stuart_pr_eval.exe` or + `edk2_pr_eval.py` to successfully execute. + + Example: Example: Overriding PrEvalSettingsManager + ```python + from edk2toolext.invocables.edk2_pr_eval import PrEvalSettingsManager + class PrEvalManager(PrEvalSettingsManager): + def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list: + filtered_packages = [] + for file in changedFilesList: + for package in potentialPackagesList: + if package.startswith(potentialPackagesList): + filtered_packages.append(package) + + return list(set(filtered_packages)) + + def GetPlatformDscAndConfig(self) -> tuple: + return None + ``` + """ def FilterPackagesToTest(self, changedFilesList: list, potentialPackagesList: list) -> list: - ''' Filter potential packages to test based on changed files. ''' + """Filter potential packages to test based on changed files. + + TIP: Optional Override in a subclass + Arguments: + changedFilesList (list): files changed in this PR + potentialPackagesList (list): packages from `GetPackagesSupported()` or from command line + option -p, --pkg, --pkg-dir from `Edk2MultiPkgAwareInvocable` + + Returns: + (list): filtered packages to test + Note: + Default implementation does zero filtering + """ # default implementation does zero filtering. return potentialPackagesList def GetPlatformDscAndConfig(self) -> tuple: - ''' If a platform desires to provide its DSC then Policy 4 will evaluate if + """Provide a platform dsc and config. + + If a platform desires to provide its DSC then Policy 4 will evaluate if any of the changes will be built in the dsc. - The tuple should be (, ) - ''' + TIP: Optional Override in a subclass + + Returns: + (tuple): (workspace relative path to dsc file, input dictionary of dsc key value pairs) + """ return None class Edk2PrEval(Edk2MultiPkgAwareInvocable): - ''' Evaluate the changes and determine what packages of the supplied packages should - be tested based on impact from the changes ''' + """Invocable to determine what packages should be tested. + + Evaluate the changes and determine what packages of the supplied packages should + be tested based on impact from the changes + """ def AddCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' + """Adds command line options to the argparser.""" parserObj.add_argument("--pr-target", dest='pr_target', type=str, default=None, help="PR Branch Target. Allows build optimizations for pull request" " validation based on files changed. If a package doesn't need testing then it will" @@ -59,25 +107,29 @@ def AddCommandLineOptions(self, parserObj): super().AddCommandLineOptions(parserObj) def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser.""" self.pr_target = args.pr_target self.output_csv_format_string = args.output_csv_format_string self.output_count_format_string = args.output_count_format_string super().RetrieveCommandLineOptions(args) def GetVerifyCheckRequired(self): - ''' Will not call self_describing_environment.VerifyEnvironment because it might not be set up yet ''' + """Will not call self_describing_environment.VerifyEnvironment because it might not be set up yet.""" return False def GetSettingsClass(self): - ''' Providing PrEvalSettingsManager ''' + """Returns the PrEvalSettingsManager class. + + WARNING: PrEvalSettingsManager must be subclassed in your platform settings file. + """ return PrEvalSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename (PREVALLOG) of where the logs for the Edk2CiBuild invocable are stored in.""" return "PREVALLOG" def Go(self): - + """Executes the core functionality of the Edk2CiBuild invocable.""" # create path obj for resolving paths. Since PR eval is run early to determine if a build is # impacted by the changes of a PR we must ignore any packages path that are not valid due to # not having their submodule or folder populated. @@ -115,6 +167,14 @@ def Go(self): return 0 def get_packages_to_build(self, possible_packages: list) -> dict: + """Returns a dictionary of packages to build. + + Arguments: + possible_packages: list of possible packages + + Returns: + (dict): filtered packages to build + """ self.parsed_dec_cache = {} (rc, files) = self._get_files_that_changed_in_this_pr(self.pr_target) if rc != 0: @@ -234,7 +294,7 @@ def get_packages_to_build(self, possible_packages: list) -> dict: return packages_to_build def _get_unique_module_infs_changed(self, files: list): - '''return a list of edk2 relative paths to modules infs that have changed files''' + """Returns a list of edk2 relative paths to modules infs that have changed files.""" modules = [] for f in files: @@ -255,7 +315,7 @@ def _get_unique_module_infs_changed(self, files: list): return modules def _does_pkg_depend_on_package(self, package_to_eval: str, support_package: str) -> bool: - ''' return if any module in package_to_eval depends on public files defined in support_package''' + """Return if any module in package_to_eval depends on public files defined in support_package.""" # get filesystem path of package_to_eval abs_pkg_path = self.edk2_path_obj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(package_to_eval) @@ -277,10 +337,11 @@ def _does_pkg_depend_on_package(self, package_to_eval: str, support_package: str return False def _get_files_that_changed_in_this_pr(self, base_branch) -> tuple: - ''' Get all the files that changed in this pr. - Return the error code and list of files - ''' + """Get all the files that changed in this pr. + Returns: + (int, list[str]): error code, list of files + """ # get file differences between pr and base output = StringIO() cmd_params = f"diff --name-only HEAD..{base_branch}" @@ -302,7 +363,7 @@ def _get_files_that_changed_in_this_pr(self, base_branch) -> tuple: return (0, files) def _parse_dec_for_package(self, path_to_package): - ''' find DEC for package and parse it''' + """Find DEC for package and parse it.""" # find DEC file path = None try: @@ -331,7 +392,7 @@ def _parse_dec_for_package(self, path_to_package): return dec def _is_public_file(self, filepath): - ''' return if file is a public files ''' + """Returns if file is a public files.""" fp = filepath.replace("\\", "/") # make consistant for easy compare self.logger.debug("Is public: " + fp) @@ -363,8 +424,7 @@ def _is_public_file(self, filepath): return False def _walk_dir_for_filetypes(self, extensionlist, directory, ignorelist=None): - ''' Walks a directory for all items ending in certain extension ''' - + """Walks a directory for all items ending in certain extension.""" if not isinstance(extensionlist, list): raise ValueError("Expected list but got " + str(type(extensionlist))) @@ -405,4 +465,5 @@ def _walk_dir_for_filetypes(self, extensionlist, directory, ignorelist=None): def main(): + """Entry point to invoke Edk2PrEval.""" Edk2PrEval().Invoke() diff --git a/edk2toolext/invocables/edk2_setup.py b/edk2toolext/invocables/edk2_setup.py index c2a4df482..4f4508985 100644 --- a/edk2toolext/invocables/edk2_setup.py +++ b/edk2toolext/invocables/edk2_setup.py @@ -5,7 +5,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Code that updates required submodules. +Contains a SetupSettingsManager that must be subclassed in a build settings +file. This provides platform specific information to Edk2PlatformSetup invocable +while allowing the invocable itself to remain platform agnostic. +""" import os import logging from io import StringIO @@ -19,34 +24,52 @@ class RequiredSubmodule(): - + """A class containing information about a git submodule.""" def __init__(self, path: str, recursive: bool = True): - ''' - Object to hold necessary information for resolving submodules + """Object to hold necessary information for resolving submodules. - path: workspace relative path to submodule that must be + Args: + path (str): workspace relative path to submodule that must be synchronized and updated - recursive: boolean if recursion should be used in this submodule - ''' + recursive (bool): if recursion should be used in this submodule + """ self.path = path self.recursive = recursive class SetupSettingsManager(MultiPkgAwareSettingsInterface): - ''' Platform settings will be accessed through this implementation. ''' + """Platform specific settings for Edk2PlatformSetup. + + Provides information necessary for `stuart_setup.exe` or `edk2_setup.py` + to successfully execute for a given platform. + Example: Example: Overriding SetupSettingsManager + ```python + from edk2toolext.invocables.edk2_setup import SetupSettingsManager, RequiredSubmodule + class PlatformManager(SetupSettingsManager): + def GetRequiredSubmodules(self) -> List[RequiredSubmodule]: + return [RequiredSubmodule('Common/MU', True)] + ``` + """ def GetRequiredSubmodules(self) -> List[RequiredSubmodule]: - ''' return iterable containing RequiredSubmodule objects. - If no RequiredSubmodules return an empty list - ''' + """Provides a list of required git submodules. + + These submodules are those that must be setup for the platform + to successfully build. + + TIP: Optional Override in a subclass + + Returns: + A list of required submodules, or an empty list + """ return [] class Edk2PlatformSetup(Edk2MultiPkgAwareInvocable): - ''' Updates git submodules listed in RequiredSubmodules ''' + """Invocable that updates git submodules listed in RequiredSubmodules.""" def AddCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' + """Adds command line options to the argparser.""" parserObj.add_argument('--force', '--FORCE', '--Force', dest="force", action='store_true', default=False) parserObj.add_argument('--omnicache', '--OMNICACHE', '--Omnicache', dest='omnicache_path', default=os.environ.get('OMNICACHE_PATH')) @@ -54,7 +77,7 @@ def AddCommandLineOptions(self, parserObj): super().AddCommandLineOptions(parserObj) def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser.""" self.force_it = args.force self.omnicache_path = args.omnicache_path if (self.omnicache_path is not None) and (not os.path.exists(self.omnicache_path)): @@ -64,17 +87,22 @@ def RetrieveCommandLineOptions(self, args): super().RetrieveCommandLineOptions(args) def GetVerifyCheckRequired(self): - ''' Will not call self_describing_environment.VerifyEnvironment because it hasn't been set up yet ''' + """Will not call self_describing_environment.VerifyEnvironment because it hasn't been set up yet.""" return False def GetSettingsClass(self): - ''' Providing SetupSettingsManager ''' + """Returns the SetupSettingsManager class. + + WARNING: SetupSettingsManager must be subclassed in your platform settings file. + """ return SetupSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename (SETUPLOG) of where the logs for the Edk2CiBuild invocable are stored in.""" return "SETUPLOG" def Go(self): + """Executes the core functionality of the Edk2PlatformSetup invocable.""" required_submodules = self.PlatformSettings.GetRequiredSubmodules() workspace_path = self.GetWorkspaceRoot() # Make sure git is installed @@ -197,4 +225,5 @@ def Go(self): def main(): + """Entry point to invoke Edk2PlatformSetup.""" Edk2PlatformSetup().Invoke() diff --git a/edk2toolext/invocables/edk2_update.py b/edk2toolext/invocables/edk2_update.py index 1e3adb6dd..fd40ce1ed 100644 --- a/edk2toolext/invocables/edk2_update.py +++ b/edk2toolext/invocables/edk2_update.py @@ -6,6 +6,12 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Updates external dependencies. + +Contains a UpdateSettingsManager that must be subclassed in a build settings +file. This provides platform specific information to Edk2Update invocable +while allowing the invocable itself to remain platform agnostic. +""" import logging from edk2toolext import edk2_logging from edk2toolext.environment import self_describing_environment @@ -14,26 +20,31 @@ class UpdateSettingsManager(MultiPkgAwareSettingsInterface): - ''' Platform settings will be accessed through this implementation. + """Platform specific settings for Edk2Update. + + Provides information necessary for `stuart_update.exe` or `edk2_update.py` + when updating the platform. - Update settings manager has no additional APIs not already defined in it's super class ''' + Update settings manager has no additional APIs not already defined in it's super class, however + The class should still be overwritten by the platform. + """ pass def build_env_changed(build_env, build_env_2): - ''' return True if build_env has changed ''' - + """Return True if build_env has changed.""" return (build_env.paths != build_env_2.paths) or \ (build_env.extdeps != build_env_2.extdeps) or \ (build_env.plugins != build_env_2.plugins) class Edk2Update(Edk2MultiPkgAwareInvocable): - ''' Updates dependencies in workspace for active scopes ''' + """Updates dependencies in workspace for active scopes.""" MAX_RETRY_COUNT = 10 def PerformUpdate(self): + """Updates the dependencies.""" ws_root = self.GetWorkspaceRoot() scopes = self.GetActiveScopes() skipped_dirs = self.GetSkippedDirectories() @@ -44,26 +55,30 @@ def PerformUpdate(self): return (build_env, shell_env, failure) def GetVerifyCheckRequired(self): - ''' Will not call self_describing_environment.VerifyEnvironment because ext_deps haven't been unpacked yet ''' + """Will not call self_describing_environment.VerifyEnvironment because ext_deps haven't been unpacked yet.""" return False def GetSettingsClass(self): - ''' Providing UpdateSettingsManger ''' + """Returns the UpdateSettingsManager class. + + WARNING: UpdateSettingsManager must be subclassed in your platform settings file. + """ return UpdateSettingsManager def GetLoggingFileName(self, loggerType): + """Returns the filename (UPDATE_LOG) of where the logs for the Edk2CiBuild invocable are stored in.""" return "UPDATE_LOG" def AddCommandLineOptions(self, parserObj): - ''' adds command line options to the argparser ''' + """Adds command line options to the argparser.""" super().AddCommandLineOptions(parserObj) def RetrieveCommandLineOptions(self, args): - ''' Retrieve command line options from the argparser ''' + """Retrieve command line options from the argparser.""" super().RetrieveCommandLineOptions(args) def Go(self): - # Get the environment set up. + """Executes the core functionality of the Edk2Update invocable.""" RetryCount = 0 failure_count = 0 logging.log(edk2_logging.SECTION, "Initial update of environment") @@ -98,4 +113,5 @@ def Go(self): def main(): + """Entry point to invoke Edk2Update.""" Edk2Update().Invoke() diff --git a/edk2toolext/nuget_publishing.py b/edk2toolext/nuget_publishing.py index 5f0607554..280124611 100644 --- a/edk2toolext/nuget_publishing.py +++ b/edk2toolext/nuget_publishing.py @@ -6,6 +6,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Provides configuration, packing, and publishing nuget packages to a release feed.""" import os import sys import argparse @@ -30,6 +31,7 @@ class NugetSupport(object): + """Support object for Nuget Publishing tool to configure NuPkg information, pack and send.""" # NOTE: This *should* have a namespace (http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd) # but ElementTree is incredibly stupid with namespaces. NUSPEC_TEMPLATE_XML = r''' @@ -56,12 +58,12 @@ class NugetSupport(object): RELEASE_NOTE_SHORT_STRING_MAX_LENGTH = 500 - # - # constructor that creates the NugetSupport object - # for new instances without existing config provide the Name parameter. - # for creating instance based on config file provide the path to the ConfigFile - # def __init__(self, Name=None, ConfigFile=None): + """Inits a new NugetSupport object. + + for new instances without existing config provide the Name parameter. + for creating instance based on config file provide the path to the ConfigFile + """ self.Name = Name self.TempFileToDelete = [] # everytime a temp is created add to list to cleanup self.NewVersion = None @@ -77,14 +79,13 @@ def __init__(self, Name=None, ConfigFile=None): self.Config = None def CleanUp(self): + """Remove all temporary files.""" logging.debug("CleanUp Called. Deleting all Temp Files") for a in self.TempFileToDelete: os.remove(a) - # - # Save the object config contents to file - # def ToConfigFile(self, filepath=None): + """Save config to a yaml file.""" if (not self.ConfigChanged): logging.debug("No Config Changes. Skip Writing config file") return 0 @@ -106,11 +107,13 @@ def ToConfigFile(self, filepath=None): return 0 def FromConfigfile(self, filepath): + """Load config from a yaml file.""" self.Config = filepath with open(self.Config, "r") as c: self.ConfigData = yaml.safe_load(c) def SetBasicData(self, authors, license, project, description, server, copyright): + """Set basic data in the config data.""" self.ConfigData["author_string"] = authors self.ConfigData["license_url"] = license self.ConfigData["project_url"] = project @@ -126,18 +129,17 @@ def SetBasicData(self, authors, license, project, description, server, copyright self.ConfigChanged = True def UpdateCopyright(self, copyright): + """Update copyright in the config data.""" self.ConfigData["copyright_string"] = copyright self.ConfigChanged = True def UpdateTags(self, tags=[]): + """Update tags in the config data.""" self.ConfigData["tags_string"] = " ".join(tags) self.ConfigChanged = True - # - # Print info about this object - # - # def Print(self): + """Print info about the Nuget Object.""" print("=======================================") print(" Name: " + self.Name) if (self.Config): @@ -156,6 +158,7 @@ def Print(self): print("=======================================") def LogObject(self): + """Logs info about Nuget Object to the logger.""" logging.debug("=======================================") logging.debug(" Name: " + self.Name) if (self.Config): @@ -235,11 +238,8 @@ def _GetNuPkgFileName(self, version): s += ".nupkg" return s - ## - # Pack the current contents into - # Nupkg - # def Pack(self, version, OutputDirectory, ContentDir, RelNotesText=None): + """Pack the current contents into Nupkg.""" self.NewVersion = version # content must be absolute path in nuspec otherwise it is assumed @@ -271,6 +271,11 @@ def Pack(self, version, OutputDirectory, ContentDir, RelNotesText=None): return ret def Push(self, nuPackage, apikey): + """Push nuget package to the server. + + Raises: + (Exception): file path is invalid + """ if (not os.path.isfile(nuPackage)): raise Exception("Invalid file path for NuPkg file") logging.debug("Pushing %s file to server %s" % (nuPackage, self.ConfigData["server_url"])) @@ -300,6 +305,7 @@ def Push(self, nuPackage, apikey): def GatherArguments(): + """Adds CLI arguments for controlling the nuget_publishing tool.""" tempparser = argparse.ArgumentParser( description='Nuget Helper Script for creating, packing, and pushing packages', add_help=False) tempparser.add_argument('--Operation', dest="op", choices=["New", "Pack", "Push", "PackAndPush"], required=True) @@ -369,6 +375,7 @@ def GatherArguments(): def main(): + """Entry point into nuget_publishing after initial configuration.""" args = GatherArguments() ret = 0 @@ -496,8 +503,8 @@ def main(): return ret -# the main method def go(): + """Main entry into the nuget publishing tool.""" # setup main console as logger logger = logging.getLogger('') logger.setLevel(logging.DEBUG) diff --git a/edk2toolext/omnicache.py b/edk2toolext/omnicache.py index d2046effc..aeefe5070 100644 --- a/edk2toolext/omnicache.py +++ b/edk2toolext/omnicache.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## # spell-checker:ignore rtags +"""Omnicache tool for lessening network usage and time of cloning repositories.""" import os import sys import logging @@ -37,14 +38,15 @@ class Omnicache(): - """ Class for managing an omnicache instance""" - + """Class for managing an omnicache instance.""" def __init__(self, cachepath, create=False, convert=True): - """Initializes an omnicache - cachepath - path to the omnicache git repo - create - if True, and omnicache doesn't already exist at cachepath, create a new omnicache instance - convert - if True, and old (but compatible) omnicache already exists, at cachepath, convert it to this - version of Omnicache + """Initializes an omnicache. + + Args: + cachepath (str): path to the omnicache git repo + create (bool): create a new omnicache instance if it doesn't exist + convert (bool): if an old (but compatible) omnicache already exists, at cachepath, convert it to this + version of Omnicache """ self.path = cachepath self._InvalidateUrlLookupCache() @@ -61,8 +63,12 @@ def __init__(self, cachepath, create=False, convert=True): def _ValidateOmnicache(self): """Validates whether self.path has a valid omnicache instance. - Returns: (valid, convertible) where "valid" is True if a compatible Omnicache exists at self.path, and - "convertible" is True if an older Omnicache that can be converted exists at self.path. + + Returns: + (bool, bool): valid, convertible + + NOTE: "valid" is True if a compatible Omnicache exists at self.path + NOTE: "convertible" is True if an older Omnicache that can be converted exists at self.path. """ logging.info("Checking if {0} is valid omnicache.".format(self.path)) if (not os.path.isdir(self.path)): @@ -92,7 +98,7 @@ def _ValidateOmnicache(self): return (False, False) def _InitOmnicache(self): - """Initializes a new omnicache instance""" + """Initializes a new omnicache instance.""" logging.critical("Initializing Omnicache in {0}".format(self.path)) os.makedirs(self.path, exist_ok=True) ret = RunCmd("git", "init --bare", workingdir=self.path) @@ -109,7 +115,7 @@ def _InitOmnicache(self): workingdir=self.path) def _ConvertOmnicache(self): - """Converts an existing bare git repo from a previous omnicache version to the current omnicache version""" + """Converts an existing bare git repo from a previous omnicache version to the current omnicache version.""" logging.info("Converting Omnicache in {0} to latest format.".format(self.path)) if (os.path.exists(os.path.join(self.path, PRE_0_11_OMNICACHE_FILENAME))): os.remove(os.path.join(self.path, PRE_0_11_OMNICACHE_FILENAME)) @@ -171,7 +177,7 @@ def _ConvertOmnicache(self): workingdir=self.path) def _RefreshUrlLookupCache(self): - """Refreshes the URL lookup cache""" + """Refreshes the URL lookup cache.""" if (len(self.urlLookupCache) == 0): logging.info("Regenerating URL lookup cache.") out = StringIO() @@ -183,21 +189,23 @@ def _RefreshUrlLookupCache(self): self.urlLookupCache[remote.split()[1]] = ".".join(remote.split()[0].split(".")[1:-1]) def _InvalidateUrlLookupCache(self): - """Invalidates the URL lookup cache""" + """Invalidates the URL lookup cache.""" logging.debug("Invalidating URL lookup cache.") self.urlLookupCache = {} def _LookupRemoteForUrl(self, url): - """Returns the git remote name for the specified URL, or None if it doesn't exist""" + """Returns the git remote name for the specified URL, or None if it doesn't exist.""" self._RefreshUrlLookupCache() if (url in self.urlLookupCache): return self.urlLookupCache[url] return None def AddRemote(self, url, name=None): - """Adds a remote for the specified URL to the omnicache - url - URL for which to add a remote - name - (optional) - provides a "display name" to be associated with this remote. + """Adds a remote for the specified URL to the omnicache. + + Args: + url (str): URL for which to add a remote + name(str, optional): provides a "display name" to be associated with this remote. """ # if we already have this remote (i.e. a remote with this URL exists), then just update. if (self._LookupRemoteForUrl(url) is not None): @@ -222,7 +230,7 @@ def AddRemote(self, url, name=None): workingdir=self.path) def RemoveRemote(self, url): - """Removes the remote for the specified url from the cache""" + """Removes the remote for the specified url from the cache.""" name = self._LookupRemoteForUrl(url) if (name is None): logging.critical("Failed to remove node for url {0}: such a remote does not exist.".format(url)) @@ -233,10 +241,12 @@ def RemoveRemote(self, url): return ret def UpdateRemote(self, oldUrl, newUrl=None, newName=None): - """Updates the remote - oldUrl - current url for the remote - newUrl - (optional) if specified, updates the remote to point to the new URL. - newName - (optional) if specified, updates the "displayname" for the remote. + """Updates the remote. + + Args: + oldUrl (str): current url for the remote + newUrl (str, optional): updates the remote to point to the new URL. + newName (str, optional): updates the "displayname" for the remote. """ remote = self._LookupRemoteForUrl(oldUrl) if (remote is None): @@ -259,7 +269,7 @@ def UpdateRemote(self, oldUrl, newUrl=None, newName=None): return 0 def Fetch(self, jobs=0): - """Fetches all remotes""" + """Fetches all remotes.""" logging.info("Fetching all remotes.") self._RefreshUrlLookupCache() # Tricky: we pass no-tags here, since we set up custom fetch refs for tags on a per-remote basis. This prevents @@ -270,10 +280,12 @@ def Fetch(self, jobs=0): return RunCmd("git", "fetch --all --no-tags", workingdir=self.path) def GetRemoteData(self): - """Gets Remote Data - Returns: a dict of dicts of the form: - {"":{"url":, "displayname":}} - note that "displayname" may not be present if the remote has no display name. + """Gets Remote Data. + + Returns: + (dict(dict)): {"":{"url":, "displayname":}} + + Note: "displayname" may not be present if the remote has no display name. """ logging.info("Retrieving all remote data") self._RefreshUrlLookupCache() @@ -296,7 +308,7 @@ def GetRemoteData(self): return remoteData def List(self): - """Prints the current set of remotes""" + """Prints the current set of remotes.""" print("List OMNICACHE content:\n") remoteData = self.GetRemoteData() if (len(remoteData) == 0): @@ -307,7 +319,9 @@ def List(self): @staticmethod def GetRemotes(path): """Gets the remotes for the git repository at the specified path. - Returns: dict of the form {:} + + Returns: + (dict): {:} """ logging.info("Retreiving remotes") remotes = {} @@ -326,7 +340,7 @@ def GetRemotes(path): @staticmethod def _IsValidUuid(val): - """Returns whether the input is valid UUID""" + """Returns whether the input is valid UUID.""" try: uuid.UUID(str(val)) return True @@ -336,7 +350,7 @@ def _IsValidUuid(val): def ProcessInputConfig(omnicache, input_config): - """Adds a list of remotes from a YAML config file to the omnicache""" + """Adds a list of remotes from a YAML config file to the omnicache.""" logging.info("Adding remotes from {0}".format(input_config)) with open(input_config) as icf: content = yaml.safe_load(icf) @@ -350,7 +364,7 @@ def ProcessInputConfig(omnicache, input_config): def ScanDirectory(omnicache, scanpath): - """Recursively scans a directory for git repositories and adds remotes and submodule remotes to omnicache""" + """Recursively scans a directory for git repositories and adds remotes and submodule remotes to omnicache.""" logging.info("Scanning {0} for remotes to add.".format(scanpath)) if (not os.path.isdir(scanpath)): logging.critical("specified scan path is invalid.") @@ -375,7 +389,7 @@ def ScanDirectory(omnicache, scanpath): def Export(omnicache, exportPath): - """Exports omnicache configuration to YAML""" + """Exports omnicache configuration to YAML.""" logging.info("Exporting omnicache config for {0} to {1}".format(omnicache.path, exportPath)) content = [] for (name, data) in omnicache.GetRemoteData().items(): @@ -393,6 +407,7 @@ def Export(omnicache, exportPath): def get_cli_options(): + """Add CLI arguments to argparse for controlling the omnicache.""" parser = argparse.ArgumentParser(description='Tool to provide easy method create and manage the OMNICACHE', ) parser.add_argument(dest="cache_dir", help="path to an existing or desired OMNICACHE directory") parser.add_argument("--scan", dest="scan", default=None, @@ -427,6 +442,7 @@ def get_cli_options(): def main(): + """Main entry point to managing the omnicache.""" # setup main console as logger logger = logging.getLogger('') logger.setLevel(logging.NOTSET) diff --git a/edk2toolext/tests/__init__.py b/edk2toolext/tests/__init__.py index 0a69010b8..a7e3a2a8b 100644 --- a/edk2toolext/tests/__init__.py +++ b/edk2toolext/tests/__init__.py @@ -3,3 +3,5 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## + +# noqa diff --git a/edk2toolext/tests/capsule/capsule_helper_test.py b/edk2toolext/tests/capsule/test_capsule_helper.py similarity index 100% rename from edk2toolext/tests/capsule/capsule_helper_test.py rename to edk2toolext/tests/capsule/test_capsule_helper.py diff --git a/edk2toolext/tests/capsule/capsule_tool_test.py b/edk2toolext/tests/capsule/test_capsule_tool.py similarity index 100% rename from edk2toolext/tests/capsule/capsule_tool_test.py rename to edk2toolext/tests/capsule/test_capsule_tool.py diff --git a/edk2toolext/tests/capsule/signing_helper_test.py b/edk2toolext/tests/capsule/test_signing_helper.py similarity index 100% rename from edk2toolext/tests/capsule/signing_helper_test.py rename to edk2toolext/tests/capsule/test_signing_helper.py diff --git a/edk2toolext/tests/uefi_tree.py b/edk2toolext/tests/uefi_tree.py index 8d88c7dab..0cfbdb2d0 100644 --- a/edk2toolext/tests/uefi_tree.py +++ b/edk2toolext/tests/uefi_tree.py @@ -4,6 +4,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent +"""Used for creating a minimal uefi tree for testing.""" import os import tempfile import json @@ -11,13 +12,21 @@ class uefi_tree: - ''' - This class represents a minimal UEFI tree that you can setup - It is configurable and offers options to modify the tree for - ''' + """A minimal UEFI tree. + + This class represents a minimal UEFI tree that you can setup. + It is configurable and offers options to modify the tree for. + + Attributes: + workspace (str): Path to a directory for the minimal tree + + Args: + workspace (str): Path to a directory for the minimal tree + create_platform (bool): create the tree or use an existing + """ def __init__(self, workspace=None, create_platform=True): - ''' copy the minimal tree to a folder, if one is not provided we will create a new temporary folder for you ''' + """Inits uefi_tree.""" if workspace is None: workspace = os.path.abspath(tempfile.mkdtemp()) self.workspace = workspace @@ -33,10 +42,11 @@ def _get_optional_folder(): return os.path.join(uefi_tree._get_src_folder(), "optional") def get_workspace(self): + """Returns the workspace path.""" return self.workspace def _create_tree(self): - ''' Creates a settings.py, test.dsc, Conf folder (with build_rule, target, and tools_def) ''' + """Creates a settings.py, test.dsc, Conf folder (with build_rule, target, and tools_def).""" settings_path = self.get_settings_provider_path() uefi_tree.write_to_file(settings_path, self._settings_file_text) dsc_path = os.path.join(self.workspace, "Test.dsc") @@ -52,24 +62,25 @@ def _create_tree(self): @staticmethod def write_to_file(path, contents, close=True): + """Writes contents to a file.""" f = open(path, "w") f.writelines(contents) if close: f.close() def get_settings_provider_path(self): - ''' gets the settings provider in the workspace ''' + """Gets the settings provider in the workspace.""" return os.path.join(self.workspace, "settings.py") def get_optional_file(self, file): - ''' gets the path of an optional file ''' + """Gets the path of an optional file.""" optional_path = os.path.join(uefi_tree._get_optional_folder(), file) if os.path.exists(optional_path): return os.path.abspath(optional_path) raise ValueError(f"Optional file not found {file}") def create_path_env(self, id=None, flags=[], var_name=None, scope="global", dir_path="", extra_data=None): - ''' creates an ext dep in your workspace ''' + """Creates an ext dep in your workspace.""" data = { "scope": scope, "flags": flags, @@ -91,10 +102,11 @@ def create_path_env(self, id=None, flags=[], var_name=None, scope="global", dir_ uefi_tree.write_to_file(output_path, text) def create_Edk2TestUpdate_ext_dep(self, version="0.0.1", extra_data=None): + """Creates an Edk2TestUpdate ext dep in your workspace.""" self.create_ext_dep("nuget", "Edk2TestUpdate", version, extra_data=extra_data) def create_ext_dep(self, dep_type, name, version, source=None, scope="global", dir_path="", extra_data=None): - ''' creates an ext dep in your workspace ''' + """Creates an ext dep in your workspace.""" dep_type = dep_type.lower() if source is None and dep_type == "nuget": source = "https://api.nuget.org/v3/index.json" diff --git a/edk2toolext/uefi/sig_db_tool.py b/edk2toolext/uefi/sig_db_tool.py index 5e1a4b63b..9cc66c658 100644 --- a/edk2toolext/uefi/sig_db_tool.py +++ b/edk2toolext/uefi/sig_db_tool.py @@ -8,8 +8,8 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent ## -""" -Tool for inspecting UEFI Secure Boot databases +"""Tool for inspecting UEFI Secure Boot databases. + The input file is a concatenations of EFI_SIGNATURE_LISTS, as is read by calling GetVariable() on Secure Boot variables (i.e. PK, KEK, db, dbx) The output can be standard human-readable or compact for easier diffing @@ -21,8 +21,11 @@ def main(): - """Parses command-line parameters using ArgumentParser, delegating to helper functions to fulfill the requests""" + """Parses command-line parameters using ArgumentParser. + Parses command-line parameters using ArgumentParser delegating to helper + functions to fulfill the requests. + """ filenameHelp = 'Filename containing a UEFI Signature Database, \ a concatenation of EFI_SIGNATURE_LISTs as read from GetVariable([PK, KEK, db, dbx])' diff --git a/edk2toolext/versioninfo/__init__.py b/edk2toolext/versioninfo/__init__.py index 0a69010b8..89c0299fb 100644 --- a/edk2toolext/versioninfo/__init__.py +++ b/edk2toolext/versioninfo/__init__.py @@ -3,3 +3,4 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""A Command-line tool to generate VERSIONINFO resource files.""" diff --git a/edk2toolext/versioninfo/versioninfo_helper.py b/edk2toolext/versioninfo/versioninfo_helper.py index 2e3286c94..dee80e7c1 100644 --- a/edk2toolext/versioninfo/versioninfo_helper.py +++ b/edk2toolext/versioninfo/versioninfo_helper.py @@ -10,17 +10,26 @@ ## # spell-checker:ignore NOFONT, ntdef +"""This module contains functions for creating VERSIONINFO resources. +This module contains helper functions for creating VERSIONINFO resources +from json files, along with the functions to output version information from +PE/PE+ files. +""" import pefile import json import logging from datetime import datetime +from typing import Tuple class PEStrings(object): - ''' - String literals for fields in PE/PE+ header and required fields for VERSIONINFO resource - ''' + """PE/PE+ field String literals. + + String literals for fields in PE/PE+ header and required fields for + VERSIONINFO resource. + """ + FILE_OS_STRINGS = { 0x00010000: "VOS_DOS", 0x00040000: "VOS_NT", @@ -246,7 +255,14 @@ class PEStrings(object): def validate_version_number(version_str: str) -> bool: - '''Helper function to check if a version string format is valid or not''' + """Helper function to check if a version string format is valid or not. + + Args: + version_str (str): the version + + Returns: + (bool): if the version string is valid or not + """ if version_str.count('.') != 3 and version_str.count(',') != 3: logging.error("Invalid version string: " + version_str + ". Version must be in form " + "\"INTEGER.INTEGER.INTEGER.INTEGER\".") @@ -271,8 +287,17 @@ def validate_version_number(version_str: str) -> bool: return True -def version_str_to_int(version_str: str) -> (int, int): - '''Given a valid version string, returns raw version number in the form (32 MS bits, 32 LS bits)''' +def version_str_to_int(version_str: str) -> Tuple[int, int]: + """Converts a valid version string to a number. + + Given a valid version string, returns raw version number + + Args: + version_str (str): version string + + Result: + (Tuple[int, int]): (32 MS bits, 32 LS bits) + """ split = None if version_str.count('.') == 3: split = version_str.split(".") @@ -285,29 +310,43 @@ def version_str_to_int(version_str: str) -> (int, int): def hex_to_version_str(val: int) -> str: - '''Helper function to convert hex version number to string''' + """Converts hex version number to string. + + Args: + val (int): value to convert + + Returns: + (str): string represention + """ return str(((val & ~0) >> 16) & 0xffff) + "." + str(val & 0xffff) class PEObject(object): - ''' - Class for parsing PE/PE+ files. Gives functionality for reading VS_VERSIONINFO metadata and .rsrc section. - ''' + """Class for parsing PE/PE+ files. + + Gives functionality for reading VS_VERSIONINFO metadata and .rsrc section. + """ _pe: pefile.PE = None def __init__(self, filepath: str) -> None: - ''' - Initializes PE parser + """Initializes PE parser. - filepath - filepath to PE/PE+ file to parse - ''' + Args: + filepath (str): filepath to PE/PE+ file to parse + """ try: self._pe = pefile.PE(filepath) except pefile.PEFormatError as e: logging.error("Error loading PE: " + str(e)) def _populate_entry(self, key: str, val: int, dict: dict) -> None: - '''Helper function to format and insert VERSIONINFO fields into dictionary''' + """Formats and inserts VERSIONINFO fields into dictionary. + + Args: + key (str): dict key for value + val (int): value to store in dict + dict (dict): dictionary to store key/value pair + """ if key.upper() == PEStrings.FILE_OS_STR: if val in PEStrings.FILE_OS_STRINGS: dict[key] = PEStrings.FILE_OS_STRINGS[val] @@ -331,14 +370,22 @@ def _populate_entry(self, key: str, val: int, dict: dict) -> None: dict[key] = hex(val) def contains_rsrc(self) -> bool: - '''Returns true if PE object contains .rsrc section, false otherwise''' + """Checks if PE object contains .rsrc section. + + Returns: + (bool): whether a .rsrc section exists in the PE + """ for section in self._pe.sections: if section.Name.decode(PEStrings.PE_ENCODING).replace("\x00", "") == PEStrings.RSRC_STR: return True return False def get_version_dict(self) -> dict: - '''Parses PE/PE+ loaded into PEObject and returns dictionary contaning metadata from header and .rsrc section''' + """Parses PE/PE+ loaded into PEObject. + + Returns: + (dict): dict containing metadata from header and .rsrc section + """ if not self._pe: logging.fatal("Cannot parse, PE not loaded.") return None @@ -411,7 +458,8 @@ def get_version_dict(self) -> dict: class VERSIONINFOGenerator(object): - ''' + """A Generator for a VERSIONINFO rc file via a given JSON settings file. + Given a JSON file, object creates generator for VERSIONINFO rc file for the given JSON. Provides validation of some basic requirements and generates VERSIONINFO.rc source to be used with rc.exe. If "Minimal" attribute is set to false, JSON defines a complete rc section. @@ -453,7 +501,7 @@ class VERSIONINFOGenerator(object): "CompanyName": "Example Company", "OriginalFilename": "ExampleApp.efi" } - ''' + """ _minimal_required_fields = { PEStrings.FILE_VERSION_STR.upper(), PEStrings.COMPANY_NAME_STR.upper(), @@ -463,11 +511,14 @@ class VERSIONINFOGenerator(object): _version_dict = None def __init__(self, filepath: str) -> None: - ''' - Initializes generator + """Initializes the VERSIONINFO generator. - filepath - filepath to JSON file containing VERSIONINFO information in format shown above - ''' + Initializes the VERSIONINFO generator using a JSON file containing + VERSIONINFO information in the format shown in the class description. + + Args: + filepath (str): filepath to the JSON file + """ with open(filepath, "r") as jsonFile: data = jsonFile.read() try: @@ -479,9 +530,11 @@ def __init__(self, filepath: str) -> None: logging.error("Invalid JSON format, " + str(e)) def validate(self) -> bool: - ''' - Returns true if loaded JSON file represents a valid VERSIONINFO resource, false otherwise - ''' + """Checks if the loaded JSON file represents a valid VERSIONINFO resource. + + Returns: + (bool): valid or not + """ valid = True # First Pass: Check to see if all required fields are present @@ -583,9 +636,11 @@ def validate(self) -> bool: return valid def validate_minimal(self) -> bool: - ''' - Returns true if loaded JSON file represents a valid minimal VERSIONINFO resource, false otherwise - ''' + """Checks if the loaded JSON file represents a valid minimal VERSIONINFO resource. + + Returns: + (bool): valid or not + """ valid = True required = self._minimal_required_fields.copy() for key in self._version_dict: @@ -601,6 +656,15 @@ def validate_minimal(self) -> bool: return validate_version_number(self._version_dict[PEStrings.FILE_VERSION_STR]) def write_minimal(self, path: str, version: str) -> bool: + """Write a minimal VERSIONINFO resource file. + + Args: + path (str): path to write + version (str): version + + Returns: + (bool): result of writing + """ if not self.validate_minimal(): logging.error("Invalid input, aborted.") return False @@ -645,12 +709,17 @@ def write_minimal(self, path: str, version: str) -> bool: return True def write(self, path: str, version: str) -> bool: - ''' - Encodes the loaded JSON and writes it to VERSION.rc, a resource file comptaible with rc.exe. - Returns true on success, false otherwise. + """Encodes the loaded JSON and writes it to VERSION.rc. + + Encodes the loaded JSON and writes it to VERSION.rc, a resource file compatible with rc.exe. + + Args: + path (str): path to the directory that VERSIONINFO.rc will be written to. + version (str): version - path - path to directory that VERSIONINFO.rc will be written to - ''' + Returns: + (bool): result of encoding and writing + """ if not self._version_dict: logging.error("Invalid input, aborted.") return False diff --git a/edk2toolext/versioninfo/versioninfo_tool.py b/edk2toolext/versioninfo/versioninfo_tool.py index d73d4b222..97947637f 100644 --- a/edk2toolext/versioninfo/versioninfo_tool.py +++ b/edk2toolext/versioninfo/versioninfo_tool.py @@ -7,7 +7,13 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""A Command-line tool to generate VERSIONINFO resource files. +Versioninfo Tool is a command-line tool to assist in generating VERSIONINFO +resource files for use with a Resource Compiler. It takes a JSON representing +versioning info and produces a resource file that once compiled will create a +standard resource section. +""" import os import sys @@ -39,10 +45,11 @@ def get_cli_options(args=None): - ''' - will parse the primary options from the command line. If provided, will take the options as + """Parse options from the command line. + + Will parse the primary options from the command line. If provided, will take the options as an array in the first parameter - ''' + """ parser = argparse.ArgumentParser(description=TOOL_DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('input_file', type=str, help='a filesystem path to a json/PE file to load') @@ -59,7 +66,14 @@ def get_cli_options(args=None): def decode_version_info(input_file): - ''' takes in a PE file (input_file) and returns the version dictionary ''' + """Takes in a PE file (input_file) and returns the version dictionary. + + Args: + input_file (str): path to a PE file + + Returns: + (Dict): version dictionary + """ if not os.path.exists(input_file): raise FileNotFoundError(input_file) pe = PEObject(input_file) @@ -67,10 +81,15 @@ def decode_version_info(input_file): def decode_version_info_dump_json(input_file, output_file): - ''' - Takes in a PE file (input_file) and dumps the output as json to (output_file) - returns True on success - ''' + """Takes in a PE file (input_file) and dumps the output as json to (output_file). + + Args: + input_file (str): Path to a PE file + output_file (str): Path to json dump + + Returns: + (Boolean): result of parsing and dumping + """ version_info = decode_version_info(input_file) if not version_info: logging.error(f"{input_file} doesn't have version info.") @@ -81,19 +100,34 @@ def decode_version_info_dump_json(input_file, output_file): def encode_version_info(input_file): - ''' takes in a JSON file (inputfile) and returns an object ''' + """Takes in a JSON file (inputfile) and returns an object. + + Args: + input_file (str): path to JSON File + + Returns: + (VERSIONINFOGenerator): VERSIONINFO rc file Generator object + """ if not os.path.exists(input_file): raise FileNotFoundError(input_file) return VERSIONINFOGenerator(input_file) def encode_version_info_dump_rc(input_file, output_file): - ''' takes in a JSON file (input_file) and outputs an RC file(output_file) ''' + """Takes in a JSON file (input_file) and outputs an RC file(output_file). + + Args: + input_file (str): path to JSON file + output_file (str): path to RC file + + Returns: + (Boolean): Result of writing to file + """ return encode_version_info(input_file).write(output_file, TOOL_VERSION) def main(): - ''' parse args, executes versioninfo_tool ''' + """Parse args, executes versioninfo_tool.""" logging.getLogger().addHandler(logging.StreamHandler()) args = get_cli_options() if not os.path.isfile(args.input_file): diff --git a/edk2toolext/windows/policy/firmware_policy_tool.py b/edk2toolext/windows/policy/firmware_policy_tool.py index a14b01a53..329f53aa5 100644 --- a/edk2toolext/windows/policy/firmware_policy_tool.py +++ b/edk2toolext/windows/policy/firmware_policy_tool.py @@ -6,13 +6,14 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""A basic command-line interface for creating and decoding windows firmware policy blobs.""" from edk2toollib.windows.policy.firmware_policy import FirmwarePolicy import argparse def PrintPolicy(filename: str) -> None: - """Attempts to parse filename as a Windows Firmware Policy and print it""" + """Attempts to parse filename as a Windows Firmware Policy and print it.""" try: with open(filename, 'rb') as f: policy = FirmwarePolicy(fs=f) @@ -24,10 +25,9 @@ def PrintPolicy(filename: str) -> None: def CreatePolicyFromParameters(filename: str, manufacturer: str, product: str, sn: str, nonce: int, oem1: str, oem2: str, devicePolicy: int) -> None: - """ - Populates a Windows FirmwarePolicy object with the provided parameters and serializes it to filename + """Populates a Windows FirmwarePolicy object with the provided parameters and serializes it to filename. - Filename must be a new file, will not overwrite existing files. + WARNING: Filename must be a new file, will not overwrite existing files. """ with open(filename, 'xb') as f: policy = FirmwarePolicy() @@ -44,7 +44,7 @@ def CreatePolicyFromParameters(filename: str, manufacturer: str, product: str, def main(): - """Parses command-line parameters using ArgumentParser, passing them to helper functions to perform the requests""" + """Parses command-line parameters using ArgumentParser, passing them to helper functions to perform the requests.""" parser = argparse.ArgumentParser(description='Firmware Policy Tool') subparsers = parser.add_subparsers(required=True, dest='action') diff --git a/requirements.txt b/requirements.txt index 015682fba..9f67b1910 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pytest == 7.1.2 pytest-html == 3.1.1 pytest-cov == 3.0.0 flake8 == 5.0.4 +pydocstyle == 6.1.1 pyopenssl == 22.0.0 pefile == 2022.5.30 semantic_version == 2.10.0 diff --git a/setup.py b/setup.py index 2d5a05cb1..04385b79d 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent ## +"""Setup info for edk2-pytool-extensions pip module.""" import setuptools from setuptools.command.sdist import sdist from setuptools.command.install import install @@ -18,7 +19,7 @@ class PostSdistCommand(sdist): """Post-sdist.""" - def run(self): + def run(self): # noqa # we need to download nuget so throw the exception if we don't get it DownloadNuget() sdist.run(self) @@ -26,7 +27,7 @@ def run(self): class PostInstallCommand(install): """Post-install.""" - def run(self): + def run(self): # noqa try: DownloadNuget() except Exception: @@ -36,7 +37,7 @@ def run(self): class PostDevCommand(develop): """Post-develop.""" - def run(self): + def run(self): # noqa try: DownloadNuget() except Exception: