From 45b5c8320fb91ff7ede9723458a08bbffa228c91 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Jan 2023 18:01:13 +0000 Subject: [PATCH 1/7] Add pre-commit config. --- .pre-commit-config.yaml | 12 ++++++++++++ CHANGES | 2 +- doc/zkg.rst | 2 +- .../scripts.packages.rot13.__load__.zeek | 2 +- .../scripts.packages.rot13.__load__.zeek | 2 +- .../packages/rot13/scripts/Demo/Rot13/__load__.zeek | 2 +- testing/packages/rot13/scripts/__load__.zeek | 2 +- testing/packages/rot13/scripts/__preload__.zeek | 10 ++++------ testing/packages/rot13/scripts/types.zeek | 1 - testing/packages/rot13/testing/tests/main | 1 - testing/scripts/setup-zeek-and-home | 1 - testing/tests/bindir-install | 1 - testing/tests/bindir-relocate | 3 +-- testing/tests/plugin | 2 +- testing/tests/plugin_tarfile | 2 +- testing/tests/test | 2 +- 16 files changed, 26 insertions(+), 21 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..54b1fc74 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + exclude: testing/baselines + - id: end-of-file-fixer + exclude: testing/baselines + - id: check-yaml + - id: check-added-large-files diff --git a/CHANGES b/CHANGES index 3d65a924..a3c3306d 100644 --- a/CHANGES +++ b/CHANGES @@ -1392,7 +1392,7 @@ 1.0.4-5 | 2017-09-14 18:56:16 -0500 - * Require extra configparser package only for Python < 3.5 + * Require extra configparser package only for Python < 3.5 where it is not included (Hilko Bengen) * Fix imports for python3 (Robert Haist) diff --git a/doc/zkg.rst b/doc/zkg.rst index 93af2b41..3af3edd4 100644 --- a/doc/zkg.rst +++ b/doc/zkg.rst @@ -50,7 +50,7 @@ remove .. note:: You may also say ``uninstall``. - + .. _purge-command: purge diff --git a/testing/baselines/tests.plugin/scripts.packages.rot13.__load__.zeek b/testing/baselines/tests.plugin/scripts.packages.rot13.__load__.zeek index 888a6190..342b04f0 100644 --- a/testing/baselines/tests.plugin/scripts.packages.rot13.__load__.zeek +++ b/testing/baselines/tests.plugin/scripts.packages.rot13.__load__.zeek @@ -2,7 +2,7 @@ # # This is loaded when a user activates the plugin. Include scripts here that should be # loaded automatically at that point. -# +# diff --git a/testing/baselines/tests.plugin_tarfile/scripts.packages.rot13.__load__.zeek b/testing/baselines/tests.plugin_tarfile/scripts.packages.rot13.__load__.zeek index 888a6190..342b04f0 100644 --- a/testing/baselines/tests.plugin_tarfile/scripts.packages.rot13.__load__.zeek +++ b/testing/baselines/tests.plugin_tarfile/scripts.packages.rot13.__load__.zeek @@ -2,7 +2,7 @@ # # This is loaded when a user activates the plugin. Include scripts here that should be # loaded automatically at that point. -# +# diff --git a/testing/packages/rot13/scripts/Demo/Rot13/__load__.zeek b/testing/packages/rot13/scripts/Demo/Rot13/__load__.zeek index 69cc270a..a5456eee 100644 --- a/testing/packages/rot13/scripts/Demo/Rot13/__load__.zeek +++ b/testing/packages/rot13/scripts/Demo/Rot13/__load__.zeek @@ -1,7 +1,7 @@ # # This is loaded when a user activates the plugin. Include scripts here that should be # loaded automatically at that point. -# +# diff --git a/testing/packages/rot13/scripts/__load__.zeek b/testing/packages/rot13/scripts/__load__.zeek index 4b38f4b9..a6528de3 100644 --- a/testing/packages/rot13/scripts/__load__.zeek +++ b/testing/packages/rot13/scripts/__load__.zeek @@ -1,7 +1,7 @@ # # This is loaded unconditionally at Zeek startup. Include scripts here that should # always be loaded. -# +# # Normally, that will be only code that initializes built-in elements. Load # your standard scripts in # scripts///__load__.zeek instead. diff --git a/testing/packages/rot13/scripts/__preload__.zeek b/testing/packages/rot13/scripts/__preload__.zeek index 9b51a8b1..a7349a7f 100644 --- a/testing/packages/rot13/scripts/__preload__.zeek +++ b/testing/packages/rot13/scripts/__preload__.zeek @@ -1,12 +1,10 @@ # # This is loaded unconditionally at Zeek startup before any of the BiFs that the -# plugin defines become available. -# +# plugin defines become available. +# # This is primarily for defining types that BiFs already depend on. If you need # to do any other unconditional initialization (usually that's just for other BiF -# elemets), that should go into __load__.zeek instead. -# +# elemets), that should go into __load__.zeek instead. +# @load ./types.zeek - - diff --git a/testing/packages/rot13/scripts/types.zeek b/testing/packages/rot13/scripts/types.zeek index 8b137891..e69de29b 100644 --- a/testing/packages/rot13/scripts/types.zeek +++ b/testing/packages/rot13/scripts/types.zeek @@ -1 +0,0 @@ - diff --git a/testing/packages/rot13/testing/tests/main b/testing/packages/rot13/testing/tests/main index 6827c616..05213bf3 100644 --- a/testing/packages/rot13/testing/tests/main +++ b/testing/packages/rot13/testing/tests/main @@ -1,3 +1,2 @@ # @TEST-EXEC: zeek rot13 > output # @TEST-EXEC: TEST_DIFF_CANONIFIER="sort -s" btest-diff output - diff --git a/testing/scripts/setup-zeek-and-home b/testing/scripts/setup-zeek-and-home index edd84ca0..cc820f7b 100755 --- a/testing/scripts/setup-zeek-and-home +++ b/testing/scripts/setup-zeek-and-home @@ -20,4 +20,3 @@ state_dir = ${zeek_zkg_state_dir} script_dir = ${zeek_site_dir} plugin_dir = ${zeek_plugins_dir} " >> ${zeek_zkg_config_dir}/config - diff --git a/testing/tests/bindir-install b/testing/tests/bindir-install index d22a6845..9f62621b 100644 --- a/testing/tests/bindir-install +++ b/testing/tests/bindir-install @@ -26,4 +26,3 @@ chmod +x x/exec1 x/exec2 git add * git commit -m 'new stuff' - diff --git a/testing/tests/bindir-relocate b/testing/tests/bindir-relocate index 8e9c0616..3472fc8f 100644 --- a/testing/tests/bindir-relocate +++ b/testing/tests/bindir-relocate @@ -4,7 +4,7 @@ # @TEST-EXEC: test -L bin/exec1 && test -L bin/exec2 # # @TEST-EXEC: sed -i.bak 's/\(^bin_dir.*\)/\1-new/g' config -# @TEST-EXEC: zkg list +# @TEST-EXEC: zkg list # @TEST-EXEC: test -L bin-new/exec1 && test -L bin-new/exec2 # @TEST-EXEC: ./bin-new/exec1 >>output # @TEST-EXEC: ./bin-new/exec2 >>output @@ -24,4 +24,3 @@ chmod +x x/exec1 x/exec2 git add * git commit -m 'new stuff' - diff --git a/testing/tests/plugin b/testing/tests/plugin index a7ff0c5e..2c6eef6f 100644 --- a/testing/tests/plugin +++ b/testing/tests/plugin @@ -3,7 +3,7 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC: zkg install rot13 -# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ +# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ # @TEST-EXEC: btest-diff scripts/packages/rot13/__load__.zeek # Unloading the package should also disable the plugin, which we diff --git a/testing/tests/plugin_tarfile b/testing/tests/plugin_tarfile index ccf76635..d0c64f78 100644 --- a/testing/tests/plugin_tarfile +++ b/testing/tests/plugin_tarfile @@ -3,7 +3,7 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC: zkg install rot13 -# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ +# @TEST-EXEC: btest-diff plugins/packages/rot13/__bro_plugin__ # @TEST-EXEC: btest-diff scripts/packages/rot13/__load__.zeek cd packages/rot13 diff --git a/testing/tests/test b/testing/tests/test index 3f1049a3..3abdb5e7 100644 --- a/testing/tests/test +++ b/testing/tests/test @@ -26,4 +26,4 @@ git commit -am 'new stuff' cd packages/rot13 echo 'hello' > testing/Baseline/tests.rot13/output git commit -am 'new stuff' -@TEST-END-FILE +@TEST-END-FILE From 9a088e43839d949c0749adfc4cf3b84ba67a9d31 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Jan 2023 18:09:33 +0000 Subject: [PATCH 2/7] Add GH action to run pre-commit. --- .github/workflows/pre-commit.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000..cc4076f6 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,16 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.7' + - uses: pre-commit/action@v3.0.0 From 99520f9201340d57d2cc9cf6f45992b994eff7c4 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Jan 2023 18:05:47 +0000 Subject: [PATCH 3/7] Reformat Python code with Black. --- .pre-commit-config.yaml | 6 + doc/conf.py | 77 +- setup.py | 40 +- testing/templates/foo/__init__.py | 33 +- zeekpkg/__init__.py | 2 +- zeekpkg/_util.py | 97 +- zeekpkg/manager.py | 1326 ++++++++++-------- zeekpkg/package.py | 97 +- zeekpkg/source.py | 54 +- zeekpkg/template.py | 185 +-- zeekpkg/uservar.py | 39 +- zkg | 2080 ++++++++++++++++------------- 12 files changed, 2326 insertions(+), 1710 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54b1fc74..34a44721 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,9 @@ repos: exclude: testing/baselines - id: check-yaml - id: check-added-large-files +- repo: https://github.com/psf/black + rev: 23.1a1 + hooks: + - id: black + +exclude: doc/ext/sphinxarg diff --git a/doc/conf.py b/doc/conf.py index a71709d2..25c799ae 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,8 +18,9 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) -sys.path.insert(0, os.path.abspath('./ext')) + +sys.path.insert(0, os.path.abspath("..")) +sys.path.insert(0, os.path.abspath("./ext")) # -- General configuration ------------------------------------------------ @@ -30,35 +31,35 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinxarg.ext', 'sphinx.ext.autodoc', 'sphinx.ext.napoleon'] +extensions = ["sphinxarg.ext", "sphinx.ext.autodoc", "sphinx.ext.napoleon"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'Zeek Package Manager' -copyright = u'2019, The Zeek Project' -author = u'The Zeek Project' +project = "Zeek Package Manager" +copyright = "2019, The Zeek Project" +author = "The Zeek Project" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -with open('../VERSION', 'r') as f: +with open("../VERSION", "r") as f: version = f.readline().strip() # The full version, including alpha/beta/rc tags. @@ -83,7 +84,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -105,7 +106,7 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -119,23 +120,24 @@ # -- Options for HTML output ---------------------------------------------- -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' + + html_theme = "sphinx_rtd_theme" html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_theme_options = { - 'collapse_navigation': False, - 'display_version': True, + "collapse_navigation": False, + "display_version": True, } # The name for this set of Sphinx documents. # " v documentation" by default. # -html_title = u'Zeek Package Manager Documentation' +html_title = "Zeek Package Manager Documentation" # A shorter title for the navigation bar. Default is the same as html_title. # @@ -155,11 +157,13 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] + def setup(app): - #app.add_javascript("custom.js") - app.add_css_file("theme_overrides.css") + # app.add_javascript("custom.js") + app.add_css_file("theme_overrides.css") + # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -180,7 +184,7 @@ def setup(app): # Custom sidebar templates, maps document names to template names. # -#html_sidebars = {'**': ['custom-sidebar.html']} +# html_sidebars = {'**': ['custom-sidebar.html']} # Additional templates that should be rendered to pages, maps page names to # template names. @@ -239,7 +243,7 @@ def setup(app): # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'ZeekPackageManagerdoc' +htmlhelp_basename = "ZeekPackageManagerdoc" # -- Options for LaTeX output --------------------------------------------- @@ -247,15 +251,12 @@ def setup(app): # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -265,8 +266,13 @@ def setup(app): # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'ZeekPackageManager.tex', u'Zeek Package Manager Documentation', - u'The Zeek Project', 'manual'), + ( + master_doc, + "ZeekPackageManager.tex", + "Zeek Package Manager Documentation", + "The Zeek Project", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -306,10 +312,7 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - ('zkg', 'zkg', u'Zeek Package Manager', - [author], 1) -] +man_pages = [("zkg", "zkg", "Zeek Package Manager", [author], 1)] # If true, show URL addresses after external links. # @@ -322,9 +325,15 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'ZeekPackageManager', u'Zeek Package Manager Documentation', - author, 'ZeekPackageManager', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "ZeekPackageManager", + "Zeek Package Manager Documentation", + author, + "ZeekPackageManager", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. diff --git a/setup.py b/setup.py index 2588d56e..e234dbf6 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,28 @@ from setuptools import setup -install_requires = ['gitpython', 'semantic_version', 'btest'] +install_requires = ["gitpython", "semantic_version", "btest"] setup( - name='zkg', - version=open('VERSION').read().replace('-', '.dev', 1).strip(), - description='The Zeek Package Manager', - long_description=open('README').read(), - license='University of Illinois/NCSA Open Source License', - keywords='zeek zeekctl zeekcontrol package manager scripts plugins security', - maintainer='The Zeek Project', - maintainer_email='info@zeek.org', - url='https://github.com/zeek/package-manager', - scripts=['zkg'], - packages=['zeekpkg'], + name="zkg", + version=open("VERSION").read().replace("-", ".dev", 1).strip(), + description="The Zeek Package Manager", + long_description=open("README").read(), + license="University of Illinois/NCSA Open Source License", + keywords="zeek zeekctl zeekcontrol package manager scripts plugins security", + maintainer="The Zeek Project", + maintainer_email="info@zeek.org", + url="https://github.com/zeek/package-manager", + scripts=["zkg"], + packages=["zeekpkg"], install_requires=install_requires, classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'License :: OSI Approved :: University of Illinois/NCSA Open Source License', - 'Operating System :: POSIX :: Linux', - 'Operating System :: MacOS :: MacOS X', - 'Programming Language :: Python :: 3', - 'Topic :: System :: Networking :: Monitoring', - 'Topic :: Utilities', + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: University of Illinois/NCSA Open Source License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + "Programming Language :: Python :: 3", + "Topic :: System :: Networking :: Monitoring", + "Topic :: Utilities", ], ) diff --git a/testing/templates/foo/__init__.py b/testing/templates/foo/__init__.py index 3dd4c9d6..777d836c 100644 --- a/testing/templates/foo/__init__.py +++ b/testing/templates/foo/__init__.py @@ -1,46 +1,47 @@ import zeekpkg.template import zeekpkg.uservar -TEMPLATE_API_VERSION = '1.0.0' +TEMPLATE_API_VERSION = "1.0.0" + class Package(zeekpkg.template.Package): def contentdir(self): - return 'package' + return "package" def needed_user_vars(self): - return ['name'] + return ["name"] def validate(self, tmpl): - if not tmpl.lookup_param('name'): - raise zeekpkg.template.InputError( - 'package requires a name') + if not tmpl.lookup_param("name"): + raise zeekpkg.template.InputError("package requires a name") class Readme(zeekpkg.template.Feature): def contentdir(self): - return 'readme' + return "readme" def needed_user_vars(self): - return ['readme'] + return ["readme"] class Template(zeekpkg.template.Template): def define_user_vars(self): return [ zeekpkg.uservar.UserVar( - 'name', desc='the name of the package, e.g. "FooBar"'), + "name", desc='the name of the package, e.g. "FooBar"' + ), zeekpkg.uservar.UserVar( - 'readme', desc='Content of the README file', - val='This is a README.'), + "readme", desc="Content of the README file", val="This is a README." + ), ] def apply_user_vars(self, uvars): for uvar in uvars: - if uvar.name() == 'name': - self.define_param('name', uvar.val()) - self.define_param('module', uvar.val().upper()) - if uvar.name() == 'readme': - self.define_param('readme', uvar.val()) + if uvar.name() == "name": + self.define_param("name", uvar.val()) + self.define_param("module", uvar.val().upper()) + if uvar.name() == "readme": + self.define_param("readme", uvar.val()) def package(self): return Package() diff --git a/zeekpkg/__init__.py b/zeekpkg/__init__.py index 65ad0f96..cd54175a 100644 --- a/zeekpkg/__init__.py +++ b/zeekpkg/__init__.py @@ -10,7 +10,7 @@ import logging __version__ = "2.13.0-9" -__all__ = ['manager', 'package', 'source', 'template', 'uservar'] +__all__ = ["manager", "package", "source", "template", "uservar"] LOG = logging.getLogger(__name__) LOG.addHandler(logging.NullHandler()) diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index 11225616..f8a5ba25 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -12,6 +12,7 @@ import git import semantic_version as semver + def make_dir(path): """Create a directory or do nothing if it already exists. @@ -31,7 +32,7 @@ def normalize_version_tag(tag): """Given version string "vX.Y.Z", returns "X.Y.Z". Returns other input strings unchanged. """ - if len(tag) > 1 and tag[0] == 'v' and tag[1].isdigit(): + if len(tag) > 1 and tag[0] == "v" and tag[1].isdigit(): return tag[1:] return tag @@ -84,6 +85,7 @@ def safe_tarfile_extractall(tfile, destdir): Raises: Exception: if the tarfile would extract outside destdir """ + def is_within_directory(directory, target): abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) @@ -94,7 +96,7 @@ def is_within_directory(directory, target): for member in tar.getmembers(): member_path = os.path.join(destdir, member.name) if not is_within_directory(destdir, member_path): - raise Exception('attempted path traversal in tarfile') + raise Exception("attempted path traversal in tarfile") tar.extractall(destdir) @@ -103,7 +105,7 @@ def find_sentence_end(s): beg = 0 while True: - period_idx = s.find('.', beg) + period_idx = s.find(".", beg) if period_idx == -1: return -1 @@ -122,21 +124,23 @@ def find_sentence_end(s): def git_clone(git_url, dst_path, shallow=False): if shallow: try: - git.Git().clone(git_url, dst_path, '--no-single-branch', recursive=True, depth=1) + git.Git().clone( + git_url, dst_path, "--no-single-branch", recursive=True, depth=1 + ) except git.exc.GitCommandError: - if not git_url.startswith('.') and not git_url.startswith('/'): + if not git_url.startswith(".") and not git_url.startswith("/"): # Not a local repo raise - if not os.path.exists(os.path.join(git_url, '.git', 'shallow')): + if not os.path.exists(os.path.join(git_url, ".git", "shallow")): raise # Some git versions cannot clone from a shallow-clone, so copy # and reset/clean it to a pristine condition. copy_over_path(git_url, dst_path) rval = git.Repo(dst_path) - rval.git.reset('--hard') - rval.git.clean('-ffdx') + rval.git.reset("--hard") + rval.git.clean("-ffdx") else: git.Git().clone(git_url, dst_path, recursive=True) @@ -149,7 +153,7 @@ def git_clone(git_url, dst_path, shallow=False): # fetching from a remote location (e.g. when unbundling). This # unintended inheritence of "origin" seems to only happen when cloning a # local git repo that has submodules ? - rval.git.remote('set-url', 'origin', git_url) + rval.git.remote("set-url", "origin", git_url) return rval @@ -165,8 +169,8 @@ def git_checkout(clone, version): git.exc.GitCommandError: if the git repo is invalid """ clone.git.checkout(version) - clone.git.submodule('sync', '--recursive') - clone.git.submodule('update', '--recursive', '--init') + clone.git.submodule("sync", "--recursive") + clone.git.submodule("update", "--recursive", "--init") def git_default_branch(repo): @@ -185,7 +189,7 @@ def git_default_branch(repo): """ try: - remote = repo.remote('origin') + remote = repo.remote("origin") except ValueError: remote = None @@ -197,20 +201,20 @@ def git_default_branch(repo): head_ref_name = None if head_ref_name: - remote_prefix = 'origin/' + remote_prefix = "origin/" if head_ref_name.startswith(remote_prefix): - return head_ref_name[len(remote_prefix):] + return head_ref_name[len(remote_prefix) :] return head_ref_name ref_names = [ref.name for ref in repo.refs] - if 'main' in ref_names: - return 'main' + if "main" in ref_names: + return "main" - if 'master' in ref_names: - return 'master' + if "master" in ref_names: + return "master" try: # See if there's a branch currently checked out @@ -249,8 +253,8 @@ def git_pull(repo): git.exc.GitCommandError: in case of git trouble """ repo.git.pull() - repo.git.submodule('sync', '--recursive') - repo.git.submodule('update', '--recursive', '--init') + repo.git.submodule("sync", "--recursive") + repo.git.submodule("update", "--recursive", "--init") def git_remote_urls(repo): @@ -262,13 +266,13 @@ def git_remote_urls(repo): cases. We use the config subsystem to query the URLs directly -- one of the fallback mechanisms in GitPython's Remote.urls() implementation. """ - remote_details = repo.git.config('--get-regexp', 'remote\..+\.url') + remote_details = repo.git.config("--get-regexp", "remote\..+\.url") remotes = {} - for line in remote_details.split('\n'): + for line in remote_details.split("\n"): try: remote, url = line.split(maxsplit=1) - remote = remote.split('.')[1] + remote = remote.split(".")[1] remotes[remote] = url except (ValueError, IndexError): pass @@ -278,14 +282,30 @@ def git_remote_urls(repo): def is_sha1(s): if not s: - return False; + return False if len(s) != 40: return False for c in s: - if c not in {'a', 'b', 'c', 'd', 'e', 'f', - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}: + if c not in { + "a", + "b", + "c", + "d", + "e", + "f", + "0", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + }: return False return True @@ -299,7 +319,7 @@ def find_program(prog_name): path, _ = os.path.split(prog_name) if path: - return prog_name if is_exe(prog_name) else '' + return prog_name if is_exe(prog_name) else "" for path in os.environ["PATH"].split(os.pathsep): path = os.path.join(path.strip('"'), prog_name) @@ -307,7 +327,7 @@ def find_program(prog_name): if is_exe(path): return path - return '' + return "" def std_encoding(stream): @@ -317,7 +337,7 @@ def std_encoding(stream): import locale if locale.getdefaultlocale()[1] is None: - return 'utf-8' + return "utf-8" return locale.getpreferredencoding() @@ -327,18 +347,23 @@ def read_zeek_config_line(stdout): def get_zeek_version(): - zeek_config = find_program('zeek-config') + zeek_config = find_program("zeek-config") if not zeek_config: - zeek_config = find_program('bro-config') + zeek_config = find_program("bro-config") if not zeek_config: - return '' + return "" import subprocess - cmd = subprocess.Popen([zeek_config, '--version'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - bufsize=1, universal_newlines=True) + + cmd = subprocess.Popen( + [zeek_config, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) return read_zeek_config_line(cmd.stdout) @@ -359,7 +384,7 @@ def load_source(filename): # Naming here is unimportant, since we access members of the new # module via the returned instance. - loader = importlib.machinery.SourceFileLoader('template_' + dirname, absname) + loader = importlib.machinery.SourceFileLoader("template_" + dirname, absname) mod = types.ModuleType(loader.name) loader.exec_module(mod) diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 0e821d2e..dab2453f 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -42,10 +42,7 @@ configparser_section_dict, safe_tarfile_extractall, ) -from .source import ( - AGGREGATE_DATA_FILE, - Source -) +from .source import AGGREGATE_DATA_FILE, Source from .package import ( METADATA_FILENAME, LEGACY_METADATA_FILENAME, @@ -61,7 +58,7 @@ Package, PackageInfo, PackageStatus, - InstalledPackage + InstalledPackage, ) from .uservar import ( UserVar, @@ -71,16 +68,17 @@ LOG, ) + class Stage(object): def __init__(self, manager, state_dir=None): self.manager = manager if state_dir: self.state_dir = state_dir - self.clone_dir = os.path.join(self.state_dir, 'clones') - self.script_dir = os.path.join(self.state_dir, 'scripts', 'packages') - self.plugin_dir = os.path.join(self.state_dir, 'plugins', 'packages') - self.bin_dir = os.path.join(self.state_dir, 'bin') + self.clone_dir = os.path.join(self.state_dir, "clones") + self.script_dir = os.path.join(self.state_dir, "scripts", "packages") + self.plugin_dir = os.path.join(self.state_dir, "plugins", "packages") + self.bin_dir = os.path.join(self.state_dir, "bin") else: # Stages not given a test directory are essentially a shortcut to # standard functionality; this doesn't require all directories: @@ -115,22 +113,27 @@ def populate(self): make_symlink(entry.path, os.path.join(self.clone_dir, entry.name)) def get_subprocess_env(self): - zeekpath = os.environ.get('ZEEKPATH') or os.environ.get('BROPATH') - pluginpath = os.environ.get('ZEEK_PLUGIN_PATH') or os.environ.get('BRO_PLUGIN_PATH') + zeekpath = os.environ.get("ZEEKPATH") or os.environ.get("BROPATH") + pluginpath = os.environ.get("ZEEK_PLUGIN_PATH") or os.environ.get( + "BRO_PLUGIN_PATH" + ) if not (zeekpath and pluginpath): - zeek_config = find_program('zeek-config') - path_option = '--zeekpath' + zeek_config = find_program("zeek-config") + path_option = "--zeekpath" if not zeek_config: - zeek_config = find_program('bro-config') - path_option = '--bropath' + zeek_config = find_program("bro-config") + path_option = "--bropath" if zeek_config: - cmd = subprocess.Popen([zeek_config, path_option, '--plugin_dir'], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=1, universal_newlines=True) + cmd = subprocess.Popen( + [zeek_config, path_option, "--plugin_dir"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) line1 = read_zeek_config_line(cmd.stdout) line2 = read_zeek_config_line(cmd.stdout) @@ -146,13 +149,13 @@ def get_subprocess_env(self): pluginpath = os.path.dirname(self.plugin_dir) + os.pathsep + pluginpath env = os.environ.copy() - env['PATH'] = self.bin_dir + os.pathsep + os.environ.get('PATH', '') - env['ZEEKPATH'] = zeekpath - env['ZEEK_PLUGIN_PATH'] = pluginpath - env['BROPATH'] = zeekpath - env['BRO_PLUGIN_PATH'] = pluginpath + env["PATH"] = self.bin_dir + os.pathsep + os.environ.get("PATH", "") + env["ZEEKPATH"] = zeekpath + env["ZEEK_PLUGIN_PATH"] = pluginpath + env["BROPATH"] = zeekpath + env["BRO_PLUGIN_PATH"] = pluginpath - return env, '' + return env, "" class Manager(object): @@ -232,8 +235,15 @@ class Manager(object): load all installed packages that have been marked as loaded. """ - def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', - user_vars=None, bin_dir=''): + def __init__( + self, + state_dir, + script_dir, + plugin_dir, + zeek_dist="", + user_vars=None, + bin_dir="", + ): """Creates a package manager instance. Args: @@ -256,7 +266,7 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', OSError: when a package manager state directory can't be created IOError: when a package manager state file can't be created """ - LOG.debug('init Manager version %s', __version__) + LOG.debug("init Manager version %s", __version__) self.sources = {} self.installed_pkgs = {} # The bro_dist attribute exists just for backward compatibility @@ -264,21 +274,20 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', self.zeek_dist = zeek_dist self.state_dir = state_dir self.user_vars = {} if user_vars is None else user_vars - self.backup_dir = os.path.join(self.state_dir, 'backups') - self.log_dir = os.path.join(self.state_dir, 'logs') - self.scratch_dir = os.path.join(self.state_dir, 'scratch') + self.backup_dir = os.path.join(self.state_dir, "backups") + self.log_dir = os.path.join(self.state_dir, "logs") + self.scratch_dir = os.path.join(self.state_dir, "scratch") self._script_dir = script_dir - self.script_dir = os.path.join(script_dir, 'packages') + self.script_dir = os.path.join(script_dir, "packages") self._plugin_dir = plugin_dir - self.plugin_dir = os.path.join(plugin_dir, 'packages') - self.bin_dir = bin_dir or os.path.join(self.state_dir, 'bin') - self.source_clonedir = os.path.join(self.state_dir, 'clones', 'source') - self.package_clonedir = os.path.join( - self.state_dir, 'clones', 'package') - self.package_testdir = os.path.join(self.state_dir, 'testing') - self.manifest = os.path.join(self.state_dir, 'manifest.json') - self.autoload_script = os.path.join(self.script_dir, 'packages.zeek') - self.autoload_package = os.path.join(self.script_dir, '__load__.zeek') + self.plugin_dir = os.path.join(plugin_dir, "packages") + self.bin_dir = bin_dir or os.path.join(self.state_dir, "bin") + self.source_clonedir = os.path.join(self.state_dir, "clones", "source") + self.package_clonedir = os.path.join(self.state_dir, "clones", "package") + self.package_testdir = os.path.join(self.state_dir, "testing") + self.manifest = os.path.join(self.state_dir, "manifest.json") + self.autoload_script = os.path.join(self.script_dir, "packages.zeek") + self.autoload_package = os.path.join(self.script_dir, "__load__.zeek") make_dir(self.state_dir) make_dir(self.log_dir) make_dir(self.scratch_dir) @@ -287,21 +296,20 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', make_dir(self.script_dir) make_dir(self.plugin_dir) make_dir(self.bin_dir) - _create_readme(os.path.join(self.script_dir, 'README')) - _create_readme(os.path.join(self.plugin_dir, 'README')) + _create_readme(os.path.join(self.script_dir, "README")) + _create_readme(os.path.join(self.plugin_dir, "README")) if not os.path.exists(self.manifest): self._write_manifest() prev_script_dir, prev_plugin_dir, prev_bin_dir = self._read_manifest() - refresh_bin_dir = False # whether we need to updates link in bin_dir - relocating_bin_dir = False # whether bin_dir has relocated + refresh_bin_dir = False # whether we need to updates link in bin_dir + relocating_bin_dir = False # whether bin_dir has relocated need_manifest_update = False if os.path.realpath(prev_script_dir) != os.path.realpath(self.script_dir): - LOG.info('relocating script_dir %s -> %s', prev_script_dir, - self.script_dir) + LOG.info("relocating script_dir %s -> %s", prev_script_dir, self.script_dir) if os.path.exists(prev_script_dir): delete_path(self.script_dir) @@ -314,19 +322,16 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', new_link = os.path.join(self.zeekpath(), pkg_name) if os.path.lexists(old_link): - LOG.info('moving package link %s -> %s', - old_link, new_link) + LOG.info("moving package link %s -> %s", old_link, new_link) shutil.move(old_link, new_link) else: - LOG.info('skip moving package link %s -> %s', - old_link, new_link) + LOG.info("skip moving package link %s -> %s", old_link, new_link) need_manifest_update = True refresh_bin_dir = True if os.path.realpath(prev_plugin_dir) != os.path.realpath(self.plugin_dir): - LOG.info('relocating plugin_dir %s -> %s', prev_plugin_dir, - self.plugin_dir) + LOG.info("relocating plugin_dir %s -> %s", prev_plugin_dir, self.plugin_dir) if os.path.exists(prev_plugin_dir): delete_path(self.plugin_dir) @@ -335,8 +340,10 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', need_manifest_update = True refresh_bin_dir = True - if prev_bin_dir and os.path.realpath(prev_bin_dir) != os.path.realpath(self.bin_dir): - LOG.info('relocating bin_dir %s -> %s', prev_bin_dir, self.bin_dir) + if prev_bin_dir and os.path.realpath(prev_bin_dir) != os.path.realpath( + self.bin_dir + ): + LOG.info("relocating bin_dir %s -> %s", prev_bin_dir, self.bin_dir) need_manifest_update = True refresh_bin_dir = True relocating_bin_dir = True @@ -357,15 +364,15 @@ def __init__(self, state_dir, script_dir, plugin_dir, zeek_dist='', self._write_manifest() self._write_autoloader() - make_symlink('packages.zeek', self.autoload_package) + make_symlink("packages.zeek", self.autoload_package) # Backward compatibility (Pre-Zeek 3.0 does not handle .zeek files) - autoload_script_fallback = os.path.join(self.script_dir, 'packages.bro') - autoload_package_fallback = os.path.join(self.script_dir, '__load__.bro') + autoload_script_fallback = os.path.join(self.script_dir, "packages.bro") + autoload_package_fallback = os.path.join(self.script_dir, "__load__.bro") delete_path(autoload_script_fallback) delete_path(autoload_package_fallback) - make_symlink('packages.zeek', autoload_script_fallback) - make_symlink('packages.zeek', autoload_package_fallback) + make_symlink("packages.zeek", autoload_script_fallback) + make_symlink("packages.zeek", autoload_package_fallback) def _write_autoloader(self): """Write the :file:`packages.zeek` loader script. @@ -373,13 +380,15 @@ def _write_autoloader(self): Raises: IOError: if :file:`packages.zeek` loader script cannot be written """ - with open(self.autoload_script, 'w') as f: - content = ('# WARNING: This file is managed by zkg.\n' - '# Do not make direct modifications here.\n') + with open(self.autoload_script, "w") as f: + content = ( + "# WARNING: This file is managed by zkg.\n" + "# Do not make direct modifications here.\n" + ) for ipkg in self.loaded_packages(): if self.has_scripts(ipkg): - content += '@load ./{}\n'.format(ipkg.package.name) + content += "@load ./{}\n".format(ipkg.package.name) f.write(content) @@ -399,25 +408,31 @@ def _write_plugin_magic(self, ipkg): directory already contains a correctly named magic file, this function does nothing. """ - magic_path = os.path.join( - self.plugin_dir, ipkg.package.name, PLUGIN_MAGIC_FILE) + magic_path = os.path.join(self.plugin_dir, ipkg.package.name, PLUGIN_MAGIC_FILE) magic_path_disabled = os.path.join( - self.plugin_dir, ipkg.package.name, PLUGIN_MAGIC_FILE_DISABLED) + self.plugin_dir, ipkg.package.name, PLUGIN_MAGIC_FILE_DISABLED + ) if ipkg.status.is_loaded: if os.path.exists(magic_path_disabled): try: os.rename(magic_path_disabled, magic_path) except OSError as exception: - LOG.warning('could not enable plugin: %s: %s'.format( - type(exception).__name__, exception)) + LOG.warning( + "could not enable plugin: %s: %s".format( + type(exception).__name__, exception + ) + ) else: if os.path.exists(magic_path): try: os.rename(magic_path, magic_path_disabled) except OSError as exception: - LOG.warning('could not disable plugin: %s: %s'.format( - type(exception).__name__, exception)) + LOG.warning( + "could not disable plugin: %s: %s".format( + type(exception).__name__, exception + ) + ) def _read_manifest(self): """Read the manifest file containing the list of installed packages. @@ -428,26 +443,26 @@ def _read_manifest(self): Raises: IOError: when the manifest file can't be read """ - with open(self.manifest, 'r') as f: + with open(self.manifest, "r") as f: data = json.load(f) - version = data['manifest_version'] - pkg_list = data['installed_packages'] + version = data["manifest_version"] + pkg_list = data["installed_packages"] self.installed_pkgs = {} for dicts in pkg_list: - pkg_dict = dicts['package_dict'] - status_dict = dicts['status_dict'] - pkg_name = pkg_dict['name'] + pkg_dict = dicts["package_dict"] + status_dict = dicts["status_dict"] + pkg_name = pkg_dict["name"] - if version == 0 and 'index_data' in pkg_dict: - del pkg_dict['index_data'] + if version == 0 and "index_data" in pkg_dict: + del pkg_dict["index_data"] - pkg_dict['canonical'] = True; + pkg_dict["canonical"] = True pkg = Package(**pkg_dict) status = PackageStatus(**status_dict) self.installed_pkgs[pkg_name] = InstalledPackage(pkg, status) - return data['script_dir'], data['plugin_dir'], data.get('bin_dir', None) + return data["script_dir"], data["plugin_dir"], data.get("bin_dir", None) def _write_manifest(self): """Writes the manifest file containing the list of installed packages. @@ -458,14 +473,22 @@ def _write_manifest(self): pkg_list = [] for _, installed_pkg in self.installed_pkgs.items(): - pkg_list.append({'package_dict': installed_pkg.package.__dict__, - 'status_dict': installed_pkg.status.__dict__}) + pkg_list.append( + { + "package_dict": installed_pkg.package.__dict__, + "status_dict": installed_pkg.status.__dict__, + } + ) - data = {'manifest_version': 1, 'script_dir': self.script_dir, - 'plugin_dir': self.plugin_dir, 'bin_dir': self.bin_dir, - 'installed_packages': pkg_list} + data = { + "manifest_version": 1, + "script_dir": self.script_dir, + "plugin_dir": self.plugin_dir, + "bin_dir": self.bin_dir, + "installed_packages": pkg_list, + } - with open(self.manifest, 'w') as f: + with open(self.manifest, "w") as f: json.dump(data, f, indent=2, sort_keys=True) def zeekpath(self): @@ -519,8 +542,9 @@ def add_source(self, name, git_url): LOG.debug('duplicate source "%s"', name) return True - return 'source already exists with different URL: {}'.format( - existing_source.git_url) + return "source already exists with different URL: {}".format( + existing_source.git_url + ) clone_path = os.path.join(self.source_clonedir, name) @@ -533,15 +557,19 @@ def add_source(self, name, git_url): # is confusing the versioning logic. Note that per the git-clone # docs git recognizes scp-style URLs only when there are no slashes # before the first colon. - colonidx, slashidx = git_url.find(':'), git_url.find('/') - - if '://' not in git_url and colonidx > 0 and (slashidx == -1 or slashidx > colonidx): - parse_result = urlparse('ssh://' + git_url.replace(':', '/', 1)) + colonidx, slashidx = git_url.find(":"), git_url.find("/") + + if ( + "://" not in git_url + and colonidx > 0 + and (slashidx == -1 or slashidx > colonidx) + ): + parse_result = urlparse("ssh://" + git_url.replace(":", "/", 1)) else: parse_result = urlparse(git_url) - if parse_result.path and '@' in parse_result.path: - git_url, version = git_url.rsplit('@', 1) + if parse_result.path and "@" in parse_result.path: + git_url, version = git_url.rsplit("@", 1) try: source = Source( @@ -551,12 +579,12 @@ def add_source(self, name, git_url): version=version, ) except git.exc.GitCommandError as error: - LOG.warning('failed to clone git repo: %s', error) - return 'failed to clone git repo' + LOG.warning("failed to clone git repo: %s", error) + return "failed to clone git repo" else: self.sources[name] = source - return '' + return "" def source_packages(self): """Return a list of :class:`.package.Package` within all sources.""" @@ -577,8 +605,10 @@ def installed_package_dependencies(self): Package-name / dependency-name / and version-requirement values are all strings. """ - return {name: ipkg.package.dependencies() - for name, ipkg in self.installed_pkgs.items()} + return { + name: ipkg.package.dependencies() + for name, ipkg in self.installed_pkgs.items() + } def loaded_packages(self): """Return list of loaded :class:`.package.InstalledPackage`.""" @@ -601,7 +631,7 @@ def package_build_log(self, pkg_path): to the package: "foo", "alice/foo", or "zeek/alice/foo". """ name = name_from_path(pkg_path) - return os.path.join(self.log_dir, '{}-build.log'.format(name)) + return os.path.join(self.log_dir, "{}-build.log".format(name)) def match_source_packages(self, pkg_path): """Return a list of :class:`.package.Package` that match a given path. @@ -663,8 +693,7 @@ def has_scripts(self, installed_pkg): Returns: bool: True if the package has installed Zeek scripts. """ - return os.path.exists(os.path.join(self.script_dir, - installed_pkg.package.name)) + return os.path.exists(os.path.join(self.script_dir, installed_pkg.package.name)) def has_plugin(self, installed_pkg): """Return whether a :class:`.package.InstalledPackage` installed a plugin. @@ -676,8 +705,7 @@ def has_plugin(self, installed_pkg): Returns: bool: True if the package has installed a Zeek plugin. """ - return os.path.exists(os.path.join(self.plugin_dir, - installed_pkg.package.name)) + return os.path.exists(os.path.join(self.plugin_dir, installed_pkg.package.name)) def save_temporary_config_files(self, installed_pkg): """Return a list of temporary package config file backups. @@ -695,8 +723,9 @@ def save_temporary_config_files(self, installed_pkg): so make use of it before doing any further operations on packages. """ import re + metadata = installed_pkg.package.metadata - config_files = re.split(',\s*', metadata.get('config_files', '')) + config_files = re.split(",\s*", metadata.get("config_files", "")) if not config_files: return [] @@ -709,11 +738,14 @@ def save_temporary_config_files(self, installed_pkg): config_file_path = os.path.join(clone_dir, config_file) if not os.path.isfile(config_file_path): - LOG.info("package '%s' claims config file at '%s'," - " but it does not exist", pkg_name, config_file) + LOG.info( + "package '%s' claims config file at '%s'," " but it does not exist", + pkg_name, + config_file, + ) continue - backup_file = os.path.join(self.scratch_dir, 'tmpcfg', config_file) + backup_file = os.path.join(self.scratch_dir, "tmpcfg", config_file) make_dir(os.path.dirname(backup_file)) shutil.copy2(config_file_path, backup_file) rval.append((config_file, backup_file)) @@ -735,8 +767,9 @@ def modified_config_files(self, installed_pkg): config file is currently installed. """ import re + metadata = installed_pkg.package.metadata - config_files = re.split(',\s*', metadata.get('config_files', '')) + config_files = re.split(",\s*", metadata.get("config_files", "")) if not config_files: return [] @@ -745,41 +778,56 @@ def modified_config_files(self, installed_pkg): script_install_dir = os.path.join(self.script_dir, pkg_name) plugin_install_dir = os.path.join(self.plugin_dir, pkg_name) clone_dir = os.path.join(self.package_clonedir, pkg_name) - script_dir = metadata.get('script_dir', '') - plugin_dir = metadata.get('plugin_dir', 'build') + script_dir = metadata.get("script_dir", "") + plugin_dir = metadata.get("plugin_dir", "build") rval = [] for config_file in config_files: their_config_file_path = os.path.join(clone_dir, config_file) if not os.path.isfile(their_config_file_path): - LOG.info("package '%s' claims config file at '%s'," - " but it does not exist", pkg_name, config_file) + LOG.info( + "package '%s' claims config file at '%s'," " but it does not exist", + pkg_name, + config_file, + ) continue if config_file.startswith(plugin_dir): our_config_file_path = os.path.join( - plugin_install_dir, config_file[len(plugin_dir):]) + plugin_install_dir, config_file[len(plugin_dir) :] + ) if not os.path.isfile(our_config_file_path): - LOG.info("package '%s' config file '%s' not found" - " in plugin_dir: %s", pkg_name, config_file, - our_config_file_path) + LOG.info( + "package '%s' config file '%s' not found" " in plugin_dir: %s", + pkg_name, + config_file, + our_config_file_path, + ) continue elif config_file.startswith(script_dir): our_config_file_path = os.path.join( - script_install_dir, config_file[len(script_dir):]) + script_install_dir, config_file[len(script_dir) :] + ) if not os.path.isfile(our_config_file_path): - LOG.info("package '%s' config file '%s' not found" - " in script_dir: %s", pkg_name, config_file, - our_config_file_path) + LOG.info( + "package '%s' config file '%s' not found" " in script_dir: %s", + pkg_name, + config_file, + our_config_file_path, + ) continue else: # Their config file is outside script/plugin install dirs, # so no way user has it even installed, much less modified. - LOG.warning("package '%s' config file '%s' not within" - " plugin_dir or script_dir", pkg_name, config_file) + LOG.warning( + "package '%s' config file '%s' not within" + " plugin_dir or script_dir", + pkg_name, + config_file, + ) continue if not filecmp.cmp(our_config_file_path, their_config_file_path): @@ -802,6 +850,7 @@ def backup_modified_files(self, backup_subdir, modified_files): `modified_files`. """ import time + rval = [] for modified_file in modified_files: @@ -809,9 +858,8 @@ def backup_modified_files(self, backup_subdir, modified_files): config_file_dir = os.path.dirname(config_file) install_path = modified_file[1] filename = os.path.basename(install_path) - backup_dir = os.path.join( - self.backup_dir, backup_subdir, config_file_dir) - timestamp = time.strftime('.%Y-%m-%d-%H:%M:%S') + backup_dir = os.path.join(self.backup_dir, backup_subdir, config_file_dir) + timestamp = time.strftime(".%Y-%m-%d-%H:%M:%S") backup_path = os.path.join(backup_dir, filename + timestamp) make_dir(backup_dir) shutil.copy2(install_path, backup_path) @@ -834,7 +882,7 @@ class SourceAggregationResults(object): the failure. """ - def __init__(self, refresh_error='', package_issues=[]): + def __init__(self, refresh_error="", package_issues=[]): self.refresh_error = refresh_error self.package_issues = package_issues @@ -896,16 +944,15 @@ def refresh_source(self, name, aggregate=False, push=False): def _refresh_source(self, name, aggregate=False, push=False): """Used by :meth:`refresh_source()` and :meth:`aggregate_source()`.""" if name not in self.sources: - return self.SourceAggregationResults('source name does not exist') + return self.SourceAggregationResults("source name does not exist") source = self.sources[name] LOG.debug('refresh "%s": pulling %s', name, source.git_url) - aggregate_file = os.path.join(source.clone.working_dir, - AGGREGATE_DATA_FILE) - agg_file_ours = os.path.join( - self.scratch_dir, AGGREGATE_DATA_FILE) - agg_file_their_orig = os.path.join(self.scratch_dir, - AGGREGATE_DATA_FILE + '.orig') + aggregate_file = os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) + agg_file_ours = os.path.join(self.scratch_dir, AGGREGATE_DATA_FILE) + agg_file_their_orig = os.path.join( + self.scratch_dir, AGGREGATE_DATA_FILE + ".orig" + ) delete_path(agg_file_ours) delete_path(agg_file_their_orig) @@ -914,18 +961,19 @@ def _refresh_source(self, name, aggregate=False, push=False): shutil.copy2(aggregate_file, agg_file_ours) source.clone.git.reset(hard=True) - source.clone.git.clean('-f', '-x', '-d') + source.clone.git.clean("-f", "-x", "-d") if os.path.isfile(aggregate_file): shutil.copy2(aggregate_file, agg_file_their_orig) try: - source.clone.git.fetch('--recurse-submodules=yes') + source.clone.git.fetch("--recurse-submodules=yes") git_pull(source.clone) except git.exc.GitCommandError as error: - LOG.error('failed to pull source %s: %s', name, error) + LOG.error("failed to pull source %s: %s", name, error) return self.SourceAggregationResults( - 'failed to pull from remote source: {}'.format(error)) + "failed to pull from remote source: {}".format(error) + ) if os.path.isfile(agg_file_ours): if os.path.isfile(aggregate_file): @@ -936,11 +984,11 @@ def _refresh_source(self, name, aggregate=False, push=False): # Their file hasn't changed, use ours. shutil.copy2(agg_file_ours, aggregate_file) LOG.debug( - "aggegrate file in source unchanged, restore local one") + "aggegrate file in source unchanged, restore local one" + ) else: # Their file changed, use theirs. - LOG.debug( - "aggegrate file in source changed, discard local one") + LOG.debug("aggegrate file in source changed, discard local one") else: # File was untracked before pull and tracked after, # use their version. @@ -969,7 +1017,7 @@ def _refresh_source(self, name, aggregate=False, push=False): urls = [] with open(index_file) as f: - urls = [line.rstrip('\n') for line in f] + urls = [line.rstrip("\n") for line in f] for url in urls: pkg_name = name_from_path(url) @@ -979,8 +1027,9 @@ def _refresh_source(self, name, aggregate=False, push=False): try: clone = git_clone(url, clonepath, shallow=True) except git.exc.GitCommandError as error: - LOG.warn('failed to clone %s, skipping aggregation: %s', - url, error) + LOG.warn( + "failed to clone %s, skipping aggregation: %s", url, error + ) aggregation_issues.append((url, repr(error))) continue @@ -994,27 +1043,38 @@ def _refresh_source(self, name, aggregate=False, push=False): try: git_checkout(clone, version) except git.exc.GitCommandError as error: - LOG.warn('failed to checkout branch/version "%s" of %s, ' - 'skipping aggregation: %s', version, url, error) + LOG.warn( + 'failed to checkout branch/version "%s" of %s, ' + "skipping aggregation: %s", + version, + url, + error, + ) msg = 'failed to checkout branch/version "{}": {}'.format( - version, repr(error)) + version, repr(error) + ) aggregation_issues.append((url, msg)) continue metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file) + metadata_parser, metadata_file + ) if invalid_reason: - LOG.warn('skipping aggregation of %s: bad metadata: %s', - url, invalid_reason) + LOG.warn( + "skipping aggregation of %s: bad metadata: %s", + url, + invalid_reason, + ) aggregation_issues.append((url, invalid_reason)) continue metadata = _get_package_metadata(metadata_parser) - index_dir = os.path.dirname(index_file)[len( - self.source_clonedir) + len(name) + 2:] + index_dir = os.path.dirname(index_file)[ + len(self.source_clonedir) + len(name) + 2 : + ] qualified_name = os.path.join(index_dir, pkg_name) parser.add_section(qualified_name) @@ -1022,35 +1082,42 @@ def _refresh_source(self, name, aggregate=False, push=False): for key, value in sorted(metadata.items()): parser.set(qualified_name, key, value) - parser.set(qualified_name, 'url', url) - parser.set(qualified_name, 'version', version) + parser.set(qualified_name, "url", url) + parser.set(qualified_name, "version", version) if qualified_name not in prev_packages: agg_adds.append(qualified_name) else: - prev_meta = configparser_section_dict(prev_parser, qualified_name) + prev_meta = configparser_section_dict( + prev_parser, qualified_name + ) new_meta = configparser_section_dict(parser, qualified_name) if prev_meta != new_meta: agg_mods.append(qualified_name) - with open(aggregate_file, 'w') as f: + with open(aggregate_file, "w") as f: parser.write(f) agg_dels = list(prev_packages.difference(set(parser.sections()))) - adds_str = ' (' + ', '.join(sorted(agg_adds)) + ')' if agg_adds else '' - mods_str = ' (' + ', '.join(sorted(agg_mods)) + ')' if agg_mods else '' - dels_str = ' (' + ', '.join(sorted(agg_dels)) + ')' if agg_dels else '' - - LOG.debug('metadata refresh: %d additions%s, %d changes%s, %d removals%s', - len(agg_adds), adds_str, - len(agg_mods), mods_str, - len(agg_dels), dels_str) - + adds_str = " (" + ", ".join(sorted(agg_adds)) + ")" if agg_adds else "" + mods_str = " (" + ", ".join(sorted(agg_mods)) + ")" if agg_mods else "" + dels_str = " (" + ", ".join(sorted(agg_dels)) + ")" if agg_dels else "" + + LOG.debug( + "metadata refresh: %d additions%s, %d changes%s, %d removals%s", + len(agg_adds), + adds_str, + len(agg_mods), + mods_str, + len(agg_dels), + dels_str, + ) if push: - if os.path.isfile(os.path.join(source.clone.working_dir, - AGGREGATE_DATA_FILE)): + if os.path.isfile( + os.path.join(source.clone.working_dir, AGGREGATE_DATA_FILE) + ): source.clone.git.add(AGGREGATE_DATA_FILE) if source.clone.is_dirty(): @@ -1061,12 +1128,13 @@ def _refresh_source(self, name, aggregate=False, push=False): # why one would use zkg for this as opposed to git # itself. source.clone.git.commit( - '--no-verify', '--message', 'Update aggregated metadata.') + "--no-verify", "--message", "Update aggregated metadata." + ) LOG.info('committed package source "%s" metadata update', name) - source.clone.git.push('--no-verify') + source.clone.git.push("--no-verify") - return self.SourceAggregationResults('', aggregation_issues) + return self.SourceAggregationResults("", aggregation_issues) def refresh_installed_packages(self): """Fetch latest git information for installed packages. @@ -1080,16 +1148,20 @@ def refresh_installed_packages(self): for ipkg in self.installed_packages(): clonepath = os.path.join(self.package_clonedir, ipkg.package.name) clone = git.Repo(clonepath) - LOG.debug('fetch package %s', ipkg.package.qualified_name()) + LOG.debug("fetch package %s", ipkg.package.qualified_name()) try: - clone.git.fetch('--recurse-submodules=yes') + clone.git.fetch("--recurse-submodules=yes") except git.exc.GitCommandError as error: - LOG.warn('failed to fetch package %s: %s', - ipkg.package.qualified_name(), error) + LOG.warn( + "failed to fetch package %s: %s", + ipkg.package.qualified_name(), + error, + ) ipkg.status.is_outdated = _is_clone_outdated( - clone, ipkg.status.current_version, ipkg.status.tracking_method) + clone, ipkg.status.current_version, ipkg.status.tracking_method + ) self._write_manifest() @@ -1183,10 +1255,10 @@ def remove(self, pkg_path): link = os.path.join(self.bin_dir, os.path.basename(exec)) if os.path.islink(link): try: - LOG.debug('removing link %s', link) + LOG.debug("removing link %s", link) os.unlink(link) except os.error as excpt: - LOG.warn('cannot remove link for %s', exec) + LOG.warn("cannot remove link for %s", exec) del self.installed_pkgs[pkg_to_remove.name] self._write_manifest() @@ -1290,31 +1362,38 @@ def load(self, pkg_path): if not ipkg: LOG.info('loading "%s": no matching package', pkg_path) - return 'no such package' + return "no such package" if ipkg.status.is_loaded: LOG.debug('loading "%s": already loaded', pkg_path) - return '' + return "" - pkg_load_script = os.path.join(self.script_dir, ipkg.package.name, - '__load__.zeek') + pkg_load_script = os.path.join( + self.script_dir, ipkg.package.name, "__load__.zeek" + ) # Check if __load__.bro exists for compatibility with older packages - pkg_load_fallback = os.path.join(self.script_dir, ipkg.package.name, - '__load__.bro') - - if (not os.path.exists(pkg_load_script) and - not os.path.exists(pkg_load_fallback) and - not self.has_plugin(ipkg)): - LOG.debug('loading "%s": %s not found and package has no plugin', - pkg_path, pkg_load_script) - return 'no __load__.zeek within package script_dir and no plugin included' + pkg_load_fallback = os.path.join( + self.script_dir, ipkg.package.name, "__load__.bro" + ) + + if ( + not os.path.exists(pkg_load_script) + and not os.path.exists(pkg_load_fallback) + and not self.has_plugin(ipkg) + ): + LOG.debug( + 'loading "%s": %s not found and package has no plugin', + pkg_path, + pkg_load_script, + ) + return "no __load__.zeek within package script_dir and no plugin included" ipkg.status.is_loaded = True self._write_autoloader() self._write_manifest() self._write_plugin_magic(ipkg) LOG.debug('loaded "%s"', pkg_path) - return '' + return "" def loaded_package_states(self): """Save "loaded" state for all installed packages. @@ -1322,7 +1401,9 @@ def loaded_package_states(self): Returns: dict: dictionary of "loaded" status for installed packages """ - return {name: ipkg.status.is_loaded for name, ipkg in self.installed_pkgs.items()} + return { + name: ipkg.status.is_loaded for name, ipkg in self.installed_pkgs.items() + } def restore_loaded_package_states(self, saved_state): """Restores state for installed packages. @@ -1359,7 +1440,7 @@ def load_with_dependencies(self, pkg_name, visited=set()): # skip loading a package if it is not installed. if not ipkg: - return [(pkg_name, 'Loading dependency failed. Package not installed.')] + return [(pkg_name, "Loading dependency failed. Package not installed.")] load_error = self.load(pkg_name) @@ -1440,6 +1521,7 @@ def unload_with_unused_dependers(self, pkg_name): Raises: IOError: if the loader script or manifest can't be written """ + def _has_all_dependers_unloaded(item, dependers): for depender in dependers: ipkg = self.find_installed_package(depender) @@ -1462,7 +1544,7 @@ def _has_all_dependers_unloaded(item, dependers): # it is possible that this dependency has been removed via zkg if not ipkg: - errors.append((pkg, 'Package not installed.')) + errors.append((pkg, "Package not installed.")) return errors if ipkg.status.is_loaded: @@ -1472,7 +1554,7 @@ def _has_all_dependers_unloaded(item, dependers): # it is possible that this package has been removed via zkg if not ipkg: - errors.append((item, 'Package not installed.')) + errors.append((item, "Package not installed.")) return errors if ipkg.status.is_loaded: @@ -1486,27 +1568,34 @@ def _has_all_dependers_unloaded(item, dependers): if ipkg and ipkg.status.is_loaded: self.unload(dep) - errors.append((dep, '')) + errors.append((dep, "")) self.unload(item) - errors.append((item, '')) + errors.append((item, "")) continue # check if all dependers are unloaded elif _has_all_dependers_unloaded(item, dep_packages): self.unload(item) - errors.append((item, '')) + errors.append((item, "")) continue # package is in use else: dep_packages = self.list_depender_pkgs(pkg_name) - dep_listing = '' + dep_listing = "" for _name in dep_packages: dep_listing += '"{}", '.format(_name) - errors.append((item, 'Package is in use by other packages --- {}.'.format(dep_listing[:-2]))) + errors.append( + ( + item, + "Package is in use by other packages --- {}.".format( + dep_listing[:-2] + ), + ) + ) return errors return errors @@ -1566,7 +1655,7 @@ def bundle_info(self, bundle_file): contained in the bundle. """ LOG.debug('getting bundle info for file "%s"', bundle_file) - bundle_dir = os.path.join(self.scratch_dir, 'bundle') + bundle_dir = os.path.join(self.scratch_dir, "bundle") delete_path(bundle_dir) make_dir(bundle_dir) infos = [] @@ -1576,31 +1665,30 @@ def bundle_info(self, bundle_file): except Exception as error: return (str(error), infos) - manifest_file = os.path.join(bundle_dir, 'manifest.txt') - config = configparser.ConfigParser(delimiters='=') + manifest_file = os.path.join(bundle_dir, "manifest.txt") + config = configparser.ConfigParser(delimiters="=") config.optionxform = str if not config.read(manifest_file): - return ('invalid bundle: no manifest file', infos) + return ("invalid bundle: no manifest file", infos) - if not config.has_section('bundle'): - return ('invalid bundle: no [bundle] section in manifest file', - infos) + if not config.has_section("bundle"): + return ("invalid bundle: no [bundle] section in manifest file", infos) - manifest = config.items('bundle') + manifest = config.items("bundle") for git_url, version in manifest: - package = Package(git_url=git_url, name=git_url.split('/')[-1], - canonical=True) + package = Package( + git_url=git_url, name=git_url.split("/")[-1], canonical=True + ) pkg_path = os.path.join(bundle_dir, package.name) LOG.debug('getting info for bundled package "%s"', package.name) - pkg_info = self.info(pkg_path, version=version, - prefer_installed=False) + pkg_info = self.info(pkg_path, version=version, prefer_installed=False) infos.append((git_url, version, pkg_info)) - return ('', infos) + return ("", infos) - def info(self, pkg_path, version='', prefer_installed=True): + def info(self, pkg_path, version="", prefer_installed=True): """Retrieves information about a package. Args: @@ -1628,7 +1716,7 @@ def info(self, pkg_path, version='', prefer_installed=True): name = name_from_path(pkg_path) if not is_valid_package_name(name): - reason = 'Package name {!r} is not valid.'.format(name) + reason = "Package name {!r} is not valid.".format(name) return PackageInfo(Package(git_url=pkg_path), invalid_reason=reason) LOG.debug('getting info on "%s"', pkg_path) @@ -1639,8 +1727,7 @@ def info(self, pkg_path, version='', prefer_installed=True): pkg_name = ipkg.package.name clonepath = os.path.join(self.package_clonedir, pkg_name) clone = git.Repo(clonepath) - return _info_from_clone(clone, ipkg.package, status, - status.current_version) + return _info_from_clone(clone, ipkg.package, status, status.current_version) else: status = None matches = self.match_source_packages(pkg_path) @@ -1651,24 +1738,30 @@ def info(self, pkg_path, version='', prefer_installed=True): try: return self._info(package, status, version) except git.exc.GitCommandError as error: - LOG.info('getting info on "%s": invalid git repo path: %s', - pkg_path, error) - - LOG.info('getting info on "%s": matched no source package', - pkg_path) - reason = ('package name not found in sources and also' - ' not a usable git URL (invalid or inaccessible,' - ' use -vvv for details)') - return PackageInfo(package=package, invalid_reason=reason, - status=status) + LOG.info( + 'getting info on "%s": invalid git repo path: %s', pkg_path, error + ) + + LOG.info('getting info on "%s": matched no source package', pkg_path) + reason = ( + "package name not found in sources and also" + " not a usable git URL (invalid or inaccessible," + " use -vvv for details)" + ) + return PackageInfo(package=package, invalid_reason=reason, status=status) if len(matches) > 1: matches_string = [match.qualified_name() for match in matches] - LOG.info('getting info on "%s": matched multiple packages: %s', - pkg_path, matches_string) - reason = str.format('"{}" matches multiple packages, try a more' - ' specific name from: {}', - pkg_path, matches_string) + LOG.info( + 'getting info on "%s": matched multiple packages: %s', + pkg_path, + matches_string, + ) + reason = str.format( + '"{}" matches multiple packages, try a more' " specific name from: {}", + pkg_path, + matches_string, + ) return PackageInfo(invalid_reason=reason, status=status) package = matches[0] @@ -1676,11 +1769,9 @@ def info(self, pkg_path, version='', prefer_installed=True): try: return self._info(package, status, version) except git.exc.GitCommandError as error: - LOG.info('getting info on "%s": invalid git repo path: %s', - pkg_path, error) - reason = 'git repository is either invalid or unreachable' - return PackageInfo(package=package, invalid_reason=reason, - status=status) + LOG.info('getting info on "%s": invalid git repo path: %s', pkg_path, error) + reason = "git repository is either invalid or unreachable" + return PackageInfo(package=package, invalid_reason=reason, status=status) def _info(self, package, status, version): """Retrieves information about a package. @@ -1696,7 +1787,6 @@ def _info(self, package, status, version): versions = git_version_tags(clone) if not version: - if len(versions): version = versions[-1] else: @@ -1705,10 +1795,8 @@ def _info(self, package, status, version): try: git_checkout(clone, version) except git.exc.GitCommandError: - reason = 'no such commit, branch, or version tag: "{}"'.format( - version) - return PackageInfo(package=package, status=status, - invalid_reason=reason) + reason = 'no such commit, branch, or version tag: "{}"'.format(version) + return PackageInfo(package=package, status=status, invalid_reason=reason) LOG.debug('checked out "%s", branch/version "%s"', package, version) return _info_from_clone(clone, package, status, version) @@ -1728,9 +1816,12 @@ def package_versions(self, installed_package): clone = git.Repo(clonepath) return git_version_tags(clone) - def validate_dependencies(self, requested_packages, - ignore_installed_packages=False, - ignore_suggestions=False): + def validate_dependencies( + self, + requested_packages, + ignore_installed_packages=False, + ignore_suggestions=False, + ): """Validates package dependencies. Args: @@ -1772,8 +1863,8 @@ def validate_dependencies(self, requested_packages, reverse iteration of the list guarantees processing of dependencies prior to the depender packages. """ - class Node(object): + class Node(object): def __init__(self, name): self.name = name self.info = None @@ -1785,12 +1876,16 @@ def __init__(self, name): def __str__(self): return str.format( - '{}\n\trequested: {}\n\tinstalled: {}\n\tdependers: {}\n\tsuggestion: {}', - self.name, self.requested_version, self.installed_version, - self.dependers, self.is_suggestion) + "{}\n\trequested: {}\n\tinstalled: {}\n\tdependers: {}\n\tsuggestion: {}", + self.name, + self.requested_version, + self.installed_version, + self.dependers, + self.is_suggestion, + ) - graph = dict() # Node.name -> Node, nodes store edges - requests = [] # List of Node, just for requested packages + graph = dict() # Node.name -> Node, nodes store edges + requests = [] # List of Node, just for requested packages # 1. Try to make nodes for everything in the dependency graph... @@ -1799,8 +1894,10 @@ def __str__(self): info = self.info(name, version=version, prefer_installed=False) if info.invalid_reason: - return ('invalid package "{}": {}'.format( - name, info.invalid_reason), []) + return ( + 'invalid package "{}": {}'.format(name, info.invalid_reason), + [], + ) node = Node(info.package.qualified_name()) node.info = info @@ -1814,39 +1911,53 @@ def __str__(self): while to_process: (_, node) = to_process.popitem() - dd = node.info.dependencies(field='depends') - ds = node.info.dependencies(field='suggests') + dd = node.info.dependencies(field="depends") + ds = node.info.dependencies(field="suggests") if dd is None: - return (str.format('package "{}" has malformed "depends" field', - node.name), []) + return ( + str.format('package "{}" has malformed "depends" field', node.name), + [], + ) all_deps = dd.copy() if not ignore_suggestions: if ds is None: - return (str.format('package "{}" has malformed "suggests" field', - node.name), []) + return ( + str.format( + 'package "{}" has malformed "suggests" field', node.name + ), + [], + ) all_deps.update(ds) for dep_name, _ in all_deps.items(): - if dep_name == 'bro' or dep_name == 'zeek': + if dep_name == "bro" or dep_name == "zeek": # A zeek node will get added later. continue - if dep_name == 'bro-pkg' or dep_name == 'zkg': + if dep_name == "bro-pkg" or dep_name == "zkg": # A zkg node will get added later. continue # Suggestion status propagates to 'depends' field of suggested packages. - is_suggestion = node.is_suggestion or dep_name in ds and dep_name not in dd + is_suggestion = ( + node.is_suggestion or dep_name in ds and dep_name not in dd + ) info = self.info(dep_name, prefer_installed=False) if info.invalid_reason: - return (str.format( - 'package "{}" has invalid dependency "{}": {}', - node.name, dep_name, info.invalid_reason), []) + return ( + str.format( + 'package "{}" has invalid dependency "{}": {}', + node.name, + dep_name, + info.invalid_reason, + ), + [], + ) dep_name = info.package.qualified_name() @@ -1873,16 +1984,17 @@ def __str__(self): zeek_version = get_zeek_version() if zeek_version: - node = Node('zeek') + node = Node("zeek") node.installed_version = (TRACKING_METHOD_VERSION, zeek_version) - graph['zeek'] = node + graph["zeek"] = node else: LOG.warning( - 'could not get zeek version: no "zeek-config" or "bro-config" in PATH ?') + 'could not get zeek version: no "zeek-config" or "bro-config" in PATH ?' + ) - node = Node('zkg') + node = Node("zkg") node.installed_version = (TRACKING_METHOD_VERSION, __version__) - graph['zkg'] = node + graph["zkg"] = node for ipkg in self.installed_packages(): name = ipkg.package.qualified_name() @@ -1895,47 +2007,55 @@ def __str__(self): graph[node.name] = node graph[name].installed_version = ( - status.tracking_method, status.current_version) + status.tracking_method, + status.current_version, + ) # 2. Fill in the edges of the graph with dependency information. for name, node in graph.items(): - if name == 'zeek': + if name == "zeek": continue - if name == 'zkg': + if name == "zkg": continue - dd = node.info.dependencies(field='depends') - ds = node.info.dependencies(field='suggests') + dd = node.info.dependencies(field="depends") + ds = node.info.dependencies(field="suggests") if dd is None: - return (str.format('package "{}" has malformed "depends" field', - node.name), []) + return ( + str.format('package "{}" has malformed "depends" field', node.name), + [], + ) all_deps = dd.copy() if not ignore_suggestions: if ds is None: - return (str.format('package "{}" has malformed "suggests" field', - node.name), []) + return ( + str.format( + 'package "{}" has malformed "suggests" field', node.name + ), + [], + ) all_deps.update(ds) for dep_name, dep_version in all_deps.items(): - if dep_name == 'bro' or dep_name == 'zeek': - if 'zeek' in graph: - graph['zeek'].dependers[name] = dep_version - node.dependees['zeek'] = dep_version - elif dep_name == 'bro-pkg' or dep_name == 'zkg': - if 'zkg' in graph: - graph['zkg'].dependers[name] = dep_version - node.dependees['zkg'] = dep_version + if dep_name == "bro" or dep_name == "zeek": + if "zeek" in graph: + graph["zeek"].dependers[name] = dep_version + node.dependees["zeek"] = dep_version + elif dep_name == "bro-pkg" or dep_name == "zkg": + if "zkg" in graph: + graph["zkg"].dependers[name] = dep_version + node.dependees["zkg"] = dep_version else: for _, dependency_node in graph.items(): - if dependency_node.name == 'zeek': + if dependency_node.name == "zeek": continue - if dependency_node.name == 'zkg': + if dependency_node.name == "zkg": continue if dependency_node.info.package.matches_path(dep_name): @@ -1976,8 +2096,9 @@ def __str__(self): continue # A new package nothing depends on -- odd? - new_pkgs.append((node.info, node.info.best_version(), - node.is_suggestion)) + new_pkgs.append( + (node.info, node.info.best_version(), node.is_suggestion) + ) continue if node.requested_version: @@ -1986,118 +2107,176 @@ def __str__(self): if track_method == TRACKING_METHOD_BRANCH: for depender_name, version_spec in node.dependers.items(): - if version_spec == '*': + if version_spec == "*": continue - if version_spec.startswith('branch='): - version_spec = version_spec[len('branch='):] + if version_spec.startswith("branch="): + version_spec = version_spec[len("branch=") :] if version_spec == required_version: continue - return (str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: requested "{}" ({}),' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) elif track_method == TRACKING_METHOD_COMMIT: for depender_name, version_spec in node.dependers.items(): - if version_spec == '*': + if version_spec == "*": continue # Could allow commit= version specification like what # is done with branches, but unsure there's a common # use-case for it. - return (str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: requested "{}" ({}),' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) else: normal_version = normalize_version_tag(required_version) req_semver = semver.Version.coerce(normal_version) for depender_name, version_spec in node.dependers.items(): - if version_spec.startswith('branch='): - version_spec = version_spec[len('branch='):] - return (str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + if version_spec.startswith("branch="): + version_spec = version_spec[len("branch=") :] + return ( + str.format( + 'unsatisfiable dependency: requested "{}" ({}),' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) else: try: semver_spec = semver.Spec(version_spec) except ValueError: - return (str.format( - 'package "{}" has invalid semver spec: {}', - depender_name, version_spec), new_pkgs) + return ( + str.format( + 'package "{}" has invalid semver spec: {}', + depender_name, + version_spec, + ), + new_pkgs, + ) if req_semver not in semver_spec: - return (str.format( - 'unsatisfiable dependency: requested "{}" ({}),' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: requested "{}" ({}),' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) elif node.installed_version: # Check that installed version doesn't conflict with dependers. track_method, required_version = node.installed_version for depender_name, version_spec in node.dependers.items(): if track_method == TRACKING_METHOD_BRANCH: - if version_spec == '*': + if version_spec == "*": continue - if version_spec.startswith('branch='): - version_spec = version_spec[len('branch='):] + if version_spec.startswith("branch="): + version_spec = version_spec[len("branch=") :] if version_spec == required_version: continue - return (str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: "{}" ({}) is installed,' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) elif track_method == TRACKING_METHOD_COMMIT: - if version_spec == '*': + if version_spec == "*": continue # Could allow commit= version specification like what # is done with branches, but unsure there's a common # use-case for it. - return (str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: "{}" ({}) is installed,' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) else: normal_version = normalize_version_tag(required_version) req_semver = semver.Version.coerce(normal_version) - if version_spec.startswith('branch='): - version_spec = version_spec[len('branch='):] - return (str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + if version_spec.startswith("branch="): + version_spec = version_spec[len("branch=") :] + return ( + str.format( + 'unsatisfiable dependency: "{}" ({}) is installed,' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) else: try: semver_spec = semver.Spec(version_spec) except ValueError: - return (str.format( - 'package "{}" has invalid semver spec: {}', - depender_name, version_spec), new_pkgs) + return ( + str.format( + 'package "{}" has invalid semver spec: {}', + depender_name, + version_spec, + ), + new_pkgs, + ) if req_semver not in semver_spec: - return (str.format( - 'unsatisfiable dependency: "{}" ({}) is installed,' - ' but "{}" requires {}', node.name, - required_version, depender_name, version_spec), - new_pkgs) + return ( + str.format( + 'unsatisfiable dependency: "{}" ({}) is installed,' + ' but "{}" requires {}', + node.name, + required_version, + depender_name, + version_spec, + ), + new_pkgs, + ) else: # Choose best version that satisfies constraints best_version = None @@ -2106,19 +2285,20 @@ def __str__(self): def no_best_version_string(node): rval = str.format( - '"{}" has no version satisfying dependencies:\n', - node.name) + '"{}" has no version satisfying dependencies:\n', node.name + ) for depender_name, version_spec in node.dependers.items(): - rval += str.format('\t"{}" requires: "{}"\n', - depender_name, version_spec) + rval += str.format( + '\t"{}" requires: "{}"\n', depender_name, version_spec + ) return rval for _, version_spec in node.dependers.items(): - if version_spec.startswith('branch='): + if version_spec.startswith("branch="): need_branch = True - elif version_spec != '*': + elif version_spec != "*": need_version = True if need_branch and need_version: @@ -2128,14 +2308,14 @@ def no_best_version_string(node): branch_name = None for depender_name, version_spec in node.dependers.items(): - if version_spec == '*': + if version_spec == "*": continue if not branch_name: - branch_name = version_spec[len('branch='):] + branch_name = version_spec[len("branch=") :] continue - if branch_name != version_spec[len('branch='):]: + if branch_name != version_spec[len("branch=") :]: return (no_best_version_string(node), new_pkgs) if branch_name: @@ -2153,9 +2333,14 @@ def no_best_version_string(node): try: semver_spec = semver.Spec(version_spec) except ValueError: - return (str.format( - 'package "{}" has invalid semver spec: {}', - depender_name, version_spec), new_pkgs) + return ( + str.format( + 'package "{}" has invalid semver spec: {}', + depender_name, + version_spec, + ), + new_pkgs, + ) if req_semver not in semver_spec: satisfied = False @@ -2185,7 +2370,7 @@ def no_best_version_string(node): seen_nodes.add(it[0].package.name) res.insert(0, it) - return ('', res) + return ("", res) def bundle(self, bundle_file, package_list, prefer_existing_clones=False): """Creates a package bundle. @@ -2206,13 +2391,13 @@ def bundle(self, bundle_file, package_list, prefer_existing_clones=False): str: empty string if the bundle is successfully created, else an error string explaining what failed. """ - bundle_dir = os.path.join(self.scratch_dir, 'bundle') + bundle_dir = os.path.join(self.scratch_dir, "bundle") delete_path(bundle_dir) make_dir(bundle_dir) - manifest_file = os.path.join(bundle_dir, 'manifest.txt') - config = configparser.ConfigParser(delimiters='=') + manifest_file = os.path.join(bundle_dir, "manifest.txt") + config = configparser.ConfigParser(delimiters="=") config.optionxform = str - config.add_section('bundle') + config.add_section("bundle") def match_package_url_and_version(git_url, version): for ipkg in self.installed_packages(): @@ -2229,18 +2414,17 @@ def match_package_url_and_version(git_url, version): for git_url, version in package_list: name = name_from_path(git_url) clonepath = os.path.join(bundle_dir, name) - config.set('bundle', git_url, version) + config.set("bundle", git_url, version) if prefer_existing_clones: ipkg = match_package_url_and_version(git_url, version) if ipkg: - src = os.path.join( - self.package_clonedir, ipkg.package.name) + src = os.path.join(self.package_clonedir, ipkg.package.name) shutil.copytree(src, clonepath, symlinks=True) clone = git.Repo(clonepath) clone.git.reset(hard=True) - clone.git.clean('-f', '-x', '-d') + clone.git.clean("-f", "-x", "-d") for modified_config in self.modified_config_files(ipkg): dst = os.path.join(clonepath, modified_config[0]) @@ -2251,15 +2435,15 @@ def match_package_url_and_version(git_url, version): try: git_clone(git_url, clonepath, shallow=(not is_sha1(version))) except git.exc.GitCommandError as error: - return 'failed to clone {}: {}'.format(git_url, error) + return "failed to clone {}: {}".format(git_url, error) - with open(manifest_file, 'w') as f: + with open(manifest_file, "w") as f: config.write(f) - archive = shutil.make_archive(bundle_dir, 'gztar', bundle_dir) + archive = shutil.make_archive(bundle_dir, "gztar", bundle_dir) delete_path(bundle_file) shutil.move(archive, bundle_file) - return '' + return "" def unbundle(self, bundle_file): """Installs all packages contained within a bundle. @@ -2272,7 +2456,7 @@ def unbundle(self, bundle_file): message indicated what went wrong. """ LOG.debug('unbundle "%s"', bundle_file) - bundle_dir = os.path.join(self.scratch_dir, 'bundle') + bundle_dir = os.path.join(self.scratch_dir, "bundle") delete_path(bundle_dir) make_dir(bundle_dir) @@ -2281,21 +2465,22 @@ def unbundle(self, bundle_file): except Exception as error: return str(error) - manifest_file = os.path.join(bundle_dir, 'manifest.txt') - config = configparser.ConfigParser(delimiters='=') + manifest_file = os.path.join(bundle_dir, "manifest.txt") + config = configparser.ConfigParser(delimiters="=") config.optionxform = str if not config.read(manifest_file): - return 'invalid bundle: no manifest file' + return "invalid bundle: no manifest file" - if not config.has_section('bundle'): - return 'invalid bundle: no [bundle] section in manifest file' + if not config.has_section("bundle"): + return "invalid bundle: no [bundle] section in manifest file" - manifest = config.items('bundle') + manifest = config.items("bundle") for git_url, version in manifest: - package = Package(git_url=git_url, name=git_url.split('/')[-1], - canonical=True) + package = Package( + git_url=git_url, name=git_url.split("/")[-1], canonical=True + ) clonepath = os.path.join(self.package_clonedir, package.name) delete_path(clonepath) shutil.move(os.path.join(bundle_dir, package.name), clonepath) @@ -2306,9 +2491,9 @@ def unbundle(self, bundle_file): if error: return error - return '' + return "" - def test(self, pkg_path, version='', test_dependencies=False): + def test(self, pkg_path, version="", test_dependencies=False): """Test a package. Args: @@ -2343,10 +2528,10 @@ def test(self, pkg_path, version='', test_dependencies=False): pkg_info = self.info(pkg_path, version=version, prefer_installed=False) if pkg_info.invalid_reason: - return (pkg_info.invalid_reason, 'False', '') + return (pkg_info.invalid_reason, "False", "") - if 'test_command' not in pkg_info.metadata: - return ('Package does not specify a test_command', False, '') + if "test_command" not in pkg_info.metadata: + return ("Package does not specify a test_command", False, "") if not version: version = pkg_info.metadata_version @@ -2363,7 +2548,7 @@ def test(self, pkg_path, version='', test_dependencies=False): env, err = stage.get_subprocess_env() if env is None: - LOG.warning('%s when running tests for %s', err, package.name) + LOG.warning("%s when running tests for %s", err, package.name) return (err, False, stage.state_dir) pkgs = [] @@ -2375,8 +2560,9 @@ def test(self, pkg_path, version='', test_dependencies=False): # Clone all packages, checkout right version, and build/install to # staging area. for info, version in reversed(pkgs): - LOG.debug('preparing "%s" for testing: version %s', - info.package.name, version) + LOG.debug( + 'preparing "%s" for testing: version %s', info.package.name, version + ) clonepath = os.path.join(stage.clone_dir, info.package.name) # After we prepared the stage, the clonepath might exist (as a @@ -2389,17 +2575,24 @@ def test(self, pkg_path, version='', test_dependencies=False): try: clone = _clone_package(info.package, clonepath, version) except git.exc.GitCommandError as error: - LOG.warning('failed to clone git repo: %s', error) - return ('failed to clone {}'.format(info.package.git_url), - False, stage.state_dir) + LOG.warning("failed to clone git repo: %s", error) + return ( + "failed to clone {}".format(info.package.git_url), + False, + stage.state_dir, + ) try: git_checkout(clone, version) except git.exc.GitCommandError as error: - LOG.warning('failed to checkout git repo version: %s', error) - return (str.format('failed to checkout {} of {}', - version, info.package.git_url), - False, stage.state_dir) + LOG.warning("failed to checkout git repo version: %s", error) + return ( + str.format( + "failed to checkout {} of {}", version, info.package.git_url + ), + False, + stage.state_dir, + ) fail_msg = self._stage(info.package, version, clone, stage, env) @@ -2416,39 +2609,57 @@ def test(self, pkg_path, version='', test_dependencies=False): LOG.info('testing "%s"', package) # Interpolate the test command: metadata, invalid_reason = self._interpolate_package_metadata( - info.metadata, stage) + info.metadata, stage + ) if invalid_reason: return (invalid_reason, False, stage.state_dir) - if 'test_command' not in metadata: - LOG.info('Skipping unit tests for "%s": no test_command in metadata', - info.package.qualified_name()) + if "test_command" not in metadata: + LOG.info( + 'Skipping unit tests for "%s": no test_command in metadata', + info.package.qualified_name(), + ) continue - test_command = metadata['test_command'] + test_command = metadata["test_command"] cwd = os.path.join(stage.clone_dir, info.package.name) - outfile = os.path.join(cwd, 'zkg.test_command.stdout') - errfile = os.path.join(cwd, 'zkg.test_command.stderr') - - LOG.debug('running test_command for %s with cwd="%s", PATH="%s",' - ' and ZEEKPATH="%s": %s', info.package.name, cwd, - env['PATH'], env['ZEEKPATH'], test_command) + outfile = os.path.join(cwd, "zkg.test_command.stdout") + errfile = os.path.join(cwd, "zkg.test_command.stderr") + + LOG.debug( + 'running test_command for %s with cwd="%s", PATH="%s",' + ' and ZEEKPATH="%s": %s', + info.package.name, + cwd, + env["PATH"], + env["ZEEKPATH"], + test_command, + ) - with open(outfile, 'w') as test_stdout, open(errfile, 'w') as test_stderr: - cmd = subprocess.Popen(test_command, shell=True, cwd=cwd, env=env, - stdout=test_stdout, stderr=test_stderr) + with open(outfile, "w") as test_stdout, open(errfile, "w") as test_stderr: + cmd = subprocess.Popen( + test_command, + shell=True, + cwd=cwd, + env=env, + stdout=test_stdout, + stderr=test_stderr, + ) rc = cmd.wait() if rc != 0: - return ('test_command failed with exit code {}'.format(rc), - False, stage.state_dir) + return ( + "test_command failed with exit code {}".format(rc), + False, + stage.state_dir, + ) - return ('', True, stage.state_dir) + return ("", True, stage.state_dir) def _get_executables(self, metadata): - return metadata.get('executables', '').split() + return metadata.get("executables", "").split() def _stage(self, package, version, clone, stage, env=None): """Stage a package. @@ -2485,8 +2696,7 @@ def _stage(self, package, version, clone, stage, env=None): LOG.debug('staging "%s": version %s', package, version) metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file) + invalid_reason = _parse_package_metadata(metadata_parser, metadata_file) if invalid_reason: return invalid_reason @@ -2495,25 +2705,31 @@ def _stage(self, package, version, clone, stage, env=None): if invalid_reason: return invalid_reason - build_command = metadata.get('build_command', '') + build_command = metadata.get("build_command", "") if build_command: - LOG.debug('building "%s": running build_command: %s', - package, build_command) + LOG.debug( + 'building "%s": running build_command: %s', package, build_command + ) bufsize = 4096 - build = subprocess.Popen(build_command, - shell=True, cwd=clone.working_dir, - env=env, bufsize=bufsize, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + build = subprocess.Popen( + build_command, + shell=True, + cwd=clone.working_dir, + env=env, + bufsize=bufsize, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) try: buildlog = self.package_build_log(clone.working_dir) - with open(buildlog, 'wb') as f: - LOG.info('installing "%s": writing build log: %s', - package, buildlog) + with open(buildlog, "wb") as f: + LOG.info( + 'installing "%s": writing build log: %s', package, buildlog + ) - f.write(u'=== STDERR ===\n'.encode(std_encoding(sys.stderr))) + f.write("=== STDERR ===\n".encode(std_encoding(sys.stderr))) while True: data = build.stderr.read(bufsize) @@ -2523,7 +2739,7 @@ def _stage(self, package, version, clone, stage, env=None): else: break - f.write(u'=== STDOUT ===\n'.encode(std_encoding(sys.stdout))) + f.write("=== STDOUT ===\n".encode(std_encoding(sys.stdout))) while True: data = build.stdout.read(bufsize) @@ -2536,75 +2752,87 @@ def _stage(self, package, version, clone, stage, env=None): except EnvironmentError as error: LOG.warning( 'installing "%s": failed to write build log %s %s: %s', - package, buildlog, error.errno, error.strerror) + package, + buildlog, + error.errno, + error.strerror, + ) returncode = build.wait() if returncode != 0: - return 'package build_command failed, see log in {}'.format( - buildlog) + return "package build_command failed, see log in {}".format(buildlog) - pkg_script_dir = metadata.get('script_dir', '') + pkg_script_dir = metadata.get("script_dir", "") script_dir_src = os.path.join(clone.working_dir, pkg_script_dir) script_dir_dst = os.path.join(stage.script_dir, package.name) if not os.path.exists(script_dir_src): - return str.format("package's 'script_dir' does not exist: {}", - pkg_script_dir) + return str.format( + "package's 'script_dir' does not exist: {}", pkg_script_dir + ) - pkgload = os.path.join(script_dir_src, '__load__.') + pkgload = os.path.join(script_dir_src, "__load__.") # Check if __load__.bro exists for compatibility with older packages - if os.path.isfile(pkgload + 'zeek') or os.path.isfile(pkgload + 'bro'): + if os.path.isfile(pkgload + "zeek") or os.path.isfile(pkgload + "bro"): try: - symlink_path = os.path.join(os.path.dirname(stage.script_dir), - package.name) - make_symlink(os.path.join('packages', package.name), - symlink_path) + symlink_path = os.path.join( + os.path.dirname(stage.script_dir), package.name + ) + make_symlink(os.path.join("packages", package.name), symlink_path) for alias in aliases(metadata): symlink_path = os.path.join( - os.path.dirname(stage.script_dir), alias) - make_symlink(os.path.join('packages', package.name), - symlink_path) + os.path.dirname(stage.script_dir), alias + ) + make_symlink(os.path.join("packages", package.name), symlink_path) except OSError as exception: - error = 'could not create symlink at {}'.format(symlink_path) - error += ': {}: {}'.format(type(exception).__name__, exception) + error = "could not create symlink at {}".format(symlink_path) + error += ": {}: {}".format(type(exception).__name__, exception) return error - error = _copy_package_dir(package, 'script_dir', - script_dir_src, script_dir_dst, - self.scratch_dir) + error = _copy_package_dir( + package, "script_dir", script_dir_src, script_dir_dst, self.scratch_dir + ) if error: return error else: - if 'script_dir' in metadata: - return str.format("no __load__.zeek file found" - " in package's 'script_dir' : {}", - pkg_script_dir) + if "script_dir" in metadata: + return str.format( + "no __load__.zeek file found" " in package's 'script_dir' : {}", + pkg_script_dir, + ) else: - LOG.warning('installing "%s": no __load__.zeek in implicit' - ' script_dir, skipped installing scripts', package) + LOG.warning( + 'installing "%s": no __load__.zeek in implicit' + " script_dir, skipped installing scripts", + package, + ) - pkg_plugin_dir = metadata.get('plugin_dir', 'build') + pkg_plugin_dir = metadata.get("plugin_dir", "build") plugin_dir_src = os.path.join(clone.working_dir, pkg_plugin_dir) plugin_dir_dst = os.path.join(stage.plugin_dir, package.name) if not os.path.exists(plugin_dir_src): - LOG.info('installing "%s": package "plugin_dir" does not exist: %s', - package, pkg_plugin_dir) + LOG.info( + 'installing "%s": package "plugin_dir" does not exist: %s', + package, + pkg_plugin_dir, + ) - if pkg_plugin_dir != 'build': + if pkg_plugin_dir != "build": # It's common for a package to not have build directory for # plugins, so don't error out in that case, just log it. - return str.format("package's 'plugin_dir' does not exist: {}", - pkg_plugin_dir) + return str.format( + "package's 'plugin_dir' does not exist: {}", pkg_plugin_dir + ) - error = _copy_package_dir(package, 'plugin_dir', - plugin_dir_src, plugin_dir_dst, - self.scratch_dir) + error = _copy_package_dir( + package, "plugin_dir", plugin_dir_src, plugin_dir_dst, self.scratch_dir + ) if error: return error @@ -2619,12 +2847,15 @@ def _stage(self, package, version, clone, stage, env=None): return str.format("file '{}' is not executable", p) if stage.bin_dir is not None: - make_symlink(full_path, os.path.join( - stage.bin_dir, os.path.basename(p)), force=True) + make_symlink( + full_path, + os.path.join(stage.bin_dir, os.path.basename(p)), + force=True, + ) - return '' + return "" - def install(self, pkg_path, version=''): + def install(self, pkg_path, version=""): """Install a package. Args: @@ -2654,18 +2885,21 @@ def install(self, pkg_path, version=''): conflict = ipkg.package if conflict.qualified_name().endswith(pkg_path): - LOG.debug('installing "%s": re-install: %s', - pkg_path, conflict) + LOG.debug('installing "%s": re-install: %s', pkg_path, conflict) clonepath = os.path.join(self.package_clonedir, conflict.name) _clone_package(conflict, clonepath, version) return self._install(conflict, version) else: LOG.info( 'installing "%s": matched already installed package: %s', - pkg_path, conflict) + pkg_path, + conflict, + ) return str.format( 'package with name "{}" ({}) is already installed', - conflict.name, conflict) + conflict.name, + conflict, + ) matches = self.match_source_packages(pkg_path) @@ -2674,25 +2908,28 @@ def install(self, pkg_path, version=''): package = Package(git_url=pkg_path) return self._install(package, version) except git.exc.GitCommandError as error: - LOG.info('installing "%s": invalid git repo path: %s', pkg_path, - error) + LOG.info('installing "%s": invalid git repo path: %s', pkg_path, error) LOG.info('installing "%s": matched no source package', pkg_path) - return 'package not found in sources and also not a valid git URL' + return "package not found in sources and also not a valid git URL" if len(matches) > 1: matches_string = [match.qualified_name() for match in matches] - LOG.info('installing "%s": matched multiple packages: %s', - pkg_path, matches_string) - return str.format('"{}" matches multiple packages, try a more' - ' specific name from: {}', - pkg_path, matches_string) + LOG.info( + 'installing "%s": matched multiple packages: %s', + pkg_path, + matches_string, + ) + return str.format( + '"{}" matches multiple packages, try a more' " specific name from: {}", + pkg_path, + matches_string, + ) try: return self._install(matches[0], version) except git.exc.GitCommandError as error: - LOG.warning('installing "%s": source package git repo is invalid', - pkg_path) + LOG.warning('installing "%s": source package git repo is invalid', pkg_path) return 'failed to clone package "{}": {}'.format(pkg_path, error) def _install(self, package, version, use_existing_clone=False): @@ -2732,8 +2969,8 @@ def _install(self, package, version, use_existing_clone=False): status.tracking_method = TRACKING_METHOD_BRANCH else: LOG.info( - 'branch "%s" not in available branches: %s', version, - branches) + 'branch "%s" not in available branches: %s', version, branches + ) return 'no such branch or version tag: "{}"'.format(version) else: @@ -2747,13 +2984,11 @@ def _install(self, package, version, use_existing_clone=False): status.current_version = version git_checkout(clone, version) status.current_hash = clone.head.object.hexsha - status.is_outdated = _is_clone_outdated( - clone, version, status.tracking_method) + status.is_outdated = _is_clone_outdated(clone, version, status.tracking_method) metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file) + invalid_reason = _parse_package_metadata(metadata_parser, metadata_file) if invalid_reason: return invalid_reason @@ -2782,7 +3017,7 @@ def _install(self, package, version, use_existing_clone=False): self._write_manifest() self._refresh_bin_dir(self.bin_dir) LOG.debug('installed "%s"', package) - return '' + return "" def _interpolate_package_metadata(self, metadata, stage): # This is a bit circular: we need to parse the user variables, if any, @@ -2794,9 +3029,9 @@ def _interpolate_package_metadata(self, metadata, stage): return None, "package has malformed 'user_vars' metadata field" substitutions = { - 'bro_dist': self.zeek_dist, - 'zeek_dist': self.zeek_dist, - 'package_base': stage.clone_dir, + "bro_dist": self.zeek_dist, + "zeek_dist": self.zeek_dist, + "package_base": stage.clone_dir, } substitutions.update(self.user_vars) @@ -2812,7 +3047,7 @@ def _interpolate_package_metadata(self, metadata, stage): # Now apply the substitutions via a new config parser: metadata_parser = configparser.ConfigParser(defaults=substitutions) - metadata_parser.read_dict({'package': metadata}) + metadata_parser.read_dict({"package": metadata}) return _get_package_metadata(metadata_parser), None @@ -2825,11 +3060,15 @@ def _refresh_bin_dir(self, bin_dir, prev_bin_dir=None): src = os.path.join(self.package_clonedir, ipkg.package.name, exec) dst = os.path.join(bin_dir, os.path.basename(exec)) - if not os.path.exists(dst) or not os.path.islink(dst) or os.path.realpath(src) != os.path.realpath(dst): - LOG.debug('creating link %s -> %s', src, dst) + if ( + not os.path.exists(dst) + or not os.path.islink(dst) + or os.path.realpath(src) != os.path.realpath(dst) + ): + LOG.debug("creating link %s -> %s", src, dst) make_symlink(src, dst, force=True) else: - LOG.debug('link %s is up to date', dst) + LOG.debug("link %s is up to date", dst) # Remove all links in bin_dir that are associated with executables # coming with any of the currently installed package. @@ -2840,9 +3079,9 @@ def _clear_bin_dir(self, bin_dir): if os.path.islink(old): try: os.unlink(old) - LOG.debug('removed link %s', old) + LOG.debug("removed link %s", old) except Exception: - LOG.warn('failed to remove link %s', old) + LOG.warn("failed to remove link %s", old) def _get_branch_names(clone): @@ -2851,10 +3090,10 @@ def _get_branch_names(clone): for ref in clone.references: branch_name = str(ref.name) - if not branch_name.startswith('origin/'): + if not branch_name.startswith("origin/"): continue - rval.append(branch_name.split('origin/')[1]) + rval.append(branch_name.split("origin/")[1]) return rval @@ -2866,7 +3105,7 @@ def _is_version_outdated(clone, version): def _is_branch_outdated(clone, branch): - it = clone.iter_commits('{0}..origin/{0}'.format(branch)) + it = clone.iter_commits("{0}..origin/{0}".format(branch)) num_commits_behind = sum(1 for c in it) return num_commits_behind > 0 @@ -2898,10 +3137,10 @@ def _copy_package_dir(package, dirname, src, dst, scratch_dir): explaining why it failed. """ if not os.path.exists(src): - return '' + return "" if os.path.isfile(src) and tarfile.is_tarfile(src): - tmp_dir = os.path.join(scratch_dir, 'untar') + tmp_dir = os.path.join(scratch_dir, "untar") delete_path(tmp_dir) make_dir(tmp_dir) @@ -2913,18 +3152,18 @@ def _copy_package_dir(package, dirname, src, dst, scratch_dir): ld = os.listdir(tmp_dir) if len(ld) != 1: - return 'failed to copy package {}: invalid tarfile'.format(dirname) + return "failed to copy package {}: invalid tarfile".format(dirname) src = os.path.join(tmp_dir, ld[0]) if not os.path.isdir(src): - return 'failed to copy package {}: not a dir or tarfile'.format(dirname) + return "failed to copy package {}: not a dir or tarfile".format(dirname) def ignore(_, files): rval = [] for f in files: - if f in {'.git', 'bro-pkg.meta', 'zkg.meta'}: + if f in {".git", "bro-pkg.meta", "zkg.meta"}: rval.append(f) return rval @@ -2937,22 +3176,21 @@ def ignore(_, files): for err in errors: src, dst, msg = err - reason = 'failed to copy {}: {} -> {}: {}'.format( - dirname, src, dst, msg) - reasons += '\n' + reason + reason = "failed to copy {}: {} -> {}: {}".format(dirname, src, dst, msg) + reasons += "\n" + reason LOG.warning('installing "%s": %s', package, reason) - return 'failed to copy package {}: {}'.format(dirname, reasons) + return "failed to copy package {}: {}".format(dirname, reasons) - return '' + return "" def _create_readme(file_path): if os.path.exists(file_path): return - with open(file_path, 'w') as f: - f.write('WARNING: This directory is managed by zkg.\n') + with open(file_path, "w") as f: + f.write("WARNING: This directory is managed by zkg.\n") f.write("Don't make direct modifications to anything within it.\n") @@ -2971,7 +3209,7 @@ def _clone_package(package, clonepath, version): def _get_package_metadata(parser): - metadata = {item[0]: item[1] for item in parser.items('package')} + metadata = {item[0]: item[1] for item in parser.items("package")} return metadata @@ -2985,18 +3223,18 @@ def _pick_metadata_file(directory): def _parse_package_metadata(parser, metadata_file): - """Return string explaining why metadata is invalid, or '' if valid. """ + """Return string explaining why metadata is invalid, or '' if valid.""" if not parser.read(metadata_file): - LOG.warning('%s: missing metadata file', metadata_file) - return 'missing {} (or {}) metadata file'.format( - METADATA_FILENAME, LEGACY_METADATA_FILENAME) + LOG.warning("%s: missing metadata file", metadata_file) + return "missing {} (or {}) metadata file".format( + METADATA_FILENAME, LEGACY_METADATA_FILENAME + ) - if not parser.has_section('package'): - LOG.warning('%s: metadata missing [package]', metadata_file) - return '{} is missing [package] section'.format( - os.path.basename(metadata_file)) + if not parser.has_section("package"): + LOG.warning("%s: metadata missing [package]", metadata_file) + return "{} is missing [package] section".format(os.path.basename(metadata_file)) - return '' + return "" def _info_from_clone(clone, package, status, version): @@ -3017,24 +3255,34 @@ def _info_from_clone(clone, package, status, version): metadata_file = _pick_metadata_file(clone.working_dir) metadata_parser = configparser.ConfigParser(interpolation=None) - invalid_reason = _parse_package_metadata( - metadata_parser, metadata_file) + invalid_reason = _parse_package_metadata(metadata_parser, metadata_file) if invalid_reason: - return PackageInfo(package=package, invalid_reason=invalid_reason, - status=status, versions=versions, - metadata_version=version, version_type=version_type, - metadata_file=metadata_file, - default_branch=default_branch) + return PackageInfo( + package=package, + invalid_reason=invalid_reason, + status=status, + versions=versions, + metadata_version=version, + version_type=version_type, + metadata_file=metadata_file, + default_branch=default_branch, + ) metadata = _get_package_metadata(metadata_parser) - return PackageInfo(package=package, invalid_reason=invalid_reason, - status=status, metadata=metadata, versions=versions, - metadata_version=version, version_type=version_type, - metadata_file=metadata_file, - default_branch=default_branch) + return PackageInfo( + package=package, + invalid_reason=invalid_reason, + status=status, + metadata=metadata, + versions=versions, + metadata_version=version, + version_type=version_type, + metadata_file=metadata_file, + default_branch=default_branch, + ) def _is_reserved_pkg_name(name): - return name == 'bro' or name == 'zeek' or name == 'zkg' + return name == "bro" or name == "zeek" or name == "zkg" diff --git a/zeekpkg/package.py b/zeekpkg/package.py index c39b3263..b2719852 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -12,26 +12,27 @@ from ._util import find_sentence_end #: The name of files used by packages to store their metadata. -METADATA_FILENAME = 'zkg.meta' -LEGACY_METADATA_FILENAME = 'bro-pkg.meta' +METADATA_FILENAME = "zkg.meta" +LEGACY_METADATA_FILENAME = "bro-pkg.meta" -TRACKING_METHOD_VERSION = 'version' -TRACKING_METHOD_BRANCH = 'branch' -TRACKING_METHOD_COMMIT = 'commit' +TRACKING_METHOD_VERSION = "version" +TRACKING_METHOD_BRANCH = "branch" +TRACKING_METHOD_COMMIT = "commit" + +PLUGIN_MAGIC_FILE = "__bro_plugin__" +PLUGIN_MAGIC_FILE_DISABLED = "__bro_plugin__.disabled" -PLUGIN_MAGIC_FILE = '__bro_plugin__' -PLUGIN_MAGIC_FILE_DISABLED = '__bro_plugin__.disabled' def name_from_path(path): """Returns the name of a package given a path to its git repository.""" - return canonical_url(path).split('/')[-1] + return canonical_url(path).split("/")[-1] def canonical_url(path): """Returns the url of a package given a path to its git repo.""" - url = path.rstrip('/') + url = path.rstrip("/") - if url.startswith('.') or url.startswith('/'): + if url.startswith(".") or url.startswith("/"): url = os.path.realpath(url) return url @@ -51,38 +52,38 @@ def is_valid_name(name): def aliases(metadata_dict): """Return a list of package aliases found in metadata's 'aliases' field.""" - if 'aliases' not in metadata_dict: + if "aliases" not in metadata_dict: return [] - return re.split(',\s*|\s+', metadata_dict['aliases']) + return re.split(",\s*|\s+", metadata_dict["aliases"]) def tags(metadata_dict): """Return a list of tag strings found in the metadata's 'tags' field.""" - if 'tags' not in metadata_dict: + if "tags" not in metadata_dict: return [] - return re.split(',\s*', metadata_dict['tags']) + return re.split(",\s*", metadata_dict["tags"]) def short_description(metadata_dict): """Returns the first sentence of the metadata's 'desciption' field.""" - if 'description' not in metadata_dict: - return '' + if "description" not in metadata_dict: + return "" - description = metadata_dict['description'] - lines = description.split('\n') - rval = '' + description = metadata_dict["description"] + lines = description.split("\n") + rval = "" for line in lines: line = line.lstrip() - rval += ' ' + rval += " " period_idx = find_sentence_end(line) if period_idx == -1: rval += line else: - rval += line[:period_idx + 1] + rval += line[: period_idx + 1] break return rval.lstrip() @@ -105,7 +106,7 @@ def user_vars(metadata_dict): return [(uvar.name(), uvar.val(), uvar.desc()) for uvar in uvars] -def dependencies(metadata_dict, field='depends'): +def dependencies(metadata_dict, field="depends"): """Returns a dictionary of (str, str) based on metadata's dependency field. The keys indicate the name of a package (shorthand name or full git URL). @@ -145,6 +146,7 @@ class InstalledPackage(object): status (:class:`PackageStatus`): the status of the installed package """ + def __init__(self, package, status): self.package = package self.status = status @@ -184,8 +186,15 @@ class PackageStatus(object): package's current version/commit. """ - def __init__(self, is_loaded=False, is_pinned=False, is_outdated=False, - tracking_method=None, current_version=None, current_hash=None): + def __init__( + self, + is_loaded=False, + is_pinned=False, + is_outdated=False, + tracking_method=None, + current_version=None, + current_hash=None, + ): self.is_loaded = is_loaded self.is_pinned = is_pinned self.is_outdated = is_outdated @@ -226,9 +235,18 @@ class PackageInfo(object): value is None. """ - def __init__(self, package=None, status=None, metadata=None, versions=None, - metadata_version='', invalid_reason='', version_type='', - metadata_file=None, default_branch=None): + def __init__( + self, + package=None, + status=None, + metadata=None, + versions=None, + metadata_version="", + invalid_reason="", + version_type="", + metadata_file=None, + default_branch=None, + ): self.package = package self.status = status self.metadata = {} if metadata is None else metadata @@ -258,7 +276,7 @@ def short_description(self): This will be the first sentence of the package's 'description' field.""" return short_description(self.metadata) - def dependencies(self, field='depends'): + def dependencies(self, field="depends"): """Returns a dictionary of dependency -> version strings. The keys indicate the name of a package (shorthand name or full git @@ -326,8 +344,16 @@ class Package(object): aggregation of the source's :file:`aggregate.meta` file (it may not be accurate/up-to-date). """ - def __init__(self, git_url, source='', directory='', metadata=None, - name=None, canonical=False): + + def __init__( + self, + git_url, + source="", + directory="", + metadata=None, + name=None, + canonical=False, + ): self.git_url = git_url self.source = source self.directory = directory @@ -382,7 +408,7 @@ def short_description(self): package has not been installed yet.""" return short_description(self.metadata) - def dependencies(self, field='depends'): + def dependencies(self, field="depends"): """Returns a dictionary of dependency -> version strings. The keys indicate the name of a package (shorthand name or full git @@ -416,7 +442,7 @@ def name_with_source_directory(self): just the package name is returned. """ if self.directory: - return '{}/{}'.format(self.directory, self.name) + return "{}/{}".format(self.directory, self.name) return self.name @@ -428,8 +454,7 @@ def qualified_name(self): git URL is returned. """ if self.source: - return '{}/{}'.format(self.source, - self.name_with_source_directory()) + return "{}/{}".format(self.source, self.name_with_source_directory()) return self.git_url @@ -439,11 +464,11 @@ def matches_path(self, path): E.g for a package with :meth:`qualified_name()` of "zeek/alice/foo", the following inputs will match: "foo", "alice/foo", "zeek/alice/foo" """ - path_parts = path.split('/') + path_parts = path.split("/") if self.source: pkg_path = self.qualified_name() - pkg_path_parts = pkg_path.split('/') + pkg_path_parts = pkg_path.split("/") for i, part in reversed(list(enumerate(path_parts))): ri = i - len(path_parts) diff --git a/zeekpkg/source.py b/zeekpkg/source.py index 02b89d73..edf1ff02 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -12,21 +12,14 @@ import shutil from . import LOG -from .package import ( - name_from_path, - Package -) -from ._util import ( - git_default_branch, - git_checkout, - git_clone -) +from .package import name_from_path, Package +from ._util import git_default_branch, git_checkout, git_clone #: The name of package index files. -INDEX_FILENAME = 'zkg.index' -LEGACY_INDEX_FILENAME = 'bro-pkg.index' +INDEX_FILENAME = "zkg.index" +LEGACY_INDEX_FILENAME = "bro-pkg.index" #: The name of the package source file where package metadata gets aggregated. -AGGREGATE_DATA_FILE = 'aggregate.meta' +AGGREGATE_DATA_FILE = "aggregate.meta" class Source(object): @@ -62,19 +55,21 @@ def __init__(self, name, clone_path, git_url, version=None): LOG.debug('creating source clone of "%s" at %s', name, clone_path) self.clone = git_clone(git_url, clone_path, shallow=True) except git.exc.InvalidGitRepositoryError: - LOG.debug('deleting invalid source clone of "%s" at %s', - name, clone_path) + LOG.debug('deleting invalid source clone of "%s" at %s', name, clone_path) shutil.rmtree(clone_path) self.clone = git_clone(git_url, clone_path, shallow=True) else: LOG.debug('found source clone of "%s" at %s', name, clone_path) - old_url = self.clone.git.config('--local', '--get', - 'remote.origin.url') + old_url = self.clone.git.config("--local", "--get", "remote.origin.url") if git_url != old_url: LOG.debug( 'url of source "%s" changed from %s to %s, reclone at %s', - name, old_url, git_url, clone_path) + name, + old_url, + git_url, + clone_path, + ) shutil.rmtree(clone_path) self.clone = git_clone(git_url, clone_path, shallow=True) @@ -91,8 +86,7 @@ def package_index_files(self): rval = [] visited_dirs = set() - for root, dirs, files in os.walk(self.clone.working_dir, - followlinks=True): + for root, dirs, files in os.walk(self.clone.working_dir, followlinks=True): stat = os.stat(root) visited_dirs.add((stat.st_dev, stat.st_ino)) dirs_to_visit_next = [] @@ -106,7 +100,7 @@ def package_index_files(self): dirs[:] = dirs_to_visit_next try: - dirs.remove('.git') + dirs.remove(".git") except ValueError: pass @@ -121,17 +115,16 @@ def packages(self): rval = [] # Use raw parser so no value interpolation takes place. parser = configparser.RawConfigParser() - aggregate_file = os.path.join( - self.clone.working_dir, AGGREGATE_DATA_FILE) + aggregate_file = os.path.join(self.clone.working_dir, AGGREGATE_DATA_FILE) parser.read(aggregate_file) for index_file in self.package_index_files(): - relative_path = index_file[len(self.clone.working_dir) + 1:] + relative_path = index_file[len(self.clone.working_dir) + 1 :] directory = os.path.dirname(relative_path) lines = [] with open(index_file) as f: - lines = [line.rstrip('\n') for line in f] + lines = [line.rstrip("\n") for line in f] for url in lines: pkg_name = name_from_path(url) @@ -139,11 +132,14 @@ def packages(self): metadata = {} if parser.has_section(agg_key): - metadata = {key: value for key, - value in parser.items(agg_key)} - - package = Package(git_url=url, source=self.name, - directory=directory, metadata=metadata) + metadata = {key: value for key, value in parser.items(agg_key)} + + package = Package( + git_url=url, + source=self.name, + directory=directory, + metadata=metadata, + ) rval.append(package) return rval diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 6bc46ded..ecac3e20 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -30,31 +30,37 @@ make_dir, ) -API_VERSION = '1.1.0' +API_VERSION = "1.1.0" + class Error(Exception): """Base class for any template-related errors.""" + class InputError(Error): """Something's amiss in the input arguments for a package.""" + class OutputError(Error): """Something's going wrong while producing template output.""" + class LoadError(Error): """Something's going wrong while retrieving a template.""" + class GitError(LoadError): """There's git trouble while producing template output.""" -class Template(): +class Template: """Base class for any template. Templates need to derive from this class in their toplevel __init__.py. Instances of this class pull together the components in a given template and capture their parameterization. """ + @staticmethod def load(config, template, version=None): """Template loader. @@ -106,7 +112,8 @@ def load(config, template, version=None): # zkg state folder's clone space and support version # requests. template_clonedir = os.path.join( - config.get('paths', 'state_dir'), 'clones', 'template') + config.get("paths", "state_dir"), "clones", "template" + ) templatedir = os.path.join(template_clonedir, name_from_path(template)) make_dir(template_clonedir) @@ -123,7 +130,7 @@ def load(config, template, version=None): for remote in repo.remotes: cur_remote_urls |= set(remote.urls) if len(cur_remote_urls) == 1 and template in cur_remote_urls: - repo.git.fetch('-f', '--recurse-submodules=yes', '--tags') + repo.git.fetch("-f", "--recurse-submodules=yes", "--tags") else: delete_path(templatedir) repo = None @@ -145,8 +152,11 @@ def load(config, template, version=None): try: git_checkout(repo, version) except git.exc.GitCommandError as error: - msg = 'failed to checkout branch/version "{}" of template {}: {}'.format( - version, template, error) + msg = ( + 'failed to checkout branch/version "{}" of template {}: {}'.format( + version, template, error + ) + ) LOG.warn(msg) raise GitError(msg) from error @@ -158,23 +168,25 @@ def load(config, template, version=None): _ = repo.active_branch git_pull(repo) except TypeError: - pass # Not on a branch, do nothing + pass # Not on a branch, do nothing except git.exc.GitCommandError as error: msg = 'failed to update branch "{}" of template {}: {}'.format( - version, template, error) + version, template, error + ) LOG.warning(msg) raise GitError(msg) from error try: - mod = load_source(os.path.join(templatedir, '__init__.py')) + mod = load_source(os.path.join(templatedir, "__init__.py")) except Exception as error: msg = 'failed to load template "{}": {}'.format(template, error) LOG.exception(msg) raise LoadError(msg) from error - if not hasattr(mod, 'TEMPLATE_API_VERSION'): - msg = 'template{} does not indicate its API version'.format( - ' version ' + version if version else '') + if not hasattr(mod, "TEMPLATE_API_VERSION"): + msg = "template{} does not indicate its API version".format( + " version " + version if version else "" + ) LOG.error(msg) raise LoadError(msg) @@ -186,13 +198,16 @@ def load(config, template, version=None): try: is_compat = Template.is_api_compatible(mod.TEMPLATE_API_VERSION) except ValueError: - raise LoadError('API version string "{}" is invalid'.format( - mod.TEMPLATE_API_VERSION)) + raise LoadError( + 'API version string "{}" is invalid'.format(mod.TEMPLATE_API_VERSION) + ) if not is_compat: - msg = 'template{} API version is incompatible with zkg ({} vs {})'.format( - ' version ' + version if version else '', - mod.TEMPLATE_API_VERSION, API_VERSION) + msg = "template{} API version is incompatible with zkg ({} vs {})".format( + " version " + version if version else "", + mod.TEMPLATE_API_VERSION, + API_VERSION, + ) LOG.error(msg) raise LoadError(msg) @@ -255,7 +270,7 @@ def __init__(self, templatedir, api_version, version=None, repo=None): self._api_version = api_version self._version = version self._repo = repo - self._params = {} # str -> str, set via self.define_param() + self._params = {} # str -> str, set via self.define_param() self._user_vars = [] def define_user_vars(self): @@ -300,7 +315,7 @@ def package(self): """ return None - def features(self): # pylint: disable=no-self-use + def features(self): # pylint: disable=no-self-use """Provides any additional features templates supported. If the template provides extra features, return each as an @@ -348,7 +363,7 @@ def version_branch(self): if self._repo and self._repo.active_branch: return self._repo.active_branch.name except TypeError: - pass # Not on a branch + pass # Not on a branch return None @@ -370,7 +385,7 @@ def define_param(self, name, val): """Defines a parameter of the given name and value.""" self._params[name] = val - def lookup_param(self, name, default=''): + def lookup_param(self, name, default=""): """Looks up a parameter, falling back to the given default.""" return self._params.get(name, default) @@ -387,8 +402,8 @@ def info(self): # In the future a template may not provide a full package, # only features overlaid in combination with another template. res = { - 'api_version': self._api_version, - 'provides_package': False, + "api_version": self._api_version, + "provides_package": False, } # XXX we should revisit the reported 'origin' value in @@ -399,46 +414,51 @@ def info(self): if self._repo is not None: try: remotes = git_remote_urls(self._repo) - res['origin'] = remotes['origin'] + res["origin"] = remotes["origin"] except KeyError: - res['origin'] = 'unavailable' - res['versions'] = git_version_tags(self._repo) - res['has_repo'] = True + res["origin"] = "unavailable" + res["versions"] = git_version_tags(self._repo) + res["has_repo"] = True else: - res['origin'] = 'not a git repository' - res['versions'] = [] - res['has_repo'] = False + res["origin"] = "not a git repository" + res["versions"] = [] + res["has_repo"] = False - pkg = self.package() # pylint: disable=assignment-from-none + pkg = self.package() # pylint: disable=assignment-from-none uvars = self.define_user_vars() feature_names = [] - res['user_vars'] = {} + res["user_vars"] = {} for uvar in uvars: - res['user_vars'][uvar.name()] = { - 'description': uvar.desc(), - 'default': uvar.default(), - 'used_by': [], + res["user_vars"][uvar.name()] = { + "description": uvar.desc(), + "default": uvar.default(), + "used_by": [], } if pkg is not None: - res['provides_package'] = True + res["provides_package"] = True for uvar_name in pkg.needed_user_vars(): try: - res['user_vars'][uvar_name]['used_by'].append('package') + res["user_vars"][uvar_name]["used_by"].append("package") except KeyError: - LOG.warning('Package requires undefined user var "%s", skipping', uvar_name) + LOG.warning( + 'Package requires undefined user var "%s", skipping', uvar_name + ) for feature in self.features(): feature_names.append(feature.name()) for uvar_name in feature.needed_user_vars(): try: - res['user_vars'][uvar_name]['used_by'].append(feature.name()) + res["user_vars"][uvar_name]["used_by"].append(feature.name()) except KeyError: - LOG.warning('Feature "%s" requires undefined user var "%s"', - feature.name(), uvar_name) + LOG.warning( + 'Feature "%s" requires undefined user var "%s"', + feature.name(), + uvar_name, + ) - res['features'] = sorted(feature_names) + res["features"] = sorted(feature_names) return res def _set_user_vars(self, user_vars): @@ -580,7 +600,7 @@ def instantiate_file(self, tmpl, orig_file, path_name, file_name, content): os.makedirs(out_dir, exist_ok=True) try: - with open(out_file, 'wb') as hdl: + with open(out_file, "wb") as hdl: hdl.write(content) shutil.copymode(orig_file, out_file) except IOError as error: @@ -618,8 +638,7 @@ def instantiate_symlink(self, tmpl, orig_file, path_name, file_name, target): delete_path(out_file) os.symlink(target, out_file) except OSError as error: - LOG.warning('OS error while creating symlink "%s": %s', - out_file, error) + LOG.warning('OS error while creating symlink "%s": %s', out_file, error) def _walk(self, tmpl): """Generator for instantiating template content. @@ -639,7 +658,7 @@ def _walk(self, tmpl): in_file = root + os.sep + fname # Substitute directory and file names - out_path = self._replace(tmpl, root[len(prefix)+1:]) + out_path = self._replace(tmpl, root[len(prefix) + 1 :]) out_file = self._replace(tmpl, fname) if os.path.islink(in_file): @@ -647,15 +666,15 @@ def _walk(self, tmpl): else: # Substitute file content. try: - with open(in_file, 'rb') as hdl: + with open(in_file, "rb") as hdl: out_content = self._replace(tmpl, hdl.read()) except IOError as error: - LOG.warning('skipping instantiation of %s: %s', in_file, error) + LOG.warning("skipping instantiation of %s: %s", in_file, error) continue yield in_file, out_path, out_file, out_content - def _replace(self, tmpl, content): # pylint: disable=no-self-use + def _replace(self, tmpl, content): # pylint: disable=no-self-use """Helper for content substitution. Args: @@ -667,10 +686,10 @@ def _replace(self, tmpl, content): # pylint: disable=no-self-use str or bytes after parameter substitution. """ for name, val in tmpl.params().items(): - pat = '@' + name + '@' + pat = "@" + name + "@" if not isinstance(content, str): - pat = bytes(pat, 'ascii') - val = bytes(val, 'ascii') + pat = bytes(pat, "ascii") + val = bytes(val, "ascii") content = re.sub(pat, val, content, flags=re.IGNORECASE) return content @@ -683,6 +702,7 @@ class Package(_Content): abstract. At a minimum, your template's Package derivative needs to implement contentdir(). """ + def do_instantiate(self, tmpl, packagedir, use_force=False): self._prepare_packagedir(packagedir) super().do_instantiate(tmpl, packagedir, use_force) @@ -698,7 +718,7 @@ def _update_metadata(self, tmpl): This information allows re-running template instantiation with identical inputs at a later time. """ - config = configparser.ConfigParser(delimiters='=') + config = configparser.ConfigParser(delimiters="=") config.optionxform = str manifest_file = os.path.join(self._packagedir, METADATA_FILENAME) @@ -706,42 +726,42 @@ def _update_metadata(self, tmpl): # content, otherwise create with just our metadata. config.read(manifest_file) - section = 'template' + section = "template" config.remove_section(section) config.add_section(section) - config.set(section, 'source', tmpl.name()) + config.set(section, "source", tmpl.name()) if tmpl.has_repo(): tmplinfo = tmpl.info() - if tmplinfo['origin'] != 'unavailable': - config.set(section, 'source', tmplinfo['origin']) + if tmplinfo["origin"] != "unavailable": + config.set(section, "source", tmplinfo["origin"]) if tmpl.version(): # If we're on a branch, disambiguate the version by also mentioning # the exact commit. if tmpl.version_branch(): - config.set(section, 'version', tmpl.version_branch()) - config.set(section, 'commit', tmpl.version_sha()[:8]) + config.set(section, "version", tmpl.version_branch()) + config.set(section, "commit", tmpl.version_sha()[:8]) else: - config.set(section, 'version', tmpl.version()) + config.set(section, "version", tmpl.version()) else: - config.set(section, 'version', tmpl.version() or 'unversioned') + config.set(section, "version", tmpl.version() or "unversioned") - config.set(section, 'zkg_version', __version__) + config.set(section, "zkg_version", __version__) if self._features: - val = ','.join(sorted([f.name() for f in self._features])) - config.set(section, 'features', val) + val = ",".join(sorted([f.name() for f in self._features])) + config.set(section, "features", val) - section = 'template_vars' + section = "template_vars" config.remove_section(section) config.add_section(section) - for uvar in tmpl._get_user_vars(): # pylint: disable=protected-access + for uvar in tmpl._get_user_vars(): # pylint: disable=protected-access if uvar.val() is not None: config.set(section, uvar.name(), uvar.val()) - with open(manifest_file, 'w') as hdl: + with open(manifest_file, "w") as hdl: config.write(hdl) def _git_init(self, tmpl): @@ -750,33 +770,37 @@ def _git_init(self, tmpl): for fname in repo.untracked_files: repo.index.add(fname) - features_info = '' + features_info = "" if self._features: names = sorted(['"' + f.name() + '"' for f in self._features]) if len(names) == 1: - features_info = ', with feature {}'.format(names[0]) + features_info = ", with feature {}".format(names[0]) else: - features_info = ', with features ' - features_info += ', '.join(names[:-1]) - features_info += ' and ' + names[-1] + features_info = ", with features " + features_info += ", ".join(names[:-1]) + features_info += " and " + names[-1] ver_info = tmpl.version() ver_sha = tmpl.version_sha() if ver_info is None: if ver_sha: - ver_info = 'version ' + ver_sha[:8] + ver_info = "version " + ver_sha[:8] else: - ver_info = 'no versioning' + ver_info = "no versioning" else: - ver_info = 'version ' + ver_info + ver_info = "version " + ver_info if ver_sha: - ver_info += ' (' + ver_sha[:8] + ')' + ver_info += " (" + ver_sha[:8] + ")" - repo.index.commit("""Initial commit. + repo.index.commit( + """Initial commit. zkg {} created this package from template "{}" -using {}{}.""".format(__version__, tmpl.name(), ver_info, features_info)) +using {}{}.""".format( + __version__, tmpl.name(), ver_info, features_info + ) + ) class Feature(_Content): @@ -786,6 +810,7 @@ class Feature(_Content): abstract. At a minimum, your template's Feature derivative needs to implement contentdir(). """ + def name(self): """A name for this feature. Defaults to its content directory.""" - return self.contentdir() or 'unnamed' + return self.contentdir() or "unnamed" diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index dd91927c..8127a5f3 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -7,6 +7,7 @@ import re import readline + def slugify(string): """Returns file-system-safe, lower-case version of the input string. @@ -14,10 +15,10 @@ def slugify(string): single underscore. If the variable has no value or the value is an empty string, returns the given default. """ - return re.sub(r'[^\w]+', '_', string, flags=re.ASCII).lower() + return re.sub(r"[^\w]+", "_", string, flags=re.ASCII).lower() -def _rlinput(prompt, prefill=''): +def _rlinput(prompt, prefill=""): """Variation of input() that supports pre-filling a value.""" readline.set_startup_hook(lambda: readline.insert_text(prefill)) try: @@ -26,7 +27,7 @@ def _rlinput(prompt, prefill=''): readline.set_startup_hook() -class UserVar(): +class UserVar: """A class representing a single user variable. User variables have a name and an optional description. They @@ -35,9 +36,10 @@ class UserVar(): cached previous values, and user input. They may come with a default value. """ + def __init__(self, name, val=None, default=None, desc=None): self._name = name - self._desc = desc or '' + self._desc = desc or "" self._val = val self._default = default if default is not None else val @@ -93,23 +95,26 @@ def resolve(self, name, config, user_var_args=None, force=False): for uvar in user_var_args: if uvar.name() == self._name: val = uvar.val() - source = 'command line' + source = "command line" break if val is None: val = os.environ.get(self._name) if val: - source = 'environment' + source = "environment" if source: - print('"{}" will use value of "{}" ({}) from {}: {}'.format( - name, self._name, self._desc, source, val)) + print( + '"{}" will use value of "{}" ({}) from {}: {}'.format( + name, self._name, self._desc, source, val + ) + ) self._val = val return val if val is None: # Try to re-use a cached value in the subsequent prompt - val = config.get('user_vars', self._name, fallback=self._default) + val = config.get("user_vars", self._name, fallback=self._default) if force: if val is None: @@ -117,10 +122,9 @@ def resolve(self, name, config, user_var_args=None, force=False): self._val = val return val - desc = ' (' + self._desc + ')' if self._desc else '' - print('"{}" requires a "{}" value{}: '.format( - name, self._name, desc)) - self._val = _rlinput(self._name + ': ', val) + desc = " (" + self._desc + ")" if self._desc else "" + print('"{}" requires a "{}" value{}: '.format(name, self._name, desc)) + self._val = _rlinput(self._name + ": ", val) return self._val @@ -128,11 +132,12 @@ def resolve(self, name, config, user_var_args=None, force=False): def parse_arg(arg): """Parser for NAME=VAL format string used in command-line args.""" try: - name, val = arg.split('=', 1) + name, val = arg.split("=", 1) return UserVar(name, val=val) except ValueError as error: - raise ValueError('invalid user var argument "{}", must be NAME=VAR' - .format(arg)) from error + raise ValueError( + 'invalid user var argument "{}", must be NAME=VAR'.format(arg) + ) from error @staticmethod def parse_dict(metadata_dict): @@ -146,7 +151,7 @@ def parse_dict(metadata_dict): list of UserVar. If the 'user_vars' field is not present, an empty list is returned. If malformed, returns None. """ - text = metadata_dict.get('user_vars') + text = metadata_dict.get("user_vars") if not text: return [] diff --git a/zkg b/zkg index 96964735..a7c22442 100755 --- a/zkg +++ b/zkg @@ -22,24 +22,26 @@ try: import git import semantic_version # noqa # pylint: disable=unused-import except ImportError as error: - print("error: zkg failed to import one or more dependencies:\n" - "\n" - "* GitPython: https://pypi.org/project/GitPython\n" - "* semantic-version: https://pypi.org/project/semantic-version\n" - "\n" - "If you use 'pip', they can be installed like:\n" - "\n" - " pip3 install GitPython semantic-version\n" - "\n" - "Also check the following exception output for possible alternate explanations:\n\n" - "{}: {}".format(type(error).__name__, error), - file=sys.stderr) + print( + "error: zkg failed to import one or more dependencies:\n" + "\n" + "* GitPython: https://pypi.org/project/GitPython\n" + "* semantic-version: https://pypi.org/project/semantic-version\n" + "\n" + "If you use 'pip', they can be installed like:\n" + "\n" + " pip3 install GitPython semantic-version\n" + "\n" + "Also check the following exception output for possible alternate explanations:\n\n" + "{}: {}".format(type(error).__name__, error), + file=sys.stderr, + ) sys.exit(1) try: # Argcomplete provides command-line completion for users of argparse. # We support it if available, but don't complain when it isn't. - import argcomplete # pylint: disable=import-error + import argcomplete # pylint: disable=import-error except ImportError: pass @@ -47,7 +49,7 @@ except ImportError: # installation. This ensures we find the matching zeekpkg module # first, avoiding potential conflicts with installations elsewhere on # the system. -ZEEK_PYTHON_DIR = '@PY_MOD_INSTALL_DIR@' +ZEEK_PYTHON_DIR = "@PY_MOD_INSTALL_DIR@" if os.path.isdir(ZEEK_PYTHON_DIR): sys.path.insert(0, os.path.abspath(ZEEK_PYTHON_DIR)) else: @@ -56,32 +58,32 @@ else: # Similarly, make Zeek's binary installation path available by # default. This helps package installations succeed that require # e.g. zeek-config for their build process. -ZEEK_BIN_DIR = '@ZEEK_BIN_DIR@' +ZEEK_BIN_DIR = "@ZEEK_BIN_DIR@" if os.path.isdir(ZEEK_BIN_DIR): try: - if ZEEK_BIN_DIR not in os.environ['PATH'].split(os.pathsep): - os.environ['PATH'] = ZEEK_BIN_DIR + os.pathsep + os.environ['PATH'] + if ZEEK_BIN_DIR not in os.environ["PATH"].split(os.pathsep): + os.environ["PATH"] = ZEEK_BIN_DIR + os.pathsep + os.environ["PATH"] except KeyError: - os.environ['PATH'] = ZEEK_BIN_DIR + os.environ["PATH"] = ZEEK_BIN_DIR else: ZEEK_BIN_DIR = None # Also when bundling with Zeek, use directories in the install tree # for storing the zkg configuration and its variable state. Support # for overrides via environment variables simplifies testing. -ZEEK_ZKG_CONFIG_DIR = os.getenv('ZEEK_ZKG_CONFIG_DIR') or '@ZEEK_ZKG_CONFIG_DIR@' +ZEEK_ZKG_CONFIG_DIR = os.getenv("ZEEK_ZKG_CONFIG_DIR") or "@ZEEK_ZKG_CONFIG_DIR@" if not os.path.isdir(ZEEK_ZKG_CONFIG_DIR): ZEEK_ZKG_CONFIG_DIR = None -ZEEK_ZKG_STATE_DIR = os.getenv('ZEEK_ZKG_STATE_DIR') or '@ZEEK_ZKG_STATE_DIR@' +ZEEK_ZKG_STATE_DIR = os.getenv("ZEEK_ZKG_STATE_DIR") or "@ZEEK_ZKG_STATE_DIR@" if not os.path.isdir(ZEEK_ZKG_STATE_DIR): ZEEK_ZKG_STATE_DIR = None # The default package source we fall back to as needed -ZKG_DEFAULT_SOURCE = 'https://github.com/zeek/packages' +ZKG_DEFAULT_SOURCE = "https://github.com/zeek/packages" # The default package template -ZKG_DEFAULT_TEMPLATE = 'https://github.com/zeek/package-template' +ZKG_DEFAULT_TEMPLATE = "https://github.com/zeek/package-template" from zeekpkg._util import ( delete_path, @@ -106,12 +108,12 @@ import zeekpkg def confirmation_prompt(prompt, default_to_yes=True): - yes = {'y', 'ye', 'yes'} + yes = {"y", "ye", "yes"} if default_to_yes: - prompt += ' [Y/n] ' + prompt += " [Y/n] " else: - prompt += ' [N/y] ' + prompt += " [N/y] " choice = input(prompt).lower() @@ -119,13 +121,13 @@ def confirmation_prompt(prompt, default_to_yes=True): if default_to_yes: return True else: - print('Abort.') + print("Abort.") return False if choice in yes: return True - print('Abort.') + print("Abort.") return False @@ -142,24 +144,31 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): for uvar in requested_user_vars: try: - answers[uvar.name()] = uvar.resolve(name, config, args.user_var, args.force) + answers[uvar.name()] = uvar.resolve( + name, config, args.user_var, args.force + ) except ValueError: - print_error(str.format('error: could not determine value of user variable "{}",' - ' provide via environment or --user-var', uvar.name())) + print_error( + str.format( + 'error: could not determine value of user variable "{}",' + " provide via environment or --user-var", + uvar.name(), + ) + ) sys.exit(1) if not args.force and answers: for key, value in answers.items(): - if not config.has_section('user_vars'): - config.add_section('user_vars') + if not config.has_section("user_vars"): + config.add_section("user_vars") - config.set('user_vars', key, value) + config.set("user_vars", key, value) if configfile: - with io.open(configfile, 'w', encoding=std_encoding(sys.stdout)) as f: + with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print('Saved answers to config file: {}'.format(configfile)) + print("Saved answers to config file: {}".format(configfile)) manager.user_vars = answers @@ -170,7 +179,7 @@ def print_error(*args, **kwargs): def config_items(config, section): # Same as config.items(section), but exclude default keys. - defaults = {key for key, _ in config.items('DEFAULT')} + defaults = {key for key, _ in config.items("DEFAULT")} items = sorted(config.items(section)) return [(key, value) for (key, value) in items if key not in defaults] @@ -181,28 +190,28 @@ def file_is_not_empty(path): def find_configfile(args): if args.user: - configfile = os.path.join(home_config_dir(), 'config') + configfile = os.path.join(home_config_dir(), "config") if file_is_not_empty(configfile): return configfile return None - configfile = os.environ.get('ZKG_CONFIG_FILE') + configfile = os.environ.get("ZKG_CONFIG_FILE") if not configfile: # For backward compatibility with old bro-pkg. - configfile = os.environ.get('BRO_PKG_CONFIG_FILE') + configfile = os.environ.get("BRO_PKG_CONFIG_FILE") if configfile and file_is_not_empty(configfile): return configfile - configfile = os.path.join(default_config_dir(), 'config') + configfile = os.path.join(default_config_dir(), "config") if file_is_not_empty(configfile): return configfile - configfile = os.path.join(legacy_config_dir(), 'config') + configfile = os.path.join(legacy_config_dir(), "config") if file_is_not_empty(configfile): return configfile @@ -211,7 +220,7 @@ def find_configfile(args): def home_config_dir(): - return os.path.join(os.path.expanduser('~'), '.zkg') + return os.path.join(os.path.expanduser("~"), ".zkg") def default_config_dir(): @@ -219,7 +228,7 @@ def default_config_dir(): def legacy_config_dir(): - return os.path.join(os.path.expanduser('~'), '.bro-pkg') + return os.path.join(os.path.expanduser("~"), ".bro-pkg") def default_state_dir(): @@ -227,7 +236,7 @@ def default_state_dir(): def legacy_state_dir(): - return os.path.join(os.path.expanduser('~'), '.bro-pkg') + return os.path.join(os.path.expanduser("~"), ".bro-pkg") def create_config(args, configfile): @@ -240,28 +249,27 @@ def create_config(args, configfile): config.read(configfile) - if not config.has_section('sources'): - config.add_section('sources') + if not config.has_section("sources"): + config.add_section("sources") - if not config.has_section('paths'): - config.add_section('paths') + if not config.has_section("paths"): + config.add_section("paths") - if not config.has_section('templates'): - config.add_section('templates') + if not config.has_section("templates"): + config.add_section("templates") if not configfile: - default = os.getenv('ZKG_DEFAULT_SOURCE', ZKG_DEFAULT_SOURCE) + default = os.getenv("ZKG_DEFAULT_SOURCE", ZKG_DEFAULT_SOURCE) if default: - config.set('sources', 'zeek', default) + config.set("sources", "zeek", default) - if not configfile or not config.has_option('templates', 'default'): - default = os.getenv('ZKG_DEFAULT_TEMPLATE', ZKG_DEFAULT_TEMPLATE) + if not configfile or not config.has_option("templates", "default"): + default = os.getenv("ZKG_DEFAULT_TEMPLATE", ZKG_DEFAULT_TEMPLATE) if default: - config.set('templates', 'default', default) + config.set("templates", "default", default) def config_option_set(config, section, option): - return config.has_option(section, option) and config.get(section, - option) + return config.has_option(section, option) and config.get(section, option) def get_option(config, section, option, default): if config_option_set(config, section, option): @@ -274,39 +282,45 @@ def create_config(args, configfile): else: def_state_dir = default_state_dir() - state_dir = get_option(config, 'paths', 'state_dir', - os.path.join(def_state_dir)) - script_dir = get_option(config, 'paths', 'script_dir', - os.path.join(state_dir, 'script_dir')) - plugin_dir = get_option(config, 'paths', 'plugin_dir', - os.path.join(state_dir, 'plugin_dir')) - bin_dir = get_option(config, 'paths', 'bin_dir', - os.path.join(state_dir, 'bin')) - zeek_dist = get_option(config, 'paths', 'zeek_dist', '') + state_dir = get_option(config, "paths", "state_dir", os.path.join(def_state_dir)) + script_dir = get_option( + config, "paths", "script_dir", os.path.join(state_dir, "script_dir") + ) + plugin_dir = get_option( + config, "paths", "plugin_dir", os.path.join(state_dir, "plugin_dir") + ) + bin_dir = get_option(config, "paths", "bin_dir", os.path.join(state_dir, "bin")) + zeek_dist = get_option(config, "paths", "zeek_dist", "") if not zeek_dist: - zeek_dist = get_option(config, 'paths', 'bro_dist', '') + zeek_dist = get_option(config, "paths", "bro_dist", "") - config.set('paths', 'state_dir', state_dir) - config.set('paths', 'script_dir', script_dir) - config.set('paths', 'plugin_dir', plugin_dir) - config.set('paths', 'bin_dir', bin_dir) - config.set('paths', 'zeek_dist', zeek_dist) + config.set("paths", "state_dir", state_dir) + config.set("paths", "script_dir", script_dir) + config.set("paths", "plugin_dir", plugin_dir) + config.set("paths", "bin_dir", bin_dir) + config.set("paths", "zeek_dist", zeek_dist) def expand_config_values(config, section): for key, value in config.items(section): value = os.path.expandvars(os.path.expanduser(value)) config.set(section, key, value) - expand_config_values(config, 'sources') - expand_config_values(config, 'paths') - expand_config_values(config, 'templates') + expand_config_values(config, "sources") + expand_config_values(config, "paths") + expand_config_values(config, "templates") - for key, value in config.items('paths'): + for key, value in config.items("paths"): if value and not os.path.isabs(value): - print_error(str.format('error: invalid config file value for key' - ' "{}" in section [paths]: "{}" is not' - ' an absolute path', key, value)) + print_error( + str.format( + "error: invalid config file value for key" + ' "{}" in section [paths]: "{}" is not' + " an absolute path", + key, + value, + ) + ) sys.exit(1) return config @@ -335,7 +349,7 @@ def active_git_branch(path): def is_local_git_repo_url(git_url): - return git_url.startswith('.') or git_url.startswith('/') + return git_url.startswith(".") or git_url.startswith("/") def is_local_git_repo(git_url): @@ -363,51 +377,60 @@ def is_local_git_repo_dirty(git_url): def check_local_git_repo(git_url): if is_local_git_repo_url(git_url): if not is_local_git_repo(git_url): - print_error('error: path {} is not a git repository'.format(git_url)) + print_error("error: path {} is not a git repository".format(git_url)) return False if is_local_git_repo_dirty(git_url): - print_error('error: local git clone at {} is dirty'.format(git_url)) + print_error("error: local git clone at {} is dirty".format(git_url)) return False return True def create_manager(args, config): - state_dir = config.get('paths', 'state_dir') - script_dir = config.get('paths', 'script_dir') - plugin_dir = config.get('paths', 'plugin_dir') - bin_dir = config.get('paths', 'bin_dir') - zeek_dist = config.get('paths', 'zeek_dist') + state_dir = config.get("paths", "state_dir") + script_dir = config.get("paths", "script_dir") + plugin_dir = config.get("paths", "plugin_dir") + bin_dir = config.get("paths", "bin_dir") + zeek_dist = config.get("paths", "zeek_dist") if state_dir == default_state_dir(): if not os.path.exists(state_dir) and os.path.exists(legacy_state_dir()): make_symlink(legacy_state_dir(), state_dir) try: - manager = zeekpkg.Manager(state_dir=state_dir, script_dir=script_dir, - plugin_dir=plugin_dir, bin_dir=bin_dir, - zeek_dist=zeek_dist) + manager = zeekpkg.Manager( + state_dir=state_dir, + script_dir=script_dir, + plugin_dir=plugin_dir, + bin_dir=bin_dir, + zeek_dist=zeek_dist, + ) except (OSError, IOError) as error: if error.errno == errno.EACCES: - print_error('{}: {}'.format(type(error).__name__, error)) + print_error("{}: {}".format(type(error).__name__, error)) def check_permission(d): if os.access(d, os.W_OK): return True - print_error( - 'error: user does not have write access in {}'.format(d)) + print_error("error: user does not have write access in {}".format(d)) return False - permissions_trouble = not all([check_permission(state_dir), - check_permission(script_dir), - check_permission(plugin_dir), - check_permission(bin_dir)]) + permissions_trouble = not all( + [ + check_permission(state_dir), + check_permission(script_dir), + check_permission(plugin_dir), + check_permission(bin_dir), + ] + ) if permissions_trouble and not args.user: print_error( - 'Consider the --user flag to manage zkg state via {}/config' - .format(home_config_dir())) + "Consider the --user flag to manage zkg state via {}/config".format( + home_config_dir() + ) + ) sys.exit(1) raise @@ -427,13 +450,15 @@ def create_manager(args, config): extra_sources.append((key, value)) - for key, value in extra_sources + config_items(config, 'sources'): + for key, value in extra_sources + config_items(config, "sources"): error = manager.add_source(name=key, git_url=value) if error: - print_error(str.format( - 'warning: skipped using package source named "{}": {}', - key, error)) + print_error( + str.format( + 'warning: skipped using package source named "{}": {}', key, error + ) + ) return manager @@ -452,7 +477,7 @@ def get_changed_state(manager, saved_state, pkg_lst): """ _lst = [zeekpkg.package.name_from_path(_pkg_path) for _pkg_path in pkg_lst] - dep_listing = '' + dep_listing = "" for _pkg_name in sorted(manager.installed_package_dependencies()): if _pkg_name in _lst: @@ -463,23 +488,21 @@ def get_changed_state(manager, saved_state, pkg_lst): if not _ipkg or _ipkg.status.is_loaded == saved_state[_pkg_name]: continue - dep_listing += ' {}\n'.format(_pkg_name) + dep_listing += " {}\n".format(_pkg_name) return dep_listing class InstallWorker(threading.Thread): - def __init__(self, manager, package_name, package_version): super(InstallWorker, self).__init__() self.manager = manager self.package_name = package_name self.package_version = package_version - self.error = '' + self.error = "" def run(self): - self.error = self.manager.install( - self.package_name, self.package_version) + self.error = self.manager.install(self.package_name, self.package_version) def wait(self, msg=None, out=sys.stdout, tty_only=True): """Blocks until this thread ends, optionally writing liveness indicators. @@ -502,7 +525,7 @@ class InstallWorker(threading.Thread): out.write(msg) out.flush() - is_tty = hasattr(out, 'isatty') and out.isatty() + is_tty = hasattr(out, "isatty") and out.isatty() while True: self.join(1.0) @@ -510,18 +533,17 @@ class InstallWorker(threading.Thread): break if out is not None and (is_tty or not tty_only): - out.write('.') + out.write(".") out.flush() if out is not None and (msg or is_tty or not tty_only): - out.write('\n') + out.write("\n") out.flush() def cmd_test(manager, args, config, configfile): if args.version and len(args.package) > 1: - print_error( - 'error: "test --version" may only be used for a single package') + print_error('error: "test --version" may only be used for a single package') sys.exit(1) package_infos = [] @@ -531,12 +553,14 @@ def cmd_test(manager, args, config, configfile): sys.exit(1) version = args.version if args.version else active_git_branch(name) - package_info = manager.info(name, version=version, - prefer_installed=False) + package_info = manager.info(name, version=version, prefer_installed=False) if package_info.invalid_reason: - print_error(str.format('error: invalid package "{}": {}', name, - package_info.invalid_reason)) + print_error( + str.format( + 'error: invalid package "{}": {}', name, package_info.invalid_reason + ) + ) sys.exit(1) if not version: @@ -549,31 +573,39 @@ def cmd_test(manager, args, config, configfile): for info, version in package_infos: name = info.package.qualified_name() - if 'test_command' not in info.metadata: - print(str.format('{}: no test_command found in metadata, skipping', - name)) + if "test_command" not in info.metadata: + print(str.format("{}: no test_command found in metadata, skipping", name)) continue error_msg, passed, test_dir = manager.test( - name, version, test_dependencies=True) + name, version, test_dependencies=True + ) if error_msg: all_passed = False - print_error(str.format('error: failed to run tests for "{}": {}', - name, error_msg)) + print_error( + str.format('error: failed to run tests for "{}": {}', name, error_msg) + ) continue if passed: - print(str.format('{}: all tests passed', name)) + print(str.format("{}: all tests passed", name)) else: all_passed = False - clone_dir = os.path.join(os.path.join(test_dir, "clones"), - info.package.name) - print_error(str.format('error: package "{}" tests failed, inspect' - ' contents of {} for details, especially' - ' any "zkg.test_command.{{stderr,stdout}}"' - ' files within {}', - name, test_dir, clone_dir)) + clone_dir = os.path.join( + os.path.join(test_dir, "clones"), info.package.name + ) + print_error( + str.format( + 'error: package "{}" tests failed, inspect' + " contents of {} for details, especially" + ' any "zkg.test_command.{{stderr,stdout}}"' + " files within {}", + name, + test_dir, + clone_dir, + ) + ) if not all_passed: sys.exit(1) @@ -581,8 +613,7 @@ def cmd_test(manager, args, config, configfile): def cmd_install(manager, args, config, configfile): if args.version and len(args.package) > 1: - print_error( - 'error: "install --version" may only be used for a single package') + print_error('error: "install --version" may only be used for a single package') sys.exit(1) package_infos = [] @@ -592,12 +623,14 @@ def cmd_install(manager, args, config, configfile): sys.exit(1) version = args.version if args.version else active_git_branch(name) - package_info = manager.info(name, version=version, - prefer_installed=False) + package_info = manager.info(name, version=version, prefer_installed=False) if package_info.invalid_reason: - print_error(str.format('error: invalid package "{}": {}', name, - package_info.invalid_reason)) + print_error( + str.format( + 'error: invalid package "{}": {}', name, package_info.invalid_reason + ) + ) sys.exit(1) if not version: @@ -610,73 +643,80 @@ def cmd_install(manager, args, config, configfile): orig_pkgs, new_pkgs = package_infos.copy(), [] if not args.nodeps: - to_validate = [(info.package.qualified_name(), version) - for info, version, _ in package_infos] + to_validate = [ + (info.package.qualified_name(), version) + for info, version, _ in package_infos + ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions) + to_validate, ignore_suggestions=args.nosuggestions + ) if invalid_reason: - print_error('error: failed to resolve dependencies:', - invalid_reason) + print_error("error: failed to resolve dependencies:", invalid_reason) sys.exit(1) if not args.force: - package_listing = '' + package_listing = "" for info, version, _ in sorted(package_infos, key=lambda x: x[0].package.name): name = info.package.qualified_name() - package_listing += ' {} ({})\n'.format(name, version) + package_listing += " {} ({})\n".format(name, version) - print('The following packages will be INSTALLED:') + print("The following packages will be INSTALLED:") print(package_listing) if new_pkgs: - dependency_listing = '' + dependency_listing = "" - for info, version, suggested in sorted(new_pkgs, key=lambda x: x[0].package.name): + for info, version, suggested in sorted( + new_pkgs, key=lambda x: x[0].package.name + ): name = info.package.qualified_name() - dependency_listing += ' {} ({})'.format( - name, version) + dependency_listing += " {} ({})".format(name, version) if suggested: - dependency_listing += ' (suggested)' + dependency_listing += " (suggested)" - dependency_listing += '\n' + dependency_listing += "\n" - print('The following dependencies will be INSTALLED:') + print("The following dependencies will be INSTALLED:") print(dependency_listing) allpkgs = package_infos + new_pkgs - extdep_listing = '' + extdep_listing = "" for info, version, _ in sorted(allpkgs, key=lambda x: x[0].package.name): name = info.package.qualified_name() - extdeps = info.dependencies(field='external_depends') + extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += ' from {} ({}):\n \n'.format( - name, version) + extdep_listing += " from {} ({}):\n \n".format( + name, version + ) continue if extdeps: - extdep_listing += ' from {} ({}):\n'.format(name, version) + extdep_listing += " from {} ({}):\n".format(name, version) for extdep, semver in sorted(extdeps.items()): - extdep_listing += ' {} {}\n'.format(extdep, semver) + extdep_listing += " {} {}\n".format(extdep, semver) if extdep_listing: - print('Verify the following REQUIRED external dependencies:\n' - '(Ensure their installation on all relevant systems before' - ' proceeding):') + print( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):" + ) print(extdep_listing) - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return package_infos += new_pkgs - prompt_for_user_vars(manager, config, configfile, args, - [info for info, _, _ in package_infos]) + prompt_for_user_vars( + manager, config, configfile, args, [info for info, _, _ in package_infos] + ) if not args.skiptests: # Iterate only over the requested packages here, skipping the @@ -684,38 +724,45 @@ def cmd_install(manager, args, config, configfile): for info, version, _ in orig_pkgs: name = info.package.qualified_name() - if 'test_command' not in info.metadata: - zeekpkg.LOG.info('Skipping unit tests for "%s": ' - 'no test_command in metadata', name) + if "test_command" not in info.metadata: + zeekpkg.LOG.info( + 'Skipping unit tests for "%s": ' "no test_command in metadata", name + ) continue print('Running unit tests for "{}"'.format(name)) - error_msg = '' + error_msg = "" # For testing we always process dependencies, since the tests might # well fail without them. If the user wants --nodeps and the tests # fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True) + name, version, test_dependencies=True + ) if error: - error_msg = str.format( - 'failed to run tests for {}: {}', name, error) + error_msg = str.format("failed to run tests for {}: {}", name, error) elif not passed: - clone_dir = os.path.join(os.path.join(test_dir, "clones"), - info.package.name) - error_msg = str.format('"{}" tests failed, inspect contents of' - ' {} for details, especially any' - ' "zkg.test_command.{{stderr,stdout}}"' - ' files within {}', - name, test_dir, clone_dir) + clone_dir = os.path.join( + os.path.join(test_dir, "clones"), info.package.name + ) + error_msg = str.format( + '"{}" tests failed, inspect contents of' + " {} for details, especially any" + ' "zkg.test_command.{{stderr,stdout}}"' + " files within {}", + name, + test_dir, + clone_dir, + ) if error_msg: - print_error('error: {}'.format(error_msg)) + print_error("error: {}".format(error_msg)) if args.force: continue - if not confirmation_prompt('Proceed to install anyway?', - default_to_yes=False): + if not confirmation_prompt( + "Proceed to install anyway?", default_to_yes=False + ): return installs_failed = [] @@ -730,8 +777,7 @@ def cmd_install(manager, args, config, configfile): is_overwriting = True modifications = manager.modified_config_files(ipkg) backup_files = manager.backup_modified_files(name, modifications) - prev_upstream_config_files = manager.save_temporary_config_files( - ipkg) + prev_upstream_config_files = manager.save_temporary_config_files(ipkg) worker = InstallWorker(manager, name, version) worker.start() @@ -780,35 +826,43 @@ def cmd_install(manager, args, config, configfile): # Now load runtime dependencies after all dependencies and suggested # packages have been installed and loaded. for info, ver, _ in sorted(orig_pkgs, key=lambda x: x[0].package.name): - _listing, saved_state = '', manager.loaded_package_states() + _listing, saved_state = "", manager.loaded_package_states() name = info.package.qualified_name() - load_error = manager.load_with_dependencies(zeekpkg.package.name_from_path(name)) + load_error = manager.load_with_dependencies( + zeekpkg.package.name_from_path(name) + ) for _name, _error in load_error: if not _error: - _listing += ' {}\n'.format(_name) + _listing += " {}\n".format(_name) if not _listing: dep_listing = get_changed_state(manager, saved_state, [name]) if dep_listing: - print('The following installed packages were additionally ' - 'loaded to satisfy runtime dependencies') + print( + "The following installed packages were additionally " + "loaded to satisfy runtime dependencies" + ) print(dep_listing) else: - print('The following installed packages could NOT be loaded ' - 'to satisfy runtime dependencies for "{}"'.format(name)) + print( + "The following installed packages could NOT be loaded " + 'to satisfy runtime dependencies for "{}"'.format(name) + ) print(_listing) manager.restore_loaded_package_states(saved_state) if installs_failed: - print_error('error: incomplete installation, the follow packages' - ' failed to be installed:') + print_error( + "error: incomplete installation, the follow packages" + " failed to be installed:" + ) for n, v in installs_failed: - print_error(' {} ({})'.format(n, v)) + print_error(" {} ({})".format(n, v)) sys.exit(1) @@ -819,18 +873,19 @@ def cmd_bundle(manager, args, config, configfile): if args.manifest: if len(args.manifest) == 1 and os.path.isfile(args.manifest[0]): - config = configparser.ConfigParser(delimiters='=') + config = configparser.ConfigParser(delimiters="=") config.optionxform = str - if config.read(args.manifest[0]) and config.has_section('bundle'): - packages = config.items('bundle') + if config.read(args.manifest[0]) and config.has_section("bundle"): + packages = config.items("bundle") else: - print_error('error: "{}" is not a valid manifest file'.format( - args.manifest[0])) + print_error( + 'error: "{}" is not a valid manifest file'.format(args.manifest[0]) + ) sys.exit(1) else: - packages = [(name, '') for name in args.manifest] + packages = [(name, "") for name in args.manifest] to_validate = [] new_pkgs = [] @@ -845,74 +900,97 @@ def cmd_bundle(manager, args, config, configfile): info = manager.info(name, version=version, prefer_installed=False) if info.invalid_reason: - print_error(str.format('error: invalid package "{}": {}', name, - info.invalid_reason)) + print_error( + str.format( + 'error: invalid package "{}": {}', name, info.invalid_reason + ) + ) sys.exit(1) if not version: version = info.best_version() to_validate.append((info.package.qualified_name(), version)) - packages_to_bundle.append((info.package.qualified_name(), - info.package.git_url, version, False, - False)) + packages_to_bundle.append( + ( + info.package.qualified_name(), + info.package.git_url, + version, + False, + False, + ) + ) if not args.nodeps: invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, True, ignore_suggestions=args.nosuggestions) + to_validate, True, ignore_suggestions=args.nosuggestions + ) if invalid_reason: - print_error('error: failed to resolve dependencies:', - invalid_reason) + print_error("error: failed to resolve dependencies:", invalid_reason) sys.exit(1) for info, version, suggested in new_pkgs: - packages_to_bundle.append((info.package.qualified_name(), - info.package.git_url, version, True, - suggested)) + packages_to_bundle.append( + ( + info.package.qualified_name(), + info.package.git_url, + version, + True, + suggested, + ) + ) else: prefer_existing_clones = True for ipkg in manager.installed_packages(): - packages_to_bundle.append((ipkg.package.qualified_name(), - ipkg.package.git_url, - ipkg.status.current_version, False, - False)) + packages_to_bundle.append( + ( + ipkg.package.qualified_name(), + ipkg.package.git_url, + ipkg.status.current_version, + False, + False, + ) + ) if not packages_to_bundle: - print_error('error: no packages to put in bundle') + print_error("error: no packages to put in bundle") sys.exit(1) if not args.force: - package_listing = '' + package_listing = "" for name, _, version, is_dependency, is_suggestion in packages_to_bundle: - package_listing += ' {} ({})'.format(name, version) + package_listing += " {} ({})".format(name, version) if is_suggestion: - package_listing += ' (suggested)' + package_listing += " (suggested)" elif is_dependency: - package_listing += ' (dependency)' + package_listing += " (dependency)" - package_listing += '\n' + package_listing += "\n" - print('The following packages will be BUNDLED into {}:'.format( - args.bundle_filename)) + print( + "The following packages will be BUNDLED into {}:".format( + args.bundle_filename + ) + ) print(package_listing) - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return - git_urls = [(git_url, version) - for _, git_url, version, _, _ in packages_to_bundle] - error = manager.bundle(args.bundle_filename, git_urls, - prefer_existing_clones=prefer_existing_clones) + git_urls = [(git_url, version) for _, git_url, version, _, _ in packages_to_bundle] + error = manager.bundle( + args.bundle_filename, git_urls, prefer_existing_clones=prefer_existing_clones + ) if error: - print_error('error: failed to create bundle: {}'.format(error)) + print_error("error: failed to create bundle: {}".format(error)) sys.exit(1) - print('Bundle successfully written: {}'.format(args.bundle_filename)) + print("Bundle successfully written: {}".format(args.bundle_filename)) def cmd_unbundle(manager, args, config, configfile): @@ -927,23 +1005,27 @@ def cmd_unbundle(manager, args, config, configfile): error, bundle_info = manager.bundle_info(args.bundle_filename) if error: - print_error('error: failed to unbundle {}: {}'.format( - args.bundle_filename, error)) + print_error( + "error: failed to unbundle {}: {}".format(args.bundle_filename, error) + ) sys.exit(1) for git_url, version, pkg_info in bundle_info: if pkg_info.invalid_reason: name = pkg_info.package.qualified_name() - print_error('error: bundle {} contains invalid package {}: {}'.format( - args.bundle_filename, name, pkg_info.invalid_reason)) + print_error( + "error: bundle {} contains invalid package {}: {}".format( + args.bundle_filename, name, pkg_info.invalid_reason + ) + ) sys.exit(1) if not bundle_info: - print('No packages in bundle.') + print("No packages in bundle.") return if not args.force: - package_listing = '' + package_listing = "" for git_url, version, _ in bundle_info: name = git_url @@ -953,12 +1035,12 @@ def cmd_unbundle(manager, args, config, configfile): name = pkg.qualified_name() break - package_listing += ' {} ({})\n'.format(name, version) + package_listing += " {} ({})\n".format(name, version) - print('The following packages will be INSTALLED:') + print("The following packages will be INSTALLED:") print(package_listing) - extdep_listing = '' + extdep_listing = "" for git_url, version, info in bundle_info: name = git_url @@ -968,36 +1050,41 @@ def cmd_unbundle(manager, args, config, configfile): name = pkg.qualified_name() break - extdeps = info.dependencies(field='external_depends') + extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += ' from {} ({}):\n \n'.format( - name, version) + extdep_listing += " from {} ({}):\n \n".format( + name, version + ) continue if extdeps: - extdep_listing += ' from {} ({}):\n'.format(name, version) + extdep_listing += " from {} ({}):\n".format(name, version) for extdep, semver in sorted(extdeps.items()): - extdep_listing += ' {} {}\n'.format(extdep, semver) + extdep_listing += " {} {}\n".format(extdep, semver) if extdep_listing: - print('Verify the following REQUIRED external dependencies:\n' - '(Ensure their installation on all relevant systems before' - ' proceeding):') + print( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):" + ) print(extdep_listing) - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return - prompt_for_user_vars(manager, config, configfile, args, - [info for _, _, info in bundle_info]) + prompt_for_user_vars( + manager, config, configfile, args, [info for _, _, info in bundle_info] + ) error = manager.unbundle(args.bundle_filename) if error: - print_error('error: failed to unbundle {}: {}'.format( - args.bundle_filename, error)) + print_error( + "error: failed to unbundle {}: {}".format(args.bundle_filename, error) + ) sys.exit(1) for git_url, _, _ in bundle_info: @@ -1025,7 +1112,8 @@ def cmd_unbundle(manager, args, config, configfile): else: print('Loaded "{}"'.format(name)) - print('Unbundling complete.') + print("Unbundling complete.") + def cmd_remove(manager, args, config, configfile): packages_to_remove = [] @@ -1041,8 +1129,7 @@ def cmd_remove(manager, args, config, configfile): ipkg = manager.find_installed_package(name) if not ipkg: - print_error( - 'error: package "{}" is not installed'.format(name)) + print_error('error: package "{}" is not installed'.format(name)) sys.exit(1) packages_to_remove.append(ipkg) @@ -1059,23 +1146,23 @@ def cmd_remove(manager, args, config, configfile): dependers_to_unload.add(ipkg.package.name) if not args.force: - print('The following packages will be REMOVED:') + print("The following packages will be REMOVED:") for ipkg in packages_to_remove: - print(' {}'.format(ipkg.package.qualified_name())) + print(" {}".format(ipkg.package.qualified_name())) print() if dependers_to_unload: - print('The following dependent packages will be UNLOADED:') + print("The following dependent packages will be UNLOADED:") for pkg_name in sorted(dependers_to_unload): ipkg = manager.find_installed_package(pkg_name) - print(' {}'.format(ipkg.package.qualified_name())) + print(" {}".format(ipkg.package.qualified_name())) print() - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return for pkg_name in sorted(dependers_to_unload): @@ -1101,10 +1188,10 @@ def cmd_remove(manager, args, config, configfile): print('Removed "{}"'.format(name)) if backup_files: - print('\tCreated backups of locally modified config files:') + print("\tCreated backups of locally modified config files:") for backup_file in backup_files: - print('\t' + backup_file) + print("\t" + backup_file) else: print('Failed removing "{}": no such package installed'.format(name)) @@ -1118,21 +1205,20 @@ def cmd_purge(manager, args, config, configfile): packages_to_remove = manager.installed_packages() if not packages_to_remove: - print('No packages to remove.') + print("No packages to remove.") return if not args.force: - package_listing = '' - names_to_remove = [ipkg.package.qualified_name() - for ipkg in packages_to_remove] + package_listing = "" + names_to_remove = [ipkg.package.qualified_name() for ipkg in packages_to_remove] for name in names_to_remove: - package_listing += ' {}\n'.format(name) + package_listing += " {}\n".format(name) - print('The following packages will be REMOVED:') + print("The following packages will be REMOVED:") print(package_listing) - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return had_failure = False @@ -1146,10 +1232,10 @@ def cmd_purge(manager, args, config, configfile): print('Removed "{}"'.format(name)) if backup_files: - print('\tCreated backups of locally modified config files:') + print("\tCreated backups of locally modified config files:") for backup_file in backup_files: - print('\t' + backup_file) + print("\t" + backup_file) else: print('Unknown error removing "{}"'.format(name)) @@ -1160,9 +1246,11 @@ def cmd_purge(manager, args, config, configfile): def outdated(manager): - return [ipkg.package.qualified_name() - for ipkg in manager.installed_packages() - if ipkg.status.is_outdated] + return [ + ipkg.package.qualified_name() + for ipkg in manager.installed_packages() + if ipkg.status.is_outdated + ] def cmd_refresh(manager, args, config, configfile): @@ -1170,19 +1258,20 @@ def cmd_refresh(manager, args, config, configfile): args.sources = list(manager.sources.keys()) if args.fail_on_aggregate_problems and not args.aggregate: - print_error('warning: --fail-on-aggregate-problems without --aggregate' - ' has no effect.') + print_error( + "warning: --fail-on-aggregate-problems without --aggregate" + " has no effect." + ) had_failure = False had_aggregation_failure = False for source in args.sources: - print('Refresh package source: {}'.format(source)) + print("Refresh package source: {}".format(source)) - src_pkgs_before = {i.qualified_name() - for i in manager.source_packages()} + src_pkgs_before = {i.qualified_name() for i in manager.source_packages()} - error = '' + error = "" aggregation_issues = [] if args.aggregate: @@ -1194,53 +1283,53 @@ def cmd_refresh(manager, args, config, configfile): if error: had_failure = True - print_error( - 'error: failed to refresh "{}": {}'.format(source, error)) + print_error('error: failed to refresh "{}": {}'.format(source, error)) continue - src_pkgs_after = {i.qualified_name() - for i in manager.source_packages()} + src_pkgs_after = {i.qualified_name() for i in manager.source_packages()} if src_pkgs_before == src_pkgs_after: - print('\tNo membership changes') + print("\tNo membership changes") else: - print('\tChanges:') + print("\tChanges:") diff = src_pkgs_before.symmetric_difference(src_pkgs_after) for name in diff: - change = 'Added' if name in src_pkgs_after else 'Removed' - print('\t\t{} {}'.format(change, name)) + change = "Added" if name in src_pkgs_after else "Removed" + print("\t\t{} {}".format(change, name)) if args.aggregate: if aggregation_issues: - print('\tWARNING: Metadata aggregated, but excludes the ' - 'following packages due to described problems:') + print( + "\tWARNING: Metadata aggregated, but excludes the " + "following packages due to described problems:" + ) for url, issue in aggregation_issues: - print('\t\t{}: {}'.format(url, issue)) + print("\t\t{}: {}".format(url, issue)) if args.fail_on_aggregate_problems: had_aggregation_failure = True else: - print('\tMetadata aggregated') + print("\tMetadata aggregated") if args.push: - print('\tPushed aggregated metadata') + print("\tPushed aggregated metadata") outdated_before = {i for i in outdated(manager)} - print('Refresh installed packages') + print("Refresh installed packages") manager.refresh_installed_packages() outdated_after = {i for i in outdated(manager)} if outdated_before == outdated_after: - print('\tNo new outdated packages') + print("\tNo new outdated packages") else: - print('\tNew outdated packages:') + print("\tNew outdated packages:") diff = outdated_before.symmetric_difference(outdated_after) for name in diff: ipkg = manager.find_installed_package(name) version_change = version_change_string(manager, ipkg) - print('\t\t{} {}'.format(name, version_change)) + print("\t\t{} {}".format(name, version_change)) if had_failure: sys.exit(1) @@ -1251,7 +1340,7 @@ def cmd_refresh(manager, args, config, configfile): def version_change_string(manager, installed_package): old_version = installed_package.status.current_version new_version = old_version - version_change = '' + version_change = "" if installed_package.status.tracking_method == TRACKING_METHOD_VERSION: versions = manager.package_versions(installed_package) @@ -1259,9 +1348,9 @@ def version_change_string(manager, installed_package): if len(versions): new_version = versions[-1] - version_change = '({} -> {})'.format(old_version, new_version) + version_change = "({} -> {})".format(old_version, new_version) else: - version_change = '({})'.format(new_version) + version_change = "({})".format(new_version) return version_change @@ -1273,7 +1362,7 @@ def cmd_upgrade(manager, args, config, configfile): pkg_list = outdated(manager) outdated_packages = [] - package_listing = '' + package_listing = "" for name in pkg_list: ipkg = manager.find_installed_package(name) @@ -1290,131 +1379,142 @@ def cmd_upgrade(manager, args, config, configfile): if not manager.match_source_packages(name): name = ipkg.package.git_url - info = manager.info(name, version=ipkg.status.current_version, - prefer_installed=False) + info = manager.info( + name, version=ipkg.status.current_version, prefer_installed=False + ) if info.invalid_reason: - print_error(str.format('error: invalid package "{}": {}', name, - info.invalid_reason)) + print_error( + str.format('error: invalid package "{}": {}', name, info.invalid_reason) + ) sys.exit(1) next_version = ipkg.status.current_version - if ( ipkg.status.tracking_method == TRACKING_METHOD_VERSION and - info.versions ): + if ipkg.status.tracking_method == TRACKING_METHOD_VERSION and info.versions: next_version = info.versions[-1] outdated_packages.append((info, next_version, False)) version_change = version_change_string(manager, ipkg) - package_listing += ' {} {}\n'.format(name, version_change) + package_listing += " {} {}\n".format(name, version_change) if not outdated_packages: - print('All packages already up-to-date.') + print("All packages already up-to-date.") return new_pkgs = [] if not args.nodeps: - to_validate = [(info.package.qualified_name(), next_version) - for info, next_version, _ in outdated_packages] + to_validate = [ + (info.package.qualified_name(), next_version) + for info, next_version, _ in outdated_packages + ] invalid_reason, new_pkgs = manager.validate_dependencies( - to_validate, ignore_suggestions=args.nosuggestions) + to_validate, ignore_suggestions=args.nosuggestions + ) if invalid_reason: - print_error('error: failed to resolve dependencies:', - invalid_reason) + print_error("error: failed to resolve dependencies:", invalid_reason) sys.exit(1) allpkgs = outdated_packages + new_pkgs if not args.force: - print('The following packages will be UPGRADED:') + print("The following packages will be UPGRADED:") print(package_listing) if new_pkgs: - dependency_listing = '' + dependency_listing = "" for info, version, suggestion in new_pkgs: name = info.package.qualified_name() - dependency_listing += ' {} ({})'.format(name, version) + dependency_listing += " {} ({})".format(name, version) if suggestion: - dependency_listing += ' (suggested)' + dependency_listing += " (suggested)" - dependency_listing += '\n' + dependency_listing += "\n" - - print('The following dependencies will be INSTALLED:') + print("The following dependencies will be INSTALLED:") print(dependency_listing) - extdep_listing = '' + extdep_listing = "" for info, version, _ in allpkgs: name = info.package.qualified_name() - extdeps = info.dependencies(field='external_depends') + extdeps = info.dependencies(field="external_depends") if extdeps is None: - extdep_listing += ' from {} ({}):\n \n'.format( - name, version) + extdep_listing += " from {} ({}):\n \n".format( + name, version + ) continue if extdeps: - extdep_listing += ' from {} ({}):\n'.format(name, version) + extdep_listing += " from {} ({}):\n".format(name, version) for extdep, semver in sorted(extdeps.items()): - extdep_listing += ' {} {}\n'.format(extdep, semver) + extdep_listing += " {} {}\n".format(extdep, semver) if extdep_listing: - print('Verify the following REQUIRED external dependencies:\n' - '(Ensure their installation on all relevant systems before' - ' proceeding):') + print( + "Verify the following REQUIRED external dependencies:\n" + "(Ensure their installation on all relevant systems before" + " proceeding):" + ) print(extdep_listing) - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): return - prompt_for_user_vars(manager, config, configfile, args, - [info for info, _, _ in allpkgs]) + prompt_for_user_vars( + manager, config, configfile, args, [info for info, _, _ in allpkgs] + ) if not args.skiptests: for info, version, _ in outdated_packages: name = info.package.qualified_name() - if 'test_command' not in info.metadata: + if "test_command" not in info.metadata: zeekpkg.LOG.info( - 'Skipping unit tests for "%s": no test_command in metadata', - name) + 'Skipping unit tests for "%s": no test_command in metadata', name + ) continue print('Running unit tests for "{}"'.format(name)) - error_msg = '' + error_msg = "" # As in cmd_install, we always process dependencies since the tests # might well fail without them. If the user wants --nodeps and the # tests fail because of it, they'll also need to say --skiptests. error, passed, test_dir = manager.test( - name, version, test_dependencies=True) + name, version, test_dependencies=True + ) if error: - error_msg = str.format( - 'failed to run tests for {}: {}', name, error) + error_msg = str.format("failed to run tests for {}: {}", name, error) elif not passed: - clone_dir = os.path.join(os.path.join(test_dir, "clones"), - info.package.name) - error_msg = str.format('"{}" tests failed, inspect contents of' - ' {} for details, especially any' - ' "zkg.test_command.{{stderr,stdout}}"' - ' files within {}', - name, test_dir, clone_dir) - + clone_dir = os.path.join( + os.path.join(test_dir, "clones"), info.package.name + ) + error_msg = str.format( + '"{}" tests failed, inspect contents of' + " {} for details, especially any" + ' "zkg.test_command.{{stderr,stdout}}"' + " files within {}", + name, + test_dir, + clone_dir, + ) if error_msg: - print_error('error: {}'.format(error_msg)) + print_error("error: {}".format(error_msg)) if args.force: continue - if not confirmation_prompt('Proceed to install anyway?', - default_to_yes=False): + if not confirmation_prompt( + "Proceed to install anyway?", default_to_yes=False + ): return for info, version, _ in reversed(new_pkgs): @@ -1460,8 +1560,7 @@ def cmd_upgrade(manager, args, config, configfile): had_failure = True else: ipkg = manager.find_installed_package(name) - print('Upgraded "{}" ({})'.format( - name, ipkg.status.current_version)) + print('Upgraded "{}" ({})'.format(name, ipkg.status.current_version)) for i, mf in enumerate(modifications): next_upstream_config_file = mf[1] @@ -1492,7 +1591,7 @@ def cmd_upgrade(manager, args, config, configfile): def cmd_load(manager, args, config, configfile): had_failure = False load_error = False - dep_error_listing = '' + dep_error_listing = "" for name in args.package: ipkg = manager.find_installed_package(name) @@ -1512,21 +1611,25 @@ def cmd_load(manager, args, config, configfile): load_error = manager.load(name) else: saved_state = manager.loaded_package_states() - dep_error_listing, load_error = '', False + dep_error_listing, load_error = "", False - loaded_dep_list = manager.load_with_dependencies(zeekpkg.package.name_from_path(name)) + loaded_dep_list = manager.load_with_dependencies( + zeekpkg.package.name_from_path(name) + ) for _name, _error in loaded_dep_list: if _error: load_error = True - dep_error_listing += ' {}: {}\n'.format(_name, _error) + dep_error_listing += " {}: {}\n".format(_name, _error) if not load_error: dep_listing = get_changed_state(manager, saved_state, [name]) if dep_listing: - print('The following installed packages were additionally loaded to satisfy' \ - ' runtime dependencies for "{}".'.format(name)) + print( + "The following installed packages were additionally loaded to satisfy" + ' runtime dependencies for "{}".'.format(name) + ) print(dep_listing) if load_error: @@ -1534,7 +1637,11 @@ def cmd_load(manager, args, config, configfile): if not args.nodeps: if dep_error_listing: - print('The following installed dependencies could not be loaded for "{}".'.format(name)) + print( + 'The following installed dependencies could not be loaded for "{}".'.format( + name + ) + ) print(dep_error_listing) manager.restore_loaded_package_states(saved_state) @@ -1580,25 +1687,24 @@ def cmd_unload(manager, args, config, configfile): if ipkg.status.is_loaded: dependers_to_unload.add(ipkg.package.name) - if packages_to_unload and not args.force: - print('The following packages will be UNLOADED:') + print("The following packages will be UNLOADED:") for ipkg in packages_to_unload: - print(' {}'.format(ipkg.package.qualified_name())) + print(" {}".format(ipkg.package.qualified_name())) print() if dependers_to_unload: - print('The following dependent packages will be UNLOADED:') + print("The following dependent packages will be UNLOADED:") for pkg_name in sorted(dependers_to_unload): ipkg = manager.find_installed_package(pkg_name) - print(' {}'.format(ipkg.package.qualified_name())) + print(" {}".format(ipkg.package.qualified_name())) print() - if not confirmation_prompt('Proceed?'): + if not confirmation_prompt("Proceed?"): if had_failure: sys.exit(1) else: @@ -1635,8 +1741,11 @@ def cmd_pin(manager, args, config, configfile): ipkg = manager.pin(name) if ipkg: - print('Pinned "{}" at version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash)) + print( + 'Pinned "{}" at version: {} ({})'.format( + name, ipkg.status.current_version, ipkg.status.current_hash + ) + ) else: had_failure = True print('Failed pinning "{}": no such package installed'.format(name)) @@ -1660,12 +1769,14 @@ def cmd_unpin(manager, args, config, configfile): ipkg = manager.unpin(name) if ipkg: - print('Unpinned "{}" from version: {} ({})'.format( - name, ipkg.status.current_version, ipkg.status.current_hash)) + print( + 'Unpinned "{}" from version: {} ({})'.format( + name, ipkg.status.current_version, ipkg.status.current_hash + ) + ) else: had_failure = True - print( - 'Failed unpinning "{}": no such package installed'.format(name)) + print('Failed unpinning "{}": no such package installed'.format(name)) if had_failure: sys.exit(1) @@ -1683,26 +1794,39 @@ def _get_filtered_packages(manager, category): if pkg_qn not in pkg_dict: pkg_dict[pkg_qn] = pkg - if category == 'all': + if category == "all": filtered_pkgs = pkg_dict - elif category == 'installed': - filtered_pkgs = {key: value for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage)} - elif category == 'not_installed': - filtered_pkgs = {key: value for key, value in pkg_dict.items() - if not isinstance(value, zeekpkg.InstalledPackage)} - elif category == 'loaded': - filtered_pkgs = {key: value for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) and - value.status.is_loaded} - elif category == 'unloaded': - filtered_pkgs = {key: value for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) and - not value.status.is_loaded} - elif category == 'outdated': - filtered_pkgs = {key: value for key, value in pkg_dict.items() - if isinstance(value, zeekpkg.InstalledPackage) and - value.status.is_outdated} + elif category == "installed": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, zeekpkg.InstalledPackage) + } + elif category == "not_installed": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if not isinstance(value, zeekpkg.InstalledPackage) + } + elif category == "loaded": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, zeekpkg.InstalledPackage) and value.status.is_loaded + } + elif category == "unloaded": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, zeekpkg.InstalledPackage) + and not value.status.is_loaded + } + elif category == "outdated": + filtered_pkgs = { + key: value + for key, value in pkg_dict.items() + if isinstance(value, zeekpkg.InstalledPackage) and value.status.is_outdated + } else: raise NotImplementedError @@ -1715,8 +1839,7 @@ def cmd_list(manager, args, config, configfile): for pkg_name, val in sorted(filtered_pkgs.items()): if isinstance(val, zeekpkg.InstalledPackage): pkg = val.package - out = '{} (installed: {})'.format( - pkg_name, val.status.current_version) + out = "{} (installed: {})".format(pkg_name, val.status.current_version) else: pkg = val out = pkg_name @@ -1725,7 +1848,7 @@ def cmd_list(manager, args, config, configfile): desc = pkg.short_description() if desc: - out += ' - ' + desc + out += " - " + desc print(out) @@ -1735,13 +1858,13 @@ def cmd_search(manager, args, config, configfile): matches = set() for search_text in args.search_text: - if search_text[0] == '/' and search_text[-1] == '/': + if search_text[0] == "/" and search_text[-1] == "/": import re try: regex = re.compile(search_text[1:-1]) except re.error as error: - print('invalid regex: {}'.format(error)) + print("invalid regex: {}".format(error)) sys.exit(1) else: for pkg in src_pkgs: @@ -1768,12 +1891,12 @@ def cmd_search(manager, args, config, configfile): ipkg = manager.find_installed_package(match.qualified_name()) if ipkg: - out += ' (installed: {})'.format(ipkg.status.current_version) + out += " (installed: {})".format(ipkg.status.current_version) desc = match.short_description() if desc: - out += ' - ' + desc + out += " - " + desc print(out) @@ -1783,8 +1906,7 @@ def cmd_search(manager, args, config, configfile): def cmd_info(manager, args, config, configfile): if args.version and len(args.package) > 1: - print_error( - 'error: "info --version" may only be used for a single package') + print_error('error: "info --version" may only be used for a single package') sys.exit(1) # Dictionary for storing package info to output as JSON @@ -1794,77 +1916,75 @@ def cmd_info(manager, args, config, configfile): if len(args.package) == 1: try: filtered_pkgs = _get_filtered_packages(manager, args.package[0]) - package_names = [pkg_name - for pkg_name, _ in sorted(filtered_pkgs.items())] + package_names = [pkg_name for pkg_name, _ in sorted(filtered_pkgs.items())] except NotImplementedError: package_names = args.package else: package_names = args.package for name in package_names: - info = manager.info(name, - version=args.version, - prefer_installed=(args.nolocal != True)) + info = manager.info( + name, version=args.version, prefer_installed=(args.nolocal != True) + ) if info.package: name = info.package.qualified_name() if args.json: pkginfo[name] = dict() - pkginfo[name]['metadata'] = dict() + pkginfo[name]["metadata"] = dict() else: print('"{}" info:'.format(name)) if info.invalid_reason: if args.json: - pkginfo[name]['invalid'] = info.invalid_reason + pkginfo[name]["invalid"] = info.invalid_reason else: - print('\tinvalid package: {}\n'.format(info.invalid_reason)) + print("\tinvalid package: {}\n".format(info.invalid_reason)) had_invalid_package = True continue if args.json: - pkginfo[name]['url'] = info.package.git_url - pkginfo[name]['versions'] = info.versions + pkginfo[name]["url"] = info.package.git_url + pkginfo[name]["versions"] = info.versions else: - print('\turl: {}'.format(info.package.git_url)) - print('\tversions: {}'.format(info.versions)) + print("\turl: {}".format(info.package.git_url)) + print("\tversions: {}".format(info.versions)) if info.status: if args.json: - pkginfo[name]['install_status'] = dict() + pkginfo[name]["install_status"] = dict() for key, value in sorted(info.status.__dict__.items()): - pkginfo[name]['install_status'][key] = value + pkginfo[name]["install_status"][key] = value else: - print('\tinstall status:') + print("\tinstall status:") for key, value in sorted(info.status.__dict__.items()): - print('\t\t{} = {}'.format(key, value)) + print("\t\t{} = {}".format(key, value)) if args.json: if info.metadata_file: - pkginfo[name]['metadata_file'] = info.metadata_file - pkginfo[name]['metadata'][info.metadata_version] = dict() + pkginfo[name]["metadata_file"] = info.metadata_file + pkginfo[name]["metadata"][info.metadata_version] = dict() else: if info.metadata_file: - print('\tmetadata file: {}'.format(info.metadata_file)) - print('\tmetadata (from version "{}"):'.format( - info.metadata_version)) + print("\tmetadata file: {}".format(info.metadata_file)) + print('\tmetadata (from version "{}"):'.format(info.metadata_version)) if len(info.metadata) == 0: if args.json != True: - print('\t\t') + print("\t\t") else: if args.json: _fill_metadata_version( - pkginfo[name]['metadata'][info.metadata_version], - info.metadata) + pkginfo[name]["metadata"][info.metadata_version], info.metadata + ) else: for key, value in sorted(info.metadata.items()): - value = value.replace('\n', '\n\t\t\t') - print('\t\t{} = {}'.format(key, value)) + value = value.replace("\n", "\n\t\t\t") + print("\t\t{} = {}".format(key, value)) # If --json and --allvers given, check for multiple versions and # add the metadata for each version to the pkginfo. @@ -1872,15 +1992,16 @@ def cmd_info(manager, args, config, configfile): for vers in info.versions: # Skip the version that was already processed if vers != info.metadata_version: - info2 = manager.info(name, - vers, - prefer_installed=(args.nolocal != True)) - pkginfo[name]['metadata'][info2.metadata_version] = dict() + info2 = manager.info( + name, vers, prefer_installed=(args.nolocal != True) + ) + pkginfo[name]["metadata"][info2.metadata_version] = dict() if info2.metadata_file: - pkginfo[name]['metadata_file'] = info2.metadata_file + pkginfo[name]["metadata_file"] = info2.metadata_file _fill_metadata_version( - pkginfo[name]['metadata'][info2.metadata_version], - info2.metadata) + pkginfo[name]["metadata"][info2.metadata_version], + info2.metadata, + ) if not args.json: print() @@ -1893,7 +2014,7 @@ def cmd_info(manager, args, config, configfile): def _fill_metadata_version(pkginfo_name_metadata_version, info_metadata): - """ Fill a dict with metadata information. + """Fill a dict with metadata information. This helper function is called by cmd_info to fill metadata information for a specific package version. @@ -1909,72 +2030,74 @@ def _fill_metadata_version(pkginfo_name_metadata_version, info_metadata): New dict entries are added to pkginfo_name_metadata_version. """ for key, value in info_metadata.items(): - if key == 'depends' or key == 'suggests': + if key == "depends" or key == "suggests": pkginfo_name_metadata_version[key] = dict() - deps = value.split('\n') + deps = value.split("\n") for i in range(1, len(deps)): - deplist = deps[i].split(' ') - pkginfo_name_metadata_version[key][deplist[0]] =\ - deplist[1] + deplist = deps[i].split(" ") + pkginfo_name_metadata_version[key][deplist[0]] = deplist[1] else: pkginfo_name_metadata_version[key] = value def cmd_config(manager, args, config, configfile): - if args.config_param == 'all': + if args.config_param == "all": out = io.StringIO() config.write(out) print(out.getvalue()) out.close() - elif args.config_param == 'sources': - for key, value in config_items(config, 'sources'): - print('{} = {}'.format(key, value)) - elif args.config_param == 'user_vars': - if config.has_section('user_vars'): - for key, value in config_items(config, 'user_vars'): - print('{} = {}'.format(key, value)) + elif args.config_param == "sources": + for key, value in config_items(config, "sources"): + print("{} = {}".format(key, value)) + elif args.config_param == "user_vars": + if config.has_section("user_vars"): + for key, value in config_items(config, "user_vars"): + print("{} = {}".format(key, value)) else: - print(config.get('paths', args.config_param)) + print(config.get("paths", args.config_param)) def cmd_autoconfig(manager, args, config, configfile): if args.user: - configfile = os.path.join(home_config_dir(), 'config') - with io.open(configfile, 'w', encoding=std_encoding(sys.stdout)) as f: + configfile = os.path.join(home_config_dir(), "config") + with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print('Successfully wrote config file to {}'.format(configfile)) + print("Successfully wrote config file to {}".format(configfile)) return - zeek_config = find_program('zeek-config') + zeek_config = find_program("zeek-config") if zeek_config: have_zeek_config = True else: have_zeek_config = False - zeek_config = find_program('bro-config') + zeek_config = find_program("bro-config") if not zeek_config: print_error('error: no "zeek-config" or "bro-config" in PATH') sys.exit(1) - dist_option = '--zeek_dist' if have_zeek_config else '--bro_dist' + dist_option = "--zeek_dist" if have_zeek_config else "--bro_dist" - cmd = subprocess.Popen([zeek_config, - '--site_dir', '--plugin_dir', '--prefix', dist_option], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - bufsize=1, universal_newlines=True) + cmd = subprocess.Popen( + [zeek_config, "--site_dir", "--plugin_dir", "--prefix", dist_option], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) script_dir = read_zeek_config_line(cmd.stdout) plugin_dir = read_zeek_config_line(cmd.stdout) - bin_dir = os.path.join(read_zeek_config_line(cmd.stdout), 'bin') + bin_dir = os.path.join(read_zeek_config_line(cmd.stdout), "bin") zeek_dist = read_zeek_config_line(cmd.stdout) if configfile: config_dir = os.path.dirname(configfile) else: config_dir = default_config_dir() - configfile = os.path.join(config_dir, 'config') + configfile = os.path.join(config_dir, "config") make_dir(config_dir) @@ -1990,52 +2113,51 @@ def cmd_autoconfig(manager, args, config, configfile): if old_value == new_value: return - prompt = u'Set "{}" config option to: {} ?'.format(option, new_value) + prompt = 'Set "{}" config option to: {} ?'.format(option, new_value) if old_value: - prompt += u'\n(previous value: {})'.format(old_value) + prompt += "\n(previous value: {})".format(old_value) if args.force or confirmation_prompt(prompt): config.set(section, option, new_value) - change_config_value(config, 'paths', 'script_dir', - script_dir, config_file_exists) - change_config_value(config, 'paths', 'plugin_dir', - plugin_dir, config_file_exists) - change_config_value(config, 'paths', 'bin_dir', - bin_dir, config_file_exists) - change_config_value(config, 'paths', 'zeek_dist', - zeek_dist, config_file_exists) + change_config_value(config, "paths", "script_dir", script_dir, config_file_exists) + change_config_value(config, "paths", "plugin_dir", plugin_dir, config_file_exists) + change_config_value(config, "paths", "bin_dir", bin_dir, config_file_exists) + change_config_value(config, "paths", "zeek_dist", zeek_dist, config_file_exists) - with io.open(configfile, 'w', encoding=std_encoding(sys.stdout)) as f: + with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print('Successfully wrote config file to {}'.format(configfile)) + print("Successfully wrote config file to {}".format(configfile)) def cmd_env(manager, args, config, configfile): - - zeek_config = find_program('zeek-config') - path_option = '--zeekpath' + zeek_config = find_program("zeek-config") + path_option = "--zeekpath" if not zeek_config: - path_option = '--bropath' - zeek_config = find_program('bro-config') + path_option = "--bropath" + zeek_config = find_program("bro-config") - zeekpath = os.environ.get('ZEEKPATH') + zeekpath = os.environ.get("ZEEKPATH") if not zeekpath: - zeekpath = os.environ.get('BROPATH') + zeekpath = os.environ.get("BROPATH") - pluginpath = os.environ.get('ZEEK_PLUGIN_PATH') + pluginpath = os.environ.get("ZEEK_PLUGIN_PATH") if not pluginpath: - pluginpath = os.environ.get('BRO_PLUGIN_PATH') + pluginpath = os.environ.get("BRO_PLUGIN_PATH") if zeek_config: - cmd = subprocess.Popen([zeek_config, path_option, '--plugin_dir'], - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - bufsize=1, universal_newlines=True) + cmd = subprocess.Popen( + [zeek_config, path_option, "--plugin_dir"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) line1 = read_zeek_config_line(cmd.stdout) line2 = read_zeek_config_line(cmd.stdout) @@ -2045,8 +2167,8 @@ def cmd_env(manager, args, config, configfile): if not pluginpath: pluginpath = line2 - zeekpaths = [p for p in zeekpath.split(':')] if zeekpath else [] - pluginpaths = [p for p in pluginpath.split(':')] if pluginpath else [] + zeekpaths = [p for p in zeekpath.split(":")] if zeekpath else [] + pluginpaths = [p for p in pluginpath.split(":")] if pluginpath else [] zeekpaths.append(manager.zeekpath()) pluginpaths.append(manager.zeek_plugin_path()) @@ -2057,31 +2179,32 @@ def cmd_env(manager, args, config, configfile): zeekpaths = remove_redundant_paths(zeekpaths) pluginpaths = remove_redundant_paths(pluginpaths) - if os.environ.get('SHELL', '').endswith('csh'): - print(u'setenv BROPATH {}'.format(':'.join(zeekpaths))) - print(u'setenv BRO_PLUGIN_PATH {}'.format(':'.join(pluginpaths))) - print(u'setenv ZEEKPATH {}'.format(':'.join(zeekpaths))) - print(u'setenv ZEEK_PLUGIN_PATH {}'.format(':'.join(pluginpaths))) - print(u'setenv PATH {}:$PATH'.format(manager.bin_dir)) + if os.environ.get("SHELL", "").endswith("csh"): + print("setenv BROPATH {}".format(":".join(zeekpaths))) + print("setenv BRO_PLUGIN_PATH {}".format(":".join(pluginpaths))) + print("setenv ZEEKPATH {}".format(":".join(zeekpaths))) + print("setenv ZEEK_PLUGIN_PATH {}".format(":".join(pluginpaths))) + print("setenv PATH {}:$PATH".format(manager.bin_dir)) else: - print(u'export BROPATH={}'.format(':'.join(zeekpaths))) - print(u'export BRO_PLUGIN_PATH={}'.format(':'.join(pluginpaths))) - print(u'export ZEEKPATH={}'.format(':'.join(zeekpaths))) - print(u'export ZEEK_PLUGIN_PATH={}'.format(':'.join(pluginpaths))) - print(u'export PATH={}:$PATH'.format(manager.bin_dir)) + print("export BROPATH={}".format(":".join(zeekpaths))) + print("export BRO_PLUGIN_PATH={}".format(":".join(pluginpaths))) + print("export ZEEKPATH={}".format(":".join(zeekpaths))) + print("export ZEEK_PLUGIN_PATH={}".format(":".join(pluginpaths))) + print("export PATH={}:$PATH".format(manager.bin_dir)) def cmd_create(manager, args, config, configfile): - - tmplname = (args.template - or config.get('templates', 'default', fallback=None) - or ZKG_DEFAULT_TEMPLATE) + tmplname = ( + args.template + or config.get("templates", "default", fallback=None) + or ZKG_DEFAULT_TEMPLATE + ) try: tmpl = Template.load(config, tmplname, args.version) except LoadError as error: - msg = 'problem while loading template {}: {}'.format(tmplname, error) + msg = "problem while loading template {}: {}".format(tmplname, error) zeekpkg.LOG.exception(msg) - print_error('error: ' + msg) + print_error("error: " + msg) sys.exit(1) try: @@ -2096,7 +2219,7 @@ def cmd_create(manager, args, config, configfile): # filter any duplicates. fnames = set() for feat in args.features: - fnames |= set([f.strip() for f in feat.split(',') if f]) + fnames |= set([f.strip() for f in feat.split(",") if f]) features = tmpl.features() for feature in features: if feature.name() in fnames: @@ -2105,11 +2228,14 @@ def cmd_create(manager, args, config, configfile): fnames.remove(feature.name()) if len(fnames) > 0: # Alert if the user requested an unknown feature. - knowns = ', '.join([f'"{f.name()}"' for f in features]) - unknowns = ', '.join([f'"{name}"' for name in fnames]) - print_error('error: the following features are unknown: {}.' - ' Template "{}" offers {}.'.format( - unknowns, tmpl.name(), knowns or 'no features')) + knowns = ", ".join([f'"{f.name()}"' for f in features]) + unknowns = ", ".join([f'"{name}"' for name in fnames]) + print_error( + "error: the following features are unknown: {}." + ' Template "{}" offers {}.'.format( + unknowns, tmpl.name(), knowns or "no features" + ) + ) sys.exit(1) # Remove user vars we don't actually require from consideration @@ -2120,8 +2246,13 @@ def cmd_create(manager, args, config, configfile): try: uvar.resolve(tmpl.name(), config, args.user_var, args.force) except ValueError: - print_error(str.format('error: could not determine value of user variable "{}",' - ' provide via environment or --user-var', uvar.name())) + print_error( + str.format( + 'error: could not determine value of user variable "{}",' + " provide via environment or --user-var", + uvar.name(), + ) + ) sys.exit(1) # Apply them to the template. After this, any parameter can be @@ -2132,47 +2263,55 @@ def cmd_create(manager, args, config, configfile): try: package.do_validate(tmpl) except zeekpkg.template.InputError as error: - print_error('error: template input invalid, ' + str(error)) + print_error("error: template input invalid, " + str(error)) sys.exit(1) # And finally, instantiate the package. try: if os.path.isdir(args.packagedir): if not args.force: - print('Package directory {} already exists.' - .format(args.packagedir)) - if not confirmation_prompt('Delete?'): + print( + "Package directory {} already exists.".format(args.packagedir) + ) + if not confirmation_prompt("Delete?"): sys.exit(1) try: delete_path(args.packagedir) - zeekpkg.LOG.info('Removed existing package directory %s', args.packagedir) + zeekpkg.LOG.info( + "Removed existing package directory %s", args.packagedir + ) except OSError as err: - print_error('error: could not remove package directory {}: {}' - .format(args.packagedir, err)) + print_error( + "error: could not remove package directory {}: {}".format( + args.packagedir, err + ) + ) sys.exit(1) package.do_instantiate(tmpl, args.packagedir, args.force) except zeekpkg.template.OutputError as error: - print_error('error: template instantiation failed, ' + str(error)) + print_error("error: template instantiation failed, " + str(error)) sys.exit(1) except Exception as error: - msg = 'problem during template instantiation: {}'.format(error) + msg = "problem during template instantiation: {}".format(error) zeekpkg.LOG.exception(msg) - print_error('error: ' + msg) + print_error("error: " + msg) sys.exit(1) def cmd_template_info(manager, args, config, configfile): - tmplname = (args.template - or config.get('templates', 'default', fallback=None) - or ZKG_DEFAULT_TEMPLATE) + tmplname = ( + args.template + or config.get("templates", "default", fallback=None) + or ZKG_DEFAULT_TEMPLATE + ) try: tmpl = Template.load(config, tmplname, args.version) except LoadError as error: - msg = 'problem while loading template {}: {}'.format(tmplname, error) + msg = "problem while loading template {}: {}".format(tmplname, error) zeekpkg.LOG.exception(msg) - print_error('error: ' + msg) + print_error("error: " + msg) sys.exit(1) tmplinfo = tmpl.info() @@ -2180,24 +2319,27 @@ def cmd_template_info(manager, args, config, configfile): if args.json: print(json.dumps(tmplinfo, indent=args.jsonpretty, sort_keys=True)) else: - print('API version: ' + tmplinfo['api_version']) - print('features: ' + ', '.join(tmplinfo['features'])) - print('origin: ' + tmplinfo['origin']) - print('provides package: ' + str(tmplinfo['provides_package']).lower()) - print('user vars:') - for uvar_name, uvar_info in tmplinfo['user_vars'].items(): - print('\t{}: {}, {}, used by {}'.format( - uvar_name, uvar_info['description'], - uvar_info['default'] or 'no default', - ', '.join(uvar_info['used_by']) or 'not used')) - print('versions: ' + ', '.join(tmplinfo['versions'])) + print("API version: " + tmplinfo["api_version"]) + print("features: " + ", ".join(tmplinfo["features"])) + print("origin: " + tmplinfo["origin"]) + print("provides package: " + str(tmplinfo["provides_package"]).lower()) + print("user vars:") + for uvar_name, uvar_info in tmplinfo["user_vars"].items(): + print( + "\t{}: {}, {}, used by {}".format( + uvar_name, + uvar_info["description"], + uvar_info["default"] or "no default", + ", ".join(uvar_info["used_by"]) or "not used", + ) + ) + print("versions: " + ", ".join(tmplinfo["versions"])) class BundleHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): # Workaround for underlying argparse bug: https://bugs.python.org/issue9338 def _format_args(self, action, default_metavar): - rval = super(BundleHelpFormatter, self)._format_args( - action, default_metavar) + rval = super(BundleHelpFormatter, self)._format_args(action, default_metavar) if action.nargs == argparse.ZERO_OR_MORE: rval += " --" @@ -2210,524 +2352,656 @@ class BundleHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): def top_level_parser(): top_parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, - description='A command-line package manager for Zeek.', - epilog='Environment Variables:\n\n' - ' ``ZKG_CONFIG_FILE``:\t' - 'Same as ``--configfile`` option, but has less precedence.\n' - ' ``ZKG_DEFAULT_SOURCE``:\t' - 'The default package source to use (normally {}).\n' - ' ``ZKG_DEFAULT_TEMPLATE``:\t' - 'The default package template to use (normally {}).\n' - .format(ZKG_DEFAULT_SOURCE, ZKG_DEFAULT_TEMPLATE) - ) - top_parser.add_argument('--version', action='version', - version='%(prog)s ' + zeekpkg.__version__) + description="A command-line package manager for Zeek.", + epilog="Environment Variables:\n\n" + " ``ZKG_CONFIG_FILE``:\t" + "Same as ``--configfile`` option, but has less precedence.\n" + " ``ZKG_DEFAULT_SOURCE``:\t" + "The default package source to use (normally {}).\n" + " ``ZKG_DEFAULT_TEMPLATE``:\t" + "The default package template to use (normally {}).\n".format( + ZKG_DEFAULT_SOURCE, ZKG_DEFAULT_TEMPLATE + ), + ) + top_parser.add_argument( + "--version", action="version", version="%(prog)s " + zeekpkg.__version__ + ) group = top_parser.add_mutually_exclusive_group() - group.add_argument('--configfile', metavar='FILE', - help='Path to Zeek Package Manager config file. Precludes --user.') - group.add_argument('--user', action='store_true', - help='Store all state in user\'s home directory. Precludes --configfile.') - - top_parser.add_argument('--verbose', '-v', action='count', default=0, - help='Increase program output for debugging.' - ' Use multiple times for more output (e.g. -vvv).') - top_parser.add_argument('--extra-source', action='append', - metavar='NAME=URL', help='Add an extra source.') + group.add_argument( + "--configfile", + metavar="FILE", + help="Path to Zeek Package Manager config file. Precludes --user.", + ) + group.add_argument( + "--user", + action="store_true", + help="Store all state in user's home directory. Precludes --configfile.", + ) + + top_parser.add_argument( + "--verbose", + "-v", + action="count", + default=0, + help="Increase program output for debugging." + " Use multiple times for more output (e.g. -vvv).", + ) + top_parser.add_argument( + "--extra-source", + action="append", + metavar="NAME=URL", + help="Add an extra source.", + ) return top_parser def argparser(): - pkg_name_help = 'The name(s) of package(s) to operate on. The package' \ - ' may be named in several ways. If the package is part' \ - ' of a package source, it may be referred to by the' \ - ' base name of the package (last component of git URL)' \ - ' or its path within the package source.' \ - ' If two packages in different package sources' \ - ' have conflicting paths, then the package source' \ - ' name may be prepended to the package path to resolve' \ - ' the ambiguity. A full git URL may also be used to refer' \ - ' to a package that does not belong to a source. E.g. for' \ - ' a package source called "zeek" that has a package named' \ - ' "foo" located in either "alice/zkg.index" or' \ - ' "alice/bro-pkg.index", the following' \ - ' names work: "foo", "alice/foo", "zeek/alice/foo".' + pkg_name_help = ( + "The name(s) of package(s) to operate on. The package" + " may be named in several ways. If the package is part" + " of a package source, it may be referred to by the" + " base name of the package (last component of git URL)" + " or its path within the package source." + " If two packages in different package sources" + " have conflicting paths, then the package source" + " name may be prepended to the package path to resolve" + " the ambiguity. A full git URL may also be used to refer" + " to a package that does not belong to a source. E.g. for" + ' a package source called "zeek" that has a package named' + ' "foo" located in either "alice/zkg.index" or' + ' "alice/bro-pkg.index", the following' + ' names work: "foo", "alice/foo", "zeek/alice/foo".' + ) def add_uservar_args(parser, force_help=None): parser.add_argument( - '--force', action='store_true', - help=force_help or "Don't prompt for confirmation or user variables.") + "--force", + action="store_true", + help=force_help or "Don't prompt for confirmation or user variables.", + ) parser.add_argument( - '--user-var', action='append', metavar='NAME=VAL', type=UserVar.parse_arg, - help='A user variable assignment. This avoids prompting' - ' for input and lets you provide a value when using --force.' - ' Use repeatedly as needed for multiple values.') + "--user-var", + action="append", + metavar="NAME=VAL", + type=UserVar.parse_arg, + help="A user variable assignment. This avoids prompting" + " for input and lets you provide a value when using --force." + " Use repeatedly as needed for multiple values.", + ) def add_json_args(parser, help_text): - parser.add_argument('--json', action='store_true', help=help_text) + parser.add_argument("--json", action="store_true", help=help_text) parser.add_argument( - '--jsonpretty', type=int, default=None, metavar='SPACES', - help='Optional number of spaces to indent for pretty-printed' - ' JSON output.') + "--jsonpretty", + type=int, + default=None, + metavar="SPACES", + help="Optional number of spaces to indent for pretty-printed" + " JSON output.", + ) top_parser = top_level_parser() command_parser = top_parser.add_subparsers( - title='commands', dest='command', - help='See `%(prog)s -h` for per-command usage info.') + title="commands", + dest="command", + help="See `%(prog)s -h` for per-command usage info.", + ) command_parser.required = True # test sub_parser = command_parser.add_parser( - 'test', - help='Runs unit tests for Zeek packages.', - description='Runs the unit tests for the specified Zeek packages.' - ' In most cases, the "zeek" and "zeek-config" programs will' - ' need to be in PATH before running this command.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "test", + help="Runs unit tests for Zeek packages.", + description="Runs the unit tests for the specified Zeek packages." + ' In most cases, the "zeek" and "zeek-config" programs will' + " need to be in PATH before running this command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_test) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) sub_parser.add_argument( - 'package', nargs='+', help=pkg_name_help) - sub_parser.add_argument( - '--version', default=None, - help='The version of the package to test. Only one package may be' - ' specified at a time when using this flag. A version tag, branch' - ' name, or commit hash may be specified here.' - ' If the package name refers to a local git repo with a working tree,' - ' then its currently active branch is used.' - ' The default for other cases is to use' - ' the latest version tag, or if a package has none,' - ' the default branch, like "main" or "master".') + "--version", + default=None, + help="The version of the package to test. Only one package may be" + " specified at a time when using this flag. A version tag, branch" + " name, or commit hash may be specified here." + " If the package name refers to a local git repo with a working tree," + " then its currently active branch is used." + " The default for other cases is to use" + " the latest version tag, or if a package has none," + ' the default branch, like "main" or "master".', + ) # install sub_parser = command_parser.add_parser( - 'install', - help='Installs Zeek packages.', - description='Installs packages from a configured package source or' - ' directly from a git URL. After installing, the package' - ' is marked as being "loaded" (see the ``load`` command).', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "install", + help="Installs Zeek packages.", + description="Installs packages from a configured package source or" + " directly from a git URL. After installing, the package" + ' is marked as being "loaded" (see the ``load`` command).', + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_install) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) sub_parser.add_argument( - 'package', nargs='+', help=pkg_name_help) - sub_parser.add_argument( - '--skiptests', action='store_true', - help='Skip running unit tests for packages before installation.') + "--skiptests", + action="store_true", + help="Skip running unit tests for packages before installation.", + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks putting your installed package collection into a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) sub_parser.add_argument( - '--nosuggestions', action='store_true', - help='Skip automatically installing suggested packages.') + "--nosuggestions", + action="store_true", + help="Skip automatically installing suggested packages.", + ) sub_parser.add_argument( - '--version', default=None, - help='The version of the package to install. Only one package may be' - ' specified at a time when using this flag. A version tag, branch' - ' name, or commit hash may be specified here.' - ' If the package name refers to a local git repo with a working tree,' - ' then its currently active branch is used.' - ' The default for other cases is to use' - ' the latest version tag, or if a package has none,' - ' the default branch, like "main" or "master".') + "--version", + default=None, + help="The version of the package to install. Only one package may be" + " specified at a time when using this flag. A version tag, branch" + " name, or commit hash may be specified here." + " If the package name refers to a local git repo with a working tree," + " then its currently active branch is used." + " The default for other cases is to use" + " the latest version tag, or if a package has none," + ' the default branch, like "main" or "master".', + ) add_uservar_args(sub_parser) # bundle sub_parser = command_parser.add_parser( - 'bundle', - help='Creates a bundle file containing a collection of Zeek packages.', - description='This command creates a bundle file containing a collection' - ' of Zeek packages. If ``--manifest`` is used, the user' - ' supplies the list of packages to put in the bundle, else' - ' all currently installed packages are put in the bundle.' - ' A bundle file can be unpacked on any target system,' - ' resulting in a repeatable/specific set of packages' - ' being installed on that target system (see the' - ' ``unbundle`` command). This command may be useful for' - ' those that want to manage packages on a system that' - ' otherwise has limited network connectivity. E.g. one can' - ' use a system with an internet connection to create a' - ' bundle, transport that bundle to the target machine' - ' using whatever means are appropriate, and finally' - ' unbundle/install it on the target machine.', - formatter_class=BundleHelpFormatter) + "bundle", + help="Creates a bundle file containing a collection of Zeek packages.", + description="This command creates a bundle file containing a collection" + " of Zeek packages. If ``--manifest`` is used, the user" + " supplies the list of packages to put in the bundle, else" + " all currently installed packages are put in the bundle." + " A bundle file can be unpacked on any target system," + " resulting in a repeatable/specific set of packages" + " being installed on that target system (see the" + " ``unbundle`` command). This command may be useful for" + " those that want to manage packages on a system that" + " otherwise has limited network connectivity. E.g. one can" + " use a system with an internet connection to create a" + " bundle, transport that bundle to the target machine" + " using whatever means are appropriate, and finally" + " unbundle/install it on the target machine.", + formatter_class=BundleHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_bundle) sub_parser.add_argument( - 'bundle_filename', - metavar='filename.bundle', - help='The path of the bundle file to create. It will be overwritten' - ' if it already exists. Note that if --manifest is used before' - ' this filename is specified, you should use a double-dash, --,' - ' to first terminate that argument list.') + "bundle_filename", + metavar="filename.bundle", + help="The path of the bundle file to create. It will be overwritten" + " if it already exists. Note that if --manifest is used before" + " this filename is specified, you should use a double-dash, --," + " to first terminate that argument list.", + ) sub_parser.add_argument( - '--force', action='store_true', - help='Skip the confirmation prompt.') + "--force", action="store_true", help="Skip the confirmation prompt." + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks creating a bundle of packages that is in a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks creating a bundle of packages that is in a" + " broken or unusable state.", + ) sub_parser.add_argument( - '--nosuggestions', action='store_true', - help='Skip automatically bundling suggested packages.') + "--nosuggestions", + action="store_true", + help="Skip automatically bundling suggested packages.", + ) sub_parser.add_argument( - '--manifest', nargs='+', - help='This may either be a file name or a list of packages to include' - ' in the bundle. If a file name is supplied, it should be in INI' - ' format with a single ``[bundle]`` section. The keys in that section' - ' correspond to package names and their values correspond to git' - ' version tags, branch names, or commit hashes. The values may be' - ' left blank to indicate that the latest available version should be' - ' used.') + "--manifest", + nargs="+", + help="This may either be a file name or a list of packages to include" + " in the bundle. If a file name is supplied, it should be in INI" + " format with a single ``[bundle]`` section. The keys in that section" + " correspond to package names and their values correspond to git" + " version tags, branch names, or commit hashes. The values may be" + " left blank to indicate that the latest available version should be" + " used.", + ) # unbundle sub_parser = command_parser.add_parser( - 'unbundle', - help='Unpacks Zeek packages from a bundle file and installs them.', - description='This command unpacks a bundle file formerly created by the' - ' ``bundle`` command and installs all the packages' - ' contained within.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "unbundle", + help="Unpacks Zeek packages from a bundle file and installs them.", + description="This command unpacks a bundle file formerly created by the" + " ``bundle`` command and installs all the packages" + " contained within.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_unbundle) sub_parser.add_argument( - 'bundle_filename', - metavar='filename.bundle', - help='The path of the bundle file to install.') + "bundle_filename", + metavar="filename.bundle", + help="The path of the bundle file to install.", + ) sub_parser.add_argument( - '--replace', action='store_true', - help='Using this flag first removes all installed packages before then' - ' installing the packages from the bundle.') + "--replace", + action="store_true", + help="Using this flag first removes all installed packages before then" + " installing the packages from the bundle.", + ) add_uservar_args(sub_parser) # remove sub_parser = command_parser.add_parser( - 'remove', - aliases=['uninstall'], - help='Uninstall a package.', - description='Unloads (see the ``unload`` command) and uninstalls a' - ' previously installed package.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "remove", + aliases=["uninstall"], + help="Uninstall a package.", + description="Unloads (see the ``unload`` command) and uninstalls a" + " previously installed package.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_remove) - sub_parser.add_argument('package', nargs='+', help=pkg_name_help) + sub_parser.add_argument("package", nargs="+", help=pkg_name_help) sub_parser.add_argument( - '--force', action='store_true', - help='Skip the confirmation prompt.') + "--force", action="store_true", help="Skip the confirmation prompt." + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks putting your installed package collection into a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) # purge sub_parser = command_parser.add_parser( - 'purge', - help='Uninstall all packages.', - description='Unloads (see the ``unload`` command) and uninstalls all' - ' previously installed packages.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "purge", + help="Uninstall all packages.", + description="Unloads (see the ``unload`` command) and uninstalls all" + " previously installed packages.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_purge) sub_parser.add_argument( - '--force', action='store_true', - help='Skip the confirmation prompt.') + "--force", action="store_true", help="Skip the confirmation prompt." + ) # refresh sub_parser = command_parser.add_parser( - 'refresh', - help='Retrieve updated package metadata.', - description='Retrieve latest package metadata from sources and checks' - ' whether any installed packages have available upgrades.' - ' Note that this does not actually upgrade any packages (see the' - ' ``upgrade`` command for that).', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "refresh", + help="Retrieve updated package metadata.", + description="Retrieve latest package metadata from sources and checks" + " whether any installed packages have available upgrades." + " Note that this does not actually upgrade any packages (see the" + " ``upgrade`` command for that).", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_refresh) sub_parser.add_argument( - '--aggregate', action='store_true', - help='Crawls the urls listed in package source zkg.index' - ' (or legacy bro-pkg.index) files and' - ' aggregates the metadata found in their zkg.meta (or legacy' - ' bro-pkg.meta) files. The aggregated metadata is stored in the local' - ' clone of the package source that zkg uses internally for locating' - ' package metadata.' - ' For each package, the metadata is taken from the highest available' - ' git version tag or the default branch, like "main" or "master", if no version tags exist') + "--aggregate", + action="store_true", + help="Crawls the urls listed in package source zkg.index" + " (or legacy bro-pkg.index) files and" + " aggregates the metadata found in their zkg.meta (or legacy" + " bro-pkg.meta) files. The aggregated metadata is stored in the local" + " clone of the package source that zkg uses internally for locating" + " package metadata." + " For each package, the metadata is taken from the highest available" + ' git version tag or the default branch, like "main" or "master", if no version tags exist', + ) sub_parser.add_argument( - '--fail-on-aggregate-problems', action='store_true', - help='When using --aggregate, exit with error when any packages trigger' - ' metadata problems. Normally such problems only cause a warning.') + "--fail-on-aggregate-problems", + action="store_true", + help="When using --aggregate, exit with error when any packages trigger" + " metadata problems. Normally such problems only cause a warning.", + ) sub_parser.add_argument( - '--push', action='store_true', - help='Push all local changes to package sources to upstream repos') - sub_parser.add_argument('--sources', nargs='+', - help='A list of package source names to operate on. If this argument' - ' is not used, then the command will operate on all configured' - ' sources.') + "--push", + action="store_true", + help="Push all local changes to package sources to upstream repos", + ) + sub_parser.add_argument( + "--sources", + nargs="+", + help="A list of package source names to operate on. If this argument" + " is not used, then the command will operate on all configured" + " sources.", + ) # upgrade sub_parser = command_parser.add_parser( - 'upgrade', - help='Upgrade installed packages to latest versions.', - description='Uprades the specified package(s) to latest available' - ' version. If no specific packages are specified, then all installed' - ' packages that are outdated and not pinned are upgraded. For packages' - ' that are installed with ``--version`` using a git branch name, the' - ' package is updated to the latest commit on that branch, else the' - ' package is updated to the highest available git version tag.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "upgrade", + help="Upgrade installed packages to latest versions.", + description="Uprades the specified package(s) to latest available" + " version. If no specific packages are specified, then all installed" + " packages that are outdated and not pinned are upgraded. For packages" + " that are installed with ``--version`` using a git branch name, the" + " package is updated to the latest commit on that branch, else the" + " package is updated to the highest available git version tag.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_upgrade) + sub_parser.add_argument("package", nargs="*", default=[], help=pkg_name_help) sub_parser.add_argument( - 'package', nargs='*', default=[], help=pkg_name_help) - sub_parser.add_argument( - '--skiptests', action='store_true', - help='Skip running unit tests for packages before installation.') + "--skiptests", + action="store_true", + help="Skip running unit tests for packages before installation.", + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks putting your installed package collection into a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) sub_parser.add_argument( - '--nosuggestions', action='store_true', - help='Skip automatically installing suggested packages.') + "--nosuggestions", + action="store_true", + help="Skip automatically installing suggested packages.", + ) add_uservar_args(sub_parser) # load sub_parser = command_parser.add_parser( - 'load', - help='Register packages to be be auto-loaded by Zeek.', - description='The Zeek Package Manager keeps track of all packages that' + "load", + help="Register packages to be be auto-loaded by Zeek.", + description="The Zeek Package Manager keeps track of all packages that" ' are marked as "loaded" and maintains a single Zeek script that, when' - ' loaded by Zeek (e.g. via ``@load packages``), will load the scripts' + " loaded by Zeek (e.g. via ``@load packages``), will load the scripts" ' from all "loaded" packages at once.' ' This command adds a set of packages to the "loaded packages" list.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_load) sub_parser.add_argument( - 'package', nargs='+', default=[], - help='Name(s) of package(s) to load.') + "package", nargs="+", default=[], help="Name(s) of package(s) to load." + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks putting your installed package collection into a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) # unload sub_parser = command_parser.add_parser( - 'unload', - help='Unregister packages to be be auto-loaded by Zeek.', - description='The Zeek Package Manager keeps track of all packages that' + "unload", + help="Unregister packages to be be auto-loaded by Zeek.", + description="The Zeek Package Manager keeps track of all packages that" ' are marked as "loaded" and maintains a single Zeek script that, when' ' loaded by Zeek, will load the scripts from all "loaded" packages at' ' once. This command removes a set of packages from the "loaded' ' packages" list.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_unload) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) sub_parser.add_argument( - 'package', nargs='+', default=[], help=pkg_name_help) - sub_parser.add_argument( - '--force', action='store_true', - help='Skip the confirmation prompt.') + "--force", action="store_true", help="Skip the confirmation prompt." + ) sub_parser.add_argument( - '--nodeps', action='store_true', - help='Skip all dependency resolution/checks. Note that using this' - ' option risks putting your installed package collection into a' - ' broken or unusable state.') + "--nodeps", + action="store_true", + help="Skip all dependency resolution/checks. Note that using this" + " option risks putting your installed package collection into a" + " broken or unusable state.", + ) # pin sub_parser = command_parser.add_parser( - 'pin', - help='Prevent packages from being automatically upgraded.', - description='Pinned packages are ignored by the ``upgrade`` command.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "pin", + help="Prevent packages from being automatically upgraded.", + description="Pinned packages are ignored by the ``upgrade`` command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_pin) - sub_parser.add_argument( - 'package', nargs='+', default=[], help=pkg_name_help) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) # unpin sub_parser = command_parser.add_parser( - 'unpin', - help='Allows packages to be automatically upgraded.', - description='Packages that are not pinned are automatically upgraded' - ' by the ``upgrade`` command', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "unpin", + help="Allows packages to be automatically upgraded.", + description="Packages that are not pinned are automatically upgraded" + " by the ``upgrade`` command", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_unpin) - sub_parser.add_argument( - 'package', nargs='+', default=[], help=pkg_name_help) + sub_parser.add_argument("package", nargs="+", default=[], help=pkg_name_help) # list sub_parser = command_parser.add_parser( - 'list', - help='Lists packages.', - description='Outputs a list of packages that match a given category.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "list", + help="Lists packages.", + description="Outputs a list of packages that match a given category.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_list) - sub_parser.add_argument('category', nargs='?', default='installed', - choices=['all', 'installed', 'not_installed', - 'loaded', 'unloaded', 'outdated'], - help='Package category used to filter listing.') sub_parser.add_argument( - '--nodesc', action='store_true', - help='Do not display description text, just the package name(s).') + "category", + nargs="?", + default="installed", + choices=["all", "installed", "not_installed", "loaded", "unloaded", "outdated"], + help="Package category used to filter listing.", + ) + sub_parser.add_argument( + "--nodesc", + action="store_true", + help="Do not display description text, just the package name(s).", + ) # search sub_parser = command_parser.add_parser( - 'search', - help='Search packages for matching names.', - description='Perform a substring search on package names and metadata' - ' tags. Surround search text with slashes to indicate it is a regular' - ' expression (e.g. ``/text/``).', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "search", + help="Search packages for matching names.", + description="Perform a substring search on package names and metadata" + " tags. Surround search text with slashes to indicate it is a regular" + " expression (e.g. ``/text/``).", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_search) sub_parser.add_argument( - 'search_text', nargs='+', default=[], - help='The text(s) or pattern(s) to look for.') + "search_text", + nargs="+", + default=[], + help="The text(s) or pattern(s) to look for.", + ) # info sub_parser = command_parser.add_parser( - 'info', - help='Display package information.', - description='Shows detailed information/metadata for given packages.' - ' If the package is currently installed, additional information about' - ' the status of it is displayed. E.g. the installed version or whether' + "info", + help="Display package information.", + description="Shows detailed information/metadata for given packages." + " If the package is currently installed, additional information about" + " the status of it is displayed. E.g. the installed version or whether" ' it is currently marked as "pinned" or "loaded."', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_info) sub_parser.add_argument( - 'package', nargs='+', default=[], help=pkg_name_help + - ' If a single name is given and matches one of the same categories' + "package", + nargs="+", + default=[], + help=pkg_name_help + + " If a single name is given and matches one of the same categories" ' as the "list" command, then it is automatically expanded to be the' - ' names of all packages which match the given category.') + " names of all packages which match the given category.", + ) sub_parser.add_argument( - '--version', default=None, - help='The version of the package metadata to inspect. A version tag,' - ' branch name, or commit hash and only one package at a time may be' - ' given when using this flag. If unspecified, the behavior depends' - ' on whether the package is currently installed. If installed,' - ' the metadata will be pulled from the installed version. If not' - ' installed, the latest version tag is used, or if a package has no' - ' version tags, the default branch, like "main" or "master", is used.') + "--version", + default=None, + help="The version of the package metadata to inspect. A version tag," + " branch name, or commit hash and only one package at a time may be" + " given when using this flag. If unspecified, the behavior depends" + " on whether the package is currently installed. If installed," + " the metadata will be pulled from the installed version. If not" + " installed, the latest version tag is used, or if a package has no" + ' version tags, the default branch, like "main" or "master", is used.', + ) sub_parser.add_argument( - '--nolocal', action='store_true', - help='Do not read information from locally installed packages.' - ' Instead read info from remote GitHub.') - add_json_args(sub_parser, 'Output package information as JSON.') + "--nolocal", + action="store_true", + help="Do not read information from locally installed packages." + " Instead read info from remote GitHub.", + ) + add_json_args(sub_parser, "Output package information as JSON.") sub_parser.add_argument( - '--allvers', action='store_true', - help='When outputting package information as JSON, show metadata for' - ' all versions. This option can be slow since remote repositories' - ' may be cloned multiple times. Also, installed packages will show' - ' metadata only for the installed version unless the --nolocal ' - ' option is given.') + "--allvers", + action="store_true", + help="When outputting package information as JSON, show metadata for" + " all versions. This option can be slow since remote repositories" + " may be cloned multiple times. Also, installed packages will show" + " metadata only for the installed version unless the --nolocal " + " option is given.", + ) # config sub_parser = command_parser.add_parser( - 'config', - help='Show Zeek Package Manager configuration info.', - description='The default output of this command is a valid package' - ' manager config file that corresponds to the one currently being used,' - ' but also with any defaulted field values filled in. This command' - ' also allows for only the value of a specific field to be output if' - ' the name of that field is given as an argument to the command.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "config", + help="Show Zeek Package Manager configuration info.", + description="The default output of this command is a valid package" + " manager config file that corresponds to the one currently being used," + " but also with any defaulted field values filled in. This command" + " also allows for only the value of a specific field to be output if" + " the name of that field is given as an argument to the command.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_config) sub_parser.add_argument( - 'config_param', nargs='?', default='all', - choices=['all', 'sources', 'user_vars', 'state_dir', 'script_dir', - 'plugin_dir', 'bin_dir', 'zeek_dist', 'bro_dist'], - help='Name of a specific config file field to output.') + "config_param", + nargs="?", + default="all", + choices=[ + "all", + "sources", + "user_vars", + "state_dir", + "script_dir", + "plugin_dir", + "bin_dir", + "zeek_dist", + "bro_dist", + ], + help="Name of a specific config file field to output.", + ) # autoconfig sub_parser = command_parser.add_parser( - 'autoconfig', - help='Generate a Zeek Package Manager configuration file.', - description='The output of this command is a valid package manager' - ' config file that is generated by using the ``zeek-config`` script' - ' that is installed along with Zeek. It is the suggested configuration' - ' to use for most Zeek installations. For this command to work, the' - ' ``zeek-config`` (or ``bro-config``) script must be in ``PATH``,' - ' unless the --user option is given, in which case this creates' - ' a config that does not touch the Zeek installation.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "autoconfig", + help="Generate a Zeek Package Manager configuration file.", + description="The output of this command is a valid package manager" + " config file that is generated by using the ``zeek-config`` script" + " that is installed along with Zeek. It is the suggested configuration" + " to use for most Zeek installations. For this command to work, the" + " ``zeek-config`` (or ``bro-config``) script must be in ``PATH``," + " unless the --user option is given, in which case this creates" + " a config that does not touch the Zeek installation.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_autoconfig) sub_parser.add_argument( - '--force', action='store_true', - help='Skip any confirmation prompt.') + "--force", action="store_true", help="Skip any confirmation prompt." + ) # env sub_parser = command_parser.add_parser( - 'env', - help='Show the value of environment variables that need to be set for' - ' Zeek to be able to use installed packages.', - description='This command returns shell commands that, when executed,' - ' will correctly set ``ZEEKPATH`` and ``ZEEK_PLUGIN_PATH`` (also' - ' ``BROPATH`` and ``BRO_PLUGIN_PATH`` for legacy compatibility) to use' - ' scripts and plugins from packages installed by the package manager.' - ' For this command to function properly, either have the ``zeek-config``' - ' script (installed by zeek) in ``PATH``, or have the ``ZEEKPATH`` and' - ' ``ZEEK_PLUGIN_PATH`` (or ``BROPATH`` and ``BRO_PLUGIN_PATH``)' - ' environment variables already set so this command' - ' can append package-specific paths to them.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "env", + help="Show the value of environment variables that need to be set for" + " Zeek to be able to use installed packages.", + description="This command returns shell commands that, when executed," + " will correctly set ``ZEEKPATH`` and ``ZEEK_PLUGIN_PATH`` (also" + " ``BROPATH`` and ``BRO_PLUGIN_PATH`` for legacy compatibility) to use" + " scripts and plugins from packages installed by the package manager." + " For this command to function properly, either have the ``zeek-config``" + " script (installed by zeek) in ``PATH``, or have the ``ZEEKPATH`` and" + " ``ZEEK_PLUGIN_PATH`` (or ``BROPATH`` and ``BRO_PLUGIN_PATH``)" + " environment variables already set so this command" + " can append package-specific paths to them.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.set_defaults(run_cmd=cmd_env) # create sub_parser = command_parser.add_parser( - 'create', help='Create a new Zeek package.', - description='This command creates a new Zeek package in the directory' - ' provided via --packagedir. If this directory exists, zkg will not' - ' modify it unless you provide --force.', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + "create", + help="Create a new Zeek package.", + description="This command creates a new Zeek package in the directory" + " provided via --packagedir. If this directory exists, zkg will not" + " modify it unless you provide --force.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) sub_parser.add_argument( - '--packagedir', metavar='DIR', required=True, - help='Output directory into which to produce the new package. Required.') + "--packagedir", + metavar="DIR", + required=True, + help="Output directory into which to produce the new package. Required.", + ) sub_parser.add_argument( - '--version', - help='The template version to use. A version tag, branch name, or' - ' commit hash may be specified here. If --template refers to a local' - ' git repo with a working tree, then zkg uses it as-is and the version' - ' is ignored. The default for other cases is to use the latest' - ' version tag, or if a template has none, the default branch, like' - ' "main" or "master".') + "--version", + help="The template version to use. A version tag, branch name, or" + " commit hash may be specified here. If --template refers to a local" + " git repo with a working tree, then zkg uses it as-is and the version" + " is ignored. The default for other cases is to use the latest" + " version tag, or if a template has none, the default branch, like" + ' "main" or "master".', + ) sub_parser.add_argument( - '--features', nargs='+', metavar='FEATURE', - help='Additional features to include in your package. Use the ``template' - ' info`` command for information about available features.') + "--features", + nargs="+", + metavar="FEATURE", + help="Additional features to include in your package. Use the ``template" + " info`` command for information about available features.", + ) sub_parser.add_argument( - '--template', metavar='URL', - help='By default, zkg uses its own package template. This makes it' - ' select an alternative.') + "--template", + metavar="URL", + help="By default, zkg uses its own package template. This makes it" + " select an alternative.", + ) sub_parser.set_defaults(run_cmd=cmd_create) add_uservar_args(sub_parser) # Template management commands - sub_parser = command_parser.add_parser( - 'template', help='Manage package templates.') + sub_parser = command_parser.add_parser("template", help="Manage package templates.") template_command_parser = sub_parser.add_subparsers( - title='template commands', dest='command', - help='See %(prog)s -h for per-command usage info.') + title="template commands", + dest="command", + help="See %(prog)s -h for per-command usage info.", + ) template_command_parser.required = True # template info sub_parser = template_command_parser.add_parser( - 'info', help='Shows information about a package template.', - description='This command shows versions and supported features for' - ' a given package.') - add_json_args(sub_parser, 'Output template information as JSON.') + "info", + help="Shows information about a package template.", + description="This command shows versions and supported features for" + " a given package.", + ) + add_json_args(sub_parser, "Output template information as JSON.") sub_parser.add_argument( - '--version', - help='The template version to report on. A version tag, branch name,' - ' or commit hash may be specified here. If the selected template' - ' refers to a local git repo, the version is ignored. The default' - ' for other cases is to use the latest version tag, or if a template' - ' has none, the default branch, like "main" or "master".') + "--version", + help="The template version to report on. A version tag, branch name," + " or commit hash may be specified here. If the selected template" + " refers to a local git repo, the version is ignored. The default" + " for other cases is to use the latest version tag, or if a template" + ' has none, the default branch, like "main" or "master".', + ) sub_parser.add_argument( - 'template', metavar='URL', nargs='?', - help='URL of a package template repository, or local path to one.' - ' When not provided, the configured default template is used.') + "template", + metavar="URL", + nargs="?", + help="URL of a package template repository, or local path to one." + " When not provided, the configured default template is used.", + ) sub_parser.set_defaults(run_cmd=cmd_template_info) - if 'argcomplete' in sys.modules: + if "argcomplete" in sys.modules: argcomplete.autocomplete(top_parser) return top_parser @@ -2738,7 +3012,8 @@ def main(): if args.verbose > 0: formatter = logging.Formatter( - '%(asctime)s %(levelname)-8s %(message)s', '%Y-%m-%d %H:%M:%S') + "%(asctime)s %(levelname)-8s %(message)s", "%Y-%m-%d %H:%M:%S" + ) handler = logging.StreamHandler() handler.setFormatter(formatter) @@ -2761,6 +3036,7 @@ def main(): args.run_cmd(manager, args, config, configfile) -if __name__ == '__main__': + +if __name__ == "__main__": main() sys.exit(0) From d29d7e8ea4c5de41e938feb0ef6c4739dc1815a0 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Tue, 10 Jan 2023 12:05:45 +0100 Subject: [PATCH 4/7] Add `.git-blame-ignore-revs` file. --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .git-blame-ignore-revs diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..fffe9a69 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# Migrate code style to Black +99520f9201340d57d2cc9cf6f45992b994eff7c4 From e5401dc2f3c9bffdeaea2d3f062638309b1b986b Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 9 Jan 2023 18:08:41 +0000 Subject: [PATCH 5/7] Automatically update Python sources to use python-3.7 syntax. --- .pre-commit-config.yaml | 5 + doc/conf.py | 3 +- zeekpkg/_util.py | 2 +- zeekpkg/manager.py | 58 +++++----- zeekpkg/package.py | 16 +-- zeekpkg/source.py | 2 +- zeekpkg/template.py | 12 +-- zeekpkg/uservar.py | 4 +- zkg | 228 +++++++++++++++++++--------------------- 9 files changed, 163 insertions(+), 167 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34a44721..44b8398f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,5 +14,10 @@ repos: rev: 23.1a1 hooks: - id: black +- repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py37-plus"] exclude: doc/ext/sphinxarg diff --git a/doc/conf.py b/doc/conf.py index 25c799ae..8dfe0ec7 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # Zeek Package Manager documentation build configuration file, created by # sphinx-quickstart on Fri Jul 15 13:46:04 2016. @@ -59,7 +58,7 @@ # built documents. # # The short X.Y version. -with open("../VERSION", "r") as f: +with open("../VERSION") as f: version = f.readline().strip() # The full version, including alpha/beta/rc tags. diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index f8a5ba25..f659a9f2 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -266,7 +266,7 @@ def git_remote_urls(repo): cases. We use the config subsystem to query the URLs directly -- one of the fallback mechanisms in GitPython's Remote.urls() implementation. """ - remote_details = repo.git.config("--get-regexp", "remote\..+\.url") + remote_details = repo.git.config("--get-regexp", r"remote\..+\.url") remotes = {} for line in remote_details.split("\n"): diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index dab2453f..ffd8af40 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -69,7 +69,7 @@ ) -class Stage(object): +class Stage: def __init__(self, manager, state_dir=None): self.manager = manager @@ -158,7 +158,7 @@ def get_subprocess_env(self): return env, "" -class Manager(object): +class Manager: """A package manager object performs various operations on packages. It uses a state directory and a manifest file within it to keep @@ -388,7 +388,7 @@ def _write_autoloader(self): for ipkg in self.loaded_packages(): if self.has_scripts(ipkg): - content += "@load ./{}\n".format(ipkg.package.name) + content += f"@load ./{ipkg.package.name}\n" f.write(content) @@ -443,7 +443,7 @@ def _read_manifest(self): Raises: IOError: when the manifest file can't be read """ - with open(self.manifest, "r") as f: + with open(self.manifest) as f: data = json.load(f) version = data["manifest_version"] pkg_list = data["installed_packages"] @@ -631,7 +631,7 @@ def package_build_log(self, pkg_path): to the package: "foo", "alice/foo", or "zeek/alice/foo". """ name = name_from_path(pkg_path) - return os.path.join(self.log_dir, "{}-build.log".format(name)) + return os.path.join(self.log_dir, f"{name}-build.log") def match_source_packages(self, pkg_path): """Return a list of :class:`.package.Package` that match a given path. @@ -725,7 +725,7 @@ def save_temporary_config_files(self, installed_pkg): import re metadata = installed_pkg.package.metadata - config_files = re.split(",\s*", metadata.get("config_files", "")) + config_files = re.split(r",\s*", metadata.get("config_files", "")) if not config_files: return [] @@ -769,7 +769,7 @@ def modified_config_files(self, installed_pkg): import re metadata = installed_pkg.package.metadata - config_files = re.split(",\s*", metadata.get("config_files", "")) + config_files = re.split(r",\s*", metadata.get("config_files", "")) if not config_files: return [] @@ -867,7 +867,7 @@ def backup_modified_files(self, backup_subdir, modified_files): return rval - class SourceAggregationResults(object): + class SourceAggregationResults: """The return value of a call to :meth:`.Manager.aggregate_source()`. Attributes: @@ -972,7 +972,7 @@ def _refresh_source(self, name, aggregate=False, push=False): except git.exc.GitCommandError as error: LOG.error("failed to pull source %s: %s", name, error) return self.SourceAggregationResults( - "failed to pull from remote source: {}".format(error) + f"failed to pull from remote source: {error}" ) if os.path.isfile(agg_file_ours): @@ -1495,7 +1495,7 @@ def list_depender_pkgs(self, pkg_path): item = queue.popleft() for _pkg_name in pkg_dependencies: - pkg_dependees = set([_pkg for _pkg in pkg_dependencies.get(_pkg_name)]) + pkg_dependees = {_pkg for _pkg in pkg_dependencies.get(_pkg_name)} if item in pkg_dependees: # check if there is a cyclic dependency @@ -1586,7 +1586,7 @@ def _has_all_dependers_unloaded(item, dependers): dep_listing = "" for _name in dep_packages: - dep_listing += '"{}", '.format(_name) + dep_listing += f'"{_name}", ' errors.append( ( @@ -1716,7 +1716,7 @@ def info(self, pkg_path, version="", prefer_installed=True): name = name_from_path(pkg_path) if not is_valid_package_name(name): - reason = "Package name {!r} is not valid.".format(name) + reason = f"Package name {name!r} is not valid." return PackageInfo(Package(git_url=pkg_path), invalid_reason=reason) LOG.debug('getting info on "%s"', pkg_path) @@ -1795,7 +1795,7 @@ def _info(self, package, status, version): try: git_checkout(clone, version) except git.exc.GitCommandError: - reason = 'no such commit, branch, or version tag: "{}"'.format(version) + reason = f'no such commit, branch, or version tag: "{version}"' return PackageInfo(package=package, status=status, invalid_reason=reason) LOG.debug('checked out "%s", branch/version "%s"', package, version) @@ -1864,7 +1864,7 @@ def validate_dependencies( prior to the depender packages. """ - class Node(object): + class Node: def __init__(self, name): self.name = name self.info = None @@ -1895,7 +1895,7 @@ def __str__(self): if info.invalid_reason: return ( - 'invalid package "{}": {}'.format(name, info.invalid_reason), + f'invalid package "{name}": {info.invalid_reason}', [], ) @@ -2435,7 +2435,7 @@ def match_package_url_and_version(git_url, version): try: git_clone(git_url, clonepath, shallow=(not is_sha1(version))) except git.exc.GitCommandError as error: - return "failed to clone {}: {}".format(git_url, error) + return f"failed to clone {git_url}: {error}" with open(manifest_file, "w") as f: config.write(f) @@ -2577,7 +2577,7 @@ def test(self, pkg_path, version="", test_dependencies=False): except git.exc.GitCommandError as error: LOG.warning("failed to clone git repo: %s", error) return ( - "failed to clone {}".format(info.package.git_url), + f"failed to clone {info.package.git_url}", False, stage.state_dir, ) @@ -2651,7 +2651,7 @@ def test(self, pkg_path, version="", test_dependencies=False): if rc != 0: return ( - "test_command failed with exit code {}".format(rc), + f"test_command failed with exit code {rc}", False, stage.state_dir, ) @@ -2749,7 +2749,7 @@ def _stage(self, package, version, clone, stage, env=None): else: break - except EnvironmentError as error: + except OSError as error: LOG.warning( 'installing "%s": failed to write build log %s %s: %s', package, @@ -2761,7 +2761,7 @@ def _stage(self, package, version, clone, stage, env=None): returncode = build.wait() if returncode != 0: - return "package build_command failed, see log in {}".format(buildlog) + return f"package build_command failed, see log in {buildlog}" pkg_script_dir = metadata.get("script_dir", "") script_dir_src = os.path.join(clone.working_dir, pkg_script_dir) @@ -2789,8 +2789,8 @@ def _stage(self, package, version, clone, stage, env=None): make_symlink(os.path.join("packages", package.name), symlink_path) except OSError as exception: - error = "could not create symlink at {}".format(symlink_path) - error += ": {}: {}".format(type(exception).__name__, exception) + error = f"could not create symlink at {symlink_path}" + error += f": {type(exception).__name__}: {exception}" return error error = _copy_package_dir( @@ -2930,7 +2930,7 @@ def install(self, pkg_path, version=""): return self._install(matches[0], version) except git.exc.GitCommandError as error: LOG.warning('installing "%s": source package git repo is invalid', pkg_path) - return 'failed to clone package "{}": {}'.format(pkg_path, error) + return f'failed to clone package "{pkg_path}": {error}' def _install(self, package, version, use_existing_clone=False): """Install a :class:`.package.Package`. @@ -2971,7 +2971,7 @@ def _install(self, package, version, use_existing_clone=False): LOG.info( 'branch "%s" not in available branches: %s', version, branches ) - return 'no such branch or version tag: "{}"'.format(version) + return f'no such branch or version tag: "{version}"' else: if len(version_tags): @@ -3152,12 +3152,12 @@ def _copy_package_dir(package, dirname, src, dst, scratch_dir): ld = os.listdir(tmp_dir) if len(ld) != 1: - return "failed to copy package {}: invalid tarfile".format(dirname) + return f"failed to copy package {dirname}: invalid tarfile" src = os.path.join(tmp_dir, ld[0]) if not os.path.isdir(src): - return "failed to copy package {}: not a dir or tarfile".format(dirname) + return f"failed to copy package {dirname}: not a dir or tarfile" def ignore(_, files): rval = [] @@ -3176,11 +3176,11 @@ def ignore(_, files): for err in errors: src, dst, msg = err - reason = "failed to copy {}: {} -> {}: {}".format(dirname, src, dst, msg) + reason = f"failed to copy {dirname}: {src} -> {dst}: {msg}" reasons += "\n" + reason LOG.warning('installing "%s": %s', package, reason) - return "failed to copy package {}: {}".format(dirname, reasons) + return f"failed to copy package {dirname}: {reasons}" return "" @@ -3232,7 +3232,7 @@ def _parse_package_metadata(parser, metadata_file): if not parser.has_section("package"): LOG.warning("%s: metadata missing [package]", metadata_file) - return "{} is missing [package] section".format(os.path.basename(metadata_file)) + return f"{os.path.basename(metadata_file)} is missing [package] section" return "" diff --git a/zeekpkg/package.py b/zeekpkg/package.py index b2719852..8a2cea24 100644 --- a/zeekpkg/package.py +++ b/zeekpkg/package.py @@ -55,7 +55,7 @@ def aliases(metadata_dict): if "aliases" not in metadata_dict: return [] - return re.split(",\s*|\s+", metadata_dict["aliases"]) + return re.split(r",\s*|\s+", metadata_dict["aliases"]) def tags(metadata_dict): @@ -63,7 +63,7 @@ def tags(metadata_dict): if "tags" not in metadata_dict: return [] - return re.split(",\s*", metadata_dict["tags"]) + return re.split(r",\s*", metadata_dict["tags"]) def short_description(metadata_dict): @@ -138,7 +138,7 @@ def dependencies(metadata_dict, field="depends"): @total_ordering -class InstalledPackage(object): +class InstalledPackage: """An installed package and its current status. Attributes: @@ -161,7 +161,7 @@ def __lt__(self, other): return str(self.package) < str(other.package) -class PackageStatus(object): +class PackageStatus: """The status of an installed package. This class contains properties of a package related to how the package @@ -203,7 +203,7 @@ def __init__( self.current_hash = current_hash -class PackageInfo(object): +class PackageInfo: """Contains information on an arbitrary package. If the package is installed, then its status is also available. @@ -314,7 +314,7 @@ def best_version(self): @total_ordering -class Package(object): +class Package: """A Zeek package. This class contains properties of a package that are defined by the package @@ -442,7 +442,7 @@ def name_with_source_directory(self): just the package name is returned. """ if self.directory: - return "{}/{}".format(self.directory, self.name) + return f"{self.directory}/{self.name}" return self.name @@ -454,7 +454,7 @@ def qualified_name(self): git URL is returned. """ if self.source: - return "{}/{}".format(self.source, self.name_with_source_directory()) + return f"{self.source}/{self.name_with_source_directory()}" return self.git_url diff --git a/zeekpkg/source.py b/zeekpkg/source.py index edf1ff02..3b07909c 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -22,7 +22,7 @@ AGGREGATE_DATA_FILE = "aggregate.meta" -class Source(object): +class Source: """A Zeek package source. This class contains properties of a package source like its name, remote git diff --git a/zeekpkg/template.py b/zeekpkg/template.py index ecac3e20..87a87fb2 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -137,7 +137,7 @@ def load(config, template, version=None): if repo is None: repo = git_clone(template, templatedir) except git.exc.GitCommandError as error: - msg = 'failed to update template "{}": {}'.format(template, error) + msg = f'failed to update template "{template}": {error}' LOG.error(msg) raise GitError(msg) from error @@ -179,7 +179,7 @@ def load(config, template, version=None): try: mod = load_source(os.path.join(templatedir, "__init__.py")) except Exception as error: - msg = 'failed to load template "{}": {}'.format(template, error) + msg = f'failed to load template "{template}": {error}' LOG.exception(msg) raise LoadError(msg) from error @@ -199,7 +199,7 @@ def load(config, template, version=None): is_compat = Template.is_api_compatible(mod.TEMPLATE_API_VERSION) except ValueError: raise LoadError( - 'API version string "{}" is invalid'.format(mod.TEMPLATE_API_VERSION) + f'API version string "{mod.TEMPLATE_API_VERSION}" is invalid' ) if not is_compat: @@ -603,7 +603,7 @@ def instantiate_file(self, tmpl, orig_file, path_name, file_name, content): with open(out_file, "wb") as hdl: hdl.write(content) shutil.copymode(orig_file, out_file) - except IOError as error: + except OSError as error: LOG.warning('I/O error while instantiating "%s": %s', out_file, error) def instantiate_symlink(self, tmpl, orig_file, path_name, file_name, target): @@ -668,7 +668,7 @@ def _walk(self, tmpl): try: with open(in_file, "rb") as hdl: out_content = self._replace(tmpl, hdl.read()) - except IOError as error: + except OSError as error: LOG.warning("skipping instantiation of %s: %s", in_file, error) continue @@ -774,7 +774,7 @@ def _git_init(self, tmpl): if self._features: names = sorted(['"' + f.name() + '"' for f in self._features]) if len(names) == 1: - features_info = ", with feature {}".format(names[0]) + features_info = f", with feature {names[0]}" else: features_info = ", with features " features_info += ", ".join(names[:-1]) diff --git a/zeekpkg/uservar.py b/zeekpkg/uservar.py index 8127a5f3..ac000715 100644 --- a/zeekpkg/uservar.py +++ b/zeekpkg/uservar.py @@ -123,7 +123,7 @@ def resolve(self, name, config, user_var_args=None, force=False): return val desc = " (" + self._desc + ")" if self._desc else "" - print('"{}" requires a "{}" value{}: '.format(name, self._name, desc)) + print(f'"{name}" requires a "{self._name}" value{desc}: ') self._val = _rlinput(self._name + ": ", val) return self._val @@ -136,7 +136,7 @@ def parse_arg(arg): return UserVar(name, val=val) except ValueError as error: raise ValueError( - 'invalid user var argument "{}", must be NAME=VAR'.format(arg) + f'invalid user var argument "{arg}", must be NAME=VAR' ) from error @staticmethod diff --git a/zkg b/zkg index a7c22442..85e21e60 100755 --- a/zkg +++ b/zkg @@ -165,10 +165,10 @@ def prompt_for_user_vars(manager, config, configfile, args, pkg_infos): config.set("user_vars", key, value) if configfile: - with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: + with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print("Saved answers to config file: {}".format(configfile)) + print(f"Saved answers to config file: {configfile}") manager.user_vars = answers @@ -244,7 +244,7 @@ def create_config(args, configfile): if configfile: if not os.path.isfile(configfile): - print_error('error: invalid config file "{}"'.format(configfile)) + print_error(f'error: invalid config file "{configfile}"') sys.exit(1) config.read(configfile) @@ -377,10 +377,10 @@ def is_local_git_repo_dirty(git_url): def check_local_git_repo(git_url): if is_local_git_repo_url(git_url): if not is_local_git_repo(git_url): - print_error("error: path {} is not a git repository".format(git_url)) + print_error(f"error: path {git_url} is not a git repository") return False if is_local_git_repo_dirty(git_url): - print_error("error: local git clone at {} is dirty".format(git_url)) + print_error(f"error: local git clone at {git_url} is dirty") return False return True @@ -405,15 +405,15 @@ def create_manager(args, config): bin_dir=bin_dir, zeek_dist=zeek_dist, ) - except (OSError, IOError) as error: + except OSError as error: if error.errno == errno.EACCES: - print_error("{}: {}".format(type(error).__name__, error)) + print_error(f"{type(error).__name__}: {error}") def check_permission(d): if os.access(d, os.W_OK): return True - print_error("error: user does not have write access in {}".format(d)) + print_error(f"error: user does not have write access in {d}") return False permissions_trouble = not all( @@ -439,13 +439,13 @@ def create_manager(args, config): for key_value in args.extra_source or []: if "=" not in key_value: - print_error('warning: invalid extra source: "{}"'.format(key_value)) + print_error(f'warning: invalid extra source: "{key_value}"') continue key, value = key_value.split("=", 1) if not key or not value: - print_error('warning: invalid extra source: "{}"'.format(key_value)) + print_error(f'warning: invalid extra source: "{key_value}"') continue extra_sources.append((key, value)) @@ -488,14 +488,14 @@ def get_changed_state(manager, saved_state, pkg_lst): if not _ipkg or _ipkg.status.is_loaded == saved_state[_pkg_name]: continue - dep_listing += " {}\n".format(_pkg_name) + dep_listing += f" {_pkg_name}\n" return dep_listing class InstallWorker(threading.Thread): def __init__(self, manager, package_name, package_version): - super(InstallWorker, self).__init__() + super().__init__() self.manager = manager self.package_name = package_name self.package_version = package_version @@ -660,7 +660,7 @@ def cmd_install(manager, args, config, configfile): for info, version, _ in sorted(package_infos, key=lambda x: x[0].package.name): name = info.package.qualified_name() - package_listing += " {} ({})\n".format(name, version) + package_listing += f" {name} ({version})\n" print("The following packages will be INSTALLED:") print(package_listing) @@ -672,7 +672,7 @@ def cmd_install(manager, args, config, configfile): new_pkgs, key=lambda x: x[0].package.name ): name = info.package.qualified_name() - dependency_listing += " {} ({})".format(name, version) + dependency_listing += f" {name} ({version})" if suggested: dependency_listing += " (suggested)" @@ -696,10 +696,10 @@ def cmd_install(manager, args, config, configfile): continue if extdeps: - extdep_listing += " from {} ({}):\n".format(name, version) + extdep_listing += f" from {name} ({version}):\n" for extdep, semver in sorted(extdeps.items()): - extdep_listing += " {} {}\n".format(extdep, semver) + extdep_listing += f" {extdep} {semver}\n" if extdep_listing: print( @@ -730,7 +730,7 @@ def cmd_install(manager, args, config, configfile): ) continue - print('Running unit tests for "{}"'.format(name)) + print(f'Running unit tests for "{name}"') error_msg = "" # For testing we always process dependencies, since the tests might # well fail without them. If the user wants --nodeps and the tests @@ -755,7 +755,7 @@ def cmd_install(manager, args, config, configfile): ) if error_msg: - print_error("error: {}".format(error_msg)) + print_error(f"error: {error_msg}") if args.force: continue @@ -781,15 +781,15 @@ def cmd_install(manager, args, config, configfile): worker = InstallWorker(manager, name, version) worker.start() - worker.wait('Installing "{}"'.format(name)) + worker.wait(f'Installing "{name}"') if worker.error: - print('Failed installing "{}": {}'.format(name, worker.error)) + print(f'Failed installing "{name}": {worker.error}') installs_failed.append((name, version)) continue ipkg = manager.find_installed_package(name) - print('Installed "{}" ({})'.format(name, ipkg.status.current_version)) + print(f'Installed "{name}" ({ipkg.status.current_version})') if is_overwriting: for i, mf in enumerate(modifications): @@ -818,9 +818,9 @@ def cmd_install(manager, args, config, configfile): load_error = manager.load(name) if load_error: - print('Failed loading "{}": {}'.format(name, load_error)) + print(f'Failed loading "{name}": {load_error}') else: - print('Loaded "{}"'.format(name)) + print(f'Loaded "{name}"') if not args.nodeps: # Now load runtime dependencies after all dependencies and suggested @@ -835,7 +835,7 @@ def cmd_install(manager, args, config, configfile): for _name, _error in load_error: if not _error: - _listing += " {}\n".format(_name) + _listing += f" {_name}\n" if not _listing: dep_listing = get_changed_state(manager, saved_state, [name]) @@ -862,7 +862,7 @@ def cmd_install(manager, args, config, configfile): ) for n, v in installs_failed: - print_error(" {} ({})".format(n, v)) + print_error(f" {n} ({v})") sys.exit(1) @@ -879,9 +879,7 @@ def cmd_bundle(manager, args, config, configfile): if config.read(args.manifest[0]) and config.has_section("bundle"): packages = config.items("bundle") else: - print_error( - 'error: "{}" is not a valid manifest file'.format(args.manifest[0]) - ) + print_error(f'error: "{args.manifest[0]}" is not a valid manifest file') sys.exit(1) else: @@ -962,7 +960,7 @@ def cmd_bundle(manager, args, config, configfile): package_listing = "" for name, _, version, is_dependency, is_suggestion in packages_to_bundle: - package_listing += " {} ({})".format(name, version) + package_listing += f" {name} ({version})" if is_suggestion: package_listing += " (suggested)" @@ -987,10 +985,10 @@ def cmd_bundle(manager, args, config, configfile): ) if error: - print_error("error: failed to create bundle: {}".format(error)) + print_error(f"error: failed to create bundle: {error}") sys.exit(1) - print("Bundle successfully written: {}".format(args.bundle_filename)) + print(f"Bundle successfully written: {args.bundle_filename}") def cmd_unbundle(manager, args, config, configfile): @@ -1005,9 +1003,7 @@ def cmd_unbundle(manager, args, config, configfile): error, bundle_info = manager.bundle_info(args.bundle_filename) if error: - print_error( - "error: failed to unbundle {}: {}".format(args.bundle_filename, error) - ) + print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") sys.exit(1) for git_url, version, pkg_info in bundle_info: @@ -1035,7 +1031,7 @@ def cmd_unbundle(manager, args, config, configfile): name = pkg.qualified_name() break - package_listing += " {} ({})\n".format(name, version) + package_listing += f" {name} ({version})\n" print("The following packages will be INSTALLED:") print(package_listing) @@ -1059,10 +1055,10 @@ def cmd_unbundle(manager, args, config, configfile): continue if extdeps: - extdep_listing += " from {} ({}):\n".format(name, version) + extdep_listing += f" from {name} ({version}):\n" for extdep, semver in sorted(extdeps.items()): - extdep_listing += " {} {}\n".format(extdep, semver) + extdep_listing += f" {extdep} {semver}\n" if extdep_listing: print( @@ -1082,9 +1078,7 @@ def cmd_unbundle(manager, args, config, configfile): error = manager.unbundle(args.bundle_filename) if error: - print_error( - "error: failed to unbundle {}: {}".format(args.bundle_filename, error) - ) + print_error(f"error: failed to unbundle {args.bundle_filename}: {error}") sys.exit(1) for git_url, _, _ in bundle_info: @@ -1096,21 +1090,21 @@ def cmd_unbundle(manager, args, config, configfile): ipkg = manager.find_installed_package(git_url) if not ipkg: - print('Skipped loading "{}": failed to install'.format(git_url)) + print(f'Skipped loading "{git_url}": failed to install') continue name = ipkg.package.qualified_name() if not need_load: - print('Skipped loading "{}"'.format(name)) + print(f'Skipped loading "{name}"') continue load_error = manager.load(name) if load_error: - print('Failed loading "{}": {}'.format(name, load_error)) + print(f'Failed loading "{name}": {load_error}') else: - print('Loaded "{}"'.format(name)) + print(f'Loaded "{name}"') print("Unbundling complete.") @@ -1129,7 +1123,7 @@ def cmd_remove(manager, args, config, configfile): ipkg = manager.find_installed_package(name) if not ipkg: - print_error('error: package "{}" is not installed'.format(name)) + print_error(f'error: package "{name}" is not installed') sys.exit(1) packages_to_remove.append(ipkg) @@ -1149,7 +1143,7 @@ def cmd_remove(manager, args, config, configfile): print("The following packages will be REMOVED:") for ipkg in packages_to_remove: - print(" {}".format(ipkg.package.qualified_name())) + print(f" {ipkg.package.qualified_name()}") print() @@ -1158,7 +1152,7 @@ def cmd_remove(manager, args, config, configfile): for pkg_name in sorted(dependers_to_unload): ipkg = manager.find_installed_package(pkg_name) - print(" {}".format(ipkg.package.qualified_name())) + print(f" {ipkg.package.qualified_name()}") print() @@ -1170,12 +1164,12 @@ def cmd_remove(manager, args, config, configfile): name = ipkg.package.qualified_name() if manager.unload(name): - print('Unloaded "{}"'.format(name)) + print(f'Unloaded "{name}"') else: # Weird that it failed, but if it's not installed and there's # nothing to unload, not worth using a non-zero exit-code to # reflect an overall failure of the package removal operation - print('Failed unloading "{}": no such package installed'.format(name)) + print(f'Failed unloading "{name}": no such package installed') had_failure = False @@ -1185,7 +1179,7 @@ def cmd_remove(manager, args, config, configfile): backup_files = manager.backup_modified_files(name, modifications) if manager.remove(name): - print('Removed "{}"'.format(name)) + print(f'Removed "{name}"') if backup_files: print("\tCreated backups of locally modified config files:") @@ -1194,7 +1188,7 @@ def cmd_remove(manager, args, config, configfile): print("\t" + backup_file) else: - print('Failed removing "{}": no such package installed'.format(name)) + print(f'Failed removing "{name}": no such package installed') had_failure = True if had_failure: @@ -1213,7 +1207,7 @@ def cmd_purge(manager, args, config, configfile): names_to_remove = [ipkg.package.qualified_name() for ipkg in packages_to_remove] for name in names_to_remove: - package_listing += " {}\n".format(name) + package_listing += f" {name}\n" print("The following packages will be REMOVED:") print(package_listing) @@ -1229,7 +1223,7 @@ def cmd_purge(manager, args, config, configfile): backup_files = manager.backup_modified_files(name, modifications) if manager.remove(name): - print('Removed "{}"'.format(name)) + print(f'Removed "{name}"') if backup_files: print("\tCreated backups of locally modified config files:") @@ -1238,7 +1232,7 @@ def cmd_purge(manager, args, config, configfile): print("\t" + backup_file) else: - print('Unknown error removing "{}"'.format(name)) + print(f'Unknown error removing "{name}"') had_failure = True if had_failure: @@ -1267,7 +1261,7 @@ def cmd_refresh(manager, args, config, configfile): had_aggregation_failure = False for source in args.sources: - print("Refresh package source: {}".format(source)) + print(f"Refresh package source: {source}") src_pkgs_before = {i.qualified_name() for i in manager.source_packages()} @@ -1283,7 +1277,7 @@ def cmd_refresh(manager, args, config, configfile): if error: had_failure = True - print_error('error: failed to refresh "{}": {}'.format(source, error)) + print_error(f'error: failed to refresh "{source}": {error}') continue src_pkgs_after = {i.qualified_name() for i in manager.source_packages()} @@ -1296,7 +1290,7 @@ def cmd_refresh(manager, args, config, configfile): for name in diff: change = "Added" if name in src_pkgs_after else "Removed" - print("\t\t{} {}".format(change, name)) + print(f"\t\t{change} {name}") if args.aggregate: if aggregation_issues: @@ -1306,7 +1300,7 @@ def cmd_refresh(manager, args, config, configfile): ) for url, issue in aggregation_issues: - print("\t\t{}: {}".format(url, issue)) + print(f"\t\t{url}: {issue}") if args.fail_on_aggregate_problems: had_aggregation_failure = True else: @@ -1329,7 +1323,7 @@ def cmd_refresh(manager, args, config, configfile): for name in diff: ipkg = manager.find_installed_package(name) version_change = version_change_string(manager, ipkg) - print("\t\t{} {}".format(name, version_change)) + print(f"\t\t{name} {version_change}") if had_failure: sys.exit(1) @@ -1348,9 +1342,9 @@ def version_change_string(manager, installed_package): if len(versions): new_version = versions[-1] - version_change = "({} -> {})".format(old_version, new_version) + version_change = f"({old_version} -> {new_version})" else: - version_change = "({})".format(new_version) + version_change = f"({new_version})" return version_change @@ -1368,7 +1362,7 @@ def cmd_upgrade(manager, args, config, configfile): ipkg = manager.find_installed_package(name) if not ipkg: - print_error('error: package "{}" is not installed'.format(name)) + print_error(f'error: package "{name}" is not installed') sys.exit(1) name = ipkg.package.qualified_name() @@ -1396,7 +1390,7 @@ def cmd_upgrade(manager, args, config, configfile): outdated_packages.append((info, next_version, False)) version_change = version_change_string(manager, ipkg) - package_listing += " {} {}\n".format(name, version_change) + package_listing += f" {name} {version_change}\n" if not outdated_packages: print("All packages already up-to-date.") @@ -1428,7 +1422,7 @@ def cmd_upgrade(manager, args, config, configfile): for info, version, suggestion in new_pkgs: name = info.package.qualified_name() - dependency_listing += " {} ({})".format(name, version) + dependency_listing += f" {name} ({version})" if suggestion: dependency_listing += " (suggested)" @@ -1451,10 +1445,10 @@ def cmd_upgrade(manager, args, config, configfile): continue if extdeps: - extdep_listing += " from {} ({}):\n".format(name, version) + extdep_listing += f" from {name} ({version}):\n" for extdep, semver in sorted(extdeps.items()): - extdep_listing += " {} {}\n".format(extdep, semver) + extdep_listing += f" {extdep} {semver}\n" if extdep_listing: print( @@ -1481,7 +1475,7 @@ def cmd_upgrade(manager, args, config, configfile): ) continue - print('Running unit tests for "{}"'.format(name)) + print(f'Running unit tests for "{name}"') error_msg = "" # As in cmd_install, we always process dependencies since the tests # might well fail without them. If the user wants --nodeps and the @@ -1507,7 +1501,7 @@ def cmd_upgrade(manager, args, config, configfile): ) if error_msg: - print_error("error: {}".format(error_msg)) + print_error(f"error: {error_msg}") if args.force: continue @@ -1521,22 +1515,22 @@ def cmd_upgrade(manager, args, config, configfile): name = info.package.qualified_name() worker = InstallWorker(manager, name, version) worker.start() - worker.wait('Installing "{}"'.format(name)) + worker.wait(f'Installing "{name}"') if worker.error: - print('Failed installing "{}": {}'.format(name, worker.error)) + print(f'Failed installing "{name}": {worker.error}') continue ipkg = manager.find_installed_package(name) - print('Installed "{}" ({})'.format(name, ipkg.status.current_version)) + print(f'Installed "{name}" ({ipkg.status.current_version})') if manager.has_scripts(ipkg): load_error = manager.load(name) if load_error: - print('Failed loading "{}": {}'.format(name, load_error)) + print(f'Failed loading "{name}": {load_error}') else: - print('Loaded "{}"'.format(name)) + print(f'Loaded "{name}"') had_failure = False @@ -1556,11 +1550,11 @@ def cmd_upgrade(manager, args, config, configfile): res = manager.upgrade(name) if res: - print('Failed upgrading "{}": {}'.format(name, res)) + print(f'Failed upgrading "{name}": {res}') had_failure = True else: ipkg = manager.find_installed_package(name) - print('Upgraded "{}" ({})'.format(name, ipkg.status.current_version)) + print(f'Upgraded "{name}" ({ipkg.status.current_version})') for i, mf in enumerate(modifications): next_upstream_config_file = mf[1] @@ -1598,11 +1592,11 @@ def cmd_load(manager, args, config, configfile): if not ipkg: had_failure = True - print('Failed to load "{}": no such package installed'.format(name)) + print(f'Failed to load "{name}": no such package installed') continue if not manager.has_scripts(ipkg): - print('The package "{}" does not contain scripts to load.'.format(name)) + print(f'The package "{name}" does not contain scripts to load.') continue name = ipkg.package.qualified_name() @@ -1620,7 +1614,7 @@ def cmd_load(manager, args, config, configfile): for _name, _error in loaded_dep_list: if _error: load_error = True - dep_error_listing += " {}: {}\n".format(_name, _error) + dep_error_listing += f" {_name}: {_error}\n" if not load_error: dep_listing = get_changed_state(manager, saved_state, [name]) @@ -1645,9 +1639,9 @@ def cmd_load(manager, args, config, configfile): print(dep_error_listing) manager.restore_loaded_package_states(saved_state) - print('Failed to load "{}": {}'.format(name, load_error)) + print(f'Failed to load "{name}": {load_error}') else: - print('Loaded "{}"'.format(name)) + print(f'Loaded "{name}"') if had_failure: sys.exit(1) @@ -1670,7 +1664,7 @@ def cmd_unload(manager, args, config, configfile): if not ipkg: had_failure = True - print('Failed to unload "{}": no such package installed'.format(name)) + print(f'Failed to unload "{name}": no such package installed') continue if not ipkg.status.is_loaded: @@ -1691,7 +1685,7 @@ def cmd_unload(manager, args, config, configfile): print("The following packages will be UNLOADED:") for ipkg in packages_to_unload: - print(" {}".format(ipkg.package.qualified_name())) + print(f" {ipkg.package.qualified_name()}") print() @@ -1700,7 +1694,7 @@ def cmd_unload(manager, args, config, configfile): for pkg_name in sorted(dependers_to_unload): ipkg = manager.find_installed_package(pkg_name) - print(" {}".format(ipkg.package.qualified_name())) + print(f" {ipkg.package.qualified_name()}") print() @@ -1717,10 +1711,10 @@ def cmd_unload(manager, args, config, configfile): name = ipkg.package.qualified_name() if manager.unload(name): - print('Unloaded "{}"'.format(name)) + print(f'Unloaded "{name}"') else: had_failure = True - print('Failed unloading "{}": no such package installed'.format(name)) + print(f'Failed unloading "{name}": no such package installed') if had_failure: sys.exit(1) @@ -1734,7 +1728,7 @@ def cmd_pin(manager, args, config, configfile): if not ipkg: had_failure = True - print('Failed to pin "{}": no such package installed'.format(name)) + print(f'Failed to pin "{name}": no such package installed') continue name = ipkg.package.qualified_name() @@ -1748,7 +1742,7 @@ def cmd_pin(manager, args, config, configfile): ) else: had_failure = True - print('Failed pinning "{}": no such package installed'.format(name)) + print(f'Failed pinning "{name}": no such package installed') if had_failure: sys.exit(1) @@ -1762,7 +1756,7 @@ def cmd_unpin(manager, args, config, configfile): if not ipkg: had_failure = True - print('Failed to unpin "{}": no such package installed'.format(name)) + print(f'Failed to unpin "{name}": no such package installed') continue name = ipkg.package.qualified_name() @@ -1776,7 +1770,7 @@ def cmd_unpin(manager, args, config, configfile): ) else: had_failure = True - print('Failed unpinning "{}": no such package installed'.format(name)) + print(f'Failed unpinning "{name}": no such package installed') if had_failure: sys.exit(1) @@ -1839,7 +1833,7 @@ def cmd_list(manager, args, config, configfile): for pkg_name, val in sorted(filtered_pkgs.items()): if isinstance(val, zeekpkg.InstalledPackage): pkg = val.package - out = "{} (installed: {})".format(pkg_name, val.status.current_version) + out = f"{pkg_name} (installed: {val.status.current_version})" else: pkg = val out = pkg_name @@ -1864,7 +1858,7 @@ def cmd_search(manager, args, config, configfile): try: regex = re.compile(search_text[1:-1]) except re.error as error: - print("invalid regex: {}".format(error)) + print(f"invalid regex: {error}") sys.exit(1) else: for pkg in src_pkgs: @@ -1891,7 +1885,7 @@ def cmd_search(manager, args, config, configfile): ipkg = manager.find_installed_package(match.qualified_name()) if ipkg: - out += " (installed: {})".format(ipkg.status.current_version) + out += f" (installed: {ipkg.status.current_version})" desc = match.short_description() @@ -1934,13 +1928,13 @@ def cmd_info(manager, args, config, configfile): pkginfo[name] = dict() pkginfo[name]["metadata"] = dict() else: - print('"{}" info:'.format(name)) + print(f'"{name}" info:') if info.invalid_reason: if args.json: pkginfo[name]["invalid"] = info.invalid_reason else: - print("\tinvalid package: {}\n".format(info.invalid_reason)) + print(f"\tinvalid package: {info.invalid_reason}\n") had_invalid_package = True continue @@ -1949,8 +1943,8 @@ def cmd_info(manager, args, config, configfile): pkginfo[name]["url"] = info.package.git_url pkginfo[name]["versions"] = info.versions else: - print("\turl: {}".format(info.package.git_url)) - print("\tversions: {}".format(info.versions)) + print(f"\turl: {info.package.git_url}") + print(f"\tversions: {info.versions}") if info.status: if args.json: @@ -1962,7 +1956,7 @@ def cmd_info(manager, args, config, configfile): print("\tinstall status:") for key, value in sorted(info.status.__dict__.items()): - print("\t\t{} = {}".format(key, value)) + print(f"\t\t{key} = {value}") if args.json: if info.metadata_file: @@ -1970,8 +1964,8 @@ def cmd_info(manager, args, config, configfile): pkginfo[name]["metadata"][info.metadata_version] = dict() else: if info.metadata_file: - print("\tmetadata file: {}".format(info.metadata_file)) - print('\tmetadata (from version "{}"):'.format(info.metadata_version)) + print(f"\tmetadata file: {info.metadata_file}") + print(f'\tmetadata (from version "{info.metadata_version}"):') if len(info.metadata) == 0: if args.json != True: @@ -1984,7 +1978,7 @@ def cmd_info(manager, args, config, configfile): else: for key, value in sorted(info.metadata.items()): value = value.replace("\n", "\n\t\t\t") - print("\t\t{} = {}".format(key, value)) + print(f"\t\t{key} = {value}") # If --json and --allvers given, check for multiple versions and # add the metadata for each version to the pkginfo. @@ -2049,11 +2043,11 @@ def cmd_config(manager, args, config, configfile): out.close() elif args.config_param == "sources": for key, value in config_items(config, "sources"): - print("{} = {}".format(key, value)) + print(f"{key} = {value}") elif args.config_param == "user_vars": if config.has_section("user_vars"): for key, value in config_items(config, "user_vars"): - print("{} = {}".format(key, value)) + print(f"{key} = {value}") else: print(config.get("paths", args.config_param)) @@ -2061,9 +2055,9 @@ def cmd_config(manager, args, config, configfile): def cmd_autoconfig(manager, args, config, configfile): if args.user: configfile = os.path.join(home_config_dir(), "config") - with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: + with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print("Successfully wrote config file to {}".format(configfile)) + print(f"Successfully wrote config file to {configfile}") return zeek_config = find_program("zeek-config") @@ -2113,10 +2107,10 @@ def cmd_autoconfig(manager, args, config, configfile): if old_value == new_value: return - prompt = 'Set "{}" config option to: {} ?'.format(option, new_value) + prompt = f'Set "{option}" config option to: {new_value} ?' if old_value: - prompt += "\n(previous value: {})".format(old_value) + prompt += f"\n(previous value: {old_value})" if args.force or confirmation_prompt(prompt): config.set(section, option, new_value) @@ -2126,10 +2120,10 @@ def cmd_autoconfig(manager, args, config, configfile): change_config_value(config, "paths", "bin_dir", bin_dir, config_file_exists) change_config_value(config, "paths", "zeek_dist", zeek_dist, config_file_exists) - with io.open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: + with open(configfile, "w", encoding=std_encoding(sys.stdout)) as f: config.write(f) - print("Successfully wrote config file to {}".format(configfile)) + print(f"Successfully wrote config file to {configfile}") def cmd_env(manager, args, config, configfile): @@ -2184,13 +2178,13 @@ def cmd_env(manager, args, config, configfile): print("setenv BRO_PLUGIN_PATH {}".format(":".join(pluginpaths))) print("setenv ZEEKPATH {}".format(":".join(zeekpaths))) print("setenv ZEEK_PLUGIN_PATH {}".format(":".join(pluginpaths))) - print("setenv PATH {}:$PATH".format(manager.bin_dir)) + print(f"setenv PATH {manager.bin_dir}:$PATH") else: print("export BROPATH={}".format(":".join(zeekpaths))) print("export BRO_PLUGIN_PATH={}".format(":".join(pluginpaths))) print("export ZEEKPATH={}".format(":".join(zeekpaths))) print("export ZEEK_PLUGIN_PATH={}".format(":".join(pluginpaths))) - print("export PATH={}:$PATH".format(manager.bin_dir)) + print(f"export PATH={manager.bin_dir}:$PATH") def cmd_create(manager, args, config, configfile): @@ -2202,7 +2196,7 @@ def cmd_create(manager, args, config, configfile): try: tmpl = Template.load(config, tmplname, args.version) except LoadError as error: - msg = "problem while loading template {}: {}".format(tmplname, error) + msg = f"problem while loading template {tmplname}: {error}" zeekpkg.LOG.exception(msg) print_error("error: " + msg) sys.exit(1) @@ -2219,7 +2213,7 @@ def cmd_create(manager, args, config, configfile): # filter any duplicates. fnames = set() for feat in args.features: - fnames |= set([f.strip() for f in feat.split(",") if f]) + fnames |= {f.strip() for f in feat.split(",") if f} features = tmpl.features() for feature in features: if feature.name() in fnames: @@ -2270,9 +2264,7 @@ def cmd_create(manager, args, config, configfile): try: if os.path.isdir(args.packagedir): if not args.force: - print( - "Package directory {} already exists.".format(args.packagedir) - ) + print(f"Package directory {args.packagedir} already exists.") if not confirmation_prompt("Delete?"): sys.exit(1) try: @@ -2293,7 +2285,7 @@ def cmd_create(manager, args, config, configfile): print_error("error: template instantiation failed, " + str(error)) sys.exit(1) except Exception as error: - msg = "problem during template instantiation: {}".format(error) + msg = f"problem during template instantiation: {error}" zeekpkg.LOG.exception(msg) print_error("error: " + msg) sys.exit(1) @@ -2309,7 +2301,7 @@ def cmd_template_info(manager, args, config, configfile): try: tmpl = Template.load(config, tmplname, args.version) except LoadError as error: - msg = "problem while loading template {}: {}".format(tmplname, error) + msg = f"problem while loading template {tmplname}: {error}" zeekpkg.LOG.exception(msg) print_error("error: " + msg) sys.exit(1) @@ -2339,7 +2331,7 @@ def cmd_template_info(manager, args, config, configfile): class BundleHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): # Workaround for underlying argparse bug: https://bugs.python.org/issue9338 def _format_args(self, action, default_metavar): - rval = super(BundleHelpFormatter, self)._format_args(action, default_metavar) + rval = super()._format_args(action, default_metavar) if action.nargs == argparse.ZERO_OR_MORE: rval += " --" From 4a28a69df9daaa317b8f7a8c075618aa54ac71bb Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Tue, 10 Jan 2023 09:56:52 +0100 Subject: [PATCH 6/7] Make sure files are properly closed in `setup.py`. Even though I am unsure this can actually lead to issues, this was flagged by CodeQL. Implementing a proper workaround instead of continously whitelisting these instances. --- setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e234dbf6..066db7c4 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,22 @@ +import pathlib from setuptools import setup install_requires = ["gitpython", "semantic_version", "btest"] + +def version(): + return pathlib.Path("VERSION").read_text().replace("-", ".dev", 1).strip() + + +def long_description(): + return pathlib.Path("README").read_text() + + setup( name="zkg", - version=open("VERSION").read().replace("-", ".dev", 1).strip(), + version=version(), description="The Zeek Package Manager", - long_description=open("README").read(), + long_description=long_description(), license="University of Illinois/NCSA Open Source License", keywords="zeek zeekctl zeekcontrol package manager scripts plugins security", maintainer="The Zeek Project", From 90a80b48a9239d8b32b4e16293039c2a64809f34 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Tue, 10 Jan 2023 13:21:58 +0100 Subject: [PATCH 7/7] Replace manual nested loops with set operation. --- zeekpkg/_util.py | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index f659a9f2..c72f07ef 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -6,6 +6,7 @@ import importlib.machinery import os import shutil +import string import tarfile import types @@ -287,28 +288,8 @@ def is_sha1(s): if len(s) != 40: return False - for c in s: - if c not in { - "a", - "b", - "c", - "d", - "e", - "f", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - }: - return False - - return True + hexdigits = set(string.hexdigits.lower()) + return all(c in hexdigits for c in s) def is_exe(path):