Update scripts

Make fetch_built_wheel work
Add new strip classifiers option to fix_thirdparty
Improve simple requirements parsing to get the latest versions

Signed-off-by: Philippe Ombredanne <pombredanne@nexb.com>

etc/scripts/fetch_built_wheels.py

@@ -5,7 +5,7 @@
 # ScanCode is a trademark of nexB Inc.
 # SPDX-License-Identifier: Apache-2.0
 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
-# See https://github.com/nexB/skeleton for support or download.
+# See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 import click
@@ -14,20 +14,50 @@
 
 
 @click.command()
+@click.option(
+    "--remote-build-log-file",
+    type=click.Path(readable=True),
+    metavar="LOG-FILE",
+    help="Path to a remote builds log file.",
+)
 @click.option(
     "-d",
     "--thirdparty-dir",
     type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
-    required=True,
-    help="Path to the thirdparty directory to check.",
+    metavar="DIR",
+    default=utils_thirdparty.THIRDPARTY_DIR,
+    show_default=True,
+    help="Path to the thirdparty directory to save built wheels.",
+)
+@click.option(
+    "--no-wait",
+    is_flag=True,
+    default=False,
+    help="Do not wait for build completion.",
+)
+@click.option(
+    "--verbose",
+    is_flag=True,
+    help="Provide verbose output.",
 )
 @click.help_option("-h", "--help")
-def check_thirdparty_dir(thirdparty_dir):
+def fetch_remote_wheels(
+    remote_build_log_file,
+    thirdparty_dir,
+    no_wait,
+    verbose,
+):
     """
-    Check a thirdparty directory for problems.
+    Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines
+    build log file.
     """
-    utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
+    utils_thirdparty.fetch_remotely_built_wheels(
+        remote_build_log_file=remote_build_log_file,
+        dest_dir=thirdparty_dir,
+        no_wait=no_wait,
+        verbose=verbose,
+    )
 
 
 if __name__ == "__main__":
-    check_thirdparty_dir()
+    fetch_remote_wheels()

etc/scripts/fix_thirdparty.py

@@ -41,12 +41,18 @@
     "do not download them either). Instead create a JSON lines log file with "
     "one entry for each build suitable to fetch the artifacts at a later time.",
 )
+@click.option(
+    "--strip-classifiers",
+    is_flag=True,
+    help="Remove danglingf classifiers",
+)
 @click.help_option("-h", "--help")
 def fix_thirdparty_dir(
     thirdparty_dir,
     build_wheels,
     build_remotely,
     remote_build_log_file,
+    strip_classifiers,
 ):
     """
     Fix a thirdparty directory of dependent package wheels and sdist.
@@ -61,35 +67,45 @@ def fix_thirdparty_dir(
     Optionally build missing binary wheels for all supported OS and Python
     version combos locally or remotely.
     """
-    print("***FETCH*** MISSING WHEELS")
-    package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir)
-    print("***FETCH*** MISSING SOURCES")
-    src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
-
-    package_envts_not_built = []
-    if build_wheels:
-        print("***BUILD*** MISSING WHEELS")
-        results = utils_thirdparty.build_missing_wheels(
-            packages_and_envts=package_envts_not_fetched,
-            build_remotely=build_remotely,
-            remote_build_log_file=remote_build_log_file,
+    if strip_classifiers:
+        print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS")
+        utils_thirdparty.add_fetch_or_update_about_and_license_files(
             dest_dir=thirdparty_dir,
+            strip_classifiers=strip_classifiers,
         )
-        package_envts_not_built, _wheel_filenames_built = results
-
-    print("***ADD*** ABOUT AND LICENSES")
-    utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir)
-
-    # report issues
-    for name, version in src_name_ver_not_fetched:
-        print(f"{name}=={version}: Failed to fetch source distribution.")
-
-    for package, envt in package_envts_not_built:
-        print(
-            f"{package.name}=={package.version}: Failed to build wheel "
-            f"on {envt.operating_system} for Python {envt.python_version}"
+    else:
+        print("***FETCH*** MISSING WHEELS")
+        package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir)
+        print("***FETCH*** MISSING SOURCES")
+        src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
+
+        package_envts_not_built = []
+        if build_wheels:
+            print("***BUILD*** MISSING WHEELS")
+            results = utils_thirdparty.build_missing_wheels(
+                packages_and_envts=package_envts_not_fetched,
+                build_remotely=build_remotely,
+                remote_build_log_file=remote_build_log_file,
+                dest_dir=thirdparty_dir,
+            )
+            package_envts_not_built, _wheel_filenames_built = results
+
+        print("***ADD*** ABOUT AND LICENSES")
+        utils_thirdparty.add_fetch_or_update_about_and_license_files(
+            dest_dir=thirdparty_dir,
+            strip_classifiers=strip_classifiers,
         )
 
+        # report issues
+        for name, version in src_name_ver_not_fetched:
+            print(f"{name}=={version}: Failed to fetch source distribution.")
+
+        for package, envt in package_envts_not_built:
+            print(
+                f"{package.name}=={package.version}: Failed to build wheel "
+                f"on {envt.operating_system} for Python {envt.python_version}"
+            )
+
     print("***FIND PROBLEMS***")
     utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
 

etc/scripts/utils_requirements.py

@@ -8,11 +8,13 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import re
 import subprocess
 
 """
 Utilities to manage requirements files and call pip.
-NOTE: this should use ONLY the standard library and not import anything else.
+NOTE: this should use ONLY the standard library and not import anything else
+becasue this is used for boostrapping.
 """
 
 
@@ -27,28 +29,69 @@ def load_requirements(requirements_file="requirements.txt", force_pinned=True):
     return get_required_name_versions(req_lines, force_pinned)
 
 
-def get_required_name_versions(requirement_lines, force_pinned=True):
+def get_required_name_versions(
+    requirement_lines,
+    force_pinned=True,
+):
     """
     Yield required (name, version) tuples given a`requirement_lines` iterable of
     requirement text lines. Every requirement versions must be pinned if
     `force_pinned` is True. Otherwise un-pinned requirements are returned with a
-    None version
+    None version.
+
     """
     for req_line in requirement_lines:
         req_line = req_line.strip()
         if not req_line or req_line.startswith("#"):
             continue
-        if "==" not in req_line and force_pinned:
-            raise Exception(f"Requirement version is not pinned: {req_line}")
+        if force_pinned:
+            if "==" not in req_line:
+                raise Exception(f"Requirement version is not pinned: {req_line}")
             name = req_line
             version = None
         else:
-            name, _, version = req_line.partition("==")
-            name = name.lower().strip()
-            version = version.lower().strip()
+            if req_line.startswith("-"):
+                print(f"Requirement skipped, is not supported: {req_line}")
+
+            if "==" in req_line:
+                name, _, version = req_line.partition("==")
+                version = version.lower().strip()
+            else:
+                # FIXME: we do not support unpinned requirements yet!
+                name = strip_reqs(req_line)
+                version = None
+
+        name = name.lower().strip()
         yield name, version
 
 
+def strip_reqs(line):
+    """
+    Return a name given a pip reuirement text ``line` striping version and
+    requirements.
+
+    For example::
+
+    >>> s = strip_reqs("foo <=12, >=13,!=12.6")
+    >>> assert s == "foo"
+    """
+    if "--" in line:
+        raise Exception(f"Unsupported requirement style: {line}")
+
+    line = line.strip()
+
+    ops = "><!=~;, []"
+
+    def has_ops(l):
+        return any(op in l for op in ops)
+
+    if not has_ops:
+        return  line
+
+    splitter = re.compile(r"[><!=~;, \[\]]+").split
+    return splitter(line)[0]
+
+
 def parse_requires(requires):
     """
     Return a list of requirement lines extracted from the `requires` text from

etc/scripts/utils_thirdparty.py

@@ -2528,7 +2528,11 @@ def hash_requirements(dest_dir=THIRDPARTY_DIR, requirements_file="requirements.t
 ################################################################################
 
 
-def add_fetch_or_update_about_and_license_files(dest_dir=THIRDPARTY_DIR, include_remote=True):
+def add_fetch_or_update_about_and_license_files(
+    dest_dir=THIRDPARTY_DIR, 
+    include_remote=True,
+    strip_classifiers=False,
+):
     """
     Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using
     best efforts:
@@ -2560,6 +2564,10 @@ def get_other_dists(_package, _dist):
             local_dist.load_about_data(dest_dir=dest_dir)
             local_dist.set_checksums(dest_dir=dest_dir)
 
+            if strip_classifiers and "classifiers" in local_dist.extra_data:
+                local_dist.extra_data.pop("classifiers", None)
+                local_dist.save_about_and_notice_files(dest_dir)
+
             # if has key data we may look to improve later, but we can move on
             if local_dist.has_key_metadata():
                 local_dist.save_about_and_notice_files(dest_dir=dest_dir)