Skip to content

Commit

Permalink
Rename V2 Configuration to FieldSet (#9668)
Browse files Browse the repository at this point in the history
This name describes the ad hoc collection of Fields needed by some rules, such as the Fields needed to create an AWS Lambda.

"Configuration" is a very overloaded term and doesn't express its relationship to the Target API / Fields.

Rejected alternatives:

* Fields -> no way to make this plural.
* FieldCollection -> clunkier than FieldSet.
* AdHocTarget -> clunky and too close to `Target`.

[ci skip-rust-tests]
[ci skip-jvm-tests]
  • Loading branch information
Eric-Arellano committed May 1, 2020
1 parent 953c448 commit cec63e0
Show file tree
Hide file tree
Showing 32 changed files with 531 additions and 537 deletions.
Expand Up @@ -12,11 +12,7 @@
from pants.engine.goal import Goal, GoalSubsystem, LineOriented
from pants.engine.rules import goal_rule
from pants.engine.selectors import Get, MultiGet
from pants.engine.target import (
Configuration,
TargetsToValidConfigurations,
TargetsToValidConfigurationsRequest,
)
from pants.engine.target import FieldSet, TargetsToValidFieldSets, TargetsToValidFieldSetsRequest
from pants.engine.unions import union


Expand All @@ -33,7 +29,7 @@ class CreatedAWSLambda:


@union
class AWSLambdaConfiguration(Configuration, metaclass=ABCMeta):
class AWSLambdaFieldSet(FieldSet, metaclass=ABCMeta):
"""The fields necessary to create an AWS Lambda from a target."""


Expand All @@ -55,16 +51,16 @@ async def create_awslambda(
buildroot: BuildRoot,
workspace: Workspace,
) -> AWSLambdaGoal:
targets_to_valid_configs = await Get[TargetsToValidConfigurations](
TargetsToValidConfigurationsRequest(
AWSLambdaConfiguration,
targets_to_valid_field_sets = await Get[TargetsToValidFieldSets](
TargetsToValidFieldSetsRequest(
AWSLambdaFieldSet,
goal_description=f"the `{options.name}` goal",
error_if_no_valid_targets=True,
)
)
awslambdas = await MultiGet(
Get[CreatedAWSLambda](AWSLambdaConfiguration, config)
for config in targets_to_valid_configs.configurations
Get[CreatedAWSLambda](AWSLambdaFieldSet, field_set)
for field_set in targets_to_valid_field_sets.field_sets
)
merged_digest = await Get[Digest](MergeDigests(awslambda.digest for awslambda in awslambdas))
result = workspace.materialize_directory(
Expand Down
Expand Up @@ -4,7 +4,7 @@
from dataclasses import dataclass

from pants.backend.awslambda.common.awslambda_common_rules import (
AWSLambdaConfiguration,
AWSLambdaFieldSet,
CreatedAWSLambda,
)
from pants.backend.awslambda.python.lambdex import Lambdex
Expand Down Expand Up @@ -43,7 +43,7 @@


@dataclass(frozen=True)
class PythonAwsLambdaConfiguration(AWSLambdaConfiguration):
class PythonAwsLambdaFieldSet(AWSLambdaFieldSet):
required_fields = (PythonAwsLambdaHandler, PythonAwsLambdaRuntime)

handler: PythonAwsLambdaHandler
Expand All @@ -57,23 +57,23 @@ class LambdexSetup:

@named_rule(desc="Create Python AWS Lambda")
async def create_python_awslambda(
config: PythonAwsLambdaConfiguration,
field_set: PythonAwsLambdaFieldSet,
lambdex_setup: LambdexSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> CreatedAWSLambda:
# Lambdas typically use the .zip suffix, so we use that instead of .pex.
pex_filename = f"{config.address.target_name}.zip"
pex_filename = f"{field_set.address.target_name}.zip"
# We hardcode the platform value to the appropriate one for each AWS Lambda runtime.
# (Running the "hello world" lambda in the example code will report the platform, and can be
# used to verify correctness of these platform strings.)
py_major, py_minor = config.runtime.to_interpreter_version()
py_major, py_minor = field_set.runtime.to_interpreter_version()
platform = f"manylinux2014_x86_64-cp-{py_major}{py_minor}-cp{py_major}{py_minor}m"
if (py_major, py_minor) == (2, 7):
platform += "u"
pex_request = TwoStepPexFromTargetsRequest(
PexFromTargetsRequest(
addresses=Addresses([config.address]),
addresses=Addresses([field_set.address]),
entry_point=None,
output_filename=pex_filename,
platforms=PexPlatforms([platform]),
Expand All @@ -86,7 +86,7 @@ async def create_python_awslambda(
)

# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
lambdex_args = ("build", "-e", config.handler.value, pex_filename)
lambdex_args = ("build", "-e", field_set.handler.value, pex_filename)
process = lambdex_setup.requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
Expand All @@ -102,7 +102,7 @@ async def create_python_awslambda(
return CreatedAWSLambda(
digest=result.output_digest,
name=pex_filename,
runtime=config.runtime.value,
runtime=field_set.runtime.value,
handler="lambdex_handler.handler",
)

Expand All @@ -126,7 +126,7 @@ def rules():
return [
create_python_awslambda,
setup_lambdex,
UnionRule(AWSLambdaConfiguration, PythonAwsLambdaConfiguration),
UnionRule(AWSLambdaFieldSet, PythonAwsLambdaFieldSet),
SubsystemRule(Lambdex),
*download_pex_bin.rules(),
*importable_python_sources.rules(),
Expand Down
Expand Up @@ -7,7 +7,7 @@
from zipfile import ZipFile

from pants.backend.awslambda.common.awslambda_common_rules import CreatedAWSLambda
from pants.backend.awslambda.python.awslambda_python_rules import PythonAwsLambdaConfiguration
from pants.backend.awslambda.python.awslambda_python_rules import PythonAwsLambdaFieldSet
from pants.backend.awslambda.python.awslambda_python_rules import rules as awslambda_python_rules
from pants.backend.awslambda.python.target_types import PythonAWSLambda
from pants.backend.python.target_types import PythonLibrary
Expand All @@ -23,7 +23,7 @@
class TestPythonAWSLambdaCreation(TestBase):
@classmethod
def rules(cls):
return (*super().rules(), *awslambda_python_rules(), RootRule(PythonAwsLambdaConfiguration))
return (*super().rules(), *awslambda_python_rules(), RootRule(PythonAwsLambdaFieldSet))

@classmethod
def target_types(cls):
Expand All @@ -34,7 +34,7 @@ def create_python_awslambda(self, addr: str) -> Tuple[str, bytes]:
created_awslambda = self.request_single_product(
CreatedAWSLambda,
Params(
PythonAwsLambdaConfiguration.create(target),
PythonAwsLambdaFieldSet.create(target),
create_options_bootstrapper(
args=["--backend-packages2=pants.backend.awslambda.python"]
),
Expand Down
26 changes: 15 additions & 11 deletions src/python/pants/backend/python/lint/bandit/rules.py
Expand Up @@ -15,7 +15,7 @@
from pants.backend.python.subsystems import python_native_code, subprocess_environment
from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
from pants.backend.python.target_types import PythonInterpreterCompatibility, PythonSources
from pants.core.goals.lint import LinterConfiguration, LinterConfigurations, LintResult
from pants.core.goals.lint import LinterFieldSet, LinterFieldSets, LintResult
from pants.core.util_rules import determine_source_files, strip_source_roots
from pants.core.util_rules.determine_source_files import (
AllSourceFilesRequest,
Expand All @@ -33,15 +33,15 @@


@dataclass(frozen=True)
class BanditConfiguration(LinterConfiguration):
class BanditFieldSet(LinterFieldSet):
required_fields = (PythonSources,)

sources: PythonSources
compatibility: PythonInterpreterCompatibility


class BanditConfigurations(LinterConfigurations):
config_type = BanditConfiguration
class BanditFieldSets(LinterFieldSets):
field_set_type = BanditFieldSet


def generate_args(*, specified_source_files: SourceFiles, bandit: Bandit) -> Tuple[str, ...]:
Expand All @@ -55,7 +55,7 @@ def generate_args(*, specified_source_files: SourceFiles, bandit: Bandit) -> Tup

@named_rule(desc="Lint using Bandit")
async def bandit_lint(
configs: BanditConfigurations,
field_sets: BanditFieldSets,
bandit: Bandit,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
Expand All @@ -66,7 +66,7 @@ async def bandit_lint(
# NB: Bandit output depends upon which Python interpreter version it's run with. See
# https://github.com/PyCQA/bandit#under-which-version-of-python-should-i-install-bandit.
interpreter_constraints = PexInterpreterConstraints.create_from_compatibility_fields(
(config.compatibility for config in configs), python_setup=python_setup
(field_set.compatibility for field_set in field_sets), python_setup=python_setup
)
requirements_pex = await Get[Pex](
PexRequest(
Expand All @@ -87,10 +87,12 @@ async def bandit_lint(
)

all_source_files = await Get[SourceFiles](
AllSourceFilesRequest(config.sources for config in configs)
AllSourceFilesRequest(field_set.sources for field_set in field_sets)
)
specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest((config.sources, config.origin) for config in configs)
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in field_sets
)
)

merged_input_files = await Get[Digest](
Expand All @@ -99,15 +101,17 @@ async def bandit_lint(
),
)

address_references = ", ".join(sorted(config.address.reference() for config in configs))
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in field_sets)
)

process = requirements_pex.create_process(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./bandit.pex",
pex_args=generate_args(specified_source_files=specified_source_files, bandit=bandit),
input_files=merged_input_files,
description=f"Run Bandit on {pluralize(len(configs), 'target')}: {address_references}.",
description=f"Run Bandit on {pluralize(len(field_sets), 'target')}: {address_references}.",
)
result = await Get[FallibleProcessResult](Process, process)
return LintResult.from_fallible_process_result(result)
Expand All @@ -117,7 +121,7 @@ def rules():
return [
bandit_lint,
SubsystemRule(Bandit),
UnionRule(LinterConfigurations, BanditConfigurations),
UnionRule(LinterFieldSets, BanditFieldSets),
*download_pex_bin.rules(),
*determine_source_files.rules(),
*pex.rules(),
Expand Down
Expand Up @@ -3,7 +3,7 @@

from typing import List, Optional

from pants.backend.python.lint.bandit.rules import BanditConfiguration, BanditConfigurations
from pants.backend.python.lint.bandit.rules import BanditFieldSet, BanditFieldSets
from pants.backend.python.lint.bandit.rules import rules as bandit_rules
from pants.backend.python.target_types import PythonInterpreterCompatibility, PythonLibrary
from pants.base.specs import FilesystemLiteralSpec, OriginSpec, SingleAddress
Expand All @@ -25,7 +25,7 @@ class BanditIntegrationTest(TestBase):

@classmethod
def rules(cls):
return (*super().rules(), *bandit_rules(), RootRule(BanditConfigurations))
return (*super().rules(), *bandit_rules(), RootRule(BanditFieldSets))

def make_target_with_origin(
self,
Expand Down Expand Up @@ -63,7 +63,7 @@ def run_bandit(
return self.request_single_product(
LintResult,
Params(
BanditConfigurations(BanditConfiguration.create(tgt) for tgt in targets),
BanditFieldSets(BanditFieldSet.create(tgt) for tgt in targets),
create_options_bootstrapper(args=args),
),
)
Expand Down
42 changes: 23 additions & 19 deletions src/python/pants/backend/python/lint/black/rules.py
Expand Up @@ -7,7 +7,7 @@
from typing import Optional, Tuple

from pants.backend.python.lint.black.subsystem import Black
from pants.backend.python.lint.python_fmt import PythonFmtConfigurations
from pants.backend.python.lint.python_fmt import PythonFmtFieldSets
from pants.backend.python.rules import download_pex_bin, pex
from pants.backend.python.rules.pex import (
Pex,
Expand All @@ -18,8 +18,8 @@
from pants.backend.python.subsystems import python_native_code, subprocess_environment
from pants.backend.python.subsystems.subprocess_environment import SubprocessEncodingEnvironment
from pants.backend.python.target_types import PythonSources
from pants.core.goals.fmt import FmtConfiguration, FmtConfigurations, FmtResult
from pants.core.goals.lint import LinterConfigurations, LintResult
from pants.core.goals.fmt import FmtFieldSet, FmtFieldSets, FmtResult
from pants.core.goals.lint import LinterFieldSets, LintResult
from pants.core.util_rules import determine_source_files, strip_source_roots
from pants.core.util_rules.determine_source_files import (
AllSourceFilesRequest,
Expand All @@ -37,19 +37,19 @@


@dataclass(frozen=True)
class BlackConfiguration(FmtConfiguration):
class BlackFieldSet(FmtFieldSet):
required_fields = (PythonSources,)

sources: PythonSources


class BlackConfigurations(FmtConfigurations):
config_type = BlackConfiguration
class BlackFieldSets(FmtFieldSets):
field_set_type = BlackFieldSet


@dataclass(frozen=True)
class SetupRequest:
configs: BlackConfigurations
field_sets: BlackFieldSets
check_only: bool


Expand Down Expand Up @@ -106,16 +106,18 @@ async def setup(
)
)

if request.configs.prior_formatter_result is None:
if request.field_sets.prior_formatter_result is None:
all_source_files = await Get[SourceFiles](
AllSourceFilesRequest(config.sources for config in request.configs)
AllSourceFilesRequest(field_set.sources for field_set in request.field_sets)
)
all_source_files_snapshot = all_source_files.snapshot
else:
all_source_files_snapshot = request.configs.prior_formatter_result
all_source_files_snapshot = request.field_sets.prior_formatter_result

specified_source_files = await Get[SourceFiles](
SpecifiedSourceFilesRequest((config.sources, config.origin) for config in request.configs)
SpecifiedSourceFilesRequest(
(field_set.sources, field_set.origin) for field_set in request.field_sets
)
)

merged_input_files = await Get[Digest](
Expand All @@ -124,7 +126,9 @@ async def setup(
),
)

address_references = ", ".join(sorted(config.address.reference() for config in request.configs))
address_references = ", ".join(
sorted(field_set.address.reference() for field_set in request.field_sets)
)

process = requirements_pex.create_process(
python_setup=python_setup,
Expand All @@ -138,26 +142,26 @@ async def setup(
input_files=merged_input_files,
output_files=all_source_files_snapshot.files,
description=(
f"Run Black on {pluralize(len(request.configs), 'target')}: {address_references}."
f"Run Black on {pluralize(len(request.field_sets), 'target')}: {address_references}."
),
)
return Setup(process, original_digest=all_source_files_snapshot.digest)


@named_rule(desc="Format using Black")
async def black_fmt(configs: BlackConfigurations, black: Black) -> FmtResult:
async def black_fmt(field_sets: BlackFieldSets, black: Black) -> FmtResult:
if black.options.skip:
return FmtResult.noop()
setup = await Get[Setup](SetupRequest(configs, check_only=False))
setup = await Get[Setup](SetupRequest(field_sets, check_only=False))
result = await Get[ProcessResult](Process, setup.process)
return FmtResult.from_process_result(result, original_digest=setup.original_digest)


@named_rule(desc="Lint using Black")
async def black_lint(configs: BlackConfigurations, black: Black) -> LintResult:
async def black_lint(field_sets: BlackFieldSets, black: Black) -> LintResult:
if black.options.skip:
return LintResult.noop()
setup = await Get[Setup](SetupRequest(configs, check_only=True))
setup = await Get[Setup](SetupRequest(field_sets, check_only=True))
result = await Get[FallibleProcessResult](Process, setup.process)
return LintResult.from_fallible_process_result(result)

Expand All @@ -168,8 +172,8 @@ def rules():
black_fmt,
black_lint,
SubsystemRule(Black),
UnionRule(PythonFmtConfigurations, BlackConfigurations),
UnionRule(LinterConfigurations, BlackConfigurations),
UnionRule(PythonFmtFieldSets, BlackFieldSets),
UnionRule(LinterFieldSets, BlackFieldSets),
*download_pex_bin.rules(),
*determine_source_files.rules(),
*pex.rules(),
Expand Down

0 comments on commit cec63e0

Please sign in to comment.