Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement RFC 3553 to add SBOM support #13709

Open
wants to merge 23 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 11 additions & 0 deletions crates/cargo-test-support/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,17 @@ impl Project {
.join(paths::get_lib_filename(name, kind))
}

/// Path to a dynamic library.
/// `kind` should be one of: "lib", "rlib", "staticlib", "dylib", "proc-macro"
/// ex: `/path/to/cargo/target/cit/t0/foo/target/debug/examples/libex.dylib`
pub fn dylib(&self, name: &str) -> PathBuf {
self.target_debug_dir().join(format!(
"{}{name}{}",
env::consts::DLL_PREFIX,
env::consts::DLL_SUFFIX
))
}

/// Path to a debug binary.
/// ex: `/path/to/cargo/target/cit/t0/foo/target/debug/foo`
pub fn bin(&self, b: &str) -> PathBuf {
Expand Down
14 changes: 14 additions & 0 deletions src/cargo/core/compiler/build_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ pub struct BuildConfig {
pub future_incompat_report: bool,
/// Which kinds of build timings to output (empty if none).
pub timing_outputs: Vec<TimingOutput>,
/// Output SBOM precursor files.
pub sbom: bool,
}

fn default_parallelism() -> CargoResult<u32> {
Expand Down Expand Up @@ -102,6 +104,17 @@ impl BuildConfig {
anyhow::bail!("-Zbuild-std requires --target");
}

// If sbom flag is set, it requires the unstable feature
let sbom = match (cfg.sbom, gctx.cli_unstable().sbom) {
(Some(sbom), true) => sbom,
(Some(_), false) => {
gctx.shell()
.warn("ignoring 'sbom' config, pass `-Zsbom` to enable it")?;
false
}
(None, _) => false,
};

Ok(BuildConfig {
requested_kinds,
jobs,
Expand All @@ -117,6 +130,7 @@ impl BuildConfig {
export_dir: None,
future_incompat_report: false,
timing_outputs: Vec::new(),
sbom,
})
}

Expand Down
27 changes: 27 additions & 0 deletions src/cargo/core/compiler/build_runner/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,10 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
}

super::output_depinfo(&mut self, unit)?;

if self.bcx.build_config.sbom {
super::output_sbom(&mut self, unit)?;
}
}

for (script_meta, output) in self.build_script_outputs.lock().unwrap().iter() {
Expand Down Expand Up @@ -419,6 +423,29 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
self.files().metadata(unit)
}

/// Returns the list of SBOM output file paths for a given [`Unit`].
///
/// Only call this function when `sbom` is active.
pub fn sbom_output_files(&self, unit: &Unit) -> CargoResult<Vec<PathBuf>> {
weihanglo marked this conversation as resolved.
Show resolved Hide resolved
const SBOM_FILE_EXTENSION: &str = ".cargo-sbom.json";

fn append_sbom_suffix(link: &PathBuf, suffix: &str) -> PathBuf {
let mut link_buf = link.clone().into_os_string();
link_buf.push(suffix);
PathBuf::from(link_buf)
}

assert!(self.bcx.build_config.sbom);
let files = self
.outputs(unit)?
.iter()
.filter(|o| matches!(o.flavor, FileFlavor::Normal | FileFlavor::Linkable))
.filter_map(|output_file| output_file.hardlink.as_ref())
.map(|link| append_sbom_suffix(link, SBOM_FILE_EXTENSION))
.collect::<Vec<_>>();
Ok(files)
}

pub fn is_primary_package(&self, unit: &Unit) -> bool {
self.primary_packages.contains(&unit.pkg.package_id())
}
Expand Down
10 changes: 9 additions & 1 deletion src/cargo/core/compiler/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ pub(crate) mod layout;
mod links;
mod lto;
mod output_depinfo;
mod output_sbom;
pub mod rustdoc;
pub mod standard_lib;
mod timings;
Expand Down Expand Up @@ -84,6 +85,7 @@ use self::job_queue::{Job, JobQueue, JobState, Work};
pub(crate) use self::layout::Layout;
pub use self::lto::Lto;
use self::output_depinfo::output_depinfo;
use self::output_sbom::output_sbom;
use self::unit_graph::UnitDep;
use crate::core::compiler::future_incompat::FutureIncompatReport;
pub use crate::core::compiler::unit::{Unit, UnitInterner};
Expand Down Expand Up @@ -672,6 +674,7 @@ where
/// completion of other units will be added later in runtime, such as flags
/// from build scripts.
fn prepare_rustc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<ProcessBuilder> {
let gctx = build_runner.bcx.gctx;
let is_primary = build_runner.is_primary_package(unit);
let is_workspace = build_runner.bcx.ws.is_member(&unit.pkg);

Expand All @@ -684,12 +687,17 @@ fn prepare_rustc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResult
build_deps_args(&mut base, build_runner, unit)?;
add_cap_lints(build_runner.bcx, unit, &mut base);
base.args(build_runner.bcx.rustflags_args(unit));
if build_runner.bcx.gctx.cli_unstable().binary_dep_depinfo {
if gctx.cli_unstable().binary_dep_depinfo {
base.arg("-Z").arg("binary-dep-depinfo");
}

if is_primary {
base.env("CARGO_PRIMARY_PACKAGE", "1");

if gctx.cli_unstable().sbom && build_runner.bcx.build_config.sbom {
justahero marked this conversation as resolved.
Show resolved Hide resolved
let file_list = std::env::join_paths(build_runner.sbom_output_files(unit)?)?;
base.env("CARGO_SBOM_PATH", file_list);
}
}

if unit.target.is_test() || unit.target.is_bench() {
Expand Down
260 changes: 260 additions & 0 deletions src/cargo/core/compiler/output_sbom.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,260 @@
//! cargo-sbom precursor files for external tools to create SBOM files from.
//! See [`output_sbom`] for more.

use std::collections::BTreeSet;
use std::io::BufWriter;
use std::path::PathBuf;

use cargo_util::paths::{self};
use cargo_util_schemas::core::PackageIdSpec;
use itertools::Itertools;
use semver::Version;
use serde::Serialize;

use crate::{
core::{profiles::Profile, Target, TargetKind},
util::Rustc,
CargoResult,
};

use super::{unit_graph::UnitDep, BuildRunner, CrateType, Unit};

#[derive(Serialize, Clone, Debug, Copy)]
#[serde(rename_all = "kebab-case")]
enum SbomBuildType {
/// A package dependency
Normal,
/// A build script dependency
Build,
}

/// Typed version of a SBOM format version number.
pub struct SbomFormatVersion<const V: u32>;

impl<const V: u32> Serialize for SbomFormatVersion<V> {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_u32(V)
}
}

/// A package dependency
#[derive(Serialize, Clone, Debug)]
struct SbomDependency {
name: String,
package_id: PackageIdSpec,
version: Option<Version>,
features: Vec<String>,
}

impl From<&UnitDep> for SbomDependency {
fn from(dep: &UnitDep) -> Self {
let package_id = dep.unit.pkg.package_id().to_spec();
let name = package_id.name().to_string();
let version = package_id.version();
let features = dep
.unit
.features
.iter()
.map(|f| f.to_string())
.collect_vec();

Self {
name,
package_id,
version,
features,
}
}
}

#[derive(Serialize, Clone, Debug)]
struct SbomPackage {
package_id: PackageIdSpec,
package: String,
/// A profile can be overriden for individual packages
///
/// See <https://doc.rust-lang.org/nightly/cargo/reference/profiles.html#overrides>
profile: Option<Profile>,
version: Option<Version>,
features: Vec<String>,
build_type: SbomBuildType,
extern_crate_name: String,
dependencies: Vec<SbomDependency>,
}

impl SbomPackage {
pub fn new(
dep: &UnitDep,
dependencies: Vec<SbomDependency>,
build_type: SbomBuildType,
root_profile: &Profile,
) -> Self {
let package_id = dep.unit.pkg.package_id().to_spec();
let package = package_id.name().to_string();
let profile = if &dep.unit.profile != root_profile {
Some(dep.unit.profile.clone())
} else {
None
};
let version = package_id.version();
let features = dep
.unit
.features
.iter()
.map(|f| f.to_string())
.collect_vec();

Self {
package_id,
package,
profile,
version,
features,
build_type,
extern_crate_name: dep.extern_crate_name.to_string(),
dependencies,
}
}
}

#[derive(Serialize)]
struct SbomTarget {
kind: TargetKind,
crate_types: Vec<CrateType>,
name: String,
edition: String,
}

impl From<&Target> for SbomTarget {
fn from(target: &Target) -> Self {
SbomTarget {
kind: target.kind().clone(),
crate_types: target.kind().rustc_crate_types().clone(),
name: target.name().to_string(),
edition: target.edition().to_string(),
}
}
}

#[derive(Serialize, Clone)]
struct SbomRustc {
justahero marked this conversation as resolved.
Show resolved Hide resolved
version: String,
wrapper: Option<PathBuf>,
justahero marked this conversation as resolved.
Show resolved Hide resolved
workspace_wrapper: Option<PathBuf>,
commit_hash: Option<String>,
host: String,
verbose_version: String,
}

impl From<&Rustc> for SbomRustc {
fn from(rustc: &Rustc) -> Self {
Self {
version: rustc.version.to_string(),
wrapper: rustc.wrapper.clone(),
workspace_wrapper: rustc.workspace_wrapper.clone(),
commit_hash: rustc.commit_hash.clone(),
host: rustc.host.to_string(),
verbose_version: rustc.verbose_version.clone(),
}
}
}

#[derive(Serialize)]
struct Sbom {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need a dependencies field for this top-level Sbom?

(Just a question. I don't really know if other SBOM formats need it to recover the dependency graph)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, yes. Copying my comment from the RFC:

Note that there are two ways of looking at dependencies: what each package needs, and the final resolved graph.

For example, if one package depends on rand with features = ["std", "getrandom"], and another with features = ["std", "simd_support"], the final resolved features will be ["std", "getrandom", "simd_support"]. Depending on the use case you may need either or both representations (direct package dependencies and the resolved graph).

cargo metadata exposes both (under "packages" and "resolve" fields), but inaccurately:

I think it would be best for the SBOM to also expose both, accurately this time.

So what I would like to see is two resolved dependency trees: one for normal dependencies and one for build dependencies, matching the way feature resolver v2 works.

format_version: SbomFormatVersion<1>,
package_id: PackageIdSpec,
name: String,
version: String,
source: String,
target: SbomTarget,
profile: Profile,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bad news. People can override profiles for individual packages. So, only a top-level profile might not be enough to represent the build.

(The truth is, I am not an SBOM expert, so just provide information for you to consider)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great point. This makes it seem like the profile needs to be captured for each package.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it sufficient to only include a package's profile when it differs from the root level one?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. It sounds like a good idea to me. This behavior just needs to be documented.

packages: Vec<SbomPackage>,
features: Vec<String>,
rustc: SbomRustc,
}

impl Sbom {
pub fn new(unit: &Unit, packages: Vec<SbomPackage>, rustc: SbomRustc) -> Self {
let package_id = unit.pkg.summary().package_id().to_spec();
let name = unit.pkg.name().to_string();
let version = unit.pkg.version().to_string();
let source = unit.pkg.package_id().source_id().to_string();
let target = (&unit.target).into();
let profile = unit.profile.clone();
let features = unit.features.iter().map(|f| f.to_string()).collect();

Self {
format_version: SbomFormatVersion,
package_id,
name,
version,
source,
target,
profile,
packages,
features,
rustc,
}
}
}

/// Saves a `<artifact>.cargo-sbom.json` file for the given [`Unit`].
///
pub fn output_sbom(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<()> {
let bcx = build_runner.bcx;
let rustc: SbomRustc = bcx.rustc().into();

let packages = collect_packages(build_runner, unit);

for sbom_output_file in build_runner.sbom_output_files(unit)? {
let sbom = Sbom::new(unit, packages.clone(), rustc.clone());

let outfile = BufWriter::new(paths::create(sbom_output_file)?);
serde_json::to_writer(outfile, &sbom)?;
}

Ok(())
}

/// Fetch all dependencies, including transitive ones. A dependency can also appear multiple times
/// if it's included with different versions.
fn collect_packages(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> Vec<SbomPackage> {
let unit_graph = &build_runner.bcx.unit_graph;
let root_deps = build_runner.unit_deps(unit);
let root_profile = &unit.profile;

let mut result = Vec::new();
let mut queue: BTreeSet<&UnitDep> = root_deps.iter().collect();
let mut visited = BTreeSet::new();

while let Some(package) = queue.pop_first() {
if visited.contains(package) {
continue;
}

let build_type = if package.unit.mode.is_run_custom_build() {
SbomBuildType::Build
} else {
SbomBuildType::Normal
};
Comment on lines +238 to +242
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you share the idea of distinguishing the build script execution?

It is a bit odd that the only "crate" dependency a build script execution has is the build script "compilation".

Let me explain it a bit: In Cargo's unit graph there are two kinds of unit for build scripts. One for build script compilation, the other for execution. See this doc for more: https://doc.rust-lang.org/nightly/nightly-rustc/cargo/core/compiler/custom_build/index.html.

Yeah we got another similar question: what do we want to track in SBOM?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just re-read the RFC and asked there rust-lang/rfcs#3553 (comment).


let mut dependencies: BTreeSet<&UnitDep> = unit_graph[&package.unit].iter().collect();
let sbom_dependencies = dependencies.iter().map(|dep| (*dep).into()).collect_vec();

result.push(SbomPackage::new(
package,
sbom_dependencies,
build_type,
root_profile,
));

visited.insert(package);

queue.append(&mut dependencies);
}

result
}