From f299395e993f7267e00bc5893fc3793501acbc28 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Thu, 31 Oct 2024 12:49:18 -0500 Subject: [PATCH 1/4] enable cargo-component lock files --- crates/wasm-pkg-core/src/lock.rs | 20 +++- crates/wasm-pkg-core/src/resolver.rs | 136 ++++++++++++++++++++++++--- crates/wasm-pkg-core/src/wit.rs | 9 +- 3 files changed, 149 insertions(+), 16 deletions(-) diff --git a/crates/wasm-pkg-core/src/lock.rs b/crates/wasm-pkg-core/src/lock.rs index 0ed59d5..bb05312 100644 --- a/crates/wasm-pkg-core/src/lock.rs +++ b/crates/wasm-pkg-core/src/lock.rs @@ -16,7 +16,7 @@ use tokio::{ }; use wasm_pkg_client::{ContentDigest, PackageRef}; -use crate::resolver::{DependencyResolution, DependencyResolutionMap}; +use crate::resolver::{DependencyResolution, DependencyResolutionMap, DEFAULT_REGISTRY_NAME}; /// The default name of the lock file. pub const LOCK_FILE_NAME: &str = "wkg.lock"; @@ -271,6 +271,17 @@ pub struct LockedPackage { pub versions: Vec, } +impl LockedPackage { + /// Gets the key used in sorting and searching the package list. + pub fn key(&self) -> (&str, &str, &str) { + ( + self.name.namespace().as_ref(), + self.name.name().as_ref(), + self.registry.as_deref().unwrap_or(DEFAULT_REGISTRY_NAME), + ) + } +} + impl Ord for LockedPackage { fn cmp(&self, other: &Self) -> Ordering { if self.name == other.name { @@ -298,6 +309,13 @@ pub struct LockedPackageVersion { pub digest: ContentDigest, } +impl LockedPackageVersion { + /// Gets the sort key for the locked package version. + pub fn key(&self) -> String { + self.requirement.to_string() + } +} + #[derive(Debug, Deserialize)] struct LockFileIntermediate { version: u64, diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 7704ef3..e91bfcf 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -13,6 +13,10 @@ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use semver::{Comparator, Op, Version, VersionReq}; +use serde::{ + de::{self, value::MapAccessDeserializer}, + Deserialize, Serialize, +}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, @@ -38,6 +42,108 @@ pub enum Dependency { Local(PathBuf), } +impl Serialize for Dependency { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + match self { + Self::Package(package) => { + if package.name.is_none() && package.registry.is_none() { + let version = package.version.to_string(); + version.trim_start_matches('^').serialize(serializer) + } else { + #[derive(Serialize)] + struct Entry<'a> { + package: Option<&'a PackageRef>, + version: &'a str, + registry: Option<&'a str>, + } + + Entry { + package: package.name.as_ref(), + version: package.version.to_string().trim_start_matches('^'), + registry: package.registry.as_deref(), + } + .serialize(serializer) + } + } + Self::Local(path) => { + #[derive(Serialize)] + struct Entry<'a> { + path: &'a PathBuf, + } + + Entry { path }.serialize(serializer) + } + } + } +} + +impl<'de> Deserialize<'de> for Dependency { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + + impl<'de> de::Visitor<'de> for Visitor { + type Value = Dependency; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a string or a table") + } + + fn visit_str(self, s: &str) -> Result + where + E: de::Error, + { + Ok(Self::Value::Package(s.parse().map_err(de::Error::custom)?)) + } + + fn visit_map(self, map: A) -> Result + where + A: de::MapAccess<'de>, + { + #[derive(Default, Deserialize)] + #[serde(default, deny_unknown_fields)] + struct Entry { + path: Option, + package: Option, + version: Option, + registry: Option, + } + + let entry = Entry::deserialize(MapAccessDeserializer::new(map))?; + + match (entry.path, entry.package, entry.version, entry.registry) { + (Some(path), None, None, None) => Ok(Self::Value::Local(path)), + (None, name, Some(version), registry) => { + Ok(Self::Value::Package(RegistryPackage { + name, + version, + registry, + })) + } + (Some(_), None, Some(_), _) => Err(de::Error::custom( + "cannot specify both `path` and `version` fields in a dependency entry", + )), + (Some(_), None, None, Some(_)) => Err(de::Error::custom( + "cannot specify both `path` and `registry` fields in a dependency entry", + )), + (Some(_), Some(_), _, _) => Err(de::Error::custom( + "cannot specify both `path` and `package` fields in a dependency entry", + )), + (None, None, _, _) => Err(de::Error::missing_field("package")), + (None, Some(_), None, _) => Err(de::Error::missing_field("version")), + } + } + } + + deserializer.deserialize_any(Visitor) + } +} + impl FromStr for Dependency { type Err = anyhow::Error; @@ -363,8 +469,10 @@ impl<'a> DependencyResolver<'a> { &mut self, name: &PackageRef, dependency: &Dependency, + is_wit: bool, ) -> Result<()> { - self.add_dependency_internal(name, dependency, false).await + self.add_dependency_internal(name, dependency, false, is_wit) + .await } /// Add a dependency to the resolver. If the dependency already exists, then it will be @@ -373,8 +481,10 @@ impl<'a> DependencyResolver<'a> { &mut self, name: &PackageRef, dependency: &Dependency, + is_wit: bool, ) -> Result<()> { - self.add_dependency_internal(name, dependency, true).await + self.add_dependency_internal(name, dependency, true, is_wit) + .await } async fn add_dependency_internal( @@ -382,6 +492,7 @@ impl<'a> DependencyResolver<'a> { name: &PackageRef, dependency: &Dependency, force_override: bool, + is_wit: bool, ) -> Result<()> { match dependency { Dependency::Package(package) => { @@ -434,13 +545,15 @@ impl<'a> DependencyResolver<'a> { return Ok(()); } - // Now that we check we haven't already inserted this dep, get the packages from the - // local dependency and add those to the resolver before adding the dependency - let (_, packages) = get_packages(p) - .context("Error getting dependent packages from local dependency")?; - Box::pin(self.add_packages(packages)) - .await - .context("Error adding packages to resolver for local dependency")?; + // // Now that we check we haven't already inserted this dep, get the packages from the + // // local dependency and add those to the resolver before adding the dependency + if is_wit { + let (_, packages) = get_packages(p) + .context("Error getting dependent packages from local dependency")?; + Box::pin(self.add_packages(packages)) + .await + .context("Error adding packages to resolver for local dependency")?; + } let prev = self.resolutions.insert(name.clone(), res); assert!(prev.is_none()); @@ -454,9 +567,9 @@ impl<'a> DependencyResolver<'a> { /// requirements to the resolver pub async fn add_packages( &mut self, - packages: impl IntoIterator, + packages: impl IntoIterator, ) -> Result<()> { - for (package, req) in packages { + for (package, req, is_wit) in packages { self.add_dependency( &package, &Dependency::Package(RegistryPackage { @@ -464,6 +577,7 @@ impl<'a> DependencyResolver<'a> { version: req, registry: None, }), + is_wit, ) .await?; } diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 059cef1..084091e 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -117,7 +117,7 @@ pub async fn fetch_dependencies( /// for resolving dependencies. pub fn get_packages( path: impl AsRef, -) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { +) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq, bool)>)> { let group = wit_parser::UnresolvedPackageGroup::parse_path(path).context("Couldn't parse package")?; @@ -137,7 +137,7 @@ pub fn get_packages( ); // Get all package refs from the main package and then from any nested packages - let packages: HashSet<(PackageRef, VersionReq)> = + let packages: HashSet<(PackageRef, VersionReq, bool)> = packages_from_foreign_deps(group.main.foreign_deps.into_keys()) .chain( group @@ -186,7 +186,7 @@ pub async fn resolve_dependencies( } }; resolver - .add_dependency(&pkg, &dep) + .add_dependency(&pkg, &dep, true) .await .context("Unable to add dependency")?; } @@ -289,7 +289,7 @@ pub async fn populate_dependencies( fn packages_from_foreign_deps( deps: impl IntoIterator, -) -> impl Iterator { +) -> impl Iterator { deps.into_iter().filter_map(|dep| { let name = PackageRef::new(dep.namespace.parse().ok()?, dep.name.parse().ok()?); let version = match dep.version { @@ -301,6 +301,7 @@ fn packages_from_foreign_deps( version .parse() .expect("Unable to parse into version request, this is programmer error"), + true, )) }) } From 9e66cb2771b793447d7984bf51f3b874e8317241 Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Thu, 31 Oct 2024 13:24:42 -0500 Subject: [PATCH 2/4] dependency kind enables component deps --- crates/wasm-pkg-core/src/resolver.rs | 61 +++++++++++++++++----------- crates/wasm-pkg-core/src/wit.rs | 17 ++++---- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index e91bfcf..a12e89d 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -30,6 +30,17 @@ use crate::{lock::LockFile, wit::get_packages}; /// The name of the default registry. pub const DEFAULT_REGISTRY_NAME: &str = "default"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum DependencyKind { + Component, + Wit, +} + +impl Default for DependencyKind { + fn default() -> Self { + Self::Wit + } +} // TODO: functions for resolving dependencies from a lock file /// Represents a WIT package dependency. @@ -39,7 +50,7 @@ pub enum Dependency { Package(RegistryPackage), /// The dependency is a path to a local directory or file. - Local(PathBuf), + Local(PathBuf, DependencyKind), } impl Serialize for Dependency { @@ -68,13 +79,14 @@ impl Serialize for Dependency { .serialize(serializer) } } - Self::Local(path) => { + Self::Local(path, kind) => { #[derive(Serialize)] struct Entry<'a> { path: &'a PathBuf, + kind: &'a DependencyKind, } - Entry { path }.serialize(serializer) + Entry { path, kind }.serialize(serializer) } } } @@ -112,30 +124,37 @@ impl<'de> Deserialize<'de> for Dependency { package: Option, version: Option, registry: Option, + kind: DependencyKind, } let entry = Entry::deserialize(MapAccessDeserializer::new(map))?; - match (entry.path, entry.package, entry.version, entry.registry) { - (Some(path), None, None, None) => Ok(Self::Value::Local(path)), - (None, name, Some(version), registry) => { + match ( + entry.path, + entry.package, + entry.version, + entry.registry, + entry.kind, + ) { + (Some(path), None, None, None, kind) => Ok(Self::Value::Local(path, kind)), + (None, name, Some(version), registry, _) => { Ok(Self::Value::Package(RegistryPackage { name, version, registry, })) } - (Some(_), None, Some(_), _) => Err(de::Error::custom( + (Some(_), None, Some(_), _, _) => Err(de::Error::custom( "cannot specify both `path` and `version` fields in a dependency entry", )), - (Some(_), None, None, Some(_)) => Err(de::Error::custom( + (Some(_), None, None, Some(_), _) => Err(de::Error::custom( "cannot specify both `path` and `registry` fields in a dependency entry", )), - (Some(_), Some(_), _, _) => Err(de::Error::custom( + (Some(_), Some(_), _, _, _) => Err(de::Error::custom( "cannot specify both `path` and `package` fields in a dependency entry", )), - (None, None, _, _) => Err(de::Error::missing_field("package")), - (None, Some(_), None, _) => Err(de::Error::missing_field("version")), + (None, None, _, _, _) => Err(de::Error::missing_field("package")), + (None, Some(_), None, _, _) => Err(de::Error::missing_field("version")), } } } @@ -469,10 +488,8 @@ impl<'a> DependencyResolver<'a> { &mut self, name: &PackageRef, dependency: &Dependency, - is_wit: bool, ) -> Result<()> { - self.add_dependency_internal(name, dependency, false, is_wit) - .await + self.add_dependency_internal(name, dependency, false).await } /// Add a dependency to the resolver. If the dependency already exists, then it will be @@ -481,10 +498,8 @@ impl<'a> DependencyResolver<'a> { &mut self, name: &PackageRef, dependency: &Dependency, - is_wit: bool, ) -> Result<()> { - self.add_dependency_internal(name, dependency, true, is_wit) - .await + self.add_dependency_internal(name, dependency, true).await } async fn add_dependency_internal( @@ -492,7 +507,6 @@ impl<'a> DependencyResolver<'a> { name: &PackageRef, dependency: &Dependency, force_override: bool, - is_wit: bool, ) -> Result<()> { match dependency { Dependency::Package(package) => { @@ -533,7 +547,7 @@ impl<'a> DependencyResolver<'a> { }, ); } - Dependency::Local(p) => { + Dependency::Local(p, kind) => { // A local path dependency, insert a resolution immediately let res = DependencyResolution::Local(LocalResolution { name: name.clone(), @@ -547,13 +561,15 @@ impl<'a> DependencyResolver<'a> { // // Now that we check we haven't already inserted this dep, get the packages from the // // local dependency and add those to the resolver before adding the dependency - if is_wit { + // if is_wit { + if kind == &DependencyKind::Wit { let (_, packages) = get_packages(p) .context("Error getting dependent packages from local dependency")?; Box::pin(self.add_packages(packages)) .await .context("Error adding packages to resolver for local dependency")?; } + // } let prev = self.resolutions.insert(name.clone(), res); assert!(prev.is_none()); @@ -567,9 +583,9 @@ impl<'a> DependencyResolver<'a> { /// requirements to the resolver pub async fn add_packages( &mut self, - packages: impl IntoIterator, + packages: impl IntoIterator, ) -> Result<()> { - for (package, req, is_wit) in packages { + for (package, req) in packages { self.add_dependency( &package, &Dependency::Package(RegistryPackage { @@ -577,7 +593,6 @@ impl<'a> DependencyResolver<'a> { version: req, registry: None, }), - is_wit, ) .await?; } diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 084091e..80b521e 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -15,8 +15,8 @@ use crate::{ config::Config, lock::LockFile, resolver::{ - DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, - DependencyResolver, LocalResolution, RegistryPackage, + DecodedDependency, Dependency, DependencyKind, DependencyResolution, + DependencyResolutionMap, DependencyResolver, LocalResolution, RegistryPackage, }, }; @@ -117,7 +117,7 @@ pub async fn fetch_dependencies( /// for resolving dependencies. pub fn get_packages( path: impl AsRef, -) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq, bool)>)> { +) -> Result<(PackageRef, HashSet<(PackageRef, VersionReq)>)> { let group = wit_parser::UnresolvedPackageGroup::parse_path(path).context("Couldn't parse package")?; @@ -137,7 +137,7 @@ pub fn get_packages( ); // Get all package refs from the main package and then from any nested packages - let packages: HashSet<(PackageRef, VersionReq, bool)> = + let packages: HashSet<(PackageRef, VersionReq)> = packages_from_foreign_deps(group.main.foreign_deps.into_keys()) .chain( group @@ -168,12 +168,12 @@ pub async fn resolve_dependencies( let dep = match (ovride.path.as_ref(), ovride.version.as_ref()) { (Some(path), None) => { let path = tokio::fs::canonicalize(path).await?; - Dependency::Local(path) + Dependency::Local(path, DependencyKind::Wit) } (Some(path), Some(_)) => { tracing::warn!("Ignoring version override for local package"); let path = tokio::fs::canonicalize(path).await?; - Dependency::Local(path) + Dependency::Local(path, DependencyKind::Wit) } (None, Some(version)) => Dependency::Package(RegistryPackage { name: Some(pkg.clone()), @@ -186,7 +186,7 @@ pub async fn resolve_dependencies( } }; resolver - .add_dependency(&pkg, &dep, true) + .add_dependency(&pkg, &dep) .await .context("Unable to add dependency")?; } @@ -289,7 +289,7 @@ pub async fn populate_dependencies( fn packages_from_foreign_deps( deps: impl IntoIterator, -) -> impl Iterator { +) -> impl Iterator { deps.into_iter().filter_map(|dep| { let name = PackageRef::new(dep.namespace.parse().ok()?, dep.name.parse().ok()?); let version = match dep.version { @@ -301,7 +301,6 @@ fn packages_from_foreign_deps( version .parse() .expect("Unable to parse into version request, this is programmer error"), - true, )) }) } From 7d97093d0868681a34392a07524d01d051af8d8d Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Fri, 1 Nov 2024 10:41:28 -0500 Subject: [PATCH 3/4] differentiate wit and component deps and remove dep serde --- crates/wasm-pkg-core/src/resolver.rs | 290 +++++++++++---------------- crates/wasm-pkg-core/src/wit.rs | 10 +- 2 files changed, 120 insertions(+), 180 deletions(-) diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index a12e89d..4df223b 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -13,10 +13,6 @@ use anyhow::{bail, Context, Result}; use futures_util::TryStreamExt; use indexmap::{IndexMap, IndexSet}; use semver::{Comparator, Op, Version, VersionReq}; -use serde::{ - de::{self, value::MapAccessDeserializer}, - Deserialize, Serialize, -}; use tokio::io::{AsyncRead, AsyncReadExt}; use wasm_pkg_client::{ caching::{CachingClient, FileCache}, @@ -30,17 +26,6 @@ use crate::{lock::LockFile, wit::get_packages}; /// The name of the default registry. pub const DEFAULT_REGISTRY_NAME: &str = "default"; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum DependencyKind { - Component, - Wit, -} - -impl Default for DependencyKind { - fn default() -> Self { - Self::Wit - } -} // TODO: functions for resolving dependencies from a lock file /// Represents a WIT package dependency. @@ -50,117 +35,7 @@ pub enum Dependency { Package(RegistryPackage), /// The dependency is a path to a local directory or file. - Local(PathBuf, DependencyKind), -} - -impl Serialize for Dependency { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - match self { - Self::Package(package) => { - if package.name.is_none() && package.registry.is_none() { - let version = package.version.to_string(); - version.trim_start_matches('^').serialize(serializer) - } else { - #[derive(Serialize)] - struct Entry<'a> { - package: Option<&'a PackageRef>, - version: &'a str, - registry: Option<&'a str>, - } - - Entry { - package: package.name.as_ref(), - version: package.version.to_string().trim_start_matches('^'), - registry: package.registry.as_deref(), - } - .serialize(serializer) - } - } - Self::Local(path, kind) => { - #[derive(Serialize)] - struct Entry<'a> { - path: &'a PathBuf, - kind: &'a DependencyKind, - } - - Entry { path, kind }.serialize(serializer) - } - } - } -} - -impl<'de> Deserialize<'de> for Dependency { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct Visitor; - - impl<'de> de::Visitor<'de> for Visitor { - type Value = Dependency; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(formatter, "a string or a table") - } - - fn visit_str(self, s: &str) -> Result - where - E: de::Error, - { - Ok(Self::Value::Package(s.parse().map_err(de::Error::custom)?)) - } - - fn visit_map(self, map: A) -> Result - where - A: de::MapAccess<'de>, - { - #[derive(Default, Deserialize)] - #[serde(default, deny_unknown_fields)] - struct Entry { - path: Option, - package: Option, - version: Option, - registry: Option, - kind: DependencyKind, - } - - let entry = Entry::deserialize(MapAccessDeserializer::new(map))?; - - match ( - entry.path, - entry.package, - entry.version, - entry.registry, - entry.kind, - ) { - (Some(path), None, None, None, kind) => Ok(Self::Value::Local(path, kind)), - (None, name, Some(version), registry, _) => { - Ok(Self::Value::Package(RegistryPackage { - name, - version, - registry, - })) - } - (Some(_), None, Some(_), _, _) => Err(de::Error::custom( - "cannot specify both `path` and `version` fields in a dependency entry", - )), - (Some(_), None, None, Some(_), _) => Err(de::Error::custom( - "cannot specify both `path` and `registry` fields in a dependency entry", - )), - (Some(_), Some(_), _, _, _) => Err(de::Error::custom( - "cannot specify both `path` and `package` fields in a dependency entry", - )), - (None, None, _, _, _) => Err(de::Error::missing_field("package")), - (None, Some(_), None, _, _) => Err(de::Error::missing_field("version")), - } - } - } - - deserializer.deserialize_any(Visitor) - } + Local(PathBuf), } impl FromStr for Dependency { @@ -484,25 +359,40 @@ impl<'a> DependencyResolver<'a> { /// Add a dependency to the resolver. If the dependency already exists, then it will be ignored. /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). + + #[deprecated(since = "0.9.0", note = "please use `new_method` instead")] pub async fn add_dependency( &mut self, name: &PackageRef, dependency: &Dependency, ) -> Result<()> { - self.add_dependency_internal(name, dependency, false).await + self.add_wit_dependency_internal(name, dependency, false) + .await } - /// Add a dependency to the resolver. If the dependency already exists, then it will be - /// overridden. - pub async fn override_dependency( + /// Add a wit dependency to the resolver. If the dependency already exists, then it will be ignored. + /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). + pub async fn add_wit_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_wit_dependency_internal(name, dependency, false) + .await + } + + /// Add a concrete component dependency to the resolver. If the dependency already exists, then it will be ignored. + /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). + pub async fn add_component_dependency( &mut self, name: &PackageRef, dependency: &Dependency, ) -> Result<()> { - self.add_dependency_internal(name, dependency, true).await + self.add_component_dependency_internal(name, dependency, false) + .await } - async fn add_dependency_internal( + async fn add_component_dependency_internal( &mut self, name: &PackageRef, dependency: &Dependency, @@ -510,44 +400,95 @@ impl<'a> DependencyResolver<'a> { ) -> Result<()> { match dependency { Dependency::Package(package) => { - // Dependency comes from a registry, add a dependency to the resolver - let registry_name = package.registry.as_deref().or_else(|| { - self.client.client().ok().and_then(|client| { - client - .config() - .resolve_registry(name) - .map(|reg| reg.as_ref()) - }) + self.add_registry_package(name, force_override, package)? + } + Dependency::Local(p) => { + // A local path dependency, insert a resolution immediately + let res = DependencyResolution::Local(LocalResolution { + name: name.clone(), + path: p.clone(), }); - let package_name = package.name.clone().unwrap_or_else(|| name.clone()); - - // Resolve the version from the lock file if there is one - let locked = match self.lock_file.as_ref().and_then(|resolver| { - resolver - .resolve(registry_name, &package_name, &package.version) - .transpose() - }) { - Some(Ok(locked)) => Some(locked), - Some(Err(e)) => return Err(e), - _ => None, - }; - - if !force_override - && (self.resolutions.contains_key(name) || self.dependencies.contains_key(name)) - { + + if !force_override && self.resolutions.contains_key(name) { tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); return Ok(()); } - self.dependencies.insert( - name.to_owned(), - RegistryDependency { - package: package_name, - version: package.version.clone(), - locked: locked.map(|l| (l.version.clone(), l.digest.clone())), - }, - ); + + let prev = self.resolutions.insert(name.clone(), res); + assert!(prev.is_none()); + } + } + + Ok(()) + } + + /// Add a dependency to the resolver. If the dependency already exists, then it will be + /// overridden. + pub async fn override_dependency( + &mut self, + name: &PackageRef, + dependency: &Dependency, + ) -> Result<()> { + self.add_wit_dependency_internal(name, dependency, true) + .await + } + + fn add_registry_package( + &mut self, + name: &PackageRef, + force_override: bool, + package: &RegistryPackage, + ) -> Result<()> { + // Dependency comes from a registry, add a dependency to the resolver + let registry_name = package.registry.as_deref().or_else(|| { + self.client.client().ok().and_then(|client| { + client + .config() + .resolve_registry(name) + .map(|reg| reg.as_ref()) + }) + }); + let package_name = package.name.clone().unwrap_or_else(|| name.clone()); + + // Resolve the version from the lock file if there is one + let locked = match self.lock_file.as_ref().and_then(|resolver| { + resolver + .resolve(registry_name, &package_name, &package.version) + .transpose() + }) { + Some(Ok(locked)) => Some(locked), + Some(Err(e)) => return Err(e), + _ => None, + }; + + if !force_override + && (self.resolutions.contains_key(name) || self.dependencies.contains_key(name)) + { + tracing::debug!(%name, "dependency already exists and override is not set, ignoring"); + return Ok(()); + } + self.dependencies.insert( + name.to_owned(), + RegistryDependency { + package: package_name, + version: package.version.clone(), + locked: locked.map(|l| (l.version.clone(), l.digest.clone())), + }, + ); + Ok(()) + } + + async fn add_wit_dependency_internal( + &mut self, + name: &PackageRef, + dependency: &Dependency, + force_override: bool, + ) -> Result<()> { + match dependency { + Dependency::Package(package) => { + self.add_registry_package(name, force_override, package)? } - Dependency::Local(p, kind) => { + Dependency::Local(p) => { // A local path dependency, insert a resolution immediately let res = DependencyResolution::Local(LocalResolution { name: name.clone(), @@ -562,13 +503,11 @@ impl<'a> DependencyResolver<'a> { // // Now that we check we haven't already inserted this dep, get the packages from the // // local dependency and add those to the resolver before adding the dependency // if is_wit { - if kind == &DependencyKind::Wit { - let (_, packages) = get_packages(p) - .context("Error getting dependent packages from local dependency")?; - Box::pin(self.add_packages(packages)) - .await - .context("Error adding packages to resolver for local dependency")?; - } + let (_, packages) = get_packages(p) + .context("Error getting dependent packages from local dependency")?; + Box::pin(self.add_packages(packages)) + .await + .context("Error adding packages to resolver for local dependency")?; // } let prev = self.resolutions.insert(name.clone(), res); @@ -586,7 +525,7 @@ impl<'a> DependencyResolver<'a> { packages: impl IntoIterator, ) -> Result<()> { for (package, req) in packages { - self.add_dependency( + self.add_wit_dependency( &package, &Dependency::Package(RegistryPackage { name: Some(package.clone()), @@ -717,10 +656,11 @@ fn find_latest_release<'a>( versions: &'a [VersionInfo], req: &VersionReq, ) -> Option<&'a VersionInfo> { - versions + let versions = versions .iter() .filter(|info| !info.yanked && req.matches(&info.version)) - .max_by(|a, b| a.version.cmp(&b.version)) + .max_by(|a, b| a.version.cmp(&b.version)); + versions } // NOTE(thomastaylor312): This is copied from the old wit package in the cargo-component and broken diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 80b521e..448b579 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -15,8 +15,8 @@ use crate::{ config::Config, lock::LockFile, resolver::{ - DecodedDependency, Dependency, DependencyKind, DependencyResolution, - DependencyResolutionMap, DependencyResolver, LocalResolution, RegistryPackage, + DecodedDependency, Dependency, DependencyResolution, DependencyResolutionMap, + DependencyResolver, LocalResolution, RegistryPackage, }, }; @@ -168,12 +168,12 @@ pub async fn resolve_dependencies( let dep = match (ovride.path.as_ref(), ovride.version.as_ref()) { (Some(path), None) => { let path = tokio::fs::canonicalize(path).await?; - Dependency::Local(path, DependencyKind::Wit) + Dependency::Local(path) } (Some(path), Some(_)) => { tracing::warn!("Ignoring version override for local package"); let path = tokio::fs::canonicalize(path).await?; - Dependency::Local(path, DependencyKind::Wit) + Dependency::Local(path) } (None, Some(version)) => Dependency::Package(RegistryPackage { name: Some(pkg.clone()), @@ -186,7 +186,7 @@ pub async fn resolve_dependencies( } }; resolver - .add_dependency(&pkg, &dep) + .add_wit_dependency(&pkg, &dep) .await .context("Unable to add dependency")?; } From 150f64e40a39c415da95fc1916f5367e6e8aafcc Mon Sep 17 00:00:00 2001 From: Daniel Macovei Date: Tue, 19 Nov 2024 14:42:18 -0600 Subject: [PATCH 4/4] shallow dependencies --- Cargo.lock | 96 +++++++++++++++++++++++++--- Cargo.toml | 6 +- crates/wasm-pkg-core/src/resolver.rs | 28 +++----- crates/wasm-pkg-core/src/wit.rs | 2 +- 4 files changed, 98 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eb4d45..b928181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2125,8 +2125,8 @@ dependencies = [ "serde_json", "sha2", "tokio", - "wit-component", - "wit-parser", + "wit-component 0.219.1", + "wit-parser 0.219.1", ] [[package]] @@ -4232,6 +4232,16 @@ dependencies = [ "wasmparser 0.219.1", ] +[[package]] +name = "wasm-encoder" +version = "0.220.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebf48234b389415b226a4daef6562933d38c7b28a8b8f64c5c4130dad1561ab7" +dependencies = [ + "leb128", + "wasmparser 0.220.0", +] + [[package]] name = "wasm-metadata" version = "0.219.1" @@ -4248,6 +4258,22 @@ dependencies = [ "wasmparser 0.219.1", ] +[[package]] +name = "wasm-metadata" +version = "0.220.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3e5f5920c5abfc45573c89b07b38efdaae1515ef86f83dad12d60e50ecd62b" +dependencies = [ + "anyhow", + "indexmap 2.6.0", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.220.0", + "wasmparser 0.220.0", +] + [[package]] name = "wasm-pkg-client" version = "0.8.2" @@ -4277,9 +4303,9 @@ dependencies = [ "warg-client", "warg-crypto", "warg-protocol", - "wasm-metadata", + "wasm-metadata 0.220.0", "wasm-pkg-common", - "wit-component", + "wit-component 0.220.0", ] [[package]] @@ -4319,12 +4345,12 @@ dependencies = [ "tokio-util", "toml", "tracing", - "wasm-metadata", + "wasm-metadata 0.220.0", "wasm-pkg-client", "wasm-pkg-common", "windows-sys 0.59.0", - "wit-component", - "wit-parser", + "wit-component 0.220.0", + "wit-parser 0.220.0", ] [[package]] @@ -4364,6 +4390,19 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmparser" +version = "0.220.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e246c2772ce3ebc83f89a2d4487ac5794cad6c309b2071818a88c7db7c36d87b" +dependencies = [ + "ahash", + "bitflags 2.6.0", + "hashbrown 0.14.5", + "indexmap 2.6.0", + "semver", +] + [[package]] name = "wasmprinter" version = "0.2.80" @@ -4665,9 +4704,28 @@ dependencies = [ "serde_derive", "serde_json", "wasm-encoder 0.219.1", - "wasm-metadata", + "wasm-metadata 0.219.1", "wasmparser 0.219.1", - "wit-parser", + "wit-parser 0.219.1", +] + +[[package]] +name = "wit-component" +version = "0.220.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ccedf54cc65f287da268d64d2bf4f7530d2cfb2296ffbe3ad5f65567e4cf53" +dependencies = [ + "anyhow", + "bitflags 2.6.0", + "indexmap 2.6.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.220.0", + "wasm-metadata 0.220.0", + "wasmparser 0.220.0", + "wit-parser 0.220.0", ] [[package]] @@ -4688,6 +4746,24 @@ dependencies = [ "wasmparser 0.219.1", ] +[[package]] +name = "wit-parser" +version = "0.220.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7117ce3adc0b4354b46dc1cf3190b00b333e65243d244c613ffcc58bdec84d" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.6.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.220.0", +] + [[package]] name = "wkg" version = "0.8.2" @@ -4708,7 +4784,7 @@ dependencies = [ "wasm-pkg-client", "wasm-pkg-common", "wasm-pkg-core", - "wit-component", + "wit-component 0.220.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d32c316..cd75fec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features = ] } wasm-pkg-common = { version = "0.8.2", path = "crates/wasm-pkg-common" } wasm-pkg-client = { version = "0.8.2", path = "crates/wasm-pkg-client" } -wasm-metadata = "0.219" -wit-component = "0.219" -wit-parser = "0.219" +wasm-metadata = "0.220" +wit-component = "0.220" +wit-parser = "0.220" wasm-pkg-core = { version = "0.8.2", path = "crates/wasm-pkg-core" } diff --git a/crates/wasm-pkg-core/src/resolver.rs b/crates/wasm-pkg-core/src/resolver.rs index 4df223b..5547030 100644 --- a/crates/wasm-pkg-core/src/resolver.rs +++ b/crates/wasm-pkg-core/src/resolver.rs @@ -360,39 +360,29 @@ impl<'a> DependencyResolver<'a> { /// Add a dependency to the resolver. If the dependency already exists, then it will be ignored. /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). - #[deprecated(since = "0.9.0", note = "please use `new_method` instead")] - pub async fn add_dependency( - &mut self, - name: &PackageRef, - dependency: &Dependency, - ) -> Result<()> { - self.add_wit_dependency_internal(name, dependency, false) - .await - } - /// Add a wit dependency to the resolver. If the dependency already exists, then it will be ignored. /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). - pub async fn add_wit_dependency( + pub async fn add_dependency( &mut self, name: &PackageRef, dependency: &Dependency, ) -> Result<()> { - self.add_wit_dependency_internal(name, dependency, false) + self.add_dependency_internal(name, dependency, false) .await } /// Add a concrete component dependency to the resolver. If the dependency already exists, then it will be ignored. /// To override an existing dependency, use [`override_dependency`](Self::override_dependency). - pub async fn add_component_dependency( + pub async fn add_shallow_dependency( &mut self, name: &PackageRef, dependency: &Dependency, ) -> Result<()> { - self.add_component_dependency_internal(name, dependency, false) + self.add_shallow_dependency_internal(name, dependency, false) .await } - async fn add_component_dependency_internal( + async fn add_shallow_dependency_internal( &mut self, name: &PackageRef, dependency: &Dependency, @@ -429,7 +419,7 @@ impl<'a> DependencyResolver<'a> { name: &PackageRef, dependency: &Dependency, ) -> Result<()> { - self.add_wit_dependency_internal(name, dependency, true) + self.add_dependency_internal(name, dependency, true) .await } @@ -478,7 +468,7 @@ impl<'a> DependencyResolver<'a> { Ok(()) } - async fn add_wit_dependency_internal( + async fn add_dependency_internal( &mut self, name: &PackageRef, dependency: &Dependency, @@ -502,13 +492,11 @@ impl<'a> DependencyResolver<'a> { // // Now that we check we haven't already inserted this dep, get the packages from the // // local dependency and add those to the resolver before adding the dependency - // if is_wit { let (_, packages) = get_packages(p) .context("Error getting dependent packages from local dependency")?; Box::pin(self.add_packages(packages)) .await .context("Error adding packages to resolver for local dependency")?; - // } let prev = self.resolutions.insert(name.clone(), res); assert!(prev.is_none()); @@ -525,7 +513,7 @@ impl<'a> DependencyResolver<'a> { packages: impl IntoIterator, ) -> Result<()> { for (package, req) in packages { - self.add_wit_dependency( + self.add_dependency( &package, &Dependency::Package(RegistryPackage { name: Some(package.clone()), diff --git a/crates/wasm-pkg-core/src/wit.rs b/crates/wasm-pkg-core/src/wit.rs index 448b579..059cef1 100644 --- a/crates/wasm-pkg-core/src/wit.rs +++ b/crates/wasm-pkg-core/src/wit.rs @@ -186,7 +186,7 @@ pub async fn resolve_dependencies( } }; resolver - .add_wit_dependency(&pkg, &dep) + .add_dependency(&pkg, &dep) .await .context("Unable to add dependency")?; }