pesde-pkg/pesde
1
1
Fork 0
mirror of https://github.com/pesde-pkg/pesde.git synced 2024-12-12 11:00:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

feat: install pesde packages before wally

Browse source
This commit is contained in:
daimond113 2024-12-02 23:39:39 +01:00
parent a4162cd300
commit b53457c42c
No known key found for this signature in database
GPG key ID: 3A8ECE51328B513C
10 changed files with 379 additions and 202 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
- Add improved CLI styling by @daimond113
- Install pesde dependencies before Wally to support scripts packages by @daimond113
### Fixed
- Link dependencies before type extraction to support more use cases
- Link dependencies before type extraction to support more use cases by @daimond113
## [0.5.0-rc.14] - 2024-11-30
### Fixed

35
src/cli/commands/execute.rs
View file

@ -5,7 +5,7 @@ use fs_err::tokio as fs;
use indicatif::MultiProgress;
use pesde::{
linking::generator::generate_bin_linking_module,
manifest::{target::TargetKind, DependencyType},
manifest::target::TargetKind,
names::PackageName,
source::{
pesde::{specifier::PesdeDependencySpecifier, PesdePackageSource},
@ -17,6 +17,7 @@ use semver::VersionReq;
use std::{
collections::HashSet, env::current_dir, ffi::OsString, io::Write, process::Command, sync::Arc,
};
use tokio::sync::Mutex;
#[derive(Debug, Args)]
pub struct ExecuteCommand {
@ -116,9 +117,17 @@ impl ExecuteCommand {
.dependency_graph(None, &mut refreshed_sources, true)
.await
.context("failed to build dependency graph")?;
let graph = Arc::new(graph);
let (rx, downloaded_graph) = project
.download_graph(&graph, &mut refreshed_sources, &reqwest, true, true)
.download_and_link(
&graph,
&Arc::new(Mutex::new(refreshed_sources)),
&reqwest,
true,
true,
|_| async { Ok::<_, std::io::Error>(()) },
)
.await
.context("failed to download dependencies")?;
@ -132,27 +141,9 @@ impl ExecuteCommand {
)
.await?;
let downloaded_graph = Arc::into_inner(downloaded_graph)
.unwrap()
.into_inner()
.unwrap();
project
.link_dependencies(
&downloaded_graph
.into_iter()
.map(|(n, v)| {
(
n,
v.into_iter()
.filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev)
.collect(),
)
})
.collect(),
)
downloaded_graph
.await
.context("failed to link dependencies")?;
.context("failed to download & link dependencies")?;
let mut caller =
tempfile::NamedTempFile::new_in(tempdir.path()).context("failed to create tempfile")?;

160
src/cli/commands/install.rs
View file

@ -9,14 +9,14 @@ use fs_err::tokio as fs;
use futures::future::try_join_all;
use indicatif::MultiProgress;
use pesde::{
lockfile::Lockfile,
manifest::{target::TargetKind, DependencyType},
Project, MANIFEST_FILE_NAME,
download_and_link::filter_graph, lockfile::Lockfile, manifest::target::TargetKind, Project,
MANIFEST_FILE_NAME,
};
use std::{
collections::{BTreeSet, HashMap, HashSet},
sync::Arc,
};
use tokio::sync::Mutex;
#[derive(Debug, Args, Copy, Clone)]
pub struct InstallCommand {
@ -81,14 +81,18 @@ stdio.ewrite(stdio.color("red") .. "binary `{alias}` not found. are you in the r
}
#[cfg(feature = "patches")]
const JOBS: u8 = 6;
#[cfg(not(feature = "patches"))]
const JOBS: u8 = 5;
#[cfg(not(feature = "patches"))]
const JOBS: u8 = 4;
fn job(n: u8) -> ColoredString {
format!("[{n}/{JOBS}]").dimmed().bold()
}
#[derive(Debug, thiserror::Error)]
#[error(transparent)]
struct CallbackError(#[from] anyhow::Error);
impl InstallCommand {
pub async fn run(
self,
@ -198,12 +202,74 @@ impl InstallCommand {
.dependency_graph(old_graph.as_ref(), &mut refreshed_sources, false)
.await
.context("failed to build dependency graph")?;
let graph = Arc::new(graph);
update_scripts_handle.await??;
let bin_folder = bin_dir().await?;
let downloaded_graph = {
let (rx, downloaded_graph) = project
.download_graph(&graph, &mut refreshed_sources, &reqwest, self.prod, true)
.download_and_link(
&graph,
&Arc::new(Mutex::new(refreshed_sources)),
&reqwest,
self.prod,
true,
|graph| {
let graph = graph.clone();
async move {
try_join_all(
graph
.values()
.flat_map(|versions| versions.values())
.filter(|node| node.target.bin_path().is_some())
.filter_map(|node| node.node.direct.as_ref())
.map(|(alias, _, _)| alias)
.filter(|alias| {
if *alias == env!("CARGO_BIN_NAME") {
log::warn!(
"package {alias} has the same name as the CLI, skipping bin link"
);
return false;
}
true
})
.map(|alias| {
let bin_folder = bin_folder.clone();
async move {
let bin_file = bin_folder.join(alias);
fs::write(&bin_file, bin_link_file(alias))
.await
.context("failed to write bin link file")?;
make_executable(&bin_file)
.await
.context("failed to make bin link executable")?;
#[cfg(windows)]
{
let bin_file = bin_file.with_extension(std::env::consts::EXE_EXTENSION);
fs::copy(
std::env::current_exe()
.context("failed to get current executable path")?,
&bin_file,
)
.await
.context("failed to copy bin link file")?;
}
Ok::<_, CallbackError>(())
}
}),
)
.await
.map(|_| ())
}
}
)
.await
.context("failed to download dependencies")?;
@ -217,33 +283,15 @@ impl InstallCommand {
)
.await?;
Arc::into_inner(downloaded_graph)
.unwrap()
.into_inner()
.unwrap()
};
let filtered_graph = if self.prod {
downloaded_graph
.clone()
.into_iter()
.map(|(n, v)| {
(
n,
v.into_iter()
.filter(|(_, n)| n.node.resolved_ty != DependencyType::Dev)
.collect(),
)
})
.collect()
} else {
downloaded_graph.clone()
.await
.context("failed to download & link dependencies")?
};
#[cfg(feature = "patches")]
{
let rx = project
.apply_patches(&filtered_graph)
.apply_patches(&filter_graph(&downloaded_graph, self.prod))
.await
.context("failed to apply patches")?;
@ -251,69 +299,13 @@ impl InstallCommand {
manifest.patches.values().map(|v| v.len() as u64).sum(),
rx,
&multi,
format!("{} 🩹 ", job(4)),
format!("{} 🩹 ", job(JOBS - 1)),
"applying patches".to_string(),
"applied patches".to_string(),
)
.await?;
}
println!("{} 🗺️ linking dependencies", job(JOBS - 1));
let bin_folder = bin_dir().await?;
try_join_all(
filtered_graph
.values()
.flat_map(|versions| versions.values())
.filter(|node| node.target.bin_path().is_some())
.filter_map(|node| node.node.direct.as_ref())
.map(|(alias, _, _)| alias)
.filter(|alias| {
if *alias == env!("CARGO_BIN_NAME") {
log::warn!(
"package {alias} has the same name as the CLI, skipping bin link"
);
return false;
}
true
})
.map(|alias| {
let bin_folder = bin_folder.clone();
async move {
let bin_file = bin_folder.join(alias);
fs::write(&bin_file, bin_link_file(alias))
.await
.context("failed to write bin link file")?;
make_executable(&bin_file)
.await
.context("failed to make bin link executable")?;
#[cfg(windows)]
{
let bin_file = bin_file.with_extension(std::env::consts::EXE_EXTENSION);
fs::copy(
std::env::current_exe()
.context("failed to get current executable path")?,
&bin_file,
)
.await
.context("failed to copy bin link file")?;
}
Ok::<_, anyhow::Error>(())
}
}),
)
.await?;
project
.link_dependencies(&filtered_graph)
.await
.context("failed to link dependencies")?;
println!("{} 🧹 finishing up", job(JOBS));
project

18
src/cli/commands/update.rs
View file

@ -5,6 +5,7 @@ use colored::Colorize;
use indicatif::MultiProgress;
use pesde::{lockfile::Lockfile, Project};
use std::{collections::HashSet, sync::Arc};
use tokio::sync::Mutex;
#[derive(Debug, Args, Copy, Clone)]
pub struct UpdateCommand {}
@ -34,6 +35,7 @@ impl UpdateCommand {
.dependency_graph(None, &mut refreshed_sources, false)
.await
.context("failed to build dependency graph")?;
let graph = Arc::new(graph);
update_scripts(&project).await?;
@ -46,7 +48,14 @@ impl UpdateCommand {
graph: {
let (rx, downloaded_graph) = project
.download_graph(&graph, &mut refreshed_sources, &reqwest, false, false)
.download_and_link(
&graph,
&Arc::new(Mutex::new(refreshed_sources)),
&reqwest,
false,
false,
|_| async { Ok::<_, std::io::Error>(()) },
)
.await
.context("failed to download dependencies")?;
@ -60,10 +69,9 @@ impl UpdateCommand {
)
.await?;
Arc::into_inner(downloaded_graph)
.unwrap()
.into_inner()
.unwrap()
downloaded_graph
.await
.context("failed to download dependencies")?
},
workspace: run_on_workspace_members(&project, |project| {

12
src/download.rs
View file

@ -16,7 +16,7 @@ use std::{
type MultithreadedGraph = Arc<Mutex<DownloadedGraph>>;
type MultithreadDownloadJob = (
pub(crate) type MultithreadDownloadJob = (
tokio::sync::mpsc::Receiver<Result<String, errors::DownloadGraphError>>,
MultithreadedGraph,
);
@ -30,6 +30,7 @@ impl Project {
reqwest: &reqwest::Client,
prod: bool,
write: bool,
wally: bool,
) -> Result<MultithreadDownloadJob, errors::DownloadGraphError> {
let manifest = self.deser_manifest().await?;
let manifest_target_kind = manifest.target.kind();
@ -53,15 +54,22 @@ impl Project {
)
.await?;
let project = Arc::new(self.clone());
for (name, versions) in graph {
for (version_id, node) in versions {
// we need to download pesde packages first, since scripts (for target finding for example) can depend on them
if node.pkg_ref.like_wally() != wally {
continue;
}
let tx = tx.clone();
let name = name.clone();
let version_id = version_id.clone();
let node = node.clone();
let project = Arc::new(self.clone());
let project = project.clone();
let reqwest = reqwest.clone();
let downloaded_graph = downloaded_graph.clone();

164
src/download_and_link.rs Normal file
View file

@ -0,0 +1,164 @@
use crate::{
lockfile::{DependencyGraph, DownloadedGraph},
manifest::DependencyType,
source::PackageSources,
Project,
};
use futures::FutureExt;
use std::{
collections::HashSet,
future::Future,
sync::{Arc, Mutex as StdMutex},
};
use tokio::sync::Mutex;
/// Filters a graph to only include production dependencies, if `prod` is `true`
pub fn filter_graph(graph: &DownloadedGraph, prod: bool) -> DownloadedGraph {
if !prod {
return graph.clone();
}
graph
.iter()
.map(|(name, versions)| {
(
name.clone(),
versions
.iter()
.filter(|(_, node)| node.node.resolved_ty != DependencyType::Dev)
.map(|(v_id, node)| (v_id.clone(), node.clone()))
.collect(),
)
})
.collect()
}
impl Project {
/// Downloads a graph of dependencies and links them in the correct order
pub async fn download_and_link<
F: FnOnce(&Arc<DownloadedGraph>) -> R + Send + 'static,
R: Future<Output = Result<(), E>> + Send,
E: Send + Sync + 'static,
>(
&self,
graph: &Arc<DependencyGraph>,
refreshed_sources: &Arc<Mutex<HashSet<PackageSources>>>,
reqwest: &reqwest::Client,
prod: bool,
write: bool,
pesde_cb: F,
) -> Result<
(
tokio::sync::mpsc::Receiver<
Result<String, crate::download::errors::DownloadGraphError>,
>,
impl Future<Output = Result<DownloadedGraph, errors::DownloadAndLinkError<E>>>,
),
errors::DownloadAndLinkError<E>,
> {
let (tx, rx) = tokio::sync::mpsc::channel(
graph
.iter()
.map(|(_, versions)| versions.len())
.sum::<usize>()
.max(1),
);
let downloaded_graph = Arc::new(StdMutex::new(DownloadedGraph::default()));
let this = self.clone();
let graph = graph.clone();
let reqwest = reqwest.clone();
let refreshed_sources = refreshed_sources.clone();
Ok((
rx,
tokio::spawn(async move {
let mut refreshed_sources = refreshed_sources.lock().await;
// step 1. download pesde dependencies
let (mut pesde_rx, pesde_graph) = this
.download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, false)
.await?;
while let Some(result) = pesde_rx.recv().await {
tx.send(result).await.unwrap();
}
let pesde_graph = Arc::into_inner(pesde_graph).unwrap().into_inner().unwrap();
// step 2. link pesde dependencies. do so without types
if write {
this.link_dependencies(&filter_graph(&pesde_graph, prod), false)
.await?;
}
let pesde_graph = Arc::new(pesde_graph);
pesde_cb(&pesde_graph)
.await
.map_err(errors::DownloadAndLinkError::PesdeCallback)?;
let pesde_graph = Arc::into_inner(pesde_graph).unwrap();
// step 3. download wally dependencies
let (mut wally_rx, wally_graph) = this
.download_graph(&graph, &mut refreshed_sources, &reqwest, prod, write, true)
.await?;
while let Some(result) = wally_rx.recv().await {
tx.send(result).await.unwrap();
}
let wally_graph = Arc::into_inner(wally_graph).unwrap().into_inner().unwrap();
{
let mut downloaded_graph = downloaded_graph.lock().unwrap();
downloaded_graph.extend(pesde_graph);
for (name, versions) in wally_graph {
for (version_id, node) in versions {
downloaded_graph
.entry(name.clone())
.or_default()
.insert(version_id, node);
}
}
}
let graph = Arc::into_inner(downloaded_graph)
.unwrap()
.into_inner()
.unwrap();
// step 4. link ALL dependencies. do so with types
if write {
this.link_dependencies(&filter_graph(&graph, prod), true)
.await?;
}
Ok(graph)
})
.map(|r| r.unwrap()),
))
}
}
/// Errors that can occur when downloading and linking dependencies
pub mod errors {
use thiserror::Error;
/// An error that can occur when downloading and linking dependencies
#[derive(Debug, Error)]
pub enum DownloadAndLinkError<E> {
/// An error occurred while downloading the graph
#[error("error downloading graph")]
DownloadGraph(#[from] crate::download::errors::DownloadGraphError),
/// An error occurred while linking dependencies
#[error("error linking dependencies")]
Linking(#[from] crate::linking::errors::LinkingError),
/// An error occurred while executing the pesde callback
#[error("error executing pesde callback")]
PesdeCallback(#[source] E),
}
}

2
src/lib.rs
View file

@ -20,6 +20,8 @@ use wax::Pattern;
/// Downloading packages
pub mod download;
/// Utility for downloading and linking in the correct order
pub mod download_and_link;
/// Linking packages
pub mod linking;
/// Lockfile

164
src/linking/mod.rs
View file

@ -47,106 +47,114 @@ impl Project {
pub async fn link_dependencies(
&self,
graph: &DownloadedGraph,
with_types: bool,
) -> Result<(), errors::LinkingError> {
let manifest = self.deser_manifest().await?;
let manifest_target_kind = manifest.target.kind();
let manifest = Arc::new(manifest);
// step 1. link all packages (and their dependencies) temporarily without types
// step 1. link all non-wally packages (and their dependencies) temporarily without types
// we do this separately to allow the required tools for the scripts to be installed
self.link(graph, &manifest, &Arc::new(Default::default()))
.await?;
// step 2. extract the types from libraries
if !with_types {
return Ok(());
}
// step 2. extract the types from libraries, prepare Roblox packages for syncing
let roblox_sync_config_gen_script = manifest
.scripts
.get(&ScriptName::RobloxSyncConfigGenerator.to_string());
let package_types = try_join_all(
graph
.iter()
.map(|(name, versions)| async move {
Ok::<_, errors::LinkingError>((name, try_join_all(versions.iter().map(|(version_id, node)| async move {
let Some(lib_file) = node.target.lib_path() else {
let package_types = try_join_all(graph.iter().map(|(name, versions)| async move {
Ok::<_, errors::LinkingError>((
name,
try_join_all(versions.iter().map(|(version_id, node)| async move {
let Some(lib_file) = node.target.lib_path() else {
return Ok((version_id, vec![]));
};
let container_folder = node.node.container_folder(
&self
.package_dir()
.join(manifest_target_kind.packages_folder(version_id.target()))
.join(PACKAGES_CONTAINER_NAME),
name,
version_id.version(),
);
let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND {
let lib_file = lib_file.to_path(&container_folder);
let contents = match fs::read_to_string(&lib_file).await {
Ok(contents) => contents,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(errors::LinkingError::LibFileNotFound(
lib_file.display().to_string(),
));
}
Err(e) => return Err(e.into()),
};
let types = match spawn_blocking(move || get_file_types(&contents))
.await
.unwrap()
{
Ok(types) => types,
Err(e) => {
return Err(errors::LinkingError::FullMoon(
lib_file.display().to_string(),
e,
))
}
};
log::debug!("{name}@{version_id} has {} exported types", types.len());
types
} else {
vec![]
};
if let Some(build_files) = Some(&node.target)
.filter(|_| !node.node.pkg_ref.like_wally())
.and_then(|t| t.build_files())
{
let Some(script_path) = roblox_sync_config_gen_script else {
log::warn!("not having a `{}` script in the manifest might cause issues with Roblox linking", ScriptName::RobloxSyncConfigGenerator);
return Ok((version_id, vec![]));
};
let container_folder = node.node.container_folder(
&self
.package_dir()
.join(manifest_target_kind.packages_folder(version_id.target()))
.join(PACKAGES_CONTAINER_NAME),
name,
version_id.version(),
);
execute_script(
ScriptName::RobloxSyncConfigGenerator,
&script_path.to_path(self.package_dir()),
std::iter::once(container_folder.as_os_str())
.chain(build_files.iter().map(OsStr::new)),
self,
false,
).await
.map_err(|e| {
errors::LinkingError::GenerateRobloxSyncConfig(
container_folder.display().to_string(),
e,
)
})?;
}
let types = if lib_file.as_str() != LINK_LIB_NO_FILE_FOUND {
let lib_file = lib_file.to_path(&container_folder);
let contents = match fs::read_to_string(&lib_file).await {
Ok(contents) => contents,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
return Err(errors::LinkingError::LibFileNotFound(
lib_file.display().to_string(),
));
}
Err(e) => return Err(e.into()),
};
let types = match spawn_blocking(move || get_file_types(&contents)).await.unwrap() {
Ok(types) => types,
Err(e) => {
return Err(errors::LinkingError::FullMoon(
lib_file.display().to_string(),
e,
))
}
};
log::debug!("{name}@{version_id} has {} exported types", types.len());
types
} else {
vec![]
};
if let Some(build_files) = Some(&node.target)
.filter(|_| !node.node.pkg_ref.like_wally())
.and_then(|t| t.build_files())
{
let Some(script_path) = roblox_sync_config_gen_script else {
log::warn!("not having a `{}` script in the manifest might cause issues with Roblox linking", ScriptName::RobloxSyncConfigGenerator);
return Ok((version_id, vec![]));
};
execute_script(
ScriptName::RobloxSyncConfigGenerator,
&script_path.to_path(self.package_dir()),
std::iter::once(container_folder.as_os_str())
.chain(build_files.iter().map(OsStr::new)),
self,
false,
)
.map_err(|e| {
errors::LinkingError::GenerateRobloxSyncConfig(
container_folder.display().to_string(),
e,
)
})?;
}
Ok((version_id, types))
})).await?.into_iter().collect::<HashMap<_, _>>()))
}
)
)
Ok((version_id, types))
}))
.await?
.into_iter()
.collect::<HashMap<_, _>>(),
))
}))
.await?
.into_iter()
.collect::<HashMap<_, _>>();
let package_types = Arc::new(package_types);
// step 3. link all packages (and their dependencies), this time with types
self.link(graph, &manifest, &package_types).await
self.link(graph, &manifest, &Arc::new(package_types)).await
}
async fn link(

20
src/scripts.rs
View file

@ -2,10 +2,12 @@ use crate::Project;
use std::{
ffi::OsStr,
fmt::{Display, Formatter},
io::{BufRead, BufReader},
path::Path,
process::{Command, Stdio},
thread::spawn,
process::Stdio,
};
use tokio::{
io::{AsyncBufReadExt, BufReader},
process::Command,
};
/// Script names used by pesde
@ -28,7 +30,7 @@ impl Display for ScriptName {
}
}
pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
pub(crate) async fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
script_name: ScriptName,
script_path: &Path,
args: A,
@ -47,14 +49,14 @@ pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
.spawn()
{
Ok(mut child) => {
let stdout = BufReader::new(child.stdout.take().unwrap());
let stderr = BufReader::new(child.stderr.take().unwrap());
let mut stdout = BufReader::new(child.stdout.take().unwrap()).lines();
let mut stderr = BufReader::new(child.stderr.take().unwrap()).lines();
let script = script_name.to_string();
let script_2 = script.to_string();
spawn(move || {
for line in stderr.lines() {
tokio::spawn(async move {
while let Some(line) = stderr.next_line().await.transpose() {
match line {
Ok(line) => {
log::error!("[{script}]: {line}");
@ -69,7 +71,7 @@ pub(crate) fn execute_script<A: IntoIterator<Item = S>, S: AsRef<OsStr>>(
let mut stdout_str = String::new();
for line in stdout.lines() {
while let Some(line) = stdout.next_line().await.transpose() {
match line {
Ok(line) => {
if return_stdout {

3
src/source/wally/compat_util.rs
View file

@ -39,7 +39,8 @@ pub(crate) async fn find_lib_path(
[package_dir],
project,
true,
)?;
)
.await?;
if let Some(result) = result.filter(|result| !result.is_empty()) {
let node: SourcemapNode = serde_json::from_str(&result)?;