pesde-pkg/pesde
1
1
Fork 0
mirror of https://github.com/pesde-pkg/pesde.git synced 2025-04-04 10:50:55 +01:00
Code Issues Projects Releases Packages Wiki Activity

feat(registry): ✨ add listing newest packages

Browse source
This commit is contained in:
daimond113 2024-03-16 17:26:58 +01:00
parent 970a8dce50
commit 9c806ba082
No known key found for this signature in database
GPG key ID: 3A8ECE51328B513C
4 changed files with 75 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
registry/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "pesde-registry"
version = "0.3.0"
version = "0.4.0"
edition = "2021"
[dependencies]
@ -27,4 +27,5 @@ log = "0.4.21"
pretty_env_logger = "0.5.0"
sentry = "0.32.2"
sentry-log = "0.32.2"
sentry-actix = "0.32.2"
sentry-actix = "0.32.2"
chrono = "0.4.34"

16
registry/src/endpoints/packages.rs
View file

@ -1,10 +1,11 @@
use actix_multipart::form::{bytes::Bytes, MultipartForm};
use actix_web::{web, HttpResponse, Responder};
use chrono::Utc;
use flate2::read::GzDecoder;
use log::error;
use reqwest::StatusCode;
use rusty_s3::S3Action;
use tantivy::{doc, Term};
use tantivy::{doc, DateTime, Term};
use tar::Archive;
use pesde::{
@ -131,10 +132,11 @@ pub async fn create_package(
search_writer.add_document(
doc!(
name_field => manifest.name.to_string(),
schema.get_field("version").unwrap() => manifest.version.to_string(),
schema.get_field("description").unwrap() => manifest.description.unwrap_or_default(),
)
name_field => manifest.name.to_string(),
schema.get_field("version").unwrap() => manifest.version.to_string(),
schema.get_field("description").unwrap() => manifest.description.unwrap_or_default(),
schema.get_field("published_at").unwrap() => DateTime::from_timestamp_secs(Utc::now().timestamp()),
)
).unwrap();
search_writer.commit().unwrap();
@ -234,8 +236,8 @@ pub async fn get_package_versions(
Some(package) => {
let versions = package
.iter()
.map(|v| v.version.to_string())
.collect::<Vec<String>>();
.map(|v| (v.version.to_string(), v.published_at.timestamp()))
.collect::<Vec<_>>();
Ok(HttpResponse::Ok().json(versions))
}

78
registry/src/endpoints/search.rs
View file

@ -1,12 +1,16 @@
use actix_web::{web, Responder};
use semver::Version;
use serde::Deserialize;
use serde_json::{json, Value};
use tantivy::{query::AllQuery, DateTime, DocAddress, Order};
use pesde::{index::Index, package_name::PackageName};
use crate::{errors, AppState};
#[derive(Deserialize)]
pub struct Query {
query: String,
query: Option<String>,
}
pub async fn search_packages(
@ -20,32 +24,64 @@ pub async fn search_packages(
let version = schema.get_field("version").unwrap();
let description = schema.get_field("description").unwrap();
let query = query.query.trim();
if query.is_empty() {
return Ok(web::Json(vec![]));
}
let query = query.query.as_deref().unwrap_or_default().trim();
let query_parser =
tantivy::query::QueryParser::for_index(&searcher.index(), vec![name, description]);
let query = query_parser.parse_query(&query)?;
let query = if query.is_empty() {
Box::new(AllQuery)
} else {
query_parser.parse_query(&query)?
};
let top_docs = searcher
.search(&query, &tantivy::collector::TopDocs::with_limit(10))
let top_docs: Vec<(DateTime, DocAddress)> = searcher
.search(
&query,
&tantivy::collector::TopDocs::with_limit(10)
.order_by_fast_field("published_at", Order::Desc),
)
.unwrap();
Ok(web::Json(
top_docs
.into_iter()
.map(|(_, doc_address)| {
let retrieved_doc = searcher.doc(doc_address).unwrap();
{
let index = app_state.index.lock().unwrap();
json!({
"name": retrieved_doc.get_first(name).unwrap().as_text().unwrap(),
"version": retrieved_doc.get_first(version).unwrap().as_text().unwrap(),
"description": retrieved_doc.get_first(description).unwrap().as_text().unwrap(),
Ok(web::Json(
top_docs
.into_iter()
.map(|(published_at, doc_address)| {
let retrieved_doc = searcher.doc(doc_address).unwrap();
let name: PackageName = retrieved_doc
.get_first(name)
.unwrap()
.as_text()
.unwrap()
.parse()
.unwrap();
let version: Version = retrieved_doc
.get_first(version)
.unwrap()
.as_text()
.unwrap()
.parse()
.unwrap();
let entry = index
.package(&name)
.unwrap()
.unwrap()
.into_iter()
.find(|v| v.version == version)
.unwrap();
json!({
"name": name,
"version": version,
"description": entry.description,
"published_at": published_at.into_timestamp_secs(),
})
})
})
.collect::<Vec<Value>>(),
))
.collect::<Vec<Value>>(),
))
}
}

13
registry/src/main.rs
View file

@ -15,7 +15,7 @@ use git2::{Cred, Signature};
use log::info;
use reqwest::{header::AUTHORIZATION, Client};
use rusty_s3::{Bucket, Credentials, UrlStyle};
use tantivy::{doc, IndexReader, IndexWriter};
use tantivy::{doc, DateTime, IndexReader, IndexWriter};
use pesde::{
index::{GitIndex, IndexFile},
@ -126,10 +126,8 @@ fn search_index(index: &GitIndex) -> (IndexReader, IndexWriter) {
schema_builder.add_text_field("name", tantivy::schema::TEXT | tantivy::schema::STORED);
let version =
schema_builder.add_text_field("version", tantivy::schema::TEXT | tantivy::schema::STORED);
let description = schema_builder.add_text_field(
"description",
tantivy::schema::TEXT | tantivy::schema::STORED,
);
let description = schema_builder.add_text_field("description", tantivy::schema::TEXT);
let published_at = schema_builder.add_date_field("published_at", tantivy::schema::FAST);
let search_index = tantivy::Index::create_in_ram(schema_builder.build());
let search_reader = search_index
@ -172,7 +170,8 @@ fn search_index(index: &GitIndex) -> (IndexReader, IndexWriter) {
.add_document(doc!(
name => package_name.to_string(),
version => entry.version.to_string(),
description => entry.description.unwrap_or_default()
description => entry.description.unwrap_or_default(),
published_at => DateTime::from_timestamp_secs(entry.published_at.timestamp()),
))
.unwrap();
}
@ -267,7 +266,7 @@ fn main() -> std::io::Result<()> {
.unwrap();
let generic_governor_config = GovernorConfigBuilder::default()
.burst_size(10)
.burst_size(50)
.per_second(10)
.use_headers()
.finish()