diff options
| -rw-r--r-- | rust/fatcat-cli/src/api.rs | 34 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/commands.rs | 189 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/download.rs | 192 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/entities.rs | 34 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/lib.rs | 19 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/main.rs | 184 | ||||
| -rw-r--r-- | rust/fatcat-cli/src/specifier.rs | 40 |
7 files changed, 520 insertions, 172 deletions
diff --git a/rust/fatcat-cli/src/api.rs b/rust/fatcat-cli/src/api.rs index cc6fa6a..2463aab 100644 --- a/rust/fatcat-cli/src/api.rs +++ b/rust/fatcat-cli/src/api.rs @@ -403,4 +403,38 @@ impl FatcatApiClient { } .with_context(|| format!("failed to update {:?}", specifier)) } + + pub fn create_editgroup(&mut self, description: Option<String>) -> Result<models::Editgroup> { + let mut eg = models::Editgroup::new(); + eg.description = description; + eg.extra = Some({ + let mut extra = std::collections::HashMap::new(); + extra.insert( + "agent".to_string(), + serde_json::Value::String("fatcat-cli".to_string()), + // TODO: version? + ); + extra + }); + let result = self.rt.block_on(self.api.create_editgroup(eg))?; + match result { + fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => Ok(eg), + other => Err(anyhow!("{:?}", other)).context("failed to create editgroup"), + } + } + + pub fn accept_editgroup(&mut self, editgroup_id: String) -> Result<models::Success> { + let result = self + .rt + .block_on(self.api.accept_editgroup(editgroup_id.clone())) + .context("accept editgroup")?; + match result { + fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => Ok(msg), + other => Err(anyhow!( + "failed to accept editgroup {}: {:?}", + editgroup_id, + other + )), + } + } } diff --git a/rust/fatcat-cli/src/commands.rs b/rust/fatcat-cli/src/commands.rs index 30fa0c4..15bfc81 100644 --- a/rust/fatcat-cli/src/commands.rs +++ b/rust/fatcat-cli/src/commands.rs @@ -3,14 +3,17 @@ use chrono_humanize::HumanTime; use fatcat_openapi::models; #[allow(unused_imports)] use log::{self, debug, info}; -use std::io::{Write, BufRead}; +use std::convert::TryInto; +use std::fs::File; +use std::io::{self, BufRead, Write}; +use std::path::PathBuf; use tabwriter::TabWriter; use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor}; -use crate::api::FatcatApiClient; -//use crate::download::download_file; -use crate::entities::{ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation, read_entity_file}; -use crate::specifier::Specifier; +use crate::{ + entity_model_from_json_str, read_entity_file, ApiModelSer, EntityType, FatcatApiClient, + Mutation, Specifier, +}; // Want to show: // - whether api_token found @@ -271,7 +274,13 @@ pub fn print_entity_histories( Ok(()) } -pub fn edit_entity_locally(api_client: &mut FatcatApiClient, specifier: Specifier, editgroup_id: String, json: bool, editing_command: String) -> Result<models::EntityEdit> { +pub fn edit_entity_locally( + api_client: &mut FatcatApiClient, + specifier: Specifier, + editgroup_id: String, + json: bool, + editing_command: String, +) -> Result<models::EntityEdit> { // TODO: fetch editgroup, check if this entity is already being updated in it. If so, // need to fetch that revision, do the edit, parse that synatx is good, then delete the // existing edit and update with the new one. @@ -312,3 +321,171 @@ pub fn edit_entity_locally(api_client: &mut FatcatApiClient, specifier: Specifi .context("updating after edit")?; Ok(ee) } + +pub enum BatchOp { + Create, + Update, + Delete, +} + +pub struct BatchGrouper { + entity_type: EntityType, + batch_size: u64, + limit: Option<u64>, + auto_accept: bool, + editgroup_description: String, + current_count: u64, + current_editgroup_id: Option<String>, + total_count: u64, +} + +// Note: should be possible to add support for the single-call batch create endpoint by storing +// edits in a batch within this object. Might need to change control flow a bit. This optimization +// was mostly intended for bootstrapping of tens of thousands of entities, so not including for +// now. +impl BatchGrouper { + pub fn new( + entity_type: EntityType, + batch_size: u64, + limit: Option<u64>, + auto_accept: bool, + ) -> Self { + let editgroup_description = "part of a fatcat-cli batch operation".to_string(); + BatchGrouper { + entity_type, + batch_size, + limit, + auto_accept, + editgroup_description, + current_count: 0, + current_editgroup_id: None, + total_count: 0, + } + } + + pub fn run( + &mut self, + api_client: &mut FatcatApiClient, + input_path: Option<PathBuf>, + op: BatchOp, + mutations: Option<Vec<Mutation>>, + ) -> Result<()> { + match input_path { + None => { + let stdin = io::stdin(); + let stdin_lock = stdin.lock(); + let lines = stdin_lock.lines(); + for line in lines { + let json_str = line?; + match op { + BatchOp::Create => self.push_create(api_client, &json_str)?, + BatchOp::Update => self.push_update( + api_client, + &json_str, + mutations.clone().unwrap_or(vec![]), + )?, + BatchOp::Delete => self.push_delete(api_client, &json_str)?, + }; + if let Some(limit) = self.limit { + if self.total_count + self.current_count >= limit { + break; + } + } + } + } + Some(path) => { + let input_file = File::open(path)?; + let buffered = io::BufReader::new(input_file); + let lines = buffered.lines(); + for line in lines { + let json_str = line?; + match op { + BatchOp::Create => self.push_create(api_client, &json_str)?, + BatchOp::Update => self.push_update( + api_client, + &json_str, + mutations.clone().unwrap_or(vec![]), + )?, + BatchOp::Delete => self.push_delete(api_client, &json_str)?, + }; + if let Some(limit) = self.limit { + if self.total_count + self.current_count >= limit { + break; + } + } + } + } + } + self.flush(api_client)?; + Ok(()) + } + + pub fn push_create( + &mut self, + api_client: &mut FatcatApiClient, + json_str: &str, + ) -> Result<models::EntityEdit> { + let editgroup_id = self.increment_editgroup(api_client)?; + api_client.create_entity_from_json(self.entity_type, json_str, editgroup_id) + } + + pub fn push_update( + &mut self, + api_client: &mut FatcatApiClient, + json_str: &str, + mutations: Vec<Mutation>, + ) -> Result<models::EntityEdit> { + let obj: serde_json::Value = serde_json::from_str(json_str)?; + let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident? + let editgroup_id = self.increment_editgroup(api_client)?; + let mut entity = entity_model_from_json_str(self.entity_type, &json_str)?; + entity.mutate(mutations)?; + api_client.update_entity_from_json( + Specifier::from_ident(self.entity_type, ident.to_string()), + &entity.to_json_string()?, + editgroup_id, + ) + } + + pub fn push_delete( + &mut self, + api_client: &mut FatcatApiClient, + json_str: &str, + ) -> Result<models::EntityEdit> { + let obj: serde_json::Value = serde_json::from_str(json_str)?; + let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident? + let editgroup_id = self.increment_editgroup(api_client)?; + api_client.delete_entity( + Specifier::from_ident(self.entity_type, ident.to_string()), + editgroup_id, + ) + } + + pub fn increment_editgroup(&mut self, api_client: &mut FatcatApiClient) -> Result<String> { + if self.current_count >= self.batch_size.try_into().unwrap() { + self.flush(api_client)?; + }; + self.current_count += 1; + if let Some(eg) = &self.current_editgroup_id { + return Ok(eg.to_string()); + } + let eg = api_client.create_editgroup(Some(self.editgroup_description.clone()))?; + self.current_editgroup_id = eg.editgroup_id; + Ok(self.current_editgroup_id.as_ref().unwrap().to_string()) + } + + pub fn flush(&mut self, api_client: &mut FatcatApiClient) -> Result<()> { + if self.current_count > 0 && self.current_editgroup_id.is_some() { + let eg_id = self.current_editgroup_id.clone().unwrap(); + if self.auto_accept { + api_client.accept_editgroup(eg_id)?; + } else { + api_client.update_editgroup_submit(eg_id, true)?; + } + self.total_count += self.current_count; + self.current_count = 0; + self.current_editgroup_id = None; + } + Ok(()) + } +} diff --git a/rust/fatcat-cli/src/download.rs b/rust/fatcat-cli/src/download.rs index 5500a7a..7821b70 100644 --- a/rust/fatcat-cli/src/download.rs +++ b/rust/fatcat-cli/src/download.rs @@ -1,23 +1,56 @@ use anyhow::{anyhow, Context, Result}; use fatcat_openapi::models::{FileEntity, ReleaseEntity}; use indicatif::ProgressBar; +use log::info; use reqwest::header::USER_AGENT; +use std::fmt; use std::fs::File; -use url::Url; +use std::io::{self, BufRead}; use std::path::Path; - +use std::path::PathBuf; +use url::Url; #[derive(Debug, PartialEq, Clone)] pub enum DownloadStatus { Exists(String), Downloaded(String), NetworkError(String), - NoPublicAccess, + HttpError(u16), + PartialExists(String), + NoPublicFile, FileMissingMetadata, WrongSize, WrongHash, } +impl DownloadStatus { + pub fn details(&self) -> Option<String> { + match self { + Self::Exists(p) => Some(p.to_string()), + Self::Downloaded(p) => Some(p.to_string()), + Self::NetworkError(p) => Some(p.to_string()), + Self::PartialExists(p) => Some(p.to_string()), + _ => None, + } + } +} + +impl fmt::Display for DownloadStatus { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Exists(_) => write!(f, "exists"), + Self::Downloaded(_) => write!(f, "success"), + Self::NetworkError(_) => write!(f, "network-error"), + Self::HttpError(p) => write!(f, "http-{}", p), + Self::PartialExists(_) => write!(f, "partial-exists"), + Self::NoPublicFile => write!(f, "no-public-file"), + Self::FileMissingMetadata => write!(f, "missing-file-metadata"), + Self::WrongSize => write!(f, "wrong-file-size"), + Self::WrongHash => write!(f, "wrong-file-hash"), + } + } +} + // eg, https://web.archive.org/web/20140802044207/http://www.geo.coop:80/sites/default/files/labs_of_oligarchy.pdf fn rewrite_wayback_url(url: Url) -> Result<Url> { // TODO: make this function correct, and add tests @@ -33,8 +66,8 @@ fn rewrite_wayback_url(url: Url) -> Result<Url> { } /// Attempts to download a file entity, including verifying checksum. -pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> { - let sha1hex = match fe.sha1 { +pub fn download_file(fe: &FileEntity) -> Result<DownloadStatus> { + let sha1hex = match &fe.sha1 { Some(v) => v, None => return Ok(DownloadStatus::FileMissingMetadata), }; @@ -48,72 +81,106 @@ pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> { Some("application/postscript") => ".pdf", Some("text/html") => ".html", Some("text/xml") => ".xml", - _ => "", + _ => ".unknown", }; // TODO: output directory let path_string = format!("{}{}", sha1hex, file_suffix); let final_path = Path::new(&path_string); + // NOTE: this isn't perfect; there could have been a race condition if final_path.exists() { - return Ok(DownloadStatus::Exists(final_path.to_string_lossy().to_string())); + return Ok(DownloadStatus::Exists( + final_path.to_string_lossy().to_string(), + )); }; let path_string = format!("{}{}.partial", sha1hex, file_suffix); let download_path = Path::new(&path_string); + // TODO: only archive.org URLs (?) let raw_url = match fe.urls.as_ref() { - None => return Ok(DownloadStatus::NoPublicAccess), - Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicAccess), + None => return Ok(DownloadStatus::NoPublicFile), + Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicFile), // TODO: remove clone (?) // TODO: better heuristic than "just try first URL" Some(url_list) => url_list[0].url.clone(), }; - // TODO: only archive.org URLs (?) - let raw_url = fe.urls.unwrap()[0].url.clone(); let mut url = Url::parse(&raw_url)?; if url.host_str() == Some("web.archive.org") { url = rewrite_wayback_url(url)?; } - // TODO: open temporary file (real file plus suffix?) - let download_file = File::create(download_path)?; + let download_file = match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(download_path) + { + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + return Ok(DownloadStatus::PartialExists( + download_path.to_string_lossy().to_string(), + )) + } + Err(e) => return Err(e).context("opening temporary download file"), + Ok(f) => f, + }; - println!("downloading: {}", url); + // TODO: print to stderr + info!("downloading: {}", url); let client = reqwest::blocking::Client::new(); - let mut resp = client + let mut resp = match client .get(url) .header(USER_AGENT, "fatcat-cli/0.0.0") - .send()?; + .send() + { + Ok(r) => r, + Err(e) => { + std::fs::remove_file(download_path)?; + return Ok(DownloadStatus::NetworkError(format!("{:?}", e))); + } + }; - // TODO: parse headers - // TODO: resp.error_for_status()?; + // TODO: parse headers, eg size (?) if !resp.status().is_success() { - return Ok(DownloadStatus::NetworkError(format!("{}", resp.status()))); + std::fs::remove_file(download_path)?; + return Ok(DownloadStatus::HttpError(resp.status().as_u16())); } // TODO: what if no filesize? // TODO: compare with resp.content_length(() -> Option<u64> let pb = ProgressBar::new(fe.size.unwrap() as u64); - let out_size = resp.copy_to(&mut pb.wrap_write(download_file))?; + let out_size = match resp.copy_to(&mut pb.wrap_write(download_file)) { + Ok(r) => r, + Err(e) => { + std::fs::remove_file(download_path)?; + return Ok(DownloadStatus::NetworkError(format!("{:?}", e))); + } + }; if out_size != expected_size { - // TODO: delete partial file? + std::fs::remove_file(download_path)?; return Ok(DownloadStatus::WrongSize); } - Ok(DownloadStatus::Downloaded(final_path.to_string_lossy().to_string())) + std::fs::rename(download_path, final_path)?; + Ok(DownloadStatus::Downloaded( + final_path.to_string_lossy().to_string(), + )) } -pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> { - let file_entities = match re.files { - None => return Err(anyhow!("expected file sub-entities to be 'expanded' on release")), +pub fn download_release(re: &ReleaseEntity) -> Result<DownloadStatus> { + let file_entities = match &re.files { + None => { + return Err(anyhow!( + "expected file sub-entities to be 'expanded' on release" + )) + } Some(list) => list, }; - let mut status = DownloadStatus::NoPublicAccess; + let mut status = DownloadStatus::NoPublicFile; for fe in file_entities { - status = download_file(fe)?; + status = download_file(&fe)?; match status { DownloadStatus::Exists(_) | DownloadStatus::Downloaded(_) => break, _ => (), @@ -121,3 +188,74 @@ pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> { } Ok(status) } + +/// Tries either file or release +fn download_entity(json_str: String) -> Result<DownloadStatus> { + let release_attempt = serde_json::from_str::<ReleaseEntity>(&json_str); + if let Ok(re) = release_attempt { + if re.ident.is_some() && (re.title.is_some() || re.files.is_some()) { + let status = download_release(&re)?; + println!( + "release_{}\t{}\t{}", + re.ident.unwrap(), + status, + status.details().unwrap_or("".to_string()) + ); + return Ok(status); + }; + } + let file_attempt = + serde_json::from_str::<FileEntity>(&json_str).context("parsing entity for download"); + match file_attempt { + Ok(fe) => { + if fe.ident.is_some() && fe.urls.is_some() { + let status = download_file(&fe)?; + println!( + "file_{}\t{}\t{}", + fe.ident.unwrap(), + status, + status.details().unwrap_or("".to_string()) + ); + return Ok(status); + } else { + Err(anyhow!("not a file entity (JSON)")) + } + } + Err(e) => Err(e), + } +} + +pub fn download_batch(input_path: Option<PathBuf>, limit: Option<u64>) -> Result<u64> { + let count = 0; + match input_path { + None => { + let stdin = io::stdin(); + let stdin_lock = stdin.lock(); + let lines = stdin_lock.lines(); + for line in lines { + let json_str = line?; + download_entity(json_str)?; + if let Some(limit) = limit { + if count >= limit { + break; + } + } + } + } + Some(path) => { + let input_file = File::open(path)?; + let buffered = io::BufReader::new(input_file); + let lines = buffered.lines(); + for line in lines { + let json_str = line?; + download_entity(json_str)?; + if let Some(limit) = limit { + if count >= limit { + break; + } + } + } + } + }; + Ok(count) +} diff --git a/rust/fatcat-cli/src/entities.rs b/rust/fatcat-cli/src/entities.rs index 314af51..d61f6dc 100644 --- a/rust/fatcat-cli/src/entities.rs +++ b/rust/fatcat-cli/src/entities.rs @@ -1,11 +1,10 @@ -use crate::Specifier; +use crate::{EntityType, Specifier}; use anyhow::{anyhow, Context, Result}; use fatcat_openapi::models; use lazy_static::lazy_static; use log::{self, info}; use regex::Regex; -use std::io::BufRead; -use std::io::Read; +use std::io::{BufRead, Read}; use std::path::PathBuf; use std::str::FromStr; @@ -94,6 +93,35 @@ pub fn read_entity_file(input_path: Option<PathBuf>) -> Result<String> { } } +pub fn entity_model_from_json_str( + entity_type: EntityType, + json_str: &str, +) -> Result<Box<dyn ApiEntityModel>> { + match entity_type { + EntityType::Release => Ok(Box::new(serde_json::from_str::<models::ReleaseEntity>( + &json_str, + )?)), + EntityType::Work => Ok(Box::new(serde_json::from_str::<models::WorkEntity>( + &json_str, + )?)), + EntityType::Container => Ok(Box::new(serde_json::from_str::<models::ContainerEntity>( + &json_str, + )?)), + EntityType::Creator => Ok(Box::new(serde_json::from_str::<models::CreatorEntity>( + &json_str, + )?)), + EntityType::File => Ok(Box::new(serde_json::from_str::<models::FileEntity>( + &json_str, + )?)), + EntityType::FileSet => Ok(Box::new(serde_json::from_str::<models::FilesetEntity>( + &json_str, + )?)), + EntityType::WebCapture => Ok(Box::new(serde_json::from_str::<models::WebcaptureEntity>( + &json_str, + )?)), + } +} + pub trait ApiModelSer { fn to_json_string(&self) -> Result<String>; fn to_toml_string(&self) -> Result<String>; diff --git a/rust/fatcat-cli/src/lib.rs b/rust/fatcat-cli/src/lib.rs index 206fd09..d648c1c 100644 --- a/rust/fatcat-cli/src/lib.rs +++ b/rust/fatcat-cli/src/lib.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Context, Result}; use data_encoding::BASE64; use macaroon::{Macaroon, Verifier}; +use std::path::PathBuf; use std::str::FromStr; mod api; @@ -12,10 +13,14 @@ mod specifier; pub use api::FatcatApiClient; pub use commands::{ - print_changelog_entries, print_editgroups, print_entity_histories, ClientStatus, edit_entity_locally, + edit_entity_locally, print_changelog_entries, print_editgroups, print_entity_histories, + BatchGrouper, BatchOp, ClientStatus, +}; +pub use download::{download_batch, download_file, download_release}; +pub use entities::{ + entity_model_from_json_str, read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer, + Mutation, }; -pub use download::{download_release, download_file}; -pub use entities::{read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation}; pub use search::crude_search; pub use specifier::Specifier; @@ -78,3 +83,11 @@ pub fn parse_macaroon_editor_id(s: &str) -> Result<String> { verifier.satisfy_exact(&format!("editor_id = {}", editor_id.to_string())); Ok(editor_id) } + +pub fn path_or_stdin(raw: Option<PathBuf>) -> Option<PathBuf> { + // treat "-" as "use stdin" + match raw { + Some(s) if s.to_string_lossy() == "-" => None, + _ => raw, + } +} diff --git a/rust/fatcat-cli/src/main.rs b/rust/fatcat-cli/src/main.rs index 79efbd3..3b0d382 100644 --- a/rust/fatcat-cli/src/main.rs +++ b/rust/fatcat-cli/src/main.rs @@ -1,7 +1,7 @@ +use crate::{path_or_stdin, BatchGrouper, BatchOp}; use anyhow::{anyhow, Context, Result}; use fatcat_cli::ApiModelSer; use fatcat_cli::*; -use fatcat_openapi::models; #[allow(unused_imports)] use log::{self, debug, info}; use std::io::Write; @@ -100,18 +100,7 @@ enum BatchCommand { batch_size: u64, #[structopt(long)] - dry_run: bool, - - #[structopt(long)] auto_accept: bool, - - #[structopt( - long = "--editgroup-id", - short, - env = "FATCAT_EDITGROUP", - hide_env_values = true - )] - editgroup_id: String, }, Update { entity_type: EntityType, @@ -121,18 +110,7 @@ enum BatchCommand { batch_size: u64, #[structopt(long)] - dry_run: bool, - - #[structopt(long)] auto_accept: bool, - - #[structopt( - long = "--editgroup-id", - short, - env = "FATCAT_EDITGROUP", - hide_env_values = true - )] - editgroup_id: String, }, Delete { entity_type: EntityType, @@ -141,18 +119,7 @@ enum BatchCommand { batch_size: u64, #[structopt(long)] - dry_run: bool, - - #[structopt(long)] auto_accept: bool, - - #[structopt( - long = "--editgroup-id", - short, - env = "FATCAT_EDITGROUP", - hide_env_values = true - )] - editgroup_id: String, }, Download {}, } @@ -231,6 +198,7 @@ enum Command { #[structopt(long)] json: bool, + #[allow(dead_code)] #[structopt(long)] toml: bool, @@ -243,7 +211,7 @@ enum Command { History { specifier: Specifier, - #[structopt(long, short = "-n", default_value = "20")] + #[structopt(long, short = "-n", default_value = "100")] limit: u64, #[structopt(long)] @@ -348,13 +316,13 @@ fn run(opt: Opt) -> Result<()> { specifier, expand, hide, - json: _, + json, toml, } => { let result = specifier.get_from_api(&mut api_client, expand, hide)?; if toml { writeln!(&mut std::io::stdout(), "{}", result.to_toml_string()?)? - } else { + } else if json || true { writeln!(&mut std::io::stdout(), "{}", result.to_json_string()?)? } } @@ -398,13 +366,16 @@ fn run(opt: Opt) -> Result<()> { toml: _, editing_command, } => { - let ee = edit_entity_locally(&mut api_client, specifier, editgroup_id, json, editing_command)?; + let ee = edit_entity_locally( + &mut api_client, + specifier, + editgroup_id, + json, + editing_command, + )?; println!("{}", serde_json::to_string(&ee)?); } - Command::Changelog { - limit, - json, - } => { + Command::Changelog { limit, json } => { let resp = api_client .rt .block_on(api_client.api.get_changelog(Some(limit))) @@ -414,9 +385,8 @@ fn run(opt: Opt) -> Result<()> { print_changelog_entries(change_list, json)?; } other => { - return Err(anyhow!("{:?}", other)).with_context(|| { - format!("failed to fetch changelogs") - }) + return Err(anyhow!("{:?}", other)) + .with_context(|| format!("failed to fetch changelogs")) } } } @@ -425,90 +395,99 @@ fn run(opt: Opt) -> Result<()> { BatchCommand::Create { entity_type, batch_size, - dry_run, auto_accept, - editgroup_id, }, input_path, limit, } => { - unimplemented!("batch create") - }, + let input_path = path_or_stdin(input_path); + let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept); + batch.run(&mut api_client, input_path, BatchOp::Create, None)?; + } Command::Batch { cmd: BatchCommand::Update { entity_type, mutations, batch_size, - dry_run, auto_accept, - editgroup_id, }, input_path, limit, } => { - unimplemented!("batch update") - }, + let input_path = path_or_stdin(input_path); + let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept); + batch.run( + &mut api_client, + input_path, + BatchOp::Update, + Some(mutations), + )?; + } Command::Batch { cmd: BatchCommand::Delete { entity_type, batch_size, - dry_run, auto_accept, - editgroup_id, }, input_path, limit, } => { - unimplemented!("batch delete") - }, + let input_path = path_or_stdin(input_path); + let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept); + batch.run(&mut api_client, input_path, BatchOp::Delete, None)?; + } Command::Batch { - cmd: - BatchCommand::Download { - // TODO - }, + cmd: BatchCommand::Download {}, input_path, limit, } => { - unimplemented!("batch create") - }, - Command::Download{specifier} => { + let input_path = path_or_stdin(input_path); + download_batch(input_path, limit)?; + } + Command::Download { specifier } => { // run lookups if necessary (inefficient) let specifier = match specifier { - Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => specifier.into_entity_specifier(&mut api_client)?, + Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => { + specifier.into_entity_specifier(&mut api_client)? + } _ => specifier, }; let status = match specifier { Specifier::Release(ident) => { - let result = api_client.rt.block_on( - api_client.api.get_release(ident.clone(), Some("files".to_string()), Some("abstracts,refs".to_string())) - )?; + let result = api_client.rt.block_on(api_client.api.get_release( + ident.clone(), + Some("files".to_string()), + Some("abstracts,refs".to_string()), + ))?; let release_entity = match result { - fatcat_openapi::GetReleaseResponse::FoundEntity(model) => { - Ok(model) - }, + fatcat_openapi::GetReleaseResponse::FoundEntity(model) => Ok(model), resp => Err(anyhow!("{:?}", resp)) .with_context(|| format!("API GET failed: {:?}", ident)), }?; - download_release(release_entity) - }, + download_release(&release_entity) + } Specifier::File(ident) => { - let result = api_client.rt.block_on( - api_client.api.get_file(ident.clone(), None, None) - )?; + let result = api_client.rt.block_on(api_client.api.get_file( + ident.clone(), + None, + None, + ))?; let file_entity = match result { - fatcat_openapi::GetFileResponse::FoundEntity(model) => { - Ok(model) - }, + fatcat_openapi::GetFileResponse::FoundEntity(model) => Ok(model), resp => Err(anyhow!("{:?}", resp)) .with_context(|| format!("API GET failed: {:?}", ident)), }?; - download_file(file_entity) - }, + download_file(&file_entity) + } other => Err(anyhow!("Don't know how to download: {:?}", other)), }?; - println!("{:?}", status); + if let Some(detail) = status.details() { + println!("{}: {}", status, detail); + } else { + println!("{}", status); + } } Command::Search { entity_type, @@ -519,7 +498,7 @@ fn run(opt: Opt) -> Result<()> { hide, } => { let limit: Option<u64> = match limit { - l if l < 0 => None, + l if l <= 0 => None, l => Some(l as u64), }; let results = fatcat_cli::crude_search(&opt.search_host, entity_type, limit, terms) @@ -618,45 +597,14 @@ fn run(opt: Opt) -> Result<()> { Command::Editgroup { cmd: EditgroupCommand::Create { description }, } => { - let mut eg = models::Editgroup::new(); - eg.description = Some(description); - eg.extra = Some({ - let mut extra = std::collections::HashMap::new(); - extra.insert( - "agent".to_string(), - serde_json::Value::String("fatcat-cli".to_string()), - ); - extra - }); - let result = api_client - .rt - .block_on(api_client.api.create_editgroup(eg))?; - match result { - fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => { - println!("{}", serde_json::to_string(&eg)?) - } - other => return Err(anyhow!("{:?}", other)).context("failed to create editgroup"), - } + let eg = api_client.create_editgroup(Some(description))?; + println!("{}", serde_json::to_string(&eg)?) } Command::Editgroup { cmd: EditgroupCommand::Accept { editgroup_id }, } => { - let result = api_client - .rt - .block_on(api_client.api.accept_editgroup(editgroup_id.clone())) - .context("accept editgroup")?; - match result { - fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => { - println!("{}", serde_json::to_string(&msg)?) - } - other => { - return Err(anyhow!( - "failed to accept editgroup {}: {:?}", - editgroup_id, - other - )) - } - } + let msg = api_client.accept_editgroup(editgroup_id.clone())?; + println!("{}", serde_json::to_string(&msg)?); } Command::Editgroup { cmd: EditgroupCommand::Submit { editgroup_id }, diff --git a/rust/fatcat-cli/src/specifier.rs b/rust/fatcat-cli/src/specifier.rs index c7a7b57..0d8d209 100644 --- a/rust/fatcat-cli/src/specifier.rs +++ b/rust/fatcat-cli/src/specifier.rs @@ -1,4 +1,4 @@ -use crate::{ApiEntityModel, FatcatApiClient}; +use crate::{ApiEntityModel, EntityType, FatcatApiClient}; use anyhow::{anyhow, Context, Result}; use lazy_static::lazy_static; use regex::Regex; @@ -50,6 +50,18 @@ pub enum Specifier { } impl Specifier { + pub fn from_ident(entity_type: EntityType, ident: String) -> Specifier { + match entity_type { + EntityType::Release => Specifier::Release(ident), + EntityType::Work => Specifier::Work(ident), + EntityType::Container => Specifier::Container(ident), + EntityType::Creator => Specifier::Creator(ident), + EntityType::File => Specifier::File(ident), + EntityType::FileSet => Specifier::FileSet(ident), + EntityType::WebCapture => Specifier::WebCapture(ident), + } + } + /// If this Specifier is a lookup, call the API to do the lookup and return the resulting /// specific entity specifier (eg, with an FCID). If already specific, just pass through. pub fn into_entity_specifier(self, api_client: &mut FatcatApiClient) -> Result<Specifier> { @@ -382,19 +394,18 @@ impl Specifier { .rt .block_on(api_client.api.get_work_history(fcid.to_string(), limit))? { - fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => { - Ok(entries) - } + fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => Ok(entries), fatcat_openapi::GetWorkHistoryResponse::NotFound(err) => { Err(anyhow!("Not Found: {}", err.message)) } resp => Err(anyhow!("{:?}", resp)) .with_context(|| format!("API GET failed: {:?}", self)), }, - Container(fcid) => match api_client - .rt - .block_on(api_client.api.get_container_history(fcid.to_string(), limit))? - { + Container(fcid) => match api_client.rt.block_on( + api_client + .api + .get_container_history(fcid.to_string(), limit), + )? { fatcat_openapi::GetContainerHistoryResponse::FoundEntityHistory(entries) => { Ok(entries) } @@ -421,9 +432,7 @@ impl Specifier { .rt .block_on(api_client.api.get_file_history(fcid.to_string(), limit))? { - fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => { - Ok(entries) - } + fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => Ok(entries), fatcat_openapi::GetFileHistoryResponse::NotFound(err) => { Err(anyhow!("Not Found: {}", err.message)) } @@ -443,10 +452,11 @@ impl Specifier { resp => Err(anyhow!("{:?}", resp)) .with_context(|| format!("API GET failed: {:?}", self)), }, - WebCapture(fcid) => match api_client - .rt - .block_on(api_client.api.get_webcapture_history(fcid.to_string(), limit))? - { + WebCapture(fcid) => match api_client.rt.block_on( + api_client + .api + .get_webcapture_history(fcid.to_string(), limit), + )? { fatcat_openapi::GetWebcaptureHistoryResponse::FoundEntityHistory(entries) => { Ok(entries) } |