From 0cbbd85d64b7de6ba89a9c39897e363f6342ee03 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 5 Feb 2021 21:05:30 -0800
Subject: lots more progress on batch commands (and more)
---
rust/fatcat-cli/src/api.rs | 34 +++++++
rust/fatcat-cli/src/commands.rs | 189 ++++++++++++++++++++++++++++++++++++--
rust/fatcat-cli/src/download.rs | 192 +++++++++++++++++++++++++++++++++------
rust/fatcat-cli/src/entities.rs | 34 ++++++-
rust/fatcat-cli/src/lib.rs | 19 +++-
rust/fatcat-cli/src/main.rs | 184 ++++++++++++++-----------------------
rust/fatcat-cli/src/specifier.rs | 40 +++++---
7 files changed, 520 insertions(+), 172 deletions(-)
(limited to 'rust')
diff --git a/rust/fatcat-cli/src/api.rs b/rust/fatcat-cli/src/api.rs
index cc6fa6a..2463aab 100644
--- a/rust/fatcat-cli/src/api.rs
+++ b/rust/fatcat-cli/src/api.rs
@@ -403,4 +403,38 @@ impl FatcatApiClient {
}
.with_context(|| format!("failed to update {:?}", specifier))
}
+
+ pub fn create_editgroup(&mut self, description: Option<String>) -> Result<models::Editgroup> {
+ let mut eg = models::Editgroup::new();
+ eg.description = description;
+ eg.extra = Some({
+ let mut extra = std::collections::HashMap::new();
+ extra.insert(
+ "agent".to_string(),
+ serde_json::Value::String("fatcat-cli".to_string()),
+ // TODO: version?
+ );
+ extra
+ });
+ let result = self.rt.block_on(self.api.create_editgroup(eg))?;
+ match result {
+ fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => Ok(eg),
+ other => Err(anyhow!("{:?}", other)).context("failed to create editgroup"),
+ }
+ }
+
+ pub fn accept_editgroup(&mut self, editgroup_id: String) -> Result<models::Success> {
+ let result = self
+ .rt
+ .block_on(self.api.accept_editgroup(editgroup_id.clone()))
+ .context("accept editgroup")?;
+ match result {
+ fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => Ok(msg),
+ other => Err(anyhow!(
+ "failed to accept editgroup {}: {:?}",
+ editgroup_id,
+ other
+ )),
+ }
+ }
}
diff --git a/rust/fatcat-cli/src/commands.rs b/rust/fatcat-cli/src/commands.rs
index 30fa0c4..15bfc81 100644
--- a/rust/fatcat-cli/src/commands.rs
+++ b/rust/fatcat-cli/src/commands.rs
@@ -3,14 +3,17 @@ use chrono_humanize::HumanTime;
use fatcat_openapi::models;
#[allow(unused_imports)]
use log::{self, debug, info};
-use std::io::{Write, BufRead};
+use std::convert::TryInto;
+use std::fs::File;
+use std::io::{self, BufRead, Write};
+use std::path::PathBuf;
use tabwriter::TabWriter;
use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
-use crate::api::FatcatApiClient;
-//use crate::download::download_file;
-use crate::entities::{ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation, read_entity_file};
-use crate::specifier::Specifier;
+use crate::{
+ entity_model_from_json_str, read_entity_file, ApiModelSer, EntityType, FatcatApiClient,
+ Mutation, Specifier,
+};
// Want to show:
// - whether api_token found
@@ -271,7 +274,13 @@ pub fn print_entity_histories(
Ok(())
}
-pub fn edit_entity_locally(api_client: &mut FatcatApiClient, specifier: Specifier, editgroup_id: String, json: bool, editing_command: String) -> Result<models::EntityEdit> {
+pub fn edit_entity_locally(
+ api_client: &mut FatcatApiClient,
+ specifier: Specifier,
+ editgroup_id: String,
+ json: bool,
+ editing_command: String,
+) -> Result<models::EntityEdit> {
// TODO: fetch editgroup, check if this entity is already being updated in it. If so,
// need to fetch that revision, do the edit, parse that synatx is good, then delete the
// existing edit and update with the new one.
@@ -312,3 +321,171 @@ pub fn edit_entity_locally(api_client: &mut FatcatApiClient, specifier: Specifi
.context("updating after edit")?;
Ok(ee)
}
+
+pub enum BatchOp {
+ Create,
+ Update,
+ Delete,
+}
+
+pub struct BatchGrouper {
+ entity_type: EntityType,
+ batch_size: u64,
+ limit: Option<u64>,
+ auto_accept: bool,
+ editgroup_description: String,
+ current_count: u64,
+ current_editgroup_id: Option<String>,
+ total_count: u64,
+}
+
+// Note: should be possible to add support for the single-call batch create endpoint by storing
+// edits in a batch within this object. Might need to change control flow a bit. This optimization
+// was mostly intended for bootstrapping of tens of thousands of entities, so not including for
+// now.
+impl BatchGrouper {
+ pub fn new(
+ entity_type: EntityType,
+ batch_size: u64,
+ limit: Option<u64>,
+ auto_accept: bool,
+ ) -> Self {
+ let editgroup_description = "part of a fatcat-cli batch operation".to_string();
+ BatchGrouper {
+ entity_type,
+ batch_size,
+ limit,
+ auto_accept,
+ editgroup_description,
+ current_count: 0,
+ current_editgroup_id: None,
+ total_count: 0,
+ }
+ }
+
+ pub fn run(
+ &mut self,
+ api_client: &mut FatcatApiClient,
+ input_path: Option<PathBuf>,
+ op: BatchOp,
+ mutations: Option<Vec<Mutation>>,
+ ) -> Result<()> {
+ match input_path {
+ None => {
+ let stdin = io::stdin();
+ let stdin_lock = stdin.lock();
+ let lines = stdin_lock.lines();
+ for line in lines {
+ let json_str = line?;
+ match op {
+ BatchOp::Create => self.push_create(api_client, &json_str)?,
+ BatchOp::Update => self.push_update(
+ api_client,
+ &json_str,
+ mutations.clone().unwrap_or(vec![]),
+ )?,
+ BatchOp::Delete => self.push_delete(api_client, &json_str)?,
+ };
+ if let Some(limit) = self.limit {
+ if self.total_count + self.current_count >= limit {
+ break;
+ }
+ }
+ }
+ }
+ Some(path) => {
+ let input_file = File::open(path)?;
+ let buffered = io::BufReader::new(input_file);
+ let lines = buffered.lines();
+ for line in lines {
+ let json_str = line?;
+ match op {
+ BatchOp::Create => self.push_create(api_client, &json_str)?,
+ BatchOp::Update => self.push_update(
+ api_client,
+ &json_str,
+ mutations.clone().unwrap_or(vec![]),
+ )?,
+ BatchOp::Delete => self.push_delete(api_client, &json_str)?,
+ };
+ if let Some(limit) = self.limit {
+ if self.total_count + self.current_count >= limit {
+ break;
+ }
+ }
+ }
+ }
+ }
+ self.flush(api_client)?;
+ Ok(())
+ }
+
+ pub fn push_create(
+ &mut self,
+ api_client: &mut FatcatApiClient,
+ json_str: &str,
+ ) -> Result<models::EntityEdit> {
+ let editgroup_id = self.increment_editgroup(api_client)?;
+ api_client.create_entity_from_json(self.entity_type, json_str, editgroup_id)
+ }
+
+ pub fn push_update(
+ &mut self,
+ api_client: &mut FatcatApiClient,
+ json_str: &str,
+ mutations: Vec<Mutation>,
+ ) -> Result<models::EntityEdit> {
+ let obj: serde_json::Value = serde_json::from_str(json_str)?;
+ let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident?
+ let editgroup_id = self.increment_editgroup(api_client)?;
+ let mut entity = entity_model_from_json_str(self.entity_type, &json_str)?;
+ entity.mutate(mutations)?;
+ api_client.update_entity_from_json(
+ Specifier::from_ident(self.entity_type, ident.to_string()),
+ &entity.to_json_string()?,
+ editgroup_id,
+ )
+ }
+
+ pub fn push_delete(
+ &mut self,
+ api_client: &mut FatcatApiClient,
+ json_str: &str,
+ ) -> Result<models::EntityEdit> {
+ let obj: serde_json::Value = serde_json::from_str(json_str)?;
+ let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident?
+ let editgroup_id = self.increment_editgroup(api_client)?;
+ api_client.delete_entity(
+ Specifier::from_ident(self.entity_type, ident.to_string()),
+ editgroup_id,
+ )
+ }
+
+ pub fn increment_editgroup(&mut self, api_client: &mut FatcatApiClient) -> Result<String> {
+ if self.current_count >= self.batch_size.try_into().unwrap() {
+ self.flush(api_client)?;
+ };
+ self.current_count += 1;
+ if let Some(eg) = &self.current_editgroup_id {
+ return Ok(eg.to_string());
+ }
+ let eg = api_client.create_editgroup(Some(self.editgroup_description.clone()))?;
+ self.current_editgroup_id = eg.editgroup_id;
+ Ok(self.current_editgroup_id.as_ref().unwrap().to_string())
+ }
+
+ pub fn flush(&mut self, api_client: &mut FatcatApiClient) -> Result<()> {
+ if self.current_count > 0 && self.current_editgroup_id.is_some() {
+ let eg_id = self.current_editgroup_id.clone().unwrap();
+ if self.auto_accept {
+ api_client.accept_editgroup(eg_id)?;
+ } else {
+ api_client.update_editgroup_submit(eg_id, true)?;
+ }
+ self.total_count += self.current_count;
+ self.current_count = 0;
+ self.current_editgroup_id = None;
+ }
+ Ok(())
+ }
+}
diff --git a/rust/fatcat-cli/src/download.rs b/rust/fatcat-cli/src/download.rs
index 5500a7a..7821b70 100644
--- a/rust/fatcat-cli/src/download.rs
+++ b/rust/fatcat-cli/src/download.rs
@@ -1,23 +1,56 @@
use anyhow::{anyhow, Context, Result};
use fatcat_openapi::models::{FileEntity, ReleaseEntity};
use indicatif::ProgressBar;
+use log::info;
use reqwest::header::USER_AGENT;
+use std::fmt;
use std::fs::File;
-use url::Url;
+use std::io::{self, BufRead};
use std::path::Path;
-
+use std::path::PathBuf;
+use url::Url;
#[derive(Debug, PartialEq, Clone)]
pub enum DownloadStatus {
Exists(String),
Downloaded(String),
NetworkError(String),
- NoPublicAccess,
+ HttpError(u16),
+ PartialExists(String),
+ NoPublicFile,
FileMissingMetadata,
WrongSize,
WrongHash,
}
+impl DownloadStatus {
+ pub fn details(&self) -> Option<String> {
+ match self {
+ Self::Exists(p) => Some(p.to_string()),
+ Self::Downloaded(p) => Some(p.to_string()),
+ Self::NetworkError(p) => Some(p.to_string()),
+ Self::PartialExists(p) => Some(p.to_string()),
+ _ => None,
+ }
+ }
+}
+
+impl fmt::Display for DownloadStatus {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Self::Exists(_) => write!(f, "exists"),
+ Self::Downloaded(_) => write!(f, "success"),
+ Self::NetworkError(_) => write!(f, "network-error"),
+ Self::HttpError(p) => write!(f, "http-{}", p),
+ Self::PartialExists(_) => write!(f, "partial-exists"),
+ Self::NoPublicFile => write!(f, "no-public-file"),
+ Self::FileMissingMetadata => write!(f, "missing-file-metadata"),
+ Self::WrongSize => write!(f, "wrong-file-size"),
+ Self::WrongHash => write!(f, "wrong-file-hash"),
+ }
+ }
+}
+
// eg, https://web.archive.org/web/20140802044207/http://www.geo.coop:80/sites/default/files/labs_of_oligarchy.pdf
fn rewrite_wayback_url(url: Url) -> Result<Url> {
// TODO: make this function correct, and add tests
@@ -33,8 +66,8 @@ fn rewrite_wayback_url(url: Url) -> Result<Url> {
}
/// Attempts to download a file entity, including verifying checksum.
-pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> {
- let sha1hex = match fe.sha1 {
+pub fn download_file(fe: &FileEntity) -> Result<DownloadStatus> {
+ let sha1hex = match &fe.sha1 {
Some(v) => v,
None => return Ok(DownloadStatus::FileMissingMetadata),
};
@@ -48,72 +81,106 @@ pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> {
Some("application/postscript") => ".pdf",
Some("text/html") => ".html",
Some("text/xml") => ".xml",
- _ => "",
+ _ => ".unknown",
};
// TODO: output directory
let path_string = format!("{}{}", sha1hex, file_suffix);
let final_path = Path::new(&path_string);
+ // NOTE: this isn't perfect; there could have been a race condition
if final_path.exists() {
- return Ok(DownloadStatus::Exists(final_path.to_string_lossy().to_string()));
+ return Ok(DownloadStatus::Exists(
+ final_path.to_string_lossy().to_string(),
+ ));
};
let path_string = format!("{}{}.partial", sha1hex, file_suffix);
let download_path = Path::new(&path_string);
+ // TODO: only archive.org URLs (?)
let raw_url = match fe.urls.as_ref() {
- None => return Ok(DownloadStatus::NoPublicAccess),
- Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicAccess),
+ None => return Ok(DownloadStatus::NoPublicFile),
+ Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicFile),
// TODO: remove clone (?)
// TODO: better heuristic than "just try first URL"
Some(url_list) => url_list[0].url.clone(),
};
- // TODO: only archive.org URLs (?)
- let raw_url = fe.urls.unwrap()[0].url.clone();
let mut url = Url::parse(&raw_url)?;
if url.host_str() == Some("web.archive.org") {
url = rewrite_wayback_url(url)?;
}
- // TODO: open temporary file (real file plus suffix?)
- let download_file = File::create(download_path)?;
+ let download_file = match std::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(download_path)
+ {
+ Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
+ return Ok(DownloadStatus::PartialExists(
+ download_path.to_string_lossy().to_string(),
+ ))
+ }
+ Err(e) => return Err(e).context("opening temporary download file"),
+ Ok(f) => f,
+ };
- println!("downloading: {}", url);
+ // TODO: print to stderr
+ info!("downloading: {}", url);
let client = reqwest::blocking::Client::new();
- let mut resp = client
+ let mut resp = match client
.get(url)
.header(USER_AGENT, "fatcat-cli/0.0.0")
- .send()?;
+ .send()
+ {
+ Ok(r) => r,
+ Err(e) => {
+ std::fs::remove_file(download_path)?;
+ return Ok(DownloadStatus::NetworkError(format!("{:?}", e)));
+ }
+ };
- // TODO: parse headers
- // TODO: resp.error_for_status()?;
+ // TODO: parse headers, eg size (?)
if !resp.status().is_success() {
- return Ok(DownloadStatus::NetworkError(format!("{}", resp.status())));
+ std::fs::remove_file(download_path)?;
+ return Ok(DownloadStatus::HttpError(resp.status().as_u16()));
}
// TODO: what if no filesize?
// TODO: compare with resp.content_length(() -> Option<u64>
let pb = ProgressBar::new(fe.size.unwrap() as u64);
- let out_size = resp.copy_to(&mut pb.wrap_write(download_file))?;
+ let out_size = match resp.copy_to(&mut pb.wrap_write(download_file)) {
+ Ok(r) => r,
+ Err(e) => {
+ std::fs::remove_file(download_path)?;
+ return Ok(DownloadStatus::NetworkError(format!("{:?}", e)));
+ }
+ };
if out_size != expected_size {
- // TODO: delete partial file?
+ std::fs::remove_file(download_path)?;
return Ok(DownloadStatus::WrongSize);
}
- Ok(DownloadStatus::Downloaded(final_path.to_string_lossy().to_string()))
+ std::fs::rename(download_path, final_path)?;
+ Ok(DownloadStatus::Downloaded(
+ final_path.to_string_lossy().to_string(),
+ ))
}
-pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> {
- let file_entities = match re.files {
- None => return Err(anyhow!("expected file sub-entities to be 'expanded' on release")),
+pub fn download_release(re: &ReleaseEntity) -> Result<DownloadStatus> {
+ let file_entities = match &re.files {
+ None => {
+ return Err(anyhow!(
+ "expected file sub-entities to be 'expanded' on release"
+ ))
+ }
Some(list) => list,
};
- let mut status = DownloadStatus::NoPublicAccess;
+ let mut status = DownloadStatus::NoPublicFile;
for fe in file_entities {
- status = download_file(fe)?;
+ status = download_file(&fe)?;
match status {
DownloadStatus::Exists(_) | DownloadStatus::Downloaded(_) => break,
_ => (),
@@ -121,3 +188,74 @@ pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> {
}
Ok(status)
}
+
+/// Tries either file or release
+fn download_entity(json_str: String) -> Result<DownloadStatus> {
+ let release_attempt = serde_json::from_str::<ReleaseEntity>(&json_str);
+ if let Ok(re) = release_attempt {
+ if re.ident.is_some() && (re.title.is_some() || re.files.is_some()) {
+ let status = download_release(&re)?;
+ println!(
+ "release_{}\t{}\t{}",
+ re.ident.unwrap(),
+ status,
+ status.details().unwrap_or("".to_string())
+ );
+ return Ok(status);
+ };
+ }
+ let file_attempt =
+ serde_json::from_str::<FileEntity>(&json_str).context("parsing entity for download");
+ match file_attempt {
+ Ok(fe) => {
+ if fe.ident.is_some() && fe.urls.is_some() {
+ let status = download_file(&fe)?;
+ println!(
+ "file_{}\t{}\t{}",
+ fe.ident.unwrap(),
+ status,
+ status.details().unwrap_or("".to_string())
+ );
+ return Ok(status);
+ } else {
+ Err(anyhow!("not a file entity (JSON)"))
+ }
+ }
+ Err(e) => Err(e),
+ }
+}
+
+pub fn download_batch(input_path: Option<PathBuf>, limit: Option<u64>) -> Result<u64> {
+ let count = 0;
+ match input_path {
+ None => {
+ let stdin = io::stdin();
+ let stdin_lock = stdin.lock();
+ let lines = stdin_lock.lines();
+ for line in lines {
+ let json_str = line?;
+ download_entity(json_str)?;
+ if let Some(limit) = limit {
+ if count >= limit {
+ break;
+ }
+ }
+ }
+ }
+ Some(path) => {
+ let input_file = File::open(path)?;
+ let buffered = io::BufReader::new(input_file);
+ let lines = buffered.lines();
+ for line in lines {
+ let json_str = line?;
+ download_entity(json_str)?;
+ if let Some(limit) = limit {
+ if count >= limit {
+ break;
+ }
+ }
+ }
+ }
+ };
+ Ok(count)
+}
diff --git a/rust/fatcat-cli/src/entities.rs b/rust/fatcat-cli/src/entities.rs
index 314af51..d61f6dc 100644
--- a/rust/fatcat-cli/src/entities.rs
+++ b/rust/fatcat-cli/src/entities.rs
@@ -1,11 +1,10 @@
-use crate::Specifier;
+use crate::{EntityType, Specifier};
use anyhow::{anyhow, Context, Result};
use fatcat_openapi::models;
use lazy_static::lazy_static;
use log::{self, info};
use regex::Regex;
-use std::io::BufRead;
-use std::io::Read;
+use std::io::{BufRead, Read};
use std::path::PathBuf;
use std::str::FromStr;
@@ -94,6 +93,35 @@ pub fn read_entity_file(input_path: Option<PathBuf>) -> Result<String> {
}
}
+pub fn entity_model_from_json_str(
+ entity_type: EntityType,
+ json_str: &str,
+) -> Result<Box<dyn ApiEntityModel>> {
+ match entity_type {
+ EntityType::Release => Ok(Box::new(serde_json::from_str::<models::ReleaseEntity>(
+ &json_str,
+ )?)),
+ EntityType::Work => Ok(Box::new(serde_json::from_str::<models::WorkEntity>(
+ &json_str,
+ )?)),
+ EntityType::Container => Ok(Box::new(serde_json::from_str::<models::ContainerEntity>(
+ &json_str,
+ )?)),
+ EntityType::Creator => Ok(Box::new(serde_json::from_str::<models::CreatorEntity>(
+ &json_str,
+ )?)),
+ EntityType::File => Ok(Box::new(serde_json::from_str::<models::FileEntity>(
+ &json_str,
+ )?)),
+ EntityType::FileSet => Ok(Box::new(serde_json::from_str::<models::FilesetEntity>(
+ &json_str,
+ )?)),
+ EntityType::WebCapture => Ok(Box::new(serde_json::from_str::<models::WebcaptureEntity>(
+ &json_str,
+ )?)),
+ }
+}
+
pub trait ApiModelSer {
fn to_json_string(&self) -> Result<String>;
fn to_toml_string(&self) -> Result<String>;
diff --git a/rust/fatcat-cli/src/lib.rs b/rust/fatcat-cli/src/lib.rs
index 206fd09..d648c1c 100644
--- a/rust/fatcat-cli/src/lib.rs
+++ b/rust/fatcat-cli/src/lib.rs
@@ -1,6 +1,7 @@
use anyhow::{anyhow, Context, Result};
use data_encoding::BASE64;
use macaroon::{Macaroon, Verifier};
+use std::path::PathBuf;
use std::str::FromStr;
mod api;
@@ -12,10 +13,14 @@ mod specifier;
pub use api::FatcatApiClient;
pub use commands::{
- print_changelog_entries, print_editgroups, print_entity_histories, ClientStatus, edit_entity_locally,
+ edit_entity_locally, print_changelog_entries, print_editgroups, print_entity_histories,
+ BatchGrouper, BatchOp, ClientStatus,
+};
+pub use download::{download_batch, download_file, download_release};
+pub use entities::{
+ entity_model_from_json_str, read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer,
+ Mutation,
};
-pub use download::{download_release, download_file};
-pub use entities::{read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation};
pub use search::crude_search;
pub use specifier::Specifier;
@@ -78,3 +83,11 @@ pub fn parse_macaroon_editor_id(s: &str) -> Result<String> {
verifier.satisfy_exact(&format!("editor_id = {}", editor_id.to_string()));
Ok(editor_id)
}
+
+pub fn path_or_stdin(raw: Option<PathBuf>) -> Option<PathBuf> {
+ // treat "-" as "use stdin"
+ match raw {
+ Some(s) if s.to_string_lossy() == "-" => None,
+ _ => raw,
+ }
+}
diff --git a/rust/fatcat-cli/src/main.rs b/rust/fatcat-cli/src/main.rs
index 79efbd3..3b0d382 100644
--- a/rust/fatcat-cli/src/main.rs
+++ b/rust/fatcat-cli/src/main.rs
@@ -1,7 +1,7 @@
+use crate::{path_or_stdin, BatchGrouper, BatchOp};
use anyhow::{anyhow, Context, Result};
use fatcat_cli::ApiModelSer;
use fatcat_cli::*;
-use fatcat_openapi::models;
#[allow(unused_imports)]
use log::{self, debug, info};
use std::io::Write;
@@ -99,19 +99,8 @@ enum BatchCommand {
#[structopt(long, default_value = "50")]
batch_size: u64,
- #[structopt(long)]
- dry_run: bool,
-
#[structopt(long)]
auto_accept: bool,
-
- #[structopt(
- long = "--editgroup-id",
- short,
- env = "FATCAT_EDITGROUP",
- hide_env_values = true
- )]
- editgroup_id: String,
},
Update {
entity_type: EntityType,
@@ -120,19 +109,8 @@ enum BatchCommand {
#[structopt(long, default_value = "50")]
batch_size: u64,
- #[structopt(long)]
- dry_run: bool,
-
#[structopt(long)]
auto_accept: bool,
-
- #[structopt(
- long = "--editgroup-id",
- short,
- env = "FATCAT_EDITGROUP",
- hide_env_values = true
- )]
- editgroup_id: String,
},
Delete {
entity_type: EntityType,
@@ -140,19 +118,8 @@ enum BatchCommand {
#[structopt(long, default_value = "50")]
batch_size: u64,
- #[structopt(long)]
- dry_run: bool,
-
#[structopt(long)]
auto_accept: bool,
-
- #[structopt(
- long = "--editgroup-id",
- short,
- env = "FATCAT_EDITGROUP",
- hide_env_values = true
- )]
- editgroup_id: String,
},
Download {},
}
@@ -231,6 +198,7 @@ enum Command {
#[structopt(long)]
json: bool,
+ #[allow(dead_code)]
#[structopt(long)]
toml: bool,
@@ -243,7 +211,7 @@ enum Command {
History {
specifier: Specifier,
- #[structopt(long, short = "-n", default_value = "20")]
+ #[structopt(long, short = "-n", default_value = "100")]
limit: u64,
#[structopt(long)]
@@ -348,13 +316,13 @@ fn run(opt: Opt) -> Result<()> {
specifier,
expand,
hide,
- json: _,
+ json,
toml,
} => {
let result = specifier.get_from_api(&mut api_client, expand, hide)?;
if toml {
writeln!(&mut std::io::stdout(), "{}", result.to_toml_string()?)?
- } else {
+ } else if json || true {
writeln!(&mut std::io::stdout(), "{}", result.to_json_string()?)?
}
}
@@ -398,13 +366,16 @@ fn run(opt: Opt) -> Result<()> {
toml: _,
editing_command,
} => {
- let ee = edit_entity_locally(&mut api_client, specifier, editgroup_id, json, editing_command)?;
+ let ee = edit_entity_locally(
+ &mut api_client,
+ specifier,
+ editgroup_id,
+ json,
+ editing_command,
+ )?;
println!("{}", serde_json::to_string(&ee)?);
}
- Command::Changelog {
- limit,
- json,
- } => {
+ Command::Changelog { limit, json } => {
let resp = api_client
.rt
.block_on(api_client.api.get_changelog(Some(limit)))
@@ -414,9 +385,8 @@ fn run(opt: Opt) -> Result<()> {
print_changelog_entries(change_list, json)?;
}
other => {
- return Err(anyhow!("{:?}", other)).with_context(|| {
- format!("failed to fetch changelogs")
- })
+ return Err(anyhow!("{:?}", other))
+ .with_context(|| format!("failed to fetch changelogs"))
}
}
}
@@ -425,90 +395,99 @@ fn run(opt: Opt) -> Result<()> {
BatchCommand::Create {
entity_type,
batch_size,
- dry_run,
auto_accept,
- editgroup_id,
},
input_path,
limit,
} => {
- unimplemented!("batch create")
- },
+ let input_path = path_or_stdin(input_path);
+ let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+ batch.run(&mut api_client, input_path, BatchOp::Create, None)?;
+ }
Command::Batch {
cmd:
BatchCommand::Update {
entity_type,
mutations,
batch_size,
- dry_run,
auto_accept,
- editgroup_id,
},
input_path,
limit,
} => {
- unimplemented!("batch update")
- },
+ let input_path = path_or_stdin(input_path);
+ let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+ batch.run(
+ &mut api_client,
+ input_path,
+ BatchOp::Update,
+ Some(mutations),
+ )?;
+ }
Command::Batch {
cmd:
BatchCommand::Delete {
entity_type,
batch_size,
- dry_run,
auto_accept,
- editgroup_id,
},
input_path,
limit,
} => {
- unimplemented!("batch delete")
- },
+ let input_path = path_or_stdin(input_path);
+ let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+ batch.run(&mut api_client, input_path, BatchOp::Delete, None)?;
+ }
Command::Batch {
- cmd:
- BatchCommand::Download {
- // TODO
- },
+ cmd: BatchCommand::Download {},
input_path,
limit,
} => {
- unimplemented!("batch create")
- },
- Command::Download{specifier} => {
+ let input_path = path_or_stdin(input_path);
+ download_batch(input_path, limit)?;
+ }
+ Command::Download { specifier } => {
// run lookups if necessary (inefficient)
let specifier = match specifier {
- Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => specifier.into_entity_specifier(&mut api_client)?,
+ Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => {
+ specifier.into_entity_specifier(&mut api_client)?
+ }
_ => specifier,
};
let status = match specifier {
Specifier::Release(ident) => {
- let result = api_client.rt.block_on(
- api_client.api.get_release(ident.clone(), Some("files".to_string()), Some("abstracts,refs".to_string()))
- )?;
+ let result = api_client.rt.block_on(api_client.api.get_release(
+ ident.clone(),
+ Some("files".to_string()),
+ Some("abstracts,refs".to_string()),
+ ))?;
let release_entity = match result {
- fatcat_openapi::GetReleaseResponse::FoundEntity(model) => {
- Ok(model)
- },
+ fatcat_openapi::GetReleaseResponse::FoundEntity(model) => Ok(model),
resp => Err(anyhow!("{:?}", resp))
.with_context(|| format!("API GET failed: {:?}", ident)),
}?;
- download_release(release_entity)
- },
+ download_release(&release_entity)
+ }
Specifier::File(ident) => {
- let result = api_client.rt.block_on(
- api_client.api.get_file(ident.clone(), None, None)
- )?;
+ let result = api_client.rt.block_on(api_client.api.get_file(
+ ident.clone(),
+ None,
+ None,
+ ))?;
let file_entity = match result {
- fatcat_openapi::GetFileResponse::FoundEntity(model) => {
- Ok(model)
- },
+ fatcat_openapi::GetFileResponse::FoundEntity(model) => Ok(model),
resp => Err(anyhow!("{:?}", resp))
.with_context(|| format!("API GET failed: {:?}", ident)),
}?;
- download_file(file_entity)
- },
+ download_file(&file_entity)
+ }
other => Err(anyhow!("Don't know how to download: {:?}", other)),
}?;
- println!("{:?}", status);
+ if let Some(detail) = status.details() {
+ println!("{}: {}", status, detail);
+ } else {
+ println!("{}", status);
+ }
}
Command::Search {
entity_type,
@@ -519,7 +498,7 @@ fn run(opt: Opt) -> Result<()> {
hide,
} => {
let limit: Option<u64> = match limit {
- l if l < 0 => None,
+ l if l <= 0 => None,
l => Some(l as u64),
};
let results = fatcat_cli::crude_search(&opt.search_host, entity_type, limit, terms)
@@ -618,45 +597,14 @@ fn run(opt: Opt) -> Result<()> {
Command::Editgroup {
cmd: EditgroupCommand::Create { description },
} => {
- let mut eg = models::Editgroup::new();
- eg.description = Some(description);
- eg.extra = Some({
- let mut extra = std::collections::HashMap::new();
- extra.insert(
- "agent".to_string(),
- serde_json::Value::String("fatcat-cli".to_string()),
- );
- extra
- });
- let result = api_client
- .rt
- .block_on(api_client.api.create_editgroup(eg))?;
- match result {
- fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => {
- println!("{}", serde_json::to_string(&eg)?)
- }
- other => return Err(anyhow!("{:?}", other)).context("failed to create editgroup"),
- }
+ let eg = api_client.create_editgroup(Some(description))?;
+ println!("{}", serde_json::to_string(&eg)?)
}
Command::Editgroup {
cmd: EditgroupCommand::Accept { editgroup_id },
} => {
- let result = api_client
- .rt
- .block_on(api_client.api.accept_editgroup(editgroup_id.clone()))
- .context("accept editgroup")?;
- match result {
- fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => {
- println!("{}", serde_json::to_string(&msg)?)
- }
- other => {
- return Err(anyhow!(
- "failed to accept editgroup {}: {:?}",
- editgroup_id,
- other
- ))
- }
- }
+ let msg = api_client.accept_editgroup(editgroup_id.clone())?;
+ println!("{}", serde_json::to_string(&msg)?);
}
Command::Editgroup {
cmd: EditgroupCommand::Submit { editgroup_id },
diff --git a/rust/fatcat-cli/src/specifier.rs b/rust/fatcat-cli/src/specifier.rs
index c7a7b57..0d8d209 100644
--- a/rust/fatcat-cli/src/specifier.rs
+++ b/rust/fatcat-cli/src/specifier.rs
@@ -1,4 +1,4 @@
-use crate::{ApiEntityModel, FatcatApiClient};
+use crate::{ApiEntityModel, EntityType, FatcatApiClient};
use anyhow::{anyhow, Context, Result};
use lazy_static::lazy_static;
use regex::Regex;
@@ -50,6 +50,18 @@ pub enum Specifier {
}
impl Specifier {
+ pub fn from_ident(entity_type: EntityType, ident: String) -> Specifier {
+ match entity_type {
+ EntityType::Release => Specifier::Release(ident),
+ EntityType::Work => Specifier::Work(ident),
+ EntityType::Container => Specifier::Container(ident),
+ EntityType::Creator => Specifier::Creator(ident),
+ EntityType::File => Specifier::File(ident),
+ EntityType::FileSet => Specifier::FileSet(ident),
+ EntityType::WebCapture => Specifier::WebCapture(ident),
+ }
+ }
+
/// If this Specifier is a lookup, call the API to do the lookup and return the resulting
/// specific entity specifier (eg, with an FCID). If already specific, just pass through.
pub fn into_entity_specifier(self, api_client: &mut FatcatApiClient) -> Result<Specifier> {
@@ -382,19 +394,18 @@ impl Specifier {
.rt
.block_on(api_client.api.get_work_history(fcid.to_string(), limit))?
{
- fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => {
- Ok(entries)
- }
+ fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => Ok(entries),
fatcat_openapi::GetWorkHistoryResponse::NotFound(err) => {
Err(anyhow!("Not Found: {}", err.message))
}
resp => Err(anyhow!("{:?}", resp))
.with_context(|| format!("API GET failed: {:?}", self)),
},
- Container(fcid) => match api_client
- .rt
- .block_on(api_client.api.get_container_history(fcid.to_string(), limit))?
- {
+ Container(fcid) => match api_client.rt.block_on(
+ api_client
+ .api
+ .get_container_history(fcid.to_string(), limit),
+ )? {
fatcat_openapi::GetContainerHistoryResponse::FoundEntityHistory(entries) => {
Ok(entries)
}
@@ -421,9 +432,7 @@ impl Specifier {
.rt
.block_on(api_client.api.get_file_history(fcid.to_string(), limit))?
{
- fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => {
- Ok(entries)
- }
+ fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => Ok(entries),
fatcat_openapi::GetFileHistoryResponse::NotFound(err) => {
Err(anyhow!("Not Found: {}", err.message))
}
@@ -443,10 +452,11 @@ impl Specifier {
resp => Err(anyhow!("{:?}", resp))
.with_context(|| format!("API GET failed: {:?}", self)),
},
- WebCapture(fcid) => match api_client
- .rt
- .block_on(api_client.api.get_webcapture_history(fcid.to_string(), limit))?
- {
+ WebCapture(fcid) => match api_client.rt.block_on(
+ api_client
+ .api
+ .get_webcapture_history(fcid.to_string(), limit),
+ )? {
fatcat_openapi::GetWebcaptureHistoryResponse::FoundEntityHistory(entries) => {
Ok(entries)
}
--
cgit v1.2.3