lots more progress on batch commands (and more)

7 files changed, 520 insertions, 172 deletions

diff --git a/rust/fatcat-cli/src/api.rs b/rust/fatcat-cli/src/api.rs
index cc6fa6a..2463aab 100644
--- a/rust/fatcat-cli/src/api.rs
+++ b/rust/fatcat-cli/src/api.rs
@@ -403,4 +403,38 @@ impl FatcatApiClient {
         }
         .with_context(|| format!("failed to update {:?}", specifier))
     }
+
+    pub fn create_editgroup(&mut self, description: Option<String>) -> Result<models::Editgroup> {
+        let mut eg = models::Editgroup::new();
+        eg.description = description;
+        eg.extra = Some({
+            let mut extra = std::collections::HashMap::new();
+            extra.insert(
+                "agent".to_string(),
+                serde_json::Value::String("fatcat-cli".to_string()),
+                // TODO: version?
+            );
+            extra
+        });
+        let result = self.rt.block_on(self.api.create_editgroup(eg))?;
+        match result {
+            fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => Ok(eg),
+            other => Err(anyhow!("{:?}", other)).context("failed to create editgroup"),
+        }
+    }
+
+    pub fn accept_editgroup(&mut self, editgroup_id: String) -> Result<models::Success> {
+        let result = self
+            .rt
+            .block_on(self.api.accept_editgroup(editgroup_id.clone()))
+            .context("accept editgroup")?;
+        match result {
+            fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => Ok(msg),
+            other => Err(anyhow!(
+                "failed to accept editgroup {}: {:?}",
+                editgroup_id,
+                other
+            )),
+        }
+    }
 }
diff --git a/rust/fatcat-cli/src/commands.rs b/rust/fatcat-cli/src/commands.rs
index 30fa0c4..15bfc81 100644
--- a/rust/fatcat-cli/src/commands.rs
+++ b/rust/fatcat-cli/src/commands.rs
@@ -3,14 +3,17 @@ use chrono_humanize::HumanTime;
 use fatcat_openapi::models;
 #[allow(unused_imports)]
 use log::{self, debug, info};
-use std::io::{Write, BufRead};
+use std::convert::TryInto;
+use std::fs::File;
+use std::io::{self, BufRead, Write};
+use std::path::PathBuf;
 use tabwriter::TabWriter;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
 
-use crate::api::FatcatApiClient;
-//use crate::download::download_file;
-use crate::entities::{ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation, read_entity_file};
-use crate::specifier::Specifier;
+use crate::{
+    entity_model_from_json_str, read_entity_file, ApiModelSer, EntityType, FatcatApiClient,
+    Mutation, Specifier,
+};
 
 // Want to show:
 // - whether api_token found
@@ -271,7 +274,13 @@ pub fn print_entity_histories(
     Ok(())
 }
 
-pub fn edit_entity_locally(api_client: &mut FatcatApiClient,  specifier: Specifier, editgroup_id: String, json: bool, editing_command: String) -> Result<models::EntityEdit> {
+pub fn edit_entity_locally(
+    api_client: &mut FatcatApiClient,
+    specifier: Specifier,
+    editgroup_id: String,
+    json: bool,
+    editing_command: String,
+) -> Result<models::EntityEdit> {
     // TODO: fetch editgroup, check if this entity is already being updated in it. If so,
     // need to fetch that revision, do the edit, parse that synatx is good, then delete the
     // existing edit and update with the new one.
@@ -312,3 +321,171 @@ pub fn edit_entity_locally(api_client: &mut FatcatApiClient,  specifier: Specifi
         .context("updating after edit")?;
     Ok(ee)
 }
+
+pub enum BatchOp {
+    Create,
+    Update,
+    Delete,
+}
+
+pub struct BatchGrouper {
+    entity_type: EntityType,
+    batch_size: u64,
+    limit: Option<u64>,
+    auto_accept: bool,
+    editgroup_description: String,
+    current_count: u64,
+    current_editgroup_id: Option<String>,
+    total_count: u64,
+}
+
+// Note: should be possible to add support for the single-call batch create endpoint by storing
+// edits in a batch within this object. Might need to change control flow a bit. This optimization
+// was mostly intended for bootstrapping of tens of thousands of entities, so not including for
+// now.
+impl BatchGrouper {
+    pub fn new(
+        entity_type: EntityType,
+        batch_size: u64,
+        limit: Option<u64>,
+        auto_accept: bool,
+    ) -> Self {
+        let editgroup_description = "part of a fatcat-cli batch operation".to_string();
+        BatchGrouper {
+            entity_type,
+            batch_size,
+            limit,
+            auto_accept,
+            editgroup_description,
+            current_count: 0,
+            current_editgroup_id: None,
+            total_count: 0,
+        }
+    }
+
+    pub fn run(
+        &mut self,
+        api_client: &mut FatcatApiClient,
+        input_path: Option<PathBuf>,
+        op: BatchOp,
+        mutations: Option<Vec<Mutation>>,
+    ) -> Result<()> {
+        match input_path {
+            None => {
+                let stdin = io::stdin();
+                let stdin_lock = stdin.lock();
+                let lines = stdin_lock.lines();
+                for line in lines {
+                    let json_str = line?;
+                    match op {
+                        BatchOp::Create => self.push_create(api_client, &json_str)?,
+                        BatchOp::Update => self.push_update(
+                            api_client,
+                            &json_str,
+                            mutations.clone().unwrap_or(vec![]),
+                        )?,
+                        BatchOp::Delete => self.push_delete(api_client, &json_str)?,
+                    };
+                    if let Some(limit) = self.limit {
+                        if self.total_count + self.current_count >= limit {
+                            break;
+                        }
+                    }
+                }
+            }
+            Some(path) => {
+                let input_file = File::open(path)?;
+                let buffered = io::BufReader::new(input_file);
+                let lines = buffered.lines();
+                for line in lines {
+                    let json_str = line?;
+                    match op {
+                        BatchOp::Create => self.push_create(api_client, &json_str)?,
+                        BatchOp::Update => self.push_update(
+                            api_client,
+                            &json_str,
+                            mutations.clone().unwrap_or(vec![]),
+                        )?,
+                        BatchOp::Delete => self.push_delete(api_client, &json_str)?,
+                    };
+                    if let Some(limit) = self.limit {
+                        if self.total_count + self.current_count >= limit {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        self.flush(api_client)?;
+        Ok(())
+    }
+
+    pub fn push_create(
+        &mut self,
+        api_client: &mut FatcatApiClient,
+        json_str: &str,
+    ) -> Result<models::EntityEdit> {
+        let editgroup_id = self.increment_editgroup(api_client)?;
+        api_client.create_entity_from_json(self.entity_type, json_str, editgroup_id)
+    }
+
+    pub fn push_update(
+        &mut self,
+        api_client: &mut FatcatApiClient,
+        json_str: &str,
+        mutations: Vec<Mutation>,
+    ) -> Result<models::EntityEdit> {
+        let obj: serde_json::Value = serde_json::from_str(json_str)?;
+        let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident?
+        let editgroup_id = self.increment_editgroup(api_client)?;
+        let mut entity = entity_model_from_json_str(self.entity_type, &json_str)?;
+        entity.mutate(mutations)?;
+        api_client.update_entity_from_json(
+            Specifier::from_ident(self.entity_type, ident.to_string()),
+            &entity.to_json_string()?,
+            editgroup_id,
+        )
+    }
+
+    pub fn push_delete(
+        &mut self,
+        api_client: &mut FatcatApiClient,
+        json_str: &str,
+    ) -> Result<models::EntityEdit> {
+        let obj: serde_json::Value = serde_json::from_str(json_str)?;
+        let ident = obj["ident"].as_str().unwrap(); // TODO: safer extraction of this ident?
+        let editgroup_id = self.increment_editgroup(api_client)?;
+        api_client.delete_entity(
+            Specifier::from_ident(self.entity_type, ident.to_string()),
+            editgroup_id,
+        )
+    }
+
+    pub fn increment_editgroup(&mut self, api_client: &mut FatcatApiClient) -> Result<String> {
+        if self.current_count >= self.batch_size.try_into().unwrap() {
+            self.flush(api_client)?;
+        };
+        self.current_count += 1;
+        if let Some(eg) = &self.current_editgroup_id {
+            return Ok(eg.to_string());
+        }
+        let eg = api_client.create_editgroup(Some(self.editgroup_description.clone()))?;
+        self.current_editgroup_id = eg.editgroup_id;
+        Ok(self.current_editgroup_id.as_ref().unwrap().to_string())
+    }
+
+    pub fn flush(&mut self, api_client: &mut FatcatApiClient) -> Result<()> {
+        if self.current_count > 0 && self.current_editgroup_id.is_some() {
+            let eg_id = self.current_editgroup_id.clone().unwrap();
+            if self.auto_accept {
+                api_client.accept_editgroup(eg_id)?;
+            } else {
+                api_client.update_editgroup_submit(eg_id, true)?;
+            }
+            self.total_count += self.current_count;
+            self.current_count = 0;
+            self.current_editgroup_id = None;
+        }
+        Ok(())
+    }
+}
diff --git a/rust/fatcat-cli/src/download.rs b/rust/fatcat-cli/src/download.rs
index 5500a7a..7821b70 100644
--- a/rust/fatcat-cli/src/download.rs
+++ b/rust/fatcat-cli/src/download.rs
@@ -1,23 +1,56 @@
 use anyhow::{anyhow, Context, Result};
 use fatcat_openapi::models::{FileEntity, ReleaseEntity};
 use indicatif::ProgressBar;
+use log::info;
 use reqwest::header::USER_AGENT;
+use std::fmt;
 use std::fs::File;
-use url::Url;
+use std::io::{self, BufRead};
 use std::path::Path;
-
+use std::path::PathBuf;
+use url::Url;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum DownloadStatus {
     Exists(String),
     Downloaded(String),
     NetworkError(String),
-    NoPublicAccess,
+    HttpError(u16),
+    PartialExists(String),
+    NoPublicFile,
     FileMissingMetadata,
     WrongSize,
     WrongHash,
 }
 
+impl DownloadStatus {
+    pub fn details(&self) -> Option<String> {
+        match self {
+            Self::Exists(p) => Some(p.to_string()),
+            Self::Downloaded(p) => Some(p.to_string()),
+            Self::NetworkError(p) => Some(p.to_string()),
+            Self::PartialExists(p) => Some(p.to_string()),
+            _ => None,
+        }
+    }
+}
+
+impl fmt::Display for DownloadStatus {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Exists(_) => write!(f, "exists"),
+            Self::Downloaded(_) => write!(f, "success"),
+            Self::NetworkError(_) => write!(f, "network-error"),
+            Self::HttpError(p) => write!(f, "http-{}", p),
+            Self::PartialExists(_) => write!(f, "partial-exists"),
+            Self::NoPublicFile => write!(f, "no-public-file"),
+            Self::FileMissingMetadata => write!(f, "missing-file-metadata"),
+            Self::WrongSize => write!(f, "wrong-file-size"),
+            Self::WrongHash => write!(f, "wrong-file-hash"),
+        }
+    }
+}
+
 // eg, https://web.archive.org/web/20140802044207/http://www.geo.coop:80/sites/default/files/labs_of_oligarchy.pdf
 fn rewrite_wayback_url(url: Url) -> Result<Url> {
     // TODO: make this function correct, and add tests
@@ -33,8 +66,8 @@ fn rewrite_wayback_url(url: Url) -> Result<Url> {
 }
 
 /// Attempts to download a file entity, including verifying checksum.
-pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> {
-    let sha1hex = match fe.sha1 {
+pub fn download_file(fe: &FileEntity) -> Result<DownloadStatus> {
+    let sha1hex = match &fe.sha1 {
         Some(v) => v,
         None => return Ok(DownloadStatus::FileMissingMetadata),
     };
@@ -48,72 +81,106 @@ pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> {
         Some("application/postscript") => ".pdf",
         Some("text/html") => ".html",
         Some("text/xml") => ".xml",
-        _ => "",
+        _ => ".unknown",
     };
 
     // TODO: output directory
     let path_string = format!("{}{}", sha1hex, file_suffix);
     let final_path = Path::new(&path_string);
 
+    // NOTE: this isn't perfect; there could have been a race condition
     if final_path.exists() {
-        return Ok(DownloadStatus::Exists(final_path.to_string_lossy().to_string()));
+        return Ok(DownloadStatus::Exists(
+            final_path.to_string_lossy().to_string(),
+        ));
     };
 
     let path_string = format!("{}{}.partial", sha1hex, file_suffix);
     let download_path = Path::new(&path_string);
 
+    // TODO: only archive.org URLs (?)
     let raw_url = match fe.urls.as_ref() {
-        None => return Ok(DownloadStatus::NoPublicAccess),
-        Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicAccess),
+        None => return Ok(DownloadStatus::NoPublicFile),
+        Some(url_list) if url_list.len() == 0 => return Ok(DownloadStatus::NoPublicFile),
         // TODO: remove clone (?)
         // TODO: better heuristic than "just try first URL"
         Some(url_list) => url_list[0].url.clone(),
     };
-    // TODO: only archive.org URLs (?)
-    let raw_url = fe.urls.unwrap()[0].url.clone();
 
     let mut url = Url::parse(&raw_url)?;
     if url.host_str() == Some("web.archive.org") {
         url = rewrite_wayback_url(url)?;
     }
 
-    // TODO: open temporary file (real file plus suffix?)
-    let download_file = File::create(download_path)?;
+    let download_file = match std::fs::OpenOptions::new()
+        .write(true)
+        .create_new(true)
+        .open(download_path)
+    {
+        Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
+            return Ok(DownloadStatus::PartialExists(
+                download_path.to_string_lossy().to_string(),
+            ))
+        }
+        Err(e) => return Err(e).context("opening temporary download file"),
+        Ok(f) => f,
+    };
 
-    println!("downloading: {}", url);
+    // TODO: print to stderr
+    info!("downloading: {}", url);
     let client = reqwest::blocking::Client::new();
-    let mut resp = client
+    let mut resp = match client
         .get(url)
         .header(USER_AGENT, "fatcat-cli/0.0.0")
-        .send()?;
+        .send()
+    {
+        Ok(r) => r,
+        Err(e) => {
+            std::fs::remove_file(download_path)?;
+            return Ok(DownloadStatus::NetworkError(format!("{:?}", e)));
+        }
+    };
 
-    // TODO: parse headers
-    // TODO: resp.error_for_status()?;
+    // TODO: parse headers, eg size (?)
     if !resp.status().is_success() {
-        return Ok(DownloadStatus::NetworkError(format!("{}", resp.status())));
+        std::fs::remove_file(download_path)?;
+        return Ok(DownloadStatus::HttpError(resp.status().as_u16()));
     }
 
     // TODO: what if no filesize?
     // TODO: compare with resp.content_length(() -> Option<u64>
     let pb = ProgressBar::new(fe.size.unwrap() as u64);
-    let out_size = resp.copy_to(&mut pb.wrap_write(download_file))?;
+    let out_size = match resp.copy_to(&mut pb.wrap_write(download_file)) {
+        Ok(r) => r,
+        Err(e) => {
+            std::fs::remove_file(download_path)?;
+            return Ok(DownloadStatus::NetworkError(format!("{:?}", e)));
+        }
+    };
 
     if out_size != expected_size {
-        // TODO: delete partial file?
+        std::fs::remove_file(download_path)?;
         return Ok(DownloadStatus::WrongSize);
     }
 
-    Ok(DownloadStatus::Downloaded(final_path.to_string_lossy().to_string()))
+    std::fs::rename(download_path, final_path)?;
+    Ok(DownloadStatus::Downloaded(
+        final_path.to_string_lossy().to_string(),
+    ))
 }
 
-pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> {
-    let file_entities = match re.files {
-        None => return Err(anyhow!("expected file sub-entities to be 'expanded' on release")),
+pub fn download_release(re: &ReleaseEntity) -> Result<DownloadStatus> {
+    let file_entities = match &re.files {
+        None => {
+            return Err(anyhow!(
+                "expected file sub-entities to be 'expanded' on release"
+            ))
+        }
         Some(list) => list,
     };
-    let mut status = DownloadStatus::NoPublicAccess;
+    let mut status = DownloadStatus::NoPublicFile;
     for fe in file_entities {
-        status = download_file(fe)?;
+        status = download_file(&fe)?;
         match status {
             DownloadStatus::Exists(_) | DownloadStatus::Downloaded(_) => break,
             _ => (),
@@ -121,3 +188,74 @@ pub fn download_release(re: ReleaseEntity) -> Result<DownloadStatus> {
     }
     Ok(status)
 }
+
+/// Tries either file or release
+fn download_entity(json_str: String) -> Result<DownloadStatus> {
+    let release_attempt = serde_json::from_str::<ReleaseEntity>(&json_str);
+    if let Ok(re) = release_attempt {
+        if re.ident.is_some() && (re.title.is_some() || re.files.is_some()) {
+            let status = download_release(&re)?;
+            println!(
+                "release_{}\t{}\t{}",
+                re.ident.unwrap(),
+                status,
+                status.details().unwrap_or("".to_string())
+            );
+            return Ok(status);
+        };
+    }
+    let file_attempt =
+        serde_json::from_str::<FileEntity>(&json_str).context("parsing entity for download");
+    match file_attempt {
+        Ok(fe) => {
+            if fe.ident.is_some() && fe.urls.is_some() {
+                let status = download_file(&fe)?;
+                println!(
+                    "file_{}\t{}\t{}",
+                    fe.ident.unwrap(),
+                    status,
+                    status.details().unwrap_or("".to_string())
+                );
+                return Ok(status);
+            } else {
+                Err(anyhow!("not a file entity (JSON)"))
+            }
+        }
+        Err(e) => Err(e),
+    }
+}
+
+pub fn download_batch(input_path: Option<PathBuf>, limit: Option<u64>) -> Result<u64> {
+    let count = 0;
+    match input_path {
+        None => {
+            let stdin = io::stdin();
+            let stdin_lock = stdin.lock();
+            let lines = stdin_lock.lines();
+            for line in lines {
+                let json_str = line?;
+                download_entity(json_str)?;
+                if let Some(limit) = limit {
+                    if count >= limit {
+                        break;
+                    }
+                }
+            }
+        }
+        Some(path) => {
+            let input_file = File::open(path)?;
+            let buffered = io::BufReader::new(input_file);
+            let lines = buffered.lines();
+            for line in lines {
+                let json_str = line?;
+                download_entity(json_str)?;
+                if let Some(limit) = limit {
+                    if count >= limit {
+                        break;
+                    }
+                }
+            }
+        }
+    };
+    Ok(count)
+}
diff --git a/rust/fatcat-cli/src/entities.rs b/rust/fatcat-cli/src/entities.rs
index 314af51..d61f6dc 100644
--- a/rust/fatcat-cli/src/entities.rs
+++ b/rust/fatcat-cli/src/entities.rs
@@ -1,11 +1,10 @@
-use crate::Specifier;
+use crate::{EntityType, Specifier};
 use anyhow::{anyhow, Context, Result};
 use fatcat_openapi::models;
 use lazy_static::lazy_static;
 use log::{self, info};
 use regex::Regex;
-use std::io::BufRead;
-use std::io::Read;
+use std::io::{BufRead, Read};
 use std::path::PathBuf;
 use std::str::FromStr;
 
@@ -94,6 +93,35 @@ pub fn read_entity_file(input_path: Option<PathBuf>) -> Result<String> {
     }
 }
 
+pub fn entity_model_from_json_str(
+    entity_type: EntityType,
+    json_str: &str,
+) -> Result<Box<dyn ApiEntityModel>> {
+    match entity_type {
+        EntityType::Release => Ok(Box::new(serde_json::from_str::<models::ReleaseEntity>(
+            &json_str,
+        )?)),
+        EntityType::Work => Ok(Box::new(serde_json::from_str::<models::WorkEntity>(
+            &json_str,
+        )?)),
+        EntityType::Container => Ok(Box::new(serde_json::from_str::<models::ContainerEntity>(
+            &json_str,
+        )?)),
+        EntityType::Creator => Ok(Box::new(serde_json::from_str::<models::CreatorEntity>(
+            &json_str,
+        )?)),
+        EntityType::File => Ok(Box::new(serde_json::from_str::<models::FileEntity>(
+            &json_str,
+        )?)),
+        EntityType::FileSet => Ok(Box::new(serde_json::from_str::<models::FilesetEntity>(
+            &json_str,
+        )?)),
+        EntityType::WebCapture => Ok(Box::new(serde_json::from_str::<models::WebcaptureEntity>(
+            &json_str,
+        )?)),
+    }
+}
+
 pub trait ApiModelSer {
     fn to_json_string(&self) -> Result<String>;
     fn to_toml_string(&self) -> Result<String>;
diff --git a/rust/fatcat-cli/src/lib.rs b/rust/fatcat-cli/src/lib.rs
index 206fd09..d648c1c 100644
--- a/rust/fatcat-cli/src/lib.rs
+++ b/rust/fatcat-cli/src/lib.rs
@@ -1,6 +1,7 @@
 use anyhow::{anyhow, Context, Result};
 use data_encoding::BASE64;
 use macaroon::{Macaroon, Verifier};
+use std::path::PathBuf;
 use std::str::FromStr;
 
 mod api;
@@ -12,10 +13,14 @@ mod specifier;
 
 pub use api::FatcatApiClient;
 pub use commands::{
-    print_changelog_entries, print_editgroups, print_entity_histories, ClientStatus, edit_entity_locally,
+    edit_entity_locally, print_changelog_entries, print_editgroups, print_entity_histories,
+    BatchGrouper, BatchOp, ClientStatus,
+};
+pub use download::{download_batch, download_file, download_release};
+pub use entities::{
+    entity_model_from_json_str, read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer,
+    Mutation,
 };
-pub use download::{download_release, download_file};
-pub use entities::{read_entity_file, ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation};
 pub use search::crude_search;
 pub use specifier::Specifier;
 
@@ -78,3 +83,11 @@ pub fn parse_macaroon_editor_id(s: &str) -> Result<String> {
     verifier.satisfy_exact(&format!("editor_id = {}", editor_id.to_string()));
     Ok(editor_id)
 }
+
+pub fn path_or_stdin(raw: Option<PathBuf>) -> Option<PathBuf> {
+    // treat "-" as "use stdin"
+    match raw {
+        Some(s) if s.to_string_lossy() == "-" => None,
+        _ => raw,
+    }
+}
diff --git a/rust/fatcat-cli/src/main.rs b/rust/fatcat-cli/src/main.rs
index 79efbd3..3b0d382 100644
--- a/rust/fatcat-cli/src/main.rs
+++ b/rust/fatcat-cli/src/main.rs
@@ -1,7 +1,7 @@
+use crate::{path_or_stdin, BatchGrouper, BatchOp};
 use anyhow::{anyhow, Context, Result};
 use fatcat_cli::ApiModelSer;
 use fatcat_cli::*;
-use fatcat_openapi::models;
 #[allow(unused_imports)]
 use log::{self, debug, info};
 use std::io::Write;
@@ -100,18 +100,7 @@ enum BatchCommand {
         batch_size: u64,
 
         #[structopt(long)]
-        dry_run: bool,
-
-        #[structopt(long)]
         auto_accept: bool,
-
-        #[structopt(
-            long = "--editgroup-id",
-            short,
-            env = "FATCAT_EDITGROUP",
-            hide_env_values = true
-        )]
-        editgroup_id: String,
     },
     Update {
         entity_type: EntityType,
@@ -121,18 +110,7 @@ enum BatchCommand {
         batch_size: u64,
 
         #[structopt(long)]
-        dry_run: bool,
-
-        #[structopt(long)]
         auto_accept: bool,
-
-        #[structopt(
-            long = "--editgroup-id",
-            short,
-            env = "FATCAT_EDITGROUP",
-            hide_env_values = true
-        )]
-        editgroup_id: String,
     },
     Delete {
         entity_type: EntityType,
@@ -141,18 +119,7 @@ enum BatchCommand {
         batch_size: u64,
 
         #[structopt(long)]
-        dry_run: bool,
-
-        #[structopt(long)]
         auto_accept: bool,
-
-        #[structopt(
-            long = "--editgroup-id",
-            short,
-            env = "FATCAT_EDITGROUP",
-            hide_env_values = true
-        )]
-        editgroup_id: String,
     },
     Download {},
 }
@@ -231,6 +198,7 @@ enum Command {
         #[structopt(long)]
         json: bool,
 
+        #[allow(dead_code)]
         #[structopt(long)]
         toml: bool,
 
@@ -243,7 +211,7 @@ enum Command {
     History {
         specifier: Specifier,
 
-        #[structopt(long, short = "-n", default_value = "20")]
+        #[structopt(long, short = "-n", default_value = "100")]
         limit: u64,
 
         #[structopt(long)]
@@ -348,13 +316,13 @@ fn run(opt: Opt) -> Result<()> {
             specifier,
             expand,
             hide,
-            json: _,
+            json,
             toml,
         } => {
             let result = specifier.get_from_api(&mut api_client, expand, hide)?;
             if toml {
                 writeln!(&mut std::io::stdout(), "{}", result.to_toml_string()?)?
-            } else {
+            } else if json || true {
                 writeln!(&mut std::io::stdout(), "{}", result.to_json_string()?)?
             }
         }
@@ -398,13 +366,16 @@ fn run(opt: Opt) -> Result<()> {
             toml: _,
             editing_command,
         } => {
-            let ee = edit_entity_locally(&mut api_client, specifier, editgroup_id, json, editing_command)?;
+            let ee = edit_entity_locally(
+                &mut api_client,
+                specifier,
+                editgroup_id,
+                json,
+                editing_command,
+            )?;
             println!("{}", serde_json::to_string(&ee)?);
         }
-        Command::Changelog {
-            limit,
-            json,
-        } => {
+        Command::Changelog { limit, json } => {
             let resp = api_client
                 .rt
                 .block_on(api_client.api.get_changelog(Some(limit)))
@@ -414,9 +385,8 @@ fn run(opt: Opt) -> Result<()> {
                     print_changelog_entries(change_list, json)?;
                 }
                 other => {
-                    return Err(anyhow!("{:?}", other)).with_context(|| {
-                        format!("failed to fetch changelogs")
-                    })
+                    return Err(anyhow!("{:?}", other))
+                        .with_context(|| format!("failed to fetch changelogs"))
                 }
             }
         }
@@ -425,90 +395,99 @@ fn run(opt: Opt) -> Result<()> {
                 BatchCommand::Create {
                     entity_type,
                     batch_size,
-                    dry_run,
                     auto_accept,
-                    editgroup_id,
                 },
             input_path,
             limit,
         } => {
-            unimplemented!("batch create")
-        },
+            let input_path = path_or_stdin(input_path);
+            let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+            batch.run(&mut api_client, input_path, BatchOp::Create, None)?;
+        }
         Command::Batch {
             cmd:
                 BatchCommand::Update {
                     entity_type,
                     mutations,
                     batch_size,
-                    dry_run,
                     auto_accept,
-                    editgroup_id,
                 },
             input_path,
             limit,
         } => {
-            unimplemented!("batch update")
-        },
+            let input_path = path_or_stdin(input_path);
+            let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+            batch.run(
+                &mut api_client,
+                input_path,
+                BatchOp::Update,
+                Some(mutations),
+            )?;
+        }
         Command::Batch {
             cmd:
                 BatchCommand::Delete {
                     entity_type,
                     batch_size,
-                    dry_run,
                     auto_accept,
-                    editgroup_id,
                 },
             input_path,
             limit,
         } => {
-            unimplemented!("batch delete")
-        },
+            let input_path = path_or_stdin(input_path);
+            let mut batch = BatchGrouper::new(entity_type, batch_size, limit, auto_accept);
+            batch.run(&mut api_client, input_path, BatchOp::Delete, None)?;
+        }
         Command::Batch {
-            cmd:
-                BatchCommand::Download {
-                    // TODO
-                },
+            cmd: BatchCommand::Download {},
             input_path,
             limit,
         } => {
-            unimplemented!("batch create")
-        },
-        Command::Download{specifier} => {
+            let input_path = path_or_stdin(input_path);
+            download_batch(input_path, limit)?;
+        }
+        Command::Download { specifier } => {
             // run lookups if necessary (inefficient)
             let specifier = match specifier {
-                Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => specifier.into_entity_specifier(&mut api_client)?,
+                Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) => {
+                    specifier.into_entity_specifier(&mut api_client)?
+                }
                 _ => specifier,
             };
             let status = match specifier {
                 Specifier::Release(ident) => {
-                    let result = api_client.rt.block_on(
-                        api_client.api.get_release(ident.clone(), Some("files".to_string()), Some("abstracts,refs".to_string()))
-                    )?;
+                    let result = api_client.rt.block_on(api_client.api.get_release(
+                        ident.clone(),
+                        Some("files".to_string()),
+                        Some("abstracts,refs".to_string()),
+                    ))?;
                     let release_entity = match result {
-                        fatcat_openapi::GetReleaseResponse::FoundEntity(model) => {
-                            Ok(model)
-                        },
+                        fatcat_openapi::GetReleaseResponse::FoundEntity(model) => Ok(model),
                         resp => Err(anyhow!("{:?}", resp))
                             .with_context(|| format!("API GET failed: {:?}", ident)),
                     }?;
-                    download_release(release_entity)
-                },
+                    download_release(&release_entity)
+                }
                 Specifier::File(ident) => {
-                    let result = api_client.rt.block_on(
-                        api_client.api.get_file(ident.clone(), None, None)
-                    )?;
+                    let result = api_client.rt.block_on(api_client.api.get_file(
+                        ident.clone(),
+                        None,
+                        None,
+                    ))?;
                     let file_entity = match result {
-                        fatcat_openapi::GetFileResponse::FoundEntity(model) => {
-                            Ok(model)
-                        },
+                        fatcat_openapi::GetFileResponse::FoundEntity(model) => Ok(model),
                         resp => Err(anyhow!("{:?}", resp))
                             .with_context(|| format!("API GET failed: {:?}", ident)),
                     }?;
-                    download_file(file_entity)
-                },
+                    download_file(&file_entity)
+                }
                 other => Err(anyhow!("Don't know how to download: {:?}", other)),
             }?;
-            println!("{:?}", status);
+            if let Some(detail) = status.details() {
+                println!("{}: {}", status, detail);
+            } else {
+                println!("{}", status);
+            }
         }
         Command::Search {
             entity_type,
@@ -519,7 +498,7 @@ fn run(opt: Opt) -> Result<()> {
             hide,
         } => {
             let limit: Option<u64> = match limit {
-                l if l < 0 => None,
+                l if l <= 0 => None,
                 l => Some(l as u64),
             };
             let results = fatcat_cli::crude_search(&opt.search_host, entity_type, limit, terms)
@@ -618,45 +597,14 @@ fn run(opt: Opt) -> Result<()> {
         Command::Editgroup {
             cmd: EditgroupCommand::Create { description },
         } => {
-            let mut eg = models::Editgroup::new();
-            eg.description = Some(description);
-            eg.extra = Some({
-                let mut extra = std::collections::HashMap::new();
-                extra.insert(
-                    "agent".to_string(),
-                    serde_json::Value::String("fatcat-cli".to_string()),
-                );
-                extra
-            });
-            let result = api_client
-                .rt
-                .block_on(api_client.api.create_editgroup(eg))?;
-            match result {
-                fatcat_openapi::CreateEditgroupResponse::SuccessfullyCreated(eg) => {
-                    println!("{}", serde_json::to_string(&eg)?)
-                }
-                other => return Err(anyhow!("{:?}", other)).context("failed to create editgroup"),
-            }
+            let eg = api_client.create_editgroup(Some(description))?;
+            println!("{}", serde_json::to_string(&eg)?)
         }
         Command::Editgroup {
             cmd: EditgroupCommand::Accept { editgroup_id },
         } => {
-            let result = api_client
-                .rt
-                .block_on(api_client.api.accept_editgroup(editgroup_id.clone()))
-                .context("accept editgroup")?;
-            match result {
-                fatcat_openapi::AcceptEditgroupResponse::MergedSuccessfully(msg) => {
-                    println!("{}", serde_json::to_string(&msg)?)
-                }
-                other => {
-                    return Err(anyhow!(
-                        "failed to accept editgroup {}: {:?}",
-                        editgroup_id,
-                        other
-                    ))
-                }
-            }
+            let msg = api_client.accept_editgroup(editgroup_id.clone())?;
+            println!("{}", serde_json::to_string(&msg)?);
         }
         Command::Editgroup {
             cmd: EditgroupCommand::Submit { editgroup_id },
diff --git a/rust/fatcat-cli/src/specifier.rs b/rust/fatcat-cli/src/specifier.rs
index c7a7b57..0d8d209 100644
--- a/rust/fatcat-cli/src/specifier.rs
+++ b/rust/fatcat-cli/src/specifier.rs
@@ -1,4 +1,4 @@
-use crate::{ApiEntityModel, FatcatApiClient};
+use crate::{ApiEntityModel, EntityType, FatcatApiClient};
 use anyhow::{anyhow, Context, Result};
 use lazy_static::lazy_static;
 use regex::Regex;
@@ -50,6 +50,18 @@ pub enum Specifier {
 }
 
 impl Specifier {
+    pub fn from_ident(entity_type: EntityType, ident: String) -> Specifier {
+        match entity_type {
+            EntityType::Release => Specifier::Release(ident),
+            EntityType::Work => Specifier::Work(ident),
+            EntityType::Container => Specifier::Container(ident),
+            EntityType::Creator => Specifier::Creator(ident),
+            EntityType::File => Specifier::File(ident),
+            EntityType::FileSet => Specifier::FileSet(ident),
+            EntityType::WebCapture => Specifier::WebCapture(ident),
+        }
+    }
+
     /// If this Specifier is a lookup, call the API to do the lookup and return the resulting
     /// specific entity specifier (eg, with an FCID). If already specific, just pass through.
     pub fn into_entity_specifier(self, api_client: &mut FatcatApiClient) -> Result<Specifier> {
@@ -382,19 +394,18 @@ impl Specifier {
                 .rt
                 .block_on(api_client.api.get_work_history(fcid.to_string(), limit))?
             {
-                fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => {
-                    Ok(entries)
-                }
+                fatcat_openapi::GetWorkHistoryResponse::FoundEntityHistory(entries) => Ok(entries),
                 fatcat_openapi::GetWorkHistoryResponse::NotFound(err) => {
                     Err(anyhow!("Not Found: {}", err.message))
                 }
                 resp => Err(anyhow!("{:?}", resp))
                     .with_context(|| format!("API GET failed: {:?}", self)),
             },
-            Container(fcid) => match api_client
-                .rt
-                .block_on(api_client.api.get_container_history(fcid.to_string(), limit))?
-            {
+            Container(fcid) => match api_client.rt.block_on(
+                api_client
+                    .api
+                    .get_container_history(fcid.to_string(), limit),
+            )? {
                 fatcat_openapi::GetContainerHistoryResponse::FoundEntityHistory(entries) => {
                     Ok(entries)
                 }
@@ -421,9 +432,7 @@ impl Specifier {
                 .rt
                 .block_on(api_client.api.get_file_history(fcid.to_string(), limit))?
             {
-                fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => {
-                    Ok(entries)
-                }
+                fatcat_openapi::GetFileHistoryResponse::FoundEntityHistory(entries) => Ok(entries),
                 fatcat_openapi::GetFileHistoryResponse::NotFound(err) => {
                     Err(anyhow!("Not Found: {}", err.message))
                 }
@@ -443,10 +452,11 @@ impl Specifier {
                 resp => Err(anyhow!("{:?}", resp))
                     .with_context(|| format!("API GET failed: {:?}", self)),
             },
-            WebCapture(fcid) => match api_client
-                .rt
-                .block_on(api_client.api.get_webcapture_history(fcid.to_string(), limit))?
-            {
+            WebCapture(fcid) => match api_client.rt.block_on(
+                api_client
+                    .api
+                    .get_webcapture_history(fcid.to_string(), limit),
+            )? {
                 fatcat_openapi::GetWebcaptureHistoryResponse::FoundEntityHistory(entries) => {
                     Ok(entries)
                 }