aboutsummaryrefslogtreecommitdiffstats
path: root/rust
diff options
context:
space:
mode:
Diffstat (limited to 'rust')
-rw-r--r--rust/Cargo.lock59
-rw-r--r--rust/fatcat-cli/Cargo.toml2
-rw-r--r--rust/fatcat-cli/src/download.rs63
-rw-r--r--rust/fatcat-cli/src/lib.rs4
-rw-r--r--rust/fatcat-cli/src/main.rs47
5 files changed, 169 insertions, 6 deletions
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 305eaf0..7f208c2 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -182,6 +182,21 @@ dependencies = [
]
[[package]]
+name = "console"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cc80946b3480f421c2f17ed1cb841753a371c7c5104f51d507e13f532c856aa"
+dependencies = [
+ "encode_unicode",
+ "lazy_static",
+ "libc",
+ "regex",
+ "terminal_size",
+ "unicode-width",
+ "winapi 0.3.8",
+]
+
+[[package]]
name = "core-foundation"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -228,6 +243,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]]
+name = "encode_unicode"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+
+[[package]]
name = "encoding_rs"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -261,6 +282,7 @@ dependencies = [
"env_logger",
"fatcat-openapi",
"hyper",
+ "indicatif",
"lazy_static",
"log 0.4.8",
"macaroon",
@@ -275,6 +297,7 @@ dependencies = [
"termcolor",
"tokio",
"toml",
+ "url",
]
[[package]]
@@ -736,6 +759,18 @@ dependencies = [
]
[[package]]
+name = "indicatif"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
+dependencies = [
+ "console",
+ "lazy_static",
+ "number_prefix",
+ "regex",
+]
+
+[[package]]
name = "iovec"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -991,6 +1026,12 @@ dependencies = [
]
[[package]]
+name = "number_prefix"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
+
+[[package]]
name = "once_cell"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1272,9 +1313,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
[[package]]
name = "regex"
-version = "1.3.9"
+version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
+checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
dependencies = [
"aho-corasick",
"memchr",
@@ -1284,9 +1325,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
-version = "0.6.18"
+version = "0.6.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
+checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
[[package]]
name = "remove_dir_all"
@@ -1657,6 +1698,16 @@ dependencies = [
]
[[package]]
+name = "terminal_size"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86ca8ced750734db02076f44132d802af0b33b09942331f4459dde8636fd2406"
+dependencies = [
+ "libc",
+ "winapi 0.3.8",
+]
+
+[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/rust/fatcat-cli/Cargo.toml b/rust/fatcat-cli/Cargo.toml
index 30aefdb..dbb114a 100644
--- a/rust/fatcat-cli/Cargo.toml
+++ b/rust/fatcat-cli/Cargo.toml
@@ -36,6 +36,8 @@ serde = "1.0"
reqwest = { version = "0.10", features = ["blocking", "json"] }
chrono-humanize = "*"
tempfile = "3"
+indicatif = "0.15"
+url = "*"
[dev-dependencies]
diff --git a/rust/fatcat-cli/src/download.rs b/rust/fatcat-cli/src/download.rs
new file mode 100644
index 0000000..c8c05fd
--- /dev/null
+++ b/rust/fatcat-cli/src/download.rs
@@ -0,0 +1,63 @@
+
+use anyhow::{anyhow, Context, Result};
+use indicatif::ProgressBar;
+use fatcat_openapi::models::FileEntity;
+use reqwest::header::USER_AGENT;
+use url::Url;
+use std::fs::File;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum DownloadStatus {
+ Exists(String),
+ Downloaded(String),
+ NetworkError(String),
+ NoAccess,
+ NotYet,
+}
+
+// eg, https://web.archive.org/web/20140802044207/http://www.geo.coop:80/sites/default/files/labs_of_oligarchy.pdf
+fn rewrite_wayback_url(url: Url) -> Result<Url> {
+ // TODO: make this function correct, and add tests
+ let mut segments: Vec<String> = url.path_segments().unwrap().map(|x| x.to_string()).collect();
+ if segments[0] == "web" && segments[1].len() == 14 {
+ segments[1] = format!("{}id_", segments[1]);
+ }
+ Ok(url)
+}
+
+/// Attempts to download a file entity, including verifying checksum.
+pub fn download_file(fe: FileEntity) -> Result<DownloadStatus> {
+
+ // TODO: check if file has sha1hex
+ // TODO: check if file already exists
+
+ // TODO: only archive.org URLs
+ let raw_url = fe.urls.unwrap()[0].url.clone();
+
+ let mut url = Url::parse(&raw_url)?;
+ if url.host_str() == Some("web.archive.org") {
+ url = rewrite_wayback_url(url)?;
+ }
+
+ // TODO: open temporary file (real file plus suffix?)
+ let out_file = File::create(format!("{}.pdf", fe.sha1.unwrap()))?;
+
+ println!("downloading: {}", url);
+ let client = reqwest::blocking::Client::new();
+ let mut resp = client.get(url)
+ .header(USER_AGENT, "fatcat-cli/0.0.0")
+ .send()?;
+
+ // TODO: parse headers
+ // TODO: resp.error_for_status()?;
+ if !resp.status().is_success() {
+ return Ok(DownloadStatus::NetworkError(format!("{}", resp.status())));
+ }
+
+ // TODO: what if no filesize?
+ // TODO: compare with resp.content_length(() -> Option<u64>
+ let pb = ProgressBar::new(fe.size.unwrap() as u64);
+ let out_size = resp.copy_to(&mut pb.wrap_write(out_file))?;
+
+ Ok(DownloadStatus::NotYet)
+}
diff --git a/rust/fatcat-cli/src/lib.rs b/rust/fatcat-cli/src/lib.rs
index fc9f209..93c17fb 100644
--- a/rust/fatcat-cli/src/lib.rs
+++ b/rust/fatcat-cli/src/lib.rs
@@ -17,11 +17,13 @@ mod api;
mod entities;
mod search;
mod specifier;
+mod download;
pub use api::FatcatApiClient;
pub use entities::{ApiEntityModel, ApiModelIdent, ApiModelSer, Mutation};
pub use search::crude_search;
pub use specifier::Specifier;
+pub use download::download_file;
// Want to show:
// - whether api_token found
@@ -138,7 +140,7 @@ impl FromStr for EntityType {
"container" => Ok(EntityType::Container),
"creator" => Ok(EntityType::Creator),
"file" => Ok(EntityType::File),
- "fileset" => Ok(EntityType::FileSet),
+ "FILESET" => Ok(EntityType::FileSet),
"webcapture" => Ok(EntityType::WebCapture),
_ => Err(anyhow!("invalid entity type : {}", s)),
}
diff --git a/rust/fatcat-cli/src/main.rs b/rust/fatcat-cli/src/main.rs
index 046a825..75ddc6a 100644
--- a/rust/fatcat-cli/src/main.rs
+++ b/rust/fatcat-cli/src/main.rs
@@ -163,8 +163,10 @@ enum Command {
#[structopt(subcommand)]
cmd: EditgroupCommand,
},
+ Download {
+ specifier: Specifier,
+ },
//Changelog
- //Download
//History
Search {
entity_type: EntityType,
@@ -324,6 +326,49 @@ fn run(opt: Opt) -> Result<()> {
.context("updating after edit")?;
println!("{}", serde_json::to_string(&ee)?);
}
+ Command::Download{specifier} => {
+ // run lookups if necessary (inefficient)
+ let specifier = match specifier {
+ Specifier::ReleaseLookup(_, _) | Specifier::FileLookup(_, _) =>
+ specifier.into_entity_specifier(&mut api_client)?,
+ _ => specifier,
+ };
+ let file_entities = match specifier {
+ Specifier::Release(ident) => {
+ let result = api_client.rt.block_on(
+ api_client.api.get_release(ident.clone(), Some("files".to_string()), Some("abstracts,refs".to_string()))
+ )?;
+ let release_entity = match result {
+ fatcat_openapi::GetReleaseResponse::FoundEntity(model) => {
+ Ok(model)
+ },
+ resp => Err(anyhow!("{:?}", resp))
+ .with_context(|| format!("API GET failed: {:?}", ident)),
+ }?;
+ // TODO: not unwrap
+ release_entity.files.unwrap()
+ },
+ Specifier::File(ident) => {
+ let result = api_client.rt.block_on(
+ api_client.api.get_file(ident.clone(), None, None)
+ )?;
+ let file_entity = match result {
+ fatcat_openapi::GetFileResponse::FoundEntity(model) => {
+ Ok(model)
+ },
+ resp => Err(anyhow!("{:?}", resp))
+ .with_context(|| format!("API GET failed: {:?}", ident)),
+ }?;
+ vec![file_entity]
+ },
+ // TODO: not panic
+ _ => panic!("TODO: can only fetch file or release"),
+ };
+ for fe in file_entities {
+ let status = download_file(fe)?;
+ println!("{:?}", status);
+ };
+ }
Command::Search {
entity_type,
terms,
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-22 02:13:19 +0000