diff options
Diffstat (limited to 'rust/src')
-rw-r--r-- | rust/src/auth.rs | 31 | ||||
-rw-r--r-- | rust/src/bin/fatcat-auth.rs | 44 | ||||
-rw-r--r-- | rust/src/bin/fatcat-export.rs | 20 | ||||
-rw-r--r-- | rust/src/bin/fatcatd.rs | 40 | ||||
-rw-r--r-- | rust/src/database_models.rs | 4 | ||||
-rw-r--r-- | rust/src/editing.rs | 140 | ||||
-rw-r--r-- | rust/src/endpoint_handlers.rs (renamed from rust/src/api_server.rs) | 20 | ||||
-rw-r--r-- | rust/src/endpoints.rs (renamed from rust/src/api_wrappers.rs) | 15 | ||||
-rw-r--r-- | rust/src/entity_crud.rs (renamed from rust/src/api_entity_crud.rs) | 178 | ||||
-rw-r--r-- | rust/src/errors.rs | 55 | ||||
-rw-r--r-- | rust/src/identifiers.rs (renamed from rust/src/api_helpers.rs) | 309 | ||||
-rw-r--r-- | rust/src/lib.rs | 203 | ||||
-rw-r--r-- | rust/src/server.rs | 81 |
13 files changed, 550 insertions, 590 deletions
diff --git a/rust/src/auth.rs b/rust/src/auth.rs index da038b6b..255da8dd 100644 --- a/rust/src/auth.rs +++ b/rust/src/auth.rs @@ -5,14 +5,16 @@ use macaroon::{Format, Macaroon, Verifier}; use std::fmt; use swagger::auth::{AuthData, Authorization, Scopes}; -use crate::api_helpers::*; -use chrono::prelude::*; use crate::database_models::*; use crate::database_schema::*; +use crate::errors::*; +use crate::identifiers::*; +use crate::server::*; +use chrono::prelude::*; use diesel; use diesel::prelude::*; -use crate::errors::*; use std::collections::HashMap; +use std::env; use std::str::FromStr; // 32 bytes max (!) @@ -468,3 +470,26 @@ pub fn print_editors(conn: &DbConn) -> Result<()> { } Ok(()) } + +pub fn env_confectionary() -> Result<AuthConfectionary> { + let auth_location = env::var("AUTH_LOCATION").expect("AUTH_LOCATION must be set"); + let auth_key = env::var("AUTH_SECRET_KEY").expect("AUTH_SECRET_KEY must be set"); + let auth_key_ident = env::var("AUTH_KEY_IDENT").expect("AUTH_KEY_IDENT must be set"); + info!("Loaded primary auth key: {}", auth_key_ident); + let mut confectionary = AuthConfectionary::new(auth_location, auth_key_ident, auth_key)?; + match env::var("AUTH_ALT_KEYS") { + Ok(var) => { + for pair in var.split(",") { + let pair: Vec<&str> = pair.split(":").collect(); + if pair.len() != 2 { + println!("{:#?}", pair); + bail!("couldn't parse keypair from AUTH_ALT_KEYS (expected 'ident:key' pairs separated by commas)"); + } + info!("Loading alt auth key: {}", pair[0]); + confectionary.add_keypair(pair[0].to_string(), pair[1].to_string())?; + } + } + Err(_) => (), + } + Ok(confectionary) +} diff --git a/rust/src/bin/fatcat-auth.rs b/rust/src/bin/fatcat-auth.rs index addd2b66..7e2a7c39 100644 --- a/rust/src/bin/fatcat-auth.rs +++ b/rust/src/bin/fatcat-auth.rs @@ -1,32 +1,16 @@ //! JSON Export Helper -//#[macro_use] -extern crate clap; -extern crate diesel; -extern crate dotenv; -#[macro_use] -extern crate error_chain; -extern crate fatcat; -//#[macro_use] -extern crate env_logger; -extern crate log; -extern crate serde_json; -extern crate uuid; - use clap::{App, SubCommand}; -use diesel::prelude::*; -use fatcat::api_helpers::FatCatId; +use fatcat::auth; +use fatcat::editing; use fatcat::errors::*; +use fatcat::identifiers::FatCatId; +use fatcat::server::*; +use std::process; use std::str::FromStr; -//use uuid::Uuid; - -//use error_chain::ChainedError; -//use std::io::{Stdout,StdoutLock}; -//use std::io::prelude::*; -//use std::io::{BufReader, BufWriter}; -fn run() -> Result<()> { +fn main() -> Result<()> { let m = App::new("fatcat-auth") .version(env!("CARGO_PKG_VERSION")) .author("Bryan Newbold <bnewbold@archive.org>") @@ -84,16 +68,14 @@ fn run() -> Result<()> { } // Then the ones that do - let db_conn = fatcat::database_worker_pool()? - .get() - .expect("database pool"); - let confectionary = fatcat::env_confectionary()?; + let db_conn = database_worker_pool()?.get().expect("database pool"); + let confectionary = auth::env_confectionary()?; match m.subcommand() { ("list-editors", Some(_subm)) => { fatcat::auth::print_editors(&db_conn)?; } ("create-editor", Some(subm)) => { - let editor = fatcat::api_helpers::create_editor( + let editor = editing::create_editor( &db_conn, subm.value_of("username").unwrap().to_string(), subm.is_present("admin"), @@ -104,10 +86,6 @@ fn run() -> Result<()> { } ("create-token", Some(subm)) => { let editor_id = FatCatId::from_str(subm.value_of("editor-id").unwrap())?; - // check that editor exists - let _ed: fatcat::database_models::EditorRow = fatcat::database_schema::editor::table - .find(&editor_id.to_uuid()) - .get_result(&db_conn)?; println!("{}", confectionary.create_token(editor_id, None)?); } ("inspect-token", Some(subm)) => { @@ -125,10 +103,8 @@ fn run() -> Result<()> { _ => { println!("Missing or unimplemented command!"); println!("{}", m.usage()); - ::std::process::exit(-1); + process::exit(-1); } } Ok(()) } - -quick_main!(run); diff --git a/rust/src/bin/fatcat-export.rs b/rust/src/bin/fatcat-export.rs index e1b930fc..889d7dff 100644 --- a/rust/src/bin/fatcat-export.rs +++ b/rust/src/bin/fatcat-export.rs @@ -2,25 +2,17 @@ #[macro_use] extern crate clap; -extern crate diesel; -extern crate dotenv; #[macro_use] extern crate error_chain; -extern crate fatcat; -extern crate fatcat_api_spec; #[macro_use] extern crate log; -extern crate crossbeam_channel; -extern crate env_logger; -extern crate num_cpus; -extern crate serde_json; -extern crate uuid; use clap::{App, Arg}; -use fatcat::api_entity_crud::*; -use fatcat::api_helpers::*; +use fatcat::entity_crud::*; use fatcat::errors::*; +use fatcat::identifiers::*; +use fatcat::server::*; use fatcat_api_spec::models::*; use std::str::FromStr; use uuid::Uuid; @@ -167,7 +159,7 @@ pub fn do_export( entity_type: ExportEntityType, redirects: bool, ) -> Result<()> { - let db_pool = fatcat::database_worker_pool()?; + let db_pool = database_worker_pool()?; let buf_input = BufReader::new(std::io::stdin()); let (row_sender, row_receiver) = channel::bounded(CHANNEL_BUFFER_LEN); let (output_sender, output_receiver) = channel::bounded(CHANNEL_BUFFER_LEN); @@ -232,7 +224,7 @@ pub fn do_export( Ok(()) } -fn run() -> Result<()> { +fn main() -> Result<()> { let m = App::new("fatcat-export") .version(env!("CARGO_PKG_VERSION")) .author("Bryan Newbold <bnewbold@archive.org>") @@ -273,5 +265,3 @@ fn run() -> Result<()> { m.is_present("include_redirects"), ) } - -quick_main!(run); diff --git a/rust/src/bin/fatcatd.rs b/rust/src/bin/fatcatd.rs index 682f5038..34652105 100644 --- a/rust/src/bin/fatcatd.rs +++ b/rust/src/bin/fatcatd.rs @@ -1,29 +1,35 @@ #![allow(missing_docs)] -extern crate chrono; -extern crate clap; -extern crate diesel; -//extern crate dotenv; -extern crate error_chain; -extern crate fatcat; -extern crate fatcat_api_spec; -extern crate futures; -extern crate iron; -extern crate iron_slog; #[macro_use] extern crate slog; -extern crate slog_async; -extern crate slog_term; +#[macro_use] +extern crate hyper; use clap::{App, Arg}; +use fatcat::errors::*; +use fatcat::server::*; +use iron::middleware::AfterMiddleware; use iron::modifiers::RedirectRaw; use iron::{status, Chain, Iron, IronResult, Request, Response}; use iron_slog::{DefaultLogFormatter, LoggerMiddleware}; use slog::{Drain, Logger}; +// HTTP header middleware +header! { (XClacksOverhead, "X-Clacks-Overhead") => [String] } + +pub struct XClacksOverheadMiddleware; + +impl AfterMiddleware for XClacksOverheadMiddleware { + fn after(&self, _req: &mut Request, mut res: Response) -> iron::IronResult<Response> { + res.headers + .set(XClacksOverhead("GNU aaronsw, jpb".to_owned())); + Ok(res) + } +} + /// Create custom server, wire it to the autogenerated router, /// and pass it to the web server. -fn main() { +fn main() -> Result<()> { let matches = App::new("server") .arg( Arg::with_name("https") @@ -38,7 +44,7 @@ fn main() { let logger = Logger::root(drain, o!()); let formatter = DefaultLogFormatter; - let server = fatcat::server().unwrap(); + let server = create_server()?; info!( logger, "using primary auth key: {}", server.auth_confectionary.identifier, @@ -59,7 +65,6 @@ fn main() { router.get("/v0/openapi2.yml", yaml_handler, "openapi2-spec-yaml"); fn root_handler(_: &mut Request) -> IronResult<Response> { - //Ok(Response::with((status::Found, Redirect(Url::parse("/swagger-ui").unwrap())))) Ok(Response::with(( status::Found, RedirectRaw("/swagger-ui".to_string()), @@ -92,7 +97,7 @@ fn main() { chain.link_before(fatcat_api_spec::server::ExtractAuthData); chain.link_before(fatcat::auth::MacaroonAuthMiddleware::new()); - chain.link_after(fatcat::XClacksOverheadMiddleware); + chain.link_after(XClacksOverheadMiddleware); if matches.is_present("https") { unimplemented!() @@ -100,6 +105,7 @@ fn main() { // Using HTTP Iron::new(chain) .http(host_port) - .expect("Failed to start HTTP server"); + .expect("failed to start HTTP server"); } + Ok(()) } diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs index ad9aaf29..4575aeaf 100644 --- a/rust/src/database_models.rs +++ b/rust/src/database_models.rs @@ -1,9 +1,9 @@ #![allow(proc_macro_derive_resolution_fallback)] -use crate::api_helpers::uuid2fcid; -use chrono; use crate::database_schema::*; use crate::errors::*; +use crate::identifiers::uuid2fcid; +use chrono; use fatcat_api_spec::models::{ChangelogEntry, Editgroup, Editor, EntityEdit}; use serde_json; use uuid::Uuid; diff --git a/rust/src/editing.rs b/rust/src/editing.rs new file mode 100644 index 00000000..e3777e24 --- /dev/null +++ b/rust/src/editing.rs @@ -0,0 +1,140 @@ +use crate::database_models::*; +use crate::database_schema::*; +use crate::entity_crud::EntityCrud; +use crate::errors::*; +use crate::identifiers::*; +use crate::server::*; +use diesel; +use diesel::prelude::*; +use fatcat_api_spec::models::*; +use uuid::Uuid; + +pub struct EditContext { + pub editor_id: FatCatId, + pub editgroup_id: FatCatId, + pub extra_json: Option<serde_json::Value>, + pub autoaccept: bool, +} + +impl EditContext { + /// This function should always be run within a transaction + pub fn check(&self, conn: &DbConn) -> Result<()> { + let count: i64 = changelog::table + .filter(changelog::editgroup_id.eq(&self.editgroup_id.to_uuid())) + .count() + .get_result(conn)?; + if count > 0 { + return Err(ErrorKind::EditgroupAlreadyAccepted(self.editgroup_id.to_string()).into()); + } + return Ok(()); + } +} + +pub fn make_edit_context( + conn: &DbConn, + editor_id: FatCatId, + editgroup_id: Option<FatCatId>, + autoaccept: bool, +) -> Result<EditContext> { + let editgroup_id: FatCatId = match (editgroup_id, autoaccept) { + (Some(eg), _) => eg, + // If autoaccept and no editgroup_id passed, always create a new one for this transaction + (None, true) => { + let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) + .values((editgroup::editor_id.eq(editor_id.to_uuid()),)) + .get_result(conn)?; + FatCatId::from_uuid(&eg_row.id) + } + (None, false) => FatCatId::from_uuid(&get_or_create_editgroup(editor_id.to_uuid(), conn)?), + }; + Ok(EditContext { + editor_id: editor_id, + editgroup_id: editgroup_id, + extra_json: None, + autoaccept: autoaccept, + }) +} + +pub fn create_editor( + conn: &DbConn, + username: String, + is_admin: bool, + is_bot: bool, +) -> Result<EditorRow> { + check_username(&username)?; + let ed: EditorRow = diesel::insert_into(editor::table) + .values(( + editor::username.eq(username), + editor::is_admin.eq(is_admin), + editor::is_bot.eq(is_bot), + )) + .get_result(conn)?; + Ok(ed) +} + +pub fn update_editor_username( + conn: &DbConn, + editor_id: FatCatId, + username: String, +) -> Result<EditorRow> { + check_username(&username)?; + diesel::update(editor::table.find(editor_id.to_uuid())) + .set(editor::username.eq(username)) + .execute(conn)?; + let editor: EditorRow = editor::table.find(editor_id.to_uuid()).get_result(conn)?; + Ok(editor) +} + +/// This function should always be run within a transaction +pub fn get_or_create_editgroup(editor_id: Uuid, conn: &DbConn) -> Result<Uuid> { + // check for current active + let ed_row: EditorRow = editor::table.find(editor_id).first(conn)?; + if let Some(current) = ed_row.active_editgroup_id { + return Ok(current); + } + + // need to insert and update + let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) + .values((editgroup::editor_id.eq(ed_row.id),)) + .get_result(conn)?; + diesel::update(editor::table.find(ed_row.id)) + .set(editor::active_editgroup_id.eq(eg_row.id)) + .execute(conn)?; + Ok(eg_row.id) +} + +/// This function should always be run within a transaction +pub fn accept_editgroup(editgroup_id: FatCatId, conn: &DbConn) -> Result<ChangelogRow> { + // check that we haven't accepted already (in changelog) + // NB: could leave this to a UNIQUE constraint + // TODO: redundant with check_edit_context + let count: i64 = changelog::table + .filter(changelog::editgroup_id.eq(editgroup_id.to_uuid())) + .count() + .get_result(conn)?; + if count > 0 { + return Err(ErrorKind::EditgroupAlreadyAccepted(editgroup_id.to_string()).into()); + } + + // copy edit columns to ident table + ContainerEntity::db_accept_edits(conn, editgroup_id)?; + CreatorEntity::db_accept_edits(conn, editgroup_id)?; + FileEntity::db_accept_edits(conn, editgroup_id)?; + FilesetEntity::db_accept_edits(conn, editgroup_id)?; + WebcaptureEntity::db_accept_edits(conn, editgroup_id)?; + ReleaseEntity::db_accept_edits(conn, editgroup_id)?; + WorkEntity::db_accept_edits(conn, editgroup_id)?; + + // append log/changelog row + let entry: ChangelogRow = diesel::insert_into(changelog::table) + .values((changelog::editgroup_id.eq(editgroup_id.to_uuid()),)) + .get_result(conn)?; + + // update any editor's active editgroup + let no_active: Option<Uuid> = None; + diesel::update(editor::table) + .filter(editor::active_editgroup_id.eq(editgroup_id.to_uuid())) + .set(editor::active_editgroup_id.eq(no_active)) + .execute(conn)?; + Ok(entry) +} diff --git a/rust/src/api_server.rs b/rust/src/endpoint_handlers.rs index 0377f970..d2576d53 100644 --- a/rust/src/api_server.rs +++ b/rust/src/endpoint_handlers.rs @@ -1,18 +1,20 @@ //! API endpoint handlers +//! +//! This module contains actual implementations of endpoints with rust-style type signatures. -use crate::api_entity_crud::EntityCrud; -use crate::api_helpers::*; -use crate::auth::*; -use chrono; use crate::database_models::*; use crate::database_schema::*; +use crate::editing::*; +use crate::entity_crud::{EntityCrud, ExpandFlags, HideFlags}; +use crate::errors::*; +use crate::identifiers::*; +use crate::server::*; +use chrono; use diesel::prelude::*; use diesel::{self, insert_into}; -use crate::errors::*; use fatcat_api_spec::models; use fatcat_api_spec::models::*; use std::str::FromStr; -use crate::ConnectionPool; macro_rules! entity_batch_handler { ($post_batch_handler:ident, $model:ident) => { @@ -40,12 +42,6 @@ macro_rules! entity_batch_handler { } } -#[derive(Clone)] -pub struct Server { - pub db_pool: ConnectionPool, - pub auth_confectionary: AuthConfectionary, -} - pub fn get_release_files( ident: FatCatId, hide_flags: HideFlags, diff --git a/rust/src/api_wrappers.rs b/rust/src/endpoints.rs index 69bdd88e..91db1027 100644 --- a/rust/src/api_wrappers.rs +++ b/rust/src/endpoints.rs @@ -1,12 +1,17 @@ -//! API endpoint handlers +//! API server endpoint request/response wrappers +//! +//! These mostly deal with type conversion between internal function signatures and API-defined +//! response types (mapping to HTTP statuses. Some contain actual endpoint implementations, but +//! most implementation lives in the server module. -use crate::api_entity_crud::EntityCrud; -use crate::api_helpers::*; -use crate::api_server::Server; use crate::auth::*; use crate::database_models::EntityEditRow; -use diesel::Connection; +use crate::editing::*; +use crate::entity_crud::{EntityCrud, ExpandFlags, HideFlags}; use crate::errors::*; +use crate::identifiers::*; +use crate::server::*; +use diesel::Connection; use fatcat_api_spec::models; use fatcat_api_spec::models::*; use fatcat_api_spec::*; diff --git a/rust/src/api_entity_crud.rs b/rust/src/entity_crud.rs index 44b421f9..d5c8081b 100644 --- a/rust/src/api_entity_crud.rs +++ b/rust/src/entity_crud.rs @@ -1,11 +1,13 @@ -use crate::api_helpers::*; -use crate::api_server::get_release_files; -use chrono; use crate::database_models::*; use crate::database_schema::*; +use crate::editing::*; +use crate::endpoint_handlers::get_release_files; +use crate::errors::*; +use crate::identifiers::*; +use crate::server::*; +use chrono; use diesel::prelude::*; use diesel::{self, insert_into}; -use crate::errors::*; use fatcat_api_spec::models::*; use sha1::Sha1; use std::marker::Sized; @@ -88,6 +90,174 @@ where fn db_insert_revs(conn: &DbConn, models: &[&Self]) -> Result<Vec<Uuid>>; } +#[derive(Clone, Copy, PartialEq)] +pub struct ExpandFlags { + pub files: bool, + pub filesets: bool, + pub webcaptures: bool, + pub container: bool, + pub releases: bool, + pub creators: bool, +} + +impl FromStr for ExpandFlags { + type Err = Error; + fn from_str(param: &str) -> Result<ExpandFlags> { + let list: Vec<&str> = param.split_terminator(",").collect(); + Ok(ExpandFlags::from_str_list(&list)) + } +} + +impl ExpandFlags { + pub fn from_str_list(list: &[&str]) -> ExpandFlags { + ExpandFlags { + files: list.contains(&"files"), + filesets: list.contains(&"filesets"), + webcaptures: list.contains(&"webcaptures"), + container: list.contains(&"container"), + releases: list.contains(&"releases"), + creators: list.contains(&"creators"), + } + } + pub fn none() -> ExpandFlags { + ExpandFlags { + files: false, + filesets: false, + webcaptures: false, + container: false, + releases: false, + creators: false, + } + } +} + +#[test] +fn test_expand_flags() { + assert!(ExpandFlags::from_str_list(&vec![]).files == false); + assert!(ExpandFlags::from_str_list(&vec!["files"]).files == true); + assert!(ExpandFlags::from_str_list(&vec!["file"]).files == false); + let all = ExpandFlags::from_str_list(&vec![ + "files", + "filesets", + "webcaptures", + "container", + "other_thing", + "releases", + "creators", + ]); + assert!( + all == ExpandFlags { + files: true, + filesets: true, + webcaptures: true, + container: true, + releases: true, + creators: true + } + ); + assert!(ExpandFlags::from_str("").unwrap().files == false); + assert!(ExpandFlags::from_str("files").unwrap().files == true); + assert!(ExpandFlags::from_str("something,,files").unwrap().files == true); + assert!(ExpandFlags::from_str("file").unwrap().files == false); + let all = + ExpandFlags::from_str("files,container,other_thing,releases,creators,filesets,webcaptures") + .unwrap(); + assert!( + all == ExpandFlags { + files: true, + filesets: true, + webcaptures: true, + container: true, + releases: true, + creators: true + } + ); +} + +#[derive(Clone, Copy, PartialEq)] +pub struct HideFlags { + // release + pub abstracts: bool, + pub refs: bool, + pub contribs: bool, + // fileset + pub manifest: bool, + // webcapture + pub cdx: bool, +} + +impl FromStr for HideFlags { + type Err = Error; + fn from_str(param: &str) -> Result<HideFlags> { + let list: Vec<&str> = param.split_terminator(",").collect(); + Ok(HideFlags::from_str_list(&list)) + } +} + +impl HideFlags { + pub fn from_str_list(list: &[&str]) -> HideFlags { + HideFlags { + abstracts: list.contains(&"abstracts"), + refs: list.contains(&"refs"), + contribs: list.contains(&"contribs"), + manifest: list.contains(&"contribs"), + cdx: list.contains(&"contribs"), + } + } + pub fn none() -> HideFlags { + HideFlags { + abstracts: false, + refs: false, + contribs: false, + manifest: false, + cdx: false, + } + } +} + +#[test] +fn test_hide_flags() { + assert!(HideFlags::from_str_list(&vec![]).abstracts == false); + assert!(HideFlags::from_str_list(&vec!["abstracts"]).abstracts == true); + assert!(HideFlags::from_str_list(&vec!["abstract"]).abstracts == false); + let all = HideFlags::from_str_list(&vec![ + "abstracts", + "refs", + "other_thing", + "contribs", + "manifest", + "cdx", + ]); + assert!( + all == HideFlags { + abstracts: true, + refs: true, + contribs: true, + manifest: true, + cdx: true, + } + ); + assert!(HideFlags::from_str("").unwrap().abstracts == false); + assert!(HideFlags::from_str("abstracts").unwrap().abstracts == true); + assert!( + HideFlags::from_str("something,,abstracts") + .unwrap() + .abstracts + == true + ); + assert!(HideFlags::from_str("file").unwrap().abstracts == false); + let all = HideFlags::from_str("abstracts,cdx,refs,manifest,other_thing,contribs").unwrap(); + assert!( + all == HideFlags { + abstracts: true, + refs: true, + contribs: true, + manifest: true, + cdx: true, + } + ); +} + macro_rules! generic_db_get { ($ident_table:ident, $rev_table:ident) => { fn db_get(conn: &DbConn, ident: FatCatId, hide: HideFlags) -> Result<Self> { diff --git a/rust/src/errors.rs b/rust/src/errors.rs new file mode 100644 index 00000000..0b966e93 --- /dev/null +++ b/rust/src/errors.rs @@ -0,0 +1,55 @@ +//! Crate-specific Result, Error, and ErrorKind types (using `error_chain`) + +error_chain! { + foreign_links { Fmt(::std::fmt::Error); + Diesel(::diesel::result::Error); + R2d2(::diesel::r2d2::Error); + Uuid(::uuid::ParseError); + Io(::std::io::Error) #[cfg(unix)]; + Serde(::serde_json::Error); + Utf8Decode(::std::string::FromUtf8Error); + StringDecode(::data_encoding::DecodeError); + } + errors { + InvalidFatcatId(id: String) { + description("invalid fatcat identifier syntax") + display("invalid fatcat identifier (expect 26-char base32 encoded): {}", id) + } + MalformedExternalId(id: String) { + description("external identifier doesn't match required pattern") + display("external identifier doesn't match required pattern: {}", id) + } + MalformedChecksum(hash: String) { + description("checksum doesn't match required pattern (hex encoding)") + display("checksum doesn't match required pattern (hex encoding): {}", hash) + } + NotInControlledVocabulary(word: String) { + description("word or type not correct for controlled vocabulary") + display("word or type not correct for controlled vocabulary") + } + EditgroupAlreadyAccepted(id: String) { + description("editgroup was already accepted") + display("attempted to accept or mutate an editgroup which was already accepted: {}", id) + } + MissingOrMultipleExternalId(message: String) { + description("external identifiers missing or multiple specified") + display("external identifiers missing or multiple specified; please supply exactly one") + } + InvalidEntityStateTransform(message: String) { + description("Invalid Entity State Transform") + display("tried to mutate an entity which was not in an appropriate state: {}", message) + } + InvalidCredentials(message: String) { + description("auth token was missing, expired, revoked, or corrupt") + display("auth token was missing, expired, revoked, or corrupt: {}", message) + } + InsufficientPrivileges(message: String) { + description("editor account doesn't have authorization") + display("editor account doesn't have authorization: {}", message) + } + OtherBadRequest(message: String) { + description("catch-all error for bad or unallowed requests") + display("broke a constraint or made an otherwise invalid request: {}", message) + } + } +} diff --git a/rust/src/api_helpers.rs b/rust/src/identifiers.rs index 55085403..adb9f413 100644 --- a/rust/src/api_helpers.rs +++ b/rust/src/identifiers.rs @@ -1,317 +1,10 @@ -use crate::api_entity_crud::EntityCrud; -use data_encoding::BASE32_NOPAD; -use crate::database_models::*; -use crate::database_schema::*; -use diesel; -use diesel::prelude::*; use crate::errors::*; -use fatcat_api_spec::models::*; +use data_encoding::BASE32_NOPAD; use regex::Regex; use serde_json; use std::str::FromStr; use uuid::Uuid; -pub type DbConn = - diesel::r2d2::PooledConnection<diesel::r2d2::ConnectionManager<diesel::PgConnection>>; - -pub struct EditContext { - pub editor_id: FatCatId, - pub editgroup_id: FatCatId, - pub extra_json: Option<serde_json::Value>, - pub autoaccept: bool, -} - -impl EditContext { - /// This function should always be run within a transaction - pub fn check(&self, conn: &DbConn) -> Result<()> { - let count: i64 = changelog::table - .filter(changelog::editgroup_id.eq(&self.editgroup_id.to_uuid())) - .count() - .get_result(conn)?; - if count > 0 { - return Err(ErrorKind::EditgroupAlreadyAccepted(self.editgroup_id.to_string()).into()); - } - return Ok(()); - } -} - -#[derive(Clone, Copy, PartialEq)] -pub struct ExpandFlags { - pub files: bool, - pub filesets: bool, - pub webcaptures: bool, - pub container: bool, - pub releases: bool, - pub creators: bool, -} - -impl FromStr for ExpandFlags { - type Err = Error; - fn from_str(param: &str) -> Result<ExpandFlags> { - let list: Vec<&str> = param.split_terminator(",").collect(); - Ok(ExpandFlags::from_str_list(&list)) - } -} - -impl ExpandFlags { - pub fn from_str_list(list: &[&str]) -> ExpandFlags { - ExpandFlags { - files: list.contains(&"files"), - filesets: list.contains(&"filesets"), - webcaptures: list.contains(&"webcaptures"), - container: list.contains(&"container"), - releases: list.contains(&"releases"), - creators: list.contains(&"creators"), - } - } - pub fn none() -> ExpandFlags { - ExpandFlags { - files: false, - filesets: false, - webcaptures: false, - container: false, - releases: false, - creators: false, - } - } -} - -#[test] -fn test_expand_flags() { - assert!(ExpandFlags::from_str_list(&vec![]).files == false); - assert!(ExpandFlags::from_str_list(&vec!["files"]).files == true); - assert!(ExpandFlags::from_str_list(&vec!["file"]).files == false); - let all = ExpandFlags::from_str_list(&vec![ - "files", - "filesets", - "webcaptures", - "container", - "other_thing", - "releases", - "creators", - ]); - assert!( - all == ExpandFlags { - files: true, - filesets: true, - webcaptures: true, - container: true, - releases: true, - creators: true - } - ); - assert!(ExpandFlags::from_str("").unwrap().files == false); - assert!(ExpandFlags::from_str("files").unwrap().files == true); - assert!(ExpandFlags::from_str("something,,files").unwrap().files == true); - assert!(ExpandFlags::from_str("file").unwrap().files == false); - let all = - ExpandFlags::from_str("files,container,other_thing,releases,creators,filesets,webcaptures") - .unwrap(); - assert!( - all == ExpandFlags { - files: true, - filesets: true, - webcaptures: true, - container: true, - releases: true, - creators: true - } - ); -} - -#[derive(Clone, Copy, PartialEq)] -pub struct HideFlags { - // release - pub abstracts: bool, - pub refs: bool, - pub contribs: bool, - // fileset - pub manifest: bool, - // webcapture - pub cdx: bool, -} - -impl FromStr for HideFlags { - type Err = Error; - fn from_str(param: &str) -> Result<HideFlags> { - let list: Vec<&str> = param.split_terminator(",").collect(); - Ok(HideFlags::from_str_list(&list)) - } -} - -impl HideFlags { - pub fn from_str_list(list: &[&str]) -> HideFlags { - HideFlags { - abstracts: list.contains(&"abstracts"), - refs: list.contains(&"refs"), - contribs: list.contains(&"contribs"), - manifest: list.contains(&"contribs"), - cdx: list.contains(&"contribs"), - } - } - pub fn none() -> HideFlags { - HideFlags { - abstracts: false, - refs: false, - contribs: false, - manifest: false, - cdx: false, - } - } -} - -#[test] -fn test_hide_flags() { - assert!(HideFlags::from_str_list(&vec![]).abstracts == false); - assert!(HideFlags::from_str_list(&vec!["abstracts"]).abstracts == true); - assert!(HideFlags::from_str_list(&vec!["abstract"]).abstracts == false); - let all = HideFlags::from_str_list(&vec![ - "abstracts", - "refs", - "other_thing", - "contribs", - "manifest", - "cdx", - ]); - assert!( - all == HideFlags { - abstracts: true, - refs: true, - contribs: true, - manifest: true, - cdx: true, - } - ); - assert!(HideFlags::from_str("").unwrap().abstracts == false); - assert!(HideFlags::from_str("abstracts").unwrap().abstracts == true); - assert!( - HideFlags::from_str("something,,abstracts") - .unwrap() - .abstracts - == true - ); - assert!(HideFlags::from_str("file").unwrap().abstracts == false); - let all = HideFlags::from_str("abstracts,cdx,refs,manifest,other_thing,contribs").unwrap(); - assert!( - all == HideFlags { - abstracts: true, - refs: true, - contribs: true, - manifest: true, - cdx: true, - } - ); -} - -pub fn make_edit_context( - conn: &DbConn, - editor_id: FatCatId, - editgroup_id: Option<FatCatId>, - autoaccept: bool, -) -> Result<EditContext> { - let editgroup_id: FatCatId = match (editgroup_id, autoaccept) { - (Some(eg), _) => eg, - // If autoaccept and no editgroup_id passed, always create a new one for this transaction - (None, true) => { - let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) - .values((editgroup::editor_id.eq(editor_id.to_uuid()),)) - .get_result(conn)?; - FatCatId::from_uuid(&eg_row.id) - } - (None, false) => FatCatId::from_uuid(&get_or_create_editgroup(editor_id.to_uuid(), conn)?), - }; - Ok(EditContext { - editor_id: editor_id, - editgroup_id: editgroup_id, - extra_json: None, - autoaccept: autoaccept, - }) -} - -pub fn create_editor( - conn: &DbConn, - username: String, - is_admin: bool, - is_bot: bool, -) -> Result<EditorRow> { - check_username(&username)?; - let ed: EditorRow = diesel::insert_into(editor::table) - .values(( - editor::username.eq(username), - editor::is_admin.eq(is_admin), - editor::is_bot.eq(is_bot), - )) - .get_result(conn)?; - Ok(ed) -} - -pub fn update_editor_username( - conn: &DbConn, - editor_id: FatCatId, - username: String, -) -> Result<EditorRow> { - check_username(&username)?; - diesel::update(editor::table.find(editor_id.to_uuid())) - .set(editor::username.eq(username)) - .execute(conn)?; - let editor: EditorRow = editor::table.find(editor_id.to_uuid()).get_result(conn)?; - Ok(editor) -} - -/// This function should always be run within a transaction -pub fn get_or_create_editgroup(editor_id: Uuid, conn: &DbConn) -> Result<Uuid> { - // check for current active - let ed_row: EditorRow = editor::table.find(editor_id).first(conn)?; - if let Some(current) = ed_row.active_editgroup_id { - return Ok(current); - } - - // need to insert and update - let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) - .values((editgroup::editor_id.eq(ed_row.id),)) - .get_result(conn)?; - diesel::update(editor::table.find(ed_row.id)) - .set(editor::active_editgroup_id.eq(eg_row.id)) - .execute(conn)?; - Ok(eg_row.id) -} - -/// This function should always be run within a transaction -pub fn accept_editgroup(editgroup_id: FatCatId, conn: &DbConn) -> Result<ChangelogRow> { - // check that we haven't accepted already (in changelog) - // NB: could leave this to a UNIQUE constraint - // TODO: redundant with check_edit_context - let count: i64 = changelog::table - .filter(changelog::editgroup_id.eq(editgroup_id.to_uuid())) - .count() - .get_result(conn)?; - if count > 0 { - return Err(ErrorKind::EditgroupAlreadyAccepted(editgroup_id.to_string()).into()); - } - - // copy edit columns to ident table - ContainerEntity::db_accept_edits(conn, editgroup_id)?; - CreatorEntity::db_accept_edits(conn, editgroup_id)?; - FileEntity::db_accept_edits(conn, editgroup_id)?; - FilesetEntity::db_accept_edits(conn, editgroup_id)?; - WebcaptureEntity::db_accept_edits(conn, editgroup_id)?; - ReleaseEntity::db_accept_edits(conn, editgroup_id)?; - WorkEntity::db_accept_edits(conn, editgroup_id)?; - - // append log/changelog row - let entry: ChangelogRow = diesel::insert_into(changelog::table) - .values((changelog::editgroup_id.eq(editgroup_id.to_uuid()),)) - .get_result(conn)?; - - // update any editor's active editgroup - let no_active: Option<Uuid> = None; - diesel::update(editor::table) - .filter(editor::active_editgroup_id.eq(editgroup_id.to_uuid())) - .set(editor::active_editgroup_id.eq(no_active)) - .execute(conn)?; - Ok(entry) -} - #[derive(Clone, Copy, PartialEq, Debug)] pub struct FatCatId(Uuid); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 18550a5d..df3d6f51 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,202 +1,25 @@ #![allow(proc_macro_derive_resolution_fallback)] #![recursion_limit = "128"] -extern crate chrono; -extern crate fatcat_api_spec; -#[macro_use] -extern crate diesel; -extern crate diesel_migrations; -extern crate dotenv; -extern crate futures; -extern crate uuid; -#[macro_use] -extern crate hyper; -extern crate swagger; #[macro_use] extern crate error_chain; -extern crate iron; -extern crate serde_json; +#[macro_use] +extern crate diesel; #[macro_use] extern crate log; -extern crate data_encoding; -extern crate regex; #[macro_use] extern crate lazy_static; -extern crate macaroon; -extern crate sha1; -extern crate rand; -pub mod api_entity_crud; -pub mod api_helpers; -pub mod api_server; -pub mod api_wrappers; pub mod auth; pub mod database_models; -pub mod database_schema; - -pub mod errors { - // Create the Error, ErrorKind, ResultExt, and Result types - error_chain! { - foreign_links { Fmt(::std::fmt::Error); - Diesel(::diesel::result::Error); - R2d2(::diesel::r2d2::Error); - Uuid(::uuid::ParseError); - Io(::std::io::Error) #[cfg(unix)]; - Serde(::serde_json::Error); - Utf8Decode(::std::string::FromUtf8Error); - StringDecode(::data_encoding::DecodeError); - } - errors { - InvalidFatcatId(id: String) { - description("invalid fatcat identifier syntax") - display("invalid fatcat identifier (expect 26-char base32 encoded): {}", id) - } - MalformedExternalId(id: String) { - description("external identifier doesn't match required pattern") - display("external identifier doesn't match required pattern: {}", id) - } - MalformedChecksum(hash: String) { - description("checksum doesn't match required pattern (hex encoding)") - display("checksum doesn't match required pattern (hex encoding): {}", hash) - } - NotInControlledVocabulary(word: String) { - description("word or type not correct for controlled vocabulary") - display("word or type not correct for controlled vocabulary") - } - EditgroupAlreadyAccepted(id: String) { - description("editgroup was already accepted") - display("attempted to accept or mutate an editgroup which was already accepted: {}", id) - } - MissingOrMultipleExternalId(message: String) { - description("external identifiers missing or multiple specified") - display("external identifiers missing or multiple specified; please supply exactly one") - } - InvalidEntityStateTransform(message: String) { - description("Invalid Entity State Transform") - display("tried to mutate an entity which was not in an appropriate state: {}", message) - } - InvalidCredentials(message: String) { - description("auth token was missing, expired, revoked, or corrupt") - display("auth token was missing, expired, revoked, or corrupt: {}", message) - } - InsufficientPrivileges(message: String) { - description("editor account doesn't have authorization") - display("editor account doesn't have authorization: {}", message) - } - OtherBadRequest(message: String) { - description("catch-all error for bad or unallowed requests") - display("broke a constraint or made an otherwise invalid request: {}", message) - } - } - } -} - -#[doc(hidden)] -pub use crate::errors::*; - -pub use self::errors::*; -use crate::auth::AuthConfectionary; -use diesel::pg::PgConnection; -use diesel::r2d2::ConnectionManager; -use dotenv::dotenv; -use iron::middleware::AfterMiddleware; -use iron::{Request, Response}; -use std::{env, thread, time}; -use std::process::Command; -use rand::Rng; - -#[cfg(feature = "postgres")] -embed_migrations!("../migrations/"); - -pub type ConnectionPool = diesel::r2d2::Pool<ConnectionManager<diesel::pg::PgConnection>>; - -/// Instantiate a new API server with a pooled database connection -pub fn database_worker_pool() -> Result<ConnectionPool> { - dotenv().ok(); - let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let manager = ConnectionManager::<PgConnection>::new(database_url); - let pool = diesel::r2d2::Pool::builder() - .build(manager) - .expect("Failed to create database pool."); - Ok(pool) -} - -pub fn env_confectionary() -> Result<AuthConfectionary> { - let auth_location = env::var("AUTH_LOCATION").expect("AUTH_LOCATION must be set"); - let auth_key = env::var("AUTH_SECRET_KEY").expect("AUTH_SECRET_KEY must be set"); - let auth_key_ident = env::var("AUTH_KEY_IDENT").expect("AUTH_KEY_IDENT must be set"); - info!("Loaded primary auth key: {}", auth_key_ident); - let mut confectionary = AuthConfectionary::new(auth_location, auth_key_ident, auth_key)?; - match env::var("AUTH_ALT_KEYS") { - Ok(var) => { - for pair in var.split(",") { - let pair: Vec<&str> = pair.split(":").collect(); - if pair.len() != 2 { - println!("{:#?}", pair); - bail!("couldn't parse keypair from AUTH_ALT_KEYS (expected 'ident:key' pairs separated by commas)"); - } - info!("Loading alt auth key: {}", pair[0]); - confectionary.add_keypair(pair[0].to_string(), pair[1].to_string())?; - } - } - Err(_) => (), - } - Ok(confectionary) -} - -/// Instantiate a new API server with a pooled database connection -pub fn server() -> Result<api_server::Server> { - dotenv().ok(); - let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let manager = ConnectionManager::<PgConnection>::new(database_url); - let pool = diesel::r2d2::Pool::builder() - .build(manager) - .expect("Failed to create database pool."); - let confectionary = env_confectionary()?; - Ok(api_server::Server { - db_pool: pool, - auth_confectionary: confectionary, - }) -} - -/// Generates a server for testing. Calls an external bash script to generate a random postgres -/// database, which will be unique to this process but common across threads and connections. The -/// database will automagically get cleaned up (deleted) after 60 seconds. -/// Currently, start times are staggered by up to 200ms to prevent internal postgres concurrency -/// errors; if this fails run the tests serially (one at a time), which is slower but more robust. -/// CI should run tests serially. -pub fn test_server() -> Result<api_server::Server> { - dotenv().ok(); - // sleep a bit so we don't have thundering herd collisions, resuliting in - // "pg_extension_name_index" or "pg_proc_proname_args_nsp_index" or "pg_type_typname_nsp_index" - // duplicate key violations. - thread::sleep(time::Duration::from_millis(rand::thread_rng().gen_range(0, 200))); - let pg_tmp = Command::new("./tests/pg_tmp.sh") - .output() - .expect("run ./tests/pg_tmp.sh to get temporary postgres DB"); - let database_url = String::from_utf8_lossy(&pg_tmp.stdout).to_string(); - env::set_var("DATABASE_URL", database_url); - - let mut server = server()?; - server.auth_confectionary = AuthConfectionary::new_dummy(); - let conn = server.db_pool.get().expect("db_pool error"); - - // run migrations; this is a fresh/bare database - diesel_migrations::run_pending_migrations(&conn).unwrap(); - Ok(server) -} - -// TODO: move this to bin/fatcatd - -/// HTTP header middleware -header! { (XClacksOverhead, "X-Clacks-Overhead") => [String] } - -pub struct XClacksOverheadMiddleware; - -impl AfterMiddleware for XClacksOverheadMiddleware { - fn after(&self, _req: &mut Request, mut res: Response) -> iron::IronResult<Response> { - res.headers - .set(XClacksOverhead("GNU aaronsw, jpb".to_owned())); - Ok(res) - } -} +pub mod database_schema; // only public for tests +pub mod editing; +mod endpoint_handlers; +mod endpoints; +pub mod entity_crud; +pub mod errors; +pub mod identifiers; +pub mod server; + +// TODO: will probably remove these as a public export? +pub use crate::server::{create_server, create_test_server}; diff --git a/rust/src/server.rs b/rust/src/server.rs new file mode 100644 index 00000000..70e667be --- /dev/null +++ b/rust/src/server.rs @@ -0,0 +1,81 @@ +//! API endpoint handlers + +use crate::auth::*; +use crate::errors::*; +use chrono; +use diesel; +use diesel::pg::PgConnection; +use diesel::r2d2::ConnectionManager; +use dotenv::dotenv; +use rand::Rng; +use std::process::Command; +use std::{env, thread, time}; + +#[cfg(feature = "postgres")] +embed_migrations!("../migrations/"); + +pub type ConnectionPool = diesel::r2d2::Pool<ConnectionManager<diesel::pg::PgConnection>>; + +pub type DbConn = + diesel::r2d2::PooledConnection<diesel::r2d2::ConnectionManager<diesel::PgConnection>>; + +/// Instantiate a new API server with a pooled database connection +pub fn database_worker_pool() -> Result<ConnectionPool> { + dotenv().ok(); + let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let manager = ConnectionManager::<PgConnection>::new(database_url); + let pool = diesel::r2d2::Pool::builder() + .build(manager) + .expect("Failed to create database pool."); + Ok(pool) +} + +#[derive(Clone)] +pub struct Server { + pub db_pool: ConnectionPool, + pub auth_confectionary: AuthConfectionary, +} + +/// Instantiate a new API server with a pooled database connection +pub fn create_server() -> Result<Server> { + dotenv().ok(); + let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let manager = ConnectionManager::<PgConnection>::new(database_url); + let pool = diesel::r2d2::Pool::builder() + .build(manager) + .expect("Failed to create database pool."); + let confectionary = env_confectionary()?; + Ok(Server { + db_pool: pool, + auth_confectionary: confectionary, + }) +} + +/// Generates a server for testing. Calls an external bash script to generate a random postgres +/// database, which will be unique to this process but common across threads and connections. The +/// database will automagically get cleaned up (deleted) after 60 seconds. +/// Currently, start times are staggered by up to 200ms to prevent internal postgres concurrency +/// errors; if this fails run the tests serially (one at a time), which is slower but more robust. +/// CI should run tests serially. +pub fn create_test_server() -> Result<Server> { + dotenv().ok(); + // sleep a bit so we don't have thundering herd collisions, resuliting in + // "pg_extension_name_index" or "pg_proc_proname_args_nsp_index" or "pg_type_typname_nsp_index" + // duplicate key violations. + thread::sleep(time::Duration::from_millis( + rand::thread_rng().gen_range(0, 200), + )); + let pg_tmp = Command::new("./tests/pg_tmp.sh") + .output() + .expect("run ./tests/pg_tmp.sh to get temporary postgres DB"); + let database_url = String::from_utf8_lossy(&pg_tmp.stdout).to_string(); + env::set_var("DATABASE_URL", database_url); + + let mut server = create_server()?; + server.auth_confectionary = AuthConfectionary::new_dummy(); + let conn = server.db_pool.get().expect("db_pool error"); + + // run migrations; this is a fresh/bare database + diesel_migrations::run_pending_migrations(&conn).unwrap(); + Ok(server) +} |