From 5a7e60632e41c7f6b7ae0b7e3ab413209c88ea4b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 24 Jul 2018 02:10:36 -0700 Subject: partial implementation of new API features --- rust/src/api_server.rs | 119 +++++++++++++++++++++++++++++++------------- rust/src/database_models.rs | 61 ++++++++++++++++++++--- rust/src/database_schema.rs | 75 +++++++++++++++++++++------- 3 files changed, 195 insertions(+), 60 deletions(-) (limited to 'rust/src') diff --git a/rust/src/api_server.rs b/rust/src/api_server.rs index b260a98a..a2a3a182 100644 --- a/rust/src/api_server.rs +++ b/rust/src/api_server.rs @@ -1,19 +1,21 @@ //! API endpoint handlers -use ConnectionPool; -use api_helpers::{accept_editgroup, get_or_create_editgroup, fcid2uuid, uuid2fcid}; +use api_helpers::{accept_editgroup, fcid2uuid, get_or_create_editgroup, uuid2fcid}; use chrono; use database_models::*; -use database_schema::{changelog, container_edit, container_ident, container_rev, creator_edit, - creator_ident, creator_rev, editgroup, editor, file_edit, file_ident, - file_release, file_rev, release_contrib, release_edit, release_ident, - release_ref, release_rev, work_edit, work_ident, work_rev}; +use database_schema::{ + changelog, container_edit, container_ident, container_rev, creator_edit, creator_ident, + creator_rev, editgroup, editor, file_edit, file_ident, file_release, file_rev, file_rev_url, + release_contrib, release_edit, release_ident, release_ref, release_rev, release_rev_abstract, + work_edit, work_ident, work_rev, +}; use diesel::prelude::*; use diesel::{self, insert_into}; use errors::*; use fatcat_api::models; use fatcat_api::models::*; use uuid::Uuid; +use ConnectionPool; type DbConn = diesel::r2d2::PooledConnection>; @@ -146,8 +148,18 @@ fn file_row2entity( let releases: Vec = file_release::table .filter(file_release::file_rev.eq(rev.id)) .get_results(conn)? - .iter() - .map(|r: &FileReleaseRow| uuid2fcid(&r.target_release_ident_id)) + .into_iter() + .map(|r: FileReleaseRow| uuid2fcid(&r.target_release_ident_id)) + .collect(); + + let urls: Vec = file_rev_url::table + .filter(file_rev_url::file_rev.eq(rev.id)) + .get_results(conn)? + .into_iter() + .map(|r: FileRevUrlRow| FileEntityUrls { + rel: r.rel, + url: r.url, + }) .collect(); Ok(FileEntity { @@ -155,7 +167,7 @@ fn file_row2entity( sha256: rev.sha256, md5: rev.md5, size: rev.size.map(|v| v as i64), - url: rev.url, + urls: Some(urls), mimetype: rev.mimetype, releases: Some(releases), state: state, @@ -186,15 +198,15 @@ fn release_row2entity( .order(release_ref::index.asc()) .get_results(conn) .expect("fetch release refs") - .iter() - .map(|r: &ReleaseRefRow| ReleaseRef { - index: r.index.clone(), - key: r.key.clone(), - raw: r.raw.clone(), - container_title: r.container_title.clone(), - year: r.year.clone(), - title: r.title.clone(), - locator: r.locator.clone(), + .into_iter() + .map(|r: ReleaseRefRow| ReleaseRef { + index: r.index, + key: r.key, + extra: r.extra_json, + container_title: r.container_title, + year: r.year, + title: r.title, + locator: r.locator, target_release_id: r.target_release_ident_id.map(|v| uuid2fcid(&v)), }) .collect(); @@ -204,15 +216,29 @@ fn release_row2entity( .order((release_contrib::role.asc(), release_contrib::index.asc())) .get_results(conn) .expect("fetch release refs") - .iter() - .map(|c: &ReleaseContribRow| ReleaseContrib { + .into_iter() + .map(|c: ReleaseContribRow| ReleaseContrib { index: c.index, - role: c.role.clone(), - raw: c.raw.clone(), + raw: c.raw, + role: c.role, + extra: c.extra_json, creator_id: c.creator_ident_id.map(|v| uuid2fcid(&v)), }) .collect(); + // XXX: join abstracts table + let abstracts: Vec = release_rev_abstract::table + .filter(release_rev_abstract::release_rev.eq(rev.id)) + .get_results(conn)? + .into_iter() + .map(|r: ReleaseRevAbstractRow| ReleaseEntityAbstracts { + sha1: Some(r.abstract_sha1), + mimetype: r.mimetype, + lang: r.lang, + content: None, + }) + .collect(); + Ok(ReleaseEntity { title: rev.title, release_type: rev.release_type, @@ -220,6 +246,8 @@ fn release_row2entity( release_date: rev.release_date .map(|v| chrono::DateTime::from_utc(v.and_hms(0, 0, 0), chrono::Utc)), doi: rev.doi, + pmid: rev.pmid, + pmcid: rev.pmcid, isbn13: rev.isbn13, volume: rev.volume, issue: rev.issue, @@ -230,6 +258,7 @@ fn release_row2entity( work_id: Some(uuid2fcid(&rev.work_ident_id)), refs: Some(refs), contribs: Some(contribs), + abstracts: Some(abstracts), state: state, ident: ident_id, revision: Some(rev.id.to_string()), @@ -249,7 +278,6 @@ fn work_row2entity(ident: Option, rev: WorkRevRow) -> Result (None, None, None), }; Ok(WorkEntity { - work_type: rev.work_type, state: state, ident: ident_id, revision: Some(rev.id.to_string()), @@ -530,20 +558,19 @@ impl Server { let edit: FileEditRow = diesel::sql_query( - "WITH rev AS ( INSERT INTO file_rev (size, sha1, sha256, md5, url, mimetype, extra_json) - VALUES ($1, $2, $3, $4, $5, $6, $7) + "WITH rev AS ( INSERT INTO file_rev (size, sha1, sha256, md5, mimetype, extra_json) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id ), ident AS ( INSERT INTO file_ident (rev_id) VALUES ((SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO file_edit (editgroup_id, ident_id, rev_id) VALUES - ($8, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($7, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::, _>(entity.size) .bind::, _>(entity.sha1) .bind::, _>(entity.sha256) .bind::, _>(entity.md5) - .bind::, _>(entity.url) .bind::, _>(entity.mimetype) .bind::, _>(entity.extra) .bind::(editgroup_id) @@ -572,6 +599,29 @@ impl Server { } }; + let _urls: Option> = match entity.urls { + None => None, + Some(url_list) => { + if url_list.len() == 0 { + Some(vec![]) + } else { + let url_rows: Vec = url_list + .into_iter() + .map(|u| FileRevUrlNewRow { + file_rev: edit.rev_id.unwrap(), + rel: u.rel, + url: u.url, + }) + .collect(); + let url_rows: Vec = insert_into(file_rev_url::table) + .values(url_rows) + .get_results(conn) + .expect("error inserting file_rev_url"); + Some(url_rows) + } + } + }; + edit.to_model() } @@ -600,7 +650,6 @@ impl Server { None => { // If a work_id wasn't passed, create a new work under the current editgroup let work_model = models::WorkEntity { - work_type: None, ident: None, revision: None, redirect: None, @@ -665,7 +714,7 @@ impl Server { year: r.year, title: r.title.clone(), locator: r.locator.clone(), - raw: r.raw.clone(), + extra_json: r.extra.clone(), }) .collect(); let ref_rows: Vec = insert_into(release_ref::table) @@ -690,9 +739,10 @@ impl Server { creator_ident_id: c.creator_id .clone() .map(|v| fcid2uuid(&v).expect("valid fatcat identifier")), + raw: c.raw.clone(), index: c.index, role: c.role.clone(), - raw: c.raw.clone(), + extra_json: c.extra.clone(), }) .collect(); let contrib_rows: Vec = insert_into(release_contrib::table) @@ -730,17 +780,16 @@ impl Server { let edit: WorkEditRow = diesel::sql_query( - "WITH rev AS ( INSERT INTO work_rev (work_type, extra_json) - VALUES ($1, $2) + "WITH rev AS ( INSERT INTO work_rev (extra_json) + VALUES ($1) RETURNING id ), ident AS ( INSERT INTO work_ident (rev_id) VALUES ((SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO work_edit (editgroup_id, ident_id, rev_id) VALUES - ($3, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($2, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", - ).bind::, _>(entity.work_type) - .bind::, _>(entity.extra) + ).bind::, _>(entity.extra) .bind::(editgroup_id) .get_result(conn)?; diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs index 167d2fbe..58dc4d42 100644 --- a/rust/src/database_models.rs +++ b/rust/src/database_models.rs @@ -43,9 +43,11 @@ macro_rules! entity_structs { pub struct $edit_struct { pub id: i64, pub editgroup_id: Uuid, + pub updated: chrono::NaiveDateTime, pub ident_id: Uuid, pub rev_id: Option, pub redirect_id: Option, + pub prev_rev: Option, pub extra_json: Option, } @@ -56,6 +58,7 @@ macro_rules! entity_structs { editgroup_id: uuid2fcid(&self.editgroup_id), revision: self.rev_id.map(|v| v.to_string()), redirect_ident: self.redirect_id.map(|v| uuid2fcid(&v)), + prev_revision: self.prev_rev.map(|v| v.to_string()), ident: uuid2fcid(&self.ident_id), edit_id: self.id, extra: self.extra_json, @@ -96,6 +99,7 @@ pub struct ContainerRevRow { pub name: String, pub publisher: Option, pub issnl: Option, + pub wikidata_qid: Option, pub abbrev: Option, pub coden: Option, } @@ -116,6 +120,7 @@ pub struct CreatorRevRow { pub given_name: Option, pub surname: Option, pub orcid: Option, + pub wikidata_qid: Option, } entity_structs!( @@ -125,6 +130,23 @@ entity_structs!( CreatorIdentRow ); +#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)] +#[table_name = "file_rev_url"] +pub struct FileRevUrlRow { + pub id: i64, + pub file_rev: Uuid, + pub rel: String, + pub url: String, +} + +#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)] +#[table_name = "file_rev_url"] +pub struct FileRevUrlNewRow { + pub file_rev: Uuid, + pub rel: String, + pub url: String, +} + #[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)] #[table_name = "file_rev"] pub struct FileRevRow { @@ -134,7 +156,6 @@ pub struct FileRevRow { pub sha1: Option, pub sha256: Option, pub md5: Option, - pub url: Option, pub mimetype: Option, } @@ -152,6 +173,9 @@ pub struct ReleaseRevRow { pub release_status: Option, pub release_date: Option, pub doi: Option, + pub pmid: Option, + pub pmcid: Option, + pub wikidata_qid: Option, pub isbn13: Option, pub volume: Option, pub issue: Option, @@ -172,21 +196,39 @@ entity_structs!( pub struct WorkRevRow { pub id: Uuid, pub extra_json: Option, - pub work_type: Option, - pub primary_release_id: Option, } entity_structs!("work_edit", WorkEditRow, "work_ident", WorkIdentRow); +#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)] +#[table_name = "release_rev_abstract"] +pub struct ReleaseRevAbstractRow { + pub id: i64, + pub release_rev: Uuid, + pub abstract_sha1: String, + pub mimetype: Option, + pub lang: Option, +} + +#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)] +#[table_name = "release_rev_abstract"] +pub struct ReleaseRevAbstractNewRow { + pub release_rev: Uuid, + pub abstract_sha1: String, + pub mimetype: Option, + pub lang: Option, +} + #[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)] #[table_name = "release_contrib"] pub struct ReleaseContribRow { pub id: i64, pub release_rev: Uuid, pub creator_ident_id: Option, + pub raw: Option, pub role: Option, pub index: Option, - pub raw: Option, + pub extra_json: Option, } #[derive(Debug, Insertable)] @@ -194,9 +236,10 @@ pub struct ReleaseContribRow { pub struct ReleaseContribNewRow { pub release_rev: Uuid, pub creator_ident_id: Option, + pub raw: Option, pub role: Option, pub index: Option, - pub raw: Option, + pub extra_json: Option, } #[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)] @@ -207,7 +250,7 @@ pub struct ReleaseRefRow { pub target_release_ident_id: Option, pub index: Option, pub key: Option, - pub raw: Option, + pub extra_json: Option, pub container_title: Option, pub year: Option, pub title: Option, @@ -221,7 +264,7 @@ pub struct ReleaseRefNewRow { pub target_release_ident_id: Option, pub index: Option, pub key: Option, - pub raw: Option, + pub extra_json: Option, pub container_title: Option, pub year: Option, pub title: Option, @@ -239,8 +282,9 @@ pub struct FileReleaseRow { #[table_name = "editgroup"] pub struct EditgroupRow { pub id: Uuid, - pub extra_json: Option, pub editor_id: Uuid, + pub created: chrono::NaiveDateTime, + pub extra_json: Option, pub description: Option, } @@ -264,6 +308,7 @@ pub struct EditorRow { pub id: Uuid, pub username: String, pub is_admin: bool, + pub registered: chrono::NaiveDateTime, pub active_editgroup_id: Option, } diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs index d99fdd3f..c23e3f83 100644 --- a/rust/src/database_schema.rs +++ b/rust/src/database_schema.rs @@ -1,3 +1,10 @@ +table! { + abstracts (sha1) { + sha1 -> Bpchar, + content -> Text, + } +} + table! { changelog (id) { id -> Int8, @@ -10,9 +17,11 @@ table! { container_edit (id) { id -> Int8, editgroup_id -> Uuid, + updated -> Timestamp, ident_id -> Uuid, rev_id -> Nullable, redirect_id -> Nullable, + prev_rev -> Nullable, extra_json -> Nullable, } } @@ -32,7 +41,8 @@ table! { extra_json -> Nullable, name -> Text, publisher -> Nullable, - issnl -> Nullable, + issnl -> Nullable, + wikidata_qid -> Nullable, abbrev -> Nullable, coden -> Nullable, } @@ -42,9 +52,11 @@ table! { creator_edit (id) { id -> Int8, editgroup_id -> Uuid, + updated -> Timestamp, ident_id -> Uuid, rev_id -> Nullable, redirect_id -> Nullable, + prev_rev -> Nullable, extra_json -> Nullable, } } @@ -65,15 +77,17 @@ table! { display_name -> Text, given_name -> Nullable, surname -> Nullable, - orcid -> Nullable, + orcid -> Nullable, + wikidata_qid -> Nullable, } } table! { editgroup (id) { id -> Uuid, - extra_json -> Nullable, editor_id -> Uuid, + created -> Timestamp, + extra_json -> Nullable, description -> Nullable, } } @@ -83,6 +97,7 @@ table! { id -> Uuid, username -> Text, is_admin -> Bool, + registered -> Timestamp, active_editgroup_id -> Nullable, } } @@ -91,9 +106,11 @@ table! { file_edit (id) { id -> Int8, editgroup_id -> Uuid, + updated -> Timestamp, ident_id -> Uuid, rev_id -> Nullable, redirect_id -> Nullable, + prev_rev -> Nullable, extra_json -> Nullable, } } @@ -119,22 +136,31 @@ table! { id -> Uuid, extra_json -> Nullable, size -> Nullable, - sha1 -> Nullable, - sha256 -> Nullable, - md5 -> Nullable, - url -> Nullable, + sha1 -> Nullable, + sha256 -> Nullable, + md5 -> Nullable, mimetype -> Nullable, } } +table! { + file_rev_url (id) { + id -> Int8, + file_rev -> Uuid, + rel -> Text, + url -> Text, + } +} + table! { release_contrib (id) { id -> Int8, release_rev -> Uuid, creator_ident_id -> Nullable, + raw -> Nullable, role -> Nullable, index -> Nullable, - raw -> Nullable, + extra_json -> Nullable, } } @@ -142,9 +168,11 @@ table! { release_edit (id) { id -> Int8, editgroup_id -> Uuid, + updated -> Timestamp, ident_id -> Uuid, rev_id -> Nullable, redirect_id -> Nullable, + prev_rev -> Nullable, extra_json -> Nullable, } } @@ -165,7 +193,7 @@ table! { target_release_ident_id -> Nullable, index -> Nullable, key -> Nullable, - raw -> Nullable, + extra_json -> Nullable, container_title -> Nullable, year -> Nullable, title -> Nullable, @@ -184,6 +212,9 @@ table! { release_status -> Nullable, release_date -> Nullable, doi -> Nullable, + pmid -> Nullable, + pmcid -> Nullable, + wikidata_qid -> Nullable, isbn13 -> Nullable, volume -> Nullable, issue -> Nullable, @@ -193,13 +224,25 @@ table! { } } +table! { + release_rev_abstract (id) { + id -> Int8, + release_rev -> Uuid, + abstract_sha1 -> Bpchar, + mimetype -> Nullable, + lang -> Nullable, + } +} + table! { work_edit (id) { id -> Int8, editgroup_id -> Uuid, + updated -> Timestamp, ident_id -> Uuid, rev_id -> Nullable, redirect_id -> Nullable, + prev_rev -> Nullable, extra_json -> Nullable, } } @@ -217,38 +260,34 @@ table! { work_rev (id) { id -> Uuid, extra_json -> Nullable, - work_type -> Nullable, - primary_release_id -> Nullable, } } joinable!(changelog -> editgroup (editgroup_id)); -joinable!(container_edit -> container_rev (rev_id)); joinable!(container_edit -> editgroup (editgroup_id)); joinable!(container_ident -> container_rev (rev_id)); -joinable!(creator_edit -> creator_rev (rev_id)); joinable!(creator_edit -> editgroup (editgroup_id)); joinable!(creator_ident -> creator_rev (rev_id)); joinable!(file_edit -> editgroup (editgroup_id)); -joinable!(file_edit -> file_rev (rev_id)); joinable!(file_ident -> file_rev (rev_id)); joinable!(file_release -> file_rev (file_rev)); joinable!(file_release -> release_ident (target_release_ident_id)); +joinable!(file_rev_url -> file_rev (file_rev)); joinable!(release_contrib -> creator_ident (creator_ident_id)); joinable!(release_contrib -> release_rev (release_rev)); joinable!(release_edit -> editgroup (editgroup_id)); -joinable!(release_edit -> release_rev (rev_id)); joinable!(release_ident -> release_rev (rev_id)); joinable!(release_ref -> release_ident (target_release_ident_id)); joinable!(release_ref -> release_rev (release_rev)); joinable!(release_rev -> container_ident (container_ident_id)); joinable!(release_rev -> work_ident (work_ident_id)); +joinable!(release_rev_abstract -> abstracts (abstract_sha1)); +joinable!(release_rev_abstract -> release_rev (release_rev)); joinable!(work_edit -> editgroup (editgroup_id)); -joinable!(work_edit -> work_rev (rev_id)); joinable!(work_ident -> work_rev (rev_id)); -joinable!(work_rev -> release_ident (primary_release_id)); allow_tables_to_appear_in_same_query!( + abstracts, changelog, container_edit, container_ident, @@ -262,11 +301,13 @@ allow_tables_to_appear_in_same_query!( file_ident, file_release, file_rev, + file_rev_url, release_contrib, release_edit, release_ident, release_ref, release_rev, + release_rev_abstract, work_edit, work_ident, work_rev, -- cgit v1.2.3