From 1e2dd5ce2d2afa1016366f9296c90759c54be623 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@robocracy.org>
Date: Wed, 26 Dec 2018 15:16:03 -0800
Subject: fileset/web basic implementation
As well as small consistency and bugfix implementations.
No new tests yet...
---
rust/src/api_entity_crud.rs | 466 ++++++++++++++++++++++++++++++++++++++++++--
rust/src/api_helpers.rs | 42 +++-
rust/src/api_server.rs | 82 +++++++-
rust/src/api_wrappers.rs | 58 ++++++
rust/src/database_models.rs | 155 ++++++++++++++-
rust/src/database_schema.rs | 163 +++++++++++++++-
6 files changed, 928 insertions(+), 38 deletions(-)
(limited to 'rust/src')
diff --git a/rust/src/api_entity_crud.rs b/rust/src/api_entity_crud.rs
index b5e37412..7220f27a 100644
--- a/rust/src/api_entity_crud.rs
+++ b/rust/src/api_entity_crud.rs
@@ -1,5 +1,6 @@
use api_helpers::*;
use api_server::get_release_files;
+use chrono;
use database_models::*;
use database_schema::*;
use diesel::prelude::*;
@@ -856,13 +857,6 @@ impl EntityCrud for FileEntity {
None => (None, None, None),
};
- let releases: Vec<FatCatId> = file_release::table
- .filter(file_release::file_rev.eq(rev_row.id))
- .get_results(conn)?
- .into_iter()
- .map(|r: FileReleaseRow| FatCatId::from_uuid(&r.target_release_ident_id))
- .collect();
-
let urls: Vec<FileEntityUrls> = file_rev_url::table
.filter(file_rev_url::file_rev.eq(rev_row.id))
.get_results(conn)?
@@ -873,14 +867,21 @@ impl EntityCrud for FileEntity {
})
.collect();
+ let release_ids: Vec<FatCatId> = file_rev_release::table
+ .filter(file_rev_release::file_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: FileRevReleaseRow| FatCatId::from_uuid(&r.target_release_ident_id))
+ .collect();
+
Ok(FileEntity {
sha1: rev_row.sha1,
sha256: rev_row.sha256,
md5: rev_row.md5,
- size: rev_row.size.map(|v| v as i64),
+ size: rev_row.size_bytes.map(|v| v as i64),
urls: Some(urls),
mimetype: rev_row.mimetype,
- release_ids: Some(releases.iter().map(|fcid| fcid.to_string()).collect()),
+ release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
state: state,
ident: ident_id,
revision: Some(rev_row.id.to_string()),
@@ -909,7 +910,7 @@ impl EntityCrud for FileEntity {
models
.iter()
.map(|model| FileRevNewRow {
- size: model.size,
+ size_bytes: model.size,
sha1: model.sha1.clone(),
sha256: model.sha256.clone(),
md5: model.md5.clone(),
@@ -921,23 +922,23 @@ impl EntityCrud for FileEntity {
.returning(file_rev::id)
.get_results(conn)?;
- let mut file_release_rows: Vec<FileReleaseRow> = vec![];
+ let mut file_rev_release_rows: Vec<FileRevReleaseRow> = vec![];
let mut file_url_rows: Vec<FileRevUrlNewRow> = vec![];
for (model, rev_id) in models.iter().zip(rev_ids.iter()) {
match &model.release_ids {
None => (),
Some(release_list) => {
- let these_release_rows: Result<Vec<FileReleaseRow>> = release_list
+ let these_release_rows: Result<Vec<FileRevReleaseRow>> = release_list
.iter()
.map(|r| {
- Ok(FileReleaseRow {
+ Ok(FileRevReleaseRow {
file_rev: *rev_id,
target_release_ident_id: FatCatId::from_str(r)?.to_uuid(),
})
})
.collect();
- file_release_rows.extend(these_release_rows?);
+ file_rev_release_rows.extend(these_release_rows?);
}
};
@@ -957,10 +958,9 @@ impl EntityCrud for FileEntity {
};
}
- if !file_release_rows.is_empty() {
- // TODO: shouldn't it be "file_rev_release"?
- insert_into(file_release::table)
- .values(file_release_rows)
+ if !file_rev_release_rows.is_empty() {
+ insert_into(file_rev_release::table)
+ .values(file_rev_release_rows)
.execute(conn)?;
}
@@ -974,6 +974,432 @@ impl EntityCrud for FileEntity {
}
}
+impl EntityCrud for FilesetEntity {
+ type EditRow = FilesetEditRow;
+ type EditNewRow = FilesetEditNewRow;
+ type IdentRow = FilesetIdentRow;
+ type IdentNewRow = FilesetIdentNewRow;
+ type RevRow = FilesetRevRow;
+
+ generic_db_get!(fileset_ident, fileset_rev);
+ generic_db_get_rev!(fileset_rev);
+ generic_db_expand!();
+ generic_db_create!(fileset_ident, fileset_edit);
+ generic_db_create_batch!(fileset_ident, fileset_edit);
+ generic_db_update!(fileset_ident, fileset_edit);
+ generic_db_delete!(fileset_ident, fileset_edit);
+ generic_db_get_history!(fileset_edit);
+ generic_db_get_edit!(fileset_edit);
+ generic_db_delete_edit!(fileset_edit);
+ generic_db_get_redirects!(fileset_ident);
+ generic_db_accept_edits_batch!("file", fileset_ident, fileset_edit);
+ generic_db_insert_rev!();
+
+ fn from_deleted_row(ident_row: Self::IdentRow) -> Result<Self> {
+ if ident_row.rev_id.is_some() {
+ bail!("called from_deleted_row with a non-deleted-state row")
+ }
+
+ Ok(FilesetEntity {
+ manifest: None,
+ urls: None,
+ release_ids: None,
+ state: Some(ident_row.state().unwrap().shortname()),
+ ident: Some(FatCatId::from_uuid(&ident_row.id).to_string()),
+ revision: ident_row.rev_id.map(|u| u.to_string()),
+ redirect: ident_row
+ .redirect_id
+ .map(|u| FatCatId::from_uuid(&u).to_string()),
+ extra: None,
+ edit_extra: None,
+ })
+ }
+
+ fn db_from_row(
+ conn: &DbConn,
+ rev_row: Self::RevRow,
+ ident_row: Option<Self::IdentRow>,
+ _hide: HideFlags,
+ ) -> Result<Self> {
+ let (state, ident_id, redirect_id) = match ident_row {
+ Some(i) => (
+ Some(i.state().unwrap().shortname()),
+ Some(FatCatId::from_uuid(&i.id).to_string()),
+ i.redirect_id.map(|u| FatCatId::from_uuid(&u).to_string()),
+ ),
+ None => (None, None, None),
+ };
+
+ let manifest: Vec<FilesetEntityManifest> = fileset_rev_file::table
+ .filter(fileset_rev_file::fileset_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: FilesetRevFileRow| FilesetEntityManifest {
+ path: r.path_name,
+ size: r.size_bytes,
+ md5: r.md5,
+ sha1: r.sha1,
+ sha256: r.sha256,
+ extra: r.extra_json,
+ })
+ .collect();
+
+ let urls: Vec<FileEntityUrls> = fileset_rev_url::table
+ .filter(fileset_rev_url::fileset_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: FilesetRevUrlRow| FileEntityUrls {
+ rel: r.rel,
+ url: r.url,
+ })
+ .collect();
+
+ let release_ids: Vec<FatCatId> = fileset_rev_release::table
+ .filter(fileset_rev_release::fileset_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: FilesetRevReleaseRow| FatCatId::from_uuid(&r.target_release_ident_id))
+ .collect();
+
+ Ok(FilesetEntity {
+ manifest: Some(manifest),
+ urls: Some(urls),
+ release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
+ state: state,
+ ident: ident_id,
+ revision: Some(rev_row.id.to_string()),
+ redirect: redirect_id,
+ extra: rev_row.extra_json,
+ edit_extra: None,
+ })
+ }
+
+ fn db_insert_revs(conn: &DbConn, models: &[&Self]) -> Result<Vec<Uuid>> {
+ // first verify hash syntax
+ for entity in models {
+ if let Some(ref manifest) = entity.manifest {
+ for file in manifest {
+ if let Some(ref hash) = file.md5 {
+ check_md5(hash)?;
+ }
+ if let Some(ref hash) = file.sha1 {
+ check_sha1(hash)?;
+ }
+ if let Some(ref hash) = file.sha256 {
+ check_sha256(hash)?;
+ }
+ }
+ }
+ }
+
+ let rev_ids: Vec<Uuid> = insert_into(fileset_rev::table)
+ .values(
+ models
+ .iter()
+ .map(|model| FilesetRevNewRow {
+ extra_json: model.extra.clone(),
+ })
+ .collect::<Vec<FilesetRevNewRow>>(),
+ )
+ .returning(fileset_rev::id)
+ .get_results(conn)?;
+
+ let mut fileset_file_rows: Vec<FilesetRevFileNewRow> = vec![];
+ let mut fileset_url_rows: Vec<FilesetRevUrlNewRow> = vec![];
+ let mut fileset_release_rows: Vec<FilesetRevReleaseRow> = vec![];
+
+ for (model, rev_id) in models.iter().zip(rev_ids.iter()) {
+ match &model.manifest {
+ None => (),
+ Some(file_list) => {
+ let these_file_rows: Vec<FilesetRevFileNewRow> = file_list
+ .into_iter()
+ .map(|f| FilesetRevFileNewRow {
+ fileset_rev: *rev_id,
+ path_name: f.path.clone(),
+ size_bytes: f.size,
+ md5: f.md5.clone(),
+ sha1: f.sha1.clone(),
+ sha256: f.sha256.clone(),
+ extra_json: f.extra.clone(),
+ })
+ .collect();
+ fileset_file_rows.extend(these_file_rows);
+ }
+ };
+
+ match &model.urls {
+ None => (),
+ Some(url_list) => {
+ let these_url_rows: Vec<FilesetRevUrlNewRow> = url_list
+ .into_iter()
+ .map(|u| FilesetRevUrlNewRow {
+ fileset_rev: *rev_id,
+ rel: u.rel.clone(),
+ url: u.url.clone(),
+ })
+ .collect();
+ fileset_url_rows.extend(these_url_rows);
+ }
+ };
+
+ match &model.release_ids {
+ None => (),
+ Some(release_list) => {
+ let these_release_rows: Result<Vec<FilesetRevReleaseRow>> = release_list
+ .iter()
+ .map(|r| {
+ Ok(FilesetRevReleaseRow {
+ fileset_rev: *rev_id,
+ target_release_ident_id: FatCatId::from_str(r)?.to_uuid(),
+ })
+ })
+ .collect();
+ fileset_release_rows.extend(these_release_rows?);
+ }
+ };
+ }
+
+ if !fileset_file_rows.is_empty() {
+ insert_into(fileset_rev_file::table)
+ .values(fileset_file_rows)
+ .execute(conn)?;
+ }
+
+ if !fileset_url_rows.is_empty() {
+ insert_into(fileset_rev_url::table)
+ .values(fileset_url_rows)
+ .execute(conn)?;
+ }
+
+ if !fileset_release_rows.is_empty() {
+ insert_into(fileset_rev_release::table)
+ .values(fileset_release_rows)
+ .execute(conn)?;
+ }
+
+ Ok(rev_ids)
+ }
+}
+
+impl EntityCrud for WebcaptureEntity {
+ type EditRow = WebcaptureEditRow;
+ type EditNewRow = WebcaptureEditNewRow;
+ type IdentRow = WebcaptureIdentRow;
+ type IdentNewRow = WebcaptureIdentNewRow;
+ type RevRow = WebcaptureRevRow;
+
+ generic_db_get!(webcapture_ident, webcapture_rev);
+ generic_db_get_rev!(webcapture_rev);
+ generic_db_expand!();
+ generic_db_create!(webcapture_ident, webcapture_edit);
+ generic_db_create_batch!(webcapture_ident, webcapture_edit);
+ generic_db_update!(webcapture_ident, webcapture_edit);
+ generic_db_delete!(webcapture_ident, webcapture_edit);
+ generic_db_get_history!(webcapture_edit);
+ generic_db_get_edit!(webcapture_edit);
+ generic_db_delete_edit!(webcapture_edit);
+ generic_db_get_redirects!(webcapture_ident);
+ generic_db_accept_edits_batch!("file", webcapture_ident, webcapture_edit);
+ generic_db_insert_rev!();
+
+ fn from_deleted_row(ident_row: Self::IdentRow) -> Result<Self> {
+ if ident_row.rev_id.is_some() {
+ bail!("called from_deleted_row with a non-deleted-state row")
+ }
+
+ Ok(WebcaptureEntity {
+ cdx: None,
+ archive_urls: None,
+ original_url: None,
+ timestamp: None,
+ release_ids: None,
+ state: Some(ident_row.state().unwrap().shortname()),
+ ident: Some(FatCatId::from_uuid(&ident_row.id).to_string()),
+ revision: ident_row.rev_id.map(|u| u.to_string()),
+ redirect: ident_row
+ .redirect_id
+ .map(|u| FatCatId::from_uuid(&u).to_string()),
+ extra: None,
+ edit_extra: None,
+ })
+ }
+
+ fn db_from_row(
+ conn: &DbConn,
+ rev_row: Self::RevRow,
+ ident_row: Option<Self::IdentRow>,
+ _hide: HideFlags,
+ ) -> Result<Self> {
+ let (state, ident_id, redirect_id) = match ident_row {
+ Some(i) => (
+ Some(i.state().unwrap().shortname()),
+ Some(FatCatId::from_uuid(&i.id).to_string()),
+ i.redirect_id.map(|u| FatCatId::from_uuid(&u).to_string()),
+ ),
+ None => (None, None, None),
+ };
+
+ let cdx: Vec<WebcaptureEntityCdx> = webcapture_rev_cdx::table
+ .filter(webcapture_rev_cdx::webcapture_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|c: WebcaptureRevCdxRow| WebcaptureEntityCdx {
+ surt: c.surt,
+ timestamp: c.timestamp,
+ url: c.url,
+ mimetype: c.mimetype,
+ status_code: c.status_code,
+ sha1: c.sha1,
+ sha256: c.sha256,
+ })
+ .collect();
+
+ let archive_urls: Vec<WebcaptureEntityArchiveUrls> = webcapture_rev_url::table
+ .filter(webcapture_rev_url::webcapture_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: WebcaptureRevUrlRow| WebcaptureEntityArchiveUrls {
+ rel: r.rel,
+ url: r.url,
+ })
+ .collect();
+
+ let release_ids: Vec<FatCatId> = webcapture_rev_release::table
+ .filter(webcapture_rev_release::webcapture_rev.eq(rev_row.id))
+ .get_results(conn)?
+ .into_iter()
+ .map(|r: WebcaptureRevReleaseRow| FatCatId::from_uuid(&r.target_release_ident_id))
+ .collect();
+
+ Ok(WebcaptureEntity {
+ cdx: Some(cdx),
+ archive_urls: Some(archive_urls),
+ original_url: Some(rev_row.original_url),
+ timestamp: Some(chrono::DateTime::from_utc(rev_row.timestamp, chrono::Utc)),
+ release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
+ state: state,
+ ident: ident_id,
+ revision: Some(rev_row.id.to_string()),
+ redirect: redirect_id,
+ extra: rev_row.extra_json,
+ edit_extra: None,
+ })
+ }
+
+ fn db_insert_revs(conn: &DbConn, models: &[&Self]) -> Result<Vec<Uuid>> {
+ // first verify hash syntax, and presence of required fields
+ for entity in models {
+ if let Some(ref cdx) = entity.cdx {
+ for row in cdx {
+ check_sha1(&row.sha1)?;
+ if let Some(ref hash) = row.sha256 {
+ check_sha256(hash)?;
+ }
+ }
+ }
+ if entity.timestamp.is_none() || entity.original_url.is_none() {
+ return Err(ErrorKind::OtherBadRequest(
+ "timestamp and original_url are required for webcapture entities".to_string(),
+ )
+ .into());
+ }
+ }
+
+ let rev_ids: Vec<Uuid> = insert_into(webcapture_rev::table)
+ .values(
+ models
+ .iter()
+ .map(|model| WebcaptureRevNewRow {
+ // these unwraps safe because of check above
+ original_url: model.original_url.clone().unwrap(),
+ timestamp: model.timestamp.unwrap().naive_utc(),
+ extra_json: model.extra.clone(),
+ })
+ .collect::<Vec<WebcaptureRevNewRow>>(),
+ )
+ .returning(webcapture_rev::id)
+ .get_results(conn)?;
+
+ let mut webcapture_cdx_rows: Vec<WebcaptureRevCdxNewRow> = vec![];
+ let mut webcapture_url_rows: Vec<WebcaptureRevUrlNewRow> = vec![];
+ let mut webcapture_release_rows: Vec<WebcaptureRevReleaseRow> = vec![];
+
+ for (model, rev_id) in models.iter().zip(rev_ids.iter()) {
+ match &model.cdx {
+ None => (),
+ Some(cdx_list) => {
+ let these_cdx_rows: Vec<WebcaptureRevCdxNewRow> = cdx_list
+ .into_iter()
+ .map(|c| WebcaptureRevCdxNewRow {
+ webcapture_rev: *rev_id,
+ surt: c.surt.clone(),
+ timestamp: c.timestamp,
+ url: c.url.clone(),
+ mimetype: c.mimetype.clone(),
+ status_code: c.status_code,
+ sha1: c.sha1.clone(),
+ sha256: c.sha256.clone(),
+ })
+ .collect();
+ webcapture_cdx_rows.extend(these_cdx_rows);
+ }
+ };
+
+ match &model.archive_urls {
+ None => (),
+ Some(url_list) => {
+ let these_url_rows: Vec<WebcaptureRevUrlNewRow> = url_list
+ .into_iter()
+ .map(|u| WebcaptureRevUrlNewRow {
+ webcapture_rev: *rev_id,
+ rel: u.rel.clone(),
+ url: u.url.clone(),
+ })
+ .collect();
+ webcapture_url_rows.extend(these_url_rows);
+ }
+ };
+
+ match &model.release_ids {
+ None => (),
+ Some(release_list) => {
+ let these_release_rows: Result<Vec<WebcaptureRevReleaseRow>> = release_list
+ .iter()
+ .map(|r| {
+ Ok(WebcaptureRevReleaseRow {
+ webcapture_rev: *rev_id,
+ target_release_ident_id: FatCatId::from_str(r)?.to_uuid(),
+ })
+ })
+ .collect();
+ webcapture_release_rows.extend(these_release_rows?);
+ }
+ };
+ }
+
+ if !webcapture_cdx_rows.is_empty() {
+ insert_into(webcapture_rev_cdx::table)
+ .values(webcapture_cdx_rows)
+ .execute(conn)?;
+ }
+
+ if !webcapture_url_rows.is_empty() {
+ insert_into(webcapture_rev_url::table)
+ .values(webcapture_url_rows)
+ .execute(conn)?;
+ }
+
+ if !webcapture_release_rows.is_empty() {
+ insert_into(webcapture_rev_release::table)
+ .values(webcapture_release_rows)
+ .execute(conn)?;
+ }
+
+ Ok(rev_ids)
+ }
+}
+
impl EntityCrud for ReleaseEntity {
type EditRow = ReleaseEditRow;
type EditNewRow = ReleaseEditNewRow;
@@ -1013,6 +1439,8 @@ impl EntityCrud for ReleaseEntity {
issue: None,
pages: None,
files: None,
+ filesets: None,
+ webcaptures: None,
container: None,
container_id: None,
publisher: None,
@@ -1275,6 +1703,8 @@ impl EntityCrud for ReleaseEntity {
issue: rev_row.issue,
pages: rev_row.pages,
files: None,
+ filesets: None,
+ webcaptures: None,
container: None,
container_id: rev_row
.container_ident_id
diff --git a/rust/src/api_helpers.rs b/rust/src/api_helpers.rs
index b837dfc2..ff164bef 100644
--- a/rust/src/api_helpers.rs
+++ b/rust/src/api_helpers.rs
@@ -38,6 +38,8 @@ impl EditContext {
#[derive(Clone, Copy, PartialEq)]
pub struct ExpandFlags {
pub files: bool,
+ pub filesets: bool,
+ pub webcaptures: bool,
pub container: bool,
pub releases: bool,
pub creators: bool,
@@ -55,6 +57,8 @@ impl ExpandFlags {
pub fn from_str_list(list: &[&str]) -> ExpandFlags {
ExpandFlags {
files: list.contains(&"files"),
+ filesets: list.contains(&"filesets"),
+ webcaptures: list.contains(&"webcaptures"),
container: list.contains(&"container"),
releases: list.contains(&"releases"),
creators: list.contains(&"creators"),
@@ -63,6 +67,8 @@ impl ExpandFlags {
pub fn none() -> ExpandFlags {
ExpandFlags {
files: false,
+ filesets: false,
+ webcaptures: false,
container: false,
releases: false,
creators: false,
@@ -77,6 +83,8 @@ fn test_expand_flags() {
assert!(ExpandFlags::from_str_list(&vec!["file"]).files == false);
let all = ExpandFlags::from_str_list(&vec![
"files",
+ "filesets",
+ "webcaptures",
"container",
"other_thing",
"releases",
@@ -85,6 +93,8 @@ fn test_expand_flags() {
assert!(
all == ExpandFlags {
files: true,
+ filesets: true,
+ webcaptures: true,
container: true,
releases: true,
creators: true
@@ -94,10 +104,14 @@ fn test_expand_flags() {
assert!(ExpandFlags::from_str("files").unwrap().files == true);
assert!(ExpandFlags::from_str("something,,files").unwrap().files == true);
assert!(ExpandFlags::from_str("file").unwrap().files == false);
- let all = ExpandFlags::from_str("files,container,other_thing,releases,creators").unwrap();
+ let all =
+ ExpandFlags::from_str("files,container,other_thing,releases,creators,filesets,webcaptures")
+ .unwrap();
assert!(
all == ExpandFlags {
files: true,
+ filesets: true,
+ webcaptures: true,
container: true,
releases: true,
creators: true
@@ -107,9 +121,14 @@ fn test_expand_flags() {
#[derive(Clone, Copy, PartialEq)]
pub struct HideFlags {
+ // release
pub abstracts: bool,
pub refs: bool,
pub contribs: bool,
+ // fileset
+ pub manifest: bool,
+ // webcapture
+ pub cdx: bool,
}
impl FromStr for HideFlags {
@@ -126,6 +145,8 @@ impl HideFlags {
abstracts: list.contains(&"abstracts"),
refs: list.contains(&"refs"),
contribs: list.contains(&"contribs"),
+ manifest: list.contains(&"contribs"),
+ cdx: list.contains(&"contribs"),
}
}
pub fn none() -> HideFlags {
@@ -133,6 +154,8 @@ impl HideFlags {
abstracts: false,
refs: false,
contribs: false,
+ manifest: false,
+ cdx: false,
}
}
}
@@ -142,12 +165,21 @@ fn test_hide_flags() {
assert!(HideFlags::from_str_list(&vec![]).abstracts == false);
assert!(HideFlags::from_str_list(&vec!["abstracts"]).abstracts == true);
assert!(HideFlags::from_str_list(&vec!["abstract"]).abstracts == false);
- let all = HideFlags::from_str_list(&vec!["abstracts", "refs", "other_thing", "contribs"]);
+ let all = HideFlags::from_str_list(&vec![
+ "abstracts",
+ "refs",
+ "other_thing",
+ "contribs",
+ "manifest",
+ "cdx",
+ ]);
assert!(
all == HideFlags {
abstracts: true,
refs: true,
contribs: true,
+ manifest: true,
+ cdx: true,
}
);
assert!(HideFlags::from_str("").unwrap().abstracts == false);
@@ -159,12 +191,14 @@ fn test_hide_flags() {
== true
);
assert!(HideFlags::from_str("file").unwrap().abstracts == false);
- let all = HideFlags::from_str("abstracts,refs,other_thing,contribs").unwrap();
+ let all = HideFlags::from_str("abstracts,cdx,refs,manifest,other_thing,contribs").unwrap();
assert!(
all == HideFlags {
abstracts: true,
refs: true,
contribs: true,
+ manifest: true,
+ cdx: true,
}
);
}
@@ -229,6 +263,8 @@ pub fn accept_editgroup(editgroup_id: FatCatId, conn: &DbConn) -> Result<Changel
ContainerEntity::db_accept_edits(conn, editgroup_id)?;
CreatorEntity::db_accept_edits(conn, editgroup_id)?;
FileEntity::db_accept_edits(conn, editgroup_id)?;
+ FilesetEntity::db_accept_edits(conn, editgroup_id)?;
+ WebcaptureEntity::db_accept_edits(conn, editgroup_id)?;
ReleaseEntity::db_accept_edits(conn, editgroup_id)?;
WorkEntity::db_accept_edits(conn, editgroup_id)?;
diff --git a/rust/src/api_server.rs b/rust/src/api_server.rs
index 5b95f149..d264afbc 100644
--- a/rust/src/api_server.rs
+++ b/rust/src/api_server.rs
@@ -59,10 +59,10 @@ pub fn get_release_files(
hide_flags: HideFlags,
conn: &DbConn,
) -> Result<Vec<FileEntity>> {
- let rows: Vec<(FileRevRow, FileIdentRow, FileReleaseRow)> = file_rev::table
+ let rows: Vec<(FileRevRow, FileIdentRow, FileRevReleaseRow)> = file_rev::table
.inner_join(file_ident::table)
- .inner_join(file_release::table)
- .filter(file_release::target_release_ident_id.eq(&ident.to_uuid()))
+ .inner_join(file_rev_release::table)
+ .filter(file_rev_release::target_release_ident_id.eq(&ident.to_uuid()))
.filter(file_ident::is_live.eq(true))
.filter(file_ident::redirect_id.is_null())
.load(conn)?;
@@ -72,6 +72,46 @@ pub fn get_release_files(
.collect()
}
+pub fn get_release_filesets(
+ ident: FatCatId,
+ hide_flags: HideFlags,
+ conn: &DbConn,
+) -> Result<Vec<FilesetEntity>> {
+ let rows: Vec<(FilesetRevRow, FilesetIdentRow, FilesetRevReleaseRow)> = fileset_rev::table
+ .inner_join(fileset_ident::table)
+ .inner_join(fileset_rev_release::table)
+ .filter(fileset_rev_release::target_release_ident_id.eq(&ident.to_uuid()))
+ .filter(fileset_ident::is_live.eq(true))
+ .filter(fileset_ident::redirect_id.is_null())
+ .load(conn)?;
+
+ rows.into_iter()
+ .map(|(rev, ident, _)| FilesetEntity::db_from_row(conn, rev, Some(ident), hide_flags))
+ .collect()
+}
+
+pub fn get_release_webcaptures(
+ ident: FatCatId,
+ hide_flags: HideFlags,
+ conn: &DbConn,
+) -> Result<Vec<WebcaptureEntity>> {
+ let rows: Vec<(
+ WebcaptureRevRow,
+ WebcaptureIdentRow,
+ WebcaptureRevReleaseRow,
+ )> = webcapture_rev::table
+ .inner_join(webcapture_ident::table)
+ .inner_join(webcapture_rev_release::table)
+ .filter(webcapture_rev_release::target_release_ident_id.eq(&ident.to_uuid()))
+ .filter(webcapture_ident::is_live.eq(true))
+ .filter(webcapture_ident::redirect_id.is_null())
+ .load(conn)?;
+
+ rows.into_iter()
+ .map(|(rev, ident, _)| WebcaptureEntity::db_from_row(conn, rev, Some(ident), hide_flags))
+ .collect()
+}
+
impl Server {
pub fn lookup_container_handler(
&self,
@@ -304,6 +344,24 @@ impl Server {
get_release_files(ident, hide_flags, conn)
}
+ pub fn get_release_filesets_handler(
+ &self,
+ ident: FatCatId,
+ hide_flags: HideFlags,
+ conn: &DbConn,
+ ) -> Result<Vec<FilesetEntity>> {
+ get_release_filesets(ident, hide_flags, conn)
+ }
+
+ pub fn get_release_webcaptures_handler(
+ &self,
+ ident: FatCatId,
+ hide_flags: HideFlags,
+ conn: &DbConn,
+ ) -> Result<Vec<WebcaptureEntity>> {
+ get_release_webcaptures(ident, hide_flags, conn)
+ }
+
pub fn get_work_releases_handler(
&self,
ident: FatCatId,
@@ -381,6 +439,22 @@ impl Server {
.map(|e: FileEditRow| e.into_model().unwrap())
.collect(),
),
+ filesets: Some(
+ fileset_edit::table
+ .filter(fileset_edit::editgroup_id.eq(editgroup_id.to_uuid()))
+ .get_results(conn)?
+ .into_iter()
+ .map(|e: FilesetEditRow| e.into_model().unwrap())
+ .collect(),
+ ),
+ webcaptures: Some(
+ webcapture_edit::table
+ .filter(webcapture_edit::editgroup_id.eq(editgroup_id.to_uuid()))
+ .get_results(conn)?
+ .into_iter()
+ .map(|e: WebcaptureEditRow| e.into_model().unwrap())
+ .collect(),
+ ),
releases: Some(
release_edit::table
.filter(release_edit::editgroup_id.eq(editgroup_id.to_uuid()))
@@ -481,6 +555,8 @@ impl Server {
entity_batch_handler!(create_container_batch_handler, ContainerEntity);
entity_batch_handler!(create_creator_batch_handler, CreatorEntity);
entity_batch_handler!(create_file_batch_handler, FileEntity);
+ entity_batch_handler!(create_fileset_batch_handler, FilesetEntity);
+ entity_batch_handler!(create_webcapture_batch_handler, WebcaptureEntity);
entity_batch_handler!(create_release_batch_handler, ReleaseEntity);
entity_batch_handler!(create_work_batch_handler, WorkEntity);
}
diff --git a/rust/src/api_wrappers.rs b/rust/src/api_wrappers.rs
index 85b698aa..aa168076 100644
--- a/rust/src/api_wrappers.rs
+++ b/rust/src/api_wrappers.rs
@@ -598,6 +598,54 @@ impl Api for Server {
GetFileRedirectsResponse,
FileEntity
);
+ wrap_entity_handlers!(
+ get_fileset,
+ GetFilesetResponse,
+ create_fileset,
+ CreateFilesetResponse,
+ create_fileset_batch,
+ create_fileset_batch_handler,
+ CreateFilesetBatchResponse,
+ update_fileset,
+ UpdateFilesetResponse,
+ delete_fileset,
+ DeleteFilesetResponse,
+ get_fileset_history,
+ GetFilesetHistoryResponse,
+ get_fileset_edit,
+ GetFilesetEditResponse,
+ delete_fileset_edit,
+ DeleteFilesetEditResponse,
+ get_fileset_revision,
+ GetFilesetRevisionResponse,
+ get_fileset_redirects,
+ GetFilesetRedirectsResponse,
+ FilesetEntity
+ );
+ wrap_entity_handlers!(
+ get_webcapture,
+ GetWebcaptureResponse,
+ create_webcapture,
+ CreateWebcaptureResponse,
+ create_webcapture_batch,
+ create_webcapture_batch_handler,
+ CreateWebcaptureBatchResponse,
+ update_webcapture,
+ UpdateWebcaptureResponse,
+ delete_webcapture,
+ DeleteWebcaptureResponse,
+ get_webcapture_history,
+ GetWebcaptureHistoryResponse,
+ get_webcapture_edit,
+ GetWebcaptureEditResponse,
+ delete_webcapture_edit,
+ DeleteWebcaptureEditResponse,
+ get_webcapture_revision,
+ GetWebcaptureRevisionResponse,
+ get_webcapture_redirects,
+ GetWebcaptureRedirectsResponse,
+ WebcaptureEntity
+ );
wrap_entity_handlers!(
get_release,
GetReleaseResponse,
@@ -665,6 +713,16 @@ impl Api for Server {
get_release_files_handler,
GetReleaseFilesResponse
);
+ wrap_fcid_hide_handler!(
+ get_release_filesets,
+ get_release_filesets_handler,
+ GetReleaseFilesetsResponse
+ );
+ wrap_fcid_hide_handler!(
+ get_release_webcaptures,
+ get_release_webcaptures_handler,
+ GetReleaseWebcapturesResponse
+ );
wrap_fcid_hide_handler!(
get_work_releases,
get_work_releases_handler,
diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs
index 2431e0fe..c913b98e 100644
--- a/rust/src/database_models.rs
+++ b/rust/src/database_models.rs
@@ -209,7 +209,7 @@ pub struct FileRevUrlNewRow {
pub struct FileRevRow {
pub id: Uuid,
pub extra_json: Option<serde_json::Value>,
- pub size: Option<i64>,
+ pub size_bytes: Option<i64>,
pub sha1: Option<String>,
pub sha256: Option<String>,
pub md5: Option<String>,
@@ -220,7 +220,7 @@ pub struct FileRevRow {
#[table_name = "file_rev"]
pub struct FileRevNewRow {
pub extra_json: Option<serde_json::Value>,
- pub size: Option<i64>,
+ pub size_bytes: Option<i64>,
pub sha1: Option<String>,
pub sha256: Option<String>,
pub md5: Option<String>,
@@ -236,6 +236,139 @@ entity_structs!(
FileIdentNewRow
);
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "fileset_rev_file"]
+pub struct FilesetRevFileRow {
+ pub id: i64,
+ pub fileset_rev: Uuid,
+ pub path_name: String,
+ pub size_bytes: i64,
+ pub md5: Option<String>,
+ pub sha1: Option<String>,
+ pub sha256: Option<String>,
+ pub extra_json: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)]
+#[table_name = "fileset_rev_file"]
+pub struct FilesetRevFileNewRow {
+ pub fileset_rev: Uuid,
+ pub path_name: String,
+ pub size_bytes: i64,
+ pub md5: Option<String>,
+ pub sha1: Option<String>,
+ pub sha256: Option<String>,
+ pub extra_json: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "fileset_rev_url"]
+pub struct FilesetRevUrlRow {
+ pub id: i64,
+ pub fileset_rev: Uuid,
+ pub rel: String,
+ pub url: String,
+}
+
+#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)]
+#[table_name = "fileset_rev_url"]
+pub struct FilesetRevUrlNewRow {
+ pub fileset_rev: Uuid,
+ pub rel: String,
+ pub url: String,
+}
+
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "fileset_rev"]
+pub struct FilesetRevRow {
+ pub id: Uuid,
+ pub extra_json: Option<serde_json::Value>,
+}
+
+#[derive(Debug, Associations, AsChangeset, Insertable)]
+#[table_name = "fileset_rev"]
+pub struct FilesetRevNewRow {
+ pub extra_json: Option<serde_json::Value>,
+}
+
+entity_structs!(
+ "fileset_edit",
+ FilesetEditRow,
+ FilesetEditNewRow,
+ "fileset_ident",
+ FilesetIdentRow,
+ FilesetIdentNewRow
+);
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "webcapture_rev_cdx"]
+pub struct WebcaptureRevCdxRow {
+ pub id: i64,
+ pub webcapture_rev: Uuid,
+ pub surt: String,
+ pub timestamp: i64,
+ pub url: String,
+ pub mimetype: Option<String>,
+ pub status_code: i64,
+ pub sha1: String,
+ pub sha256: Option<String>,
+}
+
+#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)]
+#[table_name = "webcapture_rev_cdx"]
+pub struct WebcaptureRevCdxNewRow {
+ pub webcapture_rev: Uuid,
+ pub surt: String,
+ pub timestamp: i64,
+ pub url: String,
+ pub mimetype: Option<String>,
+ pub status_code: i64,
+ pub sha1: String,
+ pub sha256: Option<String>,
+}
+
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "webcapture_rev_url"]
+pub struct WebcaptureRevUrlRow {
+ pub id: i64,
+ pub webcapture_rev: Uuid,
+ pub rel: String,
+ pub url: String,
+}
+
+#[derive(Debug, Queryable, Associations, AsChangeset, Insertable)]
+#[table_name = "webcapture_rev_url"]
+pub struct WebcaptureRevUrlNewRow {
+ pub webcapture_rev: Uuid,
+ pub rel: String,
+ pub url: String,
+}
+
+#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
+#[table_name = "webcapture_rev"]
+pub struct WebcaptureRevRow {
+ pub id: Uuid,
+ pub extra_json: Option<serde_json::Value>,
+ pub original_url: String,
+ pub timestamp: chrono::NaiveDateTime,
+}
+
+#[derive(Debug, Associations, AsChangeset, Insertable)]
+#[table_name = "webcapture_rev"]
+pub struct WebcaptureRevNewRow {
+ pub extra_json: Option<serde_json::Value>,
+ pub original_url: String,
+ pub timestamp: chrono::NaiveDateTime,
+}
+
+entity_structs!(
+ "webcapture_edit",
+ WebcaptureEditRow,
+ WebcaptureEditNewRow,
+ "webcapture_ident",
+ WebcaptureIdentRow,
+ WebcaptureIdentNewRow
+);
+
#[derive(Debug, Queryable, Identifiable, Associations, AsChangeset)]
#[table_name = "release_rev"]
pub struct ReleaseRevRow {
@@ -388,12 +521,26 @@ pub struct ReleaseRefNewRow {
}
#[derive(Debug, Queryable, Insertable, Associations, AsChangeset)]
-#[table_name = "file_release"]
-pub struct FileReleaseRow {
+#[table_name = "file_rev_release"]
+pub struct FileRevReleaseRow {
pub file_rev: Uuid,
pub target_release_ident_id: Uuid,
}
+#[derive(Debug, Queryable, Insertable, Associations, AsChangeset)]
+#[table_name = "fileset_rev_release"]
+pub struct FilesetRevReleaseRow {
+ pub fileset_rev: Uuid,
+ pub target_release_ident_id: Uuid,
+}
+
+#[derive(Debug, Queryable, Insertable, Associations, AsChangeset)]
+#[table_name = "webcapture_rev_release"]
+pub struct WebcaptureRevReleaseRow {
+ pub webcapture_rev: Uuid,
+ pub target_release_ident_id: Uuid,
+}
+
#[derive(Debug, Queryable, Insertable, Associations, AsChangeset)]
#[table_name = "abstracts"]
pub struct AbstractsRow {
diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs
index 6c1fb929..436f2989 100644
--- a/rust/src/database_schema.rs
+++ b/rust/src/database_schema.rs
@@ -125,28 +125,86 @@ table! {
}
table! {
- file_release (file_rev, target_release_ident_id) {
+ file_rev (id) {
+ id -> Uuid,
+ extra_json -> Nullable<Jsonb>,
+ size_bytes -> Nullable<Int8>,
+ sha1 -> Nullable<Text>,
+ sha256 -> Nullable<Text>,
+ md5 -> Nullable<Text>,
+ mimetype -> Nullable<Text>,
+ }
+}
+
+table! {
+ file_rev_release (file_rev, target_release_ident_id) {
file_rev -> Uuid,
target_release_ident_id -> Uuid,
}
}
table! {
- file_rev (id) {
+ file_rev_url (id) {
+ id -> Int8,
+ file_rev -> Uuid,
+ rel -> Text,
+ url -> Text,
+ }
+}
+
+table! {
+ fileset_edit (id) {
+ id -> Int8,
+ editgroup_id -> Uuid,
+ updated -> Timestamptz,
+ ident_id -> Uuid,
+ rev_id -> Nullable<Uuid>,
+ redirect_id -> Nullable<Uuid>,
+ prev_rev -> Nullable<Uuid>,
+ extra_json -> Nullable<Jsonb>,
+ }
+}
+
+table! {
+ fileset_ident (id) {
+ id -> Uuid,
+ is_live -> Bool,
+ rev_id -> Nullable<Uuid>,
+ redirect_id -> Nullable<Uuid>,
+ }
+}
+
+table! {
+ fileset_rev (id) {
id -> Uuid,
extra_json -> Nullable<Jsonb>,
- size -> Nullable<Int8>,
+ }
+}
+
+table! {
+ fileset_rev_file (id) {
+ id -> Int8,
+ fileset_rev -> Uuid,
+ path_name -> Text,
+ size_bytes -> Int8,
+ md5 -> Nullable<Text>,
sha1 -> Nullable<Text>,
sha256 -> Nullable<Text>,
- md5 -> Nullable<Text>,
- mimetype -> Nullable<Text>,
+ extra_json -> Nullable<Jsonb>,
}
}
table! {
- file_rev_url (id) {
+ fileset_rev_release (fileset_rev, target_release_ident_id) {
+ fileset_rev -> Uuid,
+ target_release_ident_id -> Uuid,
+ }
+}
+
+table! {
+ fileset_rev_url (id) {
id -> Int8,
- file_rev -> Uuid,
+ fileset_rev -> Uuid,
rel -> Text,
url -> Text,
}
@@ -236,6 +294,67 @@ table! {
}
}
+table! {
+ webcapture_edit (id) {
+ id -> Int8,
+ editgroup_id -> Uuid,
+ updated -> Timestamptz,
+ ident_id -> Uuid,
+ rev_id -> Nullable<Uuid>,
+ redirect_id -> Nullable<Uuid>,
+ prev_rev -> Nullable<Uuid>,
+ extra_json -> Nullable<Jsonb>,
+ }
+}
+
+table! {
+ webcapture_ident (id) {
+ id -> Uuid,
+ is_live -> Bool,
+ rev_id -> Nullable<Uuid>,
+ redirect_id -> Nullable<Uuid>,
+ }
+}
+
+table! {
+ webcapture_rev (id) {
+ id -> Uuid,
+ extra_json -> Nullable<Jsonb>,
+ original_url -> Text,
+ timestamp -> Timestamptz,
+ }
+}
+
+table! {
+ webcapture_rev_cdx (id) {
+ id -> Int8,
+ webcapture_rev -> Uuid,
+ surt -> Text,
+ timestamp -> Int8,
+ url -> Text,
+ mimetype -> Nullable<Text>,
+ status_code -> Int8,
+ sha1 -> Text,
+ sha256 -> Nullable<Text>,
+ }
+}
+
+table! {
+ webcapture_rev_release (webcapture_rev, target_release_ident_id) {
+ webcapture_rev -> Uuid,
+ target_release_ident_id -> Uuid,
+ }
+}
+
+table! {
+ webcapture_rev_url (id) {
+ id -> Int8,
+ webcapture_rev -> Uuid,
+ rel -> Text,
+ url -> Text,
+ }
+}
+
table! {
work_edit (id) {
id -> Int8,
@@ -272,9 +391,15 @@ joinable!(creator_edit -> editgroup (editgroup_id));
joinable!(creator_ident -> creator_rev (rev_id));
joinable!(file_edit -> editgroup (editgroup_id));
joinable!(file_ident -> file_rev (rev_id));
-joinable!(file_release -> file_rev (file_rev));
-joinable!(file_release -> release_ident (target_release_ident_id));
+joinable!(file_rev_release -> file_rev (file_rev));
+joinable!(file_rev_release -> release_ident (target_release_ident_id));
joinable!(file_rev_url -> file_rev (file_rev));
+joinable!(fileset_edit -> editgroup (editgroup_id));
+joinable!(fileset_ident -> fileset_rev (rev_id));
+joinable!(fileset_rev_file -> fileset_rev (fileset_rev));
+joinable!(fileset_rev_release -> fileset_rev (fileset_rev));
+joinable!(fileset_rev_release -> release_ident (target_release_ident_id));
+joinable!(fileset_rev_url -> fileset_rev (fileset_rev));
joinable!(release_contrib -> creator_ident (creator_ident_id));
joinable!(release_contrib -> release_rev (release_rev));
joinable!(release_edit -> editgroup (editgroup_id));
@@ -285,6 +410,12 @@ joinable!(release_rev -> container_ident (container_ident_id));
joinable!(release_rev -> work_ident (work_ident_id));
joinable!(release_rev_abstract -> abstracts (abstract_sha1));
joinable!(release_rev_abstract -> release_rev (release_rev));
+joinable!(webcapture_edit -> editgroup (editgroup_id));
+joinable!(webcapture_ident -> webcapture_rev (rev_id));
+joinable!(webcapture_rev_cdx -> webcapture_rev (webcapture_rev));
+joinable!(webcapture_rev_release -> release_ident (target_release_ident_id));
+joinable!(webcapture_rev_release -> webcapture_rev (webcapture_rev));
+joinable!(webcapture_rev_url -> webcapture_rev (webcapture_rev));
joinable!(work_edit -> editgroup (editgroup_id));
joinable!(work_ident -> work_rev (rev_id));
@@ -301,15 +432,27 @@ allow_tables_to_appear_in_same_query!(
editor,
file_edit,
file_ident,
- file_release,
file_rev,
+ file_rev_release,
file_rev_url,
+ fileset_edit,
+ fileset_ident,
+ fileset_rev,
+ fileset_rev_file,
+ fileset_rev_release,
+ fileset_rev_url,
release_contrib,
release_edit,
release_ident,
release_ref,
release_rev,
release_rev_abstract,
+ webcapture_edit,
+ webcapture_ident,
+ webcapture_rev,
+ webcapture_rev_cdx,
+ webcapture_rev_release,
+ webcapture_rev_url,
work_edit,
work_ident,
work_rev,
--
cgit v1.2.3