aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-09-09 10:05:20 -0700
committerBryan Newbold <bnewbold@robocracy.org>2018-09-09 10:05:20 -0700
commit109f82169ef2a141b5d859e23d3ba052cec99fcb (patch)
tree5518b9aa4be19e3bd1065d191a393d81d0e0f9db
parent7354899493f6448bed5698ad6ade1dbebcf39379 (diff)
downloadfatcat-109f82169ef2a141b5d859e23d3ba052cec99fcb.tar.gz
fatcat-109f82169ef2a141b5d859e23d3ba052cec99fcb.zip
parallelize abstracts insertion
-rw-r--r--rust/src/database_entity_crud.rs35
1 files changed, 20 insertions, 15 deletions
diff --git a/rust/src/database_entity_crud.rs b/rust/src/database_entity_crud.rs
index 9a47c2de..88c89e84 100644
--- a/rust/src/database_entity_crud.rs
+++ b/rust/src/database_entity_crud.rs
@@ -40,7 +40,8 @@ pub trait EntityCrud
where
Self: Sized,
{
- // TODO: could these be generic structs? Or do they need to be bound to a specific table?
+ // TODO: could EditRow and IdentRow be generic structs? Or do they need to be bound to a
+ // specific table?
type EditRow; // EntityEditRow
type EditNewRow;
type IdentRow; // EntityIdentRow
@@ -85,7 +86,7 @@ where
fn db_insert_revs(conn: &DbConn, models: &[&Self]) -> Result<Vec<Uuid>>;
}
-// TODO: this could be a separate trait on all entities?
+// TODO: this could be a separate trait on all entities
macro_rules! generic_parse_editgroup_id{
() => {
fn parse_editgroup_id(&self) -> Result<Option<FatCatId>> {
@@ -832,6 +833,8 @@ impl EntityCrud for ReleaseEntity {
let mut release_ref_rows: Vec<ReleaseRefNewRow> = vec![];
let mut release_contrib_rows: Vec<ReleaseContribNewRow> = vec![];
+ let mut abstract_rows: Vec<AbstractsRow> = vec![];
+ let mut release_abstract_rows: Vec<ReleaseRevAbstractNewRow> = vec![];
for (model, rev_id) in models.iter().zip(rev_ids.iter()) {
match &model.refs {
@@ -883,7 +886,6 @@ impl EntityCrud for ReleaseEntity {
}
};
- // TODO: this part still isn't parallelized
if let Some(abstract_list) = &model.abstracts {
// For rows that specify content, we need to insert the abstract if it doesn't exist
// already
@@ -895,15 +897,8 @@ impl EntityCrud for ReleaseEntity {
content: c.content.clone().unwrap(),
})
.collect();
- if !new_abstracts.is_empty() {
- // Sort of an "upsert"; only inserts new abstract rows if they don't already exist
- insert_into(abstracts::table)
- .values(&new_abstracts)
- .on_conflict(abstracts::sha1)
- .do_nothing()
- .execute(conn)?;
- }
- let release_abstract_rows: Vec<ReleaseRevAbstractNewRow> = abstract_list
+ abstract_rows.extend(new_abstracts);
+ let new_release_abstract_rows: Vec<ReleaseRevAbstractNewRow> = abstract_list
.into_iter()
.map(|c| {
Ok(ReleaseRevAbstractNewRow {
@@ -920,9 +915,7 @@ impl EntityCrud for ReleaseEntity {
})
})
.collect::<Result<Vec<ReleaseRevAbstractNewRow>>>()?;
- insert_into(release_rev_abstract::table)
- .values(release_abstract_rows)
- .execute(conn)?;
+ release_abstract_rows.extend(new_release_abstract_rows);
}
}
@@ -938,6 +931,18 @@ impl EntityCrud for ReleaseEntity {
.execute(conn)?;
}
+ if !abstract_rows.is_empty() {
+ // Sort of an "upsert"; only inserts new abstract rows if they don't already exist
+ insert_into(abstracts::table)
+ .values(&abstract_rows)
+ .on_conflict(abstracts::sha1)
+ .do_nothing()
+ .execute(conn)?;
+ insert_into(release_rev_abstract::table)
+ .values(release_abstract_rows)
+ .execute(conn)?;
+ }
+
Ok(rev_ids)
}
}