From f4ded2f6249364c60f6e96324cd92ad9ab29f340 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:15:40 -0700 Subject: autoaccept flag for batch inserts --- fatcat-openapi2.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/fatcat-openapi2.yml b/fatcat-openapi2.yml index 524333ce..fda630dd 100644 --- a/fatcat-openapi2.yml +++ b/fatcat-openapi2.yml @@ -57,6 +57,10 @@ x-entity-props: &ENTITYPROPS extra: type: object additionalProperties: {} +# TODO: +# edit_extra: +# type: object +# additionalProperties: {} definitions: error_response: @@ -438,6 +442,11 @@ paths: post: operationId: "create_container_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -526,6 +535,11 @@ paths: post: operationId: "create_creator_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -630,6 +644,11 @@ paths: post: operationId: "create_file_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -718,6 +737,11 @@ paths: post: operationId: "create_release_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -822,6 +846,11 @@ paths: post: operationId: "create_work_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true -- cgit v1.2.3 From bf4a949f83b30a8e9bb501871bc62224472c59aa Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:15:50 -0700 Subject: codegen --- rust/fatcat-api/README.md | 2 +- rust/fatcat-api/api.yaml | 29 +++++++++++ rust/fatcat-api/api/swagger.yaml | 40 +++++++++++++++ rust/fatcat-api/examples/client.rs | 10 ++-- rust/fatcat-api/examples/server_lib/server.rs | 46 +++++++++++++---- rust/fatcat-api/src/client.rs | 72 +++++++++++++++++++++++---- rust/fatcat-api/src/lib.rs | 45 +++++++++-------- rust/fatcat-api/src/server.rs | 30 +++++++++-- 8 files changed, 223 insertions(+), 51 deletions(-) diff --git a/rust/fatcat-api/README.md b/rust/fatcat-api/README.md index 1e382370..d0b266aa 100644 --- a/rust/fatcat-api/README.md +++ b/rust/fatcat-api/README.md @@ -13,7 +13,7 @@ To see how to make this your own, look here: [README](https://github.com/swagger-api/swagger-codegen/blob/master/README.md) - API version: 0.1.0 -- Build date: 2018-08-16T16:42:27.600Z +- Build date: 2018-08-20T08:47:01.260Z This autogenerated project defines an API crate `fatcat` which contains: * An `Api` trait defining the API in Rust. diff --git a/rust/fatcat-api/api.yaml b/rust/fatcat-api/api.yaml index 524333ce..fda630dd 100644 --- a/rust/fatcat-api/api.yaml +++ b/rust/fatcat-api/api.yaml @@ -57,6 +57,10 @@ x-entity-props: &ENTITYPROPS extra: type: object additionalProperties: {} +# TODO: +# edit_extra: +# type: object +# additionalProperties: {} definitions: error_response: @@ -438,6 +442,11 @@ paths: post: operationId: "create_container_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -526,6 +535,11 @@ paths: post: operationId: "create_creator_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -630,6 +644,11 @@ paths: post: operationId: "create_file_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -718,6 +737,11 @@ paths: post: operationId: "create_release_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true @@ -822,6 +846,11 @@ paths: post: operationId: "create_work_batch" parameters: + - name: autoaccept + in: query + type: boolean + required: false + description: "If true, and editor is authorized, batch is accepted all at once" - name: entity_list in: body required: true diff --git a/rust/fatcat-api/api/swagger.yaml b/rust/fatcat-api/api/swagger.yaml index 021594c0..11f789dc 100644 --- a/rust/fatcat-api/api/swagger.yaml +++ b/rust/fatcat-api/api/swagger.yaml @@ -76,6 +76,14 @@ paths: post: operationId: "create_container_batch" parameters: + - name: "autoaccept" + in: "query" + description: "If true, and editor is authorized, batch is accepted all at\ + \ once" + required: false + type: "boolean" + formatString: "{:?}" + example: "Some(true)" - in: "body" name: "entity_list" required: true @@ -371,6 +379,14 @@ paths: post: operationId: "create_creator_batch" parameters: + - name: "autoaccept" + in: "query" + description: "If true, and editor is authorized, batch is accepted all at\ + \ once" + required: false + type: "boolean" + formatString: "{:?}" + example: "Some(true)" - in: "body" name: "entity_list" required: true @@ -720,6 +736,14 @@ paths: post: operationId: "create_file_batch" parameters: + - name: "autoaccept" + in: "query" + description: "If true, and editor is authorized, batch is accepted all at\ + \ once" + required: false + type: "boolean" + formatString: "{:?}" + example: "Some(true)" - in: "body" name: "entity_list" required: true @@ -1012,6 +1036,14 @@ paths: post: operationId: "create_release_batch" parameters: + - name: "autoaccept" + in: "query" + description: "If true, and editor is authorized, batch is accepted all at\ + \ once" + required: false + type: "boolean" + formatString: "{:?}" + example: "Some(true)" - in: "body" name: "entity_list" required: true @@ -1358,6 +1390,14 @@ paths: post: operationId: "create_work_batch" parameters: + - name: "autoaccept" + in: "query" + description: "If true, and editor is authorized, batch is accepted all at\ + \ once" + required: false + type: "boolean" + formatString: "{:?}" + example: "Some(true)" - in: "body" name: "entity_list" required: true diff --git a/rust/fatcat-api/examples/client.rs b/rust/fatcat-api/examples/client.rs index f98518cc..06519232 100644 --- a/rust/fatcat-api/examples/client.rs +++ b/rust/fatcat-api/examples/client.rs @@ -95,7 +95,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateContainerBatch") => { - let result = client.create_container_batch(&Vec::new()).wait(); + let result = client.create_container_batch(&Vec::new(), Some(true)).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -105,7 +105,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateCreatorBatch") => { - let result = client.create_creator_batch(&Vec::new()).wait(); + let result = client.create_creator_batch(&Vec::new(), Some(true)).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -121,7 +121,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateFileBatch") => { - let result = client.create_file_batch(&Vec::new()).wait(); + let result = client.create_file_batch(&Vec::new(), Some(true)).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -131,7 +131,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateReleaseBatch") => { - let result = client.create_release_batch(&Vec::new()).wait(); + let result = client.create_release_batch(&Vec::new(), Some(true)).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -141,7 +141,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateWorkBatch") => { - let result = client.create_work_batch(&Vec::new()).wait(); + let result = client.create_work_batch(&Vec::new(), Some(true)).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } diff --git a/rust/fatcat-api/examples/server_lib/server.rs b/rust/fatcat-api/examples/server_lib/server.rs index 572025f0..32c7e97f 100644 --- a/rust/fatcat-api/examples/server_lib/server.rs +++ b/rust/fatcat-api/examples/server_lib/server.rs @@ -34,11 +34,17 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_container_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send> { + fn create_container_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + context: &Context, + ) -> Box + Send> { let context = context.clone(); println!( - "create_container_batch({:?}) - X-Span-ID: {:?}", + "create_container_batch({:?}, {:?}) - X-Span-ID: {:?}", entity_list, + autoaccept, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) @@ -50,9 +56,14 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_creator_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send> { + fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { let context = context.clone(); - println!("create_creator_batch({:?}) - X-Span-ID: {:?}", entity_list, context.x_span_id.unwrap_or(String::from("")).clone()); + println!( + "create_creator_batch({:?}, {:?}) - X-Span-ID: {:?}", + entity_list, + autoaccept, + context.x_span_id.unwrap_or(String::from("")).clone() + ); Box::new(futures::failed("Generic failure".into())) } @@ -68,9 +79,14 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_file_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send> { + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { let context = context.clone(); - println!("create_file_batch({:?}) - X-Span-ID: {:?}", entity_list, context.x_span_id.unwrap_or(String::from("")).clone()); + println!( + "create_file_batch({:?}, {:?}) - X-Span-ID: {:?}", + entity_list, + autoaccept, + context.x_span_id.unwrap_or(String::from("")).clone() + ); Box::new(futures::failed("Generic failure".into())) } @@ -80,9 +96,14 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_release_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send> { + fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { let context = context.clone(); - println!("create_release_batch({:?}) - X-Span-ID: {:?}", entity_list, context.x_span_id.unwrap_or(String::from("")).clone()); + println!( + "create_release_batch({:?}, {:?}) - X-Span-ID: {:?}", + entity_list, + autoaccept, + context.x_span_id.unwrap_or(String::from("")).clone() + ); Box::new(futures::failed("Generic failure".into())) } @@ -92,9 +113,14 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_work_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send> { + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { let context = context.clone(); - println!("create_work_batch({:?}) - X-Span-ID: {:?}", entity_list, context.x_span_id.unwrap_or(String::from("")).clone()); + println!( + "create_work_batch({:?}, {:?}) - X-Span-ID: {:?}", + entity_list, + autoaccept, + context.x_span_id.unwrap_or(String::from("")).clone() + ); Box::new(futures::failed("Generic failure".into())) } diff --git a/rust/fatcat-api/src/client.rs b/rust/fatcat-api/src/client.rs index bc1992de..d71c9dab 100644 --- a/rust/fatcat-api/src/client.rs +++ b/rust/fatcat-api/src/client.rs @@ -290,8 +290,20 @@ impl Api for Client { Box::new(futures::done(result)) } - fn create_container_batch(&self, param_entity_list: &Vec, context: &Context) -> Box + Send> { - let url = format!("{}/v0/container/batch", self.base_path); + fn create_container_batch( + &self, + param_entity_list: &Vec, + param_autoaccept: Option, + context: &Context, + ) -> Box + Send> { + // Query parameters + let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + + let url = format!( + "{}/v0/container/batch?{autoaccept}", + self.base_path, + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -420,8 +432,20 @@ impl Api for Client { Box::new(futures::done(result)) } - fn create_creator_batch(&self, param_entity_list: &Vec, context: &Context) -> Box + Send> { - let url = format!("{}/v0/creator/batch", self.base_path); + fn create_creator_batch( + &self, + param_entity_list: &Vec, + param_autoaccept: Option, + context: &Context, + ) -> Box + Send> { + // Query parameters + let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + + let url = format!( + "{}/v0/creator/batch?{autoaccept}", + self.base_path, + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -608,8 +632,16 @@ impl Api for Client { Box::new(futures::done(result)) } - fn create_file_batch(&self, param_entity_list: &Vec, context: &Context) -> Box + Send> { - let url = format!("{}/v0/file/batch", self.base_path); + fn create_file_batch( + &self, + param_entity_list: &Vec, + param_autoaccept: Option, + context: &Context, + ) -> Box + Send> { + // Query parameters + let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + + let url = format!("{}/v0/file/batch?{autoaccept}", self.base_path, autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET)); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -738,8 +770,20 @@ impl Api for Client { Box::new(futures::done(result)) } - fn create_release_batch(&self, param_entity_list: &Vec, context: &Context) -> Box + Send> { - let url = format!("{}/v0/release/batch", self.base_path); + fn create_release_batch( + &self, + param_entity_list: &Vec, + param_autoaccept: Option, + context: &Context, + ) -> Box + Send> { + // Query parameters + let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + + let url = format!( + "{}/v0/release/batch?{autoaccept}", + self.base_path, + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -868,8 +912,16 @@ impl Api for Client { Box::new(futures::done(result)) } - fn create_work_batch(&self, param_entity_list: &Vec, context: &Context) -> Box + Send> { - let url = format!("{}/v0/work/batch", self.base_path); + fn create_work_batch( + &self, + param_entity_list: &Vec, + param_autoaccept: Option, + context: &Context, + ) -> Box + Send> { + // Query parameters + let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + + let url = format!("{}/v0/work/batch?{autoaccept}", self.base_path, autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET)); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); diff --git a/rust/fatcat-api/src/lib.rs b/rust/fatcat-api/src/lib.rs index fac8ecac..044b934b 100644 --- a/rust/fatcat-api/src/lib.rs +++ b/rust/fatcat-api/src/lib.rs @@ -442,25 +442,30 @@ pub trait Api { fn create_container(&self, entity: models::ContainerEntity, context: &Context) -> Box + Send>; - fn create_container_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send>; + fn create_container_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + context: &Context, + ) -> Box + Send>; fn create_creator(&self, entity: models::CreatorEntity, context: &Context) -> Box + Send>; - fn create_creator_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send>; + fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; fn create_editgroup(&self, entity: models::Editgroup, context: &Context) -> Box + Send>; fn create_file(&self, entity: models::FileEntity, context: &Context) -> Box + Send>; - fn create_file_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send>; + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; fn create_release(&self, entity: models::ReleaseEntity, context: &Context) -> Box + Send>; - fn create_release_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send>; + fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; fn create_work(&self, entity: models::WorkEntity, context: &Context) -> Box + Send>; - fn create_work_batch(&self, entity_list: &Vec, context: &Context) -> Box + Send>; + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; fn get_changelog(&self, limit: Option, context: &Context) -> Box + Send>; @@ -515,25 +520,25 @@ pub trait ApiNoContext { fn create_container(&self, entity: models::ContainerEntity) -> Box + Send>; - fn create_container_batch(&self, entity_list: &Vec) -> Box + Send>; + fn create_container_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; fn create_creator(&self, entity: models::CreatorEntity) -> Box + Send>; - fn create_creator_batch(&self, entity_list: &Vec) -> Box + Send>; + fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; fn create_editgroup(&self, entity: models::Editgroup) -> Box + Send>; fn create_file(&self, entity: models::FileEntity) -> Box + Send>; - fn create_file_batch(&self, entity_list: &Vec) -> Box + Send>; + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; fn create_release(&self, entity: models::ReleaseEntity) -> Box + Send>; - fn create_release_batch(&self, entity_list: &Vec) -> Box + Send>; + fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; fn create_work(&self, entity: models::WorkEntity) -> Box + Send>; - fn create_work_batch(&self, entity_list: &Vec) -> Box + Send>; + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; fn get_changelog(&self, limit: Option) -> Box + Send>; @@ -606,16 +611,16 @@ impl<'a, T: Api> ApiNoContext for ContextWrapper<'a, T> { self.api().create_container(entity, &self.context()) } - fn create_container_batch(&self, entity_list: &Vec) -> Box + Send> { - self.api().create_container_batch(entity_list, &self.context()) + fn create_container_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { + self.api().create_container_batch(entity_list, autoaccept, &self.context()) } fn create_creator(&self, entity: models::CreatorEntity) -> Box + Send> { self.api().create_creator(entity, &self.context()) } - fn create_creator_batch(&self, entity_list: &Vec) -> Box + Send> { - self.api().create_creator_batch(entity_list, &self.context()) + fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { + self.api().create_creator_batch(entity_list, autoaccept, &self.context()) } fn create_editgroup(&self, entity: models::Editgroup) -> Box + Send> { @@ -626,24 +631,24 @@ impl<'a, T: Api> ApiNoContext for ContextWrapper<'a, T> { self.api().create_file(entity, &self.context()) } - fn create_file_batch(&self, entity_list: &Vec) -> Box + Send> { - self.api().create_file_batch(entity_list, &self.context()) + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { + self.api().create_file_batch(entity_list, autoaccept, &self.context()) } fn create_release(&self, entity: models::ReleaseEntity) -> Box + Send> { self.api().create_release(entity, &self.context()) } - fn create_release_batch(&self, entity_list: &Vec) -> Box + Send> { - self.api().create_release_batch(entity_list, &self.context()) + fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { + self.api().create_release_batch(entity_list, autoaccept, &self.context()) } fn create_work(&self, entity: models::WorkEntity) -> Box + Send> { self.api().create_work(entity, &self.context()) } - fn create_work_batch(&self, entity_list: &Vec) -> Box + Send> { - self.api().create_work_batch(entity_list, &self.context()) + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { + self.api().create_work_batch(entity_list, autoaccept, &self.context()) } fn get_changelog(&self, limit: Option) -> Box + Send> { diff --git a/rust/fatcat-api/src/server.rs b/rust/fatcat-api/src/server.rs index 68e08515..4e41b5e9 100644 --- a/rust/fatcat-api/src/server.rs +++ b/rust/fatcat-api/src/server.rs @@ -298,6 +298,10 @@ where context.auth_data = req.extensions.remove::(); context.authorization = req.extensions.remove::(); + // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) + let query_params = req.get::().unwrap_or_default(); + let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for // any unused fields. @@ -322,7 +326,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_container_batch(param_entity_list.as_ref(), context).wait() { + match api.create_container_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { Ok(rsp) => match rsp { CreateContainerBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -510,6 +514,10 @@ where context.auth_data = req.extensions.remove::(); context.authorization = req.extensions.remove::(); + // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) + let query_params = req.get::().unwrap_or_default(); + let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for // any unused fields. @@ -534,7 +542,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_creator_batch(param_entity_list.as_ref(), context).wait() { + match api.create_creator_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { Ok(rsp) => match rsp { CreateCreatorBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -815,6 +823,10 @@ where context.auth_data = req.extensions.remove::(); context.authorization = req.extensions.remove::(); + // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) + let query_params = req.get::().unwrap_or_default(); + let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for // any unused fields. @@ -839,7 +851,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_file_batch(param_entity_list.as_ref(), context).wait() { + match api.create_file_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { Ok(rsp) => match rsp { CreateFileBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -1027,6 +1039,10 @@ where context.auth_data = req.extensions.remove::(); context.authorization = req.extensions.remove::(); + // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) + let query_params = req.get::().unwrap_or_default(); + let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for // any unused fields. @@ -1051,7 +1067,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_release_batch(param_entity_list.as_ref(), context).wait() { + match api.create_release_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { Ok(rsp) => match rsp { CreateReleaseBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -1239,6 +1255,10 @@ where context.auth_data = req.extensions.remove::(); context.authorization = req.extensions.remove::(); + // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) + let query_params = req.get::().unwrap_or_default(); + let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for // any unused fields. @@ -1263,7 +1283,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_work_batch(param_entity_list.as_ref(), context).wait() { + match api.create_work_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { Ok(rsp) => match rsp { CreateWorkBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); -- cgit v1.2.3 From 2f7cff3c597af42c8dd02313bdceb98a11c2e414 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:16:08 -0700 Subject: implement autoaccept Need to review: - changelog creation - clobbering of existing editgroup flag --- rust/src/api_server.rs | 61 ++++++++++++++++++++++++++++++++++-------------- rust/src/api_wrappers.rs | 5 ++-- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/rust/src/api_server.rs b/rust/src/api_server.rs index b0dff173..0cb07e81 100644 --- a/rust/src/api_server.rs +++ b/rust/src/api_server.rs @@ -25,11 +25,23 @@ macro_rules! entity_batch_handler { pub fn $post_batch_handler( &self, entity_list: &[models::$model], + autoaccept: bool, conn: &DbConn, ) -> Result> { let mut ret: Vec = vec![]; + let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth + let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) + .values((editgroup::editor_id.eq(editor_id),)) + .get_result(conn)?; for entity in entity_list { - ret.push(self.$post_handler(entity.clone(), conn)?); + let mut e = entity.clone(); + e.editgroup_id = Some(uuid2fcid(&eg_row.id)); + ret.push(self.$post_handler(e, autoaccept, conn)?); + } + if autoaccept { + let _clr: ChangelogRow = diesel::insert_into(changelog::table) + .values((changelog::editgroup_id.eq(eg_row.id),)) + .get_result(conn)?; } Ok(ret) } @@ -216,7 +228,10 @@ fn release_row2entity( let contribs: Vec = release_contrib::table .filter(release_contrib::release_rev.eq(rev.id)) - .order((release_contrib::role.asc(), release_contrib::index_val.asc())) + .order(( + release_contrib::role.asc(), + release_contrib::index_val.asc(), + )) .get_results(conn) .expect("fetch release refs") .into_iter() @@ -495,6 +510,7 @@ impl Server { pub fn create_container_handler( &self, entity: models::ContainerEntity, + autoaccept: bool, conn: &DbConn, ) -> Result { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth @@ -513,11 +529,11 @@ impl Server { "WITH rev AS ( INSERT INTO container_rev (name, publisher, issnl, wikidata_qid, abbrev, coden, extra_json) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING id ), - ident AS ( INSERT INTO container_ident (rev_id) - VALUES ((SELECT rev.id FROM rev)) + ident AS ( INSERT INTO container_ident (is_live, rev_id) + VALUES ($8, (SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO container_edit (editgroup_id, ident_id, rev_id) VALUES - ($8, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($9, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::(entity.name) .bind::, _>(entity.publisher) @@ -526,6 +542,7 @@ impl Server { .bind::, _>(entity.abbrev) .bind::, _>(entity.coden) .bind::, _>(entity.extra) + .bind::(autoaccept) .bind::(editgroup_id) .get_result(conn)?; @@ -535,6 +552,7 @@ impl Server { pub fn create_creator_handler( &self, entity: models::CreatorEntity, + autoaccept: bool, conn: &DbConn, ) -> Result { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth @@ -553,11 +571,11 @@ impl Server { "WITH rev AS ( INSERT INTO creator_rev (display_name, given_name, surname, orcid, wikidata_qid, extra_json) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id ), - ident AS ( INSERT INTO creator_ident (rev_id) - VALUES ((SELECT rev.id FROM rev)) + ident AS ( INSERT INTO creator_ident (is_live, rev_id) + VALUES ($7, (SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO creator_edit (editgroup_id, ident_id, rev_id) VALUES - ($7, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($8, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::(entity.display_name) .bind::, _>(entity.given_name) @@ -565,6 +583,7 @@ impl Server { .bind::, _>(entity.orcid) .bind::, _>(entity.wikidata_qid) .bind::, _>(entity.extra) + .bind::(autoaccept) .bind::(editgroup_id) .get_result(conn)?; @@ -574,6 +593,7 @@ impl Server { pub fn create_file_handler( &self, entity: models::FileEntity, + autoaccept: bool, conn: &DbConn, ) -> Result { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth @@ -587,11 +607,11 @@ impl Server { "WITH rev AS ( INSERT INTO file_rev (size, sha1, sha256, md5, mimetype, extra_json) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id ), - ident AS ( INSERT INTO file_ident (rev_id) - VALUES ((SELECT rev.id FROM rev)) + ident AS ( INSERT INTO file_ident (is_live, rev_id) + VALUES ($7, (SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO file_edit (editgroup_id, ident_id, rev_id) VALUES - ($7, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($8, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::, _>(entity.size) .bind::, _>(entity.sha1) @@ -599,6 +619,7 @@ impl Server { .bind::, _>(entity.md5) .bind::, _>(entity.mimetype) .bind::, _>(entity.extra) + .bind::(autoaccept) .bind::(editgroup_id) .get_result(conn)?; @@ -654,6 +675,7 @@ impl Server { pub fn create_release_handler( &self, entity: models::ReleaseEntity, + autoaccept: bool, conn: &DbConn, ) -> Result { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth @@ -686,7 +708,7 @@ impl Server { editgroup_id: Some(uuid2fcid(&editgroup_id)), extra: None, }; - let new_entity = self.create_work_handler(work_model, conn)?; + let new_entity = self.create_work_handler(work_model, autoaccept, conn)?; fcid2uuid(&new_entity.ident)? } }; @@ -700,11 +722,11 @@ impl Server { "WITH rev AS ( INSERT INTO release_rev (title, release_type, release_status, release_date, doi, pmid, pmcid, wikidata_qid, isbn13, core_id, volume, issue, pages, work_ident_id, container_ident_id, publisher, language, extra_json) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING id ), - ident AS ( INSERT INTO release_ident (rev_id) - VALUES ((SELECT rev.id FROM rev)) + ident AS ( INSERT INTO release_ident (is_live, rev_id) + VALUES ($19, (SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO release_edit (editgroup_id, ident_id, rev_id) VALUES - ($19, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($20, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::(entity.title) .bind::, _>(entity.release_type) @@ -725,6 +747,7 @@ impl Server { .bind::, _>(entity.publisher) .bind::, _>(entity.language) .bind::, _>(entity.extra) + .bind::(autoaccept) .bind::(editgroup_id) .get_result(conn)?; @@ -829,6 +852,7 @@ impl Server { pub fn create_work_handler( &self, entity: models::WorkEntity, + autoaccept: bool, conn: &DbConn, ) -> Result { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth @@ -842,13 +866,14 @@ impl Server { "WITH rev AS ( INSERT INTO work_rev (extra_json) VALUES ($1) RETURNING id ), - ident AS ( INSERT INTO work_ident (rev_id) - VALUES ((SELECT rev.id FROM rev)) + ident AS ( INSERT INTO work_ident (is_live, rev_id) + VALUES ($2, (SELECT rev.id FROM rev)) RETURNING id ) INSERT INTO work_edit (editgroup_id, ident_id, rev_id) VALUES - ($2, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) + ($3, (SELECT ident.id FROM ident), (SELECT rev.id FROM rev)) RETURNING *", ).bind::, _>(entity.extra) + .bind::(autoaccept) .bind::(editgroup_id) .get_result(conn)?; diff --git a/rust/src/api_wrappers.rs b/rust/src/api_wrappers.rs index 95336d3f..4f126385 100644 --- a/rust/src/api_wrappers.rs +++ b/rust/src/api_wrappers.rs @@ -60,7 +60,7 @@ macro_rules! wrap_entity_handlers { _context: &Context, ) -> Box + Send> { let conn = self.db_pool.get().expect("db_pool error"); - let ret = match conn.transaction(|| self.$post_handler(entity, &conn)) { + let ret = match conn.transaction(|| self.$post_handler(entity, false, &conn)) { Ok(edit) => $post_resp::CreatedEntity(edit), Err(Error(ErrorKind::Diesel(e), _)) => @@ -83,10 +83,11 @@ macro_rules! wrap_entity_handlers { fn $post_batch_fn( &self, entity_list: &Vec, + autoaccept: Option, _context: &Context, ) -> Box + Send> { let conn = self.db_pool.get().expect("db_pool error"); - let ret = match conn.transaction(|| self.$post_batch_handler(entity_list, &conn)) { + let ret = match conn.transaction(|| self.$post_batch_handler(entity_list, autoaccept.unwrap_or(false), &conn)) { Ok(edit) => $post_batch_resp::CreatedEntities(edit), Err(Error(ErrorKind::Diesel(e), _)) => -- cgit v1.2.3 From 0c400f377bb02c3ea6290de865650f00f1f1abbd Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:22:32 -0700 Subject: python autoaccept implementation More to review: - duplicate containers getting created in batch mode? eg, if multiple releases with same container. should push+autoaccept those anyways? --- python/fatcat/crossref_importer.py | 2 +- python/fatcat/importer_common.py | 3 --- python/fatcat/issn_importer.py | 2 +- python/fatcat/orcid_importer.py | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/python/fatcat/crossref_importer.py b/python/fatcat/crossref_importer.py index d3e525a4..b12851dc 100644 --- a/python/fatcat/crossref_importer.py +++ b/python/fatcat/crossref_importer.py @@ -152,4 +152,4 @@ class FatcatCrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch) + self.api.create_release_batch(release_batch, autoaccept=True) diff --git a/python/fatcat/importer_common.py b/python/fatcat/importer_common.py index e084d8c4..9949d847 100644 --- a/python/fatcat/importer_common.py +++ b/python/fatcat/importer_common.py @@ -43,10 +43,7 @@ class FatcatImporter: def process_batch(self, source, size=50): """Reads and processes in batches (not API-call-per-)""" for rows in grouper(source, size): - eg = self.api.create_editgroup( - fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) self.create_batch(rows, eg.id) - self.api.accept_editgroup(eg.id) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) diff --git a/python/fatcat/issn_importer.py b/python/fatcat/issn_importer.py index 181137ac..6b806b40 100644 --- a/python/fatcat/issn_importer.py +++ b/python/fatcat/issn_importer.py @@ -70,4 +70,4 @@ class FatcatIssnImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_container_batch(objects) + self.api.create_container_batch(objects, autoaccept=True) diff --git a/python/fatcat/orcid_importer.py b/python/fatcat/orcid_importer.py index e57703d5..cc12d50d 100644 --- a/python/fatcat/orcid_importer.py +++ b/python/fatcat/orcid_importer.py @@ -71,4 +71,4 @@ class FatcatOrcidImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_creator_batch(objects) + self.api.create_creator_batch(objects, autoaccept=True) -- cgit v1.2.3 From 693b9a0bb99aa6a691357c1e7c2b018150a3380b Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:34:10 -0700 Subject: fix bug in new batch create python importer --- python/fatcat/importer_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fatcat/importer_common.py b/python/fatcat/importer_common.py index 9949d847..c1566e65 100644 --- a/python/fatcat/importer_common.py +++ b/python/fatcat/importer_common.py @@ -43,7 +43,7 @@ class FatcatImporter: def process_batch(self, source, size=50): """Reads and processes in batches (not API-call-per-)""" for rows in grouper(source, size): - self.create_batch(rows, eg.id) + self.create_batch(rows) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) -- cgit v1.2.3 From 6207a12627830f1e524f1ea36d02871510b2bd28 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 02:34:28 -0700 Subject: codegen python --- python/fatcat_client/api/default_api.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/python/fatcat_client/api/default_api.py b/python/fatcat_client/api/default_api.py index acd33c88..914f747c 100644 --- a/python/fatcat_client/api/default_api.py +++ b/python/fatcat_client/api/default_api.py @@ -245,6 +245,7 @@ class DefaultApi(object): :param async bool :param list[ContainerEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -266,12 +267,13 @@ class DefaultApi(object): :param async bool :param list[ContainerEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list'] # noqa: E501 + all_params = ['entity_list', 'autoaccept'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -296,6 +298,8 @@ class DefaultApi(object): path_params = {} query_params = [] + if 'autoaccept' in params: + query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 header_params = {} @@ -439,6 +443,7 @@ class DefaultApi(object): :param async bool :param list[CreatorEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -460,12 +465,13 @@ class DefaultApi(object): :param async bool :param list[CreatorEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list'] # noqa: E501 + all_params = ['entity_list', 'autoaccept'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -490,6 +496,8 @@ class DefaultApi(object): path_params = {} query_params = [] + if 'autoaccept' in params: + query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 header_params = {} @@ -730,6 +738,7 @@ class DefaultApi(object): :param async bool :param list[FileEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -751,12 +760,13 @@ class DefaultApi(object): :param async bool :param list[FileEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list'] # noqa: E501 + all_params = ['entity_list', 'autoaccept'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -781,6 +791,8 @@ class DefaultApi(object): path_params = {} query_params = [] + if 'autoaccept' in params: + query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 header_params = {} @@ -924,6 +936,7 @@ class DefaultApi(object): :param async bool :param list[ReleaseEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -945,12 +958,13 @@ class DefaultApi(object): :param async bool :param list[ReleaseEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list'] # noqa: E501 + all_params = ['entity_list', 'autoaccept'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -975,6 +989,8 @@ class DefaultApi(object): path_params = {} query_params = [] + if 'autoaccept' in params: + query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 header_params = {} @@ -1118,6 +1134,7 @@ class DefaultApi(object): :param async bool :param list[WorkEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -1139,12 +1156,13 @@ class DefaultApi(object): :param async bool :param list[WorkEntity] entity_list: (required) + :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list'] # noqa: E501 + all_params = ['entity_list', 'autoaccept'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -1169,6 +1187,8 @@ class DefaultApi(object): path_params = {} query_params = [] + if 'autoaccept' in params: + query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 header_params = {} -- cgit v1.2.3 From 3c7d392bc4b36e8e5890d2ccb292f74e6b988e55 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 14:49:06 -0700 Subject: notes on recent bulk import --- notes/import_timing_20180815.txt | 292 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 292 insertions(+) create mode 100644 notes/import_timing_20180815.txt diff --git a/notes/import_timing_20180815.txt b/notes/import_timing_20180815.txt new file mode 100644 index 00000000..1206cc41 --- /dev/null +++ b/notes/import_timing_20180815.txt @@ -0,0 +1,292 @@ + +Schema changes since previous imports: +- more fields (identifiers+indexes) +- timestamps +- UUIDs more places +- fixed some crossref import bugs? +- abstracts +- file_urls as table (not single value) +- timestamps +- TEXT -> CHAR in a few places +- removed many work fields + +## Containers + +(python)webcrawl@wbgrp-svc500:/srv/fatcat/src/python$ time ./fatcat_import.py import-issn /srv/datasets/journal_extra_metadata.csv + +real 1m25.292s +user 0m12.640s +sys 0m0.412s + +## Creators + +time parallel --bar --pipepart -j8 -a /srv/datasets/public_profiles_1_2_json.all.json ./fatcat_import.py import-orcid - + +(times very invalid due to hangs; got 3537837 creators, which is most of the way, so *shrug*) +real 22m2.465s +user 26m41.924s +sys 1m33.844s + +## Releases + +xzcat /srv/datasets/crossref-works.2018-01-21.json.xz | time parallel -j20 --round-robin --pipe ./fatcat_import.py import-crossref - /srv/datasets/20180216.ISSN-to-ISSN-L.txt + + 128516.30 user + 3905.14 system + 44:17:05 elapsed + 83% CPU + +Almost 44 hours... I think I remember more like 36 hours last time? Things +slowed down a lot towards the end, many more ORCID cross-references? + +looking in htop, postgres seems to be primary bottleneck. At something like 12 +hours in, had 44 million release_ident rows, which is 1000/second. + +Note: seems like the more frequently `count(*)` is run, the more performant. +Because in-memory? + + 2018-08-16 16:54:16.977 UTC [17996] postgres@fatcat_prod LOG: duration: 42949.549 ms statement: select count(id) from release_ident; + + fatcat_prod=# select count(*) from release_ident; + count + ---------- + 44185608 + (1 row) + + Time: 2753.916 ms (00:02.754) + fatcat_prod=# select count(*) from release_ident; + count + ---------- + 44187937 + (1 row) + + Time: 2711.670 ms (00:02.712) + +As expected, autovacuum very busy. Only ~150 TPS; but that includes batch +writes? 75061172 rows. + +## Files + + time ./fatcat_import.py import-manifest /srv/datasets/idents_files_urls.sqlite + + Done! Inserted 6607075 + + real 2152m28.822s => 36 hours (!) + user 401m46.464s + sys 21m45.724s + + +Going pretty slow, < 100 transactions/sec. Lots of SELECTs, which seem slow, on the abstract table? + + SELECT "release_rev_abstract"."id", "release_rev_abstract"."release_rev", "release_rev_abstract"."abstract_sha1", "release_rev_abstract"."mimetype", "release_rev_abstract"."lang", "abstracts"."sha1", "abstracts"."content" FROM ("release_rev_abstract" INNER JOIN "abstracts" ON "release_rev_abstract"."abstract_sha1" = "abstracts"."sha1") WHERE "release_rev_abstract"."release_rev" = 'ffffffc0-4dd2-47ce-a51d-44051f3699ce'; + +Created index: + + CREATE INDEX release_rev_abstract_rev_idx ON release_rev_abstract(release_rev); + +... and things sped way up. Re-ran some crossref imports to EXPLAIN and didn't +see non-indexed queries. Maybe an ANALYZE does need to happen? + +This being single-threaded is going to be a problem in the future. ~50 million +files would be ~2 weeks. + +## Post-Import Status + + Size: 358.89G (postgres self-reported) + Mem.: 57.10% - 16.85G/49.14G + +Was 184G last time in late June; doubled in size (!). + + bnewbold@wbgrp-svc500$ df -h / + Filesystem Size Used Avail Use% Mounted on + /dev/vda1 858G 529G 286G 65% / + + bnewbold@wbgrp-svc500$ sudo du -sh /var/lib/postgresql/ /srv/datasets/ /srv/elastic-blah/ + 361G /var/lib/postgresql/ + 83G /srv/datasets/ + 77G /srv/elastic-blah/ + + fatcat_prod=# select count(*) from changelog; => 2,085,067 + + SELECT + table_name, + pg_size_pretty(table_size) AS table_size, + pg_size_pretty(indexes_size) AS indexes_size, + pg_size_pretty(total_size) AS total_size + FROM ( + SELECT + table_name, + pg_table_size(table_name) AS table_size, + pg_indexes_size(table_name) AS indexes_size, + pg_total_relation_size(table_name) AS total_size + FROM ( + SELECT ('"' || table_schema || '"."' || table_name || '"') AS table_name + FROM information_schema.tables + ) AS all_tables + ORDER BY total_size DESC + ) AS pretty_sizes; + + table_name | table_size | indexes_size | total_size +--------------------------------------------------------------+------------+--------------+------------ + "public"."release_ref" | 159 GB | 47 GB | 206 GB + "public"."release_rev" | 40 GB | 10 GB | 51 GB + "public"."release_contrib" | 19 GB | 20 GB | 39 GB + "public"."release_ident" | 5797 MB | 6597 MB | 12 GB + "public"."work_ident" | 5787 MB | 6394 MB | 12 GB + "public"."release_edit" | 6674 MB | 4646 MB | 11 GB + "public"."work_edit" | 6674 MB | 4646 MB | 11 GB + "public"."work_rev" | 3175 MB | 2939 MB | 6114 MB + "public"."file_rev_url" | 1995 MB | 275 MB | 2270 MB + "public"."abstracts" | 1665 MB | 135 MB | 1800 MB + "public"."file_rev" | 829 MB | 954 MB | 1783 MB + "public"."file_ident" | 498 MB | 532 MB | 1030 MB + "public"."file_release" | 369 MB | 642 MB | 1011 MB + "public"."file_edit" | 591 MB | 410 MB | 1002 MB + "public"."creator_rev" | 337 MB | 318 MB | 655 MB + "public"."creator_ident" | 280 MB | 297 MB | 577 MB + "public"."creator_edit" | 316 MB | 220 MB | 536 MB + "public"."release_rev_abstract" | 183 MB | 84 MB | 267 MB + "public"."changelog" | 123 MB | 125 MB | 249 MB + "public"."editgroup" | 139 MB | 81 MB | 220 MB + "public"."container_rev" | 19 MB | 6912 kB | 26 MB + "public"."container_ident" | 6896 kB | 7016 kB | 14 MB + "public"."container_edit" | 8056 kB | 5240 kB | 13 MB + +In context, the full uncompressed crossref 2018-01-21 dump is about 285 GB. + +For many of these indexes, and the _ident tables, switching from UUID to +BIGSERIAL would half the size. + +## Exports + + time ./fatcat_export.py changelog - | pv -l | wc + + As of: + + 159k 1:17:35 [34.3 /s] + 159,740 lines + 2,427,277,881 chars (bytes; 2.4GB) + + real 77m35.183s + user 15m36.208s + sys 0m31.484s + +Running at about 100/sec; estimate 6 hours for completion. Could shard using +start/end flags, but am not here. + +Running `quick_dump.sql` (identifier tables, in a transaction): + + 251M Aug 19 23:08 fatcat_ident_creators.tsv + 5.9M Aug 19 23:08 fatcat_ident_containers.tsv + 467M Aug 19 23:08 fatcat_ident_files.tsv + 5.2G Aug 19 23:10 fatcat_ident_releases.tsv + 5.2G Aug 19 23:11 fatcat_ident_works.tsv + 12K Aug 19 23:11 . + 1.8G Aug 19 23:12 fatcat_abstracts.json + +Work and Release tables in under 2 minutes each; say 5 minutes total. + + time ./fatcat_export.py releases /tmp/fatcat_ident_releases.tsv - | pv -l | wc + + 172k 1:07:08 [42.7 /s] + 172181 lines + 1,118,166,293 chars (bytes; 1.1 GB) + + real 67m8.340s + user 10m21.988s + sys 0m34.612s + +Running at only 10/sec or so, this would take forever even if sharded. :( + +Both exports/dumps are running in parallel. "Expand" queries might help with speed? + +## Postgres Analysis + +SELECT * +FROM + pg_stat_statements +ORDER BY + total_time DESC LIMIT 5; + +Summary: + + SELECT "creator_ident" by ORCID + 1,295,864 calls + 930,305,208 total time + 717.9 mean time <= this should be less than a ms! + + INSERT INTO release_rev + 75144055 calls + 111470961 total time + 1.483 mean time + + INSERT INTO work_rev + 75,144,055 calls + 82693994 total time + 1.1 mean time + + INSERT INTO release_contrib (creator_ident_id = DEFAULT) RETURNING * + 26,008,280 calls <= why so few? different query depending on number + of rows inserted + 18955782 total time + 0.728 mean time + + SELECT container_ident + 78,4143 calls + 17683156 total time + 22.55 mean time <= why so slow? + + INSERT INTO release_contrib + 15,072,820 calls + + INSERT INTO "release_contrib + + + relname | too_much_seq | case | rel_size | seq_scan | idx_scan +----------------------+--------------+----------------+--------------+----------+----------- + file_rev_url | 2391 | Missing Index? | 2091147264 | 2391 | 0 + file_release | -30670 | OK | 386899968 | 2 | 30672 + container_rev | -979948 | OK | 20242432 | 784146 | 1764094 + file_edit | -2206807 | OK | 619896832 | 6 | 2206813 + creator_edit | -2206810 | OK | 331079680 | 11 | 2206821 + work_edit | -2206811 | OK | 6996566016 | 14 | 2206825 + release_edit | -2206811 | OK | 6996582400 | 14 | 2206825 + container_edit | -2206816 | OK | 8216576 | 5 | 2206821 + changelog | -2209659 | OK | 129286144 | 10 | 2209669 + abstracts | -3486466 | OK | 1706237952 | 8 | 3486474 + release_rev_abstract | -4975493 | OK | 191602688 | 42919 | 5018412 + release_ref | -5032717 | OK | 170494861312 | 3 | 5032720 + release_contrib | -5032744 | OK | 20370251776 | 3 | 5032747 + creator_rev | -8400410 | OK | 353583104 | 1296507 | 9696917 + file_ident | -13483224 | OK | 522190848 | 7 | 13483231 + creator_ident | -16686744 | OK | 293625856 | 3 | 16686747 + file_rev | -32405557 | OK | 868515840 | 4 | 32405561 + container_ident | -69162337 | OK | 7028736 | 3 | 69162340 + work_rev | -150288161 | OK | 3328589824 | 1 | 150288162 + editgroup | -162783807 | OK | 146112512 | 9 | 162783816 + release_ident | -165676917 | OK | 6076841984 | 52 | 165676969 + work_ident | -229439828 | OK | 6066814976 | 3 | 229439831 + release_rev | -930140217 | OK | 43360542720 | 9 | 930140226 + +TODO changes: +- don't return all as often; in particular, inserting release_contrib, release_ref +x missing an index somewhere on file_rev_url, release_rev_abstract +x why so many seq_scan on container_rev, creator_rev + => running/EXPLAIN same query on psql hits index, not seq_scan + => seemed to be an issue with VALUE params getting sent separately; query + planner only looked at query and wasn't using index on ORCID/ISSN-L because + it didn't know those values were not-NULL? + => adding NOT NULL to query seems to have sped up case of there being a + "hit", but no hit still slow. might need to change indices or something for + the (perhaps common in future) case of DOI lookups with invalid DOIs (eg, + CORE import) + +random DEBUG queries: + + EXPLAIN ANALYSE SELECT "creator_ident"."id", "creator_ident"."is_live", "creator_ident"."rev_id", "creator_ident"."redirect_id", "creator_rev"."id", "creator_rev"."extra_json", "creator_rev"."display_name", "creator_rev"."given_name", "creator_rev"."surname", "creator_rev"."orcid", "creator_rev"."wikidata_qid" FROM ("creator_ident" INNER JOIN "creator_rev" ON "creator_ident"."rev_id" = "creator_rev"."id") WHERE "creator_rev"."orcid" = '0000-0002-8867-1663' AND "creator_ident"."is_live" = true AND "creator_ident"."redirect_id" IS NULL LIMIT 1; + + EXPLAIN VERBOSE SELECT "creator_ident"."id", "creator_ident"."is_live", "creator_ident"."rev_id", "creator_ident"."redirect_id", "creator_rev"."id", "creator_rev"."extra_json", "creator_rev"."display_name", "creator_rev"."given_name", "creator_rev"."surname", "creator_rev"."orcid", "creator_rev"."wikidata_qid" FROM ("creator_ident" INNER JOIN "creator_rev" ON "creator_ident"."rev_id" = "creator_rev"."id") WHERE "creator_rev"."orcid" = $1 AND "creator_ident"."is_live" = true AND "creator_ident"."redirect_id" IS NULL VALUES ('0000-0002-8867-1669') LIMIT 1; + + EXPLAIN SELECT "container_ident"."id", "container_ident"."is_live", "container_ident"."rev_id", "container_ident"."redirect_id", "container_rev"."id", "container_rev"."extra_json", "container_rev"."name", "container_rev"."publisher", "container_rev"."issnl", "container_rev"."wikidata_qid", "container_rev"."abbrev", "container_rev"."coden" FROM ("container_ident" INNER JOIN "container_rev" ON "container_ident"."rev_id" = "container_rev"."id") WHERE "container_rev"."issnl" = '0001-0782' AND "container_ident"."is_live" = true AND "container_ident"."redirect_id" IS NULL LIMIT 1; + + SELECT "creator_ident"."id", "creator_ident"."is_live", "creator_ident"."rev_id", "creator_ident"."redirect_id", "creator_rev"."id", "creator_rev"."extra_json", "creator_rev"."display_name", "creator_rev"."given_name", "creator_rev"."surname", "creator_rev"."orcid", "creator_rev"."wikidata_qid" FROM ("creator_ident" INNER JOIN "creator_rev" ON "creator_ident"."rev_id" = "creator_rev"."id") WHERE "creator_rev"."orcid" = '0000-0002-8867-1663' AND "creator_ident"."is_live" = 't' AND "creator_ident"."redirect_id" IS NULL LIMIT 1; -- cgit v1.2.3 From 54fbdd96193f82adeb3d92095a6955656c67f5e3 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 20 Aug 2018 14:49:22 -0700 Subject: cost notes on hosting a fatcat mirror --- notes/cloud_instances.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 notes/cloud_instances.txt diff --git a/notes/cloud_instances.txt b/notes/cloud_instances.txt new file mode 100644 index 00000000..4582c431 --- /dev/null +++ b/notes/cloud_instances.txt @@ -0,0 +1,8 @@ + +digital ocean + 48 GB RAM, 12 cores, 960 GB $240/month + (or more) + +aws + i3.2xlarge 61 GB RAM, 8 cores, 1900 GB NVMe, $455/month + -- cgit v1.2.3 From 8528f06157b0e60842c860f81e3f2a69aa07aae9 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 24 Aug 2018 12:59:28 -0700 Subject: WIP: autoaccept --- TODO | 34 +++++----- fatcat-openapi2.yml | 25 +++++++ notes/cloud_instances.txt | 2 + rust/fatcat-api/README.md | 2 +- rust/fatcat-api/api.yaml | 25 +++++++ rust/fatcat-api/api/swagger.yaml | 40 ++++++++++++ rust/fatcat-api/examples/client.rs | 10 +-- rust/fatcat-api/examples/server_lib/server.rs | 48 +++++++++++--- rust/fatcat-api/src/client.rs | 39 ++++++++--- rust/fatcat-api/src/lib.rs | 93 +++++++++++++++++++++------ rust/fatcat-api/src/server.rs | 15 +++-- 11 files changed, 269 insertions(+), 64 deletions(-) diff --git a/TODO b/TODO index d5e10629..b7aa470a 100644 --- a/TODO +++ b/TODO @@ -1,38 +1,43 @@ ## Next Up -- some significant slow-down has happened? transactions, or regexes? +summer roadmap: +- PUT/UPDATE, DELETE, and merge code paths +- faster UPDATE-free bulk import code path +- container import (extra?): lang, region, subject +- basic API+webface creation, editing, merging, editgroup approval +- elastic schema/transform for releases; bulk and continuous scripts features: - fast database dump command: both changelog-based and entity-based (rust) => lighter, more complete dumps for each entity type? +- guide skeleton (mdbook; guide.fatcat.wiki) importers: +- CORE +- wikidata cross-ref (if they have a dump) - manifest: multiple URLs per SHA1 -- pubmed (medline) +- pubmed (medline), if not in CORE => and/or, use pubmed ID lookups on crossref import -- core - semantic scholar (up to 39 million; author de-dupe) -- wikidata (if they have a dump) bugs: - test: release pointing to a collection that has been deleted/redirected => UI crash? -july roadmap: -- complete and test this round of schema changes -- container import (extra?): lang, region, subject -- re-run imports -- basic API+webface creation, editing, merging, editgroup approval -- elastic schema/transform for releases; bulk and continuous scripts - ## Schema / Alignment / Scope - "container" -> "venue"? -- release_type, release_status, url.rel enums (and others?) +- release_type, release_status, url.rel write-time schema(and others?) name ref: https://www.w3.org/International/questions/qa-personal-names +## API + +- how to send edit "extra" metadata? +- hydrate entities in API + ? "expand" query param + ## High-Level Priorities - full database dump (export) @@ -50,11 +55,6 @@ name ref: https://www.w3.org/International/questions/qa-personal-names - batch inserts automerge: create editgroup and changelog, mark all edits as accepted, all in a single transaction -## API - -- hydrate entities in API - ? "expand" query param - ## Other - basic python hbase/elastic matcher diff --git a/fatcat-openapi2.yml b/fatcat-openapi2.yml index fda630dd..ea17e982 100644 --- a/fatcat-openapi2.yml +++ b/fatcat-openapi2.yml @@ -447,6 +447,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -540,6 +545,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -649,6 +659,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -742,6 +757,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -851,6 +871,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true diff --git a/notes/cloud_instances.txt b/notes/cloud_instances.txt index 4582c431..b7071758 100644 --- a/notes/cloud_instances.txt +++ b/notes/cloud_instances.txt @@ -6,3 +6,5 @@ digital ocean aws i3.2xlarge 61 GB RAM, 8 cores, 1900 GB NVMe, $455/month +OVH + MG-128 128 GB RAM, 16 cores, 2880 GB SSD (RAID), 500mbps unlimited b/w, $315/month diff --git a/rust/fatcat-api/README.md b/rust/fatcat-api/README.md index d0b266aa..7e4a2ec8 100644 --- a/rust/fatcat-api/README.md +++ b/rust/fatcat-api/README.md @@ -13,7 +13,7 @@ To see how to make this your own, look here: [README](https://github.com/swagger-api/swagger-codegen/blob/master/README.md) - API version: 0.1.0 -- Build date: 2018-08-20T08:47:01.260Z +- Build date: 2018-08-22T00:54:09.323Z This autogenerated project defines an API crate `fatcat` which contains: * An `Api` trait defining the API in Rust. diff --git a/rust/fatcat-api/api.yaml b/rust/fatcat-api/api.yaml index fda630dd..ea17e982 100644 --- a/rust/fatcat-api/api.yaml +++ b/rust/fatcat-api/api.yaml @@ -447,6 +447,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -540,6 +545,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -649,6 +659,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -742,6 +757,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true @@ -851,6 +871,11 @@ paths: type: boolean required: false description: "If true, and editor is authorized, batch is accepted all at once" + - name: editgroup + in: query + type: string + required: false + description: "Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True)" - name: entity_list in: body required: true diff --git a/rust/fatcat-api/api/swagger.yaml b/rust/fatcat-api/api/swagger.yaml index 11f789dc..3b8ed6e3 100644 --- a/rust/fatcat-api/api/swagger.yaml +++ b/rust/fatcat-api/api/swagger.yaml @@ -84,6 +84,14 @@ paths: type: "boolean" formatString: "{:?}" example: "Some(true)" + - name: "editgroup" + in: "query" + description: "Editgroup to auto-accept and apply to all entities (required\ + \ if 'autoaccept' is True)" + required: false + type: "string" + formatString: "{:?}" + example: "Some(\"editgroup_example\".to_string())" - in: "body" name: "entity_list" required: true @@ -387,6 +395,14 @@ paths: type: "boolean" formatString: "{:?}" example: "Some(true)" + - name: "editgroup" + in: "query" + description: "Editgroup to auto-accept and apply to all entities (required\ + \ if 'autoaccept' is True)" + required: false + type: "string" + formatString: "{:?}" + example: "Some(\"editgroup_example\".to_string())" - in: "body" name: "entity_list" required: true @@ -744,6 +760,14 @@ paths: type: "boolean" formatString: "{:?}" example: "Some(true)" + - name: "editgroup" + in: "query" + description: "Editgroup to auto-accept and apply to all entities (required\ + \ if 'autoaccept' is True)" + required: false + type: "string" + formatString: "{:?}" + example: "Some(\"editgroup_example\".to_string())" - in: "body" name: "entity_list" required: true @@ -1044,6 +1068,14 @@ paths: type: "boolean" formatString: "{:?}" example: "Some(true)" + - name: "editgroup" + in: "query" + description: "Editgroup to auto-accept and apply to all entities (required\ + \ if 'autoaccept' is True)" + required: false + type: "string" + formatString: "{:?}" + example: "Some(\"editgroup_example\".to_string())" - in: "body" name: "entity_list" required: true @@ -1398,6 +1430,14 @@ paths: type: "boolean" formatString: "{:?}" example: "Some(true)" + - name: "editgroup" + in: "query" + description: "Editgroup to auto-accept and apply to all entities (required\ + \ if 'autoaccept' is True)" + required: false + type: "string" + formatString: "{:?}" + example: "Some(\"editgroup_example\".to_string())" - in: "body" name: "entity_list" required: true diff --git a/rust/fatcat-api/examples/client.rs b/rust/fatcat-api/examples/client.rs index 06519232..34653196 100644 --- a/rust/fatcat-api/examples/client.rs +++ b/rust/fatcat-api/examples/client.rs @@ -95,7 +95,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateContainerBatch") => { - let result = client.create_container_batch(&Vec::new(), Some(true)).wait(); + let result = client.create_container_batch(&Vec::new(), Some(true), Some("editgroup_example".to_string())).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -105,7 +105,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateCreatorBatch") => { - let result = client.create_creator_batch(&Vec::new(), Some(true)).wait(); + let result = client.create_creator_batch(&Vec::new(), Some(true), Some("editgroup_example".to_string())).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -121,7 +121,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateFileBatch") => { - let result = client.create_file_batch(&Vec::new(), Some(true)).wait(); + let result = client.create_file_batch(&Vec::new(), Some(true), Some("editgroup_example".to_string())).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -131,7 +131,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateReleaseBatch") => { - let result = client.create_release_batch(&Vec::new(), Some(true)).wait(); + let result = client.create_release_batch(&Vec::new(), Some(true), Some("editgroup_example".to_string())).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } @@ -141,7 +141,7 @@ fn main() { // println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); // }, Some("CreateWorkBatch") => { - let result = client.create_work_batch(&Vec::new(), Some(true)).wait(); + let result = client.create_work_batch(&Vec::new(), Some(true), Some("editgroup_example".to_string())).wait(); println!("{:?} (X-Span-ID: {:?})", result, client.context().x_span_id.clone().unwrap_or(String::from(""))); } diff --git a/rust/fatcat-api/examples/server_lib/server.rs b/rust/fatcat-api/examples/server_lib/server.rs index 32c7e97f..60e19847 100644 --- a/rust/fatcat-api/examples/server_lib/server.rs +++ b/rust/fatcat-api/examples/server_lib/server.rs @@ -38,13 +38,15 @@ impl Api for Server { &self, entity_list: &Vec, autoaccept: Option, + editgroup: Option, context: &Context, ) -> Box + Send> { let context = context.clone(); println!( - "create_container_batch({:?}, {:?}) - X-Span-ID: {:?}", + "create_container_batch({:?}, {:?}, {:?}) - X-Span-ID: {:?}", entity_list, autoaccept, + editgroup, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) @@ -56,12 +58,19 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { + fn create_creator_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send> { let context = context.clone(); println!( - "create_creator_batch({:?}, {:?}) - X-Span-ID: {:?}", + "create_creator_batch({:?}, {:?}, {:?}) - X-Span-ID: {:?}", entity_list, autoaccept, + editgroup, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) @@ -79,12 +88,19 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { + fn create_file_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send> { let context = context.clone(); println!( - "create_file_batch({:?}, {:?}) - X-Span-ID: {:?}", + "create_file_batch({:?}, {:?}, {:?}) - X-Span-ID: {:?}", entity_list, autoaccept, + editgroup, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) @@ -96,12 +112,19 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { + fn create_release_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send> { let context = context.clone(); println!( - "create_release_batch({:?}, {:?}) - X-Span-ID: {:?}", + "create_release_batch({:?}, {:?}, {:?}) - X-Span-ID: {:?}", entity_list, autoaccept, + editgroup, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) @@ -113,12 +136,19 @@ impl Api for Server { Box::new(futures::failed("Generic failure".into())) } - fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send> { + fn create_work_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send> { let context = context.clone(); println!( - "create_work_batch({:?}, {:?}) - X-Span-ID: {:?}", + "create_work_batch({:?}, {:?}, {:?}) - X-Span-ID: {:?}", entity_list, autoaccept, + editgroup, context.x_span_id.unwrap_or(String::from("")).clone() ); Box::new(futures::failed("Generic failure".into())) diff --git a/rust/fatcat-api/src/client.rs b/rust/fatcat-api/src/client.rs index d71c9dab..628d8894 100644 --- a/rust/fatcat-api/src/client.rs +++ b/rust/fatcat-api/src/client.rs @@ -294,15 +294,18 @@ impl Api for Client { &self, param_entity_list: &Vec, param_autoaccept: Option, + param_editgroup: Option, context: &Context, ) -> Box + Send> { // Query parameters let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + let query_editgroup = param_editgroup.map_or_else(String::new, |query| format!("editgroup={editgroup}&", editgroup = query.to_string())); let url = format!( - "{}/v0/container/batch?{autoaccept}", + "{}/v0/container/batch?{autoaccept}{editgroup}", self.base_path, - autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET), + editgroup = utf8_percent_encode(&query_editgroup, QUERY_ENCODE_SET) ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -436,15 +439,18 @@ impl Api for Client { &self, param_entity_list: &Vec, param_autoaccept: Option, + param_editgroup: Option, context: &Context, ) -> Box + Send> { // Query parameters let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + let query_editgroup = param_editgroup.map_or_else(String::new, |query| format!("editgroup={editgroup}&", editgroup = query.to_string())); let url = format!( - "{}/v0/creator/batch?{autoaccept}", + "{}/v0/creator/batch?{autoaccept}{editgroup}", self.base_path, - autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET), + editgroup = utf8_percent_encode(&query_editgroup, QUERY_ENCODE_SET) ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -636,12 +642,19 @@ impl Api for Client { &self, param_entity_list: &Vec, param_autoaccept: Option, + param_editgroup: Option, context: &Context, ) -> Box + Send> { // Query parameters let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + let query_editgroup = param_editgroup.map_or_else(String::new, |query| format!("editgroup={editgroup}&", editgroup = query.to_string())); - let url = format!("{}/v0/file/batch?{autoaccept}", self.base_path, autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET)); + let url = format!( + "{}/v0/file/batch?{autoaccept}{editgroup}", + self.base_path, + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET), + editgroup = utf8_percent_encode(&query_editgroup, QUERY_ENCODE_SET) + ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -774,15 +787,18 @@ impl Api for Client { &self, param_entity_list: &Vec, param_autoaccept: Option, + param_editgroup: Option, context: &Context, ) -> Box + Send> { // Query parameters let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + let query_editgroup = param_editgroup.map_or_else(String::new, |query| format!("editgroup={editgroup}&", editgroup = query.to_string())); let url = format!( - "{}/v0/release/batch?{autoaccept}", + "{}/v0/release/batch?{autoaccept}{editgroup}", self.base_path, - autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET) + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET), + editgroup = utf8_percent_encode(&query_editgroup, QUERY_ENCODE_SET) ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); @@ -916,12 +932,19 @@ impl Api for Client { &self, param_entity_list: &Vec, param_autoaccept: Option, + param_editgroup: Option, context: &Context, ) -> Box + Send> { // Query parameters let query_autoaccept = param_autoaccept.map_or_else(String::new, |query| format!("autoaccept={autoaccept}&", autoaccept = query.to_string())); + let query_editgroup = param_editgroup.map_or_else(String::new, |query| format!("editgroup={editgroup}&", editgroup = query.to_string())); - let url = format!("{}/v0/work/batch?{autoaccept}", self.base_path, autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET)); + let url = format!( + "{}/v0/work/batch?{autoaccept}{editgroup}", + self.base_path, + autoaccept = utf8_percent_encode(&query_autoaccept, QUERY_ENCODE_SET), + editgroup = utf8_percent_encode(&query_editgroup, QUERY_ENCODE_SET) + ); let body = serde_json::to_string(¶m_entity_list).expect("impossible to fail to serialize"); diff --git a/rust/fatcat-api/src/lib.rs b/rust/fatcat-api/src/lib.rs index 044b934b..5de3647b 100644 --- a/rust/fatcat-api/src/lib.rs +++ b/rust/fatcat-api/src/lib.rs @@ -446,26 +446,51 @@ pub trait Api { &self, entity_list: &Vec, autoaccept: Option, + editgroup: Option, context: &Context, ) -> Box + Send>; fn create_creator(&self, entity: models::CreatorEntity, context: &Context) -> Box + Send>; - fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; + fn create_creator_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send>; fn create_editgroup(&self, entity: models::Editgroup, context: &Context) -> Box + Send>; fn create_file(&self, entity: models::FileEntity, context: &Context) -> Box + Send>; - fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; + fn create_file_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send>; fn create_release(&self, entity: models::ReleaseEntity, context: &Context) -> Box + Send>; - fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; + fn create_release_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send>; fn create_work(&self, entity: models::WorkEntity, context: &Context) -> Box + Send>; - fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, context: &Context) -> Box + Send>; + fn create_work_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + context: &Context, + ) -> Box + Send>; fn get_changelog(&self, limit: Option, context: &Context) -> Box + Send>; @@ -520,25 +545,40 @@ pub trait ApiNoContext { fn create_container(&self, entity: models::ContainerEntity) -> Box + Send>; - fn create_container_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; + fn create_container_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send>; fn create_creator(&self, entity: models::CreatorEntity) -> Box + Send>; - fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; + fn create_creator_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send>; fn create_editgroup(&self, entity: models::Editgroup) -> Box + Send>; fn create_file(&self, entity: models::FileEntity) -> Box + Send>; - fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, editgroup: Option) -> Box + Send>; fn create_release(&self, entity: models::ReleaseEntity) -> Box + Send>; - fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; + fn create_release_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send>; fn create_work(&self, entity: models::WorkEntity) -> Box + Send>; - fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send>; + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, editgroup: Option) -> Box + Send>; fn get_changelog(&self, limit: Option) -> Box + Send>; @@ -611,16 +651,26 @@ impl<'a, T: Api> ApiNoContext for ContextWrapper<'a, T> { self.api().create_container(entity, &self.context()) } - fn create_container_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { - self.api().create_container_batch(entity_list, autoaccept, &self.context()) + fn create_container_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send> { + self.api().create_container_batch(entity_list, autoaccept, editgroup, &self.context()) } fn create_creator(&self, entity: models::CreatorEntity) -> Box + Send> { self.api().create_creator(entity, &self.context()) } - fn create_creator_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { - self.api().create_creator_batch(entity_list, autoaccept, &self.context()) + fn create_creator_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send> { + self.api().create_creator_batch(entity_list, autoaccept, editgroup, &self.context()) } fn create_editgroup(&self, entity: models::Editgroup) -> Box + Send> { @@ -631,24 +681,29 @@ impl<'a, T: Api> ApiNoContext for ContextWrapper<'a, T> { self.api().create_file(entity, &self.context()) } - fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { - self.api().create_file_batch(entity_list, autoaccept, &self.context()) + fn create_file_batch(&self, entity_list: &Vec, autoaccept: Option, editgroup: Option) -> Box + Send> { + self.api().create_file_batch(entity_list, autoaccept, editgroup, &self.context()) } fn create_release(&self, entity: models::ReleaseEntity) -> Box + Send> { self.api().create_release(entity, &self.context()) } - fn create_release_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { - self.api().create_release_batch(entity_list, autoaccept, &self.context()) + fn create_release_batch( + &self, + entity_list: &Vec, + autoaccept: Option, + editgroup: Option, + ) -> Box + Send> { + self.api().create_release_batch(entity_list, autoaccept, editgroup, &self.context()) } fn create_work(&self, entity: models::WorkEntity) -> Box + Send> { self.api().create_work(entity, &self.context()) } - fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option) -> Box + Send> { - self.api().create_work_batch(entity_list, autoaccept, &self.context()) + fn create_work_batch(&self, entity_list: &Vec, autoaccept: Option, editgroup: Option) -> Box + Send> { + self.api().create_work_batch(entity_list, autoaccept, editgroup, &self.context()) } fn get_changelog(&self, limit: Option) -> Box + Send> { diff --git a/rust/fatcat-api/src/server.rs b/rust/fatcat-api/src/server.rs index 4e41b5e9..1ba9a218 100644 --- a/rust/fatcat-api/src/server.rs +++ b/rust/fatcat-api/src/server.rs @@ -301,6 +301,7 @@ where // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) let query_params = req.get::().unwrap_or_default(); let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + let param_editgroup = query_params.get("editgroup").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for @@ -326,7 +327,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_container_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { + match api.create_container_batch(param_entity_list.as_ref(), param_autoaccept, param_editgroup, context).wait() { Ok(rsp) => match rsp { CreateContainerBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -517,6 +518,7 @@ where // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) let query_params = req.get::().unwrap_or_default(); let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + let param_editgroup = query_params.get("editgroup").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for @@ -542,7 +544,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_creator_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { + match api.create_creator_batch(param_entity_list.as_ref(), param_autoaccept, param_editgroup, context).wait() { Ok(rsp) => match rsp { CreateCreatorBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -826,6 +828,7 @@ where // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) let query_params = req.get::().unwrap_or_default(); let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + let param_editgroup = query_params.get("editgroup").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for @@ -851,7 +854,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_file_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { + match api.create_file_batch(param_entity_list.as_ref(), param_autoaccept, param_editgroup, context).wait() { Ok(rsp) => match rsp { CreateFileBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -1042,6 +1045,7 @@ where // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) let query_params = req.get::().unwrap_or_default(); let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + let param_editgroup = query_params.get("editgroup").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for @@ -1067,7 +1071,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_release_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { + match api.create_release_batch(param_entity_list.as_ref(), param_autoaccept, param_editgroup, context).wait() { Ok(rsp) => match rsp { CreateReleaseBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); @@ -1258,6 +1262,7 @@ where // Query parameters (note that non-required or collection query parameters will ignore garbage values, rather than causing a 400 response) let query_params = req.get::().unwrap_or_default(); let param_autoaccept = query_params.get("autoaccept").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); + let param_editgroup = query_params.get("editgroup").and_then(|list| list.first()).and_then(|x| x.parse::().ok()); // Body parameters (note that non-required body parameters will ignore garbage // values, rather than causing a 400 response). Produce warning header and logs for @@ -1283,7 +1288,7 @@ where }; let param_entity_list = param_entity_list.ok_or_else(|| Response::with((status::BadRequest, "Missing required body parameter entity_list".to_string())))?; - match api.create_work_batch(param_entity_list.as_ref(), param_autoaccept, context).wait() { + match api.create_work_batch(param_entity_list.as_ref(), param_autoaccept, param_editgroup, context).wait() { Ok(rsp) => match rsp { CreateWorkBatchResponse::CreatedEntities(body) => { let body_string = serde_json::to_string(&body).expect("impossible to fail to serialize"); -- cgit v1.2.3 From cc11d5000910b5fd53e40bfd01056c0db60c35aa Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 16:52:03 -0700 Subject: editgroup parameter for batch inserts --- rust/src/api_server.rs | 24 +++++++++++++++++++----- rust/src/api_wrappers.rs | 3 ++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/rust/src/api_server.rs b/rust/src/api_server.rs index 0cb07e81..e8e7d062 100644 --- a/rust/src/api_server.rs +++ b/rust/src/api_server.rs @@ -26,21 +26,35 @@ macro_rules! entity_batch_handler { &self, entity_list: &[models::$model], autoaccept: bool, + editgroup: Option, conn: &DbConn, ) -> Result> { let mut ret: Vec = vec![]; let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth - let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) - .values((editgroup::editor_id.eq(editor_id),)) - .get_result(conn)?; + // editgroup override logic based on parameters + let eg_id: Option = match (editgroup, autoaccept) { + (Some(eg_string), _) => Some(Uuid::parse_str(&eg_string)?), + (None, true) => { + let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) + .values((editgroup::editor_id.eq(editor_id),)) + .get_result(conn)?; + Some(eg_row.id) + }, + (None, false) => None + }; for entity in entity_list { let mut e = entity.clone(); - e.editgroup_id = Some(uuid2fcid(&eg_row.id)); + // override individual editgroup IDs (if set earlier) + if let Some(inner_id) = eg_id { + e.editgroup_id = Some(uuid2fcid(&inner_id)); + } + // actual wrapped function call here ret.push(self.$post_handler(e, autoaccept, conn)?); } if autoaccept { + // if autoaccept, eg_id is always Some let _clr: ChangelogRow = diesel::insert_into(changelog::table) - .values((changelog::editgroup_id.eq(eg_row.id),)) + .values((changelog::editgroup_id.eq(eg_id.unwrap()),)) .get_result(conn)?; } Ok(ret) diff --git a/rust/src/api_wrappers.rs b/rust/src/api_wrappers.rs index 4f126385..da6139d2 100644 --- a/rust/src/api_wrappers.rs +++ b/rust/src/api_wrappers.rs @@ -84,10 +84,11 @@ macro_rules! wrap_entity_handlers { &self, entity_list: &Vec, autoaccept: Option, + editgroup: Option, _context: &Context, ) -> Box + Send> { let conn = self.db_pool.get().expect("db_pool error"); - let ret = match conn.transaction(|| self.$post_batch_handler(entity_list, autoaccept.unwrap_or(false), &conn)) { + let ret = match conn.transaction(|| self.$post_batch_handler(entity_list, autoaccept.unwrap_or(false), editgroup, &conn)) { Ok(edit) => $post_batch_resp::CreatedEntities(edit), Err(Error(ErrorKind::Diesel(e), _)) => -- cgit v1.2.3 From 43cb2cdcd3dae5e054f093120940b9aa1e09718a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 19:21:08 -0700 Subject: minimal autoaccept tests --- rust/tests/test_api_server.rs | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/rust/tests/test_api_server.rs b/rust/tests/test_api_server.rs index 02c77413..340fb996 100644 --- a/rust/tests/test_api_server.rs +++ b/rust/tests/test_api_server.rs @@ -918,3 +918,44 @@ fn test_contribs() { None, ); } + +#[test] +fn test_post_batch_autoaccept() { + let (headers, router, _conn) = setup(); + + // "true" + check_response( + request::post( + "http://localhost:9411/v0/container/batch?autoaccept=true", + headers.clone(), + r#"[{"name": "test journal"}, {"name": "another test journal"}]"#, + &router, + ), + status::Created, + None, + ); + + // "n" + check_response( + request::post( + "http://localhost:9411/v0/container/batch?autoaccept=n", + headers.clone(), + r#"[{"name": "test journal"}, {"name": "another test journal"}]"#, + &router, + ), + status::Created, + None, + ); + + // editgroup + check_response( + request::post( + "http://localhost:9411/v0/container/batch?autoaccept=yes&editgroup=asdf", + headers.clone(), + r#"[{"name": "test journal"}, {"name": "another test journal"}]"#, + &router, + ), + status::BadRequest, + None, + ); +} -- cgit v1.2.3 From d046bee7a76576cf14a2815b787247fca8c44433 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 19:36:33 -0700 Subject: editgroup IDs are fcid, not uuid --- rust/src/api_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/api_server.rs b/rust/src/api_server.rs index e8e7d062..88fd7063 100644 --- a/rust/src/api_server.rs +++ b/rust/src/api_server.rs @@ -33,7 +33,7 @@ macro_rules! entity_batch_handler { let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001")?; // TODO: auth // editgroup override logic based on parameters let eg_id: Option = match (editgroup, autoaccept) { - (Some(eg_string), _) => Some(Uuid::parse_str(&eg_string)?), + (Some(eg_string), _) => Some(fcid2uuid(&eg_string)?), (None, true) => { let eg_row: EditgroupRow = diesel::insert_into(editgroup::table) .values((editgroup::editor_id.eq(editor_id),)) -- cgit v1.2.3 From 1b10d844f20df008fa9848d4ee83e294ed9c6523 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 19:37:04 -0700 Subject: importers: create editgroup and pass --- python/fatcat/crossref_importer.py | 2 +- python/fatcat/importer_common.py | 4 +++- python/fatcat/issn_importer.py | 2 +- python/fatcat/orcid_importer.py | 2 +- python/fatcat_client/api/default_api.py | 30 +++++++++++++++++++++++++----- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/python/fatcat/crossref_importer.py b/python/fatcat/crossref_importer.py index b12851dc..7540cf61 100644 --- a/python/fatcat/crossref_importer.py +++ b/python/fatcat/crossref_importer.py @@ -152,4 +152,4 @@ class FatcatCrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch, autoaccept=True) + self.api.create_release_batch(release_batch, autoaccept=True, editgroup=editgroup_id) diff --git a/python/fatcat/importer_common.py b/python/fatcat/importer_common.py index c1566e65..0b02d175 100644 --- a/python/fatcat/importer_common.py +++ b/python/fatcat/importer_common.py @@ -43,7 +43,9 @@ class FatcatImporter: def process_batch(self, source, size=50): """Reads and processes in batches (not API-call-per-)""" for rows in grouper(source, size): - self.create_batch(rows) + eg = self.api.create_editgroup( + fatcat_client.Editgroup(editor_id='aaaaaaaaaaaabkvkaaaaaaaaae')) + self.create_batch(rows, editgroup_id=eg.id) def process_csv_source(self, source, group_size=100, delimiter=','): reader = csv.DictReader(source, delimiter=delimiter) diff --git a/python/fatcat/issn_importer.py b/python/fatcat/issn_importer.py index 6b806b40..2cb954e9 100644 --- a/python/fatcat/issn_importer.py +++ b/python/fatcat/issn_importer.py @@ -70,4 +70,4 @@ class FatcatIssnImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_container_batch(objects, autoaccept=True) + self.api.create_container_batch(objects, autoaccept=True, editgroup=editgroup_id) diff --git a/python/fatcat/orcid_importer.py b/python/fatcat/orcid_importer.py index cc12d50d..0b9860eb 100644 --- a/python/fatcat/orcid_importer.py +++ b/python/fatcat/orcid_importer.py @@ -71,4 +71,4 @@ class FatcatOrcidImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_creator_batch(objects, autoaccept=True) + self.api.create_creator_batch(objects, autoaccept=True, editgroup=editgroup_id) diff --git a/python/fatcat_client/api/default_api.py b/python/fatcat_client/api/default_api.py index 914f747c..a0298750 100644 --- a/python/fatcat_client/api/default_api.py +++ b/python/fatcat_client/api/default_api.py @@ -246,6 +246,7 @@ class DefaultApi(object): :param async bool :param list[ContainerEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -268,12 +269,13 @@ class DefaultApi(object): :param async bool :param list[ContainerEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list', 'autoaccept'] # noqa: E501 + all_params = ['entity_list', 'autoaccept', 'editgroup'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -300,6 +302,8 @@ class DefaultApi(object): query_params = [] if 'autoaccept' in params: query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 + if 'editgroup' in params: + query_params.append(('editgroup', params['editgroup'])) # noqa: E501 header_params = {} @@ -444,6 +448,7 @@ class DefaultApi(object): :param async bool :param list[CreatorEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -466,12 +471,13 @@ class DefaultApi(object): :param async bool :param list[CreatorEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list', 'autoaccept'] # noqa: E501 + all_params = ['entity_list', 'autoaccept', 'editgroup'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -498,6 +504,8 @@ class DefaultApi(object): query_params = [] if 'autoaccept' in params: query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 + if 'editgroup' in params: + query_params.append(('editgroup', params['editgroup'])) # noqa: E501 header_params = {} @@ -739,6 +747,7 @@ class DefaultApi(object): :param async bool :param list[FileEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -761,12 +770,13 @@ class DefaultApi(object): :param async bool :param list[FileEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list', 'autoaccept'] # noqa: E501 + all_params = ['entity_list', 'autoaccept', 'editgroup'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -793,6 +803,8 @@ class DefaultApi(object): query_params = [] if 'autoaccept' in params: query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 + if 'editgroup' in params: + query_params.append(('editgroup', params['editgroup'])) # noqa: E501 header_params = {} @@ -937,6 +949,7 @@ class DefaultApi(object): :param async bool :param list[ReleaseEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -959,12 +972,13 @@ class DefaultApi(object): :param async bool :param list[ReleaseEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list', 'autoaccept'] # noqa: E501 + all_params = ['entity_list', 'autoaccept', 'editgroup'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -991,6 +1005,8 @@ class DefaultApi(object): query_params = [] if 'autoaccept' in params: query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 + if 'editgroup' in params: + query_params.append(('editgroup', params['editgroup'])) # noqa: E501 header_params = {} @@ -1135,6 +1151,7 @@ class DefaultApi(object): :param async bool :param list[WorkEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. @@ -1157,12 +1174,13 @@ class DefaultApi(object): :param async bool :param list[WorkEntity] entity_list: (required) :param bool autoaccept: If true, and editor is authorized, batch is accepted all at once + :param str editgroup: Editgroup to auto-accept and apply to all entities (required if 'autoaccept' is True) :return: list[EntityEdit] If the method is called asynchronously, returns the request thread. """ - all_params = ['entity_list', 'autoaccept'] # noqa: E501 + all_params = ['entity_list', 'autoaccept', 'editgroup'] # noqa: E501 all_params.append('async') all_params.append('_return_http_data_only') all_params.append('_preload_content') @@ -1189,6 +1207,8 @@ class DefaultApi(object): query_params = [] if 'autoaccept' in params: query_params.append(('autoaccept', params['autoaccept'])) # noqa: E501 + if 'editgroup' in params: + query_params.append(('editgroup', params['editgroup'])) # noqa: E501 header_params = {} -- cgit v1.2.3 From 6006c7406716ae15d0101a8980a2ae955de71690 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 19:59:38 -0700 Subject: "true" is the only truthy value (oh no) --- python/fatcat/crossref_importer.py | 2 +- python/fatcat/issn_importer.py | 2 +- python/fatcat/orcid_importer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/fatcat/crossref_importer.py b/python/fatcat/crossref_importer.py index 7540cf61..54a3e84f 100644 --- a/python/fatcat/crossref_importer.py +++ b/python/fatcat/crossref_importer.py @@ -152,4 +152,4 @@ class FatcatCrossrefImporter(FatcatImporter): re.container_id = container.ident self._issnl_id_map[ce.issnl] = container.ident release_batch.append(re) - self.api.create_release_batch(release_batch, autoaccept=True, editgroup=editgroup_id) + self.api.create_release_batch(release_batch, autoaccept="true", editgroup=editgroup_id) diff --git a/python/fatcat/issn_importer.py b/python/fatcat/issn_importer.py index 2cb954e9..eb8a50ba 100644 --- a/python/fatcat/issn_importer.py +++ b/python/fatcat/issn_importer.py @@ -70,4 +70,4 @@ class FatcatIssnImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_container_batch(objects, autoaccept=True, editgroup=editgroup_id) + self.api.create_container_batch(objects, autoaccept="true", editgroup=editgroup_id) diff --git a/python/fatcat/orcid_importer.py b/python/fatcat/orcid_importer.py index 0b9860eb..fe76b02c 100644 --- a/python/fatcat/orcid_importer.py +++ b/python/fatcat/orcid_importer.py @@ -71,4 +71,4 @@ class FatcatOrcidImporter(FatcatImporter): objects = [o for o in objects if o != None] for o in objects: o.editgroup_id = editgroup_id - self.api.create_creator_batch(objects, autoaccept=True, editgroup=editgroup_id) + self.api.create_creator_batch(objects, autoaccept="true", editgroup=editgroup_id) -- cgit v1.2.3 From 8cccbcdef11e7ddc761ec494cb894a8d49a0d510 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 30 Aug 2018 20:02:26 -0700 Subject: autoaccept notes --- notes/autoaccept_api.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 notes/autoaccept_api.txt diff --git a/notes/autoaccept_api.txt b/notes/autoaccept_api.txt new file mode 100644 index 00000000..b7e0a824 --- /dev/null +++ b/notes/autoaccept_api.txt @@ -0,0 +1,31 @@ + +Currently only on batch creation (POST) for entities. + +For all bulk operations, optional 'editgroup' query parameter overrides +individual editgroup parameters. + +If autoaccept flag is set and editgroup is not, a new editgroup is +automatically created and overrides for all entities inserted. Note +that this is different behavior from the "use current or create new" +default behavior for regular creation. + +Unfortunately, "true" and "false" are the only values acceptable for boolean +rust/openapi2 query parameters + +THOUGHT: doing an UPDATE in a transaction is probably not expensive + +Intent: +- check can_autoaccept flag on editor table + +--------- + +Crude benchmarking... + +cat /data/crossref/crossref-works.2018-01-21.badsample_5k.json | time ./fatcat_import.py import-crossref - /data/issn/20180216.ISSN-to-ISSN-L.txt + +autoaccept: 7.47user 0.48system 0:30.64elapsed 25%CPU +master: 5.70user 0.34system 0:25.61elapsed 23%CPU + batch creation: ~153ms+ + accept: ~5ms + +uh... -- cgit v1.2.3