From 6d9bab59919352bbdd5dd354c89c95d78a8f04a2 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 26 Dec 2018 22:50:38 -0800 Subject: rust impl last webcapture schema tweaks --- rust/fatcat-api-spec/README.md | 2 +- rust/fatcat-api-spec/api.yaml | 6 ++---- rust/fatcat-api-spec/api/swagger.yaml | 20 +++++++++----------- rust/fatcat-api-spec/src/models.rs | 9 +++++---- rust/src/api_entity_crud.rs | 2 +- rust/src/bin/fatcat-export.rs | 12 ++++++------ rust/src/database_models.rs | 8 ++++---- rust/src/database_schema.rs | 4 ++-- rust/tests/test_api_server_http.rs | 2 +- 9 files changed, 31 insertions(+), 34 deletions(-) diff --git a/rust/fatcat-api-spec/README.md b/rust/fatcat-api-spec/README.md index bf7d63e6..babde90a 100644 --- a/rust/fatcat-api-spec/README.md +++ b/rust/fatcat-api-spec/README.md @@ -13,7 +13,7 @@ To see how to make this your own, look here: [README](https://github.com/swagger-api/swagger-codegen/blob/master/README.md) - API version: 0.1.0 -- Build date: 2018-12-26T23:15:13.714Z +- Build date: 2018-12-27T06:46:04.412Z This autogenerated project defines an API crate `fatcat` which contains: * An `Api` trait defining the API in Rust. diff --git a/rust/fatcat-api-spec/api.yaml b/rust/fatcat-api-spec/api.yaml index 3eada7e4..eb0528d5 100644 --- a/rust/fatcat-api-spec/api.yaml +++ b/rust/fatcat-api-spec/api.yaml @@ -238,16 +238,14 @@ definitions: - surt - timestamp - url - - status_code - sha1 properties: surt: type: string example: "org,asheesh)/apus/ch1/node15.html" timestamp: - type: integer - example: 20020429162520 - format: int64 + type: string + example: "20020429162520" url: type: string example: "http://www.asheesh.org:80/APUS/ch1/node15.html" diff --git a/rust/fatcat-api-spec/api/swagger.yaml b/rust/fatcat-api-spec/api/swagger.yaml index 4299efb4..908ff1ae 100644 --- a/rust/fatcat-api-spec/api/swagger.yaml +++ b/rust/fatcat-api-spec/api/swagger.yaml @@ -5928,14 +5928,14 @@ definitions: sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" - sha1: "3f242a192acc258bdfdb151943419437f440c313" surt: "org,asheesh)/apus/ch1/node15.html" status_code: 200 sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" ident: "q3nouwy3nnbsvo3h5klxsx4a7y" extra: "{}" state: "wip" @@ -6087,14 +6087,14 @@ definitions: sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" - sha1: "3f242a192acc258bdfdb151943419437f440c313" surt: "org,asheesh)/apus/ch1/node15.html" status_code: 200 sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" ident: "q3nouwy3nnbsvo3h5klxsx4a7y" extra: "{}" state: "wip" @@ -6118,14 +6118,14 @@ definitions: sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" - sha1: "3f242a192acc258bdfdb151943419437f440c313" surt: "org,asheesh)/apus/ch1/node15.html" status_code: 200 sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" ident: "q3nouwy3nnbsvo3h5klxsx4a7y" extra: "{}" state: "wip" @@ -7064,7 +7064,6 @@ definitions: webcapture_entity_cdx: required: - "sha1" - - "status_code" - "surt" - "timestamp" - "url" @@ -7073,9 +7072,8 @@ definitions: type: "string" example: "org,asheesh)/apus/ch1/node15.html" timestamp: - type: "integer" - format: "int64" - example: 20020429162520 + type: "string" + example: "20020429162520" url: type: "string" example: "http://www.asheesh.org:80/APUS/ch1/node15.html" @@ -7099,7 +7097,7 @@ definitions: sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832" mimetype: "text/html" url: "http://www.asheesh.org:80/APUS/ch1/node15.html" - timestamp: 20020429162520 + timestamp: "20020429162520" upperCaseName: "WEBCAPTURE_ENTITY_CDX" release_entity_abstracts: properties: diff --git a/rust/fatcat-api-spec/src/models.rs b/rust/fatcat-api-spec/src/models.rs index c6fc3ae1..a7c1375f 100644 --- a/rust/fatcat-api-spec/src/models.rs +++ b/rust/fatcat-api-spec/src/models.rs @@ -943,7 +943,7 @@ pub struct WebcaptureEntityCdx { pub surt: String, #[serde(rename = "timestamp")] - pub timestamp: i64, + pub timestamp: String, #[serde(rename = "url")] pub url: String, @@ -953,7 +953,8 @@ pub struct WebcaptureEntityCdx { pub mimetype: Option, #[serde(rename = "status_code")] - pub status_code: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub status_code: Option, #[serde(rename = "sha1")] pub sha1: String, @@ -964,13 +965,13 @@ pub struct WebcaptureEntityCdx { } impl WebcaptureEntityCdx { - pub fn new(surt: String, timestamp: i64, url: String, status_code: i64, sha1: String) -> WebcaptureEntityCdx { + pub fn new(surt: String, timestamp: String, url: String, sha1: String) -> WebcaptureEntityCdx { WebcaptureEntityCdx { surt: surt, timestamp: timestamp, url: url, mimetype: None, - status_code: status_code, + status_code: None, sha1: sha1, sha256: None, } diff --git a/rust/src/api_entity_crud.rs b/rust/src/api_entity_crud.rs index 19817540..cab6f58f 100644 --- a/rust/src/api_entity_crud.rs +++ b/rust/src/api_entity_crud.rs @@ -1334,7 +1334,7 @@ impl EntityCrud for WebcaptureEntity { .map(|c| WebcaptureRevCdxNewRow { webcapture_rev: *rev_id, surt: c.surt.clone(), - timestamp: c.timestamp, + timestamp: c.timestamp.clone(), url: c.url.clone(), mimetype: c.mimetype.clone(), status_code: c.status_code, diff --git a/rust/src/bin/fatcat-export.rs b/rust/src/bin/fatcat-export.rs index 8b7c4442..ec66ed4c 100644 --- a/rust/src/bin/fatcat-export.rs +++ b/rust/src/bin/fatcat-export.rs @@ -207,12 +207,12 @@ pub fn do_export( ExportEntityType::File => { thread::spawn(move || loop_work_file(row_receiver, output_sender, &db_conn, expand)) } - ExportEntityType::Fileset => { - thread::spawn(move || loop_work_fileset(row_receiver, output_sender, &db_conn, expand)) - } - ExportEntityType::Webcapture=> { - thread::spawn(move || loop_work_webcapture(row_receiver, output_sender, &db_conn, expand)) - } + ExportEntityType::Fileset => thread::spawn(move || { + loop_work_fileset(row_receiver, output_sender, &db_conn, expand) + }), + ExportEntityType::Webcapture => thread::spawn(move || { + loop_work_webcapture(row_receiver, output_sender, &db_conn, expand) + }), ExportEntityType::Release => thread::spawn(move || { loop_work_release(row_receiver, output_sender, &db_conn, expand) }), diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs index c913b98e..d2f6c3c2 100644 --- a/rust/src/database_models.rs +++ b/rust/src/database_models.rs @@ -305,10 +305,10 @@ pub struct WebcaptureRevCdxRow { pub id: i64, pub webcapture_rev: Uuid, pub surt: String, - pub timestamp: i64, + pub timestamp: String, pub url: String, pub mimetype: Option, - pub status_code: i64, + pub status_code: Option, pub sha1: String, pub sha256: Option, } @@ -318,10 +318,10 @@ pub struct WebcaptureRevCdxRow { pub struct WebcaptureRevCdxNewRow { pub webcapture_rev: Uuid, pub surt: String, - pub timestamp: i64, + pub timestamp: String, pub url: String, pub mimetype: Option, - pub status_code: i64, + pub status_code: Option, pub sha1: String, pub sha256: Option, } diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs index 436f2989..5db14de5 100644 --- a/rust/src/database_schema.rs +++ b/rust/src/database_schema.rs @@ -330,10 +330,10 @@ table! { id -> Int8, webcapture_rev -> Uuid, surt -> Text, - timestamp -> Int8, + timestamp -> Text, url -> Text, mimetype -> Nullable, - status_code -> Int8, + status_code -> Nullable, sha1 -> Text, sha256 -> Nullable, } diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs index 443c32ee..2160a0a0 100644 --- a/rust/tests/test_api_server_http.rs +++ b/rust/tests/test_api_server_http.rs @@ -685,7 +685,7 @@ fn test_post_webcapture() { "timestamp": "2018-12-28T05:06:07Z", "cdx": [ {"surt": "org,asheesh,)/robots.txt", - "timestamp": 20181228050607, + "timestamp": "20181228050607", "url": "https://asheesh.org/robots.txt", "status_code": 200, "mimetype": "text/html", -- cgit v1.2.3