rust impl last webcapture schema tweaks

9 files changed, 31 insertions, 34 deletions

diff --git a/rust/fatcat-api-spec/README.md b/rust/fatcat-api-spec/README.md
index bf7d63e6..babde90a 100644
--- a/rust/fatcat-api-spec/README.md
+++ b/rust/fatcat-api-spec/README.md
@@ -13,7 +13,7 @@ To see how to make this your own, look here:
 [README](https://github.com/swagger-api/swagger-codegen/blob/master/README.md)
 
 - API version: 0.1.0
-- Build date: 2018-12-26T23:15:13.714Z
+- Build date: 2018-12-27T06:46:04.412Z
 
 This autogenerated project defines an API crate `fatcat` which contains:
 * An `Api` trait defining the API in Rust.
diff --git a/rust/fatcat-api-spec/api.yaml b/rust/fatcat-api-spec/api.yaml
index 3eada7e4..eb0528d5 100644
--- a/rust/fatcat-api-spec/api.yaml
+++ b/rust/fatcat-api-spec/api.yaml
@@ -238,16 +238,14 @@ definitions:
             - surt
             - timestamp
             - url
-            - status_code
             - sha1
           properties:
             surt:
               type: string
               example: "org,asheesh)/apus/ch1/node15.html"
             timestamp:
-              type: integer
-              example: 20020429162520
-              format: int64
+              type: string
+              example: "20020429162520"
             url:
               type: string
               example: "http://www.asheesh.org:80/APUS/ch1/node15.html"
diff --git a/rust/fatcat-api-spec/api/swagger.yaml b/rust/fatcat-api-spec/api/swagger.yaml
index 4299efb4..908ff1ae 100644
--- a/rust/fatcat-api-spec/api/swagger.yaml
+++ b/rust/fatcat-api-spec/api/swagger.yaml
@@ -5928,14 +5928,14 @@ definitions:
         sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
         mimetype: "text/html"
         url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-        timestamp: 20020429162520
+        timestamp: "20020429162520"
       - sha1: "3f242a192acc258bdfdb151943419437f440c313"
         surt: "org,asheesh)/apus/ch1/node15.html"
         status_code: 200
         sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
         mimetype: "text/html"
         url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-        timestamp: 20020429162520
+        timestamp: "20020429162520"
       ident: "q3nouwy3nnbsvo3h5klxsx4a7y"
       extra: "{}"
       state: "wip"
@@ -6087,14 +6087,14 @@ definitions:
           sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
           mimetype: "text/html"
           url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-          timestamp: 20020429162520
+          timestamp: "20020429162520"
         - sha1: "3f242a192acc258bdfdb151943419437f440c313"
           surt: "org,asheesh)/apus/ch1/node15.html"
           status_code: 200
           sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
           mimetype: "text/html"
           url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-          timestamp: 20020429162520
+          timestamp: "20020429162520"
         ident: "q3nouwy3nnbsvo3h5klxsx4a7y"
         extra: "{}"
         state: "wip"
@@ -6118,14 +6118,14 @@ definitions:
           sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
           mimetype: "text/html"
           url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-          timestamp: 20020429162520
+          timestamp: "20020429162520"
         - sha1: "3f242a192acc258bdfdb151943419437f440c313"
           surt: "org,asheesh)/apus/ch1/node15.html"
           status_code: 200
           sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
           mimetype: "text/html"
           url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-          timestamp: 20020429162520
+          timestamp: "20020429162520"
         ident: "q3nouwy3nnbsvo3h5klxsx4a7y"
         extra: "{}"
         state: "wip"
@@ -7064,7 +7064,6 @@ definitions:
   webcapture_entity_cdx:
     required:
     - "sha1"
-    - "status_code"
     - "surt"
     - "timestamp"
     - "url"
@@ -7073,9 +7072,8 @@ definitions:
         type: "string"
         example: "org,asheesh)/apus/ch1/node15.html"
       timestamp:
-        type: "integer"
-        format: "int64"
-        example: 20020429162520
+        type: "string"
+        example: "20020429162520"
       url:
         type: "string"
         example: "http://www.asheesh.org:80/APUS/ch1/node15.html"
@@ -7099,7 +7097,7 @@ definitions:
       sha256: "a77e4c11a57f1d757fca5754a8f83b5d4ece49a2d28596889127c1a2f3f28832"
       mimetype: "text/html"
       url: "http://www.asheesh.org:80/APUS/ch1/node15.html"
-      timestamp: 20020429162520
+      timestamp: "20020429162520"
     upperCaseName: "WEBCAPTURE_ENTITY_CDX"
   release_entity_abstracts:
     properties:
diff --git a/rust/fatcat-api-spec/src/models.rs b/rust/fatcat-api-spec/src/models.rs
index c6fc3ae1..a7c1375f 100644
--- a/rust/fatcat-api-spec/src/models.rs
+++ b/rust/fatcat-api-spec/src/models.rs
@@ -943,7 +943,7 @@ pub struct WebcaptureEntityCdx {
     pub surt: String,
 
     #[serde(rename = "timestamp")]
-    pub timestamp: i64,
+    pub timestamp: String,
 
     #[serde(rename = "url")]
     pub url: String,
@@ -953,7 +953,8 @@ pub struct WebcaptureEntityCdx {
     pub mimetype: Option<String>,
 
     #[serde(rename = "status_code")]
-    pub status_code: i64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub status_code: Option<i64>,
 
     #[serde(rename = "sha1")]
     pub sha1: String,
@@ -964,13 +965,13 @@ pub struct WebcaptureEntityCdx {
 }
 
 impl WebcaptureEntityCdx {
-    pub fn new(surt: String, timestamp: i64, url: String, status_code: i64, sha1: String) -> WebcaptureEntityCdx {
+    pub fn new(surt: String, timestamp: String, url: String, sha1: String) -> WebcaptureEntityCdx {
         WebcaptureEntityCdx {
             surt: surt,
             timestamp: timestamp,
             url: url,
             mimetype: None,
-            status_code: status_code,
+            status_code: None,
             sha1: sha1,
             sha256: None,
         }
diff --git a/rust/src/api_entity_crud.rs b/rust/src/api_entity_crud.rs
index 19817540..cab6f58f 100644
--- a/rust/src/api_entity_crud.rs
+++ b/rust/src/api_entity_crud.rs
@@ -1334,7 +1334,7 @@ impl EntityCrud for WebcaptureEntity {
                         .map(|c| WebcaptureRevCdxNewRow {
                             webcapture_rev: *rev_id,
                             surt: c.surt.clone(),
-                            timestamp: c.timestamp,
+                            timestamp: c.timestamp.clone(),
                             url: c.url.clone(),
                             mimetype: c.mimetype.clone(),
                             status_code: c.status_code,
diff --git a/rust/src/bin/fatcat-export.rs b/rust/src/bin/fatcat-export.rs
index 8b7c4442..ec66ed4c 100644
--- a/rust/src/bin/fatcat-export.rs
+++ b/rust/src/bin/fatcat-export.rs
@@ -207,12 +207,12 @@ pub fn do_export(
             ExportEntityType::File => {
                 thread::spawn(move || loop_work_file(row_receiver, output_sender, &db_conn, expand))
             }
-            ExportEntityType::Fileset => {
-                thread::spawn(move || loop_work_fileset(row_receiver, output_sender, &db_conn, expand))
-            }
-            ExportEntityType::Webcapture=> {
-                thread::spawn(move || loop_work_webcapture(row_receiver, output_sender, &db_conn, expand))
-            }
+            ExportEntityType::Fileset => thread::spawn(move || {
+                loop_work_fileset(row_receiver, output_sender, &db_conn, expand)
+            }),
+            ExportEntityType::Webcapture => thread::spawn(move || {
+                loop_work_webcapture(row_receiver, output_sender, &db_conn, expand)
+            }),
             ExportEntityType::Release => thread::spawn(move || {
                 loop_work_release(row_receiver, output_sender, &db_conn, expand)
             }),
diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs
index c913b98e..d2f6c3c2 100644
--- a/rust/src/database_models.rs
+++ b/rust/src/database_models.rs
@@ -305,10 +305,10 @@ pub struct WebcaptureRevCdxRow {
     pub id: i64,
     pub webcapture_rev: Uuid,
     pub surt: String,
-    pub timestamp: i64,
+    pub timestamp: String,
     pub url: String,
     pub mimetype: Option<String>,
-    pub status_code: i64,
+    pub status_code: Option<i64>,
     pub sha1: String,
     pub sha256: Option<String>,
 }
@@ -318,10 +318,10 @@ pub struct WebcaptureRevCdxRow {
 pub struct WebcaptureRevCdxNewRow {
     pub webcapture_rev: Uuid,
     pub surt: String,
-    pub timestamp: i64,
+    pub timestamp: String,
     pub url: String,
     pub mimetype: Option<String>,
-    pub status_code: i64,
+    pub status_code: Option<i64>,
     pub sha1: String,
     pub sha256: Option<String>,
 }
diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs
index 436f2989..5db14de5 100644
--- a/rust/src/database_schema.rs
+++ b/rust/src/database_schema.rs
@@ -330,10 +330,10 @@ table! {
         id -> Int8,
         webcapture_rev -> Uuid,
         surt -> Text,
-        timestamp -> Int8,
+        timestamp -> Text,
         url -> Text,
         mimetype -> Nullable<Text>,
-        status_code -> Int8,
+        status_code -> Nullable<Int8>,
         sha1 -> Text,
         sha256 -> Nullable<Text>,
     }
diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs
index 443c32ee..2160a0a0 100644
--- a/rust/tests/test_api_server_http.rs
+++ b/rust/tests/test_api_server_http.rs
@@ -685,7 +685,7 @@ fn test_post_webcapture() {
                 "timestamp": "2018-12-28T05:06:07Z",
                 "cdx": [
                     {"surt": "org,asheesh,)/robots.txt",
-                     "timestamp": 20181228050607,
+                     "timestamp": "20181228050607",
                      "url": "https://asheesh.org/robots.txt",
                      "status_code": 200,
                      "mimetype": "text/html",