rust: implement content_scope

5 files changed, 22 insertions, 0 deletions

diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs
index 76c8675d..0427f9c8 100644
--- a/rust/src/database_models.rs
+++ b/rust/src/database_models.rs
@@ -220,6 +220,7 @@ pub struct FileRevRow {
     pub sha256: Option<String>,
     pub md5: Option<String>,
     pub mimetype: Option<String>,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -231,6 +232,7 @@ pub struct FileRevNewRow {
     pub sha256: Option<String>,
     pub md5: Option<String>,
     pub mimetype: Option<String>,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
@@ -291,12 +293,14 @@ pub struct FilesetRevUrlNewRow {
 pub struct FilesetRevRow {
     pub id: Uuid,
     pub extra_json: Option<serde_json::Value>,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
 #[table_name = "fileset_rev"]
 pub struct FilesetRevNewRow {
     pub extra_json: Option<serde_json::Value>,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
@@ -360,6 +364,7 @@ pub struct WebcaptureRevRow {
     pub extra_json: Option<serde_json::Value>,
     pub original_url: String,
     pub timestamp: chrono::NaiveDateTime,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -368,6 +373,7 @@ pub struct WebcaptureRevNewRow {
     pub extra_json: Option<serde_json::Value>,
     pub original_url: String,
     pub timestamp: chrono::NaiveDateTime,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs
index e0a54233..e3d16202 100644
--- a/rust/src/database_schema.rs
+++ b/rust/src/database_schema.rs
@@ -163,6 +163,7 @@ table! {
         sha256 -> Nullable<Text>,
         md5 -> Nullable<Text>,
         mimetype -> Nullable<Text>,
+        content_scope -> Nullable<Text>,
     }
 }
 
@@ -208,6 +209,7 @@ table! {
     fileset_rev (id) {
         id -> Uuid,
         extra_json -> Nullable<Jsonb>,
+        content_scope -> Nullable<Text>,
     }
 }
 
@@ -372,6 +374,7 @@ table! {
         extra_json -> Nullable<Jsonb>,
         original_url -> Text,
         timestamp -> Timestamptz,
+        content_scope -> Nullable<Text>,
     }
 }
 
diff --git a/rust/src/entity_crud.rs b/rust/src/entity_crud.rs
index 19cb58ea..f48246a5 100644
--- a/rust/src/entity_crud.rs
+++ b/rust/src/entity_crud.rs
@@ -1042,6 +1042,7 @@ impl EntityCrud for FileEntity {
             size: None,
             urls: None,
             mimetype: None,
+            content_scope: None,
             release_ids: None,
             releases: None,
             state: Some(ident_row.state().unwrap().shortname()),
@@ -1125,6 +1126,7 @@ impl EntityCrud for FileEntity {
             size: rev_row.size_bytes,
             urls: Some(urls),
             mimetype: rev_row.mimetype,
+            content_scope: rev_row.content_scope,
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
             releases: None,
             state,
@@ -1160,6 +1162,7 @@ impl EntityCrud for FileEntity {
                         sha256: model.sha256.clone(),
                         md5: model.md5.clone(),
                         mimetype: model.mimetype.clone(),
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<FileRevNewRow>>(),
@@ -1245,6 +1248,7 @@ impl EntityCrud for FilesetEntity {
         }
 
         Ok(FilesetEntity {
+            content_scope: None,
             manifest: None,
             urls: None,
             release_ids: None,
@@ -1340,6 +1344,7 @@ impl EntityCrud for FilesetEntity {
             .collect();
 
         Ok(FilesetEntity {
+            content_scope: rev_row.content_scope.clone(),
             manifest: Some(manifest),
             urls: Some(urls),
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
@@ -1376,6 +1381,7 @@ impl EntityCrud for FilesetEntity {
                 models
                     .iter()
                     .map(|model| FilesetRevNewRow {
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<FilesetRevNewRow>>(),
@@ -1492,6 +1498,7 @@ impl EntityCrud for WebcaptureEntity {
             archive_urls: None,
             original_url: None,
             timestamp: None,
+            content_scope: None,
             release_ids: None,
             releases: None,
             state: Some(ident_row.state().unwrap().shortname()),
@@ -1590,6 +1597,7 @@ impl EntityCrud for WebcaptureEntity {
             archive_urls: Some(archive_urls),
             original_url: Some(rev_row.original_url),
             timestamp: Some(chrono::DateTime::from_utc(rev_row.timestamp, chrono::Utc)),
+            content_scope: rev_row.content_scope,
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
             releases: None,
             state,
@@ -1628,6 +1636,7 @@ impl EntityCrud for WebcaptureEntity {
                         // these unwraps safe because of check above
                         original_url: model.original_url.clone().unwrap(),
                         timestamp: model.timestamp.unwrap().naive_utc(),
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<WebcaptureRevNewRow>>(),
diff --git a/rust/src/server.rs b/rust/src/server.rs
index 4d30ecbe..fb55f03c 100644
--- a/rust/src/server.rs
+++ b/rust/src/server.rs
@@ -72,6 +72,7 @@ pub fn create_test_server() -> Result<Server> {
     diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::revert_latest_migration(&conn).unwrap();
+    diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::run_pending_migrations(&conn).unwrap();
     Ok(server)
 }
diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs
index 8f691e0d..0601a26b 100644
--- a/rust/tests/test_api_server_http.rs
+++ b/rust/tests/test_api_server_http.rs
@@ -636,6 +636,7 @@ fn test_post_file() {
                     {"url": "http://web.archive.org/2/http://archive.org/asdf.txt", "rel": "webarchive" }
                 ],
                 "mimetype": "application/pdf",
+                "content_scope": "article",
                 "release_ids": [
                     "aaaaaaaaaaaaarceaaaaaaaaae",
                     "aaaaaaaaaaaaarceaaaaaaaaai"
@@ -711,6 +712,7 @@ fn test_post_fileset() {
                     "aaaaaaaaaaaaarceaaaaaaaaae",
                     "aaaaaaaaaaaaarceaaaaaaaaai"
                 ],
+                "content_scope": "dataset",
                 "extra": { "source": "speculation" }
                 }"#,
             &router,
@@ -764,6 +766,7 @@ fn test_post_webcapture() {
             headers.clone(),
             r#"{"original_url": "https://bnewbold.net/",
                 "timestamp": "2018-12-28T05:06:07Z",
+                "content_scope": "landing-page",
                 "cdx": [
                     {"surt": "org,asheesh,)/robots.txt",
                      "timestamp": "2018-12-28T05:06:07Z",