diff options
| -rw-r--r-- | rust/src/database_models.rs | 6 | ||||
| -rw-r--r-- | rust/src/database_schema.rs | 3 | ||||
| -rw-r--r-- | rust/src/entity_crud.rs | 9 | ||||
| -rw-r--r-- | rust/src/server.rs | 1 | ||||
| -rw-r--r-- | rust/tests/test_api_server_http.rs | 3 |
5 files changed, 22 insertions, 0 deletions
diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs index 76c8675d..0427f9c8 100644 --- a/rust/src/database_models.rs +++ b/rust/src/database_models.rs @@ -220,6 +220,7 @@ pub struct FileRevRow { pub sha256: Option<String>, pub md5: Option<String>, pub mimetype: Option<String>, + pub content_scope: Option<String>, } #[derive(Debug, Associations, AsChangeset, Insertable)] @@ -231,6 +232,7 @@ pub struct FileRevNewRow { pub sha256: Option<String>, pub md5: Option<String>, pub mimetype: Option<String>, + pub content_scope: Option<String>, } entity_structs!( @@ -291,12 +293,14 @@ pub struct FilesetRevUrlNewRow { pub struct FilesetRevRow { pub id: Uuid, pub extra_json: Option<serde_json::Value>, + pub content_scope: Option<String>, } #[derive(Debug, Associations, AsChangeset, Insertable)] #[table_name = "fileset_rev"] pub struct FilesetRevNewRow { pub extra_json: Option<serde_json::Value>, + pub content_scope: Option<String>, } entity_structs!( @@ -360,6 +364,7 @@ pub struct WebcaptureRevRow { pub extra_json: Option<serde_json::Value>, pub original_url: String, pub timestamp: chrono::NaiveDateTime, + pub content_scope: Option<String>, } #[derive(Debug, Associations, AsChangeset, Insertable)] @@ -368,6 +373,7 @@ pub struct WebcaptureRevNewRow { pub extra_json: Option<serde_json::Value>, pub original_url: String, pub timestamp: chrono::NaiveDateTime, + pub content_scope: Option<String>, } entity_structs!( diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs index e0a54233..e3d16202 100644 --- a/rust/src/database_schema.rs +++ b/rust/src/database_schema.rs @@ -163,6 +163,7 @@ table! { sha256 -> Nullable<Text>, md5 -> Nullable<Text>, mimetype -> Nullable<Text>, + content_scope -> Nullable<Text>, } } @@ -208,6 +209,7 @@ table! { fileset_rev (id) { id -> Uuid, extra_json -> Nullable<Jsonb>, + content_scope -> Nullable<Text>, } } @@ -372,6 +374,7 @@ table! { extra_json -> Nullable<Jsonb>, original_url -> Text, timestamp -> Timestamptz, + content_scope -> Nullable<Text>, } } diff --git a/rust/src/entity_crud.rs b/rust/src/entity_crud.rs index 19cb58ea..f48246a5 100644 --- a/rust/src/entity_crud.rs +++ b/rust/src/entity_crud.rs @@ -1042,6 +1042,7 @@ impl EntityCrud for FileEntity { size: None, urls: None, mimetype: None, + content_scope: None, release_ids: None, releases: None, state: Some(ident_row.state().unwrap().shortname()), @@ -1125,6 +1126,7 @@ impl EntityCrud for FileEntity { size: rev_row.size_bytes, urls: Some(urls), mimetype: rev_row.mimetype, + content_scope: rev_row.content_scope, release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()), releases: None, state, @@ -1160,6 +1162,7 @@ impl EntityCrud for FileEntity { sha256: model.sha256.clone(), md5: model.md5.clone(), mimetype: model.mimetype.clone(), + content_scope: model.content_scope.clone(), extra_json: model.extra.clone(), }) .collect::<Vec<FileRevNewRow>>(), @@ -1245,6 +1248,7 @@ impl EntityCrud for FilesetEntity { } Ok(FilesetEntity { + content_scope: None, manifest: None, urls: None, release_ids: None, @@ -1340,6 +1344,7 @@ impl EntityCrud for FilesetEntity { .collect(); Ok(FilesetEntity { + content_scope: rev_row.content_scope.clone(), manifest: Some(manifest), urls: Some(urls), release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()), @@ -1376,6 +1381,7 @@ impl EntityCrud for FilesetEntity { models .iter() .map(|model| FilesetRevNewRow { + content_scope: model.content_scope.clone(), extra_json: model.extra.clone(), }) .collect::<Vec<FilesetRevNewRow>>(), @@ -1492,6 +1498,7 @@ impl EntityCrud for WebcaptureEntity { archive_urls: None, original_url: None, timestamp: None, + content_scope: None, release_ids: None, releases: None, state: Some(ident_row.state().unwrap().shortname()), @@ -1590,6 +1597,7 @@ impl EntityCrud for WebcaptureEntity { archive_urls: Some(archive_urls), original_url: Some(rev_row.original_url), timestamp: Some(chrono::DateTime::from_utc(rev_row.timestamp, chrono::Utc)), + content_scope: rev_row.content_scope, release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()), releases: None, state, @@ -1628,6 +1636,7 @@ impl EntityCrud for WebcaptureEntity { // these unwraps safe because of check above original_url: model.original_url.clone().unwrap(), timestamp: model.timestamp.unwrap().naive_utc(), + content_scope: model.content_scope.clone(), extra_json: model.extra.clone(), }) .collect::<Vec<WebcaptureRevNewRow>>(), diff --git a/rust/src/server.rs b/rust/src/server.rs index 4d30ecbe..fb55f03c 100644 --- a/rust/src/server.rs +++ b/rust/src/server.rs @@ -72,6 +72,7 @@ pub fn create_test_server() -> Result<Server> { diesel_migrations::revert_latest_migration(&conn).unwrap(); diesel_migrations::revert_latest_migration(&conn).unwrap(); diesel_migrations::revert_latest_migration(&conn).unwrap(); + diesel_migrations::revert_latest_migration(&conn).unwrap(); diesel_migrations::run_pending_migrations(&conn).unwrap(); Ok(server) } diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs index 8f691e0d..0601a26b 100644 --- a/rust/tests/test_api_server_http.rs +++ b/rust/tests/test_api_server_http.rs @@ -636,6 +636,7 @@ fn test_post_file() { {"url": "http://web.archive.org/2/http://archive.org/asdf.txt", "rel": "webarchive" } ], "mimetype": "application/pdf", + "content_scope": "article", "release_ids": [ "aaaaaaaaaaaaarceaaaaaaaaae", "aaaaaaaaaaaaarceaaaaaaaaai" @@ -711,6 +712,7 @@ fn test_post_fileset() { "aaaaaaaaaaaaarceaaaaaaaaae", "aaaaaaaaaaaaarceaaaaaaaaai" ], + "content_scope": "dataset", "extra": { "source": "speculation" } }"#, &router, @@ -764,6 +766,7 @@ fn test_post_webcapture() { headers.clone(), r#"{"original_url": "https://bnewbold.net/", "timestamp": "2018-12-28T05:06:07Z", + "content_scope": "landing-page", "cdx": [ {"surt": "org,asheesh,)/robots.txt", "timestamp": "2018-12-28T05:06:07Z", |