summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--rust/src/database_models.rs6
-rw-r--r--rust/src/database_schema.rs3
-rw-r--r--rust/src/entity_crud.rs9
-rw-r--r--rust/src/server.rs1
-rw-r--r--rust/tests/test_api_server_http.rs3
5 files changed, 22 insertions, 0 deletions
diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs
index 76c8675d..0427f9c8 100644
--- a/rust/src/database_models.rs
+++ b/rust/src/database_models.rs
@@ -220,6 +220,7 @@ pub struct FileRevRow {
pub sha256: Option<String>,
pub md5: Option<String>,
pub mimetype: Option<String>,
+ pub content_scope: Option<String>,
}
#[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -231,6 +232,7 @@ pub struct FileRevNewRow {
pub sha256: Option<String>,
pub md5: Option<String>,
pub mimetype: Option<String>,
+ pub content_scope: Option<String>,
}
entity_structs!(
@@ -291,12 +293,14 @@ pub struct FilesetRevUrlNewRow {
pub struct FilesetRevRow {
pub id: Uuid,
pub extra_json: Option<serde_json::Value>,
+ pub content_scope: Option<String>,
}
#[derive(Debug, Associations, AsChangeset, Insertable)]
#[table_name = "fileset_rev"]
pub struct FilesetRevNewRow {
pub extra_json: Option<serde_json::Value>,
+ pub content_scope: Option<String>,
}
entity_structs!(
@@ -360,6 +364,7 @@ pub struct WebcaptureRevRow {
pub extra_json: Option<serde_json::Value>,
pub original_url: String,
pub timestamp: chrono::NaiveDateTime,
+ pub content_scope: Option<String>,
}
#[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -368,6 +373,7 @@ pub struct WebcaptureRevNewRow {
pub extra_json: Option<serde_json::Value>,
pub original_url: String,
pub timestamp: chrono::NaiveDateTime,
+ pub content_scope: Option<String>,
}
entity_structs!(
diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs
index e0a54233..e3d16202 100644
--- a/rust/src/database_schema.rs
+++ b/rust/src/database_schema.rs
@@ -163,6 +163,7 @@ table! {
sha256 -> Nullable<Text>,
md5 -> Nullable<Text>,
mimetype -> Nullable<Text>,
+ content_scope -> Nullable<Text>,
}
}
@@ -208,6 +209,7 @@ table! {
fileset_rev (id) {
id -> Uuid,
extra_json -> Nullable<Jsonb>,
+ content_scope -> Nullable<Text>,
}
}
@@ -372,6 +374,7 @@ table! {
extra_json -> Nullable<Jsonb>,
original_url -> Text,
timestamp -> Timestamptz,
+ content_scope -> Nullable<Text>,
}
}
diff --git a/rust/src/entity_crud.rs b/rust/src/entity_crud.rs
index 19cb58ea..f48246a5 100644
--- a/rust/src/entity_crud.rs
+++ b/rust/src/entity_crud.rs
@@ -1042,6 +1042,7 @@ impl EntityCrud for FileEntity {
size: None,
urls: None,
mimetype: None,
+ content_scope: None,
release_ids: None,
releases: None,
state: Some(ident_row.state().unwrap().shortname()),
@@ -1125,6 +1126,7 @@ impl EntityCrud for FileEntity {
size: rev_row.size_bytes,
urls: Some(urls),
mimetype: rev_row.mimetype,
+ content_scope: rev_row.content_scope,
release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
releases: None,
state,
@@ -1160,6 +1162,7 @@ impl EntityCrud for FileEntity {
sha256: model.sha256.clone(),
md5: model.md5.clone(),
mimetype: model.mimetype.clone(),
+ content_scope: model.content_scope.clone(),
extra_json: model.extra.clone(),
})
.collect::<Vec<FileRevNewRow>>(),
@@ -1245,6 +1248,7 @@ impl EntityCrud for FilesetEntity {
}
Ok(FilesetEntity {
+ content_scope: None,
manifest: None,
urls: None,
release_ids: None,
@@ -1340,6 +1344,7 @@ impl EntityCrud for FilesetEntity {
.collect();
Ok(FilesetEntity {
+ content_scope: rev_row.content_scope.clone(),
manifest: Some(manifest),
urls: Some(urls),
release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
@@ -1376,6 +1381,7 @@ impl EntityCrud for FilesetEntity {
models
.iter()
.map(|model| FilesetRevNewRow {
+ content_scope: model.content_scope.clone(),
extra_json: model.extra.clone(),
})
.collect::<Vec<FilesetRevNewRow>>(),
@@ -1492,6 +1498,7 @@ impl EntityCrud for WebcaptureEntity {
archive_urls: None,
original_url: None,
timestamp: None,
+ content_scope: None,
release_ids: None,
releases: None,
state: Some(ident_row.state().unwrap().shortname()),
@@ -1590,6 +1597,7 @@ impl EntityCrud for WebcaptureEntity {
archive_urls: Some(archive_urls),
original_url: Some(rev_row.original_url),
timestamp: Some(chrono::DateTime::from_utc(rev_row.timestamp, chrono::Utc)),
+ content_scope: rev_row.content_scope,
release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
releases: None,
state,
@@ -1628,6 +1636,7 @@ impl EntityCrud for WebcaptureEntity {
// these unwraps safe because of check above
original_url: model.original_url.clone().unwrap(),
timestamp: model.timestamp.unwrap().naive_utc(),
+ content_scope: model.content_scope.clone(),
extra_json: model.extra.clone(),
})
.collect::<Vec<WebcaptureRevNewRow>>(),
diff --git a/rust/src/server.rs b/rust/src/server.rs
index 4d30ecbe..fb55f03c 100644
--- a/rust/src/server.rs
+++ b/rust/src/server.rs
@@ -72,6 +72,7 @@ pub fn create_test_server() -> Result<Server> {
diesel_migrations::revert_latest_migration(&conn).unwrap();
diesel_migrations::revert_latest_migration(&conn).unwrap();
diesel_migrations::revert_latest_migration(&conn).unwrap();
+ diesel_migrations::revert_latest_migration(&conn).unwrap();
diesel_migrations::run_pending_migrations(&conn).unwrap();
Ok(server)
}
diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs
index 8f691e0d..0601a26b 100644
--- a/rust/tests/test_api_server_http.rs
+++ b/rust/tests/test_api_server_http.rs
@@ -636,6 +636,7 @@ fn test_post_file() {
{"url": "http://web.archive.org/2/http://archive.org/asdf.txt", "rel": "webarchive" }
],
"mimetype": "application/pdf",
+ "content_scope": "article",
"release_ids": [
"aaaaaaaaaaaaarceaaaaaaaaae",
"aaaaaaaaaaaaarceaaaaaaaaai"
@@ -711,6 +712,7 @@ fn test_post_fileset() {
"aaaaaaaaaaaaarceaaaaaaaaae",
"aaaaaaaaaaaaarceaaaaaaaaai"
],
+ "content_scope": "dataset",
"extra": { "source": "speculation" }
}"#,
&router,
@@ -764,6 +766,7 @@ fn test_post_webcapture() {
headers.clone(),
r#"{"original_url": "https://bnewbold.net/",
"timestamp": "2018-12-28T05:06:07Z",
+ "content_scope": "landing-page",
"cdx": [
{"surt": "org,asheesh,)/robots.txt",
"timestamp": "2018-12-28T05:06:07Z",