Merge branch 'bnewbold-content-scope'

12 files changed, 81 insertions, 7 deletions

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 716f2b6b..6fa62c5f 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -607,7 +607,7 @@ dependencies = [
 
 [[package]]
 name = "fatcat"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "cadence",
  "chrono 0.4.6",
@@ -648,7 +648,7 @@ dependencies = [
 
 [[package]]
 name = "fatcat-openapi"
-version = "0.4.0"
+version = "0.5.0"
 dependencies = [
  "bodyparser",
  "chrono 0.4.6",
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index e2ce5e41..b521316e 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "fatcat"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2018"
 authors = ["Bryan Newbold <bnewbold@archive.org>"]
 description = "A scalable, versioned, API-oriented catalog for bibliographic entities and file metadata"
diff --git a/rust/fatcat-openapi/Cargo.toml b/rust/fatcat-openapi/Cargo.toml
index 7f417242..f532b780 100644
--- a/rust/fatcat-openapi/Cargo.toml
+++ b/rust/fatcat-openapi/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "fatcat-openapi"
-version = "0.4.0"
+version = "0.5.0"
 edition = "2018"
 authors = ["Bryan Newbold <bnewbold@archive.org>"]
 description = "Fatcat is an editable bibliographic database. This OpenAPI code-generated crate container HTTP API models, endpoints, and other auto-generated types useful for both client and server implementations of the catalog API."
diff --git a/rust/fatcat-openapi/README.md b/rust/fatcat-openapi/README.md
index 327fe3a8..0ed53d3d 100644
--- a/rust/fatcat-openapi/README.md
+++ b/rust/fatcat-openapi/README.md
@@ -12,8 +12,8 @@ To see how to make this your own, look here:
 
 [README](https://github.com/swagger-api/swagger-codegen/blob/master/README.md)
 
-- API version: 0.4.0
-- Build date: 2021-10-12T23:51:46.767Z
+- API version: 0.5.0
+- Build date: 2021-11-17T22:18:19.232Z
 For more information, please visit [https://fatcat.wiki](https://fatcat.wiki)
 
 This autogenerated project defines an API crate `fatcat` which contains:
@@ -172,7 +172,7 @@ The server example is designed to form the basis for implementing your own serve
 
 * Set up a new Rust project, e.g., with `cargo init --bin`.
 * Insert `fatcat` into the `members` array under [workspace] in the root `Cargo.toml`, e.g., `members = [ "fatcat" ]`.
-* Add `fatcat = {version = "0.4.0", path = "fatcat"}` under `[dependencies]` in the root `Cargo.toml`.
+* Add `fatcat = {version = "0.5.0", path = "fatcat"}` under `[dependencies]` in the root `Cargo.toml`.
 * Copy the `[dependencies]` and `[dev-dependencies]` from `fatcat/Cargo.toml` into the root `Cargo.toml`'s `[dependencies]` section.
   * Copy all of the `[dev-dependencies]`, but only the `[dependencies]` that are required by the example server. These should be clearly indicated by comments.
   * Remove `"optional = true"` from each of these lines if present.
diff --git a/rust/fatcat-openapi/src/models.rs b/rust/fatcat-openapi/src/models.rs
index ca203c61..36c9105f 100644
--- a/rust/fatcat-openapi/src/models.rs
+++ b/rust/fatcat-openapi/src/models.rs
@@ -628,6 +628,10 @@ pub struct FileEntity {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub release_ids: Option<Vec<String>>,
 
+    #[serde(rename = "content_scope")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content_scope: Option<String>,
+
     #[serde(rename = "mimetype")]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub mimetype: Option<String>,
@@ -692,6 +696,7 @@ impl FileEntity {
         FileEntity {
             releases: None,
             release_ids: None,
+            content_scope: None,
             mimetype: None,
             urls: None,
             sha256: None,
@@ -763,6 +768,10 @@ pub struct FilesetEntity {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub manifest: Option<Vec<models::FilesetFile>>,
 
+    #[serde(rename = "content_scope")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content_scope: Option<String>,
+
     // Note: inline enums are not fully supported by swagger-codegen
     #[serde(rename = "state")]
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -801,6 +810,7 @@ impl FilesetEntity {
             release_ids: None,
             urls: None,
             manifest: None,
+            content_scope: None,
             state: None,
             ident: None,
             revision: None,
@@ -1453,6 +1463,10 @@ pub struct WebcaptureEntity {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub release_ids: Option<Vec<String>>,
 
+    #[serde(rename = "content_scope")]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub content_scope: Option<String>,
+
     /// Same format as CDX line timestamp (UTC, etc). Corresponds to the overall capture timestamp. Should generally be the timestamp of capture of the primary resource URL.
     #[serde(rename = "timestamp")]
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -1507,6 +1521,7 @@ impl WebcaptureEntity {
         WebcaptureEntity {
             releases: None,
             release_ids: None,
+            content_scope: None,
             timestamp: None,
             original_url: None,
             archive_urls: None,
diff --git a/rust/migrations/2021-11-17-222046_content_scope/down.sql b/rust/migrations/2021-11-17-222046_content_scope/down.sql
new file mode 100644
index 00000000..b2d55321
--- /dev/null
+++ b/rust/migrations/2021-11-17-222046_content_scope/down.sql
@@ -0,0 +1,10 @@
+-- This file should undo anything in `up.sql`
+
+ALTER TABLE file_rev
+DROP COLUMN content_scope;
+
+ALTER TABLE fileset_rev
+DROP COLUMN content_scope;
+
+ALTER TABLE webcapture_rev
+DROP COLUMN content_scope;
diff --git a/rust/migrations/2021-11-17-222046_content_scope/up.sql b/rust/migrations/2021-11-17-222046_content_scope/up.sql
new file mode 100644
index 00000000..82c5f2e6
--- /dev/null
+++ b/rust/migrations/2021-11-17-222046_content_scope/up.sql
@@ -0,0 +1,27 @@
+-- This is the v0.5.0 schema
+-- Add `content_scope` field to file, fileset, webcapture
+
+ALTER TABLE file_rev
+ADD COLUMN content_scope       TEXT CHECK (octet_length(content_scope) >= 1);
+
+ALTER TABLE fileset_rev
+ADD COLUMN content_scope       TEXT CHECK (octet_length(content_scope) >= 1);
+
+ALTER TABLE webcapture_rev
+ADD COLUMN content_scope       TEXT CHECK (octet_length(content_scope) >= 1);
+
+-------------------- Update Test Revs --------------------------------------
+-- IMPORTANT: don't create new entities here, only mutate existing
+
+BEGIN;
+
+UPDATE file_rev SET content_scope = 'article'
+WHERE id = '00000000-0000-0000-3333-FFF000000003';
+
+UPDATE fileset_rev SET content_scope = 'dataset'
+WHERE id = '00000000-0000-0000-6666-fff000000003';
+
+UPDATE webcapture_rev SET content_scope = 'webpage'
+WHERE id = '00000000-0000-0000-7777-FFF000000003';
+
+COMMIT;
diff --git a/rust/src/database_models.rs b/rust/src/database_models.rs
index 76c8675d..0427f9c8 100644
--- a/rust/src/database_models.rs
+++ b/rust/src/database_models.rs
@@ -220,6 +220,7 @@ pub struct FileRevRow {
     pub sha256: Option<String>,
     pub md5: Option<String>,
     pub mimetype: Option<String>,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -231,6 +232,7 @@ pub struct FileRevNewRow {
     pub sha256: Option<String>,
     pub md5: Option<String>,
     pub mimetype: Option<String>,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
@@ -291,12 +293,14 @@ pub struct FilesetRevUrlNewRow {
 pub struct FilesetRevRow {
     pub id: Uuid,
     pub extra_json: Option<serde_json::Value>,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
 #[table_name = "fileset_rev"]
 pub struct FilesetRevNewRow {
     pub extra_json: Option<serde_json::Value>,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
@@ -360,6 +364,7 @@ pub struct WebcaptureRevRow {
     pub extra_json: Option<serde_json::Value>,
     pub original_url: String,
     pub timestamp: chrono::NaiveDateTime,
+    pub content_scope: Option<String>,
 }
 
 #[derive(Debug, Associations, AsChangeset, Insertable)]
@@ -368,6 +373,7 @@ pub struct WebcaptureRevNewRow {
     pub extra_json: Option<serde_json::Value>,
     pub original_url: String,
     pub timestamp: chrono::NaiveDateTime,
+    pub content_scope: Option<String>,
 }
 
 entity_structs!(
diff --git a/rust/src/database_schema.rs b/rust/src/database_schema.rs
index e0a54233..e3d16202 100644
--- a/rust/src/database_schema.rs
+++ b/rust/src/database_schema.rs
@@ -163,6 +163,7 @@ table! {
         sha256 -> Nullable<Text>,
         md5 -> Nullable<Text>,
         mimetype -> Nullable<Text>,
+        content_scope -> Nullable<Text>,
     }
 }
 
@@ -208,6 +209,7 @@ table! {
     fileset_rev (id) {
         id -> Uuid,
         extra_json -> Nullable<Jsonb>,
+        content_scope -> Nullable<Text>,
     }
 }
 
@@ -372,6 +374,7 @@ table! {
         extra_json -> Nullable<Jsonb>,
         original_url -> Text,
         timestamp -> Timestamptz,
+        content_scope -> Nullable<Text>,
     }
 }
 
diff --git a/rust/src/entity_crud.rs b/rust/src/entity_crud.rs
index 19cb58ea..f48246a5 100644
--- a/rust/src/entity_crud.rs
+++ b/rust/src/entity_crud.rs
@@ -1042,6 +1042,7 @@ impl EntityCrud for FileEntity {
             size: None,
             urls: None,
             mimetype: None,
+            content_scope: None,
             release_ids: None,
             releases: None,
             state: Some(ident_row.state().unwrap().shortname()),
@@ -1125,6 +1126,7 @@ impl EntityCrud for FileEntity {
             size: rev_row.size_bytes,
             urls: Some(urls),
             mimetype: rev_row.mimetype,
+            content_scope: rev_row.content_scope,
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
             releases: None,
             state,
@@ -1160,6 +1162,7 @@ impl EntityCrud for FileEntity {
                         sha256: model.sha256.clone(),
                         md5: model.md5.clone(),
                         mimetype: model.mimetype.clone(),
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<FileRevNewRow>>(),
@@ -1245,6 +1248,7 @@ impl EntityCrud for FilesetEntity {
         }
 
         Ok(FilesetEntity {
+            content_scope: None,
             manifest: None,
             urls: None,
             release_ids: None,
@@ -1340,6 +1344,7 @@ impl EntityCrud for FilesetEntity {
             .collect();
 
         Ok(FilesetEntity {
+            content_scope: rev_row.content_scope.clone(),
             manifest: Some(manifest),
             urls: Some(urls),
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
@@ -1376,6 +1381,7 @@ impl EntityCrud for FilesetEntity {
                 models
                     .iter()
                     .map(|model| FilesetRevNewRow {
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<FilesetRevNewRow>>(),
@@ -1492,6 +1498,7 @@ impl EntityCrud for WebcaptureEntity {
             archive_urls: None,
             original_url: None,
             timestamp: None,
+            content_scope: None,
             release_ids: None,
             releases: None,
             state: Some(ident_row.state().unwrap().shortname()),
@@ -1590,6 +1597,7 @@ impl EntityCrud for WebcaptureEntity {
             archive_urls: Some(archive_urls),
             original_url: Some(rev_row.original_url),
             timestamp: Some(chrono::DateTime::from_utc(rev_row.timestamp, chrono::Utc)),
+            content_scope: rev_row.content_scope,
             release_ids: Some(release_ids.iter().map(|fcid| fcid.to_string()).collect()),
             releases: None,
             state,
@@ -1628,6 +1636,7 @@ impl EntityCrud for WebcaptureEntity {
                         // these unwraps safe because of check above
                         original_url: model.original_url.clone().unwrap(),
                         timestamp: model.timestamp.unwrap().naive_utc(),
+                        content_scope: model.content_scope.clone(),
                         extra_json: model.extra.clone(),
                     })
                     .collect::<Vec<WebcaptureRevNewRow>>(),
diff --git a/rust/src/server.rs b/rust/src/server.rs
index 4d30ecbe..fb55f03c 100644
--- a/rust/src/server.rs
+++ b/rust/src/server.rs
@@ -72,6 +72,7 @@ pub fn create_test_server() -> Result<Server> {
     diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::revert_latest_migration(&conn).unwrap();
+    diesel_migrations::revert_latest_migration(&conn).unwrap();
     diesel_migrations::run_pending_migrations(&conn).unwrap();
     Ok(server)
 }
diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs
index 8f691e0d..0601a26b 100644
--- a/rust/tests/test_api_server_http.rs
+++ b/rust/tests/test_api_server_http.rs
@@ -636,6 +636,7 @@ fn test_post_file() {
                     {"url": "http://web.archive.org/2/http://archive.org/asdf.txt", "rel": "webarchive" }
                 ],
                 "mimetype": "application/pdf",
+                "content_scope": "article",
                 "release_ids": [
                     "aaaaaaaaaaaaarceaaaaaaaaae",
                     "aaaaaaaaaaaaarceaaaaaaaaai"
@@ -711,6 +712,7 @@ fn test_post_fileset() {
                     "aaaaaaaaaaaaarceaaaaaaaaae",
                     "aaaaaaaaaaaaarceaaaaaaaaai"
                 ],
+                "content_scope": "dataset",
                 "extra": { "source": "speculation" }
                 }"#,
             &router,
@@ -764,6 +766,7 @@ fn test_post_webcapture() {
             headers.clone(),
             r#"{"original_url": "https://bnewbold.net/",
                 "timestamp": "2018-12-28T05:06:07Z",
+                "content_scope": "landing-page",
                 "cdx": [
                     {"surt": "org,asheesh,)/robots.txt",
                      "timestamp": "2018-12-28T05:06:07Z",