From 9555367d693ccb894eb8960c98507315e581f60a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 26 Dec 2018 18:14:47 -0800 Subject: most basic fileset/web access tests --- README.md | 2 +- rust/migrations/2018-05-12-001226_init/up.sql | 12 +- rust/src/api_entity_crud.rs | 4 +- rust/tests/test_api_server_http.rs | 183 +++++++++++++++++++++++++- 4 files changed, 191 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4873af8d..1433e62b 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ Internet Archive's internal (not public) infrastructure. - [x] one-to-many and many-to-many entities - [x] JSON(B) "extra" metadata fields - [x] full rev1 schema for all entities + - [x] file sets and web captures - [ ] editgroup review: comments? actions? - - [ ] file sets and web captures - HTTP API Server - [x] base32 encoding of UUID identifiers - [x] inverse many-to-many helpers (files-by-release, release-by-creator) diff --git a/rust/migrations/2018-05-12-001226_init/up.sql b/rust/migrations/2018-05-12-001226_init/up.sql index 713075f6..9b6a00a6 100644 --- a/rust/migrations/2018-05-12-001226_init/up.sql +++ b/rust/migrations/2018-05-12-001226_init/up.sql @@ -546,9 +546,9 @@ INSERT INTO fileset_rev_url (fileset_rev, rel, url) VALUES ('00000000-0000-0000-6666-FFF000000003', 'archive', 'https://archive.org/download/random-dataset/'); INSERT INTO fileset_ident (id, is_live, rev_id, redirect_id) VALUES - ('00000000-0000-0000-6666-000000000001', true, '00000000-0000-0000-6666-FFF000000001', null), -- aaaaaaaaaaaaaztg77yaaaaaae - ('00000000-0000-0000-6666-000000000002', true, '00000000-0000-0000-6666-FFF000000002', null), -- aaaaaaaaaaaaaztg77yaaaaaai - ('00000000-0000-0000-6666-000000000003', true, '00000000-0000-0000-6666-FFF000000003', null); -- aaaaaaaaaaaaaztg77yaaaaaam + ('00000000-0000-0000-6666-000000000001', true, '00000000-0000-0000-6666-FFF000000001', null), -- aaaaaaaaaaaaaztgaaaaaaaaam + ('00000000-0000-0000-6666-000000000002', true, '00000000-0000-0000-6666-FFF000000002', null), -- aaaaaaaaaaaaaztgaaaaaaaaai + ('00000000-0000-0000-6666-000000000003', true, '00000000-0000-0000-6666-FFF000000003', null); -- aaaaaaaaaaaaaztgaaaaaaaaam INSERT INTO fileset_edit (ident_id, rev_id, redirect_id, editgroup_id, prev_rev) VALUES ('00000000-0000-0000-6666-000000000001', '00000000-0000-0000-6666-FFF000000001', null, '00000000-0000-0000-BBBB-000000000003', null), @@ -571,9 +571,9 @@ INSERT INTO webcapture_rev_url (webcapture_rev, rel, url) VALUES ('00000000-0000-0000-7777-FFF000000003', 'warc', 'https://example.org/something.warc.gz'); INSERT INTO webcapture_ident (id, is_live, rev_id, redirect_id) VALUES - ('00000000-0000-0000-7777-000000000001', true, '00000000-0000-0000-7777-FFF000000001', null), -- aaaaaaaaaaaaa53x77yaaaaaae - ('00000000-0000-0000-7777-000000000002', true, '00000000-0000-0000-7777-FFF000000002', null), -- aaaaaaaaaaaaa53x77yaaaaaai - ('00000000-0000-0000-7777-000000000003', true, '00000000-0000-0000-7777-FFF000000003', null); -- aaaaaaaaaaaaa53x77yaaaaaam + ('00000000-0000-0000-7777-000000000001', true, '00000000-0000-0000-7777-FFF000000001', null), -- aaaaaaaaaaaaa53xaaaaaaaaae + ('00000000-0000-0000-7777-000000000002', true, '00000000-0000-0000-7777-FFF000000002', null), -- aaaaaaaaaaaaa53xaaaaaaaaai + ('00000000-0000-0000-7777-000000000003', true, '00000000-0000-0000-7777-FFF000000003', null); -- aaaaaaaaaaaaa53xaaaaaaaaam INSERT INTO webcapture_edit (ident_id, rev_id, redirect_id, editgroup_id, prev_rev) VALUES ('00000000-0000-0000-7777-000000000001', '00000000-0000-0000-7777-FFF000000001', null, '00000000-0000-0000-BBBB-000000000003', null), diff --git a/rust/src/api_entity_crud.rs b/rust/src/api_entity_crud.rs index 7220f27a..19817540 100644 --- a/rust/src/api_entity_crud.rs +++ b/rust/src/api_entity_crud.rs @@ -992,7 +992,7 @@ impl EntityCrud for FilesetEntity { generic_db_get_edit!(fileset_edit); generic_db_delete_edit!(fileset_edit); generic_db_get_redirects!(fileset_ident); - generic_db_accept_edits_batch!("file", fileset_ident, fileset_edit); + generic_db_accept_edits_batch!("fileset", fileset_ident, fileset_edit); generic_db_insert_rev!(); fn from_deleted_row(ident_row: Self::IdentRow) -> Result { @@ -1200,7 +1200,7 @@ impl EntityCrud for WebcaptureEntity { generic_db_get_edit!(webcapture_edit); generic_db_delete_edit!(webcapture_edit); generic_db_get_redirects!(webcapture_ident); - generic_db_accept_edits_batch!("file", webcapture_ident, webcapture_edit); + generic_db_accept_edits_batch!("webcapture", webcapture_ident, webcapture_edit); generic_db_insert_rev!(); fn from_deleted_row(ident_row: Self::IdentRow) -> Result { diff --git a/rust/tests/test_api_server_http.rs b/rust/tests/test_api_server_http.rs index 6e4adda6..443c32ee 100644 --- a/rust/tests/test_api_server_http.rs +++ b/rust/tests/test_api_server_http.rs @@ -68,6 +68,26 @@ fn test_entity_gets() { Some("archive.org"), ); + check_http_response( + request::get( + "http://localhost:9411/v0/fileset/aaaaaaaaaaaaaztgaaaaaaaaam", + headers.clone(), + &router, + ), + status::Ok, + Some(".tar.gz"), + ); + + check_http_response( + request::get( + "http://localhost:9411/v0/webcapture/aaaaaaaaaaaaa53xaaaaaaaaam", + headers.clone(), + &router, + ), + status::Ok, + Some("asheesh.org"), + ); + check_http_response( request::get( "http://localhost:9411/v0/release/aaaaaaaaaaaaarceaaaaaaaaai", @@ -152,7 +172,27 @@ fn test_entity_history() { check_http_response( request::get( - "http://localhost:9411/v0/file/aaaaaaaaaaaaamztaaaaaaaaai/history", + "http://localhost:9411/v0/file/aaaaaaaaaaaaamztaaaaaaaaam/history", + headers.clone(), + &router, + ), + status::Ok, + Some("changelog"), + ); + + check_http_response( + request::get( + "http://localhost:9411/v0/fileset/aaaaaaaaaaaaaztgaaaaaaaaam/history", + headers.clone(), + &router, + ), + status::Ok, + Some("changelog"), + ); + + check_http_response( + request::get( + "http://localhost:9411/v0/webcapture/aaaaaaaaaaaaa53xaaaaaaaaam/history", headers.clone(), &router, ), @@ -418,6 +458,26 @@ fn test_reverse_lookups() { Some("7d97e98f8af710c7e7fe703abc8f639e0ee507c4"), ); + check_http_response( + request::get( + "http://localhost:9411/v0/release/aaaaaaaaaaaaarceaaaaaaaaai/filesets", + headers.clone(), + &router, + ), + status::Ok, + Some("README.md"), + ); + + check_http_response( + request::get( + "http://localhost:9411/v0/release/aaaaaaaaaaaaarceaaaaaaaaai/webcaptures", + headers.clone(), + &router, + ), + status::Ok, + Some("http://example.org"), + ); + check_http_response( request::get( "http://localhost:9411/v0/work/aaaaaaaaaaaaavkvaaaaaaaaai/releases", @@ -544,6 +604,127 @@ fn test_post_file() { ); } +#[test] +fn test_post_fileset() { + let (headers, router, conn) = setup_http(); + + check_http_response( + request::post( + "http://localhost:9411/v0/fileset", + headers.clone(), + r#"{ }"#, + &router, + ), + status::Created, + None, + ); + + check_http_response( + request::post( + "http://localhost:9411/v0/fileset", + headers.clone(), + r#"{"manifest": [ + {"path": "new_file.txt", "size": 12345, "sha1": "e9dd75237c94b209dc3ccd52722de6931a310ba3" }, + {"path": "output/bogus.hdf5", "size": 43210, "sha1": "e9dd75237c94b209dc3ccd52722de6931a310ba3", "extra": {"some": "other value"} } + ], + "urls": [ + {"url": "http://archive.org/download/dataset-0123/", "rel": "archive" }, + {"url": "http://homepage.name/dataset/", "rel": "web" } + ], + "releases": [ + "aaaaaaaaaaaaarceaaaaaaaaae", + "aaaaaaaaaaaaarceaaaaaaaaai" + ], + "extra": { "source": "speculation" } + }"#, + &router, + ), + status::Created, + None, + ); + + let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001").unwrap(); + let editgroup_id = get_or_create_editgroup(editor_id, &conn).unwrap(); + check_http_response( + request::post( + &format!( + "http://localhost:9411/v0/editgroup/{}/accept", + uuid2fcid(&editgroup_id) + ), + headers.clone(), + "", + &router, + ), + status::Ok, + None, + ); + // TODO: there is no lookup for filesets +} + +#[test] +fn test_post_webcapture() { + let (headers, router, conn) = setup_http(); + + check_http_response( + request::post( + "http://localhost:9411/v0/webcapture", + headers.clone(), + r#"{ "original_url": "https://fatcat.wiki", + "timestamp": "2018-12-28T11:11:11Z" }"#, + &router, + ), + status::Created, + None, + ); + + check_http_response( + request::post( + "http://localhost:9411/v0/webcapture", + headers.clone(), + r#"{"original_url": "https://bnewbold.net/", + "timestamp": "2018-12-28T05:06:07Z", + "cdx": [ + {"surt": "org,asheesh,)/robots.txt", + "timestamp": 20181228050607, + "url": "https://asheesh.org/robots.txt", + "status_code": 200, + "mimetype": "text/html", + "sha1": "e9dd75237c94b209dc3ccd52722de6931a310ba3" } + ], + "archive_urls": [ + {"url": "http://archive.org/download/dataset-0123/", "rel": "archive" }, + {"url": "http://homepage.name/dataset/", "rel": "web" } + ], + "releases": [ + "aaaaaaaaaaaaarceaaaaaaaaae", + "aaaaaaaaaaaaarceaaaaaaaaai" + ], + "extra": { "source": "speculation" } + }"#, + &router, + ), + status::Created, + None, + ); + + let editor_id = Uuid::parse_str("00000000-0000-0000-AAAA-000000000001").unwrap(); + let editgroup_id = get_or_create_editgroup(editor_id, &conn).unwrap(); + check_http_response( + request::post( + &format!( + "http://localhost:9411/v0/editgroup/{}/accept", + uuid2fcid(&editgroup_id) + ), + headers.clone(), + "", + &router, + ), + status::Ok, + None, + ); + // TODO: there is no lookup for filesets +} + #[test] fn test_post_release() { let (headers, router, _conn) = setup_http(); -- cgit v1.2.3