From 55f9b88ba1d63db75ef9cde3cf94e5c98526ad2f Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Tue, 25 Aug 2020 18:09:18 -0700 Subject: significant increase in parse and test coverage --- plan.txt | 73 +-- src/parse.rs | 494 ++++++++++++++++++++- tests/files/GET_range.txt | 12 + tests/files/GET_search_agg_filter.txt | 28 ++ tests/files/GET_search_agg_filter_simple.txt | 11 + tests/files/GET_search_agg_histogram.txt | 13 + tests/files/GET_search_agg_max.txt | 11 + tests/files/GET_search_agg_nested.txt | 16 + tests/files/GET_search_exists.txt | 8 + .../files/GET_search_fatcat_preservation_year.txt | 2 + tests/files/GET_search_fatcat_scholar_basic.txt | 2 + tests/files/GET_search_ids.txt | 8 + tests/files/GET_search_match_all.txt | 6 + tests/files/GET_search_match_none.txt | 6 + tests/files/GET_search_multi_match_tie.txt | 11 + tests/files/GET_search_prefix.txt | 10 + tests/files/GET_search_querystring.txt | 14 + tests/files/GET_search_simplequerystring.txt | 10 + tests/files/GET_search_term.txt | 11 + tests/files/GET_search_term_short.txt | 8 + tests/files/GET_search_wildcard.txt | 12 + tests/files/POST_search_inner.txt | 17 - tests/files/POST_search_inner.txt.disabled | 17 + tests/files/scroll/DELETE_scroll_multi.txt | 7 + tests/parse_es_requests.rs | 4 +- 25 files changed, 754 insertions(+), 57 deletions(-) create mode 100644 tests/files/GET_range.txt create mode 100644 tests/files/GET_search_agg_filter.txt create mode 100644 tests/files/GET_search_agg_filter_simple.txt create mode 100644 tests/files/GET_search_agg_histogram.txt create mode 100644 tests/files/GET_search_agg_max.txt create mode 100644 tests/files/GET_search_agg_nested.txt create mode 100644 tests/files/GET_search_exists.txt create mode 100644 tests/files/GET_search_fatcat_preservation_year.txt create mode 100644 tests/files/GET_search_fatcat_scholar_basic.txt create mode 100644 tests/files/GET_search_ids.txt create mode 100644 tests/files/GET_search_match_all.txt create mode 100644 tests/files/GET_search_match_none.txt create mode 100644 tests/files/GET_search_multi_match_tie.txt create mode 100644 tests/files/GET_search_prefix.txt create mode 100644 tests/files/GET_search_querystring.txt create mode 100644 tests/files/GET_search_simplequerystring.txt create mode 100644 tests/files/GET_search_term.txt create mode 100644 tests/files/GET_search_term_short.txt create mode 100644 tests/files/GET_search_wildcard.txt delete mode 100644 tests/files/POST_search_inner.txt create mode 100644 tests/files/POST_search_inner.txt.disabled create mode 100644 tests/files/scroll/DELETE_scroll_multi.txt diff --git a/plan.txt b/plan.txt index 9ab837a..a88eb2f 100644 --- a/plan.txt +++ b/plan.txt @@ -1,18 +1,31 @@ -TODO: see what other requests the default python and javascript client libraries use - -## basics - -- config: TOML, env, args -- filter requests by method and endpoint -- filter query parameters -- parse request bodies (queries) -- method/body for denied requests -- async streaming responses +x filter requests by method and endpoint +x parse query parameters (typed) +x re-serialize query parameters into upstream request URL +x method/body for denied requests +- create full request objects in tests +- bad request tests (parse error, disabled query types, etc) +- "completeness" over all the below query types and parameters +- real fatcat+scholar request tests - minimize tokio feature flags +- see what other requests the default python and javascript client libraries use +- put ProxyConfig in Box? less cloning? +- config: TOML, env, args + => separate parse schema from options schema + => example file +- refactor unwrap() into error handling +- package/release + => generate .deb file + => Makefile + => manpage + => example config? + => README + => push git repo +- test in with https://search.qa.fatcat.wiki and observable (javascript) + +parse method+path (index) -> which validation method +method+path+params+body -> status, upstream request -factoring: -- validate query method (method, path, query, body) ## general endpoints @@ -29,7 +42,7 @@ factoring: ## per-index endpoints - basic info; mapping - (?) + GET //_mapping - count GET //_count - get document @@ -40,6 +53,7 @@ factoring: - search GET //_search POST //_search +- mapping later: @@ -65,31 +79,34 @@ fulltext: (bare str allowed) value (str) - multi_match -- query_string +x query_string - simple_query_string term-level: -- range +x exists + field (str) +x ids + values (array of str) +x prefix +x range gt, gte, lt, lte: str or number -- term +- NOT regex (?) +x term value: str or number -- terms +x terms + does not support "lookup" from other indices (array of str or number) -- wildcard +x wildcard value (str) boost (float, optional) rewrite (str, optional) -- exists - field (str) -- ids - values (array of str) -- match_all +x match_all boost (float, optional) -- match_none +x match_none boost (float, optional) @@ -100,6 +117,10 @@ TODO: ## additional stuff -- HTTP content-encoding: gzip -- content-type header; always JSON? +- headers + - HTTP content-encoding: gzip + - content-type header; always JSON? + - set "Via" header in responses (indicating proxy version/context) - https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html +- logging +- switch from 'url' to 'percent-encoding' (?) diff --git a/src/parse.rs b/src/parse.rs index 0bd1eeb..2a4c0a5 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -12,46 +12,85 @@ pub struct ApiRequest { #[derive(Serialize, Deserialize, Debug, Default)] #[serde(deny_unknown_fields)] pub struct UrlQueryParams { + #[serde(skip_serializing_if="Option::is_none")] pub allow_no_indices: Option, + #[serde(skip_serializing_if="Option::is_none")] pub allow_partial_search_results: Option, + #[serde(skip_serializing_if="Option::is_none")] pub batched_reduce_size: Option, + #[serde(skip_serializing_if="Option::is_none")] pub ccs_minimize_roundtrips: Option, + #[serde(skip_serializing_if="Option::is_none")] pub docvalue_fields: Option, // array of strings, comma-separated + #[serde(skip_serializing_if="Option::is_none")] pub expand_wildcards: Option, + #[serde(skip_serializing_if="Option::is_none")] pub explain: Option, + #[serde(skip_serializing_if="Option::is_none")] pub from: Option, + #[serde(skip_serializing_if="Option::is_none")] pub ignore_throttled: Option, + #[serde(skip_serializing_if="Option::is_none")] pub ignore_unavailable: Option, + #[serde(skip_serializing_if="Option::is_none")] pub max_concurrent_shard_requests: Option, + #[serde(skip_serializing_if="Option::is_none")] pub pre_filter_shard_size: Option, + #[serde(skip_serializing_if="Option::is_none")] pub preference: Option, + #[serde(skip_serializing_if="Option::is_none")] pub q: Option, + #[serde(skip_serializing_if="Option::is_none")] pub request_cache: Option, + #[serde(skip_serializing_if="Option::is_none")] pub rest_total_hits_as_int: Option, + #[serde(skip_serializing_if="Option::is_none")] pub routing: Option, + #[serde(skip_serializing_if="Option::is_none")] pub scroll: Option, // string is "time value" + #[serde(skip_serializing_if="Option::is_none")] pub search_type: Option, + #[serde(skip_serializing_if="Option::is_none")] pub seq_no_primary_term: Option, + #[serde(skip_serializing_if="Option::is_none")] pub size: Option, + #[serde(skip_serializing_if="Option::is_none")] pub sort: Option, // array of strings, comma-separated + #[serde(skip_serializing_if="Option::is_none")] pub _source: Option, // TODO: bool or string + #[serde(skip_serializing_if="Option::is_none")] pub _source_excludes: Option, // array of strings, comma-separated + #[serde(skip_serializing_if="Option::is_none")] pub _source_includes: Option, // array of strings, comma-separated + #[serde(skip_serializing_if="Option::is_none")] pub stats: Option, + #[serde(skip_serializing_if="Option::is_none")] pub stored_fields: Option, // array of strings, comma-separated + #[serde(skip_serializing_if="Option::is_none")] pub suggest_field: Option, + #[serde(skip_serializing_if="Option::is_none")] pub suggest_text: Option, + #[serde(skip_serializing_if="Option::is_none")] pub terminate_after: Option, + #[serde(skip_serializing_if="Option::is_none")] pub timeout: Option, // string is "time units" + #[serde(skip_serializing_if="Option::is_none")] pub track_scores: Option, + #[serde(skip_serializing_if="Option::is_none")] pub track_total_hits: Option, // XXX: bool or integer + #[serde(skip_serializing_if="Option::is_none")] pub typed_keys: Option, + #[serde(skip_serializing_if="Option::is_none")] pub version: Option, // additional generic params + #[serde(skip_serializing_if="Option::is_none")] pub human: Option, + #[serde(skip_serializing_if="Option::is_none")] pub pretty: Option, + #[serde(skip_serializing_if="Option::is_none")] pub filter_path: Option, + #[serde(skip_serializing_if="Option::is_none")] pub error_trace: Option, } @@ -59,74 +98,142 @@ pub struct UrlQueryParams { #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct SearchBody { + #[serde(skip_serializing_if="Option::is_none")] pub query: Option, + #[serde(skip_serializing_if="Option::is_none")] pub highlight: Option, + #[serde(skip_serializing_if="Option::is_none")] pub collapse: Option, + #[serde(skip_serializing_if="Option::is_none")] pub post_filter: Option, // TODO: leaf query only? + #[serde(skip_serializing_if="Option::is_none")] pub rescore: Option, // TODO: single or an array of rescore objects // script_fields disabled + #[serde(skip_serializing_if="Option::is_none")] + pub aggs: Option>, + #[serde(skip_serializing_if="Option::is_none")] + pub aggregations: Option>, // https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html + #[serde(skip_serializing_if="Option::is_none")] pub sort: Option>, + #[serde(skip_serializing_if="Option::is_none")] pub slice: Option, + #[serde(skip_serializing_if="Option::is_none")] pub stored_fields: Option, // array of strings, or "_none_" // overlap with URL query parameters + #[serde(skip_serializing_if="Option::is_none")] pub docvalue_fields: Option>, + #[serde(skip_serializing_if="Option::is_none")] pub explain: Option, + #[serde(skip_serializing_if="Option::is_none")] pub from: Option, - pub min_score: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub min_score: Option, + #[serde(skip_serializing_if="Option::is_none")] pub seq_no_primary_term: Option, + #[serde(skip_serializing_if="Option::is_none")] pub size: Option, + #[serde(skip_serializing_if="Option::is_none")] pub _source: Option, // XXX: bool, string, or object + #[serde(skip_serializing_if="Option::is_none")] pub terminate_after: Option, + #[serde(skip_serializing_if="Option::is_none")] pub timeout: Option, // string is "time units" } #[derive(Serialize, Deserialize, Debug)] #[serde(deny_unknown_fields)] pub struct ScrollBody { - pub scroll_id: String, + pub scroll_id: StringOrArray, + #[serde(skip_serializing_if="Option::is_none")] pub scroll: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ApiSlice { id: u32, max: u32, + #[serde(skip_serializing_if="Option::is_none")] field: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ApiRescore{ + #[serde(skip_serializing_if="Option::is_none")] pub query: Option, + #[serde(skip_serializing_if="Option::is_none")] pub window_size: Option, } // TODO: could revert to having query types as an enum, with flattening #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ApiQuery { // compound queries #[serde(rename = "bool")] + #[serde(skip_serializing_if="Option::is_none")] bool_query: Option, + #[serde(skip_serializing_if="Option::is_none")] boosting: Option, + #[serde(skip_serializing_if="Option::is_none")] constant_score: Option, // fulltext (leaf) queries - - // term-level (leaf) queries #[serde(rename = "match")] - match_query: Option>, + #[serde(skip_serializing_if="Option::is_none")] + match_query: Option>, + #[serde(skip_serializing_if="Option::is_none")] match_phrase: Option>, - query_string: Option, + #[serde(skip_serializing_if="Option::is_none")] + multi_match: Option, + #[serde(skip_serializing_if="Option::is_none")] + query_string: Option, + #[serde(skip_serializing_if="Option::is_none")] + simple_query_string: Option, + + // term-level (leaf) queries + #[serde(skip_serializing_if="Option::is_none")] + exists: Option, + #[serde(skip_serializing_if="Option::is_none")] + match_all: Option, + #[serde(skip_serializing_if="Option::is_none")] + match_none: Option, + #[serde(skip_serializing_if="Option::is_none")] + ids: Option, + #[serde(skip_serializing_if="Option::is_none")] + wildcard: Option>, // also works for wildcard + #[serde(skip_serializing_if="Option::is_none")] + prefix: Option>, // also works for prefix query + #[serde(skip_serializing_if="Option::is_none")] + range: Option>, + #[serde(skip_serializing_if="Option::is_none")] + term: Option>, + #[serde(skip_serializing_if="Option::is_none")] + // TODO: boost in terms query + terms: Option>>, // other + #[serde(skip_serializing_if="Option::is_none")] nested: Option, + #[serde(skip_serializing_if="Option::is_none")] rescore_query: Option>, + + // fields as part of a rescore query + #[serde(skip_serializing_if="Option::is_none")] + score_mode: Option, + #[serde(skip_serializing_if="Option::is_none")] + query_weight: Option, + #[serde(skip_serializing_if="Option::is_none")] + rescore_query_weight: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ApiHighlight{ // TODO: fields could also be an array of strings? fields: HashMap, @@ -136,13 +243,19 @@ pub struct ApiHighlight{ } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum SortMapValue { String(String), - Object { order: String, mode: Option }, + Object { + order: String, + #[serde(skip_serializing_if="Option::is_none")] + mode: Option, + }, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum SortElement{ String(String), @@ -150,13 +263,208 @@ pub enum SortElement{ } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum DocValOrString { String(String), - Object {field: String, format: Option}, + Object { + field: String, + #[serde(skip_serializing_if="Option::is_none")] + format: Option, + }, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum MatchQueryOrString { + Object(MatchQuery), + String(String), +} + +// https://www.elastic.co/guide/en/elasticsearch/reference/7.9/query-dsl-match-query.html +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct MatchQuery { + query: String, + + #[serde(flatten)] + options: MatchOptions, +} + +// https://www.elastic.co/guide/en/elasticsearch/reference/7.9/query-dsl-multi-match-query.html +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct MultiMatchQuery { + query: String, + fields: Vec, + #[serde(skip_serializing_if="Option::is_none")] + #[serde(rename="type")] + query_type: Option, + #[serde(skip_serializing_if="Option::is_none")] + tie_breaker: Option, + #[serde(flatten)] + options: MatchOptions, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct MatchOptions { + #[serde(skip_serializing_if="Option::is_none")] + analyzer: Option, + #[serde(skip_serializing_if="Option::is_none")] + auto_generate_synonyms_phrase_query: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzziness: Option, + #[serde(skip_serializing_if="Option::is_none")] + max_expansions: Option, + #[serde(skip_serializing_if="Option::is_none")] + prefix_length: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzzy_transpositions: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzzy_rewrite: Option, + #[serde(skip_serializing_if="Option::is_none")] + lenient: Option, + #[serde(skip_serializing_if="Option::is_none")] + operator: Option, + #[serde(skip_serializing_if="Option::is_none")] + minimum_should_match: Option, + #[serde(skip_serializing_if="Option::is_none")] + zero_terms_query: Option, +} + +// https://www.elastic.co/guide/en/elasticsearch/reference/7.9/query-dsl-query-string-query.html +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct QueryStringQuery { + query: String, + #[serde(skip_serializing_if="Option::is_none")] + #[serde(rename="type")] + query_type: Option, + #[serde(skip_serializing_if="Option::is_none")] + default_field: Option, + #[serde(skip_serializing_if="Option::is_none")] + allow_leading_wildcard: Option, + #[serde(skip_serializing_if="Option::is_none")] + analyze_wildcard: Option, + #[serde(skip_serializing_if="Option::is_none")] + analyzer: Option, + #[serde(skip_serializing_if="Option::is_none")] + auto_generate_synonyms_phrase_query: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, + #[serde(skip_serializing_if="Option::is_none")] + default_operator: Option, + #[serde(skip_serializing_if="Option::is_none")] + enable_position_increments: Option, + #[serde(skip_serializing_if="Option::is_none")] + fields: Option>, + #[serde(skip_serializing_if="Option::is_none")] + fuzziness: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzzy_max_expansions: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzzy_prefix_length: Option, + #[serde(skip_serializing_if="Option::is_none")] + fuzzy_transpositions: Option, + #[serde(skip_serializing_if="Option::is_none")] + lenient: Option, + #[serde(skip_serializing_if="Option::is_none")] + max_determinized_states: Option, + #[serde(skip_serializing_if="Option::is_none")] + minimum_should_match: Option, + #[serde(skip_serializing_if="Option::is_none")] + quote_analyzer: Option, + #[serde(skip_serializing_if="Option::is_none")] + phrase_slop: Option, + #[serde(skip_serializing_if="Option::is_none")] + quote_field_suffix: Option, + #[serde(skip_serializing_if="Option::is_none")] + rewrite: Option, + #[serde(skip_serializing_if="Option::is_none")] + time_zone: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct SimpleBoost { + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct IdsQuery { + values: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum TermQueryOrString { + String(String), + Object(TermQuery), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct TermQuery { + value: String, + #[serde(skip_serializing_if="Option::is_none")] + rewrite: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum StringOrNum { + String(String), + Int(u64), + Float(f64), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum Num { + Int(u64), + Float(f64), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum StringOrArray { + String(String), + Array(Vec), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct RangeQuery { + #[serde(skip_serializing_if="Option::is_none")] + gt: Option, + #[serde(skip_serializing_if="Option::is_none")] + gte: Option, + #[serde(skip_serializing_if="Option::is_none")] + lt: Option, + #[serde(skip_serializing_if="Option::is_none")] + lte: Option, + #[serde(skip_serializing_if="Option::is_none")] + format: Option, + #[serde(skip_serializing_if="Option::is_none")] + relation: Option, + #[serde(skip_serializing_if="Option::is_none")] + timezone: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum QueryFieldOrString { Object(QueryField), @@ -164,77 +472,128 @@ pub enum QueryFieldOrString { } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct QueryField{ query: String, + #[serde(skip_serializing_if="Option::is_none")] fuzziness: Option, + #[serde(skip_serializing_if="Option::is_none")] slop: Option, - boost: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct BoolQuery { + #[serde(skip_serializing_if="Option::is_none")] must: Option>, + #[serde(skip_serializing_if="Option::is_none")] filter: Option>, + #[serde(skip_serializing_if="Option::is_none")] should: Option>, + #[serde(skip_serializing_if="Option::is_none")] must_not: Option>, - minimum_should_match: Option, - boost: Option, + #[serde(skip_serializing_if="Option::is_none")] + minimum_should_match: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct NestedQuery { path: String, query: Box, + #[serde(skip_serializing_if="Option::is_none")] score_mode: Option, + #[serde(skip_serializing_if="Option::is_none")] ignore_unmapped: Option, + #[serde(skip_serializing_if="Option::is_none")] + inner_hits: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct BoostingQuery { positive: Box, negative: Box, - negative_boost: f64, + negative_boost: Num, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ConstantScoreQuery { filter: Box, - boost: Option, + #[serde(skip_serializing_if="Option::is_none")] + boost: Option, } // https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct HighlightField{ + #[serde(skip_serializing_if="Option::is_none")] boundary_chars: Option, + #[serde(skip_serializing_if="Option::is_none")] boundary_max_scan: Option, + #[serde(skip_serializing_if="Option::is_none")] boundary_scanner: Option, + #[serde(skip_serializing_if="Option::is_none")] boundary_scanner_locale: Option, + #[serde(skip_serializing_if="Option::is_none")] encoder: Option, + #[serde(skip_serializing_if="Option::is_none")] force_source: Option, + #[serde(skip_serializing_if="Option::is_none")] fragmenter: Option, + #[serde(skip_serializing_if="Option::is_none")] fragment_offset: Option, + #[serde(skip_serializing_if="Option::is_none")] fragment_size: Option, + #[serde(skip_serializing_if="Option::is_none")] highlight_query: Option, + #[serde(skip_serializing_if="Option::is_none")] matched_fields: Option>, + #[serde(skip_serializing_if="Option::is_none")] no_match_size: Option, + #[serde(skip_serializing_if="Option::is_none")] number_of_fragments: Option, + #[serde(skip_serializing_if="Option::is_none")] order: Option, + #[serde(skip_serializing_if="Option::is_none")] phrase_limit: Option, + #[serde(skip_serializing_if="Option::is_none")] pre_tags: Option>, + #[serde(skip_serializing_if="Option::is_none")] post_tags: Option>, + #[serde(skip_serializing_if="Option::is_none")] require_field_match: Option, + #[serde(skip_serializing_if="Option::is_none")] tags_schema: Option, #[serde(rename = "type")] + #[serde(skip_serializing_if="Option::is_none")] highlight_type: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +#[serde(untagged)] +pub enum SimpleFieldOrString { + String(String), + Object { field: String }, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct ApiCollapse{ field: String, + #[serde(skip_serializing_if="Option::is_none")] inner_hits: Option, } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum InnerHitsOneOrMore { Single(InnerHits), @@ -242,9 +601,118 @@ pub enum InnerHitsOneOrMore { } #[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] pub struct InnerHits { + #[serde(skip_serializing_if="Option::is_none")] from: Option, + #[serde(skip_serializing_if="Option::is_none")] size: Option, + #[serde(skip_serializing_if="Option::is_none")] sort: Option>, + #[serde(skip_serializing_if="Option::is_none")] name: Option, + #[serde(skip_serializing_if="Option::is_none")] + collapse: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct ApiAggregation { + // bucket type aggregations + #[serde(skip_serializing_if="Option::is_none")] + nested: Option, + #[serde(skip_serializing_if="Option::is_none")] + filter: Option, + #[serde(skip_serializing_if="Option::is_none")] + histogram: Option, + #[serde(skip_serializing_if="Option::is_none")] + terms: Option, + #[serde(skip_serializing_if="Option::is_none")] + significant_terms: Option, + + // metrics type aggregations + #[serde(skip_serializing_if="Option::is_none")] + avg: Option, + #[serde(skip_serializing_if="Option::is_none")] + min: Option, + #[serde(skip_serializing_if="Option::is_none")] + max: Option, + #[serde(skip_serializing_if="Option::is_none")] + sum: Option, + #[serde(skip_serializing_if="Option::is_none")] + value_count: Option, + #[serde(skip_serializing_if="Option::is_none")] + stats: Option, + #[serde(skip_serializing_if="Option::is_none")] + percentiles: Option, + + // nested aggregations + #[serde(skip_serializing_if="Option::is_none")] + aggs: Option>>, + #[serde(skip_serializing_if="Option::is_none")] + aggregations: Option>>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct NestedAggregation{ + path: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct SimpleAggregation{ + field: String, + #[serde(skip_serializing_if="Option::is_none")] + interval: Option, + #[serde(skip_serializing_if="Option::is_none")] + missing: Option, + #[serde(skip_serializing_if="Option::is_none")] + keyed: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct DateHistogramAggregation{ + field: String, + #[serde(skip_serializing_if="Option::is_none")] + fixed_interval: Option, + #[serde(skip_serializing_if="Option::is_none")] + calendar_interval: Option, + #[serde(skip_serializing_if="Option::is_none")] + format: Option, + #[serde(skip_serializing_if="Option::is_none")] + time_zone: Option, + #[serde(skip_serializing_if="Option::is_none")] + offset: Option, + #[serde(skip_serializing_if="Option::is_none")] + order: Option>, + #[serde(skip_serializing_if="Option::is_none")] + keyed: Option, + #[serde(skip_serializing_if="Option::is_none")] + missing: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct TermsAggregation { + field: String, + #[serde(skip_serializing_if="Option::is_none")] + size: Option, + #[serde(skip_serializing_if="Option::is_none")] + shard_size: Option, + #[serde(skip_serializing_if="Option::is_none")] + min_doc_count: Option, + #[serde(skip_serializing_if="Option::is_none")] + show_term_doc_count_error: Option, + #[serde(skip_serializing_if="Option::is_none")] + order: Option>, + #[serde(skip_serializing_if="Option::is_none")] + include: Option, + #[serde(skip_serializing_if="Option::is_none")] + exclude: Option, + #[serde(skip_serializing_if="Option::is_none")] + execution_hint: Option, + #[serde(skip_serializing_if="Option::is_none")] + missing: Option, } diff --git a/tests/files/GET_range.txt b/tests/files/GET_range.txt new file mode 100644 index 0000000..a5321c3 --- /dev/null +++ b/tests/files/GET_range.txt @@ -0,0 +1,12 @@ +GET /some-index/_search +{ + "query": { + "range": { + "age": { + "gte": 10, + "lte": 20, + "boost": 2.0 + } + } + } +} diff --git a/tests/files/GET_search_agg_filter.txt b/tests/files/GET_search_agg_filter.txt new file mode 100644 index 0000000..538380a --- /dev/null +++ b/tests/files/GET_search_agg_filter.txt @@ -0,0 +1,28 @@ +GET /shirts/_search +{ + "query": { + "bool": { + "filter": { + "term": { "brand": "gucci" } + } + } + }, + "aggs": { + "colors": { + "terms": { "field": "color" } + }, + "color_red": { + "filter": { + "term": { "color": "red" } + }, + "aggs": { + "models": { + "terms": { "field": "model" } + } + } + } + }, + "post_filter": { + "term": { "color": "red" } + } +} diff --git a/tests/files/GET_search_agg_filter_simple.txt b/tests/files/GET_search_agg_filter_simple.txt new file mode 100644 index 0000000..71a9777 --- /dev/null +++ b/tests/files/GET_search_agg_filter_simple.txt @@ -0,0 +1,11 @@ +POST /sales/_search?size=0 +{ + "aggs": { + "t_shirts": { + "filter": { "term": { "type": "t-shirt" } }, + "aggs": { + "avg_price": { "avg": { "field": "price" } } + } + } + } +} diff --git a/tests/files/GET_search_agg_histogram.txt b/tests/files/GET_search_agg_histogram.txt new file mode 100644 index 0000000..4689ad1 --- /dev/null +++ b/tests/files/GET_search_agg_histogram.txt @@ -0,0 +1,13 @@ +POST /sales/_search?size=0 +{ + "aggs": { + "quantity": { + "histogram": { + "field": "quantity", + "interval": 10, + "keyed": true, + "missing": 0 + } + } + } +} diff --git a/tests/files/GET_search_agg_max.txt b/tests/files/GET_search_agg_max.txt new file mode 100644 index 0000000..1377afc --- /dev/null +++ b/tests/files/GET_search_agg_max.txt @@ -0,0 +1,11 @@ +POST /sales/_search +{ + "aggs" : { + "grade_max" : { + "max" : { + "field" : "grade", + "missing": 10 + } + } + } +} diff --git a/tests/files/GET_search_agg_nested.txt b/tests/files/GET_search_agg_nested.txt new file mode 100644 index 0000000..8e5d1c1 --- /dev/null +++ b/tests/files/GET_search_agg_nested.txt @@ -0,0 +1,16 @@ +GET /products/_search +{ + "query": { + "match": { "name": "led tv" } + }, + "aggs": { + "resellers": { + "nested": { + "path": "resellers" + }, + "aggs": { + "min_price": { "min": { "field": "resellers.price" } } + } + } + } +} diff --git a/tests/files/GET_search_exists.txt b/tests/files/GET_search_exists.txt new file mode 100644 index 0000000..fe7d722 --- /dev/null +++ b/tests/files/GET_search_exists.txt @@ -0,0 +1,8 @@ +GET /some-index/_search +{ + "query": { + "exists": { + "field": "user" + } + } +} diff --git a/tests/files/GET_search_fatcat_preservation_year.txt b/tests/files/GET_search_fatcat_preservation_year.txt new file mode 100644 index 0000000..539677b --- /dev/null +++ b/tests/files/GET_search_fatcat_preservation_year.txt @@ -0,0 +1,2 @@ +GET /fatcat_release/_search +{"query": {"bool": {"filter": [{"term": {"container_id": "yh4zdhfsobdolesogv6czydwqi"}}, {"range": {"release_year": {"gte": 1771, "lte": 2020}}}]}}, "aggs": {"year_preservation": {"composite": {"size": 1500, "sources": [{"year": {"histogram": {"field": "release_year", "interval": 1}}}, {"preservation": {"terms": {"field": "preservation"}}}]}}}, "from": 0, "size": 0} diff --git a/tests/files/GET_search_fatcat_scholar_basic.txt b/tests/files/GET_search_fatcat_scholar_basic.txt new file mode 100644 index 0000000..c60e62f --- /dev/null +++ b/tests/files/GET_search_fatcat_scholar_basic.txt @@ -0,0 +1,2 @@ +GET /scholar_fulltext/_search +{"query": {"bool": {"filter": [{"terms": {"type": ["article-journal", "paper-conference", "chapter"]}}, {"terms": {"access_type": ["wayback", "ia_file", "ia_sim"]}}], "must": [{"boosting": {"positive": {"bool": {"must": [{"query_string": {"query": "coffee", "default_operator": "AND", "analyze_wildcard": true, "allow_leading_wildcard": false, "lenient": true, "quote_field_suffix": ".exact", "fields": ["title^5", "biblio_all^3", "abstracts.body^2", "fulltext.body", "everything"]}}], "should": [{"terms": {"access_type": ["ia_sim", "ia_file", "wayback"]}}]}}, "negative": {"bool": {"should": [{"bool": {"must_not": [{"exists": {"field": "title"}}]}}, {"bool": {"must_not": [{"exists": {"field": "year"}}]}}, {"bool": {"must_not": [{"exists": {"field": "type"}}]}}, {"bool": {"must_not": [{"exists": {"field": "stage"}}]}}, {"bool": {"must_not": [{"exists": {"field": "biblio.container_ident"}}]}}]}}, "negative_boost": 0.5}}]}}, "collapse": {"field": "collapse_key", "inner_hits": {"name": "more_pages", "size": 0}}, "from": 0, "size": 15, "highlight": {"fields": {"abstracts.body": {"number_of_fragments": 2, "fragment_size": 300}, "fulltext.body": {"number_of_fragments": 2, "fragment_size": 300}, "fulltext.acknowledgment": {"number_of_fragments": 2, "fragment_size": 300}, "fulltext.annex": {"number_of_fragments": 2, "fragment_size": 300}}}} diff --git a/tests/files/GET_search_ids.txt b/tests/files/GET_search_ids.txt new file mode 100644 index 0000000..f22c8b1 --- /dev/null +++ b/tests/files/GET_search_ids.txt @@ -0,0 +1,8 @@ +GET /some-index/_search +{ + "query": { + "ids" : { + "values" : ["1", "4", "100"] + } + } +} diff --git a/tests/files/GET_search_match_all.txt b/tests/files/GET_search_match_all.txt new file mode 100644 index 0000000..1d9622c --- /dev/null +++ b/tests/files/GET_search_match_all.txt @@ -0,0 +1,6 @@ +GET /some-index/_search +{ + "query": { + "match_all": { "boost" : 1.2 } + } +} diff --git a/tests/files/GET_search_match_none.txt b/tests/files/GET_search_match_none.txt new file mode 100644 index 0000000..12f50fc --- /dev/null +++ b/tests/files/GET_search_match_none.txt @@ -0,0 +1,6 @@ +GET /some-index/_search +{ + "query": { + "match_none": {} + } +} diff --git a/tests/files/GET_search_multi_match_tie.txt b/tests/files/GET_search_multi_match_tie.txt new file mode 100644 index 0000000..54f71db --- /dev/null +++ b/tests/files/GET_search_multi_match_tie.txt @@ -0,0 +1,11 @@ +GET /some-index/_search +{ + "query": { + "multi_match" : { + "query": "brown fox", + "type": "best_fields", + "fields": [ "subject", "message" ], + "tie_breaker": 0.3 + } + } +} diff --git a/tests/files/GET_search_prefix.txt b/tests/files/GET_search_prefix.txt new file mode 100644 index 0000000..3e35556 --- /dev/null +++ b/tests/files/GET_search_prefix.txt @@ -0,0 +1,10 @@ +GET /some-index/_search +{ + "query": { + "prefix": { + "user.id": { + "value": "ki" + } + } + } +} diff --git a/tests/files/GET_search_querystring.txt b/tests/files/GET_search_querystring.txt new file mode 100644 index 0000000..1c6e31e --- /dev/null +++ b/tests/files/GET_search_querystring.txt @@ -0,0 +1,14 @@ +GET /some-index/_search +{ + "query": { + "query_string": { + "fields": [ + "title", + "content" + ], + "query": "this OR that OR thus", + "type": "cross_fields", + "minimum_should_match": 2 + } + } +} diff --git a/tests/files/GET_search_simplequerystring.txt b/tests/files/GET_search_simplequerystring.txt new file mode 100644 index 0000000..2ae5507 --- /dev/null +++ b/tests/files/GET_search_simplequerystring.txt @@ -0,0 +1,10 @@ +GET /some-index/_search +{ + "query": { + "simple_query_string" : { + "query": "\"fried eggs\" +(eggplant | potato) -frittata", + "fields": ["title^5", "body"], + "default_operator": "and" + } + } +} diff --git a/tests/files/GET_search_term.txt b/tests/files/GET_search_term.txt new file mode 100644 index 0000000..149587f --- /dev/null +++ b/tests/files/GET_search_term.txt @@ -0,0 +1,11 @@ +GET /some-value/_search +{ + "query": { + "term": { + "user.id": { + "value": "kimchy", + "boost": 1.0 + } + } + } +} diff --git a/tests/files/GET_search_term_short.txt b/tests/files/GET_search_term_short.txt new file mode 100644 index 0000000..937a7b5 --- /dev/null +++ b/tests/files/GET_search_term_short.txt @@ -0,0 +1,8 @@ +GET /my-index-000001/_search?pretty=true +{ + "query": { + "match": { + "full_text": "Quick Brown Foxes!" + } + } +} diff --git a/tests/files/GET_search_wildcard.txt b/tests/files/GET_search_wildcard.txt new file mode 100644 index 0000000..6eb199d --- /dev/null +++ b/tests/files/GET_search_wildcard.txt @@ -0,0 +1,12 @@ +GET /some-index/_search +{ + "query": { + "wildcard": { + "user.id": { + "value": "ki*y", + "boost": 1.0, + "rewrite": "constant_score" + } + } + } +} diff --git a/tests/files/POST_search_inner.txt b/tests/files/POST_search_inner.txt deleted file mode 100644 index 56c5acc..0000000 --- a/tests/files/POST_search_inner.txt +++ /dev/null @@ -1,17 +0,0 @@ -POST /test/_search -{ - "query": { - "nested": { - "path": "comments", - "query": { - "match": { "comments.text": "words" } - }, - "inner_hits": { - "_source": false, - "docvalue_fields": [ - "comments.text.keyword" - ] - } - } - } -} diff --git a/tests/files/POST_search_inner.txt.disabled b/tests/files/POST_search_inner.txt.disabled new file mode 100644 index 0000000..56c5acc --- /dev/null +++ b/tests/files/POST_search_inner.txt.disabled @@ -0,0 +1,17 @@ +POST /test/_search +{ + "query": { + "nested": { + "path": "comments", + "query": { + "match": { "comments.text": "words" } + }, + "inner_hits": { + "_source": false, + "docvalue_fields": [ + "comments.text.keyword" + ] + } + } + } +} diff --git a/tests/files/scroll/DELETE_scroll_multi.txt b/tests/files/scroll/DELETE_scroll_multi.txt new file mode 100644 index 0000000..43ba8e2 --- /dev/null +++ b/tests/files/scroll/DELETE_scroll_multi.txt @@ -0,0 +1,7 @@ +DELETE /_search/scroll +{ + "scroll_id" : [ + "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAAAD4WYm9laVYtZndUQlNsdDcwakFMNjU1QQ==", + "DnF1ZXJ5VGhlbkZldGNoBQAAAAAAAAABFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAAAxZrUllkUVlCa1NqNmRMaUhiQlZkMWFBAAAAAAAAAAIWa1JZZFFZQmtTajZkTGlIYkJWZDFhQQAAAAAAAAAFFmtSWWRRWUJrU2o2ZExpSGJCVmQxYUEAAAAAAAAABBZrUllkUVlCa1NqNmRMaUhiQlZkMWFB" + ] +} diff --git a/tests/parse_es_requests.rs b/tests/parse_es_requests.rs index d453392..8e84148 100644 --- a/tests/parse_es_requests.rs +++ b/tests/parse_es_requests.rs @@ -36,7 +36,9 @@ fn parse_search_bodies() { if let Some(body) = parts.body { println!("parsing: {}", path.display()); println!("BODY: {}", body); - let _parsed: SearchBody = serde_json::from_str(&body).unwrap(); + let parsed: SearchBody = serde_json::from_str(&body).unwrap(); + let raw_val: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(raw_val, serde_json::to_value(parsed).unwrap()); } } } -- cgit v1.2.3