From 3aaa3d1e80284b4c4dd39cd83b73f586e2831d5d Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Mon, 24 Aug 2020 21:37:17 -0700
Subject: move request body parsing to module/file

---
 src/lib.rs                 | 243 +--------------------------------------------
 src/parse.rs               | 242 ++++++++++++++++++++++++++++++++++++++++++++
 tests/parse_es_requests.rs |   2 +-
 3 files changed, 245 insertions(+), 242 deletions(-)
 create mode 100644 src/parse.rs

diff --git a/src/lib.rs b/src/lib.rs
index 9f5b8a2..5fd54c9 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1,9 +1,9 @@
 
+use serde::{Serialize, Deserialize};
 use hyper::{Request, Body, Method, Uri};
 use http::request;
-use serde::{Serialize, Deserialize};
-use std::collections::HashMap;
 
+pub mod parse;
 
 #[derive(Default, Deserialize, Debug, Clone)]
 pub struct ProxyConfig {
@@ -29,245 +29,6 @@ impl ProxyConfig {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiRequest {
-    pub method: String,
-    pub path_and_query: String,
-    pub body: Option<SearchBody>,
-}
-
-pub struct UrlQueryParams {
-    pub allow_no_indices: Option<bool>,
-    pub allow_partial_search_results: Option<bool>,
-    pub batched_reduce_size: Option<u32>,
-    pub ccs_minimize_roundtrips: Option<bool>,
-    pub docvalue_fields: Option<String>, // array of strings, comma-separated
-    pub expand_wildcards: Option<String>,
-    pub explain: Option<bool>,
-    pub from: Option<u32>,
-    pub ignore_throttled: Option<bool>,
-    pub ignore_unavailable: Option<bool>,
-    pub max_concurrent_shard_requests: Option<u32>,
-    pub pre_filter_shard_size: Option<u32>,
-    pub preference: Option<String>,
-    pub q: Option<String>,
-    pub request_cache: Option<bool>,
-    pub rest_total_hits_as_int: Option<bool>,
-    pub routing: Option<String>,
-    pub scroll: Option<String>, // string is "time value"
-    pub search_type: Option<String>,
-    pub seq_no_primary_term: Option<bool>,
-    pub size: Option<u32>,
-    pub sort: Option<String>, // array of strings, comma-separated
-    pub _source: Option<bool>, // TODO: bool or string
-    pub _source_excludes: Option<String>, // array of strings, comma-separated
-    pub _source_includes: Option<String>, // array of strings, comma-separated
-    pub stats: Option<String>,
-    pub stored_fields: Option<String>, // array of strings, comma-separated
-    pub suggest_field: Option<String>,
-    pub suggest_text: Option<String>,
-    pub terminate_after: Option<u32>,
-    pub timeout: Option<String>, // string is "time units"
-    pub track_scores: Option<bool>,
-    pub track_total_hits: Option<bool>, // XXX: bool or integer
-    pub typed_keys: Option<bool>,
-    pub version: Option<bool>,
-}
-
-// https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(deny_unknown_fields)]
-pub struct SearchBody {
-    pub query: Option<ApiQuery>,
-    pub highlight: Option<ApiHighlight>,
-    pub collapse: Option<ApiCollapse>,
-    pub post_filter: Option<ApiQuery>, // TODO: leaf query only?
-    pub rescore: Option<ApiRescore>, // TODO: single or an array of rescore objects
-    // script_fields disabled
-
-    // https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html
-    pub sort: Option<Vec<SortElement>>,
-
-    pub slice: Option<ApiSlice>,
-    pub stored_fields: Option<String>, // array of strings, or "_none_"
-
-    // overlap with URL query parameters
-    pub docvalue_fields: Option<Vec<DocValOrString>>,
-    pub explain: Option<bool>,
-    pub from: Option<u32>,
-    pub min_score: Option<f64>,
-    pub seq_no_primary_term: Option<bool>,
-    pub size: Option<u32>,
-    pub _source: Option<bool>, // XXX: bool, string, or object
-    pub terminate_after: Option<u32>,
-    pub timeout: Option<String>, // string is "time units"
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(deny_unknown_fields)]
-pub struct ScrollBody {
-    pub scroll_id: String,
-    pub scroll: Option<String>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiSlice {
-    id: u32,
-    max: u32,
-    field: Option<String>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiRescore{
-    pub query: Option<ApiQuery>,
-    pub window_size: Option<u32>,
-}
-
-// TODO: could revert to having query types as an enum, with flattening
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiQuery {
-    // compound queries
-    #[serde(rename = "bool")]
-    bool_query: Option<BoolQuery>,
-    boosting: Option<BoostingQuery>,
-    constant_score: Option<ConstantScoreQuery>,
-
-    // fulltext (leaf) queries
-
-    // term-level (leaf) queries
-    #[serde(rename = "match")]
-    match_query: Option<HashMap<String, QueryFieldOrString>>,
-    match_phrase: Option<HashMap<String, QueryFieldOrString>>,
-    query_string: Option<QueryField>,
-
-    // other
-    nested: Option<NestedQuery>,
-    rescore_query: Option<Box<ApiQuery>>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiHighlight{
-    // TODO: fields could also be an array of strings?
-    fields: HashMap<String, HighlightField>,
-
-    #[serde(flatten)]
-    settings: HighlightField,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(untagged)]
-pub enum SortMapValue {
-    String(String),
-    Object { order: String, mode: Option<String> },
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(untagged)]
-pub enum SortElement{
-    String(String),
-    Object(HashMap<String, SortMapValue>),
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(untagged)]
-pub enum DocValOrString {
-    String(String),
-    Object {field: String, format: Option<String>},
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(untagged)]
-pub enum QueryFieldOrString {
-    Object(QueryField),
-    String(String),
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct QueryField{
-    query: String,
-    fuzziness: Option<String>,
-    slop: Option<u32>,
-    boost: Option<f64>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BoolQuery {
-    must: Option<Box<ApiQuery>>,
-    filter: Option<Box<ApiQuery>>,
-    should: Option<Box<ApiQuery>>,
-    must_not: Option<Box<ApiQuery>>,
-    minimum_should_match: Option<u32>,
-    boost: Option<f64>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct NestedQuery {
-    path: String,
-    query: Box<ApiQuery>,
-    score_mode: Option<String>,
-    ignore_unmapped: Option<bool>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct BoostingQuery {
-    positive: Box<ApiQuery>,
-    negative: Box<ApiQuery>,
-    negative_boost: f64,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ConstantScoreQuery {
-    filter: Box<ApiQuery>,
-    boost: Option<f64>,
-}
-
-// https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html
-#[derive(Serialize, Deserialize, Debug)]
-pub struct HighlightField{
-    boundary_chars: Option<String>,
-    boundary_max_scan: Option<u32>,
-    boundary_scanner: Option<String>,
-    boundary_scanner_locale: Option<String>,
-    encoder: Option<String>,
-    force_source: Option<bool>,
-    fragmenter: Option<String>,
-    fragment_offset: Option<u32>,
-    fragment_size: Option<u32>,
-    highlight_query: Option<ApiQuery>,
-    matched_fields: Option<Vec<String>>,
-    no_match_size: Option<u32>,
-    number_of_fragments: Option<u32>,
-    order: Option<String>,
-    phrase_limit: Option<u32>,
-    pre_tags: Option<Vec<String>>,
-    post_tags: Option<Vec<String>>,
-    require_field_match: Option<bool>,
-    tags_schema: Option<String>,
-    #[serde(rename = "type")]
-    highlight_type: Option<String>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct ApiCollapse{
-    field: String,
-    inner_hits: Option<InnerHitsOneOrMore>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(untagged)]
-pub enum InnerHitsOneOrMore {
-    Single(InnerHits),
-    Multiple(Vec<InnerHits>),
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct InnerHits {
-    from: Option<u32>,
-    size: Option<u32>,
-    sort: Option<Vec<SortElement>>,
-    name: Option<String>,
-}
-
 #[derive(Debug)]
 pub enum ParsedRequest {
     Malformed(String),
diff --git a/src/parse.rs b/src/parse.rs
new file mode 100644
index 0000000..a4cdc3c
--- /dev/null
+++ b/src/parse.rs
@@ -0,0 +1,242 @@
+
+use serde::{Serialize, Deserialize};
+use std::collections::HashMap;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiRequest {
+    pub method: String,
+    pub path_and_query: String,
+    pub body: Option<SearchBody>,
+}
+
+pub struct UrlQueryParams {
+    pub allow_no_indices: Option<bool>,
+    pub allow_partial_search_results: Option<bool>,
+    pub batched_reduce_size: Option<u32>,
+    pub ccs_minimize_roundtrips: Option<bool>,
+    pub docvalue_fields: Option<String>, // array of strings, comma-separated
+    pub expand_wildcards: Option<String>,
+    pub explain: Option<bool>,
+    pub from: Option<u32>,
+    pub ignore_throttled: Option<bool>,
+    pub ignore_unavailable: Option<bool>,
+    pub max_concurrent_shard_requests: Option<u32>,
+    pub pre_filter_shard_size: Option<u32>,
+    pub preference: Option<String>,
+    pub q: Option<String>,
+    pub request_cache: Option<bool>,
+    pub rest_total_hits_as_int: Option<bool>,
+    pub routing: Option<String>,
+    pub scroll: Option<String>, // string is "time value"
+    pub search_type: Option<String>,
+    pub seq_no_primary_term: Option<bool>,
+    pub size: Option<u32>,
+    pub sort: Option<String>, // array of strings, comma-separated
+    pub _source: Option<bool>, // TODO: bool or string
+    pub _source_excludes: Option<String>, // array of strings, comma-separated
+    pub _source_includes: Option<String>, // array of strings, comma-separated
+    pub stats: Option<String>,
+    pub stored_fields: Option<String>, // array of strings, comma-separated
+    pub suggest_field: Option<String>,
+    pub suggest_text: Option<String>,
+    pub terminate_after: Option<u32>,
+    pub timeout: Option<String>, // string is "time units"
+    pub track_scores: Option<bool>,
+    pub track_total_hits: Option<bool>, // XXX: bool or integer
+    pub typed_keys: Option<bool>,
+    pub version: Option<bool>,
+}
+
+// https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct SearchBody {
+    pub query: Option<ApiQuery>,
+    pub highlight: Option<ApiHighlight>,
+    pub collapse: Option<ApiCollapse>,
+    pub post_filter: Option<ApiQuery>, // TODO: leaf query only?
+    pub rescore: Option<ApiRescore>, // TODO: single or an array of rescore objects
+    // script_fields disabled
+
+    // https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html
+    pub sort: Option<Vec<SortElement>>,
+
+    pub slice: Option<ApiSlice>,
+    pub stored_fields: Option<String>, // array of strings, or "_none_"
+
+    // overlap with URL query parameters
+    pub docvalue_fields: Option<Vec<DocValOrString>>,
+    pub explain: Option<bool>,
+    pub from: Option<u32>,
+    pub min_score: Option<f64>,
+    pub seq_no_primary_term: Option<bool>,
+    pub size: Option<u32>,
+    pub _source: Option<bool>, // XXX: bool, string, or object
+    pub terminate_after: Option<u32>,
+    pub timeout: Option<String>, // string is "time units"
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(deny_unknown_fields)]
+pub struct ScrollBody {
+    pub scroll_id: String,
+    pub scroll: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiSlice {
+    id: u32,
+    max: u32,
+    field: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiRescore{
+    pub query: Option<ApiQuery>,
+    pub window_size: Option<u32>,
+}
+
+// TODO: could revert to having query types as an enum, with flattening
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiQuery {
+    // compound queries
+    #[serde(rename = "bool")]
+    bool_query: Option<BoolQuery>,
+    boosting: Option<BoostingQuery>,
+    constant_score: Option<ConstantScoreQuery>,
+
+    // fulltext (leaf) queries
+
+    // term-level (leaf) queries
+    #[serde(rename = "match")]
+    match_query: Option<HashMap<String, QueryFieldOrString>>,
+    match_phrase: Option<HashMap<String, QueryFieldOrString>>,
+    query_string: Option<QueryField>,
+
+    // other
+    nested: Option<NestedQuery>,
+    rescore_query: Option<Box<ApiQuery>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiHighlight{
+    // TODO: fields could also be an array of strings?
+    fields: HashMap<String, HighlightField>,
+
+    #[serde(flatten)]
+    settings: HighlightField,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum SortMapValue {
+    String(String),
+    Object { order: String, mode: Option<String> },
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum SortElement{
+    String(String),
+    Object(HashMap<String, SortMapValue>),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum DocValOrString {
+    String(String),
+    Object {field: String, format: Option<String>},
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum QueryFieldOrString {
+    Object(QueryField),
+    String(String),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct QueryField{
+    query: String,
+    fuzziness: Option<String>,
+    slop: Option<u32>,
+    boost: Option<f64>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct BoolQuery {
+    must: Option<Box<ApiQuery>>,
+    filter: Option<Box<ApiQuery>>,
+    should: Option<Box<ApiQuery>>,
+    must_not: Option<Box<ApiQuery>>,
+    minimum_should_match: Option<u32>,
+    boost: Option<f64>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct NestedQuery {
+    path: String,
+    query: Box<ApiQuery>,
+    score_mode: Option<String>,
+    ignore_unmapped: Option<bool>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct BoostingQuery {
+    positive: Box<ApiQuery>,
+    negative: Box<ApiQuery>,
+    negative_boost: f64,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ConstantScoreQuery {
+    filter: Box<ApiQuery>,
+    boost: Option<f64>,
+}
+
+// https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html
+#[derive(Serialize, Deserialize, Debug)]
+pub struct HighlightField{
+    boundary_chars: Option<String>,
+    boundary_max_scan: Option<u32>,
+    boundary_scanner: Option<String>,
+    boundary_scanner_locale: Option<String>,
+    encoder: Option<String>,
+    force_source: Option<bool>,
+    fragmenter: Option<String>,
+    fragment_offset: Option<u32>,
+    fragment_size: Option<u32>,
+    highlight_query: Option<ApiQuery>,
+    matched_fields: Option<Vec<String>>,
+    no_match_size: Option<u32>,
+    number_of_fragments: Option<u32>,
+    order: Option<String>,
+    phrase_limit: Option<u32>,
+    pre_tags: Option<Vec<String>>,
+    post_tags: Option<Vec<String>>,
+    require_field_match: Option<bool>,
+    tags_schema: Option<String>,
+    #[serde(rename = "type")]
+    highlight_type: Option<String>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ApiCollapse{
+    field: String,
+    inner_hits: Option<InnerHitsOneOrMore>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(untagged)]
+pub enum InnerHitsOneOrMore {
+    Single(InnerHits),
+    Multiple(Vec<InnerHits>),
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct InnerHits {
+    from: Option<u32>,
+    size: Option<u32>,
+    sort: Option<Vec<SortElement>>,
+    name: Option<String>,
+}
diff --git a/tests/parse_es_requests.rs b/tests/parse_es_requests.rs
index a4950c0..c34dfad 100644
--- a/tests/parse_es_requests.rs
+++ b/tests/parse_es_requests.rs
@@ -1,7 +1,7 @@
 
 use std::fs;
 use std::ffi::OsStr;
-use es_public_proxy::{ScrollBody, SearchBody};
+use es_public_proxy::parse::{ScrollBody, SearchBody};
 
 mod common;
 
-- 
cgit v1.2.3