Merge branch 'martin-datacite-import'

Pipfile.lock is broken.

* martin-datacite-import: (68 commits)
  datacite: pass in doi into factored out method
  datacite: reformat test cases and use jq . --sort-keys
  datacite: factor out contributor handling
  datacite: catch type mismatch in language detection
  datacite: adjust tests for release_month
  datacite: name extra.month, extra.release_month
  datacite: mark additional files as stub
  datacite: CCDC are entries, mostly
  datacite: use more specific release_type, if possible
  datacite: ignore certain names
  datacite: over 3% records have the same title: stub
  datacite: fill a few more release_type gaps
  datacite: adding datacite-specific extra metadata
  datacite: apply pylint suggestions
  datacite: fix typos
  datacite: set release_stage to published by default
  datacite: month field should be top-level
  datacite: include month in extra
  datacite: indicate mismatched file in test
  datacite: clean abstracts, use unknown value tokens
  ...

1 files changed, 598 insertions, 0 deletions

diff --git a/python/tests/files/datacite/datacite_doc_05.json b/python/tests/files/datacite/datacite_doc_05.json
new file mode 100644
index 00000000..76fb73a8
--- /dev/null
+++ b/python/tests/files/datacite/datacite_doc_05.json
@@ -0,0 +1,598 @@
+{
+  "attributes": {
+    "container": {},
+    "contentUrl": null,
+    "contributors": [
+      {
+        "affiliation": [],
+        "name": "Kessy Abarenkov"
+      },
+      {
+        "affiliation": [],
+        "name": "NHM UT-University Of Tartu; Natural History Museum And Botanic Garden"
+      }
+    ],
+    "created": "2015-06-05T10:23:18.000Z",
+    "creators": [
+      {
+        "affiliation": [],
+        "familyName": "Kõljalg",
+        "givenName": "Urmas",
+        "name": "Kõljalg, Urmas",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Abarenkov",
+        "givenName": "Kessy",
+        "name": "Abarenkov, Kessy",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Nilsson",
+        "givenName": "R. Henrik",
+        "name": "Nilsson, R. Henrik",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Larsson",
+        "givenName": "Karl-Henrik",
+        "name": "Larsson, Karl-Henrik",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Aas",
+        "givenName": "Anders Bjørnsgard",
+        "name": "Aas, Anders Bjørnsgard",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Adams",
+        "givenName": "Rachel",
+        "name": "Adams, Rachel",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Alves",
+        "givenName": "Artur",
+        "name": "Alves, Artur",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Ammirati",
+        "givenName": "Joseph F.",
+        "name": "Ammirati, Joseph F.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Arnold",
+        "givenName": "A. Elizabeth",
+        "name": "Arnold, A. Elizabeth",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Bahram",
+        "givenName": "Mohammad",
+        "name": "Bahram, Mohammad",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Bengtsson-Palme",
+        "givenName": "Johan",
+        "name": "Bengtsson-Palme, Johan",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Berlin",
+        "givenName": "Anna",
+        "name": "Berlin, Anna",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Botnen",
+        "givenName": "Synnøve",
+        "name": "Botnen, Synnøve",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Bourlat",
+        "givenName": "Sarah",
+        "name": "Bourlat, Sarah",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Cheeke",
+        "givenName": "Tanya",
+        "name": "Cheeke, Tanya",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Dima",
+        "givenName": "Bálint",
+        "name": "Dima, Bálint",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Drenkhan",
+        "givenName": "Rein",
+        "name": "Drenkhan, Rein",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Duarte",
+        "givenName": "Camila",
+        "name": "Duarte, Camila",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Dueñas",
+        "givenName": "Margarita",
+        "name": "Dueñas, Margarita",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Eberhardt",
+        "givenName": "Ursula",
+        "name": "Eberhardt, Ursula",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Friberg",
+        "givenName": "Hanna",
+        "name": "Friberg, Hanna",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Frøslev",
+        "givenName": "Tobias G.",
+        "name": "Frøslev, Tobias G.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Garnica",
+        "givenName": "Sigisfredo",
+        "name": "Garnica, Sigisfredo",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Geml",
+        "givenName": "József",
+        "name": "Geml, József",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Ghobad-Nejhad",
+        "givenName": "Masoomeh",
+        "name": "Ghobad-Nejhad, Masoomeh",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Grebenc",
+        "givenName": "Tine",
+        "name": "Grebenc, Tine",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Griffith",
+        "givenName": "Gareth W.",
+        "name": "Griffith, Gareth W.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Hampe",
+        "givenName": "Felix",
+        "name": "Hampe, Felix",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Kennedy",
+        "givenName": "Peter",
+        "name": "Kennedy, Peter",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Khomich",
+        "givenName": "Maryia",
+        "name": "Khomich, Maryia",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Kohout",
+        "givenName": "Petr",
+        "name": "Kohout, Petr",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Kollom",
+        "givenName": "Anu",
+        "name": "Kollom, Anu",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Larsson",
+        "givenName": "Ellen",
+        "name": "Larsson, Ellen",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Laszlo",
+        "givenName": "Irinyi",
+        "name": "Laszlo, Irinyi",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Leavitt",
+        "givenName": "Steven",
+        "name": "Leavitt, Steven",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Liimatainen",
+        "givenName": "Kare",
+        "name": "Liimatainen, Kare",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Lindahl",
+        "givenName": "Björn",
+        "name": "Lindahl, Björn",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Lodge",
+        "givenName": "Deborah J.",
+        "name": "Lodge, Deborah J.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Lumbsch",
+        "givenName": "Helge Thorsten",
+        "name": "Lumbsch, Helge Thorsten",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Martín Esteban",
+        "givenName": "María Paz",
+        "name": "Martín Esteban, María Paz",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Meyer",
+        "givenName": "Wieland",
+        "name": "Meyer, Wieland",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Miettinen",
+        "givenName": "Otto",
+        "name": "Miettinen, Otto",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Nguyen",
+        "givenName": "Nhu",
+        "name": "Nguyen, Nhu",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Niskanen",
+        "givenName": "Tuula",
+        "name": "Niskanen, Tuula",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Oono",
+        "givenName": "Ryoko",
+        "name": "Oono, Ryoko",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Öpik",
+        "givenName": "Maarja",
+        "name": "Öpik, Maarja",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Ordynets",
+        "givenName": "Alexander",
+        "name": "Ordynets, Alexander",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Pawłowska",
+        "givenName": "Julia",
+        "name": "Pawłowska, Julia",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Peintner",
+        "givenName": "Ursula",
+        "name": "Peintner, Ursula",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Pereira",
+        "givenName": "Olinto Liparini",
+        "name": "Pereira, Olinto Liparini",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Pinho",
+        "givenName": "Danilo Batista",
+        "name": "Pinho, Danilo Batista",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Põldmaa",
+        "givenName": "Kadri",
+        "name": "Põldmaa, Kadri",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Runnel",
+        "givenName": "Kadri",
+        "name": "Runnel, Kadri",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Ryberg",
+        "givenName": "Martin",
+        "name": "Ryberg, Martin",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Saar",
+        "givenName": "Irja",
+        "name": "Saar, Irja",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Sanli",
+        "givenName": "Kemal",
+        "name": "Sanli, Kemal",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Scott",
+        "givenName": "James",
+        "name": "Scott, James",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Spirin",
+        "givenName": "Viacheslav",
+        "name": "Spirin, Viacheslav",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Suija",
+        "givenName": "Ave",
+        "name": "Suija, Ave",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Svantesson",
+        "givenName": "Sten",
+        "name": "Svantesson, Sten",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Tadych",
+        "givenName": "Mariusz",
+        "name": "Tadych, Mariusz",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Takamatsu",
+        "givenName": "Susumu",
+        "name": "Takamatsu, Susumu",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Tamm",
+        "givenName": "Heidi",
+        "name": "Tamm, Heidi",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Taylor",
+        "givenName": "AFS.",
+        "name": "Taylor, AFS.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Tedersoo",
+        "givenName": "Leho",
+        "name": "Tedersoo, Leho",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Telleria",
+        "givenName": "M.T.",
+        "name": "Telleria, M.T.",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Udayanga",
+        "givenName": "Dhanushka",
+        "name": "Udayanga, Dhanushka",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Unterseher",
+        "givenName": "Martin",
+        "name": "Unterseher, Martin",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Volobuev",
+        "givenName": "Sergey",
+        "name": "Volobuev, Sergey",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Weiss",
+        "givenName": "Michael",
+        "name": "Weiss, Michael",
+        "nameType": "Personal"
+      },
+      {
+        "affiliation": [],
+        "familyName": "Wurzbacher",
+        "givenName": "Christian",
+        "name": "Wurzbacher, Christian",
+        "nameType": "Personal"
+      }
+    ],
+    "dates": [
+      {
+        "date": "2016-04-22",
+        "dateType": "Updated"
+      },
+      {
+        "date": "2014-10-05",
+        "dateType": "Created"
+      },
+      {
+        "date": "2015",
+        "dateType": "Issued"
+      }
+    ],
+    "descriptions": [
+      {
+        "description": "UNITE provides a unified way for delimiting, identifying, communicating, and working with DNA-based Species Hypotheses (SH). All fungal ITS sequences in the international nucleotide sequence databases are clustered to approximately the species level by applying a set of dynamic distance values (<0.5 - 3.0%). All species hypotheses are given a unique, stable name in the form of a DOI, and their taxonomic and ecological annotations are verified through distributed, web-based third-party annotation efforts. SHs are connected to a taxon name and its classification as far as possible (phylum, class, order, etc.) by taking into account identifications for all sequences in the SH. An automatically or manually designated sequence is chosen to represent each such SH. These sequences are released (https://unite.ut.ee/repository.php) for use by the scientific community in, for example, local sequence similarity searches and next-generation sequencing analysis pipelines. The system and the data are updated automatically as the number of public fungal ITS sequences grows.",
+        "descriptionType": "Abstract"
+      }
+    ],
+    "doi": "10.15156/bio/sh409843.07fu",
+    "formats": [
+      "application/json"
+    ],
+    "fundingReferences": [],
+    "geoLocations": [],
+    "identifiers": [
+      {
+        "identifier": "https://doi.org/10.15156/bio/sh409843.07fu",
+        "identifierType": "DOI"
+      }
+    ],
+    "isActive": true,
+    "language": "eng",
+    "metadataVersion": 1,
+    "publicationYear": 2015,
+    "published": "2015",
+    "publisher": "UNITE Community",
+    "reason": null,
+    "registered": "2015-06-05T10:23:19.000Z",
+    "relatedIdentifiers": [],
+    "rightsList": [
+      {
+        "rights": "Attribution-NonCommercial (CC BY-NC)",
+        "rightsUri": "http://creativecommons.org/licenses/by-nc/4.0"
+      }
+    ],
+    "schemaVersion": "http://datacite.org/schema/kernel-3",
+    "sizes": [],
+    "source": null,
+    "state": "findable",
+    "subjects": [],
+    "titles": [
+      {
+        "title": "SH409843.07FU"
+      },
+      {
+        "title": "Gomphales",
+        "titleType": "Subtitle"
+      }
+    ],
+    "types": {
+      "bibtex": "misc",
+      "citeproc": "dataset",
+      "resourceType": "Dataset/UNITE Species Hypothesis",
+      "resourceTypeGeneral": "Dataset",
+      "ris": "DATA",
+      "schemaOrg": "Dataset"
+    },
+    "updated": "2019-08-02T07:45:28.000Z",
+    "url": "https://plutof.ut.ee/#/datacite/10.15156/BIO/SH409843.07FU",
+    "version": null
+  },
+  "id": "10.15156/bio/sh409843.07fu",
+  "relationships": {
+    "client": {
+      "data": {
+        "id": "estdoi.bio",
+        "type": "clients"
+      }
+    }
+  },
+  "type": "dois"
+}