diff options
| author | Bryan Newbold <bnewbold@robocracy.org> | 2020-01-30 12:15:09 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@robocracy.org> | 2020-02-13 22:24:20 -0800 |
| commit | 87029cb13d244381f915fe66e40760477edb5675 (patch) | |
| tree | 6d4a9e0f8c1ec7327df03e9c77dd451e3e453710 /python/fatcat_tools/importers | |
| parent | e59d1b617d4abd5f002d9e59b6bbaebc9ff30993 (diff) | |
| download | fatcat-87029cb13d244381f915fe66e40760477edb5675.tar.gz fatcat-87029cb13d244381f915fe66e40760477edb5675.zip | |
shadow import: more filtering of file_meta fields
Diffstat (limited to 'python/fatcat_tools/importers')
| -rw-r--r-- | python/fatcat_tools/importers/shadow.py | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/python/fatcat_tools/importers/shadow.py b/python/fatcat_tools/importers/shadow.py index 21a18837..cfe1b1cf 100644 --- a/python/fatcat_tools/importers/shadow.py +++ b/python/fatcat_tools/importers/shadow.py @@ -43,6 +43,16 @@ class ShadowLibraryImporter(EntityImporter): self.default_link_rel = kwargs.get("default_link_rel", "web") def want(self, raw_record): + """ + Only want to import records with complete file-level metadata + """ + fm = raw_record['file_meta'] + if not (fm['mimetype'] and fm['md5hex'] and fm['sha256hex'] and fm['size_bytes']): + self.counts['skip-file-meta-incomplete'] += 1 + return False + if fm['mimetype'] != 'application/pdf': + self.counts['skip-not-pdf'] += 1 + return False return True def parse_record(self, obj): |