diff options
| -rw-r--r-- | python/sandcrawler/fileset_platforms.py | 18 | ||||
| -rw-r--r-- | python/sandcrawler/fileset_strategies.py | 9 |
2 files changed, 21 insertions, 6 deletions
diff --git a/python/sandcrawler/fileset_platforms.py b/python/sandcrawler/fileset_platforms.py index cc07948..134ae7c 100644 --- a/python/sandcrawler/fileset_platforms.py +++ b/python/sandcrawler/fileset_platforms.py @@ -54,6 +54,7 @@ class FilesetPlatformHelper(): class DataverseHelper(FilesetPlatformHelper): def __init__(self): + super().__init__() self.platform_name = 'dataverse' self.session = requests.Session() @@ -324,6 +325,7 @@ def test_parse_dataverse_persistentid(): class FigshareHelper(FilesetPlatformHelper): def __init__(self): + super().__init__() self.platform_name = 'figshare' self.session = requests.Session() @@ -337,15 +339,20 @@ class FigshareHelper(FilesetPlatformHelper): Raises a ValueError if not a figshare URL """ # eg: /articles/Optimized_protocol_to_isolate_high_quality_genomic_DNA_from_different_tissues_of_a_palm_species/8987858/1 + # /articles/dataset/STable_1_U-Pb_geochronologic_analyses_on_samples_xls/12127176/4 comp = path.split('/') if len(comp) < 4 or comp[1] != 'articles': raise ValueError(f"not a figshare URL: {path}") - if len(comp) == 5 and comp[3].isdigit() and comp[4].isdigit(): - return (comp[3], comp[4]) - elif len(comp) == 4 and comp[3].isdigit(): - return (comp[3], None) + comp = comp[2:] + if comp[0] in ['dataset',]: + comp = comp[1:] + + if len(comp) == 3 and comp[1].isdigit() and comp[2].isdigit(): + return (comp[1], comp[2]) + elif len(comp) == 2 and comp[1].isdigit(): + return (comp[1], None) else: raise ValueError(f"couldn't find figshare identiier: {path}") @@ -455,6 +462,7 @@ def test_parse_figshare_url_path(): "/articles/Optimized_protocol_to_isolate_high_quality_genomic_DNA_from_different_tissues_of_a_palm_species/8987858/1": ("8987858", "1"), "/articles/Optimized_protocol_to_isolate_high_quality_genomic_DNA_from_different_tissues_of_a_palm_species/8987858": ("8987858", None), "/articles/CIBERSORT_p-value_0_05/8217188/1": ("8217188", "1"), + "/articles/dataset/STable_1_U-Pb_geochronologic_analyses_on_samples_xls/12127176/4": ("12127176", "4"), } invalid = [ @@ -474,6 +482,7 @@ def test_parse_figshare_url_path(): class ZenodoHelper(FilesetPlatformHelper): def __init__(self): + super().__init__() self.platform_name = 'zenodo' self.session = requests.Session() @@ -619,6 +628,7 @@ class ArchiveOrgHelper(FilesetPlatformHelper): } def __init__(self): + super().__init__() self.platform_name = 'archiveorg' self.session = internetarchive.get_session() diff --git a/python/sandcrawler/fileset_strategies.py b/python/sandcrawler/fileset_strategies.py index f2f2fcc..d12fc15 100644 --- a/python/sandcrawler/fileset_strategies.py +++ b/python/sandcrawler/fileset_strategies.py @@ -32,6 +32,7 @@ class FilesetIngestStrategy(): class ArchiveorgFilesetStrategy(FilesetIngestStrategy): def __init__(self, **kwargs): + super().__init__() self.ingest_strategy = IngestStrategy.ArchiveorgFileset # TODO: enable cleanup when confident (eg, safe path parsing) @@ -195,10 +196,12 @@ class ArchiveorgFileStrategy(ArchiveorgFilesetStrategy): class WebFilesetStrategy(FilesetIngestStrategy): def __init__(self, **kwargs): + super().__init__() self.ingest_strategy = IngestStrategy.WebFileset self.wayback_client = WaybackClient() self.try_spn2 = True self.spn_client = SavePageNowClient(spn_cdx_retry_sec=kwargs.get('spn_cdx_retry_sec', 9.0)) + self.max_spn_manifest = 20 def process(self, item: FilesetPlatformItem) -> ArchiveStrategyResult: """ @@ -219,10 +222,12 @@ class WebFilesetStrategy(FilesetIngestStrategy): via = "wayback" resource = self.wayback_client.lookup_resource(fetch_url, m.mimetype) - if self.try_spn2 and (resource == None or (resource and resource.status == 'no-capture')): + if len(item.manifest) > self.max_spn_manifest: + m.status = 'too-much-spn' + continue via = "spn2" - resource = self.spn_client.crawl_resource(fetch_url, self.wayback_client) + resource = self.spn_client.crawl_resource(fetch_url, self.wayback_client, force_simple_get=True) print("[FETCH {:>6}] {} {}".format( via, |