From 07e8a199766be77f4e89561d03e9b4e995ab7396 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 6 Oct 2021 15:13:46 -0700 Subject: fileset ingest progress for dataverse --- python/sandcrawler/ingest_fileset.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'python/sandcrawler/ingest_fileset.py') diff --git a/python/sandcrawler/ingest_fileset.py b/python/sandcrawler/ingest_fileset.py index 50e3c05..3b55793 100644 --- a/python/sandcrawler/ingest_fileset.py +++ b/python/sandcrawler/ingest_fileset.py @@ -262,19 +262,21 @@ class IngestFilesetWorker(IngestFileWorker): if resource: terminal_url = resource.terminal_url dataset_meta = platform_helper.process_request(request, terminal_url, html_biblio) - print(dataset_meta, file=sys.stderr) + #print(dataset_meta, file=sys.stderr) platform = dataset_meta.platform_name result['platform'] = dataset_meta.platform_name result['platform_id'] = dataset_meta.platform_id result['item_name'] = dataset_meta.archiveorg_item_name - if not dataset_meta.manifest: + result['item_meta'] = dataset_meta.archiveorg_item_meta + + if dataset_meta.manifest: + result['manifest'] = [m.dict() for m in dataset_meta.manifest] + result['file_count'] = len(dataset_meta.manifest) + result['total_size'] = sum([m.size for m in dataset_meta.manifest if m.size]) + else: result['status'] = 'no-manifest' return result - result['manifest'] = dataset_meta.manifest or None - result['file_count'] = len(dataset_meta.manifest) or None - result['total_size'] = sum([m.size for m in dataset_meta.manifest if m.size]) or None - ingest_strategy = platform_helper.chose_strategy(dataset_meta) result['ingest_strategy'] = ingest_strategy -- cgit v1.2.3