#!/usr/bin/env python3
"""
This script is intended to be used for backfill ingest of old crawls. It can
also be used as a fast path for getting freshly crawled content into fatcat if
the crawl was a hit and the arabesque JSON was exported conservatively.

Run like:

    ./arabesque2ingestrequest.py example_arabesque.json --link-source pmc --extid-type pmcid > ingest_requests.json

Can then run through requests using that tool, or dump into kafka queue.
"""

import argparse
import json
import sys


def run(args):
    for l in args.json_file:
        if not l.strip():
            continue
        row = json.loads(l)
        if not row["hit"]:
            continue

        request = {
            "base_url": row["final_url"],
            "ingest_type": args.ingest_type,
            "link_source": args.link_source,
            "link_source_id": row["identifier"],
            "ingest_request_source": args.ingest_request_source,
            "ext_ids": {
                args.extid_type: row["identifier"],
            },
        }
        if args.release_stage:
            assert args.release_stage in (
                "published",
                "submitted",
                "accepted",
                "draft",
                "update",
            )
            request["release_stage"] = args.release_stage

        print("{}".format(json.dumps(request, sort_keys=True)))


def main():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument(
        "--link-source", required=True, help="link_source to include in request"
    )
    parser.add_argument("--extid-type", required=True, help="extid to encode identifier as")
    parser.add_argument(
        "--ingest-type", default="pdf", help="ingest type (pdf, html, xml, etc)"
    )
    parser.add_argument(
        "--ingest-request-source", default="arabesque", help="to include in request"
    )
    parser.add_argument("--release-stage", default=None, help="to include in request")
    parser.add_argument(
        "json_file", help="arabesque output file to use", type=argparse.FileType("r")
    )
    subparsers = parser.add_subparsers()

    args = parser.parse_args()

    run(args)


if __name__ == "__main__":
    main()