path: root/notes/ingest/2021-09-02_oai_pmh_patch.md

Just a "patch" of previous OAI-PMH crawl/ingest: re-ingesting and potentially
re-crawling content which failed to ingest the first time.

May fold this in with more general patch crawling.

## Basic Counts

    SELECT ingest_file_result.status, COUNT(*)
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    GROUP BY status
    ORDER BY COUNT DESC
    LIMIT 20;

            status         |  count   
    -----------------------+----------
     success               | 14145387
     no-pdf-link           | 12063022
     no-capture            |  5485640
     redirect-loop         |  2092705
     terminal-bad-status   |   747372
     wrong-mimetype        |   597219
     link-loop             |   542144
     null-body             |    93566
     cdx-error             |    19798
     petabox-error         |    17943
                           |    15283
     wayback-error         |    13897
     gateway-timeout       |      511
     skip-url-blocklist    |      184
     wayback-content-error |      146
     bad-redirect          |      137
     redirects-exceeded    |      120
     bad-gzip-encoding     |      116
     timeout               |       80
     blocked-cookie        |       64
    (20 rows)

    SELECT
        oai_prefix,
        COUNT(CASE WHEN status = 'success' THEN 1 END) as success,
        COUNT(*) as total
    FROM (
        SELECT
            ingest_file_result.status as status,
            -- eg "oai:cwi.nl:4881"
            substring(ingest_request.link_source_id FROM 'oai:([^:]+):.*') AS oai_prefix
        FROM ingest_request
        LEFT JOIN ingest_file_result
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE 
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
            AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
            AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    ) t1
    GROUP BY oai_prefix
    ORDER BY total DESC
    LIMIT 40;


            oai_prefix         | success |  total
    ---------------------------+---------+---------
     repec                     | 1133175 | 2783448
     hal                       |  573218 | 1049607
     www.irgrid.ac.cn          |   18007 |  748828
     cds.cern.ch               |   74078 |  688091
     americanae.aecid.es       |   71310 |  572792
     juser.fz-juelich.de       |   23026 |  518551
     espace.library.uq.edu.au  |    6649 |  508960
     igi.indrastra.com         |   59629 |  478577
     archive.ugent.be          |   65306 |  424014
     hrcak.srce.hr             |  404085 |  414897
     zir.nsk.hr                |  156753 |  397200
     renati.sunedu.gob.pe      |   79362 |  388355
     hypotheses.org            |       3 |  374296
     rour.neicon.ru            |    7997 |  354529
     generic.eprints.org       |  263566 |  340470
     invenio.nusl.cz           |    6340 |  325867
     evastar-karlsruhe.de      |   62282 |  317952
     quod.lib.umich.edu        |       5 |  309135
     diva.org                  |   67917 |  298348
     t2r2.star.titech.ac.jp    |    1085 |  289388
     edpsciences.org           |  139495 |  284972
     repository.ust.hk         |   10245 |  283417
     revues.org                |  151156 |  277497
     pure.atira.dk             |   13492 |  260754
     bibliotecadigital.jcyl.es |   50606 |  254134
     escholarship.org/ark      |  140835 |  245203
     ojs.pkp.sfu.ca            |  168029 |  229387
     lup.lub.lu.se             |   49358 |  226602
     library.wur.nl            |   15051 |  216738
     digitalrepository.unm.edu |  111704 |  211749
     infoscience.tind.io       |   60166 |  207299
     edoc.mpg.de               |       0 |  205252
     erudit.org                |  168490 |  197803
     delibra.bg.polsl.pl       |   38666 |  196652
     n/a                       |       0 |  193814
     aleph.bib-bvb.de          |    4349 |  186666
     serval.unil.ch            |   41643 |  186372
     orbi.ulg.ac.be            |    2400 |  184551
     digitalcommons.unl.edu    |  144025 |  184372
     bib-pubdb1.desy.de        |   33525 |  182717
    (40 rows)

Top counts by OAI prefix and status:

    SELECT
        oai_prefix,
        status,
        COUNT((oai_prefix,status))
    FROM (
        SELECT
            ingest_file_result.status as status,
            -- eg "oai:cwi.nl:4881"
            substring(ingest_request.link_source_id FROM 'oai:([^:]+):.*') AS oai_prefix
        FROM ingest_request
        LEFT JOIN ingest_file_result
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE 
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
            AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
            AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    ) t1
    GROUP BY oai_prefix, status
    ORDER BY COUNT DESC
    LIMIT 50;

            oai_prefix         |    status     |  count  
    ---------------------------+---------------+---------
     repec                     | success       | 1133175
     repec                     | no-pdf-link   |  638105
     hal                       | success       |  573218
     cds.cern.ch               | no-capture    |  540380
     repec                     | redirect-loop |  516451
     juser.fz-juelich.de       | no-pdf-link   |  477881
     americanae.aecid.es       | no-pdf-link   |  417766
     hrcak.srce.hr             | success       |  404085
     www.irgrid.ac.cn          | no-pdf-link   |  370908
     hal                       | no-pdf-link   |  359252
     www.irgrid.ac.cn          | no-capture    |  355532
     espace.library.uq.edu.au  | no-pdf-link   |  320479
     igi.indrastra.com         | no-pdf-link   |  318242
     repec                     | no-capture    |  316981
     invenio.nusl.cz           | no-pdf-link   |  309802
     rour.neicon.ru            | redirect-loop |  300911
     hypotheses.org            | no-pdf-link   |  300251
     renati.sunedu.gob.pe      | no-capture    |  282800
     t2r2.star.titech.ac.jp    | no-pdf-link   |  272045
     generic.eprints.org       | success       |  263566
     quod.lib.umich.edu        | no-pdf-link   |  259661
     archive.ugent.be          | no-capture    |  256127
     evastar-karlsruhe.de      | no-pdf-link   |  248939
     zir.nsk.hr                | link-loop     |  226919
     repository.ust.hk         | no-pdf-link   |  208569
     edoc.mpg.de               | no-pdf-link   |  199758
     bibliotecadigital.jcyl.es | no-pdf-link   |  188433
     orbi.ulg.ac.be            | no-pdf-link   |  172373
     diva.org                  | no-capture    |  171115
     lup.lub.lu.se             | no-pdf-link   |  168652
     erudit.org                | success       |  168490
     ojs.pkp.sfu.ca            | success       |  168029
     lib.dr.iastate.edu        | success       |  158494
     zir.nsk.hr                | success       |  156753
     digital.kenyon.edu        | success       |  154900
     revues.org                | success       |  151156
     books.openedition.org     | no-pdf-link   |  149607
     freidok.uni-freiburg.de   | no-pdf-link   |  146837
     digitalcommons.unl.edu    | success       |  144025
     escholarship.org/ark      | success       |  140835
     culeuclid                 | link-loop     |  140291
     edpsciences.org           | success       |  139495
     serval.unil.ch            | no-pdf-link   |  138644
     bib-pubdb1.desy.de        | no-pdf-link   |  133815
     krm.or.kr                 | no-pdf-link   |  132461
     pure.atira.dk             | no-pdf-link   |  132179
     oai-gms.dimdi.de          | redirect-loop |  131409
     aleph.bib-bvb.de          | no-capture    |  128261
     library.wur.nl            | no-pdf-link   |  124718
     lirias2repo.kuleuven.be   | no-capture    |  123106
    (50 rows)

Note: could just delete the "excluded" rows? and not harvest them in the
future, and filter them at ingest time (in transform script).



## Investigate no-pdf-link sandcrawler improvements

Do some spot-sampling of 'no-pdf-link' domains, see if newer sandcrawler works:

    SELECT
        ingest_request.link_source_id AS oai_id,
        ingest_request.base_url as base_url ,
        ingest_file_result.terminal_url as terminal_url 
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_file_result.status = 'no-pdf-link'
        AND ingest_request.link_source_id LIKE 'oai:library.wur.nl:%'
    ORDER BY random()
    LIMIT 10;

Random sampling of *all* 'no-pdf-link' URLs (see if newer sandcrawler works):

    \x auto

    SELECT
        ingest_request.link_source_id AS oai_id,
        ingest_request.base_url as base_url ,
        ingest_file_result.terminal_url as terminal_url 
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_file_result.status = 'no-pdf-link'
    ORDER BY random()
    LIMIT 30;

### repec (SKIP-PREFIX)

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:eee:jmacro:v:54:y:2017:i:pb:p:332-351
base_url     | http://www.sciencedirect.com/science/article/pii/S0164070417301593
terminal_url | http://www.sciencedirect.com/science/article/pii/S0164070417301593
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:eee:jomega:v:16:y:1988:i:2:p:107-115
base_url     | http://www.sciencedirect.com/science/article/pii/0305-0483(88)90041-2
terminal_url | https://www.sciencedirect.com/science/article/abs/pii/0305048388900412
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:sgm:pzwzuw:v:14:i:59:y:2016:p:73-92
base_url     | http://pz.wz.uw.edu.pl/en
terminal_url | http://pz.wz.uw.edu.pl:80/en
-[ RECORD 1 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:eee:jmacro:v:54:y:2017:i:pb:p:332-351
base_url     | http://www.sciencedirect.com/science/article/pii/S0164070417301593
terminal_url | http://www.sciencedirect.com/science/article/pii/S0164070417301593
-[ RECORD 2 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:eee:jomega:v:16:y:1988:i:2:p:107-115
base_url     | http://www.sciencedirect.com/science/article/pii/0305-0483(88)90041-2
terminal_url | https://www.sciencedirect.com/science/article/abs/pii/0305048388900412
-[ RECORD 3 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:sgm:pzwzuw:v:14:i:59:y:2016:p:73-92
base_url     | http://pz.wz.uw.edu.pl/en
terminal_url | http://pz.wz.uw.edu.pl:80/en
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:erv:rccsrc:y:2016:i:2016_11:35
base_url     | http://www.eumed.net/rev/caribe/2016/11/estructura.html
terminal_url | http://www.eumed.net:80/rev/caribe/2016/11/estructura.html
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:pio:envira:v:33:y:2001:i:4:p:629-647
base_url     | http://www.envplan.com/epa/fulltext/a33/a3319.pdf
terminal_url | http://uk.sagepub.com:80/en-gb/eur/pion-journals-published
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:tpr:qjecon:v:100:y:1985:i:3:p:651-75
base_url     | http://links.jstor.org/sici?sici=0033-5533%28198508%29100%3A3%3C651%3ATCOCEA%3E2.0.CO%3B2-2&origin=repec
terminal_url | https://www.jstor.org/stable/1884373

Huh! This is just a catalog of other domains. Should probably skip

DONE: skip/filter repec

### juser.fz-juelich.de (SCOPE)

-[ RECORD 1 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:132217
base_url     | http://juser.fz-juelich.de/record/132217
terminal_url | http://juser.fz-juelich.de/record/132217

Poster; no files.

-[ RECORD 2 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:268598
base_url     | http://juser.fz-juelich.de/record/268598
terminal_url | http://juser.fz-juelich.de/record/268598

Journal.

-[ RECORD 3 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:126613
base_url     | http://juser.fz-juelich.de/record/126613
terminal_url | http://juser.fz-juelich.de/record/126613

-[ RECORD 4 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:67362
base_url     | http://juser.fz-juelich.de/record/67362
terminal_url | http://juser.fz-juelich.de/record/67362
-[ RECORD 5 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:869189
base_url     | http://juser.fz-juelich.de/record/869189
terminal_url | http://juser.fz-juelich.de/record/869189
-[ RECORD 6 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:810746
base_url     | http://juser.fz-juelich.de/record/810746
terminal_url | http://juser.fz-juelich.de/record/810746
-[ RECORD 7 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:52897
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-52897%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-52897%22
-[ RECORD 8 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:114755
base_url     | http://juser.fz-juelich.de/record/114755
terminal_url | http://juser.fz-juelich.de/record/114755
-[ RECORD 9 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:58025
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-58025%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-58025%22

The search URLs seem redundant? Not going to try to handle those.

"Powered by Invenio v1.1.7"

All of these examples seem to be not papers. Maybe we can filter these better
at the harvest or transform stage?

### americanae.aecid.es (MIXED)

-[ RECORD 1 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:502896
base_url     | http://biblioteca.clacso.edu.ar/gsdl/cgi-bin/library.cgi?a=d&c=mx/mx-010&d=60327292007oai
terminal_url | http://biblioteca.clacso.edu.ar/gsdl/cgi-bin/library.cgi?a=d&c=mx/mx-010&d=60327292007oai

just a metadata record? links to redalyc

METADATA-ONLY

-[ RECORD 2 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:534600
base_url     | http://bdh-rd.bne.es/viewer.vm?id=0000077778&page=1
terminal_url | http://bdh-rd.bne.es/viewer.vm?id=0000077778&page=1
-[ RECORD 3 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:524567
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=524567
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=524567

NOT-FOUND (404)

-[ RECORD 4 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:378914
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=378914
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=378914

Some single-page image archival thing? bespoke, skipping.

SKIP-BESPOKE

-[ RECORD 5 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:526142
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=526142
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=526142

NOT-FOUND (404)

-[ RECORD 6 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:373408
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=373408
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=373408

NOT-FOUND (404)

### www.irgrid.ac.cn (SKIP-PREFIX)

Chinese Academy of Sciences Institutional Repositories Grid

-[ RECORD 1 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1749980
base_url     | http://www.irgrid.ac.cn/handle/1471x/1749980
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1749980

Can't access

FORBIDDEN

-[ RECORD 2 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/857397
base_url     | http://www.irgrid.ac.cn/handle/1471x/857397
terminal_url | http://www.irgrid.ac.cn/handle/1471x/857397

Just linking to another IR; skip it.

http://ir.ipe.ac.cn/handle/122111/10608

requires login

DONE: '/password-login;jsessionid' as a loginwall URL pattern
    http://ir.ipe.ac.cn/handle/122111/10608
    http://ir.ipe.ac.cn/bitstream/122111/10608/2/%e9%92%9d%e9%a1%b6%e8%9e%ba%e6%97%8b%e8%97%bb%e5%9c%a8%e4%b8%8d%e5%90%8c%e5%85%89%e7%85%a7%e6%9d%a1%e4%bb%b6%e4%b8%8b%e7%9a%84%e6%94%be%e6%b0%a7%e7%89%b9%e6%80%a7_%e8%96%9b%e5%8d%87%e9%95%bf.pdf

-[ RECORD 3 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1060447
base_url     | http://www.irgrid.ac.cn/handle/1471x/1060447
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1060447
-[ RECORD 4 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1671377
base_url     | http://ir.iggcas.ac.cn/handle/132A11/68622
terminal_url | http://ir.iggcas.ac.cn/handle/132A11/68622
-[ RECORD 5 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1178430
base_url     | http://www.irgrid.ac.cn/handle/1471x/1178430
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1178430
-[ RECORD 6 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/2488017
base_url     | http://www.irgrid.ac.cn/handle/1471x/2488017
terminal_url | http://www.irgrid.ac.cn/handle/1471x/2488017
-[ RECORD 7 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/977147
base_url     | http://www.irgrid.ac.cn/handle/1471x/977147
terminal_url | http://www.irgrid.ac.cn/handle/1471x/977147
-[ RECORD 8 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/2454503
base_url     | http://ir.nwipb.ac.cn/handle/363003/9957
terminal_url | http://ir.nwipb.ac.cn/handle/363003/9957

this domain is a disapointment :(

should continue crawling, as the metadata is open and good. but won't get fulltext?

### hal (FIXED-PARTIAL)

-[ RECORD 1 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00744951v1
base_url     | https://hal.archives-ouvertes.fr/hal-00744951
terminal_url | https://hal.archives-ouvertes.fr/hal-00744951

Off-site OA link.

FIXED-HAL

-[ RECORD 2 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-01065398v1
base_url     | https://hal.archives-ouvertes.fr/hal-01065398/file/AbstractSGE14_B_assaad.pdf
terminal_url | https://hal.archives-ouvertes.fr/index/index
-[ RECORD 3 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:lirmm-00371599v1
base_url     | https://hal-lirmm.ccsd.cnrs.fr/lirmm-00371599
terminal_url | https://hal-lirmm.ccsd.cnrs.fr/lirmm-00371599

To elsevier :(

-[ RECORD 4 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00284780v1
base_url     | https://hal.archives-ouvertes.fr/hal-00284780
terminal_url | https://hal.archives-ouvertes.fr/hal-00284780

METADATA-ONLY

-[ RECORD 5 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00186151v1
base_url     | https://hal.archives-ouvertes.fr/hal-00186151
terminal_url | https://hal.archives-ouvertes.fr/hal-00186151

METADATA-ONLY

-[ RECORD 6 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00399754v1
base_url     | https://hal.archives-ouvertes.fr/hal-00399754
terminal_url | https://hal.archives-ouvertes.fr/hal-00399754

METADATA-ONLY


### espace.library.uq.edu.au (SKIP)

-[ RECORD 1 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:136497
base_url     | https://espace.library.uq.edu.au/view/UQ:136497
terminal_url | https://espace.library.uq.edu.au/view/UQ:136497
-[ RECORD 2 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:411389
base_url     | https://espace.library.uq.edu.au/view/UQ:411389
terminal_url | https://espace.library.uq.edu.au/view/UQ:411389
-[ RECORD 3 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:401773
base_url     | https://espace.library.uq.edu.au/view/UQ:401773
terminal_url | https://espace.library.uq.edu.au/view/UQ:401773
-[ RECORD 4 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:675334
base_url     | https://espace.library.uq.edu.au/view/UQ:675334
terminal_url | https://espace.library.uq.edu.au/view/UQ:675334
-[ RECORD 5 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:312311
base_url     | https://espace.library.uq.edu.au/view/UQ:312311
terminal_url | https://espace.library.uq.edu.au/view/UQ:312311
-[ RECORD 6 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:209401
base_url     | https://espace.library.uq.edu.au/view/UQ:209401
terminal_url | https://espace.library.uq.edu.au/view/UQ:209401
-[ RECORD 7 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:327188
base_url     | https://espace.library.uq.edu.au/view/UQ:327188
terminal_url | https://espace.library.uq.edu.au/view/UQ:327188

Very javascript heavy (skeletal HTML). And just links to fulltext on publisher
sites.

### igi.indrastra.com (METADATA-ONLY)

-[ RECORD 1 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:267221
base_url     | http://igi.indrastra.com/items/show/267221
terminal_url | http://igi.indrastra.com/items/show/267221
-[ RECORD 2 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:181799
base_url     | http://igi.indrastra.com/items/show/181799
terminal_url | http://igi.indrastra.com/items/show/181799
-[ RECORD 3 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:125382
base_url     | http://igi.indrastra.com/items/show/125382
terminal_url | http://igi.indrastra.com/items/show/125382
-[ RECORD 4 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:47266
base_url     | http://igi.indrastra.com/items/show/47266
terminal_url | http://igi.indrastra.com/items/show/47266
-[ RECORD 5 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:12872
base_url     | http://igi.indrastra.com/items/show/12872
terminal_url | http://igi.indrastra.com/items/show/12872
-[ RECORD 6 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:231620
base_url     | http://igi.indrastra.com/items/show/231620
terminal_url | http://igi.indrastra.com/items/show/231620

"Proudly powered by Omeka"

### invenio.nusl.cz (METADATA-ONLY)

           oai_id           |              base_url              |             terminal_url
----------------------------+------------------------------------+--------------------------------------
 oai:invenio.nusl.cz:237409 | http://www.nusl.cz/ntk/nusl-237409 | http://invenio.nusl.cz/record/237409
 oai:invenio.nusl.cz:180783 | http://www.nusl.cz/ntk/nusl-180783 | http://invenio.nusl.cz/record/180783
 oai:invenio.nusl.cz:231961 | http://www.nusl.cz/ntk/nusl-231961 | http://invenio.nusl.cz/record/231961
 oai:invenio.nusl.cz:318800 | http://www.nusl.cz/ntk/nusl-318800 | http://invenio.nusl.cz/record/318800
 oai:invenio.nusl.cz:259695 | http://www.nusl.cz/ntk/nusl-259695 | http://invenio.nusl.cz/record/259695
 oai:invenio.nusl.cz:167393 | http://www.nusl.cz/ntk/nusl-167393 | http://invenio.nusl.cz/record/167393
 oai:invenio.nusl.cz:292987 | http://www.nusl.cz/ntk/nusl-292987 | http://invenio.nusl.cz/record/292987
 oai:invenio.nusl.cz:283396 | http://www.nusl.cz/ntk/nusl-283396 | http://invenio.nusl.cz/record/283396
 oai:invenio.nusl.cz:241512 | http://www.nusl.cz/ntk/nusl-241512 | http://invenio.nusl.cz/record/241512
 oai:invenio.nusl.cz:178631 | http://www.nusl.cz/ntk/nusl-178631 | http://invenio.nusl.cz/record/178631

Metadata only (at least this set)

### hypotheses.org

-[ RECORD 1 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:mittelalter/9529
base_url     | http://mittelalter.hypotheses.org/9529
terminal_url | https://mittelalter.hypotheses.org/9529
-[ RECORD 2 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivalia/18638
base_url     | http://archivalia.hypotheses.org/18638
terminal_url | https://archivalia.hypotheses.org/18638
-[ RECORD 3 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivalia/13614
base_url     | http://archivalia.hypotheses.org/13614
terminal_url | https://archivalia.hypotheses.org/13614
-[ RECORD 4 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:teteschercheuses/2785
base_url     | http://teteschercheuses.hypotheses.org/2785
terminal_url | https://teteschercheuses.hypotheses.org/2785
-[ RECORD 5 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:altervsego/608
base_url     | http://altervsego.hypotheses.org/608
terminal_url | http://altervsego.hypotheses.org/608
-[ RECORD 6 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivewk1/21905
base_url     | http://archivewk1.hypotheses.org/21905
terminal_url | https://archivewk1.hypotheses.org/21905
-[ RECORD 7 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:slkdiaspo/3321
base_url     | http://slkdiaspo.hypotheses.org/3321
terminal_url | https://slkdiaspo.hypotheses.org/3321
-[ RECORD 8 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:diga/280
base_url     | http://diga.hypotheses.org/280
terminal_url | https://diga.hypotheses.org/280

These are all a big mix... basically blogs. Should continue crawling, but expect no yield.

### t2r2.star.titech.ac.jp (METADATA-ONLY)

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00105099
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100499795
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100499795
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00101346
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100495549
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100495549
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50161100
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100632554
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100632554
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00232407
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100527528
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100527528
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50120040
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100612598
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100612598
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50321440
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100713492
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100713492
-[ RECORD 7 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50235666
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100668778
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100668778


### quod.lib.umich.edu

-[ RECORD 1 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acf2679.0015.003-2
base_url     | http://name.umdl.umich.edu/acf2679.0015.003
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acf2679.0015.003
-[ RECORD 2 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:b14970.0001.001
base_url     | http://name.umdl.umich.edu/B14970.0001.001
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=eebo2;idno=B14970.0001.001
-[ RECORD 3 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acf2679.0009.010-3
base_url     | http://name.umdl.umich.edu/ACF2679-1623SOUT-209
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acf2679.0009.010;node=acf2679.0009.010:3
-[ RECORD 4 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg2248.1-16.006-43
base_url     | http://name.umdl.umich.edu/acg2248.1-16.006
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg2248.1-16.006
-[ RECORD 5 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg2248.1-14.011-9
base_url     | http://name.umdl.umich.edu/ACG2248-1489LADI-364
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg2248.1-14.011;node=acg2248.1-14.011:9
-[ RECORD 6 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg1336.1-24.006-9
base_url     | http://name.umdl.umich.edu/acg1336.1-24.006
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg1336.1-24.006
-[ RECORD 7 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:africanamer.0002.32a
base_url     | http://name.umdl.umich.edu/africanamer.0002.32a
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=africanamer;idno=africanamer.0002.32a

These are... issues of journals? Should continue to crawl, but not expect much.

### evastar-karlsruhe.de (METADATA-ONLY)

-[ RECORD 1 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:270011444
base_url     | https://publikationen.bibliothek.kit.edu/270011444
terminal_url | https://publikationen.bibliothek.kit.edu/270011444
-[ RECORD 2 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:1000050117
base_url     | https://publikationen.bibliothek.kit.edu/1000050117
terminal_url | https://publikationen.bibliothek.kit.edu/1000050117
-[ RECORD 3 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:362296
base_url     | https://publikationen.bibliothek.kit.edu/362296
terminal_url | https://publikationen.bibliothek.kit.edu/362296
-[ RECORD 4 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:23042000
base_url     | https://publikationen.bibliothek.kit.edu/23042000
terminal_url | https://publikationen.bibliothek.kit.edu/23042000
-[ RECORD 5 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:1000069945
base_url     | https://publikationen.bibliothek.kit.edu/1000069945
terminal_url | https://publikationen.bibliothek.kit.edu/1000069945


### repository.ust.hk

-[ RECORD 1 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-67233
base_url     | http://repository.ust.hk/ir/Record/1783.1-67233
terminal_url | http://repository.ust.hk/ir/Record/1783.1-67233
-[ RECORD 2 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-63232
base_url     | http://gateway.isiknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=LinksAMR&SrcApp=PARTNER_APP&DestLinkType=FullRecord&DestApp=WOS&KeyUT=A1981KV47900017
terminal_url | http://login.webofknowledge.com/error/Error?Src=IP&Alias=WOK5&Error=IPError&Params=DestParams%3D%253FUT%253DWOS%253AA1981KV47900017%2526customersID%253DLinksAMR%2526product%253DWOS%2526action%253Dretrieve%2526mode%253DFullRecord%26DestApp%3DWOS%26SrcApp%3DPARTNER_APP%26SrcAuth%3DLinksAMR&PathInfo=%2F&RouterURL=http%3A%2F%2Fwww.webofknowledge.com%2F&Domain=.webofknowledge.com
-[ RECORD 3 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-2891
base_url     | http://gateway.isiknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=LinksAMR&SrcApp=PARTNER_APP&DestLinkType=FullRecord&DestApp=WOS&KeyUT=000240035400103
terminal_url | https://login.webofknowledge.com/error/Error?Src=IP&Alias=WOK5&Error=IPError&Params=DestParams%3D%253FUT%253DWOS%253A000240035400103%2526customersID%253DLinksAMR%2526product%253DWOS%2526action%253Dretrieve%2526mode%253DFullRecord%26DestApp%3DWOS%26SrcApp%3DPARTNER_APP%26SrcAuth%3DLinksAMR&PathInfo=%2F&RouterURL=https%3A%2F%2Fwww.webofknowledge.com%2F&Domain=.webofknowledge.com
-[ RECORD 4 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-56231
base_url     | http://repository.ust.hk/ir/Record/1783.1-56231
terminal_url | http://repository.ust.hk/ir/Record/1783.1-56231

[...]

-[ RECORD 6 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-24872
base_url     | http://repository.ust.hk/ir/Record/1783.1-24872
terminal_url | http://repository.ust.hk/ir/Record/1783.1-24872
-[ RECORD 7 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-3457
base_url     | http://lbdiscover.ust.hk/uresolver?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=0003-6870&rft.volume=40&rft.issue=2&rft.date=2009&rft.spage=267&rft.epage=279&rft.aulast=Witana&rft.aufirst=Channa+R.&rft.atitle=Effects+of+surface+characteristics+on+the+plantar+shape+of+feet+and+subjects'+perceived+sensations
terminal_url | http://lbdiscover.ust.hk/uresolver/?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=0003-6870&rft.volume=40&rft.issue=2&rft.date=2009&rft.spage=267&rft.epage=279&rft.aulast=Witana&rft.aufirst=Channa+R.&rft.atitle=Effects+of+surface+characteristics+on+the+plantar+shape+of+feet+and+subjects'+perceived+sensations
-[ RECORD 8 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-73215
base_url     | http://repository.ust.hk/ir/Record/1783.1-73215
terminal_url | http://repository.ust.hk/ir/Record/1783.1-73215

DONE: gateway.isiknowledge.com is bogus/blocking?


### edoc.mpg.de (SKIP-DEPRECATED)

         oai_id         |         base_url          |       terminal_url
------------------------+---------------------------+---------------------------
 oai:edoc.mpg.de:416650 | http://edoc.mpg.de/416650 | http://edoc.mpg.de/416650
 oai:edoc.mpg.de:8195   | http://edoc.mpg.de/8195   | http://edoc.mpg.de/8195
 oai:edoc.mpg.de:379655 | http://edoc.mpg.de/379655 | http://edoc.mpg.de/379655
 oai:edoc.mpg.de:641179 | http://edoc.mpg.de/641179 | http://edoc.mpg.de/641179
 oai:edoc.mpg.de:607141 | http://edoc.mpg.de/607141 | http://edoc.mpg.de/607141
 oai:edoc.mpg.de:544412 | http://edoc.mpg.de/544412 | http://edoc.mpg.de/544412
 oai:edoc.mpg.de:314531 | http://edoc.mpg.de/314531 | http://edoc.mpg.de/314531
 oai:edoc.mpg.de:405047 | http://edoc.mpg.de/405047 | http://edoc.mpg.de/405047
 oai:edoc.mpg.de:239650 | http://edoc.mpg.de/239650 | http://edoc.mpg.de/239650
 oai:edoc.mpg.de:614852 | http://edoc.mpg.de/614852 | http://edoc.mpg.de/614852

This whole instance seems to have been replaced

### bibliotecadigital.jcyl.es (SKIP-DIGITIZED)

-[ RECORD 1 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:10000039962
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10044664
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10044664
-[ RECORD 2 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:14075
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14075
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14075
-[ RECORD 3 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:4842
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=4842
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=4842
-[ RECORD 4 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:14799
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14799
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14799
-[ RECORD 5 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:821
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=1003474
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=1003474

Digitized images as pages; too much to deal with for now.

### orbi.ulg.ac.be

-[ RECORD 1 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/128079
base_url     | https://orbi.uliege.be/handle/2268/128079
terminal_url | https://orbi.uliege.be/handle/2268/128079
-[ RECORD 2 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/67659
base_url     | https://orbi.uliege.be/handle/2268/67659
terminal_url | https://orbi.uliege.be/handle/2268/67659
-[ RECORD 3 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/35521
base_url     | https://orbi.uliege.be/handle/2268/35521
terminal_url | https://orbi.uliege.be/handle/2268/35521
-[ RECORD 4 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/107922
base_url     | https://orbi.uliege.be/handle/2268/107922
terminal_url | https://orbi.uliege.be/handle/2268/107922
-[ RECORD 5 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/215694
base_url     | https://orbi.uliege.be/handle/2268/215694
terminal_url | https://orbi.uliege.be/handle/2268/215694

Described below.

### library.wur.nl (FIXED-BESPOKE)

                  oai_id               |                    base_url                    |                  terminal_url                  
    -----------------------------------+------------------------------------------------+------------------------------------------------
     oai:library.wur.nl:wurpubs/440939 | https://library.wur.nl/WebQuery/wurpubs/440939 | https://library.wur.nl/WebQuery/wurpubs/440939
     oai:library.wur.nl:wurpubs/427707 | https://library.wur.nl/WebQuery/wurpubs/427707 | https://library.wur.nl/WebQuery/wurpubs/427707
     oai:library.wur.nl:wurpubs/359208 | https://library.wur.nl/WebQuery/wurpubs/359208 | https://library.wur.nl/WebQuery/wurpubs/359208
     oai:library.wur.nl:wurpubs/433378 | https://library.wur.nl/WebQuery/wurpubs/433378 | https://library.wur.nl/WebQuery/wurpubs/433378
     oai:library.wur.nl:wurpubs/36416  | https://library.wur.nl/WebQuery/wurpubs/36416  | https://library.wur.nl/WebQuery/wurpubs/36416
     oai:library.wur.nl:wurpubs/469930 | https://library.wur.nl/WebQuery/wurpubs/469930 | https://library.wur.nl/WebQuery/wurpubs/469930
     oai:library.wur.nl:wurpubs/350076 | https://library.wur.nl/WebQuery/wurpubs/350076 | https://library.wur.nl/WebQuery/wurpubs/350076
     oai:library.wur.nl:wurpubs/19109  | https://library.wur.nl/WebQuery/wurpubs/19109  | https://library.wur.nl/WebQuery/wurpubs/19109
     oai:library.wur.nl:wurpubs/26146  | https://library.wur.nl/WebQuery/wurpubs/26146  | https://library.wur.nl/WebQuery/wurpubs/26146
     oai:library.wur.nl:wurpubs/529922 | https://library.wur.nl/WebQuery/wurpubs/529922 | https://library.wur.nl/WebQuery/wurpubs/529922
    (10 rows)

Seems like a one-off site? But added a pattern.

### pure.atira.dk

-[ RECORD 1 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/a27762fd-0919-4753-af55-00b9b26d02e0
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/hightech-cities-and-the-primitive-jungle-visionary-urbanism-in-europe-and-japan-of-the-1960s(a27762fd-0919-4753-af55-00b9b26d02e0).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/hightech-cities-and-the-primitive-jungle-visionary-urbanism-in-europe-and-japan-of-the-1960s(a27762fd-0919-4753-af55-00b9b26d02e0).html
-[ RECORD 2 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/215c8b96-a821-4947-bee4-c7470e9fbaf8
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/service-recovery-in-health-services--understanding-the-desired-qualities-and-behaviours-of-general-practitioners-during-service-recovery-encounters(215c8b96-a821-4947-bee4-c7470e9fbaf8).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/service-recovery-in-health-services--understanding-the-desired-qualities-and-behaviours-of-general-practitioners-during-service-recovery-encounters(215c8b96-a821-4947-bee4-c7470e9fbaf8).html
-[ RECORD 3 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/95d4920a-12c7-4e25-b86c-5f075ea23a38
base_url     | https://www.tandfonline.com/doi/full/10.1080/03057070.2016.1197694
terminal_url | https://www.tandfonline.com/action/cookieAbsent
-[ RECORD 4 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/8a2508ee-14c9-4c6a-851a-6db442090f41
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/microstructure-and-grain-size-dependence-of-ferroelectric-properties-of-batio3-thin-films-on-lanio3-buffered-si(8a2508ee-14c9-4c6a-851a-6db442090f41).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/microstructure-and-grain-size-dependence-of-ferroelectric-properties-of-batio3-thin-films-on-lanio3-buffered-si(8a2508ee-14c9-4c6a-851a-6db442090f41).html

Metadata only

DONE: /cookieAbsent is cookie block
    https://www.tandfonline.com/action/cookieAbsent

### bib-pubdb1.desy.de (FIXED-INVENIO)

-[ RECORD 2 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:96756
base_url     | http://bib-pubdb1.desy.de/record/96756
terminal_url | http://bib-pubdb1.desy.de/record/96756

Metadata only.

-[ RECORD 3 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:416556
base_url     | http://bib-pubdb1.desy.de/record/416556
terminal_url | http://bib-pubdb1.desy.de/record/416556

Fixed!

-[ RECORD 4 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:414545
base_url     | http://bib-pubdb1.desy.de/search?p=id:%22PUBDB-2018-04027%22
terminal_url | http://bib-pubdb1.desy.de/search?p=id:%22PUBDB-2018-04027%22
-[ RECORD 5 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:170169
base_url     | http://bib-pubdb1.desy.de/record/170169
terminal_url | http://bib-pubdb1.desy.de/record/170169
-[ RECORD 6 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:191154
base_url     | http://bib-pubdb1.desy.de/record/191154
terminal_url | http://bib-pubdb1.desy.de/record/191154

Metadata only

-[ RECORD 7 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:155092
base_url     | http://bib-pubdb1.desy.de/record/155092
terminal_url | http://bib-pubdb1.desy.de/record/155092

Fixed!

-[ RECORD 8 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:97158
base_url     | http://bib-pubdb1.desy.de/record/97158
terminal_url | http://bib-pubdb1.desy.de/record/97158

Metadata only

"Powered by Invenio v1.1.7"

Can/should skip the "search" URLs

### serval.unil.ch

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_60346fc75171
base_url     | https://serval.unil.ch/notice/serval:BIB_60346FC75171
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_60346FC75171
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_4db47fc4b593
base_url     | https://serval.unil.ch/notice/serval:BIB_4DB47FC4B593
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_4DB47FC4B593
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_57aac24fe115
base_url     | http://nbn-resolving.org/urn/resolver.pl?urn=urn:nbn:ch:serval-BIB_57AAC24FE1154
terminal_url | https://nbn-resolving.org/urn/resolver.pl?urn=urn:nbn:ch:serval-BIB_57AAC24FE1154
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_deabae6baf6c
base_url     | https://serval.unil.ch/notice/serval:BIB_DEABAE6BAF6C
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_DEABAE6BAF6C
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_a5ec0df1370f
base_url     | https://serval.unil.ch/notice/serval:BIB_A5EC0DF1370F
terminal_url | https://wayf.switch.ch/SWITCHaai/WAYF?entityID=https%3A%2F%2Fmy.unil.ch%2Fshibboleth&return=https%3A%2F%2Fserval.unil.ch%2FShibboleth.sso%2FLogin%3FSAMLDS%3D1%26target%3Dss%253Amem%253Aed270c26d4a36cefd1bf6a840472abe0ee5556cb5f3b42de708f3ea984775dfd
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_080300c2e23c
base_url     | https://serval.unil.ch/resource/serval:BIB_080300C2E23C.P001/REF.pdf
terminal_url | https://wayf.switch.ch/SWITCHaai/WAYF?entityID=https%3A%2F%2Fmy.unil.ch%2Fshibboleth&return=https%3A%2F%2Fserval.unil.ch%2FShibboleth.sso%2FLogin%3FSAMLDS%3D1%26target%3Dss%253Amem%253A154453d78a0fb75ffa220f7b6fe73b29447fa6ed048addf31897b41001f44679
-[ RECORD 7 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_de777dd2b07f
base_url     | https://serval.unil.ch/notice/serval:BIB_DE777DD2B07F
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_DE777DD2B07F
-[ RECORD 8 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_5e824e244c27
base_url     | https://serval.unil.ch/notice/serval:BIB_5E824E244C27
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_5E824E244C27

Metadata only? See elsewhere.

### Random Links

-[ RECORD 1 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dbc.wroc.pl:41031
base_url     | https://dbc.wroc.pl/dlibra/docmetadata?showContent=true&id=41031
terminal_url | https://dbc.wroc.pl/dlibra/docmetadata?showContent=true&id=41031

This is some platform/package thing. PDF is in an iframe. Platform is "DLibra".
FIXED-DLIBRA

-[ RECORD 2 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/174291
base_url     | https://orbi.uliege.be/handle/2268/174291
terminal_url | https://orbi.uliege.be/handle/2268/174291

DSpace platform. There are multiple files, and little to "select" on.

https://orbi.uliege.be/handle/2268/174200 has only single PDF and easier to work with

PARTIAL-DSPACE

-[ RECORD 3 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.tue.nl:664163
base_url     | http://repository.tue.nl/664163
terminal_url | http://repository.tue.nl/664163

Ah, this is the Pure platform from Elsevier.
Redirects to: https://research.tue.nl/en/publications/lowering-the-threshold-for-computers-in-early-design-some-advance

FIXED-PURE


-[ RECORD 4 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:49579
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-49579%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-49579%22

(handled above)

-[ RECORD 5 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.mit.edu:1721.1/97937
base_url     | https://orcid.org/0000-0002-2066-2082
terminal_url | https://orcid.org/0000-0002-2066-2082

ORCID! Skip it.

DONE: skip orcid.org in `terminal_url`, and/or at harvest/transform time.

-[ RECORD 6 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:edoc.mpg.de:360269
base_url     | http://edoc.mpg.de/360269
terminal_url | http://edoc.mpg.de/360269

Seems like this whole repo has disapeared, or been replaced by... pure? maybe a different pure?

DONE: edoc.mpg.de -> pure.mpg.de

-[ RECORD 7 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:books.openedition.org:msha/17716
base_url     | http://books.openedition.org/msha/17716
terminal_url | https://books.openedition.org/msha/17716

Open edition is free to read HTML, but not PDF (or epub, etc).

TODO: for some? all? openedition books records, try HTML ingest (not PDF ingest)

HTML-WORKED

-[ RECORD 8 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:epub.oeaw.ac.at:0x003aba48
base_url     | http://epub.oeaw.ac.at/?arp=8609-0inhalt/B02_2146_FP_Flores%20Castillo.pdf
terminal_url | http://epub.oeaw.ac.at/?arp=8609-0inhalt/B02_2146_FP_Flores%20Castillo.pdf

requires login

FORBIDDEN

-[ RECORD 9 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.mit.edu:1721.1/88986
base_url     | https://orcid.org/0000-0002-4147-2560
terminal_url | https://orcid.org/0000-0002-4147-2560

DONE: skip orcids

-[ RECORD 10 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-28786
base_url     | http://repository.ust.hk/ir/Record/1783.1-28786
terminal_url | http://repository.ust.hk/ir/Record/1783.1-28786

Generator: VuFind 5.1.1
just a metadata record

METADATA-ONLY

-[ RECORD 11 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:rcin.org.pl:50797
base_url     | http://195.187.71.10/ipac20/ipac.jsp?profile=iblpan&index=BOCLC&term=cc95215472
terminal_url | http://195.187.71.10/ipac20/ipac.jsp?profile=iblpan&index=BOCLC&term=cc95215472

Seems like a software platform? not sure.

METADATA-ONLY

-[ RECORD 12 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dea.lib.unideb.hu:2437/69641
base_url     | http://webpac.lib.unideb.hu:8082/WebPac/CorvinaWeb?action=cclfind&resultview=long&ccltext=idno+bibFSZ1008709
terminal_url | https://webpac.lib.unideb.hu/WebPac/CorvinaWeb?action=cclfind&resultview=long&ccltext=idno+bibFSZ1008709

-[ RECORD 13 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:unsworks.library.unsw.edu.au:1959.4/64871
base_url     | http://handle.unsw.edu.au/1959.4/64871
terminal_url | https://www.unsworks.unsw.edu.au/primo-explore/fulldisplay?vid=UNSWORKS&docid=unsworks_62832&context=L

-[ RECORD 14 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:www.wbc.poznan.pl:225930
base_url     | https://www.wbc.poznan.pl/dlibra/docmetadata?showContent=true&id=225930
terminal_url | https://www.wbc.poznan.pl/dlibra/docmetadata?showContent=true&id=225930

SOFT-404

-[ RECORD 15 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.erciyes.edu.tr:105
base_url     | http://repository.erciyes.edu.tr/bilimname/items/show/105
terminal_url | http://repository.erciyes.edu.tr:80/bilimname/items/show/105

GONE (domain not registered)

-[ RECORD 16 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digi.ub.uni-heidelberg.de:37500
base_url     | https://archivum-laureshamense-digital.de/view/sad_a1_nr_20_13
terminal_url | https://archivum-laureshamense-digital.de/view/sad_a1_nr_20_13

Seems like a bespoke site

SKIP-BESPOKE

-[ RECORD 17 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50401364
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100758313
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100758313

METADATA-ONLY

-[ RECORD 18 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:epubs.cclrc.ac.uk:work/4714
base_url     | http://purl.org/net/epubs/work/4714
terminal_url | https://epubs.stfc.ac.uk/work/4714

It's got a purl! haha.

METADATA-ONLY

------

Another batch! With some repeat domains removed.

-[ RECORD 1 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:cris.vtt.fi:persons/142c030f-ba7b-491a-8669-a361088355cc
base_url     | https://cris.vtt.fi/en/persons/142c030f-ba7b-491a-8669-a361088355cc
terminal_url | https://cris.vtt.fi/en/persons/oleg-antropov

SKIP

-[ RECORD 2 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:etd.adm.unipi.it:etd-05302014-183910
base_url     | http://etd.adm.unipi.it/theses/available/etd-05302014-183910/
terminal_url | https://etd.adm.unipi.it/theses/available/etd-05302014-183910/

Some software platform? Pretty basic/bespoke

FIXED-PARTIAL

-[ RECORD 3 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:10000098246
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10316451
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10316451

SKIP (see elsewhere)

-[ RECORD 7 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:elektra.cdaea.es:documento.29259
base_url     | https://www.juntadeandalucia.es/cultura/cdaea/elektra/catalogo_execute.html?tipoObjeto=1&id=29259
terminal_url | https://www.juntadeandalucia.es/cultura/cdaea/elektra/catalogo_execute.html?tipoObjeto=1&id=29259

Photo.

SKIP-SCOPE

-[ RECORD 9 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:unsworks.library.unsw.edu.au:1959.4/unsworks_60829
base_url     | http://handle.unsw.edu.au/1959.4/unsworks_60829
terminal_url | https://www.unsworks.unsw.edu.au/primo-explore/fulldisplay?vid=UNSWORKS&docid=unsworks_modsunsworks_60829&context=L

METADATA-ONLY

-[ RECORD 12 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.leuphana.de:publications/7d040cf2-b3b5-4671-8906-76b5bc8d870a
base_url     | http://fox.leuphana.de/portal/de/publications/studies-in-childrens-literature-1500--2000-editors-celia-keenan-(7d040cf2-b3b5-4671-8906-76b5bc8d870a).html
terminal_url | http://fox.leuphana.de/portal/de/publications/studies-in-childrens-literature-1500--2000-editors-celia-keenan-(7d040cf2-b3b5-4671-8906-76b5bc8d870a).html

unsure

-[ RECORD 16 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.wur.nl:wurpubs/369344
base_url     | https://library.wur.nl/WebQuery/wurpubs/369344
terminal_url | https://library.wur.nl/WebQuery/wurpubs/369344

this specific record not OA (but site is fine/fixed)

-[ RECORD 17 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:escholarship.umassmed.edu:oapubs-2146
base_url     | https://escholarship.umassmed.edu/oapubs/1147
terminal_url | http://escholarship.umassmed.edu/oapubs/1147/

just links to publisher (no content in repo)

-[ RECORD 18 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digitalcommons.usu.edu:wild_facpub-1010
base_url     | https://digitalcommons.usu.edu/wild_facpub/11
terminal_url | http://digitalcommons.usu.edu/wild_facpub/11/

also just links to publisher (no content in repo)

-[ RECORD 25 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:igi.indrastra.com:306768
base_url     | http://igi.indrastra.com/items/show/306768
terminal_url | http://igi.indrastra.com/items/show/306768

(see elsewhere)

-[ RECORD 26 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:fau.digital.flvc.org:fau_9804
base_url     | http://purl.flvc.org/fcla/dt/12932
terminal_url | http://fau.digital.flvc.org/islandora/object/fau%3A9804

Islandora.

-[ RECORD 27 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.lu.lv:7/16019
base_url     | https://dspace.lu.lv/dspace/handle/7/16019
terminal_url | https://dspace.lu.lv/dspace/handle/7/16019

LOGINWALL

-[ RECORD 28 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:zir.nsk.hr:umas_218
base_url     | https://repozitorij.svkst.unist.hr/islandora/object/umas:218
terminal_url | https://repozitorij.svkst.unist.hr/islandora/object/umas:218

REMOVED


-[ RECORD 29 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digi.ub.uni-heidelberg.de:36390
base_url     | https://digi.hadw-bw.de/view/sbhadwmnkl_a_1917_5
terminal_url | https://digi.hadw-bw.de/view/sbhadwmnkl_a_1917_5

Book, with chapters, not an individual work.

-[ RECORD 2 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:krm.or.kr:10056135m201r
base_url     | https://www.krm.or.kr/krmts/link.html?dbGubun=SD&m201_id=10056135&res=y
terminal_url | https://www.krm.or.kr/krmts/search/detailview/research.html?dbGubun=SD&category=Research&m201_id=10056135

research results repository; keep crawling

SKIP-SCOPE

-[ RECORD 3 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:www.db-thueringen.de:dbt_mods_00005191
base_url     | https://www.db-thueringen.de/receive/dbt_mods_00005191
terminal_url | https://www.db-thueringen.de/receive/dbt_mods_00005191

powered by "MyCoRe"

FIXED-MYCORE

-[ RECORD 6 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bibliotecavirtualandalucia.juntadeandalucia.es:1017405
base_url     | http://www.bibliotecavirtualdeandalucia.es/catalogo/es/consulta/registro.cmd?id=1017405
terminal_url | http://www.bibliotecavirtualdeandalucia.es/catalogo/es/consulta/registro.cmd?id=1017405

seems to be a general purpose regional library? not research-specific

SKIP-UNSURE

-[ RECORD 7 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:etd.adm.unipi.it:etd-02272019-123644
base_url     | http://etd.adm.unipi.it/theses/available/etd-02272019-123644/
terminal_url | https://etd.adm.unipi.it/theses/available/etd-02272019-123644/

This specific URL is not available (FORBIDDEN)

others have multiple files, not just a single PDF:
https://etd.adm.unipi.it/t/etd-09102013-124430/

SKIP-UNSURE

-[ RECORD 9 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:commons.ln.edu.hk:sw_master-5408
base_url     | https://commons.ln.edu.hk/sw_master/4408
terminal_url | https://commons.ln.edu.hk/sw_master/4408/

worth crawling I guess

METADATA-ONLY

-[ RECORD 10 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:mouseion.jax.org:ssbb1976-1224
base_url     | https://mouseion.jax.org/ssbb1976/225
terminal_url | https://mouseion.jax.org/ssbb1976/225/

METADATA-ONLY

-[ RECORD 13 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:aleph.bib-bvb.de:bvb01-016604343
base_url     | http://bvbm1.bib-bvb.de/webclient/DeliveryManager?pid=176332&custom_att_2=simple_viewer
terminal_url | http://digital.bib-bvb.de/view/action/singleViewer.do?dvs=1593269021002~476&locale=en_US&VIEWER_URL=/view/action/singleViewer.do?&DELIVERY_RULE_ID=31&frameId=1&usePid1=true&usePid2=true

SOFT-404 / FORBIDDEN (cookie timeout)

-[ RECORD 14 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bivaldi.gva.es:11740
base_url     | https://bivaldi.gva.es/es/consulta/registro.do?id=11740
terminal_url | https://bivaldi.gva.es/es/consulta/registro.do?id=11740


-[ RECORD 16 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.wur.nl:wurpubs/443282
base_url     | https://library.wur.nl/WebQuery/wurpubs/443282
terminal_url | https://library.wur.nl/WebQuery/wurpubs/443282

DIGIBIS platform (like some others)

FIXED-PARTIAL

-[ RECORD 18 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:hal:in2p3-00414135v1
base_url     | http://hal.in2p3.fr/in2p3-00414135
terminal_url | http://hal.in2p3.fr:80/in2p3-00414135

METADATA-ONLY

-[ RECORD 19 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:aaltodoc.aalto.fi:123456789/13201
base_url     | https://aaltodoc.aalto.fi/handle/123456789/13201
terminal_url | https://aaltodoc.aalto.fi/handle/123456789/13201

This specific record is not accessible.
Another: https://aaltodoc.aalto.fi/handle/123456789/38002

DSpace 5.4

Worked (from recent changes)


-[ RECORD 20 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:sedici.unlp.edu.ar:10915/40144
base_url     | http://xjornadaslc.fahce.unlp.edu.ar/actas/Ramon_Esteban_Chaparro.pdf/view
terminal_url | http://xjornadaslc.fahce.unlp.edu.ar/actas/Ramon_Esteban_Chaparro.pdf/view

This is a journal! Cool. Plone software platform.

FIXED

## Top no-capture Domains

Top terminal no-capture domains:

    SELECT domain, COUNT(domain)
    FROM (
        SELECT
            ingest_file_result.ingest_type,
            ingest_file_result.status,
            substring(ingest_file_result.terminal_url FROM '[^/]+://([^/]*)') AS domain
        FROM ingest_file_result
        LEFT JOIN ingest_request
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_file_result.status = 'no-capture'
    ) t1
    WHERE t1.domain != ''
        AND t1.status != 'success'
    GROUP BY domain
    ORDER BY COUNT DESC
    LIMIT 30;

                  domain               | count 
    -----------------------------------+-------
     digitalrepository.unm.edu         | 94087
     escholarship.org                  | 80632
     ir.opt.ac.cn                      | 70504
     idus.us.es                        | 67908
     www.cambridge.org                 | 56376
     www.ssoar.info                    | 52534
     rep.bntu.by                       | 52127
     scholarworks.umt.edu              | 48546
     publikationen.ub.uni-frankfurt.de | 46987
     dk.um.si                          | 45753
     repositorio.uladech.edu.pe        | 37028
     uu.diva-portal.org                | 34929
     digitalcommons.law.byu.edu        | 31732
     sedici.unlp.edu.ar                | 31233
     elib.sfu-kras.ru                  | 29131
     jyx.jyu.fi                        | 28144
     www.repository.cam.ac.uk          | 27728
     nagoya.repo.nii.ac.jp             | 26673
     www.duo.uio.no                    | 25258
     www.persee.fr                     | 24968
     www2.senado.leg.br                | 24426
     tesis.ucsm.edu.pe                 | 24049
     digitalcommons.unl.edu            | 21974
     www.degruyter.com                 | 21940
     www.igi-global.com                | 20736
     thekeep.eiu.edu                   | 20712
     docs.lib.purdue.edu               | 20538
     repositorio.cepal.org             | 20280
     elib.bsu.by                       | 19620
     minds.wisconsin.edu               | 19473
    (30 rows)

These all seem worth crawling. A couple publishers (cambridge.org), and
persee.fr will probably fail, but not too many URLs.

## Summary of Filtered Prefixes and Domains (OAI-PMH)

oai:kb.dk:
    too large and generic
oai:bdr.oai.bsb-muenchen.de:
    too large and generic
oai:hispana.mcu.es:
    too large and generic
oai:bnf.fr:
    too large and generic
oai:ukm.si:
    too large and generic
oai:biodiversitylibrary.org:
    redundant with other ingest and archive.org content
oai:hsp.org:
    large; historical content only
oai:repec:
    large; mostly (entirely?) links to publisher sites
oai:n/a:
    meta?
oai:quod.lib.umich.edu:
    entire issues? hard to crawl so skip for now
oai:hypotheses.org:
    HTML, not PDF
oai:americanae.aecid.es:
    large, complex. skip for now
oai:www.irgrid.ac.cn:
    aggregator of other IRs
oai:espace.library.uq.edu.au:
    large; metadata only; javascript heavy (poor heritrix crawling)
oai:edoc.mpg.de:
    deprecated domain, with no redirects
oai:bibliotecadigital.jcyl.es:
    digitized historical docs; hard to crawl, skip for now
oai:repository.erciyes.edu.tr:
    gone (domain lapsed)
oai:krm.or.kr:
    "research results repository" (metadata only)

www.kb.dk
    large, general purpose, scope
kb-images.kb.dk
    deprecated
mdz-nbn-resolving.de
    multiple prefixes end up here. historical docs, scope
aggr.ukm.um.si
    large, out of scope
edoc.mpg.de
    deprecated domain
doaj.org
    index (metadata only)
orcid.org
    out of scope
gateway.isiknowledge.com
    clarivate login/payall (skipping in ingest)

Needs filtering to a subset of records (by 'set' or other filtering?):

oai:igi.indrastra.com:
oai:invenio.nusl.cz:
oai:t2r2.star.titech.ac.jp:
oai:evastar-karlsruhe.de:
oai:repository.ust.hk:
oai:serval.unil.ch:
oai:pure.atira.dk:

FIlters in SQL syntax:

    AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:repec:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:n/a:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:quod.lib.umich.edu:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:americanae.aecid.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:www.irgrid.ac.cn:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:espace.library.uq.edu:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:edoc.mpg.de:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bibliotecadigital.jcyl.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:repository.erciyes.edu.tr:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:krm.or.kr:%'
    AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
    AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
    AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
    AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    AND ingest_request.base_url NOT LIKE '%edoc.mpg.de%'
    AND ingest_request.base_url NOT LIKE '%doaj.org%'
    AND ingest_request.base_url NOT LIKE '%orcid.org%'
    AND ingest_request.base_url NOT LIKE '%gateway.isiknowledge.com%'

and in some contexts (PDFs; switch to HTML):

    AND ingest_request.link_source_id NOT LIKE 'oai:hypotheses.org:%'

## Overall Summary of OAI-PMH Stuff

Big picture is that the majority of `no-pdf-link` crawl status are because of
repository scope, record scope, or content format issues. That being said,
there was a sizable fraction of sites which were platforms (like DSpace) which
were not ingesting well.

A significant fraction of records are "metadata only" (of papers), or non-paper
entity types (like persons, grants, or journal titles), and a growing fraction
(?) are metadata plus link to OA publisher fulltext (offsite). Might be
possible to detect these at ingest time, or earlier at OAI-PMH
harvest/transform time and filter them out.

It may be worthwhile to attempt ingest of multiple existing captures
(timestamps) in the ingest pipeline.  Eg, instead of chosing a single "best"
capture, if there are multiple HTTP 200 status captures, try ingest with each
(or at least a couple).  This is because repository software gets upgraded, so
old "no-capture" or "not found" or "link loop" type captures may work when
recrawled.

New summary with additional filters:

    SELECT ingest_file_result.status, COUNT(*)
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:repec:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:n/a:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:quod.lib.umich.edu:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:americanae.aecid.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:www.irgrid.ac.cn:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:espace.library.uq.edu:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:edoc.mpg.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bibliotecadigital.jcyl.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:repository.erciyes.edu.tr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:krm.or.kr:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_request.base_url NOT LIKE '%edoc.mpg.de%'
        AND ingest_request.base_url NOT LIKE '%doaj.org%'
        AND ingest_request.base_url NOT LIKE '%orcid.org%'
        AND ingest_request.base_url NOT LIKE '%gateway.isiknowledge.com%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hypotheses.org:%'
    GROUP BY status
    ORDER BY COUNT DESC
    LIMIT 20;

            status         |  count
    -----------------------+----------
     success               | 12872279
     no-pdf-link           |  9329602
     no-capture            |  4696362
     redirect-loop         |  1541458
     terminal-bad-status   |   660418
     link-loop             |   452831
     wrong-mimetype        |   434868
     null-body             |    71065
     cdx-error             |    17005
                           |    15275
     petabox-error         |    12743
     wayback-error         |    11759
     skip-url-blocklist    |      182
     gateway-timeout       |      122
     redirects-exceeded    |      120
     bad-redirect          |      117
     bad-gzip-encoding     |      111
     wayback-content-error |      102
     timeout               |       72
     blocked-cookie        |       62
    (20 rows)