aboutsummaryrefslogtreecommitdiffstats
path: root/notes/ingest/2021-09-02_oai_pmh_patch.md
blob: fded7b3b081f43b17c499c2bbbc8bd76edff6d78 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578

Just a "patch" of previous OAI-PMH crawl/ingest: re-ingesting and potentially
re-crawling content which failed to ingest the first time.

May fold this in with more general patch crawling.

## Basic Counts

    SELECT ingest_file_result.status, COUNT(*)
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    GROUP BY status
    ORDER BY COUNT DESC
    LIMIT 20;

            status         |  count   
    -----------------------+----------
     success               | 14145387
     no-pdf-link           | 12063022
     no-capture            |  5485640
     redirect-loop         |  2092705
     terminal-bad-status   |   747372
     wrong-mimetype        |   597219
     link-loop             |   542144
     null-body             |    93566
     cdx-error             |    19798
     petabox-error         |    17943
                           |    15283
     wayback-error         |    13897
     gateway-timeout       |      511
     skip-url-blocklist    |      184
     wayback-content-error |      146
     bad-redirect          |      137
     redirects-exceeded    |      120
     bad-gzip-encoding     |      116
     timeout               |       80
     blocked-cookie        |       64
    (20 rows)

    SELECT
        oai_prefix,
        COUNT(CASE WHEN status = 'success' THEN 1 END) as success,
        COUNT(*) as total
    FROM (
        SELECT
            ingest_file_result.status as status,
            -- eg "oai:cwi.nl:4881"
            substring(ingest_request.link_source_id FROM 'oai:([^:]+):.*') AS oai_prefix
        FROM ingest_request
        LEFT JOIN ingest_file_result
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE 
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
            AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
            AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    ) t1
    GROUP BY oai_prefix
    ORDER BY total DESC
    LIMIT 40;


            oai_prefix         | success |  total
    ---------------------------+---------+---------
     repec                     | 1133175 | 2783448
     hal                       |  573218 | 1049607
     www.irgrid.ac.cn          |   18007 |  748828
     cds.cern.ch               |   74078 |  688091
     americanae.aecid.es       |   71310 |  572792
     juser.fz-juelich.de       |   23026 |  518551
     espace.library.uq.edu.au  |    6649 |  508960
     igi.indrastra.com         |   59629 |  478577
     archive.ugent.be          |   65306 |  424014
     hrcak.srce.hr             |  404085 |  414897
     zir.nsk.hr                |  156753 |  397200
     renati.sunedu.gob.pe      |   79362 |  388355
     hypotheses.org            |       3 |  374296
     rour.neicon.ru            |    7997 |  354529
     generic.eprints.org       |  263566 |  340470
     invenio.nusl.cz           |    6340 |  325867
     evastar-karlsruhe.de      |   62282 |  317952
     quod.lib.umich.edu        |       5 |  309135
     diva.org                  |   67917 |  298348
     t2r2.star.titech.ac.jp    |    1085 |  289388
     edpsciences.org           |  139495 |  284972
     repository.ust.hk         |   10245 |  283417
     revues.org                |  151156 |  277497
     pure.atira.dk             |   13492 |  260754
     bibliotecadigital.jcyl.es |   50606 |  254134
     escholarship.org/ark      |  140835 |  245203
     ojs.pkp.sfu.ca            |  168029 |  229387
     lup.lub.lu.se             |   49358 |  226602
     library.wur.nl            |   15051 |  216738
     digitalrepository.unm.edu |  111704 |  211749
     infoscience.tind.io       |   60166 |  207299
     edoc.mpg.de               |       0 |  205252
     erudit.org                |  168490 |  197803
     delibra.bg.polsl.pl       |   38666 |  196652
     n/a                       |       0 |  193814
     aleph.bib-bvb.de          |    4349 |  186666
     serval.unil.ch            |   41643 |  186372
     orbi.ulg.ac.be            |    2400 |  184551
     digitalcommons.unl.edu    |  144025 |  184372
     bib-pubdb1.desy.de        |   33525 |  182717
    (40 rows)

Top counts by OAI prefix and status:

    SELECT
        oai_prefix,
        status,
        COUNT((oai_prefix,status))
    FROM (
        SELECT
            ingest_file_result.status as status,
            -- eg "oai:cwi.nl:4881"
            substring(ingest_request.link_source_id FROM 'oai:([^:]+):.*') AS oai_prefix
        FROM ingest_request
        LEFT JOIN ingest_file_result
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE 
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
            AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
            AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
            AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
            AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    ) t1
    GROUP BY oai_prefix, status
    ORDER BY COUNT DESC
    LIMIT 50;

            oai_prefix         |    status     |  count  
    ---------------------------+---------------+---------
     repec                     | success       | 1133175
     repec                     | no-pdf-link   |  638105
     hal                       | success       |  573218
     cds.cern.ch               | no-capture    |  540380
     repec                     | redirect-loop |  516451
     juser.fz-juelich.de       | no-pdf-link   |  477881
     americanae.aecid.es       | no-pdf-link   |  417766
     hrcak.srce.hr             | success       |  404085
     www.irgrid.ac.cn          | no-pdf-link   |  370908
     hal                       | no-pdf-link   |  359252
     www.irgrid.ac.cn          | no-capture    |  355532
     espace.library.uq.edu.au  | no-pdf-link   |  320479
     igi.indrastra.com         | no-pdf-link   |  318242
     repec                     | no-capture    |  316981
     invenio.nusl.cz           | no-pdf-link   |  309802
     rour.neicon.ru            | redirect-loop |  300911
     hypotheses.org            | no-pdf-link   |  300251
     renati.sunedu.gob.pe      | no-capture    |  282800
     t2r2.star.titech.ac.jp    | no-pdf-link   |  272045
     generic.eprints.org       | success       |  263566
     quod.lib.umich.edu        | no-pdf-link   |  259661
     archive.ugent.be          | no-capture    |  256127
     evastar-karlsruhe.de      | no-pdf-link   |  248939
     zir.nsk.hr                | link-loop     |  226919
     repository.ust.hk         | no-pdf-link   |  208569
     edoc.mpg.de               | no-pdf-link   |  199758
     bibliotecadigital.jcyl.es | no-pdf-link   |  188433
     orbi.ulg.ac.be            | no-pdf-link   |  172373
     diva.org                  | no-capture    |  171115
     lup.lub.lu.se             | no-pdf-link   |  168652
     erudit.org                | success       |  168490
     ojs.pkp.sfu.ca            | success       |  168029
     lib.dr.iastate.edu        | success       |  158494
     zir.nsk.hr                | success       |  156753
     digital.kenyon.edu        | success       |  154900
     revues.org                | success       |  151156
     books.openedition.org     | no-pdf-link   |  149607
     freidok.uni-freiburg.de   | no-pdf-link   |  146837
     digitalcommons.unl.edu    | success       |  144025
     escholarship.org/ark      | success       |  140835
     culeuclid                 | link-loop     |  140291
     edpsciences.org           | success       |  139495
     serval.unil.ch            | no-pdf-link   |  138644
     bib-pubdb1.desy.de        | no-pdf-link   |  133815
     krm.or.kr                 | no-pdf-link   |  132461
     pure.atira.dk             | no-pdf-link   |  132179
     oai-gms.dimdi.de          | redirect-loop |  131409
     aleph.bib-bvb.de          | no-capture    |  128261
     library.wur.nl            | no-pdf-link   |  124718
     lirias2repo.kuleuven.be   | no-capture    |  123106
    (50 rows)

Note: could just delete the "excluded" rows? and not harvest them in the
future, and filter them at ingest time (in transform script).



## Investigate no-pdf-link sandcrawler improvements

Do some spot-sampling of 'no-pdf-link' domains, see if newer sandcrawler works:

    SELECT
        ingest_request.link_source_id AS oai_id,
        ingest_request.base_url as base_url ,
        ingest_file_result.terminal_url as terminal_url 
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_file_result.status = 'no-pdf-link'
        AND ingest_request.link_source_id LIKE 'oai:library.wur.nl:%'
    ORDER BY random()
    LIMIT 10;

Random sampling of *all* 'no-pdf-link' URLs (see if newer sandcrawler works):

    \x auto

    SELECT
        ingest_request.link_source_id AS oai_id,
        ingest_request.base_url as base_url ,
        ingest_file_result.terminal_url as terminal_url 
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_file_result.status = 'no-pdf-link'
    ORDER BY random()
    LIMIT 30;

### repec (SKIP-PREFIX)

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:eee:jmacro:v:54:y:2017:i:pb:p:332-351
base_url     | http://www.sciencedirect.com/science/article/pii/S0164070417301593
terminal_url | http://www.sciencedirect.com/science/article/pii/S0164070417301593
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:eee:jomega:v:16:y:1988:i:2:p:107-115
base_url     | http://www.sciencedirect.com/science/article/pii/0305-0483(88)90041-2
terminal_url | https://www.sciencedirect.com/science/article/abs/pii/0305048388900412
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:sgm:pzwzuw:v:14:i:59:y:2016:p:73-92
base_url     | http://pz.wz.uw.edu.pl/en
terminal_url | http://pz.wz.uw.edu.pl:80/en
-[ RECORD 1 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:eee:jmacro:v:54:y:2017:i:pb:p:332-351
base_url     | http://www.sciencedirect.com/science/article/pii/S0164070417301593
terminal_url | http://www.sciencedirect.com/science/article/pii/S0164070417301593
-[ RECORD 2 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:eee:jomega:v:16:y:1988:i:2:p:107-115
base_url     | http://www.sciencedirect.com/science/article/pii/0305-0483(88)90041-2
terminal_url | https://www.sciencedirect.com/science/article/abs/pii/0305048388900412
-[ RECORD 3 ]+--------------------------------------------------------------------------------------------------------
--------------------------------------
oai_id       | oai:repec:sgm:pzwzuw:v:14:i:59:y:2016:p:73-92
base_url     | http://pz.wz.uw.edu.pl/en
terminal_url | http://pz.wz.uw.edu.pl:80/en
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:erv:rccsrc:y:2016:i:2016_11:35
base_url     | http://www.eumed.net/rev/caribe/2016/11/estructura.html
terminal_url | http://www.eumed.net:80/rev/caribe/2016/11/estructura.html
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:pio:envira:v:33:y:2001:i:4:p:629-647
base_url     | http://www.envplan.com/epa/fulltext/a33/a3319.pdf
terminal_url | http://uk.sagepub.com:80/en-gb/eur/pion-journals-published
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repec:tpr:qjecon:v:100:y:1985:i:3:p:651-75
base_url     | http://links.jstor.org/sici?sici=0033-5533%28198508%29100%3A3%3C651%3ATCOCEA%3E2.0.CO%3B2-2&origin=repec
terminal_url | https://www.jstor.org/stable/1884373

Huh! This is just a catalog of other domains. Should probably skip

DONE: skip/filter repec

### juser.fz-juelich.de (SCOPE)

-[ RECORD 1 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:132217
base_url     | http://juser.fz-juelich.de/record/132217
terminal_url | http://juser.fz-juelich.de/record/132217

Poster; no files.

-[ RECORD 2 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:268598
base_url     | http://juser.fz-juelich.de/record/268598
terminal_url | http://juser.fz-juelich.de/record/268598

Journal.

-[ RECORD 3 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:126613
base_url     | http://juser.fz-juelich.de/record/126613
terminal_url | http://juser.fz-juelich.de/record/126613

-[ RECORD 4 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:67362
base_url     | http://juser.fz-juelich.de/record/67362
terminal_url | http://juser.fz-juelich.de/record/67362
-[ RECORD 5 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:869189
base_url     | http://juser.fz-juelich.de/record/869189
terminal_url | http://juser.fz-juelich.de/record/869189
-[ RECORD 6 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:810746
base_url     | http://juser.fz-juelich.de/record/810746
terminal_url | http://juser.fz-juelich.de/record/810746
-[ RECORD 7 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:52897
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-52897%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-52897%22
-[ RECORD 8 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:114755
base_url     | http://juser.fz-juelich.de/record/114755
terminal_url | http://juser.fz-juelich.de/record/114755
-[ RECORD 9 ]+------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:58025
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-58025%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-58025%22

The search URLs seem redundant? Not going to try to handle those.

"Powered by Invenio v1.1.7"

All of these examples seem to be not papers. Maybe we can filter these better
at the harvest or transform stage?

### americanae.aecid.es (MIXED)

-[ RECORD 1 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:502896
base_url     | http://biblioteca.clacso.edu.ar/gsdl/cgi-bin/library.cgi?a=d&c=mx/mx-010&d=60327292007oai
terminal_url | http://biblioteca.clacso.edu.ar/gsdl/cgi-bin/library.cgi?a=d&c=mx/mx-010&d=60327292007oai

just a metadata record? links to redalyc

METADATA-ONLY

-[ RECORD 2 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:534600
base_url     | http://bdh-rd.bne.es/viewer.vm?id=0000077778&page=1
terminal_url | http://bdh-rd.bne.es/viewer.vm?id=0000077778&page=1
-[ RECORD 3 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:524567
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=524567
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=524567

NOT-FOUND (404)

-[ RECORD 4 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:378914
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=378914
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=378914

Some single-page image archival thing? bespoke, skipping.

SKIP-BESPOKE

-[ RECORD 5 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:526142
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=526142
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=526142

NOT-FOUND (404)

-[ RECORD 6 ]+---------------------------------------------------------------------------------------------
oai_id       | oai:americanae.aecid.es:373408
base_url     | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=373408
terminal_url | http://americanae.aecid.es/americanae/es/registros/registro.do?tipoRegistro=MTD&idBib=373408

NOT-FOUND (404)

### www.irgrid.ac.cn (SKIP-PREFIX)

Chinese Academy of Sciences Institutional Repositories Grid

-[ RECORD 1 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1749980
base_url     | http://www.irgrid.ac.cn/handle/1471x/1749980
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1749980

Can't access

FORBIDDEN

-[ RECORD 2 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/857397
base_url     | http://www.irgrid.ac.cn/handle/1471x/857397
terminal_url | http://www.irgrid.ac.cn/handle/1471x/857397

Just linking to another IR; skip it.

http://ir.ipe.ac.cn/handle/122111/10608

requires login

DONE: '/password-login;jsessionid' as a loginwall URL pattern
    http://ir.ipe.ac.cn/handle/122111/10608
    http://ir.ipe.ac.cn/bitstream/122111/10608/2/%e9%92%9d%e9%a1%b6%e8%9e%ba%e6%97%8b%e8%97%bb%e5%9c%a8%e4%b8%8d%e5%90%8c%e5%85%89%e7%85%a7%e6%9d%a1%e4%bb%b6%e4%b8%8b%e7%9a%84%e6%94%be%e6%b0%a7%e7%89%b9%e6%80%a7_%e8%96%9b%e5%8d%87%e9%95%bf.pdf

-[ RECORD 3 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1060447
base_url     | http://www.irgrid.ac.cn/handle/1471x/1060447
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1060447
-[ RECORD 4 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1671377
base_url     | http://ir.iggcas.ac.cn/handle/132A11/68622
terminal_url | http://ir.iggcas.ac.cn/handle/132A11/68622
-[ RECORD 5 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/1178430
base_url     | http://www.irgrid.ac.cn/handle/1471x/1178430
terminal_url | http://www.irgrid.ac.cn/handle/1471x/1178430
-[ RECORD 6 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/2488017
base_url     | http://www.irgrid.ac.cn/handle/1471x/2488017
terminal_url | http://www.irgrid.ac.cn/handle/1471x/2488017
-[ RECORD 7 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/977147
base_url     | http://www.irgrid.ac.cn/handle/1471x/977147
terminal_url | http://www.irgrid.ac.cn/handle/1471x/977147
-[ RECORD 8 ]+---------------------------------------------
oai_id       | oai:www.irgrid.ac.cn:1471x/2454503
base_url     | http://ir.nwipb.ac.cn/handle/363003/9957
terminal_url | http://ir.nwipb.ac.cn/handle/363003/9957

this domain is a disapointment :(

should continue crawling, as the metadata is open and good. but won't get fulltext?

### hal (FIXED-PARTIAL)

-[ RECORD 1 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00744951v1
base_url     | https://hal.archives-ouvertes.fr/hal-00744951
terminal_url | https://hal.archives-ouvertes.fr/hal-00744951

Off-site OA link.

FIXED-HAL

-[ RECORD 2 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-01065398v1
base_url     | https://hal.archives-ouvertes.fr/hal-01065398/file/AbstractSGE14_B_assaad.pdf
terminal_url | https://hal.archives-ouvertes.fr/index/index
-[ RECORD 3 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:lirmm-00371599v1
base_url     | https://hal-lirmm.ccsd.cnrs.fr/lirmm-00371599
terminal_url | https://hal-lirmm.ccsd.cnrs.fr/lirmm-00371599

To elsevier :(

-[ RECORD 4 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00284780v1
base_url     | https://hal.archives-ouvertes.fr/hal-00284780
terminal_url | https://hal.archives-ouvertes.fr/hal-00284780

METADATA-ONLY

-[ RECORD 5 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00186151v1
base_url     | https://hal.archives-ouvertes.fr/hal-00186151
terminal_url | https://hal.archives-ouvertes.fr/hal-00186151

METADATA-ONLY

-[ RECORD 6 ]+------------------------------------------------------------------------------
oai_id       | oai:hal:hal-00399754v1
base_url     | https://hal.archives-ouvertes.fr/hal-00399754
terminal_url | https://hal.archives-ouvertes.fr/hal-00399754

METADATA-ONLY


### espace.library.uq.edu.au (SKIP)

-[ RECORD 1 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:136497
base_url     | https://espace.library.uq.edu.au/view/UQ:136497
terminal_url | https://espace.library.uq.edu.au/view/UQ:136497
-[ RECORD 2 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:411389
base_url     | https://espace.library.uq.edu.au/view/UQ:411389
terminal_url | https://espace.library.uq.edu.au/view/UQ:411389
-[ RECORD 3 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:401773
base_url     | https://espace.library.uq.edu.au/view/UQ:401773
terminal_url | https://espace.library.uq.edu.au/view/UQ:401773
-[ RECORD 4 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:675334
base_url     | https://espace.library.uq.edu.au/view/UQ:675334
terminal_url | https://espace.library.uq.edu.au/view/UQ:675334
-[ RECORD 5 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:312311
base_url     | https://espace.library.uq.edu.au/view/UQ:312311
terminal_url | https://espace.library.uq.edu.au/view/UQ:312311
-[ RECORD 6 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:209401
base_url     | https://espace.library.uq.edu.au/view/UQ:209401
terminal_url | https://espace.library.uq.edu.au/view/UQ:209401
-[ RECORD 7 ]+------------------------------------------------
oai_id       | oai:espace.library.uq.edu.au:uq:327188
base_url     | https://espace.library.uq.edu.au/view/UQ:327188
terminal_url | https://espace.library.uq.edu.au/view/UQ:327188

Very javascript heavy (skeletal HTML). And just links to fulltext on publisher
sites.

### igi.indrastra.com (METADATA-ONLY)

-[ RECORD 1 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:267221
base_url     | http://igi.indrastra.com/items/show/267221
terminal_url | http://igi.indrastra.com/items/show/267221
-[ RECORD 2 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:181799
base_url     | http://igi.indrastra.com/items/show/181799
terminal_url | http://igi.indrastra.com/items/show/181799
-[ RECORD 3 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:125382
base_url     | http://igi.indrastra.com/items/show/125382
terminal_url | http://igi.indrastra.com/items/show/125382
-[ RECORD 4 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:47266
base_url     | http://igi.indrastra.com/items/show/47266
terminal_url | http://igi.indrastra.com/items/show/47266
-[ RECORD 5 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:12872
base_url     | http://igi.indrastra.com/items/show/12872
terminal_url | http://igi.indrastra.com/items/show/12872
-[ RECORD 6 ]+---------------------------------------------------------
oai_id       | oai:igi.indrastra.com:231620
base_url     | http://igi.indrastra.com/items/show/231620
terminal_url | http://igi.indrastra.com/items/show/231620

"Proudly powered by Omeka"

### invenio.nusl.cz (METADATA-ONLY)

           oai_id           |              base_url              |             terminal_url
----------------------------+------------------------------------+--------------------------------------
 oai:invenio.nusl.cz:237409 | http://www.nusl.cz/ntk/nusl-237409 | http://invenio.nusl.cz/record/237409
 oai:invenio.nusl.cz:180783 | http://www.nusl.cz/ntk/nusl-180783 | http://invenio.nusl.cz/record/180783
 oai:invenio.nusl.cz:231961 | http://www.nusl.cz/ntk/nusl-231961 | http://invenio.nusl.cz/record/231961
 oai:invenio.nusl.cz:318800 | http://www.nusl.cz/ntk/nusl-318800 | http://invenio.nusl.cz/record/318800
 oai:invenio.nusl.cz:259695 | http://www.nusl.cz/ntk/nusl-259695 | http://invenio.nusl.cz/record/259695
 oai:invenio.nusl.cz:167393 | http://www.nusl.cz/ntk/nusl-167393 | http://invenio.nusl.cz/record/167393
 oai:invenio.nusl.cz:292987 | http://www.nusl.cz/ntk/nusl-292987 | http://invenio.nusl.cz/record/292987
 oai:invenio.nusl.cz:283396 | http://www.nusl.cz/ntk/nusl-283396 | http://invenio.nusl.cz/record/283396
 oai:invenio.nusl.cz:241512 | http://www.nusl.cz/ntk/nusl-241512 | http://invenio.nusl.cz/record/241512
 oai:invenio.nusl.cz:178631 | http://www.nusl.cz/ntk/nusl-178631 | http://invenio.nusl.cz/record/178631

Metadata only (at least this set)

### hypotheses.org

-[ RECORD 1 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:mittelalter/9529
base_url     | http://mittelalter.hypotheses.org/9529
terminal_url | https://mittelalter.hypotheses.org/9529
-[ RECORD 2 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivalia/18638
base_url     | http://archivalia.hypotheses.org/18638
terminal_url | https://archivalia.hypotheses.org/18638
-[ RECORD 3 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivalia/13614
base_url     | http://archivalia.hypotheses.org/13614
terminal_url | https://archivalia.hypotheses.org/13614
-[ RECORD 4 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:teteschercheuses/2785
base_url     | http://teteschercheuses.hypotheses.org/2785
terminal_url | https://teteschercheuses.hypotheses.org/2785
-[ RECORD 5 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:altervsego/608
base_url     | http://altervsego.hypotheses.org/608
terminal_url | http://altervsego.hypotheses.org/608
-[ RECORD 6 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:archivewk1/21905
base_url     | http://archivewk1.hypotheses.org/21905
terminal_url | https://archivewk1.hypotheses.org/21905
-[ RECORD 7 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:slkdiaspo/3321
base_url     | http://slkdiaspo.hypotheses.org/3321
terminal_url | https://slkdiaspo.hypotheses.org/3321
-[ RECORD 8 ]+---------------------------------------------
oai_id       | oai:hypotheses.org:diga/280
base_url     | http://diga.hypotheses.org/280
terminal_url | https://diga.hypotheses.org/280

These are all a big mix... basically blogs. Should continue crawling, but expect no yield.

### t2r2.star.titech.ac.jp (METADATA-ONLY)

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00105099
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100499795
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100499795
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00101346
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100495549
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100495549
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50161100
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100632554
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100632554
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:00232407
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100527528
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100527528
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50120040
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100612598
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100612598
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50321440
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100713492
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100713492
-[ RECORD 7 ]+----------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50235666
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100668778
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100668778


### quod.lib.umich.edu

-[ RECORD 1 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acf2679.0015.003-2
base_url     | http://name.umdl.umich.edu/acf2679.0015.003
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acf2679.0015.003
-[ RECORD 2 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:b14970.0001.001
base_url     | http://name.umdl.umich.edu/B14970.0001.001
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=eebo2;idno=B14970.0001.001
-[ RECORD 3 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acf2679.0009.010-3
base_url     | http://name.umdl.umich.edu/ACF2679-1623SOUT-209
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acf2679.0009.010;node=acf2679.0009.010:3
-[ RECORD 4 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg2248.1-16.006-43
base_url     | http://name.umdl.umich.edu/acg2248.1-16.006
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg2248.1-16.006
-[ RECORD 5 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg2248.1-14.011-9
base_url     | http://name.umdl.umich.edu/ACG2248-1489LADI-364
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg2248.1-14.011;node=acg2248.1-14.011:9
-[ RECORD 6 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:acg1336.1-24.006-9
base_url     | http://name.umdl.umich.edu/acg1336.1-24.006
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=moajrnl;idno=acg1336.1-24.006
-[ RECORD 7 ]+-------------------------------------------------------------------------------------------------------
oai_id       | oai:quod.lib.umich.edu:africanamer.0002.32a
base_url     | http://name.umdl.umich.edu/africanamer.0002.32a
terminal_url | https://quod.lib.umich.edu/cgi/t/text/text-idx?c=africanamer;idno=africanamer.0002.32a

These are... issues of journals? Should continue to crawl, but not expect much.

### evastar-karlsruhe.de (METADATA-ONLY)

-[ RECORD 1 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:270011444
base_url     | https://publikationen.bibliothek.kit.edu/270011444
terminal_url | https://publikationen.bibliothek.kit.edu/270011444
-[ RECORD 2 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:1000050117
base_url     | https://publikationen.bibliothek.kit.edu/1000050117
terminal_url | https://publikationen.bibliothek.kit.edu/1000050117
-[ RECORD 3 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:362296
base_url     | https://publikationen.bibliothek.kit.edu/362296
terminal_url | https://publikationen.bibliothek.kit.edu/362296
-[ RECORD 4 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:23042000
base_url     | https://publikationen.bibliothek.kit.edu/23042000
terminal_url | https://publikationen.bibliothek.kit.edu/23042000
-[ RECORD 5 ]+----------------------------------------------------
oai_id       | oai:evastar-karlsruhe.de:1000069945
base_url     | https://publikationen.bibliothek.kit.edu/1000069945
terminal_url | https://publikationen.bibliothek.kit.edu/1000069945


### repository.ust.hk

-[ RECORD 1 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-67233
base_url     | http://repository.ust.hk/ir/Record/1783.1-67233
terminal_url | http://repository.ust.hk/ir/Record/1783.1-67233
-[ RECORD 2 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-63232
base_url     | http://gateway.isiknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=LinksAMR&SrcApp=PARTNER_APP&DestLinkType=FullRecord&DestApp=WOS&KeyUT=A1981KV47900017
terminal_url | http://login.webofknowledge.com/error/Error?Src=IP&Alias=WOK5&Error=IPError&Params=DestParams%3D%253FUT%253DWOS%253AA1981KV47900017%2526customersID%253DLinksAMR%2526product%253DWOS%2526action%253Dretrieve%2526mode%253DFullRecord%26DestApp%3DWOS%26SrcApp%3DPARTNER_APP%26SrcAuth%3DLinksAMR&PathInfo=%2F&RouterURL=http%3A%2F%2Fwww.webofknowledge.com%2F&Domain=.webofknowledge.com
-[ RECORD 3 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-2891
base_url     | http://gateway.isiknowledge.com/gateway/Gateway.cgi?GWVersion=2&SrcAuth=LinksAMR&SrcApp=PARTNER_APP&DestLinkType=FullRecord&DestApp=WOS&KeyUT=000240035400103
terminal_url | https://login.webofknowledge.com/error/Error?Src=IP&Alias=WOK5&Error=IPError&Params=DestParams%3D%253FUT%253DWOS%253A000240035400103%2526customersID%253DLinksAMR%2526product%253DWOS%2526action%253Dretrieve%2526mode%253DFullRecord%26DestApp%3DWOS%26SrcApp%3DPARTNER_APP%26SrcAuth%3DLinksAMR&PathInfo=%2F&RouterURL=https%3A%2F%2Fwww.webofknowledge.com%2F&Domain=.webofknowledge.com
-[ RECORD 4 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-56231
base_url     | http://repository.ust.hk/ir/Record/1783.1-56231
terminal_url | http://repository.ust.hk/ir/Record/1783.1-56231

[...]

-[ RECORD 6 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-24872
base_url     | http://repository.ust.hk/ir/Record/1783.1-24872
terminal_url | http://repository.ust.hk/ir/Record/1783.1-24872
-[ RECORD 7 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-3457
base_url     | http://lbdiscover.ust.hk/uresolver?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=0003-6870&rft.volume=40&rft.issue=2&rft.date=2009&rft.spage=267&rft.epage=279&rft.aulast=Witana&rft.aufirst=Channa+R.&rft.atitle=Effects+of+surface+characteristics+on+the+plantar+shape+of+feet+and+subjects'+perceived+sensations
terminal_url | http://lbdiscover.ust.hk/uresolver/?url_ver=Z39.88-2004&rft_val_fmt=info:ofi/fmt:kev:mtx:journal&rfr_id=info:sid/HKUST:SPI&rft.genre=article&rft.issn=0003-6870&rft.volume=40&rft.issue=2&rft.date=2009&rft.spage=267&rft.epage=279&rft.aulast=Witana&rft.aufirst=Channa+R.&rft.atitle=Effects+of+surface+characteristics+on+the+plantar+shape+of+feet+and+subjects'+perceived+sensations
-[ RECORD 8 ]+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-73215
base_url     | http://repository.ust.hk/ir/Record/1783.1-73215
terminal_url | http://repository.ust.hk/ir/Record/1783.1-73215

DONE: gateway.isiknowledge.com is bogus/blocking?


### edoc.mpg.de (SKIP-DEPRECATED)

         oai_id         |         base_url          |       terminal_url
------------------------+---------------------------+---------------------------
 oai:edoc.mpg.de:416650 | http://edoc.mpg.de/416650 | http://edoc.mpg.de/416650
 oai:edoc.mpg.de:8195   | http://edoc.mpg.de/8195   | http://edoc.mpg.de/8195
 oai:edoc.mpg.de:379655 | http://edoc.mpg.de/379655 | http://edoc.mpg.de/379655
 oai:edoc.mpg.de:641179 | http://edoc.mpg.de/641179 | http://edoc.mpg.de/641179
 oai:edoc.mpg.de:607141 | http://edoc.mpg.de/607141 | http://edoc.mpg.de/607141
 oai:edoc.mpg.de:544412 | http://edoc.mpg.de/544412 | http://edoc.mpg.de/544412
 oai:edoc.mpg.de:314531 | http://edoc.mpg.de/314531 | http://edoc.mpg.de/314531
 oai:edoc.mpg.de:405047 | http://edoc.mpg.de/405047 | http://edoc.mpg.de/405047
 oai:edoc.mpg.de:239650 | http://edoc.mpg.de/239650 | http://edoc.mpg.de/239650
 oai:edoc.mpg.de:614852 | http://edoc.mpg.de/614852 | http://edoc.mpg.de/614852

This whole instance seems to have been replaced

### bibliotecadigital.jcyl.es (SKIP-DIGITIZED)

-[ RECORD 1 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:10000039962
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10044664
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10044664
-[ RECORD 2 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:14075
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14075
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14075
-[ RECORD 3 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:4842
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=4842
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=4842
-[ RECORD 4 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:14799
base_url     | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14799
terminal_url | http://bibliotecadigital.jcyl.es/i18n/consulta/registro.cmd?id=14799
-[ RECORD 5 ]+--------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:821
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=1003474
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=1003474

Digitized images as pages; too much to deal with for now.

### orbi.ulg.ac.be

-[ RECORD 1 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/128079
base_url     | https://orbi.uliege.be/handle/2268/128079
terminal_url | https://orbi.uliege.be/handle/2268/128079
-[ RECORD 2 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/67659
base_url     | https://orbi.uliege.be/handle/2268/67659
terminal_url | https://orbi.uliege.be/handle/2268/67659
-[ RECORD 3 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/35521
base_url     | https://orbi.uliege.be/handle/2268/35521
terminal_url | https://orbi.uliege.be/handle/2268/35521
-[ RECORD 4 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/107922
base_url     | https://orbi.uliege.be/handle/2268/107922
terminal_url | https://orbi.uliege.be/handle/2268/107922
-[ RECORD 5 ]+----------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/215694
base_url     | https://orbi.uliege.be/handle/2268/215694
terminal_url | https://orbi.uliege.be/handle/2268/215694

Described below.

### library.wur.nl (FIXED-BESPOKE)

                  oai_id               |                    base_url                    |                  terminal_url                  
    -----------------------------------+------------------------------------------------+------------------------------------------------
     oai:library.wur.nl:wurpubs/440939 | https://library.wur.nl/WebQuery/wurpubs/440939 | https://library.wur.nl/WebQuery/wurpubs/440939
     oai:library.wur.nl:wurpubs/427707 | https://library.wur.nl/WebQuery/wurpubs/427707 | https://library.wur.nl/WebQuery/wurpubs/427707
     oai:library.wur.nl:wurpubs/359208 | https://library.wur.nl/WebQuery/wurpubs/359208 | https://library.wur.nl/WebQuery/wurpubs/359208
     oai:library.wur.nl:wurpubs/433378 | https://library.wur.nl/WebQuery/wurpubs/433378 | https://library.wur.nl/WebQuery/wurpubs/433378
     oai:library.wur.nl:wurpubs/36416  | https://library.wur.nl/WebQuery/wurpubs/36416  | https://library.wur.nl/WebQuery/wurpubs/36416
     oai:library.wur.nl:wurpubs/469930 | https://library.wur.nl/WebQuery/wurpubs/469930 | https://library.wur.nl/WebQuery/wurpubs/469930
     oai:library.wur.nl:wurpubs/350076 | https://library.wur.nl/WebQuery/wurpubs/350076 | https://library.wur.nl/WebQuery/wurpubs/350076
     oai:library.wur.nl:wurpubs/19109  | https://library.wur.nl/WebQuery/wurpubs/19109  | https://library.wur.nl/WebQuery/wurpubs/19109
     oai:library.wur.nl:wurpubs/26146  | https://library.wur.nl/WebQuery/wurpubs/26146  | https://library.wur.nl/WebQuery/wurpubs/26146
     oai:library.wur.nl:wurpubs/529922 | https://library.wur.nl/WebQuery/wurpubs/529922 | https://library.wur.nl/WebQuery/wurpubs/529922
    (10 rows)

Seems like a one-off site? But added a pattern.

### pure.atira.dk

-[ RECORD 1 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/a27762fd-0919-4753-af55-00b9b26d02e0
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/hightech-cities-and-the-primitive-jungle-visionary-urbanism-in-europe-and-japan-of-the-1960s(a27762fd-0919-4753-af55-00b9b26d02e0).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/hightech-cities-and-the-primitive-jungle-visionary-urbanism-in-europe-and-japan-of-the-1960s(a27762fd-0919-4753-af55-00b9b26d02e0).html
-[ RECORD 2 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/215c8b96-a821-4947-bee4-c7470e9fbaf8
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/service-recovery-in-health-services--understanding-the-desired-qualities-and-behaviours-of-general-practitioners-during-service-recovery-encounters(215c8b96-a821-4947-bee4-c7470e9fbaf8).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/service-recovery-in-health-services--understanding-the-desired-qualities-and-behaviours-of-general-practitioners-during-service-recovery-encounters(215c8b96-a821-4947-bee4-c7470e9fbaf8).html
-[ RECORD 3 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/95d4920a-12c7-4e25-b86c-5f075ea23a38
base_url     | https://www.tandfonline.com/doi/full/10.1080/03057070.2016.1197694
terminal_url | https://www.tandfonline.com/action/cookieAbsent
-[ RECORD 4 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.atira.dk:publications/8a2508ee-14c9-4c6a-851a-6db442090f41
base_url     | https://www.research.manchester.ac.uk/portal/en/publications/microstructure-and-grain-size-dependence-of-ferroelectric-properties-of-batio3-thin-films-on-lanio3-buffered-si(8a2508ee-14c9-4c6a-851a-6db442090f41).html
terminal_url | https://www.research.manchester.ac.uk/portal/en/publications/microstructure-and-grain-size-dependence-of-ferroelectric-properties-of-batio3-thin-films-on-lanio3-buffered-si(8a2508ee-14c9-4c6a-851a-6db442090f41).html

Metadata only

DONE: /cookieAbsent is cookie block
    https://www.tandfonline.com/action/cookieAbsent

### bib-pubdb1.desy.de (FIXED-INVENIO)

-[ RECORD 2 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:96756
base_url     | http://bib-pubdb1.desy.de/record/96756
terminal_url | http://bib-pubdb1.desy.de/record/96756

Metadata only.

-[ RECORD 3 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:416556
base_url     | http://bib-pubdb1.desy.de/record/416556
terminal_url | http://bib-pubdb1.desy.de/record/416556

Fixed!

-[ RECORD 4 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:414545
base_url     | http://bib-pubdb1.desy.de/search?p=id:%22PUBDB-2018-04027%22
terminal_url | http://bib-pubdb1.desy.de/search?p=id:%22PUBDB-2018-04027%22
-[ RECORD 5 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:170169
base_url     | http://bib-pubdb1.desy.de/record/170169
terminal_url | http://bib-pubdb1.desy.de/record/170169
-[ RECORD 6 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:191154
base_url     | http://bib-pubdb1.desy.de/record/191154
terminal_url | http://bib-pubdb1.desy.de/record/191154

Metadata only

-[ RECORD 7 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:155092
base_url     | http://bib-pubdb1.desy.de/record/155092
terminal_url | http://bib-pubdb1.desy.de/record/155092

Fixed!

-[ RECORD 8 ]+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bib-pubdb1.desy.de:97158
base_url     | http://bib-pubdb1.desy.de/record/97158
terminal_url | http://bib-pubdb1.desy.de/record/97158

Metadata only

"Powered by Invenio v1.1.7"

Can/should skip the "search" URLs

### serval.unil.ch

-[ RECORD 1 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_60346fc75171
base_url     | https://serval.unil.ch/notice/serval:BIB_60346FC75171
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_60346FC75171
-[ RECORD 2 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_4db47fc4b593
base_url     | https://serval.unil.ch/notice/serval:BIB_4DB47FC4B593
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_4DB47FC4B593
-[ RECORD 3 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_57aac24fe115
base_url     | http://nbn-resolving.org/urn/resolver.pl?urn=urn:nbn:ch:serval-BIB_57AAC24FE1154
terminal_url | https://nbn-resolving.org/urn/resolver.pl?urn=urn:nbn:ch:serval-BIB_57AAC24FE1154
-[ RECORD 4 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_deabae6baf6c
base_url     | https://serval.unil.ch/notice/serval:BIB_DEABAE6BAF6C
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_DEABAE6BAF6C
-[ RECORD 5 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_a5ec0df1370f
base_url     | https://serval.unil.ch/notice/serval:BIB_A5EC0DF1370F
terminal_url | https://wayf.switch.ch/SWITCHaai/WAYF?entityID=https%3A%2F%2Fmy.unil.ch%2Fshibboleth&return=https%3A%2F%2Fserval.unil.ch%2FShibboleth.sso%2FLogin%3FSAMLDS%3D1%26target%3Dss%253Amem%253Aed270c26d4a36cefd1bf6a840472abe0ee5556cb5f3b42de708f3ea984775dfd
-[ RECORD 6 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_080300c2e23c
base_url     | https://serval.unil.ch/resource/serval:BIB_080300C2E23C.P001/REF.pdf
terminal_url | https://wayf.switch.ch/SWITCHaai/WAYF?entityID=https%3A%2F%2Fmy.unil.ch%2Fshibboleth&return=https%3A%2F%2Fserval.unil.ch%2FShibboleth.sso%2FLogin%3FSAMLDS%3D1%26target%3Dss%253Amem%253A154453d78a0fb75ffa220f7b6fe73b29447fa6ed048addf31897b41001f44679
-[ RECORD 7 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_de777dd2b07f
base_url     | https://serval.unil.ch/notice/serval:BIB_DE777DD2B07F
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_DE777DD2B07F
-[ RECORD 8 ]+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:serval.unil.ch:bib_5e824e244c27
base_url     | https://serval.unil.ch/notice/serval:BIB_5E824E244C27
terminal_url | https://serval.unil.ch/en/notice/serval:BIB_5E824E244C27

Metadata only? See elsewhere.

### Random Links

-[ RECORD 1 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dbc.wroc.pl:41031
base_url     | https://dbc.wroc.pl/dlibra/docmetadata?showContent=true&id=41031
terminal_url | https://dbc.wroc.pl/dlibra/docmetadata?showContent=true&id=41031

This is some platform/package thing. PDF is in an iframe. Platform is "DLibra".
FIXED-DLIBRA

-[ RECORD 2 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:orbi.ulg.ac.be:2268/174291
base_url     | https://orbi.uliege.be/handle/2268/174291
terminal_url | https://orbi.uliege.be/handle/2268/174291

DSpace platform. There are multiple files, and little to "select" on.

https://orbi.uliege.be/handle/2268/174200 has only single PDF and easier to work with

PARTIAL-DSPACE

-[ RECORD 3 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.tue.nl:664163
base_url     | http://repository.tue.nl/664163
terminal_url | http://repository.tue.nl/664163

Ah, this is the Pure platform from Elsevier.
Redirects to: https://research.tue.nl/en/publications/lowering-the-threshold-for-computers-in-early-design-some-advance

FIXED-PURE


-[ RECORD 4 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:juser.fz-juelich.de:49579
base_url     | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-49579%22
terminal_url | http://juser.fz-juelich.de/search?p=id:%22PreJuSER-49579%22

(handled above)

-[ RECORD 5 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.mit.edu:1721.1/97937
base_url     | https://orcid.org/0000-0002-2066-2082
terminal_url | https://orcid.org/0000-0002-2066-2082

ORCID! Skip it.

DONE: skip orcid.org in `terminal_url`, and/or at harvest/transform time.

-[ RECORD 6 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:edoc.mpg.de:360269
base_url     | http://edoc.mpg.de/360269
terminal_url | http://edoc.mpg.de/360269

Seems like this whole repo has disapeared, or been replaced by... pure? maybe a different pure?

DONE: edoc.mpg.de -> pure.mpg.de

-[ RECORD 7 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:books.openedition.org:msha/17716
base_url     | http://books.openedition.org/msha/17716
terminal_url | https://books.openedition.org/msha/17716

Open edition is free to read HTML, but not PDF (or epub, etc).

TODO: for some? all? openedition books records, try HTML ingest (not PDF ingest)

HTML-WORKED

-[ RECORD 8 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:epub.oeaw.ac.at:0x003aba48
base_url     | http://epub.oeaw.ac.at/?arp=8609-0inhalt/B02_2146_FP_Flores%20Castillo.pdf
terminal_url | http://epub.oeaw.ac.at/?arp=8609-0inhalt/B02_2146_FP_Flores%20Castillo.pdf

requires login

FORBIDDEN

-[ RECORD 9 ]+---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.mit.edu:1721.1/88986
base_url     | https://orcid.org/0000-0002-4147-2560
terminal_url | https://orcid.org/0000-0002-4147-2560

DONE: skip orcids

-[ RECORD 10 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.ust.hk:1783.1-28786
base_url     | http://repository.ust.hk/ir/Record/1783.1-28786
terminal_url | http://repository.ust.hk/ir/Record/1783.1-28786

Generator: VuFind 5.1.1
just a metadata record

METADATA-ONLY

-[ RECORD 11 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:rcin.org.pl:50797
base_url     | http://195.187.71.10/ipac20/ipac.jsp?profile=iblpan&index=BOCLC&term=cc95215472
terminal_url | http://195.187.71.10/ipac20/ipac.jsp?profile=iblpan&index=BOCLC&term=cc95215472

Seems like a software platform? not sure.

METADATA-ONLY

-[ RECORD 12 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dea.lib.unideb.hu:2437/69641
base_url     | http://webpac.lib.unideb.hu:8082/WebPac/CorvinaWeb?action=cclfind&resultview=long&ccltext=idno+bibFSZ1008709
terminal_url | https://webpac.lib.unideb.hu/WebPac/CorvinaWeb?action=cclfind&resultview=long&ccltext=idno+bibFSZ1008709

-[ RECORD 13 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:unsworks.library.unsw.edu.au:1959.4/64871
base_url     | http://handle.unsw.edu.au/1959.4/64871
terminal_url | https://www.unsworks.unsw.edu.au/primo-explore/fulldisplay?vid=UNSWORKS&docid=unsworks_62832&context=L

-[ RECORD 14 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:www.wbc.poznan.pl:225930
base_url     | https://www.wbc.poznan.pl/dlibra/docmetadata?showContent=true&id=225930
terminal_url | https://www.wbc.poznan.pl/dlibra/docmetadata?showContent=true&id=225930

SOFT-404

-[ RECORD 15 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:repository.erciyes.edu.tr:105
base_url     | http://repository.erciyes.edu.tr/bilimname/items/show/105
terminal_url | http://repository.erciyes.edu.tr:80/bilimname/items/show/105

GONE (domain not registered)

-[ RECORD 16 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digi.ub.uni-heidelberg.de:37500
base_url     | https://archivum-laureshamense-digital.de/view/sad_a1_nr_20_13
terminal_url | https://archivum-laureshamense-digital.de/view/sad_a1_nr_20_13

Seems like a bespoke site

SKIP-BESPOKE

-[ RECORD 17 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:t2r2.star.titech.ac.jp:50401364
base_url     | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100758313
terminal_url | http://t2r2.star.titech.ac.jp/cgi-bin/publicationinfo.cgi?q_publication_content_number=CTT100758313

METADATA-ONLY

-[ RECORD 18 ]---------------------------------------------------------------------------------------------------------------------
oai_id       | oai:epubs.cclrc.ac.uk:work/4714
base_url     | http://purl.org/net/epubs/work/4714
terminal_url | https://epubs.stfc.ac.uk/work/4714

It's got a purl! haha.

METADATA-ONLY

------

Another batch! With some repeat domains removed.

-[ RECORD 1 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:cris.vtt.fi:persons/142c030f-ba7b-491a-8669-a361088355cc
base_url     | https://cris.vtt.fi/en/persons/142c030f-ba7b-491a-8669-a361088355cc
terminal_url | https://cris.vtt.fi/en/persons/oleg-antropov

SKIP

-[ RECORD 2 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:etd.adm.unipi.it:etd-05302014-183910
base_url     | http://etd.adm.unipi.it/theses/available/etd-05302014-183910/
terminal_url | https://etd.adm.unipi.it/theses/available/etd-05302014-183910/

Some software platform? Pretty basic/bespoke

FIXED-PARTIAL

-[ RECORD 3 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bibliotecadigital.jcyl.es:10000098246
base_url     | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10316451
terminal_url | http://bibliotecadigital.jcyl.es/i18n/catalogo_imagenes/grupo.cmd?path=10316451

SKIP (see elsewhere)

-[ RECORD 7 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:elektra.cdaea.es:documento.29259
base_url     | https://www.juntadeandalucia.es/cultura/cdaea/elektra/catalogo_execute.html?tipoObjeto=1&id=29259
terminal_url | https://www.juntadeandalucia.es/cultura/cdaea/elektra/catalogo_execute.html?tipoObjeto=1&id=29259

Photo.

SKIP-SCOPE

-[ RECORD 9 ]+-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:unsworks.library.unsw.edu.au:1959.4/unsworks_60829
base_url     | http://handle.unsw.edu.au/1959.4/unsworks_60829
terminal_url | https://www.unsworks.unsw.edu.au/primo-explore/fulldisplay?vid=UNSWORKS&docid=unsworks_modsunsworks_60829&context=L

METADATA-ONLY

-[ RECORD 12 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:pure.leuphana.de:publications/7d040cf2-b3b5-4671-8906-76b5bc8d870a
base_url     | http://fox.leuphana.de/portal/de/publications/studies-in-childrens-literature-1500--2000-editors-celia-keenan-(7d040cf2-b3b5-4671-8906-76b5bc8d870a).html
terminal_url | http://fox.leuphana.de/portal/de/publications/studies-in-childrens-literature-1500--2000-editors-celia-keenan-(7d040cf2-b3b5-4671-8906-76b5bc8d870a).html

unsure

-[ RECORD 16 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.wur.nl:wurpubs/369344
base_url     | https://library.wur.nl/WebQuery/wurpubs/369344
terminal_url | https://library.wur.nl/WebQuery/wurpubs/369344

this specific record not OA (but site is fine/fixed)

-[ RECORD 17 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:escholarship.umassmed.edu:oapubs-2146
base_url     | https://escholarship.umassmed.edu/oapubs/1147
terminal_url | http://escholarship.umassmed.edu/oapubs/1147/

just links to publisher (no content in repo)

-[ RECORD 18 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digitalcommons.usu.edu:wild_facpub-1010
base_url     | https://digitalcommons.usu.edu/wild_facpub/11
terminal_url | http://digitalcommons.usu.edu/wild_facpub/11/

also just links to publisher (no content in repo)

-[ RECORD 25 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:igi.indrastra.com:306768
base_url     | http://igi.indrastra.com/items/show/306768
terminal_url | http://igi.indrastra.com/items/show/306768

(see elsewhere)

-[ RECORD 26 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:fau.digital.flvc.org:fau_9804
base_url     | http://purl.flvc.org/fcla/dt/12932
terminal_url | http://fau.digital.flvc.org/islandora/object/fau%3A9804

Islandora.

-[ RECORD 27 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:dspace.lu.lv:7/16019
base_url     | https://dspace.lu.lv/dspace/handle/7/16019
terminal_url | https://dspace.lu.lv/dspace/handle/7/16019

LOGINWALL

-[ RECORD 28 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:zir.nsk.hr:umas_218
base_url     | https://repozitorij.svkst.unist.hr/islandora/object/umas:218
terminal_url | https://repozitorij.svkst.unist.hr/islandora/object/umas:218

REMOVED


-[ RECORD 29 ]-----------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:digi.ub.uni-heidelberg.de:36390
base_url     | https://digi.hadw-bw.de/view/sbhadwmnkl_a_1917_5
terminal_url | https://digi.hadw-bw.de/view/sbhadwmnkl_a_1917_5

Book, with chapters, not an individual work.

-[ RECORD 2 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:krm.or.kr:10056135m201r
base_url     | https://www.krm.or.kr/krmts/link.html?dbGubun=SD&m201_id=10056135&res=y
terminal_url | https://www.krm.or.kr/krmts/search/detailview/research.html?dbGubun=SD&category=Research&m201_id=10056135

research results repository; keep crawling

SKIP-SCOPE

-[ RECORD 3 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:www.db-thueringen.de:dbt_mods_00005191
base_url     | https://www.db-thueringen.de/receive/dbt_mods_00005191
terminal_url | https://www.db-thueringen.de/receive/dbt_mods_00005191

powered by "MyCoRe"

FIXED-MYCORE

-[ RECORD 6 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bibliotecavirtualandalucia.juntadeandalucia.es:1017405
base_url     | http://www.bibliotecavirtualdeandalucia.es/catalogo/es/consulta/registro.cmd?id=1017405
terminal_url | http://www.bibliotecavirtualdeandalucia.es/catalogo/es/consulta/registro.cmd?id=1017405

seems to be a general purpose regional library? not research-specific

SKIP-UNSURE

-[ RECORD 7 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:etd.adm.unipi.it:etd-02272019-123644
base_url     | http://etd.adm.unipi.it/theses/available/etd-02272019-123644/
terminal_url | https://etd.adm.unipi.it/theses/available/etd-02272019-123644/

This specific URL is not available (FORBIDDEN)

others have multiple files, not just a single PDF:
https://etd.adm.unipi.it/t/etd-09102013-124430/

SKIP-UNSURE

-[ RECORD 9 ]+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:commons.ln.edu.hk:sw_master-5408
base_url     | https://commons.ln.edu.hk/sw_master/4408
terminal_url | https://commons.ln.edu.hk/sw_master/4408/

worth crawling I guess

METADATA-ONLY

-[ RECORD 10 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:mouseion.jax.org:ssbb1976-1224
base_url     | https://mouseion.jax.org/ssbb1976/225
terminal_url | https://mouseion.jax.org/ssbb1976/225/

METADATA-ONLY

-[ RECORD 13 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:aleph.bib-bvb.de:bvb01-016604343
base_url     | http://bvbm1.bib-bvb.de/webclient/DeliveryManager?pid=176332&custom_att_2=simple_viewer
terminal_url | http://digital.bib-bvb.de/view/action/singleViewer.do?dvs=1593269021002~476&locale=en_US&VIEWER_URL=/view/action/singleViewer.do?&DELIVERY_RULE_ID=31&frameId=1&usePid1=true&usePid2=true

SOFT-404 / FORBIDDEN (cookie timeout)

-[ RECORD 14 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:bivaldi.gva.es:11740
base_url     | https://bivaldi.gva.es/es/consulta/registro.do?id=11740
terminal_url | https://bivaldi.gva.es/es/consulta/registro.do?id=11740


-[ RECORD 16 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:library.wur.nl:wurpubs/443282
base_url     | https://library.wur.nl/WebQuery/wurpubs/443282
terminal_url | https://library.wur.nl/WebQuery/wurpubs/443282

DIGIBIS platform (like some others)

FIXED-PARTIAL

-[ RECORD 18 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:hal:in2p3-00414135v1
base_url     | http://hal.in2p3.fr/in2p3-00414135
terminal_url | http://hal.in2p3.fr:80/in2p3-00414135

METADATA-ONLY

-[ RECORD 19 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:aaltodoc.aalto.fi:123456789/13201
base_url     | https://aaltodoc.aalto.fi/handle/123456789/13201
terminal_url | https://aaltodoc.aalto.fi/handle/123456789/13201

This specific record is not accessible.
Another: https://aaltodoc.aalto.fi/handle/123456789/38002

DSpace 5.4

Worked (from recent changes)


-[ RECORD 20 ]------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
oai_id       | oai:sedici.unlp.edu.ar:10915/40144
base_url     | http://xjornadaslc.fahce.unlp.edu.ar/actas/Ramon_Esteban_Chaparro.pdf/view
terminal_url | http://xjornadaslc.fahce.unlp.edu.ar/actas/Ramon_Esteban_Chaparro.pdf/view

This is a journal! Cool. Plone software platform.

FIXED

## Top no-capture Domains

Top terminal no-capture domains:

    SELECT domain, COUNT(domain)
    FROM (
        SELECT
            ingest_file_result.ingest_type,
            ingest_file_result.status,
            substring(ingest_file_result.terminal_url FROM '[^/]+://([^/]*)') AS domain
        FROM ingest_file_result
        LEFT JOIN ingest_request
            ON ingest_file_result.ingest_type = ingest_request.ingest_type
            AND ingest_file_result.base_url = ingest_request.base_url
        WHERE
            ingest_request.ingest_type = 'pdf'
            AND ingest_request.link_source = 'oai'
            AND ingest_file_result.status = 'no-capture'
    ) t1
    WHERE t1.domain != ''
        AND t1.status != 'success'
    GROUP BY domain
    ORDER BY COUNT DESC
    LIMIT 30;

                  domain               | count 
    -----------------------------------+-------
     digitalrepository.unm.edu         | 94087
     escholarship.org                  | 80632
     ir.opt.ac.cn                      | 70504
     idus.us.es                        | 67908
     www.cambridge.org                 | 56376
     www.ssoar.info                    | 52534
     rep.bntu.by                       | 52127
     scholarworks.umt.edu              | 48546
     publikationen.ub.uni-frankfurt.de | 46987
     dk.um.si                          | 45753
     repositorio.uladech.edu.pe        | 37028
     uu.diva-portal.org                | 34929
     digitalcommons.law.byu.edu        | 31732
     sedici.unlp.edu.ar                | 31233
     elib.sfu-kras.ru                  | 29131
     jyx.jyu.fi                        | 28144
     www.repository.cam.ac.uk          | 27728
     nagoya.repo.nii.ac.jp             | 26673
     www.duo.uio.no                    | 25258
     www.persee.fr                     | 24968
     www2.senado.leg.br                | 24426
     tesis.ucsm.edu.pe                 | 24049
     digitalcommons.unl.edu            | 21974
     www.degruyter.com                 | 21940
     www.igi-global.com                | 20736
     thekeep.eiu.edu                   | 20712
     docs.lib.purdue.edu               | 20538
     repositorio.cepal.org             | 20280
     elib.bsu.by                       | 19620
     minds.wisconsin.edu               | 19473
    (30 rows)

These all seem worth crawling. A couple publishers (cambridge.org), and
persee.fr will probably fail, but not too many URLs.

## Summary of Filtered Prefixes and Domains (OAI-PMH)

oai:kb.dk:
    too large and generic
oai:bdr.oai.bsb-muenchen.de:
    too large and generic
oai:hispana.mcu.es:
    too large and generic
oai:bnf.fr:
    too large and generic
oai:ukm.si:
    too large and generic
oai:biodiversitylibrary.org:
    redundant with other ingest and archive.org content
oai:hsp.org:
    large; historical content only
oai:repec:
    large; mostly (entirely?) links to publisher sites
oai:n/a:
    meta?
oai:quod.lib.umich.edu:
    entire issues? hard to crawl so skip for now
oai:hypotheses.org:
    HTML, not PDF
oai:americanae.aecid.es:
    large, complex. skip for now
oai:www.irgrid.ac.cn:
    aggregator of other IRs
oai:espace.library.uq.edu.au:
    large; metadata only; javascript heavy (poor heritrix crawling)
oai:edoc.mpg.de:
    deprecated domain, with no redirects
oai:bibliotecadigital.jcyl.es:
    digitized historical docs; hard to crawl, skip for now
oai:repository.erciyes.edu.tr:
    gone (domain lapsed)
oai:krm.or.kr:
    "research results repository" (metadata only)

www.kb.dk
    large, general purpose, scope
kb-images.kb.dk
    deprecated
mdz-nbn-resolving.de
    multiple prefixes end up here. historical docs, scope
aggr.ukm.um.si
    large, out of scope
edoc.mpg.de
    deprecated domain
doaj.org
    index (metadata only)
orcid.org
    out of scope
gateway.isiknowledge.com
    clarivate login/payall (skipping in ingest)

Needs filtering to a subset of records (by 'set' or other filtering?):

oai:igi.indrastra.com:
oai:invenio.nusl.cz:
oai:t2r2.star.titech.ac.jp:
oai:evastar-karlsruhe.de:
oai:repository.ust.hk:
oai:serval.unil.ch:
oai:pure.atira.dk:

FIlters in SQL syntax:

    AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:repec:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:n/a:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:quod.lib.umich.edu:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:americanae.aecid.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:www.irgrid.ac.cn:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:espace.library.uq.edu:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:edoc.mpg.de:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:bibliotecadigital.jcyl.es:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:repository.erciyes.edu.tr:%'
    AND ingest_request.link_source_id NOT LIKE 'oai:krm.or.kr:%'
    AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
    AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
    AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
    AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
    AND ingest_request.base_url NOT LIKE '%edoc.mpg.de%'
    AND ingest_request.base_url NOT LIKE '%doaj.org%'
    AND ingest_request.base_url NOT LIKE '%orcid.org%'
    AND ingest_request.base_url NOT LIKE '%gateway.isiknowledge.com%'

and in some contexts (PDFs; switch to HTML):

    AND ingest_request.link_source_id NOT LIKE 'oai:hypotheses.org:%'

## Overall Summary of OAI-PMH Stuff

Big picture is that the majority of `no-pdf-link` crawl status are because of
repository scope, record scope, or content format issues. That being said,
there was a sizable fraction of sites which were platforms (like DSpace) which
were not ingesting well.

A significant fraction of records are "metadata only" (of papers), or non-paper
entity types (like persons, grants, or journal titles), and a growing fraction
(?) are metadata plus link to OA publisher fulltext (offsite). Might be
possible to detect these at ingest time, or earlier at OAI-PMH
harvest/transform time and filter them out.

It may be worthwhile to attempt ingest of multiple existing captures
(timestamps) in the ingest pipeline.  Eg, isntead of chosing a single "best"
capture, if therea are multiple HTTP 200 status captures, try ingest with each
(or at least a couple).  This is because repository software gets upgraded, so
old "no-capture" or "not found" or "link loop" type captures may work when
recrawled.

New summary with additional filters:

    SELECT ingest_file_result.status, COUNT(*)
    FROM ingest_request
    LEFT JOIN ingest_file_result
        ON ingest_file_result.ingest_type = ingest_request.ingest_type
        AND ingest_file_result.base_url = ingest_request.base_url
    WHERE 
        ingest_request.ingest_type = 'pdf'
        AND ingest_request.link_source = 'oai'
        AND ingest_request.link_source_id NOT LIKE 'oai:kb.dk:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bdr.oai.bsb-muenchen.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hispana.mcu.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bnf.fr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:ukm.si:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:biodiversitylibrary.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hsp.org:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:repec:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:n/a:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:quod.lib.umich.edu:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:americanae.aecid.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:www.irgrid.ac.cn:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:espace.library.uq.edu:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:edoc.mpg.de:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:bibliotecadigital.jcyl.es:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:repository.erciyes.edu.tr:%'
        AND ingest_request.link_source_id NOT LIKE 'oai:krm.or.kr:%'
        AND ingest_request.base_url NOT LIKE '%www.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%kb-images.kb.dk%'
        AND ingest_request.base_url NOT LIKE '%mdz-nbn-resolving.de%'
        AND ingest_request.base_url NOT LIKE '%aggr.ukm.um.si%'
        AND ingest_request.base_url NOT LIKE '%edoc.mpg.de%'
        AND ingest_request.base_url NOT LIKE '%doaj.org%'
        AND ingest_request.base_url NOT LIKE '%orcid.org%'
        AND ingest_request.base_url NOT LIKE '%gateway.isiknowledge.com%'
        AND ingest_request.link_source_id NOT LIKE 'oai:hypotheses.org:%'
    GROUP BY status
    ORDER BY COUNT DESC
    LIMIT 20;

            status         |  count
    -----------------------+----------
     success               | 12872279
     no-pdf-link           |  9329602
     no-capture            |  4696362
     redirect-loop         |  1541458
     terminal-bad-status   |   660418
     link-loop             |   452831
     wrong-mimetype        |   434868
     null-body             |    71065
     cdx-error             |    17005
                           |    15275
     petabox-error         |    12743
     wayback-error         |    11759
     skip-url-blocklist    |      182
     gateway-timeout       |      122
     redirects-exceeded    |      120
     bad-redirect          |      117
     bad-gzip-encoding     |      111
     wayback-content-error |      102
     timeout               |       72
     blocked-cookie        |       62
    (20 rows)