diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-10-17 17:19:34 +0100 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2019-10-17 17:19:34 +0100 |
| commit | 54dabe601eaa19d0495d9a102b34e9daa056457d (patch) | |
| tree | 392e3ba4fa6a6c9d4fdda2de0e7b4656ead18f83 /pig/tests/files | |
| parent | 04e1ae4f903af98ef174be9110aaae5e1ab81360 (diff) | |
| download | sandcrawler-54dabe601eaa19d0495d9a102b34e9daa056457d.tar.gz sandcrawler-54dabe601eaa19d0495d9a102b34e9daa056457d.zip | |
new/additional GWB CDX filter scripts
Diffstat (limited to 'pig/tests/files')
| -rw-r--r-- | pig/tests/files/sourcecode.cdx | 6 | ||||
| -rw-r--r-- | pig/tests/files/tarballs.cdx | 10 |
2 files changed, 16 insertions, 0 deletions
diff --git a/pig/tests/files/sourcecode.cdx b/pig/tests/files/sourcecode.cdx new file mode 100644 index 0000000..eeb397c --- /dev/null +++ b/pig/tests/files/sourcecode.cdx @@ -0,0 +1,6 @@ +# match +edu,cmu,cs,adm,reports-archive)/anon/usr0/ftp/usr0/anon/2002/cmu-cs-02-119.java 20170706005950 http://reports-archive.adm.cs.cmu.edu/anon/usr0/ftp/usr0/anon/2002/CMU-CS-02-119.java text/plain 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 361006 17120058 CITESEERX-CRAWL-2017-06-20-20170706004100259-00924-00932-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170706005946792-00926-31209~wbgrp-svc284.us.archive.org~8443.warc.gz +# no +fi,tkk,lib)/diss/2001/isbn951225459x/isbn951225459x.pyc 20170705074926 http://lib.tkk.fi/Diss/2001/isbn951225459X/isbn951225459X.pyc text/plain 200 KJBCOT7LGBNIAVGEGPUELK5OK6RTFORR - - 344175 255650124 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705074843696-00134-31209~wbgrp-svc284.us.archive.org~8443.warc.gz +# no +org,oxfordjournals,nar)/cgi/reprint/gkl1060v1.pdf 20170706035441 http://nar.oxfordjournals.org/cgi/reprint/gkl1060v1.pdf text/html 301 OX6MLVDFURLT2KSYCXUYW2PZNOVFSEVF - - 697 49346051 CITESEERX-CRAWL-2017-06-20-20170706034741172-00140-00149-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706035435634-00148-3671~wbgrp-svc285.us.archive.org~8443.warc.gz diff --git a/pig/tests/files/tarballs.cdx b/pig/tests/files/tarballs.cdx new file mode 100644 index 0000000..7a81b79 --- /dev/null +++ b/pig/tests/files/tarballs.cdx @@ -0,0 +1,10 @@ +#http://research.fit.edu/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf +#http://ijs.sgmjournals.org:80/cgi/reprint/54/6/2217.pdf +#http://eprints.ecs.soton.ac.uk/12020/1/mind-the-semantic-gap.pdf +#http://eprint.uq.edu.au/archive/00004120/01/R103_Forrester_pp.pdf + +# should match 2: + +edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf 20170706005950 http://mit.edu/file.pdf application/pdf 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz +edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/octet-stream 200 NQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz +org,sgmjournals,ijs)//cgi/reprint/54/6/2217.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/gzip 200 TQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 123 456 CRAWL/CRAWL.warc.gz |