aboutsummaryrefslogtreecommitdiffstats
path: root/skate/url_test.go
blob: 5e36cae5777fb4fbc11c4afc8db3dda412f1ae06 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
package skate

import "testing"

func TestSanitizeURL(t *testing.T) {
	var cases = []struct {
		in  string
		out string
	}{
		{"", ""},
		{"a", ""},
		{"???", ""},
		{"???***", ""},
		{"???***___123", ""},
		{"http://abc.com", "http://abc.com"},
		{"http://!!abc.com", "http://abc.com"},
		{`http://"www.phaelos.com/oubre.html`, `http://www.phaelos.com/oubre.html`},
		{`http://!www.rkm=journal.de/archives/13383`, `http://www.rkm=journal.de/archives/13383`},
		{`http:///en.m.wikipedia.org/ChenLong`, `http://en.m.wikipedia.org/ChenLong`},
		{`http://10.1111/joim.12348`, `https://doi.org/10.1111/joim.12348`},
		{`http://10.1113/jphysiol.2002.026047`, `https://doi.org/10.1113/jphysiol.2002.026047`},
		{`http://10.30.3.16/moodle/course/view.php?id=25`, `http://10.30.3.16/moodle/course/view.php?id=25`},
		{`http://10.3266/RevEspEndocrinolPediatr.pre2015.Nov.330`, `https://doi.org/10.3266/RevEspEndocrinolPediatr.pre2015.Nov.330`},
		{`http://120.107.180.177/1832/9901/099-2-07p.pdf.Accessed`, `http://120.107.180.177/1832/9901/099-2-07p.pdf`},
		{`http://120cartas.ig.com.br/wp/maio-de-2008-um-aniversario-de-120-anos/.Acessoem:set`,
			`http://120cartas.ig.com.br/wp/maio-de-2008-um-aniversario-de-120-anos/`},
		{`http://122.53.86.125/NNS/8thNNS.pdf.Accessed`, `http://122.53.86.125/NNS/8thNNS.pdf`},
		{`http://122.53.86.125/facts_figures2011.pdf.Accessedon`,
			`http://122.53.86.125/facts_figures2011.pdf`},
		{`http://129.3.20.41/eps/fin/papers/0507/0507016.pdf.diaksespadatanggal23Januari`,
			`http://129.3.20.41/eps/fin/papers/0507/0507016.pdf`},
		{`http://129.3.20.41/eps/hew/papers/0512/0512001.pdfAccessed1`,
			`http://129.3.20.41/eps/hew/papers/0512/0512001.pdf`},
		{`http://140.120.197.173/Ecology/Download/Timing-MSChart.zipJournalofInsectScience`,
			`http://140.120.197.173/Ecology/Download/Timing-MSChart.zip`},
		{`141.213.232.243/bitstream/handle/2027.42/86336/apterc_1.pdf?sequence=1`,
			`http://141.213.232.243/bitstream/handle/2027.42/86336/apterc_1.pdf?sequence=1`},
		{`http://141.232.10.32/pm/recover/recover_docs/perf_measures/062812_rec_pm_scs_salinity_flbay.pdfRECOVER`,
			`http://141.232.10.32/pm/recover/recover_docs/perf_measures/062812_rec_pm_scs_salinity_flbay.pdf`},
		{`http://2010.census.gov/news/releases/operations/cb11-cn125.html.lastaccessed4`,
			`http://2010.census.gov/news/releases/operations/cb11-cn125.html`},
		{`http://2014hit.blogspot.com.tr/2014/12/george-gerbnerin-tv-arastrmas-ve-ekme.htmladresindenedinilmiştir`,
			`http://2014hit.blogspot.com.tr/2014/12/george-gerbnerin-tv-arastrmas-ve-ekme.html`},
		{`http://2015.ses.org.tr/wp-ontent/uploads/toplumsalcinsiyetrolleri.pdfsayfasındanulaşıl-mıştır`,
			`http://2015.ses.org.tr/wp-ontent/uploads/toplumsalcinsiyetrolleri.pdf`},
		{`http://2015.veneziabiennale-japanpavilion.jp/en/Consultadael20deoctubrealas14`,
			`http://2015.veneziabiennale-japanpavilion.jp/en/`},
		{`http://-annalsofneurosciences.org/journal/index.php/annal/article/view/43/67`,
			`http://annalsofneurosciences.org/journal/index.php/annal/article/view/43/67`},
		{`http://-www.gifted.uconn.edu/Siegle/Dissertations/Eric%20Mann.pdf.Diunduh15`,
			`http://www.gifted.uconn.edu/Siegle/Dissertations/Eric%20Mann.pdf`},
		{`http://-www.suparlan.com/pages/posts/.Diakses15Pebruari`,
			`http://www.suparlan.com/pages/posts/`},
		{`http://...books.google.com/books?isbn=0873552601`,
			`http://books.google.com/books?isbn=0873552601`},
		{`http://.R-project.org`,
			`http://R-project.org`},
		{`http://.amazona.com/academia.edu.documents//autogestion.pdfRecibido:24demayode2017`,
			`http://amazona.com/academia.edu.documents//autogestion.pdf`},
		{`http://10.1007/s00779-012-0615-1`,
			`https://doi.org/10.1007/s00779-012-0615-1`},
		{`http://20.132.48.254/PDFS/ED495503.pdf.Accessedat`,
			`http://20.132.48.254/PDFS/ED495503.pdf`},
		{`http://82.198.195.82/presse/mitteilungen/2007/Stellungnahme_dsn_BDAG_Internet_20071219.pdf,abgerufenam19`,
			`http://82.198.195.82/presse/mitteilungen/2007/Stellungnahme_dsn_BDAG_Internet_20071219.pdf`},
		{`http://CRAN.R-project.org/package=RTextTools.Zugegriffen:6Juni`,
			`http://CRAN.R-project.org/package=RTextTools`},
		{`http://189.28.128.99/provab/docs/geral/edital_28_02_2012_resultado_provab.pdf.Acessoem19/11/2014`,
			`http://189.28.128.99/provab/docs/geral/edital_28_02_2012_resultado_provab.pdf`},
		{`http://195.20.232.142/img/Schwerpunktnewsletter_Oesterreich_Bibliotheken.pdf.Stanzdnia13.04`,
			`http://195.20.232.142/img/Schwerpunktnewsletter_Oesterreich_Bibliotheken.pdf`},
		{`http://aalc07.psu.edu/papers/jn_typol_class3.pdf.Stanford`,
			`http://aalc07.psu.edu/papers/jn_typol_class3.pdf`},
		{`http://aboriginalhealth.flinders.edu.au/Newsletters/2010/Downloads/SHRP%20FINAL%20REPORT%20PART%20TWO%20July%202009.pdfAccessed14/12/2012`,
			`http://aboriginalhealth.flinders.edu.au/Newsletters/2010/Downloads/SHRP%20FINAL%20REPORT%20PART%20TWO%20July%202009.pdf`},
		{`http://about-air.ru/svojstva-vozduha/davlenie-vozduha/normalnoe-atmosfernoe-davlenie.html,доступ-свободный,датаобращения15.04.2017`,
			`http://about-air.ru/svojstva-vozduha/davlenie-vozduha/normalnoe-atmosfernoe-davlenie.html`},
		{`http://acl.ldc.upenn.edu/W/W98/W98-1120.pdfDateofaccess`,
			`http://acl.ldc.upenn.edu/W/W98/W98-1120.pdf`},
		{`http://acl.mit.edu/pa-pers/2012-uber-conference-submitted.pdf//49thIEEE`,
			`http://acl.mit.edu/pa-pers/2012-uber-conference-submitted.pdf`},
		{`http://acoss.org.au/policy/community_services/emergency_relief_handbook/,accessed1st`,
			`http://acoss.org.au/policy/community_services/emergency_relief_handbook/`},
		{`http://acrf.com.au/2012/world-firsthpv-vaccina-tion-plan-will-protect-young-australian-men-from-cancer/.Ac-cessedon06/12/2016`,
			`http://acrf.com.au/2012/world-firsthpv-vaccina-tion-plan-will-protect-young-australian-men-from-cancer/`},
		{`http://acta.uta.fi/pdf/951-44-4701-6.pdfRHEINDORF`,
			`http://acta.uta.fi/pdf/951-44-4701-6.pdf`},
		{`http://admi.net/jo/20080423/ECE-C0771649A.html.Pageconsultéele25septembre`,
			`http://admi.net/jo/20080423/ECE-C0771649A.html`},
		{`http://admin.localgov.co.uk/his_localgov/view/images/uploaded/Image/childrensblackpool.PDF.Lastaccess8`,
			`http://admin.localgov.co.uk/his_localgov/view/images/uploaded/Image/childrensblackpool.PDF`},
		{`http://aec.ifas.ufl.edu/abrams/step/critical_litreview.pdfİndirme`,
			`http://aec.ifas.ufl.edu/abrams/step/critical_litreview.pdf`},
		{`http://aem.asm.org/Downloadedfrom`, `http://aem.asm.org/`},
		{`http://aem.asm.org/content/67/6/2766.full.pdf+htmlWITTWER`,
			`http://aem.asm.org/content/67/6/2766.full.pdf+html`},
		{`http://agris.fao.org/agris-search/search.do?recordID=BR2013800115https://doi.org/10.5747/ca.2010.v06.n1.a044`,
			`http://agris.fao.org/agris-search/search.do?recordID=BR2013800115`},
		{`http://ailab.ist.psu.edu/bcpred/SVMTriP:http://sysbio.unl.edu/SVMTriP/prediction.phpBcell`,
			`http://ailab.ist.psu.edu/bcpred/SVMTriP`},
		{`http://aim.bmj.com/content/31/1/23.full.pdf+htmlAcessoem:15Dez`,
			`http://aim.bmj.com/content/31/1/23.full.pdf+html`},
		{`http://ainfo.cnptia.embrapa.br/digital/bitstream/CNPAT-2010/8608/1/Ci-017.pdfAcessed06`,
			`http://ainfo.cnptia.embrapa.br/digital/bitstream/CNPAT-2010/8608/1/Ci-017.pdf`},
		{`12s`, ``},
		{`12spoaspdop`, ``},
		{`0.0.www.epcglobalinc.org/standards_technology/Secure/v1.0/UHF-class1.pdf`,
			`http://www.epcglobalinc.org/standards_technology/Secure/v1.0/UHF-class1.pdf`},
		{`CRAN.R-project.org/package=vegan`,
			`http://CRAN.R-project.org/package=vegan`},
		{`CRD42014009228.www.crd.york.ac.uk/PROSPERO/display_record.asp?ID=CRD42014009228`,
			`http://www.crd.york.ac.uk/PROSPERO/display_record.asp?ID=CRD42014009228`},
		{`ftp://ftp.ncbi.nih.gov/genomes/Bacteria/`,
			`ftp://ftp.ncbi.nih.gov/genomes/Bacteria/`},
		{`ftp-eng.cisco.com/sobgp/index.html`,
			`http://ftp-eng.cisco.com/sobgp/index.html`},
		{`ftp.cdc.gov/pub/Publications/mmwr/SS/SS4703.pdf`,
			`http://ftp.cdc.gov/pub/Publications/mmwr/SS/SS4703.pdf`},
		{`ftpftp.inria.fr`,
			`http://ftpftp.inria.fr`},
		{`http.bglink.com/personal/batakovic`, `http://bglink.com/personal/batakovic`},
		{`http.kalsel.bps.go.id`, `http://kalsel.bps.go.id`},
		{`http.www.admhmao.ru/people/frame.htm`, `http://www.admhmao.ru/people/frame.htm`},
		{`http.worldbank.org/sq`, `http://worldbank.org/sq`},
		{`httpwww.sun.com`, `http://www.sun.com`},
		{`httpswww.unos.org`, `http://www.unos.org`},
		{`ics.uci.edu/pub/ietf/`, `http://ics.uci.edu/pub/ietf/`},
		{`ISSN-2177-4129periodicos.ufpel.edu.br/ojs2/index.php/Memoriahttp://dx.doi.org/10.15210/rmr.v8i14.7485`,
			`http://dx.doi.org/10.15210/rmr.v8i14.7485`},
		{`Shttp://hdl.handle.net/1765/1163`,
			`http://hdl.handle.net/1765/1163`},
		{`cdec.water.ca.gov/misc/DailyPrecip.html`,
			`http://cdec.water.ca.gov/misc/DailyPrecip.html`},
	}
	for _, c := range cases {
		out := SanitizeURL(c.in)
		if out != c.out {
			t.Fatalf("got %v, want %v", out, c.out)
		}
	}
}

func BenchmarkSanitizeURL(b *testing.B) {
	var bms = []struct {
		name string
		in   string
	}{
		{`http`, `http://acrf.com.au/2012/world-firsthpv-vaccina-tion-plan-will-protect-young-australian-men-from-cancer/`},
		{`plain`, `0.0.www.epcglobalinc.org/standards_technology/Secure/v1.0/UHF-class1.pdf`},
	}
	for _, bm := range bms {
		b.Run(bm.name, func(b *testing.B) {
			for n := 0; n < b.N; n++ {
				SanitizeURL(bm.in)
			}
		})
	}
}