1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
import pytest
import collections
import tempfile
from fuzzycat.cluster import release_key_title, release_key_title_normalized, release_key_title_nysiis, sort_by_column, group_by
import os
Case = collections.namedtuple("Case", 'input output')
def test_release_key_title():
with pytest.raises(KeyError):
release_key_title({})
with pytest.raises(KeyError, match='title'):
release_key_title({'ident': '123'})
with pytest.raises(KeyError, match='ident'):
release_key_title({'title': 'deep learning backdoor'})
with pytest.raises(ValueError, match='title.*missing'):
release_key_title({'ident': '', 'title': ''})
cases = (
Case(input={
'ident': '',
'title': 'simhash'
}, output=('', 'simhash')),
Case(input={
'ident': '',
'title': 'Simhash'
}, output=('', 'Simhash')),
Case(input={
'ident': '',
'title': 'Sim hash'
}, output=('', 'Sim hash')),
)
for case in cases:
assert case.output == release_key_title(case.input)
def test_release_key_title_normalized():
cases = (
Case(input={
'ident': '',
'title': 'simhash'
}, output=('', 'simhash')),
Case(input={
'ident': '',
'title': 'Simhash'
}, output=('', 'simhash')),
Case(input={
'ident': '',
'title': 'Sim hash'
}, output=('', 'simhash')),
Case(input={
'ident': '',
'title': 'THE year 1929'
}, output=('', 'theyear1929')),
Case(input={
'ident': '',
'title': '2019?'
}, output=('', '2019')),
Case(input={
'ident': '123',
'title': 'H~~2019?'
}, output=('123', 'h2019')),
)
for case in cases:
assert case.output == release_key_title_normalized(case.input), 'failed case {}'.format(
case.input)
def test_release_key_title_nysiis():
cases = (
Case(input={
'ident': '',
'title': 'simhash'
}, output=('', 'SANAS')),
Case(input={
'ident': '',
'title': 'Simhash'
}, output=('', 'SANAS')),
Case(input={
'ident': '',
'title': 'Sim hash'
}, output=('', 'SANAS')),
Case(input={
'ident': '',
'title': 'THE year 1929'
}, output=('', 'TAR')),
Case(input={
'ident': '',
'title': '2019?'
}, output=('', '')),
Case(input={
'ident': '123',
'title': 'H~~2019?'
}, output=('123', 'H')),
Case(input={
'ident': '123',
'title': '世界'
}, output=('123', '')),
)
for case in cases:
assert case.output == release_key_title_nysiis(case.input), 'failed case {}'.format(
case.input)
def test_release_key_title_authors_ngram():
pass
def test_sort_by_column():
with tempfile.NamedTemporaryFile(delete=False, mode="w") as tf:
tf.write("2 b\n")
tf.write("2 a\n")
tf.write("9 d\n")
tf.write("1 c\n")
fn = sort_by_column(tf.name, opts='-k 2')
with open(fn) as f:
lines = [v.strip() for v in f]
assert lines == ['2 a', '2 b', '1 c', '9 d']
fn = sort_by_column(tf.name, opts='-k 1')
with open(fn) as f:
lines = [v.strip() for v in f]
assert lines == ['1 c', '2 a', '2 b', '9 d']
fn = sort_by_column(tf.name, opts='-k 3')
with open(fn) as f:
lines = [v.strip() for v in f]
assert lines == ['1 c', '2 a', '2 b', '9 d']
def test_group_by():
Case = collections.namedtuple("Case", "seq keyfunc valuefunc result")
cases = (
Case(["0", "1"], lambda v: v, lambda v: v, [{
'k': '0',
'v': ['0']
}, {
'k': '1',
'v': ['1']
}]),
Case(["a 1", "a 2", "b 3"], lambda v: v.split()[0], lambda v: v.split()[1], [{
'k': 'a',
'v': ['1', '2']
}, {
'k': 'b',
'v': ['3']
}]),
)
for case in cases:
assert case.result == list(group_by(case.seq, case.keyfunc, case.valuefunc))
|
