summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/fatcat_tools/importers/common.py10
1 files changed, 6 insertions, 4 deletions
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index 79425618..43ca1b10 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -147,7 +147,8 @@ def is_cjk(s):
return False
for c in s:
if c.isalpha():
- return unicodedata.name(c).startswith("CJK")
+ lang_prefix = unicodedata.name(c).split()[0]
+ return lang_prefix in ('CJK', 'HIRAGANA', 'KATAKANA', 'HANGUL')
return False
def test_is_cjk():
@@ -159,9 +160,10 @@ def test_is_cjk():
assert is_cjk('菊') == True
assert is_cjk('岡, 鹿, 梨, 阜, 埼 with eng after') == True
assert is_cjk('水道') == True
- # TODO: assert is_cjk('ひヒ') == True
- # TODO: assert is_cjk('き゚ゅ') == True
- # TODO: assert is_cjk('ㄴ, ㄹ, ㅁ, ㅂ, ㅅ') == True
+ assert is_cjk('オウ, イク') == True # kanji
+ assert is_cjk('ひヒ') == True
+ assert is_cjk('き゚ゅ') == True
+ assert is_cjk('ㄴ, ㄹ, ㅁ, ㅂ, ㅅ') == True
DOMAIN_REL_MAP = {
"archive.org": "archive",