From b9279efacdee9bf8038203d6efe9dc105cc4dce3 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@robocracy.org>
Date: Tue, 8 Oct 2019 16:11:09 -0700
Subject: refactor duplicated b32_hex function in importers

---
 python/fatcat_tools/importers/arabesque.py      | 12 +-----------
 python/fatcat_tools/importers/common.py         |  9 +++++++++
 python/fatcat_tools/importers/wayback_static.py | 11 +----------
 3 files changed, 11 insertions(+), 21 deletions(-)

(limited to 'python')

diff --git a/python/fatcat_tools/importers/arabesque.py b/python/fatcat_tools/importers/arabesque.py
index 65597d5c..7017c56c 100644
--- a/python/fatcat_tools/importers/arabesque.py
+++ b/python/fatcat_tools/importers/arabesque.py
@@ -1,20 +1,10 @@
 
 import sys
 import json
-import base64
 import sqlite3
 import itertools
 import fatcat_openapi_client
-from .common import EntityImporter, clean, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS
-
-
-def b32_hex(s):
-    s = s.strip().split()[0].lower()
-    if s.startswith("sha1:"):
-        s = s[5:]
-    if len(s) != 32:
-        return s
-    return base64.b16encode(base64.b32decode(s.upper())).lower().decode('utf-8')
+from .common import EntityImporter, clean, make_rel_url, SANE_MAX_RELEASES, SANE_MAX_URLS, b32_hex
 
 
 ARABESQUE_MATCH_WHERE_CLAUSE='WHERE hit = 1 AND identifier IS NOT NULL'
diff --git a/python/fatcat_tools/importers/common.py b/python/fatcat_tools/importers/common.py
index a25c3196..74595790 100644
--- a/python/fatcat_tools/importers/common.py
+++ b/python/fatcat_tools/importers/common.py
@@ -4,6 +4,7 @@ import sys
 import csv
 import json
 import ftfy
+import base64
 import sqlite3
 import subprocess
 import unicodedata
@@ -141,6 +142,14 @@ def test_clean():
     assert clean('<b>a&amp;b</b>') == '<b>a&amp;b</b>'
     assert clean('<b>a&amp;b</b>', force_xml=True) == '<b>a&b</b>'
 
+def b32_hex(s):
+    s = s.strip().split()[0].lower()
+    if s.startswith("sha1:"):
+        s = s[5:]
+    if len(s) != 32:
+        return s
+    return base64.b16encode(base64.b32decode(s.upper())).lower().decode('utf-8')
+
 def is_cjk(s):
     if not s:
         return False
diff --git a/python/fatcat_tools/importers/wayback_static.py b/python/fatcat_tools/importers/wayback_static.py
index fa0bd4ef..5b23347f 100755
--- a/python/fatcat_tools/importers/wayback_static.py
+++ b/python/fatcat_tools/importers/wayback_static.py
@@ -8,7 +8,6 @@ Works as a stand-alone script (for debugging) or as library routines.
 
 import sys
 import json
-import base64
 import hashlib
 import requests
 import datetime
@@ -17,21 +16,13 @@ import subprocess
 from bs4 import BeautifulSoup
 
 from fatcat_openapi_client import *
+from .common import b32_hex
 
 CDX_API_BASE = "https://web.archive.org/cdx/search/cdx"
 GWB_URL_BASE = "https://web.archive.org/web"
 REQ_SESSION = requests.Session()
 
 
-def b32_hex(s):
-    """copy/pasta from elsewhere"""
-    s = s.strip().split()[0].lower()
-    if s.startswith("sha1:"):
-        s = s[5:]
-    if len(s) != 32:
-        return s
-    return base64.b16encode(base64.b32decode(s.upper())).lower().decode('utf-8')
-
 def parse_wbm_url(url):
     """Takes a wayback machine URL, and returns a tuple:
 
-- 
cgit v1.2.3