From ab1870f57afd756fffd5bfd3173b3e6a269ddaa9 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 26 Jul 2019 15:35:25 -0700 Subject: chocula: more host/domain fixes --- chocula.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/chocula.py b/chocula.py index 37d4c8b..50dff49 100755 --- a/chocula.py +++ b/chocula.py @@ -270,10 +270,12 @@ def parse_url(url): url = str(urlcanon.semantic_precise(url)) url_surt = surt.surt(url) tld = tldextract.extract(url) - domain = '.'.join(tld[:]) + host = '.'.join(tld) + if host.startswith('.'): + host = host[1:] return dict(url=url, url_surt=url_surt, - host='.'.join(tld), + host=host, registered_domain=tld.registered_domain, suffix=tld.suffix) @@ -283,6 +285,9 @@ def test_parse_url(): assert parse_url("http://thing.core.ac.uk")['host'] == 'thing.core.ac.uk' assert parse_url("http://thing.core.ac.uk")['suffix'] == 'ac.uk' + assert parse_url("google.com")['suffix'] == 'com' + assert parse_url("google.com")['host'] == 'google.com' + assert parse_url("mailto:bnewbold@bogus.com") == None assert parse_url("thing.com")['url'] == 'http://thing.com/' assert parse_url("Http://thing.com///")['url'] == 'http://thing.com/' @@ -385,7 +390,7 @@ class ChoculaDatabase(): self.c.execute("INSERT OR REPLACE INTO homepage (issnl, surt, url, host, domain, suffix) VALUES (?,?,?,?,?,?)", (issnl, meta['url_surt'], meta['url'], meta['host'], - meta['registered_domain'], meta['suffix'].suffix)) + meta['registered_domain'], meta['suffix'])) def index_entrez(self, args): path = args.input_file or ENTREZ_FILE -- cgit v1.2.3