fix: code cleanups and refactors

Author: Greg Lindahl

README.md

@@ -101,7 +101,6 @@ example programs need to be surt_host_names -- e.g. org,commoncrawl
 for commoncrawl.org, or org,commoncrawl, for *.commoncrawl.org.
 
 - host.py -- generates a bunch of pngs, csvs, and a webpage summarizing a host or wildcarded host
-- count-hosts.py -- given a file containing a bunch of surt hostnames, plot fetch\_200 over time
 
 ## Example SQL queries
 
@@ -200,12 +199,13 @@ LIMIT 10
 ## Expected changes in test v3
 
 - warc\_record\_length\_av will be renamed to \_avg (that was a typo)
-- more _pct columns
+- more \_pct columns
 - addition of indegree and outdegree for all hosts from the web graph
 - add unicode block information, similar to languages
 - improve language details to be more than only LOTE and LOTE_pct
 - prank10 needs its power law touched up (hcrank10 is much better)
 - there's a sort problem that .com shards have a smattering of not-.com hosts. This hurts performance.
+- add domain prank/hcrank
 - CI running against S3
 
 ## Contributing

duck_utils.py

@@ -72,4 +72,10 @@ def open_host_index(paths=None, bucket='https://data.commoncrawl.org', grep=None
     else:
         if verbose:
             print(f'{len(paths)} paths found')
-    return duckdb.read_parquet(paths, hive_partitioning=True)
+    try:
+        return duckdb.read_parquet(paths, hive_partitioning=True)
+    except Exception:
+        print('exception:')
+        print('HOST_INDEX=', os.environ['HOST_INDEX'])
+        print('HOST_INDEX_BUCKET=', os.environ['HOST_INDEX_BUCKET'])
+        raise

host.py renamed to graph.py

@@ -15,19 +15,30 @@
 SELECT
   {cols}
 FROM host_index
-WHERE surt_host_name = '{surt_host_name}'
+WHERE surt_host_name = '{surt_host_name}'{and_tld}
 ORDER BY crawl ASC
 '''
 
-sub_host_sql = '''
+subdomain_sql = '''
 SELECT
   {cols}
 FROM host_index
-WHERE surt_host_name LIKE '{surt_host_name}'
+WHERE surt_host_name LIKE '{surt_host_name}'{and_tld}
 GROUP BY crawl
 ORDER BY crawl ASC
 '''
 
+many_host_sql = '''
+SELECT
+  crawl,
+  CAST(SUM(fetch_200) AS INT64) AS sum_fetch_200
+FROM host_index
+WHERE contains(ARRAY [{surt_list}], surt_host_name){and_tld}
+GROUP BY crawl
+ORDER BY crawl ASC
+'''
+
+
 host_columns = {
     # list order does matter for the graphs
     'rank': ['crawl', 'fetch_200', 'fetch_200_lote', 'prank10', 'hcrank10'],
@@ -39,6 +50,9 @@
 domain_columns = {
     'sum': ['crawl', 'fetch_200', 'fetch_200_lote'],
 }
+many_host_columns = {
+    'sum': ['crawl', 'fetch_200'],
+}
 
 def surt_host_name_to_title(surt_host_name):
     parts = list(reversed(surt_host_name.split(',')))
@@ -48,14 +62,31 @@ def surt_host_name_to_title(surt_host_name):
 
 
 def get_surt_host_name_values(host_index, surt_host_name, col_names):
+    if not isinstance(surt_host_name, str):
+        # if not a string, it's a list of strings
+        surt_list = ','.join(f"'{s}'" for s in surt_host_name)
+
+        tlds = set([s.split(',', 1)[0] for s in surt_host_name])
+        if len(tlds) == 1:
+            tld = next(iter(tlds))
+            and_tld = f" AND url_host_tld = '{tld}'"
+        else:
+            and_tld = ''
+
+        sql = many_host_sql.format(surt_list=surt_list, and_tld=and_tld)
+        return duckdb.sql(sql).arrow()
+
+    tld = surt_host_name.split(',', 1)[0]
+    and_tld = f" AND url_host_tld = '{tld}'"
     if surt_host_name.endswith(','):
+        col_names.remove('crawl')
         cols = ', '.join(f'CAST(SUM({col}) AS INT64) AS sum_{col}' for col in col_names)
         cols = 'crawl, '+cols
-        sql = sub_host_sql.format(cols=cols, surt_host_name=surt_host_name+'%')
+        sql = subdomain_sql.format(cols=cols, surt_host_name=surt_host_name+'%', and_tld=and_tld)
         print(sql)
     else:
         cols = ', '.join(col_names)
-        sql = host_sql.format(cols=cols, surt_host_name=surt_host_name)
+        sql = host_sql.format(cols=cols, surt_host_name=surt_host_name, and_tld=and_tld)
         print(sql)
     return duckdb.sql(sql).arrow()
 
@@ -81,7 +112,8 @@ def host_plot_values(table, col_names, title):
             side = 'r'
         else:
             side = 'l'
-        lines.append(['crawl', name, side, 'o', name])
+        # x, y, side, marker, label
+        lines.append(['crawl', name, side, None, name])
     return do_plot(df, lines, title)
 
 
@@ -99,7 +131,9 @@ def do_plot(df, lines, title):
             continue
         xvalues = df[x].astype(str)
         xvalues = [x.replace('CC-MAIN-', '') for x in xvalues]
-        color, ls = graph_utils.get_color(i)
+        ls = None
+        #color, ls = graph_utils.get_color_ls(i)
+        color, marker = graph_utils.get_color_marker(i)
         if side == 'l':
             our_line, = ax1.plot(xvalues, yvalues, marker=marker, label=label, color=color, ls=ls)
         else:
@@ -145,7 +179,7 @@ def get_host(host_index, surt_host_name, title):
 
 
 def plot_host(host_index, surt_host_name, title,
-              do_csv=False, do_png=False, do_html=False, html_template = 'host.html'):
+              do_csv=False, do_png=False, do_html=False, verbose=0, html_template = 'host.html'):
     tables, plots = get_host(host_index, surt_host_name, title)
     for key in tables:
         if do_csv:
@@ -168,7 +202,7 @@ def plot_host(host_index, surt_host_name, title,
             f.write(page)
 
 
-def get_domain(host_index, surt_host_name, title):
+def get_domain(host_index, surt_host_name, title, verbose=0):
     plots = {}
     tables = {}
     for key, cols in domain_columns.items():
@@ -183,11 +217,11 @@ def get_domain(host_index, surt_host_name, title):
 
 
 def plot_domain(host_index, surt_host_name, title,
-                do_csv=False, do_png=False, do_html=False, html_template = 'domain.html'):
-    tables, plots = get_domain(host_index, surt_host_name, title)
+                do_csv=False, do_png=False, do_html=False, verbose=0, html_template = 'domain.html'):
+    tables, plots = get_domain(host_index, surt_host_name, title, verbose=verbose)
     for key in tables:
         if do_csv:
-            out = surt_host_name + '_' + key
+            out = title + '_' + key
             host_csv(tables[key], out+'.csv')
         if do_png:
             with open (out+'.png', 'wb') as fd:
@@ -202,19 +236,19 @@ def plot_domain(host_index, surt_host_name, title,
         )
         template = env.get_template(html_template)
         page = template.render(title=title, plots=plots)
-        with open(surt_host_name + '.html', 'w') as f:
+        with open(title + '.html', 'w') as f:
             f.write(page)
 
 
-def make_plot(surt_host_name, host_index):
-    title = surt_host_name_to_title(surt_host_name)
-    if surt_host_name.endswith(','):
+def make_plot(surt_host_name, host_index, title, verbose=0):
+    if not isinstance(surt_host_name, str) or surt_host_name.endswith(','):
         plot_domain(host_index, surt_host_name, title,
-                    do_csv=True, do_png=True, do_html=True)
+                    do_csv=False, do_png=False, do_html=True, verbose=verbose)
         return
 
+    print('plot host')
     plot_host(host_index, surt_host_name, title,
-              do_csv=True, do_png=True, do_html=True)
+              do_csv=True, do_png=True, do_html=True, verbose=verbose)
 
 
 def main():
@@ -223,11 +257,25 @@ def main():
     grep = None
     #grep = 'CC-MAIN-2022'
     host_index = duck_utils.open_host_index(grep=grep, verbose=verbose)
+    if len(sys.argv) > 2 and sys.argv[1] == '-f':
+        assert len(sys.argv) == 3
+        surts = []
+        title =  sys.argv[2]
+        with open(sys.argv[2], encoding='utf8') as fd:
+            for thing in fd:
+                surt_host_name = utils.thing_to_surt_host_name(thing.rstrip(), verbose=verbose)
+                if surt_host_name:
+                    surts.append(surt_host_name)
+        if verbose:
+            print(f'making a plot for {len(surts)} hosts')
+        make_plot(surts, host_index, title, verbose=verbose)
+        return
     for thing in sys.argv[1:]:
         surt_host_name = utils.thing_to_surt_host_name(thing)
         if not surt_host_name:
             continue
-        make_plot(surt_host_name, host_index)
+        title = surt_host_name_to_title(surt_host_name)
+        make_plot(surt_host_name, host_index, title, verbose=verbose)
 
 
 if __name__ == '__main__':

graph_utils.py

@@ -16,11 +16,16 @@
 combinations = len(color_list) * len(ls_list)
 
 
-def get_color(i):
+def get_color_ls(i):
     assert i < combinations
     return color_list[i % 5], ls_list[i // 5]
 
 
+def get_color_marker(i):
+    assert i < combinations
+    return color_list[i % 5], marker_list[i // 5]
+
+
 def png_to_embed(png):
     png_b64 = base64.b64encode(png).decode('utf8')
     return '<img src="data:image/jpeg;base64,'+png_b64+'">'

test_utils.py

@@ -5,7 +5,7 @@
 
 def test_thing_to_surt_host_name():
     tests = [
-        ('va', 'va'),
+        ('va', 'va,'),
         ('va,', 'va,'),
         ('va,*', 'va,'),
 
@@ -17,10 +17,11 @@ def test_thing_to_surt_host_name():
         #('.example.com/', 'com,example,'),  # python's library drops that leading dot
         ('sub.example.com', 'com,example,sub'),
         ('*.sub.example.com', 'com,example,sub,'),
+
+        ('example.com/foo', None),
     ]
 
     value_error = [
-        ('example.com/foo', ''),
         ('example.com,', ''),
     ]
 

utils.py

@@ -1,7 +1,7 @@
 import surt
 
 
-def thing_to_surt_host_name(thing):
+def thing_to_surt_host_name(thing, verbose=0):
     '''convert a thing, probably an url of some kind, to a surt host name'''
     orig = thing
     if '/' not in thing and '.' not in thing:
@@ -11,10 +11,15 @@ def thing_to_surt_host_name(thing):
             raise ValueError('unexpected * in '+thing)
         if ',,' in thing:
             raise ValueError('unexpected ,, in '+thing)
+        if ',' not in thing:
+            # assume it is a tld wildcard
+            thing += ','
         return thing
     surt_host_name, extra = surt.surt(thing).split(')/', 1)
     if extra:
-        raise ValueError(f'skipping {orig} because extra is {extra}')
+        if verbose:
+            print(f'skipping {orig} because extra is {extra}')
+        return
     if surt_host_name.endswith(',*'):
         surt_host_name = surt_host_name[:-1]
     if ',,' in  surt_host_name: