feat: bigger csv output for hugging face

Author: Greg Lindahl

Makefile

@@ -17,8 +17,8 @@ tmp/commoncrawl.bib: $(BIBSRC)
 html/commoncrawl.html: tmp/commoncrawl.bib
 	mkdir -p html; cd html; bibtex2html --charset utf-8 ../tmp/commoncrawl.bib
 
-# CSV export to import papers into wordpress
-tmp/commoncrawl_site_wp.csv: tmp/commoncrawl.bib
+# CSV export for Hugging Face 🤗
+tmp/commoncrawl_annotated.csv: tmp/commoncrawl.bib
 	python3 export-csv.py $< >$@
 
 # format .bib file

export-csv.py

@@ -14,17 +14,32 @@
 unescape_pattern = re.compile('\\\\([{}%])')
 
 csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL)
-csvwriter.writerow(["cc_project_author","post_title","cc_project_url",
-                    "cc_project_category","post_date"])
+column_names = [
+    # old website csv
+    "cc_project_author",
+    "post_title",
+    "cc_project_url",
+    "cc_project_category",  # always 'papers'
+    "post_date",
+    # full csv
+    "keywords",
+    "abstract",
+    "cc_author_affiliation",  # removed from cc_project_author
+    "cc_class",
+    "cc_snippet",
+    "cc_dataset_used",
+    "cc_derived_dataset_about",
+    "cc_derived_dataset_used",
+    "cc_derived_dataset_cited",
+]
+csvwriter.writerow(column_names)
 
 for bib_id in bibdata.entries:
     entry = bibdata.entries[bib_id]
     b = entry.fields
     authorstr = ''
-    title = None
-    year = None
-    affiliation = ''
     url = ''
+
     # try to fill all required fields
     authors = entry.persons['author']
     for author in authors:
@@ -41,17 +56,23 @@
         if authorstr != '':
             authorstr += ' '
         authorstr += ' '.join(author.last_names)
-    if 'year' in b:
-        year = b['year']
-    for f in ['title']:
-        if f in b:
-            title = b[f]
-            break
     for f in ['URL', 'pdf', 'doi']:
         if f in b:
             url = b[f]
             break
 
+    year = b.get('year')
+    title = b.get('title')
+    keywords = b.get('keywords')
+    abstract = b.get('abstract')
+    cc_author_affiliation = b.get('cc-author-affiliation')
+    cc_class = b.get('cc-class')
+    cc_snippet = b.get('cc-snippet')
+    cc_dataset_used = b.get('cc-dataset-used')
+    cc_derived_dataset_about = b.get('cc-derived-dataset-about')
+    cc_derived_dataset_used = b.get('cc-derived-dataset-used')
+    cc_derived_dataset_cited = b.get('cc-derived-dataset-cited')
+
     if authorstr == '':
         sys.stderr.write("No author: {} - {}\n".format(bib_id))
         continue
@@ -73,4 +94,16 @@
     title = re.sub(clean_pattern, '', title)
     title = re.sub(unescape_pattern, '\1', title)
 
-    csvwriter.writerow([authorstr, title, url, 'papers', '{}0101Z00:00:00'.format(year)])
+    row = [
+        authorstr, title, url, 'papers', '{}0101Z00:00:00'.format(year),
+        keywords,
+        abstract,
+        cc_author_affiliation,
+        cc_class,
+        cc_snippet,
+        cc_dataset_used,
+        cc_derived_dataset_about,
+        cc_derived_dataset_used,
+        cc_derived_dataset_cited,
+    ]
+    csvwriter.writerow(row)