Changes and fixes for May/June/July 2018 web graphs

- fix creation of webgraph/bvgraph with input
  from multiple edge files
- update configuration

Author: sebastian-nagel

src/script/webgraph_ranking/process_webgraph.sh

@@ -169,6 +169,10 @@ set -exo pipefail
 
 if [ -d $EDGES ]; then
     # edges is a directory with multiple files
+    sort_input=""
+    for e in $EDGES/part-*.gz; do
+        sort_input="$sort_input <(zcat $e)"
+    done
     if ${USE_WEBGRAPH_BIG:-false}; then
         ## TODO:
         ##    * option  --threads  not available in webgraph-big
@@ -184,15 +188,11 @@ if [ -d $EDGES ]; then
         ##               at it.unimi.dsi.big.webgraph.ArcListASCIIGraph.load(ArcListASCIIGraph.java:283)
         ##               at it.unimi.dsi.big.webgraph.ArcListASCIIGraph.load(ArcListASCIIGraph.java:279)
         ##               at it.unimi.dsi.big.webgraph.ArcListASCIIGraph.loadOffline(ArcListASCIIGraph.java:255)
-        sort_input=""
-        for e in $EDGES/part-*.gz; do
-            sort_input="$sort_input <(zcat $e)"
-        done
         _step bvgraph \
               bash -c "eval \"sort --batch-size=$SORT_BATCHES -t$'\t' -k1,1n -k2,2n --stable --merge $sort_input\" | $WG $WGP.BVGraph --once -g $WGP.ArcListASCIIGraph - $FULLNAME"
     else
         _step bvgraph \
-              $WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(zcat $EDGES/*.gz) $FULLNAME
+              bash -c "$WG $WGP.BVGraph --threads $THREADS -g $WGP.ArcListASCIIGraph <(eval \"sort --batch-size=$SORT_BATCHES -t$'\t' -k1,1n -k2,2n --stable --merge $sort_input\") $FULLNAME"
     fi
 else
     if ${USE_WEBGRAPH_BIG:-false}; then

src/script/webgraph_ranking/webgraph_config.sh

@@ -24,9 +24,15 @@ JOIN_RANKS_IN_MEMORY=${JOIN_RANKS_IN_MEMORY:-true}
 # => it's only an empirical value and possibly needs to be adjusted
 THREADS=2
 HYP_REG=4
+## on r4.8xlarge (244 GB)
+#THREADS=32
+#HYP_REG=4   # 4-5 for hostgraph, 10 for domain graph
 ## on r4.16xlarge (488 GB)
-THREADS=64
-HYP_REG=5   # 4-6 for hostgraph, 10 for domain graph
+#THREADS=64
+#HYP_REG=5   # 4-6 for hostgraph, 10 for domain graph
+## on r5.12xlarge (384 GB)
+#THREADS=48
+#HYP_REG=5   # 4-6 for hostgraph, 10 for domain graph
 ## on x1.16xlarge (976 GB)
 #THREADS=64
 #HYP_REG=9
@@ -62,5 +68,5 @@ export SORT_BUFFER_SIZE=${SORT_BUFFER_SIZE:-$MEM_10PERC}
 
 # max. number of merge inputs
 # (should be not less than number of vertices / edges files to be merged)
-export SORT_BATCHES=${SORT_BATCHES:-160}
+export SORT_BATCHES=${SORT_BATCHES:-240}