#!/bin/bash
export AWS_ACCESS_KEY_ID="<s3 id>"
export AWS_SECRET_ACCESS_KEY="<s3 secret>"

export WARC_MANIFEST="<warc input file>"
export WARC_CDX_BUCKET="<cdx output bucket>"

export WARC_CDX="<per warc cdx directory>"

export SPLIT_FILE="<shard split file location>"
export ZIPNUM_CLUSTER_DIR="<final zipnum cluster directory>"