Skip to content

Commit 97a84aa

Browse files
committed
refactor gcs to use query plan and set temp schedule
1 parent c5b3364 commit 97a84aa

File tree

4 files changed

+202
-319
lines changed

4 files changed

+202
-319
lines changed

.github/workflows/fetch.yml renamed to .github/workflows/1-fetch.yml

+10-13
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,13 @@ name: Fetch Data
22

33
on:
44
schedule:
5-
# at 01:15 on days 1-20 in first month of each quarter
6-
- cron: '15 1 1-20 1,4,7,10 *'
5+
# Normal schedule
6+
# # at 01:15 on all days in first month of each quarter
7+
# - cron: '15 1 * 1,4,7,10 *'
8+
# Temp schedule
9+
# at 01:15 on all days in all months
10+
- cron: '15 1 * * *'
11+
712
workflow_dispatch:
813

914
jobs:
@@ -44,14 +49,6 @@ jobs:
4449
run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
4550

4651
- name: Run fetch script
47-
run:
48-
# ./scripts/1-fetch/deviantart_fetched.py
49-
# ./scripts/1-fetch/flickr_fetched.py
50-
# ./scripts/1-fetch/github_fetched.py
51-
# ./scripts/1-fetch/internetarchive_fetched.py
52-
# ./scripts/1-fetch/metmuseum_fetched.py
53-
# ./scripts/1-fetch/vimeo_fetched.py
54-
# ./scripts/1-fetch/wikicommons_fetched.py
55-
# ./scripts/1-fetch/wikipedia_fetched.py
56-
# ./scripts/1-fetch/youtube_fetched.py
57-
./scripts/1-fetch/gcs_fetched.py
52+
run: |
53+
./scripts/1-fetch/gcs_fetched.py \
54+
--enable-git --enable-save --limit=100

0 commit comments

Comments
 (0)