Skip to content

Commit cba0219

Browse files
naishasinhanaishasinha
committed
initial shared module call switch / error state update
1 parent 204866d commit cba0219

File tree

6 files changed

+108
-80
lines changed

6 files changed

+108
-80
lines changed

‎.github/workflows/fetch.yml

+17-17
Original file line numberDiff line numberDiff line change
@@ -37,20 +37,20 @@ jobs:
3737
run: |
3838
python scripts/1-fetch/gcs_fetched.py
3939
40-
- name: Fetch and merge changes
41-
env:
42-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
43-
run: |
44-
python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
45-
46-
- name: Add and commit changes
47-
env:
48-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
49-
run: |
50-
python scripts/shared.py --operation add_and_commit --message "Automated data fetch and commit"
51-
52-
- name: Push changes
53-
env:
54-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55-
run: |
56-
python scripts/shared.py --operation push
40+
# - name: Fetch and merge changes
41+
# env:
42+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
43+
# run: |
44+
# python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
45+
46+
# - name: Add and commit changes
47+
# env:
48+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
49+
# run: |
50+
# python scripts/shared.py --operation add_and_commit --message "Automated data fetch and commit"
51+
52+
# - name: Push changes
53+
# env:
54+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
55+
# run: |
56+
# python scripts/shared.py --operation push

‎.github/workflows/report.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Generate Report
22

33
on:
4-
push:
4+
# push:
55
schedule:
66
# 1:15am onwards, days 1-20, third month of each quarter
77
- cron: '15 1,5,9,13,17,21,23 1-20 3,6,9,12 *'

‎Pipfile.lock

+12-12
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎data/2024Q3/1-fetch/gcs_fetched.csv

-4
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,3 @@ https://creativecommons.org/licenses/by-nc-sa/3.0,35800000,29800000,18000,5760,1
1616
https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
1717
https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
1818
https://creativecommons.org/licenses/by-nc-sa/2.0,20400000,13800000,21400,3730,65000,21700,9270,19100000,92600,773000,4470,20400000,5740
19-
https://creativecommons.org/licenses/by-sa/2.0,0,0,0,0,0,0,0,0,0,0,0,0,0
20-
https://creativecommons.org/licenses/nd-nc/1.0,0,0,0,0,0,0,0,0,0,0,0,0,0
21-
https://creativecommons.org/licenses/by-nd/2.0,0,0,0,0,0,0,0,0,0,0,0,0,0
22-
https://creativecommons.org/licenses/by/2.0,0,0,0,0,0,0,0,0,0,0,0,0,0

‎scripts/1-fetch/gcs_fetched.py

+21
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,13 @@ def record_results(results):
333333

334334
def main():
335335

336+
try:
337+
# Fetch and merge changes
338+
shared.fetch_and_merge(PATHS["repo"])
339+
except shared.GitOperationError as e:
340+
LOGGER.error(f"Fetch and merge failed: {e}")
341+
sys.exit(e.exit_code)
342+
336343
args = parse_arguments()
337344
state = load_state()
338345
total_records_retrieved = state["total_records_retrieved"]
@@ -371,6 +378,20 @@ def main():
371378
state["total_records_retrieved"] = total_records_retrieved
372379
save_state(state)
373380

381+
try:
382+
# Add and commit changes
383+
shared.add_and_commit(PATHS["repo"], "Fetched and updated new data")
384+
except shared.GitOperationError as e:
385+
LOGGER.error(f"Add and commit failed: {e}")
386+
sys.exit(e.exit_code)
387+
388+
try:
389+
# Push changes
390+
shared.push_changes(PATHS["repo"])
391+
except shared.GitOperationError as e:
392+
LOGGER.error(f"Push changes failed: {e}")
393+
sys.exit(e.exit_code)
394+
374395

375396
if __name__ == "__main__":
376397
try:

‎scripts/shared.py

+57-46
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Standard library
2-
import argparse
2+
# import argparse
3+
# Standard library
34
import logging
45
import os
56
from datetime import datetime, timezone
@@ -9,6 +10,12 @@
910
from pandas import PeriodIndex
1011

1112

13+
class GitOperationError(Exception):
14+
def __init__(self, message, exit_code):
15+
super().__init__(message)
16+
self.exit_code = exit_code
17+
18+
1219
def setup(current_file):
1320
# Set up logging
1421
logging.basicConfig(
@@ -52,19 +59,19 @@ def log_paths(logger, paths):
5259
logger.info(f"PATHS:{paths_list}")
5360

5461

55-
def fetch_and_merge(repo_path, branch="fetch-automation"):
62+
def fetch_and_merge(repo_path, branch="refine-automation"):
5663
try:
5764
repo = Repo(repo_path)
5865
origin = repo.remote(name="origin")
5966
origin.fetch()
6067
repo.git.merge(f"origin/{branch}", allow_unrelated_histories=True)
6168
logging.info(f"Fetched and merged latest changes from {branch}")
6269
except InvalidGitRepositoryError:
63-
logging.error(f"Invalid Git repository at {repo_path}")
70+
raise GitOperationError(f"Invalid Git repository at {repo_path}", 2)
6471
except NoSuchPathError:
65-
logging.error(f"No such path: {repo_path}")
72+
raise GitOperationError(f"No such path: {repo_path}", 3)
6673
except Exception as e:
67-
logging.error(f"Error during fetch and merge: {e}")
74+
raise GitOperationError(f"Error during fetch and merge: {e}", 1)
6875

6976

7077
def add_and_commit(repo_path, message):
@@ -77,9 +84,11 @@ def add_and_commit(repo_path, message):
7784
repo.index.commit(message)
7885
logging.info("Changes committed")
7986
except InvalidGitRepositoryError:
80-
logging.error(f"Invalid Git repository at {repo_path}")
87+
raise GitOperationError(f"Invalid Git repository at {repo_path}", 2)
8188
except NoSuchPathError:
82-
logging.error(f"No such path: {repo_path}")
89+
raise GitOperationError(f"No such path: {repo_path}", 3)
90+
except Exception as e:
91+
raise GitOperationError(f"Error during add and commit: {e}", 1)
8392

8493

8594
def push_changes(repo_path):
@@ -89,43 +98,45 @@ def push_changes(repo_path):
8998
origin.push()
9099
logging.info("Changes pushed")
91100
except InvalidGitRepositoryError:
92-
logging.error(f"Invalid Git repository at {repo_path}")
101+
raise GitOperationError(f"Invalid Git repository at {repo_path}", 2)
93102
except NoSuchPathError:
94-
logging.error(f"No such path: {repo_path}")
95-
96-
97-
def main():
98-
parser = argparse.ArgumentParser(description="Git operations script")
99-
parser.add_argument(
100-
"--operation",
101-
type=str,
102-
required=True,
103-
help="Operation to perform: fetch_and_merge, add_and_commit, push",
104-
)
105-
parser.add_argument("--message", type=str, help="Commit message")
106-
parser.add_argument(
107-
"--branch",
108-
type=str,
109-
default="fetch-automation",
110-
help="Branch to fetch and merge from",
111-
)
112-
args = parser.parse_args()
113-
114-
repo_path = os.getcwd() # Assuming the script runs in the root of the repo
115-
116-
if args.operation == "fetch_and_merge":
117-
fetch_and_merge(repo_path, args.branch)
118-
elif args.operation == "add_and_commit":
119-
if not args.message:
120-
raise ValueError(
121-
"Commit message is required for add_and_commit operation"
122-
)
123-
add_and_commit(repo_path, args.message)
124-
elif args.operation == "push":
125-
push_changes(repo_path)
126-
else:
127-
raise ValueError("Unsupported operation")
128-
129-
130-
if __name__ == "__main__":
131-
main()
103+
raise GitOperationError(f"No such path: {repo_path}", 3)
104+
except Exception as e:
105+
raise GitOperationError(f"Error during push changes: {e}", 1)
106+
107+
108+
# def main():
109+
# parser = argparse.ArgumentParser(description="Git operations script")
110+
# parser.add_argument(
111+
# "--operation",
112+
# type=str,
113+
# required=True,
114+
# help="Operation to perform: fetch_and_merge, add_and_commit, push",
115+
# )
116+
# parser.add_argument("--message", type=str, help="Commit message")
117+
# parser.add_argument(
118+
# "--branch",
119+
# type=str,
120+
# default="refine-automation",
121+
# help="Branch to fetch and merge from",
122+
# )
123+
# args = parser.parse_args()
124+
125+
# repo_path = os.getcwd() # Assuming the script runs in repo root
126+
127+
# if args.operation == "fetch_and_merge":
128+
# fetch_and_merge(repo_path, args.branch)
129+
# elif args.operation == "add_and_commit":
130+
# if not args.message:
131+
# raise ValueError(
132+
# "Commit message is required for add_and_commit operation"
133+
# )
134+
# add_and_commit(repo_path, args.message)
135+
# elif args.operation == "push":
136+
# push_changes(repo_path)
137+
# else:
138+
# raise ValueError("Unsupported operation")
139+
140+
141+
# if __name__ == "__main__":
142+
# main()

0 commit comments

Comments
 (0)