Merge pull request #121 from creativecommons/fetch-automation

Merge fetch-automation into gsoc2024-dev-1 branch

Author: naishasinha

.github/workflows/fetch.yml

@@ -2,10 +2,14 @@ name: Fetch Data
 
 on:
   push:
-    branches:
-      - automation-workflow
   schedule:
     - cron: '15 1 1-20 1,4,7,10 *'
+    - cron: '15 5 1-20 1,4,7,10 *'
+    - cron: '15 9 1-20 1,4,7,10 *'
+    - cron: '15 13 1-20 1,4,7,10 *'
+    - cron: '15 17 1-20 1,4,7,10 *'
+    - cron: '15 21 1-20 1,4,7,10 *'
+    - cron: '15 23 1-20 1,4,7,10 *'
   workflow_dispatch:
 
 jobs:
@@ -39,9 +43,20 @@ jobs:
         run: |
           python scripts/1-fetch/gcs_fetched.py
 
-      - name: Commit and push changes if needed
+      - name: Fetch and merge changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
+
+      - name: Add and commit changes
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python scripts/shared.py --operation add_and_commit --message "Automated data fetch and commit"
+
+      - name: Push changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
           python scripts/shared.py --operation push

.github/workflows/process.yml

@@ -1,11 +1,15 @@
 name: Process Data
 
 on:
-#   push:
-#     branches:
-#         - automation-workflow
+  # push:
   schedule:
-    - cron: '15 1 1-20 2,5,8,11 *'
+    - cron: '15 1 1-20 1,4,7,10 *'
+    - cron: '15 5 1-20 1,4,7,10 *'
+    - cron: '15 9 1-20 1,4,7,10 *'
+    - cron: '15 13 1-20 1,4,7,10 *'
+    - cron: '15 17 1-20 1,4,7,10 *'
+    - cron: '15 21 1-20 1,4,7,10 *'
+    - cron: '15 23 1-20 1,4,7,10 *'
   workflow_dispatch:
 
 jobs:
@@ -26,14 +30,29 @@ jobs:
           pip install --upgrade pip
           pip install pipenv
           pipenv sync --system
+          pip install -r requirements.txt
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
 
       - name: Run process script
         run: |
           python scripts/2-process/gcs_processed.py
 
-      - name: Commit and push changes if needed
+      - name: Fetch and merge changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
+
+      - name: Add and commit changes
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           python scripts/shared.py --operation add_and_commit --message "Automated data processing and commit"
+
+      - name: Push changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
           python scripts/shared.py --operation push

.github/workflows/report.yml

@@ -1,11 +1,15 @@
 name: Generate Report
 
 on:
-#   push:
-#     branches:
-#         - automation-workflow
+  push:
   schedule:
-    - cron: '15 1 1-20 3,6,9,12 *'
+    - cron: '15 1 1-20 1,4,7,10 *'
+    - cron: '15 5 1-20 1,4,7,10 *'
+    - cron: '15 9 1-20 1,4,7,10 *'
+    - cron: '15 13 1-20 1,4,7,10 *'
+    - cron: '15 17 1-20 1,4,7,10 *'
+    - cron: '15 21 1-20 1,4,7,10 *'
+    - cron: '15 23 1-20 1,4,7,10 *'
   workflow_dispatch:
 
 jobs:
@@ -26,14 +30,29 @@ jobs:
           pip install --upgrade pip
           pip install pipenv
           pipenv sync --system
+          pip install -r requirements.txt
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=./scripts" >> $GITHUB_ENV
 
       - name: Run report script
         run: |
           python scripts/3-report/gcs_reports.py
 
-      - name: Commit and push changes if needed
+      - name: Fetch and merge changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python scripts/shared.py --operation fetch_and_merge --branch fetch-automation
+
+      - name: Add and commit changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python scripts/shared.py --operation add_and_commit --message "Automated data report and commit"
+
+      - name: Push changes
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          python scripts/shared.py --operation add_and_commit --message "Automated report generation and commit"
           python scripts/shared.py --operation push

data/2024Q3/1-fetch/gcs_fetched.csv

@@ -16,3 +16,4 @@ https://creativecommons.org/licenses/by-nc-sa/3.0,35800000,29800000,18000,5760,1
 https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
 https://creativecommons.org/licenses/sa/1.0,120000000,101000000,23500,16600,167000,56500,24400,116000000,247000,314000,29100,120000000,213000
 https://creativecommons.org/licenses/by-nc-sa/2.0,20400000,13800000,21400,3730,65000,21700,9270,19100000,92600,773000,4470,20400000,5740
+https://creativecommons.org/licenses/by-sa/2.0,0,0,0,0,0,0,0,0,0,0,0,0,0

data/2024Q3/3-report/gcs_country_report.png

@@ -4,6 +4,18 @@
 ## Data Source: Google Custom Search
 
 
+### Country Report
+![Number of Google Webpages Licensed by Country](3-report/gcs_country_report.png)
+Number of Google Webpages Licensed by Country
+
+### License Type Report
+![Number of Webpages Licensed by License Type](3-report/gcs_licensetype_report.png)
+Number of Webpages Licensed by License Type
+
+### Language Report
+![Number of Google Webpages Licensed by Language](3-report/gcs_language_report.png)
+Number of Google Webpages Licensed by Language
+
 ### Country Report
 ![Number of Google Webpages Licensed by Country](3-report/gcs_country_report.png)
 Number of Google Webpages Licensed by Country

data/2024Q3/3-report/gcs_language_report.png

@@ -1 +1 @@
-total_records_retrieved: 234
+total_records_retrieved: 325

data/2024Q3/3-report/gcs_licensetype_report.png

@@ -52,21 +52,46 @@ def log_paths(logger, paths):
     logger.info(f"PATHS:{paths_list}")
 
 
-def commit_changes(message):
-    repo_path = os.getcwd()
+def fetch_and_merge(repo_path, branch="fetch-automation"):
     try:
         repo = Repo(repo_path)
+        origin = repo.remote(name="origin")
+        origin.fetch()
+        repo.git.merge(f"origin/{branch}", allow_unrelated_histories=True)
+        logging.info(f"Fetched and merged latest changes from {branch}")
     except InvalidGitRepositoryError:
         logging.error(f"Invalid Git repository at {repo_path}")
-        return
     except NoSuchPathError:
         logging.error(f"No such path: {repo_path}")
-        return
+    except Exception as e:
+        logging.error(f"Error during fetch and merge: {e}")
 
-    repo.git.add(update=True)
-    repo.index.commit(message)
-    origin = repo.remote(name="origin")
-    origin.push()
+
+def add_and_commit(repo_path, message):
+    try:
+        repo = Repo(repo_path)
+        if not repo.is_dirty(untracked_files=True):
+            logging.info("No changes to commit")
+            return
+        repo.git.add(update=True)
+        repo.index.commit(message)
+        logging.info("Changes committed")
+    except InvalidGitRepositoryError:
+        logging.error(f"Invalid Git repository at {repo_path}")
+    except NoSuchPathError:
+        logging.error(f"No such path: {repo_path}")
+
+
+def push_changes(repo_path):
+    try:
+        repo = Repo(repo_path)
+        origin = repo.remote(name="origin")
+        origin.push()
+        logging.info("Changes pushed")
+    except InvalidGitRepositoryError:
+        logging.error(f"Invalid Git repository at {repo_path}")
+    except NoSuchPathError:
+        logging.error(f"No such path: {repo_path}")
 
 
 def main():
@@ -75,16 +100,29 @@ def main():
         "--operation",
         type=str,
         required=True,
-        help="Operation to perform: commit",
+        help="Operation to perform: fetch_and_merge, add_and_commit, push",
     )
+    parser.add_argument("--message", type=str, help="Commit message")
     parser.add_argument(
-        "--message", type=str, required=True, help="Commit message"
+        "--branch",
+        type=str,
+        default="fetch-automation",
+        help="Branch to fetch and merge from",
     )
-
     args = parser.parse_args()
 
-    if args.operation == "commit":
-        commit_changes(args.message)
+    repo_path = os.getcwd()  # Assuming the script runs in the root of the repo
+
+    if args.operation == "fetch_and_merge":
+        fetch_and_merge(repo_path, args.branch)
+    elif args.operation == "add_and_commit":
+        if not args.message:
+            raise ValueError(
+                "Commit message is required for add_and_commit operation"
+            )
+        add_and_commit(repo_path, args.message)
+    elif args.operation == "push":
+        push_changes(repo_path)
     else:
         raise ValueError("Unsupported operation")