complete base codebase for gcs_reports.py

Author: naishasinha

data/2024Q2/1-fetched/gcs_fetched.csv

@@ -13,3 +13,10 @@ https://creativecommons.org/publicdomain/zero/1.0,32700000,30000000,20200,18200,
 https://creativecommons.org/licenses/nc-sa/2.0,25000000,17700000,19700,4640,80700,14700,11900,23400000,368000,1340000,1490,25000000,36100
 https://creativecommons.org/licenses/sa/1.0,108000000,91400000,23000,17100,159000,38200,21600,104000000,270000,337000,28600,108000000,224000
 https://creativecommons.org/licenses/by-sa/1.0,102000000,85900000,21900,15800,150000,36100,21200,97400000,267000,330000,28300,102000000,219000
+https://creativecommons.org/licenses/by-nd/2.5,49500000,35500000,45100,8640,133000,24600,23600,47800000,367000,37900,2200,49500000,11200
+https://creativecommons.org/licenses/by-nd-nc/1.0,15800000,10300000,15600,2940,66800,10200,21400,15300000,62400,29000,1980,15800000,27500
+https://creativecommons.org/licenses/nd/1.0,220000000,194000000,89900,56500,424000,147000,88400,213000000,490000,297000,69500,220000000,2570000
+https://creativecommons.org/licenses/by-nc-sa/4.0,62900000,44700000,49900,32900,163000,32900,79400,58900000,1020000,3610000,11500,62900000,102000
+https://creativecommons.org/licenses/sampling+/1.0,169000000,151000000,27300,22900,274000,57300,34700,168000000,123000,62800,1310,169000000,200000
+https://creativecommons.org/licenses/by-nc-sa/2.5,31600000,22800000,20100,5400,83100,15200,10400,29900000,258000,250000,1570,31600000,7290
+https://creativecommons.org/licenses/nc-sa/1.0,24100000,17500000,10400,3820,58800,11800,10600,23300000,67500,280000,1200,24100000,42400

data/2024Q2/3-reports/gcs_country_report.png

@@ -1 +1 @@
-total_records_retrieved: 182
+total_records_retrieved: 273

data/2024Q2/3-reports/gcs_language_report.png

@@ -0,0 +1,8 @@
+LICENSE TYPE, No Priori, United States, Canada, India, United Kingdom, Australia, Japan, English, Spanish, French, Arabic, Chinese (Simplified), Indonesian
+https://creativecommons.org/licenses/by/2.5,235000000,208000000,112000,82400,406000,701000,76800,225000000,3940000,835000,88300,235000000,115000
+https://creativecommons.org/licenses/by/4.0,412000000,334000000,702000,360000,7250000,770000,675000,381000000,16100000,5260000,255000,412000000,11900000
+https://creativecommons.org/licenses/by-nc-sa/2.5,31400000,22700000,21200,5400,83000,15000,10300,29800000,254000,248000,1580,31400000,7480
+https://creativecommons.org/licenses/nc/1.0,54600000,43400000,25100,8390,128000,22200,30500,53600000,136000,479000,3760,54500000,81500
+https://creativecommons.org/licenses/by-nc-sa/1.0,24000000,17400000,11300,3810,59100,11900,10600,23300000,66900,276000,1220,24000000,42900
+https://creativecommons.org/licenses/by/2.1,219000000,189000000,132000,56200,477000,139000,269000,211000000,494000,232000,19800,219000000,119000
+https://creativecommons.org/licenses/nc-sampling+/1.0,41200000,32000000,12900,4590,82300,14600,18800,41000000,52800,37800,381,41200000,31100

data/2024Q2/3-reports/gcs_licensetype_report.png

@@ -0,0 +1 @@
+total_records_retrieved: 91

data/2024Q2/state.yaml

@@ -335,7 +335,7 @@ def main():
     state = load_state()
     total_records_retrieved = state["total_records_retrieved"]
     LOGGER.info(f"Initial total_records_retrieved: {total_records_retrieved}")
-    goal_records = 200  # Set goal number of records
+    goal_records = 1000  # Set goal number of records
 
     if total_records_retrieved >= goal_records:
         LOGGER.info(

data/2024Q3/1-fetched/gcs_fetched.csv

@@ -3,13 +3,16 @@
 This file is dedicated to visualizing and analyzing the data collected.
 """
 # Standard library
+import argparse
 import os
 import sys
 import traceback
 
 # Third-party
 import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
 import pandas as pd
+import seaborn as sns
 
 # Add parent directory so shared can be imported
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
@@ -21,11 +24,31 @@
 LOGGER, PATHS = shared.setup(__file__)
 
 
-def load_data():
+def parse_arguments():
+    """
+    Parses command-line arguments, returns parsed arguments.
+    """
+    LOGGER.info("Parsing command-line arguments")
+    parser = argparse.ArgumentParser(description="Google Custom Search Report")
+    parser.add_argument(
+        "--quarter",
+        type=str,
+        required=True,
+        help="Data quarter in format YYYYQx, e.g., 2024Q2",
+    )
+    return parser.parse_args()
+
+
+def load_data(args):
     """
     Load the collected data from the CSV file.
     """
-    file_path = os.path.join(PATHS["data_phase"], "gcs_fetched.csv")
+    selected_quarter = args.quarter
+
+    file_path = os.path.join(
+        PATHS["data"], f"{selected_quarter}", "1-fetched", "gcs_fetched.csv"
+    )
+
     if not os.path.exists(file_path):
         LOGGER.error(f"Data file not found: {file_path}")
         return pd.DataFrame()
@@ -35,38 +58,205 @@ def load_data():
     return data
 
 
-def process_data(data):
+# By country, by license type, by license language
+
+
+def visualize_by_country(data, args):
     """
-    Process the data to prepare it for visualization.
+    Create a bar chart for the number of webpages licensed by country.
     """
-    # are we supposed to take from phase 2?
-    return data
+    LOGGER.info(
+        "Creating a bar chart for the number of webpages licensed by country."
+    )
+
+    selected_quarter = args.quarter
+
+    # Get the list of country columns dynamically
+    columns = [col.strip() for col in data.columns.tolist()]
+
+    start_index = columns.index("United States")
+    end_index = columns.index("Japan") + 1
+
+    countries = columns[start_index:end_index]
+
+    data.columns = data.columns.str.strip()
+
+    LOGGER.info(f"Cleaned Columns: {data.columns.tolist()}")
 
+    # Aggregate the data by summing the counts for each country
+    country_data = data[countries].sum()
 
-def visualize_data(data):
+    plt.figure(figsize=(12, 8))
+    ax = sns.barplot(x=country_data.index, y=country_data.values)
+    plt.title(
+        f"Number of Google Webpages Licensed by Country ({selected_quarter})"
+    )
+    plt.xlabel("Country")
+    plt.ylabel("Number of Webpages")
+    plt.xticks(rotation=45)
+
+    # Add value numbers to the top of each bar
+    for p in ax.patches:
+        ax.annotate(
+            format(p.get_height(), ",.0f"),
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="center",
+            xytext=(0, 9),
+            textcoords="offset points",
+        )
+
+    # Format the y-axis to display numbers without scientific notation
+    ax.get_yaxis().get_major_formatter().set_scientific(False)
+    ax.get_yaxis().set_major_formatter(
+        plt.FuncFormatter(lambda x, loc: "{:,}".format(int(x)))
+    )
+
+    output_directory = os.path.join(
+        PATHS["data"], f"{selected_quarter}", "3-reports"
+    )
+
+    LOGGER.info(f"Output directory: {output_directory}")
+
+    # Create the directory if it does not exist
+    os.makedirs(output_directory, exist_ok=True)
+    plt.savefig(os.path.join(output_directory, "gcs_country_report.png"))
+
+    plt.show()
+
+    LOGGER.info("Visualization by country created.")
+
+
+def visualize_by_license_type(data, args):
     """
-    Create visualizations for the data.
+    Create a bar chart for the number of webpages licensed by license type
     """
-    plt.figure(figsize=(10, 6))
+    LOGGER.info(
+        "Creating a bar chart for the number of"
+        " webpages licensed by license type."
+    )
+
+    selected_quarter = args.quarter
 
-    # Example - fix later
-    license_counts = data["LICENSE TYPE"].value_counts()
-    license_counts.plot(kind="bar")
-    plt.title("License Counts")
+    # Strip any leading/trailing spaces from the columns
+    data.columns = data.columns.str.strip()
+
+    # Sum the values across all columns except the first one ('LICENSE TYPE')
+    license_data = data.set_index("LICENSE TYPE").sum(axis=1)
+
+    plt.figure(figsize=(12, 8))
+    ax = sns.barplot(x=license_data.index, y=license_data.values)
+    plt.title(
+        f"Number of Webpages Licensed by License Type ({selected_quarter})"
+    )
     plt.xlabel("License Type")
-    plt.ylabel("Count")
+    plt.ylabel("Number of Webpages")
+    plt.xticks(rotation=45, ha="right")
+
+    # Use the millions formatter for y-axis
+    def millions_formatter(x, pos):
+        "The two args are the value and tick position"
+        return f"{x * 1e-6:.1f}M"
+
+    ax.yaxis.set_major_formatter(ticker.FuncFormatter(millions_formatter))
+
+    plt.tight_layout()
+
+    output_directory = os.path.join(
+        PATHS["data"], f"{selected_quarter}", "3-reports"
+    )
+
+    LOGGER.info(f"Output directory: {output_directory}")
+
+    # Create the directory if it does not exist
+    os.makedirs(output_directory, exist_ok=True)
+    plt.savefig(os.path.join(output_directory, "gcs_licensetype_report.png"))
+
     plt.show()
 
-    LOGGER.info("Visualization created.")
+    LOGGER.info("Visualization by license type created.")
+
+
+def visualize_by_language(data, args):
+    """
+    Create a bar chart for the number of webpages licensed by language.
+    """
+    LOGGER.info(
+        "Creating a bar chart for the number of webpages licensed by language."
+    )
+
+    selected_quarter = args.quarter
+
+    # Get the list of country columns dynamically
+    columns = [col.strip() for col in data.columns.tolist()]
+
+    start_index = columns.index("English")
+    end_index = columns.index("Indonesian") + 1
+
+    languages = columns[start_index:end_index]
+
+    data.columns = data.columns.str.strip()
+
+    LOGGER.info(f"Cleaned Columns: {data.columns.tolist()}")
+
+    # Aggregate the data by summing the counts for each country
+    language_data = data[languages].sum()
+
+    plt.figure(figsize=(12, 8))
+    ax = sns.barplot(x=language_data.index, y=language_data.values)
+    plt.title(
+        f"Number of Google Webpages Licensed by Country ({selected_quarter})"
+    )
+    plt.xlabel("Country")
+    plt.ylabel("Number of Webpages")
+    plt.xticks(rotation=45)
+
+    # Add value numbers to the top of each bar
+    for p in ax.patches:
+        ax.annotate(
+            format(p.get_height(), ",.0f"),
+            (p.get_x() + p.get_width() / 2.0, p.get_height()),
+            ha="center",
+            va="center",
+            xytext=(0, 9),
+            textcoords="offset points",
+        )
+
+    # Format the y-axis to display numbers without scientific notation
+    ax.get_yaxis().get_major_formatter().set_scientific(False)
+    ax.get_yaxis().set_major_formatter(
+        plt.FuncFormatter(lambda x, loc: "{:,}".format(int(x)))
+    )
+
+    output_directory = os.path.join(
+        PATHS["data"], f"{selected_quarter}", "3-reports"
+    )
+
+    LOGGER.info(f"Output directory: {output_directory}")
+
+    # Create the directory if it does not exist
+    os.makedirs(output_directory, exist_ok=True)
+    plt.savefig(os.path.join(output_directory, "gcs_language_report.png"))
+
+    plt.show()
+
+    LOGGER.info("Visualization by language created.")
 
 
 def main():
-    data = load_data()
+
+    args = parse_arguments()
+
+    data = load_data(args)
     if data.empty:
         return
 
-    processed_data = process_data(data)
-    visualize_data(processed_data)
+    current_directory = os.getcwd()
+    LOGGER.info(f"Current working directory: {current_directory}")
+
+    visualize_by_country(data, args)
+    visualize_by_license_type(data, args)
+    visualize_by_language(data, args)
 
 
 if __name__ == "__main__":

data/2024Q3/3-reports/gcs_country_report.png

@@ -36,6 +36,8 @@ def setup(current_file):
     paths["state"] = os.path.join(data_quarter, "state.yaml")
     paths["data_phase"] = os.path.join(data_quarter, phase)
 
+    paths["data_quarter"] = data_quarter
+
     return logger, paths