Merge pull request #170 from creativecommons/logging-sync-etc

Logging sync etc

Author: TimidRobot

ccos/data/get_community_team_data.py

@@ -15,33 +15,31 @@
 import asana
 
 # First-party/Local
-from ccos import log
+import ccos.log
 
 ASANA_WORKSPACE_GID = "133733285600979"
 ASANA_PROJECT_GID = "1172465506923661"
 
 log_name = os.path.basename(os.path.splitext(inspect.stack()[-1].filename)[0])
-logger = logging.getLogger(log_name)
-log.reset_handler()
+LOG = logging.getLogger(log_name)
+ccos.log.reset_handler()
 
 
 def setup_asana_client():
-    logger.log(logging.INFO, "Setting up Asana client...")
+    LOG.info("Setting up Asana client...")
     try:
         asana_token = os.environ["ADMIN_ASANA_TOKEN"]
     except KeyError:
-        logger.critical("missin ADMIN_ASANA_TOKEN environment variable")
+        LOG.critical("missin ADMIN_ASANA_TOKEN environment variable")
         sys.exit(1)
     asana_client = asana.Client.access_token(asana_token)
     try:
         # Perform simple API operation to test authentication
         asana_client.workspaces.get_workspace(ASANA_WORKSPACE_GID)
     except asana.error.NoAuthorizationError as e:
-        logger.critical(
-            f"{e.status} {e.message} (is ADMIN_ASANA_TOKEN valid?)"
-        )
+        LOG.critical(f"{e.status} {e.message} (is ADMIN_ASANA_TOKEN valid?)")
         sys.exit(1)
-    logger.log(logging.INFO, "done.")
+    LOG.info("done.")
     return asana_client
 
 
@@ -80,15 +78,15 @@ def generate_databag(asana_client):
     }
     """
 
-    logger.log(logging.INFO, "Pulling from Asana and generating databag...")
+    LOG.info("Pulling from Asana and generating databag...")
     databag = {"projects": [], "community_builders": []}
 
     members = asana_client.tasks.find_by_section(
         ASANA_PROJECT_GID, opt_fields=["name", "custom_fields"]
     )
-    logger.log(logging.INFO, "Team members pulled.")
+    LOG.info("Team members pulled.")
 
-    logger.log(logging.INFO, "Processing team members...")
+    LOG.info("Processing team members...")
     for member in members:
         if member["name"] == "":
             continue  # Sometimes blank names come up
@@ -123,7 +121,7 @@ def generate_databag(asana_client):
                     )
                     break
 
-    logger.log(logging.INFO, "Done.")
+    LOG.info("Done.")
     return databag
 
 
@@ -183,8 +181,7 @@ def verify_databag(databag, repo_names):
         databag_repos = project["repos"].split(",")
         for databag_repo in databag_repos:
             if databag_repo not in repo_names:
-                logger.log(
-                    logging.ERROR,
+                LOG.error(
                     f'"{project["name"]}" contains invalid reposiotry:'
                     f' "{databag_repo}"',
                 )

ccos/data/get_repo_data.py

@@ -13,36 +13,36 @@
 from github.GithubException import GithubException, UnknownObjectException
 
 # First-party/Local
-from ccos import log
+import ccos.log
 from ccos.data.push_data_via_git import GITHUB_ORGANIZATION, GITHUB_TOKEN
 
 CC_METADATA_FILE_NAME = ".cc-metadata.yml"
 
 log_name = os.path.basename(os.path.splitext(inspect.stack()[-1].filename)[0])
-logger = logging.getLogger(log_name)
-log.reset_handler()
+LOG = logging.getLogger(log_name)
+ccos.log.reset_handler()
 
 
 def set_up_github_client():
-    logger.log(logging.INFO, "Setting up GitHub client...")
+    LOG.info("Setting up GitHub client...")
     github_client = Github(GITHUB_TOKEN)
     return github_client
 
 
 def get_cc_organization(github_client):
-    logger.log(logging.INFO, "Getting CC's GitHub organization...")
+    LOG.info("Getting CC's GitHub organization...")
     cc = github_client.get_organization(GITHUB_ORGANIZATION)
     return cc
 
 
 def get_repositories(organization):
-    logger.log(logging.INFO, "Getting CC's repos...")
+    LOG.info("Getting CC's repos...")
     repos = organization.get_repos()
     return repos
 
 
 def get_repo_github_data(repo):
-    logger.log(logging.INFO, "Getting data for this repo...")
+    LOG.info("Getting data for this repo...")
     repo_github_data = {
         "id": repo.id,
         "name": repo.name,
@@ -69,7 +69,7 @@ def get_repo_github_data(repo):
 
 
 def get_repo_cc_metadata(repo):
-    logger.log(logging.INFO, "Getting CC metadata for this repo...")
+    LOG.info("Getting CC metadata for this repo...")
     try:
         cc_metadata_file = repo.get_contents(CC_METADATA_FILE_NAME)
     except (UnknownObjectException, GithubException):
@@ -89,9 +89,7 @@ def get_repo_data_list(repos):
     total = repos.totalCount
 
     for repo in repos:
-        logger.log(
-            logging.INFO, f"Processing {count} of {total} – {repo.name}"
-        )
+        LOG.info(f"Processing {count} of {total} – {repo.name}")
         if not repo.private:
             repo_cc_metadata = get_repo_cc_metadata(repo)
             is_engineering_project = repo_cc_metadata.get(
@@ -104,9 +102,7 @@ def get_repo_data_list(repos):
                 repo_data = {**repo_github_data, **repo_cc_metadata}
                 repo_data_list.append(repo_data)
             else:
-                logger.log(
-                    logging.INFO, "Not an active engineering project, skipping"
-                )
+                LOG.info("Not an active engineering project, skipping")
         count += 1
     return sorted(repo_data_list, key=lambda k: k["name"].lower())
 

ccos/data/push_data_via_git.py

@@ -6,13 +6,13 @@
 import json
 import logging
 import os
-import tempfile
+from tempfile import TemporaryDirectory
 
 # Third-party
 import git
 
 # First-party/Local
-from ccos import log
+import ccos.log
 
 GIT_USER_NAME = "CC creativecommons.github.io Bot"
 GIT_USER_EMAIL = "cc-creativecommons-github-io-bot@creativecommons.org"
@@ -30,34 +30,34 @@
 JSON_FILE_DIR = "databags"
 
 log_name = os.path.basename(os.path.splitext(inspect.stack()[-1].filename)[0])
-logger = logging.getLogger(log_name)
-log.reset_handler()
+LOG = logging.getLogger(log_name)
+ccos.log.reset_handler()
 
 
 def set_up_repo(git_working_dir):
     if not os.path.isdir(git_working_dir):
-        logger.log(logging.INFO, "Cloning repo...")
+        LOG.info("Cloning repo...")
         repo = git.Repo.clone_from(
             url=GITHUB_REPO_URL_WITH_CREDENTIALS, to_path=git_working_dir
         )
     else:
-        logger.log(logging.INFO, "Setting up repo...")
+        LOG.info("Setting up repo...")
         repo = git.Repo(git_working_dir)
     origin = repo.remotes.origin
-    logger.log(logging.INFO, "Pulling latest code...")
+    LOG.info("Pulling latest code...")
     origin.pull()
 
 
 def set_up_git_user():
-    logger.log(logging.INFO, "Setting up git user...")
+    LOG.info("Setting up git user...")
     os.environ["GIT_AUTHOR_NAME"] = GIT_USER_NAME
     os.environ["GIT_AUTHOR_EMAIL"] = GIT_USER_EMAIL
     os.environ["GIT_COMMITTER_NAME"] = GIT_USER_NAME
     os.environ["GIT_COMMITTER_EMAIL"] = GIT_USER_EMAIL
 
 
 def generate_json_file(git_working_dir, data, filename):
-    logger.log(logging.INFO, "Generating JSON file...")
+    LOG.info("Generating JSON file...")
     json_filename = os.path.join(git_working_dir, JSON_FILE_DIR, filename)
     with open(json_filename, "w") as json_file:
         json.dump(data, json_file, sort_keys=True, indent=4)
@@ -71,14 +71,14 @@ def commit_and_push_changes(git_working_dir, json_filename):
         repo.index.add(items=f"{json_filename}")
         repo.index.commit(message="Syncing new data changes.")
         origin = repo.remotes.origin
-        logger.log(logging.INFO, "Pushing latest code...")
+        LOG.info("Pushing latest code...")
         origin.push()
     else:
-        logger.log(logging.INFO, "No changes to push...")
+        LOG.info("No changes to push...")
 
 
 def push_data(data, filename):
-    with tempfile.TemporaryDirectory() as temp_dir:
+    with TemporaryDirectory() as temp_dir:
         git_working_dir = os.path.join(temp_dir, GITHUB_REPO_NAME)
         set_up_repo(git_working_dir)
         set_up_git_user()

ccos/gh_utils.py

@@ -10,41 +10,48 @@
 from github.GithubException import BadCredentialsException
 
 # First-party/Local
-from ccos import log
+import ccos.log
 
 GITHUB_ORGANIZATION = "creativecommons"
-GITHUB_USERNAME = "cc-creativecommons-github-io-bot"
-GITHUB_TOKEN = None
+GITHUB_USERNAME_DEFAULT = "cc-creativecommons-github-io-bot"
 
 log_name = os.path.basename(os.path.splitext(inspect.stack()[-1].filename)[0])
-logger = logging.getLogger(log_name)
-log.reset_handler()
+LOG = logging.getLogger(log_name)
+ccos.log.reset_handler()
 
 
-def set_up_github_client():
-    global GITHUB_TOKEN
+def get_credentials():
     try:
-        GITHUB_TOKEN = os.environ["ADMIN_GITHUB_TOKEN"]
+        github_token = os.environ["ADMIN_GITHUB_TOKEN"]
     except KeyError:
-        logger.critical("missing ADMIN_GITHUB_TOKEN environment variable")
+        LOG.critical("missing ADMIN_GITHUB_TOKEN environment variable")
         sys.exit(1)
-    logger.log(logging.INFO, "Setting up GitHub client...")
-    github_client = Github(GITHUB_TOKEN)
-    logger.log(log.SUCCESS, "done.")
+    try:
+        github_username = os.environ["ADMIN_GITHUB_USERNAME"]
+    except KeyError:
+        github_username = GITHUB_USERNAME_DEFAULT
+    return github_username, github_token
+
+
+def set_up_github_client():
+    _, github_token = get_credentials()
+    LOG.info("Setting up GitHub client...")
+    github_client = Github(github_token)
+    LOG.log(ccos.log.SUCCESS, "done.")
     return github_client
 
 
 def get_cc_organization(github_client):
-    logger.log(logging.INFO, "Getting CC's GitHub organization...")
+    LOG.info("Getting CC's GitHub organization...")
     try:
         cc = github_client.get_organization(GITHUB_ORGANIZATION)
     except BadCredentialsException as e:
-        logger.critical(
+        LOG.critical(
             f"{e.status} {e.data['message']} (see"
             f" {e.data['documentation_url']})"
         )
         sys.exit(1)
-    logger.log(log.SUCCESS, "done.")
+    LOG.log(ccos.log.SUCCESS, "done.")
     return cc
 
 

ccos/log.py

@@ -18,6 +18,7 @@ class IndentFormatter(logging.Formatter):
         logging.WARNING: 33,  # yellow
         SUCCESS: 32,  # green
         logging.INFO: 34,  # blue
+        logging.DEBUG: 35,  # magenta
     }
 
     @staticmethod
@@ -64,10 +65,13 @@ def update_format(self, record):
             "%(asctime)s │ "
             f"{prefix}{color}%(levelname)-8s{prefix}{reset} │ "
         )
-        self._style._fmt += (
-            f"%(indent)s{prefix}{bold}%(function)s{prefix}{reset}: "
-            "%(message)s"
-        )
+        if hasattr(record, "function"):
+            self._style._fmt += (
+                f"%(indent)s{prefix}{bold}%(function)s{prefix}{reset}: "
+                "%(message)s"
+            )
+        else:
+            self._style._fmt += "%(indent)s%(message)s"
 
     def format(self, record):
         """
@@ -81,19 +85,21 @@ def format(self, record):
             self.baseline = depth
         if self.cut is None:
             filenames = map(lambda x: x.filename, stack)
-            self.cut = IndentFormatter.identify_cut(filenames)
+            self.cut = self.identify_cut(filenames)
 
         # Inject custom information into the record
         record.indent = ". " * (depth - self.baseline + self.manual_push)
-        record.function = stack[self.cut].function
+        if depth > self.cut:
+            record.function = stack[self.cut].function
 
         # Format the record using custom information
         self.update_format(record)
         out = super().format(record)
 
         # Remove custom information from the record
         del record.indent
-        del record.function
+        if hasattr(record, "function"):
+            del record.function
 
         return out