9
9
import os
10
10
import sys
11
11
import traceback
12
- import logging
13
12
14
13
# Third-party
15
14
import pandas as pd
18
17
from requests .adapters import HTTPAdapter
19
18
from urllib3 .util .retry import Retry
20
19
21
- # Set up logging configuration
22
- logging .basicConfig (level = logging .DEBUG , format = '%(asctime)s - %(levelname)s - %(message)s' )
23
-
24
20
# Set up current working directory (CWD) and root_path
25
21
CWD = os .path .dirname (os .path .abspath (__file__ ))
26
22
root_path = os .path .dirname (CWD )
41
37
API_KEYS = os .getenv ("GOOGLE_API_KEYS" ).split ("," )
42
38
PSE_KEY = os .getenv ("PSE_KEY" )
43
39
40
+
44
41
def get_license_list ():
45
42
"""
46
43
Provides the list of license from 2018's record of Creative Commons.
@@ -64,6 +61,7 @@ def get_license_list():
64
61
)
65
62
return license_list [4 :]
66
63
64
+
67
65
def get_request_url (license ):
68
66
"""
69
67
Provides the API Endpoint URL for specified parameter combinations.
@@ -84,10 +82,11 @@ def get_request_url(license):
84
82
)
85
83
except Exception as e :
86
84
if isinstance (e , IndexError ):
87
- logging . error ("Depleted all API Keys provided" )
85
+ print ("Depleted all API Keys provided" , file = sys . stderr )
88
86
else :
89
87
raise e
90
88
89
+
91
90
def get_response_elems (license ):
92
91
"""
93
92
Provides the metadata for query of specified parameters
@@ -123,17 +122,21 @@ def get_response_elems(license):
123
122
# If quota limit exceeded, switch to the next API key
124
123
global API_KEYS_IND
125
124
API_KEYS_IND += 1
126
- logging .error ("Changing API KEYS due to depletion of quota" )
125
+ print (
126
+ "Changing API KEYS due to depletion of quota" , file = sys .stderr
127
+ )
127
128
return get_response_elems (license )
128
129
else :
129
130
raise e
130
131
132
+
131
133
def set_up_data_file ():
132
134
# Writes the header row to the file to contain DeviantArt data.
133
135
header_title = "LICENSE TYPE,Document Count"
134
136
with open (DATA_WRITE_FILE , "w" ) as f :
135
137
f .write (f"{ header_title } \n " )
136
138
139
+
137
140
def record_license_data (license_type ):
138
141
"""Writes the row for LICENSE_TYPE to the file to contain DeviantArt data.
139
142
Args:
@@ -150,6 +153,7 @@ def record_license_data(license_type):
150
153
with open (DATA_WRITE_FILE , "a" ) as f :
151
154
f .write (f"{ data_log } \n " )
152
155
156
+
153
157
def record_all_licenses ():
154
158
"""
155
159
Records the data for all available license types listed in the license
@@ -161,19 +165,21 @@ def record_all_licenses():
161
165
for license_type in license_list :
162
166
record_license_data (license_type )
163
167
168
+
164
169
def main ():
165
170
set_up_data_file ()
166
171
record_all_licenses ()
167
172
173
+
168
174
if __name__ == "__main__" :
169
175
try :
170
176
main ()
171
177
except SystemExit as e :
172
- logging .error (f"System exit with code { e .code } " )
173
178
sys .exit (e .code )
174
179
except KeyboardInterrupt :
175
- logging . info ("INFO (130) Halted via KeyboardInterrupt." )
180
+ print ("INFO (130) Halted via KeyboardInterrupt." , file = sys . stderr )
176
181
sys .exit (130 )
177
182
except Exception :
178
- logging .exception ("ERROR (1) Unhandled exception:" )
179
- sys .exit (1 )
183
+ print ("ERROR (1) Unhandled exception:" , file = sys .stderr )
184
+ print (traceback .print_exc (), file = sys .stderr )
185
+ sys .exit (1 )
0 commit comments