Migrate to Python 3

analytics/apps.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 
 from django.apps import AppConfig
 

analytics/request_counter.py

@@ -8,6 +8,7 @@
 """
 
 from .models import *
+from django.utils.deprecation import MiddlewareMixin
 
 EXCLUDE_PATH_PREFIXES = [
     "/favicon.ico",
@@ -24,7 +25,7 @@
     "spider"
 ]
 
-class RequestCounterMiddleware(object):
+class RequestCounterMiddleware(MiddlewareMixin):
     """A middleware that saves a RequestCount object each time a page is requested."""
 
     def process_response(self, request, response):
@@ -42,7 +43,7 @@ def process_response(self, request, response):
         if len(user_agent) > 150:
             user_agent = user_agent[:150]
         tally.user_agent = user_agent
-        if hasattr(request, "user") and request.user and request.user.is_authenticated():
+        if hasattr(request, "user") and request.user and request.user.is_authenticated:
             tally.is_authenticated = True
             try:
                 student = request.user.student

analytics/urls.py

@@ -1,13 +1,13 @@
-from django.conf.urls import url
+from django.urls import re_path
 
 from . import views
 
 urlpatterns = [
-    url(r'total_requests/(?P<time_frame>[a-z-]*)', views.total_requests, name="total_requests"),
-    url(r'user_agents/(?P<time_frame>[a-z-]*)', views.user_agents, name="user_agents"),
-    url(r'logged_in_users/(?P<time_frame>[a-z-]*)', views.logged_in_users, name="logged_in_users"),
-    url(r'user_semesters/(?P<time_frame>[a-z-]*)', views.user_semesters, name="user_semesters"),
-    url(r'request_paths/(?P<time_frame>[a-z-]*)', views.request_paths, name="request_paths"),
-    url(r'active_documents/(?P<time_frame>[a-z-]*)', views.active_documents, name="active_documents"),
-    url(r'^$', views.dashboard, name='analytics_dashboard'),
+    re_path(r'total_requests/(?P<time_frame>[a-z-]*)', views.total_requests, name="total_requests"),
+    re_path(r'user_agents/(?P<time_frame>[a-z-]*)', views.user_agents, name="user_agents"),
+    re_path(r'logged_in_users/(?P<time_frame>[a-z-]*)', views.logged_in_users, name="logged_in_users"),
+    re_path(r'user_semesters/(?P<time_frame>[a-z-]*)', views.user_semesters, name="user_semesters"),
+    re_path(r'request_paths/(?P<time_frame>[a-z-]*)', views.request_paths, name="request_paths"),
+    re_path(r'active_documents/(?P<time_frame>[a-z-]*)', views.active_documents, name="active_documents"),
+    re_path(r'^$', views.dashboard, name='analytics_dashboard'),
 ]

analytics/views.py

@@ -83,7 +83,7 @@ def total_requests(request, time_frame=None):
     timezone.activate(DISPLAY_TIME_ZONE)
     early_time, delta, format = get_time_bounds(time_frame)
     data = RequestCount.tabulate_requests(early_time, delta, lambda _: 1)
-    labels, counts = itertools.izip(*((format_date(t, format), item.get(1, 0)) for t, item in data))
+    labels, counts = list(zip(*((format_date(t, format), item.get(1, 0)) for t, item in data)))
     return HttpResponse(json.dumps({"labels": labels, "data": counts, "total": "{:,}".format(sum(counts))}), content_type="application/json")
 
 USER_AGENT_TYPES = [
@@ -127,7 +127,7 @@ def logged_in_users(request, time_frame=None):
     early_time, delta, format = get_time_bounds(time_frame)
     data = RequestCount.tabulate_requests(early_time, delta, lambda _: 1, distinct_users=True)
     total_data = RequestCount.tabulate_requests(early_time, None, lambda _: 1, distinct_users=True)
-    labels, counts = itertools.izip(*((format_date(t, format), item.get(1, 0)) for t, item in data))
+    labels, counts = list(zip(*((format_date(t, format), item.get(1, 0)) for t, item in data)))
     return HttpResponse(json.dumps({"labels": labels, "data": counts, "total": "{:,}".format(total_data.get(1, 0))}), content_type="application/json")
 
 SEMESTERS = [
@@ -170,7 +170,7 @@ def user_semesters(request, time_frame=None):
     labels = SEMESTERS
 
     semester_buckets = [0 for _ in SEMESTERS]
-    for semester, count in data.items():
+    for semester, count in list(data.items()):
         if not semester or semester < 0 or semester >= len(semester_buckets):
             continue
         semester_buckets[semester] += count
@@ -184,7 +184,7 @@ def request_paths(request, time_frame=None):
     data = RequestCount.tabulate_requests(early_time, None, lambda request: request.path)
     labels = set(data.keys()) - set([None])
     counts = {label: data.get(label, 0) for label in labels}
-    labels, counts = itertools.izip(*sorted(counts.items(), key=lambda x: x[1], reverse=True))
+    labels, counts = list(zip(*sorted(list(counts.items()), key=lambda x: x[1], reverse=True)))
     if len(labels) > 15:
         labels = labels[:15]
         counts = counts[:15]

catalog/apps.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 
 from django.apps import AppConfig
 

catalog/models.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+
 from django.core.exceptions import ObjectDoesNotExist
 
 from django.db import models
@@ -33,7 +33,7 @@ def combine(cls, attrs, unique_id):
         are combined together to form a course that satisfies each attribute's
         requirement, and a unique id is assigned to the course only if it makes
         sense for each attribute to exist multiple times in a list of courses"""
-        new_attr = cls(" ".join(map(lambda a: a.requirement,attrs)),True)
+        new_attr = cls(" ".join([a.requirement for a in attrs]),True)
         for attr in attrs:
             new_attr.course = attr.modify_course(new_attr.course)
             new_attr.needs_unique_id = new_attr.needs_unique_id and attr.needs_unique_id
@@ -203,7 +203,7 @@ def list_converter(value):
     "Custom Color":                             (CourseFields.custom_color, string_converter)
 }
 
-FIELD_TO_CSV = {field_name: csv_header for csv_header, (field_name, _) in CSV_HEADERS.items()}
+FIELD_TO_CSV = {field_name: csv_header for csv_header, (field_name, _) in list(CSV_HEADERS.items())}
 
 # Create your models here.
 class Course(models.Model):

catalog/tests.py

@@ -88,23 +88,23 @@ def test_json_object(self):
         course = Course.objects.get(subject_id="21M.030")
         result = course.to_json_object(full=False)
         expected = {
-            CourseFields.subject_id:            u"21M.030",
-            CourseFields.title:                 u"World Music",
+            CourseFields.subject_id:            "21M.030",
+            CourseFields.title:                 "World Music",
             CourseFields.total_units:           12,
             CourseFields.offered_fall:          True,
             CourseFields.offered_IAP:           False,
             CourseFields.offered_spring:        True,
             CourseFields.offered_summer:        False,
             CourseFields.public:                True,
-            CourseFields.level:                 u"U",
-            CourseFields.joint_subjects:        [u"21M.830", u"21M.290"],
-            CourseFields.equivalent_subjects:   [u"21M.031"],
-            CourseFields.quarter_information:   u"1,march 1",
-            CourseFields.not_offered_year:      u"2019-2020",
-            CourseFields.instructors:           [u"E. Zimmer", u"C. Smith"],
-            CourseFields.communication_requirement: u"CI-H",
-            CourseFields.hass_attribute:        u"HASS-A",
-            CourseFields.gir_attribute:         u"REST",
+            CourseFields.level:                 "U",
+            CourseFields.joint_subjects:        ["21M.830", "21M.290"],
+            CourseFields.equivalent_subjects:   ["21M.031"],
+            CourseFields.quarter_information:   "1,march 1",
+            CourseFields.not_offered_year:      "2019-2020",
+            CourseFields.instructors:           ["E. Zimmer", "C. Smith"],
+            CourseFields.communication_requirement: "CI-H",
+            CourseFields.hass_attribute:        "HASS-A",
+            CourseFields.gir_attribute:         "REST",
         }
         self.maxDiff = None
         self.assertDictEqual(expected, result)
@@ -113,23 +113,23 @@ def test_json_object_full(self):
         course = Course.objects.get(subject_id="21M.030")
         result = course.to_json_object()
         expected = {
-            CourseFields.subject_id:            u"21M.030",
-            CourseFields.title:                 u"World Music",
+            CourseFields.subject_id:            "21M.030",
+            CourseFields.title:                 "World Music",
             CourseFields.total_units:           12,
             CourseFields.offered_fall:          True,
             CourseFields.offered_IAP:           False,
             CourseFields.offered_spring:        True,
             CourseFields.offered_summer:        False,
             CourseFields.public:                True,
-            CourseFields.level:                 u"U",
-            CourseFields.joint_subjects:        [u"21M.830", u"21M.290"],
-            CourseFields.equivalent_subjects:   [u"21M.031"],
-            CourseFields.quarter_information:   u"1,march 1",
-            CourseFields.not_offered_year:      u"2019-2020",
-            CourseFields.instructors:           [u"E. Zimmer", u"C. Smith"],
-            CourseFields.communication_requirement: u"CI-H",
-            CourseFields.hass_attribute:        u"HASS-A",
-            CourseFields.gir_attribute:         u"REST",
+            CourseFields.level:                 "U",
+            CourseFields.joint_subjects:        ["21M.830", "21M.290"],
+            CourseFields.equivalent_subjects:   ["21M.031"],
+            CourseFields.quarter_information:   "1,march 1",
+            CourseFields.not_offered_year:      "2019-2020",
+            CourseFields.instructors:           ["E. Zimmer", "C. Smith"],
+            CourseFields.communication_requirement: "CI-H",
+            CourseFields.hass_attribute:        "HASS-A",
+            CourseFields.gir_attribute:         "REST",
             CourseFields.lecture_units:         5,
             CourseFields.lab_units:             0,
             CourseFields.design_units:          0,
@@ -138,11 +138,11 @@ def test_json_object_full(self):
             CourseFields.is_half_class:         False,
             CourseFields.pdf_option:            False,
             CourseFields.has_final:             False,
-            CourseFields.description: u"Test description of 21M.030",
-            CourseFields.prerequisites: u"21M.051/''permission of instructor''",
-            CourseFields.schedule: u"Lecture,4-364/MW/0/9.30-11,4-364/MW/0/11-12.30",
-            CourseFields.url: u"http://student.mit.edu/catalog/m21Ma.html#21M.030",
-            CourseFields.related_subjects: [u"21M.011", u"21M.031"],
+            CourseFields.description: "Test description of 21M.030",
+            CourseFields.prerequisites: "21M.051/''permission of instructor''",
+            CourseFields.schedule: "Lecture,4-364/MW/0/9.30-11,4-364/MW/0/11-12.30",
+            CourseFields.url: "http://student.mit.edu/catalog/m21Ma.html#21M.030",
+            CourseFields.related_subjects: ["21M.011", "21M.031"],
             CourseFields.rating:                5.0,
             CourseFields.enrollment_number:     45.0,
             CourseFields.in_class_hours:        3.0,
@@ -194,8 +194,8 @@ def test_lookup_subject(self):
         response = views.lookup(request, subject_id="2.001")
         self.assertEqual(200, response.status_code)
         self.assertDictContainsSubset({
-            CourseFields.subject_id: u"2.001",
-            CourseFields.title: u"Foo"
+            CourseFields.subject_id: "2.001",
+            CourseFields.title: "Foo"
         }, json.loads(response.content))
 
     def test_lookup_subject_not_existing(self):

catalog/urls.py

@@ -1,10 +1,10 @@
-from django.conf.urls import url
+from django.urls import re_path
 
 from . import views
 
 urlpatterns = [
-    url(r'lookup/(?P<subject_id>[A-z0-9.]+)', views.lookup, name='lookup'),
-    url(r'search/(?P<search_term>[^?]+)', views.search, name='search'),
-    url(r'dept/(?P<dept>[A-z0-9.]+)', views.department, name='department'),
-    url(r'all', views.list_all, name='list_all')
+    re_path(r'lookup/(?P<subject_id>[A-z0-9.]+)', views.lookup, name='lookup'),
+    re_path(r'search/(?P<search_term>[^?]+)', views.search, name='search'),
+    re_path(r'dept/(?P<dept>[A-z0-9.]+)', views.department, name='department'),
+    re_path(r'all', views.list_all, name='list_all')
 ]

catalog_parse/catalog_parser.py

@@ -31,9 +31,9 @@
 
 # For type checking str or unicode in Python 2 and 3
 try:
-    basestring
+    str
 except NameError:
-    basestring = str
+    str = str
 
 COURSE_NUMBERS = [
     "1", "2", "3", "4",
@@ -83,7 +83,7 @@ def load_course_elements(url):
     course_ids = []
     courses = []
     for element in course_elements[0].getchildren():
-        if element.tag == "a" and "name" in element.keys():
+        if element.tag == "a" and "name" in list(element.keys()):
             subject_id = element.get("name")
             course_ids.append(subject_id)
             courses.append([])
@@ -105,7 +105,7 @@ def load_course_elements(url):
 
 def get_inner_html(node):
     """Gets the inner HTML of a node, including tags."""
-    children = ''.join(etree.tostring(e).decode('utf-8') for e in node)
+    children = ''.join(etree.tostring(e, encoding="unicode") for e in node)
     if node.text is None:
         return children
     return node.text + children
@@ -119,10 +119,10 @@ def recursively_extract_info(node):
     contents = get_inner_html(node)
 
     if node.tag == "img":
-        if "title" in node.keys() and len(node.get("title")):
+        if "title" in list(node.keys()) and len(node.get("title")):
             info_items.append(node.get("title"))
     elif node.tag == "a":
-        if "name" in node.keys():
+        if "name" in list(node.keys()):
             return (info_items, True)
         text = node.text_content().strip()
         if len(text):
@@ -410,7 +410,7 @@ def merge_duplicates(courses):
 
         if len(course_dict[subject_id]) > 1:
             total_course = {}
-            keys = set().union(*(other.keys() for other in course_dict[subject_id]))
+            keys = set().union(*(list(other.keys()) for other in course_dict[subject_id]))
             for key in keys:
                 vals = [other.get(key, '') for other in course_dict[subject_id]]
 
@@ -478,7 +478,7 @@ def courses_from_dept_code(dept_code, **options):
                 schedules = {other_id: schedules[""] for other_id in subject_ids}
 
             for other_id in subject_ids:
-                copied_course = {key: val for key, val in attribs.items()}
+                copied_course = {key: val for key, val in list(attribs.items())}
                 copied_course[CourseAttribute.subjectID] = other_id
                 if other_id in schedules:
                     copied_course[CourseAttribute.schedule] = schedules[other_id]
@@ -511,7 +511,7 @@ def writing_description_for_attribute(course, attribute):
         return ""
 
     item = course[attribute]
-    if isinstance(item, basestring):
+    if isinstance(item, str):
         return '"' + item.replace('"', "'").replace('\n', '\\n') + '"'
     elif isinstance(item, bool):
         return "Y" if item == True else "N"
@@ -524,7 +524,7 @@ def writing_description_for_attribute(course, attribute):
     elif isinstance(item, list):
         return '"' + ",".join(item) + '"'
     else:
-        print("Don't have a way to represent attribute {}: {} ({})".format(attribute, item, type(item)))
+        print(("Don't have a way to represent attribute {}: {} ({})".format(attribute, item, type(item))))
         return str(item)
 
 def write_courses(courses, filepath, attributes):
@@ -536,10 +536,7 @@ def write_courses(courses, filepath, attributes):
         csv_comps.append([writing_description_for_attribute(course, attrib) for attrib in attributes])
 
     with open(filepath, 'w') as file:
-        if sys.version_info > (3, 0):
-            file.write("\n".join(",".join(item) for item in csv_comps))
-        else:
-            file.write("\n".join(",".join(item) for item in csv_comps).encode('utf-8'))
+        file.write("\n".join(",".join(item) for item in csv_comps))
 
 ### Main method
 
@@ -583,7 +580,7 @@ def parse(output_dir, equivalences_path=None, write_related=True,
             if len(addl_courses) == 0:
                 continue
 
-            print("======", total_code)
+            print(("======", total_code))
             dept_courses += addl_courses
             if original_html is None:
                 original_html = LAST_PAGE_HTML

catalog_parse/consensus_catalog.py

@@ -34,7 +34,7 @@ def make_corrections(corrections, consensus):
                     if correction[col]:
                         if col not in consensus.columns:
                             consensus[col] = ""
-                        print("Correction for {}: {} ==> {}".format(idx, col, correction[col]))
+                        print(("Correction for {}: {} ==> {}".format(idx, col, correction[col])))
                         consensus.ix[idx][col] = correction[col]
 
         elif subject_id in consensus.index:
@@ -43,12 +43,12 @@ def make_corrections(corrections, consensus):
             for col in correction:
                 if col == "Subject Id": continue
                 if correction[col]:
-                    print("Correction for {}: {} ==> {}".format(subject_id, col, correction[col]))
+                    print(("Correction for {}: {} ==> {}".format(subject_id, col, correction[col])))
                     consensus_row[col] = correction[col]
 
         else:
             # Add the subject
-            print("Correction: adding subject {}".format(subject_id))
+            print(("Correction: adding subject {}".format(subject_id)))
             consensus.loc[subject_id] = {col: correction.get(col, None) for col in consensus.columns}
 
 
@@ -70,7 +70,7 @@ def build_consensus(base_path, out_path, corrections=None,
         semester_data[semester] = all_courses
 
     # Sort in reverse chronological order
-    semester_data = sorted(semester_data.items(), key=lambda x: semester_sort_key(x[0]), reverse=True)
+    semester_data = sorted(list(semester_data.items()), key=lambda x: semester_sort_key(x[0]), reverse=True)
     if len(semester_data) == 0:
         print("No raw semester data found.")
         return
@@ -109,7 +109,7 @@ def build_consensus(base_path, out_path, corrections=None,
             consensus = pd.concat([consensus, data], sort=False)
 
         consensus = consensus.drop_duplicates(subset=[CourseAttribute.subjectID], keep='first')
-        print("Added {} courses with {}.".format(len(consensus) - last_size, semester))
+        print(("Added {} courses with {}.".format(len(consensus) - last_size, semester)))
         last_size = len(consensus)
 
     consensus.set_index(CourseAttribute.subjectID, inplace=True)
@@ -169,7 +169,7 @@ def write_df(df, path):
         eval_path = None
 
     if os.path.exists(out_path):
-        print("Fatal: the directory {} already exists. Please delete it or choose a different location.".format(out_path))
+        print(("Fatal: the directory {} already exists. Please delete it or choose a different location.".format(out_path)))
         exit(1)
 
     build_consensus(in_path, out_path, evaluations_path=eval_path)