eb_event_collector.py changing it to not getting all the pages, just the first one, to avoid rate limiting. We will not keep this code but I'm pushing it to production for now.

Author: Kimeiga

src/mappening/api/utils/eventbrite/eb_event_collector.py

@@ -8,6 +8,10 @@
 from tqdm import tqdm
 import json
 import requests
+from flask import jsonify
+import eventbrite
+
+# eventbrite = eventbrite.Eventbrite(EVENTBRITE_USER_KEY)
 
 from definitions import CENTER_LATITUDE, CENTER_LONGITUDE
 
@@ -45,48 +49,79 @@ def get_raw_events(days_back_in_time):
 
     session = requests.Session()
 
+    eventbrite2 = eventbrite.Eventbrite("GUD57Y44YXYVNP4VMQII")
+
     personal_token = EVENTBRITE_USER_KEY
     base_endpoint = 'https://www.eventbriteapi.com/v3'
     sample_headers = {
-        'Authorization': 'Bearer ' + personal_token
+        'Authorization': 'Bearer ' + personal_token,
     }
 
     # Most events on 1 page = 50, want more
-    page_num = 1
+    # page_num = 3
     request_new_results = True
 
     events_search_ep = '/events/search'
     search_args = {
-        'location.latitude': CENTER_LATITUDE,
-        'location.longitude': CENTER_LONGITUDE,
-        'location.within': '1mi',
-        'start_date.range_start': past_bound,
-        'start_date.range_end': future_bound,
-        'sort_by': 'best'
+        "location.latitude": CENTER_LATITUDE,
+        "location.longitude": CENTER_LONGITUDE,
+        "location.within": "1mi",
+        "start_date.range_start": past_bound,
+        "start_date.range_end": future_bound,
+        "sort_by": "best"
     }
 
+    # response = eventbrite2.event_search(**search_args)
+
+    # print(response)
+
     # Loop through returned pages of events until no more, or enough
-    all_events = []
-    while request_new_results and page_num <= 20:
-        # There's always a 1st page result that works
-        search_args['page'] = str(page_num)  
-        response = session.get(
-            base_endpoint + events_search_ep,
-            headers = sample_headers,
-            verify = True,  # Verify SSL certificate
-            params = search_args,
-        ).json()
+    # all_events = response['events']
+
+    # request_more_events = response['pagination']['has_more_items']
+
+    # while request_more_events:
+
+        #  search_args["page"]
+
+        # request_more_events = response['pagination']['has_more_items']
+
+    response = eventbrite2.event_search(**search_args)
+
+    print(response)
+
+    all_events = response.get('events')
+    # while request_new_results and page_num <= 20:
+
+    #     response = eventbrite2.event_search(**search_args)
+
+    #     print(response)
+ 
+    #     # There's always a 1st page result that works
+    #     search_args["page"] = page_num
+    #     print("search_args")
+    #     print(search_args)
+    #     # responseSession = session.get(
+    #     #     base_endpoint + events_search_ep,
+    #     #     headers = sample_headers,
+    #     #     verify = True,  # Verify SSL certificate
+    #     #     params = search_args,
+    #     # ).json()
+
+
+    #     # print(responseSession)
+    #     # print(responseSession.text)
 
-        # Extend, not append!
-        # combines elements of two lists as expected vs adds in the new list as ONE element
-        all_events.extend(response.get('events'))
-        if 'pagination' in response and response['pagination']['has_more_items']:
-            request_new_results = True
-            page_num += 1
-        else:
-            request_new_results = False
-
-    print('Finished collecting Eventbrite events!')
+    #     # Extend, not append!
+    #     # combines elements of two lists as expected vs adds in the new list as ONE element
+    #     all_events.extend(response.get('events'))
+    #     if 'pagination' in response and response['pagination']['has_more_items']:
+    #         request_new_results = True
+    #         page_num += 1
+    #     else:
+    #         request_new_results = False
+
+    # print('Finished collecting Eventbrite events!')
     return all_events
 
 # Database updating 
@@ -102,6 +137,8 @@ def update_database(all_events):
     events_eventbrite_collection.delete_many({})
     events_eventbrite_collection.insert_many(all_events)
 
+    print('inserted properly')
+
     all_cat_ep = '/categories'
     session = requests.Session()
     cat_resp = session.get(

src/mappening/utils/database.py

@@ -3,7 +3,7 @@
 from pymongo import MongoClient
 
 # Standard URI format: mongodb://[dbuser:dbpassword@]host:port/dbname
-events_uri = 'mongodb://{0}:{1}@{2}/events'.format(MLAB_USERNAME, MLAB_PASSWORD, MLAB_HOST)
+events_uri = 'mongodb://{0}:{1}@{2}/events?retryWrites=false'.format(MLAB_USERNAME, MLAB_PASSWORD, MLAB_HOST)
 locations_uri = 'mongodb://{0}:{1}@{2}/locations'.format(MLAB_USERNAME, MLAB_PASSWORD, MLAB_HOST)
 users_uri = 'mongodb://{0}:{1}@{2}/users'.format(MLAB_USERNAME, MLAB_PASSWORD, MLAB_HOST)
 tkinter_uri = 'mongodb://{0}:{1}@{2}/tkinter'.format(MLAB_USERNAME, MLAB_PASSWORD, MLAB_HOST)

src/requirements.txt

@@ -16,3 +16,4 @@ fuzzywuzzy
 unidecode
 psycopg2
 Flask-SQLAlchemy
+eventbrite