Merge pull request #3 from dana93os/master

Config requests pause

Author: vlad-stack

outscraper/api_client.py

@@ -22,12 +22,14 @@ class ApiClient(object):
     _api_headers = {}
 
     _max_ttl = 60 * 12
+    _requests_pause = 5
 
-    def __init__(self, api_key: str) -> None:
+    def __init__(self, api_key: str, requests_pause: int = 5) -> None:
         self._api_headers = {
             'X-API-KEY': api_key,
             'client': f'Python SDK {VERSION}'
         }
+        self._requests_pause = requests_pause
 
     def get_request_archive(self, request_id: str) -> dict:
         '''
@@ -46,17 +48,17 @@ def get_request_archive(self, request_id: str) -> dict:
 
         raise Exception(f'Response status code: {response.status_code}')
 
-    def _wait_request_archive(self, request_id: str, requests_pause: int = 5) -> dict:
-        ttl = self._max_ttl / requests_pause
+    def _wait_request_archive(self, request_id: str) -> dict:
+        ttl = self._max_ttl / self._requests_pause
 
         while ttl > 0:
             ttl -= 1
 
+            sleep(self._requests_pause)
+
             result = self.get_request_archive(request_id)
             if result['status'] != 'Pending': return result
 
-            sleep(requests_pause)
-
         raise Exception('Timeout exceeded')
 
     def google_search(self, query: list, language: str = 'en', region: str = None) -> dict:
@@ -78,7 +80,6 @@ def google_search(self, query: list, language: str = 'en', region: str = None) -
         }, headers=self._api_headers)
 
         if 199 < response.status_code < 300:
-            sleep(10)
             return self._wait_request_archive(response.json()['id'])
 
         raise Exception(f'Response status code: {response.status_code}')
@@ -112,7 +113,6 @@ def google_maps_search(self, query: list, limit: int = 400, extract_contacts: bo
         }, headers=self._api_headers)
 
         if 199 < response.status_code < 300:
-            sleep(15)
             return self._wait_request_archive(response.json()['id']).get('data', [])
 
         raise Exception(f'Response status code: {response.status_code}')
@@ -157,7 +157,6 @@ def google_maps_reviews(self, query: list, reviewsLimit: int = 100, limit: int =
         }, headers=self._api_headers)
 
         if 199 < response.status_code < 300:
-            sleep(15)
             return self._wait_request_archive(response.json()['id']).get('data', [])
 
         raise Exception(f'Response status code: {response.status_code}')
@@ -188,7 +187,6 @@ def google_maps_photos(self, query: list, photosLimit: int = 100, limit: int = 1
         }, headers=self._api_headers)
 
         if 199 < response.status_code < 300:
-            sleep(15)
             return self._wait_request_archive(response.json()['id']).get('data', [])
 
         raise Exception(f'Response status code: {response.status_code}')
@@ -223,7 +221,6 @@ def google_play_reviews(self, query: list, reviewsLimit: int = 100, sort: str =
         }, headers=self._api_headers)
 
         if 199 < response.status_code < 300:
-            sleep(15)
             return self._wait_request_archive(response.json()['id']).get('data', [])
 
         raise Exception(f'Response status code: {response.status_code}')