added support for new filters

himanshu · himanshu · commit 94b4ed6db592 · 2023-08-25T11:06:09.000+05:30
diff --git a/README.md b/README.md
@@ -25,7 +25,157 @@ pip install newsdataapi
 
 ## Documentation
 
-Newsdataapi docs can be seen [here](https://newsdata.io/docs).
+Newsdataapi docs can be seen [here](https://newsdata.io/documentation).
+
+<br />
+
+### Latest News API
+
+`GET /1/news`
+
+```
+# To get latest news use our news_api method.
+
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey='YOUR_API_KEY')
+response = api.news_api(q='entertainment')
+print(response)
+
+# Latest news with page parameter
+
+response = api.news_api(q='entertainment',page='nextPage_value')
+print(response)
+
+# To scroll through all latest news
+
+response = api.news_api(q='entertainment',page='nextPage_value',scroll=True)
+print(response)
+
+```
+<br />
+
+### News Archive API
+
+`GET /1/archive`
+
+```
+# To get archive news use our archive_api method.
+
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey='YOUR_API_KEY')
+response = api.archive_api(q='olympic',from_date='2021-01-01',to_date='2021-06-06')
+print(response)
+
+# Archive news with page parameter
+
+response = api.archive_api(q='olympic',from_date='2021-01-01',to_date='2021-06-06',page='nextPage_value')
+print(response)
+
+# To scroll through all archive news
+
+response = api.archive_api(q='olympic',from_date='2021-01-01',to_date='2021-06-06',page='nextPage_value',scroll=True)
+print(response)
+
+```
+<br />
+
+
+### News Sources API
+
+`GET /1/sources`
+
+```
+# To get sources use our sources_api method.
+
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey="YOUR_API_KEY")
+response = api.sources_api()
+print(response)
+
+```
+<br />
+
+### Crypto News API
+
+`GET /1/crypto`
+
+```
+# To get crypto news use our crypto_api method.
+
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey='YOUR_API_KEY')
+response = api.crypto_api(q='bitcoin')
+print(response)
+
+# Crypto with page parameter
+
+response = api.crypto_api(q='bitcoin',page='nextPage_value')
+print(response)
+
+# To scroll through all crypto news
+
+response = api.crypto_api(q='bitcoin',page='nextPage_value',scroll=True)
+print(response)
+
+```
+<br />
+
+### News API with Pagination
+
+`GET /1/news`
+
+```
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey="YOUR_API_KEY")
+response = api.news_api()
+
+# You can go to next page by providing Page parameter
+
+response = api.news_api(page = "nextPage value")
+
+# You can paginate till last page by providing Page parameter in Loop
+
+page=None
+while True:
+    response = api.news_api(page = page)
+    page = response.get('nextPage',None)
+    if not page:
+        break
+
+```
+
+<br />
+
+### News API with Scrolling
+
+```
+# Note: Scrolling through all result will counsume api as per your defined size. you can also define max_result to stop scrolling at desired result size.scroll is avaliable in news_archive,news_api and news_crypto, it will return all result when scrolling is compleated.
+
+from newsdataapi import NewsDataApiClient
+
+# API key authorization, Initialize the client with your API key
+
+api = NewsDataApiClient(apikey="YOUR_API_KEY")
+
+response = api.news_api(q='entertainment',page='nextPage_value',scroll=True,max_result=1000)
+print(response)
+
+```
 
 <br />
 
@@ -40,4 +190,4 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-```
+```
diff --git a/newsdataapi/constants.py b/newsdataapi/constants.py
@@ -16,4 +16,6 @@
 # Default request values
 DEFAULT_REQUEST_TIMEOUT = 300
 DEFAULT_MAX_RETRIES = 5
-DEFAULT_RETRY_DELAY = 30
+DEFAULT_RETRY_DELAY = 1800
+DEFAULT_RETRY_DELAY_TooManyRequests = 10
+DEFAULT_RETRY_DELAY_RateLimitExceeded = 900
diff --git a/newsdataapi/newsdataapi_client.py b/newsdataapi/newsdataapi_client.py
@@ -1,4 +1,5 @@
 import requests,time
+from datetime import datetime
 from newsdataapi import constants
 from typing import Optional,Union
 from urllib.parse import urlencode, quote
@@ -8,7 +9,7 @@ class NewsDataApiClient:
 
     def __init__(
             self, apikey:str, session:bool= False, max_retries:int= constants.DEFAULT_MAX_RETRIES, retry_delay:int= constants.DEFAULT_RETRY_DELAY,
-            proxies:Optional[dict]=None, request_timeout:int= constants.DEFAULT_REQUEST_TIMEOUT,max_result:int=10**10
+            proxies:Optional[dict]=None, request_timeout:int= constants.DEFAULT_REQUEST_TIMEOUT,max_result:int=10**10, debug:Optional[bool]=False
         ):
         """Initializes newsdata client object for access Newsdata APIs."""
         self.apikey = apikey
@@ -19,6 +20,7 @@ def __init__(
         self.proxies = proxies
         self.request_timeout = request_timeout
         self.recursive_retry = max_retries
+        self.is_debug = debug
 
     def set_retries( self, max_retries:int, retry_delay:int)->None:
         """ API maximum retry and delay"""
@@ -36,7 +38,7 @@ def api_proxies( self, proxies:dict)->None:
     def __validate_parms(self,param:str,value:Union[list,int,str,bool])->dict:
         bool_params = {'full_content','image','video'}
         int_params = {'size','prioritydomain','timeframe'}
-        string_params = {'q','qInTitle','country','category','language','domain','domainurl','excludedomain','timezone','page','from_date','to_date','apikey'}
+        string_params = {'q','qInTitle','country','category','language','domain','domainurl','excludedomain','timezone','page','from_date','to_date','apikey','qInMeta'}
 
         if param in string_params:
             if isinstance(value,list):
@@ -58,18 +60,35 @@ def __get_feeds(self,url:str)-> dict:
             if self.recursive_retry <= 0:
                 raise  NewsdataException('maximum retry limit reached.')
             response = self.request_method.get(url=url,proxies=self.proxies,timeout=self.request_timeout)
+            if self.is_debug == True:
+                headers = response.headers
+                print(f'Debug | {datetime.utcnow().replace(microsecond=0)} | x_rate_limit_remaining: {headers.get("x_rate_limit_remaining")} | x_api_limit_remaining: {headers.get("x_api_limit_remaining")}')
             feeds_data:dict = response.json()
             if response.status_code != 200:
-                if response.status_code == 500 or feeds_data.get('results',{}).get('code') == 'TooManyRequests' or feeds_data.get('results',{}).get('code') == 'RateLimitExceeded':
+                if response.status_code == 500:
+                    if self.is_debug == True:
+                        print(f"Debug | {datetime.utcnow().replace(microsecond=0)} | Encountered 'ServerError' going to sleep for: {self.retry_delay} seconds.")
                     time.sleep(self.retry_delay)
                     self.recursive_retry-=1
                     return self.__get_feeds(url=url)
-                raise NewsdataException(response.json())
+                elif feeds_data.get('results',{}).get('code') == 'TooManyRequests':
+                    if self.is_debug == True:
+                        print(f"Debug | {datetime.utcnow().replace(microsecond=0)} | Encountered 'TooManyRequests' going to sleep for: {constants.DEFAULT_RETRY_DELAY_TooManyRequests} seconds.")
+                    time.sleep(constants.DEFAULT_RETRY_DELAY_TooManyRequests)
+                    self.recursive_retry-=1
+                    return self.__get_feeds(url=url)
+                elif feeds_data.get('results',{}).get('code') == 'RateLimitExceeded':
+                    if self.is_debug == True:
+                        print(f"Debug | {datetime.utcnow().replace(microsecond=0)} | Encountered 'RateLimitExceeded' going to sleep for: {constants.DEFAULT_RETRY_DELAY_RateLimitExceeded} seconds.")
+                    time.sleep(constants.DEFAULT_RETRY_DELAY_RateLimitExceeded)
+                    self.recursive_retry-=1
+                    return self.__get_feeds(url=url)
+                else:
+                    raise NewsdataException(response.json())
             else:
                 self.recursive_retry = self.max_retries
                 return feeds_data
         except requests.exceptions.ConnectionError:
-            time.sleep(self.retry_delay)
             if isinstance(self.request_method,requests.Session):
                 self.request_method = requests.Session()
             self.recursive_retry-=1
@@ -88,12 +107,11 @@ def __get_feeds_all(self,url:str)-> dict:
             response = self.__get_feeds(url=f'{url}&page={data.get("nextPage")}' if data.get('results') else url)
             data['totalResults'] = response.get('totalResults')
             results = response.get('results')
-            for feed in results:
-                data['results'].append(feed)
-                feeds_count+=1
-                if feeds_count >= self.max_result:
-                    return data
+            data['results'].extend(results)
             data['nextPage'] = response.get('nextPage')
+            feeds_count+=len(results)
+            if feeds_count >= self.max_result:
+                return data
             time.sleep(0.5)
         return data
 
@@ -102,7 +120,7 @@ def news_api(
             language:Optional[Union[str, list]]=None, domain:Optional[Union[str, list]]=None, timeframe:Optional[int]=None, size:Optional[int]=None,
             domainurl:Optional[Union[str, list]]=None, excludedomain:Optional[Union[str, list]]=None, timezone:Optional[str]=None, full_content:Optional[bool]=None,
             image:Optional[bool]=None, video:Optional[bool]=None, prioritydomain:Optional[int]=None, page:Optional[str]=None, scroll:Optional[bool]=False,
-            max_result:Optional[int]=None
+            max_result:Optional[int]=None, qInMeta:Optional[str]=None
         )->dict:
         """ 
         Sending GET request to the news api.
@@ -111,7 +129,7 @@ def news_api(
         params = {
             'apikey':self.apikey,'q':q,'qInTitle':qInTitle,'country':country,'category':category,'language':language,'domain':domain,'timeframe':timeframe,'size':size,
             'domainurl':domainurl,'excludedomain':excludedomain,'timezone':timezone,'full_content':full_content,'image':image,'video':video,'prioritydomain':prioritydomain,
-            'page':page,
+            'page':page,'qInMeta':qInMeta
         }
 
         URL_parameters = {}
@@ -132,7 +150,7 @@ def archive_api(
             language:Optional[Union[str, list]]=None, domain:Optional[Union[str, list]]=None, size:Optional[int]=None,domainurl:Optional[Union[str, list]]=None,
             excludedomain:Optional[Union[str, list]]=None, timezone:Optional[str]=None, full_content:Optional[bool]=None,image:Optional[bool]=None,
             video:Optional[bool]=None,prioritydomain:Optional[int]=None, page:Optional[str]=None, scroll:Optional[bool]=False, max_result:Optional[int]=None,
-            from_date:Optional[str]=None, to_date:Optional[str]=None
+            from_date:Optional[str]=None, to_date:Optional[str]=None, qInMeta:Optional[str]=None
     ) -> dict:
         """
         Sending GET request to the archive api
@@ -141,7 +159,7 @@ def archive_api(
         params = {
             'q':q,'qInTitle':qInTitle,'country':country,'category':category,'language':language,'domain':domain,'size':size,'domainurl':domainurl,'excludedomain':excludedomain,
             'timezone':timezone,'full_content':full_content,'image':image,'video':video,'prioritydomain':prioritydomain,'page':page,'from_date':from_date,'to_date':to_date,
-            'apikey':self.apikey
+            'apikey':self.apikey,'qInMeta':qInMeta
         }
         URL_parameters = {}
         for key,value in params.items():
@@ -177,7 +195,7 @@ def crypto_api(
             language:Optional[Union[str, list]]=None, domain:Optional[Union[str, list]]=None, timeframe:Optional[int]=None, size:Optional[int]=None,
             domainurl:Optional[Union[str, list]]=None, excludedomain:Optional[Union[str, list]]=None, timezone:Optional[str]=None, full_content:Optional[bool]=None,
             image:Optional[bool]=None, video:Optional[bool]=None, prioritydomain:Optional[int]=None, page:Optional[str]=None, scroll:Optional[bool]=False,
-            max_result:Optional[int]=None
+            max_result:Optional[int]=None, qInMeta:Optional[str]=None
         )->dict:
         """ 
         Sending GET request to the crypto api
@@ -186,7 +204,8 @@ def crypto_api(
 
         params = {
             'apikey':self.apikey,'q':q,'qInTitle':qInTitle,'country':country,'category':category,'language':language,'domain':domain,'size':size,'domainurl':domainurl,
-            'excludedomain':excludedomain,'timezone':timezone,'full_content':full_content,'image':image,'video':video,'prioritydomain':prioritydomain,'page':page,'timeframe':timeframe
+            'excludedomain':excludedomain,'timezone':timezone,'full_content':full_content,'image':image,'video':video,'prioritydomain':prioritydomain,'page':page,
+            'timeframe':timeframe,'qInMeta':qInMeta
         }
 
         URL_parameters = {}
