From 0c67029dfddba2ed24fd3284da1e8414dc808182 Mon Sep 17 00:00:00 2001 From: oscatcher Date: Mon, 16 Mar 2026 11:48:44 +0100 Subject: [PATCH 1/2] chore: remove README and ci.yml from .fernignore --- .fernignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.fernignore b/.fernignore index d75d43d..cc7e199 100644 --- a/.fernignore +++ b/.fernignore @@ -1,6 +1,6 @@ LICENSE .gitignore -README.md +# README.md # Ignore custom SDK code src/newscatcher/client.py @@ -15,4 +15,4 @@ test-mock-mode.sh mypy.ini # CI/CD -.github/workflows/ci.yml \ No newline at end of file +# .github/workflows/ci.yml \ No newline at end of file From 8050d3067a1ed30ee67e697338cd59f67df72897 Mon Sep 17 00:00:00 2001 From: oscatcher Date: Mon, 16 Mar 2026 11:57:15 +0100 Subject: [PATCH 2/2] fix: remove stale Articles import incompatible with new generated types --- src/newscatcher/client.py | 493 ++++++++++++++++++++++++-------------- 1 file changed, 312 insertions(+), 181 deletions(-) diff --git a/src/newscatcher/client.py b/src/newscatcher/client.py index 1132863..f631102 100644 --- a/src/newscatcher/client.py +++ b/src/newscatcher/client.py @@ -6,7 +6,6 @@ from typing import Optional, Union, List, Set, Tuple, Any from .base_client import BaseNewscatcherApi, AsyncBaseNewscatcherApi -from .types.articles import Articles from .utils import ( parse_time_parameters, create_time_chunks, @@ -23,8 +22,18 @@ class QueryValidator: def __init__(self): """Initialize validator with validation rules.""" self.not_allowed_characters = [ - "[", "]", "/", "\\", "%5B", "%5D", "%2F", "%5C", - ":", "%3A", "^", "%5E" + "[", + "]", + "/", + "\\", + "%5B", + "%5D", + "%2F", + "%5C", + ":", + "%3A", + "^", + "%5E", ] self.open_char = ["(", "%28"] self.close_char = [")", "%29"] @@ -54,7 +63,7 @@ def validate_query(self, query: str) -> Tuple[bool, str]: result, message = self._check_middle(query, "q") if not result: return False, message - + result, message = self._check_same_level_operators(query, "q") if not result: return False, message @@ -65,16 +74,18 @@ def validate_query(self, query: str) -> Tuple[bool, str]: return True, "" - def _check_allowed_characters(self, query: str, variable_name: str) -> Tuple[bool, str]: + def _check_allowed_characters( + self, query: str, variable_name: str + ) -> Tuple[bool, str]: """Check the query for any unallowed characters.""" # Special handling: allow \" for exact phrase escaping - temp_query = query.replace('\\\"', '___ESCAPED_QUOTE___') - + temp_query = query.replace('\\"', "___ESCAPED_QUOTE___") + if any(ext in temp_query for ext in self.not_allowed_characters): return ( False, f"[{variable_name}] parameter must not include following characters " - f"{str(self.not_allowed_characters)}. Please remove them from [{variable_name}] parameter" + f"{str(self.not_allowed_characters)}. Please remove them from [{variable_name}] parameter", ) return True, "" @@ -82,14 +93,14 @@ def _check_asterisk(self, query: str, variable_name: str) -> Tuple[bool, str]: """Check the query for proper asterisk usage.""" if query == "*": return True, "" - + # Use original regex pattern but fix the multiple asterisk check matches = re.search(r"^[\*]{2,}$|[\s]\*|^\*[^\s]", query) if matches: return ( False, f"The wildcard (*) character in [{variable_name}] parameter must be preceded " - f"by at least one alphabet or number. Please modify the query." + f"by at least one alphabet or number. Please modify the query.", ) return True, "" @@ -97,24 +108,87 @@ def _check_start_end(self, query: str, variable_name: str) -> Tuple[bool, str]: """Check for invalid operators at query boundaries.""" # Operators that cannot appear at the end - include missing ones end_operators = [ - "OR ", "%7C%7C", "%7C%7C ", "AND ", "%26%26", "%26%26 ", - "&&", "&& ", "||", "|| ", "NOT", "NOT ", "%21", "%21 ", - "!", "! ", "%2B", "%2B ", "-", "- ", "OR(", "OR (", - "%7C%7C(", "%7C%7C (", "AND(", "AND( ", "%26%26(", - "%26%26 (", "&&(", "&& (", "||(", "|| (", "OR )", - "%7C%7C)", "%7C%7C )", "AND )", "%26%26)", "%26%26 )", - "&&)", "&& )", "||)", "|| )", + "OR ", + "%7C%7C", + "%7C%7C ", + "AND ", + "%26%26", + "%26%26 ", + "&&", + "&& ", + "||", + "|| ", + "NOT", + "NOT ", + "%21", + "%21 ", + "!", + "! ", + "%2B", + "%2B ", + "-", + "- ", + "OR(", + "OR (", + "%7C%7C(", + "%7C%7C (", + "AND(", + "AND( ", + "%26%26(", + "%26%26 (", + "&&(", + "&& (", + "||(", + "|| (", + "OR )", + "%7C%7C)", + "%7C%7C )", + "AND )", + "%26%26)", + "%26%26 )", + "&&)", + "&& )", + "||)", + "|| )", # Add missing operators that API rejects - "OR", "AND" + "OR", + "AND", ] # Operators that cannot appear at the start start_operators = [ - " OR", "%7C%7C", " %7C%7C", " AND", "%26%26", " %26%26", - "&&", " &&", " ||", "||", "( OR", "(%7C%7C", "( %7C%7C", - "( AND", "(%26%26", "( %26%26", "(&&", "( &&", "( ||", - "(||", ")OR", ") OR", ")%7C%7C", ") %7C%7C", ")AND", - ") AND", ")%26%26", ") %26%26", ")&&", ") &&", " )||", ") ||" + " OR", + "%7C%7C", + " %7C%7C", + " AND", + "%26%26", + " %26%26", + "&&", + " &&", + " ||", + "||", + "( OR", + "(%7C%7C", + "( %7C%7C", + "( AND", + "(%26%26", + "( %26%26", + "(&&", + "( &&", + "( ||", + "(||", + ")OR", + ") OR", + ")%7C%7C", + ") %7C%7C", + ")AND", + ") AND", + ")%26%26", + ") %26%26", + ")&&", + ") &&", + " )||", + ") ||", ] # Check end operators @@ -123,7 +197,7 @@ def _check_start_end(self, query: str, variable_name: str) -> Tuple[bool, str]: return ( False, f"[{variable_name}] parameter ends with an operator {str(op)}. " - f"Please remove an unused operator." + f"Please remove an unused operator.", ) # Special check for word operators at start (AND, OR, NOT followed by space) @@ -132,7 +206,7 @@ def _check_start_end(self, query: str, variable_name: str) -> Tuple[bool, str]: operator = re.match(r"^(AND|OR|NOT)", query, re.IGNORECASE).group(1) return ( False, - f"Syntax error in input : unexpected \"{operator}\" at position 0!" + f'Syntax error in input : unexpected "{operator}" at position 0!', ) # Check other start operators @@ -141,7 +215,7 @@ def _check_start_end(self, query: str, variable_name: str) -> Tuple[bool, str]: return ( False, f"[{variable_name}] parameter starts with an operator {str(op)}. " - f"The query must not start with such operator. Please remove it." + f"The query must not start with such operator. Please remove it.", ) return True, "" @@ -149,78 +223,104 @@ def _check_start_end(self, query: str, variable_name: str) -> Tuple[bool, str]: def _check_middle(self, query: str, variable_name: str) -> Tuple[bool, str]: """Check for invalid operator combinations.""" invalid_combinations = [ - " OR OR ", "%7C%7C %7C%7C", "|| ||", "|| (||", "||) ||", - " AND AND ", "%26%26 %26%26", "&& &&", "&& (&&", "&&) &&", - " NOT NOT ", "! !", "%21 %21", "- -", "--", " OR AND ", - " AND OR ", "%7C%7C %26%26", "%26%26 %7C%7C", " OR (AND ", - " AND (OR ", "%7C%7C (%26%26", "%26%26 (%7C%7C", " OR) AND ", - " AND) OR ", "%7C%7C) %26%26", "%26%26) %7C%7C", "()" + " OR OR ", + "%7C%7C %7C%7C", + "|| ||", + "|| (||", + "||) ||", + " AND AND ", + "%26%26 %26%26", + "&& &&", + "&& (&&", + "&&) &&", + " NOT NOT ", + "! !", + "%21 %21", + "- -", + "--", + " OR AND ", + " AND OR ", + "%7C%7C %26%26", + "%26%26 %7C%7C", + " OR (AND ", + " AND (OR ", + "%7C%7C (%26%26", + "%26%26 (%7C%7C", + " OR) AND ", + " AND) OR ", + "%7C%7C) %26%26", + "%26%26) %7C%7C", + "()", ] for combo in invalid_combinations: if combo in query: return ( False, - f"[{variable_name}] parameter contains operator \" {str(combo)} \" used without " - f"keywords. Please add keywords or remove one of the operators" + f'[{variable_name}] parameter contains operator " {str(combo)} " used without ' + f"keywords. Please add keywords or remove one of the operators", ) return True, "" - - def _check_same_level_operators(self, query: str, variable_name: str) -> Tuple[bool, str]: + + def _check_same_level_operators( + self, query: str, variable_name: str + ) -> Tuple[bool, str]: """ Check for AND/OR at same level after automatic AND insertion. - - The API automatically inserts AND operators between standalone terms that aren't - within quotes or connected by explicit operators. This creates same-level operator + + The API automatically inserts AND operators between standalone terms that aren't + within quotes or connected by explicit operators. This creates same-level operator violations when OR is mixed with these implicit ANDs. - + Examples: - "AI OR artificial intelligence" → "AI OR artificial AND intelligence" (INVALID) - "AI OR \"artificial intelligence\"" → "AI OR \"artificial intelligence\"" (VALID) """ - + # Quick check: if no OR/AND operators, no same-level issue - if not any(op in query for op in [' OR ', ' AND ', '||', '&&', '%7C%7C', '%26%26']): + if not any( + op in query for op in [" OR ", " AND ", "||", "&&", "%7C%7C", "%26%26"] + ): return True, "" - + # Tokenize the query while preserving quoted phrases and operators tokens = self._tokenize_query_for_same_level_check(query) - + # Simulate automatic AND insertion tokens_with_implicit_and = self._simulate_and_insertion(tokens) - + # Check for same-level AND/OR violations if self._has_same_level_and_or(tokens_with_implicit_and): return ( False, f'in [{variable_name}] "AND" and "OR" operator not allowed at same level, ' - f'Please use parentheses to group terms correctly, such as ' - f'`(elon AND musk) OR twitter`.' + f"Please use parentheses to group terms correctly, such as " + f"`(elon AND musk) OR twitter`.", ) - + return True, "" def _tokenize_query_for_same_level_check(self, query: str) -> List[str]: """ Tokenize query preserving quoted phrases, operators, and parentheses. - - Returns tokens like: ['AI', 'OR', 'artificial', 'intelligence'] + + Returns tokens like: ['AI', 'OR', 'artificial', 'intelligence'] or ['AI', 'OR', '"artificial intelligence"'] """ tokens = [] i = 0 current_token = "" - + while i < len(query): char = query[i] - + # Handle quoted phrases (including escaped quotes) if char == '"': if current_token: tokens.append(current_token.strip()) current_token = "" - + # Collect the entire quoted phrase quoted_phrase = '"' i += 1 @@ -229,28 +329,30 @@ def _tokenize_query_for_same_level_check(self, query: str) -> List[str]: quoted_phrase += '"' i += 1 break - elif query[i] == '\\' and i + 1 < len(query) and query[i + 1] == '"': + elif ( + query[i] == "\\" and i + 1 < len(query) and query[i + 1] == '"' + ): # Handle escaped quotes - quoted_phrase += query[i:i+2] + quoted_phrase += query[i : i + 2] i += 2 else: quoted_phrase += query[i] i += 1 - + tokens.append(quoted_phrase) continue - + # Handle operators and parentheses - elif char in '()': + elif char in "()": if current_token: tokens.append(current_token.strip()) current_token = "" tokens.append(char) i += 1 continue - + # Handle spaces - potential token boundaries - elif char == ' ': + elif char == " ": if current_token: token = current_token.strip() if token: @@ -258,23 +360,23 @@ def _tokenize_query_for_same_level_check(self, query: str) -> List[str]: current_token = "" i += 1 continue - + else: current_token += char i += 1 - + # Add final token if current_token: token = current_token.strip() if token: tokens.append(token) - + return tokens def _simulate_and_insertion(self, tokens: List[str]) -> List[str]: """ Simulate automatic AND insertion between adjacent terms. - + Rules: - Insert AND between adjacent words that aren't operators - Don't insert AND around parentheses or quoted phrases @@ -282,94 +384,100 @@ def _simulate_and_insertion(self, tokens: List[str]) -> List[str]: """ if len(tokens) <= 1: return tokens - + result = [] - operators = {'AND', 'OR', 'NOT', '&&', '||', '%26%26', '%7C%7C', '!', '-'} - brackets = {'(', ')'} - + operators = {"AND", "OR", "NOT", "&&", "||", "%26%26", "%7C%7C", "!", "-"} + brackets = {"(", ")"} + for i, token in enumerate(tokens): result.append(token) - + # Check if we should insert AND after this token if i < len(tokens) - 1: current = token next_token = tokens[i + 1] - + # Don't insert AND if current or next is operator/bracket - if (current.upper() in operators or - next_token.upper() in operators or - current in brackets or - next_token in brackets or - current.startswith('"') or # Quoted phrase - next_token.startswith('"')): + if ( + current.upper() in operators + or next_token.upper() in operators + or current in brackets + or next_token in brackets + or current.startswith('"') # Quoted phrase + or next_token.startswith('"') + ): continue - + # Insert implicit AND between standalone words - if (not current.upper() in operators and - not next_token.upper() in operators and - current not in brackets and - next_token not in brackets): - result.append('AND') - + if ( + not current.upper() in operators + and not next_token.upper() in operators + and current not in brackets + and next_token not in brackets + ): + result.append("AND") + return result def _has_same_level_and_or(self, tokens: List[str]) -> bool: """ Check if tokens contain AND/OR at the same precedence level. - + This is a simplified check that looks for AND and OR operators not properly separated by parentheses. """ - # Simple heuristic: if we have both AND and OR operators outside of + # Simple heuristic: if we have both AND and OR operators outside of # properly grouped parentheses, it's likely a same-level violation - - has_and = any(token.upper() in ['AND', '&&', '%26%26'] for token in tokens) - has_or = any(token.upper() in ['OR', '||', '%7C%7C'] for token in tokens) - + + has_and = any(token.upper() in ["AND", "&&", "%26%26"] for token in tokens) + has_or = any(token.upper() in ["OR", "||", "%7C%7C"] for token in tokens) + # If we have both AND and OR, we need to check grouping if has_and and has_or: - # For now, use a simple heuristic: + # For now, use a simple heuristic: # If there are no parentheses to group operations, assume violation - if '(' not in tokens and ')' not in tokens: + if "(" not in tokens and ")" not in tokens: return True - + # More sophisticated parsing would check proper grouping # This is a basic implementation that catches common cases return self._check_operator_grouping(tokens) - + return False def _check_operator_grouping(self, tokens: List[str]) -> bool: """ Check if AND/OR operators are properly grouped with parentheses. - + This is a simplified implementation that catches common violations. A full parser would be more accurate but more complex. """ # Simple rule: if we see patterns like "term AND term OR term" without # parentheses properly separating the different operator types, it's invalid - + operators = [] paren_depth = 0 current_level_ops = [] - + for token in tokens: - if token == '(': + if token == "(": # Starting new group - save current level operators if current_level_ops: operators.append(current_level_ops.copy()) current_level_ops = [] paren_depth += 1 - elif token == ')': + elif token == ")": paren_depth -= 1 # Check current level when closing group if len(set(current_level_ops)) > 1: # Mixed operators at this level return True current_level_ops = [] - elif token.upper() in ['AND', 'OR', '&&', '||', '%26%26', '%7C%7C']: - normalized_op = 'AND' if token.upper() in ['AND', '&&', '%26%26'] else 'OR' + elif token.upper() in ["AND", "OR", "&&", "||", "%26%26", "%7C%7C"]: + normalized_op = ( + "AND" if token.upper() in ["AND", "&&", "%26%26"] else "OR" + ) current_level_ops.append(normalized_op) - + # Check final level return len(set(current_level_ops)) > 1 @@ -387,23 +495,24 @@ def _check_quotes(self, query: str, variable_name: str) -> Tuple[bool, str]: if len(all_open) != len(all_closed): return ( False, - f"[{variable_name}] parameter contains an unclosed round bracket \"(\" or \")\". " - f"Please close the bracket before proceeding." + f'[{variable_name}] parameter contains an unclosed round bracket "(" or ")". ' + f"Please close the bracket before proceeding.", ) # Check quotes all_quotes = [] - for o in ["\"", "%22"]: + for o in ['"', "%22"]: all_quotes.extend(re.findall(re.escape(o), query)) if len(all_quotes) % 2 == 0: return True, "" # Fixed: return "" instead of 0 return ( False, - f"[{variable_name}] parameter contains an unclosed quote (\"). " - f"Please close the quote before proceeding." + f'[{variable_name}] parameter contains an unclosed quote ("). ' + f"Please close the quote before proceeding.", ) + class NewscatcherMixin: """Common functionality for both synchronous and asynchronous Newscatcher API clients.""" @@ -422,7 +531,9 @@ def prepare_time_chunks(self, endpoint_type, **kwargs): from_date, to_date, chunk_delta = parse_time_parameters(endpoint_type, **kwargs) time_chunks = create_time_chunks(from_date, to_date, chunk_delta) - desc = f"Fetching {'article' if endpoint_type == 'search' else 'headlines'} chunks" + desc = ( + f"Fetching {'article' if endpoint_type == 'search' else 'headlines'} chunks" + ) is_test = "pytest" in sys.modules or "TEST_MODE" in os.environ chunks_iter = setup_progress_tracking( @@ -449,10 +560,12 @@ def prepare_request_params(self, params, endpoint_params=None): return request_params - def _process_articles(self, articles_data, seen_ids, deduplicate, max_articles, current_count): + def _process_articles( + self, articles_data, seen_ids, deduplicate, max_articles, current_count + ): """Process articles with deduplication and limits.""" processed_articles = [] - + for article in articles_data: if current_count >= max_articles: return processed_articles, current_count, False @@ -468,11 +581,11 @@ def _process_articles(self, articles_data, seen_ids, deduplicate, max_articles, current_count += 1 return processed_articles, current_count, True - + def log_completion(self, show_progress: bool, article_count: int): """ Log completion message if progress tracking is enabled. - + Args: show_progress: Whether progress tracking is enabled article_count: Number of articles retrieved @@ -500,10 +613,10 @@ def get_all_articles( deduplicate: bool = True, validate_query: bool = True, concurrency: int = 3, - **kwargs + **kwargs, ) -> List[Any]: """Retrieve all articles matching search criteria, bypassing the 10,000 limit.""" - + if validate_query: is_valid, error_message = self.validate_query(q) if not is_valid: @@ -527,16 +640,12 @@ def get_all_articles( for chunk_start, chunk_end in chunks_iter: chunk_from = format_datetime(chunk_start) chunk_to = format_datetime(chunk_end) - + request_params = self.prepare_request_params(kwargs) try: first_page_response = self.search.post( - q=q, - from_=chunk_from, - to=chunk_to, - page=1, - **request_params + q=q, from_=chunk_from, to=chunk_to, page=1, **request_params ) articles_data = safe_get_articles(first_page_response) @@ -560,7 +669,7 @@ def get_all_articles( break total_pages = getattr(first_page_response, "total_pages", 1) - + if total_pages > 1: for page in range(2, min(total_pages + 1, 11)): if current_count >= max_articles: @@ -571,7 +680,7 @@ def get_all_articles( from_=chunk_from, to=chunk_to, page=page, - **request_params + **request_params, ) page_articles = safe_get_articles(page_response) @@ -600,7 +709,7 @@ def get_all_articles( print(f"\nCompleted: Retrieved {len(all_articles)} articles") return all_articles - + def get_all_headlines( self, when: Optional[Union[datetime.datetime, str]] = None, @@ -609,19 +718,21 @@ def get_all_headlines( show_progress: bool = False, deduplicate: bool = True, **kwargs, - ) -> Articles: + ) -> List[Any]: """ Fetch all latest headlines by splitting the request into multiple time-based chunks to overcome the 10,000 article limit. """ - + # Set defaults if max_articles is None: max_articles = self.DEFAULT_MAX_ARTICLES # Parse time parameters - this converts when="1d" to proper time ranges - from_date, to_date, chunk_delta = parse_time_parameters("latestheadlines", when=when, time_chunk_size=time_chunk_size) - + from_date, to_date, chunk_delta = parse_time_parameters( + "latestheadlines", when=when, time_chunk_size=time_chunk_size + ) + # Create time chunks time_chunks = create_time_chunks(from_date, to_date, chunk_delta) @@ -653,7 +764,9 @@ def get_all_headlines( try: # Convert chunk times to the expected format using calculate_when_param - when_param = calculate_when_param(chunk_end, chunk_start) # This creates "1d", "2h", etc. + when_param = calculate_when_param( + chunk_end, chunk_start + ) # This creates "1d", "2h", etc. # Make the first request first_response = self.latestheadlines.post( @@ -680,12 +793,14 @@ def get_all_headlines( # Stop if we've reached the limit if not should_continue: if show_progress: - print(f"\nReached maximum article limit ({max_articles}). Stopping.") + print( + f"\nReached maximum article limit ({max_articles}). Stopping." + ) break # Get total pages for pagination total_pages = getattr(first_response, "total_pages", 1) - + # If there are more pages, fetch them if total_pages > 1: for page in range(2, min(total_pages + 1, 11)): @@ -702,14 +817,16 @@ def get_all_headlines( # Process articles if any were found if page_articles: - processed_articles, current_count, should_continue = ( - self._process_articles( - page_articles, - seen_ids, - deduplicate, - max_articles, - current_count, - ) + ( + processed_articles, + current_count, + should_continue, + ) = self._process_articles( + page_articles, + seen_ids, + deduplicate, + max_articles, + current_count, ) all_articles.extend(processed_articles) @@ -717,7 +834,9 @@ def get_all_headlines( # Stop if we've reached the limit if not should_continue: if show_progress: - print(f"\nReached maximum article limit ({max_articles}). Stopping.") + print( + f"\nReached maximum article limit ({max_articles}). Stopping." + ) break except Exception as e: @@ -755,10 +874,10 @@ async def get_all_articles( deduplicate: bool = True, validate_query: bool = True, concurrency: int = 3, - **kwargs + **kwargs, ) -> List[Any]: """Asynchronously retrieve all articles matching search criteria.""" - + if validate_query: is_valid, error_message = self.validate_query(q) if not is_valid: @@ -782,16 +901,12 @@ async def get_all_articles( for chunk_start, chunk_end in chunks_iter: chunk_from = format_datetime(chunk_start) chunk_to = format_datetime(chunk_end) - + request_params = self.prepare_request_params(kwargs) try: first_page_response = await self.search.post( - q=q, - from_=chunk_from, - to=chunk_to, - page=1, - **request_params + q=q, from_=chunk_from, to=chunk_to, page=1, **request_params ) articles_data = safe_get_articles(first_page_response) @@ -815,11 +930,11 @@ async def get_all_articles( break total_pages = getattr(first_page_response, "total_pages", 1) - + if total_pages > 1: tasks = [] semaphore = asyncio.Semaphore(concurrency) - + async def fetch_page(page_num): async with semaphore: return await self.search.post( @@ -827,7 +942,7 @@ async def fetch_page(page_num): from_=chunk_from, to=chunk_to, page=page_num, - **request_params + **request_params, ) for page in range(2, min(total_pages + 1, 11)): @@ -836,25 +951,29 @@ async def fetch_page(page_num): tasks.append(fetch_page(page)) if tasks: - page_responses = await asyncio.gather(*tasks, return_exceptions=True) - + page_responses = await asyncio.gather( + *tasks, return_exceptions=True + ) + for page_response in page_responses: if isinstance(page_response, Exception): continue - + if current_count >= max_articles: break page_articles = safe_get_articles(page_response) if page_articles: - processed_articles, current_count, should_continue = ( - self._process_articles( - page_articles, - seen_ids, - deduplicate, - max_articles, - current_count, - ) + ( + processed_articles, + current_count, + should_continue, + ) = self._process_articles( + page_articles, + seen_ids, + deduplicate, + max_articles, + current_count, ) all_articles.extend(processed_articles) @@ -871,7 +990,7 @@ async def fetch_page(page_num): print(f"\nCompleted: Retrieved {len(all_articles)} articles") return all_articles - + async def get_all_headlines( self, when: Optional[Union[datetime.datetime, str]] = None, @@ -881,18 +1000,20 @@ async def get_all_headlines( deduplicate: bool = True, concurrency: int = 3, # Default concurrency for page fetching **kwargs, - ) -> Articles: + ) -> List[Any]: """ Async version: Fetch all latest headlines by splitting the request into multiple time-based chunks to overcome the 10,000 article limit. - """ + """ # Set defaults if max_articles is None: max_articles = self.DEFAULT_MAX_ARTICLES # Parse time parameters - from_date, to_date, chunk_delta = parse_time_parameters("latestheadlines", when=when, time_chunk_size=time_chunk_size) - + from_date, to_date, chunk_delta = parse_time_parameters( + "latestheadlines", when=when, time_chunk_size=time_chunk_size + ) + # Create time chunks time_chunks = create_time_chunks(from_date, to_date, chunk_delta) @@ -923,7 +1044,7 @@ async def get_all_headlines( break try: - # Convert chunk times to the expected format using calculate_when_param + # Convert chunk times to the expected format using calculate_when_param when_param = calculate_when_param(chunk_end, chunk_start) # Make the first request @@ -951,12 +1072,14 @@ async def get_all_headlines( # Stop if we've reached the limit if not should_continue: if show_progress: - print(f"\nReached maximum article limit ({max_articles}). Stopping.") + print( + f"\nReached maximum article limit ({max_articles}). Stopping." + ) break # Get total pages for pagination total_pages = getattr(first_response, "total_pages", 1) - + # If there are more pages, fetch them with concurrency if total_pages > 1: page_tasks = [] @@ -973,14 +1096,18 @@ async def fetch_page(page_num): return None # Create tasks for all pages - for page in range(2, min(total_pages + 1, 11)): # Limit to 10 pages max + for page in range( + 2, min(total_pages + 1, 11) + ): # Limit to 10 pages max if current_count >= max_articles: break page_tasks.append(fetch_page(page)) # Execute page requests with concurrency if page_tasks: - page_responses = await asyncio.gather(*page_tasks, return_exceptions=True) + page_responses = await asyncio.gather( + *page_tasks, return_exceptions=True + ) for response in page_responses: if current_count >= max_articles: @@ -988,23 +1115,27 @@ async def fetch_page(page_num): if response and not isinstance(response, Exception): page_articles = safe_get_articles(response) - + if page_articles: - processed_articles, current_count, should_continue = ( - self._process_articles( - page_articles, - seen_ids, - deduplicate, - max_articles, - current_count, - ) + ( + processed_articles, + current_count, + should_continue, + ) = self._process_articles( + page_articles, + seen_ids, + deduplicate, + max_articles, + current_count, ) all_articles.extend(processed_articles) if not should_continue: if show_progress: - print(f"\nReached maximum article limit ({max_articles}). Stopping.") + print( + f"\nReached maximum article limit ({max_articles}). Stopping." + ) break # Stop processing chunks if we've reached the limit @@ -1016,4 +1147,4 @@ async def fetch_page(page_num): # Continue with next chunk self.log_completion(show_progress, len(all_articles)) - return all_articles \ No newline at end of file + return all_articles