""" Page parsing tools with intelligent attribute extraction """ import json from typing import Dict, Any, List from bs4 import BeautifulSoup from mcp.types import TextContent from ..config import session, authenticated_session, logger async def parse_page_tool(args: Dict[str, Any]) -> List[TextContent]: """Complete page parsing with intelligent attribute extraction and fallback selectors.""" url = args["url"] extract_title = args.get("extract_title", True) extract_meta = args.get("extract_meta", True) extract_links = args.get("extract_links", False) extract_images = args.get("extract_images", False) custom_selectors = args.get("custom_selectors", {}) verbose = args.get("verbose", True) smart_extraction = args.get("smart_extraction", True) try: # Session info will be set later if authenticated # Fetch the page using efficient API with browser session cookies from ..utils.api_client import get_api_client api_client = get_api_client() # Update API client with latest browser session cookies api_client.ensure_session_updated() result_data = api_client.get(url, timeout=10, headers=headers, debug=verbose) if not result_data.get("success", False): error_msg = result_data.get("error", f"HTTP {result_data.get('status_code', 0)}: Failed to fetch {url}") return [TextContent(type="text", text=error_msg)] # Extract HTML from API response html = result_data.get("content", "") if result_data.get("json_data"): # If response is JSON, try to extract HTML from it json_data = result_data["json_data"] if isinstance(json_data, dict): html = json_data.get("html", json_data.get("content", html)) soup = BeautifulSoup(html, 'lxml') result = { "url": url, "status_code": result_data.get("status_code", 200), "content_type": result_data.get("content_type", ""), "page_size": len(html) } # Add session information if authenticated if authenticated_session.get("active"): cookies_info = api_client.get_cookies_info() if cookies_info["cookies_count"] > 0: session_info = { "source": "api_with_browser_session", "login_url": authenticated_session.get("login_url"), "cookies_count": cookies_info["cookies_count"], "session_age_hours": ( (time.time() - authenticated_session.get("login_timestamp", time.time())) / 3600 if authenticated_session.get("login_timestamp") else 0 ), "api_response_time": result_data.get("response_time_seconds", 0) } result["session_used"] = session_info # Extract title with fallbacks if extract_title: title_candidates = [ soup.find('title'), soup.find('h1'), soup.find('meta', attrs={'property': 'og:title'}), soup.find('meta', attrs={'name': 'title'}) ] title = "" title_source = "not_found" for i, candidate in enumerate(title_candidates): if candidate: if candidate.name == 'title' or candidate.name == 'h1': title = candidate.get_text(strip=True) title_source = candidate.name elif candidate.name == 'meta': title = candidate.get('content', '') title_source = f"meta_{candidate.get('property', candidate.get('name', 'unknown'))}" if title: break result["title"] = { "value": title, "source": title_source, "alternatives": [ { "source": "h1", "value": soup.find('h1').get_text(strip=True) if soup.find('h1') else "" }, { "source": "og:title", "value": soup.find('meta', attrs={'property': 'og:title'}).get('content', '') if soup.find('meta', attrs={'property': 'og:title'}) else "" } ] if verbose else [] } # Extract meta tags with categorization if extract_meta: meta_tags = { "standard": {}, "open_graph": {}, "twitter": {}, "other": {} } for meta in soup.find_all('meta'): name = meta.get('name') or meta.get('property') or meta.get('http-equiv') content = meta.get('content') if name and content: if name.startswith('og:'): meta_tags["open_graph"][name] = content elif name.startswith('twitter:'): meta_tags["twitter"][name] = content elif name in ['description', 'keywords', 'author', 'robots', 'viewport']: meta_tags["standard"][name] = content else: meta_tags["other"][name] = content result["meta"] = meta_tags # Extract links with detailed analysis if extract_links: links = [] internal_links = 0 external_links = 0 for link in soup.find_all('a', href=True): href = link['href'] text = link.get_text(strip=True) # Determine if internal or external if href.startswith('http'): if url in href: internal_links += 1 link_type = "internal" else: external_links += 1 link_type = "external" else: internal_links += 1 link_type = "relative" links.append({ "href": href, "text": text, "type": link_type, "title": link.get('title', ''), "target": link.get('target', '') }) result["links"] = { "data": links[:50], # Limit to first 50 links "stats": { "total": len(links), "internal": internal_links, "external": external_links, "showing": min(50, len(links)) } } # Extract images with detailed metadata if extract_images: images = [] for img in soup.find_all('img'): src = img.get('src') or img.get('data-src') or img.get('data-lazy') if src: images.append({ "src": src, "alt": img.get('alt', ''), "title": img.get('title', ''), "width": img.get('width', ''), "height": img.get('height', ''), "loading": img.get('loading', ''), "srcset": img.get('srcset', '') }) result["images"] = { "data": images[:50], # Limit to first 50 images "stats": { "total": len(images), "with_alt": len([img for img in images if img['alt']]), "showing": min(50, len(images)) } } # Enhanced custom selectors with intelligent extraction if custom_selectors: custom_data = {} extraction_stats = {} for key, selector_config in custom_selectors.items(): # Handle both string selectors and detailed configs if isinstance(selector_config, str): selectors = [selector_config] extract_type = "text" fallback_selectors = [] elif isinstance(selector_config, dict): selectors = selector_config.get("selectors", [selector_config.get("selector", "")]) if isinstance(selectors, str): selectors = [selectors] extract_type = selector_config.get("extract", "text") # text, attribute, html attribute_name = selector_config.get("attribute", "value") fallback_selectors = selector_config.get("fallbacks", []) else: selectors = [str(selector_config)] extract_type = "text" fallback_selectors = [] # Add common fallback selectors for form elements if smart_extraction and not fallback_selectors: if any(term in selectors[0].lower() for term in ['csrf', 'token']): fallback_selectors = [ "input[name*='csrf']", "input[name*='token']", "meta[name='csrf-token']", "meta[name='_token']" ] elif any(term in selectors[0].lower() for term in ['login', 'user', 'email']): fallback_selectors = [ "input[type='email']", "input[name*='user']", "input[name*='login']", "input[name*='email']" ] all_selectors = selectors + fallback_selectors elements_found = [] selector_used = None # Try each selector until we find elements for selector in all_selectors: try: elements = soup.select(selector) if elements: selector_used = selector for element in elements[:10]: # Limit to 10 elements if extract_type == "text": value = element.get_text(strip=True) elif extract_type == "attribute": value = element.get(attribute_name, '') elif extract_type == "html": value = str(element) else: # Smart extraction based on element type if element.name == 'input': value = element.get('value', '') if not value and element.get('type') == 'checkbox': value = element.has_attr('checked') elif element.name == 'select': selected = element.find('option', selected=True) value = selected.get('value', '') if selected else '' elif element.name == 'textarea': value = element.get_text(strip=True) elif element.name in ['meta']: value = element.get('content', '') elif element.name == 'a': value = element.get('href', '') elif element.name == 'img': value = element.get('src', '') else: value = element.get_text(strip=True) if value: # Only include non-empty values elements_found.append(value) break # Found elements, stop trying selectors except Exception as selector_error: logger.warning(f"Selector error for {selector}: {selector_error}") continue # Build result for this selector custom_data[key] = { "values": elements_found, "count": len(elements_found), "selector_used": selector_used, "selectors_tried": len(all_selectors), "success": len(elements_found) > 0 } # Add detailed stats if verbose if verbose: custom_data[key]["diagnostics"] = { "primary_selectors": selectors, "fallback_selectors": fallback_selectors, "extract_type": extract_type, "attribute_name": attribute_name if extract_type == "attribute" else None } extraction_stats[key] = { "found": len(elements_found), "selector": selector_used, "success": len(elements_found) > 0 } result["custom"] = custom_data if verbose: result["extraction_stats"] = { "custom_selectors_processed": len(custom_selectors), "successful_extractions": len([k for k, v in extraction_stats.items() if v["success"]]), "failed_extractions": len([k for k, v in extraction_stats.items() if not v["success"]]), "total_values_extracted": sum(v["found"] for v in extraction_stats.values()) } # Add page structure analysis if verbose if verbose: structure_info = { "total_elements": len(soup.find_all()), "forms": len(soup.find_all('form')), "inputs": len(soup.find_all('input')), "links": len(soup.find_all('a')), "images": len(soup.find_all('img')), "scripts": len(soup.find_all('script')), "stylesheets": len(soup.find_all('link', rel='stylesheet')), "headings": { f"h{i}": len(soup.find_all(f'h{i}')) for i in range(1, 7) } } result["page_structure"] = structure_info # Add recommendations for better extraction recommendations = [] if custom_selectors: failed_selectors = [k for k, v in custom_data.items() if not v["success"]] if failed_selectors: recommendations.extend([ f"Failed to find elements for: {', '.join(failed_selectors)}", "Try inspecting the page source to verify CSS selectors", "Consider using more specific or alternative selectors", "Check if content is loaded dynamically (use fetch_html with use_browser=true)" ]) if recommendations: result["recommendations"] = recommendations return [TextContent(type="text", text=json.dumps(result, indent=2, ensure_ascii=False))] except Exception as e: error_result = { "url": url, "error": str(e), "error_type": type(e).__name__, "diagnostics": { "error_details": f"Critical error during page parsing: {str(e)}", "custom_selectors_provided": len(custom_selectors) if custom_selectors else 0 }, "recommendations": [ "Check if URL is accessible and returns valid HTML", "Verify CSS selectors are correctly formatted", "Try using fetch_html tool first to check page content", "For JavaScript-heavy pages, use fetch_html with use_browser=true" ] } return [TextContent(type="text", text=json.dumps(error_result, indent=2, ensure_ascii=False))]