Step 1 - Mapping Relevant Pages

We generate a search keyword for the target objective using Watsonx, then map the website with Firecrawl. 

def debug_firecrawl_map(url, search_param): 
    """Debug function to test Firecrawl mapping""" 
    # Test without search parameter first 
    map_result_no_search = app.map_url(url) 
    # Test with search parameter 
    map_result_with_search = app.map_url(url, search=search_param) 
    return map_result_no_search, map_result_with_search 

find_relevant_page_via_map_fixed(objective, url, app, llm):

• WatsonX generates a search keyword from your objective.
• Firecrawl maps the URL, optionally with the search keyword.
• Extracts and ranks up to 10 links, selecting the top 3 most relevant. 

def find_relevant_page_via_map_fixed(objective, url, app, llm): 
    """Fixed version with structured debugging and no try/except""" 
    print(f"{Colors.CYAN}Objective: {objective}{Colors.RESET}") 
    # Step A: Generate search keyword 
    map_prompt = f""" 
    Based on this objective: {objective} 
    Generate a simple 1-2 word search term that would help find relevant pages. 
    Examples: If looking for founding date, use "founded" or "history" 
    If looking for contact info, use "contact" or "about" 
    Only respond with 1-2 words, nothing else. 
    """ 
    map_search_parameter = llm.invoke(map_prompt).strip() 
    print(f"{Colors.GREEN}Search keyword: {map_search_parameter}{Colors.RESET}") 
    # Step B: Debug the mapping process 
    no_search_result, with_search_result = debug_firecrawl_map(url, map_search_parameter) 
    # Step C: Extract links from API response 
    links = [] 
    # 1. Try with search parameter 
    if with_search_result and isinstance(with_search_result, dict): 
        if with_search_result.get('status') == 'success': 
            links = with_search_result.get('links', []) 
            print(f"{Colors.GREEN}Found {len(links)} links with search parameter{Colors.RESET}") 
    # 2. If no results, try without search parameter 
    if not links and no_search_result and isinstance(no_search_result, dict): 
        print(f"{Colors.YELLOW}No results with search, trying without search parameter{Colors.RESET}") 
        if no_search_result.get('status') == 'success': 
            links = no_search_result.get('links', []) 
            print(f"{Colors.GREEN}Found {len(links)} links without search parameter{Colors.RESET}") 
    # 3. If still nothing, handle MapResponse object format 
    if not links: 
        print(f"{Colors.YELLOW}Trying to extract from response objects{Colors.RESET}") 
        for result in [with_search_result, no_search_result]: 
            if result and hasattr(result, 'links'): 
                links = result.links 
                break 
            elif result and hasattr(result, '__dict__') and 'links' in result.__dict__: 
                links = result.__dict__['links'] 
                break 
    # If no links at all, fallback to original URL 
    if not links: 
        return [url] 
    print(f"{Colors.GREEN}Found {len(links)} links{Colors.RESET}") 
    # Limit to reasonable number for ranking 
    if len(links) > 10: 
        links = links[:10] 
    # Step D: Rank URLs 
    rank_prompt = f""" 
    Given this list of URLs and the objective: {objective} 
    Rank the top 3 most relevant ones based on URL structure and content hints. 
    Return ONLY a JSON array of 3 objects (or fewer if less than 3 URLs available) with: 
    - "url": the URL string 
    - "relevance_score": number from 1-10 
    - "reason": brief explanation 
    URLs to rank: 
    {json.dumps(links, indent=2)} 
    Respond with valid JSON only. 
    """ 
    ranked_raw = llm.invoke(rank_prompt).strip() 
    # Clean up possible ``` 
    if ranked_raw.startswith("```json"): 
        ranked_raw = ranked_raw.split("``` 
    if ranked_raw.endswith("```"): 
        ranked_raw = ranked_raw.rsplit("``` 
    ranked_raw = ranked_raw.strip() 
    # Load the JSON (now will raise naturally if invalid) 
    ranked_results = json.loads(ranked_raw) 
    print(f"{Colors.MAGENTA}Ranked Results:{Colors.RESET}") 
    for i, r in enumerate(ranked_results): 
        print(f"{Colors.GREEN}{i+1}. URL: {r['url']}{Colors.RESET}") 
        print(f"   {Colors.YELLOW}Score: {r['relevance_score']}{Colors.RESET}") 
        print(f"   {Colors.BLUE}Reason: {r['reason']}{Colors.RESET}") 
        print("---") 
    return [r["url"] for r in ranked_results] 

We loop through top-ranked URLs, scrape contents, and pass them to Watsonx for relevance validation & data extraction.

find_objective_in_top_pages_fixed(links, objective, app, llm):