Python Forum

Hi, I am trying to get the data from this website ino an excel (Actually libre office) file. https://sc2pulse.nephest.com/sc2/?season...ladder-top

This website shows the top players, and their mmr (match making rank). That's all I Want, a list of their names + their mmr number. I'm new to coding, I just have this code into a notepad file, converted it into a python file (.py) and run it from cmd. I have python installed. It does make the file, but the file has zero data on it. What do I need to change? Thank you very much.

import os
import requests
from bs4 import BeautifulSoup
import pandas as pd

# Define the URL of the website
url = "https://sc2pulse.nephest.com/sc2/?season=56&queue=LOTV_1V1&team-type=ARRANGED&us=true&eu=true&kr=true&cn=true&bro=true&sil=true&gol=true&pla=true&dia=true&mas=true&gra=true&page=0&type=ladder&ratingAnchor=99999&idAnchor=0&count=1#ladder-top"

try:
    # Send an HTTP request and get the webpage content
    response = requests.get(url)

    if response.status_code == 200:
        # Parse the HTML content of the page
        soup = BeautifulSoup(response.text, "html.parser")

        # Find the data you want by inspecting the webpage's HTML structure
        # Extract and store the data in a list or a pandas DataFrame

        # Example: Extracting player names and MMR
        players = []
        mmr = []

        for player_row in soup.select(".ladder-player-row"):
            player_name = player_row.find("span", class_="player-name").text
            player_mmr = player_row.find("span", class_="player-mmr").text

            players.append(player_name)
            mmr.append(player_mmr)

        # Create a pandas DataFrame
        df = pd.DataFrame({"Player Name": players, "MMR": mmr})

        # Define the path to save the Excel file in "My Documents"
        my_documents_path = os.path.expanduser("~\Documents")
        excel_file_path = os.path.join(my_documents_path, "top_100_players_mmr.xlsx")

        # Save the DataFrame to an Excel file in "My Documents"
        df.to_excel(excel_file_path, index=False)
        print(f"Data has been successfully saved to {excel_file_path}")
    else:
        print("Failed to fetch the webpage. Status code:", response.status_code)
except Exception as e:
    print("An error occurred:", str(e))

EDIT: Sorry it made two threads for some reason

The whole table is generated bye JavaScript,so the way you do it now will not work.
Also should look if site has an API to get date more easy.

Can use Selenium ,here a working setup.
Also when use this so most this path Service(r"C:\cmder\bin\chromedriver.exe") be in OS environment Path.
If not sure of Path just put chromedriver.exe in windows folder Service(r"C:\Windows\chromedriver.exe").

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
import time

# Setup
# Cromedriver most use version that your browser use
#https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/118.0.5939.0/win64/chromedriver-win64.zip
options = Options()
#options.add_argument("--headless=new")
ser = Service(r"C:\cmder\bin\chromedriver.exe")
browser = webdriver.Chrome(service=ser, options=options)
# Parse or automation
url = 'https://sc2pulse.nephest.com/sc2/?season=56&queue=LOTV_1V1&team-type=ARRANGED&us=true&eu=true&kr=true&cn=true&bro=true&sil=true&gol=true&pla=true&dia=true&mas=true&gra=true&page=0&type=ladder&ratingAnchor=99999&idAnchor=0&count=1#ladder-top'
browser.get(url)
time.sleep(2)
first_row = browser.find_element(By.CSS_SELECTOR, '#ladder > tbody > tr:nth-child(7)')
print(first_row.text)

Output:1
7239
BasiliskSerralllllllllllllrevealed
31 87

awere34a4e3

snippsat