Python 使用BeautifulSoup刮ESPN梦幻足球

我看过很多关于ESPN梦幻足球联赛的例子。我对网页抓取非常陌生，但在发布之前，我已经对此进行了广泛的研究。不过，我很难进入我的联赛并得到任何有用的东西。我想你应该在请求时传递cookies，以表明你正在进入一个私人联盟

import requests
from bs4 import BeautifulSoup

page = requests.get('https://fantasy.espn.com/football/league?leagueId=########',
                    cookies={'SWID': '#######', 'espn_s2': '#######'}
)
soup = BeautifulSoup(page.text, 'html.parser')
test = soup.find_all(class_ = 'team-scores')

print(len(test))
print(type(test))
print(test)

0

类“bs4.element.ResultSet”

[]

虽然基于本文中引用的一些帖子以及文章本身，cookies似乎很重要，但在没有cookies的情况下执行请求会得到相同的结果。我比较了汤，如果饼干用过和没用过，结果都是一样的

我知道有一些API可以在ESPN上使用，但我无法让任何代码为我工作。我希望从每支球队的名单中筛选出结果，然后运行每一个可能的时间表，以获得结果的分布，并查看每支球队在我的联赛中是多么幸运或不幸。我也很好奇与雅虎合作。在这一点上，我可以很容易地手动获取数据，因为它不太多，但我希望有一个更普遍的形式

---

对于没有经验的网络刮板，如果您有任何建议或帮助，我将不胜感激。

您必须共享您的联盟ID供我测试，但这里有一些代码可以对联盟进行一些数据操作。基本上，您将获得以json格式返回的数据，然后需要对其进行解析，以根据每周积分计算出胜负。然后，您可以排序并创建一个最终表格，将常规赛记录与总体记录进行比较，并根据时间表查看哪些球队的表现高于/低于计划：

import requests
import pandas as pd



s = requests.Session()
r = s.get('https://www.espn.com')

swid = s.cookies.get_dict()['SWID']


league_id = 31181


url = 'https://fantasy.espn.com/apis/v3/games/ffl/seasons/2019/segments/0/leagues/%s' %league_id


r = requests.get(url, cookies={"swid": swid}).json()

#Get Team IDs
teamId = {}
for team in r['teams']:
    teamId[team['id']] = team['location'].strip() + ' ' + team['nickname'].strip()


#Get each team's weekly points and calculate their head-to-head records
weeklyPoints = {}
r = requests.get(url, cookies={"swid": swid}, params={"view": "mMatchup"}).json()

weeklyPts = pd.DataFrame()
for each in r['schedule']:
    #each = r['schedule'][0]

    week = each['matchupPeriodId']
    if week >= 14:
        continue

    homeTm = teamId[each['home']['teamId']]
    homeTmPts = each['home']['totalPoints']

    try:
        awayTm = teamId[each['away']['teamId']]
        awayTmPts = each['away']['totalPoints']
    except:
        homeTmPts = 'BYE'
        continue

    temp_df = pd.DataFrame(list(zip([homeTm, awayTm], [homeTmPts, awayTmPts], [week, week])), columns=['team','pts','week'])

    if homeTmPts > awayTmPts:
        temp_df.loc[0,'win'] = 1
        temp_df.loc[0,'loss'] = 0
        temp_df.loc[0,'tie'] = 0

        temp_df.loc[1,'win'] = 0
        temp_df.loc[1,'loss'] = 1
        temp_df.loc[1,'tie'] = 0

    elif homeTmPts < awayTmPts:
        temp_df.loc[0,'win'] = 0
        temp_df.loc[0,'loss'] = 1
        temp_df.loc[0,'tie'] = 0

        temp_df.loc[1,'win'] = 1
        temp_df.loc[1,'loss'] = 0
        temp_df.loc[1,'tie'] = 0

    elif homeTmPts == awayTmPts:
        temp_df.loc[0,'win'] = 0
        temp_df.loc[0,'loss'] = 0
        temp_df.loc[0,'tie'] = 1

        temp_df.loc[1,'win'] = 0
        temp_df.loc[1,'loss'] = 0
        temp_df.loc[1,'tie'] = 1

    weeklyPts = weeklyPts.append(temp_df, sort=True).reset_index(drop=True)

weeklyPts['win'] = weeklyPts.groupby(['team'])['win'].cumsum()
weeklyPts['loss'] = weeklyPts.groupby(['team'])['loss'].cumsum()
weeklyPts['tie'] = weeklyPts.groupby(['team'])['tie'].cumsum()



# Calculate each teams record compared to all other teams points week to week
cumWeeklyRecord = {}   
for week in weeklyPts[weeklyPts['pts'] > 0]['week'].unique():
    df = weeklyPts[weeklyPts['week'] == week]

    cumWeeklyRecord[week] = {}
    for idx, row in df.iterrows():
        team = row['team']
        pts = row['pts']
        win = len(df[df['pts'] < pts])
        loss = len(df[df['pts'] > pts])
        tie = len(df[df['pts'] == pts])

        cumWeeklyRecord[week][team] = {}
        cumWeeklyRecord[week][team]['win'] = win
        cumWeeklyRecord[week][team]['loss'] = loss
        cumWeeklyRecord[week][team]['tie'] = tie-1

# Combine those cumluative records to get an overall season record      
overallRecord = {}     
for each in cumWeeklyRecord.items():
    for team in each[1].keys():
        if team not in overallRecord.keys():
            overallRecord[team] = {} 

        win = each[1][team]['win']
        loss = each[1][team]['loss']
        tie = each[1][team]['tie']

        if 'win' not in overallRecord[team].keys():
            overallRecord[team]['win'] = win
        else:
            overallRecord[team]['win'] += win

        if 'loss' not in overallRecord[team].keys():
            overallRecord[team]['loss'] = loss
        else:
            overallRecord[team]['loss'] += loss

        if 'tie' not in overallRecord[team].keys():
            overallRecord[team]['tie'] = tie
        else:
            overallRecord[team]['tie'] += tie


# Little cleaning up of the data nd calculating win %
overallRecord_df = pd.DataFrame(overallRecord).T
overallRecord_df = overallRecord_df.rename_axis('team').reset_index()
overallRecord_df = overallRecord_df.rename(columns={'win':'overall_win', 'loss':'overall_loss','tie':'overall_tie'})
overallRecord_df['overall_win%'] = overallRecord_df['overall_win'] / (overallRecord_df['overall_win'] + overallRecord_df['overall_loss'] + overallRecord_df['overall_tie'])
overallRecord_df['overall_rank'] = overallRecord_df['overall_win%'].rank(ascending=False, method='min')




regularSeasRecord = weeklyPts[weeklyPts['week'] == 13][['team','win','loss', 'tie']]
regularSeasRecord['win%'] = regularSeasRecord['win'] / (regularSeasRecord['win'] + regularSeasRecord['loss'] + regularSeasRecord['tie'])
regularSeasRecord['rank'] = regularSeasRecord['win%'].rank(ascending=False, method='min')



final_df = overallRecord_df.merge(regularSeasRecord, how='left', on=['team'])

---

我不确定你的页面中是否有你想要的数据。文本，请打印出来，然后进行编辑look@FeilNarley. 我很愿意帮你，但是今年ESPN里我没有一个幻想团队（只有NFL.com平台，而且没有访问该数据的问题）。所以我需要你的证件（登录等）。你能给我发联赛ID吗？其次，如果你访问一个API，它通常会以json格式响应，因此使用beautifulsoup解析html将不会返回任何内容。看起来我不需要凭据，只需要联赛ID。我必须添加espn_s2 cookie，因为联赛是私有的。非常感谢你的帮助。如果你对此感兴趣，我在这里做了一个图表，显示可能结果的分布（）。哦，太好了！谢谢

print (final_df.sort_values('rank').to_string())
                      team  overall_loss  overall_tie  overall_win  overall_win%  overall_rank   win  loss  tie      win%  rank
0             Luck Dynasty            39            0          104      0.727273           1.0  12.0   1.0  0.0  0.923077   1.0
10     Warsaw Widow Makers            48            0           95      0.664336           3.0  10.0   3.0  0.0  0.769231   2.0
2              Team Powell            60            0           83      0.580420           5.0   8.0   5.0  0.0  0.615385   3.0
1               Team White            46            0           97      0.678322           2.0   7.0   6.0  0.0  0.538462   4.0
3   The SouthWest Slingers            55            0           88      0.615385           4.0   7.0   6.0  0.0  0.538462   4.0
5               U MAD BRO?            71            0           72      0.503497           6.0   7.0   6.0  0.0  0.538462   4.0
11            Team Troxell            88            0           55      0.384615           9.0   7.0   6.0  0.0  0.538462   4.0
6          Organized Chaos            72            0           71      0.496503           7.0   6.0   7.0  0.0  0.461538   8.0
7         Jobobes Jabronis            88            0           55      0.384615           9.0   6.0   7.0  0.0  0.461538   8.0
4             Killa Bees!!            98            0           45      0.314685          11.0   4.0   9.0  0.0  0.307692  10.0
9             Faceless Men            86            0           57      0.398601           8.0   3.0  10.0  0.0  0.230769  11.0
8     Rollin with Mahomies           107            0           36      0.251748          12.0   1.0  12.0  0.0  0.076923  12.0