kate/hltv-api
1
0
Fork 0
mirror of https://github.com/SocksPls/hltv-api synced 2025-04-30 15:43:07 +00:00
Code Issues Releases Wiki Activity

Merge pull request #30 from SocksPls/date-formatting

Browse source 
fix(app): housekeeping/dateformat
This commit is contained in:
David Silva 2022-07-14 13:45:25 +02:00 • committed by GitHub
commit 16eec2314f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 110 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

226
main.py
View file

@ -4,6 +4,36 @@ import datetime
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from python_utils import converters from python_utils import converters
import time import time
import zoneinfo
import tzlocal
HLTV_COOKIE_TIMEZONE = "Europe/Copenhagen"
HLTV_ZONEINFO=zoneinfo.ZoneInfo(HLTV_COOKIE_TIMEZONE)
LOCAL_TIMEZONE_NAME = tzlocal.get_localzone_name()
LOCAL_ZONEINFO = zoneinfo.ZoneInfo(LOCAL_TIMEZONE_NAME)
def padIfNeeded(numberStr: str):
if int(numberStr) < 10:
return str(numberStr).zfill(2)
else:
return str(numberStr)
def monthNameToNumber(monthName: str):
months = {
'January': 1,
'February': 2,
'March': 3,
'April': 4,
'May': 5,
'June': 6,
'July': 7,
'August': 8,
'September': 9,
'October': 10,
'November': 11,
'December': 12
}
return months[monthName]
def get_parsed_page(url, delay=0.5): def get_parsed_page(url, delay=0.5):
# This fixes a blocked by cloudflare error i've encountered # This fixes a blocked by cloudflare error i've encountered
@ -12,24 +42,26 @@ def get_parsed_page(url, delay=0.5):
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
} }
cookies = {
"hltvTimeZone": HLTV_COOKIE_TIMEZONE
}
time.sleep(delay) time.sleep(delay)
return BeautifulSoup(requests.get(url, headers=headers).text, "lxml") return BeautifulSoup(requests.get(url, headers=headers, cookies=cookies).text, "lxml")
def top5teams(): def top5teams():
home = get_parsed_page("http://hltv.org/") home = get_parsed_page("https://hltv.org/")
count = 0
teams = [] teams = []
for team in home.find_all("div", {"class": ["col-box rank"], }): for team in home.find_all("div", {"class": ["col-box rank"], }):
count += 1 team = {'name': team.text[3:], 'url': "https://hltv.org" + team.find_all("a")[1]["href"]}
teamname = team.text[3:] teams.append(team)
teams.append(teamname)
return teams return teams
def top30teams(): def top30teams():
page = get_parsed_page("http://www.hltv.org/ranking/teams/") page = get_parsed_page("https://www.hltv.org/ranking/teams/")
teams = page.find("div", {"class": "ranking"}) teams = page.find("div", {"class": "ranking"})
teamlist = [] teamlist = []
for team in teams.find_all("div", {"class": "ranked-team standard-box"}): for team in teams.find_all("div", {"class": "ranked-team standard-box"}):
@ -37,11 +69,14 @@ def top30teams():
'rank': converters.to_int(team.select('.position')[0].text.strip(), regexp=True), 'rank': converters.to_int(team.select('.position')[0].text.strip(), regexp=True),
'rank-points': converters.to_int(team.find('span', {'class': 'points'}).text, regexp=True), 'rank-points': converters.to_int(team.find('span', {'class': 'points'}).text, regexp=True),
'team-id': converters.to_int(team.find('a', {'class': 'details moreLink'})['href'].split('/')[-1]), 'team-id': converters.to_int(team.find('a', {'class': 'details moreLink'})['href'].split('/')[-1]),
'team-url': "https://hltv.org/team/" + team.find('a', {'class': 'details moreLink'})['href'].split('/')[-1] + "/" + team.find('div', {"class": "ranking-header"}).select('.name')[0].text.strip(),
'stats-url': "https://www.hltv.org" + team.find('a', {'class': 'details moreLink'})['href'],
'team-players': []} 'team-players': []}
for player_div in team.find_all("td", {"class": "player-holder"}): for player_div in team.find_all("td", {"class": "player-holder"}):
player = {} player = {}
player['name'] = player_div.find('img', {'class': 'playerPicture'})['title'] player['name'] = player_div.find('img', {'class': 'playerPicture'})['title']
player['player-id'] = converters.to_int(player_div.select('.pointer')[0]['href'].split("/")[-2]) player['player-id'] = converters.to_int(player_div.select('.pointer')[0]['href'].split("/")[-2])
player['url'] = "https://www.hltv.org" + player_div.select('.pointer')[0]['href']
newteam['team-players'].append(player) newteam['team-players'].append(player)
teamlist.append(newteam) teamlist.append(newteam)
return teamlist return teamlist
@ -53,66 +88,31 @@ def top_players():
playersArray = [] playersArray = []
for player in players.find_all("div", {"class": "top-x-box standard-box"}): for player in players.find_all("div", {"class": "top-x-box standard-box"}):
playerObj = {} playerObj = {}
playerObj['country'] = player.find_all('img')[1]['alt'].encode('utf8') playerObj['country'] = player.find_all('img')[1]['alt']
buildName = player.find('img', {'class': 'img'})['alt'].split("'") buildName = player.find('img', {'class': 'img'})['alt'].split("'")
playerObj['name'] = buildName[0].rstrip() + buildName[2] playerObj['name'] = buildName[0].rstrip() + buildName[2]
playerObj['nickname'] = player.find('a', {'class': 'name'}).text.encode('utf8') playerObj['nickname'] = player.find('a', {'class': 'name'}).text
playerObj['rating'] = player.find('div', {'class': 'rating'}).find('span', {'class': 'bold'}).text.encode('utf8') playerObj['rating'] = player.find('div', {'class': 'rating'}).find('span', {'class': 'bold'}).text
playerObj['maps-played'] = player.find('div', {'class': 'average gtSmartphone-only'}).find('span', {'class': 'bold'}).text.encode('utf8') playerObj['maps-played'] = player.find('div', {'class': 'average gtSmartphone-only'}).find('span', {'class': 'bold'}).text
playerObj['url'] = "https://hltv.org" + player.find('a', {'class': 'name'}).get('href')
playersArray.append(playerObj) playersArray.append(playerObj)
return playersArray return playersArray
def get_players(teamid): def get_players(teamid):
page = get_parsed_page("http://www.hltv.org/?pageid=362&teamid=" + str(teamid)) page = get_parsed_page("https://www.hltv.org/?pageid=362&teamid=" + str(teamid))
titlebox = page.find("div", {"class": "bodyshot-team"}) titlebox = page.find("div", {"class": "bodyshot-team"})
players = [] players = []
for player_link in titlebox.find_all("a"): for player_link in titlebox.find_all("a"):
players.append({ players.append({
'id': player_link["href"].split("/")[2], 'id': player_link["href"].split("/")[2],
'nickname': player_link["title"], 'nickname': player_link["title"],
'name': player_link.find("img")['title'] 'name': player_link.find("img")['title'],
'url': "https://hltv.org" + player_link["href"]
}) })
return players return players
def get_player_info(player_id):
"""
:param player_id: integer (or string consisting of integers)
:return: dictionary of player
example player id: 7398 (dupreeh)
"""
page = get_parsed_page(f"https://www.hltv.org/stats/players/{player_id}/a")
statistics = page.find("div", {"class": "statistics"}).find_all("div", {"class": "stats-row"})
player_info = {
'nickname': page.find("h1", {"class": "summaryNickname text-ellipsis"}).text.encode('utf8'),
'name': page.find("div", {"class": "text-ellipsis"}).text[1:-1].encode('utf8'),
'country': page.find("img", {"class": "flag"})["alt"],
'team': page.find("div", {"class": "SummaryTeamname text-ellipsis"}).text.encode('utf8'),
'age': page.find("div", {"class": "summaryPlayerAge"}).text[:2],
'stats': {
'total_kills': statistics[0].find_all("span")[1].text,
'headshot_percent': statistics[1].find_all("span")[1].text,
'total_deaths': statistics[2].find_all("span")[1].text,
'kd_ratio': statistics[3].find_all("span")[1].text,
'dmg_per_round': statistics[4].find_all("span")[1].text,
'grenade_dmg_per_round': statistics[5].find_all("span")[1].text,
'maps_played': statistics[6].find_all("span")[1].text,
'rounds_played': statistics[7].find_all("span")[1].text,
'kills_per_round': statistics[8].find_all("span")[1].text,
'assists_per_round': statistics[9].find_all("span")[1].text,
'deaths_per_round': statistics[10].find_all("span")[1].text,
'saved_by_teammate_per_round': statistics[11].find_all("span")[1].text,
'saved_teammates_per_round': statistics[12].find_all("span")[1].text,
'rating_1': statistics[13].find_all("span")[1].text,
}
}
return player_info
def get_team_info(teamid): def get_team_info(teamid):
""" """
:param teamid: integer (or string consisting of integers) :param teamid: integer (or string consisting of integers)
@ -120,10 +120,10 @@ def get_team_info(teamid):
example team id: 5378 (virtus pro) example team id: 5378 (virtus pro)
""" """
page = get_parsed_page("http://www.hltv.org/?pageid=179&teamid=" + str(teamid)) page = get_parsed_page("https://www.hltv.org/?pageid=179&teamid=" + str(teamid))
team_info = {} team_info = {}
team_info['team-name']=page.find("div", {"class": "context-item"}).text.encode('utf8') team_info['team-name']=page.find("div", {"class": "context-item"}).text
current_lineup = _get_current_lineup(page.find_all("div", {"class": "col teammate"})) current_lineup = _get_current_lineup(page.find_all("div", {"class": "col teammate"}))
team_info['current-lineup'] = current_lineup team_info['current-lineup'] = current_lineup
@ -138,55 +138,16 @@ def get_team_info(teamid):
stats = columns.find_all("div", {"class": "col standard-box big-padding"}) stats = columns.find_all("div", {"class": "col standard-box big-padding"})
for stat in stats: for stat in stats:
stat_value = stat.find("div", {"class": "large-strong"}).text.encode('utf8') stat_value = stat.find("div", {"class": "large-strong"}).text
stat_title = stat.find("div", {"class": "small-label-below"}).text.encode('utf8') stat_title = stat.find("div", {"class": "small-label-below"}).text
team_stats[stat_title] = stat_value team_stats[stat_title] = stat_value
team_info['stats'] = team_stats team_info['stats'] = team_stats
team_info['url'] = "https://hltv.org/stats/team/" + str(teamid) + "/" + str(team_info['team-name'])
return team_info return team_info
def get_match_info(match_id):
"""
Will get match data, NOT map data
:param match_id: integer (or string consisting of integers)
:return dictionary of match
example match id: 65090 (fnatic-vs-nip)
"""
page = get_parsed_page(f"https://www.hltv.org/stats/matches/{match_id}/a")
match_info = {
"team1": {
"name": page.find_all("table", {"class": "stats-table"})[0].find("th", {"class": "st-teamname"}).text
},
"team2": {
"name": page.find_all("table", {"class": "stats-table"})[1].find("th", {"class": "st-teamname"}).text
}
}
match_info["team1"]["players"] = [player.text for player in page.find_all("table", {"class": "stats-table"})[0].find_all("td", {"class": "st-player"})]
match_info["team2"]["players"] = [player.text for player in page.find_all("table", {"class": "stats-table"})[1].find_all("td", {"class": "st-player"})]
for team in ["team1", "team2"]:
for count, player in enumerate(match_info[team]["players"]):
stats_table = page.find_all("table", {"class": "stats-table"})[0]
match_info[team][player] = {
"kills": stats_table.find_all("td", {"class": "st-kills"})[count].text.split()[0],
"headshots": stats_table.find_all("td", {"class": "st-kills"})[count].text.split("(")[-1][:-1],
"assists": stats_table.find_all("td", {"class": "st-assists"})[count].text.split()[0],
"flash_assists": stats_table.find_all("td", {"class": "st-assists"})[count].text.split("(")[-1][:-1],
"deaths": stats_table.find_all("td", {"class": "st-deaths"})[count].text,
"kast": stats_table.find_all("td", {"class": "st-kdratio"})[count].text,
"kd_diff": stats_table.find_all("td", {"class": "st-kddiff"})[count].text,
"adr": stats_table.find_all("td", {"class": "st-adr"})[count].text,
"fk_diff": stats_table.find_all("td", {"class": "st-fkdiff"})[count].text,
"rating": stats_table.find_all("td", {"class": "st-rating"})[count].text,
}
return match_info
def _get_current_lineup(player_anchors): def _get_current_lineup(player_anchors):
""" """
helper function for function above helper function for function above
@ -200,6 +161,7 @@ def _get_current_lineup(player_anchors):
player['name'] = buildName[0].rstrip() + buildName[2] player['name'] = buildName[0].rstrip() + buildName[2]
player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text
player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group()) player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group())
player['url'] = "https://hltv.org" + player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href")
players.append(player) players.append(player)
return players return players
@ -212,18 +174,18 @@ def _get_historical_lineup(player_anchors):
for player_anchor in player_anchors[5::]: for player_anchor in player_anchors[5::]:
player = {} player = {}
buildName = player_anchor.find("img", {"class": "container-width"})["alt"].split('\'') buildName = player_anchor.find("img", {"class": "container-width"})["alt"].split('\'')
player['country'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("img", {"class": "flag"})["alt"].encode('utf8') player['country'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("img", {"class": "flag"})["alt"]
player['name'] = buildName[0].rstrip() + buildName[2] player['name'] = buildName[0].rstrip() + buildName[2]
player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text.encode('utf8') player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text
player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group()) player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group())
player['url'] = "https://hltv.org" + player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href")
players.append(player) players.append(player)
return players return players
def get_matches(): def get_matches():
matches = get_parsed_page("http://www.hltv.org/matches/") matches = get_parsed_page("https://www.hltv.org/matches/")
matches_list = [] matches_list = []
upcomingmatches = matches.find("div", {"class": "upcomingMatchesSection"})
matchdays = matches.find_all("div", {"class": "upcomingMatchesSection"}) matchdays = matches.find_all("div", {"class": "upcomingMatchesSection"})
@ -233,17 +195,29 @@ def get_matches():
for getMatch in matchDetails: for getMatch in matchDetails:
matchObj = {} matchObj = {}
matchObj['date'] = date
matchObj['url'] = "https://hltv.org" + getMatch.find("a", {"class": "match a-reset"}).get("href") matchObj['url'] = "https://hltv.org" + getMatch.find("a", {"class": "match a-reset"}).get("href")
matchObj['time'] = getMatch.find("div", {"class": "matchTime"}).text
if (date and getMatch.find("div", {"class": "matchTime"})):
timeFromHLTV = datetime.datetime.strptime(date + " " + getMatch.find("div", {"class": "matchTime"}).text,'%Y-%m-%d %H:%M').replace(tzinfo=HLTV_ZONEINFO)
timeFromHLTV = timeFromHLTV.astimezone(LOCAL_ZONEINFO)
matchObj['date'] = timeFromHLTV.strftime('%Y-%m-%d')
matchObj['time'] = timeFromHLTV.strftime('%H:%M')
timenow = datetime.datetime.now().astimezone(LOCAL_ZONEINFO).strftime('%Y-%m-%d %H:%M')
deadline = date + " " + getMatch.find("div", {"class": "matchTime"}).text
currentTime = datetime.datetime.strptime(timenow,'%Y-%m-%d %H:%M')
ends = datetime.datetime.strptime(deadline, '%Y-%m-%d %H:%M')
if currentTime < ends:
matchObj['countdown'] = str(ends - currentTime)
if getMatch.find("div", {"class": "matchEvent"}): if getMatch.find("div", {"class": "matchEvent"}):
matchObj['event'] = getMatch.find("div", {"class": "matchEvent"}).text.encode('utf8').strip() matchObj['event'] = getMatch.find("div", {"class": "matchEvent"}).text.strip()
else: else:
matchObj['event'] = getMatch.find("div", {"class": "matchInfoEmpty"}).text.encode('utf8').strip() matchObj['event'] = getMatch.find("div", {"class": "matchInfoEmpty"}).text.strip()
if (getMatch.find_all("div", {"class": "matchTeams"})): if (getMatch.find_all("div", {"class": "matchTeams"})):
matchObj['team1'] = getMatch.find_all("div", {"class": "matchTeam"})[0].text.encode('utf8').lstrip().rstrip() matchObj['team1'] = getMatch.find_all("div", {"class": "matchTeam"})[0].text.lstrip().rstrip()
matchObj['team2'] = getMatch.find_all("div", {"class": "matchTeam"})[1].text.encode('utf8').lstrip().rstrip() matchObj['team2'] = getMatch.find_all("div", {"class": "matchTeam"})[1].text.lstrip().rstrip()
else: else:
matchObj['team1'] = None matchObj['team1'] = None
matchObj['team2'] = None matchObj['team2'] = None
@ -253,7 +227,7 @@ def get_matches():
return matches_list return matches_list
def get_results(): def get_results():
results = get_parsed_page("http://www.hltv.org/results/") results = get_parsed_page("https://www.hltv.org/results/")
results_list = [] results_list = []
@ -263,28 +237,36 @@ def get_results():
resultDiv = result.find_all("div", {"class": "result-con"}) resultDiv = result.find_all("div", {"class": "result-con"})
for res in resultDiv: for res in resultDiv:
getRes = res.find("div", {"class": "result"}).find("table")
resultObj = {} resultObj = {}
resultObj['url'] = "https://hltv.org" + res.find("a", {"class": "a-reset"}).get("href")
if (res.parent.find("span", {"class": "standard-headline"})): if (res.parent.find("span", {"class": "standard-headline"})):
resultObj['date'] = res.parent.find("span", {"class": "standard-headline"}).text.encode('utf8') dateText = res.parent.find("span", {"class": "standard-headline"}).text.replace("Results for ", "").replace("th", "")
dateArr = dateText.split()
dateTextFromArrPadded = padIfNeeded(dateArr[2]) + "-" + padIfNeeded(monthNameToNumber(dateArr[0])) + "-" + padIfNeeded(dateArr[1])
dateFromHLTV = datetime.datetime.strptime(dateTextFromArrPadded,'%Y-%m-%d').replace(tzinfo=HLTV_ZONEINFO)
dateFromHLTV = dateFromHLTV.astimezone(LOCAL_ZONEINFO)
resultObj['date'] = dateFromHLTV.strftime('%Y-%m-%d')
else: else:
dt = datetime.date.today() dt = datetime.date.today()
resultObj['date'] = str(dt.day) + '/' + str(dt.month) + '/' + str(dt.year) resultObj['date'] = str(dt.day) + '/' + str(dt.month) + '/' + str(dt.year)
if (res.find("td", {"class": "placeholder-text-cell"})): if (res.find("td", {"class": "placeholder-text-cell"})):
resultObj['event'] = res.find("td", {"class": "placeholder-text-cell"}).text.encode('utf8') resultObj['event'] = res.find("td", {"class": "placeholder-text-cell"}).text
elif (res.find("td", {"class": "event"})): elif (res.find("td", {"class": "event"})):
resultObj['event'] = res.find("td", {"class": "event"}).text.encode('utf8') resultObj['event'] = res.find("td", {"class": "event"}).text
else: else:
resultObj['event'] = None resultObj['event'] = None
if (res.find_all("td", {"class": "team-cell"})): if (res.find_all("td", {"class": "team-cell"})):
resultObj['team1'] = res.find_all("td", {"class": "team-cell"})[0].text.encode('utf8').lstrip().rstrip() resultObj['team1'] = res.find_all("td", {"class": "team-cell"})[0].text.lstrip().rstrip()
resultObj['team1score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[0].text.encode('utf8').lstrip().rstrip()) resultObj['team1score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[0].text.lstrip().rstrip())
resultObj['team2'] = res.find_all("td", {"class": "team-cell"})[1].text.encode('utf8').lstrip().rstrip() resultObj['team2'] = res.find_all("td", {"class": "team-cell"})[1].text.lstrip().rstrip()
resultObj['team2score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[1].text.encode('utf8').lstrip().rstrip()) resultObj['team2score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[1].text.lstrip().rstrip())
else: else:
resultObj['team1'] = None resultObj['team1'] = None
resultObj['team2'] = None resultObj['team2'] = None
@ -318,10 +300,20 @@ def get_results_by_date(start_date, end_date):
t2_score = int(team_cols[1].find_all(attrs={"class": "score"})[0].text.strip()[1:-1]) t2_score = int(team_cols[1].find_all(attrs={"class": "score"})[0].text.strip()[1:-1])
map = result.find(attrs={"class": "statsDetail"}).find(attrs={"class": "dynamic-map-name-full"}).text map = result.find(attrs={"class": "statsDetail"}).find(attrs={"class": "dynamic-map-name-full"}).text
event = result.find(attrs={"class": "event-col"}).text event = result.find(attrs={"class": "event-col"}).text
date = result.find(attrs={"class": "date-col"}).find("a").find("div").text dateText = result.find(attrs={"class": "date-col"}).find("a").find("div").text
url = "https://hltv.org" + result.find(attrs={"class": "date-col"}).find("a").get("href")
dateArr = dateText.split("/")
# TODO: yes, this shouldn't be hardcoded, but I'll be very surprised if this API is still a thing in 21XX
startingTwoDigitsOfYear = "20"
dateTextFromArrPadded = startingTwoDigitsOfYear + padIfNeeded(dateArr[2]) + "-" + padIfNeeded(dateArr[1]) + "-" + padIfNeeded(dateArr[0])
dateFromHLTV = datetime.datetime.strptime(dateTextFromArrPadded,'%Y-%m-%d').replace(tzinfo=HLTV_ZONEINFO)
dateFromHLTV = dateFromHLTV.astimezone(LOCAL_ZONEINFO)
date = dateFromHLTV.strftime('%Y-%m-%d')
result_dict = {"team1": t1, "team2": t2, "team1score": t1_score, result_dict = {"team1": t1, "team2": t2, "team1score": t1_score,
"team2score": t2_score, "date": date, "map": map, "event": event} "team2score": t2_score, "date": date, "map": map, "event": event, "url": url}
# Add this pages results to the result list # Add this pages results to the result list
results_list.append(result_dict) results_list.append(result_dict)

1
requirements.txt
View file

@ -4,3 +4,4 @@ lxml==4.9.1
python-utils==2.0.0 python-utils==2.0.0
requests==2.20.0 requests==2.20.0
six==1.10.0 six==1.10.0
tzlocal==4.2