From c438a70a30ed482107af6ed429b9ecc8e1f2bdca Mon Sep 17 00:00:00 2001 From: David Silva Date: Thu, 14 Jul 2022 15:25:59 +0200 Subject: [PATCH] feat(main): ids and mappers --- README.md | 157 +++++++++++++++++++++++++++++++++++++++++++++--------- main.py | 58 +++++++++++++++----- 2 files changed, 179 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 0a5bbd5..32ce902 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,13 @@ Provides an API for HLTV ```python >>> import main as hltv >>> hltv.top5teams() -[{'name': 'Natus Vincere', 'url': 'https://hltv.org/team/4608/natus-vincere'}, - {'name': 'FaZe', 'url': 'https://hltv.org/team/6667/faze'}, - {'name': 'ENCE', 'url': 'https://hltv.org/team/4869/ence'}, - {'name': 'Cloud9', 'url': 'https://hltv.org/team/5752/cloud9'}, - {'name': 'Vitality', 'url': 'https://hltv.org/team/9565/vitality'}] +[{'id': 4608, + 'name': 'Natus Vincere', + 'url': 'https://hltv.org/team/4608/natus-vincere'}, + {'id': 6667, 'name': 'FaZe', 'url': 'https://hltv.org/team/6667/faze'}, + {'id': 4869, 'name': 'ENCE', 'url': 'https://hltv.org/team/4869/ence'}, + {'id': 5752, 'name': 'Cloud9', 'url': 'https://hltv.org/team/5752/cloud9'}, + {'id': 9565, 'name': 'Vitality', 'url': 'https://hltv.org/team/9565/vitality'}] ``` ## `top30teams` @@ -48,6 +50,7 @@ Provides an API for HLTV ```python >>> hltv.top_players() [{'country': 'France', + 'id': 11893, 'maps-played': '1020', 'name': 'Mathieu Herbaut', 'nickname': 'ZywOo', @@ -61,23 +64,23 @@ Provides an API for HLTV ```python >>> hltv.get_players("6665") -[{'id': '4954', +[{'id': 4954, 'name': "Andreas 'Xyp9x' Højsleth", 'nickname': 'Xyp9x', 'url': 'https://hltv.org/player/4954/xyp9x'}, - {'id': '7412', + {'id': 7412, 'name': "Lukas 'gla1ve' Rossander", 'nickname': 'gla1ve', 'url': 'https://hltv.org/player/7412/gla1ve'}, - {'id': '9078', + {'id': 9078, 'name': "Kristian 'k0nfig' Wienecke", 'nickname': 'k0nfig', 'url': 'https://hltv.org/player/9078/k0nfig'}, - {'id': '13300', + {'id': 13300, 'name': "Asger 'Farlig' Jensen", 'nickname': 'Farlig', 'url': 'https://hltv.org/player/13300/farlig'}, - {'id': '15165', + {'id': 15165, 'name': "Benjamin 'blameF' Bremer", 'nickname': 'blameF', 'url': 'https://hltv.org/player/15165/blamef'}] @@ -88,53 +91,150 @@ Provides an API for HLTV ```python >>> hltv.get_team_info("6665") {'current-lineup': [{'country': 'Denmark', + 'id': 4954, 'maps-played': 1239, 'name': 'Andreas Højsleth', 'nickname': 'Xyp9x', 'url': 'https://hltv.org/stats/players/4954/xyp9x'}, {'country': 'Denmark', + 'id': 7412, 'maps-played': 1094, 'name': 'Lukas Rossander', 'nickname': 'gla1ve', 'url': 'https://hltv.org/stats/players/7412/gla1ve'}, {'country': 'Denmark', + 'id': 9078, 'maps-played': 139, 'name': 'Kristian Wienecke', 'nickname': 'k0nfig', 'url': 'https://hltv.org/stats/players/9078/k0nfig'}, {'country': 'Denmark', + 'id': 13300, 'maps-played': 82, 'name': 'Asger Jensen', 'nickname': 'Farlig', 'url': 'https://hltv.org/stats/players/13300/farlig'}, {'country': 'Denmark', + 'id': 15165, 'maps-played': 139, 'name': 'Benjamin Bremer', 'nickname': 'blameF', 'url': 'https://hltv.org/stats/players/15165/blamef'}], 'historical-players': [{'country': 'Denmark', + 'id': 9612, 'maps-played': 86, 'name': 'Lucas Andersen', 'nickname': 'Bubzkji', 'url': 'https://hltv.org/stats/players/9612/bubzkji'}, {'country': 'Denmark', + 'id': 9612, 'maps-played': 86, 'name': 'Lucas Andersen', 'nickname': 'Bubzkji', 'url': 'https://hltv.org/stats/players/9612/bubzkji'}, {'country': 'Denmark', + 'id': 7592, 'maps-played': 1076, 'name': 'Nicolai Reedtz', 'nickname': 'device', 'url': 'https://hltv.org/stats/players/7592/device'}, - ... - ], + {'country': 'Denmark', + 'id': 7398, + 'maps-played': 1179, + 'name': 'Peter Rasmussen', + 'nickname': 'dupreeh', + 'url': 'https://hltv.org/stats/players/7398/dupreeh'}, + {'country': 'Denmark', + 'id': 8611, + 'maps-played': 53, + 'name': 'Patrick Hansen', + 'nickname': 'es3tag', + 'url': 'https://hltv.org/stats/players/8611/es3tag'}, + {'country': 'Denmark', + 'id': 429, + 'maps-played': 185, + 'name': 'Finn Andersen', + 'nickname': 'karrigan', + 'url': 'https://hltv.org/stats/players/429/karrigan'}, + {'country': 'Denmark', + 'id': 8394, + 'maps-played': 355, + 'name': 'Markus Kjærbye', + 'nickname': 'Kjaerbye', + 'url': 'https://hltv.org/stats/players/8394/kjaerbye'}, + {'country': 'Denmark', + 'id': 9032, + 'maps-played': 732, + 'name': 'Emil Reif', + 'nickname': 'Magisk', + 'url': 'https://hltv.org/stats/players/9032/magisk'}, + {'country': 'Denmark', + 'id': 21, + 'maps-played': 12, + 'name': 'Danny Sørensen', + 'nickname': 'zonic', + 'url': 'https://hltv.org/stats/players/21/zonic'}, + {'country': 'Denmark', + 'id': 2469, + 'maps-played': 90, + 'name': 'René Borg', + 'nickname': 'cajunb', + 'url': 'https://hltv.org/stats/players/2469/cajunb'}, + {'country': 'Denmark', + 'id': 13843, + 'maps-played': 103, + 'name': 'Philip Ewald', + 'nickname': 'Lucky', + 'url': 'https://hltv.org/stats/players/13843/lucky'}, + {'country': 'Norway', + 'id': 1485, + 'maps-played': 10, + 'name': 'Ruben Villarroel', + 'nickname': 'RUBINO', + 'url': 'https://hltv.org/stats/players/1485/rubino'}, + {'country': 'Denmark', + 'id': 16718, + 'maps-played': 1, + 'name': 'Frederik Sørensen', + 'nickname': 'Fessor', + 'url': 'https://hltv.org/stats/players/16718/fessor'}, + {'country': 'Denmark', + 'id': 9903, + 'maps-played': 4, + 'name': 'Anton Pedersen', + 'nickname': 'notaN', + 'url': 'https://hltv.org/stats/players/9903/notan'}, + {'country': 'Denmark', + 'id': 8783, + 'maps-played': 15, + 'name': 'Jakob Hansen', + 'nickname': 'JUGi', + 'url': 'https://hltv.org/stats/players/8783/jugi'}, + {'country': 'Denmark', + 'id': 7154, + 'maps-played': 4, + 'name': 'Jacob Winneche', + 'nickname': 'Pimp', + 'url': 'https://hltv.org/stats/players/7154/pimp'}, + {'country': 'Denmark', + 'id': 922, + 'maps-played': 9, + 'name': 'Marco Pfeiffer', + 'nickname': 'Snappi', + 'url': 'https://hltv.org/stats/players/922/snappi'}, + {'country': 'Sweden', + 'id': 1146, + 'maps-played': 8, + 'name': 'Dennis Edman', + 'nickname': 'dennis', + 'url': 'https://hltv.org/stats/players/1146/dennis'}], 'stats': {'K/D Ratio': '1.10', 'Maps played': '1323', 'Rounds played': '34752', 'Total deaths': '110203', 'Total kills': '120679', 'Wins / draws / losses': '860 / 2 / 461'}, + 'team-id': 6665, 'team-name': 'Astralis', 'url': 'https://hltv.org/stats/team/6665/Astralis'} ``` @@ -143,13 +243,16 @@ Provides an API for HLTV ```python >>> hltv.get_matches() -[{'countdown': '1:15:00', +[{'countdown': '0:13:00', 'date': '2022-07-14', - 'event': 'ESL Challenger Melbourne 2022 Asia Closed Qualifier', - 'team1': 'IHC', - 'team2': 'Wings Up', - 'time': '15:00', - 'url': 'https://hltv.org/matches/2357388/ihc-vs-wings-up-esl-challenger-melbourne-2022-asia-closed-qualifier'}, + 'event': 'ESL Challenger Melbourne 2022 Europe Closed Qualifier', + 'match-id': 2357360, + 'team1': 'HEET', + 'team1-id': 11501, + 'team2': 'Entropiq', + 'team2-id': 10831, + 'time': '15:30', + 'url': 'https://hltv.org/matches/2357360/heet-vs-entropiq-esl-challenger-melbourne-2022-europe-closed-qualifier'}, ... ] ``` @@ -160,9 +263,12 @@ Provides an API for HLTV >>> hltv.get_results() [{'date': '2022-07-14', 'event': 'ESL Challenger Melbourne 2022 Asia Closed Qualifier', + 'match-id': 2357387, 'team1': 'YK', + 'team1-id': 11542, 'team1score': 0, 'team2': 'Wings Up', + 'team2-id': 10851, 'team2score': 2, 'url': 'https://hltv.org/matches/2357387/yk-vs-wings-up-esl-challenger-melbourne-2022-asia-closed-qualifier'}, ... @@ -174,13 +280,16 @@ Provides an API for HLTV ```python >>> hltv.get_results_by_date() [{'date': '2022-07-14', - 'event': 'ESL Challenger Melbourne 2022 Asia Closed Qualifier', - 'map': 'Overpass', - 'team1': 'Wings Up', + 'event': 'ESL Challenger Melbourne 2022 Oceania Closed Qualifier', + 'map': 'Dust2', + 'match-id': 140752, + 'team1': 'VERTEX', + 'team1-id': 10672, 'team1score': 16, - 'team2': 'YK', - 'team2score': 8, - 'url': 'https://hltv.org/stats/matches/mapstatsid/140750/wings-up-vs-yk?startDate=2022-07-14&endDate=2022-07-14'}, + 'team2': 'Encore', + 'team2-id': 11726, + 'team2score': 6, + 'url': 'https://hltv.org/stats/matches/mapstatsid/140752/vertex-vs-encore?startDate=2022-07-14&endDate=2022-07-14'}, ... ] ``` diff --git a/main.py b/main.py index 8b34f37..16a38cd 100644 --- a/main.py +++ b/main.py @@ -12,13 +12,28 @@ HLTV_ZONEINFO=zoneinfo.ZoneInfo(HLTV_COOKIE_TIMEZONE) LOCAL_TIMEZONE_NAME = tzlocal.get_localzone_name() LOCAL_ZONEINFO = zoneinfo.ZoneInfo(LOCAL_TIMEZONE_NAME) -def padIfNeeded(numberStr: str): +TEAM_MAP_FOR_RESULTS = [] +def _get_all_teams(): + if not TEAM_MAP_FOR_RESULTS: + teams = get_parsed_page("https://www.hltv.org/stats/teams?minMapCount=0") + for team in teams.find_all("td", {"class": ["teamCol-teams-overview"], }): + team = {'id': converters.to_int(team.find("a")["href"].split("/")[-2]), 'name': team.find("a").text, 'url': "https://hltv.org" + team.find("a")["href"]} + TEAM_MAP_FOR_RESULTS.append(team) + +def _findTeamId(teamName: str): + _get_all_teams() + for team in TEAM_MAP_FOR_RESULTS: + if team['name'] == teamName: + return team['id'] + return None + +def _padIfNeeded(numberStr: str): if int(numberStr) < 10: return str(numberStr).zfill(2) else: return str(numberStr) -def monthNameToNumber(monthName: str): +def _monthNameToNumber(monthName: str): months = { 'January': 1, 'February': 2, @@ -50,12 +65,11 @@ def get_parsed_page(url, delay=0.5): return BeautifulSoup(requests.get(url, headers=headers, cookies=cookies).text, "lxml") - def top5teams(): home = get_parsed_page("https://hltv.org/") teams = [] for team in home.find_all("div", {"class": ["col-box rank"], }): - team = {'name': team.text[3:], 'url': "https://hltv.org" + team.find_all("a")[1]["href"]} + team = {'id': _findTeamId(team.text[3:]), 'name': team.text[3:], 'url': "https://hltv.org" + team.find_all("a")[1]["href"]} teams.append(team) return teams @@ -68,7 +82,7 @@ def top30teams(): newteam = {'name': team.find('div', {"class": "ranking-header"}).select('.name')[0].text.strip(), 'rank': converters.to_int(team.select('.position')[0].text.strip(), regexp=True), 'rank-points': converters.to_int(team.find('span', {'class': 'points'}).text, regexp=True), - 'team-id': converters.to_int(team.find('a', {'class': 'details moreLink'})['href'].split('/')[-1]), + 'team-id': _findTeamId(team.find('div', {"class": "ranking-header"}).select('.name')[0].text.strip()), 'team-url': "https://hltv.org/team/" + team.find('a', {'class': 'details moreLink'})['href'].split('/')[-1] + "/" + team.find('div', {"class": "ranking-header"}).select('.name')[0].text.strip(), 'stats-url': "https://www.hltv.org" + team.find('a', {'class': 'details moreLink'})['href'], 'team-players': []} @@ -95,6 +109,7 @@ def top_players(): playerObj['rating'] = player.find('div', {'class': 'rating'}).find('span', {'class': 'bold'}).text playerObj['maps-played'] = player.find('div', {'class': 'average gtSmartphone-only'}).find('span', {'class': 'bold'}).text playerObj['url'] = "https://hltv.org" + player.find('a', {'class': 'name'}).get('href') + playerObj['id'] = converters.to_int(player.find('a', {'class': 'name'}).get('href').split("/")[-2]) playersArray.append(playerObj) return playersArray @@ -105,7 +120,7 @@ def get_players(teamid): players = [] for player_link in titlebox.find_all("a"): players.append({ - 'id': player_link["href"].split("/")[2], + 'id': converters.to_int(player_link["href"].split("/")[2]), 'nickname': player_link["title"], 'name': player_link.find("img")['title'], 'url': "https://hltv.org" + player_link["href"] @@ -123,7 +138,9 @@ def get_team_info(teamid): page = get_parsed_page("https://www.hltv.org/?pageid=179&teamid=" + str(teamid)) team_info = {} - team_info['team-name']=page.find("div", {"class": "context-item"}).text + team_info['team-name'] = page.find("div", {"class": "context-item"}).text + + team_info['team-id'] = _findTeamId(page.find("div", {"class": "context-item"}).text) current_lineup = _get_current_lineup(page.find_all("div", {"class": "col teammate"})) team_info['current-lineup'] = current_lineup @@ -162,6 +179,7 @@ def _get_current_lineup(player_anchors): player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group()) player['url'] = "https://hltv.org" + player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href") + player['id'] = converters.to_int(player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href").split("/")[-2]) players.append(player) return players @@ -179,10 +197,10 @@ def _get_historical_lineup(player_anchors): player['nickname'] = player_anchor.find("div", {"class": "teammate-info standard-box"}).find("div", {"class": "text-ellipsis"}).text player['maps-played'] = int(re.search(r'\d+', player_anchor.find("div", {"class": "teammate-info standard-box"}).find("span").text).group()) player['url'] = "https://hltv.org" + player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href") + player['id'] = converters.to_int(player_anchor.find("div", {"class": "teammate-info standard-box"}).find("a").get("href").split("/")[-2]) players.append(player) return players - def get_matches(): matches = get_parsed_page("https://www.hltv.org/matches/") matches_list = [] @@ -196,7 +214,8 @@ def get_matches(): matchObj = {} matchObj['url'] = "https://hltv.org" + getMatch.find("a", {"class": "match a-reset"}).get("href") - + matchObj['match-id'] = converters.to_int(getMatch.find("a", {"class": "match a-reset"}).get("href").split("/")[-2]) + if (date and getMatch.find("div", {"class": "matchTime"})): timeFromHLTV = datetime.datetime.strptime(date + " " + getMatch.find("div", {"class": "matchTime"}).text,'%Y-%m-%d %H:%M').replace(tzinfo=HLTV_ZONEINFO) timeFromHLTV = timeFromHLTV.astimezone(LOCAL_ZONEINFO) @@ -217,10 +236,14 @@ def get_matches(): if (getMatch.find_all("div", {"class": "matchTeams"})): matchObj['team1'] = getMatch.find_all("div", {"class": "matchTeam"})[0].text.lstrip().rstrip() + matchObj['team1-id'] = _findTeamId(getMatch.find_all("div", {"class": "matchTeam"})[0].text.lstrip().rstrip()) matchObj['team2'] = getMatch.find_all("div", {"class": "matchTeam"})[1].text.lstrip().rstrip() + matchObj['team2-id'] = _findTeamId(getMatch.find_all("div", {"class": "matchTeam"})[1].text.lstrip().rstrip()) else: matchObj['team1'] = None + matchObj['team1-id'] = None matchObj['team2'] = None + matchObj['team2-id'] = None matches_list.append(matchObj) @@ -240,13 +263,15 @@ def get_results(): resultObj = {} resultObj['url'] = "https://hltv.org" + res.find("a", {"class": "a-reset"}).get("href") + + resultObj['match-id'] = converters.to_int(res.find("a", {"class": "a-reset"}).get("href").split("/")[-2]) if (res.parent.find("span", {"class": "standard-headline"})): dateText = res.parent.find("span", {"class": "standard-headline"}).text.replace("Results for ", "").replace("th", "") dateArr = dateText.split() - dateTextFromArrPadded = padIfNeeded(dateArr[2]) + "-" + padIfNeeded(monthNameToNumber(dateArr[0])) + "-" + padIfNeeded(dateArr[1]) + dateTextFromArrPadded = _padIfNeeded(dateArr[2]) + "-" + _padIfNeeded(_monthNameToNumber(dateArr[0])) + "-" + _padIfNeeded(dateArr[1]) dateFromHLTV = datetime.datetime.strptime(dateTextFromArrPadded,'%Y-%m-%d').replace(tzinfo=HLTV_ZONEINFO) dateFromHLTV = dateFromHLTV.astimezone(LOCAL_ZONEINFO) @@ -265,11 +290,17 @@ def get_results(): if (res.find_all("td", {"class": "team-cell"})): resultObj['team1'] = res.find_all("td", {"class": "team-cell"})[0].text.lstrip().rstrip() resultObj['team1score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[0].text.lstrip().rstrip()) + resultObj['team1-id'] = _findTeamId(res.find_all("td", {"class": "team-cell"})[0].text.lstrip().rstrip()) resultObj['team2'] = res.find_all("td", {"class": "team-cell"})[1].text.lstrip().rstrip() + resultObj['team2-id'] = _findTeamId(res.find_all("td", {"class": "team-cell"})[1].text.lstrip().rstrip()) resultObj['team2score'] = converters.to_int(res.find("td", {"class": "result-score"}).find_all("span")[1].text.lstrip().rstrip()) else: resultObj['team1'] = None + resultObj['team1-id'] = None + resultObj['team1score'] = None resultObj['team2'] = None + resultObj['team2-id'] = None + resultObj['team2score'] = None results_list.append(resultObj) @@ -295,24 +326,27 @@ def get_results_by_date(start_date, end_date): for result in pastresults: team_cols = result.find_all("td", {"class": "team-col"}) t1 = team_cols[0].find("a").text + t1_id = _findTeamId(team_cols[0].find("a").text) t2 = team_cols[1].find("a").text + t2_id = _findTeamId(team_cols[1].find("a").text) t1_score = int(team_cols[0].find_all(attrs={"class": "score"})[0].text.strip()[1:-1]) t2_score = int(team_cols[1].find_all(attrs={"class": "score"})[0].text.strip()[1:-1]) map = result.find(attrs={"class": "statsDetail"}).find(attrs={"class": "dynamic-map-name-full"}).text event = result.find(attrs={"class": "event-col"}).text dateText = result.find(attrs={"class": "date-col"}).find("a").find("div").text url = "https://hltv.org" + result.find(attrs={"class": "date-col"}).find("a").get("href") + match_id = converters.to_int(result.find(attrs={"class": "date-col"}).find("a").get("href").split("/")[-2]) dateArr = dateText.split("/") # TODO: yes, this shouldn't be hardcoded, but I'll be very surprised if this API is still a thing in 21XX startingTwoDigitsOfYear = "20" - dateTextFromArrPadded = startingTwoDigitsOfYear + padIfNeeded(dateArr[2]) + "-" + padIfNeeded(dateArr[1]) + "-" + padIfNeeded(dateArr[0]) + dateTextFromArrPadded = startingTwoDigitsOfYear + _padIfNeeded(dateArr[2]) + "-" + _padIfNeeded(dateArr[1]) + "-" + _padIfNeeded(dateArr[0]) dateFromHLTV = datetime.datetime.strptime(dateTextFromArrPadded,'%Y-%m-%d').replace(tzinfo=HLTV_ZONEINFO) dateFromHLTV = dateFromHLTV.astimezone(LOCAL_ZONEINFO) date = dateFromHLTV.strftime('%Y-%m-%d') - result_dict = {"team1": t1, "team2": t2, "team1score": t1_score, + result_dict = {"match-id": match_id, "team1": t1, "team1-id": t1_id, "team2": t2, "team2-id": t2_id, "team1score": t1_score, "team2score": t2_score, "date": date, "map": map, "event": event, "url": url} # Add this pages results to the result list