Kate
bcd06fb05d
-this only applies to the 14 biggest groups - is needed to stay under the 16MB document limit in mongodb
103 lines
2.9 KiB
Python
103 lines
2.9 KiB
Python
import requests, time
|
|
from bs4 import BeautifulSoup
|
|
from ratelimit import limits, sleep_and_retry
|
|
|
|
# Set up rate limiter, one request per second
|
|
CALLS = 1
|
|
RATE_LIMIT = 10
|
|
|
|
|
|
@sleep_and_retry
|
|
@limits(calls=CALLS, period=RATE_LIMIT)
|
|
def make_request(url):
|
|
try:
|
|
response = requests.get(url)
|
|
if response.status_code == 429:
|
|
print("HTTP 429 Too Many Requests received. Pausing for 30 seconds.")
|
|
time.sleep(30)
|
|
return make_request(url)
|
|
response.raise_for_status()
|
|
return response
|
|
except requests.HTTPError as http_err:
|
|
print(f"HTTP error occurred: {http_err}")
|
|
return None
|
|
except Exception as err:
|
|
print(f"Other error occurred: {err}")
|
|
return None
|
|
|
|
|
|
def get_group_details(group_url_name):
|
|
# Regular group page URL
|
|
group_url = f"https://steamcommunity.com/groups/{group_url_name}"
|
|
|
|
group_page_response = make_request(group_url)
|
|
|
|
if not group_page_response:
|
|
return
|
|
|
|
group_page_soup = BeautifulSoup(group_page_response.text, "lxml")
|
|
|
|
# Group Tag
|
|
tag_span = group_page_soup.find("span", {"class": "grouppage_header_abbrev"})
|
|
tag = tag_span.text.strip() if tag_span else "No tag"
|
|
|
|
# Initialize an empty list to store all members
|
|
all_members = []
|
|
|
|
# Start with the first page
|
|
next_page_url = (
|
|
f"https://steamcommunity.com/groups/{group_url_name}/memberslistxml/?xml=1"
|
|
)
|
|
|
|
# Initialize a counter for pages
|
|
page_counter = 0
|
|
|
|
while next_page_url and page_counter < 500:
|
|
# Group details XML page URL
|
|
group_details_url = next_page_url
|
|
|
|
group_details_response = make_request(group_details_url)
|
|
|
|
if not group_details_response:
|
|
return
|
|
|
|
try:
|
|
print(f"[*] Getting page {next_page_url}...")
|
|
group_details_soup = BeautifulSoup(
|
|
group_details_response.content, "lxml-xml"
|
|
)
|
|
|
|
# Group Name
|
|
group_name = group_details_soup.find("groupName").text
|
|
|
|
# Group ID64
|
|
group_id64 = group_details_soup.find("groupID64").text
|
|
|
|
# Member List
|
|
members = [
|
|
member.text for member in group_details_soup.find_all("steamID64")
|
|
]
|
|
all_members.extend(members)
|
|
|
|
# Get the URL for the next page, if there is one
|
|
next_page_link = group_details_soup.find("nextPageLink")
|
|
next_page_url = next_page_link.text if next_page_link else None
|
|
|
|
# Increment page counter
|
|
page_counter += 1
|
|
|
|
except Exception as err:
|
|
print(f"Error occurred during parsing of group details XML page: {err}")
|
|
|
|
return {
|
|
"id64": group_id64,
|
|
"name": group_name,
|
|
"url": group_url,
|
|
"tag": tag,
|
|
"members": all_members,
|
|
}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Replace 'ilovebloop' with the desired group URL name
|
|
print(get_group_details("steamworks"))
|