Initial commit

devtools/create_achievements/steam_achievements.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+
+# This script takes two parameters
+#   - the first is the Steam game id,
+#   - the second (optional) is the GOG Galaxy game id.
+#
+# Example:
+#   ./steam_achievements.py --steamid 631570 [--galaxyid 1845001352]
+#
+# It will extract the Steam stats information to create the ScummVM ini-files for achievements.dat
+# For GOG versions of the games we assume that they are using same achievements as Steam versions
+
+import os
+import sys
+import codecs
+import argparse
+import requests
+import logging
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import requests
+from selenium.webdriver.common.by import By
+from lxml import etree
+
+# For Stats only English strings exists on Steam side.
+#   - stats info is collected from SteamDB, we collect lists of <id, [comment], initial value>
+
+# For Achievements some games have only English strings, and some games have additional translations on Steam side.
+# This script is currently using various sources of information for Steam achievements:
+#   - achievements info collection starts from SteamDB, we collect lists of <id, title, [comment], hidden flag>
+#   - if there are hidden achievements, we look up their English "comment" values on 3rd party site (achievementstats.com)
+#   - there is an additional call to SteamDB to collect list of achievements translations
+#   - if there are no hidden achievements, we use Global Statistics at steamcommunity.com to collect non-English strings
+#   - if there are hidden achievements, we use User Statistics at steamcommunity.com to collect non-English strings
+
+# Hidden achievement descriptions are available only at user accounts for those users who have them completed
+
+STEAM_USERS = {
+	"283880"  : "lb_ii",
+	"370910"  : "lb_ii",
+	"378630"  : "lb_ii",
+	"420180"  : "lb_ii",
+	"551840"  : "lb_ii",
+	"631570"  : "lb_ii",
+	"761460"  : "lb_ii",
+	"1251910" : "lb_ii",
+	"1270590" : "lb_ii",
+}
+
+# Format is: <SteamDB language name>: (<Steam API language code>, <unixLocale>)
+# For <SteamDB language name>, see verbose output of this tool
+# For <Steam API language code>, see https://partner.steamgames.com/doc/store/localization
+# For <unixLocale>, see "common/language.cpp"
+
+LANGUAGES = {
+	"English":             ("english",    "en"),
+	"Dutch":               ("dutch",      "nl_NL"),
+	"German":              ("german",     "de_DE"),
+	"French":              ("french",     "fr_FR"),
+	"Hungarian":           ("hungarian",  "hu_HU"),
+	"Italian":             ("italian",    "it_IT"),
+	"Polish":              ("polish",     "pl_PL"),
+	"Portuguese":          ("portuguese", "pt_PT"),
+	"Portuguese - Brazil": ("brazilian",  "pt_BR"),
+	"Russian":             ("russian",    "ru_RU"),
+	"Spanish - Spain":     ("spanish",    "es_ES"),
+	"Simplified Chinese":  ("schinese",   "zh_CN"),
+}
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--steamid", required=True, default="631570", type=int, help="Steam game id")
+parser.add_argument("--saveasgalaxyid", type=int, help="GOG Galaxy game id")
+parser.add_argument("-v", "--verbose", action="store_true")
+parser.add_argument("-d", "--debug", action="store_true")
+args = parser.parse_args()
+service = Service("/usr/bin/chromedriver")
+driver = webdriver.Chrome(service=service)
+stringify = etree.XPath("string()")
+
+if args.debug:
+	logging.basicConfig(level=logging.DEBUG)
+
+def log(msg):
+	global args
+	if args.verbose:
+		sys.stderr.write(msg + "\n")
+
+def err(msg):
+	sys.stderr.write(msg + "\n")
+	sys.exit(127)
+
+def create_xpath_object(html_body):
+	tree = etree.HTML(html_body)
+	return etree.XPathEvaluator(tree)
+
+def get(url):
+	driver.get(url)
+	return driver.page_source
+
+def xpath(response, xpath):
+	x = create_xpath_object(response)
+	return x(xpath)
+
+def html(response):
+	return repr(response)
+
+def parse_steamdb_info(url):
+	log("query {0}".format(url))
+	response = get(url)
+
+	info_rows = xpath(response, "//div[@id='info']/table/tbody/tr/td")
+	info_columns = 2 # id, text,
+	info_entries = int(len(info_rows) / info_columns)
+	if info_entries == 0:
+		log(html(response))
+		err("found NO information data for {0}".format(url))
+
+	FORMAT_CHECKER_STRING = "Store Release Date"
+	is_format_ok = False
+
+	langs = "English"
+	for i in range(info_entries):
+		idx        = info_columns * i
+		info_key   = stringify(info_rows[idx + 0]).strip()
+		info_value = stringify(info_rows[idx + 1]).strip()
+		if info_key == FORMAT_CHECKER_STRING or info_key == "Steam Release Date":
+			is_format_ok = True
+		if info_key == "Achievement Languages":
+			langs = info_value
+
+	if not is_format_ok:
+		log(html(response))
+		err("found NO {0}\nEntries: {1}".format(FORMAT_CHECKER_STRING, [i.text for i in info_rows][::2]))
+
+	return langs.split(", ")
+
+def parse_steamdb_stats(url):
+	log("query {0}".format(url))
+	response = get(url)
+
+	achievements_rows = xpath(response, "//tr[starts-with(@id, 'achievement-')]/td")
+	achievements_columns = 3 # name, text, img
+	achievements_entries = int(len(achievements_rows) / achievements_columns)
+	if achievements_entries == 0:
+		log(html(response))
+		err("found NO achievements for {0}".format(url))
+
+	stats_rows = xpath(response, "//tr[starts-with(@id, 'stat-')]/td")
+	stats_columns = 3 # name, text, default value
+	stats_entries = int(len(stats_rows) / stats_columns)
+
+	stats_en = {}
+	for i in range(stats_entries):
+		idx   = stats_columns * i
+		name  = stats_rows[idx + 0].text.strip()
+		descr = stats_rows[idx + 1].text.strip()
+		start = stats_rows[idx + 2].text.strip()
+		if descr == "no name":
+			descr = ""
+		stats_en[i] = (name, descr, start)
+
+	achievements_en = {}
+	for i in range(achievements_entries):
+		idx   = achievements_columns * i
+		name  = achievements_rows[idx + 0].text.strip()
+		texts = stringify(achievements_rows[idx + 1]).strip().replace("\n\n", "\n").split("\n")
+
+		if len(texts) != 2:
+			log(html(response))
+			err("Unexpected description format: {0}".format(texts))
+
+		title = texts[0]
+		descr = texts[1]
+		hide  = descr == "Hidden."
+		if descr in ["No description.", "Hidden."]:
+			descr = ""
+		achievements_en[i] = (name, title, descr, hide)
+
+	return achievements_en, stats_en
+
+def parse_steamcommunity_stats(url):
+	log("query {0}".format(url))
+	response = get(url)
+
+	achievements_rows = xpath(response, "//div[@class='achieveRow']")
+	achievements_entries = len(achievements_rows)
+	if achievements_entries == 0:
+		log(html(response))
+		err("found NO achievements for {0}".format(url))
+
+	translation = {}
+	for idx in range(achievements_entries):
+		imgs  = achievements_rows[idx].xpath("//img/@src")
+		titles = achievements_rows[idx].xpath(".//div[@class='achieveTxt']/h3/text()")
+		descrs = achievements_rows[idx].xpath(".//div[@class='achieveTxt']/h5/text()")
+
+		if len(imgs) != 1:
+			log(html(response))
+			err("Unexpected xpath result: expected exactly one img tag per achievement")
+		if len(titles) != 1:
+			log(html(response))
+			err("Unexpected xpath result: expected exactly one h3 tag per achievement")
+		if len(descrs) > 1:
+			log(html(response))
+			err("Unexpected xpath result: expected zero or one h5 tag per achievement")
+
+		translation[imgs[0]] = (titles[0].strip(), descrs[0].strip() if descrs else None)
+
+	return translation
+
+def parse_achievementstats_stats(url):
+	log("query {0}".format(url))
+	response = get(url)
+
+	tables = xpath(response, "//table")
+	if len(tables) != 1:
+		log(html(response))
+		err("Unexpected xpath result: expected exactly one table tag on page")
+
+	achievements_rows = xpath(response, "//tbody/tr/td")
+	achievements_columns = 6 # icon, name, text, date, point, report
+	achievements_entries = int(len(achievements_rows) / achievements_columns)
+	if achievements_entries == 0:
+		log(html(response))
+		err("found NO achievements")
+
+	result = {}
+	for i in range(achievements_entries):
+		idx   = achievements_columns * i
+		name  = achievements_rows[idx + 1].text.strip()
+		descr = achievements_rows[idx + 2].text.strip()
+		result[name] = descr
+	return result
+
+def join_achievements_translation(achievements_en, translations):
+	achievements = {"en": achievements_en}
+
+	entitle2img = {}
+	for img, (title, descr) in translations["English"].items():
+		entitle2img[title] = img
+
+	for l in translations.keys():
+		lang_id = LANGUAGES[l][1]
+		if lang_id == "en":
+			continue
+
+		achievements[lang_id] = {}
+		for i, (name, title, descr, hide) in achievements_en.items():
+			if  not title in entitle2img:
+				err("Can't find '{0}' at {1}".format(title, entitle2img))
+
+			t = translations[l][entitle2img[title]]
+			achievements[lang_id][i] = (name, t[0], t[1] if t[1] else descr, hide)
+
+		completely_same = True
+		for i, it in achievements_en.items():
+			if achievements[lang_id][i] != it:
+				completely_same = False
+				break
+		if completely_same:
+			print("WARNING: Translation is available for {0}, but is completely the same as English for game {1}".format(l, args.steamid))
+			del achievements[lang_id]
+
+	return achievements
+
+def join_achievements_descr(achievements_en, descs):
+	result = {}
+	for i, (name, title, descr, hide) in achievements_en.items():
+		ext_descr = descrs[title]
+		if descr and descr != ext_descr:
+			err("Unexpected difference between {0} and {1} for {2}".format(descr, ext_descr, title))
+		result[i] = (name, title, descr if descr else ext_descr, hide)
+	return result
+
+def write_ini(fname, achievements, stats):
+	log("writing: {0}".format(fname))
+	with codecs.open(fname, "w", encoding="utf-8") as out:
+		for lang, it in stats.items():
+			out.write("[stats:{0}]\n".format(lang))
+			for i, (name, descr, start) in it.items():
+				out.write("item_%d_id=%s\n" % (i, name))
+				if descr:
+					out.write("item_%d_comment=%s\n" % (i, descr))
+				out.write("item_%d_start=%s\n" % (i, start))
+		for lang, it in achievements.items():
+			out.write("[achievements:{0}]\n".format(lang))
+			for i, (name, title, descr, hide) in it.items():
+				out.write("item_%d_id=%s\n" % (i, name))
+				out.write("item_%d_title=%s\n" % (i, title))
+				if descr:
+					out.write("item_%d_comment=%s\n" % (i, descr))
+				if hide:
+					out.write("item_%d_hidden=true\n" % i)
+
+try:
+	STATS_URL = "https://steamdb.info/app/{0}/stats/".format(args.steamid)
+	achievements_en, stats_en = parse_steamdb_stats(STATS_URL)
+	log("found {0} achievements".format(len(achievements_en)))
+	log("found {0} stats".format(len(stats_en)))
+
+	hidden_achievements = [it for it in achievements_en.values() if it[3]]
+	log("found {0} hidden achievements".format(len(hidden_achievements)))
+
+	if hidden_achievements:
+		HIDDEN_STATS_URL = "https://www.achievementstats.com/index.php?action=games&gameId={0}".format(args.steamid)
+		descrs = parse_achievementstats_stats(HIDDEN_STATS_URL)
+		achievements_en = join_achievements_descr(achievements_en, descrs)
+
+	INFO_URL = "https://steamdb.info/app/{0}/info/".format(args.steamid)
+	langs = parse_steamdb_info(INFO_URL)
+	log("found langs: {0}".format(langs))
+
+	translations = {"English":{}}
+	if len(langs) > 1:
+		for l in langs:
+			if not hidden_achievements or len(langs) == 1:
+				TRANSLATION_URL = "https://steamcommunity.com/stats/{0}/achievements?l={1}".format(args.steamid, LANGUAGES[l][0])
+				translations[l] = parse_steamcommunity_stats(TRANSLATION_URL)
+			elif str(args.steamid) in STEAM_USERS:
+				TRANSLATION_URL = "https://steamcommunity.com/id/{0}/stats/{1}/?l={2}".format(STEAM_USERS[str(args.steamid)], args.steamid, LANGUAGES[l][0])
+				translations[l] = parse_steamcommunity_stats(TRANSLATION_URL)
+			else:
+				print("WARNING: ignoring {0} localization for game {1} since it has hidden achievements not listed at STEAM_USERS, please buy, complete and add this game!".format(l, args.steamid))
+
+	achievements = join_achievements_translation(achievements_en, translations)
+	stats = {"en": stats_en} if stats_en else {}
+
+	if args.saveasgalaxyid:
+		FNAME = "galaxy-{0}.ini".format(args.saveasgalaxyid)
+	else:
+		FNAME = "steam-{0}.ini".format(args.steamid)
+	write_ini(os.path.join("gen", FNAME), achievements, stats)
+	driver.quit()
+
+except requests.exceptions.RequestException as e:
+	print(e)
+	driver.quit()
+	sys.exit(127)