import re import subprocess import sys import os import pathlib import shutil import argparse import json import zipfile from typing import List from urllib.request import Request, urlopen from warnings import catch_warnings class Globals: SEVEN_ZIP_EXECUTABLE = None def findWorkingExecutablePath(executable_paths, flags): #type: (List[str], List[str]) -> str """ Try to execute each path in executable_paths to see which one can be called and returns exit code 0 The 'flags' argument is any extra flags required to make the executable return 0 exit code :param executable_paths: a list [] of possible executable paths (eg. "./7za", "7z") :param flags: a list [] of any extra flags like "-h" required to make the executable have a 0 exit code :return: the path of the valid executable, or None if no valid executables found """ with open(os.devnull, 'w') as os_devnull: for path in executable_paths: try: if subprocess.call([path] + flags, stdout=os_devnull, stderr=os_devnull) == 0: return path except: pass return None # Get the github ref GIT_TAG = None GIT_REF = os.environ.get("GITHUB_REF") # Github Tag / Version info if GIT_REF is not None: GIT_TAG = GIT_REF.split("/")[-1] print(f"--- Git Ref: {GIT_REF} Git Tag: {GIT_TAG} ---") chapter_to_chapter_number = { "onikakushi": 1, "watanagashi": 2, "tatarigoroshi": 3, "himatsubushi": 4, "meakashi": 5, "tsumihoroboshi": 6, "minagoroshi": 7, "matsuribayashi": 8, "rei": 9, "hou-plus": 10, } class BuildVariant: def __init__(self, short_description, chapter, unity, system, target_crc32=None, translation_default=False): self.chapter = chapter self.unity = unity self.system = system self.target_crc32 = target_crc32 self.chapter_number = chapter_to_chapter_number[chapter] self.data_dir = f"HigurashiEp{self.chapter_number:02}_Data" self.translation_default = translation_default self.short_description = short_description def get_build_command(self) -> str: args = [self.chapter, self.unity, self.system] if self.target_crc32 is not None: args.append(self.target_crc32) return " ".join(args) def get_translation_sharedassets_name(self) -> str: operatingSystem = None if self.system == "win": operatingSystem = "Windows" elif self.system == "unix": operatingSystem = "LinuxMac" elif self.system == "mac": operatingSystem = "Mac" else: raise Exception(f"Unknown system {self.system}") args = [operatingSystem, self.short_description, self.unity] if self.target_crc32 is not None: args.append(self.target_crc32) name_no_ext = "-".join(args) return f"{name_no_ext}.languagespecificassets" # List of build variants for any given chapter # # There must be a corresponding vanilla sharedassets0.assets file located at: # assets\vanilla\{CHAPTER_NAME}[-{CRC32}]\{OS}-{UNITY_VERSION}\sharedassets0.assets # for each entry. chapter_to_build_variants = { "onikakushi": [ BuildVariant("GOG-MG-Steam", "onikakushi", "5.2.2f1", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "onikakushi", "5.2.2f1", "unix"), ], "watanagashi": [ BuildVariant("GOG-MG-Steam", "watanagashi", "5.2.2f1", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "watanagashi", "5.2.2f1", "unix"), ], "tatarigoroshi": [ BuildVariant("GOG-Steam", "tatarigoroshi", "5.4.0f1", "win", translation_default=True), BuildVariant("GOG-Steam", "tatarigoroshi", "5.4.0f1", "unix"), BuildVariant("MG", "tatarigoroshi", "5.3.5f1", "win"), BuildVariant("Legacy", "tatarigoroshi", "5.3.4p1", "win"), BuildVariant("MG", "tatarigoroshi", "5.3.4p1", "unix"), ], "himatsubushi": [ BuildVariant("GOG-MG-Steam", "himatsubushi", "5.4.1f1", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "himatsubushi", "5.4.1f1", "unix"), ], "meakashi": [ BuildVariant("MG-Steam-GOG_old", "meakashi", "5.5.3p3", "win", translation_default=True), #also used by GOG old? BuildVariant("MG-Steam-GOG_old", "meakashi", "5.5.3p3", "unix"), #also used by GOG old? BuildVariant("GOG", "meakashi", "5.5.3p1", "win"), BuildVariant("GOG", "meakashi", "5.5.3p1", "unix"), ], "tsumihoroboshi": [ BuildVariant("GOG-MG-Steam", "tsumihoroboshi", "5.5.3p3", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "tsumihoroboshi", "5.5.3p3", "unix"), # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version. #'tsumihoroboshi 5.6.7f1 win' ], "minagoroshi": [ BuildVariant("GOG-MG-Steam", "minagoroshi", "5.6.7f1", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "minagoroshi", "5.6.7f1", "unix"), # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version. # 'matsuribayashi 5.6.7f1 win' # 'matsuribayashi 5.6.7f1 unix' ], "matsuribayashi": [ # Based on the GOG MacOS sharedassets, but works on Linux too. # Working on: # - Linux Steam (2023-07-09) # - Linux GOG (2023-07-09) # - MacOS GOG (2023-07-09) BuildVariant("GOG-MG-Steam", "matsuribayashi", "2017.2.5", "unix"), # NOTE: I'm 99% certain this file is no longer used, as we just upgrade the entire GOG/Mangagamer game # Special version for GOG/Mangagamer Linux with SHA256: # A200EC2A85349BC03B59C8E2F106B99ED0CBAAA25FC50928BB8BA2E2AA90FCE9 # CRC32L 51100D6D # BuildVariant("GOG-MG", "matsuribayashi", "2017.2.5", "unix", "51100D6D"), # TO BE REMOVED BuildVariant("GOG-MG-Steam", "matsuribayashi", "2017.2.5", "win", translation_default=True), ], 'rei': [ BuildVariant("GOG-Steam-MG_old", "rei", "2019.4.3", "win", translation_default=True), BuildVariant("MG", "rei", "2019.4.4", "win"), BuildVariant("GOG-Steam-MG_old", "rei", "2019.4.3", "unix"), BuildVariant("MG", "rei", "2019.4.4", "unix"), ], 'hou-plus': [ BuildVariant("GOG-MG-Steam", "hou-plus", "2019.4.4", "win", translation_default=True), BuildVariant("GOG-MG-Steam", "hou-plus", "2019.4.4", "unix"), ], } def is_windows(): return sys.platform == "win32" def call(args, **kwargs): print("running: {}".format(args)) retcode = subprocess.call( args, shell=is_windows(), **kwargs ) # use shell on windows if retcode != 0: raise Exception(f"ERROR: {args} exited with retcode: {retcode}") def download(url): print(f"Starting download of URL: {url}") call(["curl", "-OJLf", url]) def seven_zip_extract(input_path, outputDir=None): args = [Globals.SEVEN_ZIP_EXECUTABLE, "x", input_path, "-y"] if outputDir: args.append("-o" + outputDir) call(args) def seven_zip_compress(input_path, output_path): args = [Globals.SEVEN_ZIP_EXECUTABLE, "a", "-md=512m", output_path, input_path, "-y"] call(args) def get_chapter_name_and_translation_from_git_tag(): returned_chapter_name = None translation = False tag_fragments_debug = 'tag fragments not extracted - maybe missing GITHUB_REF?' #type: str if GIT_TAG is None: raise Exception( "'github_actions' was selected, but environment variable GITHUB_REF was not set - are you sure you're running this script from Github Actions?" ) else: # Look for the chapter name to build in the git tag tag_fragments = [x.lower() for x in re.split(r"_", GIT_REF)] tag_fragments_debug = str(tag_fragments) if "all" in tag_fragments: returned_chapter_name = "all" else: for chapter_name in chapter_to_build_variants.keys(): if chapter_name.lower() in tag_fragments: returned_chapter_name = chapter_name break if "translation" in tag_fragments: translation = True return returned_chapter_name, translation, tag_fragments_debug def get_build_variants(selected_chapter: str) -> List[BuildVariant]: if selected_chapter == "all": commands = [] for command in chapter_to_build_variants.values(): commands.extend(command) return commands elif selected_chapter in chapter_to_build_variants: return chapter_to_build_variants[selected_chapter] else: raise Exception( f"Unknown Chapter {selected_chapter} - please update the build.py script" ) def check_7z(): Globals.SEVEN_ZIP_EXECUTABLE = findWorkingExecutablePath(["7za", "7z"], ['-h']) if Globals.SEVEN_ZIP_EXECUTABLE is None: seven_zip_filename = '7z_x64_23-06-20.zip' seven_zip_url = f"https://github.com/07th-mod/ui-editing-scripts/releases/download/v1.0.0/{seven_zip_filename}" print(">>>> NOTE: Downloading 7zip as can't find 7zip as '7z' or '7za'") if os.path.exists(seven_zip_filename): os.remove(seven_zip_filename) print(f"Downloading and Extracting 7-zip from {seven_zip_url}...") download(seven_zip_url) with zipfile.ZipFile(seven_zip_filename, 'r') as zip_ref: zip_ref.extractall('.') os.remove(seven_zip_filename) Globals.SEVEN_ZIP_EXECUTABLE = findWorkingExecutablePath(["7za", "7z"], ['-h']) if Globals.SEVEN_ZIP_EXECUTABLE is None: print(">>>> ERROR: Can't find 7zip as '7z' or '7za', even after downloading it!") print("Try running this script again. If it still fails, report this issue to 07th-mod") # Check that 7zip is 64-bit seven_zip_bitness = None seven_zip_info = subprocess.check_output(Globals.SEVEN_ZIP_EXECUTABLE, text=True) for line in seven_zip_info.splitlines(): if line.strip().startswith('7-Zip'): if 'x64' in line: seven_zip_bitness = 64 elif 'x86' in line: seven_zip_bitness = 32 break if seven_zip_bitness == 64: print("7zip is 64-bit - OK") else: print(f">>>> ERROR: Unacceptable 7zip bitness '{seven_zip_bitness}' - need 64 bit.\n\n Please make sure your 7zip is 64-bit, or manually edit this script to use 128mb 7z dictionary size") exit(-1) class LastModifiedManager: savePath = 'lastModified.json' def __init__(self) -> None: self.lastModifiedDict = {} if os.path.exists(LastModifiedManager.savePath): with open(LastModifiedManager.savePath, 'r') as handle: self.lastModifiedDict = json.load(handle) def getRemoteLastModified(url: str): httpResponse = urlopen(Request(url, headers={"User-Agent": ""})) return httpResponse.getheader("Last-Modified").strip() def isRemoteModifiedAndUpdateMemory(self, url: str): """ Checks whether a URL has been modified compared to the in-memory database, and updates the in-memory database with the new date modified time. NOTE: calling this function twice will return true the first time, then false the second time (assuming remote has not been updated), as the first call updates the in-memory database """ remoteLastModified = LastModifiedManager.getRemoteLastModified(url) localLastModified = self.lastModifiedDict.get(url) if localLastModified is not None and localLastModified == remoteLastModified: print(f"LastModifiedManager [{url}]: local and remote dates the same {localLastModified}") return False print(f"LastModifiedManager [{url}]: local {localLastModified} and remote {remoteLastModified} are different") self.lastModifiedDict[url] = remoteLastModified return True def save(self): """ Save the in-memory database to file, so it persists even when the program is closed. """ with open(LastModifiedManager.savePath, 'w') as handle: json.dump(self.lastModifiedDict, handle) if sys.version_info < (2, 7): print(">>>> ERROR: This script does not work on Python 2.7") exit(-1) lastModifiedManager = LastModifiedManager() # Parse command line arguments parser = argparse.ArgumentParser( description="Download and Install dependencies for ui editing scripts, then run build" ) parser.add_argument( "chapter", help='The chapter to build, or "all" for all chapters', choices=["all", "github_actions"] + list(chapter_to_build_variants.keys()), ) parser.add_argument("--force-download", default=False, action='store_true') parser.add_argument("--disable-translation", default=False, action='store_true') args = parser.parse_args() force_download = args.force_download # Get chapter name from git tag if "github_actions" specified as the chapter chapter_name = args.chapter if chapter_name == "github_actions": chapter_name, translation, tag_fragments_debug = get_chapter_name_and_translation_from_git_tag() if chapter_name is None: print( f">>>> WARNING: No chapter name was found in git tag {GIT_TAG} parsed as {tag_fragments_debug} - skipping building .assets" ) print(f">>>> Should contain one of {chapter_to_build_variants.keys()} or 'all'") exit(0) # NOTE: For now, translation archive output is enabled by default, as most of the time this script will be used for translators translation = True if args.disable_translation: translation = False # Get a list of build variants (like 'onikakushi 5.2.2f1 win') depending on commmand line arguments build_variants = get_build_variants(chapter_name) build_variants_list = "\n - ".join([b.get_build_command() for b in build_variants]) print(f"-------- Build Started --------") print(f"Chapter: [{chapter_name}] | Translation Archive Output: [{('Enabled' if translation else 'Disabled')}]") print(f"Variants:") print(f" - {build_variants_list}") print(f"-------------------------------") print() # Add the current folder to PATH (temporarily), so that any processes spawned # by this one can see the 7zip executable (downloaded if 7zip not found) os.environ['PATH'] += os.getcwd() # Install 7zip if required check_7z() # Install python dependencies print("Installing python dependencies") call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]) # Download and extract the vanilla assets assets_path = "assets" vanilla_archive = "vanilla.7z" assets_url = f"https://github.com/07th-mod/patch-releases/releases/download/developer-v1.0/{vanilla_archive}" vanilla_folder_path = os.path.join(assets_path, "vanilla") vanilla_fully_extracted = os.path.exists(vanilla_folder_path) and not os.path.exists(vanilla_archive) if lastModifiedManager.isRemoteModifiedAndUpdateMemory(assets_url) or force_download or not vanilla_fully_extracted: print("Downloading and Extracting Vanilla assets") pathlib.Path(vanilla_archive).unlink(missing_ok=True) if os.path.exists(vanilla_folder_path): shutil.rmtree(vanilla_folder_path) download(assets_url) seven_zip_extract(vanilla_archive) # Remove the archive to indicate extraction was successful pathlib.Path(vanilla_archive).unlink(missing_ok=True) lastModifiedManager.save() else: print("Vanilla archive already extracted - skipping") # Download and extract UABE uabe_folder = "64bit" uabe_archive = "AssetsBundleExtractor_2.2stabled_64bit_with_VC2010.zip" uabe_url = f"https://github.com/07th-mod/patch-releases/releases/download/developer-v1.0/{uabe_archive}" uabe_fully_extracted = os.path.exists(uabe_folder) and not os.path.exists(uabe_archive) if lastModifiedManager.isRemoteModifiedAndUpdateMemory(uabe_url) or force_download or not uabe_fully_extracted: print("Downloading and Extracting UABE") pathlib.Path(uabe_archive).unlink(missing_ok=True) if os.path.exists(uabe_folder): shutil.rmtree(uabe_folder) # The default Windows github runner doesn't have the 2010 VC++ redistributable preventing UABE from running # This zip file bundles the required DLLs (msvcr100.dll & msvcp100.dll) so it's not required download(uabe_url) seven_zip_extract(uabe_archive) # Remove the archive to indicate extraction was successful pathlib.Path(uabe_archive).unlink(missing_ok=True) lastModifiedManager.save() else: print("UABE already extracted - skipping") # Add UABE to PATH uabe_folder = os.path.abspath(uabe_folder) os.environ["PATH"] += os.pathsep + os.pathsep.join([uabe_folder]) # If rust is not installed, download binary release of ui comopiler # This is mainly for users running this script on their own computer working_cargo = False try: subprocess.check_output("cargo -v") print( "Found working Rust/cargo - will compile ui-compiler.exe using repository sources" ) working_cargo = True except: print("No working Rust/cargo found - download binary release of UI compiler...") download( "https://github.com/07th-mod/ui-editing-scripts/releases/latest/download/ui-compiler.exe" ) # Build all the requested variants for build_variant in build_variants: print(f"Building .assets for {build_variant.get_build_command()}...") if working_cargo: call(f"cargo run {build_variant.get_build_command()}") else: call(f"ui-compiler.exe {build_variant.get_build_command()}") if translation: source_sharedassets = os.path.join("output", build_variant.data_dir, "sharedassets0.assets") translation_data_dir = os.path.join("output/translation", build_variant.data_dir) destination_sharedassets = os.path.join(translation_data_dir, build_variant.get_translation_sharedassets_name()) os.makedirs(translation_data_dir, exist_ok=True) shutil.copyfile(source_sharedassets, destination_sharedassets) if build_variant.translation_default: destination_default_sharedassets = os.path.join(translation_data_dir, "sharedassets0.assets") shutil.copyfile(source_sharedassets, destination_default_sharedassets) if translation: containing_folder = "output" output_path = "output/translation.7z" if os.path.exists(output_path): os.remove(output_path) seven_zip_compress('output/translation', output_path)