diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..9035786 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,37 @@ +name: Build ui-compiler.exe and optionally Build Assets + +on: + push: + tags: + - '*' + +jobs: + windows_build: + name: Windows Build + runs-on: windows-latest + steps: + # Download the repository + - uses: actions/checkout@v2 + + # Caching for Rust + - name: Cache rust builds + uses: Swatinem/rust-cache@v2 + + - name: Build ui-compiler.exe + run: cargo build + + - name: Download and install dependencies, then build assets + run: python build.py github_actions + + # Publish a release (tagged commits) + # For more info on options see: https://github.com/softprops/action-gh-release + - name: Release (tag) + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') # only publish tagged commits + with: + files: | + output/*.7z + target/debug/ui-compiler.exe + draft: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index b703566..1ba2d67 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /output /target **/*.rs.bk +/64bit \ No newline at end of file diff --git a/README.md b/README.md index 296ad16..ad40389 100644 --- a/README.md +++ b/README.md @@ -16,40 +16,54 @@ Please note that documentation is in two places: ## Usage instructions for translators and dev team -### Prerequisites +### Setup (Windows Only) -1. Install Python 3 -2. Run `pip install numpy Pillow unitypack` to install the required Python packages -3. Install Rust -4. Download exactly this version of UABE https://github.com/SeriousCache/UABE/releases/tag/2.2stabled -5. Download 7zip (standalone console version) from here https://www.7-zip.org/download.html -6. Extract UABE and 7z and make sure **both** are on your `PATH` - - On Windows, you need to restart your terminal window to update your `PATH`. - - Try caling `AssetBundleExtractor.exe` - - Try caling `7za.exe` -7. Download the [vanilla UI archive](http://07th-mod.com/archive/vanilla.7z) and unpack it into the repository root (creating the folder `assets/vanilla`). +The below instructions only work on Windows! -### Using the tool to generate sharedassets0.assets +1. Install Python 3 from the [Python Download Page](https://www.python.org/downloads/) +2. For translators, fork this repository ([Github forking instructions]( https://docs.github.com/en/get-started/quickstart/fork-a-repo)) + - A fork is recommended for translators as you can check in your changes to github. It also allows you to use Github for building and hosting assets +3. Clone the repository (either this repository, or the one you forked) to your computer -To build all chapters: - - On Linux open a terminal and run the below command - - On Windows with "Git for Windows" installed, right click the folder and click "Git Bash Here", then use that terminal +### Using the tool to generate sharedassets0.assets (Windows Only) -``` -./compileall.sh -``` +To list the supported Higurashi chapters for this tool, run -To build a particular chapter/version run the following command +```python build.py``` -``` -cargo run -``` +which should show an error message complaining about a missing `chapter` argument -`` is simply `onikakushi`, `watanagashi` and so on. +Then run -`` is the unity version, like `5.5.3p3` or `2017.2.5`. Note that for version `2017.2.5f1`, you just enter `2017.2.5` (currently only support the first 8 characters of the unity version) +```python build.py onikakushi``` -`` is `win` or `unix`. +for example, to build the sharedassets required for onikakushi. You can also run `python build.py all` to build all chapters. + +**NOTE: If a new game is released, or a game is updated, you may need to re-download the vanilla assets. To do this, add the '--force-download' option like so:** + +```python build.py rei --force-download``` + +You may encounter the following problems: +- Windows Defender may block/delete our precompiled `ui-compiler.exe`. In this case, you can either try to unblock it, or install Rust to make the script compile it on your own computer. Contact us if you have this issue. +- For any other error, likely we just need to update the build script, so please contact us. + +### Modifying Assets + +Assets are located in the `assets` folder. Replace any file in the `assets` folder, then run the script again, and it should be included in the generated assets files. + +### Building assets using Github Actions + +#### Note for forks/translators + +Github actions might be disabled for your forks. Clicking on the 'Actions' tab should allow you to enable it. Please do this before proceeding. + +#### Building a release + +To use Github Actions to build a release, create a tag like `v1.0.6_onikakushi` which contains the chapter name (separated by an underscore) you want to build (or 'all' for all chapters). + +Click on the 'Actions' tab to observe the build process. + +Once the build is complete, go to the 'Releases' page, and a new draft release should appear. You can check everything is OK before publishing the release, or just download the files without publishing the release. ---- @@ -71,6 +85,9 @@ You'll need to extract the 'msgothic' font files from the stock `.assets` file b 2. Rename them as `msgothic_0.dat` and `msgothic_2.dat` 3. Move them to `assets/vanilla//msgothic_0.dat` & `assets/vanilla//msgothic_2.dat` +## Building `ui-compiler.exe` + +To build just the `ui-compiler.exe`, push any tag to the repository. ## Extra Notes diff --git a/build.py b/build.py new file mode 100644 index 0000000..cca4b04 --- /dev/null +++ b/build.py @@ -0,0 +1,232 @@ +import re +import subprocess +import sys +import os +import pathlib +import shutil +import argparse +from typing import List + +# Get the github ref +GIT_TAG = None +GIT_REF = os.environ.get("GITHUB_REF") # Github Tag / Version info +if GIT_REF is not None: + GIT_TAG = GIT_REF.split("/")[-1] + print(f"--- Git Ref: {GIT_REF} Git Tag: {GIT_TAG} ---") + +# List of build variants for any given chapter +# +# There must be a corresponding vanilla sharedassets0.assets file located at: +# assets\vanilla\{CHAPTER_NAME}[-{CRC32}]\{OS}-{UNITY_VERSION}\sharedassets0.assets +# for each entry. +chapter_to_build_variants = { + "onikakushi": [ + "onikakushi 5.2.2f1 win", + "onikakushi 5.2.2f1 unix", + ], + "watanagashi": [ + "watanagashi 5.2.2f1 win", + "watanagashi 5.2.2f1 unix" + ], + "tatarigoroshi": [ + "tatarigoroshi 5.4.0f1 win", + "tatarigoroshi 5.4.0f1 unix", + "tatarigoroshi 5.3.5f1 win", + "tatarigoroshi 5.3.4p1 win", + "tatarigoroshi 5.3.4p1 unix", + ], + "himatsubushi": [ + "himatsubushi 5.4.1f1 win", + "himatsubushi 5.4.1f1 unix" + ], + "meakashi": [ + "meakashi 5.5.3p3 win", + "meakashi 5.5.3p3 unix", + "meakashi 5.5.3p1 win", + "meakashi 5.5.3p1 unix", + ], + "tsumihoroboshi": [ + "tsumihoroboshi 5.5.3p3 win", + "tsumihoroboshi 5.5.3p3 unix" + # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version. + #'tsumihoroboshi 5.6.7f1 win' + ], + "minagoroshi": [ + "minagoroshi 5.6.7f1 win", + "minagoroshi 5.6.7f1 unix" + # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version. + # 'matsuribayashi 5.6.7f1 win' + # 'matsuribayashi 5.6.7f1 unix' + ], + "matsuribayashi": [ + "matsuribayashi 2017.2.5 unix", + # Special version for GOG/Mangagamer Linux with SHA256: + # A200EC2A85349BC03B59C8E2F106B99ED0CBAAA25FC50928BB8BA2E2AA90FCE9 + # CRC32L 51100D6D + "matsuribayashi 2017.2.5 unix 51100D6D", + "matsuribayashi 2017.2.5 win", + ], + 'rei': [ + 'rei 2019.4.3 win', + 'rei 2019.4.3 unix', + ], +} + + +def is_windows(): + return sys.platform == "win32" + + +def call(args, **kwargs): + print("running: {}".format(args)) + retcode = subprocess.call( + args, shell=is_windows(), **kwargs + ) # use shell on windows + if retcode != 0: + raise Exception(f"ERROR: {args} exited with retcode: {retcode}") + + +def download(url): + print(f"Starting download of URL: {url}") + call(["curl", "-OJLf", url]) + + +def seven_zip_extract(input_path, outputDir=None): + args = ["7z", "x", input_path, "-y"] + if outputDir: + args.append("-o" + outputDir) + + call(args) + + +def get_chapter_name_from_git_tag(): + if GIT_TAG is None: + raise Exception( + "'github_actions' was selected, but environment variable GIT_REF was not set - are you sure you're running this script from Github Actions?" + ) + else: + # Look for the chapter name to build in the git tag + tag_fragments = [x.lower() for x in re.split("[\W_]", GIT_REF)] + + if "all" in tag_fragments: + return "all" + else: + for chapter_name in chapter_to_build_variants.keys(): + if chapter_name.lower() in tag_fragments: + return chapter_name + + return None + + +def get_build_variants(selected_chapter: str) -> List[str]: + if selected_chapter == "all": + commands = [] + for command in chapter_to_build_variants.values(): + commands.extend(command) + return commands + elif selected_chapter in chapter_to_build_variants: + return chapter_to_build_variants[selected_chapter] + else: + raise Exception( + f"Unknown Chapter {selected_chapter} - please update the build.py script" + ) + + +# Parse command line arguments +parser = argparse.ArgumentParser( + description="Download and Install dependencies for ui editing scripts, then run build" +) +parser.add_argument( + "chapter", + help='The chapter to build, or "all" for all chapters', + choices=["all", "github_actions"] + list(chapter_to_build_variants.keys()), +) +parser.add_argument("--force-download", default=False, action='store_true') +args = parser.parse_args() + +force_download = args.force_download + +# Get chapter name from git tag if "github_actions" specified as the chapter +chapter_name = args.chapter +if chapter_name == "github_actions": + chapter_name = get_chapter_name_from_git_tag() + if chapter_name is None: + print( + f">>>> WARNING: No chapter name (or 'all') was found in git tag {GIT_TAG} - skipping building .assets" + ) + exit(0) + +# Get a list of build variants (like 'onikakushi 5.2.2f1 win') depending on commmand line arguments +build_variants = get_build_variants(chapter_name) +print(f"For chapter '{chapter_name}' building: {build_variants}") + +# Install python dependencies +print("Installing python dependencies") +call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]) + +# Download and extract the vanilla assets +assets_path = "assets" +vanilla_archive = "vanilla.7z" +vanilla_folder_path = os.path.join(assets_path, "vanilla") +vanilla_fully_extracted = os.path.exists(vanilla_folder_path) and not os.path.exists(vanilla_archive) +if force_download or not vanilla_fully_extracted: + print("Downloading and Extracting Vanilla assets") + pathlib.Path(vanilla_archive).unlink(missing_ok=True) + if os.path.exists(vanilla_folder_path): + shutil.rmtree(vanilla_folder_path) + + download("http://07th-mod.com/archive/vanilla.7z") + seven_zip_extract(vanilla_archive) + + # Remove the archive to indicate extraction was successful + pathlib.Path(vanilla_archive).unlink(missing_ok=True) +else: + print("Vanilla archive already extracted - skipping") + +# Download and extract UABE +uabe_folder = "64bit" +uabe_archive = "AssetsBundleExtractor_2.2stabled_64bit_with_VC2010.zip" +uabe_fully_extracted = os.path.exists(uabe_folder) and not os.path.exists(uabe_archive) +if force_download or not uabe_fully_extracted: + print("Downloading and Extracting UABE") + pathlib.Path(uabe_archive).unlink(missing_ok=True) + if os.path.exists(uabe_folder): + shutil.rmtree(uabe_folder) + + # The default Windows github runner doesn't have the 2010 VC++ redistributable preventing UABE from running + # This zip file bundles the required DLLs (msvcr100.dll & msvcp100.dll) so it's not required + download(f"http://07th-mod.com/archive/{uabe_archive}") + seven_zip_extract(uabe_archive) + + # Remove the archive to indicate extraction was successful + pathlib.Path(uabe_archive).unlink(missing_ok=True) +else: + print("UABE already extracted - skipping") + + +# Add UABE to PATH +uabe_folder = os.path.abspath(uabe_folder) +os.environ["PATH"] += os.pathsep + os.pathsep.join([uabe_folder]) + +# If rust is not installed, download binary release of ui comopiler +# This is mainly for users running this script on their own computer +working_cargo = False +try: + subprocess.check_output("cargo -v") + print( + "Found working Rust/cargo - will compile ui-compiler.exe using repository sources" + ) + working_cargo = True +except: + print("No working Rust/cargo found - download binary release of UI compiler...") + download( + "https://github.com/07th-mod/ui-editing-scripts/releases/latest/download/ui-compiler.exe" + ) + +# Build all the requested variants +for command in build_variants: + print(f"Building .assets for {command}...") + if working_cargo: + call(f"cargo run {command}") + else: + call(f"ui-compiler.exe {command}") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1fc36b0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +numpy +Pillow +unitypack \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 3d5d28b..2836512 100644 --- a/src/main.rs +++ b/src/main.rs @@ -13,6 +13,11 @@ fn main() { let chapter = &args[1]; let unity = &args[2]; let system = &args[3]; + let checksum = if args.len() > 4 { + Some(&args[4]) + } else { + None + }; let mut chapters = HashMap::new(); chapters.insert("onikakushi", 1); @@ -26,16 +31,18 @@ fn main() { chapters.insert("rei", 9); if !chapters.contains_key(&chapter[..]) { - println!("Unknown chapter"); + println!("Unknown chapter, should be one of {:?}", chapters.keys()); process::exit(1); } let arc_number = chapters.get(&chapter[..]).unwrap().clone(); - let assets = format!("assets/vanilla/{}/{}-{}/sharedassets0.assets", &chapter, &system, &unity); + let assets = format!("assets/vanilla/{}/{}-{}{}/sharedassets0.assets", &chapter, &system, &unity, &format_checksum(checksum, "-")); + println!("Looking for vanilla assets at [{}]", assets); let directory_assets = "output/assets"; let directory_data = format!("output/HigurashiEp{:02}_Data", arc_number); let emip = format!("{}/{}_{}_{}.emip", &directory_data, &chapter, &unity, &system); - let archive = format!("{}-UI_{}_{}.7z", &chapter.to_title_case(), &unity, &system); + //to_title_case() replaces hyphens and underscores with spaces. If this happens, revert it by replacing spaces with hyphens. + let archive = format!("{}-UI_{}_{}{}.7z", &chapter.to_title_case().replace(" ", "-"), &unity, &system, &format_checksum(checksum, "_")); if Path::new(&emip).exists() { fs::remove_file(&emip).expect("Failed to remove file"); @@ -62,6 +69,10 @@ fn main() { let version = String::from_utf8_lossy(&output.stdout).into_owned(); + if unity != &version.trim() { + println!("ERROR: Expected unity version {} but got version {}. If 'nothing found' then check the vanilla folder actually contains the required vanilla sharedassets!", unity, &version.trim()); + } + assert_eq!(unity, &version.trim()); // 1. texts @@ -180,6 +191,8 @@ fn main() { .status() .expect("failed to execute AssetBundleExtractor"); + println!("AssetBundleExtractor Status: {}", status); + assert!(status.success()); fs::remove_file(format!("{}/sharedassets0.assets.bak0000", &directory_data)).expect("Failed to remove file"); @@ -190,16 +203,37 @@ fn main() { fs::remove_file(&emip).expect("Failed to remove file"); // 7. pack with 7zip - let status = Command::new("7za") - .current_dir("output") - .arg("a") - .arg("-t7z") - .arg(&archive) - .arg(format!("../{}", &directory_data)) - .status() - .expect("failed to execute 7ze"); + let result_7za = pack_7zip("7za", &archive, &directory_data); - assert!(status.success()); + let status: std::io::Result = match result_7za { + Ok(ok) => Ok(ok), + Err(err) => match err.kind() { + std::io::ErrorKind::NotFound => { + println!("Warning: '7za' not found - trying '7z' instead"); + pack_7zip("7z", &archive, &directory_data) + }, + _ => Err(err), + } + }; + + let exit_status = status.expect("failed to execute 7za or 7z"); + + assert!(exit_status.success()); +} + +fn format_checksum(checksum: Option<&String>, sep: &str) -> String +{ + return checksum.map_or("".to_string(), |c| format!("{}{}", sep, c)); +} + +fn pack_7zip(command: &str, archive: &String, directory_data: &String) -> std::io::Result { + Command::new(command) + .current_dir("output") + .arg("a") + .arg("-t7z") + .arg(archive) + .arg(format!("../{}", directory_data)) + .status() } fn copy_images(from: &str, to: &str) {