Add github actions build and easy build script

.github/workflows/build.yml

@@ -0,0 +1,37 @@
+name: Build ui-compiler.exe and optionally Build Assets
+
+on:
+ push:
+   tags:
+     - '*'
+
+jobs:
+  windows_build:
+    name: Windows Build
+    runs-on: windows-latest
+    steps:
+      # Download the repository
+      - uses: actions/checkout@v2
+
+      # Caching for Rust
+      - name: Cache rust builds
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build ui-compiler.exe
+        run: cargo build
+
+      - name: Download and install dependencies, then build assets
+        run: python build.py github_actions
+
+      # Publish a release (tagged commits)
+      # For more info on options see: https://github.com/softprops/action-gh-release
+      - name: Release (tag)
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/') # only publish tagged commits
+        with:
+          files: |
+            output/*.7z
+            target/debug/ui-compiler.exe
+          draft: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -2,3 +2,4 @@
 /output
 /target
 **/*.rs.bk
+/64bit

README.md

@@ -16,40 +16,54 @@ Please note that documentation is in two places:
 
 ## Usage instructions for translators and dev team
 
-### Prerequisites
+### Setup (Windows Only)
 
-1. Install Python 3
+The below instructions only work on Windows!
-2. Run `pip install numpy Pillow unitypack` to install the required Python packages
-3. Install Rust
-4. Download exactly this version of UABE https://github.com/SeriousCache/UABE/releases/tag/2.2stabled
-5. Download 7zip (standalone console version) from here https://www.7-zip.org/download.html
-6. Extract UABE and 7z and make sure **both** are on your `PATH`
-   - On Windows, you need to restart your terminal window to update your `PATH`.
-     - Try caling `AssetBundleExtractor.exe`
-     - Try caling `7za.exe`
-7. Download the [vanilla UI archive](http://07th-mod.com/archive/vanilla.7z) and unpack it into the repository root (creating the folder `assets/vanilla`).
 
-### Using the tool to generate sharedassets0.assets
+1. Install Python 3 from the [Python Download Page](https://www.python.org/downloads/)
+2. For translators, fork this repository ([Github forking instructions]( https://docs.github.com/en/get-started/quickstart/fork-a-repo))
+     - A fork is recommended for translators as you can check in your changes to github. It also allows you to use Github for building and hosting assets
+3. Clone the repository (either this repository, or the one you forked) to your computer
 
-To build all chapters:
+### Using the tool to generate sharedassets0.assets (Windows Only)
- - On Linux open a terminal and run the below command
- - On Windows with "Git for Windows" installed, right click the folder and click "Git Bash Here", then use that terminal
 
-```
+To list the supported Higurashi chapters for this tool, run
-./compileall.sh
-```
 
-To build a particular chapter/version run the following command
+```python build.py```
 
-```
+which should show an error message complaining about a missing `chapter` argument
-cargo run <chapter> <unityversion> <system>
-```
 
-`<chapter>` is simply `onikakushi`, `watanagashi` and so on.
+Then run
 
-`<unityversion>` is the unity version, like `5.5.3p3` or `2017.2.5`. Note that for version `2017.2.5f1`, you just enter `2017.2.5` (currently only support the first 8 characters of the unity version)
+```python build.py onikakushi```
 
-`<system>` is `win` or `unix`.
+for example, to build the sharedassets required for onikakushi. You can also run `python build.py all` to build all chapters.
+
+**NOTE: If a new game is released, or a game is updated, you may need to re-download the vanilla assets. To do this, add the '--force-download' option like so:**
+
+```python build.py rei --force-download```
+
+You may encounter the following problems:
+- Windows Defender may block/delete our precompiled `ui-compiler.exe`. In this case, you can either try to unblock it, or install Rust to make the script compile it on your own computer. Contact us if you have this issue.
+- For any other error, likely we just need to update the build script, so please contact us.
+
+### Modifying Assets
+
+Assets are located in the `assets` folder. Replace any file in the `assets` folder, then run the script again, and it should be included in the generated assets files.
+
+### Building assets using Github Actions
+
+#### Note for forks/translators
+
+Github actions might be disabled for your forks. Clicking on the 'Actions' tab should allow you to enable it. Please do this before proceeding.
+
+#### Building a release
+
+To use Github Actions to build a release, create a tag like `v1.0.6_onikakushi` which contains the chapter name (separated by an underscore) you want to build (or 'all' for all chapters).
+
+Click on the 'Actions' tab to observe the build process.
+
+Once the build is complete, go to the 'Releases' page, and a new draft release should appear. You can check everything is OK before publishing the release, or just download the files without publishing the release.
 
 ----
 
@@ -71,6 +85,9 @@ You'll need to extract the 'msgothic' font files from the stock `.assets` file b
 2. Rename them as `msgothic_0.dat` and `msgothic_2.dat`
 3. Move them to `assets/vanilla/<chapter>/msgothic_0.dat` & `assets/vanilla/<chapter>/msgothic_2.dat`
 
+## Building `ui-compiler.exe`
+
+To build just the `ui-compiler.exe`, push any tag to the repository.
 
 ## Extra Notes
 

build.py

@@ -0,0 +1,232 @@
+import re
+import subprocess
+import sys
+import os
+import pathlib
+import shutil
+import argparse
+from typing import List
+
+# Get the github ref
+GIT_TAG = None
+GIT_REF = os.environ.get("GITHUB_REF")  # Github Tag / Version info
+if GIT_REF is not None:
+    GIT_TAG = GIT_REF.split("/")[-1]
+    print(f"--- Git Ref: {GIT_REF} Git Tag: {GIT_TAG} ---")
+
+# List of build variants for any given chapter
+#
+# There must be a corresponding vanilla sharedassets0.assets file located at:
+# assets\vanilla\{CHAPTER_NAME}[-{CRC32}]\{OS}-{UNITY_VERSION}\sharedassets0.assets
+# for each entry.
+chapter_to_build_variants = {
+    "onikakushi": [
+        "onikakushi 5.2.2f1 win",
+        "onikakushi 5.2.2f1 unix",
+    ],
+    "watanagashi": [
+        "watanagashi 5.2.2f1 win",
+        "watanagashi 5.2.2f1 unix"
+    ],
+    "tatarigoroshi": [
+        "tatarigoroshi 5.4.0f1 win",
+        "tatarigoroshi 5.4.0f1 unix",
+        "tatarigoroshi 5.3.5f1 win",
+        "tatarigoroshi 5.3.4p1 win",
+        "tatarigoroshi 5.3.4p1 unix",
+    ],
+    "himatsubushi": [
+        "himatsubushi 5.4.1f1 win",
+        "himatsubushi 5.4.1f1 unix"
+    ],
+    "meakashi": [
+        "meakashi 5.5.3p3 win",
+        "meakashi 5.5.3p3 unix",
+        "meakashi 5.5.3p1 win",
+        "meakashi 5.5.3p1 unix",
+    ],
+    "tsumihoroboshi": [
+        "tsumihoroboshi 5.5.3p3 win",
+        "tsumihoroboshi 5.5.3p3 unix"
+        # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version.
+        #'tsumihoroboshi 5.6.7f1 win'
+    ],
+    "minagoroshi": [
+        "minagoroshi 5.6.7f1 win",
+        "minagoroshi 5.6.7f1 unix"
+        # While GOG Windows is ver 5.6.7f1, we actually downgrade back to 5.5.3p3 in the installer, so we don't need this version.
+        # 'matsuribayashi 5.6.7f1 win'
+        # 'matsuribayashi 5.6.7f1 unix'
+    ],
+    "matsuribayashi": [
+        "matsuribayashi 2017.2.5 unix",
+        # Special version for GOG/Mangagamer Linux with SHA256:
+        # A200EC2A85349BC03B59C8E2F106B99ED0CBAAA25FC50928BB8BA2E2AA90FCE9
+        # CRC32L 51100D6D
+        "matsuribayashi 2017.2.5 unix 51100D6D",
+        "matsuribayashi 2017.2.5 win",
+    ],
+    'rei': [
+        'rei 2019.4.3 win',
+        'rei 2019.4.3 unix',
+    ],
+}
+
+
+def is_windows():
+    return sys.platform == "win32"
+
+
+def call(args, **kwargs):
+    print("running: {}".format(args))
+    retcode = subprocess.call(
+        args, shell=is_windows(), **kwargs
+    )  # use shell on windows
+    if retcode != 0:
+        raise Exception(f"ERROR: {args} exited with retcode: {retcode}")
+
+
+def download(url):
+    print(f"Starting download of URL: {url}")
+    call(["curl", "-OJLf", url])
+
+
+def seven_zip_extract(input_path, outputDir=None):
+    args = ["7z", "x", input_path, "-y"]
+    if outputDir:
+        args.append("-o" + outputDir)
+
+    call(args)
+
+
+def get_chapter_name_from_git_tag():
+    if GIT_TAG is None:
+        raise Exception(
+            "'github_actions' was selected, but environment variable GIT_REF was not set - are you sure you're running this script from Github Actions?"
+        )
+    else:
+        # Look for the chapter name to build in the git tag
+        tag_fragments = [x.lower() for x in re.split("[\W_]", GIT_REF)]
+
+        if "all" in tag_fragments:
+            return "all"
+        else:
+            for chapter_name in chapter_to_build_variants.keys():
+                if chapter_name.lower() in tag_fragments:
+                    return chapter_name
+
+    return None
+
+
+def get_build_variants(selected_chapter: str) -> List[str]:
+    if selected_chapter == "all":
+        commands = []
+        for command in chapter_to_build_variants.values():
+            commands.extend(command)
+        return commands
+    elif selected_chapter in chapter_to_build_variants:
+        return chapter_to_build_variants[selected_chapter]
+    else:
+        raise Exception(
+            f"Unknown Chapter {selected_chapter} - please update the build.py script"
+        )
+
+
+# Parse command line arguments
+parser = argparse.ArgumentParser(
+    description="Download and Install dependencies for ui editing scripts, then run build"
+)
+parser.add_argument(
+    "chapter",
+    help='The chapter to build, or "all" for all chapters',
+    choices=["all", "github_actions"] + list(chapter_to_build_variants.keys()),
+)
+parser.add_argument("--force-download", default=False, action='store_true')
+args = parser.parse_args()
+
+force_download = args.force_download
+
+# Get chapter name from git tag if "github_actions" specified as the chapter
+chapter_name = args.chapter
+if chapter_name == "github_actions":
+    chapter_name = get_chapter_name_from_git_tag()
+    if chapter_name is None:
+        print(
+            f">>>> WARNING: No chapter name (or 'all') was found in git tag {GIT_TAG} - skipping building .assets"
+        )
+        exit(0)
+
+# Get a list of build variants (like 'onikakushi 5.2.2f1 win') depending on commmand line arguments
+build_variants = get_build_variants(chapter_name)
+print(f"For chapter '{chapter_name}' building: {build_variants}")
+
+# Install python dependencies
+print("Installing python dependencies")
+call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
+
+# Download and extract the vanilla assets
+assets_path = "assets"
+vanilla_archive = "vanilla.7z"
+vanilla_folder_path = os.path.join(assets_path, "vanilla")
+vanilla_fully_extracted = os.path.exists(vanilla_folder_path) and not os.path.exists(vanilla_archive)
+if force_download or not vanilla_fully_extracted:
+    print("Downloading and Extracting Vanilla assets")
+    pathlib.Path(vanilla_archive).unlink(missing_ok=True)
+    if os.path.exists(vanilla_folder_path):
+        shutil.rmtree(vanilla_folder_path)
+
+    download("http://07th-mod.com/archive/vanilla.7z")
+    seven_zip_extract(vanilla_archive)
+
+    # Remove the archive to indicate extraction was successful
+    pathlib.Path(vanilla_archive).unlink(missing_ok=True)
+else:
+    print("Vanilla archive already extracted - skipping")
+
+# Download and extract UABE
+uabe_folder = "64bit"
+uabe_archive = "AssetsBundleExtractor_2.2stabled_64bit_with_VC2010.zip"
+uabe_fully_extracted = os.path.exists(uabe_folder) and not os.path.exists(uabe_archive)
+if force_download or not uabe_fully_extracted:
+    print("Downloading and Extracting UABE")
+    pathlib.Path(uabe_archive).unlink(missing_ok=True)
+    if os.path.exists(uabe_folder):
+        shutil.rmtree(uabe_folder)
+
+    # The default Windows github runner doesn't have the 2010 VC++ redistributable preventing UABE from running
+    # This zip file bundles the required DLLs (msvcr100.dll & msvcp100.dll) so it's not required
+    download(f"http://07th-mod.com/archive/{uabe_archive}")
+    seven_zip_extract(uabe_archive)
+
+    # Remove the archive to indicate extraction was successful
+    pathlib.Path(uabe_archive).unlink(missing_ok=True)
+else:
+    print("UABE already extracted - skipping")
+
+
+# Add UABE to PATH
+uabe_folder = os.path.abspath(uabe_folder)
+os.environ["PATH"] += os.pathsep + os.pathsep.join([uabe_folder])
+
+# If rust is not installed, download binary release of ui comopiler
+# This is mainly for users running this script on their own computer
+working_cargo = False
+try:
+    subprocess.check_output("cargo -v")
+    print(
+        "Found working Rust/cargo - will compile ui-compiler.exe using repository sources"
+    )
+    working_cargo = True
+except:
+    print("No working Rust/cargo found - download binary release of UI compiler...")
+    download(
+        "https://github.com/07th-mod/ui-editing-scripts/releases/latest/download/ui-compiler.exe"
+    )
+
+# Build all the requested variants
+for command in build_variants:
+    print(f"Building .assets for {command}...")
+    if working_cargo:
+        call(f"cargo run {command}")
+    else:
+        call(f"ui-compiler.exe {command}")

requirements.txt

@@ -0,0 +1,3 @@
+numpy
+Pillow
+unitypack

src/main.rs

@@ -13,6 +13,11 @@ fn main() {
     let chapter = &args[1];
     let unity = &args[2];
     let system = &args[3];
+    let checksum = if args.len() > 4 {
+        Some(&args[4])
+    } else {
+        None
+    };
 
     let mut chapters = HashMap::new();
     chapters.insert("onikakushi", 1);
@@ -26,16 +31,18 @@ fn main() {
     chapters.insert("rei", 9);
 
     if !chapters.contains_key(&chapter[..]) {
-        println!("Unknown chapter");
+        println!("Unknown chapter, should be one of {:?}", chapters.keys());
         process::exit(1);
     }
 
     let arc_number = chapters.get(&chapter[..]).unwrap().clone();
-    let assets = format!("assets/vanilla/{}/{}-{}/sharedassets0.assets", &chapter, &system, &unity);
+    let assets = format!("assets/vanilla/{}/{}-{}{}/sharedassets0.assets", &chapter, &system, &unity, &format_checksum(checksum, "-"));
+    println!("Looking for vanilla assets at [{}]", assets);
     let directory_assets = "output/assets";
     let directory_data = format!("output/HigurashiEp{:02}_Data", arc_number);
     let emip = format!("{}/{}_{}_{}.emip", &directory_data, &chapter, &unity, &system);
-    let archive = format!("{}-UI_{}_{}.7z", &chapter.to_title_case(), &unity, &system);
+    //to_title_case() replaces hyphens and underscores with spaces. If this happens, revert it by replacing spaces with hyphens.
+    let archive = format!("{}-UI_{}_{}{}.7z", &chapter.to_title_case().replace(" ", "-"), &unity, &system, &format_checksum(checksum, "_"));
 
     if Path::new(&emip).exists() {
         fs::remove_file(&emip).expect("Failed to remove file");
@@ -62,6 +69,10 @@ fn main() {
 
     let version = String::from_utf8_lossy(&output.stdout).into_owned();
 
+    if unity != &version.trim() {
+        println!("ERROR: Expected unity version {} but got version {}. If 'nothing found' then check the vanilla folder actually contains the required vanilla sharedassets!", unity,  &version.trim());
+    }
+
     assert_eq!(unity, &version.trim());
 
     // 1. texts
@@ -180,6 +191,8 @@ fn main() {
         .status()
         .expect("failed to execute AssetBundleExtractor");
 
+    println!("AssetBundleExtractor Status: {}", status);
+
     assert!(status.success());
 
     fs::remove_file(format!("{}/sharedassets0.assets.bak0000", &directory_data)).expect("Failed to remove file");
@@ -190,16 +203,37 @@ fn main() {
     fs::remove_file(&emip).expect("Failed to remove file");
 
     // 7. pack with 7zip
-    let status = Command::new("7za")
+    let result_7za = pack_7zip("7za", &archive, &directory_data);
+
+    let status: std::io::Result<process::ExitStatus> = match result_7za {
+        Ok(ok) => Ok(ok),
+        Err(err) => match err.kind() {
+            std::io::ErrorKind::NotFound => {
+                println!("Warning: '7za' not found - trying '7z' instead");
+                pack_7zip("7z", &archive, &directory_data)
+            },
+            _ => Err(err),
+        }
+    };
+
+    let exit_status = status.expect("failed to execute 7za or 7z");
+
+    assert!(exit_status.success());
+}
+
+fn format_checksum(checksum: Option<&String>, sep: &str) -> String
+{
+    return checksum.map_or("".to_string(), |c| format!("{}{}", sep, c));
+}
+
+fn pack_7zip(command: &str, archive: &String, directory_data: &String) -> std::io::Result<process::ExitStatus> {
+    Command::new(command)
     .current_dir("output")
     .arg("a")
     .arg("-t7z")
-        .arg(&archive)
+    .arg(archive)
-        .arg(format!("../{}", &directory_data))
+    .arg(format!("../{}", directory_data))
     .status()
-        .expect("failed to execute 7ze");
-
-    assert!(status.success());
 }
 
 fn copy_images(from: &str, to: &str) {