Replace KanjiFinder with a better asset bundle string searcher
This commit is contained in:
Tellow Krinkle
121
scripts/CharacterInfoExtraction/AssetBundleStringExtractor.swift
Normal file
121
scripts/CharacterInfoExtraction/AssetBundleStringExtractor.swift
Normal file
@@ -0,0 +1,121 @@
|
|||||||
|
import Foundation
|
||||||
|
|
||||||
|
var standardError = FileHandle.standardError
|
||||||
|
|
||||||
|
extension FileHandle : TextOutputStream {
|
||||||
|
public func write(_ string: String) {
|
||||||
|
guard let data = string.data(using: .utf8) else { return }
|
||||||
|
self.write(data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
guard CommandLine.arguments.count > 1 else {
|
||||||
|
print("""
|
||||||
|
Usage: \(CommandLine.arguments[0]) assetBundle1.assets [assetBundle2.assets ...]
|
||||||
|
Use - to read from stdin
|
||||||
|
Finds strings in Unity asset bundles
|
||||||
|
""", to: &standardError)
|
||||||
|
exit(EXIT_FAILURE)
|
||||||
|
}
|
||||||
|
|
||||||
|
#if !swift(>=4.2)
|
||||||
|
extension Collection {
|
||||||
|
func firstIndex(where predicate: (Element) throws -> Bool) rethrows -> Index? {
|
||||||
|
return try self.index(where: predicate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
let inFiles: [String] = Array(CommandLine.arguments[1...])
|
||||||
|
|
||||||
|
extension Collection {
|
||||||
|
subscript(offset offset: Int) -> Element {
|
||||||
|
get {
|
||||||
|
let i = index(startIndex, offsetBy: offset)
|
||||||
|
return self[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isUTF8Continuation(_ byte: UInt8) -> Bool {
|
||||||
|
return byte & 0b1100_0000 == 0b1000_0000
|
||||||
|
}
|
||||||
|
|
||||||
|
func isValidUTF8<C: Collection>(data: C) -> Bool where C.Element == UInt8 {
|
||||||
|
var slice = data[...]
|
||||||
|
while !slice.isEmpty {
|
||||||
|
let byte = slice.first!
|
||||||
|
slice = slice.dropFirst()
|
||||||
|
if byte & 0b1000_0000 == 0 { // Single byte utf-8
|
||||||
|
// Strings shouldn't contain ASCII controll chars
|
||||||
|
guard byte > 8 else { return false }
|
||||||
|
guard !(0xe...0x1f).contains(byte) else { return false }
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
else if byte & 0b1110_0000 == 0b1100_0000 { // Two byte utf-8
|
||||||
|
guard let cont = slice.first else { return false }
|
||||||
|
guard isUTF8Continuation(cont) else { return false }
|
||||||
|
slice = slice.dropFirst()
|
||||||
|
}
|
||||||
|
else if byte & 0b1111_0000 == 0b1110_0000 { // Three byte utf-8
|
||||||
|
guard slice.count > 1 else { return false }
|
||||||
|
guard isUTF8Continuation(slice[offset: 0]) else { return false }
|
||||||
|
guard isUTF8Continuation(slice[offset: 1]) else { return false }
|
||||||
|
slice = slice.dropFirst(2)
|
||||||
|
}
|
||||||
|
else if byte & 0b1111_1000 == 0b1111_0000 { // Four byte utf-8
|
||||||
|
guard slice.count > 2 else { return false }
|
||||||
|
guard isUTF8Continuation(slice[offset: 0]) else { return false }
|
||||||
|
guard isUTF8Continuation(slice[offset: 1]) else { return false }
|
||||||
|
guard isUTF8Continuation(slice[offset: 2]) else { return false }
|
||||||
|
slice = slice.dropFirst(3)
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func stringFinder(data: Data, maxStringLength: Int = 100) -> [String] {
|
||||||
|
return data.withUnsafeBytes { (ptr: UnsafePointer<UInt8>) -> [String] in
|
||||||
|
let buffer = UnsafeRawBufferPointer(start: UnsafeRawPointer(ptr), count: data.count)
|
||||||
|
guard data.count % 4 == 0 else { fatalError("Input file wasn't 4-byte aligned, it's probably not an asset bundle") }
|
||||||
|
var out: [String] = []
|
||||||
|
let ints = buffer.bindMemory(to: UInt32.self)
|
||||||
|
for (index, int) in ints.lazy.map({ $0.littleEndian }).enumerated() {
|
||||||
|
guard int > 1 && int < maxStringLength && int < ((ints.count - index - 1) * 4) else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
let uint32Length = Int((int &+ 3) / 4)
|
||||||
|
let padding = (4 - (int % 4))
|
||||||
|
let unicode = UnsafeBufferPointer(rebasing: ints[(index + 1)...].prefix(uint32Length))
|
||||||
|
// Ensure padding is all 0s
|
||||||
|
guard unicode.last!.littleEndian &>> (padding * 8) == 0 else { continue }
|
||||||
|
let optionalStr = unicode.withMemoryRebound(to: UInt8.self) { (unicode) -> String? in
|
||||||
|
let stringUnicode = unicode[..<Int(int)]
|
||||||
|
guard isValidUTF8(data: stringUnicode) else { return nil }
|
||||||
|
return String(decoding: stringUnicode, as: UTF8.self)
|
||||||
|
}
|
||||||
|
guard let str = optionalStr else { continue }
|
||||||
|
out.append(str)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if inFiles == ["-"] {
|
||||||
|
print(stringFinder(data: FileHandle.standardInput.readDataToEndOfFile()).joined(separator: "\n"))
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
for path in inFiles {
|
||||||
|
guard FileManager.default.fileExists(atPath: path) else {
|
||||||
|
fatalError("No file exists at \(path)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for path in inFiles {
|
||||||
|
let data = try Data(contentsOf: URL(fileURLWithPath: path))
|
||||||
|
print(stringFinder(data: data).joined(separator: "\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
import Foundation
|
|
||||||
|
|
||||||
var standardError = FileHandle.standardError
|
|
||||||
|
|
||||||
extension FileHandle : TextOutputStream {
|
|
||||||
public func write(_ string: String) {
|
|
||||||
guard let data = string.data(using: .utf8) else { return }
|
|
||||||
self.write(data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
guard CommandLine.arguments.count > 1 else {
|
|
||||||
print("""
|
|
||||||
Usage: \(CommandLine.arguments[0]) [-filter filterFile.txt] assetBundle1.assets [assetBundle2.assets ...]
|
|
||||||
Use - to read from stdin
|
|
||||||
Finds 3-byte unicode characters (like kanji) in files
|
|
||||||
If a filter is supplied, only characters also in the filter will be outputted
|
|
||||||
""", to: &standardError)
|
|
||||||
exit(EXIT_FAILURE)
|
|
||||||
}
|
|
||||||
|
|
||||||
#if !swift(>=4.2)
|
|
||||||
extension Collection {
|
|
||||||
func firstIndex(where predicate: (Element) throws -> Bool) rethrows -> Index? {
|
|
||||||
return try self.index(where: predicate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|
||||||
var filter: String? = nil
|
|
||||||
var inFiles: [String] = Array(CommandLine.arguments[1...])
|
|
||||||
|
|
||||||
if let filterIndex = inFiles.firstIndex(where: { $0.lowercased() == "-filter" }) {
|
|
||||||
if filterIndex + 1 < inFiles.endIndex {
|
|
||||||
filter = try String(contentsOf: URL(fileURLWithPath: inFiles[filterIndex + 1]))
|
|
||||||
inFiles[filterIndex...filterIndex+1] = []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let bundles: [Data]
|
|
||||||
if inFiles == ["-"] {
|
|
||||||
bundles = [FileHandle.standardInput.readDataToEndOfFile()]
|
|
||||||
} else {
|
|
||||||
bundles = try inFiles.map { try Data(contentsOf: URL(fileURLWithPath: $0)) }
|
|
||||||
}
|
|
||||||
|
|
||||||
extension UTF8.CodeUnit {
|
|
||||||
var isStart3: Bool {
|
|
||||||
return self & 0b11110000 == 0b11100000
|
|
||||||
}
|
|
||||||
var isContinuation: Bool {
|
|
||||||
return self & 0b11000000 == 0b10000000
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func unicodeFinder(data: [UInt8], minLength: Int = 2) -> String {
|
|
||||||
var out = [UInt8]()
|
|
||||||
var left = data[...]
|
|
||||||
while true {
|
|
||||||
guard let index = left.firstIndex(where: { ($0 & 0b11110000) == 0b11100000 }) else { break }
|
|
||||||
left = left[index...]
|
|
||||||
guard left.count > 5 else { break }
|
|
||||||
var good = 0
|
|
||||||
for i in stride(from: left.startIndex, to: left.endIndex, by: 3) {
|
|
||||||
if left[i].isStart3 && left[i+1].isContinuation && left[i+2].isContinuation {
|
|
||||||
good += 1
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
if good >= minLength {
|
|
||||||
out.append(contentsOf: left[..<i])
|
|
||||||
good = 0
|
|
||||||
}
|
|
||||||
left = left[(i+1)...]
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if good >= minLength {
|
|
||||||
out.append(contentsOf: left.prefix(left.count / 3 * 3))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return String(decoding: out, as: UTF8.self)
|
|
||||||
}
|
|
||||||
|
|
||||||
let unicodeStrings = bundles.map({ unicodeFinder(data: Array($0)) })
|
|
||||||
var chars = unicodeStrings.map({ Set($0.unicodeScalars) }).reduce(Set(), { $0.union($1) })
|
|
||||||
if let filter = filter {
|
|
||||||
chars.formIntersection(filter.unicodeScalars)
|
|
||||||
}
|
|
||||||
|
|
||||||
print(String(chars.sorted().lazy.map(Character.init)), terminator: "")
|
|
||||||
Reference in New Issue
Block a user