Add fast handlers

src/engine/BinParser.v3

@@ -541,6 +541,20 @@ class BinParser(extensions: Extension.set, limits: Limits, err: ErrorGen, filena
 		}
 		var decl = readExportWithoutName();
 		module.exports.put(name, decl);
+		// assign fast functions
+		if (FastIntTuning.useFastFunctions && Strings.startsWith(name, "fast:") && FuncDecl.?(decl)) {
+			def fdecl = FuncDecl.!(decl);
+			def fast_funcs = module.fast_funcs;
+			if (fdecl.fast_call_idx >= 0) {
+				err.rel(decoder, pt).DuplicateFastExport(fdecl.func_index);
+			} else if (fast_funcs.length >= Opcodes.FAST_CALL_OPCODES) {
+				// No fast-call slot available; fall back to a regular call for this function.
+				if (Trace.validation) Trace.OUT.put1("    dropping fast: export, no FAST_CALL slots remain (max %d)\n", Opcodes.FAST_CALL_OPCODES);
+			} else {
+				fdecl.fast_call_idx = fast_funcs.length;
+				fast_funcs.put(fdecl);
+			}
+		}
 	}
 	def readExportWithoutName() -> Decl {
 		var pt = decoder.pos;

src/engine/BytecodeIterator.v3

@@ -779,6 +779,48 @@ class BytecodeIterator {
 			RESUME_THROW => 		v.visit_RESUME_THROW(read_CONT(), read_TAG(), read_HANDLERS());
 			RESUME_THROW_REF => 		v.visit_RESUME_THROW_REF(read_CONT(), read_HANDLERS());
 			SWITCH => 			v.visit_SWITCH(read_CONT(), read_TAG());
+
+			// replacing CALL with FAST_CALL does not touch the operand so that the original function can still be recovered
+			FAST_CALL0  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL1  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL2  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL3  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL4  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL5  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL6  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL7  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL8  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL9  => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL10 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL11 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL12 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL13 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL14 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL15 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL16 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL17 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL18 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL19 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL20 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL21 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL22 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL23 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL24 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL25 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL26 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL27 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL28 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL29 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL30 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL31 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL32 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL33 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL34 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL35 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL36 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL37 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL38 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
+			FAST_CALL39 => v.visit_FAST_CALL(Opcodes.fastCallToIndex(opcode), read_FUNC());
 		}
 	}
 	def trace(out: StringBuilder, module: Module, tracer: InstrTracer) {

src/engine/CodeValidator.v3

@@ -420,6 +420,12 @@ class CodeValidator(extensions: Extension.set, limits: Limits, module: Module, e
 					var func = parser.readFuncRef();
 					if (func == null) return;
 					checkSignature(func.sig);
+
+					// fast call: if function is exported with fast name, replace the bytecode with FAST_CALL
+					if (FastIntTuning.useFastFunctions && func.fast_call_idx >= 0) {
+						if (Trace.validation) Trace.OUT.put1("    replacing with FAST_CALL%d\n", func.fast_call_idx);
+						this.func.replaceCall(opcode_pos, func.fast_call_idx);
+					}
 				}
 				CALL_INDIRECT => {
 					var sig = parser.readSigRef();

src/engine/Module.v3

@@ -17,6 +17,7 @@ class Module(filename: string) {
 	def exports = Vector<(string, Decl)>.new();
 	def elems = Vector<ElemDecl>.new();
 	def data = Vector<DataDecl>.new();
+	def fast_funcs = Vector<FuncDecl>.new();
 	def custom_sections = Vector<CustomSection>.new();
 	var probes: Array<Array<Probe>>;
 	var dyn_probes: Vector<(int, int, Probe)>;
@@ -143,6 +144,8 @@ class FuncDecl(sig_index: int) extends Decl {
 	var target_code: TargetCode;
 	var tierup_trigger: int = int.max;
 	var handlers = FuncHandlerInfo.new();
+	var fast_target_code: TargetCode;
+	var fast_call_idx: int = -1;
 
 	def render(names: NameSection, buf: StringBuilder) -> StringBuilder {
 		var name = if (names != null, names.getFuncName(func_index));
@@ -154,6 +157,7 @@ class FuncDecl(sig_index: int) extends Decl {
 		var tc: TargetCode;
 		var tr: TargetCode;
 		target_code = tc; // reset target code as well
+		fast_target_code = tc;
 		sidetable = Sidetables.NO_SIDETABLE;
 		cbd_sidetable = null;
 	}
@@ -168,6 +172,11 @@ class FuncDecl(sig_index: int) extends Decl {
 		if (cur_bytecode == orig_bytecode) return;
 		cur_bytecode[pc] = orig_bytecode[pc];
 	}
+	def replaceCall(pc: int, idx: int) {
+		// copy bytecode for modification
+		if (cur_bytecode == orig_bytecode) orig_bytecode = Arrays.dup(orig_bytecode);
+		cur_bytecode[pc] = byte.!(Opcodes.indexToFastCall(idx).code);
+	}
 	def reset() -> this {
 		if (cur_bytecode == orig_bytecode) return;
 		ArrayUtil.copyInto(cur_bytecode, 0, orig_bytecode, 0, orig_bytecode.length);
@@ -183,6 +192,7 @@ class FuncDecl(sig_index: int) extends Decl {
 		n.sidetable = this.sidetable;
 		n.num_locals = this.num_locals;
 		n.target_code = this.target_code;
+		n.fast_target_code = this.fast_target_code;
 		return n;
 	}
 	def findExHandler(instance: Instance, tag: Tag, throw_pc: int) -> ExHandler {

src/engine/Opcodes.v3

@@ -609,8 +609,51 @@ enum Opcode(prefix: u8, code: u16, mnemonic: string, imms: Array<ImmKind>, sig:
 	RESUME_THROW			(0x00, 0xE4, "resume_throw", imm.CONT_TAG_HANDLE, null),
 	RESUME_THROW_REF		(0x00, 0xE5, "resume_throw_ref", imm.CONT_HANDLE, null),
 	SWITCH				(0x00, 0xE6, "switch", imm.CONT_TAG, null)
+
+	// fast call instructions
+	FAST_CALL0               (0x00, 0xC5, "fast_call0",  imm.FUNC, null),
+	FAST_CALL1               (0x00, 0xC6, "fast_call1",  imm.FUNC, null),
+	FAST_CALL2               (0x00, 0xC7, "fast_call2",  imm.FUNC, null),
+	FAST_CALL3               (0x00, 0xC8, "fast_call3",  imm.FUNC, null),
+	FAST_CALL4               (0x00, 0xC9, "fast_call4",  imm.FUNC, null),
+	FAST_CALL5               (0x00, 0xCA, "fast_call5",  imm.FUNC, null),
+	FAST_CALL6               (0x00, 0xCB, "fast_call6",  imm.FUNC, null),
+	FAST_CALL7               (0x00, 0xCC, "fast_call7",  imm.FUNC, null),
+	FAST_CALL8               (0x00, 0xCD, "fast_call8",  imm.FUNC, null),
+	FAST_CALL9               (0x00, 0xCE, "fast_call9",  imm.FUNC, null),
+	FAST_CALL10              (0x00, 0xCF, "fast_call10", imm.FUNC, null),
+	FAST_CALL11              (0x00, 0xD7, "fast_call11", imm.FUNC, null),
+	FAST_CALL12              (0x00, 0xD8, "fast_call12", imm.FUNC, null),
+	FAST_CALL13              (0x00, 0xD9, "fast_call13", imm.FUNC, null),
+	FAST_CALL14              (0x00, 0xDA, "fast_call14", imm.FUNC, null),
+	FAST_CALL15              (0x00, 0xDB, "fast_call15", imm.FUNC, null),
+	FAST_CALL16              (0x00, 0xDC, "fast_call16", imm.FUNC, null),
+	FAST_CALL17              (0x00, 0xDD, "fast_call17", imm.FUNC, null),
+	FAST_CALL18              (0x00, 0xDE, "fast_call18", imm.FUNC, null),
+	FAST_CALL19              (0x00, 0xDF, "fast_call19", imm.FUNC, null),
+	FAST_CALL20              (0x00, 0xE7, "fast_call20", imm.FUNC, null),
+	FAST_CALL21              (0x00, 0xE8, "fast_call21", imm.FUNC, null),
+	FAST_CALL22              (0x00, 0xE9, "fast_call22", imm.FUNC, null),
+	FAST_CALL23              (0x00, 0xEA, "fast_call23", imm.FUNC, null),
+	FAST_CALL24              (0x00, 0xEB, "fast_call24", imm.FUNC, null),
+	FAST_CALL25              (0x00, 0xEC, "fast_call25", imm.FUNC, null),
+	FAST_CALL26              (0x00, 0xED, "fast_call26", imm.FUNC, null),
+	FAST_CALL27              (0x00, 0xEE, "fast_call27", imm.FUNC, null),
+	FAST_CALL28              (0x00, 0xEF, "fast_call28", imm.FUNC, null),
+	FAST_CALL29              (0x00, 0xF2, "fast_call29", imm.FUNC, null),
+	FAST_CALL30              (0x00, 0xF3, "fast_call30", imm.FUNC, null),
+	FAST_CALL31              (0x00, 0xF4, "fast_call31", imm.FUNC, null),
+	FAST_CALL32              (0x00, 0xF5, "fast_call32", imm.FUNC, null),
+	FAST_CALL33              (0x00, 0xF6, "fast_call33", imm.FUNC, null),
+	FAST_CALL34              (0x00, 0xF7, "fast_call34", imm.FUNC, null),
+	FAST_CALL35              (0x00, 0xF8, "fast_call35", imm.FUNC, null),
+	FAST_CALL36              (0x00, 0xF9, "fast_call36", imm.FUNC, null),
+	FAST_CALL37              (0x00, 0xFA, "fast_call37", imm.FUNC, null),
+	FAST_CALL38              (0x00, 0x17, "fast_call38", imm.FUNC, null),
+	FAST_CALL39              (0x00, 0x27, "fast_call39", imm.FUNC, null),
 }
 
+
 // Enumeration of the different kinds of immediates to opcodes.
 enum ImmKind {
 	ARRAY_TYPE_INDEX,	// ARRAYT
@@ -798,9 +841,16 @@ component Opcodes {
 	def code_pages = [page_FB, page_FC, page_FD, page_FE];
 	def var longestName: int;
 	def var num_subpages: int;
+	def FAST_CALL_OPCODES = 40;
+	def var fast_calls: Array<Opcode>;
 	private var nameMap: HashMap<string, Opcode>;
 
 	new() {
+
+		fast_calls = Array.new(FAST_CALL_OPCODES);
+		for (i < FAST_CALL_OPCODES) {
+			fast_calls[i] = indexToFastCall(i);
+		}
 		for (op in Opcode) {
 			if (op == Opcode.INVALID) continue;
 			init(op);
@@ -809,6 +859,7 @@ component Opcodes {
 		attributes[InternalOpcode.PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE;
 		attributes[InternalOpcode.WHAMM_PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE;
 		attributes[InternalOpcode.BREAK_PROBE.code] = OpcodeAttribute.INTERNAL | OpcodeAttribute.PROBE;
+		for (op in fast_calls) attributes[op.tag] = OpcodeAttribute.INTERNAL;
 
 		for (op in [Opcode.END, Opcode.I32_CONST, Opcode.I64_CONST, Opcode.F32_CONST, Opcode.F64_CONST, Opcode.GLOBAL_GET,
 				Opcode.REF_NULL, Opcode.REF_FUNC, Opcode.STRUCT_NEW, Opcode.STRUCT_NEW_DEFAULT,
@@ -1121,6 +1172,100 @@ component Opcodes {
 			}
 		}
 	}
+	def indexToFastCall(index: int) -> Opcode {
+		var op: Opcode;
+		match (index) {
+			0  => op = Opcode.FAST_CALL0;
+			1  => op = Opcode.FAST_CALL1;
+			2  => op = Opcode.FAST_CALL2;
+			3  => op = Opcode.FAST_CALL3;
+			4  => op = Opcode.FAST_CALL4;
+			5  => op = Opcode.FAST_CALL5;
+			6  => op = Opcode.FAST_CALL6;
+			7  => op = Opcode.FAST_CALL7;
+			8  => op = Opcode.FAST_CALL8;
+			9  => op = Opcode.FAST_CALL9;
+			10 => op = Opcode.FAST_CALL10;
+			11 => op = Opcode.FAST_CALL11;
+			12 => op = Opcode.FAST_CALL12;
+			13 => op = Opcode.FAST_CALL13;
+			14 => op = Opcode.FAST_CALL14;
+			15 => op = Opcode.FAST_CALL15;
+			16 => op = Opcode.FAST_CALL16;
+			17 => op = Opcode.FAST_CALL17;
+			18 => op = Opcode.FAST_CALL18;
+			19 => op = Opcode.FAST_CALL19;
+			20 => op = Opcode.FAST_CALL20;
+			21 => op = Opcode.FAST_CALL21;
+			22 => op = Opcode.FAST_CALL22;
+			23 => op = Opcode.FAST_CALL23;
+			24 => op = Opcode.FAST_CALL24;
+			25 => op = Opcode.FAST_CALL25;
+			26 => op = Opcode.FAST_CALL26;
+			27 => op = Opcode.FAST_CALL27;
+			28 => op = Opcode.FAST_CALL28;
+			29 => op = Opcode.FAST_CALL29;
+			30 => op = Opcode.FAST_CALL30;
+			31 => op = Opcode.FAST_CALL31;
+			32 => op = Opcode.FAST_CALL32;
+			33 => op = Opcode.FAST_CALL33;
+			34 => op = Opcode.FAST_CALL34;
+			35 => op = Opcode.FAST_CALL35;
+			36 => op = Opcode.FAST_CALL36;
+			37 => op = Opcode.FAST_CALL37;
+			38 => op = Opcode.FAST_CALL38;
+			39 => op = Opcode.FAST_CALL39;
+			_ => System.error("indexToFastCall", "out of range");
+		}
+		return op;
+	}
+	def fastCallToIndex(op: Opcode) -> int {
+		var idx: int;
+		match (op) {
+			FAST_CALL0  => idx = 0;
+			FAST_CALL1  => idx = 1;
+			FAST_CALL2  => idx = 2;
+			FAST_CALL3  => idx = 3;
+			FAST_CALL4  => idx = 4;
+			FAST_CALL5  => idx = 5;
+			FAST_CALL6  => idx = 6;
+			FAST_CALL7  => idx = 7;
+			FAST_CALL8  => idx = 8;
+			FAST_CALL9  => idx = 9;
+			FAST_CALL10 => idx = 10;
+			FAST_CALL11 => idx = 11;
+			FAST_CALL12 => idx = 12;
+			FAST_CALL13 => idx = 13;
+			FAST_CALL14 => idx = 14;
+			FAST_CALL15 => idx = 15;
+			FAST_CALL16 => idx = 16;
+			FAST_CALL17 => idx = 17;
+			FAST_CALL18 => idx = 18;
+			FAST_CALL19 => idx = 19;
+			FAST_CALL20 => idx = 20;
+			FAST_CALL21 => idx = 21;
+			FAST_CALL22 => idx = 22;
+			FAST_CALL23 => idx = 23;
+			FAST_CALL24 => idx = 24;
+			FAST_CALL25 => idx = 25;
+			FAST_CALL26 => idx = 26;
+			FAST_CALL27 => idx = 27;
+			FAST_CALL28 => idx = 28;
+			FAST_CALL29 => idx = 29;
+			FAST_CALL30 => idx = 30;
+			FAST_CALL31 => idx = 31;
+			FAST_CALL32 => idx = 32;
+			FAST_CALL33 => idx = 33;
+			FAST_CALL34 => idx = 34;
+			FAST_CALL35 => idx = 35;
+			FAST_CALL36 => idx = 36;
+			FAST_CALL37 => idx = 37;
+			FAST_CALL38 => idx = 38;
+			FAST_CALL39 => idx = 39;
+			_ => System.error("fastCallToIndex", "not a FAST_CALL instruction");
+		}
+		return idx;
+	}
 }
 
 // Renders instructions as text.