Default user text to UTF-8

src/core/Chef.mjs

@@ -7,7 +7,7 @@
 import Dish from "./Dish.mjs";
 import Recipe from "./Recipe.mjs";
 import log from "loglevel";
-import { isWorkerEnvironment } from "./Utils.mjs";
+import Utils, { isWorkerEnvironment } from "./Utils.mjs";
 
 /**
  * The main controller for CyberChef.
@@ -48,7 +48,13 @@ class Chef {
         if (containsFc && isWorkerEnvironment()) self.setOption("attemptHighlight", false);
 
         // Load data
-        const type = input instanceof ArrayBuffer ? Dish.ARRAY_BUFFER : Dish.STRING;
+        let type = Dish.STRING;
+        if (input instanceof ArrayBuffer) {
+            type = Dish.ARRAY_BUFFER;
+        } else if (typeof input === "string" && recipe.firstActiveInputType() === "ArrayBuffer") {
+            input = Utils.strToUtf8ArrayBuffer(input);
+            type = Dish.ARRAY_BUFFER;
+        }
         this.dish.set(input, type);
 
         try {

src/core/Recipe.mjs

@@ -169,6 +169,17 @@ class Recipe  {
     }
 
 
+    /**
+     * Returns the input type for the first active operation in the recipe.
+     *
+     * @returns {string|null}
+     */
+    firstActiveInputType() {
+        const firstActiveOp = this.opList.find(op => !op.disabled);
+        return firstActiveOp ? OperationConfig[firstActiveOp.name]?.inputType || null : null;
+    }
+
+
     /**
      * Executes each operation in the recipe over the given Dish.
      *

src/core/operations/BLAKE3.mjs

@@ -52,9 +52,9 @@ class BLAKE3 extends Operation {
         const key = args[1];
         const size = args[0];
         const opts = { dkLen: size };
-        const inputBytes = new Uint8Array(Utils.strToArrayBuffer(input));
+        const inputBytes = new Uint8Array(Utils.strToUtf8ArrayBuffer(input));
         if (key !== "") {
-            const keyBytes = new Uint8Array(Utils.strToArrayBuffer(key));
+            const keyBytes = new Uint8Array(Utils.strToUtf8ArrayBuffer(key));
             if (keyBytes.length !== 32) {
                 throw new OperationError("The key must be exactly 32 bytes long");
             }

src/node/NodeRecipe.mjs

@@ -83,6 +83,17 @@ class NodeRecipe {
         this.opList = recipeConfig.map((ing) => this._validateIngredient(ing));
     }
 
+    /**
+     * Returns the inputType of the first operation in the opList
+     * @returns String
+     */
+    firstActiveInputType() {
+        const first = this.opList[0];
+        if (!first) return null;
+        const op = Object.prototype.hasOwnProperty.call(first, "op") ? first.op : first;
+        return op.inputType || null;
+    }
+
     /**
      * Run the dish through each operation, one at a time.
      * @param {NodeDish} dish

src/node/api.mjs

@@ -10,6 +10,8 @@
 
 import NodeDish from "./NodeDish.mjs";
 import NodeRecipe from "./NodeRecipe.mjs";
+import Dish from "../core/Dish.mjs";
+import Utils from "../core/Utils.mjs";
 import OperationConfig from "../core/config/OperationConfig.json" with { type: "json" };
 import { sanitise, removeSubheadingsFromArray, sentenceToCamelCase } from "./apiUtils.mjs";
 import ExcludedOperationError from "../core/errors/ExcludedOperationError.mjs";
@@ -118,7 +120,9 @@ function ensureIsDish(input) {
  * @param args - operation args
  */
 function prepareOp(opInstance, input, args) {
-    const dish = ensureIsDish(input);
+    const dish = typeof input === "string" && opInstance.inputType === "ArrayBuffer" ?
+        new NodeDish(Utils.strToUtf8ArrayBuffer(input), Dish.ARRAY_BUFFER) :
+        ensureIsDish(input);
     // Transform object-style args to original args array
     const transformedArgs = transformArgs(opInstance.args, args);
     const transformedInput = dish.get(opInstance.inputType);
@@ -242,6 +246,7 @@ export function _wrap(OpClass) {
 
     // used in chef.help
     wrapped.opName = OpClass.name;
+    wrapped.inputType = opInstance.inputType;
     wrapped.args = createArgInfo(opInstance);
     // Used in NodeRecipe to check for flowControl ops
     wrapped.flowControl = isFlowControl;
@@ -329,7 +334,9 @@ export function help(input) {
  */
 export async function bake(input, recipeConfig) {
     const recipe = new NodeRecipe(recipeConfig);
-    const dish = ensureIsDish(input);
+    const dish = typeof input === "string" && recipe.firstActiveInputType() === "ArrayBuffer" ?
+        new NodeDish(Utils.strToUtf8ArrayBuffer(input), Dish.ARRAY_BUFFER) :
+        ensureIsDish(input);
     return await recipe.execute(dish);
 }
 

src/web/App.mjs

@@ -537,12 +537,17 @@ class App {
         if (this.uriParams.input) {
             try {
                 let inputVal;
-                const inputChrEnc = this.manager.input.getChrEnc();
                 const inputData = fromBase64(this.uriParams.input, null, "byteArray");
-                if (inputChrEnc > 0) {
-                    inputVal = cptable.utils.decode(inputChrEnc, inputData);
+                if (this.uriParams.ienc) {
+                    const inputChrEnc = this.manager.input.getChrEnc();
+                    if (inputChrEnc > 0) {
+                        inputVal = cptable.utils.decode(inputChrEnc, inputData);
+                    } else {
+                        inputVal = Utils.byteArrayToChars(inputData);
+                    }
                 } else {
                     inputVal = Utils.byteArrayToChars(inputData);
+                    this.manager.input.chrEncChange(0, true, true);
                 }
                 this.setInput(inputVal);
             } catch (err) {}

src/web/waiters/InputWaiter.mjs

@@ -656,10 +656,9 @@ class InputWaiter {
             this.setInput(inputVal, silent);
 
             // Set URL to current input
-            if (inputVal.length >= 0 && inputVal.length <= 51200) {
-                const inputStr = toBase64(inputVal, "A-Za-z0-9+/");
-                this.app.updateURL(true, inputStr);
-            }
+            const inputStr = inputData.buffer.byteLength < 51200 ? toBase64(inputData.buffer, "A-Za-z0-9+/") : "";
+            const includeInput = inputStr.length > 0 && inputData.buffer.byteLength < 51200;
+            this.app.updateURL(includeInput, inputStr);
         }.bind(this));
     }
 

src/web/workers/InputWorker.mjs

@@ -756,7 +756,7 @@ self.addInput = function(
         file: null,
         status: "pending",
         progress: 0,
-        encoding: 0,
+        encoding: type === "userinput" ? 65001 : 0,
         eolSequence: "\u000a"
     };
 

tests/browser/01_io.js

@@ -125,7 +125,7 @@ module.exports = {
         browser.waitForElementVisible("#input-text .cm-status-bar .stats-lines-value")
             .expect.element("#input-text .cm-status-bar .stats-lines-value").text.to.equal("1");
         browser.waitForElementVisible("#input-text .cm-status-bar .chr-enc-value")
-            .expect.element("#input-text .cm-status-bar .chr-enc-value").text.to.equal("Raw Bytes");
+            .expect.element("#input-text .cm-status-bar .chr-enc-value").text.to.equal("UTF-8");
         browser.waitForElementVisible("#input-text .cm-status-bar .eol-value")
             .expect.element("#input-text .cm-status-bar .eol-value").text.to.equal("LF");
 
@@ -149,7 +149,7 @@ module.exports = {
 
         browser.expect.element("#input-text .cm-status-bar .stats-length-value").text.to.equal("301");
         browser.expect.element("#input-text .cm-status-bar .stats-lines-value").text.to.equal("3");
-        browser.expect.element("#input-text .cm-status-bar .chr-enc-value").text.to.equal("Raw Bytes");
+        browser.expect.element("#input-text .cm-status-bar .chr-enc-value").text.to.equal("UTF-8");
         browser.expect.element("#input-text .cm-status-bar .eol-value").text.to.equal("LF");
 
         browser.expect.element("#output-text .cm-status-bar .stats-length-value").text.to.equal("0");
@@ -167,6 +167,22 @@ module.exports = {
         browser.expect.element("#output-text .cm-status-bar .eol-value").text.to.equal("LF");
     },
 
+    "Manual input To Hex uses selected encoding": browser => {
+        utils.loadRecipe(browser, "To Hex", "á", ["Space", 0]);
+        utils.bake(browser);
+        utils.expectOutput(browser, "c3 a1", true);
+
+        utils.setChrEnc(browser, "input", "Raw Bytes");
+        browser.execute(text => {
+            window.app.setInput(text);
+        }, ["á"]);
+        utils.expectInput(browser, "á");
+        utils.bake(browser);
+        utils.expectOutput(browser, "e1", true);
+
+        utils.setChrEnc(browser, "input", "UTF-8");
+    },
+
     "Autobaking the latest input": browser => {
         // Use the sleep recipe to simulate a long running task
         utils.loadRecipe(browser, "Sleep", "input", [2000]);
@@ -433,7 +449,7 @@ module.exports = {
         browser
             .click("#btn-new-tab")
             .waitForElementVisible("#input-tabs li:nth-of-type(2).active-input-tab");
-        browser.expect.element("#input-text .chr-enc-value").text.that.equals("Raw Bytes");
+        browser.expect.element("#input-text .chr-enc-value").text.that.equals("UTF-8");
         browser.expect.element("#output-text .chr-enc-value").text.that.equals("Raw Bytes");
 
         utils.setChrEnc(browser, "input", "UTF-7");

tests/node/tests/operations.mjs

@@ -980,6 +980,18 @@ smothering ampersand abreast`;
         assert.strictEqual(result.toString(), "73:6f:6d:65:20:69:6e:70:75:74");
     }),
 
+    it("toHex: Latin-1 text input uses UTF-8", () => {
+        const result = toHex("á", {
+            delimiter: "Space",
+        });
+        assert.strictEqual(result.toString(), "c3 a1");
+    }),
+
+    it("bake: byte-first Latin-1 text input uses UTF-8", async () => {
+        const result = await chef.bake("á", "to hex");
+        assert.strictEqual(result.toString(), "c3 a1");
+    }),
+
     it("To Kebab case", () => {
         assert.strictEqual(chef.toKebabCase("Elfin Gold").toString(), "elfin-gold");
     }),

tests/operations/tests/AvroToJSON.mjs

@@ -8,6 +8,8 @@
 
 import TestRegister from "../../lib/TestRegister.mjs";
 
+const rawStringToArrayBuffer = str => Uint8Array.from(str, c => c.charCodeAt(0)).buffer;
+
 TestRegister.addTests([
     {
         name: "Avro to JSON: no input (force JSON true)",
@@ -33,12 +35,12 @@ TestRegister.addTests([
     },
     {
         name: "Avro to JSON: small (force JSON true)",
-        input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
+        input: rawStringToArrayBuffer("\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
             "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
             "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
             "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
             "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
-            "\x15\x41",
+            "\x15\x41"),
         expectedOutput: "{\n    \"name\": \"myname\"\n}",
         recipeConfig: [
             {
@@ -49,12 +51,12 @@ TestRegister.addTests([
     },
     {
         name: "Avro to JSON: small (force JSON false)",
-        input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
+        input: rawStringToArrayBuffer("\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
             "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
             "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
             "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
             "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
-            "\x15\x41",
+            "\x15\x41"),
         expectedOutput: "{\"name\":\"myname\"}\n",
         recipeConfig: [
             {

tests/operations/tests/BLAKE3.mjs

@@ -25,6 +25,15 @@ TestRegister.addTests([
                 "args": [16, ""] }
         ]
     },
+    {
+        name: "BLAKE3: 16 - Latin-1 text input uses UTF-8",
+        input: "á",
+        expectedOutput: "bb9c95e4b863990afb5259512240f785",
+        recipeConfig: [
+            { "op": "BLAKE3",
+                "args": [16, ""] }
+        ]
+    },
     {
         name: "BLAKE3: 32 - Hello world",
         input: "Hello world",

tests/operations/tests/Base32.mjs

@@ -36,6 +36,8 @@ const ALL_BYTES = [
     "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 ].join("");
 
+const rawStringToArrayBuffer = str => Uint8Array.from(str, c => c.charCodeAt(0)).buffer;
+
 const ALL_BYTES_EXTENDED_OUT = "000G40O40K30E209185GO38E1S8124GJ2GAHC5OO34D1M70T3OFI08924CI2A9H750KIKAPC5KN2UC1H68PJ8D9M6SS3IEHR7GUJSFQ085146H258P3KGIAA9D64QJIFA18L4KQKALB5EM2PB9DLONAUBTG62OJ3CHIMCPR8D5L6MR3DDPNN0SBIEDQ7ATJNF1SNKURSFLV7V041GA1O91C6GU48J2KBHI6OT3SGI699754LIQBPH6CQJEE9R7KVK2GQ58T4KMJAFA59LALQPBDELUOB3CLJMIQRDDTON6TBNF5TNQVS1GE2OF2CBHM7P34SLIUCPN7CVK6HQB9T9LEMQVCDJMMRRJETTNV0S7HE7P75SRJUHQFATFMERRNFU3OV5SVKUNRFFU7PVBTVPVFUVS======";
 const ALL_BYTES_STANDARD_OUT = "AAAQEAYEAUDAOCAJBIFQYDIOB4IBCEQTCQKRMFYYDENBWHA5DYPSAIJCEMSCKJRHFAUSUKZMFUXC6MBRGIZTINJWG44DSOR3HQ6T4P2AIFBEGRCFIZDUQSKKJNGE2TSPKBIVEU2UKVLFOWCZLJNVYXK6L5QGCYTDMRSWMZ3INFVGW3DNNZXXA4LSON2HK5TXPB4XU634PV7H7AEBQKBYJBMGQ6EITCULRSGY5D4QSGJJHFEVS2LZRGM2TOOJ3HU7UCQ2FI5EUWTKPKFJVKV2ZLNOV6YLDMVTWS23NN5YXG5LXPF5X274BQOCYPCMLRWHZDE4VS6MZXHM7UGR2LJ5JVOW27MNTWW33TO55X7A4HROHZHF43T6R2PK5PWO33XP6DY7F47U6X3PP6HZ7L57Z7P674======";
 
@@ -130,7 +132,7 @@ TestRegister.addTests([
     },
     {
         name: "To Base32 Hex Standard: All Bytes",
-        input: ALL_BYTES,
+        input: rawStringToArrayBuffer(ALL_BYTES),
         expectedOutput: ALL_BYTES_STANDARD_OUT,
         recipeConfig: [
             {
@@ -141,7 +143,7 @@ TestRegister.addTests([
     },
     {
         name: "To Base32 Hex Extended: All Bytes",
-        input: ALL_BYTES,
+        input: rawStringToArrayBuffer(ALL_BYTES),
         expectedOutput: ALL_BYTES_EXTENDED_OUT,
         recipeConfig: [
             {

tests/operations/tests/Base64.mjs

@@ -27,6 +27,8 @@ const ALL_BYTES = [
     "\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff",
 ].join("");
 
+const rawStringToArrayBuffer = str => Uint8Array.from(str, c => c.charCodeAt(0)).buffer;
+
 TestRegister.addTests([
     {
         name: "To Base64: nothing",
@@ -63,7 +65,7 @@ TestRegister.addTests([
     },
     {
         name: "To Base64: All bytes",
-        input: ALL_BYTES,
+        input: rawStringToArrayBuffer(ALL_BYTES),
         expectedOutput: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/w==",
         recipeConfig: [
             {
@@ -118,7 +120,7 @@ TestRegister.addTests([
     },
     {
         name: "Show Base64 offsets: escapes static output",
-        input: "\x00\x10\x83\x10\x51\x87",
+        input: rawStringToArrayBuffer("\x00\x10\x83\x10\x51\x87"),
         expectedOutput: "<script>\n<AQmsBRk66\n<ia1AEIM6",
         recipeConfig: [
             {

tests/operations/tests/Base85.mjs

@@ -7,6 +7,8 @@
  */
 import TestRegister from "../../lib/TestRegister.mjs";
 
+const rawStringToArrayBuffer = str => Uint8Array.from(str, c => c.charCodeAt(0)).buffer;
+
 // Example from Wikipedia
 const wpExample = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure.";
 // Escape newline, quote & backslash
@@ -51,7 +53,7 @@ TestRegister.addTests([
     },
     {
         name: "To Base85",
-        input: allZeroExample,
+        input: rawStringToArrayBuffer(allZeroExample),
         expectedOutput: allZeroOutput,
         recipeConfig: [
             { "op": "To Base85",