Skip to content
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
npm
68 changes: 68 additions & 0 deletions build_npm.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { build, emptyDir } from "https://deno.land/x/dnt/mod.ts";

const version = Deno.args[0];

if (!version) {
throw new Error("Please specify a version.");
}

await emptyDir("./npm");

await build({
entryPoints: ["./mod.ts"], // Replace with your actual entry point
outDir: "./npm",
shims: {
deno: {
test: "dev",
},
custom: [
{
package: {
name: "web-streams-polyfill",

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question: Can we just import these from node:stream/web which is built into Node.js? That would keep the overall npm package size smaller, which is a goal for this project (small and no third party dependencies outside std lib).

@riderx riderx Feb 16, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it seems node:stream/web doesn't implement stream the same way:
CleanShot 2024-02-17 at 00 21 25@2x

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, it might be just a type error; I wonder if it works if the type checking is disabled. But in any case, seems like it's not a straightforward change. Thanks for checking into it anyways.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i will try to run integration test

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I managed to run integration test in Node.js and this doesn't work, it's not only the type:
CleanShot 2024-02-17 at 00 41 16@2x

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok in fact integration test fail as well with polyfill

version: "^3.1.1",
},
globalNames: ["ReadableStream", "WritableStream", "TransformStream"],
},
],
},
package: {
// Update with your package details
name: "s3-lite-client",
version: version,
description: "This is a lightweight S3 client for Node.js and Deno.",
license: "MIT",
repository: {
type: "git",
url: "git+https://github.com/bradenmacdonald/deno-s3-lite-client.git",
},
bugs: {
url: "https://github.com/bradenmacdonald/deno-s3-lite-client/issues",
},
engines: {
"node": ">=16"
},
author: {
"name": "Braden MacDonald",
"url": "https://github.com/bradenmacdonald"
},
contributors: [
"Martin Donadieu <martindonadieu@gmail.com> (https://martin.solos.ventures/)",
],
keywords: [
"api",
"lite",
"amazon",
"minio",
"cloud",
"s3",
"storage"
]
},
postBuild() {
// Copy additional files to the npm directory if needed
Deno.copyFileSync("LICENSE", "npm/LICENSE");
Deno.copyFileSync("README.md", "npm/README.md");
},
});

console.log("Build complete. Run `cd npm && npm publish`.");
1 change: 0 additions & 1 deletion deps.ts

This file was deleted.

11 changes: 5 additions & 6 deletions signing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,13 +153,12 @@ function getHeadersToSign(headers: Headers): string[] {
"content-type",
"user-agent",
];
const headersToSign = [];
for (const key of headers.keys()) {
if (ignoredHeaders.includes(key.toLowerCase())) {
continue; // Ignore this header
const headersToSign: string[] = [];
headers.forEach((value, key) => {
if (!ignoredHeaders.includes(key.toLowerCase())) {
headersToSign.push(key);
}
headersToSign.push(key);
}
});
headersToSign.sort();
return headersToSign;
}
Expand Down
2 changes: 1 addition & 1 deletion transform-chunk-sizes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { TransformChunkSizes } from "./transform-chunk-sizes.ts";
*/
class NumberSource extends ReadableStream<Uint8Array> {
constructor(delayMs: number, chunksCount: number, bytesPerChunk = 1) {
let intervalTimer: number;
let intervalTimer: any;

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
let intervalTimer: any;
let intervalTimer: ReturnType<typeof setTimeout>;

Better to use a specific type.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok fixed

let i = 0;
super({
start(controller) {
Expand Down
43 changes: 23 additions & 20 deletions transform-chunk-sizes.ts
Original file line number Diff line number Diff line change
@@ -1,36 +1,39 @@
import { Buffer } from "./deps.ts";

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unfortunately something about this changed TransformChunkSizes is causing the integration tests to fail, which means there must be a bug in how it's processing the data.

Then I discovered There is a better alternative to Buffer available in every runtime: https://sindresorhus.com/blog/goodbye-nodejs-buffer

That article is about Node's Buffer. This repo uses the Deno Buffer which explicitly says:

  • Buffer is NOT the same thing as Node's Buffer. Node's Buffer was created in
  • 2009 before JavaScript had the concept of ArrayBuffers. It's simply a
  • non-standard ArrayBuffer.

What is wrong with using Deno's Buffer implementation? It doesn't use any Deno-specific APIs as far as I know. If you just leave it in, what doesn't work?

BTW at one point Deno std decided to deprecate their Buffer, so I actually created a branch of this that works without it, by copying its code into this repo: https://github.com/bradenmacdonald/deno-s3-lite-client/compare/no-buffer . But they changed their mind, and so I'm still using Deno std Buffer.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wasn't sure dnt will make it work, I can try


/**
* This stream transform will buffer the data it receives until it has enough to form
* a chunk of the specified size, then pass on the data in chunks of the specified size.
*/
export class TransformChunkSizes extends TransformStream<Uint8Array, Uint8Array> {
constructor(outChunkSize: number) {
// This large buffer holds all the incoming data we receive until we reach at least outChunkSize, which we then pass on.
const buffer = new Buffer();
buffer.grow(outChunkSize);
const buffer = new Uint8Array(outChunkSize * 2); // Buffer size is twice the chunk size to ensure there's enough space
let offset = 0; // Offset to keep track of the current position in the buffer

super({
start() {}, // required
async transform(chunk, controller) {
buffer.write(chunk);
start(_controller) {
// No initialization needed here since we've already initialized buffer and offset in the constructor.
},
transform(chunk, controller) {
let chunkOffset = 0;

while (buffer.length >= outChunkSize) {
const outChunk = new Uint8Array(outChunkSize);
const readFromBuffer = await buffer.read(outChunk);
if (readFromBuffer !== outChunkSize) {
throw new Error(
`Unexpectedly read ${readFromBuffer} bytes from transform buffer when trying to read ${outChunkSize} bytes.`,
);
}
// Now "outChunk" holds the next chunk of data - pass it on to the output:
controller.enqueue(outChunk);
// If the incoming chunk won't fit in the remaining buffer space, we need to process what's in the buffer first
while (offset + chunk.length - chunkOffset > outChunkSize) {
// Calculate how much of the incoming chunk we can fit into the buffer
const spaceLeft = outChunkSize - offset;
buffer.set(chunk.subarray(chunkOffset, chunkOffset + spaceLeft), offset);
controller.enqueue(buffer.subarray(0, outChunkSize));
offset = 0;
chunkOffset += spaceLeft;
}

// Put the remaining chunk into the buffer
buffer.set(chunk.subarray(chunkOffset), offset);
offset += chunk.length - chunkOffset;
},
flush(controller) {
if (buffer.length) {
// The buffer still contains some data, send it now even though it's smaller than the desired chunk size.
controller.enqueue(buffer.bytes());
if (offset > 0) {
// Send any remaining data in the buffer
controller.enqueue(buffer.subarray(0, offset));
offset = 0;
}
},
});
Expand Down