feat: define where a beacon event lives in the log table

docs/log-table/README.md

@@ -160,6 +160,35 @@ and this was not one of them, because the field joined the set afterwards. A Par
 spells it some other way answers every query with a column of nulls and reports success. Read an
 object back before trusting a visitor count over Parquet.
 
+## A beacon row is a row like any other
+
+The [optional beacon](https://github.com/KensioSoftware/rainlytics/issues/100) sends a GET to a path
+on the site's own domain and puts its payload in the query string. CloudFront records `cs-uri-query`
+whatever the cache key and origin forwarding are set to. The event lands in the same objects, the
+same partitions and this same table. Layer 2 is more rows in the dataset layer 1 already writes.
+
+The payload stays in `cs_uri_query` and is read at query time.
+
+```sql
+SELECT url_decode(url_extract_parameter(cs_uri_stem || '?' || cs_uri_query, 'e')) AS event,
+       count(*) AS events
+  FROM rainlytics.cloudfront_logs
+ WHERE cs_uri_stem = '/_rainlytics'
+```
+
+A column of its own was never available. This table's columns are the fields the delivery was
+configured with, and CloudFront has no field carrying somebody else's payload. A view or a second
+table over the same objects meets the same wall, since no SerDe parses a query string. The
+[searches](../searches/) rollup already reads a search term out of the same column the same way.
+
+`beaconQueryString`, `beaconEventColumn` and `aBeaconEvent` are exported from the package root, so
+the browser writing the payload and the SQL reading it hold one definition between them.
+
+Partitions written before the beacon shipped answer no rows at all. A beacon row is identified by
+the path it was sent to. A query for that path over an older day matches nothing, where a column
+added later would have answered nulls. That is the one shape of schema change an immutable store
+takes without argument.
+
 ## Several distributions, one table
 
 `distributionid` partitions before time does, so one table covers every site delivering into the

src/beacon-events.test.ts

@@ -0,0 +1,84 @@
+import { faker } from "@faker-js/faker";
+import { describe, expect, it } from "vitest";
+
+import {
+  aBeaconEvent,
+  beaconEventColumn,
+  beaconParameters,
+  beaconQueryString,
+  beaconSchemaVersion,
+  defaultBeaconPath,
+} from "./beacon-events.js";
+
+describe("the beacon event envelope", () => {
+  it("stamps every event with the version it was written under", () => {
+    // Given an event the beacon is about to send.
+    const sent = beaconQueryString({
+      event: "route",
+      page: "/guides/",
+    });
+
+    // Then the version rides with it. The raw store keeps whatever was
+    // written into it, so a row has to say which shape it is rather than
+    // leave a reader to infer one from the date.
+    expect(sent).toContain(
+      `${beaconParameters.version}=${String(beaconSchemaVersion)}`,
+    );
+  });
+
+  it("carries the page the event happened on, since the path cannot", () => {
+    // Given a route change on a page whose address the request never names.
+    // Every beacon request goes to the same path, so the path in the log says
+    // where the beacon is and never where the reader was.
+    const page = `/${faker.word.noun()}/`;
+
+    // When it is sent.
+    const sent = beaconQueryString({ event: "route", page });
+
+    // Then the page travels in the payload.
+    expect(sent).toContain(
+      `${beaconParameters.page}=${encodeURIComponent(page)}`,
+    );
+  });
+
+  it("encodes a value that would otherwise end the query string", () => {
+    // Given a page whose address holds the characters that separate one
+    // parameter from the next, which a router with a catch-all route can
+    // produce.
+    const sent = beaconQueryString({
+      event: "route",
+      page: "/search/?q=a&b=c",
+    });
+
+    // Then they arrive as text rather than as three more parameters. Read
+    // back, the page is the address the reader was on.
+    expect(sent).toContain("%3Fq%3Da%26b%3Dc");
+    expect(sent.split("&")).toHaveLength(3);
+  });
+
+  it("reads a row's event back off the column CloudFront wrote it to", () => {
+    // Given the SQL a rollup selects the event with.
+    // Then it reads the query string, which is where the payload is. No
+    // column of the table holds it, because a table column is a CloudFront
+    // field and CloudFront has no field for somebody else's payload.
+    expect(beaconEventColumn).toContain("cs_uri_query");
+    expect(beaconEventColumn).toContain(`'${beaconParameters.event}'`);
+  });
+
+  it("counts only requests carrying an envelope", () => {
+    // Given the conditions a rollup filters beacon rows with.
+    // Then a request to the beacon's path with no version parameter is left
+    // out. A crawler that found the URL in a page's source sends one of
+    // those, and counting it would report an event nobody caused.
+    expect(aBeaconEvent).toContain("cs_uri_query <> '-'");
+    expect(aBeaconEvent.join(" ")).toContain(`'${beaconParameters.version}'`);
+  });
+
+  it("sends to a path a site is unlikely to serve already", () => {
+    // Given the default path.
+    // Then it is one path, absolute, and marked as not a page. Pointing the
+    // beacon at a published page would count every event as a view of it and
+    // download that page a second time.
+    expect(defaultBeaconPath).toMatch(/^\/_/u);
+  });
+});

src/beacon-events.ts

@@ -0,0 +1,156 @@
+// What a beacon event is, and how it survives the round trip through a
+// CloudFront access log.
+//
+// The beacon sends a GET to a path on the site's own domain and puts its
+// payload in the query string. CloudFront records `cs-uri-query` whatever the
+// cache key and origin forwarding are set to, so the event lands in the same
+// objects, the same partitions and the same table as every page request.
+// Layer 2 is more rows in the dataset layer 1 already writes, and that is what
+// makes the beacon nearly free.
+//
+// The rows carry different information all the same, and this module is the
+// one place saying what. KensioSoftware/rainlytics#100 asked where that
+// definition lives and settled three things.
+//
+// **The fields stay in `cs_uri_query`.** A Glue column on the log table would
+// have to be a CloudFront field, because `LogTable` builds its columns from
+// what the delivery was configured with, and CloudFront has no field carrying
+// somebody else's payload. A view or a second table over the same objects
+// runs into the same wall, since no SerDe parses a query string. So the
+// payload is read at query time, the way `searches` already reads a search
+// term out of the same column.
+//
+// **Nothing has to be backfilled.** A beacon row is identified by the path it
+// was sent to. A query over partitions written before the beacon shipped
+// therefore matches no rows at all, rather than answering nulls for a column
+// added later. This is the one shape of schema change an immutable store
+// takes without argument.
+//
+// **The envelope is versioned and the payload is not, yet.** Every event
+// carries the three parameters below. What each event type puts beside them
+// is for the beacon's own issue, and `version` is what lets that arrive
+// without reinterpreting rows already written.
+//
+// Both halves of the package read this. The beacon builds a query string from
+// it in a browser, so nothing here may reach a Node built-in or `aws-cdk-lib`,
+// and a rollup reads the same parameters back as SQL.
+
+import { decodedParameter } from "./log-encoding.js";
+
+/**
+ * The path a beacon reports to, where a site chooses none.
+ *
+ * One path, under a leading underscore so it sorts away from a site's own
+ * pages and collides with nothing a router already serves. A site names its
+ * own where that is taken.
+ *
+ * The path is what tells a beacon row from a page request, so it has to be a
+ * path nothing else answers. Pointing the beacon at a page the site publishes
+ * would count every event as a view of it, and download the page body a
+ * second time.
+ */
+export const defaultBeaconPath = "/_rainlytics";
+
+/**
+ * The version of the envelope below.
+ *
+ * Written into every event and read back off every row. The raw store is
+ * immutable, so a row written today is still read under today's rules in a
+ * year. A query that has to tell two shapes apart has this to tell them
+ * apart by.
+ */
+export const beaconSchemaVersion = 1;
+
+/**
+ * The query-string parameters every event carries.
+ *
+ * One letter each. The whole query string is written into `cs-uri-query` on
+ * every event, percent-encoded, and stored for as long as the log objects
+ * last. Long names would be paid for on every row and scanned by every query
+ * reading the column.
+ */
+export const beaconParameters = {
+  /** The envelope version, being {@link beaconSchemaVersion}. */
+  version: "v",
+
+  /** What happened, such as a route change or a web vital. */
+  event: "e",
+
+  /** The page it happened on, which the beacon's own path cannot say. */
+  page: "p",
+} as const;
+
+/** One event, as the beacon reports it. */
+export interface BeaconEvent {
+  /**
+   * What happened.
+   *
+   * A short name a rollup groups by. The set of them belongs to the beacon
+   * rather than to this envelope.
+   */
+  readonly event: string;
+
+  /**
+   * The page it happened on, as a path.
+   *
+   * The request's own path is the beacon's path, so the page has to travel in
+   * the payload. A single-page app changing route is the case this exists
+   * for, where the address bar has moved and no request was made.
+   */
+  readonly page: string;
+}
+
+/**
+ * One event as a query string, ready to be sent.
+ *
+ * The browser's own encoding, which is the single pass a request carries.
+ * CloudFront adds its own on the way into the record, and
+ * {@link beaconEventColumn} reads both back off.
+ *
+ * No leading `?`. The caller joins it to the path it is sending to.
+ *
+ * ```typescript
+ * new Image().src = `${path}?${beaconQueryString({ event: "route", page })}`;
+ * ```
+ */
+export function beaconQueryString(event: BeaconEvent): string {
+  return [
+    [beaconParameters.version, String(beaconSchemaVersion)],
+    [beaconParameters.event, event.event],
+    [beaconParameters.page, event.page],
+  ]
+    .map(
+      ([name, value]) => `${String(name)}=${encodeURIComponent(String(value))}`,
+    )
+    .join("&");
+}
+
+/** The envelope version a row was written under, as SQL. */
+export const beaconVersionColumn = decodedParameter(beaconParameters.version);
+
+/** What happened, as SQL. */
+export const beaconEventColumn = decodedParameter(beaconParameters.event);
+
+/** The page it happened on, as SQL. */
+export const beaconPageColumn = decodedParameter(beaconParameters.page);
+
+/**
+ * The rows a beacon event is, as conditions for `rowsFor`.
+ *
+ * The path is not among them. A rollup narrows to the beacon's path through
+ * the request's own `paths`, the way any other question narrows to a section
+ * of a site, and a site that moved its beacon then says so in one place.
+ *
+ * These leave out anything else reaching the same path. A crawler following
+ * a beacon URL out of a page's source carries no version parameter, and the
+ * bot filter `rowsFor` applies has already taken most of them.
+ *
+ * ```typescript
+ * rowsFor({ ...request, paths: [defaultBeaconPath] }, aBeaconEvent);
+ * ```
+ */
+export const aBeaconEvent: readonly string[] = [
+  "cs_method = 'GET'",
+  "cs_uri_query <> '-'",
+  `${beaconVersionColumn} <> ''`,
+];

src/cdk/log-table.test.ts

@@ -10,6 +10,13 @@ import { describe, expect, it } from "vitest";
 import { deployStacks, simStartedAt } from "#test/simulated-deployment.js";
 
 import { defaultLogDataset, qualifiedTableName } from "../dataset.js";
+import {
+  beaconEventColumn,
+  beaconPageColumn,
+  beaconQueryString,
+  beaconVersionColumn,
+  defaultBeaconPath,
+} from "../beacon-events.js";
 import {
   deliveredLogColumnNames,
   logFieldNamesWithoutAddress,
@@ -404,6 +411,39 @@ describe("the Glue table over delivered logs", () => {
     ]);
   });
 
+  it("reads a beacon event back out of the query string", async () => {
+    // Given the query string a beacon sends, delivered the way CloudFront
+    // writes one. The event is a route change on a page the request never
+    // names, which is the case a payload in the query string exists for.
+    const deployed = await deployTable();
+    const [distributionId = ""] = deployed.distributionIds;
+    const sent = beaconQueryString({ event: "route", page: "/guides/好/" });
+    await putDelivered(deployed, distributionId, simStartedAt, [
+      {
+        "timestamp(ms)": String(simStartedAt.getTime()),
+        "cs-method": "GET",
+        "cs-uri-stem": defaultBeaconPath,
+        "cs-uri-query": asCloudFrontWrites(sent),
+        "cs(User-Agent)": "Mozilla/5.0",
+      },
+    ]);
+    await enableQueryEngine(deployed);
+
+    // When the envelope is selected the way a rollup selects it.
+    const answered = await queryRows(
+      deployed,
+      `SELECT ${beaconEventColumn}, ${beaconPageColumn},` +
+        ` ${beaconVersionColumn} FROM ${table()} WHERE year = '2026'` +
+        ` AND month = '08' AND day = '23' AND hour = '09'`,
+    );
+
+    // Then the event arrives as the beacon meant it, through the browser's
+    // encoding and CloudFront's on top of it. No column of this table holds
+    // any of the three, and nothing had to be added to it for the beacon.
+    expect(answered.rows).toStrictEqual([["route", "/guides/好/", "1"]]);
+    expect(answered.answeredBy).toBe("engine");
+  });
+
   it("describes no address where the delivery asked for none", async () => {
     // Given a site delivering the field set that holds no personal data.
     const table = catalogTable(
@@ -599,6 +639,24 @@ describe("the Glue table over delivered logs", () => {
       .enable();
   };
 
+  /**
+   * A query string as CloudFront writes it into a record.
+   *
+   * CloudFront percent-encodes each value it writes and the browser has
+   * already encoded it once, so `家` reaches the log as `%25E5%25AE%25B6`.
+   * The separators survive, which is what leaves `url_extract_parameter` a
+   * query string to split. `docs/searches/` has the same worked example.
+   */
+  const asCloudFrontWrites = (queryString: string): string =>
+    queryString
+      .split("&")
+      .map((pair) => {
+        const [name = "", value = ""] = pair.split("=");
+
+        return `${name}=${encodeURIComponent(value)}`;
+      })
+      .join("&");
+
   /**
    * The rows one query answered with, and what answered it.
    *

src/index.ts

@@ -72,6 +72,17 @@ export {
   windowedSql,
   windowRange,
 } from "./summary-runs.js";
+export {
+  aBeaconEvent,
+  type BeaconEvent,
+  beaconEventColumn,
+  beaconPageColumn,
+  beaconParameters,
+  beaconQueryString,
+  beaconSchemaVersion,
+  beaconVersionColumn,
+  defaultBeaconPath,
+} from "./beacon-events.js";
 export { decodedColumn, decodedParameter } from "./log-encoding.js";
 export {
   lastRange,