diff --git a/docs/log-table/README.md b/docs/log-table/README.md index adee1bf..2930efe 100644 --- a/docs/log-table/README.md +++ b/docs/log-table/README.md @@ -160,6 +160,35 @@ and this was not one of them, because the field joined the set afterwards. A Par spells it some other way answers every query with a column of nulls and reports success. Read an object back before trusting a visitor count over Parquet. +## A beacon row is a row like any other + +The [optional beacon](https://github.com/KensioSoftware/rainlytics/issues/100) sends a GET to a path +on the site's own domain and puts its payload in the query string. CloudFront records `cs-uri-query` +whatever the cache key and origin forwarding are set to. The event lands in the same objects, the +same partitions and this same table. Layer 2 is more rows in the dataset layer 1 already writes. + +The payload stays in `cs_uri_query` and is read at query time. + +```sql +SELECT url_decode(url_extract_parameter(cs_uri_stem || '?' || cs_uri_query, 'e')) AS event, + count(*) AS events + FROM rainlytics.cloudfront_logs + WHERE cs_uri_stem = '/_rainlytics' +``` + +A column of its own was never available. This table's columns are the fields the delivery was +configured with, and CloudFront has no field carrying somebody else's payload. A view or a second +table over the same objects meets the same wall, since no SerDe parses a query string. The +[searches](../searches/) rollup already reads a search term out of the same column the same way. + +`beaconQueryString`, `beaconEventColumn` and `aBeaconEvent` are exported from the package root, so +the browser writing the payload and the SQL reading it hold one definition between them. + +Partitions written before the beacon shipped answer no rows at all. A beacon row is identified by +the path it was sent to. A query for that path over an older day matches nothing, where a column +added later would have answered nulls. That is the one shape of schema change an immutable store +takes without argument. + ## Several distributions, one table `distributionid` partitions before time does, so one table covers every site delivering into the diff --git a/src/beacon-events.test.ts b/src/beacon-events.test.ts new file mode 100644 index 0000000..6c31f93 --- /dev/null +++ b/src/beacon-events.test.ts @@ -0,0 +1,84 @@ +import { faker } from "@faker-js/faker"; +import { describe, expect, it } from "vitest"; + +import { + aBeaconEvent, + beaconEventColumn, + beaconParameters, + beaconQueryString, + beaconSchemaVersion, + defaultBeaconPath, +} from "./beacon-events.js"; + +describe("the beacon event envelope", () => { + it("stamps every event with the version it was written under", () => { + // Given an event the beacon is about to send. + const sent = beaconQueryString({ + event: "route", + page: "/guides/", + }); + + // Then the version rides with it. The raw store keeps whatever was + // written into it, so a row has to say which shape it is rather than + // leave a reader to infer one from the date. + expect(sent).toContain( + `${beaconParameters.version}=${String(beaconSchemaVersion)}`, + ); + }); + + it("carries the page the event happened on, since the path cannot", () => { + // Given a route change on a page whose address the request never names. + // Every beacon request goes to the same path, so the path in the log says + // where the beacon is and never where the reader was. + const page = `/${faker.word.noun()}/`; + + // When it is sent. + const sent = beaconQueryString({ event: "route", page }); + + // Then the page travels in the payload. + expect(sent).toContain( + `${beaconParameters.page}=${encodeURIComponent(page)}`, + ); + }); + + it("encodes a value that would otherwise end the query string", () => { + // Given a page whose address holds the characters that separate one + // parameter from the next, which a router with a catch-all route can + // produce. + const sent = beaconQueryString({ + event: "route", + page: "/search/?q=a&b=c", + }); + + // Then they arrive as text rather than as three more parameters. Read + // back, the page is the address the reader was on. + expect(sent).toContain("%3Fq%3Da%26b%3Dc"); + expect(sent.split("&")).toHaveLength(3); + }); + + it("reads a row's event back off the column CloudFront wrote it to", () => { + // Given the SQL a rollup selects the event with. + // Then it reads the query string, which is where the payload is. No + // column of the table holds it, because a table column is a CloudFront + // field and CloudFront has no field for somebody else's payload. + expect(beaconEventColumn).toContain("cs_uri_query"); + expect(beaconEventColumn).toContain(`'${beaconParameters.event}'`); + }); + + it("counts only requests carrying an envelope", () => { + // Given the conditions a rollup filters beacon rows with. + // Then a request to the beacon's path with no version parameter is left + // out. A crawler that found the URL in a page's source sends one of + // those, and counting it would report an event nobody caused. + expect(aBeaconEvent).toContain("cs_uri_query <> '-'"); + expect(aBeaconEvent.join(" ")).toContain(`'${beaconParameters.version}'`); + }); + + it("sends to a path a site is unlikely to serve already", () => { + // Given the default path. + // Then it is one path, absolute, and marked as not a page. Pointing the + // beacon at a published page would count every event as a view of it and + // download that page a second time. + expect(defaultBeaconPath).toMatch(/^\/_/u); + }); +}); diff --git a/src/beacon-events.ts b/src/beacon-events.ts new file mode 100644 index 0000000..4e74ed8 --- /dev/null +++ b/src/beacon-events.ts @@ -0,0 +1,156 @@ +// What a beacon event is, and how it survives the round trip through a +// CloudFront access log. +// +// The beacon sends a GET to a path on the site's own domain and puts its +// payload in the query string. CloudFront records `cs-uri-query` whatever the +// cache key and origin forwarding are set to, so the event lands in the same +// objects, the same partitions and the same table as every page request. +// Layer 2 is more rows in the dataset layer 1 already writes, and that is what +// makes the beacon nearly free. +// +// The rows carry different information all the same, and this module is the +// one place saying what. KensioSoftware/rainlytics#100 asked where that +// definition lives and settled three things. +// +// **The fields stay in `cs_uri_query`.** A Glue column on the log table would +// have to be a CloudFront field, because `LogTable` builds its columns from +// what the delivery was configured with, and CloudFront has no field carrying +// somebody else's payload. A view or a second table over the same objects +// runs into the same wall, since no SerDe parses a query string. So the +// payload is read at query time, the way `searches` already reads a search +// term out of the same column. +// +// **Nothing has to be backfilled.** A beacon row is identified by the path it +// was sent to. A query over partitions written before the beacon shipped +// therefore matches no rows at all, rather than answering nulls for a column +// added later. This is the one shape of schema change an immutable store +// takes without argument. +// +// **The envelope is versioned and the payload is not, yet.** Every event +// carries the three parameters below. What each event type puts beside them +// is for the beacon's own issue, and `version` is what lets that arrive +// without reinterpreting rows already written. +// +// Both halves of the package read this. The beacon builds a query string from +// it in a browser, so nothing here may reach a Node built-in or `aws-cdk-lib`, +// and a rollup reads the same parameters back as SQL. + +import { decodedParameter } from "./log-encoding.js"; + +/** + * The path a beacon reports to, where a site chooses none. + * + * One path, under a leading underscore so it sorts away from a site's own + * pages and collides with nothing a router already serves. A site names its + * own where that is taken. + * + * The path is what tells a beacon row from a page request, so it has to be a + * path nothing else answers. Pointing the beacon at a page the site publishes + * would count every event as a view of it, and download the page body a + * second time. + */ +export const defaultBeaconPath = "/_rainlytics"; + +/** + * The version of the envelope below. + * + * Written into every event and read back off every row. The raw store is + * immutable, so a row written today is still read under today's rules in a + * year. A query that has to tell two shapes apart has this to tell them + * apart by. + */ +export const beaconSchemaVersion = 1; + +/** + * The query-string parameters every event carries. + * + * One letter each. The whole query string is written into `cs-uri-query` on + * every event, percent-encoded, and stored for as long as the log objects + * last. Long names would be paid for on every row and scanned by every query + * reading the column. + */ +export const beaconParameters = { + /** The envelope version, being {@link beaconSchemaVersion}. */ + version: "v", + + /** What happened, such as a route change or a web vital. */ + event: "e", + + /** The page it happened on, which the beacon's own path cannot say. */ + page: "p", +} as const; + +/** One event, as the beacon reports it. */ +export interface BeaconEvent { + /** + * What happened. + * + * A short name a rollup groups by. The set of them belongs to the beacon + * rather than to this envelope. + */ + readonly event: string; + + /** + * The page it happened on, as a path. + * + * The request's own path is the beacon's path, so the page has to travel in + * the payload. A single-page app changing route is the case this exists + * for, where the address bar has moved and no request was made. + */ + readonly page: string; +} + +/** + * One event as a query string, ready to be sent. + * + * The browser's own encoding, which is the single pass a request carries. + * CloudFront adds its own on the way into the record, and + * {@link beaconEventColumn} reads both back off. + * + * No leading `?`. The caller joins it to the path it is sending to. + * + * ```typescript + * new Image().src = `${path}?${beaconQueryString({ event: "route", page })}`; + * ``` + */ +export function beaconQueryString(event: BeaconEvent): string { + return [ + [beaconParameters.version, String(beaconSchemaVersion)], + [beaconParameters.event, event.event], + [beaconParameters.page, event.page], + ] + .map( + ([name, value]) => `${String(name)}=${encodeURIComponent(String(value))}`, + ) + .join("&"); +} + +/** The envelope version a row was written under, as SQL. */ +export const beaconVersionColumn = decodedParameter(beaconParameters.version); + +/** What happened, as SQL. */ +export const beaconEventColumn = decodedParameter(beaconParameters.event); + +/** The page it happened on, as SQL. */ +export const beaconPageColumn = decodedParameter(beaconParameters.page); + +/** + * The rows a beacon event is, as conditions for `rowsFor`. + * + * The path is not among them. A rollup narrows to the beacon's path through + * the request's own `paths`, the way any other question narrows to a section + * of a site, and a site that moved its beacon then says so in one place. + * + * These leave out anything else reaching the same path. A crawler following + * a beacon URL out of a page's source carries no version parameter, and the + * bot filter `rowsFor` applies has already taken most of them. + * + * ```typescript + * rowsFor({ ...request, paths: [defaultBeaconPath] }, aBeaconEvent); + * ``` + */ +export const aBeaconEvent: readonly string[] = [ + "cs_method = 'GET'", + "cs_uri_query <> '-'", + `${beaconVersionColumn} <> ''`, +]; diff --git a/src/cdk/log-table.test.ts b/src/cdk/log-table.test.ts index 9557a15..7024d6e 100644 --- a/src/cdk/log-table.test.ts +++ b/src/cdk/log-table.test.ts @@ -10,6 +10,13 @@ import { describe, expect, it } from "vitest"; import { deployStacks, simStartedAt } from "#test/simulated-deployment.js"; import { defaultLogDataset, qualifiedTableName } from "../dataset.js"; +import { + beaconEventColumn, + beaconPageColumn, + beaconQueryString, + beaconVersionColumn, + defaultBeaconPath, +} from "../beacon-events.js"; import { deliveredLogColumnNames, logFieldNamesWithoutAddress, @@ -404,6 +411,39 @@ describe("the Glue table over delivered logs", () => { ]); }); + it("reads a beacon event back out of the query string", async () => { + // Given the query string a beacon sends, delivered the way CloudFront + // writes one. The event is a route change on a page the request never + // names, which is the case a payload in the query string exists for. + const deployed = await deployTable(); + const [distributionId = ""] = deployed.distributionIds; + const sent = beaconQueryString({ event: "route", page: "/guides/好/" }); + await putDelivered(deployed, distributionId, simStartedAt, [ + { + "timestamp(ms)": String(simStartedAt.getTime()), + "cs-method": "GET", + "cs-uri-stem": defaultBeaconPath, + "cs-uri-query": asCloudFrontWrites(sent), + "cs(User-Agent)": "Mozilla/5.0", + }, + ]); + await enableQueryEngine(deployed); + + // When the envelope is selected the way a rollup selects it. + const answered = await queryRows( + deployed, + `SELECT ${beaconEventColumn}, ${beaconPageColumn},` + + ` ${beaconVersionColumn} FROM ${table()} WHERE year = '2026'` + + ` AND month = '08' AND day = '23' AND hour = '09'`, + ); + + // Then the event arrives as the beacon meant it, through the browser's + // encoding and CloudFront's on top of it. No column of this table holds + // any of the three, and nothing had to be added to it for the beacon. + expect(answered.rows).toStrictEqual([["route", "/guides/好/", "1"]]); + expect(answered.answeredBy).toBe("engine"); + }); + it("describes no address where the delivery asked for none", async () => { // Given a site delivering the field set that holds no personal data. const table = catalogTable( @@ -599,6 +639,24 @@ describe("the Glue table over delivered logs", () => { .enable(); }; + /** + * A query string as CloudFront writes it into a record. + * + * CloudFront percent-encodes each value it writes and the browser has + * already encoded it once, so `家` reaches the log as `%25E5%25AE%25B6`. + * The separators survive, which is what leaves `url_extract_parameter` a + * query string to split. `docs/searches/` has the same worked example. + */ + const asCloudFrontWrites = (queryString: string): string => + queryString + .split("&") + .map((pair) => { + const [name = "", value = ""] = pair.split("="); + + return `${name}=${encodeURIComponent(value)}`; + }) + .join("&"); + /** * The rows one query answered with, and what answered it. * diff --git a/src/index.ts b/src/index.ts index c1ffe29..c4f657f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -72,6 +72,17 @@ export { windowedSql, windowRange, } from "./summary-runs.js"; +export { + aBeaconEvent, + type BeaconEvent, + beaconEventColumn, + beaconPageColumn, + beaconParameters, + beaconQueryString, + beaconSchemaVersion, + beaconVersionColumn, + defaultBeaconPath, +} from "./beacon-events.js"; export { decodedColumn, decodedParameter } from "./log-encoding.js"; export { lastRange,