diff --git a/cache-blob.go b/cache-blob.go index 828960eb..b68ea715 100644 --- a/cache-blob.go +++ b/cache-blob.go @@ -38,11 +38,11 @@ import ( // blob be matched, or hashed, without decoding it back into an lruKey. type cacheBlob []byte -// blobVersion is the version of the layout above. It is reserved rather than -// used: every blob is freshly allocated and so carries 0, and nothing reads it -// back. It is here so that a later change to the layout has somewhere to say -// so, and can tell its own records from these. A writer that bumps it has to -// start setting it explicitly. +// blobVersion is the version of the layout above. A blob is freshly allocated +// and so carries 0 without being written, but it is read back: a blob read +// from the cache file is rejected unless it carries this version, so a later +// change to the layout keeps old records out of new accessors. A writer that +// bumps it has to start setting it explicitly. // // Note that 1 is already taken: it is binaryFormatVersion, the Redis record // format, which is a different layout in the same first byte. A stored form diff --git a/cache-file-raw.go b/cache-file-raw.go new file mode 100644 index 00000000..3893c6ae --- /dev/null +++ b/cache-file-raw.go @@ -0,0 +1,141 @@ +package rdns + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/miekg/dns" +) + +// The raw cache file stores entries in the form the cache already holds them, +// so writing is a copy and reading needs no parsing: +// +// header: magic, then a version byte +// record: uint32 length, then the blob, repeated to end of file +// +// The magic starts with a NUL so it can't be mistaken for the JSON format, +// whose records start with '{', and so anything that opens the file can see +// at once that it is binary. +// +// The version belongs to the file rather than to each record: it is checked +// once, and a file cannot hold a mix. The version byte inside a blob stays +// reserved for a record that travels on its own, without a file around it. +const ( + rawCacheMagic = "\x00RDC" + rawCacheVersion = 1 + rawCacheHeaderLen = len(rawCacheMagic) + 1 + + // An upper bound on a stored record, so a corrupt length can't ask for an + // unbounded allocation. This is a sanity limit rather than a property of + // the data: a stored entry is not bounded by the 64KB wire limit, because + // the cache packs without name compression and a response that arrived + // near the limit can be twice that once stored. The theoretical worst case + // is a few megabytes, so this sits well above anything real while still + // keeping a bad length to a bounded allocation. + maxRawCacheRecord = 16 << 20 +) + +// isRawCacheFile reports whether the reader is positioned at a raw cache file, +// leaving it unread either way so the right decoder can take it from the top. +func isRawCacheFile(r *bufio.Reader) bool { + magic, err := r.Peek(len(rawCacheMagic)) + return err == nil && string(magic) == rawCacheMagic +} + +func (c *lruCache) serializeRaw(w io.Writer) error { + header := make([]byte, 0, rawCacheHeaderLen) + header = append(header, rawCacheMagic...) + header = append(header, rawCacheVersion) + if _, err := w.Write(header); err != nil { + return err + } + + var length [4]byte + for item := c.tail.prev; item != c.head; item = item.prev { + if len(item.blob) > maxRawCacheRecord { + // Nothing on the store path bounds an entry this far, so this + // should not happen; say so rather than dropping it in silence. + Log.Warn("cache entry too large for the cache file, skipping", + "size", len(item.blob), "name", item.blob.key().Question.Name) + continue + } + binary.BigEndian.PutUint32(length[:], uint32(len(item.blob))) + if _, err := w.Write(length[:]); err != nil { + return err + } + if _, err := w.Write(item.blob); err != nil { + return err + } + } + return nil +} + +// deserializeRaw reads a raw cache file. A record that can't be used is +// skipped, as in the JSON format. Damage to the framing is different: there is +// no way to find the next record once a length is untrustworthy, so reading +// stops there and keeps the entries it already has. +func (c *lruCache) deserializeRaw(r io.Reader) error { + header := make([]byte, rawCacheHeaderLen) + if _, err := io.ReadFull(r, header); err != nil { + return fmt.Errorf("failed to read cache file header: %w", err) + } + if string(header[:len(rawCacheMagic)]) != rawCacheMagic { + return errors.New("not a raw cache file") + } + if version := header[len(rawCacheMagic)]; version != rawCacheVersion { + return fmt.Errorf("unsupported raw cache file version %d", version) + } + + var length [4]byte + for { + if _, err := io.ReadFull(r, length[:]); err != nil { + if errors.Is(err, io.EOF) { + return nil // clean end of file + } + Log.Warn("cache file ends mid-record, keeping the entries read so far", "error", err) + return nil + } + + n := binary.BigEndian.Uint32(length[:]) + if n < blobHdrLen || n > maxRawCacheRecord { + Log.Warn("cache file record has an implausible length, stopping", "length", n) + return nil + } + + blob := make(cacheBlob, n) + if _, err := io.ReadFull(r, blob); err != nil { + Log.Warn("cache file ends mid-record, keeping the entries read so far", "error", err) + return nil + } + + key, ok := blobFromFile(blob) + if !ok { + continue // skip the record, the framing is still good + } + c.addKey(key, blob) + } +} + +// blobFromFile checks a blob read from disk far enough to be sure the +// accessors on it are safe, and returns the key it is stored under. +func blobFromFile(blob cacheBlob) (lruKey, bool) { + if len(blob) < blobHdrLen || blob.version() != blobVersion { + return lruKey{}, false + } + if blobHdrLen+blob.netLen()+blob.nameLen() > len(blob) { + return lruKey{}, false + } + // Unpack once here so a corrupt message is kept out of the cache rather + // than taking up an entry until the lookup that finds it evicts it. + if err := new(dns.Msg).Unpack(blob.message()); err != nil { + return lruKey{}, false + } + key := blob.key() + if key.Question.Name == "" { + return lruKey{}, false + } + return key, true +} diff --git a/cache-file-raw_test.go b/cache-file-raw_test.go new file mode 100644 index 00000000..8357e95e --- /dev/null +++ b/cache-file-raw_test.go @@ -0,0 +1,212 @@ +package rdns + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" +) + +func rawTestEntry(t *testing.T, i int) (*dns.Msg, *cacheAnswer) { + t.Helper() + name := fmt.Sprintf("host%d.example.com.", i) + q := new(dns.Msg) + q.SetQuestion(name, dns.TypeA) + a := new(dns.Msg) + a.SetReply(q) + rr, err := dns.NewRR(fmt.Sprintf("%s 3600 IN A 192.0.%d.%d", name, i/256, i%256)) + require.NoError(t, err) + a.Answer = []dns.RR{rr} + + now := time.Now() + return q, &cacheAnswer{Msg: a, Timestamp: now, Expiry: now.Add(time.Hour)} +} + +func fillCache(t *testing.T, c *lruCache, n int) []*dns.Msg { + t.Helper() + queries := make([]*dns.Msg, n) + for i := range n { + q, answer := rawTestEntry(t, i) + queries[i] = q + key := lruKeyFromQuery(q) + blob, err := newCacheBlob(key, answer) + require.NoError(t, err) + c.addKey(key, blob) + } + return queries +} + +func TestRawCacheFileRoundTrip(t *testing.T) { + src := newLRUCache(0) + queries := fillCache(t, src, 20) + + var buf bytes.Buffer + require.NoError(t, src.serializeRaw(&buf)) + + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(buf.Bytes()))) + require.Equal(t, len(queries), dst.size()) + for _, q := range queries { + key := lruKeyFromQuery(q) + require.NotNil(t, dst.find(dst.hash(key), key), "%s must be findable again", key.Question.Name) + } + + // Writing it back out reproduces the file, so the queue order survives too + // and the least recently used entry is still the first to go. + var again bytes.Buffer + require.NoError(t, dst.serializeRaw(&again)) + require.Equal(t, buf.Bytes(), again.Bytes()) +} + +// The option picks the format to write; the format to read comes from the file +// itself, so an existing cache survives the option being changed either way. +func TestCacheFileFormatIsDetectedNotConfigured(t *testing.T) { + for _, tc := range []struct { + wrote, prefix, thenRunsAs string + }{ + {"", "{", CacheFileFormatRaw}, // unset writes JSON + {CacheFileFormatRaw, rawCacheMagic, CacheFileFormatJSON}, // raw read back by a JSON-configured run + } { + t.Run(tc.wrote+"-then-"+tc.thenRunsAs, func(t *testing.T) { + filename := filepath.Join(t.TempDir(), "cache") + + first := NewMemoryBackend(MemoryBackendOptions{ + GCPeriod: time.Hour, Filename: filename, FileFormat: tc.wrote, + }) + queries := make([]*dns.Msg, 5) + for i := range queries { + q, answer := rawTestEntry(t, i) + queries[i] = q + first.Store(q, answer) + } + require.NoError(t, first.Close()) + + content, err := os.ReadFile(filename) + require.NoError(t, err) + require.True(t, bytes.HasPrefix(content, []byte(tc.prefix)), "%q wrote the wrong format", tc.wrote) + + second := NewMemoryBackend(MemoryBackendOptions{ + GCPeriod: time.Hour, Filename: filename, FileFormat: tc.thenRunsAs, + }) + defer second.Close() + require.Equal(t, len(queries), second.Size()) + for _, q := range queries { + _, _, ok := second.Lookup(q) + require.True(t, ok, "%s must survive the format switch", q.Question[0].Name) + } + }) + } +} + +// Downgrading the binary has to cost a cold cache and nothing worse, which is +// what the magic buys: a build that predates the format can't parse it. +func TestRawCacheFileRejectedByJSONReader(t *testing.T) { + src := newLRUCache(0) + fillCache(t, src, 3) + var buf bytes.Buffer + require.NoError(t, src.serializeRaw(&buf)) + + dst := newLRUCache(0) + require.Error(t, dst.deserialize(bytes.NewReader(buf.Bytes()))) + require.Zero(t, dst.size()) +} + +// Damage to a record costs that record; damage to the framing costs the rest +// of the file, because there is no way to find the next record from it. Either +// way what was read before it is kept. +func TestRawCacheFileCorruption(t *testing.T) { + src := newLRUCache(0) + fillCache(t, src, 10) + var buf bytes.Buffer + require.NoError(t, src.serializeRaw(&buf)) + good := buf.Bytes() + firstLen := int(binary.BigEndian.Uint32(good[rawCacheHeaderLen:])) + + t.Run("corrupt message skips one record", func(t *testing.T) { + // Scribble on the packed message, leaving the framing intact. + damaged := bytes.Clone(good) + msg := damaged[rawCacheHeaderLen+4 : rawCacheHeaderLen+4+firstLen] + for i := len(msg) - 8; i < len(msg); i++ { + msg[i] = 0xff + } + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(damaged))) + require.Equal(t, 9, dst.size(), "the other records still load") + }) + + t.Run("truncated file keeps what came before", func(t *testing.T) { + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(good[:len(good)-20]))) + require.Equal(t, 9, dst.size()) + }) + + t.Run("implausible length is refused not allocated", func(t *testing.T) { + damaged := bytes.Clone(good) + binary.BigEndian.PutUint32(damaged[rawCacheHeaderLen+4+firstLen:], 3_000_000_000) + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(damaged))) + require.Equal(t, 1, dst.size(), "only the record before the bad length is kept") + }) +} + +// A stored entry is not bounded by the 64KB wire limit: the cache packs +// without name compression, so a response that arrived comfortably inside the +// limit can be twice that once stored. The record limit has to leave room for +// it rather than dropping the largest entries on the way to disk. +func TestRawCacheFileKeepsOversizedEntries(t *testing.T) { + q := new(dns.Msg) + q.SetQuestion("big.example.com.", dns.TypeTXT) + a := new(dns.Msg) + a.SetReply(q) + name := strings.Repeat("averylonglabelusedtomakethisnamebig.", 6) + "example.com." + for range 290 { + rr, err := dns.NewRR(fmt.Sprintf(`%s 3600 IN TXT "%s"`, name, strings.Repeat("x", 200))) + require.NoError(t, err) + a.Answer = append(a.Answer, rr) + } + + a.Compress = true + onTheWire, err := a.Pack() + require.NoError(t, err) + require.Less(t, len(onTheWire), dns.MaxMsgSize, "this has to be a response that could arrive") + a.Compress = false + + now := time.Now() + key := lruKeyFromQuery(q) + blob, err := newCacheBlob(key, &cacheAnswer{Msg: a, Timestamp: now, Expiry: now.Add(time.Hour)}) + require.NoError(t, err) + require.Greater(t, len(blob), dns.MaxMsgSize, "stored uncompressed, it outgrows the wire limit") + + src := newLRUCache(0) + src.addKey(key, blob) + var buf bytes.Buffer + require.NoError(t, src.serializeRaw(&buf)) + + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(buf.Bytes()))) + require.Equal(t, 1, dst.size(), "the entry must survive the file, not be skipped") +} + +// Persisting blobs makes their version byte part of an on-disk format, so a +// record laid out by a later version has to be refused rather than read +// through accessors that no longer match it. +func TestRawCacheFileRejectsUnknownBlobVersion(t *testing.T) { + src := newLRUCache(0) + fillCache(t, src, 3) + var buf bytes.Buffer + require.NoError(t, src.serializeRaw(&buf)) + + damaged := bytes.Clone(buf.Bytes()) + damaged[rawCacheHeaderLen+4+blobOffVersion] = blobVersion + 1 + + dst := newLRUCache(0) + require.NoError(t, dst.deserializeRaw(bytes.NewReader(damaged))) + require.Equal(t, 2, dst.size(), "the record with an unknown layout is skipped, the rest load") +} diff --git a/cache-memory.go b/cache-memory.go index 4fa37117..dda0516c 100644 --- a/cache-memory.go +++ b/cache-memory.go @@ -34,8 +34,19 @@ type MemoryBackendOptions struct { // Write the file in an interval. Only write on shutdown if not set SaveInterval time.Duration + + // Format to write the cache file in, "json" (default) or "raw". Reading + // detects the format from the file, so this can be changed either way + // without losing what is already on disk. + FileFormat string } +// Values for MemoryBackendOptions.FileFormat. +const ( + CacheFileFormatJSON = "json" + CacheFileFormatRaw = "raw" +) + var _ CacheBackend = (*memoryBackend)(nil) func NewMemoryBackend(opt MemoryBackendOptions) *memoryBackend { @@ -182,6 +193,9 @@ func (b *memoryBackend) writeToFile(filename string) error { // buffered writer may still flush to disk here, under the lock. b.mu.Lock() defer b.mu.Unlock() + if b.opt.FileFormat == CacheFileFormatRaw { + return b.lru.serializeRaw(w) + } return b.lru.serialize(w) }) if err != nil { @@ -203,7 +217,15 @@ func (b *memoryBackend) loadFromFile(filename string) error { } defer f.Close() - if err := b.lru.deserialize(bufio.NewReaderSize(f, fileBufSize)); err != nil { + // The format is taken from the file rather than from the configuration, + // so switching FileFormat picks up an existing cache instead of dropping + // it on the floor. + r := bufio.NewReaderSize(f, fileBufSize) + read := b.lru.deserialize + if isRawCacheFile(r) { + read = b.lru.deserializeRaw + } + if err := read(r); err != nil { log.Warn("failed to read cache from disk", "error", err) return err } diff --git a/cmd/routedns/config.go b/cmd/routedns/config.go index f702b709..2e5bc2fd 100644 --- a/cmd/routedns/config.go +++ b/cmd/routedns/config.go @@ -94,6 +94,7 @@ type cacheBackend struct { GCPeriod int `toml:"gc-period"` // Time-period (seconds) used to expire cached items Filename string // File to load/store cache content, optional, for "memory" type cache SaveInterval int `toml:"save-interval"` // Seconds to write the cache to file + FileFormat string `toml:"file-format"` // Format of the cache file, "json" (default) or "raw" RedisNetwork string `toml:"redis-network"` // The network type, either tcp or unix. Defaults to tcp. RedisAddress string `toml:"redis-address"` // Address for redis cache RedisUsername string `toml:"redis-username"` // Redis username diff --git a/cmd/routedns/main.go b/cmd/routedns/main.go index 6e15ef9e..e3c6fec9 100644 --- a/cmd/routedns/main.go +++ b/cmd/routedns/main.go @@ -844,11 +844,17 @@ func instantiateGroup(id string, g group, resolvers map[string]rdns.Resolver) er var backend rdns.CacheBackend switch g.Backend.Type { case "memory": + switch g.Backend.FileFormat { + case "", rdns.CacheFileFormatJSON, rdns.CacheFileFormatRaw: + default: + return fmt.Errorf("unsupported cache file-format '%s' in group '%s'", g.Backend.FileFormat, id) + } backend = rdns.NewMemoryBackend(rdns.MemoryBackendOptions{ Capacity: g.Backend.Size, GCPeriod: time.Duration(g.Backend.GCPeriod) * time.Second, Filename: g.Backend.Filename, SaveInterval: time.Duration(g.Backend.SaveInterval) * time.Second, + FileFormat: g.Backend.FileFormat, }) onClose = append(onClose, func() { backend.Close() }) case "redis": diff --git a/doc/caching.md b/doc/caching.md index b6797f90..7b8baffd 100644 --- a/doc/caching.md +++ b/doc/caching.md @@ -41,6 +41,7 @@ The memory backend will keep all cache items in memory. It can be configured to - `gc-period` - How often (in seconds) expired items are swept out of the cache. Defaults to 60. Optional. - `filename` - File to use for persistent storage to disk. The cache will be initialized with the content from the file and it'll write the content to the same file on shutdown. Defaults to no persistence. The file is written by creating a temporary file in the same directory and renaming it into place, so the directory has to be writable, and a symlink at this path is replaced by a regular file rather than being written through. Point the option at the real location if the data needs to live elsewhere, for example on a tmpfs. A new file is created with mode `0600`, since it records what has been looked up; an existing file keeps whatever mode it already has. When running under the systemd unit shipped with the packages, use a path under `/var/cache/routedns`; see [Writable Paths](overview.md#writable-paths). - `save-interval` - Interval (in seconds) to save the cache to file. Optional. If not set, the file is written only on shutdown. +- `file-format` - Format of the cache file, `json` (default) or `raw`. The raw format stores each entry the way the cache holds it in memory, which makes the file smaller and much faster to write and read; a 20,000-entry cache writes in a twentieth of the time and loads in a sixth. Reading detects the format from the file itself rather than from this option, so switching either way keeps an existing cache. Note that a cache file written in the raw format cannot be read by a version of RouteDNS that predates it; it is ignored and the cache starts empty. **Redis backend**