Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 56 additions & 11 deletions internal/machine/dns/resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ import (
"sync"
"time"

"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/network"
"github.com/psviderski/uncloud/internal/machine/store"
)

Expand All @@ -19,7 +21,9 @@ type ClusterResolver struct {
store *store.Store
// serviceIPs maps service names to container IPs.
serviceIPs map[string][]netip.Addr
// mu protects the serviceIPs map.
// machineIPs maps machine IDs and names to their IP.
machineIPs map[string]netip.Addr
// mu protects the serviceIPs and machineIPs map.
mu sync.RWMutex
// lastUpdate tracks when records were last updated.
lastUpdate time.Time
Expand All @@ -46,6 +50,13 @@ func (r *ClusterResolver) Run(ctx context.Context) error {
// TODO: implement machine membership check using Corrossion Admin client to filter available containers.
r.updateServiceIPs(containers)

machines, mchanges, err := r.store.SubscribeMachines(ctx)
if err != nil {
return fmt.Errorf("subscribe to machine changes: %w", err)
}
r.log.Info("Subscribed to machine changes in the clsuter to keep machine DNS records updated.")
r.updateMachineIPs(machines)

for {
select {
case _, ok := <-changes:
Expand All @@ -59,9 +70,22 @@ func (r *ClusterResolver) Run(ctx context.Context) error {
r.log.Error("Failed to list containers.", "err", err)
continue
}

// TODO: implement machine membership check using Corrossion Admin client to filter available containers.
r.updateServiceIPs(containers)

case _, ok := <-mchanges:
if !ok {
return fmt.Errorf("machine subscription failed")
}
r.log.Debug("Cluster machines changed, updating DNS records.")

machines, err := r.store.ListMachines(ctx)
if err != nil {
r.log.Error("Failed to list machines.", "err", err)
continue
}
r.updateMachineIPs(machines)

case <-ctx.Done():
return nil
}
Expand Down Expand Up @@ -121,19 +145,40 @@ func (r *ClusterResolver) updateServiceIPs(containers []store.ContainerRecord) {
r.log.Info("DNS records updated.", "services", len(newServiceIPs)/3, "containers", containersCount)
}

// Resolve returns IP addresses of the service containers.
func (r *ClusterResolver) Resolve(serviceName string) []netip.Addr {
func (r *ClusterResolver) updateMachineIPs(machines []*pb.MachineInfo) {
newMachineIPs := make(map[string]netip.Addr, len(machines))

for _, machine := range machines {
subnet, err := machine.Network.Subnet.ToPrefix()
if err != nil {
continue
}
addr := network.MachineIP(subnet)
newMachineIPs[machine.Name+".m"] = addr
newMachineIPs[machine.Id+".m"] = addr
}
r.mu.Lock()
r.machineIPs = newMachineIPs
r.mu.Unlock()

r.log.Info("DNS records updated.", "machines", len(machines))
}

// Resolve returns IP addresses of the service containers or machines.
func (r *ClusterResolver) Resolve(name string) []netip.Addr {
r.mu.RLock()
defer r.mu.RUnlock()

ips, ok := r.serviceIPs[serviceName]
if !ok || len(ips) == 0 {
return nil
// Return a copy of the IPs slice to prevent modification of the original.
ips, ok := r.serviceIPs[name]
if ok && len(ips) > 0 {
return slices.Clone(ips)
}

// Return a copy of the IPs slice to prevent modification of the original.
ipsCopy := make([]netip.Addr, len(ips))
copy(ipsCopy, ips)
ip, ok := r.machineIPs[name]
if ok {
return slices.Clone([]netip.Addr{ip})
}

return ipsCopy
return nil
}
29 changes: 29 additions & 0 deletions internal/machine/dns/resolver_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package dns

import (
"net/netip"
"reflect"
"testing"

"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/psviderski/uncloud/internal/machine/api/pb"
"github.com/psviderski/uncloud/internal/machine/store"
"github.com/psviderski/uncloud/pkg/api"
"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -75,3 +77,30 @@ func newRecord(serviceID, serviceName, ip, machineID string) store.ContainerReco
MachineID: machineID,
}
}

func TestClusterResolver_UpdateMachineIPs(t *testing.T) {
t.Parallel()

machines := []*pb.MachineInfo{
newMachineRecord("x0y0z0", "mach-1", "10.210.0.0/24"),
newMachineRecord("x1y1z1", "mach-2", "10.210.1.0/24"),
newMachineRecord("x2y2z2", "mach-3", "10.210.2.0/24"),
}

r := NewClusterResolver(nil)
r.updateMachineIPs(machines)

assert.NotEmpty(t, r.Resolve("mach-1.m"))
assert.NotEmpty(t, r.Resolve("mach-3.m"))
assert.NotEmpty(t, r.Resolve("x0y0z0.m"))
}

func newMachineRecord(machineID, machineName, prefix string) *pb.MachineInfo {
return &pb.MachineInfo{
Id: machineID,
Name: machineName,
Network: &pb.NetworkConfig{
Subnet: pb.NewIPPrefix(netip.MustParsePrefix(prefix)),
},
}
}
8 changes: 4 additions & 4 deletions internal/machine/dns/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,13 +290,13 @@ func (s *Server) forwardRequest(req *dns.Msg, proto string) (*dns.Msg, error) {
// handleAQuery processes an A query for the internal domain and returns A records for the requested name.
// The internal domain suffix is already stripped from the name. An empty list is returned if no records are found.
func (s *Server) handleAQuery(name string) []dns.RR {
serviceName, mode := extractModeFromDomain(trimInternalDomain(name))
ips := s.resolver.Resolve(serviceName)
unname, mode := extractModeFromDomain(trimInternalDomain(name))
ips := s.resolver.Resolve(unname)
if len(ips) == 0 {
s.log.Debug("Failed to resolve service name.", "service", serviceName)
s.log.Debug("Failed to resolve internal name.", "name", name)
return nil
}
s.log.Debug("Resolved service name.", "service", serviceName, "ips", ips)
s.log.Debug("Resolved service internal name.", "name", name, "ips", ips)

if len(ips) > 1 {
// Shuffle the IPs to approximate round-robin.
Expand Down
34 changes: 33 additions & 1 deletion website/docs/3-concepts/6-services/1-internal-dns.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
Services can be addressed on the internal WireGuard network by service name, service ID, or a machine-scoped service name:

## Service name

```
$ nslookup nats.internal
Server: 127.0.0.11
Expand Down Expand Up @@ -30,6 +31,7 @@ Address: 10.210.1.4
```

## Service ID

```
$ nslookup 3ecb3a8bbec5fd3f46efb056a934714a.internal
Server: 127.0.0.11
Expand All @@ -40,6 +42,7 @@ Address: 10.210.0.4
```

## Machine ID scoped service name

```
$ nslookup 0903f0ee483aa97d559eeeaac5e22283.m.nats.internal
Server: 127.0.0.11
Expand All @@ -64,7 +67,8 @@ Address: 10.210.1.4

Additionally, the IP ordering preference can be specified with a `rr` (round-robin) or `nearest` subdomain prefix.

### `rr` (round-robin) *current default*
### `rr` (round-robin) _current default_

Randomly shuffled order on each lookup.

```
Expand Down Expand Up @@ -96,9 +100,11 @@ Address: 10.210.0.3
```

## Nearest scope

Returns machine-local instances first.

`machine-a`:

```
$ nslookup nearest.worker.internal
Server: 127.0.0.11
Expand All @@ -115,6 +121,7 @@ Address: 10.210.1.4
```

`machine-b`:

```
$ nslookup nearest.worker.internal
Server: 127.0.0.11
Expand All @@ -131,3 +138,28 @@ Address: 10.210.0.4
```

The prefixes can be used with service ID and machine-scoped service names, as well (e.g. `nearest.3ecb3a8bbec5fd3f46efb056a934714a.internal` or `rr.0903f0ee483aa97d559eeeaac5e22283.m.worker.internal`).

## Machine name

Machines can be addressed too on the internal WireGuard network, either by name of by ID, these names are
available from the `m.internal` zone.

```
$ nslookup machine-1.m.internal
Server: 127.0.0.11
Address: 127.0.0.11#53

Name: machine-1.m.internal
Address: 10.210.0.1
```

## Machine ID

```
$ nslookup c337f00600de51ef4375c9a9a267dba5.m.internal
Server: 127.0.0.11
Address: 127.0.0.11#53

Name: c337f00600de51ef4375c9a9a267dba5.m.internal
Address: 10.210.0.1
```
Loading