diff --git a/Apps/AdvancedBlockingApp/App.cs b/Apps/AdvancedBlockingApp/App.cs index fcb5a1642..5753b7303 100644 --- a/Apps/AdvancedBlockingApp/App.cs +++ b/Apps/AdvancedBlockingApp/App.cs @@ -610,6 +610,28 @@ string GetBlockingReport() return blockingReport; } + DnsQueryLogMetadata GetLogMetadata() + { + Dictionary values = new Dictionary(6, StringComparer.OrdinalIgnoreCase) + { + ["source"] = "advanced-blocking-app", + ["group"] = group.Name + }; + + if (blockListUrl?.Uri is not null) + values["blockListUrl"] = blockListUrl.Uri.AbsoluteUri; + + if (blockedRegex is null) + values["domain"] = blockedDomain ?? question.Name; + else + values["regex"] = blockedRegex; + + return new DnsQueryLogMetadata(values); + } + + DnsQueryLogMetadata logMetadata = GetLogMetadata(); + DnsServerResponseMetadata responseMetadata = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata); + if (group.AllowTxtBlockingReport && (question.Type == DnsResourceRecordType.TXT)) { //return meta data @@ -617,7 +639,7 @@ string GetBlockingReport() DnsResourceRecord[] answer = [new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _blockingAnswerTtl, new DnsTXTRecordData(blockingReport))]; - return Task.FromResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer)); + return Task.FromResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = responseMetadata }); } else { @@ -702,7 +724,7 @@ string GetBlockingReport() } } - return Task.FromResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options)); + return Task.FromResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options) { Tag = responseMetadata }); } } diff --git a/Apps/LogExporterApp/App.cs b/Apps/LogExporterApp/App.cs index a04bcd7ee..0f3248b96 100644 --- a/Apps/LogExporterApp/App.cs +++ b/Apps/LogExporterApp/App.cs @@ -31,7 +31,7 @@ You should have received a copy of the GNU General Public License namespace LogExporter { - public sealed class App : IDnsApplication, IDnsQueryLogger + public sealed class App : IDnsApplication, IDnsQueryLoggerEx { #region variables @@ -142,11 +142,16 @@ public Task InitializeAsync(IDnsServer dnsServer, string? config) } public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null); + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata) { if (_enableLogging) { if (_queuedLogs.Count < _config!.MaxQueueSize) - _queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging)); + _queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging, metadata)); } return Task.CompletedTask; diff --git a/Apps/LogExporterApp/LogEntry.cs b/Apps/LogExporterApp/LogEntry.cs index eabe6ca4e..caa8f34ff 100644 --- a/Apps/LogExporterApp/LogEntry.cs +++ b/Apps/LogExporterApp/LogEntry.cs @@ -34,7 +34,7 @@ namespace LogExporter { public class LogEntry { - public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false) + public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false, DnsQueryLogMetadata? metadata = null) { // Assign timestamp and ensure it's in UTC Timestamp = timestamp.Kind == DateTimeKind.Utc ? timestamp : timestamp.ToUniversalTime(); @@ -42,7 +42,9 @@ public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol pr // Extract client information ClientIp = remoteEP.Address.ToString(); Protocol = protocol; - ResponseType = response.Tag == null ? DnsServerResponseType.Recursive : (DnsServerResponseType)response.Tag; + ResponseType = DnsServerResponseTag.GetResponseType(response.Tag); + DnsQueryLogMetadata? logMetadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag); + BlockingMetadata = logMetadata?.Values; if ((ResponseType == DnsServerResponseType.Recursive) && (response.Metadata is not null)) ResponseRtt = response.Metadata.RoundTripTime; @@ -85,18 +87,27 @@ public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol pr foreach (EDnsOption extendedErrorLog in response.EDNS.Options.Where(o => o.Code == EDnsOptionCode.EXTENDED_DNS_ERROR)) { - string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", StringSplitOptions.TrimEntries); + string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", 2, StringSplitOptions.TrimEntries); + string? errType = null; + string? message = null; + + if (extractedData.Length > 0) + errType = extractedData[0]; + + if (extractedData.Length > 1) + message = extractedData[1]; EDNS.Add(new EDNSLog { - ErrType = extractedData[0], - Message = extractedData[1] + ErrType = errType, + Message = message }); } } public List Answers { get; private set; } public string ClientIp { get; private set; } + public IReadOnlyDictionary? BlockingMetadata { get; private set; } public List EDNS { get; private set; } public DnsTransportProtocol Protocol { get; private set; } public DnsQuestion? Question { get; private set; } diff --git a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs index 194f37f62..d91466853 100644 --- a/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs +++ b/Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs @@ -164,6 +164,15 @@ private static LogEvent Convert(LogEntry log) } } + // Add blocking metadata + if ((log.BlockingMetadata is not null) && (log.BlockingMetadata.Count > 0)) + { + foreach (KeyValuePair item in log.BlockingMetadata) + properties.Add(new LogEventProperty("blocking_" + item.Key, new ScalarValue(item.Value))); + + properties.Add(new LogEventProperty("blockingMetadataSummary", new ScalarValue(string.Join(", ", log.BlockingMetadata.Select(kv => kv.Key + "=" + kv.Value))))); + } + // Define the message template to match the original summary format const string templateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]"; diff --git a/Apps/QueryLogsMySqlApp/App.cs b/Apps/QueryLogsMySqlApp/App.cs index 3af1e55d3..44b019b0a 100644 --- a/Apps/QueryLogsMySqlApp/App.cs +++ b/Apps/QueryLogsMySqlApp/App.cs @@ -34,7 +34,7 @@ You should have received a copy of the GNU General Public License namespace QueryLogsMySql { - public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs + public sealed class App : IDnsApplication, IDnsQueryLoggerEx, IDnsQueryLogs { #region variables @@ -236,14 +236,14 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) await using (MySqlCommand command = connection.CreateCommand()) { sb.Length = 0; - sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer) VALUES "); + sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer, blocking_metadata) VALUES "); for (int i = 0; i < logs.Count; i++) { if (i == 0) - sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})"); + sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})"); else - sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})"); + sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})"); } command.CommandText = sb.ToString(); @@ -263,18 +263,14 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) MySqlParameter paramQtype = command.Parameters.Add("@qtype" + i, MySqlDbType.UInt16); MySqlParameter paramQclass = command.Parameters.Add("@qclass" + i, MySqlDbType.Int16); MySqlParameter paramAnswer = command.Parameters.Add("@answer" + i, MySqlDbType.VarChar); + MySqlParameter paramBlockingMetadata = command.Parameters.Add("@blocking_metadata" + i, MySqlDbType.Text); paramServer.Value = _dnsServer?.ServerDomain; paramTimestamp.Value = log.Timestamp; paramClientIp.Value = log.RemoteEP.Address.ToString(); paramProtocol.Value = (byte)log.Protocol; - DnsServerResponseType responseType; - - if (log.Response.Tag == null) - responseType = DnsServerResponseType.Recursive; - else - responseType = (DnsServerResponseType)log.Response.Tag; + DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag); paramResponseType.Value = (byte)responseType; @@ -328,6 +324,11 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) paramAnswer.Value = answer; } + + if (log.Metadata is null) + paramBlockingMetadata.Value = DBNull.Value; + else + paramBlockingMetadata.Value = JsonSerializer.Serialize(log.Metadata.Values); } await command.ExecuteNonQueryAsync(); @@ -402,7 +403,8 @@ client_ip VARCHAR(39) NOT NULL, qname VARCHAR(255), qtype SMALLINT UNSIGNED, qclass SMALLINT, - answer VARCHAR(4000) + answer VARCHAR(4000), + blocking_metadata TEXT ); "; @@ -445,6 +447,18 @@ answer VARCHAR(4000) { } } + await using (MySqlCommand command = connection.CreateCommand()) + { + command.CommandText = "ALTER TABLE dns_logs ADD blocking_metadata TEXT;"; + + try + { + await command.ExecuteNonQueryAsync(); + } + catch + { } + } + await using (MySqlCommand command = connection.CreateCommand()) { command.CommandText = "CREATE INDEX index_server ON dns_logs (server);"; @@ -705,9 +719,14 @@ answer VARCHAR(4000) } public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null); + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata) { if (_enableLogging) - _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response)); + _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response, metadata)); return Task.CompletedTask; } @@ -924,18 +943,20 @@ readonly struct LogEntry public readonly IPEndPoint RemoteEP; public readonly DnsTransportProtocol Protocol; public readonly DnsDatagram Response; + public readonly DnsQueryLogMetadata? Metadata; #endregion #region constructor - public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata = null) { Timestamp = timestamp; Request = request; RemoteEP = remoteEP; Protocol = protocol; Response = response; + Metadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag); } #endregion diff --git a/Apps/QueryLogsSqlServerApp/App.cs b/Apps/QueryLogsSqlServerApp/App.cs index 2bf09c64d..078f5b439 100644 --- a/Apps/QueryLogsSqlServerApp/App.cs +++ b/Apps/QueryLogsSqlServerApp/App.cs @@ -34,7 +34,7 @@ You should have received a copy of the GNU General Public License namespace QueryLogsSqlServer { - public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs + public sealed class App : IDnsApplication, IDnsQueryLoggerEx, IDnsQueryLogs { #region variables @@ -52,7 +52,7 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs Channel? _channel; ChannelWriter? _channelWriter; Thread? _consumerThread; - const int BULK_INSERT_COUNT = 190; //sql server supports a maximum of 2100 parameters per query + const int BULK_INSERT_COUNT = 175; //sql server supports a maximum of 2100 parameters per query const int BULK_INSERT_ERROR_DELAY = 10000; const int BULK_REMOVE_COUNT = 10000; @@ -236,14 +236,14 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) await using (SqlCommand command = connection.CreateCommand()) { sb.Length = 0; - sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer) VALUES "); + sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer, blocking_metadata) VALUES "); for (int i = 0; i < logs.Count; i++) { if (i == 0) - sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})"); + sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})"); else - sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})"); + sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})"); } command.CommandText = sb.ToString(); @@ -263,18 +263,14 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) SqlParameter paramQtype = command.Parameters.Add("@qtype" + i, SqlDbType.Int); SqlParameter paramQclass = command.Parameters.Add("@qclass" + i, SqlDbType.SmallInt); SqlParameter paramAnswer = command.Parameters.Add("@answer" + i, SqlDbType.VarChar); + SqlParameter paramBlockingMetadata = command.Parameters.Add("@blocking_metadata" + i, SqlDbType.NVarChar, -1); paramServer.Value = _dnsServer?.ServerDomain; paramTimestamp.Value = log.Timestamp; paramClientIp.Value = log.RemoteEP.Address.ToString(); paramProtocol.Value = (byte)log.Protocol; - DnsServerResponseType responseType; - - if (log.Response.Tag == null) - responseType = DnsServerResponseType.Recursive; - else - responseType = (DnsServerResponseType)log.Response.Tag; + DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag); paramResponseType.Value = (byte)responseType; @@ -328,6 +324,11 @@ private async Task BulkInsertLogsAsync(List logs, StringBuilder sb) paramAnswer.Value = answer; } + + if (log.Metadata is null) + paramBlockingMetadata.Value = DBNull.Value; + else + paramBlockingMetadata.Value = JsonSerializer.Serialize(log.Metadata.Values); } await command.ExecuteNonQueryAsync(); @@ -414,7 +415,8 @@ client_ip VARCHAR(39) NOT NULL, qname VARCHAR(255), qtype INT, qclass SMALLINT, - answer VARCHAR(4000) + answer VARCHAR(4000), + blocking_metadata NVARCHAR(MAX) ); END @@ -423,6 +425,22 @@ IF NOT EXISTS(SELECT * FROM sys.columns WHERE name = 'server' AND object_id = OB ALTER TABLE dns_logs ADD server varchar(255); END +IF NOT EXISTS(SELECT * FROM sys.columns WHERE name = 'blocking_metadata' AND object_id = OBJECT_ID('dns_logs')) +BEGIN + ALTER TABLE dns_logs ADD blocking_metadata NVARCHAR(MAX); +END +ELSE IF EXISTS +( + SELECT * + FROM sys.columns + WHERE object_id = OBJECT_ID('dns_logs') + AND name = 'blocking_metadata' + AND system_type_id = 167 +) +BEGIN + ALTER TABLE dns_logs ALTER COLUMN blocking_metadata NVARCHAR(MAX); +END + IF NOT EXISTS(SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='dns_logs' AND COLUMN_NAME='qtype' AND DATA_TYPE='INT') BEGIN DROP INDEX index_qtype ON dns_logs; @@ -597,9 +615,14 @@ IF NOT EXISTS(SELECT * FROM sys.indexes WHERE name = 'index_all' AND object_id = } public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null); + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata) { if (_enableLogging) - _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response)); + _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response, metadata)); return Task.CompletedTask; } @@ -817,18 +840,20 @@ readonly struct LogEntry public readonly IPEndPoint RemoteEP; public readonly DnsTransportProtocol Protocol; public readonly DnsDatagram Response; + public readonly DnsQueryLogMetadata? Metadata; #endregion #region constructor - public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata = null) { Timestamp = timestamp; Request = request; RemoteEP = remoteEP; Protocol = protocol; Response = response; + Metadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag); } #endregion diff --git a/Apps/QueryLogsSqliteApp/App.cs b/Apps/QueryLogsSqliteApp/App.cs index dff3de728..1837a0c02 100644 --- a/Apps/QueryLogsSqliteApp/App.cs +++ b/Apps/QueryLogsSqliteApp/App.cs @@ -35,7 +35,7 @@ You should have received a copy of the GNU General Public License namespace QueryLogsSqlite { - public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs + public sealed class App : IDnsApplication, IDnsQueryLoggerEx, IDnsQueryLogs { #region variables @@ -264,7 +264,7 @@ private async Task BulkInsertLogsAsync(List logs) { await using (SqliteCommand command = connection.CreateCommand()) { - command.CommandText = "INSERT INTO dns_logs (timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer) VALUES (@timestamp, @client_ip, @protocol, @response_type, @response_rtt, @rcode, @qname, @qtype, @qclass, @answer);"; + command.CommandText = "INSERT INTO dns_logs (timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer, blocking_metadata) VALUES (@timestamp, @client_ip, @protocol, @response_type, @response_rtt, @rcode, @qname, @qtype, @qclass, @answer, @blocking_metadata);"; SqliteParameter paramTimestamp = command.Parameters.Add("@timestamp", SqliteType.Text); SqliteParameter paramClientIp = command.Parameters.Add("@client_ip", SqliteType.Text); @@ -276,6 +276,7 @@ private async Task BulkInsertLogsAsync(List logs) SqliteParameter paramQtype = command.Parameters.Add("@qtype", SqliteType.Integer); SqliteParameter paramQclass = command.Parameters.Add("@qclass", SqliteType.Integer); SqliteParameter paramAnswer = command.Parameters.Add("@answer", SqliteType.Text); + SqliteParameter paramBlockingMetadata = command.Parameters.Add("@blocking_metadata", SqliteType.Text); foreach (LogEntry log in logs) { @@ -283,12 +284,7 @@ private async Task BulkInsertLogsAsync(List logs) paramClientIp.Value = log.RemoteEP.Address.ToString(); paramProtocol.Value = (int)log.Protocol; - DnsServerResponseType responseType; - - if (log.Response.Tag == null) - responseType = DnsServerResponseType.Recursive; - else - responseType = (DnsServerResponseType)log.Response.Tag; + DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag); paramResponseType.Value = (int)responseType; @@ -340,6 +336,11 @@ private async Task BulkInsertLogsAsync(List logs) paramAnswer.Value = answer; } + if (log.Metadata is null) + paramBlockingMetadata.Value = DBNull.Value; + else + paramBlockingMetadata.Value = JsonSerializer.Serialize(log.Metadata.Values); + await command.ExecuteNonQueryAsync(); } @@ -430,7 +431,8 @@ client_ip VARCHAR(39) NOT NULL, qname VARCHAR(255), qtype SMALLINT, qclass SMALLINT, - answer TEXT + answer TEXT, + blocking_metadata TEXT ); "; await command.ExecuteNonQueryAsync(); @@ -447,6 +449,17 @@ answer TEXT catch { } + try + { + await using (SqliteCommand command = connection.CreateCommand()) + { + command.CommandText = "ALTER TABLE dns_logs ADD COLUMN blocking_metadata TEXT;"; + await command.ExecuteNonQueryAsync(); + } + } + catch + { } + await using (SqliteCommand command = connection.CreateCommand()) { command.CommandText = "CREATE INDEX IF NOT EXISTS index_timestamp ON dns_logs (timestamp);"; @@ -567,9 +580,14 @@ answer TEXT } public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + { + return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null); + } + + public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata) { if (_enableLogging) - _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response)); + _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response, metadata)); return Task.CompletedTask; } @@ -786,18 +804,20 @@ readonly struct LogEntry public readonly IPEndPoint RemoteEP; public readonly DnsTransportProtocol Protocol; public readonly DnsDatagram Response; + public readonly DnsQueryLogMetadata? Metadata; #endregion #region constructor - public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response) + public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata = null) { Timestamp = timestamp; Request = request; RemoteEP = remoteEP; Protocol = protocol; Response = response; + Metadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag); } #endregion diff --git a/DnsServerCore.ApplicationCommon/IDnsQueryLogger.cs b/DnsServerCore.ApplicationCommon/IDnsQueryLogger.cs index d23b31f6a..459e8a8ab 100644 --- a/DnsServerCore.ApplicationCommon/IDnsQueryLogger.cs +++ b/DnsServerCore.ApplicationCommon/IDnsQueryLogger.cs @@ -18,12 +18,114 @@ You should have received a copy of the GNU General Public License */ using System; +using System.Collections.Generic; +using System.Linq; using System.Net; using System.Threading.Tasks; using TechnitiumLibrary.Net.Dns; namespace DnsServerCore.ApplicationCommon { + public sealed class DnsQueryLogMetadata + { + public DnsQueryLogMetadata(IReadOnlyDictionary? values = null) + { + Values = values is null ? new Dictionary(0, StringComparer.OrdinalIgnoreCase) : new Dictionary(values, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Structured metadata key/value pairs (example keys: source, domain, blockListUrl). + /// + public IReadOnlyDictionary Values { get; } + + public string ToReportString() + { + if (Values.Count < 1) + return string.Empty; + + return string.Join("; ", Values.Select(kv => Uri.EscapeDataString(kv.Key) + "=" + Uri.EscapeDataString(kv.Value))); + } + + public static DnsQueryLogMetadata? ParseReportString(string? report) + { + if (string.IsNullOrWhiteSpace(report)) + return null; + + string[] parts = report.Split(';', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + if (parts.Length < 1) + return null; + + Dictionary values = new Dictionary(parts.Length, StringComparer.OrdinalIgnoreCase); + + foreach (string part in parts) + { + int separatorIndex = part.IndexOf('='); + if ((separatorIndex < 1) || (separatorIndex >= (part.Length - 1))) + continue; + + string key = part[..separatorIndex].Trim(); + string value = part[(separatorIndex + 1)..].Trim(); + + try + { + key = Uri.UnescapeDataString(key); + value = Uri.UnescapeDataString(value); + } + catch + { + continue; + } + + if ((key.Length < 1) || (value.Length < 1)) + continue; + + values[key] = value; + } + + if (values.Count < 1) + return null; + + return new DnsQueryLogMetadata(values); + } + } + + public sealed class DnsServerResponseMetadata + { + public DnsServerResponseMetadata(DnsServerResponseType responseType, DnsQueryLogMetadata? logMetadata = null) + { + ResponseType = responseType; + LogMetadata = logMetadata; + } + + public DnsServerResponseType ResponseType { get; } + public DnsQueryLogMetadata? LogMetadata { get; } + } + + public static class DnsServerResponseTag + { + public static DnsServerResponseType GetResponseType(object? tag) + { + if (tag is null) + return DnsServerResponseType.Recursive; + + if (tag is DnsServerResponseType responseType) + return responseType; + + if (tag is DnsServerResponseMetadata responseMetadata) + return responseMetadata.ResponseType; + + return DnsServerResponseType.Recursive; + } + + public static DnsQueryLogMetadata? GetLogMetadata(object? tag) + { + if (tag is DnsServerResponseMetadata responseMetadata) + return responseMetadata.LogMetadata; + + return null; + } + } + public enum DnsServerResponseType : byte { Authoritative = 1, @@ -50,4 +152,21 @@ public interface IDnsQueryLogger /// The DNS response that was sent. Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response); } + + /// + /// Allows a DNS App to receive extended query log metadata. + /// + public interface IDnsQueryLoggerEx : IDnsQueryLogger + { + /// + /// Allows a DNS App to log incoming DNS requests and responses, including metadata that may not be present in the wire response. + /// + /// The time stamp of the log entry. + /// The incoming DNS request that was received. + /// The end point (IP address and port) of the client making the request. + /// The protocol using which the request was received. + /// The DNS response that was sent. + /// Optional metadata for logging. + Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata); + } } diff --git a/DnsServerCore/Dns/DnsServer.cs b/DnsServerCore/Dns/DnsServer.cs index c031cbe9e..ef29d64a4 100644 --- a/DnsServerCore/Dns/DnsServer.cs +++ b/DnsServerCore/Dns/DnsServer.cs @@ -4357,7 +4357,18 @@ private async Task IsAllowedAsync(DnsDatagram request, IPEndPoint remoteEP return false; } - private async Task ProcessBlockedQueryAsync(DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol) + sealed class BlockedQueryResult + { + public BlockedQueryResult(DnsDatagram response) + { + Response = response; + } + + public DnsDatagram Response { get; } + public DnsQueryLogMetadata? LogMetadata => DnsServerResponseTag.GetLogMetadata(Response.Tag); + } + + private async Task ProcessBlockedQueryAsync(DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol) { if (_enableBlocking) { @@ -4365,12 +4376,21 @@ private async Task ProcessBlockedQueryAsync(DnsDatagram request, IP if (response is null) { //domain not blocked in blocked zone - response = _blockListZoneManager.Query(request); //check in block list zone + response = _blockListZoneManager.Query(request, out _); //check in block list zone if (response is not null) { //domain is blocked in block list zone - response.Tag = DnsServerResponseType.Blocked; - return response; + if (response.Tag is null) + { + string blockedDomain = request.Question[0].Name; + DnsResourceRecord firstAuthority = response.FindFirstAuthorityRecord(); + if ((firstAuthority is not null) && (firstAuthority.Type == DnsResourceRecordType.SOA)) + blockedDomain = firstAuthority.Name; + + response.Tag = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, new DnsQueryLogMetadata(new Dictionary(2, StringComparer.OrdinalIgnoreCase) { ["source"] = "block-list-zone", ["domain"] = blockedDomain })); + } + + return new BlockedQueryResult(response); } //domain not blocked in block list zone; continue to check app blocking handlers @@ -4393,20 +4413,21 @@ string GetBlockedDomain() { //return meta data string blockedDomain = GetBlockedDomain(); + DnsQueryLogMetadata logMetadata = new DnsQueryLogMetadata(new Dictionary(2, StringComparer.OrdinalIgnoreCase) { ["source"] = "blocked-zone", ["domain"] = blockedDomain }); - IReadOnlyList answer = [new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _blockingAnswerTtl, new DnsTXTRecordData("source=blocked-zone; domain=" + blockedDomain))]; + IReadOnlyList answer = [new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _blockingAnswerTtl, new DnsTXTRecordData(logMetadata.ToReportString()))]; - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = DnsServerResponseType.Blocked }; + return new BlockedQueryResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata) }); } else { - string blockedDomain = null; + string blockedDomain = GetBlockedDomain(); EDnsOption[] options = null; + DnsQueryLogMetadata logMetadata = new DnsQueryLogMetadata(new Dictionary(2, StringComparer.OrdinalIgnoreCase) { ["source"] = "blocked-zone", ["domain"] = blockedDomain }); if (_allowTxtBlockingReport && (request.EDNS is not null)) { - blockedDomain = GetBlockedDomain(); - options = [new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, "source=blocked-zone; domain=" + blockedDomain))]; + options = [new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, logMetadata.ToReportString()))]; } IReadOnlyCollection aRecords; @@ -4425,14 +4446,11 @@ string GetBlockedDomain() break; case DnsServerBlockingType.NxDomain: - if (blockedDomain is null) - blockedDomain = GetBlockedDomain(); - string parentDomain = AuthZoneManager.GetParentZone(blockedDomain); if (parentDomain is null) parentDomain = string.Empty; - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NxDomain, request.Question, null, [new DnsResourceRecord(parentDomain, DnsResourceRecordType.SOA, question.Class, _blockingAnswerTtl, _blockedZoneManager.DnsSOARecord)], null, request.EDNS is null ? ushort.MinValue : _udpPayloadSize, EDnsHeaderFlags.None, options) { Tag = DnsServerResponseType.Blocked }; + return new BlockedQueryResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NxDomain, request.Question, null, [new DnsResourceRecord(parentDomain, DnsResourceRecordType.SOA, question.Class, _blockingAnswerTtl, _blockedZoneManager.DnsSOARecord)], null, request.EDNS is null ? ushort.MinValue : _udpPayloadSize, EDnsHeaderFlags.None, options) { Tag = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata) }); default: throw new InvalidOperationException(); @@ -4489,7 +4507,7 @@ string GetBlockedDomain() break; } - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _udpPayloadSize, EDnsHeaderFlags.None, options) { Tag = DnsServerResponseType.Blocked }; + return new BlockedQueryResult(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _udpPayloadSize, EDnsHeaderFlags.None, options) { Tag = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata) }); } } } @@ -4502,9 +4520,9 @@ string GetBlockedDomain() if (appBlockedResponse is not null) { if (appBlockedResponse.Tag is null) - appBlockedResponse.Tag = DnsServerResponseType.Blocked; + appBlockedResponse.Tag = new DnsServerResponseMetadata(DnsServerResponseType.Blocked); - return appBlockedResponse; + return new BlockedQueryResult(appBlockedResponse); } } catch (Exception ex) @@ -4531,9 +4549,9 @@ private async Task ProcessRecursiveQueryAsync(DnsDatagram request, isAllowed = await IsAllowedAsync(request, remoteEP, protocol); if (!isAllowed) { - DnsDatagram blockedResponse = await ProcessBlockedQueryAsync(request, remoteEP, protocol); + BlockedQueryResult blockedResponse = await ProcessBlockedQueryAsync(request, remoteEP, protocol); if (blockedResponse is not null) - return blockedResponse; + return blockedResponse.Response; } } @@ -4579,32 +4597,37 @@ private async Task ProcessRecursiveQueryAsync(DnsDatagram request, break; //CNAME is in allowed zone //check blocked zone and block list zone - DnsDatagram blockedResponse = await ProcessBlockedQueryAsync(newRequest, remoteEP, protocol); + BlockedQueryResult blockedResponse = await ProcessBlockedQueryAsync(newRequest, remoteEP, protocol); if (blockedResponse is not null) { //found cname cloaking - List answer = new List(i + 1 + blockedResponse.Answer.Count); + List answer = new List(i + 1 + blockedResponse.Response.Answer.Count); //copy current and previous CNAME records for (int j = 0; j <= i; j++) answer.Add(response.Answer[j]); //copy last response answers - answer.AddRange(blockedResponse.Answer); + answer.AddRange(blockedResponse.Response.Answer); //include blocked response additional section to pass on Extended DNS Errors - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, true, true, false, false, blockedResponse.RCODE, request.Question, answer, blockedResponse.Authority, blockedResponse.Additional) { Tag = blockedResponse.Tag }; + return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, true, true, false, false, blockedResponse.Response.RCODE, request.Question, answer, blockedResponse.Response.Authority, blockedResponse.Response.Additional) + { + Tag = blockedResponse.LogMetadata is null ? blockedResponse.Response.Tag : new DnsServerResponseMetadata(DnsServerResponseType.Blocked, blockedResponse.LogMetadata) + }; } } } } + DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(response.Tag); + if (response.Tag is null) { if (response.IsBlockedResponse()) response.Tag = DnsServerResponseType.UpstreamBlocked; } - else if ((DnsServerResponseType)response.Tag == DnsServerResponseType.Cached) + else if (responseType == DnsServerResponseType.Cached) { if (response.IsBlockedResponse()) response.Tag = DnsServerResponseType.UpstreamBlockedCached; diff --git a/DnsServerCore/Dns/StatsManager.cs b/DnsServerCore/Dns/StatsManager.cs index a29ca6481..566258e6d 100644 --- a/DnsServerCore/Dns/StatsManager.cs +++ b/DnsServerCore/Dns/StatsManager.cs @@ -149,10 +149,8 @@ public StatsManager(DnsServer dnsServer) if (item._response is null) responseType = DnsServerResponseType.Dropped; - else if (item._response.Tag is null) - responseType = DnsServerResponseType.Recursive; else - responseType = (DnsServerResponseType)item._response.Tag; + responseType = DnsServerResponseTag.GetResponseType(item._response.Tag); UpdateLifetimeCounters(responseCode, responseType, item._remoteEP.Address); @@ -176,7 +174,12 @@ public StatsManager(DnsServer dnsServer) { try { - _ = logger.InsertLogAsync(item._timestamp, item._request, item._remoteEP, item._protocol, item._response); + DnsQueryLogMetadata? logMetadata = DnsServerResponseTag.GetLogMetadata(item._response.Tag); + + if (logger is IDnsQueryLoggerEx loggerEx) + _ = loggerEx.InsertLogAsync(item._timestamp, item._request, item._remoteEP, item._protocol, item._response, logMetadata); + else + _ = logger.InsertLogAsync(item._timestamp, item._request, item._remoteEP, item._protocol, item._response); } catch (Exception ex) { diff --git a/DnsServerCore/Dns/ZoneManagers/BlockListZoneManager.cs b/DnsServerCore/Dns/ZoneManagers/BlockListZoneManager.cs index 689f9350a..917c09ac8 100644 --- a/DnsServerCore/Dns/ZoneManagers/BlockListZoneManager.cs +++ b/DnsServerCore/Dns/ZoneManagers/BlockListZoneManager.cs @@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License */ +using DnsServerCore.ApplicationCommon; using System; using System.Collections.Generic; using System.IO; @@ -847,6 +848,13 @@ public bool IsAllowed(DnsDatagram request) public DnsDatagram Query(DnsDatagram request) { + return Query(request, out _); + } + + public DnsDatagram Query(DnsDatagram request, out DnsQueryLogMetadata? logMetadata) + { + logMetadata = null; + if (_blockListZone.Count < 1) return null; @@ -856,6 +864,32 @@ public DnsDatagram Query(DnsDatagram request) if (blockLists is null) return null; //zone not blocked + Dictionary metadataValues = new Dictionary(4, StringComparer.OrdinalIgnoreCase) + { + ["source"] = "block-list-zone", + ["domain"] = blockedDomain + }; + + if (blockLists.Count > 0) + { + metadataValues["blockListUrl"] = blockLists[0].AbsoluteUri; + metadataValues["blockListCount"] = blockLists.Count.ToString(System.Globalization.CultureInfo.InvariantCulture); + } + + DnsQueryLogMetadata CreateLogMetadata(Uri? blockList = null) + { + Dictionary values = new Dictionary(metadataValues, StringComparer.OrdinalIgnoreCase); + + if (blockList is not null) + values["blockListUrl"] = blockList.AbsoluteUri; + + return new DnsQueryLogMetadata(values); + } + + DnsQueryLogMetadata responseLogMetadata = new DnsQueryLogMetadata(metadataValues); + DnsServerResponseMetadata responseMetadata = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, responseLogMetadata); + logMetadata = responseLogMetadata; + //zone is blocked if (_dnsServer.AllowTxtBlockingReport && (question.Type == DnsResourceRecordType.TXT)) { @@ -863,9 +897,9 @@ public DnsDatagram Query(DnsDatagram request) DnsResourceRecord[] answer = new DnsResourceRecord[blockLists.Count]; for (int i = 0; i < answer.Length; i++) - answer[i] = new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _dnsServer.BlockingAnswerTtl, new DnsTXTRecordData("source=block-list-zone; blockListUrl=" + blockLists[i].AbsoluteUri + "; domain=" + blockedDomain)); + answer[i] = new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _dnsServer.BlockingAnswerTtl, new DnsTXTRecordData(CreateLogMetadata(blockLists[i]).ToReportString())); - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer); + return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = responseMetadata }; } else { @@ -876,7 +910,7 @@ public DnsDatagram Query(DnsDatagram request) options = new EDnsOption[blockLists.Count]; for (int i = 0; i < options.Length; i++) - options[i] = new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, "source=block-list-zone; blockListUrl=" + blockLists[i].AbsoluteUri + "; domain=" + blockedDomain)); + options[i] = new EDnsOption(EDnsOptionCode.EXTENDED_DNS_ERROR, new EDnsExtendedDnsErrorOptionData(EDnsExtendedDnsErrorCode.Blocked, CreateLogMetadata(blockLists[i]).ToReportString())); } IReadOnlyCollection aRecords; @@ -899,7 +933,7 @@ public DnsDatagram Query(DnsDatagram request) if (parentDomain is null) parentDomain = string.Empty; - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NxDomain, request.Question, null, [new DnsResourceRecord(parentDomain, DnsResourceRecordType.SOA, question.Class, _dnsServer.BlockingAnswerTtl, _soaRecord)], null, request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, EDnsHeaderFlags.None, options); + return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NxDomain, request.Question, null, [new DnsResourceRecord(parentDomain, DnsResourceRecordType.SOA, question.Class, _dnsServer.BlockingAnswerTtl, _soaRecord)], null, request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, EDnsHeaderFlags.None, options) { Tag = responseMetadata }; default: throw new InvalidOperationException(); @@ -965,7 +999,7 @@ public DnsDatagram Query(DnsDatagram request) break; } - return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, EDnsHeaderFlags.None, options); + return new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer.UdpPayloadSize, EDnsHeaderFlags.None, options) { Tag = responseMetadata }; } }