cotugno/AdGuardHome
2
Fork 0
mirror of https://github.com/AdguardTeam/AdGuardHome.git synced 2024-11-17 02:48:28 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge: * querylog: refactor: move HTTP handlers to querylog/

Browse Source 
* commit '90db91b0fd347d168ef9589405ba812c0cfc0c2d':
  * querylog: refactor: move HTTP handlers to querylog/
This commit is contained in:
Simon Zolin 2019-10-09 19:43:00 +03:00
commit b43c076c4d
11 changed files with 276 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dnsforward/dnsforward.go
View File

@ -550,7 +550,7 @@ func (s *Server) handleDNSRequest(p *proxy.Proxy, d *proxy.DNSContext) error {
s.RLock()
// Synchronize access to s.queryLog and s.stats so they won't be suddenly uninitialized while in use.
// This can happen after proxy server has been stopped, but its workers haven't yet exited.
if s.conf.QueryLogEnabled && shouldLog && s.queryLog != nil {
if shouldLog && s.queryLog != nil {
upstreamAddr := ""
if d.Upstream != nil {
upstreamAddr = d.Upstream.Address()

1
dnsforward/dnsforward_test.go
View File

@ -380,7 +380,6 @@ func createTestServer(t *testing.T) *Server {
s.conf.UDPListenAddr = &net.UDPAddr{Port: 0}
s.conf.TCPListenAddr = &net.TCPAddr{Port: 0}
s.conf.QueryLogEnabled = true
s.conf.FilteringConfig.FilteringEnabled = true
s.conf.FilteringConfig.ProtectionEnabled = true
s.conf.FilteringConfig.SafeBrowsingEnabled = true

11
home/config.go
View File

@ -278,10 +278,6 @@ func parseConfig() error {
config.DNS.FiltersUpdateIntervalHours = 24
}
if !checkQueryLogInterval(config.DNS.QueryLogInterval) {
config.DNS.QueryLogInterval = 1
}
for _, cy := range config.Clients {
cli := Client{
Name: cy.Name,
@ -364,6 +360,13 @@ func (c *configuration) write() error {
config.DNS.StatsInterval = sdc.Interval
}
if config.queryLog != nil {
dc := querylog.DiskConfig{}
config.queryLog.WriteDiskConfig(&dc)
config.DNS.QueryLogEnabled = dc.Enabled
config.DNS.QueryLogInterval = dc.Interval
}
configFile := config.getConfigFilename()
log.Debug("Writing YAML file: %s", configFile)
yamlText, err := yaml.Marshal(&config)

2
home/control.go
View File

@ -111,7 +111,6 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
"http_port": config.BindPort,
"dns_port": config.DNS.Port,
"protection_enabled": config.DNS.ProtectionEnabled,
"querylog_enabled": config.DNS.QueryLogEnabled,
"running": isRunning(),
"bootstrap_dns": config.DNS.BootstrapDNS,
"upstream_dns": config.DNS.UpstreamDNS,
@ -568,7 +567,6 @@ func registerControlHandlers() {
RegisterClientsHandlers()
registerRewritesHandlers()
RegisterBlockedServicesHandlers()
RegisterQueryLogHandlers()
RegisterAuthHandlers()
http.HandleFunc("/dns-query", postInstall(handleDOH))

160
home/control_querylog.go
View File

@ -1,160 +0,0 @@
package home
import (
"encoding/json"
"net/http"
"time"
"github.com/AdguardTeam/AdGuardHome/querylog"
"github.com/miekg/dns"
)
type qlogFilterJSON struct {
Domain string `json:"domain"`
Client string `json:"client"`
QuestionType string `json:"question_type"`
ResponseStatus string `json:"response_status"`
}
type queryLogRequest struct {
OlderThan string `json:"older_than"`
Filter qlogFilterJSON `json:"filter"`
}
// "value" -> value, return TRUE
func getDoubleQuotesEnclosedValue(s *string) bool {
t := *s
if len(t) >= 2 && t[0] == '"' && t[len(t)-1] == '"' {
*s = t[1 : len(t)-1]
return true
}
return false
}
func handleQueryLog(w http.ResponseWriter, r *http.Request) {
req := queryLogRequest{}
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
httpError(w, http.StatusBadRequest, "json decode: %s", err)
return
}
params := querylog.GetDataParams{
Domain: req.Filter.Domain,
Client: req.Filter.Client,
}
if len(req.OlderThan) != 0 {
params.OlderThan, err = time.Parse(time.RFC3339Nano, req.OlderThan)
if err != nil {
httpError(w, http.StatusBadRequest, "invalid time stamp: %s", err)
return
}
}
if getDoubleQuotesEnclosedValue(&params.Domain) {
params.StrictMatchDomain = true
}
if getDoubleQuotesEnclosedValue(&params.Client) {
params.StrictMatchClient = true
}
if len(req.Filter.QuestionType) != 0 {
qtype, ok := dns.StringToType[req.Filter.QuestionType]
if !ok {
httpError(w, http.StatusBadRequest, "invalid question_type")
return
}
params.QuestionType = qtype
}
if len(req.Filter.ResponseStatus) != 0 {
switch req.Filter.ResponseStatus {
case "filtered":
params.ResponseStatus = querylog.ResponseStatusFiltered
default:
httpError(w, http.StatusBadRequest, "invalid response_status")
return
}
}
data := config.queryLog.GetData(params)
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
func handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
config.queryLog.Clear()
returnOK(w)
}
type qlogConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"`
}
// Get configuration
func handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = config.DNS.QueryLogEnabled
resp.Interval = config.DNS.QueryLogInterval
jsonVal, err := json.Marshal(resp)
if err != nil {
httpError(w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(w, http.StatusInternalServerError, "http write: %s", err)
}
}
// Set configuration
func handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
reqData := qlogConfig{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
httpError(w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkQueryLogInterval(reqData.Interval) {
httpError(w, http.StatusBadRequest, "Unsupported interval")
return
}
config.DNS.QueryLogEnabled = reqData.Enabled
config.DNS.QueryLogInterval = reqData.Interval
_ = config.write()
conf := querylog.Config{
Interval: config.DNS.QueryLogInterval * 24,
}
config.queryLog.Configure(conf)
returnOK(w)
}
func checkQueryLogInterval(i uint32) bool {
return i == 1 || i == 7 || i == 30 || i == 90
}
// RegisterQueryLogHandlers - register handlers
func RegisterQueryLogHandlers() {
httpRegister("POST", "/control/querylog", handleQueryLog)
httpRegister(http.MethodGet, "/control/querylog_info", handleQueryLogInfo)
httpRegister(http.MethodPost, "/control/querylog_clear", handleQueryLogClear)
httpRegister(http.MethodPost, "/control/querylog_config", handleQueryLogConfig)
}

7
home/dns.go
View File

@ -48,8 +48,11 @@ func initDNSServer() {
log.Fatal("Couldn't initialize statistics module")
}
conf := querylog.Config{
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval * 24,
Enabled: config.DNS.QueryLogEnabled,
BaseDir: baseDir,
Interval: config.DNS.QueryLogInterval,
ConfigModified: onConfigModified,
HTTPRegister: httpRegister,
}
config.queryLog = querylog.New(conf)
config.dnsServer = dnsforward.NewServer(config.stats, config.queryLog)

92
querylog/qlog.go
View File

@ -16,7 +16,7 @@ import (
)
const (
logBufferCap = 5000 // maximum capacity of logBuffer before it's flushed to disk
logBufferCap = 5000 // maximum capacity of buffer before it's flushed to disk
queryLogFileName = "querylog.json" // .gz added during compression
getDataLimit = 500 // GetData(): maximum log entries to return
@ -29,10 +29,11 @@ type queryLog struct {
conf Config
logFile string // path to the log file
logBufferLock sync.RWMutex
logBuffer []*logEntry
bufferLock sync.RWMutex
buffer []*logEntry
fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
flushPending bool // don't start another goroutine while the previous one is still running
fileWriteLock sync.Mutex
}
// create a new instance of the query log
@ -40,7 +41,13 @@ func newQueryLog(conf Config) *queryLog {
l := queryLog{}
l.logFile = filepath.Join(conf.BaseDir, queryLogFileName)
l.conf = conf
go l.periodicQueryLogRotate()
if !checkInterval(l.conf.Interval) {
l.conf.Interval = 1
}
if l.conf.HTTPRegister != nil {
l.initWeb()
}
go l.periodicRotate()
return &l
}
@ -48,18 +55,30 @@ func (l *queryLog) Close() {
_ = l.flushLogBuffer(true)
}
func (l *queryLog) Configure(conf Config) {
l.conf = conf
func checkInterval(days uint32) bool {
return days == 1 || days == 7 || days == 30 || days == 90
}
func (l *queryLog) Clear() {
// Set new configuration at runtime
func (l *queryLog) configure(conf Config) {
l.conf.Enabled = conf.Enabled
l.conf.Interval = conf.Interval
}
func (l *queryLog) WriteDiskConfig(dc *DiskConfig) {
dc.Enabled = l.conf.Enabled
dc.Interval = l.conf.Interval
}
// Clear memory buffer and remove log files
func (l *queryLog) clear() {
l.fileFlushLock.Lock()
defer l.fileFlushLock.Unlock()
l.logBufferLock.Lock()
l.logBuffer = nil
l.bufferLock.Lock()
l.buffer = nil
l.flushPending = false
l.logBufferLock.Unlock()
l.bufferLock.Unlock()
err := os.Remove(l.logFile + ".1")
if err != nil && !os.IsNotExist(err) {
@ -96,6 +115,10 @@ func getIPString(addr net.Addr) string {
}
func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string) {
if !l.conf.Enabled {
return
}
var q []byte
var a []byte
var err error
@ -132,16 +155,16 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res
Upstream: upstream,
}
l.logBufferLock.Lock()
l.logBuffer = append(l.logBuffer, &entry)
l.bufferLock.Lock()
l.buffer = append(l.buffer, &entry)
needFlush := false
if !l.flushPending {
needFlush = len(l.logBuffer) >= logBufferCap
needFlush = len(l.buffer) >= logBufferCap
if needFlush {
l.flushPending = true
}
}
l.logBufferLock.Unlock()
l.bufferLock.Unlock()
// if buffer needs to be flushed to disk, do it now
if needFlush {
@ -152,11 +175,9 @@ func (l *queryLog) Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Res
}
// Return TRUE if this entry is needed
func isNeeded(entry *logEntry, params GetDataParams) bool {
if params.ResponseStatus != 0 {
if params.ResponseStatus == ResponseStatusFiltered && !entry.Result.IsFiltered {
return false
}
func isNeeded(entry *logEntry, params getDataParams) bool {
if params.ResponseStatus == responseStatusFiltered && !entry.Result.IsFiltered {
return false
}
if len(params.Domain) != 0 || params.QuestionType != 0 {
@ -193,7 +214,7 @@ func isNeeded(entry *logEntry, params GetDataParams) bool {
return true
}
func (l *queryLog) readFromFile(params GetDataParams) ([]*logEntry, int) {
func (l *queryLog) readFromFile(params getDataParams) ([]*logEntry, int) {
entries := []*logEntry{}
olderThan := params.OlderThan
totalChunks := 0
@ -247,7 +268,28 @@ func (l *queryLog) readFromFile(params GetDataParams) ([]*logEntry, int) {
return entries, total
}
func (l *queryLog) GetData(params GetDataParams) []map[string]interface{} {
// Parameters for getData()
type getDataParams struct {
OlderThan time.Time // return entries that are older than this value
Domain string // filter by domain name in question
Client string // filter by client IP
QuestionType uint16 // filter by question type
ResponseStatus responseStatusType // filter by response status
StrictMatchDomain bool // if Domain value must be matched strictly
StrictMatchClient bool // if Client value must be matched strictly
}
// Response status
type responseStatusType int32
// Response status constants
const (
responseStatusAll responseStatusType = iota + 1
responseStatusFiltered
)
// Get log entries
func (l *queryLog) getData(params getDataParams) []map[string]interface{} {
var data = []map[string]interface{}{}
if len(params.Domain) != 0 && params.StrictMatchDomain {
@ -266,9 +308,9 @@ func (l *queryLog) GetData(params GetDataParams) []map[string]interface{} {
}
// add from memory buffer
l.logBufferLock.Lock()
total += len(l.logBuffer)
for _, entry := range l.logBuffer {
l.bufferLock.Lock()
total += len(l.buffer)
for _, entry := range l.buffer {
if !isNeeded(entry, params) {
continue
@ -283,7 +325,7 @@ func (l *queryLog) GetData(params GetDataParams) []map[string]interface{} {
}
entries = append(entries, entry)
}
l.logBufferLock.Unlock()
l.bufferLock.Unlock()
// process the elements from latest to oldest
for i := len(entries) - 1; i >= 0; i-- {

162
querylog/qlog_http.go Normal file
View File

@ -0,0 +1,162 @@
package querylog
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/AdguardTeam/golibs/log"
"github.com/miekg/dns"
)
func httpError(r *http.Request, w http.ResponseWriter, code int, format string, args ...interface{}) {
text := fmt.Sprintf(format, args...)
log.Info("QueryLog: %s %s: %s", r.Method, r.URL, text)
http.Error(w, text, code)
}
type filterJSON struct {
Domain string `json:"domain"`
Client string `json:"client"`
QuestionType string `json:"question_type"`
ResponseStatus string `json:"response_status"`
}
type request struct {
OlderThan string `json:"older_than"`
Filter filterJSON `json:"filter"`
}
// "value" -> value, return TRUE
func getDoubleQuotesEnclosedValue(s *string) bool {
t := *s
if len(t) >= 2 && t[0] == '"' && t[len(t)-1] == '"' {
*s = t[1 : len(t)-1]
return true
}
return false
}
func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
req := request{}
err := json.NewDecoder(r.Body).Decode(&req)
if err != nil {
httpError(r, w, http.StatusBadRequest, "json decode: %s", err)
return
}
params := getDataParams{
Domain: req.Filter.Domain,
Client: req.Filter.Client,
ResponseStatus: responseStatusAll,
}
if len(req.OlderThan) != 0 {
params.OlderThan, err = time.Parse(time.RFC3339Nano, req.OlderThan)
if err != nil {
httpError(r, w, http.StatusBadRequest, "invalid time stamp: %s", err)
return
}
}
if getDoubleQuotesEnclosedValue(&params.Domain) {
params.StrictMatchDomain = true
}
if getDoubleQuotesEnclosedValue(&params.Client) {
params.StrictMatchClient = true
}
if len(req.Filter.QuestionType) != 0 {
qtype, ok := dns.StringToType[req.Filter.QuestionType]
if !ok {
httpError(r, w, http.StatusBadRequest, "invalid question_type")
return
}
params.QuestionType = qtype
}
if len(req.Filter.ResponseStatus) != 0 {
switch req.Filter.ResponseStatus {
case "filtered":
params.ResponseStatus = responseStatusFiltered
default:
httpError(r, w, http.StatusBadRequest, "invalid response_status")
return
}
}
data := l.getData(params)
jsonVal, err := json.Marshal(data)
if err != nil {
httpError(r, w, http.StatusInternalServerError, "Couldn't marshal data into json: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(r, w, http.StatusInternalServerError, "Unable to write response json: %s", err)
}
}
func (l *queryLog) handleQueryLogClear(w http.ResponseWriter, r *http.Request) {
l.clear()
}
type qlogConfig struct {
Enabled bool `json:"enabled"`
Interval uint32 `json:"interval"`
}
// Get configuration
func (l *queryLog) handleQueryLogInfo(w http.ResponseWriter, r *http.Request) {
resp := qlogConfig{}
resp.Enabled = l.conf.Enabled
resp.Interval = l.conf.Interval
jsonVal, err := json.Marshal(resp)
if err != nil {
httpError(r, w, http.StatusInternalServerError, "json encode: %s", err)
return
}
w.Header().Set("Content-Type", "application/json")
_, err = w.Write(jsonVal)
if err != nil {
httpError(r, w, http.StatusInternalServerError, "http write: %s", err)
}
}
// Set configuration
func (l *queryLog) handleQueryLogConfig(w http.ResponseWriter, r *http.Request) {
reqData := qlogConfig{}
err := json.NewDecoder(r.Body).Decode(&reqData)
if err != nil {
httpError(r, w, http.StatusBadRequest, "json decode: %s", err)
return
}
if !checkInterval(reqData.Interval) {
httpError(r, w, http.StatusBadRequest, "Unsupported interval")
return
}
conf := Config{
Enabled: reqData.Enabled,
Interval: reqData.Interval,
}
l.configure(conf)
l.conf.ConfigModified()
}
// Register web handlers
func (l *queryLog) initWeb() {
l.conf.HTTPRegister("POST", "/control/querylog", l.handleQueryLog)
l.conf.HTTPRegister("GET", "/control/querylog_info", l.handleQueryLogInfo)
l.conf.HTTPRegister("POST", "/control/querylog_clear", l.handleQueryLogClear)
l.conf.HTTPRegister("POST", "/control/querylog_config", l.handleQueryLogConfig)
}

47
querylog/querylog.go
View File

@ -2,58 +2,45 @@ package querylog
import (
"net"
"net/http"
"time"
"github.com/AdguardTeam/AdGuardHome/dnsfilter"
"github.com/miekg/dns"
)
// DiskConfig - configuration settings that are stored on disk
type DiskConfig struct {
Enabled bool
Interval uint32
}
// QueryLog - main interface
type QueryLog interface {
// Close query log object
Close()
// Set new configuration at runtime
// Currently only 'Interval' field is supported.
Configure(conf Config)
// Add a log entry
Add(question *dns.Msg, answer *dns.Msg, result *dnsfilter.Result, elapsed time.Duration, addr net.Addr, upstream string)
// Get log entries
GetData(params GetDataParams) []map[string]interface{}
// Clear memory buffer and remove log files
Clear()
// WriteDiskConfig - write configuration
WriteDiskConfig(dc *DiskConfig)
}
// Config - configuration object
type Config struct {
Enabled bool
BaseDir string // directory where log file is stored
Interval uint32 // interval to rotate logs (in hours)
Interval uint32 // interval to rotate logs (in days)
// Called when the configuration is changed by HTTP request
ConfigModified func()
// Register an HTTP handler
HTTPRegister func(string, string, func(http.ResponseWriter, *http.Request))
}
// New - create a new instance of the query log
func New(conf Config) QueryLog {
return newQueryLog(conf)
}
// GetDataParams - parameters for GetData()
type GetDataParams struct {
OlderThan time.Time // return entries that are older than this value
Domain string // filter by domain name in question
Client string // filter by client IP
QuestionType uint16 // filter by question type
ResponseStatus ResponseStatusType // filter by response status
StrictMatchDomain bool // if Domain value must be matched strictly
StrictMatchClient bool // if Client value must be matched strictly
}
// ResponseStatusType - response status
type ResponseStatusType int32
// Response status constants
const (
ResponseStatusAll ResponseStatusType = iota + 1
ResponseStatusFiltered
)

31
querylog/querylog_file.go
View File

@ -7,17 +7,12 @@ import (
"fmt"
"io"
"os"
"sync"
"time"
"github.com/AdguardTeam/golibs/log"
"github.com/go-test/deep"
)
var (
fileWriteLock sync.Mutex
)
const enableGzip = false
const maxEntrySize = 1000
@ -27,16 +22,16 @@ func (l *queryLog) flushLogBuffer(fullFlush bool) error {
defer l.fileFlushLock.Unlock()
// flush remainder to file
l.logBufferLock.Lock()
needFlush := len(l.logBuffer) >= logBufferCap
l.bufferLock.Lock()
needFlush := len(l.buffer) >= logBufferCap
if !needFlush && !fullFlush {
l.logBufferLock.Unlock()
l.bufferLock.Unlock()
return nil
}
flushBuffer := l.logBuffer
l.logBuffer = nil
flushBuffer := l.buffer
l.buffer = nil
l.flushPending = false
l.logBufferLock.Unlock()
l.bufferLock.Unlock()
err := l.flushToFile(flushBuffer)
if err != nil {
log.Error("Saving querylog to file failed: %s", err)
@ -98,8 +93,8 @@ func (l *queryLog) flushToFile(buffer []*logEntry) error {
zb = b
}
fileWriteLock.Lock()
defer fileWriteLock.Unlock()
l.fileWriteLock.Lock()
defer l.fileWriteLock.Unlock()
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Error("failed to create file \"%s\": %s", filename, err)
@ -146,7 +141,7 @@ func checkBuffer(buffer []*logEntry, b bytes.Buffer) error {
return nil
}
func (l *queryLog) rotateQueryLog() error {
func (l *queryLog) rotate() error {
from := l.logFile
to := l.logFile + ".1"
@ -171,9 +166,9 @@ func (l *queryLog) rotateQueryLog() error {
return nil
}
func (l *queryLog) periodicQueryLogRotate() {
for range time.Tick(time.Duration(l.conf.Interval) * time.Hour) {
err := l.rotateQueryLog()
func (l *queryLog) periodicRotate() {
for range time.Tick(time.Duration(l.conf.Interval) * 24 * time.Hour) {
err := l.rotate()
if err != nil {
log.Error("Failed to rotate querylog: %s", err)
// do nothing, continue rotating
@ -219,7 +214,7 @@ func (l *queryLog) OpenReader() *Reader {
r := Reader{}
r.ql = l
r.now = time.Now()
r.validFrom = r.now.Unix() - int64(l.conf.Interval*60*60)
r.validFrom = r.now.Unix() - int64(l.conf.Interval*24*60*60)
r.validFrom *= 1000000000
r.files = []string{
r.ql.logFile,

7
querylog/querylog_test.go
View File

@ -12,9 +12,10 @@ import (
func TestQueryLog(t *testing.T) {
conf := Config{
Enabled: true,
Interval: 1,
}
l := New(conf)
l := newQueryLog(conf)
q := dns.Msg{}
q.Question = append(q.Question, dns.Question{
@ -37,10 +38,10 @@ func TestQueryLog(t *testing.T) {
res := dnsfilter.Result{}
l.Add(&q, &a, &res, 0, nil, "upstream")
params := GetDataParams{
params := getDataParams{
OlderThan: time.Now(),
}
d := l.GetData(params)
d := l.getData(params)
m := d[0]
mq := m["question"].(map[string]interface{})
assert.True(t, mq["host"].(string) == "example.org")