package querylog import ( "encoding/base64" "io" "strconv" "strings" "time" "github.com/AdguardTeam/AdGuardHome/dnsfilter" "github.com/AdguardTeam/AdGuardHome/util" "github.com/AdguardTeam/golibs/log" "github.com/miekg/dns" ) // searchFiles reads log entries from all log files and applies the specified search criteria. // IMPORTANT: this method does not scan more than "maxSearchEntries" so you // may need to call it many times. // // it returns: // * an array of log entries that we have read // * time of the oldest processed entry (even if it was discarded) // * total number of processed entries (including discarded). func (l *queryLog) searchFiles(params getDataParams) ([]*logEntry, time.Time, int) { entries := make([]*logEntry, 0) oldest := time.Time{} r, err := l.openReader() if err != nil { log.Error("Failed to open qlog reader: %v", err) return entries, oldest, 0 } defer r.Close() if params.OlderThan.IsZero() { err = r.SeekStart() } else { err = r.Seek(params.OlderThan.UnixNano()) if err == nil { // Read to the next record right away // The one that was specified in the "oldest" param is not needed, // we need only the one next to it _, err = r.ReadNext() } } if err != nil { log.Debug("Cannot Seek() to %v: %v", params.OlderThan, err) return entries, oldest, 0 } total := 0 oldestNano := int64(0) // Do not scan more than 50k at once for total <= maxSearchEntries { entry, ts, err := l.readNextEntry(r, params) if err == io.EOF { // there's nothing to read anymore break } oldestNano = ts total++ if entry != nil { entries = append(entries, entry) if len(entries) == getDataLimit { // Do not read more than "getDataLimit" records at once break } } } oldest = time.Unix(0, oldestNano) return entries, oldest, total } // readNextEntry - reads the next log entry and checks if it matches the search criteria (getDataParams) // // returns: // * log entry that matches search criteria or null if it was discarded (or if there's nothing to read) // * timestamp of the processed log entry // * error if we can't read anymore func (l *queryLog) readNextEntry(r *QLogReader, params getDataParams) (*logEntry, int64, error) { line, err := r.ReadNext() if err != nil { return nil, 0, err } // Read the log record timestamp right away timestamp := readQLogTimestamp(line) // Quick check without deserializing log entry if !quickMatchesGetDataParams(line, params) { return nil, timestamp, nil } entry := logEntry{} decodeLogEntry(&entry, line) // Full check of the deserialized log entry if !matchesGetDataParams(&entry, params) { return nil, timestamp, nil } return &entry, timestamp, nil } // openReader - opens QLogReader instance func (l *queryLog) openReader() (*QLogReader, error) { files := make([]string, 0) if util.FileExists(l.logFile + ".1") { files = append(files, l.logFile+".1") } if util.FileExists(l.logFile) { files = append(files, l.logFile) } return NewQLogReader(files) } // quickMatchesGetDataParams - quickly checks if the line matches getDataParams // this method does not guarantee anything and the reason is to do a quick check // without deserializing anything func quickMatchesGetDataParams(line string, params getDataParams) bool { if params.ResponseStatus == responseStatusFiltered { boolVal, ok := readJSONBool(line, "IsFiltered") if !ok || !boolVal { return false } } if len(params.Domain) != 0 { val := readJSONValue(line, "QH") if len(val) == 0 { return false } if (params.StrictMatchDomain && val != params.Domain) || (!params.StrictMatchDomain && strings.Index(val, params.Domain) == -1) { return false } } if len(params.QuestionType) != 0 { val := readJSONValue(line, "QT") if val != params.QuestionType { return false } } if len(params.Client) != 0 { val := readJSONValue(line, "IP") if len(val) == 0 { log.Debug("QueryLog: failed to decodeLogEntry") return false } if (params.StrictMatchClient && val != params.Client) || (!params.StrictMatchClient && strings.Index(val, params.Client) == -1) { return false } } return true } // matchesGetDataParams - returns true if the entry matches the search parameters func matchesGetDataParams(entry *logEntry, params getDataParams) bool { if params.ResponseStatus == responseStatusFiltered && !entry.Result.IsFiltered { return false } if len(params.QuestionType) != 0 { if entry.QType != params.QuestionType { return false } } if len(params.Domain) != 0 { if (params.StrictMatchDomain && entry.QHost != params.Domain) || (!params.StrictMatchDomain && strings.Index(entry.QHost, params.Domain) == -1) { return false } } if len(params.Client) != 0 { if (params.StrictMatchClient && entry.IP != params.Client) || (!params.StrictMatchClient && strings.Index(entry.IP, params.Client) == -1) { return false } } return true } // decodeLogEntry - decodes query log entry from a line // nolint (gocyclo) func decodeLogEntry(ent *logEntry, str string) { var b bool var i int var err error for { k, v, t := readJSON(&str) if t == jsonTErr { break } switch k { case "IP": if len(ent.IP) == 0 { ent.IP = v } case "T": ent.Time, err = time.Parse(time.RFC3339, v) case "QH": ent.QHost = v case "QT": ent.QType = v case "QC": ent.QClass = v case "Answer": ent.Answer, err = base64.StdEncoding.DecodeString(v) case "OrigAnswer": ent.OrigAnswer, err = base64.StdEncoding.DecodeString(v) case "IsFiltered": b, err = strconv.ParseBool(v) ent.Result.IsFiltered = b case "Rule": ent.Result.Rule = v case "FilterID": i, err = strconv.Atoi(v) ent.Result.FilterID = int64(i) case "Reason": i, err = strconv.Atoi(v) ent.Result.Reason = dnsfilter.Reason(i) case "Upstream": ent.Upstream = v case "Elapsed": i, err = strconv.Atoi(v) ent.Elapsed = time.Duration(i) // pre-v0.99.3 compatibility: case "Question": var qstr []byte qstr, err = base64.StdEncoding.DecodeString(v) if err != nil { break } q := new(dns.Msg) err = q.Unpack(qstr) if err != nil { break } ent.QHost = q.Question[0].Name if len(ent.QHost) == 0 { break } ent.QHost = ent.QHost[:len(ent.QHost)-1] ent.QType = dns.TypeToString[q.Question[0].Qtype] ent.QClass = dns.ClassToString[q.Question[0].Qclass] case "Time": ent.Time, err = time.Parse(time.RFC3339, v) } if err != nil { log.Debug("decodeLogEntry err: %s", err) break } } } // Get bool value from "key":bool func readJSONBool(s, name string) (bool, bool) { i := strings.Index(s, "\""+name+"\":") if i == -1 { return false, false } start := i + 1 + len(name) + 2 b := false if strings.HasPrefix(s[start:], "true") { b = true } else if !strings.HasPrefix(s[start:], "false") { return false, false } return b, true } // Get value from "key":"value" func readJSONValue(s, name string) string { i := strings.Index(s, "\""+name+"\":\"") if i == -1 { return "" } start := i + 1 + len(name) + 3 i = strings.IndexByte(s[start:], '"') if i == -1 { return "" } end := start + i return s[start:end] } const ( jsonTErr = iota jsonTObj jsonTStr jsonTNum jsonTBool ) // Parse JSON key-value pair // e.g.: "key":VALUE where VALUE is "string", true|false (boolean), or 123.456 (number) // Note the limitations: // . doesn't support whitespace // . doesn't support "null" // . doesn't validate boolean or number // . no proper handling of {} braces // . no handling of [] brackets // Return (key, value, type) func readJSON(ps *string) (string, string, int32) { s := *ps k := "" v := "" t := int32(jsonTErr) q1 := strings.IndexByte(s, '"') if q1 == -1 { return k, v, t } q2 := strings.IndexByte(s[q1+1:], '"') if q2 == -1 { return k, v, t } k = s[q1+1 : q1+1+q2] s = s[q1+1+q2+1:] if len(s) < 2 || s[0] != ':' { return k, v, t } if s[1] == '"' { q2 = strings.IndexByte(s[2:], '"') if q2 == -1 { return k, v, t } v = s[2 : 2+q2] t = jsonTStr s = s[2+q2+1:] } else if s[1] == '{' { t = jsonTObj s = s[1+1:] } else { sep := strings.IndexAny(s[1:], ",}") if sep == -1 { return k, v, t } v = s[1 : 1+sep] if s[1] == 't' || s[1] == 'f' { t = jsonTBool } else if s[1] == '.' || (s[1] >= '0' && s[1] <= '9') { t = jsonTNum } s = s[1+sep+1:] } *ps = s return k, v, t }