From 234ab23557f7c0dc8da4c76159d2d1e915d3ce3c Mon Sep 17 00:00:00 2001
From: Andrey Meshkov <am@adguard.com>
Date: Fri, 26 Oct 2018 14:13:31 +0300
Subject: [PATCH 1/9] Fix gitignore

---
 .gitignore                               | 1 +
 client/src/helpers/trackers/adguard.json | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 4f695bdf..86a6ee69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .DS_Store
 .vscode
+.idea
 debug
 /AdGuardHome
 /AdGuardHome.yaml
diff --git a/client/src/helpers/trackers/adguard.json b/client/src/helpers/trackers/adguard.json
index 1272827e..1ae36e5b 100644
--- a/client/src/helpers/trackers/adguard.json
+++ b/client/src/helpers/trackers/adguard.json
@@ -61,6 +61,11 @@
             "name": "Branch.io",
             "categoryId": 101,
             "url": "https://branch.io/"
+        },
+        "adocean": {
+            "name": "Gemius Adocean",
+            "categoryId": 4,
+            "url": "https://adocean-global.com/"
         }
     },
     "trackerDomains": {
@@ -72,6 +77,7 @@
         "appsflyer.com": "appsflyer",
         "appmetrica.yandex.com": "yandex_appmetrica",
         "adjust.com": "adjust",
-        "mobileapptracking.com": "branch"
+        "mobileapptracking.com": "branch",
+        "adocean.cz": "adocean"
     }
 }
\ No newline at end of file

From 90ed48e9fbc140ef3ab09f4ae006a4b4d433e518 Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Sat, 27 Oct 2018 21:17:48 +0300
Subject: [PATCH 2/9] Fix graph corruption on Edge browser

Closes #433
---
 client/src/components/ui/Line.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/client/src/components/ui/Line.js b/client/src/components/ui/Line.js
index af8fe982..5e94d35b 100644
--- a/client/src/components/ui/Line.js
+++ b/client/src/components/ui/Line.js
@@ -19,11 +19,11 @@ const Line = props => (
             curve='linear'
             axisBottom={{
                 tickSize: 0,
-                tickPadding: 0,
+                tickPadding: 10,
             }}
             axisLeft={{
                 tickSize: 0,
-                tickPadding: 0,
+                tickPadding: 10,
             }}
             enableGridX={false}
             enableGridY={false}

From 3cd3b93511033d9948ddaf5b6ed8281537634994 Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Mon, 29 Oct 2018 13:12:04 +0300
Subject: [PATCH 3/9] Fix IP address sorting

Closes #437
---
 client/src/components/Dashboard/Clients.js | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/client/src/components/Dashboard/Clients.js b/client/src/components/Dashboard/Clients.js
index 633ef08b..57d97e7f 100644
--- a/client/src/components/Dashboard/Clients.js
+++ b/client/src/components/Dashboard/Clients.js
@@ -23,6 +23,22 @@ class Clients extends Component {
         Header: 'IP',
         accessor: 'ip',
         Cell: ({ value }) => (<div className="logs__row logs__row--overflow"><span className="logs__text" title={value}>{value}</span></div>),
+        sortMethod: (a, b) => {
+            const nextValue = a.split('.');
+            const prevValue = b.split('.');
+
+            for (let i = 0; i < nextValue.length; i += 1) {
+                const nextNumber = parseInt(nextValue[i], 10);
+                const prevNumber = parseInt(prevValue[i], 10);
+
+                if (nextNumber < prevNumber) {
+                    return -1;
+                } else if (nextNumber > prevNumber) {
+                    return 1;
+                }
+            }
+            return 0;
+        },
     }, {
         Header: 'Requests count',
         accessor: 'count',

From c427034e27e888dab33e635926e6f828b752e265 Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Mon, 29 Oct 2018 13:13:32 +0300
Subject: [PATCH 4/9] Fix tooltips over-lapping

Closes #439
---
 client/src/components/Dashboard/BlockedDomains.js | 4 ++--
 client/src/components/Dashboard/QueriedDomains.js | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/src/components/Dashboard/BlockedDomains.js b/client/src/components/Dashboard/BlockedDomains.js
index 707007a7..d2a81029 100644
--- a/client/src/components/Dashboard/BlockedDomains.js
+++ b/client/src/components/Dashboard/BlockedDomains.js
@@ -20,8 +20,8 @@ class BlockedDomains extends Component {
             const trackerData = getTrackerData(value);
 
             return (
-                <div className="logs__row" title={value}>
-                    <div className="logs__text">
+                <div className="logs__row">
+                    <div className="logs__text" title={value}>
                         {value}
                     </div>
                     {trackerData && <Popover data={trackerData} />}
diff --git a/client/src/components/Dashboard/QueriedDomains.js b/client/src/components/Dashboard/QueriedDomains.js
index 045ceb6c..7ffc2555 100644
--- a/client/src/components/Dashboard/QueriedDomains.js
+++ b/client/src/components/Dashboard/QueriedDomains.js
@@ -29,8 +29,8 @@ class QueriedDomains extends Component {
             const trackerData = getTrackerData(value);
 
             return (
-                <div className="logs__row" title={value}>
-                    <div className="logs__text">
+                <div className="logs__row">
+                    <div className="logs__text" title={value}>
                         {value}
                     </div>
                     {trackerData && <Popover data={trackerData} />}

From 9173b0ee7a94527870e9b35478e03d6030c26846 Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Mon, 29 Oct 2018 14:38:01 +0300
Subject: [PATCH 5/9] Move tiny-version-compare module to helpers

---
 client/package-lock.json             | 11 ++---
 client/package.json                  |  1 -
 client/src/helpers/versionCompare.js | 63 ++++++++++++++++++++++++++++
 client/src/reducers/index.js         |  2 +-
 4 files changed, 67 insertions(+), 10 deletions(-)
 create mode 100644 client/src/helpers/versionCompare.js

diff --git a/client/package-lock.json b/client/package-lock.json
index 1e9b0f5d..a4170f6e 100644
--- a/client/package-lock.json
+++ b/client/package-lock.json
@@ -6551,7 +6551,7 @@
     },
     "html-webpack-plugin": {
       "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
+      "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
       "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=",
       "dev": true,
       "requires": {
@@ -6601,7 +6601,7 @@
         },
         "readable-stream": {
           "version": "1.0.34",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
+          "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
           "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
           "dev": true,
           "requires": {
@@ -14930,7 +14930,7 @@
     },
     "through": {
       "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
+      "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
       "dev": true
     },
@@ -14965,11 +14965,6 @@
         "setimmediate": "^1.0.4"
       }
     },
-    "tiny-version-compare": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/tiny-version-compare/-/tiny-version-compare-0.9.1.tgz",
-      "integrity": "sha512-kYim94l7ptSmj9rqxUMkrcMCJ448CS+hwqjA7OFcRi0ISdi0zjgdSUklQ4velVVECCjCo5frU3tNZ3oSgIKzsA=="
-    },
     "tmp": {
       "version": "0.0.33",
       "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
diff --git a/client/package.json b/client/package.json
index 38e18a30..9e3a16e8 100644
--- a/client/package.json
+++ b/client/package.json
@@ -30,7 +30,6 @@
     "redux-actions": "^2.4.0",
     "redux-thunk": "^2.3.0",
     "svg-url-loader": "^2.3.2",
-    "tiny-version-compare": "^0.9.1",
     "whatwg-fetch": "2.0.3"
   },
   "devDependencies": {
diff --git a/client/src/helpers/versionCompare.js b/client/src/helpers/versionCompare.js
new file mode 100644
index 00000000..47302d6f
--- /dev/null
+++ b/client/src/helpers/versionCompare.js
@@ -0,0 +1,63 @@
+/*
+* Project: tiny-version-compare https://github.com/bfred-it/tiny-version-compare
+* License (MIT) https://github.com/bfred-it/tiny-version-compare/blob/master/LICENSE
+*/
+const split = v => String(v).replace(/^[vr]/, '') // Drop initial 'v' or 'r'
+    .replace(/([a-z]+)/gi, '.$1.') // Sort each word separately
+    .replace(/[-.]+/g, '.') // Consider dashes as separators (+ trim multiple separators)
+    .split('.');
+
+// Development versions are considered "negative",
+// but localeCompare doesn't handle negative numbers.
+// This offset is applied to reset the lowest development version to 0
+const offset = (part) => {
+    // Not numeric, return as is
+    if (Number.isNaN(part)) {
+        return part;
+    }
+    return 5 + Number(part);
+};
+
+const parsePart = (part) => {
+    // Missing, consider it zero
+    if (typeof part === 'undefined') {
+        return 0;
+    }
+    // Sort development versions
+    switch (part.toLowerCase()) {
+        case 'dev':
+            return -5;
+        case 'alpha':
+            return -4;
+        case 'beta':
+            return -3;
+        case 'rc':
+            return -2;
+        case 'pre':
+            return -1;
+        default:
+    }
+    // Return as is, it’s either a plain number or text that will be sorted alphabetically
+    return part;
+};
+
+const versionCompare = (prev, next) => {
+    const a = split(prev);
+    const b = split(next);
+    for (let i = 0; i < a.length || i < b.length; i += 1) {
+        const ai = offset(parsePart(a[i]));
+        const bi = offset(parsePart(b[i]));
+        const sort = String(ai).localeCompare(bi, 'en', {
+            numeric: true,
+        });
+        // Once the difference is found,
+        // stop comparing the rest of the parts
+        if (sort !== 0) {
+            return sort;
+        }
+    }
+    // No difference found
+    return 0;
+};
+
+export default versionCompare;
diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js
index 9fbd8f73..2e32be98 100644
--- a/client/src/reducers/index.js
+++ b/client/src/reducers/index.js
@@ -1,8 +1,8 @@
 import { combineReducers } from 'redux';
 import { handleActions } from 'redux-actions';
 import { loadingBarReducer } from 'react-redux-loading-bar';
-import versionCompare from 'tiny-version-compare';
 import nanoid from 'nanoid';
+import versionCompare from '../helpers/versionCompare';
 
 import * as actions from '../actions';
 

From c39831abbcc3942bc7af9f8fd6c816b96770ee3e Mon Sep 17 00:00:00 2001
From: Ildar Kamalov <i.kamalov@adguard.com>
Date: Mon, 29 Oct 2018 15:24:35 +0300
Subject: [PATCH 6/9] Fix sorting method

---
 client/src/components/Dashboard/Clients.js | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/client/src/components/Dashboard/Clients.js b/client/src/components/Dashboard/Clients.js
index 57d97e7f..a9fa4792 100644
--- a/client/src/components/Dashboard/Clients.js
+++ b/client/src/components/Dashboard/Clients.js
@@ -23,22 +23,7 @@ class Clients extends Component {
         Header: 'IP',
         accessor: 'ip',
         Cell: ({ value }) => (<div className="logs__row logs__row--overflow"><span className="logs__text" title={value}>{value}</span></div>),
-        sortMethod: (a, b) => {
-            const nextValue = a.split('.');
-            const prevValue = b.split('.');
-
-            for (let i = 0; i < nextValue.length; i += 1) {
-                const nextNumber = parseInt(nextValue[i], 10);
-                const prevNumber = parseInt(prevValue[i], 10);
-
-                if (nextNumber < prevNumber) {
-                    return -1;
-                } else if (nextNumber > prevNumber) {
-                    return 1;
-                }
-            }
-            return 0;
-        },
+        sortMethod: (a, b) => parseInt(a.replace(/\./g, ''), 10) - parseInt(b.replace(/\./g, ''), 10),
     }, {
         Header: 'Requests count',
         accessor: 'count',

From 2b2a797cf79c49e4b3b2e9234a0a06475677b1ee Mon Sep 17 00:00:00 2001
From: Andrey Meshkov <am@adguard.com>
Date: Mon, 29 Oct 2018 15:46:58 +0300
Subject: [PATCH 7/9] Moved hosts-syntax matching to DnsFilter

---
 .gitignore                            |  11 ++-
 coredns_plugin/coredns_plugin.go      |  69 ++++-----------
 coredns_plugin/coredns_plugin_test.go |  35 --------
 dnsfilter/dnsfilter.go                | 120 +++++++++++++++++++++++---
 dnsfilter/dnsfilter_test.go           |  28 ++++++
 dnsfilter/helpers.go                  |   6 ++
 6 files changed, 166 insertions(+), 103 deletions(-)

diff --git a/.gitignore b/.gitignore
index 86a6ee69..a863c779 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,13 @@ debug
 /Corefile
 /dnsfilter.txt
 /querylog.json
-/querylog.json.1
\ No newline at end of file
+/querylog.json.1
+
+# Test output
+dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage1.pprof
+dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage2.pprof
+dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage3.pprof
+dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage1.pprof
+dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage2.pprof
+dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage3.pprof
+tests/top-1m.csv
diff --git a/coredns_plugin/coredns_plugin.go b/coredns_plugin/coredns_plugin.go
index 8bd8b74c..d12031ae 100644
--- a/coredns_plugin/coredns_plugin.go
+++ b/coredns_plugin/coredns_plugin.go
@@ -62,7 +62,6 @@ type plug struct {
 	d        *dnsfilter.Dnsfilter
 	Next     plugin.Handler
 	upstream upstream.Upstream
-	hosts    map[string]net.IP
 	settings plugSettings
 
 	sync.RWMutex
@@ -82,7 +81,6 @@ func setupPlugin(c *caddy.Controller) (*plug, error) {
 	p := &plug{
 		settings: defaultPluginSettings,
 		d:        dnsfilter.New(),
-		hosts:    make(map[string]net.IP),
 	}
 
 	filterFileNames := []string{}
@@ -149,9 +147,7 @@ func setupPlugin(c *caddy.Controller) (*plug, error) {
 		scanner := bufio.NewScanner(file)
 		for scanner.Scan() {
 			text := scanner.Text()
-			if p.parseEtcHosts(text) {
-				continue
-			}
+
 			err = p.d.AddRule(text, uint32(i))
 			if err == dnsfilter.ErrInvalidSyntax {
 				continue
@@ -231,28 +227,6 @@ func setup(c *caddy.Controller) error {
 	return nil
 }
 
-func (p *plug) parseEtcHosts(text string) bool {
-	if pos := strings.IndexByte(text, '#'); pos != -1 {
-		text = text[0:pos]
-	}
-	fields := strings.Fields(text)
-	if len(fields) < 2 {
-		return false
-	}
-	addr := net.ParseIP(fields[0])
-	if addr == nil {
-		return false
-	}
-	for _, host := range fields[1:] {
-		// debug logging for duplicate values, pretty common if you subscribe to many hosts files
-		// if val, ok := p.hosts[host]; ok {
-		// 	log.Printf("warning: host %s already has value %s, will overwrite it with %s", host, val, addr)
-		// }
-		p.hosts[host] = addr
-	}
-	return true
-}
-
 func (p *plug) onShutdown() error {
 	p.Lock()
 	p.d.Destroy()
@@ -432,27 +406,6 @@ func (p *plug) serveDNSInternal(ctx context.Context, w dns.ResponseWriter, r *dn
 		}
 		p.RUnlock()
 
-		// is it in hosts?
-		if val, ok := p.hosts[host]; ok {
-			// it is, if it's a loopback host, reply with NXDOMAIN
-			// TODO: research if it's better than 127.0.0.1
-			if false && val.IsLoopback() {
-				rcode, err := p.writeNXdomain(ctx, w, r)
-				if err != nil {
-					return rcode, dnsfilter.Result{}, err
-				}
-				return rcode, dnsfilter.Result{Reason: dnsfilter.FilteredInvalid}, err
-			}
-			// it's not a loopback host, replace it with value specified
-			rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, val.String(), question)
-			if err != nil {
-				return rcode, dnsfilter.Result{}, err
-			}
-			// TODO: This must be handled in the dnsfilter and not here!
-			rule := val.String() + " " + host
-			return rcode, dnsfilter.Result{Reason: dnsfilter.FilteredBlackList, Rule: rule}, err
-		}
-
 		// needs to be filtered instead
 		p.RLock()
 		result, err := p.d.CheckHost(host)
@@ -482,12 +435,22 @@ func (p *plug) serveDNSInternal(ctx context.Context, w dns.ResponseWriter, r *dn
 				}
 				return rcode, result, err
 			case dnsfilter.FilteredBlackList:
-				// return NXdomain
-				rcode, err := p.writeNXdomain(ctx, w, r)
-				if err != nil {
-					return rcode, dnsfilter.Result{}, err
+
+				if result.Ip == nil {
+					// return NXDomain
+					rcode, err := p.writeNXdomain(ctx, w, r)
+					if err != nil {
+						return rcode, dnsfilter.Result{}, err
+					}
+					return rcode, result, err
+				} else {
+					// This is a hosts-syntax rule
+					rcode, err := p.replaceHostWithValAndReply(ctx, w, r, host, result.Ip.String(), question)
+					if err != nil {
+						return rcode, dnsfilter.Result{}, err
+					}
+					return rcode, result, err
 				}
-				return rcode, result, err
 			case dnsfilter.FilteredInvalid:
 				// return NXdomain
 				rcode, err := p.writeNXdomain(ctx, w, r)
diff --git a/coredns_plugin/coredns_plugin_test.go b/coredns_plugin/coredns_plugin_test.go
index 17210b63..2f65cf9a 100644
--- a/coredns_plugin/coredns_plugin_test.go
+++ b/coredns_plugin/coredns_plugin_test.go
@@ -40,41 +40,6 @@ func TestSetup(t *testing.T) {
 	}
 }
 
-func TestEtcHostsParse(t *testing.T) {
-	addr := "216.239.38.120"
-	text := []byte(fmt.Sprintf("   %s  google.com www.google.com   # enforce google's safesearch   ", addr))
-	tmpfile, err := ioutil.TempFile("", "")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if _, err = tmpfile.Write(text); err != nil {
-		t.Fatal(err)
-	}
-	if err = tmpfile.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	defer os.Remove(tmpfile.Name())
-
-	c := caddy.NewTestController("dns", fmt.Sprintf("dnsfilter %s", tmpfile.Name()))
-	p, err := setupPlugin(c)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if len(p.hosts) != 2 {
-		t.Fatal("Expected p.hosts to have two keys")
-	}
-
-	val, ok := p.hosts["google.com"]
-	if !ok {
-		t.Fatal("Expected google.com to be set in p.hosts")
-	}
-	if !val.Equal(net.ParseIP(addr)) {
-		t.Fatalf("Expected google.com's value %s to match %s", val, addr)
-	}
-}
-
 func TestEtcHostsFilter(t *testing.T) {
 	text := []byte("127.0.0.1 doubleclick.net\n" + "127.0.0.1 example.org example.net www.example.org www.example.net")
 	tmpfile, err := ioutil.TempFile("", "")
diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go
index 3b3627f8..49ee721c 100644
--- a/dnsfilter/dnsfilter.go
+++ b/dnsfilter/dnsfilter.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"net/http"
 	"regexp"
 	"strings"
@@ -56,6 +57,7 @@ type rule struct {
 	text         string // text without @@ decorators or $ options
 	shortcut     string // for speeding up lookup
 	originalText string // original text for reporting back to applications
+	ip           net.IP // IP address (for the case when we're matching a hosts file)
 
 	// options
 	options []string // optional options after $
@@ -94,7 +96,7 @@ type Stats struct {
 
 // Dnsfilter holds added rules and performs hostname matches against the rules
 type Dnsfilter struct {
-	storage      map[string]*rule // rule storage, not used for matching, needs to be key->value
+	storage      map[string]bool // rule storage, not used for matching, just for filtering out duplicates
 	storageMutex sync.RWMutex
 
 	// rules are checked against these lists in the order defined here
@@ -121,11 +123,11 @@ const (
 	NotFilteredError                   // there was a transitive error during check
 
 	// reasons for filtering
-	FilteredBlackList    // the host was matched to be advertising host
-	FilteredSafeBrowsing // the host was matched to be malicious/phishing
-	FilteredParental     // the host was matched to be outside of parental control settings
-	FilteredInvalid      // the request was invalid and was not processed
-	FilteredSafeSearch   // the host was replaced with safesearch variant
+	FilteredBlackList     // the host was matched to be advertising host
+	FilteredSafeBrowsing  // the host was matched to be malicious/phishing
+	FilteredParental      // the host was matched to be outside of parental control settings
+	FilteredInvalid       // the request was invalid and was not processed
+	FilteredSafeSearch    // the host was replaced with safesearch variant
 )
 
 // these variables need to survive coredns reload
@@ -137,9 +139,11 @@ var (
 
 // Result holds state of hostname check
 type Result struct {
-	IsFiltered bool   `json:",omitempty"`
-	Reason     Reason `json:",omitempty"`
-	Rule       string `json:",omitempty"`
+	IsFiltered bool   `json:",omitempty"` // True if the host name is filtered
+	Reason     Reason `json:",omitempty"` // Reason for blocking / unblocking
+	Rule       string `json:",omitempty"` // Original rule text
+	Ip         net.IP `json:",omitempty"` // Not nil only in the case of a hosts file syntax
+	FilterID   uint32 `json:",omitempty"` // Filter ID the rule belongs to
 }
 
 // Matched can be used to see if any match at all was found, no matter filtered or not
@@ -199,6 +203,7 @@ func (d *Dnsfilter) CheckHost(host string) (Result, error) {
 //
 
 type rulesTable struct {
+	rulesByHost     map[string]*rule
 	rulesByShortcut map[string][]*rule
 	rulesLeftovers  []*rule
 	sync.RWMutex
@@ -206,6 +211,7 @@ type rulesTable struct {
 
 func newRulesTable() *rulesTable {
 	return &rulesTable{
+		rulesByHost:     make(map[string]*rule),
 		rulesByShortcut: make(map[string][]*rule),
 		rulesLeftovers:  make([]*rule, 0),
 	}
@@ -213,16 +219,27 @@ func newRulesTable() *rulesTable {
 
 func (r *rulesTable) Add(rule *rule) {
 	r.Lock()
-	if len(rule.shortcut) == shortcutLength && enableFastLookup {
+
+	if rule.ip != nil {
+		// Hosts syntax
+		r.rulesByHost[rule.text] = rule
+	} else if //noinspection GoBoolExpressions
+	len(rule.shortcut) == shortcutLength && enableFastLookup {
+
+		// Adblock syntax with a shortcut
 		r.rulesByShortcut[rule.shortcut] = append(r.rulesByShortcut[rule.shortcut], rule)
 	} else {
+
+		// Adblock syntax -- too short to have a shortcut
 		r.rulesLeftovers = append(r.rulesLeftovers, rule)
 	}
 	r.Unlock()
 }
 
 func (r *rulesTable) matchByHost(host string) (Result, error) {
-	res, err := r.searchShortcuts(host)
+
+	// First: examine the hosts-syntax rules
+	res, err := r.searchByHost(host)
 	if err != nil {
 		return res, err
 	}
@@ -230,6 +247,16 @@ func (r *rulesTable) matchByHost(host string) (Result, error) {
 		return res, nil
 	}
 
+	// Second: examine the adblock-syntax rules with shortcuts
+	res, err = r.searchShortcuts(host)
+	if err != nil {
+		return res, err
+	}
+	if res.Reason.Matched() {
+		return res, nil
+	}
+
+	// Third: examine the others
 	res, err = r.searchLeftovers(host)
 	if err != nil {
 		return res, err
@@ -241,6 +268,17 @@ func (r *rulesTable) matchByHost(host string) (Result, error) {
 	return Result{}, nil
 }
 
+func (r *rulesTable) searchByHost(host string) (Result, error) {
+
+	rule, ok := r.rulesByHost[host]
+
+	if ok {
+		return rule.match(host)
+	}
+
+	return Result{}, nil
+}
+
 func (r *rulesTable) searchShortcuts(host string) (Result, error) {
 	// check in shortcuts first
 	for i := 0; i < len(host); i++ {
@@ -424,8 +462,21 @@ func (rule *rule) compile() error {
 	return nil
 }
 
+// Checks if the rule matches the specified host and returns a corresponding Result object
 func (rule *rule) match(host string) (Result, error) {
 	res := Result{}
+
+	if rule.ip != nil && rule.text == host {
+		// This is a hosts-syntax rule -- just check that the hostname matches and return the result
+		return Result{
+			IsFiltered: true,
+			Reason:     FilteredBlackList,
+			Rule:       rule.originalText,
+			Ip:         rule.ip,
+			FilterID:   rule.listID,
+		}, nil
+	}
+
 	err := rule.compile()
 	if err != nil {
 		return res, err
@@ -686,6 +737,7 @@ func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
 	d.storageMutex.RUnlock()
 	if exists {
 		// already added
+		// TODO: Change the error type
 		return ErrInvalidSyntax
 	}
 
@@ -693,13 +745,20 @@ func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
 		return ErrInvalidSyntax
 	}
 
+	// First, check if this is a hosts-syntax rule
+	if d.parseEtcHosts(input, filterListID) {
+		// This is a valid hosts-syntax rule, no need for further parsing
+		return nil
+	}
+
+	// Start parsing the rule
 	rule := rule{
 		text:         input, // will be modified
 		originalText: input,
 		listID:       filterListID,
 	}
 
-	// mark rule as whitelist if it starts with @@
+	// Mark rule as whitelist if it starts with @@
 	if strings.HasPrefix(rule.text, "@@") {
 		rule.isWhitelist = true
 		rule.text = rule.text[2:]
@@ -712,6 +771,7 @@ func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
 
 	rule.extractShortcut()
 
+	//noinspection GoBoolExpressions
 	if !enableDelayedCompilation {
 		err := rule.compile()
 		if err != nil {
@@ -727,12 +787,44 @@ func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
 	}
 
 	d.storageMutex.Lock()
-	d.storage[input] = &rule
+	d.storage[input] = true
 	d.storageMutex.Unlock()
 	destination.Add(&rule)
 	return nil
 }
 
+// Parses the hosts-syntax rules. Returns false if the input string is not of hosts-syntax.
+func (d *Dnsfilter) parseEtcHosts(input string, filterListID uint32) bool {
+	// Strip the trailing comment
+	ruleText := input
+	if pos := strings.IndexByte(ruleText, '#'); pos != -1 {
+		ruleText = ruleText[0:pos]
+	}
+	fields := strings.Fields(ruleText)
+	if len(fields) < 2 {
+		return false
+	}
+	addr := net.ParseIP(fields[0])
+	if addr == nil {
+		return false
+	}
+
+	d.storageMutex.Lock()
+	d.storage[input] = true
+	d.storageMutex.Unlock()
+
+	for _, host := range fields[1:] {
+		rule := rule{
+			text:         host,
+			originalText: input,
+			listID:       filterListID,
+			ip:           addr,
+		}
+		d.blackList.Add(&rule)
+	}
+	return true
+}
+
 // matchHost is a low-level way to check only if hostname is filtered by rules, skipping expensive safebrowsing and parental lookups
 func (d *Dnsfilter) matchHost(host string) (Result, error) {
 	lists := []*rulesTable{
@@ -761,7 +853,7 @@ func (d *Dnsfilter) matchHost(host string) (Result, error) {
 func New() *Dnsfilter {
 	d := new(Dnsfilter)
 
-	d.storage = make(map[string]*rule)
+	d.storage = make(map[string]bool)
 	d.important = newRulesTable()
 	d.whiteList = newRulesTable()
 	d.blackList = newRulesTable()
diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go
index 19ae1c77..735d086e 100644
--- a/dnsfilter/dnsfilter_test.go
+++ b/dnsfilter/dnsfilter_test.go
@@ -281,6 +281,20 @@ func (d *Dnsfilter) checkMatch(t *testing.T, hostname string) {
 	}
 }
 
+func (d *Dnsfilter) checkMatchIp(t *testing.T, hostname string, ip string) {
+	t.Helper()
+	ret, err := d.CheckHost(hostname)
+	if err != nil {
+		t.Errorf("Error while matching host %s: %s", hostname, err)
+	}
+	if !ret.IsFiltered {
+		t.Errorf("Expected hostname %s to match", hostname)
+	}
+	if ret.Ip == nil || ret.Ip.String() != ip {
+		t.Errorf("Expected ip %s to match, actual: %v", ip, ret.Ip)
+	}
+}
+
 func (d *Dnsfilter) checkMatchEmpty(t *testing.T, hostname string) {
 	t.Helper()
 	ret, err := d.CheckHost(hostname)
@@ -345,6 +359,20 @@ func TestSanityCheck(t *testing.T) {
 	d.checkAddRuleFail(t, "lkfaojewhoawehfwacoefawr$@#$@3413841384")
 }
 
+func TestEtcHostsMatching(t *testing.T) {
+	d := NewForTest()
+	defer d.Destroy()
+
+	addr := "216.239.38.120"
+	text := fmt.Sprintf("   %s  google.com www.google.com   # enforce google's safesearch   ", addr)
+
+	d.checkAddRule(t, text)
+	d.checkMatchIp(t, "google.com", addr)
+	d.checkMatchIp(t, "www.google.com", addr)
+	d.checkMatchEmpty(t, "subdomain.google.com")
+	d.checkMatchEmpty(t, "example.org")
+}
+
 func TestSuffixMatching1(t *testing.T) {
 	d := NewForTest()
 	defer d.Destroy()
diff --git a/dnsfilter/helpers.go b/dnsfilter/helpers.go
index fb34cff9..68d4ba26 100644
--- a/dnsfilter/helpers.go
+++ b/dnsfilter/helpers.go
@@ -19,11 +19,17 @@ func isValidRule(rule string) bool {
 		return false
 	}
 
+	// Filter out all sorts of cosmetic rules:
+	// https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#cosmetic-rules
 	masks := []string{
 		"##",
 		"#@#",
+		"#?#",
+		"#@?#",
 		"#$#",
 		"#@$#",
+		"#?$#",
+		"#@?$#",
 		"$$",
 		"$@$",
 		"#%#",

From abb51ddb8ab1f1ec15e0ba648ba265fdccbff111 Mon Sep 17 00:00:00 2001
From: Andrey Meshkov <am@adguard.com>
Date: Mon, 29 Oct 2018 16:17:18 +0300
Subject: [PATCH 8/9] Add ErrAlreadyExists

---
 .gitignore                       | 7 +------
 control.go                       | 7 ++++---
 coredns_plugin/coredns_plugin.go | 6 ++++--
 dnsfilter/dnsfilter.go           | 8 +++++---
 dnsfilter/dnsfilter_test.go      | 6 ++++--
 5 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index a863c779..e22df4e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,10 +13,5 @@ debug
 /querylog.json.1
 
 # Test output
-dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage1.pprof
-dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage2.pprof
-dnsfilter/dnsfilter.TestLotsOfRulesLotsOfHostsMemoryUsage3.pprof
-dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage1.pprof
-dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage2.pprof
-dnsfilter/dnsfilter.TestLotsOfRulesMemoryUsage3.pprof
+dnsfilter/dnsfilter.TestLotsOfRules*.pprof
 tests/top-1m.csv
diff --git a/control.go b/control.go
index 91eb8c68..1afd4e24 100644
--- a/control.go
+++ b/control.go
@@ -731,12 +731,13 @@ func (filter *filter) update(now time.Time) (bool, error) {
 			}
 		} else if len(line) != 0 {
 			err = d.AddRule(line, 0)
-			if err == dnsfilter.ErrInvalidSyntax {
+			if err == dnsfilter.ErrAlreadyExists || err == dnsfilter.ErrInvalidSyntax {
 				continue
 			}
 			if err != nil {
-				log.Printf("Couldn't add rule %s: %s", filter.URL, err)
-				return false, err
+				log.Printf("Cannot add rule %s from %s: %s", line, filter.URL, err)
+				// Just ignore invalid rules
+				continue
 			}
 			rulesCount++
 		}
diff --git a/coredns_plugin/coredns_plugin.go b/coredns_plugin/coredns_plugin.go
index d12031ae..31b147cf 100644
--- a/coredns_plugin/coredns_plugin.go
+++ b/coredns_plugin/coredns_plugin.go
@@ -149,11 +149,13 @@ func setupPlugin(c *caddy.Controller) (*plug, error) {
 			text := scanner.Text()
 
 			err = p.d.AddRule(text, uint32(i))
-			if err == dnsfilter.ErrInvalidSyntax {
+			if err == dnsfilter.ErrAlreadyExists || err == dnsfilter.ErrInvalidSyntax {
 				continue
 			}
 			if err != nil {
-				return nil, err
+				log.Printf("Cannot add rule %s: %s", text, err)
+				// Just ignore invalid rules
+				continue
 			}
 			count++
 		}
diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go
index 49ee721c..1eaaeffa 100644
--- a/dnsfilter/dnsfilter.go
+++ b/dnsfilter/dnsfilter.go
@@ -33,9 +33,12 @@ const defaultSafebrowsingURL = "http://%s/safebrowsing-lookup-hash.html?prefixes
 const defaultParentalServer = "pctrl.adguard.com"
 const defaultParentalURL = "http://%s/check-parental-control-hash?prefixes=%s&sensitivity=%d"
 
-// ErrInvalidSyntax is returned by AddRule when rule is invalid
+// ErrInvalidSyntax is returned by AddRule when the rule is invalid
 var ErrInvalidSyntax = errors.New("dnsfilter: invalid rule syntax")
 
+// ErrInvalidSyntax is returned by AddRule when the rule was already added to the filter
+var ErrAlreadyExists = errors.New("dnsfilter: rule was already added")
+
 // ErrInvalidParental is returned by EnableParental when sensitivity is not a valid value
 var ErrInvalidParental = errors.New("dnsfilter: invalid parental sensitivity, must be either 3, 10, 13 or 17")
 
@@ -737,8 +740,7 @@ func (d *Dnsfilter) AddRule(input string, filterListID uint32) error {
 	d.storageMutex.RUnlock()
 	if exists {
 		// already added
-		// TODO: Change the error type
-		return ErrInvalidSyntax
+		return ErrAlreadyExists
 	}
 
 	if !isValidRule(input) {
diff --git a/dnsfilter/dnsfilter_test.go b/dnsfilter/dnsfilter_test.go
index 735d086e..f186ecce 100644
--- a/dnsfilter/dnsfilter_test.go
+++ b/dnsfilter/dnsfilter_test.go
@@ -261,7 +261,7 @@ func (d *Dnsfilter) checkAddRule(t *testing.T, rule string) {
 func (d *Dnsfilter) checkAddRuleFail(t *testing.T, rule string) {
 	t.Helper()
 	err := d.AddRule(rule, 0)
-	if err == ErrInvalidSyntax {
+	if err == ErrInvalidSyntax || err == ErrAlreadyExists {
 		return
 	}
 	if err != nil {
@@ -318,7 +318,7 @@ func loadTestRules(d *Dnsfilter) error {
 	for scanner.Scan() {
 		rule := scanner.Text()
 		err = d.AddRule(rule, 0)
-		if err == ErrInvalidSyntax {
+		if err == ErrInvalidSyntax || err == ErrAlreadyExists {
 			continue
 		}
 		if err != nil {
@@ -724,6 +724,7 @@ func BenchmarkAddRule(b *testing.B) {
 		err := d.AddRule(rule, 0)
 		switch err {
 		case nil:
+		case ErrAlreadyExists: // ignore rules which were already added
 		case ErrInvalidSyntax: // ignore invalid syntax
 		default:
 			b.Fatalf("Error while adding rule %s: %s", rule, err)
@@ -743,6 +744,7 @@ func BenchmarkAddRuleParallel(b *testing.B) {
 		}
 		switch err {
 		case nil:
+		case ErrAlreadyExists: // ignore rules which were already added
 		case ErrInvalidSyntax: // ignore invalid syntax
 		default:
 			b.Fatalf("Error while adding rule %s: %s", rule, err)

From 1e1ce606c552d83daa4272a301d373561182820b Mon Sep 17 00:00:00 2001
From: Andrey Meshkov <am@adguard.com>
Date: Mon, 29 Oct 2018 16:29:30 +0300
Subject: [PATCH 9/9] gofmt instead of goland's format

---
 dnsfilter/dnsfilter.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/dnsfilter/dnsfilter.go b/dnsfilter/dnsfilter.go
index 1eaaeffa..96ba2512 100644
--- a/dnsfilter/dnsfilter.go
+++ b/dnsfilter/dnsfilter.go
@@ -126,11 +126,11 @@ const (
 	NotFilteredError                   // there was a transitive error during check
 
 	// reasons for filtering
-	FilteredBlackList     // the host was matched to be advertising host
-	FilteredSafeBrowsing  // the host was matched to be malicious/phishing
-	FilteredParental      // the host was matched to be outside of parental control settings
-	FilteredInvalid       // the request was invalid and was not processed
-	FilteredSafeSearch    // the host was replaced with safesearch variant
+	FilteredBlackList    // the host was matched to be advertising host
+	FilteredSafeBrowsing // the host was matched to be malicious/phishing
+	FilteredParental     // the host was matched to be outside of parental control settings
+	FilteredInvalid      // the request was invalid and was not processed
+	FilteredSafeSearch   // the host was replaced with safesearch variant
 )
 
 // these variables need to survive coredns reload