Add AdGuard DNS filter support

This commit is contained in:
世界 2024-07-26 08:03:08 +08:00
parent 9dc3bb975a
commit a9209bb3e5
No known key found for this signature in database
GPG Key ID: CD109927C34A63C4
9 changed files with 723 additions and 0 deletions

View File

@ -0,0 +1,88 @@
package main
import (
"io"
"os"
"strings"
"github.com/sagernet/sing-box/cmd/sing-box/internal/convertor/adguard"
"github.com/sagernet/sing-box/common/srs"
"github.com/sagernet/sing-box/log"
"github.com/sagernet/sing-box/option"
E "github.com/sagernet/sing/common/exceptions"
"github.com/spf13/cobra"
)
var (
flagRuleSetConvertType string
flagRuleSetConvertOutput string
)
var commandRuleSetConvert = &cobra.Command{
Use: "convert [source-path]",
Short: "Convert adguard DNS filter to rule-set",
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
err := convertRuleSet(args[0])
if err != nil {
log.Fatal(err)
}
},
}
func init() {
commandRuleSet.AddCommand(commandRuleSetConvert)
commandRuleSetConvert.Flags().StringVarP(&flagRuleSetConvertType, "type", "t", "", "Source type, available: adguard")
commandRuleSetConvert.Flags().StringVarP(&flagRuleSetConvertOutput, "output", "o", flagRuleSetCompileDefaultOutput, "Output file")
}
func convertRuleSet(sourcePath string) error {
var (
reader io.Reader
err error
)
if sourcePath == "stdin" {
reader = os.Stdin
} else {
reader, err = os.Open(sourcePath)
if err != nil {
return err
}
}
var rules []option.HeadlessRule
switch flagRuleSetConvertType {
case "adguard":
rules, err = adguard.Convert(reader)
case "":
return E.New("source type is required")
default:
return E.New("unsupported source type: ", flagRuleSetConvertType)
}
if err != nil {
return err
}
var outputPath string
if flagRuleSetConvertOutput == flagRuleSetCompileDefaultOutput {
if strings.HasSuffix(sourcePath, ".txt") {
outputPath = sourcePath[:len(sourcePath)-4] + ".srs"
} else {
outputPath = sourcePath + ".srs"
}
} else {
outputPath = flagRuleSetConvertOutput
}
outputFile, err := os.Create(outputPath)
if err != nil {
return err
}
defer outputFile.Close()
err = srs.Write(outputFile, option.PlainRuleSet{Rules: rules}, true)
if err != nil {
outputFile.Close()
os.Remove(outputPath)
return err
}
outputFile.Close()
return nil
}

View File

@ -0,0 +1,346 @@
package adguard
import (
"bufio"
"io"
"net/netip"
"os"
"strconv"
"strings"
C "github.com/sagernet/sing-box/constant"
"github.com/sagernet/sing-box/log"
"github.com/sagernet/sing-box/option"
"github.com/sagernet/sing/common"
E "github.com/sagernet/sing/common/exceptions"
M "github.com/sagernet/sing/common/metadata"
)
type agdguardRuleLine struct {
ruleLine string
isRawDomain bool
isExclude bool
isSuffix bool
hasStart bool
hasEnd bool
isRegexp bool
isImportant bool
}
func Convert(reader io.Reader) ([]option.HeadlessRule, error) {
scanner := bufio.NewScanner(reader)
var (
ruleLines []agdguardRuleLine
ignoredLines int
)
parseLine:
for scanner.Scan() {
ruleLine := scanner.Text()
if ruleLine == "" || ruleLine[0] == '!' || ruleLine[0] == '#' {
continue
}
originRuleLine := ruleLine
if M.IsDomainName(ruleLine) {
ruleLines = append(ruleLines, agdguardRuleLine{
ruleLine: ruleLine,
isRawDomain: true,
})
continue
}
hostLine, err := parseAdGuardHostLine(ruleLine)
if err == nil {
if hostLine != "" {
ruleLines = append(ruleLines, agdguardRuleLine{
ruleLine: hostLine,
isRawDomain: true,
hasStart: true,
hasEnd: true,
})
}
continue
}
if strings.HasSuffix(ruleLine, "|") {
ruleLine = ruleLine[:len(ruleLine)-1]
}
var (
isExclude bool
isSuffix bool
hasStart bool
hasEnd bool
isRegexp bool
isImportant bool
)
if !strings.HasPrefix(ruleLine, "/") && strings.Contains(ruleLine, "$") {
params := common.SubstringAfter(ruleLine, "$")
for _, param := range strings.Split(params, ",") {
paramParts := strings.Split(param, "=")
var ignored bool
if len(paramParts) > 0 && len(paramParts) <= 2 {
switch paramParts[0] {
case "app", "network":
// maybe support by package_name/process_name
case "dnstype":
// maybe support by query_type
case "important":
ignored = true
isImportant = true
case "dnsrewrite":
if len(paramParts) == 2 && M.ParseAddr(paramParts[1]).IsUnspecified() {
ignored = true
}
}
}
if !ignored {
ignoredLines++
log.Debug("ignored unsupported rule with modifier: ", paramParts[0], ": ", ruleLine)
continue parseLine
}
}
ruleLine = common.SubstringBefore(ruleLine, "$")
}
if strings.HasPrefix(ruleLine, "@@") {
ruleLine = ruleLine[2:]
isExclude = true
}
if strings.HasSuffix(ruleLine, "|") {
ruleLine = ruleLine[:len(ruleLine)-1]
}
if strings.HasPrefix(ruleLine, "||") {
ruleLine = ruleLine[2:]
isSuffix = true
} else if strings.HasPrefix(ruleLine, "|") {
ruleLine = ruleLine[1:]
hasStart = true
}
if strings.HasSuffix(ruleLine, "^") {
ruleLine = ruleLine[:len(ruleLine)-1]
hasEnd = true
}
if strings.HasPrefix(ruleLine, "/") && strings.HasSuffix(ruleLine, "/") {
ruleLine = ruleLine[1 : len(ruleLine)-1]
if ignoreIPCIDRRegexp(ruleLine) {
ignoredLines++
log.Debug("ignored unsupported rule with IPCIDR regexp: ", ruleLine)
continue
}
isRegexp = true
} else {
if strings.Contains(ruleLine, "://") {
ruleLine = common.SubstringAfter(ruleLine, "://")
}
if strings.Contains(ruleLine, "/") {
ignoredLines++
log.Debug("ignored unsupported rule with path: ", ruleLine)
continue
}
if strings.Contains(ruleLine, "##") {
ignoredLines++
log.Debug("ignored unsupported rule with element hiding: ", ruleLine)
continue
}
if strings.Contains(ruleLine, "#$#") {
ignoredLines++
log.Debug("ignored unsupported rule with element hiding: ", ruleLine)
continue
}
var domainCheck string
if strings.HasPrefix(ruleLine, ".") || strings.HasPrefix(ruleLine, "-") {
domainCheck = "r" + ruleLine
} else {
domainCheck = ruleLine
}
if ruleLine == "" {
ignoredLines++
log.Debug("ignored unsupported rule with empty domain", originRuleLine)
continue
} else {
domainCheck = strings.ReplaceAll(domainCheck, "*", "x")
if !M.IsDomainName(domainCheck) {
_, ipErr := parseADGuardIPCIDRLine(ruleLine)
if ipErr == nil {
ignoredLines++
log.Debug("ignored unsupported rule with IPCIDR: ", ruleLine)
continue
}
if M.ParseSocksaddr(domainCheck).Port != 0 {
log.Debug("ignored unsupported rule with port: ", ruleLine)
} else {
log.Debug("ignored unsupported rule with invalid domain: ", ruleLine)
}
ignoredLines++
continue
}
}
}
ruleLines = append(ruleLines, agdguardRuleLine{
ruleLine: ruleLine,
isExclude: isExclude,
isSuffix: isSuffix,
hasStart: hasStart,
hasEnd: hasEnd,
isRegexp: isRegexp,
isImportant: isImportant,
})
}
if len(ruleLines) == 0 {
return nil, E.New("AdGuard rule-set is empty or all rules are unsupported")
}
if common.All(ruleLines, func(it agdguardRuleLine) bool {
return it.isRawDomain
}) {
return []option.HeadlessRule{
{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
Domain: common.Map(ruleLines, func(it agdguardRuleLine) string {
return it.ruleLine
}),
},
},
}, nil
}
mapDomain := func(it agdguardRuleLine) string {
ruleLine := it.ruleLine
if it.isSuffix {
ruleLine = "||" + ruleLine
} else if it.hasStart {
ruleLine = "|" + ruleLine
}
if it.hasEnd {
ruleLine += "^"
}
return ruleLine
}
importantDomain := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return it.isImportant && !it.isRegexp && !it.isExclude }), mapDomain)
importantDomainRegex := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return it.isImportant && it.isRegexp && !it.isExclude }), mapDomain)
importantExcludeDomain := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return it.isImportant && !it.isRegexp && it.isExclude }), mapDomain)
importantExcludeDomainRegex := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return it.isImportant && it.isRegexp && it.isExclude }), mapDomain)
domain := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return !it.isImportant && !it.isRegexp && !it.isExclude }), mapDomain)
domainRegex := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return !it.isImportant && it.isRegexp && !it.isExclude }), mapDomain)
excludeDomain := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return !it.isImportant && !it.isRegexp && it.isExclude }), mapDomain)
excludeDomainRegex := common.Map(common.Filter(ruleLines, func(it agdguardRuleLine) bool { return !it.isImportant && it.isRegexp && it.isExclude }), mapDomain)
currentRule := option.HeadlessRule{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
AdGuardDomain: domain,
DomainRegex: domainRegex,
},
}
if len(excludeDomain) > 0 || len(excludeDomainRegex) > 0 {
currentRule = option.HeadlessRule{
Type: C.RuleTypeLogical,
LogicalOptions: option.LogicalHeadlessRule{
Mode: C.LogicalTypeAnd,
Rules: []option.HeadlessRule{
{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
AdGuardDomain: excludeDomain,
DomainRegex: excludeDomainRegex,
Invert: true,
},
},
currentRule,
},
},
}
}
if len(importantDomain) > 0 || len(importantDomainRegex) > 0 {
currentRule = option.HeadlessRule{
Type: C.RuleTypeLogical,
LogicalOptions: option.LogicalHeadlessRule{
Mode: C.LogicalTypeOr,
Rules: []option.HeadlessRule{
{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
AdGuardDomain: importantDomain,
DomainRegex: importantDomainRegex,
},
},
currentRule,
},
},
}
}
if len(importantExcludeDomain) > 0 || len(importantExcludeDomainRegex) > 0 {
currentRule = option.HeadlessRule{
Type: C.RuleTypeLogical,
LogicalOptions: option.LogicalHeadlessRule{
Mode: C.LogicalTypeAnd,
Rules: []option.HeadlessRule{
{
Type: C.RuleTypeDefault,
DefaultOptions: option.DefaultHeadlessRule{
AdGuardDomain: importantExcludeDomain,
DomainRegex: importantExcludeDomainRegex,
Invert: true,
},
},
currentRule,
},
},
}
}
log.Info("parsed rules: ", len(ruleLines), "/", len(ruleLines)+ignoredLines)
return []option.HeadlessRule{currentRule}, nil
}
func ignoreIPCIDRRegexp(ruleLine string) bool {
if strings.HasPrefix(ruleLine, "(http?:\\/\\/)") {
ruleLine = ruleLine[12:]
} else if strings.HasPrefix(ruleLine, "(https?:\\/\\/)") {
ruleLine = ruleLine[13:]
} else if strings.HasPrefix(ruleLine, "^") {
ruleLine = ruleLine[1:]
} else {
return false
}
_, parseErr := strconv.ParseUint(common.SubstringBefore(ruleLine, "\\."), 10, 8)
return parseErr == nil
}
func parseAdGuardHostLine(ruleLine string) (string, error) {
idx := strings.Index(ruleLine, " ")
if idx == -1 {
return "", os.ErrInvalid
}
address, err := netip.ParseAddr(ruleLine[:idx])
if err != nil {
return "", err
}
if !address.IsUnspecified() {
return "", nil
}
domain := ruleLine[idx+1:]
if !M.IsDomainName(domain) {
return "", E.New("invalid domain name: ", domain)
}
return domain, nil
}
func parseADGuardIPCIDRLine(ruleLine string) (netip.Prefix, error) {
var isPrefix bool
if strings.HasSuffix(ruleLine, ".") {
isPrefix = true
ruleLine = ruleLine[:len(ruleLine)-1]
}
ruleStringParts := strings.Split(ruleLine, ".")
if len(ruleStringParts) > 4 || len(ruleStringParts) < 4 && !isPrefix {
return netip.Prefix{}, os.ErrInvalid
}
ruleParts := make([]uint8, 0, len(ruleStringParts))
for _, part := range ruleStringParts {
rulePart, err := strconv.ParseUint(part, 10, 8)
if err != nil {
return netip.Prefix{}, err
}
ruleParts = append(ruleParts, uint8(rulePart))
}
bitLen := len(ruleParts) * 8
for len(ruleParts) < 4 {
ruleParts = append(ruleParts, 0)
}
return netip.PrefixFrom(netip.AddrFrom4(*(*[4]byte)(ruleParts)), bitLen), nil
}

View File

@ -0,0 +1,140 @@
package adguard
import (
"strings"
"testing"
"github.com/sagernet/sing-box/adapter"
"github.com/sagernet/sing-box/route"
"github.com/stretchr/testify/require"
)
func TestConverter(t *testing.T) {
t.Parallel()
rules, err := Convert(strings.NewReader(`
||example.org^
|example.com^
example.net^
||example.edu
||example.edu.tw^
|example.gov
example.arpa
@@|sagernet.example.org|
||sagernet.org^$important
@@|sing-box.sagernet.org^$important
`))
require.NoError(t, err)
require.Len(t, rules, 1)
rule, err := route.NewHeadlessRule(nil, rules[0])
require.NoError(t, err)
matchDomain := []string{
"example.org",
"www.example.org",
"example.com",
"example.net",
"isexample.net",
"www.example.net",
"example.edu",
"example.edu.cn",
"example.edu.tw",
"www.example.edu",
"www.example.edu.cn",
"example.gov",
"example.gov.cn",
"example.arpa",
"www.example.arpa",
"isexample.arpa",
"example.arpa.cn",
"www.example.arpa.cn",
"isexample.arpa.cn",
"sagernet.org",
"www.sagernet.org",
}
notMatchDomain := []string{
"example.org.cn",
"notexample.org",
"example.com.cn",
"www.example.com.cn",
"example.net.cn",
"notexample.edu",
"notexample.edu.cn",
"www.example.gov",
"notexample.gov",
"sagernet.example.org",
"sing-box.sagernet.org",
}
for _, domain := range matchDomain {
require.True(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
for _, domain := range notMatchDomain {
require.False(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
}
func TestHosts(t *testing.T) {
t.Parallel()
rules, err := Convert(strings.NewReader(`
127.0.0.1 localhost
::1 localhost #[IPv6]
0.0.0.0 google.com
`))
require.NoError(t, err)
require.Len(t, rules, 1)
rule, err := route.NewHeadlessRule(nil, rules[0])
require.NoError(t, err)
matchDomain := []string{
"google.com",
}
notMatchDomain := []string{
"www.google.com",
"notgoogle.com",
"localhost",
}
for _, domain := range matchDomain {
require.True(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
for _, domain := range notMatchDomain {
require.False(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
}
func TestSimpleHosts(t *testing.T) {
t.Parallel()
rules, err := Convert(strings.NewReader(`
example.com
www.example.org
`))
require.NoError(t, err)
require.Len(t, rules, 1)
rule, err := route.NewHeadlessRule(nil, rules[0])
require.NoError(t, err)
matchDomain := []string{
"example.com",
"www.example.org",
}
notMatchDomain := []string{
"example.com.cn",
"www.example.com",
"notexample.com",
"example.org",
}
for _, domain := range matchDomain {
require.True(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
for _, domain := range notMatchDomain {
require.False(t, rule.Match(&adapter.InboundContext{
Domain: domain,
}), domain)
}
}

View File

@ -36,6 +36,7 @@ const (
ruleItemPackageName
ruleItemWIFISSID
ruleItemWIFIBSSID
ruleItemAdGuardDomain
ruleItemFinal uint8 = 0xFF
)
@ -212,6 +213,17 @@ func readDefaultRule(reader varbin.Reader, recover bool) (rule option.DefaultHea
rule.WIFISSID, err = readRuleItemString(reader)
case ruleItemWIFIBSSID:
rule.WIFIBSSID, err = readRuleItemString(reader)
case ruleItemAdGuardDomain:
if recover {
err = E.New("unable to decompile binary AdGuard rules to rule-set")
return
}
var matcher *domain.AdGuardMatcher
matcher, err = domain.ReadAdGuardMatcher(reader)
if err != nil {
return
}
rule.AdGuardDomainMatcher = matcher
case ruleItemFinal:
err = binary.Read(reader, binary.BigEndian, &rule.Invert)
return
@ -332,6 +344,16 @@ func writeDefaultRule(writer varbin.Writer, rule option.DefaultHeadlessRule, gen
return err
}
}
if len(rule.AdGuardDomain) > 0 {
err = binary.Write(writer, binary.BigEndian, ruleItemAdGuardDomain)
if err != nil {
return err
}
err = domain.NewAdGuardMatcher(rule.AdGuardDomain).Write(writer)
if err != nil {
return err
}
}
err = binary.Write(writer, binary.BigEndian, ruleItemFinal)
if err != nil {
return err

View File

@ -0,0 +1,71 @@
---
icon: material/new-box
---
# AdGuard DNS Filter
!!! question "Since sing-box 1.10.0"
sing-box supports some rule-set formats from other projects which cannot be fully translated to sing-box,
currently only AdGuard DNS Filter.
These formats are not directly supported as source formats,
instead you need to convert them to binary rule-set.
## Convert
Use `sing-box rule-set convert --type adguard [--output <file-name>.srs] <file-name>.txt` to convert to binary rule-set.
## Performance
AdGuard keeps all rules in memory and matches them sequentially,
while sing-box chooses high performance and smaller memory usage.
As a trade-off, you cannot know which rule item is matched.
## Compatibility
Almost all rules in [AdGuardSDNSFilter](https://github.com/AdguardTeam/AdGuardSDNSFilter)
and rules in rule-sets listed in [adguard-filter-list](https://github.com/ppfeufer/adguard-filter-list)
are supported.
## Supported formats
### AdGuard Filter
#### Basic rule syntax
| Syntax | Supported |
|--------|------------------|
| `@@` | :material-check: |
| `\|\|` | :material-check: |
| `\|` | :material-check: |
| `^` | :material-check: |
| `*` | :material-check: |
#### Host syntax
| Syntax | Example | Supported |
|-------------|--------------------------|--------------------------|
| Scheme | `https://` | :material-alert: Ignored |
| Domain Host | `example.org` | :material-check: |
| IP Host | `1.1.1.1`, `10.0.0.` | :material-close: |
| Regexp | `/regexp/` | :material-check: |
| Port | `example.org:80` | :material-close: |
| Path | `example.org/path/ad.js` | :material-close: |
#### Modifier syntax
| Modifier | Supported |
|-----------------------|--------------------------|
| `$important` | :material-check: |
| `$dnsrewrite=0.0.0.0` | :material-alert: Ignored |
| Any other modifiers | :material-close: |
### Hosts
Only items with `0.0.0.0` IP addresses will be accepted.
### Simple
When all rule lines are valid domains, they are treated as simple line-by-line domain rules which,
like hosts, only match the exact same domain.

View File

@ -91,6 +91,7 @@ nav:
- configuration/rule-set/index.md
- Source Format: configuration/rule-set/source-format.md
- Headless Rule: configuration/rule-set/headless-rule.md
- AdGuard DNS Filer: configuration/rule-set/adguard.md
- Experimental:
- configuration/experimental/index.md
- Cache File: configuration/experimental/cache-file.md

View File

@ -166,6 +166,9 @@ type DefaultHeadlessRule struct {
DomainMatcher *domain.Matcher `json:"-"`
SourceIPSet *netipx.IPSet `json:"-"`
IPSet *netipx.IPSet `json:"-"`
AdGuardDomain Listable[string] `json:"-"`
AdGuardDomainMatcher *domain.AdGuardMatcher `json:"-"`
}
func (r DefaultHeadlessRule) IsValid() bool {

View File

@ -142,6 +142,15 @@ func NewDefaultHeadlessRule(router adapter.Router, options option.DefaultHeadles
rule.allItems = append(rule.allItems, item)
}
}
if len(options.AdGuardDomain) > 0 {
item := NewAdGuardDomainItem(options.AdGuardDomain)
rule.destinationAddressItems = append(rule.destinationAddressItems, item)
rule.allItems = append(rule.allItems, item)
} else if options.AdGuardDomainMatcher != nil {
item := NewRawAdGuardDomainItem(options.AdGuardDomainMatcher)
rule.destinationAddressItems = append(rule.destinationAddressItems, item)
rule.allItems = append(rule.allItems, item)
}
return rule, nil
}

View File

@ -0,0 +1,43 @@
package route
import (
"strings"
"github.com/sagernet/sing-box/adapter"
"github.com/sagernet/sing/common/domain"
)
var _ RuleItem = (*AdGuardDomainItem)(nil)
type AdGuardDomainItem struct {
matcher *domain.AdGuardMatcher
}
func NewAdGuardDomainItem(ruleLines []string) *AdGuardDomainItem {
return &AdGuardDomainItem{
domain.NewAdGuardMatcher(ruleLines),
}
}
func NewRawAdGuardDomainItem(matcher *domain.AdGuardMatcher) *AdGuardDomainItem {
return &AdGuardDomainItem{
matcher,
}
}
func (r *AdGuardDomainItem) Match(metadata *adapter.InboundContext) bool {
var domainHost string
if metadata.Domain != "" {
domainHost = metadata.Domain
} else {
domainHost = metadata.Destination.Fqdn
}
if domainHost == "" {
return false
}
return r.matcher.Match(strings.ToLower(domainHost))
}
func (r *AdGuardDomainItem) String() string {
return "!adguard_domain_rules=<binary>"
}