diff --git a/README.md b/README.md index 4c9f22a96b..67b564359e 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ Flags: -b, --tcp-buffer="4KB" Size of TCP buffer to use. -i, --prefer-ip="prefer-ipv6" IP preference. By default we prefer IPv6 with fallback to IPv4. -p, --domain-fronting-port=443 A port to access for domain fronting. - -n, --doh-ip=9.9.9.9 IP address of DNS-over-HTTP to use. + -n, --doh-ip=1.1.1.1 IP address of DNS-over-HTTP to use. -t, --timeout=10s Network timeout to use -a, --antireplay-cache-size="1MB" A size of anti-replay cache to use. ``` diff --git a/example.config.toml b/example.config.toml index bdec7b73f6..eb2c62c3db 100644 --- a/example.config.toml +++ b/example.config.toml @@ -71,6 +71,15 @@ tolerate-time-skewness = "5s" # Otherwise, chose a new DC. allow-fallback-on-unknown-dc = false +# Telegram uses different DCs for different purposes. Unfortunately, most of +# DCs are not public, and dependent on a location of the current user, so +# mtg cannot know upfront about all of them, and how to access them. It has +# a default list of DCs, including some CDN IPs, but it is possible that some +# of them are not working for you. In this case, you can override them here. +[[dc-overrides]] +dc = 101 +ips = ["127.0.0.1:443"] + # network defines different network-related settings [network] # please be aware that mtg needs to do some external requests. For @@ -84,8 +93,8 @@ allow-fallback-on-unknown-dc = false # resolver of the operating system and uses DOH instead. This is a host # it has to access. # -# By default we use Quad9. -doh-ip = "9.9.9.9" +# By default we use Cloudflare. +doh-ip = "1.1.1.1" # mtg can work via proxies (for now, we support only socks5). Proxy # configuration is done via list. So, you can specify many proxies diff --git a/go.mod b/go.mod index b9e46e8ca7..b4c34f76f6 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/jarcoal/httpmock v1.0.8 github.com/mccutchen/go-httpbin v1.1.1 github.com/panjf2000/ants/v2 v2.11.5 - github.com/pelletier/go-toml v1.9.5 github.com/prometheus/client_golang v1.23.2 github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect @@ -28,6 +27,7 @@ require ( ) require ( + github.com/pelletier/go-toml/v2 v2.2.4 github.com/txthinking/socks5 v0.0.0-20251011041537-5c31f201a10e github.com/yl2chen/cidranger v1.0.2 ) @@ -36,6 +36,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -43,8 +44,10 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/txthinking/runnergroup v0.0.0-20250224021307-5864ffeb65ae // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/sync v0.19.0 // indirect + golang.org/x/tools v0.41.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index aac55d1a6d..c7d814b8b9 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k= github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -55,8 +55,8 @@ github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -68,8 +68,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -106,8 +106,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -140,8 +141,9 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/internal/cli/access.go b/internal/cli/access.go index 766a793f1c..2e99c98adf 100644 --- a/internal/cli/access.go +++ b/internal/cli/access.go @@ -129,7 +129,7 @@ func (a *Access) getIP(ntw mtglib.Network, protocol string) net.IP { defer func() { io.Copy(io.Discard, resp.Body) //nolint: errcheck - resp.Body.Close() //nolint: errcheck + resp.Body.Close() //nolint: errcheck }() data, err := io.ReadAll(resp.Body) diff --git a/internal/cli/run_proxy.go b/internal/cli/run_proxy.go index c8532129a7..5357309ae8 100644 --- a/internal/cli/run_proxy.go +++ b/internal/cli/run_proxy.go @@ -240,6 +240,14 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen return fmt.Errorf("cannot build ip allowlist: %w", err) } + dcOverrides := map[int][]string{} + for _, override := range conf.DCOverrides { + dcid := override.DC.Get() + for _, addr := range override.IPs { + dcOverrides[dcid] = append(dcOverrides[dcid], addr.Get("")) + } + } + opts := mtglib.ProxyOpts{ Logger: logger, Network: ntw, @@ -254,6 +262,7 @@ func runProxy(conf *config.Config, version string) error { //nolint: funlen AllowFallbackOnUnknownDC: conf.AllowFallbackOnUnknownDC.Get(false), TolerateTimeSkewness: conf.TolerateTimeSkewness.Value, + DCOverrides: dcOverrides, } proxy, err := mtglib.NewProxy(opts) diff --git a/internal/cli/simple_run.go b/internal/cli/simple_run.go index 02d6dc3328..53deb613e9 100644 --- a/internal/cli/simple_run.go +++ b/internal/cli/simple_run.go @@ -18,7 +18,7 @@ type SimpleRun struct { TCPBuffer string `kong:"name='tcp-buffer',short='b',default='4KB',help='Deprecated and ignored'"` //nolint: lll PreferIP string `kong:"name='prefer-ip',short='i',default='prefer-ipv6',help='IP preference. By default we prefer IPv6 with fallback to IPv4.'"` //nolint: lll DomainFrontingPort uint64 `kong:"name='domain-fronting-port',short='p',default='443',help='A port to access for domain fronting.'"` //nolint: lll - DOHIP net.IP `kong:"name='doh-ip',short='n',default='9.9.9.9',help='IP address of DNS-over-HTTP to use.'"` //nolint: lll + DOHIP net.IP `kong:"name='doh-ip',short='n',default='1.1.1.1',help='IP address of DNS-over-HTTP to use.'"` //nolint: lll Timeout time.Duration `kong:"name='timeout',short='t',default='10s',help='Network timeout to use'"` //nolint: lll Socks5Proxies []string `kong:"name='socks5-proxy',short='s',help='Socks5 proxies to use for network access.'"` //nolint: lll AntiReplayCacheSize string `kong:"name='antireplay-cache-size',short='a',default='1MB',help='A size of anti-replay cache to use.'"` //nolint: lll diff --git a/internal/config/config.go b/internal/config/config.go index 9b69a94023..3bab778391 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,10 @@ type Config struct { MetricPrefix TypeMetricPrefix `json:"metricPrefix"` } `json:"prometheus"` } `json:"stats"` + DCOverrides []struct { + DC TypeDC `json:"dc"` + IPs []TypeHostPort `json:"ips"` + } `json:"dcOverrides"` } func (c *Config) Validate() error { diff --git a/internal/config/parse.go b/internal/config/parse.go index 591e6bf1e5..fc1a14845a 100644 --- a/internal/config/parse.go +++ b/internal/config/parse.go @@ -5,7 +5,7 @@ import ( "encoding/json" "fmt" - "github.com/pelletier/go-toml" + "github.com/pelletier/go-toml/v2" ) type tomlConfig struct { @@ -59,6 +59,10 @@ type tomlConfig struct { MetricPrefix string `toml:"metric-prefix" json:"metricPrefix,omitempty"` } `toml:"prometheus" json:"prometheus,omitempty"` } `toml:"stats" json:"stats,omitempty"` + DCOverrides []struct { + DC uint `toml:"dc" json:"dc"` + IPs []string `toml:"ips" json:"ips"` + } `toml:"dc-overrides" json:"dcOverrides,omitempty"` } func Parse(rawData []byte) (*Config, error) { diff --git a/internal/config/type_dc.go b/internal/config/type_dc.go new file mode 100644 index 0000000000..1d3beb522f --- /dev/null +++ b/internal/config/type_dc.go @@ -0,0 +1,41 @@ +package config + +import ( + "fmt" + "strconv" +) + +type TypeDC struct { + Value int +} + +func (t *TypeDC) Set(value string) error { + parsed, err := strconv.ParseInt(value, 10, 16) + if err != nil { + return fmt.Errorf("cannot parse dc: %w", err) + } + + if parsed < 0 { + parsed = -parsed + } + + t.Value = int(parsed) + + return nil +} + +func (t *TypeDC) UnmarshalJSON(data []byte) error { + return t.Set(string(data)) +} + +func (t TypeDC) MarshalJSON() ([]byte, error) { + return []byte(t.String()), nil +} + +func (t TypeDC) String() string { + return strconv.Itoa(t.Value) +} + +func (t TypeDC) Get() int { + return t.Value +} diff --git a/internal/config/type_dc_test.go b/internal/config/type_dc_test.go new file mode 100644 index 0000000000..ebda9a2bbb --- /dev/null +++ b/internal/config/type_dc_test.go @@ -0,0 +1,96 @@ +package config_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/9seconds/mtg/v2/internal/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type typeDCTestStruct struct { + Value config.TypeDC `json:"value"` +} + +type TypeDCTestSuite struct { + suite.Suite +} + +func (suite *TypeDCTestSuite) TestUnmarshalFail() { + testData := []string{ + "-1s", + "1202002020202", + "xxx", + "-11111111111111", + "", + } + + for _, v := range testData { + data, err := json.Marshal(map[string]string{ + "value": v, + }) + suite.NoError(err) + + suite.T().Run(v, func(t *testing.T) { + assert.Error(t, json.Unmarshal(data, &typeDCTestStruct{})) + }) + } +} + +func (suite *TypeDCTestSuite) TestUnmarshalOk() { + testData := map[int]int{ + 1: 1, + -1: 1, + 203: 203, + } + + for value, expected := range testData { + data, err := json.Marshal(map[string]int{ + "value": value, + }) + suite.NoError(err) + + suite.T().Run(strconv.Itoa(value), func(t *testing.T) { + testStruct := &typeDCTestStruct{} + + assert.NoError(t, json.Unmarshal(data, testStruct)) + assert.Equal(t, expected, testStruct.Value.Value) + assert.Equal(t, expected, testStruct.Value.Get()) + }) + } +} + +func (suite *TypeDCTestSuite) TestMarshalOk() { + testData := map[string]int{ + "1": 1, + "203": 203, + } + + for k, v := range testData { + value := k + expected := v + + suite.T().Run(value, func(t *testing.T) { + testStruct := &typeDCTestStruct{} + + assert.NoError(t, testStruct.Value.Set(value)) + + data, err := json.Marshal(testStruct) + assert.NoError(t, err) + + expectedJSON, err := json.Marshal(map[string]int{ + "value": expected, + }) + assert.NoError(t, err) + + assert.JSONEq(t, string(expectedJSON), string(data)) + }) + } +} + +func TestTypeDC(t *testing.T) { + t.Parallel() + suite.Run(t, &TypeDCTestSuite{}) +} diff --git a/ipblocklist/files/http.go b/ipblocklist/files/http.go index 84bbfbce51..40fb6210e2 100644 --- a/ipblocklist/files/http.go +++ b/ipblocklist/files/http.go @@ -23,7 +23,7 @@ func (h httpFile) Open(ctx context.Context) (io.ReadCloser, error) { if err != nil { if response != nil { io.Copy(io.Discard, response.Body) //nolint: errcheck - response.Body.Close() //nolint: errcheck + response.Body.Close() //nolint: errcheck } return nil, fmt.Errorf("cannot get url %s: %w", h.url, err) diff --git a/mtglib/internal/dc/addr.go b/mtglib/internal/dc/addr.go new file mode 100644 index 0000000000..8e433e3a5f --- /dev/null +++ b/mtglib/internal/dc/addr.go @@ -0,0 +1,10 @@ +package dc + +type Addr struct { + Network string + Address string +} + +func (d Addr) String() string { + return d.Address +} diff --git a/mtglib/internal/dc/addr_set.go b/mtglib/internal/dc/addr_set.go new file mode 100644 index 0000000000..f1229a7a46 --- /dev/null +++ b/mtglib/internal/dc/addr_set.go @@ -0,0 +1,33 @@ +package dc + +import "math/rand/v2" + +type dcAddrSet struct { + v4 map[int][]Addr + v6 map[int][]Addr +} + +func (d dcAddrSet) getV4(dc int) []Addr { + if d.v4 == nil { + return nil + } + return d.get(d.v4[dc]) +} + +func (d dcAddrSet) getV6(dc int) []Addr { + if d.v6 == nil { + return nil + } + return d.get(d.v6[dc]) +} + +func (d dcAddrSet) get(addrs []Addr) []Addr { + otherSet := make([]Addr, 0, len(addrs)) + otherSet = append(otherSet, addrs...) + + rand.Shuffle(len(otherSet), func(i, j int) { + otherSet[i], otherSet[j] = otherSet[j], otherSet[i] + }) + + return otherSet +} diff --git a/mtglib/internal/dc/addr_test.go b/mtglib/internal/dc/addr_test.go new file mode 100644 index 0000000000..c288c0d432 --- /dev/null +++ b/mtglib/internal/dc/addr_test.go @@ -0,0 +1,16 @@ +package dc_test + +import ( + "testing" + + "github.com/9seconds/mtg/v2/mtglib/internal/dc" + "github.com/stretchr/testify/assert" +) + +func TestAddr(t *testing.T) { + t.Parallel() + + addr := dc.Addr{Network: "tcp4", Address: "127.0.0.1:443"} + + assert.Equal(t, "127.0.0.1:443", addr.String()) +} diff --git a/mtglib/internal/dc/init.go b/mtglib/internal/dc/init.go new file mode 100644 index 0000000000..c253f6e037 --- /dev/null +++ b/mtglib/internal/dc/init.go @@ -0,0 +1,73 @@ +package dc + +type preferIP uint8 + +const ( + preferIPOnlyIPv4 preferIP = iota + preferIPOnlyIPv6 + preferIPPreferIPv4 + preferIPPreferIPv6 +) + +const ( + DefaultDC = 2 +) + +type Logger interface { + Info(msg string) + WarningError(msg string, err error) +} + +var ( + // https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30 + defaultDCAddrSet = dcAddrSet{ + v4: map[int][]Addr{ + 1: { + {Network: "tcp4", Address: "149.154.175.50:443"}, + }, + 2: { + {Network: "tcp4", Address: "149.154.167.51:443"}, + {Network: "tcp4", Address: "95.161.76.100:443"}, + }, + 3: { + {Network: "tcp4", Address: "149.154.175.100:443"}, + }, + 4: { + {Network: "tcp4", Address: "149.154.167.91:443"}, + }, + 5: { + {Network: "tcp4", Address: "149.154.171.5:443"}, + }, + }, + v6: map[int][]Addr{ + 1: { + {Network: "tcp6", Address: "[2001:b28:f23d:f001::a]:443"}, + }, + 2: { + {Network: "tcp6", Address: "[2001:67c:04e8:f002::a]:443"}, + }, + 3: { + {Network: "tcp6", Address: "[2001:b28:f23d:f003::a]:443"}, + }, + 4: { + {Network: "tcp6", Address: "[2001:67c:04e8:f004::a]:443"}, + }, + 5: { + {Network: "tcp6", Address: "[2001:b28:f23f:f005::a]:443"}, + }, + }, + } + + defaultDCOverridesAddrSet = dcAddrSet{ + v4: map[int][]Addr{ + 203: { + {Network: "tcp4", Address: "91.105.192.100:443"}, + }, + }, + v6: map[int][]Addr{ + 203: { + {Network: "tcp6", Address: "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"}, + }, + }, + } +) diff --git a/mtglib/internal/dc/telegram.go b/mtglib/internal/dc/telegram.go new file mode 100644 index 0000000000..e9f261f4e1 --- /dev/null +++ b/mtglib/internal/dc/telegram.go @@ -0,0 +1,79 @@ +package dc + +import ( + "fmt" + "net" + "strings" +) + +type Telegram struct { + view dcView + preferIP preferIP +} + +func (t *Telegram) GetAddresses(dc int) []Addr { + switch t.preferIP { + case preferIPOnlyIPv4: + return t.view.getV4(dc) + case preferIPOnlyIPv6: + return t.view.getV4(dc) + case preferIPPreferIPv4: + return append(t.view.getV4(dc), t.view.getV6(dc)...) + } + + return append(t.view.getV6(dc), t.view.getV4(dc)...) +} + +func New(ipPreference string, userOverrides map[int][]string) (*Telegram, error) { + var pref preferIP + + switch strings.ToLower(ipPreference) { + case "prefer-ipv4": + pref = preferIPPreferIPv4 + case "prefer-ipv6": + pref = preferIPPreferIPv6 + case "only-ipv4": + pref = preferIPOnlyIPv4 + case "only-ipv6": + pref = preferIPOnlyIPv6 + default: + return nil, fmt.Errorf("unknown ip preference %s", ipPreference) + } + + overrides := dcAddrSet{ + v4: map[int][]Addr{}, + v6: map[int][]Addr{}, + } + for dc, addrs := range userOverrides { + for _, addr := range addrs { + host, _, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("incorrect host %s: %w", addr, err) + } + + parsed := net.ParseIP(host) + if parsed == nil { + return nil, fmt.Errorf("incorrect host %s", addr) + } + + if parsed.To4() != nil { + overrides.v4[dc] = append(overrides.v4[dc], Addr{ + Network: "tcp4", + Address: addr, + }) + } else { + overrides.v6[dc] = append(overrides.v6[dc], Addr{ + Network: "tcp6", + Address: addr, + }) + } + } + } + + return &Telegram{ + view: dcView{ + overrides: overrides, + }, + preferIP: pref, + }, nil +} diff --git a/mtglib/internal/dc/view.go b/mtglib/internal/dc/view.go new file mode 100644 index 0000000000..efee1c031b --- /dev/null +++ b/mtglib/internal/dc/view.go @@ -0,0 +1,21 @@ +package dc + +type dcView struct { + overrides dcAddrSet +} + +func (d dcView) getV4(dc int) []Addr { + addrs := d.overrides.getV4(dc) + addrs = append(addrs, defaultDCOverridesAddrSet.getV4(dc)...) + addrs = append(addrs, defaultDCAddrSet.getV4(dc)...) + + return addrs +} + +func (d dcView) getV6(dc int) []Addr { + addrs := d.overrides.getV6(dc) + addrs = append(addrs, defaultDCOverridesAddrSet.getV6(dc)...) + addrs = append(addrs, defaultDCAddrSet.getV6(dc)...) + + return addrs +} diff --git a/mtglib/internal/dc/view_test.go b/mtglib/internal/dc/view_test.go new file mode 100644 index 0000000000..74d46cb6de --- /dev/null +++ b/mtglib/internal/dc/view_test.go @@ -0,0 +1,81 @@ +package dc + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ViewTestSuite struct { + suite.Suite + + view dcView +} + +func (suite *ViewTestSuite) SetupSuite() { + suite.view = dcView{ + overrides: dcAddrSet{ + v4: map[int][]Addr{ + 111: { + {Network: "tcp4", Address: "127.0.0.1:443"}, + }, + 203: { + {Network: "tcp4", Address: "127.0.0.2:443"}, + }, + }, + v6: map[int][]Addr{ + 203: { + {Network: "tcp6", Address: "xxx"}, + }, + }, + }, + } +} + +func (suite *ViewTestSuite) TestGetV4() { + testData := map[int][]Addr{ + 111: { + {"tcp4", "127.0.0.1:443"}, + }, + 203: { + {"tcp4", "127.0.0.2:443"}, + {"tcp4", "91.105.192.100:443"}, + }, + 2: { + {"tcp4", "149.154.167.51:443"}, + {"tcp4", "95.161.76.100:443"}, + }, + } + + for dc, addresses := range testData { + suite.T().Run(fmt.Sprintf("dc%d", dc), func(t *testing.T) { + assert.ElementsMatch(t, addresses, suite.view.getV4(dc)) + }) + } +} + +func (suite *ViewTestSuite) TestGetV6() { + testData := map[int][]Addr{ + 111: {}, + 203: { + {"tcp6", "xxx"}, + {"tcp6", "[2a0a:f280:0203:000a:5000:0000:0000:0100]:443"}, + }, + 1: { + {"tcp6", "[2001:b28:f23d:f001::a]:443"}, + }, + } + + for dc, addresses := range testData { + suite.T().Run(fmt.Sprintf("dc%d", dc), func(t *testing.T) { + assert.ElementsMatch(t, addresses, suite.view.getV6(dc)) + }) + } +} + +func TestView(t *testing.T) { + t.Parallel() + suite.Run(t, &ViewTestSuite{}) +} diff --git a/mtglib/internal/relay/relay.go b/mtglib/internal/relay/relay.go index ac35cf94d3..060a05cd32 100644 --- a/mtglib/internal/relay/relay.go +++ b/mtglib/internal/relay/relay.go @@ -10,7 +10,7 @@ import ( func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials.Conn) { defer telegramConn.Close() //nolint: errcheck - defer clientConn.Close() //nolint: errcheck + defer clientConn.Close() //nolint: errcheck ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -18,7 +18,7 @@ func Relay(ctx context.Context, log Logger, telegramConn, clientConn essentials. go func() { <-ctx.Done() telegramConn.Close() //nolint: errcheck - clientConn.Close() //nolint: errcheck + clientConn.Close() //nolint: errcheck }() closeChan := make(chan struct{}) diff --git a/mtglib/internal/telegram/address_pool.go b/mtglib/internal/telegram/address_pool.go deleted file mode 100644 index 2a22371a20..0000000000 --- a/mtglib/internal/telegram/address_pool.go +++ /dev/null @@ -1,41 +0,0 @@ -package telegram - -import "math/rand" - -type addressPool struct { - v4 [][]tgAddr - v6 [][]tgAddr -} - -func (a addressPool) isValidDC(dc int) bool { - return dc > 0 && dc <= len(a.v4) && dc <= len(a.v6) -} - -func (a addressPool) getRandomDC() int { - return 1 + rand.Intn(len(a.v4)) -} - -func (a addressPool) getV4(dc int) []tgAddr { - return a.get(a.v4, dc-1) -} - -func (a addressPool) getV6(dc int) []tgAddr { - return a.get(a.v6, dc-1) -} - -func (a addressPool) get(addresses [][]tgAddr, dc int) []tgAddr { - if dc < 0 || dc >= len(addresses) { - return nil - } - - rv := make([]tgAddr, len(addresses[dc])) - copy(rv, addresses[dc]) - - if len(rv) > 1 { - rand.Shuffle(len(rv), func(i, j int) { - rv[i], rv[j] = rv[j], rv[i] - }) - } - - return rv -} diff --git a/mtglib/internal/telegram/init.go b/mtglib/internal/telegram/init.go deleted file mode 100644 index 78874dafaa..0000000000 --- a/mtglib/internal/telegram/init.go +++ /dev/null @@ -1,90 +0,0 @@ -package telegram - -import ( - "context" - "errors" - - "github.com/9seconds/mtg/v2/essentials" -) - -var errNoAddresses = errors.New("no addresses") - -type preferIP uint8 - -const ( - preferIPOnlyIPv4 preferIP = iota - preferIPOnlyIPv6 - preferIPPreferIPv4 - preferIPPreferIPv6 -) - -type tgAddr struct { - network string - address string -} - -// https://github.com/telegramdesktop/tdesktop/blob/master/Telegram/SourceFiles/mtproto/mtproto_dc_options.cpp#L30 -var ( - productionV4Addresses = [][]tgAddr{ - { // dc1 - {network: "tcp4", address: "149.154.175.50:443"}, - }, - { // dc2 - {network: "tcp4", address: "149.154.167.51:443"}, - {network: "tcp4", address: "95.161.76.100:443"}, - }, - { // dc3 - {network: "tcp4", address: "149.154.175.100:443"}, - }, - { // dc4 - {network: "tcp4", address: "149.154.167.91:443"}, - }, - { // dc5 - {network: "tcp4", address: "149.154.171.5:443"}, - }, - } - productionV6Addresses = [][]tgAddr{ - { // dc1 - {network: "tcp6", address: "[2001:b28:f23d:f001::a]:443"}, - }, - { // dc2 - {network: "tcp6", address: "[2001:67c:04e8:f002::a]:443"}, - }, - { // dc3 - {network: "tcp6", address: "[2001:b28:f23d:f003::a]:443"}, - }, - { // dc4 - {network: "tcp6", address: "[2001:67c:04e8:f004::a]:443"}, - }, - { // dc5 - {network: "tcp6", address: "[2001:b28:f23f:f005::a]:443"}, - }, - } - - testV4Addresses = [][]tgAddr{ - { // dc1 - {network: "tcp4", address: "149.154.175.10:443"}, - }, - { // dc2 - {network: "tcp4", address: "149.154.167.40:443"}, - }, - { // dc3 - {network: "tcp4", address: "149.154.175.117:443"}, - }, - } - testV6Addresses = [][]tgAddr{ - { // dc1 - {network: "tcp6", address: "[2001:b28:f23d:f001::e]:443"}, - }, - { // dc2 - {network: "tcp6", address: "[2001:67c:04e8:f002::e]:443"}, - }, - { // dc3 - {network: "tcp6", address: "[2001:b28:f23d:f003::e]:443"}, - }, - } -) - -type Dialer interface { - DialContext(ctx context.Context, network, address string) (essentials.Conn, error) -} diff --git a/mtglib/internal/telegram/telegram.go b/mtglib/internal/telegram/telegram.go deleted file mode 100644 index 926c7a00eb..0000000000 --- a/mtglib/internal/telegram/telegram.go +++ /dev/null @@ -1,83 +0,0 @@ -package telegram - -import ( - "context" - "fmt" - "strings" - - "github.com/9seconds/mtg/v2/essentials" -) - -type Telegram struct { - dialer Dialer - preferIP preferIP - pool addressPool -} - -func (t Telegram) Dial(ctx context.Context, dc int) (essentials.Conn, error) { - var addresses []tgAddr - - switch t.preferIP { - case preferIPOnlyIPv4: - addresses = t.pool.getV4(dc) - case preferIPOnlyIPv6: - addresses = t.pool.getV6(dc) - case preferIPPreferIPv4: - addresses = append(t.pool.getV4(dc), t.pool.getV6(dc)...) - case preferIPPreferIPv6: - addresses = append(t.pool.getV6(dc), t.pool.getV4(dc)...) - } - - var conn essentials.Conn - - err := errNoAddresses - - for _, v := range addresses { - conn, err = t.dialer.DialContext(ctx, v.network, v.address) - if err == nil { - return conn, nil - } - } - - return nil, fmt.Errorf("cannot dial to %d dc: %w", dc, err) -} - -func (t Telegram) IsKnownDC(dc int) bool { - return t.pool.isValidDC(dc) -} - -func (t Telegram) GetFallbackDC() int { - return t.pool.getRandomDC() -} - -func New(dialer Dialer, ipPreference string, useTestDCs bool) (*Telegram, error) { - var pref preferIP - - switch strings.ToLower(ipPreference) { - case "prefer-ipv4": - pref = preferIPPreferIPv4 - case "prefer-ipv6": - pref = preferIPPreferIPv6 - case "only-ipv4": - pref = preferIPOnlyIPv4 - case "only-ipv6": - pref = preferIPOnlyIPv6 - default: - return nil, fmt.Errorf("unknown ip preference %s", ipPreference) - } - - pool := addressPool{ - v4: productionV4Addresses, - v6: productionV6Addresses, - } - if useTestDCs { - pool.v4 = testV4Addresses - pool.v6 = testV6Addresses - } - - return &Telegram{ - dialer: dialer, - preferIP: pref, - pool: pool, - }, nil -} diff --git a/mtglib/internal/telegram/telegram_internal_test.go b/mtglib/internal/telegram/telegram_internal_test.go deleted file mode 100644 index 794e97adc2..0000000000 --- a/mtglib/internal/telegram/telegram_internal_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package telegram - -import ( - "context" - "errors" - "io" - "net" - "strconv" - "testing" - - "github.com/9seconds/mtg/v2/internal/testlib" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" -) - -type TelegramTestSuite struct { - suite.Suite - - dialerMock *testlib.MtglibNetworkMock - t *Telegram -} - -func (suite *TelegramTestSuite) SetupTest() { - suite.dialerMock = &testlib.MtglibNetworkMock{} - suite.t, _ = New(suite.dialerMock, "prefer-ipv4", false) -} - -func (suite *TelegramTestSuite) TearDownTest() { - suite.dialerMock.AssertExpectations(suite.T()) -} - -func (suite *TelegramTestSuite) TestUnknownDC() { - testData := []int{ - -1, - 0, - 6, - 100, - } - - for _, v := range testData { - value := v - - suite.T().Run(strconv.Itoa(value), func(t *testing.T) { - _, err := suite.t.Dial(context.Background(), value) - assert.Error(t, err) - assert.False(t, suite.t.IsKnownDC(value)) - }) - } -} - -func (suite *TelegramTestSuite) TestDialToCorrectIPs() { - testData := map[int][]tgAddr{} - - for i := 1; i <= 5; i++ { - testData[i] = []tgAddr{} - testData[i] = append(testData[i], productionV4Addresses[i-1]...) - testData[i] = append(testData[i], productionV6Addresses[i-1]...) - } - - for i, v := range testData { - idx := i - addresses := v - - suite.T().Run(strconv.Itoa(idx), func(t *testing.T) { - for _, addr := range addresses { - suite.dialerMock. - On("DialContext", mock.Anything, addr.network, addr.address). - Once(). - Return((*net.TCPConn)(nil), io.EOF) - } - - _, err := suite.t.Dial(context.Background(), idx) - assert.True(t, errors.Is(err, io.EOF)) - assert.True(t, suite.t.IsKnownDC(idx)) - }) - } -} - -func (suite *TelegramTestSuite) TestDialPreferIPRange() { - testData := map[string][]tgAddr{ - "prefer-ipv4": {testV4Addresses[0][0], testV6Addresses[0][0]}, - "prefer-ipv6": {testV6Addresses[0][0], testV4Addresses[0][0]}, - "only-ipv4": {testV4Addresses[0][0]}, - "only-ipv6": {testV6Addresses[0][0]}, - } - - for k, v := range testData { - name := k - addresses := v - - suite.T().Run(name, func(t *testing.T) { - for _, addr := range addresses { - suite.dialerMock. - On("DialContext", mock.Anything, addr.network, addr.address). - Once(). - Return((*net.TCPConn)(nil), io.EOF) - } - - tg, _ := New(suite.dialerMock, name, true) - _, err := tg.Dial(context.Background(), 1) - - assert.True(t, errors.Is(err, io.EOF)) - }) - } -} - -func (suite *TelegramTestSuite) TestDialPreferIPPriority() { - testData := map[string]tgAddr{ - "prefer-ipv4": productionV4Addresses[0][0], - "prefer-ipv6": productionV6Addresses[0][0], - } - - for k, v := range testData { - name := k - addr := v - - suite.T().Run(name, func(t *testing.T) { - conn := &net.TCPConn{} - - suite.dialerMock. - On("DialContext", mock.Anything, addr.network, addr.address). - Once(). - Return(conn, nil) - - tg, _ := New(suite.dialerMock, name, false) - - res, err := tg.Dial(context.Background(), 1) - assert.NoError(t, err) - assert.Equal(t, conn, res) - }) - } -} - -func (suite *TelegramTestSuite) TestUnknownPreferIP() { - _, err := New(suite.dialerMock, "xxx", false) - suite.Error(err) -} - -func (suite *TelegramTestSuite) TestFallbackDC() { - dcs := make([]int, 10) - - for i := 0; i < len(dcs); i++ { - dcs[i] = suite.t.GetFallbackDC() - } - - for _, v := range dcs { - value := v - - suite.T().Run(strconv.Itoa(value), func(t *testing.T) { - assert.True(t, suite.t.IsKnownDC(value)) - }) - } -} - -func TestTelegram(t *testing.T) { - t.Parallel() - suite.Run(t, &TelegramTestSuite{}) -} diff --git a/mtglib/proxy.go b/mtglib/proxy.go index e63f6ba29f..830a2f9af1 100644 --- a/mtglib/proxy.go +++ b/mtglib/proxy.go @@ -10,11 +10,11 @@ import ( "time" "github.com/9seconds/mtg/v2/essentials" + "github.com/9seconds/mtg/v2/mtglib/internal/dc" "github.com/9seconds/mtg/v2/mtglib/internal/faketls" "github.com/9seconds/mtg/v2/mtglib/internal/faketls/record" "github.com/9seconds/mtg/v2/mtglib/internal/obfuscated2" "github.com/9seconds/mtg/v2/mtglib/internal/relay" - "github.com/9seconds/mtg/v2/mtglib/internal/telegram" "github.com/panjf2000/ants/v2" ) @@ -28,7 +28,7 @@ type Proxy struct { tolerateTimeSkewness time.Duration domainFrontingPort int workerPool *ants.PoolWithFunc - telegram *telegram.Telegram + telegram *dc.Telegram secret Secret network Network @@ -219,18 +219,26 @@ func (p *Proxy) doObfuscated2Handshake(ctx *streamContext) error { } func (p *Proxy) doTelegramCall(ctx *streamContext) error { - dc := ctx.dc - - if p.allowFallbackOnUnknownDC && !p.telegram.IsKnownDC(dc) { - dc = p.telegram.GetFallbackDC() - ctx.logger = ctx.logger.BindInt("fallback_dc", dc) + dcid := ctx.dc + addresses := p.telegram.GetAddresses(dcid) + if len(addresses) == 0 && p.allowFallbackOnUnknownDC { + ctx.logger = ctx.logger.BindInt("fallback_dc", dc.DefaultDC) ctx.logger.Warning("unknown DC, fallbacks") + addresses = p.telegram.GetAddresses(dc.DefaultDC) } - conn, err := p.telegram.Dial(ctx, dc) + var conn essentials.Conn + var err error + + for _, addr := range addresses { + conn, err = p.network.Dial(addr.Network, addr.Address) + if err == nil { + break + } + } if err != nil { - return fmt.Errorf("cannot dial to Telegram: %w", err) + return fmt.Errorf("no addresses to call: %w", err) } encryptor, decryptor, err := obfuscated2.ServerHandshake(conn) @@ -292,9 +300,9 @@ func NewProxy(opts ProxyOpts) (*Proxy, error) { return nil, fmt.Errorf("invalid settings: %w", err) } - tg, err := telegram.New(opts.Network, opts.getPreferIP(), opts.UseTestDCs) + tg, err := dc.New(opts.getPreferIP(), opts.DCOverrides) if err != nil { - return nil, fmt.Errorf("cannot build telegram dialer: %w", err) + return nil, fmt.Errorf("cannot build telegram dc fetcher: %w", err) } ctx, cancel := context.WithCancel(context.Background()) diff --git a/mtglib/proxy_opts.go b/mtglib/proxy_opts.go index deb3fb0bb2..53f34348a2 100644 --- a/mtglib/proxy_opts.go +++ b/mtglib/proxy_opts.go @@ -110,7 +110,15 @@ type ProxyOpts struct { // Telegram-related projects. // // This is an optional setting. + // + // OBSOLETE and DEPRECATED. Ignored. UseTestDCs bool + + // DCOverrides defines a set of IP addresses that should be used + // with a higher priority to those that are calculated somehow by mtg. + // + // This is an optional setting + DCOverrides map[int][]string } func (p ProxyOpts) valid() error { diff --git a/network/init.go b/network/init.go index 52831c25e7..3baa8b0193 100644 --- a/network/init.go +++ b/network/init.go @@ -63,7 +63,7 @@ const ( // DefaultDOHHostname defines a default IP address for DOH host. Since mtg is // simple, please pass IP address here. We do not have bootstrap servers here // embedded. - DefaultDOHHostname = "9.9.9.9" + DefaultDOHHostname = "1.1.1.1" // DNSTimeout defines a timeout for DNS queries. DNSTimeout = 5 * time.Second diff --git a/stats/statsd_test.go b/stats/statsd_test.go index a8759bacf6..5587345b00 100644 --- a/stats/statsd_test.go +++ b/stats/statsd_test.go @@ -100,7 +100,7 @@ func (suite *StatsdTestSuite) SetupTest() { func (suite *StatsdTestSuite) TearDownTest() { suite.statsd.Shutdown() - suite.factory.Close() //nolint: errcheck + suite.factory.Close() //nolint: errcheck suite.statsdServer.Close() //nolint: errcheck }