feat(tcp): add Minecraft analyzer for handshake detection
feat(tcp): implement FETAnalyzer with segment buffering feat(tcp): enhance TCP engine with window scale detection test(tcp): add tests for Minecraft and FET analyzers docs: add documentation for Minecraft analyzer and TCP evasion
This commit is contained in:
+51
-6
@@ -1,6 +1,21 @@
|
||||
package tcp
|
||||
|
||||
import "github.com/apernet/OpenGFW/analyzer"
|
||||
import (
|
||||
"github.com/apernet/OpenGFW/analyzer"
|
||||
"github.com/apernet/OpenGFW/analyzer/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
// fetMinDataLen is how many bytes we want before classifying a stream.
|
||||
// The metrics below are meaningless on a handful of bytes - a peer can
|
||||
// force a tiny first segment by advertising a small TCP receive window in
|
||||
// its SYN-ACK, which used to be enough to make us exempt the connection.
|
||||
fetMinDataLen = 16
|
||||
// fetMaxSegments bounds how long we wait for fetMinDataLen bytes. Without
|
||||
// it, a peer that sends one byte and then goes quiet would keep the stream
|
||||
// active - and therefore out of the connmark bypass - until the TCP timeout.
|
||||
fetMaxSegments = 4
|
||||
)
|
||||
|
||||
var _ analyzer.TCPAnalyzer = (*FETAnalyzer)(nil)
|
||||
|
||||
@@ -25,6 +40,8 @@ func (a *FETAnalyzer) NewTCP(info analyzer.TCPInfo, logger analyzer.Logger) anal
|
||||
|
||||
type fetStream struct {
|
||||
logger analyzer.Logger
|
||||
buf utils.ByteBuffer
|
||||
segs int
|
||||
}
|
||||
|
||||
func newFETStream(logger analyzer.Logger) *fetStream {
|
||||
@@ -38,6 +55,38 @@ func (s *fetStream) Feed(rev, start, end bool, skip int, data []byte) (u *analyz
|
||||
if len(data) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
if rev {
|
||||
// We classify what the client sends first. If the server speaks before
|
||||
// the client does, this isn't a handshake we can say anything about.
|
||||
if s.buf.Len() == 0 {
|
||||
return nil, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
s.segs++
|
||||
if s.buf.Len() == 0 && len(data) >= fetMinDataLen {
|
||||
// Enough to decide on, and nothing buffered yet - classify in place.
|
||||
return fetResult(data), true
|
||||
}
|
||||
s.buf.Append(data)
|
||||
if s.buf.Len() < fetMinDataLen && s.segs < fetMaxSegments {
|
||||
return nil, false
|
||||
}
|
||||
u = fetResult(s.buf.Buf)
|
||||
s.buf.Reset()
|
||||
return u, true
|
||||
}
|
||||
|
||||
func (s *fetStream) Close(limited bool) *analyzer.PropUpdate {
|
||||
if s.buf.Len() == 0 {
|
||||
return nil
|
||||
}
|
||||
u := fetResult(s.buf.Buf)
|
||||
s.buf.Reset()
|
||||
return u
|
||||
}
|
||||
|
||||
func fetResult(data []byte) *analyzer.PropUpdate {
|
||||
ex1 := averagePopCount(data)
|
||||
ex2 := isFirstSixPrintable(data)
|
||||
ex3 := printablePercentage(data)
|
||||
@@ -54,11 +103,7 @@ func (s *fetStream) Feed(rev, start, end bool, skip int, data []byte) (u *analyz
|
||||
"ex5": ex5,
|
||||
"yes": !exempt,
|
||||
},
|
||||
}, true
|
||||
}
|
||||
|
||||
func (s *fetStream) Close(limited bool) *analyzer.PropUpdate {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func popCount(b byte) int {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/apernet/OpenGFW/analyzer"
|
||||
)
|
||||
|
||||
// feedFET runs a sequence of client-to-server segments through a fresh stream
|
||||
// and returns the props it settled on, or nil if it never reached a verdict.
|
||||
func feedFET(t *testing.T, segments ...[]byte) analyzer.PropMap {
|
||||
t.Helper()
|
||||
s := newFETStream(nil)
|
||||
for _, seg := range segments {
|
||||
u, done := s.Feed(false, false, false, 0, seg)
|
||||
if done {
|
||||
if u == nil {
|
||||
return nil
|
||||
}
|
||||
return u.M
|
||||
}
|
||||
}
|
||||
if u := s.Close(false); u != nil {
|
||||
return u.M
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// encryptedSample returns deterministic high-entropy bytes, standing in for the
|
||||
// first segment of a fully encrypted proxy protocol such as Shadowsocks.
|
||||
func encryptedSample(n int) []byte {
|
||||
r := rand.New(rand.NewSource(1))
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(r.Intn(256))
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func tlsClientHelloSample() []byte {
|
||||
return append([]byte{0x16, 0x03, 0x01, 0x02, 0x00, 0x01, 0x00, 0x01, 0xfc}, encryptedSample(64)...)
|
||||
}
|
||||
|
||||
// A peer can force a one-byte first segment by advertising a tiny TCP receive
|
||||
// window in its SYN-ACK. The verdict must not depend on where the boundary
|
||||
// falls: classifying the first segment alone used to exempt these streams.
|
||||
func TestFETVerdictIsSegmentationIndependent(t *testing.T) {
|
||||
samples := map[string][]byte{
|
||||
"encrypted": encryptedSample(64),
|
||||
"tls": tlsClientHelloSample(),
|
||||
"http": []byte("GET /index.html HTTP/1.1\r\nHost: example.com\r\n\r\n"),
|
||||
}
|
||||
for name, data := range samples {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
whole := feedFET(t, data)
|
||||
if whole == nil {
|
||||
t.Fatal("no verdict on unsplit data")
|
||||
}
|
||||
split := feedFET(t, data[:1], data[1:])
|
||||
if got, want := split.Get("yes"), whole.Get("yes"); got != want {
|
||||
t.Errorf("split verdict yes = %v, unsplit = %v", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFETDetectsEncryptedFirstSegment(t *testing.T) {
|
||||
data := encryptedSample(64)
|
||||
if got := feedFET(t, data).Get("yes"); got != true {
|
||||
t.Fatalf("unsplit: yes = %v, want true", got)
|
||||
}
|
||||
if got := feedFET(t, data[:1], data[1:]).Get("yes"); got != true {
|
||||
t.Errorf("one-byte first segment: yes = %v, want true", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFETExemptsTLS(t *testing.T) {
|
||||
data := tlsClientHelloSample()
|
||||
if got := feedFET(t, data).Get("yes"); got != false {
|
||||
t.Fatalf("unsplit: yes = %v, want false", got)
|
||||
}
|
||||
if got := feedFET(t, data[:1], data[1:]).Get("yes"); got != false {
|
||||
t.Errorf("one-byte first segment: yes = %v, want false", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Waiting for more data must be bounded, or a peer that sends one byte and then
|
||||
// stops would hold the stream open until the TCP timeout, keeping every packet
|
||||
// of the connection in userspace.
|
||||
func TestFETGivesUpAfterMaxSegments(t *testing.T) {
|
||||
s := newFETStream(nil)
|
||||
for i := 0; i < fetMaxSegments-1; i++ {
|
||||
if _, done := s.Feed(false, false, false, 0, []byte{0x00}); done {
|
||||
t.Fatalf("verdict reached after %d segments, want to keep waiting", i+1)
|
||||
}
|
||||
}
|
||||
u, done := s.Feed(false, false, false, 0, []byte{0x00})
|
||||
if !done || u == nil {
|
||||
t.Fatalf("no verdict after %d segments", fetMaxSegments)
|
||||
}
|
||||
}
|
||||
|
||||
// The metrics only describe a client's opening bytes, so a server that speaks
|
||||
// first should end the stream rather than be classified as if it were a client.
|
||||
func TestFETServerSpeaksFirst(t *testing.T) {
|
||||
s := newFETStream(nil)
|
||||
u, done := s.Feed(true, false, false, 0, encryptedSample(64))
|
||||
if !done {
|
||||
t.Error("stream not done after server-first data")
|
||||
}
|
||||
if u != nil {
|
||||
t.Errorf("props = %#v, want none", u.M)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/apernet/OpenGFW/analyzer"
|
||||
"github.com/apernet/OpenGFW/analyzer/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
minecraftMaxHandshakePacketLen = 2048
|
||||
minecraftMaxServerAddrLen = 255 * 4
|
||||
|
||||
minecraftPacketHandshake = 0
|
||||
|
||||
minecraftNextStateStatus = 1
|
||||
minecraftNextStateLogin = 2
|
||||
minecraftNextStateTransfer = 3
|
||||
)
|
||||
|
||||
var _ analyzer.TCPAnalyzer = (*MinecraftAnalyzer)(nil)
|
||||
|
||||
type MinecraftAnalyzer struct{}
|
||||
|
||||
func (a *MinecraftAnalyzer) Name() string {
|
||||
return "minecraft"
|
||||
}
|
||||
|
||||
func (a *MinecraftAnalyzer) Limit() int {
|
||||
return minecraftMaxHandshakePacketLen + 5
|
||||
}
|
||||
|
||||
func (a *MinecraftAnalyzer) NewTCP(info analyzer.TCPInfo, logger analyzer.Logger) analyzer.TCPStream {
|
||||
return newMinecraftStream(logger)
|
||||
}
|
||||
|
||||
type minecraftStream struct {
|
||||
logger analyzer.Logger
|
||||
reqBuf *utils.ByteBuffer
|
||||
}
|
||||
|
||||
func newMinecraftStream(logger analyzer.Logger) *minecraftStream {
|
||||
return &minecraftStream{logger: logger, reqBuf: &utils.ByteBuffer{}}
|
||||
}
|
||||
|
||||
func (s *minecraftStream) Feed(rev, start, end bool, skip int, data []byte) (u *analyzer.PropUpdate, done bool) {
|
||||
if skip != 0 {
|
||||
return minecraftResult(false, false, nil), true
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
if rev {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
s.reqBuf.Append(data)
|
||||
m, needMore, valid := parseMinecraftHandshake(s.reqBuf.Buf)
|
||||
if needMore {
|
||||
return minecraftResult(false, true, nil), false
|
||||
}
|
||||
if !valid {
|
||||
return minecraftResult(false, false, nil), true
|
||||
}
|
||||
|
||||
return minecraftResult(true, false, m), true
|
||||
}
|
||||
|
||||
func (s *minecraftStream) Close(limited bool) *analyzer.PropUpdate {
|
||||
s.reqBuf.Reset()
|
||||
return minecraftResult(false, false, nil)
|
||||
}
|
||||
|
||||
func minecraftResult(yes, candidate bool, m analyzer.PropMap) *analyzer.PropUpdate {
|
||||
if m == nil {
|
||||
m = analyzer.PropMap{}
|
||||
}
|
||||
m["yes"] = yes
|
||||
m["candidate"] = candidate
|
||||
return &analyzer.PropUpdate{
|
||||
Type: analyzer.PropUpdateReplace,
|
||||
M: m,
|
||||
}
|
||||
}
|
||||
|
||||
func parseMinecraftHandshake(buf []byte) (m analyzer.PropMap, needMore, valid bool) {
|
||||
packetLen, off, ok, invalid := readMinecraftVarInt(buf, 0)
|
||||
if invalid || packetLen <= 0 || packetLen > minecraftMaxHandshakePacketLen {
|
||||
return nil, false, false
|
||||
}
|
||||
if !ok {
|
||||
return nil, true, false
|
||||
}
|
||||
|
||||
packetEnd := off + int(packetLen)
|
||||
if len(buf) < packetEnd {
|
||||
return nil, true, false
|
||||
}
|
||||
|
||||
packet := buf[off:packetEnd]
|
||||
packetID, pos, ok, invalid := readMinecraftVarInt(packet, 0)
|
||||
if invalid || !ok || packetID != minecraftPacketHandshake {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
protocolVersion, pos, ok, invalid := readMinecraftVarInt(packet, pos)
|
||||
if invalid || !ok {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
addrLen, pos, ok, invalid := readMinecraftVarInt(packet, pos)
|
||||
if invalid || !ok || addrLen <= 0 || addrLen > minecraftMaxServerAddrLen {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
addrEnd := pos + int(addrLen)
|
||||
if addrEnd+2 > len(packet) {
|
||||
return nil, false, false
|
||||
}
|
||||
addrBytes := packet[pos:addrEnd]
|
||||
if !utf8.Valid(addrBytes) {
|
||||
return nil, false, false
|
||||
}
|
||||
pos = addrEnd
|
||||
|
||||
serverPort := uint16(packet[pos])<<8 | uint16(packet[pos+1])
|
||||
pos += 2
|
||||
|
||||
nextState, pos, ok, invalid := readMinecraftVarInt(packet, pos)
|
||||
if invalid || !ok || !isMinecraftNextState(nextState) || pos != len(packet) {
|
||||
return nil, false, false
|
||||
}
|
||||
|
||||
return analyzer.PropMap{
|
||||
"protocol": int(protocolVersion),
|
||||
"server_addr": string(addrBytes),
|
||||
"server_port": serverPort,
|
||||
"next_state": int(nextState),
|
||||
"next_state_name": minecraftNextStateName(nextState),
|
||||
"packet_length": int(packetLen),
|
||||
}, false, true
|
||||
}
|
||||
|
||||
func readMinecraftVarInt(buf []byte, offset int) (value int32, next int, ok bool, invalid bool) {
|
||||
var result int32
|
||||
for i := 0; i < 5; i++ {
|
||||
if offset+i >= len(buf) {
|
||||
return 0, offset, false, false
|
||||
}
|
||||
b := buf[offset+i]
|
||||
result |= int32(b&0x7f) << (7 * i)
|
||||
if b&0x80 == 0 {
|
||||
return result, offset + i + 1, true, false
|
||||
}
|
||||
}
|
||||
return 0, offset, false, true
|
||||
}
|
||||
|
||||
func isMinecraftNextState(state int32) bool {
|
||||
switch state {
|
||||
case minecraftNextStateStatus, minecraftNextStateLogin, minecraftNextStateTransfer:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func minecraftNextStateName(state int32) string {
|
||||
switch state {
|
||||
case minecraftNextStateStatus:
|
||||
return "status"
|
||||
case minecraftNextStateLogin:
|
||||
return "login"
|
||||
case minecraftNextStateTransfer:
|
||||
return "transfer"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/apernet/OpenGFW/analyzer"
|
||||
)
|
||||
|
||||
func TestMinecraftParsing_HandshakeLogin(t *testing.T) {
|
||||
handshake := buildMinecraftHandshake(t, 767, "211.136.162.178", 25565, minecraftNextStateLogin)
|
||||
want := analyzer.PropMap{
|
||||
"yes": true,
|
||||
"candidate": false,
|
||||
"protocol": 767,
|
||||
"server_addr": "211.136.162.178",
|
||||
"server_port": uint16(25565),
|
||||
"next_state": minecraftNextStateLogin,
|
||||
"next_state_name": "login",
|
||||
"packet_length": len(handshake) - 1,
|
||||
}
|
||||
|
||||
u, done := newMinecraftStream(nil).Feed(false, false, false, 0, handshake)
|
||||
if !done {
|
||||
t.Fatal("stream not done after full handshake")
|
||||
}
|
||||
if !reflect.DeepEqual(u.M, want) {
|
||||
t.Errorf("parsed = %#v, want %#v", u.M, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinecraftParsing_FragmentedHandshake(t *testing.T) {
|
||||
handshake := buildMinecraftHandshake(t, 767, "mc.example.com", 25565, minecraftNextStateStatus)
|
||||
s := newMinecraftStream(nil)
|
||||
|
||||
u, done := s.Feed(false, false, false, 0, handshake[:2])
|
||||
if done {
|
||||
t.Fatal("stream done on partial handshake")
|
||||
}
|
||||
if got := u.M.Get("candidate"); got != true {
|
||||
t.Fatalf("candidate = %v, want true", got)
|
||||
}
|
||||
|
||||
u, done = s.Feed(false, false, false, 0, handshake[2:])
|
||||
if !done {
|
||||
t.Fatal("stream not done after remaining handshake")
|
||||
}
|
||||
if got := u.M.Get("yes"); got != true {
|
||||
t.Fatalf("yes = %v, want true", got)
|
||||
}
|
||||
if got := u.M.Get("server_addr"); got != "mc.example.com" {
|
||||
t.Fatalf("server_addr = %v, want mc.example.com", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinecraftParsing_InvalidAfterCandidate(t *testing.T) {
|
||||
s := newMinecraftStream(nil)
|
||||
u, done := s.Feed(false, false, false, 0, []byte{0x04})
|
||||
if done {
|
||||
t.Fatal("stream done on partial packet")
|
||||
}
|
||||
if got := u.M.Get("candidate"); got != true {
|
||||
t.Fatalf("candidate = %v, want true", got)
|
||||
}
|
||||
|
||||
u, done = s.Feed(false, false, false, 0, []byte{0x01, 0x02, 0x03, 0x04})
|
||||
if !done {
|
||||
t.Fatal("stream not done on invalid packet")
|
||||
}
|
||||
if got := u.M.Get("candidate"); got != false {
|
||||
t.Fatalf("candidate = %v, want false", got)
|
||||
}
|
||||
if got := u.M.Get("yes"); got != false {
|
||||
t.Fatalf("yes = %v, want false", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinecraftParsing_RejectsInvalidUTF8Address(t *testing.T) {
|
||||
packet := []byte{0x00}
|
||||
packet = append(packet, encodeMinecraftVarInt(767)...)
|
||||
packet = append(packet, 0x01, 0xff)
|
||||
packet = append(packet, 0x63, 0xdd)
|
||||
packet = append(packet, encodeMinecraftVarInt(minecraftNextStateLogin)...)
|
||||
data := append(encodeMinecraftVarInt(int32(len(packet))), packet...)
|
||||
|
||||
u, done := newMinecraftStream(nil).Feed(false, false, false, 0, data)
|
||||
if !done {
|
||||
t.Fatal("stream not done on invalid UTF-8 address")
|
||||
}
|
||||
if got := u.M.Get("yes"); got != false {
|
||||
t.Fatalf("yes = %v, want false", got)
|
||||
}
|
||||
}
|
||||
|
||||
func buildMinecraftHandshake(t *testing.T, protocol int32, addr string, port uint16, nextState int32) []byte {
|
||||
t.Helper()
|
||||
packet := []byte{0x00}
|
||||
packet = append(packet, encodeMinecraftVarInt(protocol)...)
|
||||
packet = append(packet, encodeMinecraftVarInt(int32(len(addr)))...)
|
||||
packet = append(packet, []byte(addr)...)
|
||||
packet = append(packet, byte(port>>8), byte(port))
|
||||
packet = append(packet, encodeMinecraftVarInt(nextState)...)
|
||||
return append(encodeMinecraftVarInt(int32(len(packet))), packet...)
|
||||
}
|
||||
|
||||
func encodeMinecraftVarInt(value int32) []byte {
|
||||
var out []byte
|
||||
for {
|
||||
b := byte(value & 0x7f)
|
||||
value = int32(uint32(value) >> 7)
|
||||
if value != 0 {
|
||||
b |= 0x80
|
||||
}
|
||||
out = append(out, b)
|
||||
if value == 0 {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user